quantum_layer_activation_visualization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Quantum layer activation visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: OMP_NUM_THREADS=1\n"
     ]
    }
   ],
   "source": [
    "# OpenMP: number of parallel threads.\n",
    "%env OMP_NUM_THREADS=1\n",
    "\n",
    "# Plotting\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# PyTorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "import torchvision\n",
    "from torchvision import datasets, models, transforms\n",
    "\n",
    "# Pennylane\n",
    "import pennylane as qml\n",
    "from pennylane import numpy as np\n",
    "\n",
    "# Other tools\n",
    "import time\n",
    "import copy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_classes = ['cat', 'dog']  # Subset of CIFAR ('plane', 'car', 'bird', 'cat','deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "n_qubits = 4                       # Number of qubits\n",
    "                                   # An enterily classical net (defined by the next parameter). \n",
    "classical_model = '512_nq_n'          # Possible choices: '512_n','512_nq_n','551_512_n'. [nq=n_qubits, n=num_filtered_classes]\n",
    "step = 0.001                       # Learning rate\n",
    "batch_size = 256                   # Number of samples for each training step\n",
    "num_epochs = 3                     # Number of training epochs\n",
    "q_depth = 5                        # Depth of the quantum circuit (number of variational layers)\n",
    "gamma_lr_scheduler = 1             # Learning rate reduction applied every 10 epochs.                       \n",
    "max_layers = 15                    # Keep 15 even if not all are used.\n",
    "q_delta = 0.01                     # Initial spread of random quantum weights\n",
    "rng_seed = 0                       # Seed for random number generator\n",
    "start_time = time.time()           # start of the computation timer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(rng_seed)\n",
    "dev = qml.device('default.qubit', wires=n_qubits)\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "#device = torch.device(\"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HKdzGz2E2Br9"
   },
   "source": [
    "## Hybrid transfer learning model (classical-to-quantum)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def H_layer(nqubits):\n",
    "    \"\"\"Layer of single-qubit Hadamard gates. \n",
    "    \"\"\"\n",
    "    for idx in range(nqubits):\n",
    "        qml.Hadamard(wires=idx)\n",
    "        \n",
    "def RY_layer(w):\n",
    "    \"\"\"Layer of parametrized qubit rotations around the y axis. \n",
    "    \"\"\"\n",
    "    for idx, element in enumerate(w):\n",
    "        qml.RY(element, wires=idx)\n",
    "\n",
    "def entangling_layer(nqubits):\n",
    "    \"\"\"Layer of CNOTs followed by another shifted layer of CNOT.\n",
    "    \"\"\"\n",
    "    # In other words it should apply something like :\n",
    "    # CNOT  CNOT  CNOT  CNOT...  CNOT\n",
    "    #   CNOT  CNOT  CNOT...  CNOT  \n",
    "    for i in range(0, nqubits - 1, 2): # Loop over even indices: i=0,2,...N-2  \n",
    "        qml.CNOT(wires=[i, i + 1])\n",
    "    for i in range(1, nqubits - 1,2): # Loop over odd indices:  i=1,3,...N-3\n",
    "        qml.CNOT(wires=[i, i + 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "@qml.qnode(dev, interface='torch')\n",
    "def q_net(q_in, q_weights_flat):\n",
    "        \n",
    "        # Reshape weights\n",
    "        q_weights = q_weights_flat.reshape(max_layers, n_qubits)\n",
    "        \n",
    "        # Start from state |+> , unbiased w.r.t. |0> and |1>\n",
    "        H_layer(n_qubits)\n",
    "        \n",
    "        # Embed features in the quantum node\n",
    "        RY_layer(q_in)\n",
    "       \n",
    "        # Sequence of trainable variational layers\n",
    "        for k in range(q_depth):\n",
    "            entangling_layer(n_qubits)\n",
    "            RY_layer(q_weights[k+1])\n",
    "\n",
    "        # Expectation values in the Z basis\n",
    "        return [qml.expval(qml.PauliZ(j)) for j in range(n_qubits)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Quantumnet(nn.Module):\n",
    "        def __init__(self):\n",
    "            super().__init__()\n",
    "            self.pre_net = nn.Linear(512, n_qubits)\n",
    "            self.q_params = nn.Parameter(q_delta * torch.randn(max_layers * n_qubits))\n",
    "            self.post_net = nn.Linear(n_qubits, len(filtered_classes))\n",
    "\n",
    "        def forward(self, input_features):\n",
    "            pre_out = self.pre_net(input_features) \n",
    "            q_in = torch.tanh(pre_out) * np.pi / 2.0   \n",
    "            \n",
    "            # Apply the quantum circuit to each element of the batch, and append to q_out\n",
    "            q_out = torch.Tensor(0, n_qubits)\n",
    "            q_out = q_out.to(device)\n",
    "            for elem in q_in:\n",
    "                q_out_elem = q_net(elem,self.q_params).float().unsqueeze(0)\n",
    "                q_out = torch.cat((q_out, q_out_elem))\n",
    "            return self.post_net(q_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Quantumnet_Cut(nn.Module):\n",
    "        def __init__(self):\n",
    "            super().__init__()\n",
    "            self.pre_net = nn.Linear(512, n_qubits)\n",
    "            self.q_params = nn.Parameter(q_delta * torch.randn(max_layers * n_qubits))\n",
    "\n",
    "        def forward(self, input_features):\n",
    "            pre_out = self.pre_net(input_features) \n",
    "            q_in = torch.tanh(pre_out) * np.pi / 2.0   \n",
    "            \n",
    "            # Apply the quantum circuit to each element of the batch, and append to q_out\n",
    "            q_out = torch.Tensor(0, n_qubits)\n",
    "            q_out = q_out.to(device)\n",
    "            for elem in q_in:\n",
    "                q_out_elem = q_net(elem,self.q_params).float().unsqueeze(0)\n",
    "                q_out = torch.cat((q_out, q_out_elem))\n",
    "            return q_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Quantumnet_Classical_Cut(nn.Module):\n",
    "        def __init__(self):\n",
    "            super().__init__()\n",
    "            self.pre_net = nn.Linear(512, n_qubits)\n",
    "            self.q_params = nn.Parameter(q_delta * torch.randn(max_layers * n_qubits))\n",
    "\n",
    "        def forward(self, input_features):\n",
    "            pre_out = self.pre_net(input_features) \n",
    "            q_in = torch.tanh(pre_out) * np.pi / 2.0   \n",
    "\n",
    "            return q_in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build Hybrid Model\n",
    "model_hybrid = torchvision.models.resnet18(pretrained=True)\n",
    "\n",
    "for param in model_hybrid.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "model_hybrid.fc = Quantumnet()\n",
    "\n",
    "# Use CUDA or CPU according to the \"device\" object.\n",
    "model_hybrid = model_hybrid.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_classical_cut = torchvision.models.resnet18(pretrained=True)\n",
    "\n",
    "model_classical_cut.fc = Quantumnet_Classical_Cut()\n",
    "\n",
    "for param in model_classical_cut.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "# Use CUDA or CPU according to the \"device\" object.\n",
    "model_classical_cut = model_classical_cut.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_cut = torchvision.models.resnet18(pretrained=True)\n",
    "\n",
    "model_cut.fc = Quantumnet_Cut()\n",
    "\n",
    "for param in model_cut.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "# Use CUDA or CPU according to the \"device\" object.\n",
    "model_cut = model_cut.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ybqXssPX2BsB"
   },
   "source": [
    "## Load model from file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load model from file\n",
    "path = './'\n",
    "model_hybrid.load_state_dict(torch.load(path+'quantum_' + filtered_classes[0] + '_' + filtered_classes[1] + '.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_dict = model_hybrid.state_dict()\n",
    "\n",
    "model_cut_dict = model_cut.state_dict()\n",
    "model_cut_dict = {k: v for k, v in model_dict.items() if k in model_cut_dict}\n",
    "model_cut.load_state_dict(model_cut_dict)\n",
    "\n",
    "model_classical_cut_dict = model_classical_cut.state_dict()\n",
    "model_classical_cut_dict = {k: v for k, v in model_dict.items() if k in model_classical_cut_dict}\n",
    "model_classical_cut.load_state_dict(model_classical_cut_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "q4qWpbZy2Br4"
   },
   "source": [
    "## Dataset loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 169926656/170498071 [00:56<00:00, 8128959.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./data/cifar-10-python.tar.gz to ./data\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# Fixed pre-processing operations\n",
    "data_transforms = {\n",
    "    'train': transforms.Compose([\n",
    "        #transforms.RandomResizedCrop(224),     # uncomment for data augmentation\n",
    "        #transforms.RandomHorizontalFlip(),     # uncomment for data augmentation\n",
    "        transforms.Resize(256),\n",
    "        transforms.CenterCrop(224),\n",
    "        transforms.ToTensor(),\n",
    "        # Normalize input channels using mean values and standard deviations of ImageNet.\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    'val': transforms.Compose([\n",
    "        transforms.Resize(256),\n",
    "        transforms.CenterCrop(224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "}\n",
    "\n",
    "\n",
    "# =================== begin CIFAR dataset loading ===================\n",
    "trainset_full = torchvision.datasets.CIFAR10(root='./data', train=True,\n",
    "                                        download=True, transform=data_transforms['train'])\n",
    "testset_full = torchvision.datasets.CIFAR10(root='./data', train=False,\n",
    "                                       download=True, transform=data_transforms['val'])\n",
    "image_datasets_full={'train': trainset_full, 'val': testset_full}\n",
    "\n",
    "# CIFAR classes\n",
    "class_names = ('plane', 'car', 'bird', 'cat',\n",
    "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "# Get indices of samples associated to filtered_classes\n",
    "filtered_labels=[class_names.index(cl) for cl in filtered_classes]\n",
    "sub_indices={'train': [], 'val': []}\n",
    "for phase in ['train', 'val']:\n",
    "    for idx, label in enumerate(image_datasets_full[phase].targets):  \n",
    "        if label in filtered_labels:\n",
    "            sub_indices[phase].append(idx)\n",
    "            \n",
    "# Initialize sub-datasets according to filtered indices\n",
    "image_datasets = {x: torch.utils.data.Subset(image_datasets_full[x], sub_indices[x])\n",
    "                for x in ['train', 'val']}\n",
    "\n",
    "def labels_to_filtered(labels):\n",
    "    \"\"\"Maps CIFAR labels (0,1,2,3,4,5,6,7,8,9) to the index of filtered_labels\"\"\"\n",
    "    return [filtered_labels.index(label) for label in labels]\n",
    "# =================== end CIFAR dataset loading ==========================\n",
    "\n",
    "# Number of samples\n",
    "dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}\n",
    "\n",
    "# Initialize dataloader\n",
    "dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], \n",
    "                  batch_size=batch_size, shuffle=True, num_workers=0) for x in ['train', 'val']}\n",
    "\n",
    "# Function to plot images from tensors\n",
    "def imshow(inp, title=None):\n",
    "    \"\"\"Imshow for Tensor.\"\"\"\n",
    "    inp = inp.numpy().transpose((1, 2, 0))\n",
    "    # We apply the inverse of the initial normalization operation.\n",
    "    mean = np.array([0.485, 0.456, 0.406])\n",
    "    std = np.array([0.229, 0.224, 0.225])\n",
    "    inp = std * inp + mean\n",
    "    inp = np.clip(inp, 0, 1)\n",
    "    plt.imshow(inp)\n",
    "    if title is not None:\n",
    "        plt.title(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "170500096it [00:59, 2886231.09it/s]                               \n"
     ]
    }
   ],
   "source": [
    "# Get a batch of training data\n",
    "X, y_cifar = next(iter(dataloaders['val']))\n",
    "y = torch.tensor(labels_to_filtered(y_cifar))\n",
    "X, y = X.to(device), y.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Slice Models\n",
    "network = model_hybrid\n",
    "layers = list(network.children())\n",
    "hybrid_feats = []\n",
    "\n",
    "for i in range(9):\n",
    "    model_sliced = nn.Sequential(*layers[: i]) #Max: 9\n",
    "    hybrid_feats.append(torch.flatten(model_sliced(X), 1, 3).detach().cpu().numpy())\n",
    "\n",
    "hybrid_feats.append(model_classical_cut(X).detach().cpu().numpy())\n",
    "hybrid_feats.append(model_cut(X).detach().cpu().numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize the model - quantum layer activation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "import umap.umap_ as umap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_layout(layout, min_percentile=1, max_percentile=99, relative_margin=0.1):\n",
    "    \"\"\"Removes outliers and scales layout to between [0,1].\"\"\"\n",
    "\n",
    "    # compute percentiles\n",
    "    mins = np.percentile(layout, min_percentile, axis=(0))\n",
    "    maxs = np.percentile(layout, max_percentile, axis=(0))\n",
    "\n",
    "    # add margins\n",
    "    mins -= relative_margin * (maxs - mins)\n",
    "    maxs += relative_margin * (maxs - mins)\n",
    "\n",
    "    # `clip` broadcasts, `[None]`s added only for readability\n",
    "    clipped = np.clip(layout, mins, maxs)\n",
    "\n",
    "    # embed within [0,1] along both axes\n",
    "    clipped -= clipped.min(axis=0)\n",
    "    clipped /= clipped.max(axis=0)\n",
    "\n",
    "    return clipped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UMAP(angular_rp_forest=True, dens_frac=0.0, dens_lambda=0.0, metric='cosine',\n",
      "     min_dist=0.01, n_neighbors=20, verbose=True)\n",
      "Construct fuzzy simplicial set\n",
      "Fri Feb 19 05:15:28 2021 Finding Nearest Neighbors\n",
      "Fri Feb 19 05:15:28 2021 Finished Nearest Neighbor Search\n",
      "Disconnection_distance = 1 has removed 6670 edges.  This is not a problem as no vertices were disconnected.\n",
      "Fri Feb 19 05:15:28 2021 Construct embedding\n",
      "\tcompleted  0  /  500 epochs\n",
      "\tcompleted  50  /  500 epochs\n",
      "\tcompleted  100  /  500 epochs\n",
      "\tcompleted  150  /  500 epochs\n",
      "\tcompleted  200  /  500 epochs\n",
      "\tcompleted  250  /  500 epochs\n",
      "\tcompleted  300  /  500 epochs\n",
      "\tcompleted  350  /  500 epochs\n",
      "\tcompleted  400  /  500 epochs\n",
      "\tcompleted  450  /  500 epochs\n",
      "Fri Feb 19 05:15:31 2021 Finished embedding\n"
     ]
    }
   ],
   "source": [
    "layout = umap.UMAP(n_components=2, verbose=True, n_neighbors=20, min_dist=0.01, metric=\"cosine\").fit_transform(result)\n",
    "\n",
    "## You can optionally use TSNE as well\n",
    "# layout = TSNE(n_components=2, verbose=True, metric=\"cosine\", learning_rate=10, perplexity=50).fit_transform(d)\n",
    "\n",
    "layout = normalize_layout(layout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlMAAAI/CAYAAABTd1zJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAo00lEQVR4nO3db4xkZ50f+u+ztlnwXWxaGUeysD1GyrSIcSKz0+C9IpqstJPLgBaMlD+C1creyKzfQDRJVkhEuU4i/GqDbqKJ1ncTX4zWXollyb6wvITgaBIia8kycrdwWOyFtpekmXGQbLNtm2QwjMlzX0zXUNOu7q7uU1XnVJ3PR0Ke7iq6HtXp6vrW8/ud3ym11gAAcDA/0/YCAADmmTAFANCAMAUA0IAwBQDQgDAFANCAMAUA0MCVbT3woUOH6s0339zWwwMAjG1tbe3FWut1o25rLUzdfPPNWV1dbevhAQDGVkrZ2Ok2ZT4AgAaEKQCABoQpAIAGhCkAgAaEKQCABoQpAIAGhCkAgAaEKQCABoQpAIAGhCkAgAaEKQCABoQpAIAGhCkAgAaEKQCABoQpAIAGhCkAgAaEKQCABoQpAIAGhCkAgAb2DFOllM+WUp4vpXxzh9tLKeVflVKeLaV8o5Ty85NfJgBAN42zM/U7SU7scvv7khzZ+t89SX67+bIAAObDnmGq1vp4kj/f5S53JHm4XvS1JG8ppVw/qQUCAHTZJHqm3prk7NDX57a+B0AL1jY2c+eDZ7K2sdn2UqAXZtqAXkq5p5SyWkpZfeGFF2b50AALaVRwOnV6PY8/82JOnV5vcWXQH5MIU88luXHo6xu2vvc6tdYHaq0rtdaV6667bgIPDdBvo4LTiVuvz9LVV+XErTouYBYmEaYeTXLn1ll9v5Dk5Vrr9ybwcwHYw8njyzl25FBOHl++9L0vrJ7N5vkL+cLq2V3+n8CkXLnXHUopv5fkF5McKqWcS/JPk1yVJLXWf53kS0nen+TZJOeT/N1pLRaAyx09vJSH77798m/Wevl/ganaM0zVWj+yx+01yccmtiIAGrn3A+/IqdPrl+1WAdOzZ5gCYL6M3K0CpsblZAAAGhCmAAAaEKYAesIwT5gOYQqgJwzzhOkQpgB6YngmlV0qmBxn8wH0xPBZfnc+eCaPP/NikjjzDxqyMwXQQy45A5MjTAH00Je/+b1snr+QL3/T1b+gKWU+gB4aTEc3JR2aE6YAesiUdJgcZT6AnnNmHzQjTAH0nPlT0IwwBdBzw/OnBuxWwfj0TAH03Kj+qcFuVWIOFexFmALgdZztB+MTpgB4HWf7wfj0TAEAu9JDtzthCgDYlTM+d6fMBwDsSg/d7oQpAGBXeuh2p8wHANCAMAUA7Jum9J9S5gMA9mVtYzMffeiJbJ6/kMRgVztTAMDYhoPU0tVXaUqPMAXsYbetfNv8sPi2v85PnV6/FKQ+c9e7cvTwUssrbJ8wBVwyKhztNl/G7BlYfNtf54MLYw8HqbWNzXzot/4oH7r/q738cKVnCrhk1MVtd5svs9NtaxubOXV6/dL3B//2CRbmz8njy3nlhxfyyquvZW1j83VjErb3Tw1e73163duZAi4ZfOIcDkdHDy9d+sO4/RPn4I/q9j+Ww59kJ717pbQI07X9NXb08FKuedNVefLsSzvuUG+ev5A3/+wVue3Gt1z6e9GnXWs7U8Alw584h3eXRu1Y7WbUjtWkmlT3uxZgf5rsUA8+WPVtYnqptbbywCsrK3V1dbWVxwb2dueDZ/L4My9e2qnqypb9cMhrey2LxPPKgN+F0Uopa7XWlZG3CVPAKP6g9stweLbj109e87vbLUzpmQJG2qkfal7orRrf2sZmXvnhhUv9LvRT3/qcJkmYAubOOEHJG8P4Tp1ez5PnXs41b7xybsMzzY06AaVt8/KhSAM6MHfGaULvWwNsE54rkrxu5EEXjPNa70J5UpgC5s44b/5dfGPoKs8VXTXOa70LZ/gKU8Dc8eYPi2+3Hafh27qws6pnCgAWwLz0F41rt77H+774dB5/5sXc98WnO3GyjDAFzK1Fe/OAJhbtpItdG+K3xjr92fM/6MTrX5gC5taivXlAE108G6+J3Xac7v3AO7J09VX5wY9+0onXvzAFc67PuzOL9uYBTXSh3DUrRw8v5TN3vaszr38T0GHOmVwNMH0moMMCszsD/dPnHekuEqZgzvVpax/6ZLfApF+wW4QpAJiQSe4Y7RaY7Eh3i6GdADAhk5zGvdswSoNru0WYAoAJ2R6ABpO6T9x6fb78ze/t6/pxAtP8EKYAYEK2B6DBTtWfPPdyNs9fSPL6HatB4Lrl+mvy+6tn84n3vj2/cvtNM103zQhTADAlgx2q4Z2p7QaB67/82ffz2v+u+fRj3xKm5owwBQBTMrxTtVNAGgSs4Z0p5ouhnQAAezC0EwA6wsDNxSNMAcAMGbi5eIQpmCCfOIG9GLi5eIQpmKC9PnEKW8yC37NucwmoxeNsPpig3SYWJ5Odjgw78XsGsyVMwQTtNbF4r7AF2w0GOu5ncrbfM5gtZT6YklGlFtv77NdBmpV3+z3bqQSoNAgHZ2cKpkSphUmY9C7TTr+Xfl/h4OxMwT6N+gQ/6nvO2KGJwe9UkonuZu70e+n3FQ7OBHTYpzsfPJPHn3kxx44cuvQJftT3oAm/U9Atu01AV+aDfVjb2MwrP7yQ2258y2Wf4DX8Mml+p2B+KPPBPpw6vZ4nz72ca9545WVlF43lTNr23ykN4tBdwhSMYW1jMx+6/6v53suv5rYbrrVbwMwNGsQ/+tATAhV0jDAFYzh1ej1Pnn0pzzz/P3PNm66yA8XMnTy+nKWrr8rm+Quu6dYhdgxJhCnY06BP6shf/Dm7UrTm6OGlfOaudznjrmNctJhEAzrsaDB5+pUfXsiT5152VhWt22vCPrPnRAESO1Owo0tDDEuxG0DrlJO6ycknJHam4HUGO1Inbr0+SfZ1TTSYFhPKobuEKdjGmxZdtFc56SAXRAYmQ5iCbfRA0EV79Uv5EDAbQiujCFOwzW5vWv6Q0lU+BMyG0MoowhTsgz+kdJUz/WZDaGUUYQr2wR9S6DehlVGEKdgHf0gB2M6cKYAFZTYVzIYwBbCgXOoEZkOZD2BB6fGD2RCmABaUHj+YDWU+AIAGhCkAgAaEKQCABoQpAIAGhCkAgAaEKQCABoQpAIAGhCkAgAaEKQCABoQpgDnmYsbQPmEKYI65mDG0b6wwVUo5UUr5dinl2VLKJ0fcflMp5SullK+XUr5RSnn/5JcKwHYnjy/n2JFDLmYMLSq11t3vUMoVSdaT/I0k55I8keQjtdanh+7zQJKv11p/u5RyS5Iv1Vpv3u3nrqys1NXV1YbLBwCYvlLKWq11ZdRt4+xMvTvJs7XW79Raf5zk80nu2HafmuSarX9fm+R/HHSxAADzZJww9dYkZ4e+Prf1vWH/LMmvllLOJflSkr83kdUBTJHmbWASJtWA/pEkv1NrvSHJ+5P8binldT+7lHJPKWW1lLL6wgsvTOihAQ5G8zYwCVeOcZ/nktw49PUNW98bdneSE0lSa/3jUsobkxxK8vzwnWqtDyR5ILnYM3XANQNMxKBpW/M20MQ4O1NPJDlSSnlbKeUNST6c5NFt9/lukl9KklLKX07yxiS2noBOO3p4KQ/ffXuOHl5qeynAHNszTNVaX0vy8SSPJfnTJF+otT5VSvlUKeWDW3f7jSS/Xkr5r0l+L8mv1b1OEwQAWADjlPlSa/1SLjaWD3/vnwz9++kk75ns0gAAus8EdACABoQpAIAGhCkAgAaEKQCABoQpAIAGhCkAgAaEKQCABoQpAC7jAtCwP8IU0GuCw+u5ADTsz1gT0AEW1SA4JMnDd9/e8mq6wQWgYX+EKaDXBIfXG1wAGhiPMh/Qa4PgcPTw0kweT1kRFo8wBTBDXexHEvCgGWU+gBnqYllR3xg0I0wBzFAX+5G6GPBgnijzAfTcbn1jSoCwN2EKgB11sccLukaZD4AdKQHC3oQpAHbUxR4v6BplPgCABoQpAIAGhCkAgAaEKYAR5nEkwDyuGRaBMAUwwmAkwEcfemJuwokxBtAOYQpghJPHl7N09VXZPH9hbsLJyePLOXbkkDEGMGOl1trKA6+srNTV1dVWHhtgHGsbmzl1ej0njy+PnA4O9EcpZa3WujLqNnOmAHZgxhIwDmU+AIAGhCkAgAaEKQCABoQpgJ4whwqmQ5gC6AlzqGA6nM0H0BOD+VPmUMFkCVMAPWHUA0yHMh8AQAPCFABAA8IUAEADwhQAQAPCFABAA8IUAEADwhQAQAPCFABAA8IUAEADwhQAQAPCFABAA8IUAEADwhQAQAPCFMABrW1s5s4Hz2RtY7OTPw+YDWEK4IBOnV7P48+8mFOn1zv58xIBDWbhyrYXADCvTh5fvuy/Xft5yU8DWpI8fPftE/u5wE+VWmsrD7yyslJXV1dbeWyAvljb2Myp0+s5eXw5Rw8vtb0cmFullLVa68qo25T5AGagrXLb0cNLefju2wUpmCJhCmAGptEPBXSDnimAGZhGPxTQDXamAPbpICU75TZYXMIUwD4p2QHDhCmAMQzvRp08vpxjRw61WrIzPwq6Q88UwBi2z2tqe2aT+VHQHcIUwBi61kDetfVAnxnaCQCwB0M7AQCmRJgCAGhAmAIAaECYAgBoQJgCGMEcJ2BcwhTACKacA+MSpgC2WdvYzCuvvpbbbrjWHCdgT8IUwDanTq/nybMv5Zo3XTU3FyZWloT2mIAOsM08Thd3eRlojzAFsM3Rw0tzF0jmMQDCohCmABbAPAZAWBR6pgAAGhCmAAAaEKYAABoQpgAAGhCmAAAaEKYAABoQpgAAGhCmAAAaEKYAABoQpgAAGhCmAAAaEKYAABoQpgAAGhCmAAAaEKYAABoQpgA6bm1jM3c+eCZrG5ttLwUYQZgC6LhTp9fz+DMv5tTp9baXAoxwZdsLAGB3J48vX/ZfoFuEKYCOO3p4KQ/ffXvbywB2MFaZr5RyopTy7VLKs6WUT+5wn79TSnm6lPJUKeVzk10mAEA37bkzVUq5Isn9Sf5GknNJniilPFprfXroPkeS/KMk76m1bpZS/uK0FgwA0CXj7Ey9O8mztdbv1Fp/nOTzSe7Ydp9fT3J/rXUzSWqtz092mQAA3TROmHprkrNDX5/b+t6w5STLpZSvllK+Vko5MakFAvSd0QjQbZNqQL8yyZEkv5jkhiSPl1L+Sq31peE7lVLuSXJPktx0000TemiAxTYYjZBEIzp00Dhh6rkkNw59fcPW94adS3Km1nohyX8rpaznYrh6YvhOtdYHkjyQJCsrK/WgiwboE6MRoNvGKfM9keRIKeVtpZQ3JPlwkke33eeRXNyVSinlUC6W/b4zuWUC9NdgNMLRw0ttLwUYYc8wVWt9LcnHkzyW5E+TfKHW+lQp5VOllA9u3e2xJN8vpTyd5CtJPlFr/f60Fg0A0BWl1naqbSsrK3V1dbWVxwaYZ2sbmzl1ej0njy/brYIZKaWs1VpXRt3m2nwAc8a1+qBbXE4GYM5oSIdusTMFMGe2N6SbQwXtEqYA5pyyH7RLmAKYY2sbm3nl1ddy2w3XKvtBS4QpgDl26vR6njz7Uq5501XO7IOWaEAHmGOa0aF9whTAHBs0owPtUeYDAGhAmAIAaECYAgBoQJgCAGhAmAIAaECYolcmfdkNl/EAQJiiVyZ92Y3Bz/voQ08IVAA9JUzRKyePL+fYkUMTG3B48vhylq6+KpvnL7guGkBPCVPMvf2U2gYDDid12Y2jh5fymbveNdGABsB8EaaYe5Mu3Y1rEOKSTDSgATBfXE6GudfWtckGIS6Jy3kA9Jgwxdxr69pkXbnA7NrGZk6dXs/J48t2xwBaoMzHSF055b8r6xhl0v1XB9VWmXNWuvw7AJAIU+zgIG/Q03jTu+8Pn8rjz7yY+/7wqYn9zEUz6TMUu2bRwyIw/5T5GOkgJayp9BCVcvl/eZ2dypyLUv7rSjkVYCfCFCMdpA9pGm969/7yLZcCAfuzKA3ybfXEAYyr1FpbeeCVlZW6urraymNDHyzKzhRAF5RS1mqtK6NuszMFC2r7js7axubF3rNScu8v3yJgAUyIBnToiVOn1/PkuZfz5NmXNHMDTJAwxVxwenxzJ48v57Ybrs1tN75FDxrABAlTzEVQafP0+Hl4fsZx9PBSHvn4X8sjH3uPEh/ABAlTzMUcnzZnKc3D89PEfsLiXvddlOAJsB8a0Hto+1le8zDHp83T4+fh+WliPyMU9rrvooxjANgPYaqHBm94f/Lcy/nMXe8yx2cPi/787Ccs7nXfRQ+eAKOYM9VDaxub+ehDT2Tz/IUcO3JooYMCAEzCbnOm9Ez10NHDS/nMXe9a6Ou5jaKfB4BpEKZ6alC66tNZXcON5ILVwXjeAF5PzxS9MdzPo1H6YDxvAK8nTNEbw43kGqUPxvMG8Hoa0IGZcgFmYB5pQKf39Pp0x6IPQQX6R5mPhTc8CiLR69M2pUJg0QhTLLxTp9ezef5Clq6+yht4Byz6EFSgf5T5WHiD6/p94r1vvzQWgdlSZgUWmTDFwhvshHz5m9/Tq9OSQZ/URx96QqACFo4wRW8MdqiU+mbv5PHlLF19VTbPXxBmgYUjTM055ZPx9XHqe1f09RJGQD9oQJ9zJlIzLzSeA4vKztScU7piXg3vqtphBeaZnak559M+82p4VzWJHVZgbglTQCtGDe+0wwrMI9fmAwDYg2vzsZD02QDQBcIUc8sFcwHoAmFqjtiJuZwzGQHoAg3oc8RMqcs5kxGALhCm5sios58AgHYJU3PETgwAdI+eKQCABoQpAIAGhCkAgAaEKQCABoQpAIAGhCkAgAaEKQCABoQpAIAGhKmecF0/AJgOYaonBtf1O3V6ve2lAMBCcTmZnnBdPwCYDjtTLWij5Da4rt/Rw0sze0wA6ANhqgVKbgCwOJT5WqDkBgCLQ5hqwaDkBgDMP2U+AIAGhCkAgAaEqQ4yYBMA5oeeqQ5Z29jMqdPreeXV1/Lk2ZeSRG8VAHScnakOGYxMSK05duSQs/0AYA7YmeqQ4ZEJhmsCwHywM7Xlc2e+m3d+6j/kc2e+O/GfPW4PlCnl+6O3DIAuEKa2fPqxb2Xz/IV8+rFvTfxnm3g+HZ5XALpAmW/LJ9779nz6sW/lE+99+8R/tonn+7e2sZn7vvh0Umvu/cA7Ru7WeV4B6IJSa23lgVdWVurq6morj0333fngmYvN+EmOHTnU6KzGwVmSetEAOKhSylqtdWXUbb0t8+m36baTx5dz241vyW03XNt450k5EIBp6m2Zb/AG+yfPvZzP3PUuOxYdc/TwUh752Hsm8rOUAwGYpt7uTJ08vpylq6/K5vkLOXV63U7VAnOWJADT1NswdfTwUj5z17suDcdUCpofgi8AXdLbMl/y0x2LRClonlyaFJ9cCsKaywFoS6/D1LDhYEW3DQff4WDl+AHQBmGKuWNHEYAuEaaYa3YUAWjbWA3opZQTpZRvl1KeLaV8cpf7/c1SSi2ljBxqBQCwaPYMU6WUK5Lcn+R9SW5J8pFSyi0j7vfmJCeTnJn0IgEAumqcnal3J3m21vqdWuuPk3w+yR0j7ndfkt9M8uoE1wcA0GnjhKm3Jjk79PW5re9dUkr5+SQ31lr/3QTXBgDQeY2HdpZSfibJv0jyG2Pc955SymopZfWFF15o+tAAAK0bJ0w9l+TGoa9v2PrewJuT3JrkP5dS/nuSX0jy6Kgm9FrrA7XWlVrrynXXXXfwVcMBmJwOwDSME6aeSHKklPK2Usobknw4yaODG2utL9daD9Vab6613pzka0k+WGtdncqK4YBcMgiAadhzzlSt9bVSyseTPJbkiiSfrbU+VUr5VJLVWuuju/8E6AYDPgGYhlJrbeWBV1ZW6uqqzSsAoPtKKWu11pFzNBs3oAMA9JkwNQaNywDAToSpMWhcBgB24kLHY9C4DADsRJgaw9HDS3n47tvbXgYA0EHKfAAADQhTAAANCFMAAA0IU7swEgEA2IswtQsjEQCAvTibbxdGIgAAe+ndztRw6W6vMt5gJMLRw0szXiUAMC96tzM1KN0NDP5tjhQAcBC9C1OjSnfKeADAQZVaaysPvLKyUldXV1t5bACA/SilrNVaV0bd1rueqZ0YgwAAHEQvw9So4GQMAgBwEL3rmUoub0IfNJ4bgwAAHEQvw9So4DQYgwAAsB+9DFOCEwAwKb3smQIAmBRhCgCgAWEKAKCBXoQpM6QAgGnpRZgyQwoAmJZenM1nhhQAMC29CFNGIQAA09KLMh8AwLQIUwAADfQuTDmzDwCYpN6FKWf2AQCT1IsG9GHO7AMAJql3YcqZfQDAJPWuzAcAMEnCFABAA8IUAEADwhQAQAPCFABAA8IUAEADwhQAQAPCFABAA8IUAEADwhQAQAPCFABAA8IUAEADCxum1jY2c+eDZ7K2sdn2UgCABbawYerU6fU8/syLOXV6ve2lAAAL7Mq2FzAtJ48vX/ZfAIBpWNidqaOHl/Lw3bfn6OGlsf8/SoMAwH4tbJg6CKVBAGC/FrbMdxBKgwDAfglTQwalQQCAcSnzAQA0IEwBADTQqzDlbD0AYNJ6FaacrQcATFqvGtCdrQcATFqvwpSz9QCASetVmQ8AYNKEKQCABoQpAIAGhCkAgAaEKdhiDhkAByFMwRZzyAA4iF6NRoDdmEMGwEH0cmdKOYdRBnPIjh5eanspAMyRXoap4XKOYAUANNHLMt9wOee+P3wqT557Oa/88EIe+fhfa3llAMC86eXO1GXlnFKSJP/rxz+xQwUdY+cYmAe9DFPD7v3lW3LsyKH8H2+4wplc0DHOsATmQS/LfMMGu1RrG5s5dXrdmVzQIc6wBOZBqbW28sArKyt1dXW1lccG+mX4w5KzNYGDKKWs1VpXRt3W+zIfs6P/hbYoFwLT1PsyH7MzeENLkofvvr3l1dAnyoXANAlTzMw8v6EpE823QW8kwDQIU8zMPL+h2VUDYCd6pmAMJ48v59iRQ/veVVvb2MyH7v9qPvRbf7TvXjE9ZvvnOQPaIEzBGPa6bt9Ob+KnTq/nybMv5clzL++7+VnT9P55zoA2KPPBBOxUBjx5fDmvvPpaUuu+d7XmucesLZ4zoA3CFEzA4M37xK3X584Hz1xqVD96eCmPfOw9B/qZ89xjBtAnynwwAYPg8+Vvfm+qZSY9QbvbqczneQOmSZiiMW9UP3XQRvVx6Qna3U7Pv+cNmCZlPhozNuCnpl2aW7SeoEnP79rp+V+05w3oFmGKxrxRzU5X+6gOGopmFcS7+rwBi0GZj8aOHl7KyePLOXV6Xamvp/ZbRhuUhk/cev1Uy6IAs2BniolQ6uu3/e5OTur3xWV+gC4QppiIwTylV354IWsbm97Yema/ZbRJlYaFeKALlPmYiKOHl3LNG6880KRv+mevifLjmvbZkwDjsDPFxGhEny+LUCLTWA50gTDFxHhjmy9KZACTMVaZr5RyopTy7VLKs6WUT464/R+WUp4upXyjlPIfSymHJ79UmJ0+DCJVIgOYjD3DVCnliiT3J3lfkluSfKSUcsu2u309yUqt9a8m+YMk/3zSC4WDOkgw6sPE7En1LfUheALsZpydqXcnebbW+p1a64+TfD7JHcN3qLV+pdZ6fuvLryW5YbLLhIM7SDCyazO+PgRPgN2M0zP11iRnh74+l2S3Bou7k/z7JotisbTd6HyQxnj9X+Nz4gHQdxNtQC+l/GqSlSR/fYfb70lyT5LcdNNNk3xoOqztRmfBaLo8v0DfjVPmey7JjUNf37D1vcuUUo4n+cdJPlhr/dGoH1RrfaDWulJrXbnuuusOsl7m0KRKZnpzAOiicXamnkhypJTytlwMUR9O8ivDdyilvDPJv0lyotb6/MRXyVyb1M5F2ztcADDKnjtTtdbXknw8yWNJ/jTJF2qtT5VSPlVK+eDW3T6d5OeS/NtSypOllEentmJ6S1M4XWGXFBhWaq2tPPDKykpdXV1t5bGZjrYbzWFW7nzwTB5/5sUcO3LILin0RCllrda6Muo2E9CZGGU4+sIZjMAwYYqJ8QZDXziDERgmTDEx3mAA6KOxrs0HbdLsC0CXCVN0nsuVANBlwhSdZyTC/gzv5NnVA5g+PVNMXdORCXqx9mf4rMokzrAEmDJhiqkzMmG2Rp1VaVcPYHqEKabOyITZ2r6TJ8DOjsG10E/CFFOnTEdf2IWFfhKmACbELiz0k7P56AxnnjHvBruwSnzQL8IUnWGeFADzSJmPzlAiAWAe2ZmiM5RIWGSjythK27AYhCmAGRhVxlbahsWgzAcwA7sNU1XahvlWaq2tPPDKykpdXV1t5bFpn+GGAMyTUsparXVl1G3KfLRCeQNG00cF80eYohUnjy/n2JFDyhvMnWmHHR80YP7omaIVLjHDvJr2JWP0UcH8EaYA9mHaYccHDZg/ynxAp3S9Z8g8NGA7YQroFD1DwLwRpoBO6cvJCV3fgRs2T2uFNuiZAjqlLz1D025kn6R5Wiu0wc4UB9bmp1WflJl3XdmBG+e11JW1QlcJUxxYm70t+moORgjtjv02sk/r2G1/La1tbOZD9381H/qtP7r0WJruYXfCFAfW5qdVn5QPRgidX/s5dvsJXttfS6dOr+fJsy/lyXMv56MPPSF4wxhcmw96xDUR59e4x25tYzO/9tkz+cGPfpLbbnxLHvnYe/b9/7/vi0/nz57/QX7wo5/k2JFD+qQgu1+bTwM69EhfmrsX0bjH7tTp9fzgRz+5+MXQh+Vxm8iPHl7KIx97z2XhazcCOghTAAvl5PHlvPLDC0kpufeXb7ns+8P/3ct+wpsz/eg7ZT4Wik/JMJ5JvVbWNjZz3x8+dSm8ed2xqHYr82lAZ6FosJ4/zjBsx6ReK0cPL+WaN12VJ8++5HVHbynzsVCmfRFaJk+ZqB2TfK143dF3ynx0gvJcv2xvbnbsga5zNh+dZ3eiX7Yfb8ccmGfCFJ2gTNAvjjewSJT5AAD24Gw+AIApEaagB4wfAJgeYQp6oOlMIWEMYGca0KEHmjZ8O9sSYGd2pqAHBtdZO+gcp5PHl3PbDdfmlVdfG2t3yk4W0CfCFK3zxtt9+71kiMv6AH2izEfrlJDmw35KheZIAX1izhStcykZALrO5WTotEE/DwDMIz1TAAANCFPMpWk1rWuGB2C/hCnm0rTOFnMWGgD7JUzRSXvtEJ08vpxjRw5N/Gyxaf3ceWSXDmA8GtDppL3GJUyraV0z/E8ZWQEwHmGKTjKnqH2OAcB4zJmCKTNHC2D+7TZnSs8UNDBOX5GmdoDFpswHDYzTV6RcBrDYhCloYJygpKkdYLEJU9CAoASAnikAgAaEKQCABoQpAIAGhCkAgAaEKeZOF64Z14U1ANANwhRzpwtDMLuwBgC6wWgE5s5gptOJW6/PnQ+eaeUyLQZxAjDg2nzMrTsfPJPHn3kxx44cMusJgKlybT4W0snjyzl25NBUdof0RAEwLmU+5tY0p4+Pc809AEiEKRhJTxQA4xKmYATX3ANgXHqmAAAaEKYAABoQpgAAGhCmAAAaEKYAABoQpgAAGhCmAAAaEKYAABoQpgAAGhCmAAAaEKYAABoQpmBOrW1s5s4Hz2RtY7PtpQD0mjAFmc9gcur0eh5/5sWcOr3e9lIAek2Yghw8mEw6hH3uzHfzzk/9h3zuzHf3vO/J48s5duRQTh5fnshjA3AwV7a9AOiCQSDZbzAZhLAkefju2xuv49OPfSub5y/k0499K79y+0273vfo4aWJPCYAzQhTkIMHk4OGsJ184r1vz6cf+1Y+8d63T+TnATB9pdbaygOvrKzU1dXVVh4bAGA/SilrtdaVUbfpmYIdzGNTOgCzJ0zBDu774tN5/JkXc98Xn257KQB02FhhqpRyopTy7VLKs6WUT464/WdLKb+/dfuZUsrNE18pzNqgBN5SKRyA+bBnmCqlXJHk/iTvS3JLko+UUm7Zdre7k2zWWv9Skn+Z5DcnvVCYtXs/8I4cO3Io937gHW0vBYAOG2dn6t1Jnq21fqfW+uMkn09yx7b73JHkoa1//0GSXyqllMktE2ZvcIbf0cNLSfbuodJjBdBP44SptyY5O/T1ua3vjbxPrfW1JC8n+QuTWCB0xW6DPdc2NvPRh54wkRygh2Y6Z6qUck+Se5Lkppt2H0gIXbPbTKlTp9ezef5Clq6+ykRygJ4ZJ0w9l+TGoa9v2PreqPucK6VcmeTaJN/f/oNqrQ8keSC5OGfqIAuGtuw22HM4aA3KggD0wzhh6okkR0opb8vF0PThJL+y7T6PJrkryR8n+VtJ/lNtaxootMClXQD6a88wVWt9rZTy8SSPJbkiyWdrrU+VUj6VZLXW+miSB5P8binl2SR/nouBCwBg4Y3VM1Vr/VKSL2373j8Z+verSf72ZJcGANB9JqADADQgTAEANCBMAQA0IEwBADQgTAEANCBMAQA0IEwBADQgTAEANCBMAQA0IEwBADQgTAEANCBMAQA0IEwBADQgTAEANCBMAQA0IEwBADQgTAEANCBMAQA0IEwBADQgTAEANFBqre08cCkvJNmY8sMcSvLilB+D/XNcuscx6SbHpXsck26axXE5XGu9btQNrYWpWSilrNZaV9peB5dzXLrHMekmx6V7HJNuavu4KPMBADQgTAEANLDoYeqBthfASI5L9zgm3eS4dI9j0k2tHpeF7pkCAJi2Rd+ZAgCYqoUIU6WUE6WUb5dSni2lfHLE7T9bSvn9rdvPlFJubmGZvTPGcfmHpZSnSynfKKX8x1LK4TbW2Sd7HZOh+/3NUkotpThracrGOSallL+z9Vp5qpTyuVmvsY/G+Pt1UynlK6WUr2/9DXt/G+vsk1LKZ0spz5dSvrnD7aWU8q+2jtk3Sik/P6u1zX2YKqVckeT+JO9LckuSj5RSbtl2t7uTbNZa/1KSf5nkN2e7yv4Z87h8PclKrfWvJvmDJP98tqvslzGPSUopb05yMsmZ2a6wf8Y5JqWUI0n+UZL31FrfkeTvz3qdfTPma+X/TvKFWus7k3w4yf8721X20u8kObHL7e9LcmTrf/ck+e0ZrCnJAoSpJO9O8myt9Tu11h8n+XySO7bd544kD239+w+S/FIppcxwjX2053GptX6l1np+68uvJblhxmvsm3FeK0lyXy5+4Hh1lovrqXGOya8nub/WupkktdbnZ7zGPhrnuNQk12z9+9ok/2OG6+ulWuvjSf58l7vckeThetHXkryllHL9LNa2CGHqrUnODn19but7I+9Ta30tyctJ/sJMVtdf4xyXYXcn+fdTXRF7HpOtbfEba63/bpYL67FxXifLSZZLKV8tpXytlLLbJ3MmY5zj8s+S/Gop5VySLyX5e7NZGrvY7/vOxFw5iweB3ZRSfjXJSpK/3vZa+qyU8jNJ/kWSX2t5KVzuylwsW/xiLu7ePl5K+Su11pfaXBT5SJLfqbX+P6WU/zPJ75ZSbq21/u+2F8bsLcLO1HNJbhz6+oat7428Tynlylzckv3+TFbXX+Mcl5RSjif5x0k+WGv90YzW1ld7HZM3J7k1yX8upfz3JL+Q5FFN6FM1zuvkXJJHa60Xaq3/Lcl6LoYrpmec43J3ki8kSa31j5O8MRevD0d7xnrfmYZFCFNPJDlSSnlbKeUNudgI+Oi2+zya5K6tf/+tJP+pGrA1bXsel1LKO5P8m1wMUvpApm/XY1JrfbnWeqjWenOt9eZc7GP7YK11tZ3l9sI4f78eycVdqZRSDuVi2e87M1xjH41zXL6b5JeSpJTyl3MxTL0w01Wy3aNJ7tw6q+8Xkrxca/3eLB547st8tdbXSikfT/JYkiuSfLbW+lQp5VNJVmutjyZ5MBe3YJ/Nxea1D7e34n4Y87h8OsnPJfm3W+cDfLfW+sHWFr3gxjwmzNCYx+SxJP9XKeXpJD9J8olaq531KRrzuPxGkv+vlPIPcrEZ/dd8SJ+uUsrv5eIHi0NbvWr/NMlVSVJr/de52Lv2/iTPJjmf5O/ObG2OPQDAwS1CmQ8AoDXCFABAA8IUAEADwhQAQAPCFABAA8IUAEADwhQAQAPCFABAA/8/nzTCGvaENscAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10, 10))\n",
    "plt.scatter(x=layout[:,0],y=layout[:,1], s=2)\n",
    "plt.savefig('quantum_activation_result.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def whiten(full_activations):\n",
    "    correl = np.matmul(full_activations.T, full_activations) / len(full_activations)\n",
    "    correl = correl.astype(\"float32\")\n",
    "    S = np.linalg.inv(correl)\n",
    "    S = S.astype(\"float32\")\n",
    "    return S"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "S = whiten(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQcAAAD8CAYAAAB6iWHJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAANEElEQVR4nO3df6jd9X3H8edr8dapKfVXIEEz7VBkpdv8ETKLMEQrqBQdzKL+YbUkZJS62rFC2w0c6z+z+6OFzlIJKtNSWot2XVYcJUNLWzadaYhW42wzYZhUZqJtrJharrz3x/nqbm8/MTbne77n3HufDzjc749Pzvtzkssr53y/3/N9p6qQpMV+a9oTkDSbDAdJTYaDpCbDQVKT4SCpyXCQ1DRWOCQ5Ocn2JD/ufp50mHGvJ9nVPbaNU1PSMDLOdQ5J/h54qapuS/Ip4KSq+mRj3CtVtXqMeUoa2Ljh8AxwcVU9n2Qd8J2qOqcxznCQlphxw+FnVXVitxzgp2+sLxo3D+wC5oHbquqbh3m+LcAWgLm5uQtOPvnko57brBr9NS1P8/Pz057CRCznf7P9+/cfqKo1rX3HHOkPJ/k3YG1j118vXKmqSnK4pDmjqvYl+V3goSQ/rKr/XjyoqrYCWwHWrl1bN9xww5Gmt+SsWrVq2lOYmAMHDkx7ChMxNzc37SlMzB133PE/h9t3xHCoqvcfbl+S/02ybsHHihcO8xz7up/PJvkOcB7wa+EgaXaMeypzG3Bjt3wj8M+LByQ5Kcmx3fKpwEXA7jHrSpqwccPhNuCyJD8G3t+tk2RDkju7Mb8H7EjyOPAwo2MOhoM04474seKtVNWLwKWN7TuAzd3yvwO/P04dScPzCklJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkpl7CIcnlSZ5JsqfrfLV4/7FJ7uv2P5rkzD7qSpqcscMhySrgi8AVwHuA65O8Z9GwTYwa3pwFfB747Lh1JU1WH+8cNgJ7qurZqvol8DXg6kVjrgbu6ZbvBy7Ncm4jJC0DfYTDacBzC9b3dtuaY6pqHjgInNJDbUkTMlMHJJNsSbIjyY5XX3112tORVrQ+wmEfsH7B+undtuaYJMcA7wJeXPxEVbW1qjZU1Ybjjz++h6lJOlp9hMNjwNlJ3p3kHcB1jNrkLbSwbd41wEM1TntvSRM3VscrGB1DSHIz8G1gFXB3VT2V5DPAjqraBtwFfDnJHuAlRgEiaYaNHQ4AVfUg8OCibbcuWP4F8ME+akkaxkwdkJQ0OwwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDUZDpKaDAdJTYaDpKahemXelGR/kl3dY3MfdSVNztg3mF3QK/MyRt2uHkuyrap2Lxp6X1XdPG49ScPo4+7Tb/bKBEjyRq/MxeHwG0nCqlWrepjebLn22munPYWJOf/886c9hYnYtGnTtKcwFUP1ygT40yRPJLk/yfrGftvhSTNkqAOS/wKcWVV/AGzn/ztu/wrb4UmzY5BemVX1YlW91q3eCVzQQ11JEzRIr8wk6xasXgU83UNdSRM0VK/MjyW5Cphn1CvzpnHrSpqsoXplfhr4dB+1JA3DKyQlNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6Smvpqh3d3kheSPHmY/Unyha5d3hNJlmf3E2kZ6eudwz8Cl7/F/iuAs7vHFuBLPdWVNCG9hENVfZfRXaUP52rg3hp5BDhx0e3qJc2YoY45vK2WebbDk2bHTB2QtB2eNDuGCocjtsyTNFuGCodtwIe6sxYXAger6vmBaks6Cr10vEryVeBi4NQke4G/AeYAquoORt2wrgT2AK8CH+6jrqTJ6asd3vVH2F/AR/uoJWkYM3VAUtLsMBwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1DdUO7+IkB5Ps6h639lFX0uT0cg9JRu3wbgfufYsx36uqD/RUT9KEDdUOT9IS09c7h7fjfUkeB34CfKKqnlo8IMkWRo12OeGEEzhw4MCA0xvG+ecv3wbjo5uMLz+bN2+e9hSmYqhw2AmcUVWvJLkS+Cajjtu/oqq2AlsB1qxZszx/06QlYpCzFVX1clW90i0/CMwlOXWI2pKOziDhkGRtknTLG7u6Lw5RW9LRGaod3jXAR5LMA4eA62q5fkCVlomh2uHdzuhUp6QlwiskJTUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkprGDock65M8nGR3kqeS3NIYkyRfSLInyRNJlm/zBmmZ6OMekvPAX1bVziTvBH6QZHtV7V4w5gpGfSrOBv4I+FL3U9KMGvudQ1U9X1U7u+WfA08Dpy0adjVwb408ApyYZN24tSVNTq/HHJKcCZwHPLpo12nAcwvW9/LrAUKSLUl2JNlx6NChPqcm6TfUWzgkWQ08AHy8ql4+mueoqq1VtaGqNhx33HF9TU3SUeglHJLMMQqGr1TVNxpD9gHrF6yf3m2TNKP6OFsR4C7g6ar63GGGbQM+1J21uBA4WFXPj1tb0uT0cbbiIuAG4IdJdnXb/gr4HXizHd6DwJXAHuBV4MM91JU0QWOHQ1V9H8gRxhTw0XFrSRqOV0hKajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNQ3VDu/iJAeT7Ooet45bV9JkDdUOD+B7VfWBHupJGsBQ7fAkLTF9vHN401u0wwN4X5LHgZ8An6iqpxp/fguwBWD16tXMzc31Ob2ZsGnTpmlPYWI2b9487SlMxFlnnTXtKUzFUO3wdgJnVNUfAv8AfLP1HLbDk2bHIO3wqurlqnqlW34QmEtyah+1JU3GIO3wkqztxpFkY1f3xXFrS5qcodrhXQN8JMk8cAi4ruuCJWlGDdUO73bg9nFrSRqOV0hKajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNfVxg9nfTvKfSR7v2uH9bWPMsUnuS7InyaNdfwtJM6yPdw6vAZd0PSnOBS5PcuGiMZuAn1bVWcDngc/2UFfSBPXRDq/e6EkBzHWPxXeWvhq4p1u+H7j0jVvVS5pNfTW1WdXdlv4FYHtVLW6HdxrwHEBVzQMHgVP6qC1pMnoJh6p6varOBU4HNiZ579E8T5ItSXYk2XHo0KE+pibpKPV6tqKqfgY8DFy+aNc+YD1AkmOAd9HoeGWvTGl29HG2Yk2SE7vl44DLgP9aNGwbcGO3fA3wkB2vpNnWRzu8dcA9SVYxCpuvV9W3knwG2FFV2xj10vxykj3AS8B1PdSVNEF9tMN7Ajivsf3WBcu/AD44bi1Jw/EKSUlNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FS01C9Mm9Ksj/Jru6xedy6kiarj7tPv9Er85Ukc8D3k/xrVT2yaNx9VXVzD/UkDaCPu08XcKRemZKWmPTRW6brWfED4Czgi1X1yUX7bwL+DtgP/Aj4i6p6rvE8W4At3eo5wDNjT+7tOxU4MGC9ofi6lp4hX9sZVbWmtaOXcHjzyUadr/4J+POqenLB9lOAV6rqtSR/BlxbVZf0VrgHSXZU1YZpz6Nvvq6lZ1Ze2yC9Mqvqxap6rVu9E7igz7qS+jdIr8wk6xasXgU8PW5dSZM1VK/MjyW5Cphn1Cvzph7q9m3rtCcwIb6upWcmXluvxxwkLR9eISmpyXCQ1LTiwyHJ5UmeSbInyaemPZ++JLk7yQtJnjzy6KUjyfokDyfZ3V2uf8u059SHt/M1hMHntJKPOXQHUX/E6AzLXuAx4Pqq2j3VifUgyR8zunL13qp677Tn05fuzNe6qtqZ5J2MLr77k6X+b5YkwAkLv4YA3NL4GsJgVvo7h43Anqp6tqp+CXwNuHrKc+pFVX2X0ZmhZaWqnq+qnd3yzxmdFj9turMaX43M1NcQVno4nAYsvIx7L8vgF22lSHImcB7w6JSn0oskq5LsAl4AtlfVVF/XSg8HLVFJVgMPAB+vqpenPZ8+VNXrVXUucDqwMclUPw6u9HDYB6xfsH56t00zrPtM/gDwlar6xrTn07fDfQ1haCs9HB4Dzk7y7iTvAK4Dtk15TnoL3YG7u4Cnq+pz055PX97O1xCGtqLDoarmgZuBbzM6sPX1qnpqurPqR5KvAv8BnJNkb5JN055TTy4CbgAuWXBnsSunPakerAMeTvIEo/+0tlfVt6Y5oRV9KlPS4a3odw6SDs9wkNRkOEhqMhwkNRkOkpoMB0lNhoOkpv8DoYUUVsQj3D0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(S, cmap='gray')\n",
    "plt.savefig('quantum_activation_whitening.png')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}