collab_xnn_features_svhn.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "av22_gwc_hOd"
      },
      "source": [
        "# Setup and install for SVHN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l247Lv_MYR8O",
        "outputId": "97e22eb0-748e-4d11-c8ec-52acfe92c698"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bxt_WMJPvnIc",
        "outputId": "b4fc8b64-227e-42e1-82f7-185589010008"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: progressbar in /usr/local/lib/python3.7/dist-packages (2.5)\n"
          ]
        }
      ],
      "source": [
        "%pip install progressbar"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yqquQY0B65AH",
        "outputId": "01ceaaa9-2708-4ac6-9283-3003f33fdffd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: gpflow in /usr/local/lib/python3.7/dist-packages (2.3.0)\n",
            "Requirement already satisfied: tensorflow>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from gpflow) (2.7.0)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.7/dist-packages (from gpflow) (0.6)\n",
            "Requirement already satisfied: tensorflow-probability>0.10.0 in /usr/local/lib/python3.7/dist-packages (from gpflow) (0.15.0)\n",
            "Requirement already satisfied: deprecated in /usr/local/lib/python3.7/dist-packages (from gpflow) (1.2.13)\n",
            "Requirement already satisfied: multipledispatch>=0.6 in /usr/local/lib/python3.7/dist-packages (from gpflow) (0.6.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from gpflow) (3.10.0.2)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from gpflow) (57.4.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from gpflow) (21.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from gpflow) (1.19.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from gpflow) (1.4.1)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.7/dist-packages (from gpflow) (0.8.9)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from multipledispatch>=0.6->gpflow) (1.15.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (1.6.3)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (0.2.0)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (3.17.3)\n",
            "Requirement already satisfied: tensorboard~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (2.7.0)\n",
            "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (1.13.3)\n",
            "Requirement already satisfied: flatbuffers<3.0,>=1.12 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (2.0)\n",
            "Requirement already satisfied: absl-py>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (0.12.0)\n",
            "Requirement already satisfied: tensorflow-estimator<2.8,~=2.7.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (2.7.0)\n",
            "Requirement already satisfied: gast<0.5.0,>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (0.4.0)\n",
            "Requirement already satisfied: libclang>=9.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (12.0.0)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (0.23.1)\n",
            "Requirement already satisfied: keras<2.8,>=2.7.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (2.7.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (1.1.0)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (3.1.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (1.43.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (1.1.2)\n",
            "Requirement already satisfied: wheel<1.0,>=0.32.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (0.37.1)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->gpflow) (3.3.0)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow>=2.2.0->gpflow) (1.5.2)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (2.23.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (0.4.6)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (1.0.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (1.8.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (3.3.6)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (1.35.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (0.6.1)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (4.2.4)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (4.8)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (4.10.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (3.7.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (0.4.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (1.24.3)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow>=2.2.0->gpflow) (3.1.1)\n",
            "Requirement already satisfied: cloudpickle>=1.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow-probability>0.10.0->gpflow) (1.3.0)\n",
            "Requirement already satisfied: dm-tree in /usr/local/lib/python3.7/dist-packages (from tensorflow-probability>0.10.0->gpflow) (0.1.6)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from tensorflow-probability>0.10.0->gpflow) (4.4.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->gpflow) (3.0.6)\n",
            "Requirement already satisfied: plotnine in /usr/local/lib/python3.7/dist-packages (0.6.0)\n",
            "Requirement already satisfied: numpy>=1.16.0 in /usr/local/lib/python3.7/dist-packages (from plotnine) (1.19.5)\n",
            "Requirement already satisfied: pandas>=0.25.0 in /usr/local/lib/python3.7/dist-packages (from plotnine) (1.1.5)\n",
            "Requirement already satisfied: patsy>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from plotnine) (0.5.2)\n",
            "Requirement already satisfied: matplotlib>=3.1.1 in /usr/local/lib/python3.7/dist-packages (from plotnine) (3.2.2)\n",
            "Requirement already satisfied: mizani>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from plotnine) (0.6.0)\n",
            "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from plotnine) (1.4.1)\n",
            "Requirement already satisfied: statsmodels>=0.9.0 in /usr/local/lib/python3.7/dist-packages (from plotnine) (0.10.2)\n",
            "Requirement already satisfied: descartes>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from plotnine) (1.1.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.1.1->plotnine) (2.8.2)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.1.1->plotnine) (0.11.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.1.1->plotnine) (1.3.2)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.1.1->plotnine) (3.0.6)\n",
            "Requirement already satisfied: palettable in /usr/local/lib/python3.7/dist-packages (from mizani>=0.6.0->plotnine) (3.3.0)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.25.0->plotnine) (2018.9)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from patsy>=0.4.1->plotnine) (1.15.0)\n"
          ]
        }
      ],
      "source": [
        "%pip install gpflow\n",
        "%pip install plotnine"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EfIU_eNp3Zio"
      },
      "outputs": [],
      "source": [
        "from plotnine import *\n",
        "from plotnine.themes import *"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZmUjYbArAuQT"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "from scipy.io import loadmat\n",
        "import random\n",
        "import math\n",
        "import tensorflow_probability as tfp"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PieVKPfHHYQ6"
      },
      "source": [
        "_paper_name_ establishes the reusable name of the paper, it represents the directory under data_papers on the google drive"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BI4p7ZKb0Qz2"
      },
      "outputs": [],
      "source": [
        "paper_name = \"gpSVHN\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "433z6V3T2rB2"
      },
      "outputs": [],
      "source": [
        "import os, sys\n",
        "import errno\n",
        "\n",
        "# make a directory if it does not exist\n",
        "def make_dir_if_not_exist(used_path):\n",
        "    if not os.path.isdir(used_path):\n",
        "        try:\n",
        "            os.mkdir(used_path)\n",
        "        except OSError as exc:\n",
        "            if exc.errno != errno.EEXIST:\n",
        "                raise exc\n",
        "            else:\n",
        "                raise ValueError(f'{used_path} directoy cannot be created because its parent directory does not exist.')\n",
        "\n",
        "# make directories if they do not exist\n",
        "\n",
        "make_dir_if_not_exist(\"/content/drive/MyDrive/data_papers/\")\n",
        "make_dir_if_not_exist(f\"/content/drive/MyDrive/data_papers/{paper_name}\")\n",
        "make_dir_if_not_exist(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features/\")\n",
        "make_dir_if_not_exist(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_checkpoints/\")\n",
        "make_dir_if_not_exist(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_history/\")\n",
        "make_dir_if_not_exist(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\")\n",
        "make_dir_if_not_exist(f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/\")\n",
        "make_dir_if_not_exist(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_predictions/\")\n",
        "make_dir_if_not_exist(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/\")\n",
        "make_dir_if_not_exist(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/\")\n",
        "make_dir_if_not_exist(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/temp/\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IufmBnPeB50P"
      },
      "source": [
        "\n",
        "This will use the [SVHN dataset](http://ufldl.stanford.edu/housenumbers/). This is an  image dataset of over 600,000 digit images in all, and is a harder dataset than MNIST as the numbers appear in the context of natural scene images. SVHN is obtained from house numbers in Google Street View images. \n",
        "\n",
        "* Y. Netzer, T. Wang, A. Coates, A. Bissacco, B. Wu and A. Y. Ng. \"Reading Digits in Natural Images with Unsupervised Feature Learning\". NIPS Workshop on Deep Learning and Unsupervised Feature Learning, 2011.\n",
        "\n",
        "[Here](https://github.com/aditya9211/SVHN-CNN/blob/master/svhn_model.ipynb) is a github project that uses the same data.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0yCKQpti4HWJ"
      },
      "outputs": [],
      "source": [
        "data_location = '/content/drive/MyDrive/mlpapers/GP_Collab_SVHN'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M7--mKqbEhZZ"
      },
      "outputs": [],
      "source": [
        "# Run this cell to load the dataset\n",
        "train = loadmat(f'{data_location}/train_32x32.mat')\n",
        "test = loadmat(f'{data_location}/test_32x32.mat')\n",
        "extra = loadmat(f'{data_location}/extra_32x32.mat')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OBiOGiseZmQc"
      },
      "outputs": [],
      "source": [
        "# we have  png files with house numbers for the extra as well, in case of other ideas\n",
        "# https://stackoverflow.com/questions/15612373/convert-image-png-to-matrix-and-then-to-1d-array\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OfOH52uQ0myu"
      },
      "outputs": [],
      "source": [
        "# Set up the imports\n",
        "from sklearn.model_selection import train_test_split\n",
        "from tensorflow.keras.models import Sequential\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "from tensorflow.keras import regularizers\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Flatten, Dense, Conv2D, MaxPooling2D, BatchNormalization, Dropout, Concatenate, Add, Activation\n",
        "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import site\n",
        "import os\n",
        "import tensorflow as tf\n",
        "import pandas as pd\n",
        "import h5py as h5\n",
        "import matplotlib.pyplot as plt\n",
        "import errno\n",
        "import numpy as np\n",
        "import itertools\n",
        "import multiprocessing\n",
        "import json\n",
        "import datetime\n",
        "import random\n",
        "from collections import defaultdict\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "from tensorflow.keras.models import Model, Sequential, load_model\n",
        "from tensorflow.keras.layers import Bidirectional\n",
        "from tensorflow.keras.layers import  Dense, Flatten, Activation, Dropout, Embedding, Conv1D, Conv2D, MaxPooling2D, MaxPooling1D, Concatenate, BatchNormalization, GaussianNoise, AveragePooling2D\n",
        "from tensorflow.keras.layers import LSTM, TimeDistributed, Permute, Reshape, Lambda, RepeatVector, Input, Multiply, SimpleRNN, GRU, LeakyReLU, Add\n",
        "from tensorflow.keras.regularizers import L2\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from tensorflow.summary import create_file_writer\n",
        "\n",
        "pd.set_option('display.width', 400)\n",
        "pd.set_option('display.max_columns', 40)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YJ0GxmKnj56g",
        "outputId": "a5a43436-0c25-4c26-ab2b-aa1ac6bb991c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[ 1  2  3  4  5  6  7  8  9 10]\n",
            "(32, 32, 3, 73257)\n",
            "(32, 32, 3, 26032)\n",
            "(32, 32, 3, 531131)\n",
            "(531131, 1)\n"
          ]
        }
      ],
      "source": [
        "print(np.unique(test['y'][:12000])) # just checking that there is no order to the test data\n",
        "print(train['X'].shape)\n",
        "print(test['X'].shape)\n",
        "print(extra['X'].shape)\n",
        "print(extra['y'].shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s1B6X2mB0myt"
      },
      "source": [
        "## Inspect and Preprocess the dataset\n",
        "* Extract the training and testing images and labels separately from the train and test dictionaries loaded for you.\n",
        "* turn test into validation and slice some extra for test\n",
        "* Select a random sample of images and corresponding labels from the dataset (at least 10), and display them in a figure.\n",
        "* Convert the training and test images to grayscale by taking the average across all colour channels for each pixel.\n",
        "* Select a random sample of the grayscale images and corresponding labels from the dataset (at least 10), and display them in a figure.\n",
        "# Why greyscale?\n",
        "* So we get worse performance and have less discriminating features, allowing us to discern the impact of the GP on the features in _collaborative_ learning better.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VXyZZbP_0myu"
      },
      "outputs": [],
      "source": [
        "train_data = train['X']\n",
        "validation_data = test['X']\n",
        "test_data = extra['X'][:,:,:,:30000]\n",
        "train_targets = train['y']\n",
        "validation_targets = test['y']\n",
        "test_targets = extra['y'][:30000]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YXfIEqzV0myv"
      },
      "outputs": [],
      "source": [
        "def plot_images(images, nrows, ncols, cls_true, cls_pred=None):\n",
        "    fig, axes = plt.subplots(nrows, ncols, figsize=(16, 2*nrows))\n",
        "    for i, ax in enumerate(axes.flat): \n",
        "        # Pretty string with actual label\n",
        "        true_number = ''.join(str(x) for x in cls_true[i] if x != 10)\n",
        "        title = \"Label: {0}\".format(true_number)\n",
        "            \n",
        "        ax.imshow(images[:,:,:,i])\n",
        "        ax.set_title(title)   \n",
        "        ax.set_xticks([]); ax.set_yticks([])\n",
        "\n",
        "def plot_sample(num_sample=10):\n",
        "    idxs = sorted(random.sample(range(train_targets.shape[0]),num_sample))\n",
        "    plot_images(train_data[:,:,:,idxs],2,math.ceil(num_sample/2),train_targets[idxs])    \n",
        "    print(idxs)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "id": "QfkGWAj8D8-f",
        "outputId": "218c2379-79f1-4d7f-8de5-2c8358eede81"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[671, 3197, 4500, 5658, 24350, 28420, 47072, 47779, 57606, 66059]\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1cAAAD7CAYAAACG9G74AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9e6wlW37X91tVtR/nffp133NfM+M7M7HBJhPsBAQWRgKDLGLxUAgWQQ4kShQZJDBBCMcIYSIlkYIIAiGEAA1IIcJWHJmJE0TkQLBhsAF7xoyvZzx37rtvd9/uPu/9qKqVP/bp+n1WdVV3n+697z197/cjXd11dtdj1a7fWrVqr+/6/kKM0YQQQgghhBBCPBrZh10BIYQQQgghhPgooJcrIYQQQgghhFgCerkSQgghhBBCiCWglyshhBBCCCGEWAJ6uRJCCCGEEEKIJaCXKyGEEEIIIYRYAh+pl6sQws+EEP7oB72vEB8UinHxUUcxLj7KKL7FRx3F+Dl9uQohfDOE8Ns/7Hr0EUL41hDC/xVCuBFCUKIwcWYegxj/IyGEKoRwiP+++8Oul3h8eAxiXP24eGgeg/j+z0IIvxBC2A8hvBVC+B9CCMWHXS/x+KAYf3jO5cvVY8DczP43M/vPP+yKCLFCfi7GuIn/fubDrpAQS0T9uPgos25mf8LMLpvZd5rZ95jZn/pQayTEcjm3Mf5YvVyFEC6EEH4qhHA9hHDrtPxca7NPhhC+dPom+5MhhIvY/7tCCD8bQrgdQvjFh/0lPsb4aozxb5nZLz/C5QhxF+clxoVYFeclxtWPi1VwjuL7r8cY/1mMcRZjfNvM/r6Z/aaHvzIhFijG789j9XJli/r+bTN7wcyeN7MTM/urrW3+sJn9oJk9bWalmf0VM7MQwrNm9o/M7C+a2UVbvN3+eAjhSvskIYTnT2/68yu6DiH6OE8x/h2nkqlfDSH8yHmZbhePPecpxoVYNuc1vn+L6YcEsRwU4/fhsXq5ijG+H2P88RjjcYzxwMx+zMx+a2uzL8QYvxJjPDKzHzGzPxBCyM3sB8zsizHGL8YY6xjjPzaznzez39VxnjdijLsxxjdWfElCJJyjGP+nZvatZvaEmf1eM/uDZvbDS7lI8bHmHMW4EEvnPMZ3COEHzezzZvY/PeLlCaEYfwAeq5erEMJ6COFvhBBeDyHs22IAuHt6w+7wJsqvm9nAFnrMF8zs95++Bd8OIdw2s99si7dqIc4F5yXGY4zfiDG+dtr5fdnM/oKZ/b6HvS4h7nBeYlyIVXDe4juE8B+b2X9vZt8bY7zxsMcR4g6K8fvzuMl8/qSZvWJm3xljvBpC+HYz+zdmFrDNJ1B+3haLlm/Y4kZ/Icb4xz6oygrxEJzXGI+tOgjxsJzXGBdiGZyb+A4h/E4z+5tm9rtPfyQTYhkoxu/DeZ65GoQQxvivMLMtW2g7b58ujvvRjv1+IITwuRDCui1+bf+HMcbKzP6emX1fCOF3hBDy02N+d8civPsSFozNbHj69ziEMHrYCxUfW85zjH9vCOHJ0/JnbDGt/5MPeZ3i48t5jnH14+JROc/x/dtsscD/98YYv/TQVyg+7ijGH4Lz/HL1RVvcvDv//Xkz+8tmtmaLt99/YWY/3bHfF8zs75jZVTMbm9kPmZnFGN80s99jZn/WzK7b4u35h63jOwiLRXSHoX8R3QundbqzcO7EzF494/UJcZ5j/HvM7JdCCEen9fwJM/tLD3GN4uPNeY5x9ePiUTnP8f0jZrZjZl8Mnqvw/3yoqxQfZxTjD0GIUbkThRBCCCGEEOJROc8zV0IIIYQQQgjx2KCXKyGEEEIIIYRYAnq5EkIIIYQQQogloJcrIYQQQgghhFgCerkSQgghhBBCiCVwpiTCa+Nx3NnauOc2NB8MSCcWW/lHY+0bzufzpjxDOQR/98tyT/xc41hlVeLc/nnAyRNHxLY7Irbre9MMPalTax6W26OcsR5WpweIPXslxeRLbDiZHN2IMV7pqbJ4CMbjtbixtXXX572Zc/sCwyy5V7E3OmJHKf0j9MRCGsX9jp9JiOGP2FM/tpUs8xbR255AaLdxng/nGAwGTXmIMnnn7TcV3ytgOCzieG2RyqmOVfN5Gme470nMteK9r1vtc6AN/nlypNDd96V9c/IP6WGTKnWfm7GYnC7DMwNHqqP31XXdve/i7/v/Ppn1nOPmzX3F+Aq4eOFCfPbZZxd/LDXted/BluW4/GDPkwfaKDDmsMUjukM/yN4831d++ZcV4yvgwsWL8ZnnTmM8do8j0mjqfpa39+8Lwb5neUI64H8A+jdqjyUenuV1AF3X/e7bb9utmzc7T3Kml6udrQ37Q9//u+/+Bzx8+FBKXohiev7p1F+K3n373ab8ztVrTTkfjZvy+s6FpjxBoLz3/m0/R+2fF4NhU55P/YWtKn1AYWZWFF7HYe51zHEdRXIdfq0zHGpe+x8DHHM08K+4LCfJuevS65XjIR1wvjx4GVWyX/rKz71uYqlsbG3Z7/7+P7D4IxmMdb+AM76tNciqEQ9V8hLV/ZJSVmhDjKXc46fIvMx2FmuU00tK4p3lsvJ9ytLbIgfca2ve/nLEdDlH4KNd53nanVQzP25Ve6w/9fSTTfm5557BuX3fH/1vf0jxvQLGayP7D77rM2ZmdhKPms8LhO84875zYJ5Td26IdzOrSt+pmiF+EWfBvJzlaFMZbjZPnqMPL/ASXng9ipi2tQz9YoX4rXHuqvZYzHAZo7EfN0M9TibeV0+nU98mS889wDMq49gCl7c+8nMMcE1//ws/rRhfAc8++6z9xE/8QzNL++6kH+8Z11X3GPC17/0d6r4Xlp7P+86Qh/7hWF3Xvf/mp/MjFz0/jM2rdPxzVh7k5Yxt+1te+axifAU889yz9g/+j580M7Oq8mcr4yTDmKTI0M9l6Q+anOjgMIaTAfPK+8A5nuVsE1nh8Rv7wjX5gTeNxazmj3p+LP44FUP//s2+jH3j+Jljr9YPdMkLZ+wsc1xmYXGB/+n3f39nHcwkCxRCCCGEEEKIpXCmmas+0pdA/JrJX+PbkqGkjH/DbECJN+qjo+OmPMFb8WyK2SD8sp+8ieI1upzPknqE6G/xGX4dzXt+hpxMfP9Z6Z9XySWgHvglNtnIzKq6+82bv2DV3L1ne7EcyrKy6zdumdk9fsngL0OURbWm2suav9h7TI/H/ks3p9HZVvhL1Np4zY8z9GDgr6VzzDxRYrv4G7NHyUyZX0fFMn7hr9EGsjkkU6hrQOMfFK345IwCZ9BQJ7anLNdvPasmhNBIMWvbbD6P+EU7w8xQ8mN9TGOLMZj8es9Z1VByD3yOOuHXxTz3/jhgJimGOcrpL68IRyvRPhOVAdqIceYV51grvG0ORz57N0Wbmla8HrMa/0a5K9ULdZ/8W6yG0D/LdIe65zbcS47UL4fuOUf/gbo/ZpXr/hmiwHbHz/E8ST5PTt2tyLj37+x+tAeRh1t1/1k28YhEs+pOn00pP+5PwZnQCrPyx6mCipOZHJ8MBt6HFYH9GfpANiR8nOHcrBP7xdCaeYolB7sY67AtFBircHlQzTr5NnnyHtAf45R3x8hZX7YXKoTqptSHRjNCCCGEEEIIsQT0ciWEEEIIIYQQS+BMssBoPguYSqI4Tc0FYNymXxJHyRCnECtMB56c+FTmpKbMD9OBWKjMRXqculxbc5mVmdn29nZT3tl0p7gMU5bTyUlTnlf7Xq5d0jQeuqRwe8ePubnp06x1CUmYmdWly3LqOSUznDqldEyywFUSoxtLjIZYnEkpRSLz9GJ7wfFsShdLlw4VWEy6vr7u+2NVPkLBCjTRYeFSJdYjumLW9k8Ok3pMZ5AtQeqUyHS5EBWSjpM5F8r69QTKxhCfdUvKgm7B4BWTLLSdTXyhbC5Z4MpZyAIXfdV8DqkH7y9DAzcupH4WVsCgogzozzP2U5Bo4GkTCsRNxvZFWZ/XaUbpqqX9YA15dmIYY5TBQu6K/Qd49lC+FzLKYPxcs1l67nnt8bsBWTrlL3P0DYPeld5i1YQ+iVDovyd9Mro+SdyDfN63dD7Zvq015HDmAfZJXIopwb2Xw63v3fsv6fUtyyVRnJ1o2ekDlnehyDFGqL0PohTwxnu3kiNdu+amcOsYHz/3/FNN+eJlH6vk6LspyQ7mY5sK61nmMx8nz2HqNijSWGR9BxgnDYY4Lvvx4MfKafaFZxHHZVzq0+6G+8I6EXR3mOPcS0as0YwQQgghhBBCLAG9XAkhhBBCCCHEEnhot8BkQjiZbqeDVIWP03k4StwoEaRzVAjdkrgq8eXnVB0riJwPkGhcurSb1OPJJzznzjYSyEZIom7f8mnUo2OXCFIgsr7tyZUvX7nYlHd3/fPQctvKoJsaDZAvBVOidHWLkLB86Uv/j4nlErJgw1N5JyV0nEOm3IKyt7rlkJRIkiBdpVyVcljmUGAOq2TSmW2I0/FoA6Mh6m2pvCnHv+WYal+DPLGsfAr/5u33/XRltxyy1/XTWrlket25WJaT2sqJLpWIc8Yi+m1IATPILYbj9HGROFYWHuPlFDI9dpKUGMKlb07ZHdrHrHTJXY08WlmV/ibIZpEN6dTqn1exOzl9zChxwTnqbglZbDtO4W9+hyVzeqGjmMvxdeUEC5a3crKZpX00e+s87x5zmKV9cSLN43iE/TVzacGxkJJnSpWYo5DK8nkrHyefO6wvr6RE/iHKtXLk2mQyNrbtcsZlGqkj5hCycUqrEikxOvt0b7Eqmv6XuSbh0sf+aHrssfHWG+8lx/n5L/1iU2a4/87v/Z6mfGHnhaZMF8Ec/eTxicfQtWs+drj69jtN+eb7N7x+yJ1lZjYufBzy1BOfaMpPf+LZprx5EePpsY9hCsjJmTM0SfybDqaSc4ckfxblg905wB5EXquZKyGEEEIIIYRYAnq5EkIIIYQQQoglcGZZYH3qrJP1Oedwuiz0u3NwWo5uNqOxS5cqOJ9MDiHHm3M6kbJAJP/C50yMtnvBnfzMzK5cvtCUN9Z8u3Lq8qjp9MiPS4csTMMHSF5Ga17vtQ13ERwMUskWDmUDJDBmAra6R4oglk8Wgo3Gi/tAV8kaDjURU84Z7nndknFMph6jTKRaQfvBhL+JEyTaTYmEwtMpJB09MpO85cAzQsxlkMeO1z3WL170NkAp7/7Bnl9DImXpttNpuwXmD+RM1XkosSLqurbjozt9G/pL9M815XS4h6OWXeAIcqMx/qlEvz1H0vY52hFNXqskITZcWpHvvZxCelW1JB3Yf4B+v2C+7gGuA4mKB5BYZZS0Q0I2RNLiMEy/A7p2Zky0iToWOHdZyy3wg6CR7VDSyf4p+VmZss/++5PI+XpcBM9aTmqBWBzlaZzNZt5gpnBfKwZMeNr9fCgg65tOMWbB8Skfb9cvlTeiveA7zLqVimJFxGhWnfYxdCCOFcfc6Hegvb52NXUL/JV/91pTHo3gbl1j7BAwdoWcL0J6eONdl/z963/5lab89Ve/3pTff/96Uz469vFF+zqee+7lpvzpz362KX/b539dU37mxSu+c45lFVjaQLkqZaxcetOGrzPJu02yBuL+81KauRJCCCGEEEKIJaCXKyGEEEIIIYRYAmd3CzydGmOyYE6dJQZhmCpuz4TX1i0lHI9dHme5lw+O4RyF42TITJlj+nsAycr2rrsArq+nSYSHQ8g6WF8kda0jJIIzJBQu/fMM048DTNUP4F41GqZfQoQUjPKZRBEAqWN9j2SH4tHJsszGp9Pia5CS0j2Jcp8Mjmdtt8ABkuPOSyTjheyEMinKAinDoNQukSdSAhsYb+l0N2VIlLHWiaTVt88w3V0gEKuyOw4pc4xZ+h1ENKg0KWboLEsWuHpiNCvLxX3aRIwP1hFPUIDQ5YwJKs3MRgPEWoGE75CKzGaI9ykkf3SyQtgcQ74yQ7meI2bKNM6YoHq07jE3HEG2CPfP9YE/V+iERunrdOLlNSSI3xmkieADnj+U+ZbwTIMpm1VR0u4PgjuSqJouYNn9ZcrtJMB9SYFJ6HHyyxGY6XHqznKEDWtbmhcT52UuqaCjsm8/hyx9CKnYAMFYFEwqj/FOmbq4USqZyqTgMEgnRg1TVk/wcQLd7TBEMDzibe+WL225dvVmcqib7x835Z0L3tflQ3fvKwY+bj46PGzKb73+dlP+hX/55ab8lV/6WlM+2PNzJ0tpWq8fJyde+Vdf/WZTfvMdr+8cLqDjrc835ctP+XIfLqsJGaToc086nA3Sc4fkfaTbITThrizfd6OZKyGEEEIIIYRYAnq5EkIIIYQQQogloJcrIYQQQgghhFgCZ15zdUdzG7lugsnsqSnnJnmqUawj16G4LnJQuLYzg6afa1Io+81hL11A/7614eusLl642JRHo3TdQA2dc52sSaFWE1r6qrtMm9aARSxcExZar7KQg1vF7zDJ8o5yuDvrvFgeIQQbjRaa4/HY43A2c60u7UKZ+X5O32gzOx752rwp1tNx3VTVp2UHvP+JnS/rncRVSwvMmME50vWJsHKN1M5De5xYCWOdYta93mBRk57fbrA/Y12Lrj4AQmgWAY1Gft+3tjd8E1iHc03dIE8fF2sj1+dHxPXUfM1GxXVTFW2xu+87rX1pK8xYumvtKdb6DVCnDaRT2Nry9QObWEtAG/fZxNt5jnbAtWYb6/49mZkN8G8nE38enFRYm2tYsztP+wmxfGL0/rTm+qSePjZJg9HyEe/bJ7U95zorWprfv0Pj8Q8P970WrRA/xDqXGtUdjL0NHx97/E4mHmfr6x7vT1xy+2oumWQ/3u62S4zPAteaoCJcA1xV6shXTQihsdiPJdY2o888OvR4eOvNq0356rvpmqsKuTe4FvZkhkDDWr0Z1qC/9to3m/LXvuaW60cH3v+tjzz+mH+I61LNzGZr/m+3bnnd373hlu1f+oVfaspPPPdkU97e/UxTHm94n1ziPaPCc2NgWJBm7TWOPWXarzdtu3/tlWauhBBCCCGEEGIJ6OVKCCGEEEIIIZbAmWSB0WJjJZ3IeTg9nFjN+jRf24CW8ogZyoMBphAx1VzCfpeZ1gOsGbMIW/bMpwYHA2SazlLpUjrt3511nXK8HLKw4ShD2eudweI01pguL9vfAmQvuR+XUkJO10d5nK6UELJGFjiCvCiVijBDPdMApMcaweZ6Biv2+cwlU0kTwpR1yLolt7Rxp2U644LbmJmVtHKn8gN277TbTiQdtJeHXCCLlMTwOC17U0pIKL1p62NPeQDXY/GIxBhtftpfV4jliH4tS2TJjJ/0BrFfHaP/mx67JOQI3e1oDe0FgZ3PEdeQ2c0Rc0dHkGC3utEw9LaWQVa+sely8O1Nl/ONIIk93D9oypT1sW0P0JbXNlJZYJHkMYBclnb0aAd1JVngygnB6tM+pq7mPZsgruv7SwfNzIwSfd5rqoV6pICUWFdYflBh+cHJiVtWHx15GzIz2zvwf2Md87xbFjhNZIGbuATfvkA7WEeMhyxtYIF28Rg/ZUwHgt/pi4fI8CPOSMQjFc/sY8TJ229eb8pvfNMt04+OUqv9rd1LTXl9x2NlztQUaEcnU4/Na9evNeUS49utTbdGHyF9xWTu+x4fuwW8mVmx5s+Q9W1I+3KP5W+89mZTfvVXX2vKn/7MC015vOHn43Aox7Mstt5IonUv63mUIYlmroQQQgghhBBiCejlSgghhBBCCCGWwJnnb+tT2V+wbrkaHdASV6fWdDsdAssSbmo90rwqmVb3bQoLnZ9zipyuKRvrPmVoljoMhoxOP3Ry4/Q3JHtGJzZMY0KJcHToU7BZSN1R0gzu/nmWV9gGdTLJAldJCB5zlH0grGwfMqLh0O/5EM5ri79dVjiE7JVxzKnpxLGqpoMljptTkggpH+o3n6fT3Zzap0rv+MSn52/v+TVlRolMnxMO23K/W06f5DZSkoPPs/5DiSVRR7PjU/nGBLdnG33fYAiJIO9b1ZILQWfKvox91mBA6azH7Ig3+5gyFToEevkAz4j5PH2W0FU2Bkqq4eyZ8VGHtg3pa4lyzr4WP0HGluwrlfPSYRBSQDwz8qAgXzkxNm6/lOOxz3wYErfWnvvIPo/npuMwl0GUcF7jvkWR1nX3gsusBgVcOtFHj0fejvb3XXJ1fOTjn1s3va/f3d1tylDNJm3ZLF3OkEgoa44Bsf8gdWQWqyE/7UNnGAC8845LAX/5q+7e9/Wvu5xufy+VnGYZ4onjHsjoAvrxGh3i3r4fa3/f42xr5AG1s4M4i3ADH6fj2QGcWOs9xDKeD7duu6Pmtff8Wg8R7xcvc4zvMVpAElvfpS3n3xjvJzLY0LN9N5q5EkIIIYQQQogloJcrIYQQQgghhFgCZ5MFRsrXKJvrSfZYdicUXuwDNyUkoJzTbRBlJgOklIPT0UwSdnjk04R7+56E7OIlTy68uArIN3CsOnZ/TolgVVHa6OXJCZwQJ0gmiWS0ZqkT1xxaQn6HlJ6NBnLhWS3BiiYpNeJz6tPS1677VPQ6prEvXryQHKkYdLtVTqceAxUcxSgboYPUGG5QI5SZHJhT3PMylZ5S6lRB0nSyh6Svcy/nPQn0Qq/8j9LWuxq5F/kx2xm7COXIXj3BzO70I1DvwKTPRiM45bG/m6axxRyQcyRrpNvlAHLZMITEOaMjE6QbOYLjhBJx78+rlqSD7k50a0tkt5Ch53Bw5faUQyUOcIljYiplyfukYjgAn3WhbSsqlk60CNdiuoB1u/3VPdLpu+mW5XNMwHs9Z2JtSAGnU28rXBJBtrbSccrmjssCGe8nxx7j21tod4XLp25c/1pTnkxuNOVLl/2ZtbsJl+YWdeIGTeksr9vjej7T8oVVE0Jmw1NH7AjH3rfe9vHJ//vP/1VTPrzp445hkcbWBGOSjR2XCA4ogYYctKr9Xh8fe8y9d9Ulp6On3KmVTpSXrnjMrd9MnTyLDY/Blz/tDoa/+OVvNOWDIz/H+zduN+Ub17z8wotPNGW2lToZw6SDjRAYy+z8vVgn7f/+Ma6ZKyGEEEIIIYRYAnq5EkIIIYQQQoglcHaNQux4H+vJtBUik++m+1HZUSVT6ZByzHzKnK5+owIOgXRHo3lfoOTOp+frlrwpZN1aJCbYSyvuxx0P4bKCCzo59kRulAseHvjUpZnZjBIxSAiYgHYTidV4PrF8QvDkoRWSVk8hCzw6PGzKaRJFn8Y2MxuP3BWHkr/JxOVNlFgwLPscp7gNJS50cJrP27JAuFFhn3mSkBpykjx0fJqWU6/A2Fk2a0+jdzsExgd0HhTLochzu7y1kBjtItnisGCSaI/XaC7rK1v3h4Z/NBgrIF8ercE1c821hwE6xAJy15OCUvCyc5u81WVniYsm49ePlSVlh/E+HsI5lonc+VxpJZ9Mk3SjLTDJLGQ0dHoTqyFYt5tfn9tfUr7nkbsHOn39NZ/9dIPl5ywXg7yzbJbKGA8O/Bm0d9uXHWxv+TOoyL1tnxx7e97bu9WUn3nqSlO+csGlYlnRkipCCjtAu8jhMpcFf97t7UkWuHJitHgq+a+5vICratB3b277/cktdTbe33epaM0lCSP2uRgPw5G1GPhxpzO6SiKh8LaPYXcueFzeTnMI29a2x9MnP/PJznNcv37TP6fkr+I4BPLfRO7HNtVKItybLbjb5fjO5/EeaYY1cyWEEEIIIYQQS0AvV0IIIYQQQgixBM4oCwwWTmWBSRJhTpfBbifDXFvetgJj0lzqLjAvX2IqvYD0YwTJ3gkcrBJZEpPwUbPSVgHSDYR1YnJiSqswbVoUPm06gRRwNk2TtN1hOkndAudwEGKCzY3CKzkcUHoo2dQqCSGz0en3XcFlb3LisooCyUiHlEi0bg2dxzbggsNp94Mjd3TKp0x+Cvc0OA3SxYZJUcdwd2OCSjOz6pg6ASSphMwkMEkknT8Df3vpTghOZ8vYTswX6cLFpKzMOEkplYkVs5AFLiRA2+vefw1xT+oKskDEYt1KoFuhT6cTZWK2hP3Zx+XoO+shpFQzP3eWJJRncKSBwuTxdP9L0gYj5IqB12mYOLAymSalJZDTthIpJ/p2Ws31lDNZYn6gUE7HcsgpF6L7cHp/s8T1sftzypzniYOw98WUg1MKOEAC7M0td59tJzyu4SZcoY0cHfm4Y30MJzYkca3g3vfW62815c++4tKrsvK6rg3Sc2d4zhUFXHAhCzzch6Puuy49FKtkcV85Vtlcd9ndK5/+VFMe5S6to7Oemdm1a+4wWKLvzzD2oNR7beSfP/Hk5aa8e9GlpeNt32b3KXe6HO16bJ28lY6HR9H/Ho98OzpZbkCquHthsylvbfn18VHBbpjPk+ouNR8/oBN61r3NAwzFNZwRQgghhBBCiCWglyshhBBCCCGEWAJndgu846yTuPEwWWhN1yluc9eRmhIlInTpqypOyffILHo+Zz2SGbyYOtlEJCqmQxw3i0mSQSbUQzJAJoGd4TuAs0rbkoQOLGNMtW5u+vQqE9XOT7oTDoolEc3sToLRRGrkUp61kU+7855lrWnimGSeRiLLZCsmM4UcD21gCOfIYY4Ef5CLDiFxqVqypUS+m7j02X0/LyFhtNA9bW7sB0LbJarbLTD2OfBI9bpyshBsdJrMd0iXvdh9fzLc0zxP7y8leJYkBe52hmRk5r3Oa16m8xplgaEtmYKUizJVljOUc2jDBxmcs9BVV4nbnG9flul3kLhUUZUOd8PBwNtwLWn3B0qfK2DstwdL4HZ9+/Rtk8YvxjtodyxzG8oFzVLH2UM41tZwh10bIs4qfz4wWf3xsS9Z4DKFCseJo3RYmOH5UqCd1xiOHBz4cd96620TqyZaOH2+DiDjfOopl+l9+7d7DG2NXJr36ldfS470xltvNOUMY48S8ctYXocD7DPP+vmef/nJpnzpijtXXnnRXSk3d/34T+w9k9SD8VtBOnuwv9eUc+i7L1/2a9qBRJCPB0rGE1/iDjfRLpLNOna511E0cyWEEEIIIYQQS0AvV0IIIYQQQgixBJYiC6T0iO5QtI1iolOzdPq8SBLTMTEYZX6coqTFH2QadbcEIEvcmloXBOlS6k7lWzABMl3WKiS5rHBuJmul+0pbFljAPZC3xToAACAASURBVGtAN5axT7uOmHjzHgnLxKNT15UdHJ46+FHKh/LGhss0h0jqHFpWd0ni4JnLL45PTjq34e8cGeKCCaVHSLya9UhL1tbcNcfMLEcySrpisT3y8wrloyOXn0S08ZoSrdAtbVwct/sPxjHbpqJ79cQQLQ4X/VaJezeE9A052pPku3nekpwGl50kUs/ElY2PGPTt6LdncD9jm2A01ZS7ttSniQIXStay5LMILn01+/NuN066xpVJks70O5jBaTNCIri9BQkvEgdXc0X5ygmhedAnkr2ke6IjGKX+6f1Jxjl0GEyyqrNf7U4cnIx3iqKzPBowZlJZ4Gwy7SxXyBqbLE2YM94xHkGi177ExnWah94yPnfQ5k9wrNu33LXw176eys7EaqhP43aI5OdPPrnblHe33T1ya7DTlA9vuczOzGx708c0cdC9FCeJ36Fvc/mKH/e5l1wWuAXnwPXLPoa5cMnle5/+7CtJPcoTxpM7Kb/91ptNOcfD6coTfn07u34NlsH9mGMsSArb0vI+lXDyOnPGrlszV0IIIYQQQgixBPRyJYQQQgghhBBL4GyywNDtstHnmEGpSNulj9CNhhKnktPTlDGVlGyhHpSHYFa84PGzNIljTlkT3bDq7vfOxI2K2gBMsc9RV8pc8tYhBwOfyhwMIQNg0kpOa7YPIJZKVVW2d3uRYI+zxhFao9HIpRsFpqirmhI/s+nEY+n4xBNInkAWOCu5j28/YKJGyEYo35vPKWuhnDVtjSMmIcY5mKA7cRiEtInJU9Nkmpxep1tgSxbIco/tTvLpWefdxZkJwexO/mlKPQtI1/JE5uSxa7HlVjqgpNP7rwCnvIDYooSUKvEZOnpKBEvEeJW49KVxkuG4iYRvBonWnPKwbilr1iMRZ+L4qkxlgXwW0ZmKuWgDHkbt549YLf1ugYZyvyMg/2acUdbat38iccW5h0MkZy26h2BlmWrzplOXAvIcHC9N4P43m3gAMvHvaOSy8YJOmUhEW7djfIjvEBLDCWRct266jOu9995rX45YMtGiVacxSLn2+obH0/Y6ku9mft/HWToWD5XH1nDdZXsjyA1rjIHK2uPs8lMuC/z0515qylu7fpydCy4R5DNgZ8tlfWZm1/ZuNOWv/+rrTfn61ZtNeXfLj/XEk0iaveExXsPGMhmr0Dk5pjFOyTpfKvpdRUPr/3ej0boQQgghhBBCLAG9XAkhhBBCCCHEEjizW+Ad0iTC+DzdqinV7em1xC0QySwTe45uKUjokRVl/AvTj4MMboQtVzdKEhMDIMij+pyt6ITV54w2GEISEtrfDtx6Zj6VuX+wl2x1hzVM84rlE2O0+Xwx5U25BqU8eU4pFGR6ZUsWOPep9pPJSed2VdWyZTolkeDh8znkGpScDGu4ObVccFJ5Y3dMUwZS4XMeK9CSDW00PqAsUF6A54MsBNs8lQkFJGkvIvtBJP6F1G0C+YhZ2kcGSIzSRMDW+XmSk5rSupLOgegfce6qSuMsq719sn0x4W8F2ROTglMynjjJ4hxMpnmXYytdZUP3c2mOvt2qthxFLJ9od2TWqUy6e2lC0i+2JacICvaHiZSa+1fdfTSTAjMBPGV9symeB63x0v7tPWznz5PxyN3hOLY5OXI579GBu76O4XA7giNhXzJjs3S5Bc2PD/e8Hjev3/ZtFOMfEIsYqQOk13jeD7HMZYjYr+cu6zMzmx8eNOWNLZfzrY18XFGV3ofNSu+LL8Kd8HNr39KUC8R7BY303vsuH719zc9rZvbNr11tyr/61W825cnMz3fpGU9IvL3j4+Gs4JIHBGnP8KRqLVNK3ilCdz/RWsTQs42jmSshhBBCCCGEWAJ6uRJCCCGEEEKIJfDQssDERSNx5IG8JMN0eUsBRbcyJghO9qfTByUomLUbQJrHKX0mJuZ0flsWyCnsjI6EkJHQhaeE7CTnuylmCQdIsjaEC2Dr1FbP/Us5OfEkfIcnPo2f4Tq2Nd2+UmKMjVyuQsAyMe8YZSaBbuXIthqST8ZAndiIUYKHOIFsiXImymctcVjzKfu85T7FOnJanM5XjCq2RV53ZPJjOlfRDa5oBThOmCQRR7tOZbn3n2oXj0YWMlsvFvc1T+5PEgVeQtLQo2ka5FPINfoSo9IxdhS7729EbFToEymnqyBFie3HVpIcksncKfcC+Dw1YPV9B2iDM34e27LbbrnrAE6iTEic6ffMlROjj0/aMmnfpsdhtd2P98ik+xIEc1yT9Ms9Sd8LSM4r1GM68fGAmdnBnssCmTh4c9elUTzujffcee21r3+jKe9ge0oEx3DBXR+l3xldBY8PvR3euHGrKb/7jjsEbm9vm1g9d4wbS/SNo5En7OV4tp7O8Xk6jjy66ZLOZ5591o+VPB+8X46QG84rj/GtHXe/nh779reueZxcfcvj5Of/xZeTerzx2jU/H6o4wDhke8fdArfhSBgxiqGzIRN/l6hr27U1eZ3B5+wa2Bdgz47PFqinF0IIIYQQQogloJcrIYQQQgghhFgCerkSQgghhBBCiCVwtjVX0DK37TqTjU5JTFBb1oep1Wh3FnXaQtM6MeKdkFapsUcfnbOcp1rLxMod61DmyOBcwaq6xhqY0Zp/fZtb6/gcdqewswx5qnWdHPs6lveveRbqgyNf47UOa8yTaWqhKZZLlme2sbm4j/kA6+agR99Yg8Yda4yqlh6XSzO4FjBpB5T6x+42wO0DdMJcD0X9f1mmixvn+Hs89FgcIi659ib25FXoMyGNvf1AeiyuS0jWayZiZ/3Ws2qCBcvyRTzX0XX4NdICJNbU6O+qaXp/yinWEq1h3VTRnWaD+vdkqVKy5qVnbR7XtLb08oMcltIF/o074UmXDQK2Z7qB7vW+WD5lw9bvkbwMLiscjLCuJg67dxAr404MJ2uqH2D9VRvuk6a14BqP7hQXTCUw8KUwaYoLxH6Bz9/fT9dcHR+6tfruzsWmvLO105TffdfXzrz2a77O6sa1603505/6fFO+sO37bq1jLfEg/T7mE/97b8/ts69f9fNdf+/9pvyJTz1j4gPgNHYC+qQScTkM3qczfUWo0rHK5tiDc4T+M2CdFVNWVIiHHJ1jiL5vxHrDa2/5Wqqf/7l/25R/7dW3k3pE83HW1q7H5njbj3vxSY/98RrSf1iPbwPW1ybtrtUXMK3IstBoRgghhBBCCCGWgF6uhBBCCCGEEGIJLMWKvVf+U3eXTzdsiml2cN+EU+yprLBnep9lSEICpjqzliwwh+Uu/2UGBd98BskMqjGCzOrKpUtNeeuC20UOIQ+hPa+Z2d5NlwIe3fbp9qMjt9ZMvpusX4IlHp0sy2x9a2EnyjhkSoGqdilVqNh8Wlnt8XcGGRNlUjNsn1j7IsH4BFPcOSxyh8iATmhfbWY2g5S0hpRwY93ljak0tlsLyNhLtumR9JrdSwqIjaiSTJumWAFVNDs87c420X8VsNeNkEGXkEsUGbRNZjYc+HaUjQwHtJr27QeIfaa6GMK6fYA6ZSew1A2+TW7tPtxPcjxxqfUIMsaNmR93yyiD8eNSXsNzDJEOI+Stdg55Cb+rwRDW9Kg7pYdiddzPir1vDNG/3OHB4Jil5FKGxPod4xHUY46UL7evuZW6mdlkz2WBlz7xqaa8s+6251+9/s2mfAMyvbWh9/tPXrnSlHcv+L7jsbcPyoXNzGZIuXCw7/XYu+0pYw4hW7xw0SVdYlWE5oGZ2o1Drhr4/MWDtjUU38bSkwH6Q0rFB4WPN7LA8YzHzQFi9J1vvtOUX/2qS1S/8bU3cAU+BjEzG294PYbrftxPvvJCU37xZS8Px0hxU2FcljRh2slDDp63Xn0CZfFdluvpsR4ka4x6eiGEEEIIIYRYAnq5EkIIIYQQQoglsAJZoG+TJ+5f6VwkXagoGUw+xz5VpDQLLlK4Aiou6NDGqf7QkpTQgS2LdBnhNKFvX2DadA0uKxd2d5vy7iWfbi8o5Wp92xkkZrvbPiV6dOjyADoIDXqkYGI5hBAsO51iL+d+b+YTl0Uc135veP/zQXpzObVMt0rGDyUrscedchr93AHtZDb341Rzj9XJJHWUnJz4VH3bSfAOlGuxzVLJG/rmwSEXrFubJErgHokgnbakClw986q2q+8v5DxPX3S50A5kGHVwaWmNWMwg2TMzW1/zWBmPPR5HY0qtqfP2Y03QvoZoO5RaR7S1ECBLydxVyswsREpZIOlAXFd4NpSQZ5dwRhtQ4ofrLnC+fJgGeU3ZCbTkJ3P/DgfBPx8N/DsXqyEE72czDBBSU0DKtrlNKglinzmvIdHHUoM5ZPxzxPWYz4fM4/fo0KWr64XH+ztvvNWUb77jDn9mZi+/6HKop3efbMqTfT/3rau+zOAGyp975XNN+dmnn2rKsfZ9ZzP/PtbXU/nv8RFlgV5+/5a7BW7vuBTwxRc/YWL11KfP6hrLFmrEa51I/Lm8pD2n4tsdTz022YfxWT4ewjG59ri++s6bTfmnf/ofN+W3XnuvKY/GkCAON5JajDa8b3z5Mx7vv+E7v60pX7js2+RwfZ3j2ULH2JCMQTC2aS9TSj2QvRR7XAjj/eelNHMlhBBCCCGEEEtAL1dCCCGEEEIIsQTOLAu8M832ICm3Ene7ui0LrDr/iYmA674ypuRikgC1W6LE6dGqVfOSeX3xTzWcn5IpREwNMrnsCM4lo6GXqeQrilRyMBz69OUY+1Oewi+nKPQuvHJOY2U29SnxvX13RZrNXPYxguxjYyN1vsnh4EPp3BgxU5Z+rBkcKSltYnwewQmNybMpYZpXqdMTJXgzyrJwfRlcBJloeI765UniSwj4YtLQknMn/xa7ZYUZ6pc9olOXuD9VVdn+aYLS7TX/vjeRHLsYeVyPDYkas7T/YoyPkHQ7h/NgDWlJibZTwdWygpxuznJJqSyk4HnahwfE1tamS7Iv7Hh5Z8flKOM1vz6GclmiTUTKxuBy2JLHTlnfie9/Anku43o8TBPJi1UQmqUAqWNpt/SHvzFnrUTmdIcsp95HZ4h3JnRP9j1ySfbxgScFvnXdJXsVYqZALL3yyivJsS5euODnRjzeeM8lV7/y715tynS4/PW/3mVVL738XFMebeCZAwnuHhwBzcwipF/TmX9vb77hSWCfe+Fpvw6tXlg50YLVp3HLJTA1OrSK0ld8HluStkM4Cq/xkY1nfo0kxHn0G3z9XXe1/Mq//eWm/B4+L8Y+5lnfcifti094zJiZvfRJl5N+5ttebMpXnvJ9spzPBLjJYlFBuvSHSYSxJKM11uB3aL1JxbnUqGcToNG6EEIIIYQQQiwBvVwJIYQQQgghxBI4kywwGJz3kqkzTD9yh0TW15pqY3LURD3EpF88FKbksEOAVC6DlK8v4W5oT/tDpdGX6LSimyGdCpO6dk8lcjp2Nk8TvJYlHROTjK1+XMgnq1m325tYDiEEGw4XU9iH0WUch3BvnOOeHZ5ANlekTmpMyjhM5FP+eViHQ+QBXP1wz5m8L9D9DNuczFC/liywgESV8qnjqbs+0YVye9MdfOiUw3rkucchY9iqtA1QSljDyjOZwkesZw+SmU88ElVZ2a33F7Kk3W2/D7tI6FggHugGNWo5qdFlMlEsR+/n2K/RtTVH/0xXWEpl53DBnE7hWlik0roc5mabcJza2fBY3oQcd8inFNzd4pwOn0hsDKe3dhL6GODmSYfRYyTv5vaVYnz1RIu2iJEkd2qPNJnP/buSDmfdcqqQWqE2JfallOYd4Rx0FKS72/a2y1if/+RLSTVODj3O3njbpYBf+tf/pim/fc0Tt37br/t8U/7kZ55vylu73lhm8aApl7DTnFdpjA9H3o4ODl9vyr/6dZchvvSKS7pG4/RZKFZA8LFyjSBPJMw9y1mmLdfgG3t7TfmZdU9QndO5FW2nhJT1V7781ab86i//SlMeDNDfbrisb/Oiu2p/+3/065N6fOpbPE7XN9Dfw702Jom5fd90vI9nS0/brqr0O+BrQeyRBSbnaL6P/v5cM1dCCCGEEEIIsQT0ciWEEEIIIYQQS+BsboHBGuez3LoTddVIjMtXt9rayfko8/ApRDqlHZ8giSST/UKmQelRagbUPR3Ylh5RpkU3qxO4qQ0xPUqHOKqg6KBWUKeCL+fgKJVs7R/xfH7dsyTBsk9fblV9LiZiGYQQLD+V/eSIC7qnzSOkgLi3ecsiiTK60aDbSZIyqQ3ImdbGXt7a8Gn0Ivc6ncwgR7rt7lOHk9TpqUJy4sRVCOUZpFiJI2HskfUmyYVBy0InkYRxHzTU3uTEYiXUZWWHNxfJP0+edAeyE6gkBomODTHeklSP0G8jb2PiLBUKSlZwLEi72daOIP8b7sO9r/AYDSGVHRXYf4POVEMmM0YAzv0cjMuMSWLxHTCs2y5TA8T4AP9GmWSZxL6JFRMtWl3fkQV2uwKynLiLtaWDWAaQ46bSWbLP1Xg09OHVFtxk1/A8GaBNbQz5nEmrcevI++Wvf/ONpvzVX/taU37yeU8Q/B/+lt/YlIfrfrBp9OdDHSHvxnUXIXW+PZn793P9+q2mPJn7+Gx9w9vg2prsAldOtCY2M45DOC6HFL+sOb5MZdVTLFeZQy6XdPdoFicnHotvvemJgw/giLlz6WJTfu4ld6j83Hd4Qutv/dZPJ/UIwePp8MATVNMxu0J/W5VIYr+BdwIkhk+X7vi1Va3voMi6lyr0uYr2KIwTNHMlhBBCCCGEEEtAL1dCCCGEEEIIsQTOnET4DpQ0BVryYLqsQtLS2Sx1yuNU5CBzKYfVdOeB5CJxY4NrGs+BKc2MshPICOt5Oh1Y0/kMdZwkCVv9fIMhJIzJ3CDqjWlFJmu9cX3PyNWr15vy/qGfbwIdyXhcoNydrFAsh2ju7EVnmKQMBQnd+4ph2pSYYHo8gnPgwLejYqXI4CII+cnOzk5TXlt3ucYh4nNaI27rVhJh6EsyJjZG+6UUYEIXQWxf193JTws6W7ZlgXTteQD5X/1AqcnFo3JH8saE6jXdzCiZgHy7qNPf4pgAmjK6VlZ43x/xlBdI6AhXtfGat4MB2hSTtraNWSnBHUCfyHBncvoqcdREnRij6NsTCUk7Tza+wwIOg2tICh/w7CoKSMbFSli4Gi/uS0jyifYlQofcp+0Uhn8rMXYoKZ/CLkPIvkeIS8qO6LBKJ0rG2VvX3BHQzOzG9f2m/N6+jyO2n7jUlD/3rZ9tyk8877KssvY+fYTnT4YE4TN/nFiRpbLAa9dcdv7Wm+825ZdefrEpP/vcE015MEzdBsUqiBZOZZ0DxHKOeC2i3+uDfZfsTeYeD2ZmF6+4PJwuelxOM8e4gFLA6zc8WTDHFJ94zt0jX37x2ab89CXv/7IS+1q6zKKYeFu4ccPr/vrb15ryYOxx+srnXGK4hmU8AYnv+Q7SUrgn7xoh6x4n8TXHHRr7xyyauRJCCCGEEEKIJaCXKyGEEEIIIYRYAmeSBUZjEq8eRw1Mq1dwvZu33DlK/kkXPDqMGaUjmLrHuWdIiDaF2x+degpMhbcT3OW0uZp6PZisrITUaspEkSxXTKyKBK9IuHZ7z6f2zczeu+aywOkJJAeQYOV0moMTllg+IXicZZQt4R6EOd10ICnK06bERKxM5Et3SxpoVjguE2kHxH2B+z+k3Ir1a9mRFZAuDrFdlkimkMS1ZvK+bndBtvcs0EGnLQtkv0CJoEM1WSZV4MrJssw2T5M6jocuMy4S2TX7cO/jirqVQBfOmRUkgzUlU4mjpu9LSS2ULDYY0HUV0m6quFrSLajxLC+4D4/Vk8QacZmxzBPw2dOSBVLCO4CEew2OsaM1SGEy9eEfBHeWLSTOX4n87+5tT/9KjhPTLMS+FY7FfjJizEM5+QjS8DXI5iomZ8USgqs3XH5nZva1r3vy3pMTl2j9ht/4HU35lVc+2ZTntbsCwkwz6cdzuG4WucdlNU9/c799y5MNX3vPJYIvveDSr93dTT/fQL/Zr5rFWOU0xul4auyH/fPDg8OmzCUvZmabcLK8sOn3MSIR+sENP9jVNzxZ9cmex9kwegyVB7793jV3/nszeuzuvZPGydaaJ9E+8erar712tSl/4623m/JLn/F459IdwjE622NRpOcOiatgT3sOWef2fagVCCGEEEIIIcQS0MuVEEIIIYQQQiyBs7kFRk+Sx1mxyESlmEaPcDEJRSrH47/RdW/OZGcol5ySx3FqztpD60RXvwGkgG1HsgqJxZJ/wUnKmm5qPqV6eOTHPTj0eczx2qhz+6MjzHW2/mZi5CH239jawHHlNLVKQshseJokeh1JVbe2XK5BqesQcUXpn5lZQbkpGkuF9kFnvhmcqOiAeYwp/PWtraZMaVQBV6rxuBUjTAJLeRNimlZvbANVT3JMivk41Z632hYTFybnoIyG0/Ftpy6xdIo8tws7i8TU2+seT6OB9zl58Ngo4Ly0lrcTZcP5su6OoUA3vkQW6OUM+yaOltSJZozF1BGzipR+QLoBaTg/zwIkv9T58VnCE6BOWZ7GaICMvTA8Z0K3nKyu0rqLVRAg26HUHzopuhLjxlMF2IYJSQOTjuK4EcsfKCOiRJVSwArOgUeQBd46dIc0M7MZnjuXn7zclF/+9ItNefeyt+ebN92JrUZ/C/V5shpjkLkc7GTSclQuvb6TY5eBbWx60uIxpe/F/SVT4tGI0aw+vYGUBdKFtUJc37zlDpMcq5qlyeH33vck0e+/646V1YlLyN97w2V6e++5ZLSu/dxv/JrL966+4zLC4dhja2MtlfINzJ9Bs2P/t1sHLk/MkCx4iKTbbF8cvNPxkMOLdhLhAbSzTDZMiWCedR23P9Y1cyWEEEIIIYQQS0AvV0IIIYQQQgixBM6YRDjanWl2uuhQosRptCxxU0tlgRmkTBFT7Jj9thLz1nQFzDgTh32ZuHFj0+V0BeQsTOBnZlbSNYTTfj0alhL1oGRrf9+dAOl4VUe6r6Vf9/qG13E6dbkIHQxHSKoZBqksRyyXkGU2Hi2mmnNjckffhrKlMdz7NtbSBM9MFhzQPqaQfhxPfbq7hHyFcroZkm1TekWJyjpkpOXcpSGLfTyumPiSCTETZ0S02ZoOhpxTT3Rffd5/ZoF/J8n4YtfHdyfwFEsnz3O7sLOIkZ11j9lR1u3SR3u8vGXIRHkn5eBGyR/dJJm0uKRUzrenXCNxm6REfJomwZyXkO3yHPztEO2x4LOIzlB0aaWcMedzIZWBMLl2gGNiXvn5ZoljbJpIXqwW9pmU4KWZqOkk2e8ixgSriasl7SoZzBljy8tHxy752z/wMcR7N30MUba6wpc/9VJTfuGF55vyxpa34Xnlz5PNLZdMTU/g6IblEkwcTPfYwwOXepmlskde3xBjuM1Nd5x7ECc18ejceVxS8hzR39IV8K2332rK33jtteQ41264/O9o6pLB17Hd4UUfV7zztssC6Vy5sebS0uMjbysHhz4GicG3L1rPk2oK58wJ3g/Q517+hEtiR6PumCuNzxD0z2izoWqNVfocQx9hTKKZKyGEEEIIIYRYAnq5EkIIIYQQQogloJcrIYQQQgghhFgCZ1xz5aQSRWiWM66/gs1y1dYuUosPau4PnT0yLTMZ82jkGmJala9jPQHXPc3mqRXuMEDbifpynVXO9QiwlOe6lcND17eOYIe9hnU5W5sXknNnmR9r79D11hkWN6xDyzxoi1TFUolVbSdHi/uY4f6PsCZpZ9O1xwU0vMO77g3tb7FO78TjhFbsJWK0YhqApN14eQCb6Y3S460ap+tRsMTLMuyfYU0Z2zLXcs1mWHPF5QpcV2n9muREx4y1O/w8yXreeySxLIoit0sXF9r4nQ2PmyJZOof1cYlVfsuiue5eUxJwJ5M1tCViHOXZDGsMJ1hXiMfTEFbxJxNfW2JmFtG+JhPv3w8Pfbth5fG+Puxe88cmnMNynbbb7Z8jueQw43ao+8G+r2O49s6+idUSY7TqNCa4BCpG9jXsg3DfWt1ZMgZBnAXc+PH6Ovbw+z6Z+Fonrk25vefxcOP92358WFl/4pnnkno8+eQTTXl9w7ebl36sEhbqA6wZXxv7Wpga7a5AvNalt5tha1QYa3+IbG74uIpratk1xOoefvZiOUQ8k3vWBc5wr4/QZ57M0/5zAK+CrYseK8W6f47QtBlu9gRrZGusIa+wzWANYw3YrR9zcGJmA8NYF+u3Dice40O0tYuXd5vy+jrGQOZtbTrFmiv06cPWAmKOb7hOODV4ONsIRTNXQgghhBBCCLEE9HIlhBBCCCGEEEvgTLLAEEIrE/KCqu62z03leOkUYInpxCKnfa7Pz1FiSP1FZJb76JdA2VxAPY4OXIoRWza+s4I2qj7td3zMqVNYVcPGdw7Z1OG+SwByWLNWcMYeDFM7+k1M79OWnc6uY1p81w9vCynuT1VVtn9qlTwqMM1MaVMilcOUeBreSeyXkFwcn3hccap+XnXblk9nvu9s5icZjzyWhsNBZ9nMrKKVe+iR5mH7kMjAvE51Itfl9t0SK7P0lxuemzLixAY5KL5XTVFkdvHSolMao5/JEqt9SENRLltynySeElkhZH6I38kR5BqwJ6/gO30E296TY6QOmPF5kT6DKNueQha4jz55NMOzgRb0ax5/gwGlsgxmph1Jf49MUxT4v2UBKTSi9yXl7KGV+OIhSGTHodtyPdzLeRl/Mz5oz19hJ6YSmM+YSobH9OMMCsTi0OPk6aefTqpx6fJ2Uz46dinhdOK27lwiMcQ4ZYY2UaI9DjLfPsm+UKdjpHLu7Whrw497adeXOYxxbqbzECsi+PccIF1LJPtYtrC+5ePLK09eSQ61BXndzkWPs8vPue15gXHy+IJL9sKGx+IcbWWOtAc1xvEBgRbztC+c4wBFgM4UD6rnXn6hKT/1lEtlk7ED5JB5suQBS5FaDZ2ppdhP5Hm3FNBTcPSPWdQKoF+rUwAAIABJREFUhBBCCCGEEGIJ6OVKCCGEEEIIIZbAmTQK0czinZlGuocZpUGQD8G5ZDpNHUoqSPsG5lPjwwGmHzENf2K+P49bziChOoG7ibns5OjQp0ezmE7z5ZByFHANogSGDm/V3M9d0QXuwKfnK0y9H8FFcAwXQbM0y7vltCvx8h2Zmpk9UrZocX+qqrK926eywIHHW6RMqvS4ygtOH7flQnBGq7vdKqeUqELaxOnrE8g+jo7cNSfPXW9aV5TvtaRb+DvWieWff04pYLI7pLjJ9Delv/5pWzCcqP/oJpf8A9uj4nvVZHmw7Z2FtKc6QSzHbslUGSmlSB8XlIhniP8KMth6imcAZB8nE8alx8AQMqnNDcYfJFmDVPpKd9YScsNjg2x3DuksZDRZDonWwK9hRMu0ujv2zdK2Oixw3BqSnLF/Phq6xEqshhBc1hkyyv+4bKDbebf9iKVcKOkaMz4TKN2GWxvGCnNIX7PcJXTbW16+cOliUx4N/XMzs6r0ZwWXUayNPH45NplwfMWlDLgett8Bjmkh1bjP5gd+vnU452671GxAR+Ug6euqCSFY3iyDobsrtkGMXrrisfXSp15MjnUAN76tSztNeecK3K3RMJ556RNNuYT73xSrdWZ0KUZ/Pef6iZiOVcq5t9XjPd/u0kWv+6c/83JTXoPbbVX5eK2sfF86b1NNTufP0y1RL7TVrFtKGNprIDrQzJUQQgghhBBCLAG9XAkhhBBCCCHEEjjz/O0dCRKlSIlbIB3UEklSmoCywrxhXfh8YlH4NOMAMosxJBuc2g4REkFMb1KCkiQ0C6mkJNDhCXOqGSz7KC1YG3e7tPE7mEM6Vh3ANa7lVMhEsGX0fWY1JZD43h5gKlI8PGVV2vs33zczs0FBuZDf8/GYSad9nnneypvIaefpDC5pkIwmTjR0uwl+sMnUZYHvXb/WlGczykRQj5b8tqAjD05XY4qbTp5MeDzHOSgLpNyAn8eWS1RZ0oGHSbkhp6S0Ur/1rJ7oTpYFJNhZ4hLJHfzzIk/7TsqKmJQ1R7+dr8OND+5k843u5wS7uCs9LmzWcnCq8CzKEWd0d2OizBGkVBaYVBXxV1JO1i9dpcKVsRzwLFnf8v7jxU+lLnBiFYRG9pfKArulryRaOk5J2kKP/JpJsCfQRmH1gkU4BNIBeG3s0rrtHSROPbqV1OPmdf+bisG1MdoUYnl6jHEHxhaDnN8Bxmfmlc2L1sMs+DUxgTFlWRnGZ7WGKSsnxtjEYIUYLRDjI/R5L3zSXfa2t2FhbWY3D33pyWDN+8/LT7gcj0sK/r1v+0xTfvopT3Z9dIwlD6gUZYGTKZbY1NARmlkG6fft993h+8K2SxWffsbrFELaVpu64v0gMXPFWL+1eqLlCojxzSMsxdFoRgghhBBCCCGWgF6uhBBCCCGEEGIJnFEWGJspt5h450AaxKR7SOo7LNL3uIA589nEHZQyyLGGmMJeX/Mp6InRGQTOVJXLmJjElZKV0K4H6k6HwKz2+jH575AOf5GyMMqsIJOkzGrakkZCNlCMIJsK3dOSddmayxRLJcbo8QRpXoR7Yw03MsqFYivBM5Nk3+1MsyDvkRtRajeH3PTw5ADbe52GlP615rvpdlPi36ZTv47pDMmMIWnNIP9jQm9KbaiSCi3JFKWyPBbrnuwTuh28xHK5I2eNBaU8iA24jpWRcryWW2DWLemk/CLpO4tuKXmdSMa7XVMzuKnWLZepivLT0p8fRebnziDBjqhrBfeqeopnAUJ8MIT8MWvLRJj0HlUv4Ojmai8braeJ5MUKiLHRSsW8u4+trbtPbssCEwJjEG6BcICdQPYdzMcssfb4K8xjYH3dJVqzmUumBq2R2WTCRNsuFS9yD67h0OWGTNhNx+GLu34+PgNizjabnnttE1LAbT9HAQnZEOOzqk6l6WL5BAvN+KNGzEbKuEd0onRp3e5uKgt8lmPozNvFaM33Z5e7PvJ7/cKzHKuiX+USBPSLs7J7jN7ecHLoMZShv97a8piLkRJcOlx2v49Qynt3outuWWDoGYs/iFxQM1dCCCGEEEIIsQT0ciWEEEIIIYQQSyCcxQ0jhHDdzF5fXXXEGXghxnjlw67ERwnF97lC8b0CFOPnCsX4ClCMnysU4ytAMX5u6I3vM71cCSGEEEIIIYToRrJAIYQQQgghhFgCerkSQgghhBBCiCXwkXq5CiH8TAjhj37Q+wrxQaEYFx91FOPio4ziW3zUUYyf05erEMI3Qwi//cOuRx8hhD8SQqhCCIf477s/7HqJx4fHIMb/kxDCqyGEvRDCtRDC3w0hbH/Y9RKPD49BjI9CCP9zCOGdEMKtEMJfCyEM7r+nEI9FfGucIh6J8x7jJITwT0IIMYRwxvy9q+Fcvlw9JvxcjHET//3Mh10hIZbIPzez3xRj3DGzl22RcPwvfrhVEmKp/Bkz+7yZfauZfYuZ/QYz+3Mfao2EWC4ap4iPPCGEP2Rm5+qHscfq5SqEcCGE8FMhhOunvzT+VAjhudZmnwwhfCmEsB9C+MkQwkXs/10hhJ8NIdwOIfyifsUR543zEuMxxjdjjDfwUWVmn3qYYwlBzkuMm9n3mdlfiTHejDFeN7O/YmY/+JDHEsLMzlV8C7ESzlOMhxB2zOxHzexPP+wxVsFj9XJli/r+bTN7wcyeN7MTM/urrW3+sC0ekE+bWWmLB6aFEJ41s39ki1/fL5rZnzKzHw8h3OVRH0J4/vSmP3+PunxHCOFGCOFXQwg/cl6mIsVjz7mJ8RDCbw4h7JnZgZn9XjP7y492aUKY2TmKcTMLrfJzpw9rIR6W8xTfGqeIVXCeYvwvmdlfN7Orj3JBy+axermKMb4fY/zxGONxjPHAzH7MzH5ra7MvxBi/EmM8MrMfMbM/EELIzewHzOyLMcYvxhjrGOM/NrOfN7Pf1XGeN2KMuzHGN3qq8k9tISV5whaDzj9oZj+8lIsUH2vOUYxbjPH/O5UFPmdm/6OZfXMpFyk+1pyjGP9pM/vjIYQrIYSnzOyHTj9fX8Jlio8p5yi+NU4RK+G8xHgI4fNm9pvM7H9Z4uUthcfq5SqEsB5C+BshhNdDCPu26Dx2T2/YHd5E+XVb6DAv2+IN+/efvgXfDiHcNrPfbIu36jMRY/xGjPG108D4spn9BTP7fQ97XULc4bzEOIkxvm2Lgej/+ijHEcLsXMX4j5nZvzGzf2tmP2tm/7uZzc3svYc4lhBmdn7iW+MUsSrOQ4yHEDIz+2tm9sdjjOWjXM8qeNymiP+kmb1iZt8ZY7waQvh2WzwcKe34BMrP2+JhecMWN/oLMcY/toJ6xVYdhHhYzmuMF2b2yRUcV3z8OBcxHmM8MbP/5vQ/CyH8F2b2CzHG+lGPLT7WnIv47kDjFLEszkOMb9vCkOgfhBDMzO682L0VQvj9McZ/9ojHfyTO88zVIIQwxn+FmW3ZQtt5+3Rx3I927PcDIYTPhRDWbfFLzT+MMVZm9vfM7PtCCL8jhJCfHvO7Oxbh3ZcQwveGEJ48LX/GFlOeP/mQ1yk+vpznGP9Dd3TOIYQXbPEr/z95yOsUH1/Oc4w/G0J4Jiz4Llv04111EaKP8xzfGqeIZXBeY3zPzJ4xs28//e+OrPDfN7N/efbLXC7n+eXqi7a4eXf++/O2WFC/Zou3339hC6lSmy+Y2d+xxeK2sZ3q6GOMb5rZ7zGzP2tm123x9vzD1vEdhMUiusPQv4jue8zsl0IIR6f1/AlbLKoT4iyc5xj/nJn97GmM/3Mze9XMVvFrqvhoc55j/JO2kAMemdnfNbM/E2P8vx/iGsXHl/Mc3xqniGVwLmM8Lrh657/TY5mZvRdjnD3sxS6LEGP8sOsghBBCCCGEEI8953nmSgghhBBCCCEeG/RyJYQQQgghhBBLQC9XQgghhBBCCLEE9HIlhBBCCCGEEEvgTHmutra24pUrV+7+h9CTOuEeXhkR//hgnhrd5wg9504+Rzlrbc6UJlVV4XOvVMAraIZjhZ46cd/kOuvYu10ffdf3+utv3IgxdtwM8bBsbG7FixcuLf5Ivnbc8577UbfuZV3XnWXGWF1hG8ZbEq84d9Zdj746tTmreU3ahHrqlGzU2j/5oLtN9PUR77z9luJ7BRRFEUfD4eIPfPeM0SzHfUNKyKqVp5HhFGp/lOS57xSynj61zrs/x6mzzDveLM+wfZqGqi79HGXF2ELMMhaDb1PjWJExjnPn+A2yHa515PWxzeN0OC73n82mivEVsLu5EZ+6dOE+W/X0Wx+IwRfP8RBppx6giskl9Z0i6YYf7LrPWttX33xHMb4C1jc24s7u7r03eoD7/uDc/5mdfNzTju41VknHzffnQcc9q2T/9m07Pj7urMiZXq6uXLliP/ZjP7b4g31T1j0Bxmdg3UrLWPY8iNIXkOQk+BQP3cwf0lnul5MVeNgXvv2wdcXlfNqU9w/3/fPSnRxHQ99/MPDzDXIezOtdlmV3eTZPzl2V3QNtBk2eDzo//6N/7L983cRSuXjhkv2JP/nnFn8k94Cx5OUa8Tmbps6fxycnTfno2Mv7+x5j+weHvj9iY4BzjEbjpjy8Myg2s8HAtykQk+0OJ/3xAANJY6PjKNmLBdrNaOTnHg09JllXbm9mluNlMKlH7W2irivs4fX47/70Dyu+V8BoOLTPfssrZmaWTf2eHE68H1zfQT+67fdwr7qRHKv0XSw/vtyUL+xs+P7rHu9JH3fs29RTv+9Z4dtsb2825eGmt4N5dZzU4/jGXlO+ueexFWcep2sBfXXmbfW4nng90M9vjNa9HLZ8G2O8mp3M/NyTuR9rMkX/kXk9ZmgHb7zxdcX4Cnjq0gX7m3/mhxZ/ZHhxN75s41eDjC/b6f2NsWds0zd47OlXk1FN7K5T/YBjxYxjqbr7BwT+EMcfDSJ+WIhoj/OaP5ykg7UsGXs9wA/CeJv7LX/8RxTjK2Bnd9d+8L/+r+69UXsm4ZS6PRjvIfBWcx+M0Xkstom+H5c5hmkzn/sYqLeO/MGt4LiH7ZTjmZ4fgtvt90EmOuLdL5h/92/9rd7tJQsUQgghhBBCiCVwppkrs34F4JmPk8xZUh7VPaOVvsnynRC/NGX+6wtlJFCp2BRlM7Oq8rfl2QyzD/gFq+a8+gNMpcfecvvL65Yuch5facg+QIJ1TqUn97Dn/rd/yUymuJNy35Hv92nrl6HkV6JuWdU965EEcnfEJr/aKhA/GkSz7HRW5aTEr474lXOU+a+LJX/RTieMLDvxfQYj32606f1oKHzGaTbH42bTZ4bWN70eOftt823GFRUKrYpc9NjcGPr5ipNRU67GPvuUD3GseLspz4fe54+G2015zXwGbcbpOjObTP26b1zzuh/M/LiHhu+jbD2AxEpJui10jokcFFJSjhvMUtVCelz0v5gBohplDvULFTac7Wd/m/UogBabMf7948CZOQyFWI+qWwBkIYMig5LzVohGjIXmVFigIjnqXs7T2T/x4dAtyr97iPMgT/Y8CZzu8UJV96hhQLIsojU7xfF3WfKdgOdAX7/p6ofhiIoiHrd7/JO1voUHkTRyXP4gr0GauRJCCCGEEEKIJaCXKyGEEEIIIYRYAmeSBYYQkunfO1Du1id7ai+E5OIwmlhwOnCGqe0Zppq5TVVRKtU9HUjZYQitemBaPcO/DSEdoSFGkWP7RI7VIxHrcR1c/I1pRkoBeRVcMPgwzkLigYkxYqqaWgrKHCgt8U853d3+u6YLZc8C0MTwgXEffKqcU+IFpCj3MrRIFlXHbvMIwuZNV8AKbnCJJPEerp9SEp4/6hjs+LSvoYlFPnB52zT6va5qmJrkLq0zS3wALOBRwm7u6LbH0Hu3XVI0L9zoYjikM4bL+tbnHteXcNDphhvEmJkdjryNbI99/401L5drvv0WZFnD+SX/B5irDAouhPa2RkMjM7MRzpdDuhXW/Puc4bhF6ZLLa++YWBH+OO1x9O1R5LelSuwz+2R7yXMcaxBy61m8j+Nw3yz0D8ci3S5oTMRlFLxWOH7mPY6ukeYASf3S7yBAJxggE+Y3S5Ox4l7yRrE0Qn33A5ZxTXnmA8nerN/JN7Jv61mSkJpYdLvEcgRyeHhoZP/A5d40AZtM/bkxHLg50FPPPNWUd2DwlbT4HifatnENZY/02OI4LjGgO71uSoLbqBUIIYQQQgghxBLQy5UQQgghhBBCLIEzuwVmXdPtSZJEyKYS85D29Fm3bGo+d4nHyQT5SI6nnZ9P4V4zn3fnz6kxlV3VaSLM0cinGXd3Xfays+VOJCXdgDDNmCeyR6ffIfAeboHJfL0Xa0kBP1B8mpvOepS9MpZ8P+YzM2u5NXFqOZk6755SrxLHTCR3RXwz31ZeQpJ1l50nYrRnijxRk+SJLrApFj0uP0m5JQd5kByGvQkuxUooY2Xvz47MzCyvXHoxhKz5BGqNeSJtSm/WoGKGYe9HD1A+mXm/fVQfeT1mHjfHnh7KMri3cvubbvxn82P/3MyshGvZTcj2YuHn3jnw8sT8hHPIScaQ+D0LGewxEiGHlixwXLiT4MZFfy49ve3fQcT3lM/9HP/qy181cR7ozse3+JsyOEiPKKnj+Ac5zSyRGvEZgBNQIpg0rzTOyp4+1+h0yL2z7n68xxg26YdjO8cQE3vjLDVzCHFz5OYUqyGYy7Lrnuf6Xfexh9QNucfBuOZ4qHspAHNNZRm36R7/TKfpmIlSwL19lwhOkEN0vOb67oszLrfozh3HCrJJtOW/fUt0Kkgr+a5RlXcO3y+x1MyVEEIIIYQQQiwBvVwJIYQQQgghxBI4m1ugefItupIxGV1koj5ODbYPVna7/00g+Ts8cPnH3r5rVQ7w+RE0JSdTl35QXpiW0ySQOzsu6yiGLzTljS2ffqx7EpFF65mKJPyeWpKt0KPNitYzdy9WTuc0b29ia5bTaeYkSWXP1HHv/cexSlrXQLqVYxo859T3PbJ807Ezo9Qj696nqLqn//tyBT6oW6Ai+sMjhGj5YNEHXtp0idp24f1drF2SdKP2/vJwBv2emeUl7mQBh0uDUx765CkkgmtwMyuiSw8DnApjdBnGpICEZD19bO1ku015NPVrmsz9HAe3/dwzyArnG0iKuuH1mEU/Tg2nwSGSFJuZDSErzOBaOBr6sYrCy4ORdLAfBN4Ndgv2Q9u6984W95D50DUyg3saO9M01ypjuce1EPK/KvbI98xsTjfZRALlFDklhrw+LtW42/FscQno1ENbFkjHY5wDm9SQ5mZBssAPln7H3uWdoXusm8MdkzFOF28yxzNgOm0nZMdyn2n30p+ApN6JFDCRuHYPShIJY3us0rNEo8RSDNZpNq9Pt5UsUAghhBBCCCFWil6uhBBCCCGEEGIJnNktsJkdxPQwncsog0tc0qwtm8I0HLeDO8cc04HTCd0CUT5xqQqn7eY4Zll1O7eZpe5/rHuG6U4mD8ww9c4pUU6RV5xWf8Bp2tgjPRQfING67xdDN1GZdCfcW2zWLQctCo8ZSoqSRNqQVXF6PXHsoeSWiYnv4RaYfJoklqTr0/3LWZ/U9Z6KJ8X0eaDIMruyvtC5jceQ7yA5YzbzGF0vISmape5ONaR9M8j2MjjBFpDmGY5b5NTU+vbHcCfcWHPH1pefWvc6XYBOz8xG5i6v2aFfxxEcPPd2DppyfujnOITcfO+WP0vKzMsFrjuP6bnn5jaG8/FNr1PudR9CchkslcKIVRAsxMVzmhKhPqkc+6Ys6x8SMVFu4sCX9G3d0j4m4o2ox7zm9t1JWM3M6giXPmyXLM9A/Xju+czHRSVkWRHtY4C2PGip+oqhfycFxjw1rqmmvFFJhFdP9HFiulTFaUvfmm0eUJmcyD4xxsgZZ4kTZXe5RowzsTFlgGZmMyQLns29nCTqTbMh43zdUtlUFtiTOdxa7QVSwAnazvGRPxNmp9fRXg5C1AqEEEIIIYQQYgno5UoIIYQQQgghloBeroQQQgghhBBiCZx5zVU8VTRmUDYW1NgmWcmxFqS15iLRF8ce7SQ0ktRCDweuma/XfJvhaIztoUWma2qW1mNzy7X8W1vQyUNnPMLahCHKBWwhDethKl53xuzULZ1nohmFRWrs1nCLFRPukuI2nzsPYquertPjmsRB4fFTDakl7tbOB6y/qnuykKdpAFp2/521bWUk55rCpNyz7pBlZmW/69zd9WK7fmABuFga4bR/OTryeDqIx025KF1nXmAtxtrA1xeZmQ0z/7vCGqoMmvWj2o+1jccE9e/HFfrRNdjDf8L7482nvbw79z7bzOzoqvedb+25zbpd8nNcedLLo108o17zZ8nJMdYV5K6vj1inclKm7b/C84QO1MfQ6k/mfr7iHta9YnncWV8Vsf4vXQPFdaP+edZaL5SOQbgh15ZjTQme/SVOWHIbpKGZYd1TsjYqpms52Ldi+ZUNsAY8WQ+FdYITpB44OfK1hxnWv4yGWGe56W3CzOzixQtNOdCCHn3/DGuDJzOtK1w10aKPlWP6+R26VxWe5SRcQ9Wdgojr0WuMYedYMzVDXB8d+XNmcpKm9qB/Av0Wki4TjTUZOiTvDT115fVYSoU2OYGnw/HxCeqHVE8zWbELIYQQQgghxAeCXq6EEEIIIYQQYgmcSRYYrcf+kVNyPdanGbV5ZpZnsP4NPm1YY3o5yUSO6cDh0KetRyO3uc0h06N96ABT3sNhWo/RmtdjfYPHHWIf32YAiRflkHWgPWr3dxNbU/19luuJyrJX/iVWwVm+Y/4ykbVkgTlu4oByDdivJ1PKSVLx0Pl5IgvsSQ/QliemKdu9nDPLOtpmf5ltuceW/UEliZ01Eh8EVV3a7cktMzMLU5f7VHgU0HB9PnKr8gtbqUfzhYi+M/peB1B77FUu71irXdo3Rf9frPm5d7Z9myevIFVB7sc5PEltqm8e7TXl24de3yzzZ8MTY5cb7uy6rDB/xvv5fOoxfvXAn0kh9+2nRRrVwzXf5+WnXmjK46FLJmtYDFcTP+473zCxImrPGdN8dlffeAfo7PKsNU6hDAlSwJqSfmij+ISnffrhFFKjI4+B6RTyvYk3nCqmaQ9Yj2zgxx2OMDZB3ec41vG+t4+jvdtNOUC+u4bjXKx2knPv7PjfOVOGYDxzMmX793gXqyLYndhOx5VnHyP2WZcnnyM1QB0KbIPxOtMooZ+bMPaPPS6PDl1yt/g3/5sW6AFxnSyZ6XkHSTzok++G15bOK5WQMTKlU4m+e44YvyODvVfaJM1cCSGEEEIIIcQS0MuVEEIIIYQQQiyBs7kFxmhVdWfqz6fDZjNIjCDNC3D4GwxSB5qIKfMJppFruHaUmBo0uPCMcayt7d2mvL7u8o0hZH2jMZz/1tJLzql0CX7uAeQfwwEdAuGgZt2Sv8CpyHtkXeeUYl7guD0SrF5Zg1gKwULy3XdvRMcZ3L/WbpQFMmYi3AIpo0vldd1ufFVN6SldrHJ8ntYjJjHXLbNl/dh+c8Z6j3NguEd8pnKD0Pl5XxZ5sRqqqrL9vX0zMxtWHosbhcvxwnizKW9tuLRusJHKmie1yz1C5THx/oH357chFcly75MDVE/r6KsvX/T+/HKBfREn701SN7J3D280ZTq0rcOJrZz4tdZxuymPL/o5ikMvTyfusDZMFCepbGyj8GfOhbUtP/emn2+O54rN1IevnOB9171kO83m2MTHNwtq9vdcsoDbmCfjAy9T9jSb+76Hxz6u2dtz974S5y7LlrQOMcRHVA35YMDzIVKidQS3wIN933nu9djd8Ta/tpa6glLWlWFJRo3nw8Gxu8C99a63R7E6GuErxxex+xmfLA5oLU/pFs5ZYtOXjFW4B9sO+tsTyFIP4BC4f9vjnTJAM7Mp4oxtoRhh7AH3Wo6HkmvKusdSHJ7wPaO1i+EUST3oAn5nica9uhfNXAkhhBBCCCHEEtDLlRBCCCGEEEIsgbPJAkNIpEJ34FQ6Z8noNtZ2EqMrT55IsbqTjeZZt0xva91lK5tbPrU9gPsNHQFZNjMLSH5Zlj5NScc/ugJm1iNvYjmw3O3WtvhH/ztJOptRLsaEspKUrJq8Q3qZ3MHknynNS3+nKHpbFq0g2ZbYVhgLkAVW3bJAuum056nrZOoc0hLKChNZ4AMkDk4kAiyn8G+p/84HwXIL5UIWl8M5NUey1QGcT3eiB/Jkkvb976MtbI8RjyNo/oJLkmbmMqQRVE8XIcG+tOmufgapIZ8FYeIyk8XflGrDMQ2xfwxZ1QnkXUlGVnwfxTpkXJU/F9YtTWB8JXcJ1W7tkimeO6ccsr77+SmWTWjuawj1fbZNXY3vTgqK5L+UQ+HZzc8rxCnDbDrz+NuHXPDGrUNsw8SraYLVDDrakFPu5fuEyrepIJ0tJ5AnniCWMc7YWPMYrepUkkg3xBL/dgJXtf+/vTNpjiPZsrN7TDmBIMiaWnrWJmst9f9/h3ZaaaU2vaqugROAzIzZtQCQ97vBCBTxXiZZLTvfho7IGDwyr3t40I+f++H9h1P5l//4LYivx5I038kCMS4Y03SMs+AQiO1LDsE9pIB8D2gRG0wW7BIFTyS4lPlRp5fn8+MhChpdsmCO0ReGzHEyIuG4L8/mr5fl+D4e+4nnRuSauRJCCCGEEEKIM6CXKyGEEEIIIYQ4Ay+SBWYxC5vNgyyCU4BdiylouNwwGWoKXg4xYM6cSiaXfLVgsjI7VwlZoJdjUe6BpGew9hl8bj7n7EfZFd3fnAMLLsFLJ04fxnk5FeV+02tHSg5oxsJrSBZ4UWL4PBlwCMuywHHBfS8En6Q3Qf6XaEVDFzJ3wflp+iGjLIVT3xYkU3esEe2Us+48sZ/yn69TdMn7wnz5C1mS04qvQRaKR4e7rLK4adA9N6VJ+XoYtqa9Ty5aDSYl2ry2ONt39qt2fMQUtn8GQ7IKjoSbtdWpQzrjGnHWjqhUCKFKJjVpOju+b80/g3MTAAAgAElEQVQBcdwgcfAajp0N3AWReP57OFS9pxwyn0Tszo4ZIEUPiZJLa3j1Cm5t4htCydS81D8E7xbIPneAjDZAOsvEuhQ9HSih+2RSwF//sKS+d0iA3bbeEbPI4J5WWMyWJZZL0DWuoZOntSM6HJdIdL3Zmhx3vfZugXymDCgf4eZ8vze54adbuw9xIaIlteaYJHNx/WUP50gpbJofl8eF8TAdsOn2R4dJbu/wfjBMJLjOkZDLE7gUCbaxyckZ55cqeJYGXBP3ZDh3F2U+uz09tu7nDLw1cyWEEEIIIYQQZ0AvV0IIIYQQQghxBl4kC4wxhKL43IWnp9QOU31jolOZnwIceiTkGpBEONF5EFPsTIaK5HdHJHscISPJa0wTFpDmVX4er8BnLK+qeVlH5DQhtIOJmivKDJzkwL/LZgsOJ94xMYivyHyi5vkf4flkuPNSEycXdC5/Vhwh/0vZQlx9oSxwcHK+eXcdxi6b2QiN4Djks/uHkbKZyf/VuGR+X9ImFOyXJmZDyFYPzl4pvjltp6Rtw4Tvo/WpRe6lSht0/Ktgx5TJJHgdJHz5YI+bbmXlFk5llLiUyeo0QNr0ofP1uEWy4W1n590Gc48tBitncPVbb+z50eCZlP1s+7RHxOgW300IYVyZnGqA/Le9gzthaXHdruUW+O2gJHtez5OeSyJPeZzbTjc9O2+LbrJDucYfx6ONffb3TKI6kb4W6LtxjWLB0TVCSsXlFSvocbdod9Uakt1yEqPox/nc4Jispdxr4gInLkP2+Oz0TtV4Zn/mfPnAVC2Y5lWB/g+O69EX9xiLtw3k2Y3JRI/YTnfBFCaJfOEaS0dM5465sJxh0TExzDNOBmwZXMML1GOzxjOrm0nWLFmgEEIIIYQQQlwWvVwJIYQQQgghxBl4kSwwhRCGRzlgwrRaXuIdbYj+gEc6Z1UWQgcpYIOpxQ7lGtOJByTF42zlsbZ9KOsbIyVQKEc/FbnG1Pjbtzen8qsrk7aMPRyhKpOqONdCOgkNlEPatT6TjlGCsJCk7UumOMWZiAuywIWp8uh+24nzjXP8m3f/y5xKBUnsOLUf4+wB0cXOM26B43z8JEzzO/ktpuozOPMMPR036Yw5L/377O+F5Mv++1aEX5oiZuGH9YNErk4mERpz+91fl7Y9DyZ7OyafvLfbWp982FoM7SHJjqP1nVmyc9WQUu1zk4rcI7twicSrxYiYG/z/CQ6duQ3mw/WpfA0J1FXFhJhw0ERc55C+HuBmGyH320afhH432rXvf7Vn0c+/WSLVe+eM6J3YxPmJgf3Kwv8fM8FqoLzf78ZxDhXQlOYxUXtPKSDGAcfOYvxQd7PlFnLBaVL6ElLdFZSpWWAs2/EVliysV2gHkP9uN1Zewy2wqCbDQtz30Nv1KP8rIHHfrn2ibXF+YoizzsZLz9DkXAC/TH7P3QaM14cOY3fI/xqM11s6BGJM32PZT5wMiCOTBRfzywXoRs5lR/PLOfyxGdvyZC+Op4qSy4OK2XL/2LaXr6uZKyGEEEIIIYQ4C3q5EkIIIYQQQogz8CJZYAgpDKdpOcgmIFfi29roEgV7OZ6bZhw51WfT5yOcqnpks6Tsjvtwhm7A9Xh+yhFDCGGzsSlzzkRGHN+3Nn3eI8HeZo2pdCQYo7OKn4GdvsvOJ0HzQinJAr8m2cyXzClk93P+k+Z2S4K4uCARDQtSwJg9Iwt0iS/nEwwyEWBCm2XwMrl3wQR/Yb7tP9Qd1XXlZQdNcVmyrAhXqx9CCCGUoyX7PGQm7ziM1seVSPx7jD4D+wh5XeHyPFofu0O3nwc77xHy8dSbk1+P7XlhdRpGu8B64hZ4BclVhzod4bj2fWGSxlhAYn6081ad9fMJCZIHyF1KU44/1Le0a/z8x4dT+df7d6dyDWmky9YsvgJ//tR0su2JzIfuZBzCcHyRIFkdGdeQC3aQ/x2PdAVEG8Iyg/XGD81eXVk8buFgFnprI32LJRKUBWKcstlZ+erKyq/fWBvcXXlZ34jvh05xlJav4Jr59uZtEF+PzI0LrJiocX1GCsj9Ri5ncHJQ9LHdvOSvwxjdyfcQM265xKStcRnCkgMy6+HGOm6YNL8kg5rf7LM1OgtjndLaGpcdPalglURYCCGEEEIIIS6MXq6EEEIIIYQQ4gy8zC0wpdA9OoU46ZLTUs27kFHuN/3bTV9SBscEeZjmHpcs+BbMyVx54lrYtzaVeX9vMpmIuvdw0kmQUHHKcVUhCR+cc9zs42QK0deRLnR0U1s6QpybGKKLuScyl0xv5AG2+bP54fkpef87L5SX9IY0DqQU18kCfXyP+IxGnkwuTNcnJxFEGXn8nJuhlzNOqksHHiYIDEvxLS7NGLNw/+gGWAaTEWVw1jsgf2kT4QCVeTlejHZMn16dysd0eyqnHDLsAfI6Jh3G9Yq9BcRuZ1KlQwfXp97OE0IIP333nV0b95HnJrOqIO/Io9UpopwQlxWkXhuWMy/rqzuTG34c3p/K7eqTXaOCm2H3QiW++Ic4/ZJ0BXR9rOF626lcyHX3kKwiJgY+M7j6AceWuf3uu83uVL5+hfaBSl1f2z4hhHBzYzG029m56r3F2f7jx1M5Y8J4yJl63J8rO/fDiVwrWMx3yIzctmzbkFWpU784KaQwPMrfkhtrWDHLZ5LehuASw4cwDfl5V8GE8XpdQ05+gCwVcsEVXCl5gQ5xQnfMEEKocfzIxNWRyxDgKOjGPRjDYPyc0y2QffdEJsnkxqwvpYAV7mn1uL9kgUIIIYQQQghxYfRyJYQQQgghhBBn4GWywDGF9lFGl0OmV0Jy4R1AlqVO3I/nWq3sXBsk+A1uCtumEzk1uHTtAW4l7USWMWIen1ODh6NNd1KCRVdASgHLgnXCNKaTTE7eZSkxc0ldOYWrKfavyXNJ4aafMwne5/HNvzi9TqnsfHnRBSfMSwHzfFnSsXQ7rAcdeOjyMzgXUCbjo5shXXb8xQpqCROksosJs+frKs5HCik04eE37iqT3RWUO3f7U7kbrB8cBy+Jq3KTKnW1Hd9Q6YS+fRhN9jEgyft9e3cq3yL0r/AsWHUWHKvXvh5vdt/beVHHu6PJE10oNlbvgonuk0mshsyOzV5b+1gV/trb3qQi310hgfFbPLvwnFihz3/3v4O4AClZYl/2yt55lTJl9M+DlypRJs5EuezDOiTv3cMV8P69yaeavbWjTWUx8/0P5qy3QnbgK7j6hRDCem3XLpEQe7WCZBVJgUe4In96by6W+4+oU7vB/tZ+Vxuf6Pot3AP3tbXVprb7vv1k2z/ivsXlGcP82JHjajo7TmHMcykAk2PXtcXv/Z31jfXBJNojxifbtcXyBmVKAe+PXmbewnmQDde1Qchr83z+9SVBFsjlGmy/nzkb41553zmWNpRIIlw0D+dVEmEhhBBCCCGEuDB6uRJCCCGEEEKIM/BCt8DxlDRshMwny+flSomyqUnuRLqHrdY2nX01mEtOXiCpHlz9xpFTn5wmhLtgRndBk3U0rXeaOtbm9nQ42HQ2JYLDyGlTO5b1CM7hb74eYerkRqkKbYZw3tGlsJVu6q/KdNJ9dNPMSGiNKXi69A1w0eGxTiG44NCZzTgczn3GGey04Bb45AYawsQJEPvzapRblcVEcltaox8T2oTi+JuR0hBS/+AqNlYm/4mtSTfyxn6fpodcMPNyITpCFa1JCdcDZEFI/jsgOfGAWPmjRlLVj4izwvb/rjQJY3UzSXKK+9h/QLL5DEkjk9V16ODeB3niMZnbWjtS5gjXzBv/HWQ/2rn6j5CgNOaeuMLD7/qNYv9rsvRtp4VkplO31YD+ms9r9tdYdRDGFuMGyJya2tpHfbQxRwMNbd8gRhvflx6gX81yyJZWVqmyRM+MvpcZvvva9j/A+S/ubVz0+ujd5F69tmPaxuqxxzEf3puk9sMHyQK/JpT8uWc2YjQyrifS1xF/d4jHAdLSw4ExixhfcBr8fET0AA374jOJfH37hHtfznG2HUlJItspxxo+2fI0WTjaAsopYx3tvk9DtGW1pWauhBBCCCGEEOIc6OVKCCGEEEIIIc6AXq6EEEIIIYQQ4gy8aM3VmMyKvUx2aEUrdq57gsYxn7zGlSV1zlwMguOx5qpobE0AlzrRGr2Ajp/bR2SkPkLX/FBh2KhizRXtqbkmhethnN4yzq+BYVboqTyT61h4T6OTxP65jlWck7nveP434Jq70f9oLma4fq/rEFc944p2qgvr95bKz6y5cu7DvAvUl7HeYm3jkoU8z5m5lAy+bTFFA9dDjov261qPcmnSmEK7f/gtN72tmehgFd3U9jsmrLMqVn4tRhktVjI8SoYc67ewvUp2LkrZs6Ot1+j+w9Yw/d2qFI5v7Txl9HHW1nYfv95anX68tvW7VWE26UzN0Y92bINlr0MwK+wdnlW74bW79v3f7aB//+WPU7nAmsgtLLZvW3+8uATptGbDrzO1sltnxXXNGYLg4UNjob9PA86FsQbXubRYc3X38d2pvEe748XKycgsjrb+JUW7xuba1h/efA9b9621oxJrI7sOttgt7nVv7eZu79elf7o/omxrb/7AOqtffv3tVL79ZPcqLodL2fLEQkofPr/TZKwyYkzSNbb+tcOawfqItazDn/sADOP8tSejkMlR/BtrfRdeU2qMVWrUu8J7QMLYqII/w3TdN8fpMdCvwepe0FfhC8YqmrkSQgghhBBCiDOglyshhBBCCCGEOAMvs2Ifx1A/SjBSsunlNTJ6c7o9L2ADnU282KNN+yW84yVWKeuwP6wWMeXopYCQjiCb8gA74D7hnGEi26PtKux3E+RbfkqV04SUbMGmntb0k1nQMabZz9wUrpSAX5EUwqNEJDmZ5rwcxJUHLydZkgVSdkfbc14jyxDHpbWzuCg9fU4WyLZljLjBHnVtUSeXrX3gND3rimn3yuoaQghVD3kYZLo5JFPpmbqL8zOmFI6PsrjxFnGGWC5f2W/95o1ZoG9zSphCaG9NitF0kBJCWdo26F9hJ51WJj3aUKIFqdLx9u5U/vvRpEbZ6DvFsbcYGgKkhzcWfwfEe5HZtYsR54Uaiuk7BvsKwqc7/x3cvzeZVKjts7GyL+E4mC37XsrXr4qXMy88i2nDPOmOEnWFtLam/TWOwUqGADVe2K6tre1QHjqMa5AmJu+9dCsbLJ4SLKFjjWfQ0WI/X6OMSnWFtYn7vS2DOB7s2r9/sJQEjyc4FT9+NCngr+9MBvvbuw+ncnOgtFxcjKcYpG05xqQcg7AdjP1krMLxCeR1Dcp3d0hT1NjvW1YWW24c4v3QT3BZRDf4sThT1nBcQEv4prc63d7d8uhTabsySfcWaTpKjPXL6TolrvZJ8+UwN8Z/pj/XyEYIIYQQQgghzoBeroQQQgghhBDiDLxMFpjSScrk3L8m7iNPcIpyOt0ec0jnRsqdbB86osUFh7EMEiM6BFaYrhwGO09TeKepjHJDZqrGVGlPV7eBErF5h7fk3lknckjiXBJRlBTwm5CCxfKiLBAx4lzHeu+kRskfpYBty2l3284mVBb2R04pC+SmXx4k81nPKRlwdYUUgFLAsbQympyT4vI802u4741Z1nHfMUozdXFiDKF8iKO2gxRwa+WrNxZnr7Zw++t9Jz7AWY2uSiX69gG/aQ+LwM6ZRsHxFf1rBWleS8ne5P8Eh0hpuH12dQX5am5ypkQZLOrXNXa98g7XgCTrQzKntxBCaCEfL3I7vhrxaIXEa9/JSe1r8CSD8m6BkE+5+GOcTJ/X6GfxWc5zYbsJkkLYXZkk6TVc/ZrapKQFJH59gzYUvHSrxBgm5HbtfAPpdWn1WKFOEe6dTcF+3653d2dxma9M4heCU0OGe7gFfrq3Yw64p5j0f/YXJ5njqh8KzEtX/YDGjx3cEgY8wxu4sO7hGMn9N2F+7N5jbMRhboPxTzsZLzjXTtYPg6O2sTp9giyw6+1c3dZk2Glrda3wfhBLv4QhOi0gtzOW6X7+cK7nXAPVCoQQQgghhBDiDOjlSgghhBBCCCHOwMtkgSGE/nGKrofMp1+Q0OUFpXJ++iwtua7BMWTJcY3T+3QkDHDfyzktDg1Akftb5mecymQCtb5bkAgu3APr56dsp1OI81OOShb8jUiWII+/IX9bOutROtp23iHJy/+sfDzatDYT3zHP3gi3m4yJ73I6YMLhaiEOp/uxPQ0u1iEFQJ24zzAgUTjik1LcrqMoxp+3LC2+CzrzOPNNyQIvTZ5i2PUPv1lfIgY2kKjmJp0eG/sN+9H3nf3aftO6Rmw5N0m7xgqPmxGJVzvIBVNGx1a6aUKSUZjcKgQvU63w2aYyecgG9egHi9lP6Gr3g7mklQe7dtHZOY/5xOFqxXuFMyJcEqvMJChX0c5FrytxXkwWmH227QFmV6eUyp/Huf1CtsRE6Dn2WcOlb4txw/W1WU620LgWSAgcOsgLM98XbtAuKjghd9i+H+br5GSLcHTNMHbq8bw71BNHZcj/Gjy/Gsq6UN1y9aJhpfhHeUqU/UX7ojiJcboF+iUC866PbszsxsMos63Q/RptbRi99HV0NwI343H+mZD2JkXlcp0SMvEVpLJNa/dZTOaV2F7yhW+US4jy0xINyQKFEEIIIYQQ4qLo5UoIIYQQQgghzsCL52+f5FKc6htGTvVRrrQkj/PyJZfErJ2XKLG85PpDiZKXRz1jxUcJn0sKuyRbZHneYUSqvv+cpJBOv7tz1hvn5XQ9ktv1vZdSdHQIROzWcOA5NObAs2BWEwq4WxY5XdUgsYL73lQWOA5LMtt5t8AO7W/APdFhLafDGmQm3cQx0X1vC/1F5hIgikuThxhuxgfpUtqZjPNui74zzDtANhPL16ayYw4H/NZwx8vobIY4zSijphSwxD4lHfsgMZ8kWMVpw/XG7ilHzGVwX8tzyPwsT3G4wz2wDWajxfhQ+RgfS9zHiETKuKc1pDA3cEb8OYhLk77EVZV2ZpNeiHKj5FzWIBGCc3JkotKNxVmBjMJFiTKSnOaI/V3ph2Y7fLZeWzyOiK3siGcT6sSErFxGEeE+26NNta2P8ZbPioFSLrRt3He18o7M4vzEMO+uuxTvbng6TcK+tLwFv3Xm2gXkpFwSxDEzXDB5zgKS0Wld6Q4+Ohthjve5XMN2GRbkiZHjqnHe3TaEEAo6k/Nev+D7XEIzV0IIIYQQQghxBvRyJYQQQgghhBBn4MWywCdHJLroECdvouPaZHrNO5SZhKqurXw4mmxqf4CEiolRAyUoVqcVHNc4ezqVbw2Ddyw5HUO3HZfAeEm8xGRq2Wx5Og0al9UIi8eIy/LkcsMYo/PN4OSCdBGcJBGmGx/kg20PuSAcBvkzF3Bro2R2cAmMMfWNBJOfuQXS1ZOywCWJIK9Hh058BwXc1nhvw+Q74PS8lyLMx7TMAi9PyrIwPErnNgVj3ORJqYakY7B+t/V5F0OCZDWHC94YTPraF5BzwxUwg8wPeVDDd2/N4e+qsA9u7+08H95Dy/dwtlOpWtt5q8LaFx0MKW3q7m3/+nht+6/tWORgDdnOPzL5FByOaIe410N1fyqvrvT/mX8F0oIOe+pgVkLu5qRDdD3DIXTQO+yRhPVgzmZ3RyvfoxwHOLWufZzlpY1ndpvNqbzC9r6wdnvoUD9KFSEtT+ivczdm8R1xwbHQCpJESHCHm52dt/PPIHF+UkDYumUydMGMc0XLPvwInbWLwuKuhPNluGPyc8r6kVibGryBslSM0RfqN4USwczJWu2Zs4Kkdo1ySdkt3ZazeQfxECbjJi5Zwi316Bu68aHtpGcEgurphRBCCCGEEOIM6OVKCCGEEEIIIc7Ay2SBMZ6m1ug048vzMji67oQwcQukvIrOgZALHjGt7hxNxvkpxwouZpz2pARqSgEZClUDfpoR06BMVJxdRtO0LEMUX4svkboOE/nmsJAM27kNLiTGdi6EY/+nZTqpTWWB3tFw/tqUNPbO4W/edYfOfy5hdvAs5s9mTGcLZXERxhjC3WNyd6g1QnGHPrJB3wwH1iHzss8sMfGi7Vfj/+zGBHcoXC/PLbaur03S8W//8sOpfFNY4tVffntn5/9oUsUQQmhQx+oaya4pZxq3dh+4v+aTyV06SKnSK7vX9c76/3/98Sd3bTaMj7/aM+rdLVxB4Sh4pwSrX5mFxMEos2+LEye1FCn3x/MevVtcSDhPN+EGbnxHJBc+QCYeBivHyTil6q0eJZK1v8Lyh9XWyt0BzxZKw+liW1s76jvIE5OP0fXK7puZXmNv7aK5Z0JxkzeKyxCjjQ2d4J4SQT5P0/yylRBCyHI6PXIMbOfa7az/HJ0MMc3uz+U3LcYgbE+dX6HjHWTxDIHhrHNP3kIeu1tb7G9Q5ti9xPcxHWnwPaJ3Sz9sHzoMnpY8PLNsRzNXQgghhBBCCHEG9HIlhBBCCCGEEGfgRRqFGGLIs4dD6MLBqTcm1KPcb+oW6BxKUCaULh3hHEinQcqsONXH+lEiOJVNcZpxtzPHm6E3qcp2a9tXC1OOTiKIqc+0IBkIYfqdfIkjobg4p2nuNLNt6lQ5X57+PS44D44L8jonmV1KAkyLqmzZLbBzx8zLCtlO00KdFtPmPed46ZyB8P84nJ6ndEuywIuThRC2j4lEe8RiNZgDXw/JHsxbQ1X6GCjw+zaUVUTrb4vB+lEmjM4r27+sINkr7djf6RB4jwSQ0frjEELYon//8bWd6wqJV4ve+u3b+1ur69Fu8KaA89rG6vrD3747lf/Hf/83d+0K39XP8fdT+X+9/3gqt509G7ZrcyQUlyctOKm5Z+yS21pwXevieVPieXns/MF8BjB/MRR3oZmMl+4hs8qOJmXNEPubCsM5nIvPjeZgzpVtAwe4ztpaGf21r5kUGJ/lg22v0daa4OXD4rK48Sa2p4V9PotxyKqL1Qr7WfzevJmX13o3bNungxNyi3HHoTG59ODV3SF0HJ+gTXJsjPmgDeq6225nt69KugWiPcbpeGb+m+OYxH2fT9ufGbJo5koIIYQQQgghzoBeroQQQgghhBDiDLzYuuhpSpEOgXk2X6bsbTpDnmV26aKw6UTKCpem7kcmgWxturxGouED5H5dZVOUzmUt+ESuBWR+FcorTDNSYsiEa37K8Z+TN7mjJRH8ijA13xdI4p6D0/ALMjjG+tRN87QdU+3DOO/wR5ndZ9JTtJWlxJleysI2N5/Ij8m6fbJt38iXpAhfUhYXIo6hyB+kQR1kfl326VQeCkghTEUUiokcLx9vbL9kEqOYm/MY+/ZyfH0qU9pXd5AUNZD/mYIkHGsktFx5J7U3P1mffP3a+u23N29O5fd//3Aqfzj8diq/+sHO+yOSCH8YTSa5fmPn/CNnMs0QruDWRllXX8N9EYleMzxXxKVI4ckl0PcplPLBCRW75IXvg5j4ekzsS2dP69RGOZ7ka8hdtytzOeshhRqYkHUSJkO0dlSjj99TZjVa/9u2SGK9t/ZYI2lxBhe3HVwsrzf+4tdIaJzj+bBK1na6K5Nl9Ue5BV6aFNDfMP6wD2N/pHRwIr/nmL3CuHnA85y/O8cCBSSFHF+0cKVsIGkN97Z/O1GPDujwh4QPE8c3tplJhHdra1MV6pSj7CSMwY+TOEbLOOeE8RdWGi3KhYlmroQQQgghhBDiDOjlSgghhBBCCCHOwD8hC/xz1zTKhKbTaJyydHI8TPWt4czHsncAshN3cBe830OmYjPhoen8lDWde+haWKwo/5ufWnyppGm6z+LxS2VxcZ5moNPMthCC030syumCl/zRSZIOk94JEMmwKRdETZjgN4ekg2q8cZIE00tZ7BpOCpNzmt+6hBH3xO1lQUkvk2pPkhM6OaRtV6x/O2JMIRYP/WQBqVIaLWEvZRF5ZlLrrPC/b4u+s6ve236QCIYWDqx0FAx23oSko7/dWfzerCxh77/8zdpNrLxsafsjkqduTY7y8fDLqfx/35u+8WODvn1jz5sax3a97d+gTr98wsMkhHCIcML61bYfc7vGq8y+26uo/8/8q+GcXScuYplbmrDgpMpk2tic4bwb9Ps3V69OZTqhdQk62DjRTEXr7ynxOtINtoej8sG2399be2Ti4AKV3UAO9mrjh4VXSCJcoo9fZSbFCt+ZpDYb/wEZvfiHWfq2pw7GT8TJYDzSLZDjggEyuhIxHuclghx70EFzgF66yCyOqxxWtCGEJtL5G3VnPGH5REwcl8+PT8piXoY9Tv7md+LG+APH+5ASh/zx3+Uxi3p6IYQQQgghhDgDerkSQgghhBBCiDOglyshhBBCCCGEOAMvWnMVYzzpGXP4Erq1Q7Sa9p7in53rCWevCNvzzcY0vbudaff9sfM66AFW1QPWqtSwLg3Br2/ZrFezZdqvc82MX4vFLNL/JFqG8s140ikv6pUXfpvn1lwxTmjxzzVQbs1VxrjiWRHrjHvEt/cIDm6dAM/F+pXIYs5Yp10xdcxf0h5CmFq2c32h7eOWswVxaVKKIY0Pv+UmYB3raL/dAZb/bWa/dZusHEIINWykU7I4KDusv2hxjd409rGwfrg72rHv3yFtxs3Pp/Lf/kbLc6+jf//eYvzdv9/aeQ+/n8q/HbGWMNm56vqPU/lQ2XnebOz72HVmM51/hK1wCOHd3mzdP32y7+eHytbV/HRj5e+ure7/M4hvhVtHy3Xik37cW7Gjzw1cy2qxwjUYjNJNCQtq9J8DxhkNxy+jrwdTyIx4VtRYr1h3FpvN3trR/s7WD7YY/2yw5nJTWf2uN76dX2/tb1py85gq47od/xwQlyE9Pkid5wGXJ2E9nl9K6OdUcvj+c1zPdpFh7DFivffY+7QYT/Q9YvRosVjXGJcPk7mdweJm6Oat2DmWP9xZLO9Ke1coI8YqWCsWmWJkUm++O/Cr4l491n71j8en6XgLaOZKCCGEEEIIIc6AXq6EEEIIIYQQ4gy8SEU5SScAAAfOSURBVBaYZTGstw/T2E4SV9CuMLr9cbQ7l7NiRy0qWKBvtybH4BQnZUnMBN3381PnAdOYtJQOIYSysr+3W5Mebnd27RWkTytn0c7M5bBypB7S2cZPbV4xFZnNT+1KK/X1SGFJFvjnEsGpLJBtooB8okyIP/z+Ll5dCoP5NACMMV57Oku9bAlv9WBMO+khTkZZIPenjHcqC8xZx7AAp+On/qji7KSUQtM9fNE3G0g9EKP3jVk390wL0E1s/kezJX8VrO8s2LdXkA6ij2uTyTjqFjKp0WROR/TtvxzNzvw4kUwdYDud1VYeca6xNKliGUw6mMYPduzqrV1jb33+/rU9P95WXjL1X17ZMdc72++msjp+b47y4dW1P15cmLjQd6PsFVOTnorxD0kS0wpEdt3o6TKIigpIrAv0saWzd0dqg9HLloYGcY0lD11n25ujxV99MMlUvYf9Otrm1dr69FdXNt65ubK2HEII27XFbOR4prJ7XSOFTpAV+7eD8vtxXn//2cqBhaFOdLJAez4kBDxTxXSdbW/QD9fHHmWL0bbxEuuugWQQH7n0MojfPVIMlG7ZgdWDY5AV5OTTtEj8DjpIEuvankEHtKMnOe4oWaAQQgghhBBCXBa9XAkhhBBCCCHEGXihLDAL68eM9k7+hzLlVNkz2eh9hnMrUl5H+d/VlclCCmRdbluTfnQdJYI2r9j1yNi8msoC7Vw7SAG3G3OUKuCgQjmVk0Zi+pHTqc/Ly5amFKUF/GY8xWVa+g2pAaHLlD9NgXhIlMvBVY3OS27qO7FtURYIRyZKUiH3m7rXjIXVt4L0KzkvK4v1spw/F69XwWVqTVfN0ru4FZQxUqHA79ZpARX3lyaGEPJH1UOCbC5DX7uG81Ic7TfsIdN7OIHJJ/K19Z3jClJAyCqOR8hJesQ14wHxXrbW5w+/wp0t93KSvKxRtrq3PeSGve0TIUfJM9v/FWQtHZ4fHztzW0tXkD+FEG52dvxP/2qugJuN7ZdBev5p+h2Ks5OSuQhH13dTG4Ui5fm97z8T1XkDD1rowyDvLilJQnlLBzM48x0hizr21jZDmMjG4cTWHW2/fm8xXuJet25cY3F5tV1ju5WvJ7LACs8X14/zeYTn1xu4Y4rLw7H4kuudl8E9J9vkuTCowW9NCz2OW3os3ekQo22L7ZC39hOZeYB7YM4xEKrB+6AL4X20Pnq9tnHIBu2rxPiHyyUez2z1gqvtEfLa+1uTId7vHyTxdHmeopkrIYQQQgghhDgDerkSQgghhBBCiDPwsiTCWTy5+bnEZXDOoaOOm4r0Wfuc+18/zDuUFQWnBm06m9K8Fk5TlAKyPHJuf+Ie5BzUVkyOCvkfp79zOrnNJ0ZNC7LAaWLaRbmZSwonqdTX5E/dAp0sYskZ0yfQzTG9TtlTzOan89lWYqAkg85/jEOcfyILLJDgcoRT4dL95QUTYhpsJ5T/rZhQeOLESbnwxJsHRckCvyYxxbB6lKa2jX3fVW6/w6oyqdsYre9sxzt3rrGG8xOUeg1ivN9bbNaI9+tXFgPfrWz77dGkF6G26+VWpZBNHltlYR+uR5NAhWjnyqI5PZWDHV8j4eQQ4VZlKqnwCS60h4/+O7jDua6/s4NWORzWOmsvXacYvzwpPD1PfXf4BTL8qdMdTwD3tYx9euL4x8p05nMSQYRvhv45H+2D2E+cVyHFbkZut9hCsw0FEsOuV5RJWYxeobyGI2BVTeXdVi8+ayiFz2iZODleXIJ4SiIclqSATsPJGF/ug9KCerB3Ca55PTdKwOXmxy1sN8UkGXyGsQeG3062y3cKugIyZrlsiEmRucRi+j7ipplc27bNdAg9yQHlFiiEEEIIIYQQl0UvV0IIIYQQQghxBl4mCwzxJA/iVPiQFiRtzzjlJUj1lhxOOF1PGVSMLM/vwzKvxaRiIXj5VkTiV7oWugTBnBZfnF1duu8vdQsU/xmIi3/42Fh01qQs0JnOxNmymxJ325+p40I9fLLvhfjGeRaTGS9IY6fHL8oCxdclZiErH6VzkLQVkHNTVlGWpvdbFT4BboM4oMQwgzNfPVrcrN68PpX/648m33u7tnj4VNux73+3JMV391aPYwvpXwjO0o3ucNdoVHR8bXOT+TWDua1FuAuuK7j6ZXbtOPj/j6wPcK/q6eiG76NmElYv9xIXIIZZhfE0cej8oX6fHFKnkQnW3TIAjEGyebkgVNVhDffIjH3hCJkUkmw/1IMSXsiyIcXuC7h3or+uVlamu+sGZScLLH07598+kb3tw2Ue8RnZmTgfT9/ykisgxxduZcNz7SC5HU9FOgGOTHzNcUTO5wbkqiuLsy267ukSBjZat9SBYx10vwlSdkpf6Sy+Wlk7yl1CcO/yF9Ggc7YvHL/dcvnSQ/Bn+XJ/rpkrIYQQQgghhDgDerkSQgghhBBCiDMQP5+ae2bnGH8PIfyfy1VHvID/llL64VtX4v8nFN9/KRTfF0Ax/pdCMX4BFON/KRTjF0Ax/pdhMb5f9HIlhBBCCCGEEGIeyQKFEEIIIYQQ4gzo5UoIIYQQQgghzoBeroQQQgghhBDiDOjlSgghhBBCCCHOgF6uhBBCCCGEEOIM6OVKCCGEEEIIIc6AXq6EEEIIIYQQ4gzo5UoIIYQQQgghzoBeroQQQgghhBDiDPw/p0+7JO76kr0AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 1152x288 with 10 Axes>"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "plot_sample(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JtZNyab1GfBr"
      },
      "source": [
        "Convert the train, test and validation data to greyscale"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0tE_Uv1R0myw",
        "outputId": "71fc4647-b3be-4d33-affd-495205f4abc5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(32, 32, 3, 73257)\n",
            "(73257, 32, 32, 1)\n"
          ]
        }
      ],
      "source": [
        "train_data_grey = train_data.mean(axis=2)\n",
        "train_data_grey = np.expand_dims(train_data_grey,axis=2)\n",
        "train_data_grey = np.moveaxis(train_data_grey, 3, 0)\n",
        "\n",
        "test_data_grey = test_data.mean(axis=2)\n",
        "test_data_grey = np.expand_dims(test_data_grey,axis=2)\n",
        "test_data_grey = np.moveaxis(test_data_grey, 3, 0)\n",
        "\n",
        "validation_data_grey = validation_data.mean(axis=2)\n",
        "validation_data_grey = np.expand_dims(validation_data_grey,axis=2)\n",
        "validation_data_grey = np.moveaxis(validation_data_grey, 3, 0)\n",
        "\n",
        "print(train_data.shape)\n",
        "print(train_data_grey.shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5x_7ZufGGoAa"
      },
      "source": [
        "Set the 10th class to zero (represents '0' digit)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fdN10zRS0myx"
      },
      "outputs": [],
      "source": [
        "train_targets[train_targets == 10] = 0\n",
        "validation_targets[validation_targets == 10] = 0\n",
        "test_targets[test_targets == 10] = 0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8dSQ_i-n0myx"
      },
      "outputs": [],
      "source": [
        "def plot_images_grey(images, nrows, ncols, cls_true):\n",
        "    fig, axes = plt.subplots(nrows, ncols, figsize=(16, 2*nrows))\n",
        "    for i, ax in enumerate(axes.flat): \n",
        "        # Pretty string with actual label\n",
        "        true_number = ''.join(str(x) for x in cls_true[i] if x != 10)\n",
        "        title = \"Label: {0}\".format(true_number)\n",
        "        ax.imshow(images[i,:,:,0],cmap='gray', vmin=0, vmax=255)\n",
        "        ax.set_title(title)   \n",
        "        ax.set_xticks([]); ax.set_yticks([])\n",
        "\n",
        "def plot_sample_grey(num_sample=10):\n",
        "    idxs = sorted(random.sample(range(train_targets.shape[0]),num_sample))\n",
        "    plot_images_grey(train_data_grey[idxs,:,:,:],2,math.ceil(num_sample/2),train_targets[idxs])    \n",
        "    print(idxs)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "id": "YEtUCVGnEQLJ",
        "outputId": "67da4b9d-bffc-4e10-829c-03a8345f11b7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[7038, 31426, 31989, 36730, 38065, 41154, 46719, 57394, 66873, 69878]\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1cAAAD7CAYAAACG9G74AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO29e4xlyX3f96vpmdmZ3Xn2PHYeOztLUdQyMiNvQAUSQCYWYgmyHTgOwNCwIcYSZPtfP0CKCIwoshXZQZQoEmzDiiU5VkwGsAwrgACbkiDRFiyZthWFDkUKAUNp37M7j573zL5mpit/dN/a76093zN17j23+/bO5wM0UH37POrc8ztVdfr3rW+lnHMAAAAAAADAfOza7goAAAAAAAC8H+DlCgAAAAAAYAR4uQIAAAAAABgBXq4AAAAAAABGgJcrAAAAAACAEeDlCgAAAAAAYATeVy9XKaXfSCn9ha3eF2CrIMbh/Q4xDu9niG94v0OML+nLVUrpxZTSd293PRwppe9PKf3fKaVbKaVXU0o/nlLavd31gp3Dssd4RERK6a+mlC5uxvn/llJ6bLvrBDuHnRDjE1JKX0wpZdpxaGUnxHdK6ZtSSv8spXQ7pbSWUvrx7a4T7ByWPcaXeSy+lC9XO4DHI+KvRMTxiPiOiPijEfGZba0RwIiklL43Iv6b2Ijt8xHxTRHxN7a1UgALIKX0fRGxZ7vrATAmKaW9EfFrEfEvIuJURDwVEZ/f1koBjMvSjsV31MtVSuno5n9hrqSUrm+Wn6o2+2BK6bc332R/KaW0Kvt/Z0rpSymlGymlr6SUvmuWeuScfzrn/Js553dyzhci4v+IiI/NfmUAGyxLjEfE90fEP8g5/17O+XpE/PcR8QMzHgugsEQxHimlwxHxIxHx2VmPAaAsUXz/QES8lnP+X3LOd3POb+Wcf3fGYwEUliXGl3ksvqNermKjvv8wNv6T/nREvBkRf7fa5s9FxA9GxOmIuB8RfzsiIqV0NiL+eUT8WESsxsbb7S+mlE7UJ0kpPb15059urNd/GhG/N/hqAN7LssT4H4qIr8jvX4mIJ1NKx2a8LoAJyxLjERF/KyJ+OiIuznNBAMKyxPd3RsSLKaVf3pQE/kZK6T+c++oAlifGa5ZmLL6jXq5yzldzzr+Yc34j53w7Iv5mRPyRarPP5Zy/lnO+GxE/HBF/OqW0EhGfiogv5Jy/kHNezzn/WkT8TkT8iY7zvJxzPpJzfvlhdUop/WBEfHtE/M9zXh7AMsX4gYi4Kb9PygfnuDyApYnxlNK3x8Z/Of/OiJcHjzjLEt+xIQP8M7ExqD0TGwPaX9qUCwLMzBLFeGHZxuI76uUqpfR4Sunvp5ReSindioh/FRFHNm/YhFek/FJsaOmPx8Yb9ic334JvpJRuRMTHY+Otetb6/JcR8T9ExB/POa/NehyACUsU43ci4pD8PinfnuFYAIVliPGU0q6I+HsR8ZdzzvfnuR4AZRnie5M3I+K3cs6/nHN+JzYGncci4j+Y4VgAhSWK8Ul9lm4svhSuGgP4dEQ8GxHfkXO+mFJ6LiL+fUQk2eaclJ+OiHsRsRYbN/pzOee/OEZFUkp/LCJ+NiL+85zzV8c4JkAsT4z/XkT84Yj4J5u//+GIuJRzvjrCseHRZhli/FBs/JfzF1JKERGTQcGrKaVP5px/c87jw6PLMsR3RMTvxpLMP4H3HcsS40s7Fl/mzNWelNI++dkdG5KkNyPixubkuB/p2O9TKaVvTSk9HhE/GhH/NOf8IDZccv5kSul7U0orm8f8ro5JeA8lpfSfxcbEuU/knH975iuER52ljfGI+EcR8ec3z3MkIv7biPj5WS4SHmmWNcZvxoZU6rnNn4kk5aMR8e+GXyY8oixrfMfmsb4zpfTdmxmFvxIbg9v/d5YLhUeWpY3xZR6LL/PL1Rdi4+ZNfv56RPxUROyPjQbi30bEr3Ts97nYGARejIh9EfGXIiJyzq9ExJ+KiL8WEVdi4+35h6LjO0gbk+juJD+J7ocj4nBEfGFzuzsppV+e6SrhUWZpYzzn/CsR8eMR8S8j4uXYSOt3NaAAfSxljOcNLk5+No8VsZGdfWfWi4VHjqWM781jfT025rf8rxFxffO4/wXxDQNZ2hiPJR6Lp5zzdtcBAAAAAABgx7PMmSsAAAAAAIAdAy9XAAAAAAAAI8DLFQAAAAAAwAjwcgUAAAAAADACvFwBAAAAAACMwKBFhA8dOpRPnjz5ns83F2F8T3kW1L2w5VhuGz3O+vq63b7lHHosLe/atavz81bctbqybv+Nb3xjLed8YvBJwbJ79+68d+/eiJjtfo6F3nONsXmfM30OXEzPc92tdWq5vtu3bxPfC2D//v358OHDEeHvu8aJsrKyMvV7SzwOja2h7fG8DL2GedFjXb58mRhfAHv27Mn79u1r3n6W++tiZejnW4271nnHL467d+8S4wvgwIEDeXV1NSLa2vG+9r3lPi5iPNT3TAx9jlz9ZhmTDNnmxo0b8cYbb3TuPOjl6uTJk/ETP/ET7/lcO93du7sP2XqR9+7d6zyWG4TpNho0epy33367s64REY899ljncR88eNB5LD3HZCAeMX1zdV+lDgA9ll6fdgzuBe57vud7Xuo8CczM3r1749lnn42IiPv375fP9T65ezZL5+niRGNSY2HPnj2d22g9avQ63nnn3eVN9JnQz3V7pe8cLdvo3/S50bJe66//+q8T3wvg8OHD8alPfSoipmP5rbfeKuW7d++WsraXBw4cmDqW3jtth/VZ0HjSsvsnlZ7P/WOpfm7095Z/WLnzub5Lv6f63C3/FHP7/+RP/iQxvgD27dsXH/3oRyOi7Z8G7vP6bxo3rqzx5MotbWmNG0C3/HPAxZ97burvw30/Lf84+dKXvkSML4DV1dX47Gc/GxHT7aq249rHv/nmm53liOk4cGNgt42j5WWnfg5anhf93PUJeo56vN+1TYRv+91zPtn/Z3/2Zzv3i0AWCAAAAAAAMAqDMlcppfL25v7z2CpdclI9/W+o/qde30z17VzfwvUNXt/a3fHr69Dzuf+0K3pczW65/9DW34d7I3f1c5kOGIfdu3fH0aNHI2I6ft54441SdlmsPtw+Lf+B1Jg8fvx4Kas81/03J2I6Fm/evFnKN27cKOXbt2+Xsj5bY8ae1lGfp2PHjpXykSNH5joHPJycc8lUuv94alvWJ31ukWi4Ns5lhpVWZYBTFij6HM0j6erLOrT8R3eWrAUsB+7+tmS0XNkdv0/l4urR1w887POhmQnYfib3zKlTtE3XMcydO3emjqP7u8yVa1cfltmpP9cY1Ta572/uedExml6T1lWP03fulveALlVF37NCSw8AAAAAADACvFwBAAAAAACMwCBZoGNehyf3N01X6iTrq1evlrKTbOkx3WS1CJ8qfOKJJ0pZJ3I7AwxX1nP3uW25CXxO6gjjs3///njuueciYvp7v3LlSilfu3atlPV+1PdGU/LOJELT7hpjTz31VCmfPn26lCcubxHT5g+tUlyV0Kos8PLly6V86dKlUtZUu8axu54+meT+/ftL+ezZs6V86tSpUj506JDdH8bhwYMHRQaqMavSOjfRvZZHO/lES9vrZIFOZtI34V6fNS1rm6rPi5Zd+zyvO9ZQ11sYl67vXOPJyezq++4MVjRuXAy5z2cxQprH0EKfZz33LFLvFoMQWDw559Jmu2kyOg1A+3L9PGJaSthiUuLad2dOpWNm3V4/j4g4ePBg5980Zp1RnY7LdBvto7ROtTmTPp86Vnn88cc76z6pU6+Jl/0LAAAAAAAANMPLFQAAAAAAwAgMlgVOUs9D09StMgvdR9OXr7/+eim/+uqrpXz9+vVSdi4hmq7sW0BNt1NntqeffrqUVbLl5FHO8bBO27sUp6br3XpEMD579+6NM2fORMR0XGiKWu/nrVu3OssRfn0f5wSoKfETJ95dc/HJJ58sZU1Xt0pDNG2t++s16bFcjDk3zD401a6SRpUCqlvgkIU/YTbu3btX2lK9j3p/NYZczEV4h9NZ1wyJmH4mtNwnwVW3S7dGl8afk4JruzvvukROMoVEcPGklDq/55a1d/ruj5P5uXhqkQ6OKQt0skeNa9cXKfXnLRJeJIJby/r6epEDtqxf2bfOle7T4mbspK8qodP2Vl2AVfo/WQR5gjog67G0Ttq+63uAyh5VIqhovNaSRB3v6xjfuTJP6qT943vOZ/8CAAAAAAAAzfByBQAAAAAAMAIzuwUOXYixxqUc1f1vbW2tlFUWqFJATWlqqk/d/lSmV8ueVFKiZZXJqLOIypgUTZ13uYrU20R4BxdN2/bJYWB8uiRAev+dxKKOK40f5xyoMjiNVxe7+pzMsoCxk1/p9WnaXuuqz6WTPNZoylzPoSl/pVVuCLNz//790q62uIW5RdojfDvnPneybe0LnNRQ2z5tpyOmnS+dw6X2Dc7VSnESsrp9cM/hUEc3WE6cfNVJo1rK88oCnTSvZbFrJy9kEeGdRc653DMda2h7rX12nyxQ93ftnotlbVdVCqiOwJOpFhHTUx5Uclf/ru2yxqbKAnWsomOsl19+uXN77XO0rhHTU38+9KEPdda9a4HlvqkMZK4AAAAAAABGgJcrAAAAAACAERhFFujKznmkRtOSKvnT9J5+rrIptyDp0aNHO89Vp0RfeeWVUtYFVFWm98ILL5SySp00xekWG3NOJxHTbicqgXSOL8gCF0tKqUg2NI7d4r16b/pkeipVUmmVxpLGq6a7W2SB+vzUuMWGNUb1eVLXQk216/mcs2UtRdFjqRuPXp/WY5ZFLWEY6jLVIv9xiwtHTMeWxrLGk5NSOXQbPbe2nfUimNo3qGTQPV8ti2O657m+hqGOaUiutpYWqZyT2fXt3zL+aZHszSILbKlHy8Lc7tmcxflvFkdNmJ1du3aV9lf7aZUCqoOxm3YSMd2uO/lfy2K82saeP3++lD/84Q+Xsrofq4tgxPQY2i0gr2Udl7lpDi+++GIpa1zX03ueffbZzrq7cf2kHs4ZN4LMFQAAAAAAwCjwcgUAAAAAADACvFwBAAAAAACMwOA5VxMtZIsuV7epbXydplLnp1y9erWUVSeq2k7VcJ4+fbqU3VyT2gbazWlS7b7OxdJ5JKrp17kx7vhqGRwRcfny5VLWVaW1vnoOnbcC45NzLvM8nNWuxpXGQj0fTmNa760+E3pvdR6SzlnRc+szpDpr1VPX2nfVNattqm7nrKn1cy07nXF9bncsZRZ7eZidlZWVokl39tDOgr+OcWfTrvfUWZK7ORqq/9fja3tct6P6N+0/FK2f0/C7uip1vVvmbzHPaudQt0HONl3jQLdxZbc8wSxzrpShc99ZFuD9gY5VtF3WualadsvBRLS1dW68oGMVHZerrbp6E+g4p25Lta/Rv+l8Kh3D6LwpnTOlc3X1u9G+oX4PcEvF6Bimq8/qm2tI5goAAAAAAGAEeLkCAAAAAAAYgZmt2GdJbbdQpywnaFpd5ViaGlSZlR5HU4N1XTWtqWU9llr9qiRF0+0q/1Npi55brTHrY+l2Kv87d+5cKa+urgYsjgcPHpTUscaY4mK9/lxjwFmduthzkhNN7WtMquywtorWtLam1LW+mnbXuNd9NdWu26tcpZYL6rPp7Ox1f6zYF8/KykppR5xsU+NMpdm1HE/vl7Nsd7Gsnzu5q8r9tKzbRLQv+dF17qEyqVZZoItlLKu3hq7pC1p296cvBvTetVhWO9ntLLbsLfHk7NTdEgN9S3g4WpYrgMWzvr5exioqd2uRAtZxpnGj/YCLLY1xldC5/l6fA61rbQmv/YvWQ4+lY2MdM+kyTHoOnTJx4cKFUq6/A62Lfm9uelFTP/PQLQAAAAAAAOCh8HIFAAAAAAAwAoNlgZPU2LxuSE6Ooa4d6gSoaU11H9GUoZOaONee+m96bpVQacpQU46aqnffgdZDpS0R0ylcTbWqo4q6vSELXCz379+fkkFN0BS1SinW1tZKuZZMaZy0xKJ+7qRyem4XkzX6fGgaXePNyVRc/ZyUpZYk6vfmrm+opAvmY/fu3UW+4WTQ6tqkcuc6zjSutexksA7dXs+tMmr9vI4Tbas1nlz8KS0yvT7plnO9bZUPw2Lokqy1yAJdnETM5xbYIgvsw7WZztFNzzFUotrXDrv2mrZ7a9EpDNouq6TNxUmNc7J08aTtqo5V1SFQ5Xu6jY55tU2PmHbl1vGJ9i06XUPH61o/HfOoXFDH33W86jOif3PPbZOE/KFbAAAAAAAAwEPh5QoAAAAAAGAEZnYLbFm8ri915rZT6ZumBjXFqa4kTrLl0uJ1nXR/55SmshMta7pSz61pVreIWf23lgWJcedZLPfu3SuOMirz05jRe67b1JLPlthwi/QqznVQ5VoqBaglIJp617/ps6U4KYuTC7jF/iKmnxU9lkvBIy1ZPLt27Sptmzo9qcRC75W2gzUa4xqP+rlzVdO40edDY1lj17XtdR1d3Vuc3pzbmlKfu6W/W5SzLjycMdsUd69bpFQtDoF9baGLx5ZFrN02TiZZn8tJAXF33T5yzqWt1LZX28kW+WiEb5NczGk/r/2GTmFRaZ72M31SRTem0XbcxZzGpW6vCxvrGKuWJOrvOuVC+zKt76Qefe0LmSsAAAAAAIAR4OUKAAAAAABgBAbLAidpsKFSwDoF6GQaKiXSlKOmDJ3sTtN5mn7UOtWpTt3flTUNqqlFlxJ030ftnKXXqudQRxQtswDlYrl371689tprEeFdzjQt7RY/jZiODSf/dM6BTnrhHKO0XNfDLW6tx3KSgRa3K7cYccR0fOtxWxa4hMWQUir3xS0irPGq1O2Pc1ZzLoJOGqpSFicN0TipJbQtfYnu0+KO5eiTgcziAgfjk3Mu92leWeA8Ms6hctBWibTbzkn7nLS8Ve7n9kfSvXz0ufd2bVPj2msn/9d21cmz9XzqHKiOghHT7svqKuhw8ad1deN1daKNmF5g2E2Z0H5x0jfV4y2FHgAAAAAAAGAEeLkCAAAAAAAYgZndAhWXkuuTC7q0oZNKqXTEpQPd4l+arqzrce3atc6yLmimaMqwxflNpSmaEq1/V7mYk3zhzrNYVlZWisuNxommqFV6qmh8RrQtlKux5J6HoRLBWtLlJF6OFucr93ktpdTr63LaqeuOtGTxpJRK++TuT4vjVL1/yyLRGosaA/q5ygL1cyflq8/tJCHaB+ixWuTts8ifcAXcXiZx5/pSR31/h7ZJLeMfFxt9ceb+1jL9oUWG3dcOu+2QcW8vk1hz8v3W+6ZSQP2bHkvbUif71n7DOXTr86GL2EdMuw22OFy2TMtRtG+5efPm1N+c8626QOv1TfomZIEAAAAAAAALhpcrAAAAAACAERgkC0wpFUmFS/W1Lr7YkrJskSLp5yqtc9ur9C8i4vXXXy9ldStRNF2pUj5NlbrvwC1oFjEte1Hp2SuvvFLKmq48c+ZMZ/1gHA4cOBAf//jHI2Ja/nflypVSvnr1ainrIsJ1mllTyxoPej/dwsEOtxixSk9r6ajGmKbR3cLIziXRPZdOOlAfS8+t1+GkabAYdBFh597nZCI1LYtMunZRny9dwNHJLPSYGu8Rvj/QfVokWi7+3HNT/9633cM+h3GZxJqTwbnFQWucHK8lnlrK7hlqXcjXxZ+jRV7Yd25XDyTdW0tKqbTfbipNi3tkhF+E2En+79y5U8o6BtJx0unTp0t5dXW1lN0UoojpxYZ1POz6pr5jTdBr0z5H6x0xPfbXMZ6+H3S5H+vxaxjNAAAAAAAAjAAvVwAAAAAAACMw2C1wkg5zcgonKanlEEMXD3WuVU5+pcfXdOCrr746dVyV4OnCYpqiPHnyZCnrwmcqO3Gubpqyrd0C3QLIly9fLmXnkgjjs2vXrnLfVeLm7qfGW+0WqGl09xwMvZ/zSo1a9l9UjLUurA2LRaXdGtfajqpMTz/vc+lzDnyKWyBYz6fPiltQvpaf6rOn/Y/i+ivnNOikV30SsqGSMFgMbhHhFgey1vurOIfVlgWqW10pWxyStxoWgN85tC5o7WJL22uV1F28eLGUn3/++VLW9lrRqTu187IeV8/X8uxoWfsZHdOr1FCljXVdtD/SPvLs2bOlPBnLu4WaI8hcAQAAAAAAjAIvVwAAAAAAACMwWJMzkV24RSNbcfu4hSOd7ETT0ZrOu379eimrzK52CdFFwvQcKgU8f/585+duEU0nVaylY05CqWVNZV64cCFgsUzuo0qP9H7o53qf63h2qeV5pH3umZlXGjLU5axVYtDiyuYkWrA4JvdC21qVT2gbqdvU8gyVfqgrqpMIaXunsg99PrQN1mOqpLquh7bhLW5Si5LvzeNICIuh5T703R+3nfb9bpziFr5uWfy0jj+3/9BF31ueg1aI6+0j5/xQR0yl7145V1/n0qdj0kuXLpWyyrj1mNrWHzx40NZV23Gth7b9LfJzlfzp2F/fCfRcEdPjNZ0SpOfWRY8n7wHaDtTwRAAAAAAAAIwAL1cAAAAAAAAjMEgWqKnIoQsu1tu3pB9dil23V5cPlc29/PLLpazpwNppStN+utiZLtj75JNPlrKmDJ07oUppNEWp9Yjw8hs9h7qr1ItnwrjknIt0U+NVY0bvsy4uXae4VeKmMVov4PcwdpK7WKtEUNHvo5bNwvhojGtcqixQF8RWSUft7OgkE3qv9Z5qG+dcCFUKqLJDPX69cKNKPNzz1SKTUlpkXPV2znUOt8CtJaXU+T23SNf6HDHdNAD93JWVFtfCVrQebqrGUDfDPmmki3fYWnLOpR1scZRulQW23F8do2vb66at6DhX+wwd20ZMt/3qyn348OHObdyUIF3496WXXipllQhqfxcxfU063tPvTT+f1L2vTSFzBQAAAAAAMAK8XAEAAAAAAIzAYLfASSpO5Rcti3z1yQI1na1pP91GJXUqW9FFzF577bXOz1VOp25/ERHnzp0rZZV5qauJc45zLi3qpqL1uHLlytS59ftRCYzKCTQlWqdRYVzW19eLjNMtbqufq1RJ5VMR09KlloVyWxaGHOrk14ruo8+1k/i6Z6CvXs4NU8tDJZMwnPX19RKrblHFWnY3QdvmiOn2yMk1VG6hx9Wy7utkgX1uhC3Pi4s5xUlinMSqPkfLIsQ4Ym4ND/ueXTtZt9VOduemLLTIAl1safvXJzN3n8/jjNjnANsia0XuuvVMYsS1ba0uwu5ZcO2kHlcliToGunr1auf22ufoNJyIiNOnT5eyjr91O33WnJO2SgFVInjt2rVSrt0CFe1r9Pp0n8n1uUXrI8hcAQAAAAAAjAIvVwAAAAAAACPAyxUAAAAAAMAIDJ5zNUG1hk736z6P8Jp0/Xxtba2UdSVonXOlFozK2bNnS1nnWdVzrvR31Uir/tnNQXC27KoF1bliOq8hYnqOl1pP6lwDZ0MJ43P//v2ipdV76/T1OpevtsnXWOrTs0/QeHPPkNP8t87jaJkb2aKzbp0n1XLdbs4BLIYHDx4Ua1xtp1Qvr/dK51XpHMOIttjUc6gVu55P57S64yi1zl3jRvd3cwld2cV+35yTofMgmZuyeFJKnW3dLHNWW+ZZDZ1zpfHrYrfGzZ9x8a6x7J5Tt2/fvELmDy4HOecSE64Pdf1p3c65OVsap/q5jkM1BnQMpG26a//q8azOrdJx+dGjR0tZr1U9DL7+9a+X8h/8wR+Ustqv980r1uvTvkn7r1deeaWUJ32oLmdSQ+YKAAAAAABgBHi5AgAAAAAAGIHBssBJWk5T4S7d7lKM9e/O7lBtFDW9p2hq8cknnyxlTSuq/K5PvuXkTk6ypdtrelBXpNa0Yv09ObtJZwM/i802tKOSKb0fKtNssVWPaLNHb5EOtUhZ+rbXtL2T9jlabIJbrV9b5F5Ofgvj8eDBgyJV1rZJ2y+VAmrsazsaMd2WurZTZdsqy9DzaXunOBv3Wgquv6ukQ9t23V/7G7f0gDKLLBC2l642t8WCv95vqI25lrXN03O7bfpsnV2cuXO7aRct/Ux9rpb+C7YWlb66flrjqXXs6GJTP9cxt06fcNNZdMrLsWPHSlmn7tS/Hzp0qJT12dFpNs8//3wpf+1rXyvl3//93y9lfYfQ6UT1s6bjbB2f6HZd05GwYgcAAAAAAFgwvFwBAAAAAACMwGBZ4CRV6JwAXYqylvzo7ypPUVdAlQKq05TKSDT9eObMmVLWVKSTRkV4iYjiJE1aJ7f6sx5f5TYR0y4oWnZpyb4UJMzPgwcPSupXpUaa+na0rmrvnpuh5VaphtvfyUaUFulgn3Oge57mOR/Mh8a4tk0aDxrvKuOoZYEay3qvnduSSgH13FrWNlXR/kLlGRERt27d6jy3ylf0cz2fc27D4W9nM6Qt0W3re+0k0K7s9m1xq9RY7JNYt8TjmNJy1++0SsJhMUzuRUs89D0PbizhxipONq7j8hMnTpSyyv1Onz5dyjqNJ8K7AqpruDr2feMb3yhllQi++uqrpax9hfYBtdOlOuHqNennXf1dX9yTuQIAAAAAABgBXq4AAAAAAABGYJAsUB1KWlKRTnIRMS27couBqbxO3dvUSaRFWqcykDo96tyiNAWoTiKaElU5ijuHO2aEX3TNyRhZZHWx5JxLzLrvWu/HvO6N88gCWySC9XYuha2fD12sUrfXZyBiWsrVItNBfrV4cs7lPjkHKJX/aZtau/rpvXcLEqsUUD93Cwprm6ro9tov1Pvodtr/aJ/hnAN1e3Ua7Hs+3PPlnDmJ8cWTc35oX+na0lp6P3QhaucsrJ9rzGksukVb62O1uBMOjbO+/qRFYqggEVw8OhZ301Za+vWINlmrcwfXz7WNVbdulQXqdJ16kW2dWqNyPp0qpO5/KhGs+4QJOsbW56t2LNbpRefOneusu34fk+dWx+01ZK4AAAAAAABGgJcrAAAAAACAERjsFtiVinTSOi3XboEqI1FZhy4SpulHde3QFJ46WznXKT13X8pbU4UuDaplTQm6hdWcZCViWjKjf9Nj6fdcywphXFQWqPejRVrXJ+PQ9LempvX+KxqjtdSu63z6/NWOlHosPbdzQHSLXbrj9MlWnROnPlssHKhY998AACAASURBVLy1pJRK+6LtjLapKgvUdrdegF3jQNumWu7Rtb3ed5V0uHbbLTpfb6d/c9JwF5dOzj2mzAnJ1OIZKgtU6vGBc/NzclLn6Ov2deW6P9F6aVzjsProMokJjQfX9mq70+c67aY9uHGojiNUQn7q1KlSVldA3b4e/1y8eLGUVfKn7n/6fqD9htZJ5Ynq/KffTS0LVHfD8+fPl/K3fMu3lHKXLFD70BoyVwAAAAAAACPAyxUAAAAAAMAIDJYFTtLpQ13M+hzNNPXu0poqI1THEHUa1HNrCk/le7W7h/6u51YJjEqtNK2pn2tZ05LqgKLXEBFx48aNUtZ0pzojatqxz5kExqFLLuKkPC1ywQgvMVXHTE3Vu1Szxphb5FTlXRHTDm8ute/crrrS4BFevlLLtdQFTnGyQJzUFs/u3buL7E8lf84VsLXNcRISJ5d2ciYnBeyTBervzgHRubG6mJtFvjfUVQ0WR5f0qeWe9I1TnIRPY9Y5pGq76rZ3C2hH+Jh17nBKS1z2yWCHPgvE/uJJKZWY0Bhw7XXfPXTSfhdn2q7qWFelgFrWhej1mdDxb0TE5cuXS1llga+99lopuyk+Ou7Rsbteg47F6+f8qaeeKuUPfvCDpfyRj3yklPUZnDz/bnpFBJkrAAAAAACAUeDlCgAAAAAAYAQGywInKUTnouPc9GpUkqIyFE0nappc04lra2ulrGlC5xDU56aiKUQnKVHJjKY41WFEj6OpUk0b1rJA/V1lgVpfvSbcAhfLyspKcZfR+z+LlMLhFhdVmYlz7NHY1fq5xagjpp3fnBTQLfrq3Cydo2DfQuFuIVUnPYDFsHv37rKQo5MyO6lRn7Oja/f1uE6+2uI6eOfOnVJWeUfEdJxpvOtClupYpXXSZ0ev28luazmJW1B7lsU8YRycW6C7p0otV3XSPo3ZlrGQ277FrbJGn5EWKeBW0CqRh/Fx02+0HW5xtKxx8m514NNFgU+fPl3K2vZqW69tdZ9zt7sO17fouFz7NX2+9BpqabmO8d14X889eVbdNKYIMlcAAAAAAACjwMsVAAAAAADACAzW4UzSYS61rWkyJ7OLmJYCaupNP3cuJprSU0mTcydzkqv6uLq/k+NpytEtOqzXc/Pmzc56REynQdU5RevRt/AZjItKppxMSu+Zi8kIv2CqW3xS77lK8JxEVOVMmnavY0SfOxe7Wj8nW3TPg1sgue+aXD36FuSD8eiSDLmFoV0M1Oh91DZcz+XcAt2C3XpMjZlauqXPlD63WlYZuh7LOQo6CVktzXayQNdOwHIwi2yuZZ+WbTR+Xbkep7gpAU5i7WJxK8AtcGtpkWH2SZtbFtR2DoHqbK1SQJXWabuvbWHttKf7nDlzpnM77RN0+7Nnz3Zuc+vWrVLWPk5l5hHTfYJzEO+SevfFOpkrAAAAAACAEeDlCgAAAAAAYAQG6cxyziW15uQQLh1dp7U11VcvfDrBOXHoOVQS4uRUWq6lg+rYp04mWl9Nfarkz7lLqQRFXarq70DrovVQKaGCW+Bi2b17d7nX6oij99k5/NULQLpFJnU7TU3r/VdZlT4Dev/1+XEugBFe6uSuw0n59HPnfFWj16THcm6IKnWExbC+vl7ui7vvTgZby0ncotQqsdD93SLYzr3KLTSsLpYR07Glba9KWVrcNfVZc3KrvkVmHa6vhMWgC6zWn09ocQ6M8Iv3bueC0fPUY97FsWE5WF9fL+2eW3jdSU7rPnuoy7Zr991C7fqsaR+gMsL6HOoa7hwG3dQi/Q4uXbpUyuoy3jdOcv1f15jO9VERZK4AAAAAAABGgZcrAAAAAACAERhsPzdJ97lUsXOsqdNnup1zCWtZLE/Tec6FTKnPpb871zXnKOWc3/TanHNWxLScRV1N9Lo15Ut6frHs2rWryAGdREjvmaara2mTSuJUxqTyK40ldYvUxak1FvQZ0vppuX5O3POodde6urJKVZ3Er5Zr3b59u7OsbkMuHQ+L4d69e3HlypWImJZY6Hfv2lQtR0xLZxW3KLAeV6V5bhFWbV9VQlvHSYuMUc/nJC5O8tfXjzmc8xuywK3hYbLA1r60ZQF5F7PO7bh16oTi5ImKO4dbqNjFcn189721jM9gMegUHTcOdfe9Hqu4mND2cJ4pKS0LzEdMt9Eq89Pr02Pp9jpNQsc2Oibpm3ag42zn7tzlLI4sEAAAAAAAYMHwcgUAAAAAADACvFwBAAAAAACMwKBJDimlTr2vm5+i1DpP1ei6/Z2O12mAnb10nx5Y51OpxtTp7/U6nIWwohrTI0eOTP1NNZxq965zWpyeG8ZH51ypxljvgbPPr1f8Vq2vixOdxzSZBxMxrTfWeS5urkjf/AE390NjTzXKeh06T6rlelSrXB9Xr0/nIR4/fryUWWpg8bzzzjvxyiuvvOdzZ5er96puv5w+3y1RoW2htruu3VY0NuolOpzlupuLpbj5VLPMk5plbhaMT0ppIfM33TyrenmXCS1ttJuPUm/vxibu2XGfa7vv5mjVuGe7Zf4WLI5JDLoxon7uYjfCW7G7e6rb6/lcnLnYqsfYbl6s1sPZwLvnQ4+jddKxW32Oa9eudW43dI4hmSsAAAAAAIAR4OUKAAAAAABgBAblznPOJR021Ba8b2V7Z03qpBnObldxsq5WCdXQNLezrWy9bk1ZOptiZIFbh8aMytvUMl/Tx2qlHuFldO6e67Fef/31UtZYUImWft4ngXGp8OvXr3eeW6WAKk/V47hnqK6H7n/16tVSVgtWTdvXVt8wPjlnK2PqQu+P3s+ItqUonIzDbdMiNawtddUSXmPILfGh6PlaJOl9cpCW/orlNLaGrn53FrtwFxPaHuq5VBrt5LFDpU11PVyd3JQFLc8i33PnJpaXg74x5oRZZM5O+qox7pam0e3VJl3r2rdsjHtGXB/S0l67qRAR0+OeCxculPLZs2dLWWXxk36xV05r/wIAAAAAAADN8HIFAAAAAAAwAqNY6sybbm9Jebt9h56rlZZrUncfJ81qlZE4B6DWFdxhflQypalvJ9lTqZvK7CK8FNC5W6pL39raWimrBEqlhkePHi3lPjmd7qN1vHz5cimrk59u4+RjGvdOuhUx/R2obFK/Q92ndqOD8XnsscfiQx/6UET4tkmlgBp/TzzxxNR2Ku9U6YeTQyktznzOSa2uh55DpYBaJ70mJ79yfc8sUii3T4uEBxaPu7/z4uJ6Xlwd9Xwqe3Kft2xT13sWiSwslpRSaRN1rKLtXEu7Wv+u7ZP23zqlwLkLax+v44gWJ9lWnGOyxrKOeVrqXW+nfYi6eJ8/f76UDx8+/J7z1tDSAwAAAAAAjAAvVwAAAAAAACMwsyzQySmcvK0Vl2p2x3KpxXnldG5htZbFhV09Wv/mHISQBS6Wt99+O55//vmImE61q0OgSuhUNldL6FwaXtPiKnXS+NGUujqyqcxJJVmaxq6fH62XOuLoNTmHQMUtHunagRpNw1+8eLGU9TmbpNphcTz22GPxzDPP9G7j7mkt+1Q5qsaykzU7qazipEra9tUyWCdj1OfCyUmce1VLv1Lv4z7HVW3r6bovLfeqT6rUch/7HP+GUI93hk6daFk83m1Tn9u5KLNw8PaRUirtm7ar2v45WWA9tm1ZcNs5B+qUiRdeeKGUVYKnLsfaT/SNZ13ddR+3KL1K/NT577XXXitlHcfV9dXz6Xej46SJXLDPeZfMFQAAAAAAwAjwcgUAAAAAADACg2SBKaWSkh4qUetzoBma2m5xqXHb1Kn6obIQTRM6mUury6GT37jUu5Mqwji88cYb8eUvfzkipr9rtwijlvuehxY5iYsrdeDRskrrWh2qnPTDpcH1mty+Wtf6O3ByA02la6r+0qVLtu4wDrt3744TJ05EhI/LPgmJohKUlmO1SIpc/Gps1YsIqzxE5X9OgtLSd43phDbLAp4ASksf4uJsnvESLCcppdLWaZvn+u++6SVOwuxiwrkIqsuxTmdw8vG6HnpuJ1/V69BpEjqlQCV+Ok5SCWPtFujkfU5OPjmHTh+pIXMFAAAAAAAwArxcAQAAAAAAjEAakg5OKV2JiJcWVx0YwPmc84ntrsT7CeJ7qSC+FwAxvlQQ4wuAGF8qiPEFQIwvDTa+B71cAQAAAAAAQDfIAgEAAAAAAEaAlysAAAAAAIAReF+9XKWUfiOl9Be2el+ArYIYh/czxDe83yHG4f0OMb6kL1cppRdTSt+93fVwpJR+IKX0IKV0R36+a7vrBTuHHRDjH0kp/WpKaS2lxMRMGMQOiO+UUvqxlNKFlNLNzQ79D213vWDnsANi/M+klL6+Gd+XU0r/e0rp0HbXC3YOyx7jSkrpiymlnFIatH7voljKl6sdwr/JOR+Qn9/Y7goBjMi9iPgnEfHnt7siAAvgkxHxgxHxn0TEakT8m4j43LbWCGBc/nVEfCznfDgivikidkfEj21vlQDGJ6X0fRGx56EbbiE76uUqpXQ0pfTPUkpXUkrXN8tPVZt9MKX02ymlWymlX0oprcr+35lS+lJK6UZK6Stkm2DZWJYYzzl/Pef8DyLi9+a4HIApliW+I+IDEfFbOefnc84PIuLzEfGtMx4LoLAsMZ5zfiXnvCYfPYiIb57lWADKssT45rEOR8SPRMRnZz3GIthRL1exUd9/GBHnI+LpiHgzIv5utc2fi43/SJ6OiPsR8bcjIlJKZyPin8fGf25WI+IzEfGLKaX3eNSnlJ7evOlP99TlP9qUTP1/KaUfXpZUJOx4linGAcZmWeL7H8dG5/8tKaU9EfH9EfErc14bQMTyxHiklD6eUroZEbcj4hMR8VPzXRpARCxRjEfE34qIn46Ii/Nc0NjsqJernPPVnPMv5pzfyDnfjoi/GRF/pNrscznnr+Wc70bED0fEn04prUTEpyLiCznnL+Sc13POvxYRvxMRf6LjPC/nnI/knF82VflXEfGRiDgZGw3Wn42IHxrlIuGRZoliHGB0lii+X4+I34qIr8fGwOCTEfFXR7lIeKRZohiPnPNvbcoCn4qI/ykiXhzlIuGRZlliPKX07RHxsYj4OyNe3ijsqJerlNLjKaW/n1J6KaV0KzZeco5s3rAJr0j5pdjQYR6PjTfsT26+Bd9IKd2IiI/Hxlv1IDalJC9sBsZXI+JHI+K/mvW6ACYsS4wDLIIliu//LiL+44g4FxH7IuJvRMS/SCk9PsOxAApLFOOFnPOF2MjM/uN5jgMQsRwxnlLaFRF/LyL+cs75/jzXswh2mpTt0xHxbER8R875YkrpuYj49xGRZJtzUn46Nibmr8XGjf5czvkvLqBeuaoDwKwsa4wDjMGyxPdzEfELOedXN3//+ZTST8XGvKvfGeH48OiyLDFeszsiPriA48KjxzLE+KGI+PaI+IWUUkTE5MXu1ZTSJ3POvznn8edimTNXe1JK++Rnd0QcjA0Jx43NyXE/0rHfp1JK37r5H8gfjYh/KhOW/2RK6XtTSiubx/yujkl4DyWl9MdTSk9ulj8cGynPX5rxOuHRZZljPKWU9kXE3s3f96WUHpv1QuGRZGnjOyL+r9j47+mTKaVdKaX/Ojb+s/r7M10pPKosbYynlL5vMlclpXQ+NqRbX5zxOuHRZVlj/GZEnImNf5Q9F+/KCj8aEf9u+GWOyzK/XH0hNm7e5Oevx8ZkzP2x8fb7b6N7AvLnIuLnY2Ny276I+EsRG845EfGnIuKvRcSV2Hh7/qHo+A7SxiS6O8lPovujEfG7KaW7m/X8P2NjUh3AEJY5xs9v1mniFvhmbMxPAWhlmeP7f4yIr0TE/xMRN2JjvtUncs43hl8mPMIsc4x/a0R8aXOc8q9jo/1G1QBDWcoYzxtcnPxsHisi4lLO+Z1ZL3YsUs6sDwoAAAAAADAvy5y5AgAAAAAA2DHwcgUAAAAAADACvFwBAAAAAACMAC9XAAAAAAAAIzBonau9e/fm/fv3v+dzNcVwBhm7dk2/x62srHT+TT9vYdPfvrdOrbjr0HNoub6msc63vr5eyg8ePOgs37p1ay3nfGKuCsAUe/bsyfv27duy87kYU9znLccc8rcxzj3msW7fvk18L4AjR47kM2fORERbPOh9a42tlv6g5Rytx3F/c21qyzbumNoGR0Tcv//u2pW6v+szlLW1NWJ8Aezfvz8fPHhwlGO1tNEt+w7dpo5XjbuW4+rYRMstcVkzTz9AjC+GAwcO5NXV1fd83nJ/689b4mOeGGitx9DnpaU87zW4Nn3C5cuX49atW50HHvRytX///vjYxz72ns/feedd18O33367lPVFqX4p08bvwIEDneXdu9+tnl6kBoNuo2in19owuY5Sz7F3795Sfuyxd5f9cTdO61o3mHo+/d7eeOONUr5161Yp37x5s5R/9Vd/9SVzGTAj+/bti+eee653G9cQ1dSDsC5cI9DyjwfdXuNKY6quR/23rnO0NEbunwp9/2zQY+nz5BrFL37xi8T3Ajhz5kx8/vOfjwj/jxsXZ3VMu38CaX/gtnHxfu/evVJ2bXiNbuf20fbVbf/mm292XoPGpbbBERFra2ud+2iMu+f5Z37mZ4jxBXDw4MH4xCc+0bx9Xzvuxh2K6+PduEO3cdvrGCAi4u7du6Wscabodeg/CbWsY5Y9e/Z07luj8avllj7u537u54jxBbC6uhqf/vSnI8L3rXp/9b7p5xHTMaHjW91Ht3FjaDd2cP1JnUhp+WeYttdvvfVW5+fah7ixeyvaJ+ixJtfxmc98xu6LLBAAAAAAAGAEBmWucs7lrVD/a+HeMt0bdcT0W6R7W9Y3Wf1vjb5Nuv9UtkjrInxmQMv6nx/dXv8D4P7735fpcPXV/7Lqf6y0DMtNX8ZygosTl5l1/zl9WOr6YXVy5x4a0/W5W5+DrjrBYkgplXbLZRCHSrPr/VsldV3na+lXapxyQnEZMf38zp07pax9TJ/aQTMMLf+tZV3JrWEsSfNQRUqLvF/jT2NXt9H4i5j+L737z7x7joY+U/Xz7/oEl5kbU04O3ezduzc+8IEPRIQfI7ROvXFtVX2+CU4R4NpY1yb3KblaMsYtCgmNSx2v19M/9Pr0fI8//ngp6/c8KTvlXASZKwAAAAAAgFHg5QoAAAAAAGAEBskC19fXiwyiRVbUl4bT3zUlp2lJN9n4pZfenSPpJnq6OtVoSlRTgIcOHercXif2aVmP05U+jHivJMSl7lWeoilVlQbA1uHiZ14pREvMuJR/i9NkvU/LM+vq1DIpdRYXIje5GxbH5HvWuHETobVdUtlHzVDzExcP9WTrh21f10v7FWfG4cw3dF9tg3X7ug2+evVqKTv5i4t32Lm0uE+6spMFahzXcaax1WLQ1WIO0NqOKy2OmkhfF8/KykoxhdO2W2PIxVYdDy2yQidfbekfNB6ceUbEdIxrDKoZnpui40y9WqWGrs/Temh9J+8KfRJ6MlcAAAAAAAAjwMsVAAAAAADACAyWBU7kEk7KoWk/TTf2uXM46ZymCVUWeO3atVLWdUc0neckiXU6UM+t6Ucnb3SyKd3Gyan60rFuf1eGxdK6ntXQY7VIATUWnJOmxnpfXfUZGioPcXVy56slXS1ug07GAIthfX29xITGg8aZtpfOhamPFldLJ1tybadzke3bp+UZcZIVJ6mp22CtuzoHuudolrVWYLH0TRtokXG2OAQ6+bPbvpZYuXXXHBrLQ91g6+O3fAfIXbeW9fX1MiZ20rQWB73695YxqdvXuYO79rmWvqqET+P38OHDpaxjdK2fHkuflevXr5fypUuXSlll33XdnZz32LFjpXzq1KmI6I97MlcAAAAAAAAjwMsVAAAAAADACAxeRLhLuuOcZjS1V8sCXdrapcZV3qTpQHX10xSdO3ctXdLfDxw4UMqaitTPncuhS6H2La7pvqsjR450HmuWxT1hXGZxCHSLAmvZueg4OZ6TWNVyEif5U1okiU6e2Lpocat7JyyWBw8exI0bNyJiOm5cu9aHc8Fz8kGVdjvJnpMquuegPlaLZMq5CKosRY/jtqn/5lymcMRcPhbVBrW4uLa4BdbPkIuhlukEbjFZJwusY9Q9X7Tj28fKykoZ++r9dfHgXPPq/Z1zoBvTDnUN1z7gtddem6rHxYsXS/n48eOl7Bxg9RxufHLlypVSfvnllzvLEd5pfOLIGBHx9NNPl/LELbBPKs/TAQAAAAAAMAK8XAEAAAAAAIzAzPZzLtWsKTx19tByhF842Mkm9PMnn3yyc1/nyqbnUvlV/buTRz3xxBMPPYdzs2qVlOi5kQVuP30Lzk3okwi6RU9bHHU09rTsnhMXh3XdXcrfnc/JAp1EoE/K4tx4XD1gMdy/fz8uX74cEdP3QeXVrt2unwP3XGg7p1IndXZVtybnxqp9idZPF3uPaFvsWnHPtusz9Hrqa3bnmNdhFLaHWe6bk8S6xUx1ioM6TLpt6nO4fkOfF/cczOKC2yIDQ+66tegiwtpWOQlnq5TfTctxMdSyWLWeW+P9xRdfnNruwoULpazPwrlz50pZ+yMne9Rr0PPdvXu3lFWeGDF9rfp9Puzdpk8aS+YKAAAAAABgBHi5AgAAAAAAGAFergAAAAAAAEZg0JyrlFLRIzotorNA75vrpFpNZ4mouny3YnN9ji7qlaadvlK1pHrcFu1pq0WxmzejdexbOR3GZ8i8tlabffe5m+/h5l/pvqoR7tPRu/leipsDpfVwz4nTOte/a9ldq1sFHsbj3r17Zc6VxoO21c5SvO/+aLutevZbt26V8uuvv17KEzv4CD9n0C25cerUqalznzhxopS1P9BjuWtqmeen1zbv0gP1vFtYDA+zCW+dK6cx78YpOj/q9u3bpazzClvmVvXN7XNzXp31tpsT78YTreMU3Uevtc+SGhbD5B67ZS0UN+6ImL6nGtcas24soDHb4sNw7dq1Utb+ICLi6tWrpezm+qr/gdZVx+h6Dp3HpfXQeVwR09+hXp/2LR/+8IdLeXV1NSL6+0QyVwAAAAAAACPAyxUAAAAAAMAIDNLh7Nq1q9jgulXCnayqL93uLCMVlxbX9KFa9LoUfi3fcmlNJ+1zaUDd3slR6n2dPMXZndb2rLA19Fmut+zTgpPvOUlHq6Wuq3uLfGDoUgN1fDqpiMa6kyHCYnjw4EGR5DlJnJMO1TGtv6sU8Pr166W8trbWWVZJUUv/oTbudT1UEuKOpW2qk2Mr2i/o99EqfXXPF/KpreFh7fS8lutO2qdybd3GtcMau722zkZmqri2u2WagZP+1b+7eJ+lj4RxcONNd0/q++vaRkVj2cmkVXbnxq0q2dN+oj6u0vLs6HXrcVRGePbs2c761XXU71PfKSZSwIh330d6ZeL2LwAAAAAAANAML1cAAAAAAAAjMLNboEsfuhRjvX2La5qTY6iLieKcyzTl1+rqpmhq0bmmqDxR06N9khcnu1JpgfIwBySYn0l8tKw23oeTdjqZlZMRupT4vNKLFomgwzlL1XHr0vxOUgxbwySGnbuTkwvV6N+0jVWHQNdWq/ufljUe9JhvvfWWPaZKDA8cOFDKzj3WPUctzoj1c+pkUu75xC1w+2h1CFT0uXCSP41TJxF00wNanYH1WXVTJNw4yrXvTu5Xt+NOMujad8YpW8Pkvrjvu2WMHuHHIbqdOl/qWNeNe3XfK1eulPLFixdLWaXeEW0uxM7ZWNE6aX+gEj91HK+Pq3Gt/Y72M5Nr7esfeQoAAAAAAABGgJcrAAAAAACAERgsC5ykm1vca5xkIsI7BLo0t6KpxZZFhPukGE56qOk+54Jy8ODBUj569Gjn587JKsJfq0vNDlngFuZjXmmDc/NrWci3RSLYct6+erQseNpS175n3Mmkhqb5YTxWVlaKDE+/byf11M/7FqhWXHw4ucbJkyc7t1F5oTpL1XGmMg5t690C3G5hWKVFujULxPjOwkn1NM5aHAIVjScdWyh1jLvpEvP0M1pvLddyPzdNQSWQ88jMYTYeNoVhFtdXt79za9UpMxqj6h576dKlUtZ2vNWVUuOsZdqRyvf03O44EdN9hXtG9JrccRSeAgAAAAAAgBHg5QoAAAAAAGAEBskCc84lDdayQFmfZEjT5y61rWlGTdu5lLxLTTsZUv03TfupS4im8V3q0123yl80hVofq8XJiHT7zsG5BTr5lStrfLuFt+fFyUlcTLt0fJ8TZ4ucqi/FDuOwsrJS2iR1gHLSH6VPxqEOZhoreo5Tp06V8jPPPFPKR44c6dxX23lta/WYEdMyEJWv6HOnTlHuHA597vrkWtrWK64tgMWQUirf89DFbWeRfbb03RpDOpZxz03rgsJDHRBd36LjndbF4HUf51QIi2PynbtxpBtr9KH7uGkrbqqKxtNkofoILwWs48TJ8VpcD3X7F154oZS/+tWvlvKxY8dKWfuD+tzqXnvixIlS7hrPIAsEAAAAAABYMLxcAQAAAAAAjMAgWWBEt0OJpvdcmqyWlKgcQ/fX7dRpT9N43/zN31zKmmJvcfyoU97qSKXSE7ePW9ysZTHUepuWfVpc5GCxzOsQpjHjZJ66jXs2nISp1QlonrKT//VJVJxcwclGiO+tRe+JtkXOTa+OP9dGtvQBLQvHa52cm2r9+9BFul1cO+lgLZ90cpuWhVthcXS12fPKAltkfur+554jHde0LORd7+8+d2XnYOgW6dZyvb9bVLllAW4Yj5RSiTs3jtT71iq51/ur91Fj1vX5Gjc6rtY4URls/az1tbMT3NhBF5bXely9erWUr1271nkN9e8qC/zABz5QyioRnDjcIgsEAAAAAABYMLxcAQAAAAAAjMDg/O0klTfUEabPrcTJRTQVqSlEdWXSNKOTWemieJo+rPdRWeDdu3c799eybqOOVS5FXjuUaN0V59JCun256JOQuMVXnVzDld1CqK3yJ7fwpZNlOYchJ93VOtWLeLvvwC1ajGRqHdRB8AAAFOFJREFUa+hqw50jqlJLNVoWT3XSUpUkaZvqZN59/U2Lo2rLIsnOSc3Vtd6uRXozr8QY2pi1LelrP12/ru2ec0NzbamTS/eNl1y76p413cb1M04iWG/nZOrE9daSUupsE11f7lwpI3zM6j1VR1fnWqzjYZUFHjx4sJRXV1dLuR4v6Phb66jxqDHn3g/0GlSmq2P0WuLu5LKufZ9IB/vaGTJXAAAAAAAAI8DLFQAAAAAAwAgM0pnp4nxKy8KIfds4eYqTxGmKU8suhd/naKYpR5XtaVlTgy5dqQtb6jk0LXn48OGpc7tFA50ECxbPIuQN89zDoYtg9uEWKnbPkMM9l63XSaxvL5M4alk82km2I7w7q5NrqKROF/vVbfbv31/KrY5RblHWFhn10IW8++QkLn5ZOHjr6YrxeY4TMR1PbjyiMaBx6SSjLYuzR/hFXIcey7lg9sqbGpwvXfsBiyHnXO6fc5x0EsE+J0onr3NjaG0bdcqNbn/69OlSdlMe+nBx6qYUaL2dDLGWvurveiyVKurCyJNrxS0QAAAAAABgwfByBQAAAAAAMAIz28+5lLBzdepLc7vFIhWX3tc0t9tG61Q79DkpoKYQnfzFOe+o/EXTpnU61rnI6TWRYt9aHiYjaZXpOclVS9q9pazx0id7HSrXGCoL0G3qZ9+dzzkSwfbhJEV63+u208Wmk+M5tyZ1ltI2WI+pkpM6ZlRK6GSBLbKqFre/un9y0hvYXsaSYjrZs2v3dBqAc/zTz90z0Ss36mnvu47lFsd2Y6f6+XXS15ZF7WExrK+vlzZR40+d+VqfgZYpNG4MrHI6lXprbGmbrrFVx66TYruyi99jx46V8rd927d17lu7vqpD4NraWilfuXKllFUWeP369Yjwz0AEmSsAAAAAAIBR4OUKAAAAAABgBGaWBbp0tEsJ16nmFpmRSvM0bedSlJpKdwtQ1qlSlZFMFgaLmHb/08XRWhYtdulHdR6p/6apU3dNsFhSSg+VqbXK2Jw0qkUu2Op0OaEvXpzEdOjivS116lvo1e3T55AFi2HyPQ91hmzFyYhU/qcSEm37VOKncaJxXLuu9klTu47laNmm/j7qhTAnuO8WGezOouU+urbU3esW976+hV5dPdz2rr1tcbrsO8fQfgrG48GDB6U91e9eZala7qNF5q/jaRenOp7V8Y+2731TAtxCvjoW1zG0tr16jqeeeqqz3BeXeu4XX3yxlL/85S+XskoEJ3LevsW+yVwBAAAAAACMAC9XAAAAAAAAIzBYFjhJQQ6VUPRtP9YiZpqi0zRm32JlLrWo8hTdX7d3cixNj6qbijpe1X/TFG7LQsqw82mVZUxokWH0PWctTpzuuM4ZTum7hqHXB4sh59x5L5wERNsclWpEeEmIlvW4zl3VLSTt3KAOHDgwVY+WfmYeB9Y+JzX9vUXi2iedhZ2Dk0/pfW+51y0SwRo3DnCfOylgqzuh0vcsdJ0DFkPOubTHLv5a+1PXz+v91T5Bx9wq2dPxrbbRrj9pXZRej+v6lieeeKJz31aZuHNV1uvT8frkO+hdfNv+BQAAAAAAAJrh5QoAAAAAAGAEeLkCAAAAAAAYgZkn8bj5TYrTE9d/c3OX3Dwmp8F0GuI+7WmLDaWzj3Q6fq2TzlNQrWpdX6cT1esYa9V5WDx9Fp0TXKy3zMVq0UnXv7tnws2naplnpces41u/AxfHbskEWAw55/esTh/hY8jN0YiYboevXr1ayqpN1zg7cuRI53FVL68xoPGj7Wj9bLk2Vq/DteEtVtZ9z4H73jTe9Xtj3uxy0NKeRfh+2X3eMj5omSPYNwdK40mP1TKv0PUzbn5jjWujWVJja0kplXn62qZoeZY5V+5Ybn6tLq+h8eRs4FvnhLtz6BworZObM6zb9I2ftd/Q+WIf/OAHS/nSpUvvqV/v9di/AAAAAAAAQDO8XAEAAAAAAIzAYI1CV8rXpZc11VZLStwqz7qPSw3euHGjlFWCovXQlZz7JIx6vlrWNAR3Pc5+uK7vUFtuWG5abM8d80gE++rRIvNrsWV3UkB9Fuu/ad31WG71dlgck/bJLXXhJD51+6VtrLbPup1K/vReazus27hlNpzlb0TE3bt3S/n27dul7GSBQ58jZ9Mb0Sa17VsKBHYmen/37t1bys5CXeNGt9dyX/vnpHZurNEytnBjlroeffbZD6sTLIaVlZU4fPhwKU/QtmYWe329v24ZDR2X37x5s5SvXLlSytevXy9llYy/+uqrncfp+v1hxzp16lRnXfUaWmzZa5yscPJ9R7x73cgCAQAAAAAAFgwvVwAAAAAAACMwSBaYc+6UVKh8w8lI6lRxi3ROJR4qBbxw4UIpqxRJz11LlCbs37+/8/OIadcpTVE6Fx4nh3SOUHVK3aUUXVq91f0Ftp8WidFY99Ol9evfnZOQk3q0xLpzdKt/d25ZKouBrWFy751M1EmbapyEW4+lDoGrq6ulrO2wygU1nlR2qNuoa1PEtGRQt1PHKufo1uIa51wv699bnF2RfO9cnLPwUKmctsMaoxrvdVvq4szFphtDOMm5q2tdX3etSAG3lj179sTp06cjYvhYtabFOVjH0yrTu3z5cik///zznZ+3Pjd6HY899lgpq+z79ddfL+Vz5851bq99i75DaFm3j5jup5zD4NGjR0t5ImXvk3yTuQIAAAAAABgBXq4AAAAAAABGYOYVDZ0TiXMVq9OSbjsnC1FZ4LVr10rZyf803a7b9MmQnNypZXE+t+iwnq9vgVfncOLkVDA+TvaqtEr53P10x2px8nOOe61SI+eW0/IsumfZuQhFeLmwptLdNcHicW6Qzl21lkA4CbhKLlSmd/DgwVJWh0BtI1UO5aRQdT20n1CJoJN6DHXs1GvrWwh+6Dlg8TgXTKXPVU3/5qSl7ny6vYtfHae4BVzr/VucAxVtY1va97r/cf2L67OI/a1hiMS4RQpdH9M5oWr86oK76qbnxq0a17Xrq5syof3JyZMnS/nYsWOlrFJAPa7KCLWf0HpHTPdBej6tb1d/0rcoPKN1AAAAAACAEeDlCgAAAAAAYARmlgW2yJJmcUZzCwq7BUmds5WTaNTSqKFSqxZpnqZNNcWoEoAIn1Ikrb68zLKQb4tz4Dz0PWca7y0ulE6SOnRh4/q4ziVoFnkjjIOTAro2sW47neTKSUjcgqmuzXeSrNrpaagEV3Gx7J6p+vja/zhpFdLXrSWlVL5/F7+ti4u2xIGWNX5VhqRSJZ2m4Mp9rpQON57QzzVe9fO+4+u4RZ89lfDWcllYLCmlcv80Lp1zYN8YwbVVur/GgMrjzp8/3/m5xr7uq3Hy1a9+daoeGltuoW2VHur2ek06bejll18uZZWMHzp0aOrc2gaofF2fW91mck197wNkrgAAAAAAAEaAlysAAAAAAIARGCQLTCmV9JtzGHPUcgqXfnSuOLp9n6vOBCfHq6V5em5Ny7e4tyn6fWidXHozwjtxOflBy/cMi2UW6Zpz6nGp+qEugn3bzLOYacuCk5oq71tQT9HtcMDcPloWQu1zmXJtuEqP9BzaLurnTgqo53Pywr7juj7KXbeTivU5oTlppGu3ifetoWucMos8s0XC7NwkVVKkC6GqZMrJSlv7+haXPtc/OJlkLR8fOhaCxZNzLvfJSbJd2903Fm9xAdcx9PHjx0tZF4lXVMp38+bNUlbJXsS0s6zDOXDqc6SLHL/00kulfOXKlVKu+xDdZ7I4c133LgdDZIEAAAAAAAALhpcrAAAAAACAERgsC5zIM9wiu604t50WiaBK/pxLjZMF6mJj9f5Optey+GCLu2Atm9LfXbqz1ZkNtoZWOclQB80W6YZztXKyqPr3lpS/w9Vbz11Lbp1M0EnFYGuY3D/XNuk96ZN/uwVJtU0dKntysr55F+jV/Z2sfOhi3/XvLfWapa+E4UzuRUufOUs8tSzM654J56bZN6ZyrsgtY5CW7Z1baP03rUdLXwaLIedc3BpbZIFK/XnLWFzbSbeIsDu3uu9p/OhC8hHTskLdTscVrq7ab6i8UB0M1Umx7n/cmEvrqJ9PrrvPaZSnAAAAAAAAYAR4uQIAAAAAABiBQZqc9fX1klprcRjRNJwuOBfhF8zT9Jym9zT9eOvWrVJW1w9N4R09erTzOLVLSMtCpy3OQFo/Pbd+XsukXCpSJY3qrqLXDVvHvLIRd6wW+Z8rOylA64KY8yy26mSBffto2UkgkUxtDZP755we9Z72uUy5NktjUxduVMc0FwMuHpwLW12vFifKFgm3k0n1ybXcM6ng+Lo1LHqBdidxbZHHKk6W2ic/1XO0SPBcvLfGYsti8rD1LGJBcndPnRy0pT66r8aruvLVaIyrtE/H8no+Hdc/88wznZ/rWLruQ1S6qO8Run/XuKfvHpC5AgAAAAAAGAFergAAAAAAAEZgkCww5zy1AN4EJzdS2Ui9n3PJcS59mk507mjqKqJ10rSkShUj2pytnOxKy5pW1DTmoUOHSlmlMzUu7ar1VVkNLDdjuQK2LGaqsdMn1ZhHxtGywGotSXQyvxZ3IlgMKaVyz5yEtHXBdm3zjh07Vspra2ulfO3atVJ2i5aqdFrjSdvmq1evlnItj9Z6OXcnJ79yCx7ruTUuazmJ1qVlQW2kVFvD5Ht27XDrfXDSOdeGueMOve99i1W34PqQlr5olgWMFaSviyfnXNqoFsdSFwNdx53Q4i7sHK/dMXWM/uyzz05td+7cuVLWNteNsxWV/Om+KiPsG19oP6DHUrrcdZ1beQSZKwAAAAAAgFHg5QoAAAAAAGAEBssCJ2kwTRm2LPxYp8/ULbA+R9f+elyVkTg5nR6/T1Ki59B04PXr10tZr1XToCrzO3z4cCmrLFDTmPUiq4rKTVyK8vbt23Z/mJ+U0kJcplpkdC3uaU7W0io5cWlxJ9dyOGlJ/d3pcd0CmS0Ly8K4dMW4k1pruXZa1fZPy3osdYnVNlXjTNtqjSeVQWs7WNdf21X37ChO+uLKTiIY4d1jndvgIhy+wDPUFbWmRRrl2kAXi7O03S3ndhLelme7r+11fZYbe/U51sI46CLCen+0jdaxaqsU0PXTLU6qbuzvXMPrttSNC7Rd1b5C5edO3q2xr8epnzs3DahF9uggcwUAAAAAADACvFwBAAAAAACMAC9XAAAAAAAAIzBozlXEu9pLp2d3+t6aFt2x6h11npVqItXi3VnmarnWFqtWU7fT+U26j16fzrM6evRo5+da79qKXc+tWlI9t9apz/YRlhc3x8Ppm/U+q0bZ2Uk7rXKE19g73bTu755Rp/mvaZkjpnFfL5MA47Nr167SJrmV6FusfSP8/FMtaxun7ZrOp1K7do1x3VfLaq9b/+7mGbQ8gy1zpmqc1bfSOrcFxuNhc6pa58Hpdq6ta1mqRePSxZlre2vccbXcMv+qr99QXHvgvjeWG1g86+vrZWyg907b1dalB1x7qPtozLo41fO5OVNuLB3RZu3vrsPFuHs++ubOur5Cr2MyVul9buxfAAAAAAAAoBlergAAAAAAAEZgkCxw165dvXbiET5lXaf8XHrZ2UrqeV0KUCVUzr5R038RPuXo0okqpTl58mQpHzt2rJQPHTpUyipZqdOg+l2pJEqvu1WCBYuj5Xuv08MtFs/uGXASujp+JmhMzyI7cvVrsUzvk9e0SH/1WMgCF8/u3bvLUhEq39P74yyW69hyS1HosdzyFioR1LbaSTr2799fytq+RvjlLpwsvSXeWyVT7hyu73PLIcDW0ippcxIhdyxnW+7krk7q3xdzLTbyLVJFV+/6O3DHcu0ELJ6ccxnXtshM++6P+1vLM6LbaIy7JQLq8bfSMvZw9vB37tzp3N5JG/Vdof6b4truyfZuSakIMlcAAAAAAACjwMsVAAAAAADACAySBaaUOmWBmi5zboF9KyI7OYaTRE1kLRHT0hRN9WlZ5UataUndTmV6KvNbXV0tZZWmuOuuvwO9Vt1HZS9ad5WawNahcdEqzWxx4Bvq/ugkRc7Rpt6n1QGtC+dO2CdbbXnGnSMcLIaVlZXibOpcwJxEsOtYE7T903bKuRBqP+KkJbqvHl8dWCOm22Tdx7ljOqmje2b75DV6rbq/k8cjn9oaJvdiaDtX3x8nxW5xT3XPQYv8uZabuukSOk1BnwMta/yqvLZF6h0x3ae0SL3ddwbjkXMu90XbaBcnri+u/+baJ3XlbpkW4MY56n7dJ6lTWqTUKgt0Y3/XB9S/u2d4aN3IXAEAAAAAAIwAL1cAAAAAAAAjMFgWOEmZuZSwSw32pcKdrMktPnb8+PFS1vS5pkE1/a3HaXULHJpq1cXbWuVNThKlqUj93mqHE9h6Wt34WlwenQxVaZFoOXlHxGwOaBNaFmF1z2i9j5MCtkp2YRxSSqUNczI2ve9uccV6H/e5ypb0OVBpd4ucTuXftVRDf3eS7BYpqusn+tw4W+Qkrn6wOIbIAfsWW3WyJ3ffndula+udJKuvLXXn0GdE48zVr9UlzkkBWySCsBju3bsXly5des/nKt9T2V3fWKNFdtey2LUb5zgX7xoXp65PcO6EzqXTPTf1/m6s0/VM9U0TIXMFAAAAAAAwArxcAQAAAAAAjMDMti4taeA+uVLf3yaoZEi30fS3S7E7t5JantgiHXGLYiq6r6t33z5ucbQ+qQAsL32LMnZ93udkM6FFNjfvYsYOV1fnyFb/3rIoINKSxZNz7pRou/anr512sgzn/ucczFy7psfsk1+42HRtsspUVDqj2+jnKrWppdm6j16rk8sgC9waZnVl7JPEuRh0siX3TDk5XassUKcmqOumk6LqsZw0qs/ZuXXBZVd3GJ+33347Xnjhhfd8rmOEWaYEuHbcSWedpFvR8beT/tV11DbXxWaLJNu1vfWz7L4rJzGcnKOvnSFzBQAAAAAAMAK8XAEAAAAAAIxAGuiqcyUiXlpcdWAA53POJ7a7Eu8niO+lgvheAMT4UkGMLwBifKkgxhcAMb402Pge9HIFAAAAAAAA3SALBAAAAAAAGAFergAAAAAAAEaAlysAAAAAAIAR4OUKAAAAAABgBHi5AgAAAAAAGAFergAAAAAAAEaAlysAAAAAAIAR4OUKAAAAAABgBHi5AgAAAAAAGIH/H6IWFa8VO4OiAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 1152x288 with 10 Axes>"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "plot_sample_grey(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SA4FUPc8ke1I"
      },
      "outputs": [],
      "source": [
        "# extracts the layer from a model using the name\n",
        "def get_layer_by_name(layers, name, return_first=True):\n",
        "    matching_named_layers = [l for l in layers if l.name == name]\n",
        "    if not matching_named_layers:\n",
        "        return None\n",
        "    return matching_named_layers[0] if return_first else matching_named_layers\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LM8K14_1HBws"
      },
      "source": [
        "Let us do a dumb sequential and see how far this can go  => 78% validation is the best.. + 85% test ?!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0juaJ1ELRdSA",
        "outputId": "90e93938-5675-4a3a-a6fd-b9e98ae336ba"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " F1 (Flatten)                (None, 1024)              0         \n",
            "                                                                 \n",
            " D1 (Dense)                  (None, 128)               131200    \n",
            "                                                                 \n",
            " D2 (Dense)                  (None, 128)               16512     \n",
            "                                                                 \n",
            " D4R (Dense)                 (None, 48)                6192      \n",
            "                                                                 \n",
            " SFTMX1 (Dense)              (None, 10)                490       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 154,394\n",
            "Trainable params: 154,394\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# # let's do a dumb sequential and see how far this can go  => 78% validation is the best.. + 85% test ?!\n",
        "\n",
        "def get_model_seq(input_shape):\n",
        "    model = Sequential([\n",
        "                Flatten(name='F1',input_shape=input_shape),\n",
        "                Dense(128, activation='relu', \n",
        "                      name = 'D1'),\n",
        "                Dense(128, activation='relu', name = 'D2'),\n",
        "                Dense(48, activation='relu', kernel_regularizer=tf.keras.regularizers.l1(0.005), name = 'D4R'),                \n",
        "                Dense(10,  activation='softmax', name = 'SFTMX1')\n",
        "    ])    \n",
        "    return model\n",
        "\n",
        "model_seq = get_model_seq(train_data_grey[0,:,:,:].shape)\n",
        "\n",
        "# model_seq.compile(optimizer=tf.keras.optimizers.Adam(),\n",
        "#               loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "#               metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])\n",
        "\n",
        "# callbacks_seq = [ EarlyStopping(monitor='val_sparse_categorical_accuracy',\n",
        "#                            mode='max',\n",
        "#                            patience=60)\n",
        "#                 ]\n",
        "\n",
        "# history_seq = model_seq.fit(  train_data_grey, \n",
        "#                       train_targets,\n",
        "#                       epochs=500, \n",
        "#                       validation_data=(validation_data_grey, validation_targets),\n",
        "#                       callbacks=callbacks_seq,\n",
        "#                       batch_size=512)\n",
        "\n",
        "# plot_history(history_seq)\n",
        "# model_seq.evaluate(test_data_grey,test_targets)\n",
        "\n",
        "model_seq.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6xqL0RbypQUF"
      },
      "outputs": [],
      "source": [
        "# plotting utilities for the history of the fit\n",
        "def plot_history(history):\n",
        "    acc_keys = [k for k in history.history.keys() if 'accuracy' in k]\n",
        "    loss_keys = [k for k in history.history.keys() if not k in acc_keys]\n",
        "    for k, v in history.history.items():\n",
        "        if k in acc_keys:\n",
        "            plt.figure(1)\n",
        "            plt.plot(v)\n",
        "        else:\n",
        "            plt.figure(2)\n",
        "            plt.plot(v)\n",
        "    plt.figure(1)\n",
        "    plt.title('Accuracy vs. epochs')\n",
        "    plt.ylabel('Accuracy')\n",
        "    plt.xlabel('Epoch')\n",
        "    plt.legend(acc_keys, loc='lower right')\n",
        "    plt.figure(2)\n",
        "    plt.title('Loss vs. epochs')\n",
        "    plt.ylabel('Loss')\n",
        "    plt.xlabel('Epoch')\n",
        "    plt.legend(loss_keys, loc='upper right')\n",
        "    plt.show()\n",
        "\n",
        "def plot_history_df(history):\n",
        "    acc_keys = [k for k in history.columns.values if 'accuracy' in k]\n",
        "    loss_keys = [k for k in history.columns.values if not k in acc_keys and not k in ['epoch']]\n",
        "    for k, v in history.items():\n",
        "        if k in acc_keys:\n",
        "            plt.figure(1)\n",
        "            plt.plot(v)\n",
        "        if k in loss_keys:\n",
        "            plt.figure(2)\n",
        "            plt.plot(v)\n",
        "    plt.figure(1)\n",
        "    plt.title('Accuracy vs. epochs')\n",
        "    plt.ylabel('Accuracy')\n",
        "    plt.xlabel('Epoch')\n",
        "    plt.legend(acc_keys, loc='lower right')\n",
        "    plt.figure(2)\n",
        "    plt.title('Loss vs. epochs')\n",
        "    plt.ylabel('Loss')\n",
        "    plt.xlabel('Epoch')\n",
        "    plt.legend(loss_keys, loc='upper right')\n",
        "    plt.show()   \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oNo_peRzfXNP"
      },
      "source": [
        "A function that will take a model construction function (with _model_name_ and _input_shape_ arguments), train and fit it using the supplied data and callbacks. The _kwargs_ are supplied to the model fit function."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P_dqIA0noEoo"
      },
      "outputs": [],
      "source": [
        "from tensorflow.keras.callbacks import CSVLogger\n",
        "import datetime\n",
        "\n",
        "def compile_and_fit_model_basic(  model_func,\n",
        "                                  model_name,\n",
        "                                  input_shape,\n",
        "                                  X_train,\n",
        "                                  Y_train,\n",
        "                                  save_max_epoch=True,\n",
        "                                  save_final=False,\n",
        "                                  patience_count = None,\n",
        "                                  early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                                  log_history = True,\n",
        "                                  verbose_level = 0,\n",
        "                                  **kwargs):\n",
        "    m = None\n",
        "    if isinstance(model_func, tf.keras.models.Model):\n",
        "        m = model_func\n",
        "        m._name = model_name\n",
        "    else:\n",
        "        m = model_func(model_name, input_shape)\n",
        "      \n",
        "    if 'validation_data' not in kwargs.keys() and 'val_' in early_stopping_obs:\n",
        "        early_stopping_obs = early_stopping_obs.replace('val_','')\n",
        "\n",
        "    callbacks_used = []\n",
        "    if save_max_epoch:\n",
        "        callbacks_used.append(ModelCheckpoint(f'/content/drive/MyDrive/data_papers/{paper_name}/model_checkpoints/{m.name}' + '_model_{epoch:03d}_{accuracy:0.3f}',\n",
        "                                              save_weights_only=False,\n",
        "                                              monitor = early_stopping_obs,\n",
        "                                              mode='max',\n",
        "                                              save_best_only=True))\n",
        "    if patience_count is not None:\n",
        "        callbacks_used.append(tf.keras.callbacks.EarlyStopping(monitor=early_stopping_obs, patience=patience_count))\n",
        "\n",
        "    if log_history:\n",
        "        callbacks_used.append(tf.keras.callbacks.CSVLogger(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_history/history_log_{model_name}_{datetime.date.today().strftime('%Y%m%d')}.csv\", append=True))\n",
        "\n",
        "    m.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(), \n",
        "              optimizer=tf.keras.optimizers.Adam(), \n",
        "              metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])\n",
        "    \n",
        "    history = m.fit(X_train, Y_train, callbacks=callbacks_used, verbose=verbose_level, **kwargs)\n",
        "    if save_final:\n",
        "        make_dir_if_not_exist(model_name)\n",
        "        m.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{m.name}_saved_model_after_fit\")  # Save the model\n",
        "    return (m, history)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J212e-sSgQEs"
      },
      "source": [
        "A function that given a model or model directory create a new model up to the _layer_name_, then write the features matching the supplied _X_ and _Y_ as numpy arrays to google drive."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DxMillEBxs5H"
      },
      "outputs": [],
      "source": [
        "def write_features_from_models(\n",
        "        model_entry,\n",
        "        layer_name,\n",
        "        X_input, Y_input,\n",
        "        reverse_one_hot=False,\n",
        "        normalize_X_func=None,\n",
        "        dataset_id = \"NA\",\n",
        "        **kwargs):\n",
        "  \n",
        "    Y_new = None\n",
        "    X_new = None\n",
        "    if reverse_one_hot:\n",
        "        Y_new = np.apply_along_axis(np.argmax, 1, Y_input) + 1\n",
        "    else:\n",
        "        Y_new = Y_input.copy()\n",
        "\n",
        "    model_here = None\n",
        "    if isinstance(model_entry, tf.keras.models.Model):\n",
        "        model_here = model_entry\n",
        "        model_file_name = model_here.name\n",
        "    else:\n",
        "        model_here = tf.keras.models.load_model(model_entry,**kwargs) \n",
        "\n",
        "    features_model = Model(model_here.input,\n",
        "                            get_layer_by_name(model_here.layers, layer_name).output)\n",
        "    if normalize_X_func is None:\n",
        "        X_new = np.array(features_model.predict(X_input), dtype='float64')\n",
        "    else:\n",
        "        X_new = np.array(normalize_X_func(features_model.predict(X_input)), dtype='float64')\n",
        "\n",
        "    np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features/{model_here.name}_features_{layer_name}_{dataset_id}_X\", X_new, \n",
        "               allow_pickle=True, \n",
        "               fix_imports=True)\n",
        "    np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features/{model_here.name}_features_{layer_name}_{dataset_id}_Y\", Y_new, \n",
        "               allow_pickle=True, \n",
        "               fix_imports=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t2O5XMTr2s5m"
      },
      "source": [
        "Some functions to get scores on the results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bhPJFtuzujdC"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import precision_recall_fscore_support, accuracy_score\n",
        "import re\n",
        "\n",
        "def get_confusion_matrix_classification(model, X, Y_true):\n",
        "    y_pred = model.predict(X)\n",
        "    y_true = np.apply_along_axis(np.argmax, 1, Y_true)\n",
        "    y_pred = np.apply_along_axis(np.argmax, 1, y_pred)\n",
        "    return (confusion_matrix(y_true, y_pred), y_pred, y_true)\n",
        "\n",
        "def construct_confusion_matrix(X, Y_true, Y_pred):\n",
        "    y_true = Y_true\n",
        "    y_pred = np.apply_along_axis(np.argmax, 1, Y_pred)\n",
        "    return (confusion_matrix(y_true, y_pred), y_pred, y_true)\n",
        "\n",
        "def pr_rc_f1_acc_from_supplied(y_pred, y_true):  \n",
        "    pr, rc, f1, _ = precision_recall_fscore_support(y_true, y_pred, average=\"weighted\")   \n",
        "    acc = accuracy_score(y_true, y_pred)\n",
        "    return pr, rc, f1, acc\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mVz_etHKO89T"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "import os\n",
        "\n",
        "def dir_has_file_with_regex(dir_name, regex_string):\n",
        "  filenames = [ f\"{dir_name}/{dir_entry.name}\" for dir_entry in os.scandir(dir_name) if os.path.isfile(f\"{dir_name}/{dir_entry.name}\") ]   \n",
        "  filenames = [ fn for fn in filenames if re.match(regex_string, fn, re.IGNORECASE) ]\n",
        "  return filenames\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PZU3TT6VZwLU"
      },
      "outputs": [],
      "source": [
        "# dir_has_file_with_regex(\"/content/drive/MyDrive/data_papers/gpSVHN/model_features\", \"^.*DNN_A_.*D3R_Test_X.*\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IiW-RCSEtkeR",
        "outputId": "a0681d77-6df1-483e-dfa7-c20853089dac"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "GPU 0: Tesla V100-SXM2-16GB (UUID: GPU-079e25da-1ef9-8a36-ff4f-49f366f518ca)\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi -L"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rLjgQs2buLYd",
        "outputId": "b2b39a90-cd2f-4504-c96f-1c82ab4cf146"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: ipython-autotime in /usr/local/lib/python3.7/dist-packages (0.3.1)\n",
            "Requirement already satisfied: ipython in /usr/local/lib/python3.7/dist-packages (from ipython-autotime) (5.5.0)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.7/dist-packages (from ipython->ipython-autotime) (5.1.1)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.7/dist-packages (from ipython->ipython-autotime) (57.4.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython->ipython-autotime) (0.7.5)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython->ipython-autotime) (2.6.1)\n",
            "Requirement already satisfied: pexpect in /usr/local/lib/python3.7/dist-packages (from ipython->ipython-autotime) (4.8.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython->ipython-autotime) (4.4.2)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython->ipython-autotime) (0.8.1)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython->ipython-autotime) (1.0.18)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython->ipython-autotime) (0.2.5)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython->ipython-autotime) (1.15.0)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.7/dist-packages (from pexpect->ipython->ipython-autotime) (0.7.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install ipython-autotime"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FY393TrFuNod",
        "outputId": "1dd60e18-62e0-4762-c8e2-05273a9fbde0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 100 µs (started: 2022-01-12 14:13:48 +00:00)\n"
          ]
        }
      ],
      "source": [
        "%load_ext autotime"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T1RHIyWyuxrh",
        "outputId": "2ed0b331-a623-4e93-df12-e75d46801720"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 923 µs (started: 2022-01-12 14:13:49 +00:00)\n"
          ]
        }
      ],
      "source": [
        "import timeit"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UrA1SY4CHTOS"
      },
      "source": [
        "# A basic DNN to fit SVHN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GHdJJrIbn3DR",
        "outputId": "864214bf-bdc9-426c-9066-680a5782b171"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 5.52 ms (started: 2022-01-10 11:52:03 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def basic_DNNTemplate_A(model_name, inshape, num_classes = 10):\n",
        "\n",
        "  base_input = Input(shape=inshape, name='base_input')\n",
        "  f1_output = Flatten(name='F1')(base_input)\n",
        "  d1_output = Dense(128, activation='relu', name = 'D1')(f1_output)\n",
        "  d2_output = Dense(128, activation='relu', name = 'D2')(d1_output)\n",
        "  d3r_output = Dense(48, activation='relu', kernel_regularizer=tf.keras.regularizers.l1(0.005), name = 'D3R')(d2_output)\n",
        "  model_output = Dense(10,  activation='softmax', name = 'SFTMX1')(d3r_output)\n",
        "  model = Model(base_input, model_output, name = model_name)\n",
        "\n",
        "  return model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TogBuHZwn8pl"
      },
      "outputs": [],
      "source": [
        "train_data_grey[0,:,:,:].shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 452
        },
        "id": "d7Q4E6VW3QaW",
        "outputId": "c010112f-5ce8-478d-e678-ea0ba71719b3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/DNN_A_1_20220110115206_saved_model_after_fit/assets\n",
            "103.02348736299973\n"
          ]
        },
        {
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-50-093c5ed7a231>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     14\u001b[0m                     \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m250\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m                     \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m                     validation_data=(validation_data_grey, validation_targets))\n\u001b[0m\u001b[1;32m     17\u001b[0m   \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdefault_timer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mstart_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;31m# # /content/drive/MyDrive/data_papers/gpSVHN/model_finals/DNN_A_6_20210923210545_saved_model_after_fit/assets\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-23-633f1114e7bd>\u001b[0m in \u001b[0;36mcompile_and_fit_model_basic\u001b[0;34m(model_func, model_name, input_shape, X_train, Y_train, save_max_epoch, save_final, patience_count, early_stopping_obs, log_history, verbose_level, **kwargs)\u001b[0m\n\u001b[1;32m     41\u001b[0m               metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])\n\u001b[1;32m     42\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 43\u001b[0;31m     \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks_used\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose_level\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     44\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0msave_final\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0mmake_dir_if_not_exist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1214\u001b[0m                 _r=1):\n\u001b[1;32m   1215\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1216\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1217\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1218\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    908\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    909\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 910\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    911\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    912\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    940\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    941\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 942\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    943\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    944\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   3129\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   3130\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 3131\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   3132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3133\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1958\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1959\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1960\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1961\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1962\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    601\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    602\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 603\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    604\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    605\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 59\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     60\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 2min 51s (started: 2022-01-10 11:52:06 +00:00)\n"
          ]
        }
      ],
      "source": [
        "# # saving 100 DNNs\n",
        "for model_count in [i+1 for i in range(100)]:\n",
        "  start_time = timeit.default_timer()\n",
        "  m1, h1 = compile_and_fit_model_basic( basic_DNNTemplate_A,  \n",
        "                    f\"DNN_A_{str(model_count)}_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    train_data_grey[0,:,:,:].shape, \n",
        "                    train_data_grey, train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(validation_data_grey, validation_targets))\n",
        "  print(timeit.default_timer()-start_time)\n",
        "# # /content/drive/MyDrive/data_papers/gpSVHN/model_finals/DNN_A_6_20210923210545_saved_model_after_fit/assets\n",
        "# plot_history(h1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8tP4KtdkyPrM"
      },
      "outputs": [],
      "source": [
        "# saving the features of 100 DNNs for the train data\n",
        "\n",
        "check_model_string = \"DNN_A_\"\n",
        "not_check_model_string = [\"arallel\",\"Collab_\"]\n",
        "for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\"):\n",
        "  if os.path.isdir(dir_entry):\n",
        "    if check_model_string in str(dir_entry) and all([(not ncs in dir_entry.name) for ncs in not_check_model_string]):\n",
        "      # f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features/{model_here.name}_features_{layer_name}_{dataset_id}_X\"\n",
        "      # f\"DNN_A_{str(model_count)}_{datetime.datetime.now():%Y%m%d%H%M%S}\"\n",
        "      if not dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{dir_entry.name.replace('_saved_model_after_fit','')}.*_features_D3R_Train.*$\"):\n",
        "        print(dir_entry.name)\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"D3R\",\n",
        "          train_data_grey, train_targets,\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=\"Train\")\n",
        "      if not dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{dir_entry.name.replace('_saved_model_after_fit','')}.*_features_SFTMX1_Train.*$\"):\n",
        "        print(dir_entry.name)\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"SFTMX1\",\n",
        "          train_data_grey, train_targets,\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=\"Train\")      "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DbyJqR9rbWlc"
      },
      "outputs": [],
      "source": [
        "# dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{check_model_string}.*_features_D3R_Train.*$\")\n",
        "\n",
        "# check_model_string = \"DNN_A_\"\n",
        "# not_check_model_string = [\"arallel\",\"Collab_\"]\n",
        "# for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\"):\n",
        "#   if os.path.isdir(dir_entry):\n",
        "#     if check_model_string in str(dir_entry) and all([(not ncs in dir_entry.name) for ncs in not_check_model_string]):\n",
        "#       print(dir_entry.name)\n",
        "#       print(dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{dir_entry.name.replace('_saved_model_after_fit','')}.*_features_D3R_Train.*$\"))\n",
        "#       break\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "r_HsBg4cIfIa"
      },
      "outputs": [],
      "source": [
        "# saving the features of 100 DNNs for the validation data\n",
        "check_model_string = \"DNN_A_\"\n",
        "not_check_model_string = [\"arallel\",\"Collab_\"]\n",
        "for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\"):\n",
        "  if os.path.isdir(dir_entry):\n",
        "    if check_model_string in str(dir_entry) and all([(not ncs in dir_entry.name) for ncs in not_check_model_string]):\n",
        "      # f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features/{model_here.name}_features_{layer_name}_{dataset_id}_X\"\n",
        "      if not dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{dir_entry.name.replace('_saved_model_after_fit','')}.*_features_D3R_Validation.*$\"):\n",
        "        print(dir_entry.name)\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"D3R\",\n",
        "          validation_data_grey, validation_targets,\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=\"Validation\")\n",
        "      if not dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{dir_entry.name.replace('_saved_model_after_fit','')}.*_features_SFTMX1_Validation.*$\"):\n",
        "        print(dir_entry.name)\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"SFTMX1\",\n",
        "          validation_data_grey, validation_targets,\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=\"Validation\")      "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "t_NjgPWEIqXy"
      },
      "outputs": [],
      "source": [
        "# saving the features of 100 DNNs for the test data\n",
        "check_model_string = \"DNN_A_\"\n",
        "not_check_model_string = [\"arallel\",\"Collab_\"]\n",
        "for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\"):\n",
        "  if os.path.isdir(dir_entry):\n",
        "    if check_model_string in str(dir_entry) and all([(not ncs in dir_entry.name) for ncs in not_check_model_string]):\n",
        "      if not dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{dir_entry.name}.*_features_D3R_Test.*$\"):\n",
        "        print(dir_entry.name)\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"D3R\",\n",
        "          test_data_grey, test_targets,\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=\"Test\")\n",
        "      if not dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{dir_entry.name}.*_features_SFTMX1_Test.*$\"):\n",
        "        print(dir_entry.name)\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"SFTMX1\",\n",
        "          test_data_grey, test_targets,\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=\"Test\")      "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gnFbW-5mqeVl"
      },
      "outputs": [],
      "source": [
        "# getting the scores for the individual 100 DNNs on the test data set\n",
        "scores_dnn_simple = []\n",
        "check_model_string = \"DNN_A_\"\n",
        "not_check_model_string = [\"arallel\",\"Collab_\"]\n",
        "for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\"):\n",
        "  if os.path.isdir(dir_entry):\n",
        "    if check_model_string in str(dir_entry) and all([ (not ncs in dir_entry.name) for ncs in not_check_model_string]):\n",
        "      print(dir_entry.name)\n",
        "      model_here = tf.keras.models.load_model(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\")  \n",
        "      y_predict_here = np.array(model_here.predict(test_data_grey), dtype='float64')\n",
        "      y_predict_here = np.apply_along_axis(np.argmax, 1, y_predict_here)\n",
        "      scores_dnn_simple.append(pr_rc_f1_acc_from_supplied(y_predict_here, test_targets))\n",
        "\n",
        "np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/individual_dnn_summary_{datetime.datetime.now():%Y%m%d%H%M%S}\", np.array(scores_dnn_simple), \n",
        "               allow_pickle=True, \n",
        "               fix_imports=True)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NUnTfSfY2pZK"
      },
      "source": [
        "# A basic CNN to fit SVHN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UjivSGDyT0xA",
        "outputId": "d2027462-c37c-4c15-daba-3c6b3a9e0dd4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 11.1 ms (started: 2022-01-10 14:06:53 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def basic_CNNTemplate_A(model_name, inshape, num_classes = 10):\n",
        "  # Input Layer\n",
        "  base_input = Input(shape=inshape, name='base_input')\n",
        "  # Convolutional Layer #1\n",
        "  c1_output = Conv2D(filters=32,kernel_size=[5, 5],padding=\"same\",activation=\"relu\", name=\"C1\")(base_input)\n",
        "\n",
        "  # Pooling Layer #1\n",
        "  mxp1_output = MaxPooling2D(pool_size=[2, 2], strides=2, name=\"MXP1\")(c1_output)\n",
        "  c2_output = Conv2D(filters=64,kernel_size=[5, 5],padding=\"same\",activation=\"relu\",name=\"C2\")(mxp1_output)\n",
        "    \n",
        "  #with tf.name_scope('Pool2 Layer'):\n",
        "  mxp2_output = MaxPooling2D(pool_size=[2, 2], strides=2, name=\"MXP2\")(c2_output)\n",
        "  f1_output = Flatten(name='F1')(mxp2_output)\n",
        "\n",
        "  # Dense Layer\n",
        "  d1_output = Dense(units=256, activation=\"relu\", name=\"D1\")(f1_output)\n",
        "  drp1_output = Dropout(rate=0.5, name=\"DRP1\")(d1_output)\n",
        "\n",
        "  model_output = Dense(num_classes,  activation='softmax', name = 'SFTMX1')(drp1_output)\n",
        "  model = Model(base_input, model_output, name = model_name)\n",
        "  return model\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 525
        },
        "id": "GqC4AbkNH2ne",
        "outputId": "790d5134-8018-4425-fc47-ab8c52ee8d92"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/CNN_A_1_20220110140657_saved_model_after_fit/assets\n",
            "141.07107983399874\n",
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/CNN_A_2_20220110140918_saved_model_after_fit/assets\n",
            "89.44230966600117\n",
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/CNN_A_3_20220110141047_saved_model_after_fit/assets\n",
            "185.23633293700004\n"
          ]
        },
        {
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-54-ea65e8203be5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     14\u001b[0m                     \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m250\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m                     \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m                     validation_data=(validation_data_grey, validation_targets))\n\u001b[0m\u001b[1;32m     17\u001b[0m   \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdefault_timer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mstart_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-23-633f1114e7bd>\u001b[0m in \u001b[0;36mcompile_and_fit_model_basic\u001b[0;34m(model_func, model_name, input_shape, X_train, Y_train, save_max_epoch, save_final, patience_count, early_stopping_obs, log_history, verbose_level, **kwargs)\u001b[0m\n\u001b[1;32m     41\u001b[0m               metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])\n\u001b[1;32m     42\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 43\u001b[0;31m     \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks_used\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose_level\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     44\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0msave_final\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0mmake_dir_if_not_exist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1206\u001b[0m         \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_epoch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1207\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcatch_stop_iteration\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1208\u001b[0;31m           \u001b[0;32mfor\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msteps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1209\u001b[0m             with tf.profiler.experimental.Trace(\n\u001b[1;32m   1210\u001b[0m                 \u001b[0;34m'train'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/data_adapter.py\u001b[0m in \u001b[0;36msteps\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1248\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_insufficient_data\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Set by `catch_stop_iteration`.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1249\u001b[0m         \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1250\u001b[0;31m       \u001b[0moriginal_spe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_steps_per_execution\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1251\u001b[0m       can_run_full_execution = (\n\u001b[1;32m   1252\u001b[0m           \u001b[0moriginal_spe\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/resource_variable_ops.py\u001b[0m in \u001b[0;36mnumpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    643\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    644\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 645\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    646\u001b[0m     raise NotImplementedError(\n\u001b[1;32m    647\u001b[0m         \"numpy() is only available when eager execution is enabled.\")\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/resource_variable_ops.py\u001b[0m in \u001b[0;36mread_value\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    718\u001b[0m     \"\"\"\n\u001b[1;32m    719\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Read\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 720\u001b[0;31m       \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_variable_op\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    721\u001b[0m     \u001b[0;31m# Return an identity so it can get placed on whatever device the context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    722\u001b[0m     \u001b[0;31m# specifies instead of the device where the variable is.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/resource_variable_ops.py\u001b[0m in \u001b[0;36m_read_variable_op\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    697\u001b[0m           \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mread_and_set_handle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    698\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 699\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mread_and_set_handle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    700\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    701\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/resource_variable_ops.py\u001b[0m in \u001b[0;36mread_and_set_handle\u001b[0;34m()\u001b[0m\n\u001b[1;32m    688\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread_and_set_handle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    689\u001b[0m       result = gen_resource_variable_ops.read_variable_op(\n\u001b[0;32m--> 690\u001b[0;31m           self.handle, self._dtype)\n\u001b[0m\u001b[1;32m    691\u001b[0m       \u001b[0m_maybe_set_handle_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    692\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/gen_resource_variable_ops.py\u001b[0m in \u001b[0;36mread_variable_op\u001b[0;34m(resource, dtype, name)\u001b[0m\n\u001b[1;32m    469\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    470\u001b[0m       _result = pywrap_tfe.TFE_Py_FastPathExecute(\n\u001b[0;32m--> 471\u001b[0;31m         _ctx, \"ReadVariableOp\", name, resource, \"dtype\", dtype)\n\u001b[0m\u001b[1;32m    472\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    473\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0m_core\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 7min 13s (started: 2022-01-10 14:06:57 +00:00)\n"
          ]
        }
      ],
      "source": [
        "# # saving 100 CNNs\n",
        "for model_count in [i+1 for i in range(100)]:\n",
        "  start_time = timeit.default_timer()\n",
        "  m1, h1 = compile_and_fit_model_basic( basic_CNNTemplate_A,  \n",
        "                    f\"CNN_A_{str(model_count)}_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    train_data_grey[0,:,:,:].shape, \n",
        "                    train_data_grey, train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(validation_data_grey, validation_targets))\n",
        "  print(timeit.default_timer()-start_time)\n",
        "\n",
        "# # /content/drive/MyDrive/data_papers/gpSVHN/model_finals/DNN_A_6_20210923210545_saved_model_after_fit/assets\n",
        "# plot_history(h1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0rhK968KxRy3"
      },
      "outputs": [],
      "source": [
        "# # saving the features of 100 CNNs for the training data\n",
        "\n",
        "check_model_string = \"CNN_A_\"\n",
        "not_check_model_string = [\"arallel\",\"Collab_\"]\n",
        "for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\"):\n",
        "  if os.path.isdir(dir_entry):\n",
        "    if check_model_string in str(dir_entry) and all([ (not ncs in dir_entry.name) for ncs in not_check_model_string]):\n",
        "      if not dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{dir_entry.name.replace('_saved_model_after_fit','')}.*_features_DRP1_Train.*$\"):\n",
        "        print(dir_entry.name)\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"DRP1\",\n",
        "          train_data_grey, train_targets,\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=\"Train\")\n",
        "      if not dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{dir_entry.name.replace('_saved_model_after_fit','')}.*_features_SFTMX1_Train.*$\"):\n",
        "        print(dir_entry.name)\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"SFTMX1\",\n",
        "          train_data_grey, train_targets,\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=\"Train\")      "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kk8HZeqOI5IZ"
      },
      "outputs": [],
      "source": [
        "# # saving the features of 100 CNNs for the Validation data\n",
        "\n",
        "# # /content/drive/MyDrive/data_papers/gpSVHN/model_finals/CNN_A_\n",
        "check_model_string = \"CNN_A_\"\n",
        "not_check_model_string = [\"arallel\",\"Collab_\"]\n",
        "for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\"):\n",
        "  if os.path.isdir(dir_entry):\n",
        "    if check_model_string in str(dir_entry) and all([ (not ncs in dir_entry.name) for ncs in not_check_model_string]):\n",
        "      if not dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{dir_entry.name.replace('_saved_model_after_fit','')}.*_features_DRP1_Validation.*$\"):\n",
        "        print(dir_entry.name)\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"DRP1\",\n",
        "          validation_data_grey, validation_targets,\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=\"Validation\")\n",
        "      if not dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{dir_entry.name.replace('_saved_model_after_fit','')}.*_features_SFTMX1_Validation.*$\"):\n",
        "        print(dir_entry.name)\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"SFTMX1\",\n",
        "          validation_data_grey, validation_targets,\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=\"Validation\")      "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XIjoa4OnJAih"
      },
      "outputs": [],
      "source": [
        "# # saving the features of 100 CNNs for the Test data\n",
        "\n",
        "# # /content/drive/MyDrive/data_papers/gpSVHN/model_finals/CNN_A_\n",
        "check_model_string = \"CNN_A_\"\n",
        "not_check_model_string = [\"arallel\",\"Collab_\"]\n",
        "for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\"):\n",
        "  if os.path.isdir(dir_entry):\n",
        "    if check_model_string in str(dir_entry) and all([ (not ncs in dir_entry.name) for ncs in not_check_model_string]):\n",
        "      if not dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{dir_entry.name.replace('_saved_model_after_fit','')}.*_features_DRP1_Test.*$\"):\n",
        "        print(dir_entry.name)\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"DRP1\",\n",
        "          test_data_grey, test_targets,\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=\"Test\")\n",
        "      if not dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{dir_entry.name.replace('_saved_model_after_fit','')}.*_features_SFTMX1_Test.*$\"):\n",
        "        print(dir_entry.name)\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"SFTMX1\",\n",
        "          test_data_grey, test_targets,\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=\"Test\")      "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xZsCz3bYySRB"
      },
      "outputs": [],
      "source": [
        "# getting the scores for the individual 100 CNNs on the test data set\n",
        "scores_cnn_simple = []\n",
        "check_model_string = \"CNN_A_\"\n",
        "not_check_model_string = [\"arallel\",\"Collab_\"]\n",
        "for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\"):\n",
        "  if os.path.isdir(dir_entry):\n",
        "    if check_model_string in str(dir_entry) and all([ (not ncs in dir_entry.name) for ncs in not_check_model_string]):\n",
        "      print(dir_entry.name)\n",
        "      model_here = tf.keras.models.load_model(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\")  \n",
        "      y_predict_here = np.array(model_here.predict(test_data_grey), dtype='float64')\n",
        "      y_predict_here = np.apply_along_axis(np.argmax, 1, y_predict_here)\n",
        "      scores_cnn_simple.append(pr_rc_f1_acc_from_supplied(y_predict_here, test_targets))\n",
        "\n",
        "np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/individual_cnn_summary_{datetime.datetime.now():%Y%m%d%H%M%S}\", np.array(scores_cnn_simple), \n",
        "               allow_pickle=True, \n",
        "               fix_imports=True)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "boCHy50O_Z-m"
      },
      "source": [
        "# ResNet50 fit SVHN grey"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zMFmsZHu_s9S"
      },
      "outputs": [],
      "source": [
        "# from tensorflow.keras.applications.vgg16 import VGG16\n",
        "from tensorflow.keras.applications.vgg19 import VGG19\n",
        "from tensorflow.keras.applications import ResNet50\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Flatten, GlobalAveragePooling2D, BatchNormalization\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0hG4HIud_Z-n"
      },
      "outputs": [],
      "source": [
        "def basic_ResNet50Template_A(model_name, inshape, num_classes = 10):\n",
        "  model = Sequential(name=model_name)\n",
        "  model.add(ResNet50(include_top=True, pooling='avg', weights=None,input_shape=inshape, classes=num_classes))\n",
        "  model.compile(loss='sparse_categorical_crossentropy',\n",
        "                optimizer='adam',      \n",
        "                metrics=['acc'])\n",
        "  return(model)\n",
        "\n",
        "def basic_VGG19Template_A(model_name, inshape, num_classes = 10):\n",
        "  model = Sequential(name=model_name)\n",
        "  model.add(VGG19(include_top=True, pooling='avg', weights=None,input_shape=inshape, classes=num_classes))\n",
        "  model.compile(loss='sparse_categorical_crossentropy',\n",
        "                optimizer='adam',      \n",
        "                metrics=['acc'])\n",
        "  return(model)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mlIRttQrBd_x"
      },
      "outputs": [],
      "source": [
        "# # saving 100 RestNet50\n",
        "for model_count in [i+1 for i in range(40)]:\n",
        "  m1, h1 = compile_and_fit_model_basic( basic_ResNet50Template_A,  \n",
        "                    f\"RestNet50_A_{str(model_count)}_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    train_data_grey[0,:,:,:].shape, \n",
        "                    train_data_grey, train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    verbose_level = 1,\n",
        "                    validation_data=(validation_data_grey, validation_targets))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AUtudPCgMrN7"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras.utils import plot_model\n",
        "# plot_model(m1)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mA5PSzi6NvjP"
      },
      "outputs": [],
      "source": [
        "#  ResNets do NOT allow feature extractions!\n",
        "# model_entry = m1\n",
        "# X_input = train_data_grey.copy()\n",
        "# Y_input = train_targets.copy()\n",
        "# layer_name = \"resnet50\"\n",
        "# m1.input\n",
        "# get_layer_by_name(m1.layers, \"resnet50\").output\n",
        "# Model(m1.input, get_layer_by_name(m1.layers, layer_name).output)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wXQ_hD5xP5yh"
      },
      "outputs": [],
      "source": [
        "# getting the scores for the individual 40 ResNets on the test data set\n",
        "scores_resnets_simple = []\n",
        "check_model_string = \"RestNet50_A\"\n",
        "not_check_model_string = [\"arallel\",\"Collab_\"]\n",
        "for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\"):\n",
        "  if os.path.isdir(dir_entry):\n",
        "    if check_model_string in str(dir_entry) and all([ (not ncs in dir_entry.name) for ncs in not_check_model_string]):\n",
        "      print(dir_entry.name)\n",
        "      model_here = tf.keras.models.load_model(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\")  \n",
        "      y_predict_here = np.array(model_here.predict(test_data_grey), dtype='float64')\n",
        "      y_predict_here = np.apply_along_axis(np.argmax, 1, y_predict_here)\n",
        "      scores_resnets_simple.append(pr_rc_f1_acc_from_supplied(y_predict_here, test_targets))\n",
        "\n",
        "np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/individual_resnets_summary_{datetime.datetime.now():%Y%m%d%H%M%S}\", np.array(scores_resnets_simple), \n",
        "               allow_pickle=True, \n",
        "               fix_imports=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WS7USmAlAZ_Q"
      },
      "source": [
        "# WideResNet fit SVHN grey"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FkC-qxwi5U1f"
      },
      "outputs": [],
      "source": [
        "import uuid\n",
        "# uuid.uuid4()\n",
        "# str(uuid.uuid4()).split(\"-\")[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kSq59Zr_AZBE"
      },
      "outputs": [],
      "source": [
        "# https://github.com/asmith26/wide_resnets_keras\n",
        "\n",
        "# Wide residual network http://arxiv.org/abs/1605.07146\n",
        "def _wide_basic(n_input_plane, n_output_plane, stride, identifier, \n",
        "                channel_axis = -1,\n",
        "                weight_decay = 0.0005,\n",
        "                weight_init=\"he_normal\",\n",
        "                use_bias = False,\n",
        "                dropout_probability = 0.0\n",
        "                ):\n",
        "    def f(net):\n",
        "        # format of conv_params:\n",
        "        #               [ [nb_col=\"kernel width\", nb_row=\"kernel height\",\n",
        "        #               subsample=\"(stride_vertical,stride_horizontal)\",\n",
        "        #               border_mode=\"same\" or \"valid\"] ]\n",
        "        # B(3,3): orignal <<basic>> block\n",
        "        conv_params = [ [3,3,stride,\"same\"],\n",
        "                        [3,3,(1,1),\"same\"] ]\n",
        "        \n",
        "        n_bottleneck_plane = n_output_plane\n",
        "\n",
        "        # Residual block\n",
        "        for i, v in enumerate(conv_params):\n",
        "            if i == 0:\n",
        "                if n_input_plane != n_output_plane:\n",
        "                    net = BatchNormalization(axis=channel_axis, name=f\"BN{str(i)}_{identifier}_{uuid.uuid4()}\")(net)\n",
        "                    net = Activation(\"relu\")(net)\n",
        "                    convs = net\n",
        "                else:\n",
        "                    convs = BatchNormalization(axis=channel_axis, name=f\"BN{str(i)}_{identifier}_{uuid.uuid4()}\")(net)\n",
        "                    convs = Activation(\"relu\")(convs)\n",
        "                convs = Conv2D(n_bottleneck_plane, \n",
        "                               (v[0],v[1]),\n",
        "                                strides=v[2],\n",
        "                                padding=v[3],\n",
        "                                kernel_initializer=weight_init,\n",
        "                                kernel_regularizer=L2(weight_decay),\n",
        "                                use_bias=use_bias,\n",
        "                                name = f\"CONV0_{identifier}_{uuid.uuid4()}\")(convs)\n",
        "            else:\n",
        "                convs = BatchNormalization(axis=channel_axis, name=f\"BN{str(i)}_{identifier}_{uuid.uuid4()}\")(convs)\n",
        "                convs = Activation(\"relu\")(convs)\n",
        "                if dropout_probability > 0:\n",
        "                   convs = Dropout(dropout_probability, name=f\"DRP{str(i)}_{identifier}_{uuid.uuid4()}\")(convs)\n",
        "                convs = Conv2D(n_bottleneck_plane, \n",
        "                               (v[0],v[1]),\n",
        "                                strides=v[2],\n",
        "                                padding=v[3],\n",
        "                                kernel_initializer=weight_init,\n",
        "                                kernel_regularizer=L2(weight_decay),\n",
        "                                use_bias=use_bias,\n",
        "                                name=f\"CONV{str(i)}_{identifier}_{uuid.uuid4()}\")(convs)\n",
        "\n",
        "        # Shortcut Conntection: identity function or 1x1 convolutional\n",
        "        #  (depends on difference between input & output shape - this\n",
        "        #   corresponds to whether we are using the first block in each\n",
        "        #   group; see _layer() ).\n",
        "        if n_input_plane != n_output_plane:\n",
        "            shortcut = Conv2D(n_output_plane, \n",
        "                              (1,1),\n",
        "                              strides=stride,\n",
        "                              padding=\"same\",\n",
        "                              kernel_initializer=weight_init,\n",
        "                              kernel_regularizer=L2(weight_decay),\n",
        "                              use_bias=use_bias,\n",
        "                              name=f\"CONVSHORTCUT_{identifier}_{uuid.uuid4()}\")(net)\n",
        "        else:\n",
        "            shortcut = net\n",
        "\n",
        "        return Add()([convs, shortcut])\n",
        "    \n",
        "    return f\n",
        "\n",
        "\n",
        "# \"Stacking Residual Units on the same stage\"\n",
        "def _layer(block, n_input_plane, n_output_plane, count, stride):\n",
        "    def f(net):\n",
        "        net = block(n_input_plane, n_output_plane, stride)(net)\n",
        "        for i in range(2,int(count+1)):\n",
        "            net = block(n_output_plane, n_output_plane, stride=(1,1))(net)\n",
        "        return net\n",
        "    \n",
        "    return f\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n14_FVnPRzTA"
      },
      "source": [
        "This will be WRN-28-10 WideResNet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2zWw-kt6RxYk"
      },
      "outputs": [],
      "source": [
        "input_shape=train_data_grey[0,:,:,:].shape\n",
        "weight_decay = 0.0005\n",
        "weight_init=\"he_normal\"\n",
        "use_bias = False\n",
        "k = 10\n",
        "depth = 28             \n",
        "n = (depth - 4) / 6\n",
        "dropout_probability = 0.0\n",
        "# batch_size = 128      \n",
        "# nb_epochs = 200\n",
        "channel_axis = -1\n",
        "\n",
        "num_wrn_models = 10\n",
        "no_classes=10\n",
        "import functools"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QuHzXzVpVkmd"
      },
      "outputs": [],
      "source": [
        "# set up 40 WRN-28-10 models \n",
        "model_wrns = []\n",
        "\n",
        "for i in range(num_wrn_models):\n",
        "  inputs_wrn = Input(shape=input_shape,name=f\"M{str(i)}_INPUT\")\n",
        "  n_stages=[16, 16*k, 32*k, 64*k]\n",
        "  conv1_wrn = Conv2D(16, \n",
        "                  (3, 3), \n",
        "                  strides=1,\n",
        "                  padding=\"same\",\n",
        "                  kernel_initializer=weight_init,\n",
        "                  kernel_regularizer=L2(weight_decay),\n",
        "                  use_bias=use_bias,\n",
        "                  name=\"C1BLOCK\")(inputs_wrn) # \"One conv at the beginning (spatial size: 32x32)\"\n",
        "  # Add wide residual blocks\n",
        "  block_fn = _wide_basic\n",
        "  conv2_wrn = _layer(functools.partial(block_fn,identifier=f\"C2BLOCK\"), n_input_plane=n_stages[0], n_output_plane=n_stages[1], count=n, stride=(1,1))(conv1_wrn)# \"Stage 1 (spatial size: 32x32)\"\n",
        "  conv3_wrn = _layer(functools.partial(block_fn,identifier=f\"C3BLOCK\"), n_input_plane=n_stages[1], n_output_plane=n_stages[2], count=n, stride=(2,2))(conv2_wrn)# \"Stage 2 (spatial size: 16x16)\"\n",
        "  conv4_wrn = _layer(functools.partial(block_fn,identifier=f\"C4BLOCK\"), n_input_plane=n_stages[2], n_output_plane=n_stages[3], count=n, stride=(2,2))(conv3_wrn)# \"Stage 3 (spatial size: 8x8)\"\n",
        "\n",
        "  batch_norm_wrn = BatchNormalization(axis=channel_axis,name=f\"M{str(i)}_BN\")(conv4_wrn)\n",
        "  relu_wrn = Activation(\"relu\")(batch_norm_wrn)\n",
        "                                          \n",
        "  # Classifier block\n",
        "  pool_wrn = AveragePooling2D(pool_size=(8, 8), strides=(1, 1), padding=\"same\", name=f\"CLASSIFIER_AVPL\")(relu_wrn)\n",
        "  flatten_wrn = Flatten(name=f\"CLASSIFIER_FL\")(pool_wrn)\n",
        "  predictions_wrn = Dense(units=no_classes, kernel_initializer=weight_init, use_bias=use_bias,\n",
        "                      kernel_regularizer=L2(weight_decay), activation=\"softmax\", name=\"CLASSIFIER_D1\")(flatten_wrn)\n",
        "  model_wrn = Model(inputs=inputs_wrn, outputs=predictions_wrn)\n",
        "  model_wrns.append(model_wrn)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Lq0CzhUGVIg_"
      },
      "outputs": [],
      "source": [
        "for model_count in [i for i in range(len(model_wrns))]:\n",
        "  mwrn, hwrn = compile_and_fit_model_basic(  model_wrns[model_count], \n",
        "                                           f\"WideResNet28-10_ID{str(uuid.uuid4()).split('-')[0]}_{str(model_count+11)}_{datetime.datetime.now():%Y%m%d%H%M%S}\",\n",
        "                                          train_data_grey[0,:,:,:].shape,\n",
        "                                          train_data_grey,\n",
        "                                          train_targets,\n",
        "                                          save_max_epoch=False,\n",
        "                                          save_final=True,\n",
        "                                          patience_count = 35,\n",
        "                                          early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                                          log_history = True,\n",
        "                                          verbose_level = 1,\n",
        "                                          batch_size=256, \n",
        "                                          epochs=250, \n",
        "                                          class_weight=None,\n",
        "                                          validation_data=(validation_data_grey, validation_targets))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "brTi7688Dx78"
      },
      "outputs": [],
      "source": [
        "# # saving the features of WideResNets for the training data\n",
        "\n",
        "check_model_string = \"WideResNet28-10_\"\n",
        "not_check_model_string = [\"arallel\",\"Collab_\"]\n",
        "\n",
        "acceptable_chunk = 5000\n",
        "idxs_for_train = np.unique(list(range(0,train_data_grey.shape[0],acceptable_chunk)) + [train_data_grey.shape[0]]).tolist()\n",
        "\n",
        "for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\"):\n",
        "  if os.path.isdir(dir_entry):\n",
        "    if check_model_string in str(dir_entry) and all([ (not ncs in dir_entry.name) for ncs in not_check_model_string]):\n",
        "      for i in range(len(idxs_for_train[:-1])):\n",
        "        print(f\"{dir_entry.name}_{idxs_for_train[i]}-{idxs_for_train[i+1]}\")\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"CLASSIFIER_FL\",\n",
        "          train_data_grey[idxs_for_train[i]:idxs_for_train[i+1],:,:,:], train_targets[idxs_for_train[i]:idxs_for_train[i+1],:],\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=f\"Train{str(i)}_{idxs_for_train[i]}-{idxs_for_train[i+1]}\")\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"CLASSIFIER_D1\",\n",
        "          train_data_grey[idxs_for_train[i]:idxs_for_train[i+1],:,:,:], train_targets[idxs_for_train[i]:idxs_for_train[i+1],:],\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=f\"Train{str(i)}_{idxs_for_train[i]}-{idxs_for_train[i+1]}\")\n",
        "        "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PzPD9hg1vTTk"
      },
      "outputs": [],
      "source": [
        "\n",
        "# # saving the features of WideResNets for the Validation data  \n",
        "\n",
        "check_model_string = \"WideResNet28-10_\"\n",
        "not_check_model_string = [\"arallel\",\"Collab_\"]\n",
        "\n",
        "acceptable_chunk = 5000\n",
        "idxs_for_validation = np.unique(list(range(0,validation_data_grey.shape[0],acceptable_chunk)) + [validation_data_grey.shape[0]]).tolist()\n",
        "\n",
        "for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\"):\n",
        "  if os.path.isdir(dir_entry):\n",
        "    if check_model_string in str(dir_entry) and all([ (not ncs in dir_entry.name) for ncs in not_check_model_string]):\n",
        "      for i in range(len(idxs_for_validation[:-1])):\n",
        "        print(f\"{dir_entry.name}__{idxs_for_validation[i]}-{idxs_for_validation[i+1]}\")\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"CLASSIFIER_FL\",\n",
        "          validation_data_grey[idxs_for_validation[i]:idxs_for_validation[i+1],:,:,:], validation_targets[idxs_for_validation[i]:idxs_for_validation[i+1],:],\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=f\"Validation{str(i)}_{idxs_for_validation[i]}-{idxs_for_validation[i+1]}\")     \n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"CLASSIFIER_D1\",\n",
        "          validation_data_grey[idxs_for_validation[i]:idxs_for_validation[i+1],:,:,:], validation_targets[idxs_for_validation[i]:idxs_for_validation[i+1],:],\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=f\"Validation{str(i)}_{idxs_for_validation[i]}-{idxs_for_validation[i+1]}\")     \n",
        "      "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "quehv3Q9fDC8"
      },
      "outputs": [],
      "source": [
        "gpus = tf.config.experimental.list_physical_devices('GPU') \n",
        "# for gpu in gpus: \n",
        "#   tf.config.experimental.set_memory_growth(gpu, True)\n",
        "gpus"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "T3g_gaA1S2iP"
      },
      "outputs": [],
      "source": [
        "\n",
        "# # saving the features of WideResNets for the Test data  \n",
        "\n",
        "check_model_string = \"WideResNet28-10_\"\n",
        "not_check_model_string = [\"arallel\",\"Collab_\"] \n",
        "\n",
        "acceptable_chunk = 5000\n",
        "idxs_for_test = np.unique(list(range(0,test_data_grey.shape[0],acceptable_chunk)) + [test_data_grey.shape[0]]).tolist()\n",
        "\n",
        "for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\"):\n",
        "  if os.path.isdir(dir_entry):\n",
        "    if check_model_string in str(dir_entry) and all([ (not ncs in dir_entry.name) for ncs in not_check_model_string]):\n",
        "      for i in range(len(idxs_for_test[:-1])):\n",
        "      # # saving the features of WideResNets for the Test data      \n",
        "        print(f\"{dir_entry.name}__{idxs_for_test[i]}-{idxs_for_test[i+1]}\")\n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"CLASSIFIER_FL\",\n",
        "          test_data_grey[idxs_for_test[i]:idxs_for_test[i+1],:,:,:], test_targets[idxs_for_test[i]:idxs_for_test[i+1],:],\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=f\"TestBatch{str(i)}_{idxs_for_test[i]}-{idxs_for_test[i+1]}\")      \n",
        "        write_features_from_models(\n",
        "          f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\",\n",
        "          \"CLASSIFIER_D1\",\n",
        "          test_data_grey[idxs_for_test[i]:idxs_for_test[i+1],:,:,:], test_targets[idxs_for_test[i]:idxs_for_test[i+1],:],\n",
        "          reverse_one_hot=False,\n",
        "          normalize_X_func=None,\n",
        "          dataset_id=f\"TestBatch{str(i)}_{idxs_for_test[i]}-{idxs_for_test[i+1]}\")              "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9TxCYEyUEnem"
      },
      "outputs": [],
      "source": [
        "# getting the scores for the individual WideResNets on the test data set\n",
        "scores_wideresnets_simple = []\n",
        "check_model_string = \"WideResNet28-10_\"\n",
        "not_check_model_string = [\"arallel\",\"Collab_\"]\n",
        "for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\"):\n",
        "  if os.path.isdir(dir_entry):\n",
        "    if check_model_string in str(dir_entry) and all([ (not ncs in dir_entry.name) for ncs in not_check_model_string]):\n",
        "      print(dir_entry.name)\n",
        "      model_here = tf.keras.models.load_model(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\")  \n",
        "      y_predict_here = np.array(model_here.predict(test_data_grey), dtype='float64')\n",
        "      y_predict_here = np.apply_along_axis(np.argmax, 1, y_predict_here)\n",
        "      print(pr_rc_f1_acc_from_supplied(y_predict_here, test_targets))\n",
        "      scores_wideresnets_simple.append(pr_rc_f1_acc_from_supplied(y_predict_here, test_targets))\n",
        "\n",
        "np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/individual_WideResNet2810_summary_{datetime.datetime.now():%Y%m%d%H%M%S}\", np.array(scores_wideresnets_simple), \n",
        "               allow_pickle=True, \n",
        "               fix_imports=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vu9-in9HfWZc"
      },
      "outputs": [],
      "source": [
        "# print(test_data_grey.shape)\n",
        "# print(test_targets.shape)\n",
        "# a=np.unique(list(range(0,test_data_grey.shape[0],5000)) + [test_data_grey.shape[0]]).tolist()\n",
        "# print(a)\n",
        "# a[:-1]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ib862JpbN7oI"
      },
      "source": [
        "# Save the validation results for all the DNN, CNN, ResNet50, WideResnets to see whether it can be used for 'selecting' models"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_09nA-cDONBT"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "\n",
        "def get_validation_acc_from_history_file(f1):\n",
        "  pd1 = pd.read_csv(f1)\n",
        "  return pd1[pd1.epoch==max(pd1.epoch)].val_sparse_categorical_accuracy.iloc[0]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rqG47BAEONen"
      },
      "outputs": [],
      "source": [
        "all_histories = os.listdir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_history\")\n",
        "\n",
        "all_dnn_histories = [f for f in all_histories if \"DNN_\" in f and \"arallel\" not in f and \"ollab\" not in f]\n",
        "all_cnn_histories = [f for f in all_histories if \"CNN_\" in f and \"arallel\" not in f and \"ollab\" not in f]\n",
        "all_resnet50_histories = [f for f in all_histories if \"RestNet50\" in f and \"arallel\" not in f and \"ollab\" not in f]\n",
        "all_WideResNet_histories = [f for f in all_histories if \"WideResNet\" in f and \"arallel\" not in f and \"ollab\" not in f]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7yHc2JWiiFpL"
      },
      "outputs": [],
      "source": [
        "dnn_val_accs = pd.DataFrame( { \"Type\": \"DNN\", \"File\": all_dnn_histories, \"ValAcc\" : [ get_validation_acc_from_history_file(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_history/{f}\") for f in all_dnn_histories  ] } )\n",
        "cnn_val_accs = pd.DataFrame( { \"Type\": \"CNN\", \"File\": all_cnn_histories, \"ValAcc\" : [ get_validation_acc_from_history_file(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_history/{f}\") for f in all_cnn_histories  ] } )\n",
        "resnet50_val_accs = pd.DataFrame( { \"Type\": \"ResNet50\", \"File\": all_resnet50_histories, \"ValAcc\" : [ get_validation_acc_from_history_file(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_history/{f}\") for f in all_resnet50_histories  ] } )\n",
        "wideResNet_val_accs = pd.DataFrame( { \"Type\": \"WideResNet\", \"File\": all_WideResNet_histories, \"ValAcc\" : [ get_validation_acc_from_history_file(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_history/{f}\") for f in all_WideResNet_histories  ] } )\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "feXi7f-Zq4vz"
      },
      "outputs": [],
      "source": [
        "import datetime\n",
        "val_accs_base_models = pd.concat([dnn_val_accs, cnn_val_accs, resnet50_val_accs, wideResNet_val_accs])\n",
        "val_accs_base_models.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/validation_accs_base_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\", index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d4NWpQ7wAS1u"
      },
      "source": [
        "# Set up ensembles of 20 DNN, 20 CNN or 10 CNN/10 DNN to predict (randomly assembled from the 100 before)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iIL2mp4cmkT0"
      },
      "outputs": [],
      "source": [
        "num_of_repeats = 20\n",
        "num_of_models = 20\n",
        "\n",
        "x_input = test_data_grey\n",
        "y_input = test_targets\n",
        "\n",
        "# create traditional ensemble of 20 DNNs \n",
        "scores_dnn20 = []\n",
        "for repc in range(num_of_repeats):\n",
        "  dnn_models_to_use = [ tf.keras.models.load_model(mfile) for mfile in sorted(random.sample(dnn_model_dirs, num_of_models))]\n",
        "  dnn_model_predictions = [ model.predict(x_input) for model in dnn_models_to_use]\n",
        "  y_ens_preds = avgfilter_ensemble_predictions(dnn_model_predictions, x_input)\n",
        "  scores_dnn20.append(pr_rc_f1_acc_from_supplied(y_ens_preds,y_input))\n",
        "\n",
        "# create traditional ensemble of 20 CNNs \n",
        "scores_cnn20 = []\n",
        "for repc in range(num_of_repeats):\n",
        "  cnn_models_to_use = [ tf.keras.models.load_model(mfile) for mfile in sorted(random.sample(cnn_model_dirs, num_of_models))]\n",
        "  cnn_model_predictions = [ model.predict(x_input) for model in cnn_models_to_use]\n",
        "  y_ens_preds = avgfilter_ensemble_predictions(cnn_model_predictions, x_input)\n",
        "  scores_cnn20.append(pr_rc_f1_acc_from_supplied(y_ens_preds,y_input))\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FROXYexX_CcB"
      },
      "outputs": [],
      "source": [
        "num_of_repeats = 20\n",
        "num_of_models = 20\n",
        "\n",
        "x_input = test_data_grey\n",
        "y_input = test_targets\n",
        "\n",
        "# create traditional ensemble of 10 CNNs and 10 DNNs\n",
        "scores_cnn10dnn10 = []\n",
        "for repc in range(num_of_repeats):\n",
        "  cnn_models_to_use = [ tf.keras.models.load_model(mfile) for mfile in sorted(random.sample(cnn_model_dirs, int(num_of_models/2)))]\n",
        "  cnn_model_predictions = [ model.predict(x_input) for model in cnn_models_to_use]\n",
        "  dnn_models_to_use = [ tf.keras.models.load_model(mfile) for mfile in sorted(random.sample(dnn_model_dirs, int(num_of_models/2)))]\n",
        "  models_to_use = cnn_models_to_use\n",
        "  models_to_use.extend(dnn_models_to_use)\n",
        "  model_predictions = [ model.predict(x_input) for model in models_to_use]\n",
        "  y_ens_preds = avgfilter_ensemble_predictions(model_predictions, x_input)\n",
        "  scores_cnn10dnn10.append(pr_rc_f1_acc_from_supplied(y_ens_preds,y_input))\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zVqJ8RVFV5pn"
      },
      "source": [
        "Save the score results of the ensembles"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DNE4dpF_V5pn"
      },
      "outputs": [],
      "source": [
        "np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/traditional_ensemble_dnn20_summary_{datetime.datetime.now():%Y%m%d%H%M%S}\", np.array(scores_dnn20), \n",
        "               allow_pickle=True, \n",
        "               fix_imports=True)\n",
        "np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/traditional_ensemble_cnn20_summary_{datetime.datetime.now():%Y%m%d%H%M%S}\", np.array(scores_cnn20), \n",
        "               allow_pickle=True, \n",
        "               fix_imports=True)\n",
        "np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/traditional_ensemble_cnn10dnn10_summary_{datetime.datetime.now():%Y%m%d%H%M%S}\", np.array(scores_cnn10dnn10), \n",
        "               allow_pickle=True, \n",
        "               fix_imports=True)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N3pww0F8PDy4"
      },
      "source": [
        "# Set up ensembles of 20 ResNets50 (randomly assembled from the 40 before) and 10 WideResNets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VctHsz0EPDy4"
      },
      "outputs": [],
      "source": [
        "num_of_repeats = 10\n",
        "num_of_models = 20\n",
        "\n",
        "x_input = test_data_grey\n",
        "y_input = test_targets\n",
        "\n",
        "# create traditional ensemble of 20 resnets \n",
        "scores_resnets = []\n",
        "for repc in range(num_of_repeats):\n",
        "  print(repc)\n",
        "  resnets_models_to_use = [ tf.keras.models.load_model(mfile) for mfile in sorted(random.sample(resnets_model_dirs, num_of_models))]\n",
        "  resnets_model_predictions = [ model.predict(x_input) for model in resnets_models_to_use]\n",
        "  y_ens_preds = avgfilter_ensemble_predictions(resnets_model_predictions, x_input)\n",
        "  scores_resnets.append(pr_rc_f1_acc_from_supplied(y_ens_preds,y_input))\n",
        "  del resnets_models_to_use\n",
        "  del resnets_model_predictions\n",
        "  del y_ens_preds\n",
        "\n",
        "\n",
        "# create a traditional ensemble of 10 wideresnets \n",
        "scores_wideresnets = []\n",
        "for repc in range(1):\n",
        "  print(repc)\n",
        "  wideresnets_models_to_use = [ tf.keras.models.load_model(mfile) for mfile in sorted(random.sample(wideresnets_model_dirs, 10))]\n",
        "  wideresnets_model_predictions = [ model.predict(x_input) for model in wideresnets_models_to_use]\n",
        "  y_ens_preds = avgfilter_ensemble_predictions(wideresnets_model_predictions, x_input)\n",
        "  scores_wideresnets.append(pr_rc_f1_acc_from_supplied(y_ens_preds,y_input))\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0VkkQpmnAdAE"
      },
      "source": [
        "Save the score results of the ensembles resnets and wideResNets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oC7uHy83spxp"
      },
      "outputs": [],
      "source": [
        "np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/traditional_ensemble_resnets_summary_{datetime.datetime.now():%Y%m%d%H%M%S}\", np.array(scores_resnets), \n",
        "               allow_pickle=True, \n",
        "               fix_imports=True)\n",
        "np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/traditional_ensemble_wideresnet28-10_summary_{datetime.datetime.now():%Y%m%d%H%M%S}\", np.array(scores_wideresnets), \n",
        "               allow_pickle=True, \n",
        "               fix_imports=True)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "05H-E-Hb21S9"
      },
      "source": [
        "# Set up the models + features file lists for CNN/DNN/ResNet50/WideResNet28-10 splits for reuse SVHN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ll_pxppktl28",
        "outputId": "1efb8802-d7c3-4a54-8e63-a8564687cdc9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 2 s (started: 2022-01-12 14:13:50 +00:00)\n"
          ]
        }
      ],
      "source": [
        "acceptable_string_grabs = [ \"CNN_A_\", \"DNN_A_\", \"RestNet50_A_\", \"WideResNet28-10_ID\"]\n",
        "not_check_model_string = [\"arallel\",\"Collab_\"]\n",
        "\n",
        "model_dirs = [  f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/{dir_entry.name}\" \n",
        "                for dir_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_finals/\") \n",
        "                if os.path.isdir(dir_entry) and any(xs in dir_entry.name for xs in acceptable_string_grabs) ]\n",
        "dnn_model_dirs = [ s for s in model_dirs if \"DNN\" in s and all([ (not ncs in s) for ncs in not_check_model_string])] \n",
        "cnn_model_dirs = [ s for s in model_dirs if \"CNN\" in s and all([ (not ncs in s) for ncs in not_check_model_string])] \n",
        "resnets_model_dirs = [ s for s in model_dirs if \"RestNet50\" in s and all([ (not ncs in s) for ncs in not_check_model_string])] \n",
        "wideresnets_model_dirs = [ s for s in model_dirs if \"WideResNet\" in s and all([ (not ncs in s) for ncs in not_check_model_string])] \n",
        "\n",
        "model_features_files = [  f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features/{file_entry.name}\" \n",
        "                for file_entry in os.scandir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features/\") \n",
        "                if os.path.isfile(file_entry) and any(xs in file_entry.name for xs in acceptable_string_grabs) ]\n",
        "\n",
        "dnn_features_files = [ s for s in model_features_files if \"DNN\" in s and all([ (not ncs in s) for ncs in not_check_model_string])] \n",
        "cnn_features_files = [ s for s in model_features_files if \"CNN\" in s and all([ (not ncs in s) for ncs in not_check_model_string])] \n",
        "resnets_features_files = [ s for s in model_features_files if \"RestNet50\" in s and all([ (not ncs in s) for ncs in not_check_model_string])] \n",
        "wideresnets_features_files = [ s for s in model_features_files if \"WideResNet\" in s and all([ (not ncs in s) for ncs in not_check_model_string])] \n",
        "\n",
        "dnn_identifier = acceptable_string_grabs[1]\n",
        "# dnn_layer_name = \"DRP1\"\n",
        "\n",
        "cnn_identifier = acceptable_string_grabs[0]\n",
        "# cnn_layer_name = \"D3R\"\n",
        "\n",
        "wideresnet_identifier = acceptable_string_grabs[3]\n",
        "# wideresnet_layer_name = \"CLASSIFIER_FL\"\n",
        "\n",
        "\n",
        "def avgfilter_ensemble_predictions(y_pred_ms, xtest):\n",
        "  y_preds_ens_prb = np.apply_along_axis(np.mean, 0, y_pred_ms)\n",
        "  y_preds_ens_idx = np.apply_along_axis(np.argmax, 1, y_preds_ens_prb) \n",
        "  # y_preds_ens_idx = y_preds_ens_idx + 1\n",
        "  return y_preds_ens_idx  \n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7NfTm6C0efVR"
      },
      "source": [
        "# Set up data for ensemble plot collection"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0N1xIkYzeOqW"
      },
      "outputs": [],
      "source": [
        "# set up the data\n",
        "ensemble_test_results = None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vlEnnUKUYQsU"
      },
      "outputs": [],
      "source": [
        "x_input = test_data_grey\n",
        "y_input = test_targets\n",
        "\n",
        "num_of_repeats = 10\n",
        "num_of_models = [3,4,7,8,13,16,19]\n",
        "# [1,2,5,10,12,15,18,20]\n",
        "\n",
        "# resnet_model_predicted_values =  dict(zip(resnets_model_dirs,[ resnet_loaded_models[model].predict(x_input) for model in resnets_model_dirs]))\n",
        "# dnn_model_predicted_values =  dict(zip(dnn_model_dirs,[ dnn_loaded_models[model].predict(x_input) for model in dnn_model_dirs]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pLSgKRgSmPGz"
      },
      "outputs": [],
      "source": [
        "from psutil import virtual_memory\n",
        "ram_gb = virtual_memory().total / 1e9\n",
        "print('Your runtime has {:.1f} gigabytes of available RAM\\n'.format(ram_gb))\n",
        "\n",
        "if ram_gb < 20:\n",
        "  print('Not using a high-RAM runtime')\n",
        "else:\n",
        "  print('You are using a high-RAM runtime!')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zarZ-ubirq5w"
      },
      "source": [
        "# Data for plot of #DNN in ensemble "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KY_OZVbUvt1p"
      },
      "outputs": [],
      "source": [
        "\n",
        "dnn_loaded_models = dict(zip(dnn_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in dnn_model_dirs]))\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iTD7m-7DJ9iu"
      },
      "outputs": [],
      "source": [
        "# from numba import jit, prange\n",
        "# @jit(nopython=True, parallel=True)\n",
        "# def parallel_sum(A):\n",
        "#     sum = 0.0\n",
        "#     for i in prange(A.shape[0]):\n",
        "#         sum += A[i]\n",
        "#     return sum"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UFsdTnB3PEHI"
      },
      "outputs": [],
      "source": [
        "x_input = test_data_grey\n",
        "y_input = test_targets\n",
        "\n",
        "num_of_repeats = 4\n",
        "num_of_models = [22,25,30,35,40]  # [2,3,4,6,7,8,9,11,12,13,14,16,19]\n",
        "\n",
        "idxCount = 0 if ensemble_test_results is None else len(ensemble_test_results.index)\n",
        "for mc in num_of_models:\n",
        "  for repc in range(num_of_repeats):\n",
        "    selected_model_names = sorted(random.sample(dnn_model_dirs, min(len(dnn_model_dirs),mc)))\n",
        "    dnn_models_to_use = [ dnn_loaded_models[mn] for mn in selected_model_names ]\n",
        "    dnn_model_predictions = [ model.predict(x_input) for model in dnn_models_to_use ]\n",
        "    y_ens_preds = avgfilter_ensemble_predictions(dnn_model_predictions, x_input)\n",
        "    pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_ens_preds,y_input)\n",
        "    print (mc, repc, pr, rc, f1, acc)\n",
        "    if ensemble_test_results is None:\n",
        "      ensemble_test_results = pd.DataFrame({\"Type\": \"DNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": len(selected_model_names), \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            \"ModelNames\": \"XOX\".join([ \"_\".join(zz.split(\"DNN_A_\")[1].split(\"_\")[:2]) for zz in selected_model_names])\n",
        "                                            }, index = [idxCount])\n",
        "    else:\n",
        "      ensemble_test_results = pd.concat([ensemble_test_results,\n",
        "                                         pd.DataFrame({\"Type\": \"DNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": len(selected_model_names), \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            \"ModelNames\": \"XOX\".join([ \"_\".join(zz.split(\"DNN_A_\")[1].split(\"_\")[:2]) for zz in selected_model_names])\n",
        "                                            }, index = [idxCount])\n",
        "                                         ])\n",
        "    idxCount = idxCount + 1\n",
        "    del dnn_models_to_use\n",
        "    del dnn_model_predictions\n",
        "    del y_ens_preds\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lfyPSdOghY4j"
      },
      "outputs": [],
      "source": [
        "ensemble_test_results.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/ensemble_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qr4uwylNzQl1"
      },
      "source": [
        "# Data for plot of #CNN in ensemble "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VRWdcBVizQl2"
      },
      "outputs": [],
      "source": [
        "\n",
        "# preload all the models\n",
        "# resnet_loaded_models = dict(zip(resnets_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in resnets_model_dirs]))\n",
        "# dnn_loaded_models = dict(zip(dnn_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in dnn_model_dirs]))\n",
        "cnn_loaded_models = dict(zip(cnn_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in cnn_model_dirs]))\n",
        "# wideresnet_loaded_models = dict(zip(wideresnets_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in wideresnets_model_dirs]))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MR3GuoTdzqGc"
      },
      "outputs": [],
      "source": [
        "# for CNN we try to memoize the predictions... ?\n",
        "# cnn_model_predictions = [ cnn_loaded_models[mn].predict(x_input) for mn in list(cnn_loaded_models.keys()) ]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WCDDvCaczQl2"
      },
      "outputs": [],
      "source": [
        "# from numba import jit, prange\n",
        "# @jit(nopython=True, parallel=True)\n",
        "# def parallel_sum(A):\n",
        "#     sum = 0.0\n",
        "#     for i in prange(A.shape[0]):\n",
        "#         sum += A[i]\n",
        "#     return sum"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lPhanQJWzQl2"
      },
      "outputs": [],
      "source": [
        "# num_of_models = [16,20]\n",
        "\n",
        "x_input = test_data_grey\n",
        "y_input = test_targets\n",
        "\n",
        "num_of_repeats = 4\n",
        "num_of_models = [22,25,30,35,40] # [2,3,4,6,7,8,9,11,12,13,14,16,19]\n",
        "\n",
        "idxCount = 0 if ensemble_test_results is None else len(ensemble_test_results.index)\n",
        "for mc in num_of_models:\n",
        "  for repc in range(num_of_repeats):\n",
        "    selected_model_names = sorted(random.sample(cnn_model_dirs, min(len(cnn_model_dirs),mc)))\n",
        "    cnn_models_to_use = [ cnn_loaded_models[mn] for mn in selected_model_names ]\n",
        "    cnn_model_predictions = [ model.predict(x_input) for model in cnn_models_to_use ]\n",
        "    y_ens_preds = avgfilter_ensemble_predictions(cnn_model_predictions, x_input)\n",
        "    pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_ens_preds,y_input)\n",
        "    print (mc, repc, pr, rc, f1, acc)\n",
        "    if ensemble_test_results is None:\n",
        "      ensemble_test_results = pd.DataFrame({\"Type\": \"CNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": len(selected_model_names), \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            \"ModelNames\": \"XOX\".join([ \"_\".join(zz.split(\"CNN_A_\")[1].split(\"_\")[:2]) for zz in selected_model_names])\n",
        "                                            }, index = [idxCount])\n",
        "    else:\n",
        "      ensemble_test_results = pd.concat([ensemble_test_results,\n",
        "                                         pd.DataFrame({\"Type\": \"CNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": len(selected_model_names), \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            \"ModelNames\": \"XOX\".join([ \"_\".join(zz.split(\"CNN_A_\")[1].split(\"_\")[:2]) for zz in selected_model_names])\n",
        "                                            }, index = [idxCount])\n",
        "                                         ])\n",
        "    idxCount = idxCount + 1\n",
        "    del cnn_models_to_use\n",
        "    del cnn_model_predictions\n",
        "    del y_ens_preds\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mqinAog5zQl2"
      },
      "outputs": [],
      "source": [
        "ensemble_test_results.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/cnn_ensemble_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nlcSr-vhOENw"
      },
      "source": [
        "# Data for plot of #ResNets in ensemble "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0sIHDYSOOENw"
      },
      "outputs": [],
      "source": [
        "\n",
        "# preload all the models\n",
        "resnet_loaded_models = dict(zip(resnets_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in resnets_model_dirs]))\n",
        "# dnn_loaded_models = dict(zip(dnn_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in dnn_model_dirs]))\n",
        "# cnn_loaded_models = dict(zip(cnn_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in cnn_model_dirs]))\n",
        "# wideresnet_loaded_models = dict(zip(wideresnets_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in wideresnets_model_dirs]))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X7C96YxfOENw"
      },
      "outputs": [],
      "source": [
        "x_input = test_data_grey\n",
        "y_input = test_targets\n",
        "\n",
        "num_of_repeats = 4\n",
        "num_of_models = [22,25,30,35,39]  # [1,2,5,10,12,15,18,20] \n",
        "# [3,4,7,8,13,16,19]\n",
        "# [1,2,5,10,12,15,18,20]\n",
        "del cnn_loaded_models"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y-Sv3xjyXv7W"
      },
      "outputs": [],
      "source": [
        "# del resnet_model_predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "J0TWDQcWOENw"
      },
      "outputs": [],
      "source": [
        "# for ResNets we try to memoize the predictions... ?\n",
        "resnet_model_predictions = dict(zip(resnets_model_dirs, [ resnet_loaded_models[mn].predict(x_input) for mn in resnets_model_dirs ]))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AW3AdzcXOENx"
      },
      "outputs": [],
      "source": [
        "\n",
        "idxCount = 0 if ensemble_test_results is None else len(ensemble_test_results.index)\n",
        "for mc in num_of_models:\n",
        "  for repc in range(num_of_repeats):\n",
        "    selected_model_names = sorted(random.sample(resnets_model_dirs, min(len(resnets_model_dirs),mc)))\n",
        "    # resnet_models_to_use = [ resnet_loaded_models[mn] for mn in selected_model_names ]\n",
        "    resnet_model_predictions_here = [ resnet_model_predictions[mn] for mn in selected_model_names ]\n",
        "    y_ens_preds = avgfilter_ensemble_predictions(resnet_model_predictions_here, x_input)\n",
        "    pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_ens_preds,y_input)\n",
        "    print (mc, repc, pr, rc, f1, acc)\n",
        "    if ensemble_test_results is None:\n",
        "      ensemble_test_results = pd.DataFrame({\"Type\": \"ResNet50\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": len(selected_model_names), \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            \"ModelNames\": \"XOX\".join([ \"_\".join(zz.split(\"RestNet50_A_\")[1].split(\"_\")[:2]) for zz in selected_model_names])\n",
        "                                            }, index = [idxCount])\n",
        "    else:\n",
        "      ensemble_test_results = pd.concat([ensemble_test_results,\n",
        "                                         pd.DataFrame({\"Type\": \"ResNet50\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": len(selected_model_names), \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            \"ModelNames\": \"XOX\".join([ \"_\".join(zz.split(\"RestNet50_A_\")[1].split(\"_\")[:2]) for zz in selected_model_names])\n",
        "                                            }, index = [idxCount])\n",
        "                                         ])\n",
        "    idxCount = idxCount + 1\n",
        "    # del resnet_models_to_use\n",
        "    del resnet_model_predictions_here\n",
        "    del y_ens_preds\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "68ex5xdfOENx"
      },
      "outputs": [],
      "source": [
        "ensemble_test_results.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/resnet_ensemble_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pVGqX79lZ3pw"
      },
      "source": [
        "# Data for plot of #WideResNets in ensemble "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "753xmge4Z3pw"
      },
      "outputs": [],
      "source": [
        "\n",
        "# preload all the models\n",
        "# resnet_loaded_models = dict(zip(resnets_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in resnets_model_dirs]))\n",
        "# dnn_loaded_models = dict(zip(dnn_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in dnn_model_dirs]))\n",
        "# cnn_loaded_models = dict(zip(cnn_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in cnn_model_dirs]))\n",
        "wideresnets_loaded_models = dict(zip(wideresnets_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in wideresnets_model_dirs]))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IdoovBqTZ3pw"
      },
      "outputs": [],
      "source": [
        "x_input = test_data_grey\n",
        "y_input = test_targets\n",
        "\n",
        "num_of_repeats = 10\n",
        "num_of_models = [1,2,3,4,5,6,7,8,9,10] \n",
        "# [3,4,7,8,13,16,19]\n",
        "# [1,2,5,10,12,15,18,20]\n",
        "# del resnet_loaded_models\n",
        "# del resnet_model_predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "V5-jUpgRZ3pw"
      },
      "outputs": [],
      "source": [
        "# for WideResNet we try to memoize the predictions... ?\n",
        "wideresnets_model_predictions = dict(zip(wideresnets_model_dirs, [ wideresnet_loaded_models[mn].predict(x_input) for mn in wideresnets_model_dirs ]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TKQun312Z3pw"
      },
      "outputs": [],
      "source": [
        "\n",
        "idxCount = 0 if ensemble_test_results is None else len(ensemble_test_results.index)\n",
        "for mc in num_of_models:\n",
        "  for repc in range(num_of_repeats):\n",
        "    selected_model_names = sorted(random.sample(wideresnets_model_dirs, min(len(wideresnets_model_dirs),mc)))\n",
        "    # resnet_models_to_use = [ resnet_loaded_models[mn] for mn in selected_model_names ]\n",
        "    wideresnet_model_predictions_here = [ wideresnets_model_predictions[mn] for mn in selected_model_names ]\n",
        "    y_ens_preds = avgfilter_ensemble_predictions(wideresnet_model_predictions_here, x_input)\n",
        "    pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_ens_preds,y_input)\n",
        "    print (mc, repc, pr, rc, f1, acc)\n",
        "    if ensemble_test_results is None:\n",
        "      ensemble_test_results = pd.DataFrame({\"Type\": \"WideResNet\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": len(selected_model_names), \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            \"ModelNames\": \"XOX\".join([ \"_\".join(zz.split(\"WideResNet28-10_\")[1].split(\"_\")[:3]) for zz in selected_model_names])\n",
        "                                            }, index = [idxCount])\n",
        "    else:\n",
        "      ensemble_test_results = pd.concat([ensemble_test_results,\n",
        "                                         pd.DataFrame({\"Type\": \"WideResNet\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": len(selected_model_names), \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            \"ModelNames\": \"XOX\".join([ \"_\".join(zz.split(\"WideResNet28-10_\")[1].split(\"_\")[:3]) for zz in selected_model_names])\n",
        "                                            }, index = [idxCount])\n",
        "                                         ])\n",
        "    idxCount = idxCount + 1\n",
        "    # del resnet_models_to_use\n",
        "    del wideresnet_model_predictions_here\n",
        "    del y_ens_preds\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uquPC-rGZ3px"
      },
      "outputs": [],
      "source": [
        "ensemble_test_results.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/wideresnet_ensemble_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5YEfUSsWG6bm"
      },
      "source": [
        "# Data for Contour Plot on Mixed CNN/WideResNet ensemble"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IuYJjSlHHT9Y"
      },
      "outputs": [],
      "source": [
        "wideresnets_loaded_models = dict(zip(wideresnets_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in wideresnets_model_dirs]))\n",
        "cnn_loaded_models = dict(zip(cnn_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in cnn_model_dirs]))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Fpy7lzitHiJ9"
      },
      "outputs": [],
      "source": [
        "x_input = test_data_grey\n",
        "y_input = test_targets\n",
        "\n",
        "num_of_repeats = 10\n",
        "num_of_models = [2,3,4,5,6,7,8,9,10,11,12] \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_-U8hTmpHiJ9"
      },
      "outputs": [],
      "source": [
        "# for WideResNet we try to memoize the predictions... ?\n",
        "wideresnets_model_predictions = [ wideresnets_loaded_models[mn].predict(x_input) for mn in wideresnets_model_dirs ]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "c_gCb_W0-1UK"
      },
      "outputs": [],
      "source": [
        "cnn_model_predictions = [ cnn_loaded_models[mn].predict(x_input) for mn in list(cnn_loaded_models.keys()) ]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xSZg_DX5bz43"
      },
      "outputs": [],
      "source": [
        "# wideresnets_model_dirs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RFU-Ro3xG6bn"
      },
      "outputs": [],
      "source": [
        "# set up the combinations we are going to try\n",
        "# len(wideresnets_features_files)\n",
        "cnn_wideresnet_ensemble_contour_data = None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1Lfkuj2vHz0B"
      },
      "outputs": [],
      "source": [
        "idxCount = 0 if cnn_wideresnet_ensemble_contour_data is None else len(cnn_wideresnet_ensemble_contour_data.index)\n",
        "for mc in num_of_models:\n",
        "  for repc in range(num_of_repeats):\n",
        "    for num_of_wrn in range(mc+1):\n",
        "      num_of_cnn = mc - num_of_wrn\n",
        "\n",
        "      ensemble_model_predictions_here = []\n",
        "      if num_of_cnn > 0:\n",
        "        ensemble_model_predictions_here.extend(random.sample(cnn_model_predictions, min(num_of_cnn,len(cnn_model_predictions)) ))\n",
        "      if num_of_wrn > 0:\n",
        "        ensemble_model_predictions_here.extend(random.sample(wideresnets_model_predictions, min(num_of_wrn,len(wideresnets_model_predictions)) ))\n",
        "\n",
        "      y_ens_preds = avgfilter_ensemble_predictions(ensemble_model_predictions_here, x_input)\n",
        "      pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_ens_preds,y_input)\n",
        "      print (mc, num_of_cnn, repc, pr, rc, f1, acc)\n",
        "      if cnn_wideresnet_ensemble_contour_data is None:\n",
        "        cnn_wideresnet_ensemble_contour_data = pd.DataFrame({\"TypeA\": \"CNN\", \n",
        "                                                             \"TypeB\": \"WideResNet\", \n",
        "                                                              \"Data\" : \"Test\",\n",
        "                                                             \"Layer\" : \"Ensemble\",\n",
        "                                                    \"NumOfA\": num_of_cnn, \n",
        "                                                    \"NumOfB\": num_of_wrn, \n",
        "                                                    \"RepC\": repc, \n",
        "                                                    \"Pr\": pr,\n",
        "                                                    \"Rc\": rc,\n",
        "                                                    \"F1\": f1,\n",
        "                                                    \"Acc\": acc\n",
        "                                              }, index = [idxCount])\n",
        "      else:\n",
        "        cnn_wideresnet_ensemble_contour_data = pd.concat([cnn_wideresnet_ensemble_contour_data,\n",
        "                                                          pd.DataFrame({\"TypeA\": \"CNN\", \n",
        "                                                                        \"TypeB\": \"WideResNet\", \n",
        "                                                                        \"Data\" : \"Test\",\n",
        "                                                                        \"Layer\" : \"Ensemble\",\n",
        "                                                                        \"NumOfA\": num_of_cnn, \n",
        "                                                                        \"NumOfB\": num_of_wrn, \n",
        "                                                                        \"RepC\": repc, \n",
        "                                                                        \"Pr\": pr,\n",
        "                                                                        \"Rc\": rc,\n",
        "                                                                        \"F1\": f1,\n",
        "                                                                        \"Acc\": acc\n",
        "                                                                        }, index = [idxCount])])\n",
        "      idxCount = idxCount + 1\n",
        "      # del resnet_models_to_use\n",
        "      del ensemble_model_predictions_here\n",
        "      del y_ens_preds\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nP3uqLTL9M2v"
      },
      "outputs": [],
      "source": [
        "cnn_wideresnet_ensemble_contour_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/cnn_wideresnet_ensemble_contour_data_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W8fThz_isxYI"
      },
      "source": [
        "# Data for Contour Plot on Mixed CNN/DNN ensemble"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IxNNRDdLsxYY"
      },
      "outputs": [],
      "source": [
        "dnn_loaded_models = dict(zip(dnn_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in dnn_model_dirs]))\n",
        "cnn_loaded_models = dict(zip(cnn_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in cnn_model_dirs]))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1iFW3cntsxYY"
      },
      "outputs": [],
      "source": [
        "x_input = test_data_grey\n",
        "y_input = test_targets\n",
        "\n",
        "num_of_repeats = 4\n",
        "num_of_models = [2,4,6,8,10,15,20,25,30] \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-jFlm_Q5sxYY"
      },
      "outputs": [],
      "source": [
        "# for WideResNet we try to memoize the predictions... ?\n",
        "dnn_model_predictions = [ dnn_loaded_models[mn].predict(x_input) for mn in dnn_model_dirs ]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CVNAmSrSsxYZ"
      },
      "outputs": [],
      "source": [
        "cnn_model_predictions = [ cnn_loaded_models[mn].predict(x_input) for mn in list(cnn_loaded_models.keys()) ]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BNPPTYjAsxYZ"
      },
      "outputs": [],
      "source": [
        "# wideresnets_model_dirs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Mk0nfl46sxYZ"
      },
      "outputs": [],
      "source": [
        "# set up the combinations we are going to try\n",
        "# len(wideresnets_features_files)\n",
        "cnn_dnn_ensemble_contour_data = None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "04d4jlM1sxYa"
      },
      "outputs": [],
      "source": [
        "idxCount = 0 if cnn_dnn_ensemble_contour_data is None else len(cnn_dnn_ensemble_contour_data.index)\n",
        "for mc in num_of_models:\n",
        "  for repc in range(num_of_repeats):\n",
        "    for num_of_dnn in range(mc+1):\n",
        "      num_of_cnn = mc - num_of_dnn\n",
        "\n",
        "      ensemble_model_predictions_here = []\n",
        "      if num_of_cnn > 0:\n",
        "        ensemble_model_predictions_here.extend(random.sample(cnn_model_predictions, min(num_of_cnn,len(cnn_model_predictions)) ))\n",
        "      if num_of_dnn > 0:\n",
        "        ensemble_model_predictions_here.extend(random.sample(dnn_model_predictions, min(num_of_dnn,len(dnn_model_predictions)) ))\n",
        "\n",
        "      y_ens_preds = avgfilter_ensemble_predictions(ensemble_model_predictions_here, x_input)\n",
        "      pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_ens_preds,y_input)\n",
        "      print (mc, num_of_cnn, repc, pr, rc, f1, acc)\n",
        "      if cnn_dnn_ensemble_contour_data is None:\n",
        "        cnn_dnn_ensemble_contour_data = pd.DataFrame({\"TypeA\": \"CNN\", \n",
        "                                                             \"TypeB\": \"DNN\", \n",
        "                                                              \"Data\" : \"Test\",\n",
        "                                                             \"Layer\" : \"Ensemble\",\n",
        "                                                    \"NumOfA\": num_of_cnn, \n",
        "                                                    \"NumOfB\": num_of_dnn, \n",
        "                                                    \"RepC\": repc, \n",
        "                                                    \"Pr\": pr,\n",
        "                                                    \"Rc\": rc,\n",
        "                                                    \"F1\": f1,\n",
        "                                                    \"Acc\": acc\n",
        "                                              }, index = [idxCount])\n",
        "      else:\n",
        "        cnn_dnn_ensemble_contour_data = pd.concat([cnn_dnn_ensemble_contour_data,\n",
        "                                                          pd.DataFrame({\"TypeA\": \"CNN\", \n",
        "                                                                        \"TypeB\": \"DNN\", \n",
        "                                                                        \"Data\" : \"Test\",\n",
        "                                                                        \"Layer\" : \"Ensemble\",\n",
        "                                                                        \"NumOfA\": num_of_cnn, \n",
        "                                                                        \"NumOfB\": num_of_dnn, \n",
        "                                                                        \"RepC\": repc, \n",
        "                                                                        \"Pr\": pr,\n",
        "                                                                        \"Rc\": rc,\n",
        "                                                                        \"F1\": f1,\n",
        "                                                                        \"Acc\": acc\n",
        "                                                                        }, index = [idxCount])])\n",
        "      idxCount = idxCount + 1\n",
        "      # del resnet_models_to_use\n",
        "      del ensemble_model_predictions_here\n",
        "      del y_ens_preds\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jEpM9VOesxYa"
      },
      "outputs": [],
      "source": [
        "cnn_dnn_ensemble_contour_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/cnn_dnn_ensemble_contour_data_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3pW2evV5b2wJ"
      },
      "source": [
        "# Data for Contour Plot on Mixed DNN/WideResNet ensemble"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ewEnZ4Xmb2wK"
      },
      "outputs": [],
      "source": [
        "wideresnets_loaded_models = dict(zip(wideresnets_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in wideresnets_model_dirs]))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DMWBmmO-cBxf"
      },
      "outputs": [],
      "source": [
        "dnn_loaded_models = dict(zip(dnn_model_dirs,[ tf.keras.models.load_model(mfile) for mfile in dnn_model_dirs]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9DtyfYapb2wL"
      },
      "outputs": [],
      "source": [
        "x_input = test_data_grey\n",
        "y_input = test_targets\n",
        "\n",
        "num_of_repeats = 10\n",
        "num_of_models = [2,3,4,5,6,7,8,9,10,11,12] "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yznjta6xb2wM"
      },
      "outputs": [],
      "source": [
        "# for WideResNet we try to memoize the predictions... ?\n",
        "wideresnets_model_predictions = [ wideresnets_loaded_models[mn].predict(x_input) for mn in wideresnets_model_dirs ]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HXlMbVhwb2wM"
      },
      "outputs": [],
      "source": [
        "dnn_model_predictions = [ dnn_loaded_models[mn].predict(x_input) for mn in list(dnn_loaded_models.keys()) ]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZIi9Y-Rab2wN"
      },
      "outputs": [],
      "source": [
        "# wideresnets_model_dirs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-RcqU1O-b2wN"
      },
      "outputs": [],
      "source": [
        "# set up the combinations we are going to try\n",
        "# len(wideresnets_features_files)\n",
        "dnn_wideresnet_ensemble_contour_data = None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZIwA5Y0Ub2wO"
      },
      "outputs": [],
      "source": [
        "idxCount = 0 if dnn_wideresnet_ensemble_contour_data is None else len(dnn_wideresnet_ensemble_contour_data.index)\n",
        "for mc in num_of_models:\n",
        "  for repc in range(num_of_repeats):\n",
        "    for num_of_wrn in range(mc+1):\n",
        "\n",
        "      num_of_dnn = mc - num_of_wrn\n",
        "\n",
        "      ensemble_model_predictions_here = []\n",
        "      if num_of_dnn > 0:\n",
        "        ensemble_model_predictions_here.extend(random.sample(dnn_model_predictions, min(num_of_dnn,len(dnn_model_predictions)) ))\n",
        "      if num_of_wrn > 0:\n",
        "        ensemble_model_predictions_here.extend(random.sample(wideresnets_model_predictions, min(num_of_wrn,len(wideresnets_model_predictions)) ))\n",
        "\n",
        "      y_ens_preds = avgfilter_ensemble_predictions(ensemble_model_predictions_here, x_input)\n",
        "      pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_ens_preds,y_input)\n",
        "      print (mc, num_of_cnn, repc, pr, rc, f1, acc)\n",
        "      if dnn_wideresnet_ensemble_contour_data is None:\n",
        "        dnn_wideresnet_ensemble_contour_data = pd.DataFrame({\"TypeA\": \"DNN\", \n",
        "                                                             \"TypeB\": \"WideResNet\", \n",
        "                                                              \"Data\" : \"Test\",\n",
        "                                                             \"Layer\" : \"Ensemble\",\n",
        "                                                    \"NumOfA\": num_of_dnn, \n",
        "                                                    \"NumOfB\": num_of_wrn, \n",
        "                                                    \"RepC\": repc, \n",
        "                                                    \"Pr\": pr,\n",
        "                                                    \"Rc\": rc,\n",
        "                                                    \"F1\": f1,\n",
        "                                                    \"Acc\": acc\n",
        "                                              }, index = [idxCount])\n",
        "      else:\n",
        "        dnn_wideresnet_ensemble_contour_data = pd.concat([dnn_wideresnet_ensemble_contour_data,\n",
        "                                                          pd.DataFrame({\"TypeA\": \"DNN\", \n",
        "                                                                        \"TypeB\": \"WideResNet\", \n",
        "                                                                        \"Data\" : \"Test\",\n",
        "                                                                        \"Layer\" : \"Ensemble\",\n",
        "                                                                        \"NumOfA\": num_of_dnn, \n",
        "                                                                        \"NumOfB\": num_of_wrn, \n",
        "                                                                        \"RepC\": repc, \n",
        "                                                                        \"Pr\": pr,\n",
        "                                                                        \"Rc\": rc,\n",
        "                                                                        \"F1\": f1,\n",
        "                                                                        \"Acc\": acc\n",
        "                                                                        }, index = [idxCount])])\n",
        "      idxCount = idxCount + 1\n",
        "      # del resnet_models_to_use\n",
        "      del ensemble_model_predictions_here\n",
        "      del y_ens_preds\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nkxTQ3E8b2wP"
      },
      "outputs": [],
      "source": [
        "dnn_wideresnet_ensemble_contour_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/dnn_wideresnet_ensemble_contour_data_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9SyehzPysxYa"
      },
      "source": [
        "# Experiment with CCA on the DNN/CNN models for selection into ensemble/collab\n",
        "### Could also be used to initialize an integrated fully parallel model of DNN/CNN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ttBlxirJsxYb",
        "outputId": "bfd10d1d-07c8-4fbb-90ba-a4619f257159"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 2.75 ms (started: 2022-01-12 14:13:52 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def get_input_features(X_data, m1, m2, m1_layer_name = \"leaky_re_lu_6\", m2_layer_name = \"dropout_7\"):\n",
        "  extractor_m1 = Model(inputs=m1.inputs, outputs=m1.get_layer(m1_layer_name).output)\n",
        "  features_m1 = extractor_m1.predict(X_data)\n",
        "  extractor_m2 = Model(inputs=m2.inputs, outputs=m2.get_layer(m2_layer_name).output)\n",
        "  features_m2 = extractor_m2.predict(X_data)\n",
        "  return tf.concat([features_m1,features_m2])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4pF0jD2vsxYb",
        "outputId": "a31a3618-b7ea-4279-d6f4-caeb2298aba4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 6.74 ms (started: 2022-01-12 14:13:52 +00:00)\n"
          ]
        }
      ],
      "source": [
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.cross_decomposition import CCA\n",
        "from sklearn.decomposition import PCA\n",
        "pd.set_option('display.max_colwidth', None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-3aOwsdSsxYb",
        "outputId": "f40c25de-ef16-48c7-e810-716e5d9829cd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 2.62 ms (started: 2022-01-12 14:13:52 +00:00)\n"
          ]
        }
      ],
      "source": [
        "available_dnn_feature_layers = list(set([ x.split(\"_Validation\")[0].split(\"_features_\")[1].split(\"_\")[0] for x in dnn_features_files if \"_X\" in x ]))\n",
        "available_cnn_feature_layers = list(set([ x.split(\"_Validation\")[0].split(\"_features_\")[1].split(\"_\")[0] for x in cnn_features_files if \"_X\" in x ]))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ny0Iy_fQgq80",
        "outputId": "8f1bfa62-bdba-4704-e15a-d7203da788bb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 1.85 ms (started: 2022-01-12 14:13:52 +00:00)\n"
          ]
        }
      ],
      "source": [
        "available_wrn_feature_layers = list(set([ x.split(\"_Validation\")[0].split(\"_features_\")[1].split(\"_\")[1] for x in wideresnets_features_files if \"_X\" in x ]))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WWo7QIj6nwup",
        "outputId": "dbfa245e-41aa-45f3-b03a-66acb8bd4797"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['D1', 'FL']"
            ]
          },
          "metadata": {},
          "execution_count": 40
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 2.38 ms (started: 2022-01-12 14:13:52 +00:00)\n"
          ]
        }
      ],
      "source": [
        "# wideresnets_features_files\n",
        "available_wrn_feature_layers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yIxHgOuZsxYb",
        "outputId": "83aa4a5b-fc63-4734-95c8-e7820fe676c9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 11.3 ms (started: 2022-01-12 14:13:52 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def get_cca1_pca1_from_file_names(f1,f2):\n",
        "\n",
        "  f1t = np.load(f1)\n",
        "  f2t = np.load(f2)  \n",
        "  scaler1 = StandardScaler()\n",
        "  scaler2 = StandardScaler()\n",
        "\n",
        "  scaler1.fit(f1t)\n",
        "  scaler2.fit(f2t)\n",
        "\n",
        "  f1tt = scaler1.transform(f1t)\n",
        "  f2tt = scaler2.transform(f2t)\n",
        "\n",
        "  pca_f1 = PCA(n_components=1)\n",
        "  pca_f1.fit(f1tt)\n",
        "  pca_f2 = PCA(n_components=1)\n",
        "  pca_f2.fit(f2tt)\n",
        "\n",
        "  ca = CCA()\n",
        "  ca.fit(f1tt, f2tt)\n",
        "  f1tt_c, f2tt_c = ca.transform(f1tt, f2tt)\n",
        "\n",
        "  f1_l = np.apply_along_axis(lambda x: np.dot(pca_f1.components_,np.expand_dims(x, axis=1)),1,f1tt)\n",
        "  f2_l = np.apply_along_axis(lambda x: np.dot(pca_f2.components_,np.expand_dims(x, axis=1)),1,f2tt)\n",
        "  \n",
        "  return (np.corrcoef(f1tt_c[:,0],f2tt_c[:,0])[0,1] , np.corrcoef(f1_l[:,0,0],f2_l[:,0,0])[0,1])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PUTtSBsJJ5rX",
        "outputId": "9598ceb3-803c-4e67-8899-d9cfa83f75aa"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 9.01 ms (started: 2022-01-12 14:13:52 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def get_wrn_validation_features(f2):\n",
        "  layer_name = f2.split(\"_CLASSIFIER_\")[1].split(\"_Validation\")[0]\n",
        "  model_type = \"WideResNet\"\n",
        "  num_of_models = 1\n",
        "\n",
        "  feature_files_used = [f2]\n",
        "\n",
        "  base_patterns_for_validations = [ f2.split(\"_Validation\")[0] ]\n",
        "  # base_patterns_for_validations = get_base_patterns_for_validation(feature_files_used, layer_name, model_type)\n",
        "\n",
        "  np_x_validation_collab = None\n",
        "  for base_val_str in base_patterns_for_validations:\n",
        "      validation_batch_files = sorted([ ff for ff in wideresnets_features_files if base_val_str in ff and \"Validation\" in ff and \"_X\" in ff])\n",
        "      # print(len(validation_batch_files))\n",
        "      np_x_validation_collab_batch = np.array([np.load(ff) for ff in validation_batch_files])\n",
        "      np_x_validation_collab_batch = np.concatenate(np_x_validation_collab_batch, axis=0)\n",
        "      if np_x_validation_collab is None:\n",
        "        np_x_validation_collab = np_x_validation_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_validation_collab = np.concatenate([np_x_validation_collab, np_x_validation_collab_batch], axis=axis_to_concat)\n",
        "\n",
        "  return np_x_validation_collab\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sIpRyxRvbU09"
      },
      "outputs": [],
      "source": [
        "# print(pca_f1.explained_variance_ratio_[0])\n",
        "# pca_f1a = PCA(n_components=10)\n",
        "# pca_f1a.fit(f1tt)\n",
        "# print(pca_f1a.explained_variance_ratio_)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4KvY8PACuz1l",
        "outputId": "35005ceb-03ac-4348-edb2-5444f0080567"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 27.3 ms (started: 2022-01-12 14:13:52 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def get_pca1_from_file_names(f1,f2,memoized_file_pca1s, add_perc=True):\n",
        "  \n",
        "  f1_l = None\n",
        "  if f1 not in memoized_file_pca1s.keys():\n",
        "    f1t = get_wrn_validation_features(f1) if \"WideResNet\" in f1 else np.load(f1) \n",
        "    scaler1 = StandardScaler()\n",
        "    scaler1.fit(f1t)\n",
        "    f1tt = scaler1.transform(f1t)\n",
        "    pca_f1 = PCA(n_components=1)\n",
        "    pca_f1.fit(f1tt)\n",
        "    f1_l = np.apply_along_axis(lambda x: np.dot(pca_f1.components_,np.expand_dims(x, axis=1)),1,f1tt)\n",
        "    exp_var1 = pca_f1.explained_variance_ratio_.round(3)[0] if add_perc else None\n",
        "    memoized_file_pca1s[f1] = (f1_l,exp_var1)\n",
        "    del f1_t\n",
        "    del pca_f1\n",
        "    del f1tt\n",
        "    del f1t\n",
        "    del scaler1    \n",
        "  f1_l,exp_var1 = memoized_file_pca1s[f1]\n",
        "\n",
        "  f2_l = None\n",
        "  if f2 not in memoized_file_pca1s.keys():\n",
        "    f2t = get_wrn_validation_features(f2) if \"WideResNet\" in f2 else np.load(f2)\n",
        "    scaler2 = StandardScaler()\n",
        "    scaler2.fit(f2t)\n",
        "    f2tt = scaler2.transform(f2t)\n",
        "    pca_f2 = PCA(n_components=1)\n",
        "    pca_f2.fit(f2tt)\n",
        "    f2_l = np.apply_along_axis(lambda x: np.dot(pca_f2.components_,np.expand_dims(x, axis=1)),1,f2tt)\n",
        "    exp_var2 = pca_f2.explained_variance_ratio_.round(3)[0] if add_perc else None\n",
        "    memoized_file_pca1s[f2] = (f2_l,exp_var2)\n",
        "    del f2_t\n",
        "    del pca_f2\n",
        "    del f2tt\n",
        "    del f2t\n",
        "    del scaler2\n",
        "  f2_l,exp_var2 = memoized_file_pca1s[f2]\n",
        "\n",
        "  return (np.corrcoef(f1_l[:,0,0],f2_l[:,0,0])[0,1], exp_var1, exp_var2)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B8jkZQjB61mZ",
        "outputId": "dc6945c3-4693-4da0-fdb4-1ba97b2bf4d2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 33.8 ms (started: 2022-01-12 14:13:52 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def get_pca1_cca_from_file_names(f1,f2,memoized_file_pca1s, memoized_file_ccas, add_perc=True):\n",
        "  \n",
        "  f1_l = None\n",
        "  f1tt = None\n",
        "  if f1 not in memoized_file_pca1s.keys():\n",
        "    f1t = get_wrn_validation_features(f1) if \"WideResNet\" in f1 else np.load(f1) \n",
        "    scaler1 = StandardScaler()\n",
        "    scaler1.fit(f1t)\n",
        "    f1tt = scaler1.transform(f1t)\n",
        "    pca_f1 = PCA(n_components=1)\n",
        "    pca_f1.fit(f1tt)\n",
        "    f1_l = np.apply_along_axis(lambda x: np.dot(pca_f1.components_,np.expand_dims(x, axis=1)),1,f1tt)\n",
        "    exp_var1 = pca_f1.explained_variance_ratio_.round(3)[0] if add_perc else None\n",
        "    memoized_file_pca1s[f1] = (f1_l,exp_var1, f1tt)\n",
        "    del f1_t\n",
        "    del pca_f1\n",
        "    del f1t\n",
        "    del scaler1    \n",
        "  f1_l,exp_var1, f1tt = memoized_file_pca1s[f1]\n",
        "\n",
        "  f2_l = None\n",
        "  f2tt = None\n",
        "  if f2 not in memoized_file_pca1s.keys():\n",
        "    f2t = get_wrn_validation_features(f2) if \"WideResNet\" in f2 else np.load(f2)\n",
        "    scaler2 = StandardScaler()\n",
        "    scaler2.fit(f2t)\n",
        "    f2tt = scaler2.transform(f2t)\n",
        "    pca_f2 = PCA(n_components=1)\n",
        "    pca_f2.fit(f2tt)\n",
        "    f2_l = np.apply_along_axis(lambda x: np.dot(pca_f2.components_,np.expand_dims(x, axis=1)),1,f2tt)\n",
        "    exp_var2 = pca_f2.explained_variance_ratio_.round(3)[0] if add_perc else None\n",
        "    memoized_file_pca1s[f2] = (f2_l,exp_var2, f2tt)\n",
        "    del f2_t\n",
        "    del pca_f2\n",
        "    del f2t\n",
        "    del scaler2\n",
        "  f2_l,exp_var2, f2tt = memoized_file_pca1s[f2]\n",
        "\n",
        "  if (f1,f2) not in memoized_file_ccas.keys():\n",
        "    ca = CCA()\n",
        "    ca.fit(f1tt, f2tt)\n",
        "    f1tt_c, f2tt_c = ca.transform(f1tt, f2tt)\n",
        "    memoized_file_ccas[(f1,f2)] = np.corrcoef(f1tt_c[:,0],f2tt_c[:,0])[0,1]\n",
        "  cca_f1f2 = memoized_file_ccas[(f1,f2)]\n",
        "\n",
        "  return (cca_f1f2 , np.corrcoef(f1_l[:,0,0],f2_l[:,0,0])[0,1], exp_var1, exp_var2 )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HSniHBhRJVor",
        "outputId": "de46e71d-fcaa-4e51-cf62-ab8e94b2fdf1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 11.8 ms (started: 2022-01-12 14:13:52 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def get_cca_from_file_names(f1,f2,memoized_file_ccas):\n",
        "\n",
        "  if (f1,f2) not in memoized_file_ccas.keys():\n",
        "    f1t = get_wrn_validation_features(f1) if \"WideResNet\" in f1 else np.load(f1) \n",
        "    scaler1 = StandardScaler()\n",
        "    scaler1.fit(f1t)\n",
        "    f1tt = scaler1.transform(f1t)\n",
        "    \n",
        "    f2t = get_wrn_validation_features(f1) if \"WideResNet\" in f2 else np.load(f2) \n",
        "    scaler2 = StandardScaler()\n",
        "    scaler2.fit(f2t)\n",
        "    f2tt = scaler2.transform(f2t)\n",
        "\n",
        "    ca = CCA()\n",
        "    ca.fit(f1tt, f2tt)\n",
        "    f1tt_c, f2tt_c = ca.transform(f1tt, f2tt)\n",
        "    memoized_file_ccas[(f1,f2)] = np.corrcoef(f1tt_c[:,0],f2tt_c[:,0])[0,1]\n",
        "\n",
        "    del f1t\n",
        "    del f2t\n",
        "    del scaler1\n",
        "    del scaler2\n",
        "    del f1tt\n",
        "    del f2tt\n",
        "    del f1tt_c\n",
        "    del f2tt_c\n",
        "    \n",
        "  return memoized_file_ccas[(f1,f2)]\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MEY9GoMrsxYc"
      },
      "source": [
        "### Create an empty summary DataFrame"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OTixLruYsxYc",
        "outputId": "e444e541-6f5c-4ced-a850-530c50095c43"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 57.2 ms (started: 2022-01-12 15:33:55 +00:00)\n"
          ]
        }
      ],
      "source": [
        "cframe = pd.DataFrame(columns=[\"f1\",\"f2\", \"CCA\", \"PCA1\", \"PercPCA1F1\", \"PercPCA1F2\"])\n",
        "\n",
        "number_per_type = 6\n",
        "\n",
        "all_relevant_dnn_feature_files = []\n",
        "for dnn_layer in available_dnn_feature_layers:\n",
        "  all_relevant_dnn_feature_files.extend(sorted(random.sample(\n",
        "      [ f for f in dnn_features_files if \"_Validation\" in f and \"_features_\" in f and \"_X\" in f and dnn_layer in f],2)))\n",
        "\n",
        "all_relevant_cnn_feature_files = []\n",
        "for cnn_layer in available_cnn_feature_layers:\n",
        "  all_relevant_cnn_feature_files.extend(sorted(random.sample(\n",
        "      [ f for f in cnn_features_files if \"_Validation\" in f and \"_features_\" in f and \"_X\" in f and cnn_layer in f],2)))\n",
        "\n",
        "all_relevant_wrn_feature_files = []\n",
        "for wrn_layer in available_wrn_feature_layers:\n",
        "  all_relevant_wrn_feature_files.extend(sorted(random.sample(\n",
        "      [ f for f in wideresnets_features_files if \"_Validation\" in f and \"_features_\" in f and \"_X\" in f and wrn_layer in f],number_per_type)))\n",
        "\n",
        "all_relevant_feature_files = all_relevant_dnn_feature_files + all_relevant_cnn_feature_files + all_relevant_wrn_feature_files\n",
        "\n",
        "for xfile in all_relevant_feature_files:\n",
        "  cframe = pd.concat([cframe,pd.DataFrame({\"f1\":[xfile]*len(all_relevant_feature_files), \"f2\": all_relevant_feature_files, \n",
        "                                            \"CCA\": np.nan, \"PCA1\": np.nan, \"PercPCA1F1\" : np.nan, \"PercPCA1F2\" : np.nan  })])      \n",
        "cframe = cframe.drop_duplicates()\n",
        "cframe.loc[ (cframe.f1==cframe.f2), [\"PCA1\"]] = [1.0]\n",
        "cframe.loc[ (cframe.f1==cframe.f2), [\"CCA\"]] = [1.0]\n",
        "\n",
        "# causes a double log in corr matrix\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ib2_MyjvwRus",
        "outputId": "a36865ea-da5c-4072-ad04-61169ce29182"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 2.21 ms (started: 2022-01-12 15:34:01 +00:00)\n"
          ]
        }
      ],
      "source": [
        "cframe = cframe.reset_index()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PM1qKukjocnM",
        "outputId": "725577a6-c989-4521-cc3a-6ee27284e890"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['index' 'f1' 'f2' 'CCA' 'PCA1' 'PercPCA1F1' 'PercPCA1F2']\n",
            "(400, 7)\n",
            "RangeIndex(start=0, stop=400, step=1)\n",
            "time: 1.84 ms (started: 2022-01-12 15:34:02 +00:00)\n"
          ]
        }
      ],
      "source": [
        "print(cframe.columns.values)\n",
        "print(cframe.shape)\n",
        "print(cframe.index)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c2fsTzarpRvm",
        "outputId": "5d0a9f68-25be-45dc-8103-6faf70fab9c5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 908 µs (started: 2022-01-12 15:34:09 +00:00)\n"
          ]
        }
      ],
      "source": [
        "memoized_file_pca1s = {}\n",
        "memoized_file_ccas = {}\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WRiJMqM4lRiZ",
        "outputId": "a0f9eaa5-801a-496a-a475-f9ed9f5ee53e"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  0% |                                                                        |\r"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_69_20211003012506_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_57_20211003010925_features_D3R_Validation_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  1% |                                                                        |\r"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_62_20211003011453_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_28_20211003032657_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_3_20211003021214_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_39_20211003040252_features_SFTMX1_Validation_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  2% |#                                                                       |\r"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_55_20211003044915_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID393ddffb_10_20211016184954_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID4cec947d_3_20211015065514_features_CLASSIFIER_D1_Validation3_15000-20000_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  3% |##                                                                      |\r"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_D1_Validation1_5000-10000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDdc182e41_7_20211016062521_features_CLASSIFIER_D1_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_D1_Validation5_25000-26032_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  4% |##                                                                      |\r"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID97463b53_4_20211015104216_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe1fdfe44_8_20211016112241_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  5% |###                                                                     |\r"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_69_20211003012506_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            ""
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_69_20211003012506_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_69_20211003012506_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID393ddffb_10_20211016184954_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_69_20211003012506_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID4cec947d_3_20211015065514_features_CLASSIFIER_D1_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_69_20211003012506_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_D1_Validation1_5000-10000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_69_20211003012506_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDdc182e41_7_20211016062521_features_CLASSIFIER_D1_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_69_20211003012506_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_D1_Validation5_25000-26032_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_69_20211003012506_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_69_20211003012506_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_69_20211003012506_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_69_20211003012506_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID97463b53_4_20211015104216_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_69_20211003012506_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe1fdfe44_8_20211016112241_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_69_20211003012506_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n",
            "40\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            ""
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_57_20211003010925_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_57_20211003010925_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID393ddffb_10_20211016184954_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_57_20211003010925_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID4cec947d_3_20211015065514_features_CLASSIFIER_D1_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_57_20211003010925_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_D1_Validation1_5000-10000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_57_20211003010925_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDdc182e41_7_20211016062521_features_CLASSIFIER_D1_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_57_20211003010925_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_D1_Validation5_25000-26032_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            ""
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_57_20211003010925_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_57_20211003010925_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_57_20211003010925_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_57_20211003010925_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID97463b53_4_20211015104216_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_57_20211003010925_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe1fdfe44_8_20211016112241_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_57_20211003010925_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            ""
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_62_20211003011453_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_62_20211003011453_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID393ddffb_10_20211016184954_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_62_20211003011453_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID4cec947d_3_20211015065514_features_CLASSIFIER_D1_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_62_20211003011453_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_D1_Validation1_5000-10000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_62_20211003011453_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDdc182e41_7_20211016062521_features_CLASSIFIER_D1_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_62_20211003011453_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_D1_Validation5_25000-26032_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            ""
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_62_20211003011453_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_62_20211003011453_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_62_20211003011453_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_62_20211003011453_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID97463b53_4_20211015104216_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_62_20211003011453_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe1fdfe44_8_20211016112241_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_62_20211003011453_features_D3R_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n",
            "80\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            ""
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_28_20211003032657_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_28_20211003032657_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID393ddffb_10_20211016184954_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_28_20211003032657_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID4cec947d_3_20211015065514_features_CLASSIFIER_D1_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_28_20211003032657_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_D1_Validation1_5000-10000_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 23% |################                                                        |\r"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_28_20211003032657_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDdc182e41_7_20211016062521_features_CLASSIFIER_D1_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_28_20211003032657_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_D1_Validation5_25000-26032_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_28_20211003032657_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_28_20211003032657_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 24% |#################                                                       |\r"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_28_20211003032657_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_28_20211003032657_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID97463b53_4_20211015104216_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_28_20211003032657_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe1fdfe44_8_20211016112241_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_28_20211003032657_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            ""
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_3_20211003021214_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_3_20211003021214_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID393ddffb_10_20211016184954_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_3_20211003021214_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID4cec947d_3_20211015065514_features_CLASSIFIER_D1_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_3_20211003021214_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_D1_Validation1_5000-10000_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 28% |####################                                                    |\r"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_3_20211003021214_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDdc182e41_7_20211016062521_features_CLASSIFIER_D1_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_3_20211003021214_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_D1_Validation5_25000-26032_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_3_20211003021214_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 29% |####################                                                    |\r"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_3_20211003021214_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_3_20211003021214_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_3_20211003021214_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID97463b53_4_20211015104216_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_3_20211003021214_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe1fdfe44_8_20211016112241_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 30% |#####################                                                   |\r"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_3_20211003021214_features_DRP1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n",
            "120\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            ""
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_39_20211003040252_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_39_20211003040252_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID393ddffb_10_20211016184954_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_39_20211003040252_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID4cec947d_3_20211015065514_features_CLASSIFIER_D1_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_39_20211003040252_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_D1_Validation1_5000-10000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_39_20211003040252_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDdc182e41_7_20211016062521_features_CLASSIFIER_D1_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_39_20211003040252_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_D1_Validation5_25000-26032_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_39_20211003040252_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_39_20211003040252_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_39_20211003040252_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_39_20211003040252_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID97463b53_4_20211015104216_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_39_20211003040252_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe1fdfe44_8_20211016112241_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_39_20211003040252_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            ""
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_55_20211003044915_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_55_20211003044915_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID393ddffb_10_20211016184954_features_CLASSIFIER_D1_Validation4_20000-25000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_55_20211003044915_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID4cec947d_3_20211015065514_features_CLASSIFIER_D1_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_55_20211003044915_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_D1_Validation1_5000-10000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_55_20211003044915_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDdc182e41_7_20211016062521_features_CLASSIFIER_D1_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_55_20211003044915_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_D1_Validation5_25000-26032_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_55_20211003044915_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_55_20211003044915_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_55_20211003044915_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID71928613_1_20211014230300_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_55_20211003044915_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID97463b53_4_20211015104216_features_CLASSIFIER_FL_Validation3_15000-20000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_55_20211003044915_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe1fdfe44_8_20211016112241_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/CNN_A_55_20211003044915_features_SFTMX1_Validation_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_IDe984f9ca_2_20211015035259_features_CLASSIFIER_FL_Validation5_25000-26032_X.npy\n",
            "160\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            ""
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "200\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            ""
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "240\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/WideResNet28-10_ID294f3208_1_20211015004120_features_CLASSIFIER_FL_Validation2_10000-15000_X.npy\n",
            "/content/drive/MyDrive/data_papers/gpSVHN/model_features/DNN_A_13_20211003001057_features_SFTMX1_Validation_X.npy\n"
          ]
        }
      ],
      "source": [
        "import datetime\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "from progressbar import ProgressBar\n",
        "pbar = ProgressBar()\n",
        "\n",
        "for rc in pbar(cframe.index):\n",
        "  xfile = cframe.loc[rc, 'f1']\n",
        "  yfile = cframe.loc[rc, 'f2']\n",
        "  if xfile != yfile:\n",
        "    try:\n",
        "      cframe.loc[rc, [\"PCA1\",\"PercPCA1F1\",\"PercPCA1F2\"]]  = get_pca1_from_file_names(xfile, yfile, memoized_file_pca1s)     \n",
        "      cframe.loc[rc, [\"CCA\"]]  = get_cca_from_file_names(xfile,yfile,memoized_file_ccas)\n",
        "      if (rc % 40) == 0:\n",
        "        print(rc)\n",
        "    except:\n",
        "      print(xfile)\n",
        "      print(yfile)\n",
        "      pass\n",
        "      \n",
        "    # print(rc,cframe.loc[rc, \"PCA1\"])      \n",
        "\n",
        "cframe.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/all_Validation_ccs_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\",index=False)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QtduTw9C7OC-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b0185c53-3f8b-49f5-e8fd-9548ebafb25d"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['cnn_Validation_ccs_20211115184258.csv',\n",
              " 'dnn_Validation_ccs_20211115040852.csv',\n",
              " 'validation_accs_base_20211117102712.csv',\n",
              " 'pca1_no_perc.zip',\n",
              " 'summary_pca1_bytypeComb.csv',\n",
              " 'summary_pca1_byTypeCombLayerComb.csv',\n",
              " 'pca1_with_perc.zip',\n",
              " 'all_Validation_ccs_20220110114058.csv',\n",
              " 'dwnld_ensemble_regdata.csv',\n",
              " 'dwnld_ensemble_regdata2.csv',\n",
              " 'dwnld_svhn1_ensemble_regdata.csv',\n",
              " 'dwnld_svhn2_ensemble_regdata.csv',\n",
              " 'dwnld_svhn_c.csv',\n",
              " 'dwnld_svhn_a.csv']"
            ]
          },
          "metadata": {},
          "execution_count": 51
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 5.41 ms (started: 2022-01-12 15:20:48 +00:00)\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "os.listdir(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# cframe\n",
        "cframe.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/all_Validation_ccs_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\",index=False)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ThVNj_-S4O26",
        "outputId": "56776995-b881-497d-fd32-478d4b870ef9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "time: 18.8 ms (started: 2022-01-12 15:21:31 +00:00)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "j3PDqCkYx5r_"
      },
      "outputs": [],
      "source": [
        "print(cframe.loc[2, 'f1'])\n",
        "print(cframe.loc[2, 'f2'])\n",
        "print(memoized_file_pca1s[cframe.loc[2, 'f1']][:,0,0])\n",
        "print(memoized_file_pca1s[cframe.loc[2, 'f2']][:,0,0])\n",
        "\n",
        "# np.corrcoef(memoized_file_pca1s[cframe.loc[2, 'f1']][:,0,0],memoized_file_pca1s[cframe.loc[2, 'f2']][:,0,0]) # [0,1]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vuN3skzK-fAy"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import re\n",
        "\n",
        "def get_base_patterns_for_validation(features_files, layer_name = \"CLASSIFIER_D1\", model_type = \"WideResNet\"):\n",
        "  base_patterns_for_validations = []\n",
        "  for ff in [ s for s in features_files if \"Validation\" in s]:\n",
        "    validation_search = re.search(f'^.*({model_type}.*_ID.*features_{layer_name}_Validation)[0-9]+.*$', ff, re.IGNORECASE)\n",
        "    if validation_search:\n",
        "        base_patterns_for_validations.append(validation_search.group(1))\n",
        "\n",
        "  base_patterns_for_validations = sorted(list(set(base_patterns_for_validations)))\n",
        "  return base_patterns_for_validations\n",
        "\n",
        "\n",
        "def get_validation_features_for_layer_using_subbatches(feature_files, layer_name, num_of_models, model_type=\"WideResNet\", axis_to_concat = 1):\n",
        " \n",
        "  feature_files_used = [ff for ff in feature_files if \"Validation\" in ff and \"_X\" in ff and f\"_{layer_name}_\" in ff ]\n",
        "  \n",
        "  # base_patterns_for_validations = []\n",
        "  # for ff in [ s for s in feature_files_used ]:\n",
        "  #   validation_search = re.search(f'^.*(_ID.*features_{layer_name}_Validation)[0-9]+.*$', ff, re.IGNORECASE)\n",
        "  #   if validation_search:\n",
        "  #       base_patterns_for_validations.append(validation_search.group(1))\n",
        "  # base_patterns_for_validations = sorted(list(set(base_patterns_for_validations)))\n",
        "\n",
        "  base_patterns_for_validations = get_base_patterns_for_validation(feature_files_used, layer_name, model_type)\n",
        "  base_patterns_for_validations = sorted(random.sample(base_patterns_for_validations, min(num_of_models,len(base_patterns_for_validations)) ))\n",
        "\n",
        "  np_x_validation_collab = None\n",
        "  for base_val_str in base_patterns_for_validations:\n",
        "      validation_batch_files = sorted([ ff for ff in feature_files_used if base_val_str in ff])\n",
        "      np_x_validation_collab_batch = np.array([np.load(ff) for ff in validation_batch_files])\n",
        "      np_x_validation_collab_batch = np.concatenate(np_x_validation_collab_batch, axis=0)\n",
        "      if np_x_validation_collab is None:\n",
        "        np_x_validation_collab = np_x_validation_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_validation_collab = np.concatenate([np_x_validation_collab, np_x_validation_collab_batch], axis=axis_to_concat)\n",
        "  \n",
        "  return np_x_validation_collab  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fxjZPT2gsxYc"
      },
      "outputs": [],
      "source": [
        "# import datetime \n",
        "\n",
        "# import warnings\n",
        "# warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "# for dnn_layer in available_dnn_feature_layers:\n",
        "#   all_relevant_feature_files =  [ f for f in dnn_features_files if \"_Validation\" in f and \"_features_\" in f and \"_X\" in f and dnn_layer in f]\n",
        "#   for xfileCount, xfile in enumerate(all_relevant_feature_files):\n",
        "#     for yfile in all_relevant_feature_files:\n",
        "#       if xfile != yfile:\n",
        "#         if np.isnan(cframe.loc[(cframe.f1==xfile) & (cframe.f2 == yfile),\"CCA1\"].iloc[0]):          \n",
        "#           cca1_val, pca1_val = get_cca1_pca1_from_file_names(xfile, yfile)          \n",
        "#           cframe.loc[ (cframe.f1==xfile) & (cframe.f2 == yfile), [\"CCA1\",\"PCA1\"] ] = [ cca1_val, pca1_val ] \n",
        "#           cframe.loc[ (cframe.f1==yfile) & (cframe.f2 == xfile),[\"CCA1\",\"PCA1\"] ] = [ cca1_val, pca1_val ] \n",
        "#     print(f\"DNN {xfileCount}/{len(all_relevant_feature_files)} {dnn_layer}\")\n",
        "\n",
        "\n",
        "# cframe.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/dnn_Validation_temp_ccs_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\",index=False)\n",
        "\n",
        "\n",
        "# for cnn_layer in available_cnn_feature_layers:\n",
        "#   all_relevant_feature_files =  [ f for f in cnn_features_files if \"_Validation\" in f and \"_features_\" in f and \"_X\" in f and cnn_layer in f]\n",
        "#   for xfileCount, xfile in enumerate(all_relevant_feature_files):\n",
        "#     for yfile in all_relevant_feature_files:\n",
        "#       if xfile != yfile:\n",
        "#         if np.isnan(cframe.loc[(cframe.f1==xfile) & (cframe.f2 == yfile),\"CCA1\"].iloc[0]):          \n",
        "#           cca1_val, pca1_val = get_cca1_pca1_from_file_names(xfile, yfile)          \n",
        "#           cframe.loc[ (cframe.f1==xfile) & (cframe.f2 == yfile), [\"CCA1\",\"PCA1\"] ] = [ cca1_val, pca1_val ] \n",
        "#           cframe.loc[ (cframe.f1==yfile) & (cframe.f2 == xfile),[\"CCA1\",\"PCA1\"] ] = [ cca1_val, pca1_val ] \n",
        "#     print(f\"CNN {xfileCount}/{len(all_relevant_feature_files)} {cnn_layer}\")\n",
        "\n",
        "\n",
        "# cframe.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/cnn_Validation_ccs_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\",index=False)\n",
        "\n",
        "\n",
        "# warnings.resetwarnings()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XtjoNp7qsxYd"
      },
      "source": [
        "### Select the most distant DNN/CNN validation that still make some sense"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3h4Kt3rJsxYd"
      },
      "outputs": [],
      "source": [
        "# cframe.loc[cframe.CCA1==1.0][\"f1\"]\n",
        "\n",
        "import os,sys\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "dnn_distresults = pd.read_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/dnn_Validation_ccs_20211115040852.csv\")\n",
        "cnn_distresults = pd.read_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/cnn_Validation_ccs_20211115184258.csv\")\n",
        "\n",
        "dnn_distresults = dnn_distresults[dnn_distresults.Type==\"DNN\"]\n",
        "# individual_dnns = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/individual_dnn_summary_20211014152626.npy\")\n",
        "# individual_cnns = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/individual_cnn_summary_20211014152108.npy\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "w7qMZNmasxYd"
      },
      "outputs": [],
      "source": [
        "# cnn_distresults.sort_values(\"CCA1\")\n",
        "# cnn_distresults[cnn_distresults.Layer == \"SFTMX1\"].sort_values(\"PCA1\")\n",
        "# cnn_distresults[cnn_distresults.Layer == \"SFTMX1\"].sort_values(\"CCA1\")[0:40]\n",
        "# dnn_distresults[dnn_distresults.Layer == \"SFTMX1\"].sort_values(\"CCA1\")\n",
        "\n",
        "cnn_avg_cca = cnn_distresults.groupby([\"Type\",\"Layer\",\"f1\"])[\"CCA1\"].mean().reset_index()\n",
        "dnn_avg_cca = dnn_distresults.groupby([\"Type\",\"Layer\",\"f1\"])[\"CCA1\"].mean().reset_index()\n",
        "cnn_avg_cca = cnn_avg_cca.sort_values('CCA1')\n",
        "dnn_avg_cca = pd.concat([dnn_avg_cca[dnn_avg_cca.CCA1 > 0.45].sort_values('CCA1'), dnn_avg_cca[dnn_avg_cca.CCA1 <= 0.45]])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yfwMt8ZrsxYd"
      },
      "outputs": [],
      "source": [
        "cnn_avg_pca = cnn_distresults.groupby([\"Type\",\"Layer\",\"f1\"])[\"PCA1\"].apply(lambda x: np.mean(np.abs(x))).reset_index()\n",
        "dnn_avg_pca = dnn_distresults.groupby([\"Type\",\"Layer\",\"f1\"])[\"PCA1\"].apply(lambda x: np.mean(np.abs(x))).reset_index()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7cAF5i8WsxYe"
      },
      "outputs": [],
      "source": [
        "# ggplot(dnn_avg_pca) + geom_density(aes('PCA1'))\n",
        "# ggplot(dnn_avg_cca) + geom_density(aes('CCA1'))\n",
        "# ggplot(cnn_avg_cca) + geom_density(aes('CCA1'))\n",
        "\n",
        "\n",
        "\n",
        "# start going through it pairwise"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8GvmAzHIp7te"
      },
      "source": [
        "### Create an empty summary DataFrame"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jBsx6oAbp-Bf"
      },
      "outputs": [],
      "source": [
        "cframe = pd.DataFrame(columns=[\"f1\",\"f2\",\"CCA1\",\"PCA1\",\"Type\",\"Layer\"])\n",
        "\n",
        "for dnn_layer in available_dnn_feature_layers:\n",
        "  all_relevant_feature_files =  [ f for f in dnn_features_files if \"_Validation\" in f and \"_features_\" in f and \"_X\" in f and dnn_layer in f]\n",
        "  for xfile in all_relevant_feature_files:\n",
        "    cframe = pd.concat([cframe,pd.DataFrame({\"f1\":[xfile]*len(all_relevant_feature_files), \"f2\": all_relevant_feature_files, \"CCA1\": np.nan, \"PCA1\": np.nan, \"Type\": \"DNN\", \"Layer\": dnn_layer  })])      \n",
        "\n",
        "for cnn_layer in available_cnn_feature_layers:\n",
        "  all_relevant_feature_files =  [ f for f in cnn_features_files if \"_Validation\" in f and \"_features_\" in f and \"_X\" in f and cnn_layer in f]\n",
        "  for xfile in all_relevant_feature_files:\n",
        "    cframe = pd.concat([cframe,pd.DataFrame({\"f1\":[xfile]*len(all_relevant_feature_files), \"f2\": all_relevant_feature_files, \"CCA1\": np.nan, \"PCA1\": np.nan, \"Type\": \"CNN\", \"Layer\": cnn_layer  })])      \n",
        "\n",
        "cframe = cframe.drop_duplicates()\n",
        "cframe.loc[ (cframe.f1==cframe.f2), [\"CCA1\",\"PCA1\"]] = [1.0,1.0]\n",
        "\n",
        "# causes a double log in corr matrix\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e96k_d6dqynr"
      },
      "outputs": [],
      "source": [
        "import datetime \n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "for dnn_layer in available_dnn_feature_layers:\n",
        "  all_relevant_feature_files =  [ f for f in dnn_features_files if \"_Validation\" in f and \"_features_\" in f and \"_X\" in f and dnn_layer in f]\n",
        "  for xfileCount, xfile in enumerate(all_relevant_feature_files):\n",
        "    for yfile in all_relevant_feature_files:\n",
        "      if xfile != yfile:\n",
        "        if np.isnan(cframe.loc[(cframe.f1==xfile) & (cframe.f2 == yfile),\"CCA1\"].iloc[0]):          \n",
        "          cca1_val, pca1_val = get_cca1_pca1_from_file_names(xfile, yfile)          \n",
        "          cframe.loc[ (cframe.f1==xfile) & (cframe.f2 == yfile), [\"CCA1\",\"PCA1\"] ] = [ cca1_val, pca1_val ] \n",
        "          cframe.loc[ (cframe.f1==yfile) & (cframe.f2 == xfile),[\"CCA1\",\"PCA1\"] ] = [ cca1_val, pca1_val ] \n",
        "    print(f\"DNN {xfileCount}/{len(all_relevant_feature_files)} {dnn_layer}\")\n",
        "\n",
        "\n",
        "cframe.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/dnn_Validation_temp_ccs_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\",index=False)\n",
        "\n",
        "\n",
        "for cnn_layer in available_cnn_feature_layers:\n",
        "  all_relevant_feature_files =  [ f for f in cnn_features_files if \"_Validation\" in f and \"_features_\" in f and \"_X\" in f and cnn_layer in f]\n",
        "  for xfileCount, xfile in enumerate(all_relevant_feature_files):\n",
        "    for yfile in all_relevant_feature_files:\n",
        "      if xfile != yfile:\n",
        "        if np.isnan(cframe.loc[(cframe.f1==xfile) & (cframe.f2 == yfile),\"CCA1\"].iloc[0]):          \n",
        "          cca1_val, pca1_val = get_cca1_pca1_from_file_names(xfile, yfile)          \n",
        "          cframe.loc[ (cframe.f1==xfile) & (cframe.f2 == yfile), [\"CCA1\",\"PCA1\"] ] = [ cca1_val, pca1_val ] \n",
        "          cframe.loc[ (cframe.f1==yfile) & (cframe.f2 == xfile),[\"CCA1\",\"PCA1\"] ] = [ cca1_val, pca1_val ] \n",
        "    print(f\"CNN {xfileCount}/{len(all_relevant_feature_files)} {cnn_layer}\")\n",
        "\n",
        "\n",
        "cframe.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/cnn_Validation_ccs_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\",index=False)\n",
        "\n",
        "\n",
        "warnings.resetwarnings()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kIiJPc3OqE-V"
      },
      "source": [
        "### Select the most distant DNN/CNN validation that still make some sense"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2Lfl0MK_UdlL"
      },
      "outputs": [],
      "source": [
        "# cframe.loc[cframe.CCA1==1.0][\"f1\"]\n",
        "\n",
        "import os,sys\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "dnn_distresults = pd.read_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/dnn_Validation_ccs_20211115040852.csv\")\n",
        "cnn_distresults = pd.read_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/cnn_Validation_ccs_20211115184258.csv\")\n",
        "\n",
        "dnn_distresults = dnn_distresults[dnn_distresults.Type==\"DNN\"]\n",
        "# individual_dnns = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/individual_dnn_summary_20211014152626.npy\")\n",
        "# individual_cnns = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/individual_cnn_summary_20211014152108.npy\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H1HIOBeUz-SL"
      },
      "outputs": [],
      "source": [
        "# cnn_distresults.sort_values(\"CCA1\")\n",
        "# cnn_distresults[cnn_distresults.Layer == \"SFTMX1\"].sort_values(\"PCA1\")\n",
        "# cnn_distresults[cnn_distresults.Layer == \"SFTMX1\"].sort_values(\"CCA1\")[0:40]\n",
        "# dnn_distresults[dnn_distresults.Layer == \"SFTMX1\"].sort_values(\"CCA1\")\n",
        "\n",
        "cnn_avg_cca = cnn_distresults.groupby([\"Type\",\"Layer\",\"f1\"])[\"CCA1\"].mean().reset_index()\n",
        "dnn_avg_cca = dnn_distresults.groupby([\"Type\",\"Layer\",\"f1\"])[\"CCA1\"].mean().reset_index()\n",
        "cnn_avg_cca = cnn_avg_cca.sort_values('CCA1')\n",
        "dnn_avg_cca = pd.concat([dnn_avg_cca[dnn_avg_cca.CCA1 > 0.45].sort_values('CCA1'), dnn_avg_cca[dnn_avg_cca.CCA1 <= 0.45]])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BWR5WCGCgJ5_"
      },
      "outputs": [],
      "source": [
        "cnn_avg_pca = cnn_distresults.groupby([\"Type\",\"Layer\",\"f1\"])[\"PCA1\"].apply(lambda x: np.mean(np.abs(x))).reset_index()\n",
        "dnn_avg_pca = dnn_distresults.groupby([\"Type\",\"Layer\",\"f1\"])[\"PCA1\"].apply(lambda x: np.mean(np.abs(x))).reset_index()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mCP0hSiL1NR2"
      },
      "outputs": [],
      "source": [
        "# ggplot(dnn_avg_pca) + geom_density(aes('PCA1'))\n",
        "# ggplot(dnn_avg_cca) + geom_density(aes('CCA1'))\n",
        "# ggplot(cnn_avg_cca) + geom_density(aes('CCA1'))\n",
        "\n",
        "\n",
        "\n",
        "# start going through it pairwise"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-nbwCVHSRRfM"
      },
      "source": [
        "# Parallel model with no GP and no collaborative but with different parallel DNN/CNN/WideResnet streams. The _best_ parallel fully integrated model."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-ntNI2qC_9Hm"
      },
      "source": [
        "function for basic parallel 20 DNN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kXWOr1f4RQbd",
        "outputId": "66d3793f-beb2-41d0-9296-808fb4a96479"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 12 ms (started: 2022-01-10 14:22:10 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def basic_ParallelDNN_A(model_name, inshape, num_classes = 10, num_of_parallels = 20):\n",
        "\n",
        "  base_input = Input(shape=inshape, name='base_input')\n",
        "  f1_output = Flatten(name='F1')(base_input)\n",
        "\n",
        "  d3rs = []\n",
        "  for pc in range(num_of_parallels):\n",
        "    d1_output = Dense(128, activation='relu', name = f'p{pc+1}_D1')(f1_output)\n",
        "    d2_output = Dense(128, activation='relu', name = f'p{pc+1}_D2')(d1_output)\n",
        "    d3r_output = Dense(48, activation='relu', kernel_regularizer=tf.keras.regularizers.l1(0.005), name = f'p{pc+1}_D3R')(d2_output)\n",
        "    d3rs.append(d3r_output)\n",
        "\n",
        "  # merging_layer = tf.keras.layers.concatenate(d3rs)\n",
        "  concat1_output = Concatenate(name=\"CONCAT1\")(d3rs)\n",
        "  model_output = Dense(num_classes,  activation='softmax', name = 'SFTMX1')(concat1_output)\n",
        "  model = Model(base_input, model_output, name = model_name)\n",
        "\n",
        "  return model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CdiHdMZbAzHK"
      },
      "source": [
        "function for basic parallel 20 CNN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "inikwtpIW4M3",
        "outputId": "cdd0a16b-2472-4e79-ea2e-ff7fd54695a5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 12.4 ms (started: 2022-01-10 14:22:14 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def basic_ParallelCNN_A(model_name, inshape, num_classes = 10, num_of_parallels = 20):\n",
        "\n",
        "  # Input Layer\n",
        "  base_input = Input(shape=inshape, name='base_input')\n",
        "\n",
        "  drp1s = []\n",
        "  for pc in range(num_of_parallels):\n",
        "    c1_output = Conv2D(filters=32,kernel_size=[5, 5],padding=\"same\",activation=\"relu\", name=f\"p{pc+1}_C1\")(base_input)\n",
        "    mxp1_output = MaxPooling2D(pool_size=[2, 2], strides=2, name=f\"p{pc+1}_MXP1\")(c1_output)\n",
        "    c2_output = Conv2D(filters=64,kernel_size=[5, 5],padding=\"same\",activation=\"relu\",name=f\"p{pc+1}_C2\")(mxp1_output)\n",
        "    mxp2_output = MaxPooling2D(pool_size=[2, 2], strides=2, name=f\"p{pc+1}_MXP2\")(c2_output)\n",
        "    f1_output = Flatten(name=f\"p{pc+1}_F1\")(mxp2_output)\n",
        "    d1_output = Dense(units=256, activation=\"relu\", name=f\"p{pc+1}_D1\")(f1_output)\n",
        "    drp1_output = Dropout(rate=0.5, name=f\"p{pc+1}_DRP1\")(d1_output)\n",
        "    drp1s.append(drp1_output)\n",
        "\n",
        "  concat1_output = Concatenate(name=\"CONCAT1\")(drp1s)\n",
        "  model_output = Dense(num_classes,  activation='softmax', name = 'SFTMX1')(concat1_output)\n",
        "  model = Model(base_input, model_output, name = model_name)\n",
        "  return model\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PJEQRsKaA2fq"
      },
      "source": [
        "function for basic parallel 10 CNN + 10 DNN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EFDcerJycgBz",
        "outputId": "6e435651-6bca-4d27-e5f6-c152ccc28cd5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 20.4 ms (started: 2022-01-10 14:22:16 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def basic_ParallelCNNDNN_A(model_name, inshape, num_classes = 10, num_of_parallels = 20):\n",
        "\n",
        "  # Input Layer\n",
        "  base_input = Input(shape=inshape, name='base_input')\n",
        "\n",
        "  to_merge = []\n",
        "  for pc in range(int(num_of_parallels/2)):\n",
        "    c1_output = Conv2D(filters=32,kernel_size=[5, 5],padding=\"same\",activation=\"relu\", name=f\"p{pc+1}_C1_CNN\")(base_input)\n",
        "    mxp1_output = MaxPooling2D(pool_size=[2, 2], strides=2, name=f\"p{pc+1}_MXP1_CNN\")(c1_output)\n",
        "    c2_output = Conv2D(filters=64,kernel_size=[5, 5],padding=\"same\",activation=\"relu\",name=f\"p{pc+1}_C2_CNN\")(mxp1_output)\n",
        "    mxp2_output = MaxPooling2D(pool_size=[2, 2], strides=2, name=f\"p{pc+1}_MXP2_CNN\")(c2_output)\n",
        "    f1_output = Flatten(name=f\"p{pc+1}_F1_CNN\")(mxp2_output)\n",
        "    d1_output = Dense(units=256, activation=\"relu\", name=f\"p{pc+1}_D1_CNN\")(f1_output)\n",
        "    drp1_output = Dropout(rate=0.5, name=f\"p{pc+1}_DRP1_CNN\")(d1_output)\n",
        "    to_merge.append(drp1_output)\n",
        "\n",
        "  for pc in range(int(num_of_parallels/2)):\n",
        "    d1_output = Dense(128, activation='relu', name = f'p{pc+1}_D1_DNN')(f1_output)\n",
        "    d2_output = Dense(128, activation='relu', name = f'p{pc+1}_D2_DNN')(d1_output)\n",
        "    d3r_output = Dense(48, activation='relu', kernel_regularizer=tf.keras.regularizers.l1(0.005), name = f'p{pc+1}_D3R_DNN')(d2_output)\n",
        "    to_merge.append(d3r_output)\n",
        "\n",
        "  concat1_output = Concatenate(name=\"CONCAT1\")(to_merge)\n",
        "  model_output = Dense(num_classes,  activation='softmax', name = 'SFTMX1')(concat1_output)\n",
        "  model = Model(base_input, model_output, name = model_name)\n",
        "  return model\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gg49OZj4Htvb"
      },
      "source": [
        "function for basic parallel 10 WideResnet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HitZBCmym3Tp",
        "outputId": "20161cae-260b-489c-a149-017e83626600"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 44.3 ms (started: 2022-01-10 14:22:21 +00:00)\n"
          ]
        }
      ],
      "source": [
        "# input_shape=train_data_grey[0,:,:,:].shape\n",
        "import functools\n",
        "\n",
        "def basic_ParallelWideResNet2810_A(model_name, inshape, num_classes = 10, num_of_parallels = 10,\n",
        "                                   weight_decay = 0.0005, weight_init=\"he_normal\",\n",
        "                                   depth=28, k = 10, dropout_probability = 0.0, channel_axis = -1,\n",
        "                                   use_bias = False):\n",
        "\n",
        "  # Input Layer\n",
        "  base_input = Input(shape=inshape, name='base_input')\n",
        "\n",
        "  n = (depth - 4) / 6\n",
        "\n",
        "  to_merge = []\n",
        "  for pc in range(num_of_parallels):\n",
        "    # inputs_wrn = Input(shape=input_shape,name=f\"P{str(pc)}_INPUT\")\n",
        "    n_stages=[16, 16*k, 32*k, 64*k]\n",
        "    conv1_wrn = Conv2D(16, \n",
        "                    (3, 3), \n",
        "                    strides=1,\n",
        "                    padding=\"same\",\n",
        "                    kernel_initializer=weight_init,\n",
        "                    kernel_regularizer=L2(weight_decay),\n",
        "                    use_bias=use_bias,\n",
        "                    name=f\"P{str(pc)}_C1BLOCK\")(base_input) # \"One conv at the beginning (spatial size: 32x32)\"\n",
        "    # Add wide residual blocks\n",
        "    block_fn = _wide_basic\n",
        "    conv2_wrn = _layer(functools.partial(block_fn,\n",
        "                                         identifier=f\"P{str(pc)}_C2BLOCK\",\n",
        "                                          channel_axis = channel_axis,\n",
        "                                          weight_decay = weight_decay,\n",
        "                                          weight_init= weight_init,\n",
        "                                          use_bias = use_bias,\n",
        "                                          dropout_probability = dropout_probability\n",
        "                                         ), \n",
        "                                         n_input_plane=n_stages[0], n_output_plane=n_stages[1], count=n, stride=(1,1)\n",
        "                       )(conv1_wrn)# \"Stage 1 (spatial size: 32x32)\"\n",
        "    conv3_wrn = _layer(functools.partial(block_fn,identifier=f\"P{str(pc)}_C3BLOCK\",\n",
        "                                          channel_axis = channel_axis,\n",
        "                                          weight_decay = weight_decay,\n",
        "                                          weight_init= weight_init,\n",
        "                                          use_bias = use_bias,\n",
        "                                          dropout_probability = dropout_probability\n",
        "                                         ), n_input_plane=n_stages[1], n_output_plane=n_stages[2], count=n, stride=(2,2)\n",
        "                       )(conv2_wrn)# \"Stage 2 (spatial size: 16x16)\"\n",
        "    conv4_wrn = _layer(functools.partial(block_fn,identifier=f\"P{str(pc)}_C4BLOCK\",\n",
        "                                          channel_axis = channel_axis,\n",
        "                                          weight_decay = weight_decay,\n",
        "                                          weight_init= weight_init,\n",
        "                                          use_bias = use_bias,\n",
        "                                          dropout_probability = dropout_probability\n",
        "                                         ), n_input_plane=n_stages[2], n_output_plane=n_stages[3], count=n, stride=(2,2)\n",
        "                       )(conv3_wrn)# \"Stage 3 (spatial size: 8x8)\"\n",
        "\n",
        "    batch_norm_wrn = BatchNormalization(axis=channel_axis,name=f\"P{str(pc)}_BN\")(conv4_wrn)\n",
        "    relu_wrn = Activation(\"relu\")(batch_norm_wrn)\n",
        "                                            \n",
        "    # Classifier block\n",
        "    pool_wrn = AveragePooling2D(pool_size=(8, 8), strides=(1, 1), padding=\"same\", name=f\"P{str(pc)}_CLASSIFIER_AVPL\")(relu_wrn)\n",
        "    flatten_wrn = Flatten(name=f\"P{str(pc)}_CLASSIFIER_FL\")(pool_wrn)\n",
        "\n",
        "    to_merge.append(flatten_wrn)\n",
        "\n",
        "    # predictions_wrn = Dense(units=no_classes, kernel_initializer=weight_init, use_bias=use_bias,\n",
        "    #                     kernel_regularizer=L2(weight_decay), activation=\"softmax\", name=\"P{str(pc)}_CLASSIFIER_D1\")(flatten_wrn)\n",
        "    # model_wrn = Model(inputs=inputs_wrn, outputs=predictions_wrn)\n",
        "\n",
        "  concat1_output = Concatenate(name=\"CONCAT1\")(to_merge)\n",
        "  model_output = Dense(num_classes,  activation='softmax', name = 'SFTMX1')(concat1_output)\n",
        "  model = Model(base_input, model_output, name = model_name)\n",
        "\n",
        "  return model\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kn-syuLCXyKh",
        "outputId": "4580c1c8-7a71-4370-b92c-7ce69ac933df"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 981 µs (started: 2022-01-10 14:23:25 +00:00)\n"
          ]
        }
      ],
      "source": [
        "import functools\n",
        "from functools import partial"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qmOuqvetypnO",
        "outputId": "e10d18fd-3788-45ee-f1dd-35cbe1cb3c1e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/ParallelDNN_A_20220110142406_saved_model_after_fit/assets\n",
            "487.11203305399977\n",
            "time: 8min 7s (started: 2022-01-10 14:24:06 +00:00)\n"
          ]
        }
      ],
      "source": [
        " # parallel fuly integrated 20 DNN\n",
        " \n",
        " start_time = timeit.default_timer()\n",
        " pdnn20, pdnn20h = compile_and_fit_model_basic( partial(basic_ParallelDNN_A,num_of_parallels=20),  \n",
        "                    f\"ParallelDNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    train_data_grey[0,:,:,:].shape, \n",
        "                    train_data_grey, train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,\n",
        "                    verbose_level = 0,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(validation_data_grey, validation_targets))\n",
        "print(timeit.default_timer()-start_time)\n",
        "\n",
        "# /content/drive/MyDrive/data_papers/gpSVHN/model_finals/ParallelDNN_A_20211002170140_saved_model_after_fit/assets  # 88% validation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "m7W5a2D_9GsK"
      },
      "outputs": [],
      "source": [
        "# parallel fuly integrated 20 CNN\n",
        "\n",
        "start_time = timeit.default_timer()\n",
        "pcnn20, pcnn20h = compile_and_fit_model_basic( partial(basic_ParallelCNN_A, num_of_parallels=20),  \n",
        "                    f\"ParallelCNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    train_data_grey[0,:,:,:].shape, \n",
        "                    train_data_grey, train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,\n",
        "                    verbose_level = 0,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(validation_data_grey, validation_targets))\n",
        "print(timeit.default_timer()-start_time)\n",
        " \n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K1bRQHOu9PjY",
        "outputId": "98c22724-63da-4bc1-b55e-a5452b57bd50"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/ParallelCNNDNN_A_20220110151356_saved_model_after_fit/assets\n",
            "539.0434264710002\n",
            "time: 8min 59s (started: 2022-01-10 15:13:56 +00:00)\n"
          ]
        }
      ],
      "source": [
        "# parallel fuly integrated 10 DNN + 10 CNN\n",
        " \n",
        "start_time = timeit.default_timer()\n",
        "pdcnn20, pdcnn20h = compile_and_fit_model_basic( partial(basic_ParallelCNNDNN_A,num_of_parallels=20),  \n",
        "                    f\"ParallelCNNDNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    train_data_grey[0,:,:,:].shape, \n",
        "                    train_data_grey, train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,\n",
        "                    verbose_level = 0,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(validation_data_grey, validation_targets))\n",
        "print(timeit.default_timer()-start_time)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KSTlw2v-7RDr"
      },
      "outputs": [],
      "source": [
        "# parallel fuly integrated 10 WideResNet \n",
        "# does not work!!! OOM\n",
        "# pwrs10, pwrs10h = compile_and_fit_model_basic( basic_ParallelWideResNet2810_A,  \n",
        "#                     f\"ParallelWideResNet_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "#                     train_data_grey[0,:,:,:].shape, \n",
        "#                     train_data_grey, train_targets,\n",
        "#                     save_max_epoch=False,\n",
        "#                     save_final=True,\n",
        "#                     patience_count = 35,\n",
        "#                     early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "#                     log_history = True,\n",
        "#                     verbose_level = 1,                             \n",
        "#                     batch_size=512, \n",
        "#                     epochs=250, \n",
        "#                     class_weight=None, \n",
        "#                     validation_data=(validation_data_grey, validation_targets))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SRolnh9r_bcK"
      },
      "outputs": [],
      "source": [
        "plot_history(pdnn20h)\n",
        "plot_history(pcnn20h)\n",
        "plot_history(pdcnn20h)\n",
        "plot_history(pwrs10h)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eDrWFkawP38Q"
      },
      "source": [
        " the summary results for the fully parallel models"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KWHWnbJj_2kH"
      },
      "outputs": [],
      "source": [
        "x_input = test_data_grey\n",
        "y_input = test_targets\n",
        "\n",
        "models_to_use = [pdnn20, pcnn20, pdcnn20, pwrs10]\n",
        "model_predictions = [ model.predict(x_input) for model in models_to_use]\n",
        "\n",
        "y_preds = [ np.apply_along_axis(np.argmax, 1, y_pred_model) for y_pred_model in model_predictions ] \n",
        "scores_pdnn20_pcnn20_pdcnn20 = [ pr_rc_f1_acc_from_supplied(y_pred,y_input) for y_pred in y_preds ]\n",
        "\n",
        "np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/full_parallel_dnn20_cnn20_cnn10dnn10_wrs10_summary_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "        np.array(scores_pdnn20_pcnn20_pdcnn20), \n",
        "        allow_pickle=True, \n",
        "        fix_imports=True)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pE8Hb3aiWxjG"
      },
      "outputs": [],
      "source": [
        "scores_pdnn20_pcnn20_pdcnn20"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1_Os-eyfeYfE"
      },
      "source": [
        "# Functions for collaborative learning (no GP) based on last dense and softmax layer features for DNN/CNN/WideResNet inputs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bDIJ2mhv525M"
      },
      "source": [
        "A sequential model for DNN on collaborative features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "giePq9Q2189i",
        "outputId": "758212f8-deb4-4798-a8a1-df5a0b836f05"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 14.6 ms (started: 2022-01-10 16:35:47 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def model_combination_of_features(model_name, input_shape,num_classes=10):\n",
        "    model = Sequential([\n",
        "        tf.keras.Input(shape=input_shape),\n",
        "        BatchNormalization(),\n",
        "        Dense(256, kernel_initializer='RandomNormal', bias_initializer='zeros'),\n",
        "        LeakyReLU(),\n",
        "        Dropout(0.6),\n",
        "        Dense(128, kernel_initializer='RandomNormal', bias_initializer='zeros', kernel_regularizer = tf.keras.regularizers.l1(1e-3)),\n",
        "        LeakyReLU(),\n",
        "        Dropout(0.6),\n",
        "        Dense(32, kernel_initializer='RandomNormal', bias_initializer='zeros', kernel_regularizer = tf.keras.regularizers.l1(1e-2)),\n",
        "        LeakyReLU(),\n",
        "        Dropout(0.5),\n",
        "        Dense(num_classes, activation='softmax')\n",
        "    ], name=model_name)\n",
        "    return model\n",
        "\n",
        "\n",
        "def model_combination_of_features_with_flatten(model_name, input_shape,num_classes=10):\n",
        "    model = Sequential([\n",
        "        tf.keras.Input(shape=input_shape),\n",
        "        Flatten(),\n",
        "        BatchNormalization(),\n",
        "        Dense(256, kernel_initializer='RandomNormal', bias_initializer='zeros'),\n",
        "        LeakyReLU(),\n",
        "        Dropout(0.6),\n",
        "        Dense(128, kernel_initializer='RandomNormal', bias_initializer='zeros', kernel_regularizer = tf.keras.regularizers.l1(1e-3)),\n",
        "        LeakyReLU(),\n",
        "        Dropout(0.6),\n",
        "        Dense(32, kernel_initializer='RandomNormal', bias_initializer='zeros', kernel_regularizer = tf.keras.regularizers.l1(1e-2)),\n",
        "        LeakyReLU(),\n",
        "        Dropout(0.5),\n",
        "        Dense(num_classes, activation='softmax')\n",
        "    ], name=model_name)\n",
        "    return model    \n",
        "\n",
        "\n",
        "# X_trains_out.append(np.array(features_model.predict(X_train_combined), dtype='float64'))\n",
        "# X_train_new = np.concatenate(tuple(X_trains_out), axis=1)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rV8G8UWixd_l",
        "outputId": "1b4247e9-53da-4c67-cacb-77f7a0672cd7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 8.27 ms (started: 2022-01-10 16:35:47 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def get_features_from_multiple_models(feature_files, axis_to_concat = 1):\n",
        " \n",
        "  validation_features_to_load = [ff for ff in feature_files if \"Validation\" in ff and \"_X\" in ff ]\n",
        "  np_x_validation_collab = np.array([np.load(ff) for ff in validation_features_to_load])\n",
        "  np_x_validation_collab = np.concatenate(np_x_validation_collab, axis=axis_to_concat)\n",
        "\n",
        "  train_features_to_load = [ff.replace(\"Validation\", \"Train\") for ff in validation_features_to_load]\n",
        "  np_x_train_collab = np.array([np.load(ff) for ff in train_features_to_load])\n",
        "  np_x_train_collab = np.concatenate(np_x_train_collab, axis=axis_to_concat)\n",
        "\n",
        "  test_features_to_load = [ff.replace(\"Validation\", \"Test\") for ff in validation_features_to_load]\n",
        "  np_x_test_collab = np.array([np.load(ff) for ff in test_features_to_load])\n",
        "  np_x_test_collab = np.concatenate(np_x_test_collab, axis=axis_to_concat)\n",
        "\n",
        "  return np_x_validation_collab, np_x_train_collab, np_x_test_collab"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0PaQyNjIsR97",
        "outputId": "e3704c08-e970-4638-c418-5c8929402237"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 11.2 ms (started: 2022-01-10 16:35:47 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def get_features_for_layer(feature_files, layer_name, num_of_models, axis_to_concat = 1):\n",
        " \n",
        "  feature_files_used = [ff for ff in feature_files if \"Validation\" in ff and \"_X\" in ff and f\"{layer_name}_\" in ff ]\n",
        "  validation_features_to_load = sorted(random.sample(feature_files_used, min(num_of_models,len(feature_files_used)) ))\n",
        "  np_x_validation_collab = np.array([np.load(ff) for ff in validation_features_to_load])\n",
        "  np_x_validation_collab = np.concatenate(np_x_validation_collab, axis=axis_to_concat)\n",
        "\n",
        "  train_features_to_load = [ff.replace(\"Validation\", \"Train\") for ff in validation_features_to_load]\n",
        "  np_x_train_collab = np.array([np.load(ff) for ff in train_features_to_load])\n",
        "  np_x_train_collab = np.concatenate(np_x_train_collab, axis=axis_to_concat)\n",
        "\n",
        "  test_features_to_load = [ff.replace(\"Validation\", \"Test\") for ff in validation_features_to_load]\n",
        "  np_x_test_collab = np.array([np.load(ff) for ff in test_features_to_load])\n",
        "  np_x_test_collab = np.concatenate(np_x_test_collab, axis=axis_to_concat)\n",
        "\n",
        "  return np_x_validation_collab, np_x_train_collab, np_x_test_collab\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7IhTGp4wUo7T"
      },
      "source": [
        "# Collaborative learning on DNN models based on last dense and softmax layer features"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YqPlplwbEhW2"
      },
      "source": [
        "Do a collaborative layer on 20 DNN on features of penultimate layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fvMFC5GHoFM3",
        "outputId": "79bd6aa4-6860-4b71-fcb6-9d2d964e033f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/Collab_D3R_DNN_A_20220110153153_saved_model_after_fit/assets\n",
            "60.19611946000077\n",
            "time: 2min 8s (started: 2022-01-10 15:30:45 +00:00)\n"
          ]
        }
      ],
      "source": [
        "# set up the data for the DNN collaborative\n",
        "\n",
        "# dnn_features_files\n",
        "# cnn_features_files\n",
        "\n",
        "import random\n",
        "import datetime\n",
        "\n",
        "num_of_models = 20\n",
        "layer_name_penultimate_dnn = \"D3R\"\n",
        "\n",
        "np_x_validation_collab_dnn_penultimate, np_y_train_collab_dnn_penultimate, np_x_test_collab_dnn_penultimate = get_features_for_layer(dnn_features_files, layer_name_penultimate_dnn, num_of_models)\n",
        "\n",
        "start_time = timeit.default_timer()\n",
        "fcd1, fcdh1 = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "                    f\"Collab_{layer_name_penultimate_dnn}_DNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    np_y_train_collab_dnn_penultimate[0,:].shape, \n",
        "                    np_y_train_collab_dnn_penultimate, \n",
        "                    train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,\n",
        "                    verbose_level=0,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(np_x_validation_collab_dnn_penultimate, validation_targets))\n",
        "                    # validation_data=(test_data_grey, test_targets))\n",
        "print(timeit.default_timer()-start_time)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DU7j_6cWhxVC"
      },
      "source": [
        "Do a collaborative layer on 20 DNN on features of softmax layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ue64ByTEhxVD",
        "outputId": "e6047c3f-8fb9-432d-c573-fa6337b309e9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/Collab_SFTMX1_DNN_A_20220110153631_saved_model_after_fit/assets\n",
            "52.93023701000129\n",
            "time: 1min 41s (started: 2022-01-10 15:35:42 +00:00)\n"
          ]
        }
      ],
      "source": [
        "# set up the data for the DNN collaborative\n",
        "\n",
        "import random\n",
        "\n",
        "num_of_models = 20\n",
        "layer_name_sfmx_dnn = \"SFTMX1\"\n",
        "\n",
        "np_x_validation_collab_dnn_sftmx, np_y_train_collab_dnn_sftmx, np_x_test_collab_dnn_sftmx = get_features_for_layer(dnn_features_files, layer_name_sfmx_dnn, num_of_models)\n",
        "\n",
        "start_time = timeit.default_timer()\n",
        "fcd2, fcdh2 = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "                    f\"Collab_{layer_name_sfmx_dnn}_DNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    np_y_train_collab_dnn_sftmx[0,:].shape, \n",
        "                    np_y_train_collab_dnn_sftmx, \n",
        "                    train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,\n",
        "                    verbose_level=0,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(np_x_validation_collab_dnn_sftmx, validation_targets))\n",
        "print(timeit.default_timer()-start_time)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mIJt0f9ZVIm5"
      },
      "source": [
        "# Collaborative learning on CNN models based on last dense and softmax layer features"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-orJHkR41Kql"
      },
      "source": [
        "Do a collaborative layer on 20 CNN of features of penultimate layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uOE0m4YR1Kqq",
        "outputId": "650a5b04-e6bf-4a15-fd6d-5919cd9fe83b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/Collab_DRP1_CNN_A_20220110153945_saved_model_after_fit/assets\n",
            "79.79652630700002\n",
            "time: 2min 52s (started: 2022-01-10 15:38:13 +00:00)\n"
          ]
        }
      ],
      "source": [
        "# set up the data for the CNN collaborative\n",
        "\n",
        "# dnn_features_files\n",
        "# cnn_features_files\n",
        "\n",
        "import random\n",
        "\n",
        "num_of_models = 20\n",
        "layer_name_penultimate_cnn = \"DRP1\"\n",
        "\n",
        "np_x_validation_collab_cnn_penultimate, np_y_train_collab_cnn_penultimate, np_x_test_collab_cnn_penultimate = get_features_for_layer(cnn_features_files, layer_name_penultimate_cnn, num_of_models)\n",
        "\n",
        "start_time = timeit.default_timer()\n",
        "fcc1, fcch1 = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "                    f\"Collab_{layer_name_penultimate_cnn}_CNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    np_y_train_collab_cnn_penultimate[0,:].shape, \n",
        "                    np_y_train_collab_cnn_penultimate, \n",
        "                    train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,\n",
        "                    verbose_level=0,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(np_x_validation_collab_cnn_penultimate, validation_targets))\n",
        "print(timeit.default_timer()-start_time)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MEetGroZmRtA"
      },
      "source": [
        "Do a collaborative layer on 20 CNN of features of softmax layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tIPxhqdImRtA",
        "outputId": "41cae43d-163e-40fb-da54-f369e52150e7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/Collab_SFTMX1_CNN_A_20220110154200_saved_model_after_fit/assets\n",
            "24.77354120000018\n",
            "time: 1min 20s (started: 2022-01-10 15:41:05 +00:00)\n"
          ]
        }
      ],
      "source": [
        "# set up the data for the CNN collaborative\n",
        "\n",
        "# dnn_features_files\n",
        "# cnn_features_files\n",
        "\n",
        "import random\n",
        "\n",
        "num_of_models = 20\n",
        "layer_name_sftmx_cnn = \"SFTMX1\"\n",
        "\n",
        "np_x_validation_collab_cnn_sftmx, np_y_train_collab_cnn_sftmx, np_x_test_collab_cnn_sftmx =  get_features_for_layer(cnn_features_files, layer_name_sftmx_cnn, num_of_models)\n",
        "\n",
        "start_time = timeit.default_timer()\n",
        "fcc2, fcch2 = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "                    f\"Collab_{layer_name_sftmx_cnn}_CNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    np_y_train_collab_cnn_sftmx[0,:].shape, \n",
        "                    np_y_train_collab_cnn_sftmx, \n",
        "                    train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,\n",
        "                    verbose_level=0,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(np_x_validation_collab_cnn_sftmx, validation_targets))\n",
        "print(timeit.default_timer()-start_time)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tgih_EEjVMdQ"
      },
      "source": [
        "# Collaborative learning on CNN & DNN models based on last dense and softmax layer features"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mwv8LJSc2Jyh"
      },
      "source": [
        "Do a collaborative layer on 10 CNN and 10 DNN on features of the penultimate layer\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "updulQYF2Jyh",
        "outputId": "679418bb-c140-4b8a-c3cd-e200c73792d9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 58 s (started: 2022-01-10 16:08:38 +00:00)\n"
          ]
        }
      ],
      "source": [
        "# set up the data for the CNN+DNN collaborative on last dense layer\n",
        "\n",
        "import random\n",
        "\n",
        "num_of_models = 20\n",
        "\n",
        "layer_name_penultimate_dnn = \"D3R\"\n",
        "layer_name_penultimate_cnn = \"DRP1\"\n",
        "\n",
        "validation_features_to_load_cnn = sorted(random.sample([ff for ff in cnn_features_files if \"Validation_X.npy\" in ff and f\"_{layer_name_penultimate_cnn}_\" in ff], int(num_of_models/2)))\n",
        "validation_features_to_load_dnn  = sorted(random.sample([ff for ff in dnn_features_files if \"Validation_X.npy\" in ff and f\"_{layer_name_penultimate_dnn}_\" in ff], int(num_of_models/2)))\n",
        "\n",
        "validation_features_cnn = np.array([np.load(ff) for ff in validation_features_to_load_cnn ])\n",
        "validation_features_dnn = np.array([np.load(ff) for ff in validation_features_to_load_dnn ])\n",
        "np_x_validation_collab_cdnn = np.concatenate([validation_features_cnn,validation_features_dnn], axis=2)\n",
        "\n",
        "train_features_to_load_cnn = [ff.replace(\"Validation\", \"Train\") for ff in validation_features_to_load_cnn]\n",
        "train_features_to_load_dnn = [ff.replace(\"Validation\", \"Train\") for ff in validation_features_to_load_dnn]\n",
        "train_features_cnn = np.array([np.load(ff) for ff in train_features_to_load_cnn ])\n",
        "train_features_dnn = np.array([np.load(ff) for ff in train_features_to_load_dnn ])\n",
        "np_x_train_collab_cdnn = np.concatenate([train_features_cnn,train_features_dnn], axis=2)\n",
        "\n",
        "test_features_to_load_cnn = [ff.replace(\"Validation\", \"Test\") for ff in validation_features_to_load_cnn]\n",
        "test_features_to_load_dnn = [ff.replace(\"Validation\", \"Test\") for ff in validation_features_to_load_dnn]\n",
        "test_features_cnn = np.array([np.load(ff) for ff in test_features_to_load_cnn ])\n",
        "test_features_dnn = np.array([np.load(ff) for ff in test_features_to_load_dnn ])\n",
        "np_x_test_collab_cdnn_penultimate = np.concatenate([test_features_cnn,test_features_dnn], axis=2)\n",
        "\n",
        "\n",
        "np_x_train_collab_cdnn = np.swapaxes(np_x_train_collab_cdnn, 0,1)\n",
        "np_x_validation_collab_cdnn = np.swapaxes(np_x_validation_collab_cdnn, 0,1)\n",
        "np_x_test_collab_cdnn_penultimate = np.swapaxes(np_x_test_collab_cdnn_penultimate, 0,1)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7mbiopJ_2Jyi",
        "outputId": "a4faa4b3-b432-4b18-a8c3-7c0f890101c9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/Collab_DRP1_D3R_DCNN_A_20220110160938_saved_model_after_fit/assets\n",
            "65.51908587999787\n",
            "time: 1min 5s (started: 2022-01-10 16:09:38 +00:00)\n"
          ]
        }
      ],
      "source": [
        "layer_name = \"DRP1_D3R\"\n",
        "\n",
        "start_time = timeit.default_timer()\n",
        "fccd1, fccdh1 = compile_and_fit_model_basic( model_combination_of_features_with_flatten,  \n",
        "                    f\"Collab_{layer_name}_DCNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    np_x_train_collab_cdnn[0,:].shape, \n",
        "                    np_x_train_collab_cdnn, \n",
        "                    train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,\n",
        "                    verbose_level=0,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(np_x_validation_collab_cdnn, validation_targets))\n",
        "print(timeit.default_timer()-start_time)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rqlhpJSF932v"
      },
      "source": [
        "Do a collaborative layer on 10 CNN and 10 DNN on features of the softmax layer\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7PeDS353932v",
        "outputId": "f31e14f5-f046-4a39-c753-a619805afe7f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 36.1 s (started: 2022-01-10 16:10:43 +00:00)\n"
          ]
        }
      ],
      "source": [
        "# set up the data for the CNN+DNN collaborative on softmax layer\n",
        "\n",
        "import random\n",
        "\n",
        "num_of_models = 20\n",
        "\n",
        "layer_name_sftmx_dnn = \"SFTMX1\"\n",
        "layer_name_sftmx_cnn = \"SFTMX1\"\n",
        "\n",
        "validation_features_to_load_cnn = sorted(random.sample([ff for ff in cnn_features_files if \"Validation_X.npy\" in ff and f\"_{layer_name_sftmx_dnn}_\" in ff], int(num_of_models/2)))\n",
        "validation_features_to_load_dnn  = sorted(random.sample([ff for ff in dnn_features_files if \"Validation_X.npy\" in ff and f\"_{layer_name_sftmx_cnn}_\" in ff], int(num_of_models/2)))\n",
        "\n",
        "validation_features_cnn = np.array([np.load(ff) for ff in validation_features_to_load_cnn ])\n",
        "validation_features_dnn = np.array([np.load(ff) for ff in validation_features_to_load_dnn ])\n",
        "np_x_validation_collab_cdnn = np.concatenate([validation_features_cnn,validation_features_dnn], axis=2)\n",
        "\n",
        "train_features_to_load_cnn = [ff.replace(\"Validation\", \"Train\") for ff in validation_features_to_load_cnn]\n",
        "train_features_to_load_dnn = [ff.replace(\"Validation\", \"Train\") for ff in validation_features_to_load_dnn]\n",
        "train_features_cnn = np.array([np.load(ff) for ff in train_features_to_load_cnn ])\n",
        "train_features_dnn = np.array([np.load(ff) for ff in train_features_to_load_dnn ])\n",
        "np_x_train_collab_cdnn = np.concatenate([train_features_cnn,train_features_dnn], axis=2)\n",
        "\n",
        "test_features_to_load_cnn = [ff.replace(\"Validation\", \"Test\") for ff in validation_features_to_load_cnn]\n",
        "test_features_to_load_dnn = [ff.replace(\"Validation\", \"Test\") for ff in validation_features_to_load_dnn]\n",
        "test_features_cnn = np.array([np.load(ff) for ff in test_features_to_load_cnn ])\n",
        "test_features_dnn = np.array([np.load(ff) for ff in test_features_to_load_dnn ])\n",
        "np_x_test_collab_cdnn_sftmx = np.concatenate([test_features_cnn,test_features_dnn], axis=2)\n",
        "\n",
        "\n",
        "np_x_train_collab_cdnn = np.swapaxes(np_x_train_collab_cdnn, 0,1)\n",
        "np_x_validation_collab_cdnn = np.swapaxes(np_x_validation_collab_cdnn, 0,1)\n",
        "np_x_test_collab_cdnn_sftmx = np.swapaxes(np_x_test_collab_cdnn_sftmx, 0,1)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZzSY-SuN932w",
        "outputId": "baf9dd56-ed95-4470-b76c-2c15dae9b308"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/Collab_SFTMX1_SFTMX1_DCNN_A_20220110161119_saved_model_after_fit/assets\n",
            "41.36772378400201\n",
            "time: 41.4 s (started: 2022-01-10 16:11:19 +00:00)\n"
          ]
        }
      ],
      "source": [
        "layer_name = \"SFTMX1_SFTMX1\"\n",
        "\n",
        "start_time = timeit.default_timer()\n",
        "fccd2, fccdh2 = compile_and_fit_model_basic( model_combination_of_features_with_flatten,  \n",
        "                    f\"Collab_{layer_name}_DCNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    np_x_train_collab_cdnn[0,:].shape, \n",
        "                    np_x_train_collab_cdnn, \n",
        "                    train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,\n",
        "                    verbose_level=0,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(np_x_validation_collab_cdnn, validation_targets))\n",
        "print(timeit.default_timer()-start_time)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eDq_scmJVVLP"
      },
      "source": [
        "# Collaborative learning on WideResNet models based on last dense (not possible) and softmax layer features"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K7EjdLpd2sXO"
      },
      "source": [
        "Do a collaborative layer on 10 WideResNet28-10 penultimate layer features\n",
        "\n",
        "Not possible to run because of memory constraints "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A10At6zo1iNq"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5NX-FaO4OmYX"
      },
      "source": [
        "Do a collaborative layer on 10 WideResNet28-10 softmax layer features\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-JhGM1AFPobp",
        "outputId": "fa54d959-984a-4136-a316-d9814622a34e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 5.79 ms (started: 2022-01-10 16:32:49 +00:00)\n"
          ]
        }
      ],
      "source": [
        "import random\n",
        "import re\n",
        "\n",
        "def get_base_patterns_for_validation(features_files, layer_name = \"CLASSIFIER_D1\", model_type = \"WideResNet\"):\n",
        "  base_patterns_for_validations = []\n",
        "  for ff in [ s for s in features_files if \"Validation\" in s]:\n",
        "    validation_search = re.search(f'^.*({model_type}.*_ID.*features_{layer_name}_Validation)[0-9]+.*$', ff, re.IGNORECASE)\n",
        "    if validation_search:\n",
        "        base_patterns_for_validations.append(validation_search.group(1))\n",
        "\n",
        "  base_patterns_for_validations = sorted(list(set(base_patterns_for_validations)))\n",
        "  return base_patterns_for_validations\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LBpPvn3SS8MW",
        "outputId": "93d038f3-6c20-4ccf-ad33-883077573404"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 34.4 ms (started: 2022-01-10 16:32:52 +00:00)\n"
          ]
        }
      ],
      "source": [
        "def get_features_for_layer_using_subbatches(feature_files, layer_name, num_of_models, model_type=\"WideResNet\", axis_to_concat = 1):\n",
        " \n",
        "  feature_files_used = [ff for ff in feature_files if \"Validation\" in ff and \"_X\" in ff and f\"_{layer_name}_\" in ff ]\n",
        "  \n",
        "  # base_patterns_for_validations = []\n",
        "  # for ff in [ s for s in feature_files_used ]:\n",
        "  #   validation_search = re.search(f'^.*(_ID.*features_{layer_name}_Validation)[0-9]+.*$', ff, re.IGNORECASE)\n",
        "  #   if validation_search:\n",
        "  #       base_patterns_for_validations.append(validation_search.group(1))\n",
        "  # base_patterns_for_validations = sorted(list(set(base_patterns_for_validations)))\n",
        "\n",
        "  base_patterns_for_validations = get_base_patterns_for_validation(feature_files_used, layer_name, model_type)\n",
        "  base_patterns_for_validations = sorted(random.sample(base_patterns_for_validations, min(num_of_models,len(base_patterns_for_validations)) ))\n",
        "\n",
        "  np_x_validation_collab = None\n",
        "  for base_val_str in base_patterns_for_validations:\n",
        "      validation_batch_files = sorted([ ff for ff in feature_files_used if base_val_str in ff])\n",
        "      np_x_validation_collab_batch = np.array([np.load(ff) for ff in validation_batch_files])\n",
        "      np_x_validation_collab_batch = np.concatenate(np_x_validation_collab_batch, axis=0)\n",
        "      if np_x_validation_collab is None:\n",
        "        np_x_validation_collab = np_x_validation_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_validation_collab = np.concatenate([np_x_validation_collab, np_x_validation_collab_batch], axis=axis_to_concat)\n",
        "  \n",
        "  np_x_test_collab = None\n",
        "  for base_val_str in [ ff.replace(\"Validation\",\"Test\") for ff in base_patterns_for_validations]:\n",
        "      test_batch_files = dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{base_val_str}.*_X.*$\")\n",
        "      np_x_test_collab_batch = np.array([np.load(ff) for ff in test_batch_files])\n",
        "      np_x_test_collab_batch = np.concatenate(np_x_test_collab_batch, axis=0)\n",
        "      if np_x_test_collab is None:\n",
        "        np_x_test_collab = np_x_test_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_test_collab = np.concatenate([np_x_test_collab, np_x_test_collab_batch], axis=axis_to_concat)\n",
        "\n",
        "  np_x_train_collab = None\n",
        "  for base_val_str in [ ff.replace(\"Validation\",\"Train\") for ff in base_patterns_for_validations]:\n",
        "      train_batch_files = dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{base_val_str}.*_X.*$\")\n",
        "      np_x_train_collab_batch = np.array([np.load(ff) for ff in train_batch_files])\n",
        "      np_x_train_collab_batch = np.concatenate(np_x_train_collab_batch, axis=0)\n",
        "      if np_x_train_collab is None:\n",
        "        np_x_train_collab = np_x_train_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_train_collab = np.concatenate([np_x_train_collab, np_x_train_collab_batch], axis=axis_to_concat)\n",
        "\n",
        "  return np_x_validation_collab, np_x_train_collab, np_x_test_collab"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nTpZ0n_PK6XR",
        "outputId": "36fb949d-8608-45d1-b5c3-d693b8ae7286"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:18: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:38: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/Collab_CLASSIFIER_D1_WideResnet28-10_A_20220110163605_saved_model_after_fit/assets\n",
            "22.104595387998415\n",
            "time: 31.6 s (started: 2022-01-10 16:35:56 +00:00)\n"
          ]
        }
      ],
      "source": [
        "import random\n",
        "import re\n",
        "\n",
        "num_of_models = 10\n",
        "layer_name = \"CLASSIFIER_D1\"\n",
        "model_type = \"WideResNet\"\n",
        "\n",
        "np_x_validation_collab_wideresnets, np_x_train_collab_wideresnets, np_x_test_collab_wideresnets =  get_features_for_layer_using_subbatches(wideresnets_features_files,layer_name,num_of_models,model_type )\n",
        "\n",
        "start_time = timeit.default_timer()\n",
        "wrsftmx, wrsftmxh = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "                    f\"Collab_{layer_name}_WideResnet28-10_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    np_x_train_collab_wideresnets[0,:].shape, \n",
        "                    np_x_train_collab_wideresnets, \n",
        "                    train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,\n",
        "                    verbose_level=0,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(np_x_validation_collab_wideresnets, validation_targets))\n",
        "print(timeit.default_timer()-start_time)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YlyIJ3CCVmCq"
      },
      "source": [
        "# Collaborative learning on WideResNet, CNN & DNN models based on softmax layer features"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WX8Z0L0QyUQb"
      },
      "source": [
        "Do a collaborative layer on 1 DNN and 4 CNN, 5 WideResNets on features of the softmax layer\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PlbJZSX9yUQb",
        "outputId": "f9f5571b-3084-4cb0-e11e-3107fe7c14e3"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:18: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:38: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 11.4 s (started: 2022-01-10 16:38:01 +00:00)\n"
          ]
        }
      ],
      "source": [
        "# set up the data for the WideResnet+CNN+DNN collaborative on softmax layer\n",
        "\n",
        "import random\n",
        "\n",
        "num_of_models_dnn = 1\n",
        "num_of_models_cnn = 4\n",
        "num_of_models_wideresnets = 5\n",
        "\n",
        "layer_name_sftmx_dnn = \"SFTMX1\"\n",
        "layer_name_sftmx_cnn = \"SFTMX1\"\n",
        "layer_name_sftmx_wideresnets = \"CLASSIFIER_D1\"\n",
        "\n",
        "np_x_validation_collab_dnn, np_x_train_collab_dnn, np_x_test_collab_dnn = get_features_for_layer(dnn_features_files,layer_name_sftmx_dnn,num_of_models_dnn)\n",
        "np_x_validation_collab_cnn, np_x_train_collab_cnn, np_x_test_collab_cnn = get_features_for_layer(cnn_features_files,layer_name_sftmx_cnn,num_of_models_cnn)\n",
        "np_x_validation_collab_wideresnets, np_x_train_collab_wideresnets, np_x_test_collab_wideresnets = get_features_for_layer_using_subbatches(wideresnets_features_files,layer_name_sftmx_wideresnets,num_of_models_wideresnets,model_type=\"WideResNet\")\n",
        "\n",
        "\n",
        "np_x_train_collab_wrcdnn = np.concatenate([np_x_train_collab_dnn,np_x_train_collab_cnn,np_x_train_collab_wideresnets], axis=1)\n",
        "np_x_validation_collab_wrcdnn = np.concatenate([np_x_validation_collab_dnn,np_x_validation_collab_cnn,np_x_validation_collab_wideresnets], axis=1)\n",
        "np_x_test_collab_wrcdnn = np.concatenate([np_x_test_collab_dnn,np_x_test_collab_cnn,np_x_test_collab_wideresnets], axis=1)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1aX3tTiDyUQc",
        "outputId": "4107aa96-a115-4b34-fb18-d2ac9223c4d8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/Collab_SFTMX1_SFTMX1_CLASSIFIER_D1_DCNNWRNTS_A_20220110163814_saved_model_after_fit/assets\n",
            "21.534225131999847\n",
            "time: 21.5 s (started: 2022-01-10 16:38:14 +00:00)\n"
          ]
        }
      ],
      "source": [
        "layer_name = \"SFTMX1_SFTMX1_CLASSIFIER_D1\"\n",
        "\n",
        "start_time = timeit.default_timer()\n",
        "wrsdcnnsftmx, wrsdcnnsftmxh = compile_and_fit_model_basic( model_combination_of_features_with_flatten,  \n",
        "                    f\"Collab_{layer_name}_DCNNWRNTS_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    np_x_train_collab_wrcdnn[0,:].shape, \n",
        "                    np_x_train_collab_wrcdnn, \n",
        "                    train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,\n",
        "                    verbose_level=0,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(np_x_validation_collab_wrcdnn, validation_targets))\n",
        "print(timeit.default_timer()-start_time)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hj3nBkOG0gzh"
      },
      "source": [
        "Do a collaborative layer on 2 DNN and 8 CNN, 10 WideResNets on features of the softmax layer\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nVTtEzUn0gzh",
        "outputId": "309ff8f5-f4bc-4cc0-8e39-1abbe5f18e86"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:18: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:38: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 12.2 s (started: 2022-01-10 16:39:15 +00:00)\n"
          ]
        }
      ],
      "source": [
        "# set up the data for the WideResnet+CNN+DNN collaborative on softmax layer\n",
        "\n",
        "import random\n",
        "\n",
        "num_of_models_dnn = 2\n",
        "num_of_models_cnn = 8\n",
        "num_of_models_wideresnets = 10\n",
        "\n",
        "layer_name_sftmx_dnn = \"SFTMX1\"\n",
        "layer_name_sftmx_cnn = \"SFTMX1\"\n",
        "layer_name_sftmx_wideresnets = \"CLASSIFIER_D1\"\n",
        "\n",
        "np_x_validation_collab_dnn, np_x_train_collab_dnn, np_x_test_collab_dnn = get_features_for_layer(dnn_features_files,layer_name_sftmx_dnn,num_of_models_dnn)\n",
        "np_x_validation_collab_cnn, np_x_train_collab_cnn, np_x_test_collab_cnn = get_features_for_layer(cnn_features_files,layer_name_sftmx_cnn,num_of_models_cnn)\n",
        "np_x_validation_collab_wideresnets, np_x_train_collab_wideresnets, np_x_test_collab_wideresnets = get_features_for_layer_using_subbatches(wideresnets_features_files,layer_name_sftmx_wideresnets,num_of_models_wideresnets,model_type=\"WideResNet\")\n",
        "\n",
        "\n",
        "np_x_train_collab_wrcdnn = np.concatenate([np_x_train_collab_dnn,np_x_train_collab_cnn,np_x_train_collab_wideresnets], axis=1)\n",
        "np_x_validation_collab_wrcdnn = np.concatenate([np_x_validation_collab_dnn,np_x_validation_collab_cnn,np_x_validation_collab_wideresnets], axis=1)\n",
        "np_x_test_collab_wrcdnn2 = np.concatenate([np_x_test_collab_dnn,np_x_test_collab_cnn,np_x_test_collab_wideresnets], axis=1)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sB1THFzz0gzh",
        "outputId": "a2863ca1-7029-4c04-d682-d8e7f03c0c03"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/Collab_SFTMX1_SFTMX1_CLASSIFIER_D1_DCNNWRNTS_A_20220110163929_saved_model_after_fit/assets\n",
            "21.000002349999704\n",
            "time: 21 s (started: 2022-01-10 16:39:29 +00:00)\n"
          ]
        }
      ],
      "source": [
        "layer_name = \"SFTMX1_SFTMX1_CLASSIFIER_D1\"\n",
        "\n",
        "start_time = timeit.default_timer()\n",
        "wrsdcnnsftmx2, wrsdcnnsftmxh2 = compile_and_fit_model_basic( model_combination_of_features_with_flatten,  \n",
        "                    f\"Collab_{layer_name}_DCNNWRNTS_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    np_x_train_collab_wrcdnn[0,:].shape, \n",
        "                    np_x_train_collab_wrcdnn, \n",
        "                    train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,\n",
        "                    verbose_level=0,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(np_x_validation_collab_wrcdnn, validation_targets))\n",
        "print(timeit.default_timer()-start_time)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FIWn1BVd_y3F"
      },
      "source": [
        "Do a collaborative layer on 5 CNN, 5 WideResNets on features of the softmax layer\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CuPo4NEK_y3G",
        "outputId": "85ad5d2b-b3d0-475d-a554-be7a9ebe000f"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:18: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:38: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "time: 5.71 s (started: 2022-01-10 16:43:51 +00:00)\n"
          ]
        }
      ],
      "source": [
        "# set up the data for the WideResnet+CNN+DNN collaborative on softmax layer\n",
        "\n",
        "import random\n",
        "\n",
        "num_of_models_cnn = 5\n",
        "num_of_models_wideresnets = 5\n",
        "\n",
        "layer_name_sftmx_cnn = \"SFTMX1\"\n",
        "layer_name_sftmx_wideresnets = \"CLASSIFIER_D1\"\n",
        "\n",
        "np_x_validation_collab_cnn, np_x_train_collab_cnn, np_x_test_collab_cnn = get_features_for_layer(cnn_features_files,layer_name_sftmx_cnn,num_of_models_cnn)\n",
        "np_x_validation_collab_wideresnets, np_x_train_collab_wideresnets, np_x_test_collab_wideresnets = get_features_for_layer_using_subbatches(wideresnets_features_files,layer_name_sftmx_wideresnets,num_of_models_wideresnets,model_type=\"WideResNet\")\n",
        "\n",
        "np_x_train_collab_wrcnn = np.concatenate([np_x_train_collab_cnn,np_x_train_collab_wideresnets], axis=1)\n",
        "np_x_validation_collab_wrcnn = np.concatenate([np_x_validation_collab_cnn,np_x_validation_collab_wideresnets], axis=1)\n",
        "np_x_test_collab_wrcnn = np.concatenate([np_x_test_collab_cnn,np_x_test_collab_wideresnets], axis=1)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HE2FGKUv_y3G",
        "outputId": "40f609cb-9e88-46c4-bca5-9e24b0e5fee3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/data_papers/gpSVHN/model_finals/Collab_SFTMX1_CLASSIFIER_D1_CNNWRNTS_A_20220110164357_saved_model_after_fit/assets\n",
            "20.701131232999614\n",
            "time: 20.7 s (started: 2022-01-10 16:43:57 +00:00)\n"
          ]
        }
      ],
      "source": [
        "layer_name = \"SFTMX1_CLASSIFIER_D1\"\n",
        "start_time = timeit.default_timer()\n",
        "wrscnnsftmx, wrscnnsftmxh = compile_and_fit_model_basic( model_combination_of_features_with_flatten,  \n",
        "                    f\"Collab_{layer_name}_CNNWRNTS_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                    np_x_train_collab_wrcnn[0,:].shape, \n",
        "                    np_x_train_collab_wrcnn, \n",
        "                    train_targets,\n",
        "                    save_max_epoch=False,\n",
        "                    save_final=True,\n",
        "                    patience_count = 35,\n",
        "                    early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                    log_history = True,\n",
        "                    verbose_level=0,                             \n",
        "                    batch_size=512, \n",
        "                    epochs=250, \n",
        "                    class_weight=None, \n",
        "                    validation_data=(np_x_validation_collab_wrcnn, validation_targets))\n",
        "print(timeit.default_timer()-start_time)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "upPlyiuGnrbK"
      },
      "source": [
        "# A summary results for all collaborative models on the last layer features (DNN 20, CNN 20, CNN10/DNN10, WideResnet, CNN/DNN/WideReset, CNN/WideResNet) using a DNN to collaborate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TBpiSG03piyX"
      },
      "outputs": [],
      "source": [
        "models_to_use_collab_dnnf = [fcd1, fcd2, fcc1, fcc2, fccd1, fccd2, wrsftmx, wrsdcnnsftmx, wrsdcnnsftmx2, wrscnnsftmx]\n",
        "model_predictions_collab_dnnf = [ fcd1.predict(np_x_test_collab_dnn_penultimate), \n",
        "                                  fcd2.predict(np_x_test_collab_dnn_sftmx), \n",
        "                                  fcc1.predict(np_x_test_collab_cnn_penultimate), \n",
        "                                  fcc2.predict(np_x_test_collab_cnn_sftmx), \n",
        "                                  fccd1.predict(np_x_test_collab_cdnn_penultimate),\n",
        "                                  fccd2.predict(np_x_test_collab_cdnn_sftmx),\n",
        "                                  wrsftmx.predict(np_x_test_collab_wideresnets),\n",
        "                                  wrsdcnnsftmx.predict(np_x_test_collab_wrcdnn),\n",
        "                                  wrsdcnnsftmx2.predict(np_x_test_collab_wrcdnn2),\n",
        "                                 wrscnnsftmx.predict(np_x_test_collab_wrcnn)]\n",
        "\n",
        "y_preds_collab_dnnf = [ np.apply_along_axis(np.argmax, 1, y_pred_model) for y_pred_model in model_predictions_collab_dnnf ] \n",
        "scores_fCollabdnnDNN20_fCollabdnnCNN20_fCollabdnnCNN10DNN10 = [ pr_rc_f1_acc_from_supplied(y_pred,test_targets) for y_pred in y_preds_collab_dnnf ]\n",
        "\n",
        "np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/feature_collabUsingDnn_dnn20_cnn20_cnn10dnn10_wideresnet2810_wrs5cnn4dnn1_wrs10cnn8dnn2_wrs5cnn5_summary_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "        np.array(scores_fCollabdnnDNN20_fCollabdnnCNN20_fCollabdnnCNN10DNN10), \n",
        "        allow_pickle=True, \n",
        "        fix_imports=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S91-VXhYXqYE"
      },
      "outputs": [],
      "source": [
        "scores_fCollabdnnDNN20_fCollabdnnCNN20_fCollabdnnCNN10DNN10"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dR603zQv3p5-"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DaQvakL4rU88"
      },
      "source": [
        "# Load CCA/PCA Data for Collaborative CNN/DNN models"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_SAYfptJrN-V"
      },
      "outputs": [],
      "source": [
        "# cframe.loc[cframe.CCA1==1.0][\"f1\"]\n",
        "\n",
        "import os,sys\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "dnn_distresults = pd.read_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/dnn_Validation_ccs_20211115040852.csv\")\n",
        "cnn_distresults = pd.read_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_ccs/cnn_Validation_ccs_20211115184258.csv\")\n",
        "\n",
        "dnn_distresults = dnn_distresults[dnn_distresults.Type==\"DNN\"]\n",
        "# individual_dnns = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/individual_dnn_summary_20211014152626.npy\")\n",
        "# individual_cnns = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/individual_cnn_summary_20211014152108.npy\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3m5dSqo9rN-V"
      },
      "outputs": [],
      "source": [
        "# cnn_distresults.sort_values(\"CCA1\")\n",
        "# cnn_distresults[cnn_distresults.Layer == \"SFTMX1\"].sort_values(\"PCA1\")\n",
        "# cnn_distresults[cnn_distresults.Layer == \"SFTMX1\"].sort_values(\"CCA1\")[0:40]\n",
        "# dnn_distresults[dnn_distresults.Layer == \"SFTMX1\"].sort_values(\"CCA1\")\n",
        "\n",
        "cnn_avg_cca = cnn_distresults.groupby([\"Type\",\"Layer\",\"f1\"])[\"CCA1\"].mean().reset_index()\n",
        "dnn_avg_cca = dnn_distresults.groupby([\"Type\",\"Layer\",\"f1\"])[\"CCA1\"].mean().reset_index()\n",
        "cnn_avg_cca = cnn_avg_cca.sort_values('CCA1')\n",
        "dnn_avg_cca = pd.concat([dnn_avg_cca[dnn_avg_cca.CCA1 > 0.45].sort_values('CCA1'), dnn_avg_cca[dnn_avg_cca.CCA1 <= 0.45]])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "L5iV3Im5rN-V"
      },
      "outputs": [],
      "source": [
        "cnn_avg_pca = cnn_distresults.groupby([\"Type\",\"Layer\",\"f1\"])[\"PCA1\"].apply(lambda x: np.mean(np.abs(x))).reset_index()\n",
        "dnn_avg_pca = dnn_distresults.groupby([\"Type\",\"Layer\",\"f1\"])[\"PCA1\"].apply(lambda x: np.mean(np.abs(x))).reset_index()\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TXy-xmor32UY"
      },
      "source": [
        "# Data for Plot on Collaborative DNN softmax layers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-Uk02GgJsT1x"
      },
      "outputs": [],
      "source": [
        "collaborativeFullyC_softmax_data = None\n",
        "\n",
        "x_input = test_data_grey\n",
        "y_input = test_targets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sAUaJo6XFvVI"
      },
      "outputs": [],
      "source": [
        "# dnn_features_files_here = [ f for f in dnn_features_files if layer_name_sfmx_dnn in f]\n",
        "# dnn_features_files_here\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lawIAx1L32UZ"
      },
      "outputs": [],
      "source": [
        "# set up the data for the DNN collaborative\n",
        "\n",
        "import random\n",
        "import datetime\n",
        "\n",
        "num_of_repeats = 4\n",
        "num_of_models = [25,30,35,40,45,50]  #[2,3,4,5,6,8,10,12,14,16,18,20]\n",
        "layer_name_sfmx_dnn = \"SFTMX1\"\n",
        "\n",
        "idxCount = 0 if collaborativeFullyC_softmax_data is None else len(collaborativeFullyC_softmax_data.index)\n",
        "for repc in range(num_of_repeats):\n",
        "  for mc in num_of_models:\n",
        "    model_name = f\"Collab_{layer_name_sfmx_dnn}_DNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\"\n",
        "    np_x_validation_collab_dnn_sftmx, np_y_train_collab_dnn_sftmx, np_x_test_collab_dnn_sftmx = get_features_for_layer(dnn_features_files, layer_name_sfmx_dnn, mc)\n",
        "    model_here, history_here = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "                        model_name, \n",
        "                        np_y_train_collab_dnn_sftmx[0,:].shape, \n",
        "                        np_y_train_collab_dnn_sftmx, \n",
        "                        train_targets,\n",
        "                        save_max_epoch = False,\n",
        "                        save_final = True,\n",
        "                        patience_count = 35,\n",
        "                        early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                        log_history = True,\n",
        "                        verbose_level=0,                             \n",
        "                        batch_size=512, \n",
        "                        epochs=250, \n",
        "                        class_weight=None, \n",
        "                        validation_data=(np_x_validation_collab_dnn_sftmx, validation_targets))\n",
        "    del np_x_validation_collab_dnn_sftmx\n",
        "    del np_y_train_collab_dnn_sftmx\n",
        "    y_pred_model = model_here.predict(np_x_test_collab_dnn_sftmx)\n",
        "    del np_x_test_collab_dnn_sftmx\n",
        "    y_pred = np.apply_along_axis(np.argmax, 1, y_pred_model) \n",
        "    pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_pred,test_targets)\n",
        "    print (mc, repc, pr, rc, f1, acc)\n",
        "    del model_here\n",
        "    del history_here\n",
        "    if collaborativeFullyC_softmax_data is None:\n",
        "      collaborativeFullyC_softmax_data = pd.DataFrame({\"Type\": \"DNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            }, index = [idxCount])\n",
        "    else:\n",
        "      collaborativeFullyC_softmax_data = pd.concat([collaborativeFullyC_softmax_data,\n",
        "                                         pd.DataFrame({\"Type\": \"DNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc\n",
        "                                            }, index = [idxCount])\n",
        "                                         ])\n",
        "    idxCount = idxCount + 1\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0fkz6m8Ly2e3"
      },
      "outputs": [],
      "source": [
        "collaborativeFullyC_softmax_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/dnn_collab_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fh84XOFV4ZZp"
      },
      "outputs": [],
      "source": [
        "dnn_avg_cca"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uLw5jkNprylq"
      },
      "source": [
        "# Data for Plot on Collaborative DNN softmax layers using CCA/PCA "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WYAa6Bd5ro78"
      },
      "outputs": [],
      "source": [
        "collaborativeFullyC_softmax_using_CCA_data = None\n",
        "\n",
        "x_input = test_data_grey\n",
        "y_input = test_targets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WBr8MZZqtcVy"
      },
      "outputs": [],
      "source": [
        "# [f for f in dnn_features_files if \"DNN_A_3_20211002235817_features_SFTMX1\" in f]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dVitC_n-ro78"
      },
      "outputs": [],
      "source": [
        "# set up the data for the DNN collaborative on the softmax using the CCA data\n",
        "\n",
        "import random\n",
        "import datetime\n",
        "\n",
        "num_of_repeats = 1\n",
        "num_of_models = [2,3,4,5,6,8,10,12,14,16,18,20,25,30,35,40,45,50] \n",
        "layer_name_sfmx_dnn = \"SFTMX1\"\n",
        "\n",
        "idxCount = 0 if collaborativeFullyC_softmax_using_CCA_data is None else len(collaborativeFullyC_softmax_using_CCA_data.index)\n",
        "for repc in range(num_of_repeats):\n",
        "  for mc in num_of_models:\n",
        "    model_name = f\"Collab_{layer_name_sfmx_dnn}_DNNCCA_A_{datetime.datetime.now():%Y%m%d%H%M%S}\"\n",
        "    \n",
        "    val_files_used = dnn_avg_cca.loc[(dnn_avg_cca.Layer==layer_name_sfmx_dnn) & (dnn_avg_cca.Type==\"DNN\"),\"f1\"].head(mc)\n",
        "    matches_required = [ vf.split(\"model_features/\")[1].split(\"_Validation\")[0] for vf in val_files_used  ] \n",
        "    feature_files_considered = [ ff for ff in dnn_features_files if any([mu in ff for mu in matches_required]) ]\n",
        "    np_x_validation_collab_dnn_sftmx, np_y_train_collab_dnn_sftmx, np_x_test_collab_dnn_sftmx = get_features_from_multiple_models(feature_files_considered)\n",
        "\n",
        "    model_here, history_here = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "                        model_name, \n",
        "                        np_y_train_collab_dnn_sftmx[0,:].shape, \n",
        "                        np_y_train_collab_dnn_sftmx, \n",
        "                        train_targets,\n",
        "                        save_max_epoch = False,\n",
        "                        save_final = True,\n",
        "                        patience_count = 35,\n",
        "                        early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                        log_history = True,\n",
        "                        verbose_level=0,                             \n",
        "                        batch_size=512, \n",
        "                        epochs=250, \n",
        "                        class_weight=None, \n",
        "                        validation_data=(np_x_validation_collab_dnn_sftmx, validation_targets))\n",
        "    del np_x_validation_collab_dnn_sftmx\n",
        "    del np_y_train_collab_dnn_sftmx\n",
        "    y_pred_model = model_here.predict(np_x_test_collab_dnn_sftmx)\n",
        "    del np_x_test_collab_dnn_sftmx\n",
        "    y_pred = np.apply_along_axis(np.argmax, 1, y_pred_model) \n",
        "    pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_pred,test_targets)\n",
        "    print (mc, repc, pr, rc, f1, acc)\n",
        "    del model_here\n",
        "    del history_here\n",
        "    if collaborativeFullyC_softmax_using_CCA_data is None:\n",
        "      collaborativeFullyC_softmax_using_CCA_data = pd.DataFrame({\"Type\": \"DNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            \"Xtra\" : \"CCAmin\"\n",
        "                                            }, index = [idxCount])\n",
        "    else:\n",
        "      collaborativeFullyC_softmax_using_CCA_data = pd.concat([collaborativeFullyC_softmax_using_CCA_data,\n",
        "                                         pd.DataFrame({\"Type\": \"DNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            \"Xtra\" : \"CCAmin\"\n",
        "                                            }, index = [idxCount])\n",
        "                                         ])\n",
        "    idxCount = idxCount + 1\n",
        "    collaborativeFullyC_softmax_using_CCA_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/temp/dnn_temp_Collab_ccaorder_sftmx_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hEyQeXSLro79"
      },
      "outputs": [],
      "source": [
        "collaborativeFullyC_softmax_using_CCA_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/dnn_collab_ccaorder_sftmx_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ul_SqJXjz_s9"
      },
      "source": [
        "# Data for Plot on Collaborative CNN softmax layers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pdqqNRyFz_s9"
      },
      "outputs": [],
      "source": [
        "# carry this over from DNN if possible (so keep commented out)\n",
        "## collaborativeFullyC_softmax_data = None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BhDBg50vz_s9"
      },
      "outputs": [],
      "source": [
        "# set up the data for the CNN collaborative\n",
        "\n",
        "import random\n",
        "import datetime\n",
        "\n",
        "num_of_repeats = 4\n",
        "num_of_models = [22,25,30,35,40,50]  # [2,3,4,5,6,8,10,12,14,16,18,20]\n",
        "layer_name_sfmx_cnn = \"SFTMX1\"\n",
        "\n",
        "# layer_name_sftmx_cnn = \"SFTMX1\"\n",
        "# layer_name_sftmx_wideresnets = \"CLASSIFIER_D1\"\n",
        "\n",
        "idxCount = 0 if collaborativeFullyC_softmax_data is None else len(collaborativeFullyC_softmax_data.index)\n",
        "for repc in range(num_of_repeats):\n",
        "  for mc in num_of_models:\n",
        "    model_name = f\"Collab_{layer_name_sfmx_cnn}_CNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\"\n",
        "    np_x_validation_collab_cnn_sftmx, np_y_train_collab_cnn_sftmx, np_x_test_collab_cnn_sftmx = get_features_for_layer(cnn_features_files, layer_name_sfmx_cnn, mc)\n",
        "    model_here, history_here = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "                        model_name, \n",
        "                        np_y_train_collab_cnn_sftmx[0,:].shape, \n",
        "                        np_y_train_collab_cnn_sftmx, \n",
        "                        train_targets,\n",
        "                        save_max_epoch = False,\n",
        "                        save_final = True,\n",
        "                        patience_count = 35,\n",
        "                        early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                        log_history = True,\n",
        "                        verbose_level=0,                             \n",
        "                        batch_size=512, \n",
        "                        epochs=250, \n",
        "                        class_weight=None, \n",
        "                        validation_data=(np_x_validation_collab_cnn_sftmx, validation_targets))\n",
        "    del np_x_validation_collab_cnn_sftmx\n",
        "    del np_y_train_collab_cnn_sftmx\n",
        "    y_pred_model = model_here.predict(np_x_test_collab_cnn_sftmx)\n",
        "    del np_x_test_collab_cnn_sftmx\n",
        "    y_pred = np.apply_along_axis(np.argmax, 1, y_pred_model) \n",
        "    pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_pred,test_targets)\n",
        "    print (mc, repc, pr, rc, f1, acc)\n",
        "    del model_here\n",
        "    del history_here\n",
        "    if collaborativeFullyC_softmax_data is None:\n",
        "      collaborativeFullyC_softmax_data = pd.DataFrame({\"Type\": \"CNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            }, index = [idxCount])\n",
        "    else:\n",
        "      collaborativeFullyC_softmax_data = pd.concat([collaborativeFullyC_softmax_data,\n",
        "                                         pd.DataFrame({\"Type\": \"CNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc\n",
        "                                            }, index = [idxCount])\n",
        "                                         ])\n",
        "    idxCount = idxCount + 1\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Xl-IjQYV2-3Q"
      },
      "outputs": [],
      "source": [
        "collaborativeFullyC_softmax_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/cnn_collab_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zv4uI-Rf_P4R"
      },
      "source": [
        "# Data for Plot on Collaborative CNN softmax layers using CCA/PCA "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QthquvcK_P4R"
      },
      "outputs": [],
      "source": [
        "collaborativeCNN_softmax_using_CCA_data = None\n",
        "\n",
        "x_input = test_data_grey\n",
        "y_input = test_targets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ESk9xrs1_P4R"
      },
      "outputs": [],
      "source": [
        "# [f for f in dnn_features_files if \"DNN_A_3_20211002235817_features_SFTMX1\" in f]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "itBFK4Kk_P4R"
      },
      "outputs": [],
      "source": [
        "# set up the data for the DNN collaborative on the softmax using the CCA data\n",
        "\n",
        "import random\n",
        "import datetime\n",
        "\n",
        "num_of_repeats = 1\n",
        "num_of_models = [2,3,4,5,6,8,10,12,14,16,18,20,25,30,35,40] \n",
        "layer_name_sfmx_cnn = \"SFTMX1\"\n",
        "\n",
        "idxCount = 0 if collaborativeCNN_softmax_using_CCA_data is None else len(collaborativeCNN_softmax_using_CCA_data.index)\n",
        "for repc in range(num_of_repeats):\n",
        "  for mc in num_of_models:\n",
        "    model_name = f\"Collab_{layer_name_sfmx_cnn}_CNNCCA_A_{datetime.datetime.now():%Y%m%d%H%M%S}\"\n",
        "    \n",
        "    val_files_used = cnn_avg_cca.loc[(cnn_avg_cca.Layer==layer_name_sfmx_cnn) & (cnn_avg_cca.Type==\"CNN\"),\"f1\"].head(mc)\n",
        "    matches_required = [ vf.split(\"model_features/\")[1].split(\"_Validation\")[0] for vf in val_files_used  ] \n",
        "    feature_files_considered = [ ff for ff in cnn_features_files if any([mu in ff for mu in matches_required]) ]\n",
        "    np_x_validation_collab_cnn_sftmx, np_y_train_collab_cnn_sftmx, np_x_test_collab_cnn_sftmx = get_features_from_multiple_models(feature_files_considered)\n",
        "\n",
        "    model_here, history_here = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "                        model_name, \n",
        "                        np_y_train_collab_cnn_sftmx[0,:].shape, \n",
        "                        np_y_train_collab_cnn_sftmx, \n",
        "                        train_targets,\n",
        "                        save_max_epoch = False,\n",
        "                        save_final = True,\n",
        "                        patience_count = 35,\n",
        "                        early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                        log_history = True,\n",
        "                        verbose_level=0,                             \n",
        "                        batch_size=512, \n",
        "                        epochs=250, \n",
        "                        class_weight=None, \n",
        "                        validation_data=(np_x_validation_collab_cnn_sftmx, validation_targets))\n",
        "    del np_x_validation_collab_cnn_sftmx\n",
        "    del np_y_train_collab_cnn_sftmx\n",
        "    y_pred_model = model_here.predict(np_x_test_collab_cnn_sftmx)\n",
        "    del np_x_test_collab_cnn_sftmx\n",
        "    y_pred = np.apply_along_axis(np.argmax, 1, y_pred_model) \n",
        "    pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_pred,test_targets)\n",
        "    print (mc, repc, pr, rc, f1, acc)\n",
        "    del model_here\n",
        "    del history_here\n",
        "    if collaborativeCNN_softmax_using_CCA_data is None:\n",
        "      collaborativeCNN_softmax_using_CCA_data = pd.DataFrame({\"Type\": \"CNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            \"Xtra\" : \"CCAmin\"\n",
        "                                            }, index = [idxCount])\n",
        "    else:\n",
        "      collaborativeCNN_softmax_using_CCA_data = pd.concat([collaborativeCNN_softmax_using_CCA_data,\n",
        "                                         pd.DataFrame({\"Type\": \"CNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            \"Xtra\" : \"CCAmin\"\n",
        "                                            }, index = [idxCount])\n",
        "                                         ])\n",
        "    idxCount = idxCount + 1\n",
        "    collaborativeCNN_softmax_using_CCA_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/temp/cnn_temp_Collab_ccaorder_sftmx_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b7WJ_0_M_P4S"
      },
      "outputs": [],
      "source": [
        "collaborativeCNN_softmax_using_CCA_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/cnn_collab_ccaorder_sftmx_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pfPdlux63Hqa"
      },
      "source": [
        "# Data for Plot on Collaborative WideResNet softmax layers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BaKSB7u93Hqa"
      },
      "outputs": [],
      "source": [
        "# carry this over from DNN/CNN if possible (so keep commented out)\n",
        "## collaborativeFullyC_softmax_data = None\n",
        "\n",
        "# wideresnets_features_files\n",
        "# cnn_features_files\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RrUkssLPG2yb"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import re\n",
        "\n",
        "def get_base_patterns_for_validation(features_files, layer_name = \"CLASSIFIER_D1\", model_type = \"WideResNet\"):\n",
        "  base_patterns_for_validations = []\n",
        "  for ff in [ s for s in features_files if \"Validation\" in s]:\n",
        "    validation_search = re.search(f'^.*({model_type}.*_ID.*features_{layer_name}_Validation)[0-9]+.*$', ff, re.IGNORECASE)\n",
        "    if validation_search:\n",
        "        base_patterns_for_validations.append(validation_search.group(1))\n",
        "\n",
        "  base_patterns_for_validations = sorted(list(set(base_patterns_for_validations)))\n",
        "  return base_patterns_for_validations\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_lRcoa4qG2yc"
      },
      "outputs": [],
      "source": [
        "def get_features_for_layer_using_subbatches(feature_files, layer_name, num_of_models, model_type=\"WideResNet\", axis_to_concat = 1):\n",
        " \n",
        "  feature_files_used = [ff for ff in feature_files if \"Validation\" in ff and \"_X\" in ff and f\"_{layer_name}_\" in ff ]\n",
        "  \n",
        "  # base_patterns_for_validations = []\n",
        "  # for ff in [ s for s in feature_files_used ]:\n",
        "  #   validation_search = re.search(f'^.*(_ID.*features_{layer_name}_Validation)[0-9]+.*$', ff, re.IGNORECASE)\n",
        "  #   if validation_search:\n",
        "  #       base_patterns_for_validations.append(validation_search.group(1))\n",
        "  # base_patterns_for_validations = sorted(list(set(base_patterns_for_validations)))\n",
        "\n",
        "  base_patterns_for_validations = get_base_patterns_for_validation(feature_files_used, layer_name, model_type)\n",
        "  base_patterns_for_validations = sorted(random.sample(base_patterns_for_validations, min(num_of_models,len(base_patterns_for_validations)) ))\n",
        "\n",
        "  np_x_validation_collab = None\n",
        "  for base_val_str in base_patterns_for_validations:\n",
        "      validation_batch_files = sorted([ ff for ff in feature_files_used if base_val_str in ff])\n",
        "      np_x_validation_collab_batch = np.array([np.load(ff) for ff in validation_batch_files])\n",
        "      np_x_validation_collab_batch = np.concatenate(np_x_validation_collab_batch, axis=0)\n",
        "      if np_x_validation_collab is None:\n",
        "        np_x_validation_collab = np_x_validation_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_validation_collab = np.concatenate([np_x_validation_collab, np_x_validation_collab_batch], axis=axis_to_concat)\n",
        "  \n",
        "  np_x_test_collab = None\n",
        "  for base_val_str in [ ff.replace(\"Validation\",\"Test\") for ff in base_patterns_for_validations]:\n",
        "      test_batch_files = dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{base_val_str}.*_X.*$\")\n",
        "      np_x_test_collab_batch = np.array([np.load(ff) for ff in test_batch_files])\n",
        "      np_x_test_collab_batch = np.concatenate(np_x_test_collab_batch, axis=0)\n",
        "      if np_x_test_collab is None:\n",
        "        np_x_test_collab = np_x_test_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_test_collab = np.concatenate([np_x_test_collab, np_x_test_collab_batch], axis=axis_to_concat)\n",
        "\n",
        "  np_x_train_collab = None\n",
        "  for base_val_str in [ ff.replace(\"Validation\",\"Train\") for ff in base_patterns_for_validations]:\n",
        "      train_batch_files = dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{base_val_str}.*_X.*$\")\n",
        "      np_x_train_collab_batch = np.array([np.load(ff) for ff in train_batch_files])\n",
        "      np_x_train_collab_batch = np.concatenate(np_x_train_collab_batch, axis=0)\n",
        "      if np_x_train_collab is None:\n",
        "        np_x_train_collab = np_x_train_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_train_collab = np.concatenate([np_x_train_collab, np_x_train_collab_batch], axis=axis_to_concat)\n",
        "\n",
        "  return np_x_validation_collab, np_x_train_collab, np_x_test_collab"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mgc3WW_Y3Hqa"
      },
      "outputs": [],
      "source": [
        "# set up the data for the WideResNet collaborative\n",
        "\n",
        "import random\n",
        "import datetime\n",
        "\n",
        "num_of_repeats = 4\n",
        "num_of_models = [1,2,3,4,5,6,7,8,9,10]\n",
        "layer_name_sftmx_wideresnets = \"CLASSIFIER_D1\"\n",
        "model_type = \"WideResNet\"\n",
        "\n",
        "idxCount = 0 if collaborativeFullyC_softmax_data is None else len(collaborativeFullyC_softmax_data.index)\n",
        "for repc in range(num_of_repeats):\n",
        "  for mc in num_of_models:\n",
        "    model_name = f\"Collab_{layer_name_sftmx_wideresnets}_WideResNet_A_{datetime.datetime.now():%Y%m%d%H%M%S}\"\n",
        "    np_x_validation_collab_wideresnets_sftmx, np_y_train_collab_wideresnets_sftmx, np_x_test_collab_wideresnets_sftmx = get_features_for_layer_using_subbatches(wideresnets_features_files,layer_name_sftmx_wideresnets,mc,model_type )\n",
        "    model_here, history_here = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "                        model_name, \n",
        "                        np_y_train_collab_wideresnets_sftmx[0,:].shape, \n",
        "                        np_y_train_collab_wideresnets_sftmx, \n",
        "                        train_targets,\n",
        "                        save_max_epoch = False,\n",
        "                        save_final = True,\n",
        "                        patience_count = 35,\n",
        "                        early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                        log_history = True,\n",
        "                        verbose_level=0,                             \n",
        "                        batch_size=512, \n",
        "                        epochs=250, \n",
        "                        class_weight=None, \n",
        "                        validation_data=(np_x_validation_collab_wideresnets_sftmx, validation_targets))\n",
        "    del np_x_validation_collab_wideresnets_sftmx\n",
        "    del np_y_train_collab_wideresnets_sftmx\n",
        "    y_pred_model = model_here.predict(np_x_test_collab_wideresnets_sftmx)\n",
        "    del np_x_test_collab_wideresnets_sftmx\n",
        "    y_pred = np.apply_along_axis(np.argmax, 1, y_pred_model) \n",
        "    pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_pred,test_targets)\n",
        "    print (mc, repc, pr, rc, f1, acc)\n",
        "    del model_here\n",
        "    del history_here\n",
        "    if collaborativeFullyC_softmax_data is None:\n",
        "      collaborativeFullyC_softmax_data = pd.DataFrame({\"Type\": \"WideResNet\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            }, index = [idxCount])\n",
        "    else:\n",
        "      collaborativeFullyC_softmax_data = pd.concat([collaborativeFullyC_softmax_data,\n",
        "                                         pd.DataFrame({\"Type\": \"WideResNet\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc\n",
        "                                            }, index = [idxCount])\n",
        "                                         ])\n",
        "    idxCount = idxCount + 1\n",
        "\n",
        "\n",
        "# np_x_validation_collab_wideresnets, np_x_train_collab_wideresnets, np_x_test_collab_wideresnets =  get_features_for_layer_using_subbatches(wideresnets_features_files,layer_name,num_of_models,model_type )\n",
        "# wrsftmx, wrsftmxh = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "#                     f\"Collab_{layer_name}_WideResnet28-10_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "#                     np_x_train_collab_wideresnets[0,:].shape, \n",
        "#                     np_x_train_collab_wideresnets, \n",
        "#                     train_targets,\n",
        "#                     save_max_epoch=False,\n",
        "#                     save_final=True,\n",
        "#                     patience_count = 35,\n",
        "#                     early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "#                     log_history = True,\n",
        "#                     verbose_level=1,                             \n",
        "#                     batch_size=512, \n",
        "#                     epochs=250, \n",
        "#                     class_weight=None, \n",
        "#                     validation_data=(np_x_validation_collab_wideresnets, validation_targets))\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-pu-2cp23Hqb"
      },
      "outputs": [],
      "source": [
        "collaborativeFullyC_softmax_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/wideresnet_collab_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qSwgurq54dx7"
      },
      "source": [
        "# Data for Contour Plot on Collaborative CNN/WideResNet softmax layers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bnWPbCsx4dx8"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import re\n",
        "\n",
        "def get_base_patterns_for_validation(features_files, layer_name = \"CLASSIFIER_D1\", model_type = \"WideResNet\"):\n",
        "  base_patterns_for_validations = []\n",
        "  for ff in [ s for s in features_files if \"Validation\" in s]:\n",
        "    validation_search = re.search(f'^.*({model_type}.*_ID.*features_{layer_name}_Validation)[0-9]+.*$', ff, re.IGNORECASE)\n",
        "    if validation_search:\n",
        "        base_patterns_for_validations.append(validation_search.group(1))\n",
        "\n",
        "  base_patterns_for_validations = sorted(list(set(base_patterns_for_validations)))\n",
        "  return base_patterns_for_validations\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ibzvZiTJ4dx8"
      },
      "outputs": [],
      "source": [
        "def get_features_for_layer_using_subbatches(feature_files, layer_name, num_of_models, model_type=\"WideResNet\", axis_to_concat = 1):\n",
        " \n",
        "  feature_files_used = [ff for ff in feature_files if \"Validation\" in ff and \"_X\" in ff and f\"_{layer_name}_\" in ff ]\n",
        "  \n",
        "  # base_patterns_for_validations = []\n",
        "  # for ff in [ s for s in feature_files_used ]:\n",
        "  #   validation_search = re.search(f'^.*(_ID.*features_{layer_name}_Validation)[0-9]+.*$', ff, re.IGNORECASE)\n",
        "  #   if validation_search:\n",
        "  #       base_patterns_for_validations.append(validation_search.group(1))\n",
        "  # base_patterns_for_validations = sorted(list(set(base_patterns_for_validations)))\n",
        "\n",
        "  base_patterns_for_validations = get_base_patterns_for_validation(feature_files_used, layer_name, model_type)\n",
        "  base_patterns_for_validations = sorted(random.sample(base_patterns_for_validations, min(num_of_models,len(base_patterns_for_validations)) ))\n",
        "\n",
        "  np_x_validation_collab = None\n",
        "  for base_val_str in base_patterns_for_validations:\n",
        "      validation_batch_files = sorted([ ff for ff in feature_files_used if base_val_str in ff])\n",
        "      np_x_validation_collab_batch = np.array([np.load(ff) for ff in validation_batch_files])\n",
        "      np_x_validation_collab_batch = np.concatenate(np_x_validation_collab_batch, axis=0)\n",
        "      if np_x_validation_collab is None:\n",
        "        np_x_validation_collab = np_x_validation_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_validation_collab = np.concatenate([np_x_validation_collab, np_x_validation_collab_batch], axis=axis_to_concat)\n",
        "  \n",
        "  np_x_test_collab = None\n",
        "  for base_val_str in [ ff.replace(\"Validation\",\"Test\") for ff in base_patterns_for_validations]:\n",
        "      test_batch_files = dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{base_val_str}.*_X.*$\")\n",
        "      np_x_test_collab_batch = np.array([np.load(ff) for ff in test_batch_files])\n",
        "      np_x_test_collab_batch = np.concatenate(np_x_test_collab_batch, axis=0)\n",
        "      if np_x_test_collab is None:\n",
        "        np_x_test_collab = np_x_test_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_test_collab = np.concatenate([np_x_test_collab, np_x_test_collab_batch], axis=axis_to_concat)\n",
        "\n",
        "  np_x_train_collab = None\n",
        "  for base_val_str in [ ff.replace(\"Validation\",\"Train\") for ff in base_patterns_for_validations]:\n",
        "      train_batch_files = dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{base_val_str}.*_X.*$\")\n",
        "      np_x_train_collab_batch = np.array([np.load(ff) for ff in train_batch_files])\n",
        "      np_x_train_collab_batch = np.concatenate(np_x_train_collab_batch, axis=0)\n",
        "      if np_x_train_collab is None:\n",
        "        np_x_train_collab = np_x_train_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_train_collab = np.concatenate([np_x_train_collab, np_x_train_collab_batch], axis=axis_to_concat)\n",
        "\n",
        "  return np_x_validation_collab, np_x_train_collab, np_x_test_collab\n",
        "\n",
        "  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HF7tZpgXioib"
      },
      "outputs": [],
      "source": [
        "# set up the combinations we are going to try\n",
        "# len(wideresnets_features_files)\n",
        "\n",
        "cnn_wideresnet_contour_data = None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AeM917nGy0Gd"
      },
      "outputs": [],
      "source": [
        "cnn_features_files"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dNlenvgshmJG"
      },
      "outputs": [],
      "source": [
        "# set up the data for the WideResnet+CNN collaborative on softmax layer\n",
        "import random\n",
        "layer_name_sftmx_cnn = \"SFTMX1\"\n",
        "layer_name_sftmx_wideresnets = \"CLASSIFIER_D1\"\n",
        "total_num_components = [2,3,4,5,6,7,8,9,10]\n",
        "num_of_repeats = 3\n",
        "\n",
        "idxCount = 0 if cnn_wideresnet_contour_data is None else len(cnn_wideresnet_contour_data.index)\n",
        "\n",
        "for repc in range(num_of_repeats):\n",
        "  for component_count in total_num_components:\n",
        "    for num_of_models_cnn in range(component_count+1):\n",
        "      num_of_models_wideresnets = component_count - num_of_models_cnn\n",
        "\n",
        "      np_x_validation_collab_cnn, np_x_train_collab_cnn, np_x_test_collab_cnn = (None,None,None)\n",
        "      np_x_validation_collab_wideresnets, np_x_train_collab_wideresnets, np_x_test_collab_wideresnets = (None,None,None)\n",
        "      np_x_validation_collab_wrcdnn, np_x_train_collab_wrcdnn, np_x_test_collab_wrcdnn = (None, None, None)\n",
        "\n",
        "      if num_of_models_cnn > 0:\n",
        "        np_x_validation_collab_cnn, np_x_train_collab_cnn, np_x_test_collab_cnn = get_features_for_layer(cnn_features_files,layer_name_sftmx_cnn,num_of_models_cnn)\n",
        "\n",
        "      if num_of_models_wideresnets > 0:\n",
        "        np_x_validation_collab_wideresnets, np_x_train_collab_wideresnets, np_x_test_collab_wideresnets = get_features_for_layer_using_subbatches(wideresnets_features_files,layer_name_sftmx_wideresnets,num_of_models_wideresnets,model_type=\"WideResNet\")\n",
        "      \n",
        "      if num_of_models_cnn > 0 and num_of_models_wideresnets > 0:        \n",
        "        np_x_train_collab_wrcdnn = np.concatenate([np_x_train_collab_cnn,np_x_train_collab_wideresnets], axis=1)\n",
        "        np_x_validation_collab_wrcdnn = np.concatenate([np_x_validation_collab_cnn,np_x_validation_collab_wideresnets], axis=1)\n",
        "        np_x_test_collab_wrcdnn = np.concatenate([np_x_test_collab_cnn,np_x_test_collab_wideresnets], axis=1)\n",
        "      else: \n",
        "        if num_of_models_cnn == 0:\n",
        "          np_x_train_collab_wrcdnn = np_x_train_collab_wideresnets\n",
        "          np_x_validation_collab_wrcdnn = np_x_validation_collab_wideresnets\n",
        "          np_x_test_collab_wrcdnn = np_x_test_collab_wideresnets\n",
        "        if num_of_models_wideresnets == 0:\n",
        "          np_x_train_collab_wrcdnn = np_x_train_collab_cnn\n",
        "          np_x_validation_collab_wrcdnn = np_x_validation_collab_cnn\n",
        "          np_x_test_collab_wrcdnn = np_x_test_collab_cnn\n",
        "      \n",
        "      layer_name = f\"{layer_name_sftmx_cnn}_{layer_name_sftmx_wideresnets}_{str(num_of_models_cnn)}-{str(num_of_models_wideresnets)}\"\n",
        "      model_here, history_here = compile_and_fit_model_basic( model_combination_of_features_with_flatten,  \n",
        "                          f\"Collab_{layer_name}_CNNWRNTS_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                          np_x_train_collab_wrcdnn[0,:].shape, \n",
        "                          np_x_train_collab_wrcdnn, \n",
        "                          train_targets,\n",
        "                          save_max_epoch=False,\n",
        "                          save_final=True,\n",
        "                          patience_count = 35,\n",
        "                          early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                          log_history = True,\n",
        "                          verbose_level=0,                             \n",
        "                          batch_size=512, \n",
        "                          epochs=250, \n",
        "                          class_weight=None, \n",
        "                          validation_data=(np_x_validation_collab_wrcdnn, validation_targets))\n",
        "      del np_x_train_collab_wrcdnn\n",
        "      del np_x_validation_collab_wrcdnn\n",
        "      del np_x_validation_collab_cnn\n",
        "      del np_x_train_collab_cnn\n",
        "      del np_x_validation_collab_wideresnets\n",
        "      del np_x_train_collab_wideresnets\n",
        "      \n",
        "      y_pred_model = model_here.predict(np_x_test_collab_wrcdnn)\n",
        "      del np_x_test_collab_cnn\n",
        "      del np_x_test_collab_wideresnets\n",
        "      del np_x_test_collab_wrcdnn\n",
        "\n",
        "      y_pred = np.apply_along_axis(np.argmax, 1, y_pred_model) \n",
        "      pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_pred,test_targets)\n",
        "      print (num_of_models_cnn, repc, pr, rc, f1, acc)\n",
        "      del model_here\n",
        "      del history_here\n",
        "      if cnn_wideresnet_contour_data is None:\n",
        "        cnn_wideresnet_contour_data = pd.DataFrame({\"TypeA\": \"CNN\", \n",
        "                                                    \"TypeB\": \"WideResNet\", \n",
        "                                                    \"Data\" : \"Test\",\n",
        "                                                    \"NumOfA\": num_of_models_cnn, \n",
        "                                                    \"NumOfB\": num_of_models_wideresnets, \n",
        "                                                    \"RepC\": repc, \n",
        "                                                    \"Pr\": pr,\n",
        "                                                    \"Rc\": rc,\n",
        "                                                    \"F1\": f1,\n",
        "                                                    \"Acc\": acc,\n",
        "                                                    }, index = [idxCount])\n",
        "      else:\n",
        "        cnn_wideresnet_contour_data = pd.concat([cnn_wideresnet_contour_data,\n",
        "                                          pd.DataFrame({\"TypeA\": \"CNN\", \n",
        "                                              \"TypeB\": \"WideResNet\", \n",
        "                                              \"Data\" : \"Test\",\n",
        "                                              \"NumOfA\": num_of_models_cnn, \n",
        "                                              \"NumOfB\": num_of_models_wideresnets, \n",
        "                                              \"RepC\": repc, \n",
        "                                              \"Pr\": pr,\n",
        "                                              \"Rc\": rc,\n",
        "                                              \"F1\": f1,\n",
        "                                              \"Acc\": acc\n",
        "                                              }, index = [idxCount])\n",
        "                                          ])\n",
        "      idxCount = idxCount + 1\n",
        "    cnn_wideresnet_contour_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/cnn_wideresnet_tempcontour_data_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n",
        "    \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_TsMTDnt4dx8"
      },
      "outputs": [],
      "source": [
        "cnn_wideresnet_contour_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/cnn_wideresnet_contour_data_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BvyzEp0EkgBD"
      },
      "source": [
        "# Data for Contour Plot on Collaborative DNN/WideResNet softmax layers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bMV-tRMnkgBE"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import re\n",
        "\n",
        "def get_base_patterns_for_validation(features_files, layer_name = \"CLASSIFIER_D1\", model_type = \"WideResNet\"):\n",
        "  base_patterns_for_validations = []\n",
        "  for ff in [ s for s in features_files if \"Validation\" in s]:\n",
        "    validation_search = re.search(f'^.*({model_type}.*_ID.*features_{layer_name}_Validation)[0-9]+.*$', ff, re.IGNORECASE)\n",
        "    if validation_search:\n",
        "        base_patterns_for_validations.append(validation_search.group(1))\n",
        "\n",
        "  base_patterns_for_validations = sorted(list(set(base_patterns_for_validations)))\n",
        "  return base_patterns_for_validations\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nPUPFfJgkgBE"
      },
      "outputs": [],
      "source": [
        "def get_features_for_layer_using_subbatches(feature_files, layer_name, num_of_models, model_type=\"WideResNet\", axis_to_concat = 1):\n",
        " \n",
        "  feature_files_used = [ff for ff in feature_files if \"Validation\" in ff and \"_X\" in ff and f\"_{layer_name}_\" in ff ]\n",
        "  \n",
        "  # base_patterns_for_validations = []\n",
        "  # for ff in [ s for s in feature_files_used ]:\n",
        "  #   validation_search = re.search(f'^.*(_ID.*features_{layer_name}_Validation)[0-9]+.*$', ff, re.IGNORECASE)\n",
        "  #   if validation_search:\n",
        "  #       base_patterns_for_validations.append(validation_search.group(1))\n",
        "  # base_patterns_for_validations = sorted(list(set(base_patterns_for_validations)))\n",
        "\n",
        "  base_patterns_for_validations = get_base_patterns_for_validation(feature_files_used, layer_name, model_type)\n",
        "  base_patterns_for_validations = sorted(random.sample(base_patterns_for_validations, min(num_of_models,len(base_patterns_for_validations)) ))\n",
        "\n",
        "  np_x_validation_collab = None\n",
        "  for base_val_str in base_patterns_for_validations:\n",
        "      validation_batch_files = sorted([ ff for ff in feature_files_used if base_val_str in ff])\n",
        "      np_x_validation_collab_batch = np.array([np.load(ff) for ff in validation_batch_files])\n",
        "      np_x_validation_collab_batch = np.concatenate(np_x_validation_collab_batch, axis=0)\n",
        "      if np_x_validation_collab is None:\n",
        "        np_x_validation_collab = np_x_validation_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_validation_collab = np.concatenate([np_x_validation_collab, np_x_validation_collab_batch], axis=axis_to_concat)\n",
        "  \n",
        "  np_x_test_collab = None\n",
        "  for base_val_str in [ ff.replace(\"Validation\",\"Test\") for ff in base_patterns_for_validations]:\n",
        "      test_batch_files = dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{base_val_str}.*_X.*$\")\n",
        "      np_x_test_collab_batch = np.array([np.load(ff) for ff in test_batch_files])\n",
        "      np_x_test_collab_batch = np.concatenate(np_x_test_collab_batch, axis=0)\n",
        "      if np_x_test_collab is None:\n",
        "        np_x_test_collab = np_x_test_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_test_collab = np.concatenate([np_x_test_collab, np_x_test_collab_batch], axis=axis_to_concat)\n",
        "\n",
        "  np_x_train_collab = None\n",
        "  for base_val_str in [ ff.replace(\"Validation\",\"Train\") for ff in base_patterns_for_validations]:\n",
        "      train_batch_files = dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{base_val_str}.*_X.*$\")\n",
        "      np_x_train_collab_batch = np.array([np.load(ff) for ff in train_batch_files])\n",
        "      np_x_train_collab_batch = np.concatenate(np_x_train_collab_batch, axis=0)\n",
        "      if np_x_train_collab is None:\n",
        "        np_x_train_collab = np_x_train_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_train_collab = np.concatenate([np_x_train_collab, np_x_train_collab_batch], axis=axis_to_concat)\n",
        "\n",
        "  return np_x_validation_collab, np_x_train_collab, np_x_test_collab\n",
        "\n",
        "  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GJsVRidtkgBE"
      },
      "outputs": [],
      "source": [
        "# set up the combinations we are going to try\n",
        "# len(wideresnets_features_files)\n",
        "dnn_wideresnet_contour_data = None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jUNzYWylkgBE"
      },
      "outputs": [],
      "source": [
        "dnn_features_files"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mMbDrr35kgBE"
      },
      "outputs": [],
      "source": [
        "# set up the data for the WideResnet+CNN collaborative on softmax layer\n",
        "import random\n",
        "layer_name_sftmx_dnn = \"SFTMX1\"\n",
        "layer_name_sftmx_wideresnets = \"CLASSIFIER_D1\"\n",
        "total_num_components = [2,3,4,5,6,7,8,9,10]\n",
        "num_of_repeats = 3\n",
        "\n",
        "idxCount = 0 if dnn_wideresnet_contour_data is None else len(dnn_wideresnet_contour_data.index)\n",
        "\n",
        "for repc in range(num_of_repeats):\n",
        "  for component_count in total_num_components:\n",
        "    for num_of_models_dnn in range(component_count+1):\n",
        "\n",
        "      num_of_models_wideresnets = component_count - num_of_models_dnn\n",
        "\n",
        "      np_x_validation_collab_dnn, np_x_train_collab_dnn, np_x_test_collab_dnn = (None,None,None)\n",
        "      np_x_validation_collab_wideresnets, np_x_train_collab_wideresnets, np_x_test_collab_wideresnets = (None,None,None)\n",
        "      np_x_validation_collab_wrcdnn, np_x_train_collab_wrcdnn, np_x_test_collab_wrcdnn = (None, None, None)\n",
        "\n",
        "      if num_of_models_dnn > 0:\n",
        "        np_x_validation_collab_dnn, np_x_train_collab_dnn, np_x_test_collab_dnn = get_features_for_layer(dnn_features_files,layer_name_sftmx_dnn,num_of_models_dnn)\n",
        "\n",
        "      if num_of_models_wideresnets > 0:\n",
        "        np_x_validation_collab_wideresnets, np_x_train_collab_wideresnets, np_x_test_collab_wideresnets = get_features_for_layer_using_subbatches(wideresnets_features_files,layer_name_sftmx_wideresnets,num_of_models_wideresnets,model_type=\"WideResNet\")\n",
        "      \n",
        "      if num_of_models_dnn > 0 and num_of_models_wideresnets > 0:        \n",
        "        np_x_train_collab_wrcdnn = np.concatenate([np_x_train_collab_dnn,np_x_train_collab_wideresnets], axis=1)\n",
        "        np_x_validation_collab_wrcdnn = np.concatenate([np_x_validation_collab_dnn,np_x_validation_collab_wideresnets], axis=1)\n",
        "        np_x_test_collab_wrcdnn = np.concatenate([np_x_test_collab_dnn,np_x_test_collab_wideresnets], axis=1)\n",
        "      else: \n",
        "        if num_of_models_dnn == 0:\n",
        "          np_x_train_collab_wrcdnn = np_x_train_collab_wideresnets\n",
        "          np_x_validation_collab_wrcdnn = np_x_validation_collab_wideresnets\n",
        "          np_x_test_collab_wrcdnn = np_x_test_collab_wideresnets\n",
        "        if num_of_models_wideresnets == 0:\n",
        "          np_x_train_collab_wrcdnn = np_x_train_collab_dnn\n",
        "          np_x_validation_collab_wrcdnn = np_x_validation_collab_dnn\n",
        "          np_x_test_collab_wrcdnn = np_x_test_collab_dnn\n",
        "      \n",
        "      layer_name = f\"{layer_name_sftmx_dnn}_{layer_name_sftmx_wideresnets}_{str(num_of_models_dnn)}-{str(num_of_models_wideresnets)}\"\n",
        "      \n",
        "      model_here, history_here = compile_and_fit_model_basic( model_combination_of_features_with_flatten,  \n",
        "                          f\"Collab_{layer_name}_DNNWRNTS_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                          np_x_train_collab_wrcdnn[0,:].shape, \n",
        "                          np_x_train_collab_wrcdnn, \n",
        "                          train_targets,\n",
        "                          save_max_epoch=False,\n",
        "                          save_final=True,\n",
        "                          patience_count = 35,\n",
        "                          early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                          log_history = True,\n",
        "                          verbose_level=0,                             \n",
        "                          batch_size=512, \n",
        "                          epochs=250, \n",
        "                          class_weight=None, \n",
        "                          validation_data=(np_x_validation_collab_wrcdnn, validation_targets))\n",
        "      del np_x_train_collab_wrcdnn\n",
        "      del np_x_validation_collab_wrcdnn\n",
        "      del np_x_validation_collab_dnn\n",
        "      del np_x_train_collab_dnn\n",
        "      del np_x_validation_collab_wideresnets\n",
        "      del np_x_train_collab_wideresnets\n",
        "      \n",
        "      y_pred_model = model_here.predict(np_x_test_collab_wrcdnn)\n",
        "      del np_x_test_collab_dnn\n",
        "      del np_x_test_collab_wideresnets\n",
        "      del np_x_test_collab_wrcdnn\n",
        "\n",
        "      y_pred = np.apply_along_axis(np.argmax, 1, y_pred_model) \n",
        "      pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_pred,test_targets)\n",
        "      print (num_of_models_dnn, repc, pr, rc, f1, acc)\n",
        "      del model_here\n",
        "      del history_here\n",
        "      if dnn_wideresnet_contour_data is None:\n",
        "        dnn_wideresnet_contour_data = pd.DataFrame({\"TypeA\": \"DNN\", \n",
        "                                                    \"TypeB\": \"WideResNet\", \n",
        "                                                    \"Data\" : \"Test\",\n",
        "                                                    \"NumOfA\": num_of_models_dnn, \n",
        "                                                    \"NumOfB\": num_of_models_wideresnets, \n",
        "                                                    \"RepC\": repc, \n",
        "                                                    \"Pr\": pr,\n",
        "                                                    \"Rc\": rc,\n",
        "                                                    \"F1\": f1,\n",
        "                                                    \"Acc\": acc,\n",
        "                                                    }, index = [idxCount])\n",
        "      else:\n",
        "        dnn_wideresnet_contour_data = pd.concat([dnn_wideresnet_contour_data,\n",
        "                                          pd.DataFrame({\"TypeA\": \"DNN\", \n",
        "                                              \"TypeB\": \"WideResNet\", \n",
        "                                              \"Data\" : \"Test\",\n",
        "                                              \"NumOfA\": num_of_models_dnn, \n",
        "                                              \"NumOfB\": num_of_models_wideresnets, \n",
        "                                              \"RepC\": repc, \n",
        "                                              \"Pr\": pr,\n",
        "                                              \"Rc\": rc,\n",
        "                                              \"F1\": f1,\n",
        "                                              \"Acc\": acc\n",
        "                                              }, index = [idxCount])\n",
        "                                          ])\n",
        "      idxCount = idxCount + 1\n",
        "    dnn_wideresnet_contour_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/dnn_wideresnet_tempcontour_data_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n",
        "    \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vrB5oFXEkgBF"
      },
      "outputs": [],
      "source": [
        "dnn_wideresnet_contour_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/dnn_wideresnet_contour_data_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nKZebazXuPRf"
      },
      "source": [
        "# Data for Contour Plot on Collaborative CNN/DNN softmax layers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rUFA68nKuPRf"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import re\n",
        "\n",
        "def get_base_patterns_for_validation(features_files, layer_name = \"CLASSIFIER_D1\", model_type = \"WideResNet\"):\n",
        "  base_patterns_for_validations = []\n",
        "  for ff in [ s for s in features_files if \"Validation\" in s]:\n",
        "    validation_search = re.search(f'^.*({model_type}.*_ID.*features_{layer_name}_Validation)[0-9]+.*$', ff, re.IGNORECASE)\n",
        "    if validation_search:\n",
        "        base_patterns_for_validations.append(validation_search.group(1))\n",
        "\n",
        "  base_patterns_for_validations = sorted(list(set(base_patterns_for_validations)))\n",
        "  return base_patterns_for_validations\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-nmVKaDguPRf"
      },
      "outputs": [],
      "source": [
        "# set up the combinations we are going to try\n",
        "# len(wideresnets_features_files)\n",
        "cnn_dnn_contour_data = None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iWjI91pDuPRf"
      },
      "outputs": [],
      "source": [
        "# set up the data for the WideResnet+CNN collaborative on softmax layer\n",
        "import random\n",
        "layer_name_sftmx_cnn = \"SFTMX1\"\n",
        "layer_name_sftmx_dnn = \"SFTMX1\"\n",
        "total_num_components = [2,4,6,8,10,12,14,16,18,20,24,28,32,36,38,40]\n",
        "num_of_repeats = 3\n",
        "\n",
        "idxCount = 0 if cnn_dnn_contour_data is None else len(cnn_dnn_contour_data.index)\n",
        "\n",
        "for repc in range(num_of_repeats):\n",
        "  for component_count in total_num_components:\n",
        "    for num_of_models_cnn in range(component_count+1):\n",
        "      num_of_models_dnn = component_count - num_of_models_cnn\n",
        "\n",
        "      np_x_validation_collab_cnn, np_x_train_collab_cnn, np_x_test_collab_cnn = (None,None,None)\n",
        "      np_x_validation_collab_dnn, np_x_train_collab_dnn, np_x_test_collab_dnn = (None,None,None)\n",
        "      np_x_validation_collab_wrcdnn, np_x_train_collab_wrcdnn, np_x_test_collab_wrcdnn = (None, None, None)\n",
        "\n",
        "      if num_of_models_cnn > 0:\n",
        "        np_x_validation_collab_cnn, np_x_train_collab_cnn, np_x_test_collab_cnn = get_features_for_layer(cnn_features_files,layer_name_sftmx_cnn,num_of_models_cnn)\n",
        "\n",
        "      if num_of_models_dnn > 0:\n",
        "        np_x_validation_collab_dnn, np_x_train_collab_dnn, np_x_test_collab_dnn = get_features_for_layer(dnn_features_files,layer_name_sftmx_dnn,num_of_models_dnn)\n",
        "      \n",
        "      if num_of_models_cnn > 0 and num_of_models_dnn > 0:        \n",
        "        np_x_train_collab_wrcdnn = np.concatenate([np_x_train_collab_cnn,np_x_train_collab_dnn], axis=1)\n",
        "        np_x_validation_collab_wrcdnn = np.concatenate([np_x_validation_collab_cnn,np_x_validation_collab_dnn], axis=1)\n",
        "        np_x_test_collab_wrcdnn = np.concatenate([np_x_test_collab_cnn,np_x_test_collab_dnn], axis=1)\n",
        "      else: \n",
        "        if num_of_models_cnn == 0:\n",
        "          np_x_train_collab_wrcdnn = np_x_train_collab_dnn\n",
        "          np_x_validation_collab_wrcdnn = np_x_validation_collab_dnn\n",
        "          np_x_test_collab_wrcdnn = np_x_test_collab_dnn\n",
        "        if num_of_models_dnn == 0:\n",
        "          np_x_train_collab_wrcdnn = np_x_train_collab_cnn\n",
        "          np_x_validation_collab_wrcdnn = np_x_validation_collab_cnn\n",
        "          np_x_test_collab_wrcdnn = np_x_test_collab_cnn\n",
        "      \n",
        "      layer_name = f\"{layer_name_sftmx_cnn}_{layer_name_sftmx_dnn}_{str(num_of_models_cnn)}-{str(num_of_models_dnn)}\"\n",
        "      model_here, history_here = compile_and_fit_model_basic( model_combination_of_features_with_flatten,  \n",
        "                          f\"Collab_{layer_name}_CNNDNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                          np_x_train_collab_wrcdnn[0,:].shape, \n",
        "                          np_x_train_collab_wrcdnn, \n",
        "                          train_targets,\n",
        "                          save_max_epoch=False,\n",
        "                          save_final=True,\n",
        "                          patience_count = 35,\n",
        "                          early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                          log_history = True,\n",
        "                          verbose_level=0,                             \n",
        "                          batch_size=512, \n",
        "                          epochs=250, \n",
        "                          class_weight=None, \n",
        "                          validation_data=(np_x_validation_collab_wrcdnn, validation_targets))\n",
        "      del np_x_train_collab_wrcdnn\n",
        "      del np_x_validation_collab_wrcdnn\n",
        "      del np_x_validation_collab_cnn\n",
        "      del np_x_train_collab_cnn\n",
        "      del np_x_validation_collab_dnn\n",
        "      del np_x_train_collab_dnn\n",
        "      \n",
        "      y_pred_model = model_here.predict(np_x_test_collab_wrcdnn)\n",
        "      del np_x_test_collab_cnn\n",
        "      del np_x_test_collab_dnn\n",
        "      del np_x_test_collab_wrcdnn\n",
        "\n",
        "      y_pred = np.apply_along_axis(np.argmax, 1, y_pred_model) \n",
        "      pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_pred,test_targets)\n",
        "      print (num_of_models_cnn, repc, pr, rc, f1, acc)\n",
        "      del model_here\n",
        "      del history_here\n",
        "      if cnn_dnn_contour_data is None:\n",
        "        cnn_dnn_contour_data = pd.DataFrame({\"TypeA\": \"CNN\", \n",
        "                                                    \"TypeB\": \"DNN\", \n",
        "                                                    \"Data\" : \"Test\",\n",
        "                                                    \"NumOfA\": num_of_models_cnn, \n",
        "                                                    \"NumOfB\": num_of_models_dnn, \n",
        "                                                    \"RepC\": repc, \n",
        "                                                    \"Pr\": pr,\n",
        "                                                    \"Rc\": rc,\n",
        "                                                    \"F1\": f1,\n",
        "                                                    \"Acc\": acc,\n",
        "                                                    }, index = [idxCount])\n",
        "      else:\n",
        "        cnn_dnn_contour_data = pd.concat([cnn_dnn_contour_data,\n",
        "                                          pd.DataFrame({\"TypeA\": \"CNN\", \n",
        "                                              \"TypeB\": \"DNN\", \n",
        "                                              \"Data\" : \"Test\",\n",
        "                                              \"NumOfA\": num_of_models_cnn, \n",
        "                                              \"NumOfB\": num_of_models_dnn, \n",
        "                                              \"RepC\": repc, \n",
        "                                              \"Pr\": pr,\n",
        "                                              \"Rc\": rc,\n",
        "                                              \"F1\": f1,\n",
        "                                              \"Acc\": acc\n",
        "                                              }, index = [idxCount])\n",
        "                                          ])\n",
        "      idxCount = idxCount + 1\n",
        "    \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2iN0aQPfuPRg"
      },
      "outputs": [],
      "source": [
        "cnn_dnn_contour_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/cnn_dnn_contour_data_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sddcE82tOFDm"
      },
      "source": [
        "# Data for Contour Plot on Collaborative CNN/DNN lastdense layers _In Progress_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TuFWFx6YOFDn"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import re\n",
        "\n",
        "def get_base_patterns_for_validation(features_files, layer_name = \"CLASSIFIER_D1\", model_type = \"WideResNet\"):\n",
        "  base_patterns_for_validations = []\n",
        "  for ff in [ s for s in features_files if \"Validation\" in s]:\n",
        "    validation_search = re.search(f'^.*({model_type}.*_ID.*features_{layer_name}_Validation)[0-9]+.*$', ff, re.IGNORECASE)\n",
        "    if validation_search:\n",
        "        base_patterns_for_validations.append(validation_search.group(1))\n",
        "\n",
        "  base_patterns_for_validations = sorted(list(set(base_patterns_for_validations)))\n",
        "  return base_patterns_for_validations\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QukEjJMcOFDn"
      },
      "outputs": [],
      "source": [
        "# set up the combinations we are going to try\n",
        "# len(wideresnets_features_files)\n",
        "cnn_dnn_ld_contour_data = None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "C6cY5BzAOFDn"
      },
      "outputs": [],
      "source": [
        "# set up the data for the DNN+CNN collaborative on lastdense layer\n",
        "import random\n",
        "layer_name_penultimate_cnn = \"DRP1\"\n",
        "layer_name_penultimate_dnn = \"D3R\"\n",
        "total_num_components = [25, 30, 35, 40] # [2,4,6,8,10,12,14,16,18,20]\n",
        "num_of_repeats = 2\n",
        "\n",
        "idxCount = 0 if cnn_dnn_ld_contour_data is None else len(cnn_dnn_ld_contour_data.index)\n",
        "\n",
        "for repc in range(num_of_repeats):\n",
        "  for component_count in total_num_components:\n",
        "    for num_of_models_cnn in range(component_count+1):\n",
        "      num_of_models_dnn = component_count - num_of_models_cnn\n",
        "\n",
        "      np_x_validation_collab_cnn, np_x_train_collab_cnn, np_x_test_collab_cnn = (None,None,None)\n",
        "      np_x_validation_collab_dnn, np_x_train_collab_dnn, np_x_test_collab_dnn = (None,None,None)\n",
        "      np_x_validation_collab_wrcdnn, np_x_train_collab_wrcdnn, np_x_test_collab_wrcdnn = (None, None, None)\n",
        "\n",
        "      if num_of_models_cnn > 0:\n",
        "        np_x_validation_collab_cnn, np_x_train_collab_cnn, np_x_test_collab_cnn = get_features_for_layer(cnn_features_files,layer_name_penultimate_cnn,num_of_models_cnn)\n",
        "\n",
        "      if num_of_models_dnn > 0:\n",
        "        np_x_validation_collab_dnn, np_x_train_collab_dnn, np_x_test_collab_dnn = get_features_for_layer(dnn_features_files,layer_name_penultimate_dnn,num_of_models_dnn)\n",
        "      \n",
        "      if num_of_models_cnn > 0 and num_of_models_dnn > 0:        \n",
        "        np_x_train_collab_wrcdnn = np.concatenate([np_x_train_collab_cnn,np_x_train_collab_dnn], axis=1)\n",
        "        np_x_validation_collab_wrcdnn = np.concatenate([np_x_validation_collab_cnn,np_x_validation_collab_dnn], axis=1)\n",
        "        np_x_test_collab_wrcdnn = np.concatenate([np_x_test_collab_cnn,np_x_test_collab_dnn], axis=1)\n",
        "      else: \n",
        "        if num_of_models_cnn == 0:\n",
        "          np_x_train_collab_wrcdnn = np_x_train_collab_dnn\n",
        "          np_x_validation_collab_wrcdnn = np_x_validation_collab_dnn\n",
        "          np_x_test_collab_wrcdnn = np_x_test_collab_dnn\n",
        "        if num_of_models_dnn == 0:\n",
        "          np_x_train_collab_wrcdnn = np_x_train_collab_cnn\n",
        "          np_x_validation_collab_wrcdnn = np_x_validation_collab_cnn\n",
        "          np_x_test_collab_wrcdnn = np_x_test_collab_cnn\n",
        "      \n",
        "      layer_name = f\"{layer_name_penultimate_cnn}_{layer_name_penultimate_dnn}_{str(num_of_models_cnn)}-{str(num_of_models_dnn)}\"\n",
        "      model_here, history_here = compile_and_fit_model_basic( model_combination_of_features_with_flatten,  \n",
        "                          f\"Collab_{layer_name}_CNNDNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "                          np_x_train_collab_wrcdnn[0,:].shape, \n",
        "                          np_x_train_collab_wrcdnn, \n",
        "                          train_targets,\n",
        "                          save_max_epoch=False,\n",
        "                          save_final=True,\n",
        "                          patience_count = 35,\n",
        "                          early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                          log_history = True,\n",
        "                          verbose_level=0,                             \n",
        "                          batch_size=512, \n",
        "                          epochs=250, \n",
        "                          class_weight=None, \n",
        "                          validation_data=(np_x_validation_collab_wrcdnn, validation_targets))\n",
        "      del np_x_train_collab_wrcdnn\n",
        "      del np_x_validation_collab_wrcdnn\n",
        "      del np_x_validation_collab_cnn\n",
        "      del np_x_train_collab_cnn\n",
        "      del np_x_validation_collab_dnn\n",
        "      del np_x_train_collab_dnn\n",
        "      \n",
        "      y_pred_model = model_here.predict(np_x_test_collab_wrcdnn)\n",
        "      del np_x_test_collab_cnn\n",
        "      del np_x_test_collab_dnn\n",
        "      del np_x_test_collab_wrcdnn\n",
        "\n",
        "      y_pred = np.apply_along_axis(np.argmax, 1, y_pred_model) \n",
        "      pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_pred,test_targets)\n",
        "      print (num_of_models_cnn, repc, pr, rc, f1, acc)\n",
        "      del model_here\n",
        "      del history_here\n",
        "      if cnn_dnn_ld_contour_data is None:\n",
        "        cnn_dnn_ld_contour_data = pd.DataFrame({\"TypeA\": \"CNN\", \n",
        "                                                    \"TypeB\": \"DNN\", \n",
        "                                                    \"Data\" : \"Test\",\n",
        "                                                    \"NumOfA\": num_of_models_cnn, \n",
        "                                                    \"NumOfB\": num_of_models_dnn, \n",
        "                                                    \"Layer\": \"LastDense\",\n",
        "                                                    \"RepC\": repc, \n",
        "                                                    \"Pr\": pr,\n",
        "                                                    \"Rc\": rc,\n",
        "                                                    \"F1\": f1,\n",
        "                                                    \"Acc\": acc,\n",
        "                                                    }, index = [idxCount])\n",
        "      else:\n",
        "        cnn_dnn_ld_contour_data = pd.concat([cnn_dnn_ld_contour_data,\n",
        "                                          pd.DataFrame({\"TypeA\": \"CNN\", \n",
        "                                              \"TypeB\": \"DNN\", \n",
        "                                              \"Data\" : \"Test\",\n",
        "                                              \"NumOfA\": num_of_models_cnn, \n",
        "                                              \"NumOfB\": num_of_models_dnn, \n",
        "                                              \"Layer\": \"LastDense\",\n",
        "                                              \"RepC\": repc, \n",
        "                                              \"Pr\": pr,\n",
        "                                              \"Rc\": rc,\n",
        "                                              \"F1\": f1,\n",
        "                                              \"Acc\": acc\n",
        "                                              }, index = [idxCount])\n",
        "                                          ])\n",
        "      idxCount = idxCount + 1\n",
        "    cnn_dnn_ld_contour_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/temp/cnn_dnn_ldtemp_contour_data_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n",
        "    \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jXbwuusKOFDo"
      },
      "outputs": [],
      "source": [
        "cnn_dnn_ld_contour_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/cnn_dnn_ld_contour_data_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FWDC54QRQhyy"
      },
      "source": [
        "# Data for Plot on Collaborative DNN last dense layers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jckmvYAKQhyy"
      },
      "outputs": [],
      "source": [
        "collaborativeFullyC_lastdense_data = None\n",
        "\n",
        "x_input = test_data_grey\n",
        "y_input = test_targets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qzVNJs1VQhyy"
      },
      "outputs": [],
      "source": [
        "# dnn_features_files_here = [ f for f in dnn_features_files if layer_name_sfmx_dnn in f]\n",
        "# dnn_features_files_here\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o51PtIPEQhyy"
      },
      "outputs": [],
      "source": [
        "# set up the data for the DNN collaborative\n",
        "\n",
        "import random\n",
        "import datetime\n",
        "\n",
        "num_of_repeats = 4\n",
        "num_of_models = [25,30,35,40]\n",
        "# [2,3,4,5,6,8,10,12,14,16,18,20]\n",
        "layer_name_penultimate_dnn = \"D3R\"\n",
        "\n",
        "idxCount = 0 if collaborativeFullyC_lastdense_data is None else len(collaborativeFullyC_lastdense_data.index)\n",
        "for repc in range(num_of_repeats):\n",
        "  for mc in num_of_models:\n",
        "    model_name = f\"Collab_{layer_name_penultimate_dnn}_DNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\"\n",
        "    np_x_validation_collab_dnn_penultimate, np_y_train_collab_dnn_penultimate, np_x_test_collab_dnn_penultimate = get_features_for_layer(dnn_features_files, layer_name_penultimate_dnn, mc)\n",
        "    model_here, history_here = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "                        model_name, \n",
        "                        np_y_train_collab_dnn_penultimate[0,:].shape, \n",
        "                        np_y_train_collab_dnn_penultimate, \n",
        "                        train_targets,\n",
        "                        save_max_epoch = False,\n",
        "                        save_final = True,\n",
        "                        patience_count = 35,\n",
        "                        early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                        log_history = True,\n",
        "                        verbose_level=0,                             \n",
        "                        batch_size=512, \n",
        "                        epochs=250, \n",
        "                        class_weight=None, \n",
        "                        validation_data=(np_x_validation_collab_dnn_penultimate, validation_targets))\n",
        "    del np_x_validation_collab_dnn_penultimate\n",
        "    del np_y_train_collab_dnn_penultimate\n",
        "    y_pred_model = model_here.predict(np_x_test_collab_dnn_penultimate)\n",
        "    del np_x_test_collab_dnn_penultimate\n",
        "    y_pred = np.apply_along_axis(np.argmax, 1, y_pred_model) \n",
        "    pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_pred,test_targets)\n",
        "    print (mc, repc, pr, rc, f1, acc)\n",
        "    del model_here\n",
        "    del history_here\n",
        "    if collaborativeFullyC_lastdense_data is None:\n",
        "      collaborativeFullyC_lastdense_data = pd.DataFrame({\"Type\": \"DNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"Layer\" : layer_name_penultimate_dnn,\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            }, index = [idxCount])\n",
        "    else:\n",
        "      collaborativeFullyC_lastdense_data = pd.concat([collaborativeFullyC_lastdense_data,\n",
        "                                         pd.DataFrame({\"Type\": \"DNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"Layer\" : layer_name_penultimate_dnn,\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc\n",
        "                                            }, index = [idxCount])\n",
        "                                         ])\n",
        "    idxCount = idxCount + 1\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "w5v1LfD_Qhyy"
      },
      "outputs": [],
      "source": [
        "collaborativeFullyC_lastdense_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/dnn_collab_lastdense_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hx1fELMRQhyy"
      },
      "source": [
        "# Data for Plot on Collaborative CNN last dense layers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jse5z9uRQhyy"
      },
      "outputs": [],
      "source": [
        "# carry this over from DNN if possible (so keep commented out)\n",
        "## collaborativeFullyC_softmax_data = None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_Ej69RilQhyz"
      },
      "outputs": [],
      "source": [
        "# set up the data for the CNN collaborative\n",
        "\n",
        "import random\n",
        "import datetime\n",
        "\n",
        "num_of_repeats = 4\n",
        "num_of_models = [35, 40] # [22, 25,30]\n",
        "# [2,3,4,5,6,8,10,12,14,16,18,20]\n",
        "layer_name_penultimate_cnn = \"DRP1\"\n",
        "\n",
        "idxCount = 0 if collaborativeFullyC_lastdense_data is None else len(collaborativeFullyC_lastdense_data.index)\n",
        "for repc in range(num_of_repeats):\n",
        "  for mc in num_of_models:\n",
        "    model_name = f\"Collab_{layer_name_penultimate_cnn}_CNN_A_{datetime.datetime.now():%Y%m%d%H%M%S}\"\n",
        "    np_x_validation_collab_cnn_penultimate, np_y_train_collab_cnn_penultimate, np_x_test_collab_cnn_penultimate = get_features_for_layer(cnn_features_files, layer_name_penultimate_cnn, mc)\n",
        "    model_here, history_here = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "                        model_name, \n",
        "                        np_y_train_collab_cnn_penultimate[0,:].shape, \n",
        "                        np_y_train_collab_cnn_penultimate, \n",
        "                        train_targets,\n",
        "                        save_max_epoch = False,\n",
        "                        save_final = True,\n",
        "                        patience_count = 35,\n",
        "                        early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                        log_history = True,\n",
        "                        verbose_level=0,                             \n",
        "                        batch_size=512, \n",
        "                        epochs=250, \n",
        "                        class_weight=None, \n",
        "                        validation_data=(np_x_validation_collab_cnn_penultimate, validation_targets))\n",
        "    del np_x_validation_collab_cnn_penultimate\n",
        "    del np_y_train_collab_cnn_penultimate\n",
        "    y_pred_model = model_here.predict(np_x_test_collab_cnn_penultimate)\n",
        "    del np_x_test_collab_cnn_penultimate\n",
        "    y_pred = np.apply_along_axis(np.argmax, 1, y_pred_model) \n",
        "    pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_pred,test_targets)\n",
        "    print (mc, repc, pr, rc, f1, acc)\n",
        "    del model_here\n",
        "    del history_here\n",
        "    if collaborativeFullyC_lastdense_data is None:\n",
        "      collaborativeFullyC_lastdense_data = pd.DataFrame({\"Type\": \"CNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"Layer\" : layer_name_penultimate_cnn,\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            }, index = [idxCount])\n",
        "    else:\n",
        "      collaborativeFullyC_lastdense_data = pd.concat([collaborativeFullyC_lastdense_data,\n",
        "                                         pd.DataFrame({\"Type\": \"CNN\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"Layer\" : layer_name_penultimate_cnn,\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc\n",
        "                                            }, index = [idxCount])\n",
        "                                         ])\n",
        "    idxCount = idxCount + 1\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "k2iGJLv1Qhyz"
      },
      "outputs": [],
      "source": [
        "collaborativeFullyC_lastdense_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/cnn_collab_lastdense_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "io3MAwHuQhyz"
      },
      "source": [
        "# Data for Plot on Collaborative WideResNet last dense layers (does not work)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XKjiiIoMQhyz"
      },
      "outputs": [],
      "source": [
        "# carry this over from DNN/CNN if possible (so keep commented out)\n",
        "## collaborativeFullyC_softmax_data = None\n",
        "\n",
        "# wideresnets_features_files\n",
        "# cnn_features_files\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KGA4KvQeQhyz"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import re\n",
        "\n",
        "def get_base_patterns_for_validation(features_files, layer_name = \"CLASSIFIER_D1\", model_type = \"WideResNet\"):\n",
        "  base_patterns_for_validations = []\n",
        "  for ff in [ s for s in features_files if \"Validation\" in s]:\n",
        "    validation_search = re.search(f'^.*({model_type}.*_ID.*features_{layer_name}_Validation)[0-9]+.*$', ff, re.IGNORECASE)\n",
        "    if validation_search:\n",
        "        base_patterns_for_validations.append(validation_search.group(1))\n",
        "\n",
        "  base_patterns_for_validations = sorted(list(set(base_patterns_for_validations)))\n",
        "  return base_patterns_for_validations\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TaD_SmR_Qhyz"
      },
      "outputs": [],
      "source": [
        "def get_features_for_layer_using_subbatches(feature_files, layer_name, num_of_models, model_type=\"WideResNet\", axis_to_concat = 1):\n",
        " \n",
        "  feature_files_used = [ff for ff in feature_files if \"Validation\" in ff and \"_X\" in ff and f\"_{layer_name}_\" in ff ]\n",
        "  \n",
        "  # base_patterns_for_validations = []\n",
        "  # for ff in [ s for s in feature_files_used ]:\n",
        "  #   validation_search = re.search(f'^.*(_ID.*features_{layer_name}_Validation)[0-9]+.*$', ff, re.IGNORECASE)\n",
        "  #   if validation_search:\n",
        "  #       base_patterns_for_validations.append(validation_search.group(1))\n",
        "  # base_patterns_for_validations = sorted(list(set(base_patterns_for_validations)))\n",
        "\n",
        "  base_patterns_for_validations = get_base_patterns_for_validation(feature_files_used, layer_name, model_type)\n",
        "  base_patterns_for_validations = sorted(random.sample(base_patterns_for_validations, min(num_of_models,len(base_patterns_for_validations)) ))\n",
        "\n",
        "  np_x_validation_collab = None\n",
        "  for base_val_str in base_patterns_for_validations:\n",
        "      validation_batch_files = sorted([ ff for ff in feature_files_used if base_val_str in ff])\n",
        "      np_x_validation_collab_batch = np.array([np.load(ff) for ff in validation_batch_files])\n",
        "      np_x_validation_collab_batch = np.concatenate(np_x_validation_collab_batch, axis=0)\n",
        "      if np_x_validation_collab is None:\n",
        "        np_x_validation_collab = np_x_validation_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_validation_collab = np.concatenate([np_x_validation_collab, np_x_validation_collab_batch], axis=axis_to_concat)\n",
        "  \n",
        "  np_x_test_collab = None\n",
        "  for base_val_str in [ ff.replace(\"Validation\",\"Test\") for ff in base_patterns_for_validations]:\n",
        "      test_batch_files = dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{base_val_str}.*_X.*$\")\n",
        "      np_x_test_collab_batch = np.array([np.load(ff) for ff in test_batch_files])\n",
        "      np_x_test_collab_batch = np.concatenate(np_x_test_collab_batch, axis=0)\n",
        "      if np_x_test_collab is None:\n",
        "        np_x_test_collab = np_x_test_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_test_collab = np.concatenate([np_x_test_collab, np_x_test_collab_batch], axis=axis_to_concat)\n",
        "\n",
        "  np_x_train_collab = None\n",
        "  for base_val_str in [ ff.replace(\"Validation\",\"Train\") for ff in base_patterns_for_validations]:\n",
        "      train_batch_files = dir_has_file_with_regex(f\"/content/drive/MyDrive/data_papers/{paper_name}/model_features\",f\".*{base_val_str}.*_X.*$\")\n",
        "      np_x_train_collab_batch = np.array([np.load(ff) for ff in train_batch_files])\n",
        "      np_x_train_collab_batch = np.concatenate(np_x_train_collab_batch, axis=0)\n",
        "      if np_x_train_collab is None:\n",
        "        np_x_train_collab = np_x_train_collab_batch.copy()\n",
        "      else:\n",
        "        np_x_train_collab = np.concatenate([np_x_train_collab, np_x_train_collab_batch], axis=axis_to_concat)\n",
        "\n",
        "  return np_x_validation_collab, np_x_train_collab, np_x_test_collab"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qka-rNJfQhyz"
      },
      "outputs": [],
      "source": [
        "# set up the data for the WideResNet collaborative\n",
        "\n",
        "import random\n",
        "import datetime\n",
        "\n",
        "num_of_repeats = 4\n",
        "num_of_models = [2,3,4,5,6,7,8,9,10]\n",
        "layer_name_penultimate_wideresnets = \"CLASSIFIER_FL\"\n",
        "model_type = \"WideResNet\"\n",
        "\n",
        "idxCount = 0 if collaborativeFullyC_lastdense_data is None else len(collaborativeFullyC_lastdense_data.index)\n",
        "for repc in range(num_of_repeats):\n",
        "  for mc in num_of_models:\n",
        "    model_name = f\"Collab_{layer_name_penultimate_wideresnets}_WideResNet_A_{datetime.datetime.now():%Y%m%d%H%M%S}\"\n",
        "    np_x_validation_collab_wideresnets_penultimate, np_y_train_collab_wideresnets_penultimate, np_x_test_collab_wideresnets_penultimate = get_features_for_layer_using_subbatches(wideresnets_features_files,layer_name_penultimate_wideresnets,mc,model_type )\n",
        "    model_here, history_here = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "                        model_name, \n",
        "                        np_y_train_collab_wideresnets_penultimate[0,:].shape, \n",
        "                        np_y_train_collab_wideresnets_penultimate, \n",
        "                        train_targets,\n",
        "                        save_max_epoch = False,\n",
        "                        save_final = True,\n",
        "                        patience_count = 35,\n",
        "                        early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "                        log_history = True,\n",
        "                        verbose_level=0,                             \n",
        "                        batch_size=512, \n",
        "                        epochs=250, \n",
        "                        class_weight=None, \n",
        "                        validation_data=(np_x_validation_collab_wideresnets_penultimate, validation_targets))\n",
        "    del np_x_validation_collab_wideresnets_penultimate\n",
        "    del np_y_train_collab_wideresnets_penultimate\n",
        "    y_pred_model = model_here.predict(np_x_test_collab_wideresnets_penultimate)\n",
        "    del np_x_test_collab_wideresnets_penultimate\n",
        "    y_pred = np.apply_along_axis(np.argmax, 1, y_pred_model) \n",
        "    pr, rc, f1, acc = pr_rc_f1_acc_from_supplied(y_pred,test_targets)\n",
        "    print (mc, repc, pr, rc, f1, acc)\n",
        "    del model_here\n",
        "    del history_here\n",
        "    if collaborativeFullyC_lastdense_data is None:\n",
        "      collaborativeFullyC_lastdense_data = pd.DataFrame({\"Type\": \"WideResNet\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"Layer\" : layer_name_penultimate_wideresnets,\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc,\n",
        "                                            }, index = [idxCount])\n",
        "    else:\n",
        "      collaborativeFullyC_lastdense_data = pd.concat([collaborativeFullyC_lastdense_data,\n",
        "                                         pd.DataFrame({\"Type\": \"WideResNet\", \n",
        "                                            \"Data\" : \"Test\",\n",
        "                                            \"Layer\" : layer_name_penultimate_wideresnets,\n",
        "                                            \"NumOfModels\": mc, \n",
        "                                            \"RepC\": repc, \n",
        "                                            \"Pr\": pr,\n",
        "                                            \"Rc\": rc,\n",
        "                                            \"F1\": f1,\n",
        "                                            \"Acc\": acc\n",
        "                                            }, index = [idxCount])\n",
        "                                         ])\n",
        "    collaborativeFullyC_lastdense_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/wideresnet_collab_lastdense_interim_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n",
        "    idxCount = idxCount + 1\n",
        "\n",
        "\n",
        "# np_x_validation_collab_wideresnets, np_x_train_collab_wideresnets, np_x_test_collab_wideresnets =  get_features_for_layer_using_subbatches(wideresnets_features_files,layer_name,num_of_models,model_type )\n",
        "# wrsftmx, wrsftmxh = compile_and_fit_model_basic( model_combination_of_features,  \n",
        "#                     f\"Collab_{layer_name}_WideResnet28-10_A_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "#                     np_x_train_collab_wideresnets[0,:].shape, \n",
        "#                     np_x_train_collab_wideresnets, \n",
        "#                     train_targets,\n",
        "#                     save_max_epoch=False,\n",
        "#                     save_final=True,\n",
        "#                     patience_count = 35,\n",
        "#                     early_stopping_obs = 'val_sparse_categorical_accuracy',\n",
        "#                     log_history = True,\n",
        "#                     verbose_level=1,                             \n",
        "#                     batch_size=512, \n",
        "#                     epochs=250, \n",
        "#                     class_weight=None, \n",
        "#                     validation_data=(np_x_validation_collab_wideresnets, validation_targets))\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KlsYre4_Qhyz"
      },
      "outputs": [],
      "source": [
        "collaborativeFullyC_lastdense_data.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/wideresnet_collab_lastdense_test_results_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cxBmUlk5Qhy0"
      },
      "source": [
        "# Plot on Collaborative CNN/DNN/WideResNet last dense layers as function of number of models"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YJBHqxkQQhy0"
      },
      "outputs": [],
      "source": [
        "all_lastdense_collab_data = pd.concat([ pd.read_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/{f}\") for f in os.listdir(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/\") if \"collab_lastdense_test_results\" in f ])\n",
        "all_lastdense_collab_data = all_lastdense_collab_data.drop_duplicates()\n",
        "all_lastdense_collab_data[\"BestComponent\"] = np.nan\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "89TQ2Q1IQhy0"
      },
      "outputs": [],
      "source": [
        "# read validation accur\n",
        "val_accs = pd.read_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/val_test_accs_20211118165251.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "go025R4GQhy0"
      },
      "outputs": [],
      "source": [
        "best_submodels = val_accs.loc[val_accs.groupby(\"Type\")['ValAcc'].idxmax()].copy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VvmWk6U1Qhy0"
      },
      "outputs": [],
      "source": [
        "for typename in best_submodels[\"Type\"]:\n",
        "  all_lastdense_collab_data.loc[all_lastdense_collab_data.Type==typename,\"BestComponent\"] = float(best_submodels.loc[best_submodels.Type==typename, \"TestAcc\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ApUSqTHOQhy0"
      },
      "outputs": [],
      "source": [
        "all_lastdense_collab_data = pd.concat([all_lastdense_collab_data.groupby([\"Type\",\"NumOfModels\"])[\"Acc\"].mean(), all_lastdense_collab_data.groupby([\"Type\",\"NumOfModels\"])[\"BestComponent\"].mean()],axis=1)\n",
        "all_lastdense_collab_data = all_lastdense_collab_data.reset_index()\n",
        "\n",
        "# all_lastdense_collab_data[all_lastdense_collab_data.NumOfModels<=2]\n",
        "# all_lastdense_collab_data = all_lastdense_collab_data[all_lastdense_collab_data.NumOfModels>2]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4Pk8mYEnQhy0"
      },
      "outputs": [],
      "source": [
        "ggplot(all_lastdense_collab_data) +  \\\n",
        "  geom_line(aes(x='NumOfModels',y='Acc',group='Type',color='Type', fill='Type'),size=2) + \\\n",
        "  geom_line(aes(x='NumOfModels',y='BestComponent',group='Type',color='Type', fill='Type'), linetype='dashed')  + \\\n",
        "  theme_bw() + ggtitle('Test Accuracy for Collab models')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fEPMLKUxQhy0"
      },
      "outputs": [],
      "source": [
        "all_lastdense_collab_data.sample(10)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JBcVku5ACyVv"
      },
      "outputs": [],
      "source": [
        "all_lastdense_collab_data[\"Layer\"] = \"LASTDENSE\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Z9SnOmMTKbj"
      },
      "source": [
        "# A GP collaborative model on the softmax layer features of input DNNX/CNNX/CNNX0.5+DNNX0.5/WideResNet28-10"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-dyXfmdTb1aU"
      },
      "outputs": [],
      "source": [
        "import gpflow\n",
        "from gpflow.utilities import ops, print_summary, set_trainable\n",
        "from gpflow.config import set_default_float, default_float, set_default_summary_fmt\n",
        "from gpflow.ci_utils import ci_niter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SUt2vbQC62W9"
      },
      "outputs": [],
      "source": [
        "import inspect\n",
        "  \n",
        "def retrieve_name(var):\n",
        "    callers_local_vars = inspect.currentframe().f_back.f_locals.items()\n",
        "    return [var_name for var_name, var_val in callers_local_vars if var_val is var]\n",
        " \n",
        "def mod_retrieve_name(var):\n",
        "    callers_local_vars = inspect.currentframe().f_back.f_back.f_locals.items()\n",
        "    return [var_name for var_name, var_val in callers_local_vars if var_val is var]\n",
        " \n",
        "def foo(bar):\n",
        "\tprint(retrieve_name(bar))\n",
        "\tprint(mod_retrieve_name(bar))\n",
        "\n",
        "# x,y,z = 1,2,3\n",
        "# foo(x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "busXdvq-TKbj"
      },
      "outputs": [],
      "source": [
        "# set up the data for the collaborative models using GPs from DNN and CNN softmax\n",
        "\n",
        "num_of_models = 20\n",
        "\n",
        "layer_name_sftmx_dnn = \"SFTMX1\"\n",
        "np_x_validation_collab_dnn, np_x_train_collab_dnn, np_x_test_collab_dnn = get_features_for_layer(dnn_features_files, layer_name_sftmx_dnn, num_of_models)\n",
        "\n",
        "layer_name_sftmx_cnn = \"SFTMX1\"\n",
        "np_x_validation_collab_cnn, np_x_train_collab_cnn, np_x_test_collab_cnn = get_features_for_layer(cnn_features_files, layer_name_sftmx_cnn, num_of_models)\n",
        "\n",
        "# set up the data for the collaborative models using GPs from WideResNet softmax\n",
        "\n",
        "layer_name_sftmx_wideresnets = \"CLASSIFIER_D1\"\n",
        "np_x_validation_collab_wideresnets, np_x_train_collab_wideresnets, np_x_test_collab_wideresnets = get_features_for_layer_using_subbatches(wideresnets_features_files, layer_name_sftmx_wideresnets, num_of_models)\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YI3qO9NDwcbj"
      },
      "source": [
        "Setup the GP-specific data structure for the DNN/CNN/WideResNet softmax features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UUDOr6jhn4qN"
      },
      "outputs": [],
      "source": [
        "def setup_feature_data_for_gp(np_x_train_collab, np_x_test_collab, train_targets, test_targets):\n",
        "  \n",
        "  num_of_data_points = np_x_train_collab.shape[0] \n",
        "  num_classes = np.unique(train_targets[:num_of_data_points,:]).size\n",
        "  num_of_functions = num_classes\n",
        "  num_of_independent_vars = np_x_train_collab.shape[-1]\n",
        "  num_of_test_data_points = min(num_of_data_points,test_targets.shape[0])\n",
        "\n",
        "  data_gp_train = np_x_train_collab[:num_of_data_points,:]\n",
        "  data_gp_test = np_x_test_collab[:num_of_test_data_points,:]\n",
        "\n",
        "  data_gp_train_target_hot = np.squeeze(np.eye(num_classes)[train_targets[:num_of_data_points,:]].astype(bool))\n",
        "  data_gp_train_target = np.apply_along_axis(np.argmax, 1, data_gp_train_target_hot)\n",
        "  data_gp_train_target = np.expand_dims(data_gp_train_target, axis=1)\n",
        "\n",
        "  data_gp_test_target_hot = np.squeeze(np.eye(num_classes)[test_targets[:num_of_test_data_points,:]].astype(bool))\n",
        "  data_gp_test_target = np.apply_along_axis(np.argmax, 1, data_gp_test_target_hot)\n",
        "  data_gp_test_target = np.expand_dims(data_gp_test_target, axis=1)\n",
        "\n",
        "  data_gp = ( data_gp_train, data_gp_train_target )\n",
        "  return data_gp, data_gp_test, data_gp_test_target\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3nog8X09wcbn"
      },
      "outputs": [],
      "source": [
        "# reproducibility:\n",
        "np.random.seed(0)\n",
        "tf.random.set_seed(123)\n",
        "\n",
        "data_gp_dnnf, data_gp_test_dnnf, data_gp_test_target_dnnf = setup_feature_data_for_gp(np_x_train_collab_dnn,  np_x_test_collab_dnn, train_targets, test_targets   )\n",
        "data_gp_cnnf, data_gp_test_cnnf, data_gp_test_target_cnnf = setup_feature_data_for_gp(np_x_train_collab_cnn,  np_x_test_collab_cnn, train_targets, test_targets   )\n",
        "data_gp_wideresnets, data_gp_test_wideresnets, data_gp_test_target_wideresnets = setup_feature_data_for_gp(np_x_train_collab_wideresnets,  np_x_test_collab_wideresnets, train_targets, test_targets   )\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "I2BuEPuRS4Tt"
      },
      "outputs": [],
      "source": [
        "def construct_kernel_list(num_of_independent_vars, base_lengthscales = [0.1]):\n",
        "  possible_kernels = []\n",
        "  for i in range(len(base_lengthscales)):\n",
        "    possible_kernels.append(gpflow.kernels.Matern12(variance=1.0, lengthscales=[base_lengthscales[i]]*num_of_independent_vars))\n",
        "    possible_kernels.append(gpflow.kernels.Matern32(variance=1.0, lengthscales=[base_lengthscales[i]]*num_of_independent_vars))\n",
        "    possible_kernels.append(gpflow.kernels.RBF(variance=1.0, lengthscales=[base_lengthscales[i]]*num_of_independent_vars))\n",
        "    # possible_kernels.append(gpflow.kernels.Matern52(variance=1.0, lengthscales=[base_lengthscales[i]]*num_of_independent_vars))\n",
        "    # possible_kernels.append(gpflow.kernels.SquaredExponential(lengthscales=[base_lengthscales[i]]*num_of_independent_vars))  \n",
        "\n",
        "  return possible_kernels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "g_qZ_vVF5KrG"
      },
      "outputs": [],
      "source": [
        "# print(np_x_validation_collab_wideresnets.shape)\n",
        "# print(np_x_test_collab_wideresnets.shape)\n",
        "# print(np_x_train_collab_wideresnets.shape)\n",
        "\n",
        "# print(np_x_validation_collab_cnn.shape)\n",
        "# print(np_x_test_collab_cnn.shape)\n",
        "# print(np_x_train_collab_cnn.shape)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6zIkvKG7qLKB"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import os,sys\n",
        "\n",
        "\n",
        "def run_gp_directly_on_input(data_gp, \n",
        "                             data_gp_test,\n",
        "                             data_gp_test_target,\n",
        "                             id_str,\n",
        "                             nth_inducing_ratio = 120, \n",
        "                             var_list = [0.1, 1.0],       \n",
        "                             early_stop_count = 100,\n",
        "                             early_stop_check_interval = 1000,\n",
        "                             early_stop_elbo_factor = 1.0001,\n",
        "                             max_niters = 200000):\n",
        "\n",
        "  num_of_independent_vars = data_gp[0].shape[1]\n",
        "  num_of_classes = np.unique(data_gp[1]).size\n",
        "  num_of_functions = num_of_classes\n",
        "  possible_kernels = construct_kernel_list(num_of_independent_vars, var_list)\n",
        "  # Robustmax Multiclass Likelihood\n",
        "  invlink = gpflow.likelihoods.RobustMax(num_of_functions)  # Robustmax inverse link function\n",
        "  likelihood = gpflow.likelihoods.MultiClass(num_of_functions, invlink=invlink)  # Multiclass likelihood\n",
        "  idxs_of_induced = sorted(random.sample(range(data_gp[0].shape[0]),int(data_gp[0].shape[0]/nth_inducing_ratio)))\n",
        "  inducing_inputs = data_gp[0][idxs_of_induced,:].copy()  # inducing inputs\n",
        "  gp_models = []\n",
        "  for kernel in possible_kernels:\n",
        "    m = gpflow.models.SVGP(\n",
        "        kernel=kernel,\n",
        "        likelihood=likelihood,\n",
        "        inducing_variable=inducing_inputs,\n",
        "        num_latent_gps=num_of_functions,\n",
        "        whiten=True,\n",
        "        q_diag=True,\n",
        "    )\n",
        "    # Only train the variational parameters\n",
        "    # set_trainable(m.kernel.kernels[1].variance, False)\n",
        "    set_trainable(m.inducing_variable, True)\n",
        "    gp_models.append(m)\n",
        "  result_dict = dict()\n",
        "  for mcount,m in enumerate(gp_models):\n",
        "      print(mcount)\n",
        "      tensor_data = tuple(map(tf.convert_to_tensor, data_gp))\n",
        "      training_loss = m.training_loss_closure(tensor_data, compile=True)\n",
        "      starting_elbo = -training_loss().numpy()\n",
        "      print(f'Starting ELBO {starting_elbo}')\n",
        "      elbos = [training_loss().numpy()]\n",
        "      optimizer = tf.optimizers.Adam()  \n",
        "\n",
        "      # optimizer = tf.optimizers.RMSprop()\n",
        "      @tf.function\n",
        "      def optim_here():\n",
        "          optimizer.minimize(training_loss, m.trainable_variables)\n",
        "\n",
        "      try:\n",
        "        for itc in range(max_niters):\n",
        "            optim_here()\n",
        "            elbo_now = -training_loss().numpy()\n",
        "            elbos.append(elbo_now)\n",
        "            if (itc % early_stop_check_interval) == 0:\n",
        "              print(f'ELBO {elbo_now}')\n",
        "              if len(elbos) > (early_stop_count+1):\n",
        "                if elbos[-early_stop_count] >= elbos[-1]*early_stop_elbo_factor:\n",
        "                    print(f'Early stopping at {itc}')\n",
        "                    break          \n",
        "            # needs at least a decent improvement\n",
        "\n",
        "        print(f'Ending ELBO {elbos[-1]}')\n",
        "\n",
        "        pY, pYv = m.predict_y(data_gp_test)  # Predict Y values at test locations\n",
        "        result_dict[f\"{gp_models[0].kernel.name}_{mcount}_{id_str}\"] = dict({'model': m, 'X': data_gp_test, 'Y': data_gp_test_target, 'pY': pY, 'pYv': pYv})\n",
        "        np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/feature_collabUsingGP_{id_str}_Test_{gp_models[mcount].kernel.name}_{mcount}_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "            np.array(pY), \n",
        "            allow_pickle=True, \n",
        "            fix_imports=True)\n",
        "        np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/feature_collabUsingGP_{id_str}_Test_Targets_{gp_models[mcount].kernel.name}_{mcount}_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "            np.array(data_gp[1]), \n",
        "            allow_pickle=True, \n",
        "            fix_imports=True)\n",
        "        tf.saved_model.save(m, f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/feature_collabUsingGP_{id_str}_model_{gp_models[mcount].kernel.name}_{mcount}_{datetime.datetime.now():%Y%m%d%H%M%S}\")\n",
        "        print_summary(m, fmt=\"notebook\")  \n",
        "      except:\n",
        "        print(f'ERROR, Ending ELBO {elbos[-1]} (NOT saved)')\n",
        "        next\n",
        "\n",
        "  return result_dict\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7en8tCffS4Tt"
      },
      "source": [
        "Do a Collobarative for the softmax features of DNN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eMoheCCYyyD5"
      },
      "outputs": [],
      "source": [
        "layer_name_sftmx_dnn = \"SFTMX1\"\n",
        "gp_dnn_sftmx_dict = run_gp_directly_on_input(data_gp_dnnf, \n",
        "                             data_gp_test_dnnf,\n",
        "                             data_gp_test_target_dnnf,\n",
        "                             f\"DNN_{layer_name_sftmx_dnn}_ID{str(uuid.uuid4()).split('-')[0]}\",\n",
        "                             nth_inducing_ratio = 120, \n",
        "                             var_list = [0.1, 1.0],       \n",
        "                             early_stop_count = 100,\n",
        "                             early_stop_check_interval = 1000,\n",
        "                             early_stop_elbo_factor = 1.0001,\n",
        "                             max_niters = 200000)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "95F5SdFc3l2v"
      },
      "source": [
        "Do a Collobarative for the softmax features of CNN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pPFjYugx3l2w"
      },
      "outputs": [],
      "source": [
        "layer_name_sftmx_cnn = \"SFTMX1\"\n",
        "gp_cnn_sftmx_dict = run_gp_directly_on_input(data_gp_cnnf, \n",
        "                             data_gp_test_cnnf,\n",
        "                             data_gp_test_target_cnnf,\n",
        "                             f\"CNN_{layer_name_sftmx_cnn}_ID{str(uuid.uuid4()).split('-')[0]}\",\n",
        "                             nth_inducing_ratio = 120, \n",
        "                             var_list = [0.1, 1.0],       \n",
        "                             early_stop_count = 100,\n",
        "                             early_stop_check_interval = 1000,\n",
        "                             early_stop_elbo_factor = 1.0001,\n",
        "                             max_niters = 200000)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V4nn00xj33dI"
      },
      "source": [
        "Do a Collobarative for the softmax features of WideResNet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0sPjC97G33dI"
      },
      "outputs": [],
      "source": [
        "layer_name_sftmx_wideresnets = \"CLASSIFIER_D1\"\n",
        "gp_wideresnets_sftmx_dict = run_gp_directly_on_input(data_gp_wideresnets, \n",
        "                             data_gp_test_wideresnets,\n",
        "                             data_gp_test_target_wideresnets,\n",
        "                             f\"WideResNet28-10_{layer_name_sftmx_wideresnets}_ID{str(uuid.uuid4()).split('-')[0]}\",\n",
        "                             nth_inducing_ratio = 120, \n",
        "                             var_list = [0.1, 1.0],       \n",
        "                             early_stop_count = 100,\n",
        "                             early_stop_check_interval = 1000,\n",
        "                             early_stop_elbo_factor = 1.0001,\n",
        "                             max_niters = 200000)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Y3rtsniIwbwP"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yHXkywirRpmM"
      },
      "source": [
        "# A collaborative GP model on the penultimate layer features of input DNNX/CNNX/CNNX0.5+DNNX0.5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tdG2Foo36ck8"
      },
      "outputs": [],
      "source": [
        "# set up the data for the collaborative models using GPs\n",
        "\n",
        "num_of_models = 20\n",
        "\n",
        "layer_name_penultimate_dnn = \"D3R\"\n",
        "layer_name_penultimate_cnn = \"DRP1\"\n",
        "\n",
        "validation_features_to_load_dnn = sorted(random.sample([ff for ff in dnn_features_files if \"Validation_X.npy\" in ff and f\"_{layer_name_penultimate_dnn}_\" in ff ], num_of_models))\n",
        "np_x_validation_collab_dnn = np.array([np.load(ff) for ff in validation_features_to_load_dnn ])\n",
        "np_x_validation_collab_dnn = np.concatenate(np_x_validation_collab_dnn, axis=1)\n",
        "\n",
        "train_features_to_load_dnn = [ff.replace(\"Validation\", \"Train\") for ff in validation_features_to_load_dnn]\n",
        "np_x_train_collab_dnn = np.array([np.load(ff) for ff in train_features_to_load_dnn ])\n",
        "np_x_train_collab_dnn = np.concatenate(np_x_train_collab_dnn, axis=1)\n",
        "\n",
        "test_features_to_load_dnn = [ff.replace(\"Validation\", \"Test\") for ff in validation_features_to_load_dnn]\n",
        "np_x_test_collab_dnn = np.array([np.load(ff) for ff in test_features_to_load_dnn ])\n",
        "np_x_test_collab_dnn = np.concatenate(np_x_test_collab_dnn, axis=1)\n",
        "\n",
        "validation_features_to_load_cnn = sorted(random.sample([ff for ff in cnn_features_files if \"Validation_X.npy\" in ff and f\"_{layer_name_penultimate_cnn}_\" in ff], num_of_models))\n",
        "np_x_validation_collab_cnn = np.array([np.load(ff) for ff in validation_features_to_load_cnn ])\n",
        "np_x_validation_collab_cnn = np.concatenate(np_x_validation_collab_cnn, axis=1)\n",
        "\n",
        "train_features_to_load_cnn = [ff.replace(\"Validation\", \"Train\") for ff in validation_features_to_load_cnn]\n",
        "np_x_train_collab_cnn = np.array([np.load(ff) for ff in train_features_to_load_cnn ])\n",
        "np_x_train_collab_cnn = np.concatenate(np_x_train_collab_cnn, axis=1)\n",
        "\n",
        "test_features_to_load_cnn = [ff.replace(\"Validation\", \"Test\") for ff in validation_features_to_load_cnn]\n",
        "np_x_test_collab_cnn = np.array([np.load(ff) for ff in test_features_to_load_cnn ])\n",
        "np_x_test_collab_cnn = np.concatenate(np_x_test_collab_cnn, axis=1)\n",
        "\n",
        "validation_features_to_load_cnn = sorted(random.sample([ff for ff in cnn_features_files if \"Validation_X.npy\" in ff and f\"_{layer_name_penultimate_cnn}_\" in ff], int(num_of_models/2)))\n",
        "validation_features_to_load_dnn  = sorted(random.sample([ff for ff in dnn_features_files if \"Validation_X.npy\" in ff and f\"_{layer_name_penultimate_dnn}_\" in ff], int(num_of_models/2)))\n",
        "\n",
        "validation_features_cnn = np.array([np.load(ff) for ff in validation_features_to_load_cnn ])\n",
        "validation_features_dnn = np.array([np.load(ff) for ff in validation_features_to_load_dnn ])\n",
        "np_x_validation_collab_cdnn = np.concatenate([validation_features_cnn,validation_features_dnn], axis=2)\n",
        "\n",
        "train_features_to_load_cnn = [ff.replace(\"Validation\", \"Train\") for ff in validation_features_to_load_cnn]\n",
        "train_features_to_load_dnn = [ff.replace(\"Validation\", \"Train\") for ff in validation_features_to_load_dnn]\n",
        "train_features_cnn = np.array([np.load(ff) for ff in train_features_to_load_cnn ])\n",
        "train_features_dnn = np.array([np.load(ff) for ff in train_features_to_load_dnn ])\n",
        "np_x_train_collab_cdnn = np.concatenate([train_features_cnn,train_features_dnn], axis=2)\n",
        "\n",
        "test_features_to_load_cnn = [ff.replace(\"Validation\", \"Test\") for ff in validation_features_to_load_cnn]\n",
        "test_features_to_load_dnn = [ff.replace(\"Validation\", \"Test\") for ff in validation_features_to_load_dnn]\n",
        "test_features_cnn = np.array([np.load(ff) for ff in test_features_to_load_cnn ])\n",
        "test_features_dnn = np.array([np.load(ff) for ff in test_features_to_load_dnn ])\n",
        "np_x_test_collab_cdnn = np.concatenate([test_features_cnn,test_features_dnn], axis=2)\n",
        "\n",
        "np_x_train_collab_cdnn = np.swapaxes(np_x_train_collab_cdnn, 0,1)\n",
        "np_x_validation_collab_cdnn = np.swapaxes(np_x_validation_collab_cdnn, 0,1)\n",
        "np_x_test_collab_cdnn = np.swapaxes(np_x_test_collab_cdnn, 0,1)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fNEi9qQeUTf1"
      },
      "outputs": [],
      "source": [
        "# np_x_train_collab_dnn\n",
        "# data_gp_train_dnnf\n",
        "# test_targets.shape\n",
        "# data_gp_test_target_dnnf.shape\n",
        "# data_gp_test_dnnf.shape\n",
        "# data_gp_train_dnnf.shape"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A2BIu6tAocat"
      },
      "source": [
        "Setup the GP data for the DNN features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tupBZqUJXG7L"
      },
      "outputs": [],
      "source": [
        "# reproducibility:\n",
        "np.random.seed(0)\n",
        "tf.random.set_seed(123)\n",
        "\n",
        "num_of_data_points_dnnf = np_x_train_collab_dnn.shape[0] # 2000\n",
        "num_classes = np.unique(train_targets[:num_of_data_points_dnnf,:]).size\n",
        "num_of_functions = num_classes\n",
        "num_of_independent_vars_dnn = np_x_train_collab_dnn.shape[-1]\n",
        "num_of_test_data_points_dnnf = min(num_of_data_points_dnnf,test_targets.shape[0])\n",
        "\n",
        "data_gp_train_dnnf = np_x_train_collab_dnn[:num_of_data_points_dnnf,:]     # np_x_train_collab_dnn # np.concatenate((data_train[0],data_train_post[0]), axis = 1)\n",
        "data_gp_test_dnnf = np_x_test_collab_dnn[:num_of_test_data_points_dnnf,:]     # np_x_train_collab_dnn # np.concatenate((data_train[0],data_train_post[0]), axis = 1)\n",
        "\n",
        "data_gp_train_target_hot_dnnf = np.squeeze(np.eye(num_classes)[train_targets[:num_of_data_points_dnnf,:]].astype(bool))\n",
        "data_gp_train_target_dnnf = np.apply_along_axis(np.argmax, 1, data_gp_train_target_hot_dnnf)\n",
        "data_gp_train_target_dnnf = np.expand_dims(data_gp_train_target_dnnf, axis=1)\n",
        "\n",
        "data_gp_test_target_hot_dnnf = np.squeeze(np.eye(num_classes)[test_targets[:num_of_test_data_points_dnnf,:]].astype(bool))\n",
        "data_gp_test_target_dnnf = np.apply_along_axis(np.argmax, 1, data_gp_test_target_hot_dnnf)\n",
        "data_gp_test_target_dnnf = np.expand_dims(data_gp_test_target_dnnf, axis=1)\n",
        "\n",
        "data_gp_dnnf = ( data_gp_train_dnnf, data_gp_train_target_dnnf )\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wp4y41yYbv1B"
      },
      "source": [
        "Setup the GP data for the CNN features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YRrCIHWlbv1B"
      },
      "outputs": [],
      "source": [
        "# reproducibility:\n",
        "np.random.seed(0)\n",
        "tf.random.set_seed(123)\n",
        "\n",
        "num_of_data_points_cnnf = np_x_train_collab_cnn.shape[0] # 2000\n",
        "num_classes = np.unique(train_targets[:num_of_data_points_cnnf,:]).size\n",
        "num_of_functions = num_classes\n",
        "num_of_independent_vars_cnn = np_x_train_collab_cnn.shape[-1]\n",
        "num_of_test_data_points_cnnf = min(num_of_data_points_cnnf,test_targets.shape[0])\n",
        "\n",
        "data_gp_train_cnnf = np_x_train_collab_cnn[:num_of_data_points_cnnf,:]\n",
        "data_gp_test_cnnf = np_x_test_collab_cnn[:num_of_test_data_points_cnnf,:]\n",
        "\n",
        "data_gp_train_target_hot_cnnf = np.squeeze(np.eye(num_classes)[train_targets[:num_of_data_points_cnnf,:]].astype(bool))\n",
        "data_gp_train_target_cnnf = np.apply_along_axis(np.argmax, 1, data_gp_train_target_hot_cnnf)\n",
        "data_gp_train_target_cnnf = np.expand_dims(data_gp_train_target_cnnf, axis=1)\n",
        "\n",
        "data_gp_test_target_hot_cnnf = np.squeeze(np.eye(num_classes)[test_targets[:num_of_test_data_points_cnnf,:]].astype(bool))\n",
        "data_gp_test_target_cnnf = np.apply_along_axis(np.argmax, 1, data_gp_test_target_hot_cnnf)\n",
        "data_gp_test_target_cnnf = np.expand_dims(data_gp_test_target_cnnf, axis=1)\n",
        "\n",
        "data_gp_cnnf = ( data_gp_train_cnnf, data_gp_train_target_cnnf )\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cFnc2Nq7ug2n"
      },
      "outputs": [],
      "source": [
        "# save this down for local analysis\n",
        "npy_to_save = [data_gp_train_dnnf, \n",
        "               data_gp_train_target_dnnf,\n",
        "               data_gp_test_dnnf,\n",
        "               data_gp_test_target_dnnf,\n",
        "               data_gp_train_cnnf, \n",
        "               data_gp_train_target_cnnf,\n",
        "               data_gp_test_cnnf,\n",
        "               data_gp_test_target_cnnf]\n",
        "\n",
        "for npx in npy_to_save:\n",
        "  print([x for x in retrieve_name(npx) if 'data' in x][0])\n",
        "  # np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/features_for_local/{[x for x in retrieve_name(npx) if 'data' in x][0]}_20models_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "  #         npx, \n",
        "  #         allow_pickle=True, \n",
        "  #         fix_imports=True)\n",
        "  # np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/features_for_local/{[x for x in retrieve_name(npx) if 'data' in x][0]}_4models_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "  #         npx, \n",
        "  #         allow_pickle=True, \n",
        "  #         fix_imports=True)\n",
        "  # np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/features_for_local/{[x for x in retrieve_name(npx) if 'data' in x][0]}_2models_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "  #         npx, \n",
        "  #         allow_pickle=True, \n",
        "  #         fix_imports=True)\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z_kpE1wX--Vf"
      },
      "outputs": [],
      "source": [
        "# np_x_train_collab_dnn.shape\n",
        "# np_x_train_collab_dnn[:1000,:].shape\n",
        "# train_targets[:1000]\n",
        "# train_targets.shape\n",
        "# data_gp_train_target_hot\n",
        "# np.apply_along_axis(np.argmax, 1, data_gp_train_target_hot)\n",
        "# data_gp_train_target\n",
        "# num_of_independent_vars\n",
        "# np_x_train_collab_dnn.shape\n",
        "# possible_kernels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wG768Pf7errr"
      },
      "outputs": [],
      "source": [
        "# cnn_features_files"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nJnGnexkeJim"
      },
      "source": [
        "Get the explained variance graph on the DNN features PCA"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LxYELqWFB0AD"
      },
      "outputs": [],
      "source": [
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.decomposition import PCA\n",
        "import tensorflow as tf\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vlXxrzjzC8CM"
      },
      "outputs": [],
      "source": [
        "sc_dnnf = StandardScaler()\n",
        "sc_dnnf.fit(data_gp_train_dnnf)\n",
        "data_gp_train_dnnf_pca = sc_dnnf.transform(data_gp_train_dnnf)\n",
        "data_gp_test_dnnf_pca = sc_dnnf.transform(data_gp_test_dnnf)\n",
        "\n",
        "pca_dnnf = PCA()\n",
        "data_gp_train_dnnf_pca = pca_dnnf.fit_transform(data_gp_train_dnnf_pca)\n",
        "data_gp_test_dnnf_pca = pca_dnnf.transform(data_gp_test_dnnf_pca)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5g4iRu2KDeRr"
      },
      "outputs": [],
      "source": [
        "exp_var_pca_dnnf = pca_dnnf.explained_variance_ratio_\n",
        "cum_sum_eigenvalues_dnnf = np.cumsum(exp_var_pca_dnnf)\n",
        "\n",
        "plt.bar(range(0,len(exp_var_pca_dnnf)), exp_var_pca_dnnf, alpha=0.5, align='center', label='Individual explained variance')\n",
        "plt.step(range(0,len(cum_sum_eigenvalues_dnnf)), cum_sum_eigenvalues_dnnf, where='mid',label='Cumulative explained variance')\n",
        "plt.ylabel('Explained variance ratio _dnnf')\n",
        "plt.xlabel('Principal component index _dnnf')\n",
        "plt.legend(loc='best')\n",
        "plt.tight_layout()\n",
        "plt.show()\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rH6ZnzMyeTga"
      },
      "source": [
        "Get the explained variance graph on the CNN features PCA"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z5394cPreZGS"
      },
      "outputs": [],
      "source": [
        "sc_cnnf = StandardScaler()\n",
        "sc_cnnf.fit(data_gp_train_cnnf)\n",
        "data_gp_train_cnnf_pca = sc_cnnf.transform(data_gp_train_cnnf)\n",
        "data_gp_test_cnnf_pca = sc_cnnf.transform(data_gp_test_cnnf)\n",
        "\n",
        "pca_cnnf = PCA()\n",
        "data_gp_train_cnnf_pca = pca_cnnf.fit_transform(data_gp_train_cnnf_pca)\n",
        "data_gp_test_cnnf_pca = pca_cnnf.transform(data_gp_test_cnnf_pca)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "awvaE0OceZGT"
      },
      "outputs": [],
      "source": [
        "exp_var_pca_cnnf = pca_cnnf.explained_variance_ratio_\n",
        "cum_sum_eigenvalues_cnnf = np.cumsum(exp_var_pca_cnnf)\n",
        "\n",
        "plt.bar(range(0,len(exp_var_pca_cnnf)), exp_var_pca_cnnf, alpha=0.5, align='center', label='Individual explained variance')\n",
        "plt.step(range(0,len(cum_sum_eigenvalues_cnnf)), cum_sum_eigenvalues_cnnf, where='mid',label='Cumulative explained variance')\n",
        "plt.ylabel('Explained variance ratio _cnnf')\n",
        "plt.xlabel('Principal component index _cnnf')\n",
        "plt.legend(loc='best')\n",
        "plt.tight_layout()\n",
        "plt.show()\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iEtkDK7OY4k2"
      },
      "source": [
        "Do a Collobarative on the X first PCAs for the DNN features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8niTwje9e7Hh"
      },
      "outputs": [],
      "source": [
        "# trying to run it at different var% explained through PCA\n",
        "vars_to_explain = [0.95]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uGLZOuaHpMvj"
      },
      "outputs": [],
      "source": [
        "def construct_kernel_list(num_of_independent_vars, base_lengthscales = [0.1]):\n",
        "  possible_kernels = []\n",
        "  for i in range(len(base_lengthscales)):\n",
        "    possible_kernels.append(gpflow.kernels.RBF(variance=1.0, lengthscales=[base_lengthscales[i]]*num_of_independent_vars))\n",
        "    possible_kernels.append(gpflow.kernels.Matern12(variance=1.0, lengthscales=[base_lengthscales[i]]*num_of_independent_vars))\n",
        "    possible_kernels.append(gpflow.kernels.Matern32(variance=1.0, lengthscales=[base_lengthscales[i]]*num_of_independent_vars))\n",
        "    # possible_kernels.append(gpflow.kernels.Matern52(variance=1.0, lengthscales=[base_lengthscales[i]]*num_of_independent_vars))\n",
        "    # possible_kernels.append(gpflow.kernels.SquaredExponential(lengthscales=[base_lengthscales[i]]*num_of_independent_vars))  \n",
        "\n",
        "  return possible_kernels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9Yl5ILQWDhHh"
      },
      "outputs": [],
      "source": [
        "\n",
        "# nth_inducing_ratio = 80\n",
        "nth_inducing_ratio = 120\n",
        "\n",
        "for min_var_explained in vars_to_explain:\n",
        "  num_of_loadings = np.min(np.where(cum_sum_eigenvalues_dnnf>min_var_explained))\n",
        "  data_gp_train_dnnf_pca = data_gp_train_dnnf_pca[:,:(num_of_loadings)]\n",
        "  data_gp_test_dnnf_pca = data_gp_test_dnnf_pca[:,:(num_of_loadings)]\n",
        "  data_gp_dnnf = ( data_gp_train_dnnf_pca, data_gp_train_target_dnnf )\n",
        "  num_of_independent_vars = data_gp_dnnf[0].shape[1]\n",
        "  num_of_classes = np.unique(data_gp_dnnf[1]).size\n",
        "  num_of_functions = num_of_classes\n",
        "  possible_kernels_dnnf = construct_kernel_list(num_of_independent_vars, [0.1, 1.0])\n",
        "  # Robustmax Multiclass Likelihood\n",
        "  invlink = gpflow.likelihoods.RobustMax(num_of_functions)  # Robustmax inverse link function\n",
        "  likelihood = gpflow.likelihoods.MultiClass(num_of_functions, invlink=invlink)  # Multiclass likelihood\n",
        "  idxs_of_induced_dnnf = sorted(random.sample(range(data_gp_dnnf[0].shape[0]),int(data_gp_dnnf[0].shape[0]/nth_inducing_ratio)))\n",
        "  inducing_inputs_dnnf = data_gp_dnnf[0][idxs_of_induced_dnnf,:].copy()  # inducing inputs\n",
        "  gp_models_dnnf = []\n",
        "  for kernel in possible_kernels_dnnf:\n",
        "    m = gpflow.models.SVGP(\n",
        "        kernel=kernel,\n",
        "        likelihood=likelihood,\n",
        "        inducing_variable=inducing_inputs_dnnf,\n",
        "        num_latent_gps=num_of_functions,\n",
        "        whiten=True,\n",
        "        q_diag=True,\n",
        "    )\n",
        "    # Only train the variational parameters\n",
        "    # set_trainable(m.kernel.kernels[1].variance, False)\n",
        "    set_trainable(m.inducing_variable, False)\n",
        "    gp_models_dnnf.append(m)\n",
        "  result_dict_dnnf = dict()\n",
        "  # tensor_data = tuple(map(tf.convert_to_tensor, data_gp_dnnf))\n",
        "  # training_loss = m.training_loss_closure(tensor_data, compile=True)\n",
        "  # starting_elbo = -training_loss().numpy()\n",
        "  # print(f'Starting ELBO {starting_elbo}')\n",
        "  for mcount,m in enumerate(gp_models_dnnf):\n",
        "      print(mcount)\n",
        "      # opt = gpflow.optimizers.Scipy()\n",
        "      # opt_logs = opt.minimize(\n",
        "      #     m.training_loss_closure(data_gp_dnnf), m.trainable_variables, options=dict(maxiter=ci_niter(500))\n",
        "      # )\n",
        "      tensor_data = tuple(map(tf.convert_to_tensor, data_gp_dnnf))\n",
        "      training_loss = m.training_loss_closure(tensor_data, compile=True)\n",
        "      starting_elbo = -training_loss().numpy()\n",
        "      print(f'Starting ELBO {starting_elbo}')\n",
        "\n",
        "      elbos = [training_loss().numpy()]\n",
        "      optimizer = tf.optimizers.Adam()  \n",
        "      # optimizer = tf.optimizers.RMSprop()\n",
        "\n",
        "      @tf.function\n",
        "      def optim_here():\n",
        "          optimizer.minimize(training_loss, m.trainable_variables)\n",
        "\n",
        "      early_stop_count = 100\n",
        "      max_niters = 200000\n",
        "      for itc in range(max_niters):\n",
        "          optim_here()\n",
        "          elbo_now = -training_loss().numpy()\n",
        "          elbos.append(elbo_now)\n",
        "          # if len(elbos) > 100:\n",
        "          if (itc % 1000) == 0:\n",
        "            print(f'ELBO {elbo_now}')\n",
        "            if len(elbos) > (early_stop_count+1):\n",
        "              if elbos[-early_stop_count] >= elbos[-1]*1.0001:\n",
        "                  print(f'Early stopping at {itc}')\n",
        "                  break          \n",
        "          # needs at least a decent improvement\n",
        "\n",
        "      print(f'Ending ELBO {elbos[-1]}')\n",
        "\n",
        "      pY, pYv = m.predict_y(data_gp_test_dnnf_pca)  # Predict Y values at test locations\n",
        "      result_dict_dnnf[f\"{gp_models_dnnf[0].kernel.name}_{mcount}_{str(int(min_var_explained*100))}\"] = dict({'model': m, 'X': data_gp_test_dnnf_pca, 'Y': data_gp_test_target_dnnf, 'pY': pY, 'pYv': pYv})\n",
        "      np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/feature_collabUsingGP_dnn20_TestPCA_{gp_models_dnnf[mcount].kernel.name}_{mcount}_{str(int(min_var_explained*1000))}_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "          np.array(pY), \n",
        "          allow_pickle=True, \n",
        "          fix_imports=True)\n",
        "      np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/feature_collabUsingGP_dnn20_TestPCA_Targets_{gp_models_dnnf[mcount].kernel.name}_{mcount}_{str(int(min_var_explained*1000))}_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "          np.array(data_gp_dnnf[1]), \n",
        "          allow_pickle=True, \n",
        "          fix_imports=True)\n",
        "      tf.saved_model.save(m, f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/feature_collabUsingGP_dnn20_model_{gp_models_dnnf[mcount].kernel.name}_{mcount}_{str(int(min_var_explained*1000))}_{datetime.datetime.now():%Y%m%d%H%M%S}\")\n",
        "      print_summary(m, fmt=\"notebook\")  \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A4hFzFmgUjSq"
      },
      "source": [
        "Do a Collobarative on the X first PCAs for the CNN\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "h9hhtojWUjSs"
      },
      "outputs": [],
      "source": [
        "# trying to run it at different var% explained through PCA\n",
        "vars_to_explain = [0.95]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Lk7BkNymel7P"
      },
      "outputs": [],
      "source": [
        "# len(cum_sum_eigenvalues_cnnf)\n",
        "# min_var_explained\n",
        "# cum_sum_eigenvalues_cnnf\n",
        "# num_of_loadings\n",
        "# data_gp_cnnf[0].shape\n",
        "# # 73257/120"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v6U4WHJxUvjW"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import os,sys\n",
        "\n",
        "# min_var_explained = 0.4\n",
        "\n",
        "# nth_inducing_ratio = 80\n",
        "nth_inducing_ratio = 120\n",
        "\n",
        "for min_var_explained in vars_to_explain:\n",
        "  num_of_loadings = np.min(np.where(cum_sum_eigenvalues_cnnf>min_var_explained))\n",
        "  data_gp_train_cnnf_pca = data_gp_train_cnnf_pca[:,:(num_of_loadings)]\n",
        "  data_gp_test_cnnf_pca = data_gp_test_cnnf_pca[:,:(num_of_loadings)]\n",
        "  data_gp_cnnf = ( data_gp_train_cnnf_pca, data_gp_train_target_cnnf )\n",
        "  num_of_independent_vars = data_gp_cnnf[0].shape[1]\n",
        "  num_of_classes = np.unique(data_gp_cnnf[1]).size\n",
        "  num_of_functions = num_of_classes\n",
        "  possible_kernels_cnnf = construct_kernel_list(num_of_independent_vars, [0.1, 1.0])\n",
        "  # Robustmax Multiclass Likelihood\n",
        "  invlink = gpflow.likelihoods.RobustMax(num_of_functions)  # Robustmax inverse link function\n",
        "  likelihood = gpflow.likelihoods.MultiClass(num_of_functions, invlink=invlink)  # Multiclass likelihood\n",
        "  idxs_of_induced_cnnf = sorted(random.sample(range(data_gp_cnnf[0].shape[0]),int(data_gp_cnnf[0].shape[0]/nth_inducing_ratio)))\n",
        "  inducing_inputs_cnnf = data_gp_cnnf[0][idxs_of_induced_cnnf,:].copy()  # inducing inputs\n",
        "  gp_models_cnnf = []\n",
        "  for kernel in possible_kernels_cnnf:\n",
        "    m = gpflow.models.SVGP(\n",
        "        kernel=kernel,\n",
        "        likelihood=likelihood,\n",
        "        inducing_variable=inducing_inputs_cnnf,\n",
        "        num_latent_gps=num_of_functions,\n",
        "        whiten=True,\n",
        "        q_diag=True,\n",
        "    )\n",
        "    # Only train the variational parameters\n",
        "    # set_trainable(m.kernel.kernels[1].variance, False)\n",
        "    set_trainable(m.inducing_variable, False)\n",
        "    gp_models_cnnf.append(m)\n",
        "  result_dict_cnnf = dict()\n",
        "  # tensor_data = tuple(map(tf.convert_to_tensor, data_gp_cnnf))\n",
        "  # training_loss = m.training_loss_closure(tensor_data, compile=True)\n",
        "  # starting_elbo = -training_loss().numpy()\n",
        "  # print(f'Starting ELBO {starting_elbo}')\n",
        "  for mcount,m in enumerate(gp_models_cnnf):\n",
        "      print(mcount)\n",
        "      # opt = gpflow.optimizers.Scipy()\n",
        "      # opt_logs = opt.minimize(\n",
        "      #     m.training_loss_closure(data_gp_cnnf), m.trainable_variables, options=dict(maxiter=ci_niter(500))\n",
        "      # )\n",
        "      tensor_data = tuple(map(tf.convert_to_tensor, data_gp_cnnf))\n",
        "      training_loss = m.training_loss_closure(tensor_data, compile=True)\n",
        "      starting_elbo = -training_loss().numpy()\n",
        "      print(f'Starting ELBO {starting_elbo}')\n",
        "\n",
        "      elbos = [training_loss().numpy()]\n",
        "      optimizer = tf.optimizers.Adam()  \n",
        "      # optimizer = tf.optimizers.RMSprop()\n",
        "\n",
        "      @tf.function\n",
        "      def optim_here():\n",
        "          optimizer.minimize(training_loss, m.trainable_variables)\n",
        "\n",
        "      early_stop_count = 100\n",
        "      max_niters = 200000\n",
        "      for itc in range(max_niters):\n",
        "          optim_here()\n",
        "          elbo_now = -training_loss().numpy()\n",
        "          elbos.append(elbo_now)\n",
        "          # if len(elbos) > 100:\n",
        "          if (itc % 1000) == 0:\n",
        "            print(f'ELBO {elbo_now}')\n",
        "            if len(elbos) > (early_stop_count+1):\n",
        "              if elbos[-early_stop_count] >= elbos[-1]*1.0001:\n",
        "                  print(f'Early stopping at {itc}')\n",
        "                  break          \n",
        "          # needs at least a decent improvement\n",
        "\n",
        "      print(f'Ending ELBO {elbos[-1]}')\n",
        "\n",
        "      pY, pYv = m.predict_y(data_gp_test_cnnf_pca)  # Predict Y values at test locations\n",
        "      result_dict_cnnf[f\"{gp_models_cnnf[0].kernel.name}_{mcount}_{str(int(min_var_explained*100))}\"] = dict({'model': m, 'X': data_gp_test_cnnf_pca, 'Y': data_gp_test_target_cnnf, 'pY': pY, 'pYv': pYv})\n",
        "      np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/feature_collabUsingGP_cnn20_TestPCA_{gp_models_cnnf[mcount].kernel.name}_{mcount}_{str(int(min_var_explained*1000))}_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "          np.array(pY), \n",
        "          allow_pickle=True, \n",
        "          fix_imports=True)\n",
        "      np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/feature_collabUsingGP_cnn20_TestPCA_Targets_{gp_models_cnnf[mcount].kernel.name}_{mcount}_{str(int(min_var_explained*1000))}_{datetime.datetime.now():%Y%m%d%H%M%S}\", \n",
        "          np.array(data_gp_cnnf[1]), \n",
        "          allow_pickle=True, \n",
        "          fix_imports=True)\n",
        "      tf.saved_model.save(m, f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/feature_collabUsingGP_cnn20_model_{gp_models_cnnf[mcount].kernel.name}_{mcount}_{str(int(min_var_explained*1000))}_{datetime.datetime.now():%Y%m%d%H%M%S}\")\n",
        "      print_summary(m, fmt=\"notebook\")  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xATuNcTfxZ5R"
      },
      "outputs": [],
      "source": [
        "# gp_models_dnnf[0].kernel.name"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kKzFaZdmn5WS"
      },
      "outputs": [],
      "source": [
        "# gp_models[0].kernel\n",
        "# gp_models[1].kernel\n",
        "\n",
        "# data_gp_test_dnnf.shape\n",
        "# data_gp_test_dnnf_pca.shape\n",
        "# data_gp_test_target_dnnf.shape\n",
        "\n",
        "# pY, pYv = m.predict_y(data_gp_test_dnnf_pca)  # Predict Y values at test locations\n",
        "# result_dict[f\"{gp_models_dnnf[0].kernel.name}_{mcount}\"] = dict({'model': m, 'X': data_gp_test_dnnf_pca, 'Y': data_gp_test_target_dnnf, 'pY': pY, 'pYv': pYv})\n",
        "# print_summary(m, fmt=\"notebook\")\n",
        "# print(pY.shape)\n",
        "# np.apply_along_axis(np.argmax, 1, pY) "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DqVhyWuv26HD"
      },
      "source": [
        "Set up a summary of all the GP collaborative"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "63Tz4R2t2zqj"
      },
      "outputs": [],
      "source": [
        "import datetime\n",
        "\n",
        "# it (\"Targets\") is reverted!!!\n",
        "predicted_y_files = [ i for i in os.listdir(f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/\") if os.path.isfile(f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/{i}\") and not \"_Targets_\" in i ]  \n",
        "# used_x_files = [ i for i in os.listdir(\"D:/papers/gp_collab/gp_collab\") if os.path.isfile(i) and \"_Targets_\" in i ]\n",
        "\n",
        "collab_summaries = []\n",
        "used_y_files = []\n",
        "\n",
        "# check that it picks up the full directory\n",
        "for predicted_y_file in predicted_y_files:\n",
        "    y_test_predicted = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/{predicted_y_file}\")\n",
        "    y_test_predicted = np.apply_along_axis(np.argmax, 1, y_test_predicted)\n",
        "    y_test_predicted = np.expand_dims(y_test_predicted, axis=1)\n",
        "    try:\n",
        "        summary_here =  pr_rc_f1_acc_from_supplied(y_test_predicted, test_targets)\n",
        "        used_y_files.append(predicted_y_file)\n",
        "        collab_summaries.append(summary_here)\n",
        "    except ValueError:\n",
        "        print(f\"FAILED {predicted_y_file}\")            \n",
        "        pass\n",
        "\n",
        "\n",
        "collab_summaries = np.array(collab_summaries)\n",
        "\n",
        "gp_collab_summary = pd.DataFrame(collab_summaries, columns=['PR','RC','F1','ACC'])\n",
        "gp_collab_summary['file'] = used_y_files\n",
        "\n",
        "gp_collab_summary.to_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/gp_collab_summary_with_file_{datetime.datetime.now():%Y%m%d%H%M%S}.csv\",index=False)\n",
        "np.save(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/gp_collab_summary_with_file_{datetime.datetime.now():%Y%m%d%H%M%S}.npy\",collab_summaries)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jh3gG7xCrTxM"
      },
      "source": [
        "# Summary analysis/display across all runs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OCIWfW-E_MKa"
      },
      "outputs": [],
      "source": [
        "import os,sys\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "# os.chdir(\"D:/papers/gp_collab/summary_results\")\n",
        "# os.listdir()\n",
        "\n",
        "# pr_rc_f1_acc_from_supplied\n",
        "# feature_collabs = np.load(\"feature_collabUsingDnn_dnn20_cnn20_cnn10dnn10_wideresnet2810_summary_20211021233740.npy\")\n",
        "# feature_collabs = np.load(\"feature_collabUsingDnn_dnn20_cnn20_cnn10dnn10_wideresnet2810_wrs5cnn4dnn1_wrs10cnn8dnn2_summary_20211027174827.npy\")\n",
        "\n",
        "feature_collabs = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/feature_collabUsingDnn_dnn20_cnn20_cnn10dnn10_wideresnet2810_wrs5cnn4dnn1_wrs10cnn8dnn2_wrs5cnn5_summary_20211028180009.npy\")\n",
        "full_parrallel = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/full_parallel_dnn20_cnn20_cnn10dnn10_summary_20211003130317.npy\")\n",
        "\n",
        "tensemble_dnn20 = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/traditional_ensemble_dnn20_summary_20211117002646.npy\")\n",
        "tensemble_cnn20 = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/traditional_ensemble_cnn20_summary_20211117002646.npy\")\n",
        "tensemble_cnn10dnn10 = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/traditional_ensemble_cnn10dnn10_summary_20211117002646.npy\")\n",
        "tensemble_resnets = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/traditional_ensemble_resnets_summary_20211117022755.npy\")\n",
        "tensemble_wideresnets = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/traditional_ensemble_wideresnet28-10_summary_20211117022755.npy\")\n",
        "\n",
        "individual_dnns = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/individual_dnn_summary_20211014152626.npy\")\n",
        "individual_cnns = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/individual_cnn_summary_20211014152108.npy\")\n",
        "individual_resnets = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/individual_resnets_summary_20211014154755.npy\")\n",
        "individual_wideresnets = np.load(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/individual_WideResNet2810_summary_20211017233136.npy\")\n",
        "\n",
        "\n",
        "\n",
        "gpresults = pd.read_csv(f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/gp_collab_summary_with_file_20211027112445.csv\")\n",
        "\n",
        "print(np.mean(tensemble_dnn20,0))\n",
        "print(np.mean(tensemble_cnn20,0))\n",
        "print(np.mean(tensemble_cnn10dnn10,0))\n",
        "# individual_dnns[np.where(individual_dnns[:,0]>0.10)[0],:]\n",
        "print(np.mean(individual_dnns,0))\n",
        "print(np.mean(individual_dnns[np.where(individual_dnns[:,0]>0.10)[0],:],0))\n",
        "print(np.where(individual_dnns[:,0]>0.10)[0].size)\n",
        "print(len(individual_cnns))\n",
        "print(np.mean(individual_cnns,0))\n",
        "print(np.mean(individual_cnns[np.where(individual_cnns[:,0]>0.10)[0],:],0))\n",
        "print(np.where(individual_cnns[:,0]>0.10)[0].size)\n",
        "print(len(individual_resnets))\n",
        "print(np.mean(individual_resnets,0))\n",
        "print(len(individual_wideresnets))\n",
        "print(np.mean(individual_wideresnets,0))\n",
        "print(len(tensemble_wideresnets))\n",
        "print(np.mean(tensemble_wideresnets,0))\n",
        "print(len(tensemble_resnets))\n",
        "print(np.mean(tensemble_resnets,0))\n",
        "\n",
        "print(feature_collabs)\n",
        "\n",
        "# import platform\n",
        "# print(platform.python_implementation())\n",
        "\n",
        "# os.listdir()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8EjlpZ2ia5SN"
      },
      "outputs": [],
      "source": [
        "# # [ i for i in os.listdir(f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/\") if not \"_Targets_\" in i and os.path.isfile(f\"/content/drive/MyDrive/data_papers/{paper_name}/gp_collab/{i}\")  ]\n",
        "# collab_summaries\n",
        "# # used_y_files\n",
        "# np.save(collab_summaries,f\"/content/drive/MyDrive/data_papers/{paper_name}/summary_results/gp_collab_summary_with_file_{datetime.datetime.now():%Y%m%d%H%M%S}.npy\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0dox3kuua3ff"
      },
      "outputs": [],
      "source": [
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oo_ykvqAencY"
      },
      "outputs": [],
      "source": [
        "# (Y_test_GP_mean_dnnf, Y_test_GP_variance_dnnf) = gp_models[0].predict_y(data_gp_test_dnnf)\n",
        "# Y_test_GP_mean_dnnf = Y_test_GP_mean_dnnf.numpy()\n",
        "# Y_test_pred_dnnf = []\n",
        "# for i in range(data_gp_test_dnnf.shape[0]):\n",
        "#   Y_test_pred_dnnf.append(np.argmax(Y_test_GP_mean_dnnf[i,]))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qFnU7LqBrmBU"
      },
      "outputs": [],
      "source": [
        "# Y_test_GP_mean_dnnf"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M_-H1KDTojCa"
      },
      "outputs": [],
      "source": [
        "# np_x_train_collab_dnn.shape\n",
        "# np_x_train_collab_dnn[:1000,:].shape\n",
        "# train_targets[:1000]\n",
        "# train_targets.shape\n",
        "# data_gp_train_target_hot\n",
        "# np.apply_along_axis(np.argmax, 1, data_gp_train_target_hot)\n",
        "# data_gp_train_target\n",
        "# num_of_independent_vars\n",
        "# np_x_train_collab_dnn.shape\n",
        "# possible_kernels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PjMVfwrK5H2s"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sy1gmVR3GSrG"
      },
      "outputs": [],
      "source": [
        "def unpacking_apply_along_axis(all_args):\n",
        "    \"\"\"\n",
        "    Like numpy.apply_along_axis(), but with arguments in a tuple\n",
        "    instead.\n",
        "\n",
        "    This function is useful with multiprocessing.Pool().map(): (1)\n",
        "    map() only handles functions that take a single argument, and (2)\n",
        "    this function can generally be imported from a module, as required\n",
        "    by map().\n",
        "    \"\"\"\n",
        "    (func1d, axis, arr, args, kwargs) = all_args\n",
        "    # return np.apply_along_axis(func1d, axis, arr, *args, **kwargs)\n",
        "\n",
        "\n",
        "def parallel_apply_along_axis(func1d, axis, arr, *args, **kwargs):\n",
        "    \"\"\"\n",
        "    Like numpy.apply_along_axis(), but takes advantage of multiple\n",
        "    cores.\n",
        "    \"\"\"\n",
        "    # Effective axis where apply_along_axis() will be applied by each\n",
        "    # worker (any non-zero axis number would work, so as to allow the use\n",
        "    # of `np.array_split()`, which is only done on axis 0):\n",
        "    effective_axis = 1 if axis == 0 else axis\n",
        "    if effective_axis != axis:\n",
        "        arr = arr.swapaxes(axis, effective_axis)\n",
        "\n",
        "    # Chunks for the mapping (only a few chunks):\n",
        "    chunks = [(func1d, effective_axis, sub_arr, args, kwargs)\n",
        "              for sub_arr in np.array_split(arr, multiprocessing.cpu_count())]\n",
        "\n",
        "    pool = multiprocessing.Pool()\n",
        "    individual_results = pool.map(unpacking_apply_along_axis, chunks)\n",
        "    # Freeing the workers:\n",
        "    pool.close()\n",
        "    pool.join()\n",
        "\n",
        "    return np.concatenate(individual_results)\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "av22_gwc_hOd",
        "UrA1SY4CHTOS",
        "NUnTfSfY2pZK",
        "boCHy50O_Z-m",
        "WS7USmAlAZ_Q",
        "ib862JpbN7oI",
        "d4NWpQ7wAS1u",
        "N3pww0F8PDy4",
        "05H-E-Hb21S9",
        "7NfTm6C0efVR",
        "zarZ-ubirq5w",
        "Qr4uwylNzQl1",
        "nlcSr-vhOENw",
        "pVGqX79lZ3pw",
        "5YEfUSsWG6bm",
        "W8fThz_isxYI",
        "3pW2evV5b2wJ",
        "XtjoNp7qsxYd",
        "8GvmAzHIp7te",
        "kIiJPc3OqE-V",
        "-nbwCVHSRRfM",
        "1_Os-eyfeYfE",
        "7IhTGp4wUo7T",
        "mIJt0f9ZVIm5",
        "tgih_EEjVMdQ",
        "eDq_scmJVVLP",
        "YlyIJ3CCVmCq",
        "upPlyiuGnrbK",
        "DaQvakL4rU88",
        "TXy-xmor32UY",
        "uLw5jkNprylq",
        "Ul_SqJXjz_s9",
        "Zv4uI-Rf_P4R",
        "pfPdlux63Hqa",
        "qSwgurq54dx7",
        "BvyzEp0EkgBD",
        "nKZebazXuPRf",
        "sddcE82tOFDm",
        "FWDC54QRQhyy",
        "hx1fELMRQhyy",
        "io3MAwHuQhyz",
        "cxBmUlk5Qhy0",
        "4Z9SnOmMTKbj",
        "yHXkywirRpmM",
        "jh3gG7xCrTxM"
      ],
      "machine_shape": "hm",
      "name": "collab_xnn_features_svhn.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}