simulation_plots.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Signaling Game Simulation Plots"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Requirements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from typing import *\n",
    "\n",
    "import plots\n",
    "import simulations\n",
    "import utils\n",
    "\n",
    "plt.style.use('seaborn-whitegrid')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load a simulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'simulations/acl_ready/extremity_game_acl_4__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__loss_type_cross_entropy/extremity_game_acl_4__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__loss_type_cross_entropy.json'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-b41d71dd7f0a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0msimulation_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"acl_ready/extremity_game_acl_4__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__loss_type_cross_entropy\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# simulation_name = \"thesis/extremity_game_acl_4__o_4__s_0__n_5__m_128__l_cross_entropy\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0msimulation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msimulations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_simulation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msimulation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mevaluations\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msimulation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluations\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# print(simulation.name)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Projects/Signaling_auto_encoder/signaling_auto_encoder/simulations.py\u001b[0m in \u001b[0;36mload_simulation\u001b[0;34m(simulation_name, subdir)\u001b[0m\n\u001b[1;32m     57\u001b[0m         \u001b[0msubdir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msimulation_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname_split\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m     return Simulation.from_json(\n\u001b[0;32m---> 59\u001b[0;31m         \u001b[0m_get_simulation_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msimulation_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubdir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     60\u001b[0m         \u001b[0;34m.\u001b[0m\u001b[0mjoinpath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{simulation_name}.json\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m         \u001b[0;34m.\u001b[0m\u001b[0mread_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/signaling_auto_encoder/lib/python3.8/pathlib.py\u001b[0m in \u001b[0;36mread_text\u001b[0;34m(self, encoding, errors)\u001b[0m\n\u001b[1;32m   1227\u001b[0m         \u001b[0mOpen\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mfile\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtext\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mread\u001b[0m \u001b[0mit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mclose\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1228\u001b[0m         \"\"\"\n\u001b[0;32m-> 1229\u001b[0;31m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'r'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencoding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1230\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1231\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/signaling_auto_encoder/lib/python3.8/pathlib.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, mode, buffering, encoding, errors, newline)\u001b[0m\n\u001b[1;32m   1213\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_closed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1214\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_raise_closed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1215\u001b[0;31m         return io.open(self, mode, buffering, encoding, errors, newline,\n\u001b[0m\u001b[1;32m   1216\u001b[0m                        opener=self._opener)\n\u001b[1;32m   1217\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/signaling_auto_encoder/lib/python3.8/pathlib.py\u001b[0m in \u001b[0;36m_opener\u001b[0;34m(self, name, flags, mode)\u001b[0m\n\u001b[1;32m   1069\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_opener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0o666\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1070\u001b[0m         \u001b[0;31m# A stub for the opener argument to built-in open()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1071\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_accessor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1072\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1073\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_raw_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0o777\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'simulations/acl_ready/extremity_game_acl_4__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__loss_type_cross_entropy/extremity_game_acl_4__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__loss_type_cross_entropy.json'"
     ]
    }
   ],
   "source": [
    "simulation_name = \"acl_ready/extremity_game_acl_4__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__loss_type_cross_entropy\"\n",
    "# simulation_name = \"thesis/extremity_game_acl_4__o_4__s_0__n_5__m_128__l_cross_entropy\"\n",
    "simulation = simulations.load_simulation(simulation_name)\n",
    "evaluations = simulation.evaluations\n",
    "# print(simulation.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "extremity_game_acl_4__o_4__s_0__n_5__m_128__l_cross_entropy\n",
      "Simulation message sizes: [2]\n",
      "Number of trials per evaluation: 10\n"
     ]
    }
   ],
   "source": [
    "print(simulation.experiment_name)\n",
    "print(f\"Simulation message sizes: {list(evaluations.keys())}\")\n",
    "print(f\"Number of trials per evaluation: {len(evaluations[2])}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Available simulations\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "text_l =    ['\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\"',\n",
    "    # O=3\n",
    "    '\"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_6__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_0__nature_includes_function_1__num_objects_6__mini_batch_size_128__num_batches_5000\"',\n",
    "\n",
    "\n",
    "\n",
    "    # O=5\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_15__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_15__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_5__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_5__mini_batch_size_128__num_batches_5000\"',\n",
    "    # O=3\n",
    "    '\"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_6__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_6__mini_batch_size_128__num_batches_5000\"',    \n",
    "\n",
    "\n",
    "    # O=5\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_1__nature_includes_function_0__num_objects_10__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_0__nature_includes_function_0__num_objects_10__mini_batch_size_128__num_batches_5000\"',\n",
    "\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_0__num_objects_5__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_0__num_objects_5__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_0__num_objects_10__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_0__num_objects_10__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_0__num_objects_15__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_0__num_objects_15__mini_batch_size_128__num_batches_5000\"',\n",
    "\n",
    "    # O=3\n",
    "    '\"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_1__nature_includes_function_0__num_objects_6__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_0__nature_includes_function_0__num_objects_6__mini_batch_size_128__num_batches_5000\"',\n",
    "    \n",
    "    '\"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_0__shared_context_1__nature_includes_function_0__num_objects_6__mini_batch_size_128__num_batches_5000\"',\n",
    "    '\"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_0__shared_context_0__nature_includes_function_0__num_objects_6__mini_batch_size_128__num_batches_5000\"']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000',\n",
       " 'acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000',\n",
       " 'acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_5__mini_batch_size_128__num_batches_5000',\n",
       " 'acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_5__mini_batch_size_128__num_batches_5000',\n",
       " 'acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000',\n",
       " 'acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000',\n",
       " 'acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_15__mini_batch_size_128__num_batches_5000',\n",
       " 'acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_15__mini_batch_size_128__num_batches_5000']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cells = [\n",
    "    [\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\",\n",
    "    \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\"],\n",
    "    \n",
    "    [\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_5__mini_batch_size_128__num_batches_5000\",\n",
    "    \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_5__mini_batch_size_128__num_batches_5000\"],\n",
    "    \n",
    "    [\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\", \n",
    "     \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\"],\n",
    "    \n",
    "    [\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_15__mini_batch_size_128__num_batches_5000\",\n",
    "    \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_15__mini_batch_size_128__num_batches_5000\"]\n",
    "]\n",
    "\n",
    "new_cells = []\n",
    "for cell in cells:\n",
    "    [new_cells.append(c) for c in cell]\n",
    "    \n",
    "new_cells"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acl_ready/extremity_game_acl_4__object_size_5__strict_context_1__shared_context_1__num_objects_10__mini_batch_size_128__loss_type_cross_entropy\n",
      "acl_ready/extremity_game_acl_4__object_size_5__strict_context_1__shared_context_0__num_objects_10__mini_batch_size_128__loss_type_cross_entropy\n",
      "acl_ready/extremity_game_acl_4__object_size_5__strict_context_0__shared_context_1__num_objects_5__mini_batch_size_128__loss_type_cross_entropy\n",
      "acl_ready/extremity_game_acl_4__object_size_5__strict_context_0__shared_context_0__num_objects_5__mini_batch_size_128__loss_type_cross_entropy\n",
      "acl_ready/extremity_game_acl_4__object_size_5__strict_context_0__shared_context_1__num_objects_10__mini_batch_size_128__loss_type_cross_entropy\n",
      "acl_ready/extremity_game_acl_4__object_size_5__strict_context_0__shared_context_0__num_objects_10__mini_batch_size_128__loss_type_cross_entropy\n",
      "acl_ready/extremity_game_acl_4__object_size_5__strict_context_0__shared_context_1__num_objects_15__mini_batch_size_128__loss_type_cross_entropy\n",
      "acl_ready/extremity_game_acl_4__object_size_5__strict_context_0__shared_context_0__num_objects_15__mini_batch_size_128__loss_type_cross_entropy\n"
     ]
    }
   ],
   "source": [
    "text = \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_0__num_objects_15__mini_batch_size_128__num_batches_5000\"\n",
    "\n",
    "\n",
    "\n",
    "def process_file_name(text_l, loss_type = \"CE\"):\n",
    "    \n",
    "    def remove_flag(text, flag_name):\n",
    "        if len(flag_name.split(\"_\"))==1:\n",
    "            start_idx = text.index(flag_name)\n",
    "            stop_idx = text.index(\"\", start_idx)\n",
    "            del text[start_idx:stop_idx]\n",
    "        else:\n",
    "            flag_names = flag_name.split(\"_\")\n",
    "            flag1 = flag_names[0]\n",
    "            flag2 = flag_names[1]\n",
    "            for i in range(len(text)-1):\n",
    "                if text[i] == flag1 and text[i+1] == flag2:\n",
    "                    start_idx = i\n",
    "                    try:\n",
    "                        stop_idx = text.index(\"\", start_idx)\n",
    "                        del text[start_idx:stop_idx]\n",
    "                        break\n",
    "                    except ValueError: # delete until the last element (including the last \"\")\n",
    "                        del text[start_idx:]\n",
    "                        del text[-1]\n",
    "                        break\n",
    "    for text in text_l:\n",
    "        text = text.split(\"_\")\n",
    "    #     print(text)\n",
    "        # insert 4 after acl\n",
    "        acl_idx = text.index(\"acl\", 2)\n",
    "        text.insert(acl_idx+1,\"4\")\n",
    "\n",
    "\n",
    "        remove_flag(text, \"o\")\n",
    "        remove_flag(text, \"nature\")\n",
    "        remove_flag(text,\"num_batches\")\n",
    "        if loss_type == \"CE\":\n",
    "            text += [\"\",\"loss\", \"type\", \"cross\", \"entropy\"]\n",
    "        else:\n",
    "            text += [\"\",\"loss\", \"type\", \"mse\"]\n",
    "\n",
    "\n",
    "        # remove duplicate \"\"\n",
    "        for i in range(len(text)-1):\n",
    "            if text[i] == \"\" and text[i+1] == \"\":\n",
    "                del text[i]\n",
    "\n",
    "        text = \"_\".join(text)\n",
    "        print(text)\n",
    "    \n",
    "\n",
    "process_file_name(new_cells)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "['a', 'b'] is not in list",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-46-166e1630f4d1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0ml\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"a\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"b\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0ml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"a\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"b\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m: ['a', 'b'] is not in list"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "EOL while scanning string literal (<ipython-input-14-b11be5ce687a>, line 10)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-14-b11be5ce687a>\"\u001b[0;36m, line \u001b[0;32m10\u001b[0m\n\u001b[0;31m    \"Strict\\nshared\\nO=4, C=(10,5)\": \"thesis/extremity_game_acl_4__o_4__s_1__n_5__m_128__l_cross_entropy,\u001b[0m\n\u001b[0m                                                                                                         ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m EOL while scanning string literal\n"
     ]
    }
   ],
   "source": [
    "strict_context_simulations = {\n",
    "    # O=5\n",
    "    \"Strict\\nshared\\nO=5, C=(10,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__loss_type_cross_entropy\",\n",
    "    \"Strict\\nnon-shared\\nO=5, C=(10,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__loss_type_cross_entropy\",\n",
    "    # O=3\n",
    "    \"Strict\\nshared\\nO=3, C=(6,3)\": \"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_6__mini_batch_size_128__loss_type_cross_entropy\",\n",
    "    \"Strict\\nnon-shared\\nO=3, C=(6,3)\": \"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_0__nature_includes_function_1__num_objects_6__mini_batch_size_128__loss_type_cross_entropy\",\n",
    "}\n",
    "    \n",
    "    # O=5\n",
    "    \"Strict\\nshared\\nO=5, C=(10,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__loss_type_cross_entropy\",\n",
    "    \"Strict\\nnon-shared\\nO=5, C=(10,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__loss_type_cross_entropy\",\n",
    "    # O=3\n",
    "    \"Strict\\nshared\\nO=3, C=(6,3)\": \"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_6__mini_batch_size_128__loss_type_cross_entropy\",\n",
    "    \"Strict\\nnon-shared\\nO=3, C=(6,3)\": \"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_0__nature_includes_function_1__num_objects_6__mini_batch_size_128__loss_type_cross_entropy\",\n",
    "}\n",
    "\n",
    "non_strict_context_simulations = {\n",
    "    # O=5\n",
    "    \"Non-strict\\nshared\\nO=5, C=(10,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__loss_type_cross_entropy\",\n",
    "    \"Non-Strict\\nnon-shared\\nO=5, C=(10,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__loss_type_cross_entropy\",\n",
    "    \"Non-strict\\nshared\\nO=5, C=(15,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_15__mini_batch_size_128__loss_type_cross_entropy\",\n",
    "    \"Non-strict\\nnon-shared\\nO=5, C=(15,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_15__mini_batch_size_128__loss_type_cross_entropy\",\n",
    "    \"Non-strict\\nshared\\nO=5, C=(5,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_5__mini_batch_size_128__loss_type_cross_entropy\",\n",
    "    \"Non-strict\\nnon-shared\\nO=5, C=(5,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_5__mini_batch_size_128__num_batches_5000\",\n",
    "    # O=3\n",
    "    \"Non-strict\\nshared\\nO=3, C=(6,3)\": \"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_6__mini_batch_size_128__num_batches_5000\",\n",
    "    \"Non-strict\\nnon-shared\\nO=3, C=(6,3)\": \"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_6__mini_batch_size_128__num_batches_5000\",    \n",
    "}\n",
    "\n",
    "# Encoder input is <c, f(c)>\n",
    "\n",
    "nature_includes_fc_simulations = {\n",
    "    # O=5\n",
    "    \"F(c) input\\nStrict\\nshared\\nO=5, C=(10,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_1__nature_includes_function_0__num_objects_10__mini_batch_size_128__num_batches_5000\",\n",
    "    \"F(c) input\\nStrict\\nnon-shared\\nO=5, C=(10,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_0__nature_includes_function_0__num_objects_10__mini_batch_size_128__num_batches_5000\",\n",
    "\n",
    "    \"F(c) input\\nNon-strict\\nshared\\nO=5\\nC=(5,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_0__num_objects_5__mini_batch_size_128__num_batches_5000\",\n",
    "    \"F(c) input\\nNon-strict\\nnon-shared\\nO=5\\nC=(5,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_0__num_objects_5__mini_batch_size_128__num_batches_5000\",\n",
    "    \"F(c) input\\nNon-strict\\nshared\\nO=5\\nC=(10,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_0__num_objects_10__mini_batch_size_128__num_batches_5000\",\n",
    "    \"F(c) input\\nNon-strict\\nnon-shared\\nO=5\\nC=(10,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_0__num_objects_10__mini_batch_size_128__num_batches_5000\",\n",
    "    \"F(c) input\\nNon-strict\\nshared\\nO=5\\nC=(15,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_0__num_objects_15__mini_batch_size_128__num_batches_5000\",\n",
    "    \"F(c) input\\nNon-strict\\nnon-shared\\nO=5\\nC=(15,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_0__num_objects_15__mini_batch_size_128__num_batches_5000\",\n",
    "\n",
    "    # O=3\n",
    "    \"F(c) input\\nStrict\\nshared\\nO=3\\nC=(6,3)\": \"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_1__nature_includes_function_0__num_objects_6__mini_batch_size_128__num_batches_5000\",\n",
    "    \"F(c) input\\nStrict\\nnon-shared\\nO=3\\nC=(6,3)\": \"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_0__nature_includes_function_0__num_objects_6__mini_batch_size_128__num_batches_5000\",\n",
    "    \n",
    "    \"F(c) input\\nNon-strict\\nshared\\nO=3\\nC=(6,3)\": \"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_0__shared_context_1__nature_includes_function_0__num_objects_6__mini_batch_size_128__num_batches_5000\",\n",
    "    \"F(c) input\\nNon-strict\\nnon-shared\\nO=3\\nC=(6,3)\": \"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_0__shared_context_0__nature_includes_function_0__num_objects_6__mini_batch_size_128__num_batches_5000\",\n",
    "}\n",
    "\n",
    "# Untrained\n",
    "\n",
    "untrained_simulations = {\n",
    "    \"Strict, shared\\nO=5, C=(10,5)\": \"acl_ready/extremity_game_acl_untrained__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_6__mini_batch_size_128__num_batches_0\",\n",
    "    \"Strict, shared\\nO=3, C=(6,3)\": \"acl_ready/extremity_game_acl_untrained__object_size_3__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_6__mini_batch_size_128__num_batches_0\",\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Available evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_losses\n",
      "object_prediction_accuracy\n",
      "detected_num_clusters\n",
      "object_prediction_by_cluster_loss\n",
      "predictions_by_average_msg_loss\n",
      "predictions_by_average_msg_accuracy\n",
      "addition_compositionality_mean_message_loss\n",
      "addition_compositionality_mean_message_cluster_accuracy\n",
      "addition_compositionality_mean_prediction_loss\n",
      "addition_compositionality_mean_prediction_accuracy\n",
      "analogy_compositionality_net_message_mean_loss\n",
      "analogy_compositionality_net_message_cluster_mean_accuracy\n",
      "analogy_compositionality_net_prediction_mean_loss\n",
      "analogy_compositionality_net_prediction_mean_accuracy\n",
      "compositionality_net_message_mean_loss\n",
      "compositionality_net_message_cluster_mean_accuracy\n",
      "compositionality_net_prediction_mean_loss\n",
      "compositionality_net_prediction_mean_accuracy\n",
      "functions_from_messages\n",
      "min_max_from_messages\n",
      "dimension_from_messages\n",
      "sanity_from_messages\n",
      "object_by_context_from_messages\n",
      "object_by_decoder_context_from_messages\n",
      "context_from_messages\n",
      "decoder_context_from_messages\n"
     ]
    }
   ],
   "source": [
    "print('\\n'.join(evaluations[2][0].keys()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "def get_evaluation_mean_and_err(simulation_name, evaluation_name, message_size, percents=False) -> float:\n",
    "    simulation = simulations.load_simulation(simulation_name)\n",
    "    vals = []\n",
    "    print(\"evaluation name: \", evaluation_name)\n",
    "    \n",
    "    for trial in simulation.evaluations[message_size]:\n",
    "        print(\"trial: \", trial.keys())\n",
    "        vals.append(trial[evaluation_name])\n",
    "    if percents:\n",
    "        vals = np.array(vals) * 100\n",
    "    return np.mean(vals), np.std(vals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "def plot_bars_for_evaluation(evaluation_name, display_names_to_simulation_names, title, y_label, width=18, msg_size=2):\n",
    "    bar_width = 1\n",
    "    num_simulation_sets = len(display_names_to_simulation_names)\n",
    "\n",
    "    fig, ax = plt.subplots(num_simulation_sets, figsize=(width,8*num_simulation_sets))\n",
    "    \n",
    "    for i, (set_name, display_name_to_simulation) in enumerate(display_names_to_simulation_names.items()):\n",
    "        for s, (display_name, file_name) in enumerate(display_name_to_simulation.items()):\n",
    "            mean, err = get_evaluation_mean_and_err(file_name, evaluation_name, msg_size)\n",
    "            ax[i].bar(s, mean, yerr=err, capsize=10, color=f\"C{i}\", label=display_name)    \n",
    "\n",
    "            ax[i].set_title(set_name)\n",
    "            ax[i].set_xticks(range(len(display_name_to_simulation)))\n",
    "            ax[i].set_xticklabels(display_name_to_simulation.keys())\n",
    "            ax[i].set_ylabel(y_label)\n",
    "\n",
    "    fig.suptitle(title, y=0.92, fontsize=15)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "def plot_bars_for_acl(evaluation_name, display_name_to_simulation, title, y_label, msg_size=2):\n",
    "    fig, ax = plt.subplots(1, figsize=(6, 4))\n",
    "    \n",
    "    for s, (display_name, file_name) in enumerate(display_name_to_simulation.items()):\n",
    "        mean, err = get_evaluation_mean_and_err(file_name, evaluation_name, msg_size)\n",
    "        ax.bar(s, mean, yerr=err, capsize=10, color=f\"C{s}\", label=display_name)    \n",
    "\n",
    "        ax.set_xticks(range(len(display_name_to_simulation)))\n",
    "        ax.set_xticklabels(display_name_to_simulation.keys())\n",
    "        ax.set_ylabel(y_label)\n",
    "\n",
    "    fig.suptitle(title, y=0.92, fontsize=15)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "def print_mean_err_for_evaluation(evaluation_name, display_names_to_simulation_names, message_size=2):\n",
    "    for name, sim_name in display_names_to_simulation_names.items():\n",
    "        mean, err = get_evaluation_mean_and_err(sim_name, evaluation_name, message_size)\n",
    "        print(f\"{name}:\\n\\tmean = {mean}\\n\\tstd = {err}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ACL Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "#                 Shared  | non-shared\n",
    "# Strict                \n",
    "# Non-strict 5\n",
    "# Non-strict 10\n",
    "# Non-strict 15\n",
    "\n",
    "# should be object size 5, here we use 4 temporarily\n",
    "\n",
    "cells = [\n",
    "    #  \n",
    "    [\"thesis/extremity_game_acl_4__o_5__s_0__n_5__m_128__l_cross_entropy\",\n",
    "    \"thesis/extremity_game_acl_4__o_5__s_0__n_5__m_128__l_mse\"],\n",
    "    \n",
    "    # should be non-strict\n",
    "    [\"thesis/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\",\n",
    "    \"thesis/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\"]\n",
    "    \n",
    "    \n",
    "#     [\"thesis/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_5__mini_batch_size_128__num_batches_5000\",\n",
    "#     \"thesis/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_5__mini_batch_size_128__num_batches_5000\"]\n",
    "    \n",
    "#     [\"thesis/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\", \n",
    "#      \"thesis/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\"],\n",
    "    \n",
    "#     [\"thesis/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_15__mini_batch_size_128__num_batches_5000\",\n",
    "#     \"thesis/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_15__mini_batch_size_128__num_batches_5000\"]\n",
    "]\n",
    "\n",
    "cells = [ [\"extremity_game_acl_4__o_15__s_1__n_5__m_2__l_mse\"],[\"extremity_game_acl_4__o_15__s_1__n_5__m_2__l_cross_entropy\"]    ]\n",
    "\n",
    "cells = np.array(cells)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "evaluation name:  addition_compositionality_mean_prediction_loss\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "evaluation name:  analogy_compositionality_net_prediction_mean_loss\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "evaluation name:  addition_compositionality_mean_prediction_loss\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "evaluation name:  analogy_compositionality_net_prediction_mean_loss\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "\n",
      "\\begin{table}[hbt!]\n",
      "\\begin{tabular}{lcccc}\n",
      "\\toprule\n",
      "                       & \\multicolumn{2}{c}{\\textbf{Compositionality by Addition}} & \\multicolumn{2}{c}{\\textbf{Composition Network}} \\\\ \\midrule\n",
      "                       & \\textbf{Shared} & \\textbf{Non-shared} & \\textbf{Shared} & \\textbf{Non-shared} \\\\ \\midrule\n",
      "\\textbf{Strict} & \\\\ \n",
      "\\emph{10 objects} &  $0.11\\pm 0.00$  & XX0,1 &  $0.11\\pm 0.00$  & YY0,1      \\\\ % \\midrule\n",
      "\\textbf{Non-strict} & \\\\\n",
      "\\emph{5 objects} &\n",
      "\t$3.43\\pm 0.00$  & XX1,1 &  $3.46\\pm 0.00$  & YY1,1      \\\\\n",
      "%\\textbf{Non-strict} & \\\\\n",
      "\\emph{10 objects} &\n",
      "\tXX2,0  & XX2,1 &  YY2,0  & YY2,1     \\\\\n",
      "\\emph{15 objects} &\n",
      "\tXX3,0  & XX3,1 &  YY3,0  & YY3,1     \\\\ \\bottomrule\n",
      "\\end{tabular}\n",
      "\\caption{Lossess}\\label{tab:composition_mse_losses}\n",
      "\\end{table}\n",
      "    \n",
      "evaluation name:  addition_compositionality_mean_prediction_loss\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "evaluation name:  analogy_compositionality_net_prediction_mean_loss\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "evaluation name:  addition_compositionality_mean_prediction_loss\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "evaluation name:  analogy_compositionality_net_prediction_mean_loss\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "\n",
      "\\begin{table}[hbt!]\n",
      "\\begin{tabular}{lcccc}\n",
      "\\toprule\n",
      "                       & \\multicolumn{2}{c}{\\textbf{Compositionality by Addition}} & \\multicolumn{2}{c}{\\textbf{Composition Network}} \\\\ \\midrule\n",
      "                       & \\textbf{Shared} & \\textbf{Non-shared} & \\textbf{Shared} & \\textbf{Non-shared} \\\\ \\midrule\n",
      "\\textbf{Strict} & \\\\ \n",
      "\\emph{10 objects} &  $0.11\\pm 0.00$  & XX0,1 &  $0.11\\pm 0.00$  & YY0,1      \\\\ % \\midrule\n",
      "\\textbf{Non-strict} & \\\\\n",
      "\\emph{5 objects} &\n",
      "\t$3.43\\pm 0.00$  & XX1,1 &  $3.46\\pm 0.00$  & YY1,1      \\\\\n",
      "%\\textbf{Non-strict} & \\\\\n",
      "\\emph{10 objects} &\n",
      "\tXX2,0  & XX2,1 &  YY2,0  & YY2,1     \\\\\n",
      "\\emph{15 objects} &\n",
      "\tXX3,0  & XX3,1 &  YY3,0  & YY3,1     \\\\ \\bottomrule\n",
      "\\end{tabular}\n",
      "\\caption{Lossess}\\label{tab:composition_mse_losses}\n",
      "\\end{table}\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "def plot_table(evaluation_name, cells, caption=\"\", label=\"\", percents=False, message_size=2):\n",
    "\n",
    "    s=r\"\"\"\n",
    "\\begin{table}[]\n",
    "\\begin{tabular}{lcc}\n",
    "\\toprule\n",
    "                       & \\textbf{Shared} & \\textbf{Non-shared} \\\\ \\midrule\n",
    "\\textbf{Strict} & \\\\\n",
    "\\emph{10 objects} & \n",
    "\tXX0,0  & XX0,1      \\\\ % \\midrule\n",
    "\\textbf{Non-strict} & \\\\\n",
    "\\emph{5 objects} &\n",
    "\tXX1,0  & XX1,1      \\\\\n",
    "%\\textbf{Non-strict} & \\\\\n",
    "\\emph{10 objects} &\n",
    "\tXX2,0  & XX2,1      \\\\\n",
    "\\emph{15 objects} \n",
    "\tXX3,0  & XX3,1      \\\\ \\bottomrule\n",
    "\\end{tabular}\n",
    "\\caption{CAPTION}\\label{tab:LABEL}\n",
    "\\end{table}\n",
    "    \"\"\"\n",
    "    \n",
    "    print(message_size)\n",
    "    for idxs, x in np.ndenumerate(cells):\n",
    "        mean, err = get_evaluation_mean_and_err(x, evaluation_name, message_size, percents)\n",
    "        filler = \"$\" + f\"{mean:.2f}\" \n",
    "        filler += r'\\%' if percents else ''\n",
    "        filler += r\"\\pm \" + f\"{err:.2f}\" + \"$\"\n",
    "        s = s.replace(F\"XX{idxs[0]},{idxs[1]}\", filler)\n",
    "        s = s.replace(\"LABEL\", label)\n",
    "        s = s.replace(\"CAPTION\", caption)\n",
    "\n",
    "    print(s)\n",
    "    \n",
    "    \n",
    "def plot_compositionality_table(evaluation_names, cells, caption=\"\", label=\"\", percents=False, message_size=2):\n",
    "\n",
    "    s=r\"\"\"\n",
    "\\begin{table}[hbt!]\n",
    "\\begin{tabular}{lcccc}\n",
    "\\toprule\n",
    "                       & \\multicolumn{2}{c}{\\textbf{Compositionality by Addition}} & \\multicolumn{2}{c}{\\textbf{Composition Network}} \\\\ \\midrule\n",
    "                       & \\textbf{Shared} & \\textbf{Non-shared} & \\textbf{Shared} & \\textbf{Non-shared} \\\\ \\midrule\n",
    "\\textbf{Strict} & \\\\ \n",
    "\\emph{10 objects} &  XX0,0  & XX0,1 &  YY0,0  & YY0,1      \\\\ % \\midrule\n",
    "\\textbf{Non-strict} & \\\\\n",
    "\\emph{5 objects} &\n",
    "\tXX1,0  & XX1,1 &  YY1,0  & YY1,1      \\\\\n",
    "%\\textbf{Non-strict} & \\\\\n",
    "\\emph{10 objects} &\n",
    "\tXX2,0  & XX2,1 &  YY2,0  & YY2,1     \\\\\n",
    "\\emph{15 objects} &\n",
    "\tXX3,0  & XX3,1 &  YY3,0  & YY3,1     \\\\ \\bottomrule\n",
    "\\end{tabular}\n",
    "\\caption{CAPTION}\\label{tab:LABEL}\n",
    "\\end{table}\n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    for idxs, x in np.ndenumerate(cells):\n",
    "        mean1, err1 = get_evaluation_mean_and_err(x, evaluation_names[0], message_size, percents)\n",
    "        mean2, err2 = get_evaluation_mean_and_err(x, evaluation_names[1], message_size, percents)\n",
    "        filler = \"$\" + f\"{mean1:.2f}\" \n",
    "        filler += r'\\%' if percents else ''\n",
    "        filler += r\"\\pm \" + f\"{err1:.2f}\" + \"$\"\n",
    "        \n",
    "        filler2 = \"$\" + f\"{mean2:.2f}\" \n",
    "        filler2 += r'\\%' if percents else ''\n",
    "        filler2 += r\"\\pm \" + f\"{err2:.2f}\" + \"$\"\n",
    "        \n",
    "        s = s.replace(F\"XX{idxs[0]},{idxs[1]}\", filler)\n",
    "        s = s.replace(F\"YY{idxs[0]},{idxs[1]}\", filler2)\n",
    "        \n",
    "        s = s.replace(\"LABEL\", label)\n",
    "        s = s.replace(\"CAPTION\", caption)\n",
    "\n",
    "    print(s)\n",
    "    \n",
    "# plot_compositionality_table([\"addition_compositionality_mean_prediction_accuracy\", \n",
    "#                              \"analogy_compositionality_net_prediction_mean_accuracy\"],\n",
    "#                            cells,\n",
    "#                            \"CAPTION\",\n",
    "#                            \"LABEL\", percents=True)\n",
    "\n",
    "# plot_compositionality_table([\"addition_compositionality_mean_prediction_accuracy\"],\n",
    "#                            cells,\n",
    "#                            \"CAPTION\",\n",
    "#                            \"LABEL\", percents=True)\n",
    "\n",
    "plot_compositionality_table([\"addition_compositionality_mean_prediction_loss\", \n",
    "                             \"analogy_compositionality_net_prediction_mean_loss\"],\n",
    "                           cells,\n",
    "                           \"Lossess\",\n",
    "                           \"composition_mse_losses\", percents=False)\n",
    "\n",
    "plot_compositionality_table([\"addition_compositionality_mean_prediction_loss\",\n",
    "                            \"analogy_compositionality_net_prediction_mean_loss\"],\n",
    "                           cells,\n",
    "                           \"Lossess\",\n",
    "                           \"composition_mse_losses\", percents=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "def plot_table_bars(evaluation_name, cells, title, y_label, y_min, y_max=None, percents=False, message_size=2):\n",
    "    plt.style.use('seaborn-whitegrid')\n",
    "    num_ticks = 5\n",
    "    #TODO\n",
    "#     fig, ax = plt.subplots(cells.shape[0], cells.shape[1], figsize=(7,10))\n",
    "    fig, ax = plt.subplots(2,2, figsize=(7,10))\n",
    "    \n",
    "    print(\"cell shape: \", cells.shape)\n",
    "    global_max = float(\"-inf\")\n",
    "    \n",
    "    for row in range(cells.shape[0]):\n",
    "        for col in range(cells.shape[1]):\n",
    "            mean, err = get_evaluation_mean_and_err(cells[row,col], evaluation_name, message_size)\n",
    "            global_max = max(global_max, mean)\n",
    "            print(\"row: ,\", row)\n",
    "            print(\"col: ,\", col)\n",
    "            print(\"ax: \", ax)\n",
    "            ax_ = ax[row,col]\n",
    "            ax_.bar([0], mean, yerr=err, capsize=10, color=\"cornflowerblue\")\n",
    "            ax_.set_xticks([])\n",
    "            ax_.set_xticklabels([])\n",
    "#             ax_.set_ylabel(y_label)\n",
    "    \n",
    "    y_max = y_max or global_max\n",
    "    \n",
    "    y_interval = round(\n",
    "        y_max / num_ticks, int(np.abs(np.floor(np.log10(y_max / num_ticks))))\n",
    "    )\n",
    "    y_ticks = [round(x, 1) for x in np.arange(0.0, y_max + y_interval, y_interval)]\n",
    "    y_tick_labels = [str(x) for x in y_ticks]\n",
    "    if percents:\n",
    "        y_tick_labels = [str(int(x*100)) + \"%\" for x in y_ticks]\n",
    "    \n",
    "    for _, curr_ax in np.ndenumerate(ax):\n",
    "        curr_ax.set_yticks(y_ticks)\n",
    "        curr_ax.set_yticklabels(y_tick_labels)\n",
    "\n",
    "#     fig.suptitle(title, fontsize=18, y=0.98)\n",
    "    \n",
    "    fig.text(0.31,0.9, \"Shared context\", fontsize=16, horizontalalignment='center')\n",
    "    fig.text(0.72,0.9, \"Non-shared context\", fontsize=16, horizontalalignment='center')\n",
    "    \n",
    "    fig.text(-0.02,0.8, \"Strict\", fontsize=16, horizontalalignment='center')\n",
    "    fig.text(-0.02,0.6, \"Non-strict,\\n5 objects\", fontsize=16, horizontalalignment='center')\n",
    "    fig.text(-0.02,0.4, \"Non-strict,\\n10 objects\", fontsize=16, horizontalalignment='center')\n",
    "    fig.text(-0.02,0.2, \"Non-strict,\\n15 objects\", fontsize=16, horizontalalignment='center')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compositionality accuracy drop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Addition network\n",
      "evaluation name:  object_prediction_accuracy\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "evaluation name:  addition_compositionality_mean_prediction_accuracy\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "evaluation name:  object_prediction_accuracy\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "evaluation name:  addition_compositionality_mean_prediction_accuracy\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "\t\t [[0.031, 0.03471428571428572], [0.028, 0.032285714285714286]]\n",
      "extremity_game_acl_4__o_15__s_1__n_5__m_2__l_mse:\n",
      "\t -0.0037142857142857186 %\n",
      "\n",
      "extremity_game_acl_4__o_15__s_1__n_5__m_2__l_cross_entropy:\n",
      "\t -0.004285714285714285 %\n",
      "\n",
      "Min: -0.004285714285714285\n",
      "Compositionality network\n",
      "evaluation name:  object_prediction_accuracy\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "evaluation name:  compositionality_net_prediction_mean_accuracy\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "evaluation name:  object_prediction_accuracy\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "evaluation name:  compositionality_net_prediction_mean_accuracy\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "\t\t [[0.031, 0.032], [0.028, 0.038000000000000006]]\n",
      "extremity_game_acl_4__o_15__s_1__n_5__m_2__l_mse:\n",
      "\t -0.0010000000000000009 %\n",
      "\n",
      "extremity_game_acl_4__o_15__s_1__n_5__m_2__l_cross_entropy:\n",
      "\t -0.010000000000000005 %\n",
      "\n",
      "Min: -0.010000000000000005\n"
     ]
    }
   ],
   "source": [
    "def get_performance_drop(evaluation_names):\n",
    "    eval_means = [[get_evaluation_mean_and_err(str(simulation_name), x, message_size=2)[0] for x in evaluation_names] \n",
    "                  for simulation_name in np.nditer(cells)]\n",
    "    print(\"\\t\\t\", eval_means)\n",
    "\n",
    "    decreases = [(x[0]-x[1]) for x in eval_means]\n",
    "\n",
    "    for sim, decrease in zip(np.nditer(cells), decreases):\n",
    "        print(f\"{sim}:\\n\\t\", decrease, \"%\\n\")\n",
    "    print(\"Min:\", min(decreases))\n",
    "\n",
    "print(f\"Addition network\")\n",
    "get_performance_drop([\"object_prediction_accuracy\", \"addition_compositionality_mean_prediction_accuracy\"])\n",
    "\n",
    "print(f\"Compositionality network\")\n",
    "get_performance_drop([\"object_prediction_accuracy\", \"compositionality_net_prediction_mean_accuracy\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compositionality Appendix Tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "evaluation name:  addition_compositionality_mean_prediction_accuracy\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "evaluation name:  addition_compositionality_mean_prediction_accuracy\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "\n",
      "\\begin{table}[]\n",
      "\\begin{tabular}{lcc}\n",
      "\\toprule\n",
      "                       & \\textbf{Shared} & \\textbf{Non-shared} \\\\ \\midrule\n",
      "\\textbf{Strict} & \\\\\n",
      "\\emph{10 objects} & \n",
      "\t$3.47\\%\\pm 0.00$  & XX0,1      \\\\ % \\midrule\n",
      "\\textbf{Non-strict} & \\\\\n",
      "\\emph{5 objects} &\n",
      "\t$3.23\\%\\pm 0.00$  & XX1,1      \\\\\n",
      "%\\textbf{Non-strict} & \\\\\n",
      "\\emph{10 objects} &\n",
      "\tXX2,0  & XX2,1      \\\\\n",
      "\\emph{15 objects} &\n",
      "\tXX3,0  & XX3,1      \\\\ \\bottomrule\n",
      "\\end{tabular}\n",
      "\\caption{addition_compositionality_mean_prediction_accuracy }\\label{tab:addition_compositionality_accuracy}\n",
      "\\end{table}\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "plot_table(\"addition_compositionality_mean_prediction_accuracy\", cells, caption=\"addition_compositionality_mean_prediction_accuracy \", label=\"addition_compositionality_accuracy\", percents=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "evaluation name:  analogy_compositionality_net_prediction_mean_accuracy\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "evaluation name:  analogy_compositionality_net_prediction_mean_accuracy\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "\n",
      "\\begin{table}[]\n",
      "\\begin{tabular}{lcc}\n",
      "\\toprule\n",
      "                       & \\textbf{Shared} & \\textbf{Non-shared} \\\\ \\midrule\n",
      "\\textbf{Strict} & \\\\\n",
      "\\emph{10 objects} & \n",
      "\t$3.20\\%\\pm 0.00$  & XX0,1      \\\\ % \\midrule\n",
      "\\textbf{Non-strict} & \\\\\n",
      "\\emph{5 objects} &\n",
      "\t$3.17\\%\\pm 0.00$  & XX1,1      \\\\\n",
      "%\\textbf{Non-strict} & \\\\\n",
      "\\emph{10 objects} &\n",
      "\tXX2,0  & XX2,1      \\\\\n",
      "\\emph{15 objects} &\n",
      "\tXX3,0  & XX3,1      \\\\ \\bottomrule\n",
      "\\end{tabular}\n",
      "\\caption{addition_compositionality_mean_prediction_accuracy }\\label{tab:addition_compositionality_accuracy}\n",
      "\\end{table}\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "plot_table(\"analogy_compositionality_net_prediction_mean_accuracy\", cells, caption=\"addition_compositionality_mean_prediction_accuracy \", label=\"addition_compositionality_accuracy\", percents=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Object Prediciton Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cell shape:  (2, 1)\n",
      "evaluation name:  object_prediction_accuracy\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "row: , 0\n",
      "col: , 0\n",
      "ax:  [[<matplotlib.axes._subplots.AxesSubplot object at 0x12f872e80>\n",
      "  <matplotlib.axes._subplots.AxesSubplot object at 0x12f7a30a0>]\n",
      " [<matplotlib.axes._subplots.AxesSubplot object at 0x12fa63cd0>\n",
      "  <matplotlib.axes._subplots.AxesSubplot object at 0x12f9be550>]]\n",
      "evaluation name:  object_prediction_accuracy\n",
      "trial:  dict_keys(['training_losses', 'object_prediction_accuracy', 'detected_num_clusters', 'object_prediction_by_cluster_loss', 'predictions_by_average_msg_loss', 'predictions_by_average_msg_accuracy', 'addition_compositionality_mean_message_loss', 'addition_compositionality_mean_message_cluster_accuracy', 'addition_compositionality_mean_prediction_loss', 'addition_compositionality_mean_prediction_accuracy', 'analogy_compositionality_net_message_mean_loss', 'analogy_compositionality_net_message_cluster_mean_accuracy', 'analogy_compositionality_net_prediction_mean_loss', 'analogy_compositionality_net_prediction_mean_accuracy', 'compositionality_net_message_mean_loss', 'compositionality_net_message_cluster_mean_accuracy', 'compositionality_net_prediction_mean_loss', 'compositionality_net_prediction_mean_accuracy', 'functions_from_messages', 'min_max_from_messages', 'dimension_from_messages', 'sanity_from_messages', 'object_by_context_from_messages', 'object_by_decoder_context_from_messages', 'context_from_messages', 'decoder_context_from_messages'])\n",
      "row: , 1\n",
      "col: , 0\n",
      "ax:  [[<matplotlib.axes._subplots.AxesSubplot object at 0x12f872e80>\n",
      "  <matplotlib.axes._subplots.AxesSubplot object at 0x12f7a30a0>]\n",
      " [<matplotlib.axes._subplots.AxesSubplot object at 0x12fa63cd0>\n",
      "  <matplotlib.axes._subplots.AxesSubplot object at 0x12f9be550>]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/murphy/anaconda3/envs/signaling_auto_encoder/lib/python3.8/site-packages/dataclasses_json/core.py:142: RuntimeWarning: `NoneType` object value of non-optional type target_function detected when decoding Simulation.\n",
      "  warnings.warn(f\"`NoneType` object {warning}.\", RuntimeWarning)\n",
      "/Users/murphy/anaconda3/envs/signaling_auto_encoder/lib/python3.8/site-packages/dataclasses_json/core.py:142: RuntimeWarning: `NoneType` object value of non-optional type context_generator detected when decoding Simulation.\n",
      "  warnings.warn(f\"`NoneType` object {warning}.\", RuntimeWarning)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgkAAAJUCAYAAACfcrXRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdfVzN9/8/8MepNHQp17OPyzkhpSu1RFQs+2WMInyEr6uIpWYutlFG0/Sdq5VYzOTqoxg+k8thRLUiV5sdm5m5GoZEpU4X5/X7o6/3nPWWpDrHPO63m9ut3q/3+3We7+P9OufR+/V+n6MQQggQERER/Y2BrgsgIiIi/cSQQERERLIYEoiIiEgWQwIRERHJYkggIiIiWQwJRC+wmrg5iTc8ET1dTY0TfRt/DAmkE+np6Rg7diy6du0KW1tb9O3bF0uWLEFeXp60zrZt22BtbY3s7GwdVlqetbU1vvzyS12XgaSkJCxdurRa+7xw4QJGjRpVrX3SXwIDA2FnZ4fff/+9XJtKpYK1tTUyMjJqv7C/ycjIgLW1NX744Qddl6LFy8sL8+bN03UZOHDgACIiIqq1z5s3b2Ls2LG4d+9etfb7vBgSqNYdOXIEY8aMQbNmzRAdHY34+HgMGTIEmzdvxrhx41BaWqrrEl8IK1euRG5ubrX2uXfvXr17Y/inUavVmDNnjq7LoOeQkJCAW7duVWufaWlpOHbsWLX2WR2MdF0AvXxWr14Nd3d3fPLJJ9IyNzc3tG3bFkFBQTh27Bh69uypwwqJao6ZmRkyMzOxZcsWDB48WNflEFWIZxKo1mVnZ8vOu7m7uyMsLAxNmzbVWv79999jwIABsLW1ha+vLw4ePKjVfvToUYwYMQIODg6wtbXFgAEDsH//fqk9JiYGgwYNwoIFC+Do6Ih33nkHAFBSUoJly5ahV69esLW1xaBBg5Cenq7V96VLlzB27Fg4ODigT58+OHr0aKX2MTExEb6+vrCzs0Pfvn2RlJQktQkhkJSUhLfffht2dnZ48803sXbtWq3tra2tsW3bNoSFhcHBwQGurq745JNPUFJSAqDstOv169exceNGWFtbS9v9+OOPGDVqFLp06YI33ngD8+fPR0FBAQDg2rVrcHBwQFhYmLT+1atXYW9vj48//hgxMTGIjY3Fw4cPpcen6ufk5ARPT09ER0fj9u3bFa577do1TJ06FW5ubnBwcMCkSZO0pioeHdvJycnw8fGBra0t/Pz8cPLkyafWsX37dvj6+sLW1hYeHh5YsGAB1Gq11jrnz5/HsGHDYGtrC29vb2zZskWr/ezZsxg/fjycnZ3RuXNn+Pj4YPPmzVL7tm3b4OrqitWrV8PV1RU9e/bEw4cPAQDr1q3Dm2++ic6dO8PX1xe7d+/W6vv27dsICQmBk5MTevTogR07djx1nwBg//79GDRoELp06QIvLy+sXLlS6/Xm22+/hZ+fH+zt7dGzZ08sXbpUGldA2dhatWoVIiIi4OLiAkdHR8ycOVOaCg0MDERmZiYOHz4Ma2trXLt2DQBw+fJlBAcHw8HBAc7Ozpg+fbo0VZqbm4sePXogICBAquX+/fvo3r07goKCsG3bNnzwwQcAyv5giomJqdS+1gpBVMs+/fRToVQqRVBQkEhOThZ//vmn7Hpff/21UCqVolu3bmLLli0iJSVFBAQECFtbW3Hnzh0hhBBnzpwRHTp0EHPnzhVpaWni22+/FQEBAaJz587i7t27QgghPv/8c9GpUycRGBgo0tLSxHfffSeEEGLWrFmiS5cu4ssvvxRHjhwR06ZNEzY2NiIrK0sIIURubq5wd3cXAwYMEN9++61ISkoS3bp1E0qlUqxevfqJ+7dmzRphbW0toqKiRGpqqliyZIlQKpUiOTlZCCHEZ599Jjp27CgWL14sjh49KhYtWiQ6dOggFi9eLPWhVCqFk5OTmDdvnkhLSxPLli0TSqVSbNy4UQghxLlz54S7u7t49913xalTp4QQQly4cEF06dJFjBo1Shw6dEhs2bJFvPHGG2LChAlSv2vXrhVKpVKkpaUJjUYjAgMDRe/evUV+fr64ceOG+PDDD4WdnZ04deqU9PxR9RkxYoSYMGGCuH79urC3txfvvvuu1PbTTz8JpVIpvv/+eyGEEDdu3BBvvPGGGDBggNi7d6/Ys2eP6Nevn3BzcxM3b94UQpQd2w4ODqJPnz7im2++Ed99953w9fUV3bt3F8XFxU+sIzMzU3To0EHExcWJjIwMsXHjRmFrays+++wzIYQQ33//vVAqlcLe3l6sXbtWpKamikmTJglra2uhUqmEEEJcv35ddOnSRYSEhIhjx46Jw4cPi6CgIKFUKsX58+eFEGVjuGPHjqJ///4iJSVF7NmzRwghRExMjOjUqZNYsmSJOHr0qIiMjBTW1tZi9+7dQgghSkpKxNtvvy08PT3Frl27RHJysvD09BSdOnUSH3/88RP3a+/evUKpVIpZs2aJlJQU8dVXXwkbGxvxxRdfCCGE2Lx5s1AqlWLu3Lni6NGjIj4+Xtja2opp06ZJfXh6egonJycRGhoqjh49KhISEoSNjY2Ijo4WQpSNs3feeUcMHTpUnDp1SqjVanH79m3RrVs3MWDAALF//36RnJwsevfuLd555x2hVquFEEJ8++23QqlUii1btgghhJg5c6ZwcXERt27dEnfv3pVeJ1JSUsSNGzcqdTzVBoYEqnVqtVrMmjVLdOjQQSiVSqFUKkXfvn3FkiVLRE5OjrTeo5Dw6M1VCCF+/PFHoVQqxYEDB4QQQmzdulXrhVaIsjdQpVIpDh06JIQoeyFVKpXi3Llz0jq//vqrUCqVIikpSWvbkSNHisDAQCGEEAkJCaJTp05aA3bfvn0VhoTS0lLh4uIiPvjgA63l7733npg7d67Izs4WNjY20ovxI5999pmwsbGR3piVSqUYN26c1jrvvPOOCAoKkn739PTUesEMCwsT3t7e0ouSEEIcP35cKJVKkZmZKdUXEBAgfH19xcaNG0WHDh3EiRMnpPU///xzYW9vL7tv9PwehQQhyo6vx4/lv4eEqKgoYW9vrxXW7t69KxwcHERUVJQQ4q9j+8yZM9I6Bw4cEEqlUvzwww9PrOOLL74QDg4OWsdKUlKS2L59uxDir5Dw6M1VCCHu3bsnlEqlSEhIEEIIcfjwYTFq1ChRVFQkrZOTkyOUSqVYv369EOKvMbx//35pnfv37wtbW1utUCyEEB988IHw9vbW2odHAViIv8Z+RSFhwIABYuTIkVrLoqOjRVBQkCgpKRFvvPGGCAsL02r/z3/+I5RKpRR+PD09ha+vr9BoNNI6kydPFv369ZN+f/z/UYiy8evk5KT1f3XlyhXRsWNH6TkVQojQ0FDh5uYm9uzZI5RKpdi5c6fU9ui50rdwzukGqnXGxsaIiorCd999h4iICPTp0wd3797FihUr0K9fP1y9elVrfQcHB+nnFi1aAIB0wZ6fnx8+//xzPHz4ED/88AN27tyJjRs3AgCKioq0+mndurX0c2ZmJgDAw8MDJSUl0r+ePXvi5MmTKCoqwsmTJ6FUKtGsWTNpO29vbxgaGj5x3y5duoScnBx4eXlpLV+0aBEiIiJw5swZFBcXo2/fvlrtvr6+KC4uxpkzZ6RlXbp00VqnadOm0qlaORkZGejWrRsMDAyk/bG3t4epqak0jWJgYIBPPvkEly9fxvz58zF69Gg4OTk9sU+qOSNGjICdnR3mzZundVfPI8ePH4erqyusrKykZVZWVnBzc5OOXwAwMjJC586dpd8fHa+PppkeP75LSkoghICjoyMePnyI/v37Y9myZTh79iz8/f2lqbhHHh97lpaWMDExwYMHDwAAPXv2xNq1a6HRaHD+/Hns3bsXX3zxBYDyY69NmzbSz6dPn4ZarUavXr206vLw8MDVq1dx9epVnDx5EhYWFrC3t5e2s7Gxkca/nMLCQqhUKnh6emotnz59OlauXInffvsN2dnZsmMPAE6cOCEts7W1hUKh0HpOnzb27O3tYW5uLu1P8+bN0a5dO60pzDlz5qC0tBRhYWHw8fFBv379ntinvuCFi6QzzZo1w/DhwzF8+HCUlJTgv//9LyIiIhAbG4uFCxdK69WtW1f62cCgLNdqNBoAwMOHDxEeHo49e/YAKHsx6tChAwDt+43r16+P+vXrS7/n5OQAKAsJcu7du4cHDx6gQYMGWssNDQ21XrT/7lG/T1rn/v37AIBGjRppLW/YsCEAaL1Z1KtXT2sdAwODCu+hzsnJQWJiIhITE8u1PT733a5dO9jY2ODUqVO8QFSHDAwMEBkZiUGDBmHRokUYMmSIVvuDBw/QsWPHcts1bNgQv/76q/S7sbGxNC4e9QuUjZFr167B29tba/t169bB1dUVcXFx+OqrrxAfH4+4uDi89tprmDt3Lnr06CGt+/jYe9T3o2OwtLQUn376KRITE1FcXIyWLVvC2dkZQPl7/R8fD4/GyNChQ2Wfl9u3b8uOPQBo3Lix7DbAX2Pr0ViqbLuZmRmMjY0rHHsKheKpY+/MmTOwsbGpsGYrKyt0794dycnJL8zYY0igWnX69GkEBwdjxYoVWn8pGxkZwc/PD4cOHcLFixcr3d/8+fORmpqK+Ph4dO3aFcbGxvj111+xc+fOCrczMzODQqHA5s2bZc8MNGjQAJaWluVqEUJILzZP6hdAuc92uHTpEu7duwdLS0sAwJ07d7Qu0Lxz5w4ASO1VYWpqCm9vbwwbNqxc2+MvuN988w1Onz4NpVKJefPmYceOHTA2Nq7y41LVWVtbY+zYsYiPj8frr7+u1WZhYSEdF4+7c+dOpY+TJk2aYOvWrVrLHv1V7+XlBS8vL+Tm5iIlJQUrVqxAWFgY0tLSKtX3ihUrkJSUhIULF6Jnz56oX78+CgoKyj3e3z0aI8uXLy93kfKj+iwtLXH37t1ybY8ChhwTExMA5cfezZs3cfnyZSmo/L3fBw8eoKio6LnHnoeHB0JCQp5YF1B2tmLXrl2wtrbGZ599Bi8vL9kwpE843UC1qnXr1sjPz8e6devKtZWWluLq1ato3759pfs7ffo0evToAXd3d+mN7tEdCBUlfycnJwghkJeXB1tbW+lfeno61q5dCyMjI7i6uuLChQtaV5Onp6eXO5X6uLZt28LS0hKHDx/WWr5s2TJER0fD1tYWderUwd69e7Xad+/eDSMjI9jZ2VV63x//6/HRPv3222/o3LmztD/NmzfHokWLcOHCBQBlL6ALFiyAv78/YmJicPXqVSxfvvyJfVLNmzx5Mlq2bInFixdrLXdyckJGRobWm152djbS09Ph6OhYqb6NjY21jm9bW1uYmpoiJiZGOnNhZmYGX19fjB07Frm5ubJTH3JOnz6Nzp0746233pLO0lVm7HXp0gV16tTB3bt3teq6cOGCdCy6uroiNzdX61T9b7/9hitXrjyxX1NTUyiVynJjb/369Xj//ffRtm1bNGjQQHbsAaj0cwo8eexZW1tL+6NUKhEbG4usrCwAZZ+PMXv2bLi5uWHdunUQQmjdBq6vY49nEqhWWVpaIiwsDFFRUcjJycHAgQPRrFkz/Pnnn9i8eTNu3bqF2NjYSvdna2uLQ4cOYfv27WjevDm+//576dMQCwsLn7hdx44d4ePjg+nTp2PKlClo164dMjMzsWLFCowbNw4GBgZ45513sGbNGkycOBFhYWEoLCzEkiVLUKdOnSf2a2RkhKCgIPzv//4vGjRoADc3Nxw/fhx79+5FbGwsrKysEBgYiC+//BKGhobo2rUrjh8/ji+//BKjR4+GhYVFpffd3Nwc586dQ2ZmJrp27Yrg4GAMHToUU6dOhZ+fH4qKihAXF4cbN26gU6dOAIDIyEgAwPvvvw9LS0uMGzcOq1atwltvvYUOHTrA3NwcBQUFOHDgAOzs7NCkSZNK10NV88orr2DevHnlPuly9OjR2L59O8aMGYNJkyYBKPvr3djY+Lk/FdPV1RXLly/H7Nmz4evri/v372PlypVwcnKqcDrtcba2tli1ahU2bNgApVKJH374AcuXL4dCoahw7D0aA59++inu378POzs7nD9/HkuWLIG3tzdMTU3h7u6Orl27Yvr06Xj//fdRv359LF26tMKxB5QFrqlTp2LOnDno27cvfvnlF6xbtw4zZsyAoaEhpkyZgvnz58PCwgLe3t74+eefERMTg759+0KpVFb6+TM3N4dKpUJGRga6dOmC//mf/8F///tfjBs3DiNHjkSdOnWwZs0anD59GqGhoQCA2NhYXL9+HStXroSlpSXef/99fPTRR3j77bfRs2dPmJubAyi7RdPd3R2vvfZapeupUTq6YJJecocOHRJjxowRrq6uolOnTsLd3V3MmDFDXLlyRVpH7mrf+/fvC6VSKb7++mshRNnV3u+++65wdnYWzs7OIiAgQBw+fFi8+eabYs6cOUKIJ1+xr1arRXR0tPDw8BA2NjbizTffFKtWrdK6qvnGjRsiODhY2NvbCw8PD7F9+3bRrVu3Cm+BFEKIDRs2iD59+ojOnTuLfv36iV27dkltpaWl4osvvhBeXl7S465du1brceXuoJg0aZIYMWKE9PvevXuFq6ursLOzk+7AOHHihBgxYoSws7MTXbt2FUFBQeKXX36RnvPHb8ESQojCwkLh7e0tBg4cKEpKSsTdu3fFoEGDhI2NjYiPj69wH+nZ/f2q+Md98MEHWnc3CCHEL7/8IsaPHy/s7e2Fk5OTCA4OFr/99pvULnds//0uiSfZtWuXGDBggOjSpYtwcXER77//vnRr8aO7G86ePau1jZOTk/j888+FEEI8fPhQfPTRR8LV1VXY29uLd955R2zfvl2MGTNGjBkzRgjx5Cv2S0tLRXx8vOjdu7ewsbERnp6eYtGiRVp3Wzx48EDMmjVLODs7izfeeEPEx8eLwYMHV3h3gxBC7N69W/Tr10/Y2NiIPn36SHdaPLJlyxbx1ltvSY+7dOlSrTs0/n7XkBBCREZGCk9PT+n3EydOiB49eojOnTtLt0xfuHBBTJgwQdjb2wsHBwcxYsQIqe3cuXOiU6dOYunSpVIfGo1GDB06VPTs2VPk5uaKhw8fijFjxggbG5un7mNtUgihZ98mQURERHpBPydBiIiISOcYEoiIiEgWQwIRERHJYkggIiIiWQwJREREJIshgYiIiGQxJBAREZEshgQiIiKSxZBAREREshgSiIiISBZDQi05c+YMAgMDAQCXL1/GsGHDMHz4cERERECj0QAo+wIQf39/DB06FGfPngUApKSkwN/fHyEhIdJ68+bNw7Vr13SzI0R6gmOKqOYxJNSCVatWYfbs2VCr1QCAqKgohIaGYtOmTRBC4ODBg9K3+W3ZsgWLFy/Gxx9/DADYtGkT1qxZgyZNmuD8+fM4f/48TE1N9ecbwoh0gGOKqHYwJNSCli1bIiYmRvr93LlzcHFxAQB4eHggLS0NWVlZ6N69OxQKBV599VWUlpYiOzsbJiYmKCwshFqtRr169bBq1SqMHz9eV7tCpBc4pohqh5GuC3gZ+Pj4aJ3KFEJAoVAAAExMTJCbm4u8vDxYWlpK6zxaHhwcjKioKFhbW+PKlStwdHREcnIyVCoVBg4cCAcHh3KPl5WVVfM7RVTDnJycntjGMUX07CoaU0/CkKADBgZ/ncDJz8+Hubk5TE1NkZ+fr7XczMwMVlZWWLJkCUpLSxEaGorIyEh8+OGHWLZsGSZNmoRVq1bJPkZVDgYiffGsb8ov05hSqVTo2LGjrsuQ6FM9+lQLoF/1VDXocrpBBzp16oSMjAwAZRdROTs7w9HREceOHYNGo8Eff/wBjUYDKysraZvExEQMHDgQAKDRaKBQKFBQUKCT+on0DccUUc3gmQQdmDlzJubMmYPFixejbdu28PHxgaGhIZydnREQEACNRoPw8HBp/by8PGRmZmLp0qUAgMaNG0tXchMRxxRRTVEIIYSui6DqlZWVpTenRomqQt+OYX2qR59OYQP6VY8+1QLoVz1VPYY53UBERESyGBKIiIhIFkMCERERyWJIICIiIlkMCURERCSLIYGIiIhkMSQQERGRLIYEIiIiksWQQERERLIYEoiIiEgWQwIRERHJYkggIiIiWQwJREREJIshgYiIiGQxJBAREZEshgQiIiKSxZBAREREshgSiIiISBZDAhEREcliSCAiIiJZDAlEREQkiyGBiIiIZDEkEBERkSyGBCIiIpLFkEBERESyGBKIiIhIFkMCERERyWJIICIiIlkMCURERCSLIYGIiIhkMSQQERGRLIYEIiIiksWQQERERLIYEoiIiEgWQwIRERHJYkggIiIiWQwJREREJIshgYiIiGQxJBAREZEshgQiIiKSpbchQQjxQvRJRET0T1XrISE9PR1jx45F165dYWtri759+2LJkiXIy8sDABQVFSEyMhIHDx6ssJ+YmBg4ODhU+nEPHDiAiIiI56qdiIjoZVKrIeHIkSMYM2YMmjVrhujoaMTHx2PIkCHYvHkzxo0bh9LSUvz5559Yv349SkpKKuxr8ODBSEhIqPRjJyQk4NatW8+7C9WmuLgY06ZNw9ChQzF8+HBcvHgRly9fxrBhwzB8+HBERERAo9FAo9EgODgYgwcPRmpqKgDg6tWriIyM1PEeEOkXjimi6mdUmw+2evVquLu745NPPpGWubm5oW3btggKCsKxY8fQrl27SvXVrFkzNGvWrKZKrXFHjhxBSUkJNm/ejNTUVCxduhTFxcUIDQ2Fq6srwsPDcfDgQbz66qto0aIFoqKiMGvWLLi7uyMuLg7Tpk3T9S4Q6RWOKaLqV6shITs7W/aN3d3dHWFhYWjatCm8vb0BAFOnToWLiwvWr18PLy8v/L//9/+QmZmJ8+fPIyQkBAUFBVizZg1OnToFACgtLcWqVauwdetW3L59G61bt8a7776L3r17IzAwEJmZmQAAa2trHDx4EK+99lrt7biMNm3aoLS0FBqNBnl5eTAyMsLp06fh4uICAPDw8EBqaipGjhwJtVqNwsJC1K9fH1lZWWjdujUaNWpUYf8qlao2doNIb7wsY6qwsFBvagH0qx59qgXQv3qqolZDgoeHB9asWYOJEyfi7bffhouLCxo3bow6depg4sSJKCoqQmxsLKZMmYL33ntPCgwA8NVXXyEkJASTJk1Cq1atsGvXLq2+o6KisHnzZgQHB8PBwQF79uxBSEgI1q1bh4iICEyfPh1169bFzJkz0aRJk9rcbVn169fH9evX8dZbb+HevXtYuXIljh8/DoVCAQAwMTFBbm4u2rRpg6ZNmyI6OhrBwcFYtmwZpk+fjoiICFhYWCA0NBQGBuVnjTp27Fjbu0RUbbKysp55m5dlTKlUKr2pBdCvevSpFkC/6qnKmAJqOSSEhYUhJycHO3bswHfffQcAaNu2LXx8fPA///M/sLCwkJ7QVq1a4fXXX5e2bdeuHYKCgmT7zcnJwaZNmzB58mQEBwcDKJvGuHTpEk6cOIGJEyfC1NQU9evXh729fQ3vZeWsXbsW3bt3x7Rp03Djxg2MGjUKxcXFUnt+fj7Mzc0BAJMnTwYA7Ny5E97e3khKSoK/vz8yMzORnp4Od3d3newDkT7hmCKqfrV64aKxsTGioqLw3XffISIiAn369MHdu3exYsUK9OvXD1evXn3itm3atHli25kzZ1BaWgovLy+t5evXr8fEiROrrf7qZG5uDjMzMwCAhYUFSkpK0KlTJ2RkZAAAUlJS4OzsLK2vVquxf/9+9O/fHwUFBTA0NIRCocDDhw91Uj+RvuGYIqp+OvmchGbNmmH48OGIjY1FWloaFixYgHv37iE2NvaJ2zRs2PCJbffv3wcAWFlZVXutNWX06NE4d+4chg8fjlGjRiEsLAzh4eGIiYlBQEAAiouL4ePjI62fkJCAwMBAKBQK+Pn5ISIiAkePHuVfPET/h2OKqPrV2nTD6dOnERwcjBUrVqBLly5/FWBkBD8/Pxw6dAgXL16sUt+P/nq4d+8emjZtKi1XqVQQQqBTp07PV3wNMDExwbJly8ot37Bhg+z6EyZMkH7u2LEjtmzZUmO1Eb2IOKaIql+tnUlo3bo18vPzsW7dunJtpaWluHr1Ktq3bw9DQ8Nn7tvOzg5GRkbSdQ6PhIeH48svvwQA2QuRiIiI6Mlq7UyCpaUlwsLCEBUVhZycHAwcOBDNmjXDn3/+ic2bN+PWrVuIjY2VzgqkpaWhdevW6NChw1P7btiwIYYOHYoVK1bAyMgInTt3xp49e6BSqRAeHg6gbL5SpVIhIyMDXbp0Qd26dWt0f4mIiF50tXp3w+jRo9GqVSts2LABkZGRyM3NRYMGDaQPWPrXv/4FABg/fjw2bNiAU6dOYefOnZXq+8MPP0SDBg2wceNG3Lt3D+3bt8eqVatga2srPXZYWBjGjRuHhIQEODo61th+EhER/RMoBL/16B8nKysLTk5Oui6DqMr07RjWp3r06d57QL/q0adaAP2qp6rHMCfqiYiISBZDAhEREcliSCAiIiJZDAlEREQkiyGBiIiIZDEkEBERkSyGBCIiIpLFkEBERESyGBKIiIhIFkMCERERyWJIICIiIlkMCURERCSLIYGIiIhkMSQQERGRLIYEIiIiksWQQERERLIYEoiIiEgWQwIRERHJYkggIiIiWQwJREREJIshgYiIiGQxJBAREZEshgQiIiKSxZBAREREshgSiIiISBZDAhEREcliSCAiIiJZDAlEREQkiyGBiIiIZDEkEBERkSyGBCIiIpLFkEBERESyGBKIiIhIFkMCERERyWJIICIiIlkMCURERCSLIYGIiIhkMSTo0BdffIGAgAAMGjQIW7ZsweXLlzFs2DAMHz4cERER0Gg00Gg0CA4OxuDBg5GamgoAuHr1KiIjI3VcPZH+4Zgiql4MCTqSkZGBU6dO4T//+Q/Wr1+PmzdvIioqCqGhodi0aROEEDh48CBUKhVatGiB1atXY8OGDQCAuLg4TJw4Ucd7QKRfOKaIqh9Dgo4cO3YMSqUSkydPxsSJE9GrVy+cO3cOLi4uAAAPDw+kpaWhfv36UKvVKCwsRP369ZGVlYXWrVujUaNGOldc1kYAACAASURBVN4DIv3CMUVU/Yx0XcDL6t69e/jjjz+wcuVKXLt2DZMmTYIQAgqFAgBgYmKC3NxctGnTBk2bNkV0dDSCg4OxbNkyTJ8+HREREbCwsEBoaCgMDMpnPZVKVdu7RKRTL8uYKiws1JtaAP2qR59qAfSvnqpgSNARS0tLtG3bFsbGxmjbti1eeeUV3Lx5U2rPz8+Hubk5AGDy5MkAgJ07d8Lb2xtJSUnw9/dHZmYm0tPT4e7uXq7/jh071s6OENWArKysZ97mZRlTKpVKb2oB9KsefaoF0K96qjKmAE436IyTkxOOHj0KIQRu3bqFgoICuLm5ISMjAwCQkpICZ2dnaX21Wo39+/ejf//+KCgogKGhIRQKBR4+fKirXSDSKxxTRNWPZxJ0xNPTE8ePH4e/vz+EEAgPD8drr72GOXPmYPHixWjbti18fHyk9RMSEhAYGAiFQgE/Pz+Eh4fD1NQUy5cv1+FeEOkPjimi6qcQQghdF0HVKysrC05OTroug6jK9O0Y1qd69OkUNqBf9ehTLYB+1VPVY5jTDURERCSLIYGIiIhkMSQQERGRLIYEIiIiksWQQERERLIYEoiIiEgWQwIRERHJYkggIiIiWQwJREREJIshgYiIiGQxJBAREZEshgQiIiKSxZBAREREshgSiIiISBZDAhEREcliSCAiIiJZDAlEREQkiyGBiIiIZDEkEBERkSyGBCIiIpLFkEBERESyGBKIiIhIFkMCERERyWJIICIiIlkMCURERCSLIYGIiIhkMSQQERGRLIYEIiIiksWQQERERLIYEoiIiEgWQwIRERHJYkggIiIiWQwJREREJIshgYiIiGQxJBAREZEshgQiIiKSxZBAREREshgSiIiISBZDAhEREcliSCAiIiJZDAlEREQkiyFBh+7evYuePXvi4sWLuHz5MoYNG4bhw4cjIiICGo0GGo0GwcHBGDx4MFJTUwEAV69eRWRkpI4rJ9JPHFNE1YshQUeKi4sRHh6OunXrAgCioqIQGhqKTZs2QQiBgwcPQqVSoUWLFli9ejU2bNgAAIiLi8PEiRN1WTqRXuKYIqp+DAk6snDhQgwdOhRNmjQBAJw7dw4uLi4AAA8PD6SlpaF+/fpQq9UoLCxE/fr1kZWVhdatW6NRo0a6LJ1IL3FMEVU/I10X8DLatm0brKys0KNHD8THxwMAhBBQKBQAABMTE+Tm5qJNmzZo2rQpoqOjERwcjGXLlmH69OmIiIiAhYUFQkNDYWAgn/NUKlWt7Q+Rrr1MY6qwsFBvagH0qx59qgXQv3qqgiFBB77++msoFAqkp6dDpVJh5syZyM7Oltrz8/Nhbm4OAJg8eTIAYOfOnfD29kZSUhL8/f2RmZmJ9PR0uLu7yz5Gx44da35HiGpIVlbWM63/Mo0plUqlN7UA+lWPPtUC6Fc9zzqmHuF0gw5s3LgRGzZswPr169GxY0csXLgQHh4eyMjIAACkpKTA2dlZWl+tVmP//v3o378/CgoKYGhoCIVCgYcPH+pqF4j0CscUUc1gSNATM2fORExMDAICAlBcXAwfHx+pLSEhAYGBgVAoFPDz80NERASOHj36xL94iIhjiqg6KIQQQtdFUPXKysqCk5OTrssgqjJ9O4b1qR59OoUN6Fc9+lQLoF/1VPUY5pkEIiIiksWQQERERLIYEoiIiEgWQwIRERHJYkggIiIiWQwJREREJIshgYiIiGQxJBAREZEshgQiIiKSxZBAREREshgSiIiISBZDAhEREcliSCAiIiJZDAlEREQkiyGBiIiIZDEkEBERkSyGBCIiIpLFkEBERESyGBKIiIhI1j8qJAghXog+iYiIXgRPDQmBgYGws7PD77//Xq5NpVLB2toaGRkZNVHbM4mNjcWmTZsqXGfbtm2wtrZGdnZ2pfo8ceIEQkJCqqM8IiKiF06lziSo1WrMmTOnpmt5LjExMSgsLKxwnV69eiExMRHm5uaV6nPr1q24dOlSdZRHRET0wqlUSDAzM0NmZia2bNlS0/XUKCsrK9jb28PIyEjXpRAREem9SoUEJycneHp6Ijo6Grdv337ieteuXcPUqVPh5uYGBwcHTJo0SWuaIiYmBoMGDUJycjJ8fHxga2sLPz8/nDx58qk1bN++Hb6+vrC1tYWHhwcWLFgAtVoNALC2tgYAREdHw8vLC0DZNMmcOXMwduxY2NnZYf78+bLTDYmJifD19YWdnR369u2LpKQkAMCsWbOwfft2XLhwQW+mVIiIiGpTpS9cDA8PR0lJCebPny/bfvPmTQwePBiXL1/G3LlzERUVhWvXrmH48OG4deuWtN7vv/+Ozz//HFOmTEFMTAzUajWmTp2KkpKSJz728ePH8eGHH6Jfv3748ssvMXHiRGzevBmxsbEAyt7ogbJg8GgZUHYNQps2bRAXF4cBAwaU6/err75CREQEevTogZUrV6Jv376YM2cOdu3aheDgYPTs2RP/+te/kJiYCBsbm8o+VURERP8IlT7v/uqrryIsLAyffPIJDh48CG9vb632tWvXorCwEGvWrIGVlRUAwMXFBb1798ZXX32FWbNmAQDy8/Oxdu1a2NnZAQBKS0sRHByM8+fPo3PnzrKPferUKdSrVw9jx46FsbExXFxcUKdOHdSpUwcAYG9vDwBo3rw5OnXqJG1nYmKCDz/8EAYGZVno119/ldo0Gg1WrlyJQYMGSbV169YNV69exYkTJ+Dr6wsrKyv88ccfUv9EREQvk2eanB8xYgR27tyJefPmwdXVVavt+PHjcHV1lQICUHYNgJubGzIzM/96QCMjrTDQrFkzAEBBQQEAlDujYGhoCEdHRzx8+BD9+/fHW2+9BU9PT/j7+0OhUFRYb8uWLaWA8HeXLl1CTk6OND3xyKJFiyrsk4iI6GXxTJ+TYGBggMjISNy5c6fcm+mDBw/QqFGjcts0bNgQ+fn50u/GxsZab9yPftZoNLh27RpsbGy0/mVmZsLZ2RlxcXFo3Lgx4uPjMXjwYPTu3RtHjx6tsN6GDRs+sS0nJwcAtEINERER/eWZL/O3trbG2LFjER8fj9dff11abmFhgTt37pRb/86dO7C0tKxU302aNMHWrVu1lrVp0wYA4OXlBS8vL+Tm5iIlJQUrVqxAWFgY0tLSYGxs/Ky7ATMzMwAo95kJly5dwr179+Do6PjMfRIREf2TVOkTFydPnoyWLVti8eLF0jInJydkZGRovelmZ2cjPT290m+4xsbGsLW11fpnamqKmJgYDBkyBEDZm7uvry/Gjh2L3Nxc5OXlle3IE6YVnqRt27awtLTE4cOHtZYvW7YM0dHRVeqTiIjon6RK74KvvPIK5s2bJ71BA8Do0aNRp04djBkzBvv27cO+ffswZswYGBsbY9SoUc9VpKurK86ePYvZs2cjPT0de/fuxcqVK+Hk5CRNF5ibmyMrKwtnzpypVJ9GRkYICgrC119/jUWLFiEtLQ3Lli3D3r17MW7cOKnPmzdvIjU1Fffv3wcAnD59GleuXHmu/SEiInoRVPlP5TfeeAN+fn7S782bN8fGjRvRpEkTzJo1Cx999BFatGiBzZs3SxcnVpWLiwsWL16MH3/8EZMmTUJERATs7OwQExMjrTNlyhRkZGRg/PjxFd5O+bgxY8Zg9uzZ2LdvH4KCgnDgwAEsXrwYvXv3BgAEBASgYcOGCAoKQmpqqrQsLi7uufaHiIjoRaAQ/AajZ5Keno49e/Zg3rx5ui7libKysuDk5KTrMoiqTN+OYX2qR6VSoWPHjrouQ6JP9ehTLYB+1VPVY5iT7s9Ao9Fg9erV8PDw0HUpRERENY5fYvAMDAwMMG3aNK0PbCIiIvqn4pmEZ8SAQERELwuGhP/DSzOIiIi0VSkk3Lt3D9bW1uX+hYSEPFcxgYGBCAoKqnAdLy+var1o8MGDB5g2bRrOnTtXbX0SERH9E1TpmoTz588DANasWQMTExNpeWU/WfF5xMbGwtzcvNr6U6lUSE5OxujRo6utTyIion+CKoWEn3/+GY0aNYK7u3t11/NUvCaAiIiodlRpuuHnn3+GtbX1M293/Phx/Pvf/4ajoyO6deuGefPmaX35E1B2m+HChQvRtWtXuLq64uOPP8bDhw+l9r9PN9y9exczZsyAi4sLHBwcMHHiRFy9elWrz/Pnz2PcuHHS437wwQfIyclBRkYGRo4cCQDw9/eXvjL6zJkz+Pe//w0HBwe4uLggJCQE169ff+b9JSIiepFVOSQUFBRg6NChsLW1hYeHB1avXl3hxX9HjhzByJEj0bhxYyxZsgTvvvsukpOTERQUBI1GI62XmpqKEydO4NNPP0VISAh27NghvXn/XWFhIUaOHImsrCzMnj0b0dHRuHPnDkaMGCF9jPL169cxfPhw5OXlITo6GrNnz0ZqaiqmTZsGGxsbhIeHAwCioqIQHByM3NxcTJgwAU2bNkVcXBzmz5+Pn376Ce+9915VnioiIqIX1jNPN5SWluLixYuoV68eZs6ciVdffRWHDx/GokWLUFhYiClTpshut2zZMtjZ2WHp0qXSstdeew3jxo3D4cOH4eXlBQCoV68eVq9eDQsLCwCAQqHAxx9/jMuXL6NVq1Zafe7YsQOXLl3Czp070a5dOwCAm5sbPD09sX79ekyZMgUJCQkwNDTE6tWrYWpqCqDsuyeio6NRXFwsfZNl+/bt0bJlS5w+fRo5OTkIDAyEg4MDAKBBgwb4/vvvodFoqu1Ln4qLi/Hhhx/i+vXrKCoqwqRJk/D6669j1qxZUCgUaN++PSIiIgCUfeT07du3ERoaCnd3d1y9ehUJCQmYPXt2tdRC9E/AMUVU/ar0jrdy5UokJiZi4MCBcHV1xcyZMzFkyBCsXr0aarW63Pr5+fn46aef0LdvX63lPXr0gIWFBY4fPy4tc3V1lQICACk8ZGVlles3IyMDrVq1QqtWrVBSUoKSkhLUrVsXTk5O+P777wEAp06dQteuXaWAAADe3t7Yt28fGjRoUK7P119/HZaWlpg4cSLmzZuHI0eOwN7eHiEhIdX6rZDffPMNLC0tsWnTJqxevRrz589HVFQUQkNDsWnTJgghcPDgQahUKrRo0QKrV6/Ghg0bAABxcXGYOHFitdVC9E/AMUVU/Z75TIKhoSHc3NzKLe/Rowc2b96My5cvQ6lUarXl5uZCCIGGDRuW287Kykrr2yQbNWpUrv1RH3+Xk5OD3377DTY2NuXaWrduDQC4f/8+OnTo8PQd+z+mpqbYsGEDli9fju3bt2Pjxo0wNzfHhAkTMH78+Er38zR9+/aFj48PgLLPaDA0NMS5c+fg4uICAPDw8EBqaipGjhwJtVqNwsJC1K9fH1lZWWjdunW55+nvVCpVtdVK9CJ4WcZUYWGh3tQC6Fc9+lQLoH/1VMUzh4Rbt27h8OHD6NOnj/QGDkA6gyD317mZmRkUCgXu3r1bru3OnTtat04+upbg8XYAWo/1eL8dOnRAZGRkuTZjY2MAZW/62dnZWm1FRUVIT0+XphP+rn379li6dCmKioqQlZWFhIQEfPbZZ3BxcUGXLl1kt3lWj24dzcvLQ0hICEJDQ7Fw4UIoFAqpPTc3F23atEHTpk0RHR2N4OBgLFu2DNOnT0dERAQsLCwQGhoqe4ZDX75UhKgq5M4cPs3LMqb06UuDAP2qR59qAfSrnqqMKaAK0w1FRUUIDw/HN998o7V83759aN26NRo3blxuGxMTE3Ts2BF79+7VWn706FHk5ubC0dFRWnb8+HGtKYv9+/fDwMAAzs7O5fp1dHTEtWvX0KJFC9ja2sLW1hadO3fG2rVrcfjwYWmd48ePa91FkZ6ejgkTJuDu3bswNDTU6jMlJQVubm7Izs6GsbEx3NzcMGfOHADAH3/8UclnqXJu3LiBkSNHYsCAAXj77be1Xpjy8/Olz4OYPHkyFi1ahJ9++gne3t5ISkqCv78/LCwskJ6eXq01Eb3IOKaIqtczh4R//etf6NevH5YtW4aEhAQcO3YMs2fPxv79+zFjxownbvfuu+/izJkzCA0NRUpKChITE/H+++/DwcFB61sV8/LyMHnyZKSmpmLt2rVYsmQJAgIC0Lx583J9+vv7w9LSEmPGjMHu3buRlpaG0NBQ7N69W5piGDVqFEpLSzFhwgQcOnQIycnJiIiIwJtvvok2bdrAzMwMQNndFxcvXoSdnR2EEJgyZQq+++47HDt2DHPnzoW5uTlcXV2f9el6ojt37mDMmDGYPn06/P39AZR9BkRGRgaAsrDyeDBSq9XYv38/+vfvj4KCAhgaGkKhUGjdHkr0MuOYIqp+ClGFLy0oLCxEXFwckpOTcfv2bbRr1w6TJ09Gnz59Ktzu4MGDWL58OX755RdYWlrCx8cHYWFh0kWFgYGBaNmyJYyNjbFjxw7UrVsXgwcPxtSpU6W/+L28vNCrVy/p1sWbN28iOjoax44dQ1FREdq3b4/g4GB4enpKj6tSqRAdHY2TJ0/CzMwMffv2RVhYGExMTKDRaDBr1izs2bMH7u7uWLlyJX788UcsWrQIP/74I4qLi2FnZ4cZM2agc+fOz/pUPVFkZCT27NmDtm3bSss++ugjREZGori4GG3btkVkZKS03/Hx8bC3t4eLiwtUKhXCw8NhamqK5cuXo379+lp9V/V7w4n0RVWO4ZdlTOnTKWxAv+rRp1oA/aqnqsdwlUKCLrm7u2PAgAEVnrV42enTCxpRVejbMaxP9ejTGw+gX/XoUy2AftVT1WO4Sh/LrAsXLlxAZmYm7ty5g5YtW+q6HCIion+8FyYk7N69G2vXrkXPnj3Rr18/XZdDRET0j/fChISpU6di6tSpui6DiIjopVF9HyFIRERE/ygMCURERCSLIYGIiIhkMSQQERGRLIYEIiIiksWQQERERLIYEoiIiEgWQwIRERHJYkggIiIiWQwJREREJIshgYiIiGQxJBAREZEshgQiIiKSxZBAREREshgSiIiISBZDAhEREcliSCAiIiJZDAlEREQkiyGBiIiIZDEkEBERkSyGBCIiIpLFkEBERESyGBKIiIhIFkMCERERyWJIICIiIlkMCURERCSLIYGIiIhkMSQQERGRLIYEIiIiksWQQERERLIYEoiIiEgWQwIRERHJYkggIiIiWUa6LoBq3/i4bF2XoLeSl7yNm7+mVVt/zV7vhn5hO6utv3+KVcFWui6BiCqBIYHoMXxDJyL6C6cbiIiISBZDAhEREcliSCAiIiJZDAl6RKPRIDw8HAEBAQgMDMTly5exZcsWDBkyBHPnzpXWmzZtGvLy8nRXKNELgmOK6PnwwkU9cuDAARQVFSExMRGnT5/Gp59+itzcXGzevBmTJ0/G/fv3cerUKTg5OcHU1FTX5RLpPY4poufDMwl6JCsrCz169AAA2Nvb48cff0TdunVRXFyM0tJSGBgY4Ouvv8aQIUN0XCnRi4Fjiuj5KIQQQtdFUJmPPvoIb775Jnr27AkA6NWrFz777DOsX78e3bt3R1FREVq0aIHz58/jxo0bGDVqFNq2bVuun6ysrNounajaOTk5PXcfHFNEf6nKmOJ0gx4xNTVFfn6+9LtGo4GzszOcnZ2Rm5uLiIgIuLm5ISUlBVOnTsUnn3yCRYsWleunOl5cif4JOKaIng+nG/SIo6MjUlJSAACnT5+GUqmU2uLj4zFhwgQUFhbCwMAACoUCDx8+1FWpRC8Ejimi58MzCXqkT58+SE1NxdChQyGEwIIFCwAA165dw4MHD9ChQwdoNBrcuHEDEyZMQGhoqI4rJtJvHFNEz4fXJBARoWwqYu7cufj5559hbGyMyMhItGrVSmpPSkrC5s2bYWRkhEmTJsHT01On9axduxa7du0CAPTs2RNTpkzRWS2P1pkwYQK8vb0xbNiwGqulMvUcOXIEy5cvhxACNjY2iIiIgEKh0Ekta9asQXJyMhQKBSZOnIg+ffrUSB1/d+bMGen6m8cdOnQIy5cvh5GREfz8/J5+0a4gIiKxb98+MXPmTCGEEKdOnRITJ06U2v7880/Rr18/oVarxYMHD6SfdVXPlStXxMCBA0VJSYnQaDQiICBAqFQqndTyyKJFi8TgwYPFpk2baqyOytSTm5srfH19xd27d4UQQsTHx0s/13Yt9+/fFz179hRqtVrk5OSIXr161Vgdj4uPjxf9+vUTgwcP1lpeVFQkevfuLXJycoRarRaDBg0St2/frrAvXpNARAT52yUfOXv2LBwcHGBsbAwzMzO0bNkS58+f11k9zZo1w+rVq2FoaAiFQoGSkhK88sorOqkFAPbu3QuFQiGtU9MqqufUqVNQKpVYuHAhhg8fjkaNGsHKqua+dbSiWurVq4dXX30VBQUFKCgoqLGzGX/XsmVLxMTElFt+8eJFtGzZEhYWFjA2NoaTkxOOHz9eYV+8JoGICEBeXp7WByoZGhqipKQERkZGyMvLg5mZmdRmYmJS45/QWFE9derUgZWVFYQQiI6ORqdOndCmTRud1PLLL78gOTkZn3/+OZYvX15jNVS2nnv37iEjIwM7duxA/fr18e9//xv29vY19vxUVAsANG/eHL6+vigtLUVQUFCN1PB3Pj4+uHbtmmytz3ocMyQQEUH+dslHL/R/b8vPz9d6sa3tegBArVbjww8/hImJCSIiInRWy44dO3Dr1i2MGjUK169fR506ddCiRQt4eHjopB5LS0vY2tqicePGAABnZ2eoVKoaCwkV1ZKSkoI///wTBw8eBACMHTsWjo6OsLOzq5FanrXWyhzHnG4gIkLFt0va2dkhKysLarUaubm5uHjxolZ7bdcjhEBwcDCsra0xb948GBoa6qyWGTNmYMuWLVi/fj0GDhyI0aNH12hAeFo9NjY2+OWXX5CdnY2SkhKcOXMGr7/+uk5qsbCwQN26dWFsbIxXXnkFZmZmePDgQY3V8jTt2rXD5cuXkZOTg6KiIpw4cQIODg4VbsMzCUREkL9d8quvvkLLli3h7e2NwMBADB8+HEIIhIWF1eg1AE+rR6PRIDMzE0VFRTh69CgA4L333nvqC35N1OLt7V0jj/k89UybNg3jxo0DAPTt27dGA93TaklLS8OQIUNgYGAAR0dHuLu711gtT7Jz5048fPgQAQEBmDVrFsaOHQshBPz8/NC0adMKt+UtkERERCSL0w1EREQkiyGBiIiIZDEkEBERkSyGBCIiIpLFkEBERESyGBKIiIhIFkMCERERyWJIICIiIlkMCURERCSLIYGIiIhkMSQQERGRLIYEIiIiksWQQERERLIYEoiIiEgWQwIRERHJYkggIiIiWQwJREREJIshgYiIiGQxJBAREZEshgQiIiKSxZBAREREshgSiIiISBZDAhEREcliSCAiIiJZDAlEREQkiyGBiIiIZDEkEBERkSyGBCIiIpLFkEBERESyGBKIiIhIFkMCERERyWJIICIiIlkMCURERCSLIYGIiIhkMSQQERGRLIaEWnLmzBkEBgYCAC5fvoxhw4Zh+PDhiIiIgEajAQDExsbC398fQ4cOxdmzZwEAKSkp8Pf3R0hIiLTevHnzcO3aNd3sCJGe4JgiqnkMCbVg1apVmD17NtRqNQAgKioKoaGh2LRpE4QQOHjwIM6dO4fMzExs2bIFixcvxscffwwA2LRpE9asWYMmTZrg/PnzOH/+PExNTfHaa6/pcpeIdIpjiqh2GOm6gJdBy5YtERMTgxkzZgAAzp07BxcXFwCAh4cHUlNT0aZNG3Tv3h0KhQKvvvoqSktLkZ2dDRMTExQWFkKtVqNevXqIjY3F3LlzK3y8rKysmt4lohrn5OT0xDaOKaJnV9GYehKGhFrg4+OjdSpTCAGFQgEAMDExQW5uLvLy8mBpaSmt82h5cHAwoqKiYG1tjStXrsDR0RHJyclQqVQYOHAgHBwcZB+zKgcDkb542pvyyzymVCoVOnbsqOsyJPpUjz7VAuhXPVUNupxu0AEDg7+e9vz8fJibm8PU1BT5+flay83MzNCuXTssWbIE48ePx9atW9GvXz8cO3YM4eHhiIuL00X5RHqHY4qoZjAk6ECnTp2QkZEBoOwiKmdnZzg6OuLYsWPQaDT4448/oNFoYGVlJW2TmJiIgQMHAgA0Gg0UCgUKCgp0Uj+RvuGYIqoZnG7QgZkzZ2LOnDlYvHgx2rZtCx8fHxgaGsLZ2RkBAQHQaDQIDw+X1s/Ly0NmZiaWLl0KAGjcuLF0JTcRcUwR1RSFEELougiqXllZWXozf0pUFfp2DOtTPfo0zw3oVz36VAugX/VU9RjmdAMRERHJYkggIiIiWQwJREREJIshgYiIiGQxJBAREZEshgQiIiKSxZBAREREshgSiIiISBZDAhEREcliSCAiIiJZDAlEREQkiyGBiIiIZDEkEBERkSyGBCIiIpL1jwoJNfGt1/wmbSIielk9NSQEBgbCzs4Ov//+e7k2lUoFa2trZGRk1ERtzyQ2NhabNm2qcJ1t27bB2toa2dnZlerzxIkTCAkJqY7yiIiIXjiVOpOgVqsxZ86cmq7lucTExKCwsLDCdXr16oXExESYm5tXqs+tW7fi0qVL1VEeERHRC6dSIcHMzAyZmZnYsmVLTddTo6ysrGBvbw8jIyNdl0JERKT3KhUSnJyc4OnpiejoaNy+ffuJ6127dg1Tp06Fm5sbHBwcMGnSJK1pipiYGAwaNAjJycnw8fGBra0t/Pz8cPLkyafWsH37dvj6+sLW1hYeHh5YsGAB1Go1AMDa2hoAEB0dDS8vLwBl0yRz5szB2LFjYWdnh/nz58tONyQmJsLX1xd2dnbo27cvkpKSAACzZs3C9u3bceHCBb2ZUiEiov/f3r1HVVXnfRx/H26agDhe85IT1iMlSSA8OqSigi1t0ViJaCGaipMIjGIu8jIJpZajlTcUiQdRV+p4yWrGatSJZ0YNWajkZeUie6aMtNSGFEUuB/CcqyNP7QAAIABJREFU5w+Xp4jthVvnoJ/XWqwl+/Lb333W+bk/7N++yK/pti9cTE5Oprq6mgULFhjOP3fuHJGRkRQWFvLKK6+waNEizpw5Q1RUFOfPn7ct980337By5UoSEhJITU3FbDYzffp0qqurb7jtQ4cOMXfuXJ588knWrl1LbGwsW7ZsYdWqVcC1Az1cCwbXp8G1axC8vb1JS0vjqaeeqtXuunXrSElJYeDAgaSnpzN8+HDmzZvHRx99RFxcHIMGDeK+++5j69at+Pr63u5HJSIicke47fPuXbp0YcaMGbz22mtkZ2cTFhZWY/769eupqKggKyuLtm3bAtC3b1+GDh3KunXrmD17NgClpaWsX78ePz8/AK5evUpcXBxffPEFjzzyiOG2jxw5wj333ENMTAxubm707dsXV1dXXF1dAfD39wegc+fO9OrVy7aeu7s7c+fOxcnpWhb697//bZtnsVhIT09n5MiRttoee+wxTp8+zeHDhwkPD6dt27Z8//33tvZFRETuJnUanI+Ojmbnzp3Mnz+ffv361Zh36NAh+vXrZwsIcO0agODgYA4ePPjTBl1caoSBe++9F4Dy8nKAWmcUnJ2d6dOnD2VlZYwYMYInnniCIUOGMGrUKEwm003r7d69uy0g/NKpU6coLi62DU9c99Zbb920TRERkbtFnZ6T4OTkxMKFCykqKqp1ML18+TLt27evtU67du0oLS21/e7m5lbjwH393xaLhTNnzuDr61vj5+DBgwQFBZGWlkaHDh3IyMggMjKSoUOHsn///pvW265duxvOKy4uBqgRakREROQndb7M38fHh5iYGDIyMnjwwQdt0728vCgqKqq1fFFREW3atLmttjt27Mi7775bY5q3tzcAoaGhhIaGUlJSwr59+1izZg0zZszgwIEDuLm51XU38PT0BKj1zIRTp05x8eJF+vTpU+c2RURE7iT1euJifHw83bt3Z+nSpbZpgYGB5OXl1TjoXrhwgdzc3Ns+4Lq5udG7d+8aPx4eHqSmpjJ69Gjg2sE9PDycmJgYSkpKuHLlyrUducGwwo306NGDNm3a8K9//avG9BUrVrBkyZJ6tSkiInInqddRsEWLFsyfP992gAaYMGECrq6uTJo0id27d7N7924mTZqEm5sbzz//fIOK7NevH8ePH+fll18mNzeXXbt2kZ6eTmBgoG24oHXr1uTn53Ps2LHbatPFxYUpU6awY8cO3nrrLQ4cOMCKFSvYtWsXkydPtrV57tw5cnJyuHTpEgBHjx7l22+/bdD+iIiINAf1/lP5d7/7HREREbbfO3fuzKZNm+jYsSOzZ8/mT3/6E127dmXLli22ixPrq2/fvixdupTPP/+cqVOnkpKSgp+fH6mpqbZlEhISyMvL4w9/+MNNb6f8uUmTJvHyyy+ze/dupkyZwieffMLSpUsZOnQoAGPGjKFdu3ZMmTKFnJwc27S0tLQG7Y+IiEhzYLLqDUZ1kpuby9///nfmz59v71JuKD8/n8DAQHuXIVJvjvYddqR6CgoKePjhh+1dho0j1eNItYBj1VPf77AG3evAYrGQmZlJSEiIvUsRERFpcnqJQR04OTkxc+bMGg9sEhERuVPpTEIdKSCIiMjdQiHhBnSphoiI3O3qFBKys7MJCAioNd1qtbJmzRoGDx7Mo48+ysSJE/nqq68aXFxoaOgtLxD08fFh7dq1Dd7WdefOnSMmJoaLFy82WpsiIiLN0W2HhM8++4ykpCTDeatXr2bNmjVMmjSJpUuXUlJSwoQJEygpKWm0Qm9k69at/P73v2+09g4cOMCnn37aaO2JiIg0V7cMCZWVlfzP//wP48ePx8Wl9nWOV65cYe3atSQkJDB+/HjCwsJYu3YtpaWltR6x3BT8/f3p2LFjk29HRETkbnPLkLBv3z4yMjJ46aWXiI6OrjX/2LFjlJWV1Xh1tJeXF3379r3lC5j+8Y9/EBERgb+/P4MGDWL58uW1HoRUUVHBnDlzCAgIYMCAASxbtqzGMr8cbigsLCQuLo6AgACCgoJISkqq9X6GvLw8xo4dS0BAACEhISxatAiz2cx7773HnDlzAAgODrY9rGnv3r2MHDmSRx99lODgYObMmWN7QZSIiMid6pYhoXfv3mRnZzN+/HjDVzN/8803ANx33301pnfr1s02z8jWrVtJSEjAz8+PVatWER0dTVZWFrNnz66x3AcffEBRURHLly8nOjqazMzMG77OuaioiKioKL7//nuWLFnCq6++ytGjR4mJiaGyshKA48ePM2nSJDw9PVm2bBl//OMfeffdd3nttdcYPHgwU6dOBSAzM5PIyEgKCwtJSEigT58+ZGRkMGvWLP75z3869MOUREREGsMtn5PQqVOnm86/cuUKbm5utd7E6O7uXuPdDj939epVli9fTnh4OCkpKQAMGDAAT09PUlJSmDx5Mg899BBw7XHPa9aswcXFhUGDBlFSUsLGjRuJj4/Hw8OjRrsbNmzAbDaTlZVle6eDn58fw4YN4+OPP+bpp5/m7bffplu3bqxevRpnZ2cAzGYz77//Pl5eXnTv3h0AX19f2rZty0cffURlZSUvvPCCbVjD3d2d77777lYfnYiISLPW4FsgrVar4RkG4IbTv/76ay5cuMDw4cNrTA8PDwfg8OHDtmmhoaE1roUYMmQIFRUVnDhxola7eXl5+Pv707p1a6qrq6murqZz58488MAD5ObmAnDkyBFCQkJsAQEgOjqaHTt21Jh2nZ+fH25ubkRGRrJ48WLy8vIIDQ1lwoQJN/hERERE7gwNfuKip6cnlZWVVFVV4erqapteWlqKp6en4TrX36jYrl27Wm25ubnVOAPRvn37GstcP0NgdOdEcXExx44dw9fXt9a8Dh062Lb9y+3ezH333cf69evJyMhg48aNZGVl0b59e5KSknj66advux0REZHmpsEh4be//S1Wq5UzZ87g7e1tm/7L33+uTZs2APz44481pl++fJnKykrbfPgpUFx3fZ3rYeHnPDw8CAkJYdq0abXmubu725b55YWMxcXFnDhxgj59+hjWGxgYyNtvv015eTm5ublkZmYyd+5cgoODbzkcIyIi0lw1eLghICCAFi1a8Mknn9imXbp0iYMHDxIcHGy4jre3N7/5zW/YtWtXjekff/wxQI2DdU5OTo2nH+7evRsPDw/DxyMHBgby9ddf4+PjQ+/evenduzc9e/Zk1apV5Ofn2+rdt28fFoulxnanTJnC1atXcXKq+ZFs376d0NBQqqqquOeeewgNDSUxMZGrV69y/vz52/2YaqmqqmLmzJk8++yzREVF8dVXX1FYWMhzzz1HVFQUKSkpWCwWLBYLcXFxREZG2l5Xffr0aRYuXFjvbYvcidSnRBpfg88kuLu7Ex0dzYoVK3BycuL+++8nPT0dDw8PIiMjDddxdnYmISGBBQsW4OXlRVhYGCdPniQ1NZXhw4fTs2dP27KnTp1i1qxZPPPMMxw6dIhNmzbx4osv0rJly1rtTpw4kb/+9a9MnjyZ8ePH4+rqSlZWFkePHiUxMRGA2NhYxo4dy7Rp0xg9ejTnzp2z3Tnh4eFB69atgWu3Z/bv35+goCCKioqYPn06UVFRVFVVsWbNGrp169agV4Du3buX6upqtmzZQk5ODsuXL6eqqorExET69etHcnIy2dnZdOnSha5du7Jo0SJmz55N//79SUtLY+bMmfXetsidSH1KpPE1ylsgX3zxRZycnMjKyqKsrIyAgAD+/Oc/3/CaBLh2sWDLli3Jyspi+/btdOzYkYkTJxIXF1djuYkTJ/Ltt98SGxtLmzZtmDVr1g0vGuzSpQubN2/mjTfeICkpCZPJhK+vL+vWrbMd0P39/Vm7di3Lli0jPj6e9u3bM27cOGJjY4Frz0cYMGAACxYsYPTo0SQnJ5Oens7KlSttwxj9+vXjzTffrHENRl15e3tz9epVLBYLV65cwcXFhaNHj9K3b18AQkJCyMnJYfz48ZjNZioqKmjVqhX5+fncf//9ta7VELnbqU+JND6TtRm/ychsNuPn50dycjJjx461dzl1cvbsWeLi4igrK+PixYukp6czbdo02yOhc3Nz2bFjB2+++SarV6/m66+/Ji4ujhUrVpCUlERmZiZeXl4kJibWGiLJz8+nVatW9tgtkUZRVlZGYGBgnda5W/pURUWF4ZlUe3GkehypFnCseurTp6CRziTYw/Hjx9m7dy+A7dkGzcn69esZMGAAM2fO5OzZszz//PNUVVXZ5peWltqGPuLj4wHYuXMnYWFhbNu2jVGjRnHw4EFyc3Pp379/rfYbMhQiYm/XryGqi7ulTxUUFDhMLeBY9ThSLeBY9dSnT0EzflX0pk2byMrK4qmnnrrhBZKOrHXr1rbhGC8vL6qrq+nVqxd5eXnAtcdhBwUF2ZY3m83s2bOHESNGUF5ejrOzMyaTibKyMrvUL+Jo1KdEGl+zPZOwePFiFi9ebO8y6m3ChAnMnTvXdjHkjBkzeOSRR5g3bx5Lly6lR48eDBs2zLb8hg0bGDduHCaTiYiICJKTk/Hw8GD16tV23AsRx6E+JdL4mvU1CWIsPz+/XmNPIo7C0b7DjlSPI53CBseqx5FqAceqp77f4WY73CAiIiJNSyFBREREDCkkiIiIiCGFBBERETGkkCAiIiKGFBJERETEkEKCiIiIGFJIEBEREUMKCSIiImJIIUFEREQMKSSIiIiIIYUEERERMaSQICIiIoYUEkRERMSQQoKIiIgYUkgQERERQwoJIiIiYkghQURERAwpJIiIiIghhQQRERExpJAgIiIihhQSRERExJBCgoiIiBhSSBARERFDCgkiIiJiSCFBREREDCkkiIiIiCGFBBERETGkkCAiIiKGFBJERETEkEKCiIiIGFJIEBEREUMKCSIiImJIIUFEREQMKSSIiIiIIYUEERERMaSQICIiIoYUEkRERMSQQoKIiIgYUkgQERERQwoJIiIiYkghwY7efvttxowZw8iRI9m+fTuFhYU899xzREVFkZKSgsViwWKxEBcXR2RkJDk5OQCcPn2ahQsX2rl6EcejPiXSuBQS7CQvL48jR47wl7/8hXfeeYdz586xaNEiEhMT2bx5M1arlezsbAoKCujatSuZmZls3LgRgLS0NGJjY+28ByKORX1KpPEpJNjJp59+Ss+ePYmPjyc2NpbBgwdz4sQJ+vbtC0BISAgHDhygVatWmM1mKioqaNWqFfn5+dx///20b9/eznsg4ljUp0Qan4u9C7hbXbx4ke+//5709HTOnDnD1KlTsVqtmEwmANzd3SkpKcHb25tOnTqxZMkS4uLiWLFiBUlJSaSkpODl5UViYiJOTrWzXkFBwa+9SyJ2dbf0qYqKCoepBRyrHkeqBRyvnvpQSLCTNm3a0KNHD9zc3OjRowctWrTg3LlztvmlpaW0bt0agPj4eAB27txJWFgY27ZtY9SoURw8eJDc3Fz69+9fq/2HH37419kRkSaQn59f53Xulj5VUFDgMLWAY9XjSLWAY9VTnz4FGm6wm8DAQPbv34/VauX8+fOUl5cTHBxMXl4eAPv27SMoKMi2vNlsZs+ePYwYMYLy8nKcnZ0xmUyUlZXZaxdEHIr6lEjj05kEOxkyZAiHDh1i1KhRWK1WkpOT6datG/PmzWPp0qX06NGDYcOG2ZbfsGED48aNw2QyERERQXJyMh4eHqxevdqOeyHiONSnRBqfyWq1Wu1dhDSu/Px8AgMD7V2GSL052nfYkepxpFPY4Fj1OFIt4Fj11Pc7rOEGERERMaSQICIiIoYUEkRERMSQQoKIiIgYUkgQERERQwoJIiIiYkghQURERAwpJIiIiIghhQQRERExpJAgIiIihhQSRERExJBCgoiIiBhSSBARERFDCgkiIiJiSCFBREREDCkkiIiIiCGFBBERETGkkCAiIiKGFBJERETEkEKCiIiIGFJIEBEREUMKCSIiImJIIUFEREQMKSSIiIiIIYUEERERMaSQICIiIoYUEkRERMSQQoKIiIgYUkgQERERQwoJIiIiYkghQURERAwpJIiIiIghhQQRERExpJAgIiIihhQSRERExJBCgoiIiBhSSBARERFDCgkiIiJiSCFBREREDCkkiIiIiCGFBBERETGkkCAiIiKGFBLs6Mcff2TQoEF89dVXFBYW8txzzxEVFUVKSgoWiwWLxUJcXByRkZHk5OQAcPr0aRYuXGjnykUck/qUSONSSLCTqqoqkpOTadmyJQCLFi0iMTGRzZs3Y7Vayc7OpqCggK5du5KZmcnGjRsBSEtLIzY21p6lizgk9SmRxudi7wLuVosXL+bZZ58lIyMDgBMnTtC3b18AQkJCyMnJYfz48ZjNZioqKmjVqhX5+fncf//9tG/f/pbtFxQUNGn9Io7mbulTFRUVDlMLOFY9jlQLOF499aGQYAfvvfcebdu2ZeDAgbb/0KxWKyaTCQB3d3dKSkrw9vamU6dOLFmyhLi4OFasWEFSUhIpKSl4eXmRmJiIk5PxyaCHH374V9sfkcaWn59fp+Xvpj5VUFDgMLWAY9XjSLWAY9VT1z51nUKCHezYsQOTyURubi4FBQXMmjWLCxcu2OaXlpbSunVrAOLj4wHYuXMnYWFhbNu2jVGjRnHw4EFyc3Pp37+/XfZBxJGoT4k0DV2TYAebNm1i48aNvPPOOzz88MMsXryYkJAQ8vLyANi3bx9BQUG25c1mM3v27GHEiBGUl5fj7OyMyWSirKzMXrsg4lDUp0SahkKCg5g1axapqamMGTOGqqoqhg0bZpu3YcMGxo0bh8lkIiIigpSUFPbv36+/eERuQn1KpOFMVqvVau8ipHHl5+cTGBho7zJE6s3RvsOOVI8jjXODY9XjSLWAY9VT3++wziSIiIiIIYUEERERMaSQICIiIoYUEkRERMSQQoKIiIgYUkgQERERQwoJIiIiYkghQURERAwpJIiIiIghhQQRERExdEeFhKZ4wrSeWi0iInerW4aEcePG4efnxzfffFNrXkFBAT4+PrY3rdnTqlWr2Lx5802Xee+99/Dx8anxCtmbOXz4MNOmTWuM8kRERJqd2zqTYDabmTdvXlPX0iCpqalUVFTcdJnBgwezdetW23vlb+Xdd9/l1KlTjVGeiIhIs3NbIcHT05ODBw+yffv2pq6nSbVt2xZ/f39cXFzsXYqIiIjDu62QEBgYyJAhQ1iyZAn/+c9/brjcmTNnmD59OsHBwQQEBDB16tQawxSpqamMHDmSDz/8kGHDhtG7d28iIiL47LPPblnD+++/T3h4OL179yYkJITXX38ds9kMgI+PDwBLliwhNDQUuDZMMm/ePGJiYvDz82PBggWGww1bt24lPDwcPz8/hg8fzrZt2wCYPXs277//Pv/3f//nMEMqIiIiv6bbvnAxOTmZ6upqFixYYDj/3LlzREZGUlhYyCuvvMKiRYs4c+YMUVFRnD9/3rbcN998w8qVK0lISCA1NRWz2cz06dOprq6+4bYPHTrE3LlzefLJJ1m7di2xsbFs2bKFVatWAdcO9HAtGFyfBteuQfD29iYtLY2nnnqqVrvr1q0jJSWFgQMHkp6ezvDhw5k3bx4fffQRcXFxDBo0iPvuu4+tW7fi6+t7ux+ViIjIHeG2z7t36dKFGTNm8Nprr5GdnU1YWFiN+evXr6eiooKsrCzatm0LQN++fRk6dCjr1q1j9uzZAJSWlrJ+/Xr8/PwAuHr1KnFxcXzxxRc88sgjhts+cuQI99xzDzExMbi5udG3b19cXV1xdXUFwN/fH4DOnTvTq1cv23ru7u7MnTsXJ6drWejf//63bZ7FYiE9PZ2RI0faanvsscc4ffo0hw8fJjw8nLZt2/L999/b2hcREbmb1GlwPjo6mp07dzJ//nz69etXY96hQ4fo16+fLSDAtWsAgoODOXjw4E8bdHGpEQbuvfdeAMrLywFqnVFwdnamT58+lJWVMWLECJ544gmGDBnCqFGjMJlMN623e/futoDwS6dOnaK4uNg2PHHdW2+9ddM2RURE7hZ1ek6Ck5MTCxcupKioqNbB9PLly7Rv377WOu3ataO0tNT2u5ubW40D9/V/WywWzpw5g6+vb42fgwcPEhQURFpaGh06dCAjI4PIyEiGDh3K/v37b1pvu3btbjivuLgYoEaoERERkZ/U+TJ/Hx8fYmJiyMjI4MEHH7RN9/LyoqioqNbyRUVFtGnT5rba7tixI++++26Nad7e3gCEhoYSGhpKSUkJ+/btY82aNcyYMYMDBw7g5uZW193A09MToNYzE06dOsXFixfp06dPndsUERG5k9TriYvx8fF0796dpUuX2qYFBgaSl5dX46B74cIFcnNzb/uA6+bmRu/evWv8eHh4kJqayujRo4FrB/fw8HBiYmIoKSnhypUr13bkBsMKN9KjRw/atGnDv/71rxrTV6xYwZIlS+rVpoiIyJ2kXkfBFi1aMH/+fNsBGmDChAm4uroyadIkdu/eze7du5k0aRJubm48//zzDSqyX79+HD9+nJdffpnc3Fx27dpFeno6gYGBtuGC1q1bk5+fz7Fjx26rTRcXF6ZMmcKOHTt46623OHDgACtWrGDXrl1MnjzZ1ua5c+fIycnh0qVLABw9epRvv/22QfsjIiLSHNT7T+Xf/e53RERE2H7v3LkzmzZtomPHjsyePZs//elPdO3alS1bttguTqyvvn37snTpUj7//HOmTp1KSkoKfn5+pKam2pZJSEggLy+PP/zhDze9nfLnJk2axMsvv8zu3buZMmUKn3zyCUuXLmXo0KEAjBkzhnbt2jFlyhRycnJs09LS0hq0PyIiIs2Byao3GNVJbm4uf//735k/f769S7mh/Px8AgMD7V2GSL052nfYkeopKCjg4YcftncZNo5UjyPVAo5VT32/wxp0rwOLxUJmZiYhISH2LkVERKTJ6SUGdeDk5MTMmTNrPLBJRETkTqUzCXWkgCAiIncLhYQb0KUaIiJyt6tTSMjOziYgIKDW9M8//xwfH59aP4sXL25QcaGhobe8QNDHx4e1a9c2aDs/d+7cOWJiYrh48WKjtSkiItIc3fY1CZ999hlJSUmG87744gtatWrFunXrakzv2LFjw6q7DVu3bqVLly6N1t6BAwf49NNPG609ERGR5uqWIaGyspINGzawYsUKWrVqRVVVVa1lTp48yX/913/Z5W2JekOjiIhI07jlcMO+ffvIyMjgpZdeIjo62nCZkydP4uPjU+eN/+Mf/yAiIgJ/f38GDRrE8uXLaz0IqaKigjlz5hAQEMCAAQNYtmxZjWV+OdxQWFhIXFwcAQEBBAUFkZSUVOv9DHl5eYwdO5aAgABCQkJYtGgRZrOZ9957jzlz5gAQHBxse1jT3r17GTlyJI8++ijBwcHMmTPH9oIoERGRO9UtQ0Lv3r3Jzs5m/PjxN3w185dffsnZs2d56qmneOSRR3j88cd5//33b9ru1q1bSUhIwM/Pj1WrVhEdHU1WVhazZ8+usdwHH3xAUVERy5cvJzo6mszMzBu+zrmoqIioqCi+//57lixZwquvvsrRo0eJiYmhsrISgOPHjzNp0iQ8PT1ZtmwZf/zjH3n33Xd57bXXGDx4MFOnTgUgMzOTyMhICgsLSUhIoE+fPmRkZDBr1iz++c9/OvTDlERERBrDLYcbOnXqdNP558+f5+LFixQWFvLiiy/i5eXFhx9+yOzZszGZTDz99NO11rl69SrLly8nPDyclJQUAAYMGICnpycpKSlMnjyZhx56CLj2uOc1a9bg4uLCoEGDKCkpYePGjcTHx+Ph4VGj3Q0bNmA2m8nKyrK908HPz49hw4bx8ccf8/TTT/P222/TrVs3Vq9ejbOzMwBms5n3338fLy8vunfvDoCvry9t27blo48+orKykhdeeMF2jYW7uzvffffdrT46ERGRZq3Bt0B6eXmxdu1aNm3axBNPPMFjjz3G66+/zsCBA1m1apXhOl9//TUXLlxg+PDhNaaHh4cDcPjwYdu00NBQXFx+yjJDhgyhoqKCEydO1Go3Ly8Pf39/WrduTXV1NdXV1XTu3JkHHniA3NxcAI4cOUJISIgtIABER0ezY8eOGtOu8/Pzw83NjcjISBYvXkxeXh6hoaFMmDDh9j8kERGRZqjBT1xs2bIlAwYMqDV94MCB7N+/n9LSUtzd3WvMu/5GxXbt2tWY7unpiZubW423S7Zv377GMtfPEJSUlNTaZnFxMceOHcPX17fWvA4dOti2/cvt3sx9993H+vXrycjIYOPGjWRlZdG+fXuSkpIMz5KIiIjcKRocEk6dOkVubi6jRo3Czc3NNt1sNtOyZUtatWpVa502bdoA8OOPP9aYfvnyZSorK23z4adAcd31da6HhZ/z8PAgJCSEadOm1Zp3Pah4eHjUupCxuLiYEydO0KdPH8N9DAwM5O2336a8vJzc3FwyMzOZO3cuwcHBtxyOERERaa4aPNxw/vx5Xn31Vfbu3WubZrVa2bNnD0FBQYYXO3p7e/Ob3/yGXbt21Zj+8ccfA9Q4WOfk5NR4+uHu3bvx8PAwfDxyYGAgX3/9NT4+PvTu3ZvevXvTs2dPVq1aRX5+PgABAQHs27cPi8VSY7tTpkzh6tWrODnV/Ei2b99OaGgoVVVV3HPPPYSGhpKYmMjVq1c5f/58XT4qERGRZqXBZxL++7//m8DAQFJSUrh06RIdOnRg27ZtnDx5kr/85S+G6zg7O5OQkMCCBQvw8vIiLCyMkydPkpqayvDhw+nZs6dt2VOnTjFr1iyeeeYZDh06xKZNm3jxxRdp2bJlrXYnTpzIX//6VyZPnsz48eNxdXUlKyuLo0ePkpiYCEBsbCxjx45l2rRpjB49mnPnztnunPDw8KB169bAtdsz+/fvT1BQEEVFRUyfPp2oqCiqqqpYs2YN3bp1c5hXgIrpNbNnAAAKcklEQVSIiDSFBocEZ2dn0tLSWLp0KStXrqS4uJhevXqxbt06HnnkkRuuFx0dTcuWLcnKymL79u107NiRiRMnEhcXV2O5iRMn8u233xIbG0ubNm2YNWvWDS8a7NKlC5s3b+aNN94gKSkJk8mEr68v69atsx3Q/f39Wbt2LcuWLSM+Pp727dszbtw4YmNjgWvPRxgwYAALFixg9OjRJCcnk56ezsqVK23DGP369ePNN9/E1dW1oR+fiIiIwzJZm/GbjMxmM35+fiQnJzN27Fh7l+Mw8vPzCQwMtHcZIvXmaN9hR6qnoKDAoc5iOlI9jlQLOFY99f0ON9u3QB4/fpyMjAwA27MNmpOqqiqSkpKIiopi1KhRZGdnU1hYyHPPPUdUVBQpKSlYLBYsFgtxcXFERkaSk5MDwOnTp1m4cKGd90DEsahPiTS+Bg832MumTZvYs2cPTz31FMHBwfYup87+9re/0aZNG9544w2Ki4t5+umneeihh0hMTKRfv34kJyeTnZ1Nly5d6Nq1K4sWLWL27Nn079+ftLQ0Zs6cae9dEHEo6lMija/ZnklYvHgxR44cYcmSJTUettRcDB8+nOnTpwPX7gZxdnbmxIkT9O3bF4CQkBAOHDhAq1atMJvNVFRU0KpVK/Lz87n//vtrPT9C5G6nPiXS+Jrf0fUOcf25DVeuXGHatGkkJiayePFi2y2j7u7ulJSU4O3tTadOnViyZAlxcXGsWLGCpKQkUlJS8PLyIjExsdZtm3BtLEzkbnK39KmKigqHqQUcqx5HqgUcr576UEiwo7NnzxIfH09UVBS///3veeONN2zzSktLbbdjxsfHA7Bz507CwsLYtm0bo0aN4uDBg+Tm5tK/f/9abTvKxTIi9XH9uSZ1dTf0KUe6GA4cqx5HqgUcq5769qlmO9zQ3BUVFTFp0iSSkpIYNWoUAL169SIvLw+49oruoKAg2/Jms5k9e/YwYsQIysvLcXZ2xmQyUVZWZpf6RRyN+pRI41NIsJP09HQuX75MWloa48aNY9y4cSQmJpKamsqYMWOoqqpi2LBhtuU3bNjAuHHjMJlMREREkJKSwv79+w3/4hG5G6lPiTS+Zv2cBDHmSPd0i9SHo32HHakeRzqFDY5VjyPVAo5Vz133nAQRERFpWgoJIiIiYkghQURERAwpJIiIiIghhQQRERExpJAgIiIihhQSRERExJBCgoiIiBhSSBARERFDCgkiIiJiSCFBREREDCkkiIiIiCGFBBERETGkkCAiIiKGFBJERETEkEKCiIiIGFJIEBEREUMKCSIiImJIIUFEREQMKSSIiIiIIYUEERERMaSQICIiIoYUEkRERMSQQoKIiIgYUkgQERERQwoJIiIiYkghQURERAwpJIiIiIghhQQRERExpJAgIiIihhQSRERExJBCgoiIiBhSSBARERFDCgkiIiJiSCFBREREDCkkiIiIiCGFBBERETHkYu8C5Nf3h7QL9i7BYX247Pec+/eBRmvv3gcf48kZOxutvTvF/8S1tXcJInIbFBJEfkYHdBGRn2i4QURERAwpJIiIiIghhQQRERExpJDgQCwWC8nJyYwZM4Zx48ZRWFjI9u3bGT16NK+88optuZkzZ3LlyhX7FSrSTKhPiTSMLlx0IJ988gmVlZVs3bqVo0eP8uc//5mSkhK2bNlCfHw8ly5d4siRIwQGBuLh4WHvckUcnvqUSMMoJDiQ/Px8Bg4cCIC/vz+ff/45Pj4+VFVVcfXqVZycnNixYwfLli27rbZuJLZfo5UsUi/5+ad+pe38On3q1+ZItYBj1eNItYDj1VNXCgkO5MqVKzX+mnF2diY2NpaXXnqJxx9/nL/97W9ERESQmZnJ2bNnef755+nRo0etdgIDA3/NskUclvqUSMPomgQH4uHhQWlpqe13i8VCUFAQK1asYPjw4eTn59O9e3d++OEHpk+fzurVq+1YrYjjU58SaRiFBAfSp08f9u3bB8DRo0fp2bOnbV5GRgYvvPACFRUVODk5YTKZKCsrs1epIs2C+pRIw2i4wYE8/vjj5OTk8Oyzz2K1Wnn99dcBOHPmDJcvX+ahhx7CYrFw9uxZXnjhBRITE+1csYhjU58SaRiT1Wq12rsIERF7s1gsvPLKK5w8eRI3NzcWLlzIb3/7W9v8bdu2sWXLFlxcXJg6dSpDhgyxaz3r16/no48+AmDQoEEkJCTYrZbry7zwwguEhYXx3HPPNVktt1PP3r17Wb16NVarFV9fX1JSUjCZTHapJSsriw8//BCTyURsbCyPP/54k9TxS8eOHePNN9/knXfeqTH9f//3f1m9ejUuLi5EREQwevTomzdkFRER6+7du62zZs2yWq1W65EjR6yxsbG2eT/88IP1ySeftJrNZuvly5dt/7ZXPd9++631mWeesVZXV1stFot1zJgx1oKCArvUct1bb71ljYyMtG7evLnJ6ridekpKSqzh4eHWH3/80Wq1Wq0ZGRm2f//atVy6dMk6aNAgq9lsthYXF1sHDx7cZHX8XEZGhvXJJ5+0RkZG1pheWVlpHTp0qLW4uNhqNputI0eOtP7nP/+5aVu6JkFEBOPbJa87fvw4AQEBuLm54enpSffu3fniiy/sVs+9995LZmYmzs7OmEwmqquradGihV1qAdi1axcmk8m2TFO7WT1HjhyhZ8+eLF68mKioKNq3b0/btk331tGb1XLPPffQpUsXysvLKS8vb7KzGb/UvXt3UlNTa03/6quv6N69O15eXri5uREYGMihQ4du2pauSRARwfh2yerqalxcXLhy5Qqenp62ee7u7k3+hMab1ePq6krbtm2xWq0sWbKEXr164e3tbZdavvzySz788ENWrlz5q90dcrN6Ll68SF5eHh988AGtWrVi7Nix+Pv7N9nnc7NaADp37kx4eDhXr15lypQpTVLDLw0bNowzZ84Y1lrX77FCgogIxrdLXv+P/pfzSktLa/xn+2vXA2A2m5k7dy7u7u6kpKTYrZYPPviA8+fP8/zzz/Pdd9/h6upK165dCQkJsUs9bdq0oXfv3nTo0AGAoKAgCgoKmiwk3KyWffv28cMPP5CdnQ1ATEwMffr0wc/Pr0lqqWutt/M91nCDiAg3v13Sz8+P/Px8zGYzJSUlfPXVVzXm/9r1WK1W4uLi8PHxYf78+Tg7O9utlpdeeont27fzzjvv8MwzzzBhwoQmDQi3qsfX15cvv/ySCxcuUF1dzbFjx3jwwQftUouXlxctW7bEzc2NFi1a4OnpyeXLl5usllt54IEHKCwspLi4mMrKSg4fPkxAQMBN19GZBBERjG+XXLduHd27dycsLIxx48YRFRWF1WplxowZTXoNwK3qsVgsHDx4kMrKSvbv3w/Aiy++eMv/8JuilrCwsCbZZkPqmTlzJpMnTwZg+PDhTRroblXLgQMHGD16NE5OTvTp04f+/fs3WS03snPnTsrKyhgzZgyzZ88mJiYGq9VKREQEnTp1uum6ugVSREREDGm4QURERAwpJIiIiIghhQQRERExpJAgIiIihhQSRERExJBCgoiIiBhSSBARERFD/w/kruzq8CJXqQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 504x720 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_table_bars(\"object_prediction_accuracy\", cells, \"Object Prediction Accuracy\", \"Accuracy\", percents=True, y_min=0, y_max=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "plot_table(\"clusterization_f_score\", cells, caption=\"Message clusterization F1 scores. \", label=\"f1_scores\", percents=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "plot_table(\"predictions_by_average_msg_accuracy\", cells, caption=\"Object prediction accuracy using average message from each function cluster. \", label=\"average_message_accuracy\", percents=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "plot_table_bars(\"clusterization_f_score\", cells, \"Object Prediction Accuracy\", \"Accuracy\", percents=False, y_min=0, y_max=1.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## General prediction accuracy\n",
    "Test whether predicted object is closest to target object.\n",
    "\n",
    "Method: Take network's predicted object, find closest object in input context C (using MSE), check if closest object is the target object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_sets = {\"Strict context\": strict_context_simulations, \n",
    "                   \"Non-strict\": non_strict_context_simulations, \n",
    "                   # \"F(c) input\": nature_includes_fc_simulations,\n",
    "                  # \"Untrained networks\": untrained_simulations\n",
    "                   }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "# simulation_sets = {\"extremity_game_acl_4__o_15__s_0__n_5__m_128__l_mse\"}\n",
    "plot_bars_for_evaluation(\"object_prediction_accuracy\", simulation_sets, \"Prediction accuracy of objects\", \"Accuracy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predictions from M\n",
    "\n",
    "Method: a simple neural network (hidden layer size 32 * ReLU) is trained to predict the various game elements from the messages M.\n",
    "For one-hot objects (i.e. F) an accuracy measure can be used. For the other elements, which are continuous, MSE is used. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plot_bars_for_evaluation(\"functions_from_messages\", simulation_sets, \"M -> F\", \"Accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "plot_bars_for_evaluation(\"context_from_messages\", simulation_sets, \"M -> C\", \"MSE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "plot_bars_for_evaluation(\"decoder_context_from_messages\", simulation_sets, \"M -> C'\", \"MSE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "plot_bars_for_evaluation(\"object_by_context_from_messages\", simulation_sets, \"M -> f(c)\", \"MSE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "plot_bars_for_evaluation(\"object_by_decoder_context_from_messages\", simulation_sets, \"M -> f(c')\", \"MSE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "plot_bars_for_evaluation(\"min_max_from_messages\", simulation_sets, \"M -> min/max\", \"Accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "plot_bars_for_evaluation(\"dimension_from_messages\", simulation_sets, \"M -> Param(F)\", \"Accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "plot_bars_for_evaluation(\"sanity_from_messages\", simulation_sets, \"Sanity Check\", \"Accuracy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training losses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Shared vs. Non-shared"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_display_name_to_file_name = {\n",
    "    \"Shared context\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\",\n",
    "    \"Non-shared context\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\",\n",
    "} \n",
    "\n",
    "plots.plot_simulation_training_loss(simulation_display_name_to_file_name, max_epochs=5000, epoch_interval=100, label_interval=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First 50 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "plots.plot_simulation_training_loss(simulation_display_name_to_file_name, max_epochs=50, epoch_interval=1, label_interval=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strict context, O=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_display_name_to_file_name = {\n",
    "    \"Strict context\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\",\n",
    "} \n",
    "\n",
    "plots.plot_simulation_training_loss(simulation_display_name_to_file_name, max_epochs=5000, epoch_interval=100, label_interval=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Non-strict, O=5, C=(15, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_display_name_to_file_name = {\n",
    "    \"Non-strict context\\nO=5, C=(15,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_15__mini_batch_size_128__num_batches_5000\",\n",
    "} \n",
    "\n",
    "plots.plot_simulation_training_loss(simulation_display_name_to_file_name, max_epochs=5000, epoch_interval=100, label_interval=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Non-strict, O=5, C=(10,5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_display_name_to_file_name = {\n",
    "    \"Non-strict context\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\",\n",
    "} \n",
    "plots.plot_simulation_training_loss(simulation_display_name_to_file_name, max_epochs=5000, epoch_interval=100, label_interval=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Non-strict, O=5, C=(5,5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_display_name_to_file_name = {\n",
    "    \"Non-strict context\\nO=5, C=(5,5)\": \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_1__nature_includes_function_1__num_objects_5__mini_batch_size_128__num_batches_5000\",\n",
    "} \n",
    "\n",
    "plots.plot_simulation_training_loss(simulation_display_name_to_file_name, max_epochs=5000, epoch_interval=100, label_interval=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clustering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cluster-function F1-score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "sets = {\"Strict context\": strict_context_simulations, \n",
    "        \"Non-strict\": non_strict_context_simulations, \n",
    "        \"F(c) input\": nature_includes_fc_simulations,\n",
    "        \"Untrained networks\": untrained_simulations}\n",
    "plot_bars_for_evaluation(\"clusterization_f_score\", sets, \"Clusterization F1-score\", \"F1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction by cluster average message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plot_bars_for_evaluation(\"predictions_by_average_msg_accuracy\", \n",
    "                         sets, \n",
    "                         \"Prediction of context objects from cluster average message\", \n",
    "                         \"Accuracy\", \n",
    "                         width=12)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compositionality Tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "compositionality_test_to_evals = {\n",
    "    \"Compositionality as addition\": {\n",
    "        \"loss_evals\": {\n",
    "            \"Perception loss (O)\": \"addition_compositionality_mean_prediction_loss\",\n",
    "            \"Production loss (M)\": \"addition_compositionality_mean_message_loss\",\n",
    "        },\n",
    "        \"accuracy_evals\": {\n",
    "            \"Perception accuracy\\n(O)\": \"addition_compositionality_mean_prediction_accuracy\",\n",
    "            \"Production accuracy\\n(M cluster)\": \"addition_compositionality_mean_message_cluster_accuracy\",\n",
    "        }\n",
    "    },\n",
    "\n",
    "    \"Compositionality as analogy\":  {       \n",
    "        \"loss_evals\": {\n",
    "            \"Perception loss (O)\": \"analogy_compositionality_net_prediction_mean_loss\",\n",
    "            \"Production loss (M)\": \"analogy_compositionality_net_message_mean_loss\"\n",
    "        },\n",
    "        \"accuracy_evals\": {\n",
    "            \"Perception accuracy (O)\": \"analogy_compositionality_net_prediction_mean_accuracy\",\n",
    "            \"Production accuracy\\n(M cluster)\": \"analogy_compositionality_net_message_cluster_mean_accuracy\"\n",
    "        }\n",
    "    },\n",
    "\n",
    "    \"Compositionality network\": { \n",
    "        \"loss_evals\": {\n",
    "            \"Perception loss (O)\": \"addition_compositionality_mean_prediction_loss\",\n",
    "            \"Production loss (M)\": \"addition_compositionality_mean_message_loss\",\n",
    "        },\n",
    "        \"accuracy_evals\": {\n",
    "            \"Perception accuracy\\n(O)\": \"addition_compositionality_mean_prediction_accuracy\",\n",
    "            \"Production accuracy\\n(M cluster)\": \"addition_compositionality_mean_message_cluster_accuracy\",\n",
    "        }\n",
    "    }\n",
    "}\n",
    "\n",
    "def plot_all_compositionality_stats(simulation_name):\n",
    "    for compo_test_name, evals in compositionality_test_to_evals.items():\n",
    "        plot_compositionality(compo_test_name, simulation_name, evals[\"loss_evals\"], evals[\"accuracy_evals\"])\n",
    "\n",
    "    \n",
    "def plot_compositionality(title, simulation_name, loss_evals, accuracy_evals, message_size=2):\n",
    "    fig, ax = plt.subplots(1,2, figsize=(10,4))\n",
    "    bar_width = 1\n",
    "\n",
    "    for i, (display_name, eval_name) in enumerate(loss_evals.items()):\n",
    "        mean, err = get_evaluation_mean_and_err(simulation_name, eval_name, message_size)\n",
    "        ax[0].bar(i, mean, yerr=err, capsize=5, color=f\"C{i}\", label=display_name)\n",
    "\n",
    "    for i, (display_name, eval_name) in enumerate(accuracy_evals.items()):\n",
    "        mean, err = get_evaluation_mean_and_err(simulation_name, eval_name, message_size)\n",
    "        ax[1].bar(i, mean, yerr=err, capsize=5, color=f\"C{i}\", label=display_name)\n",
    "\n",
    "    ax[0].set_xticks(range(len(accuracy_evals)))\n",
    "    ax[0].set_xticklabels(list(loss_evals.keys()))\n",
    "\n",
    "    ax[1].set_xticks(range(len(loss_evals)))\n",
    "    ax[1].set_xticklabels(list(accuracy_evals.keys()))\n",
    "\n",
    "    ax[0].set_ylabel(\"MSE loss\")\n",
    "    ax[1].set_ylabel(\"Accuracy\")\n",
    "    fig.suptitle(title)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strict, Shared context, O=5, Encoder input = <c, f>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation = \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\"\n",
    "plot_all_compositionality_stats(simulation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strict, Non-shared context, O=5, Encoder input = <c, f>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation = \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\"\n",
    "plot_all_compositionality_stats(simulation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strict, Shared context, O=5, Encoder input = <c, f(c)>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation = \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_1__nature_includes_function_0__num_objects_10__mini_batch_size_128__num_batches_5000\"\n",
    "plot_all_compositionality_stats(simulation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strict, Non-shared context, O=5, Encoder input = <c, f(c)>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation=\"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_0__nature_includes_function_0__num_objects_10__mini_batch_size_128__num_batches_5000\"\n",
    "plot_all_compositionality_stats(simulation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strict, Shared context, O=3, Encoder input = <c, f>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation=\"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_6__mini_batch_size_128__num_batches_5000\"\n",
    "plot_all_compositionality_stats(simulation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strict, Non-shared context, O=3, Encoder input = <c, f>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation=\"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_0__nature_includes_function_1__num_objects_6__mini_batch_size_128__num_batches_5000\"\n",
    "plot_all_compositionality_stats(simulation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strict, Shared context, O=3, Encoder input = <c, f(c)>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation=\"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_1__nature_includes_function_0__num_objects_6__mini_batch_size_128__num_batches_5000\"\n",
    "plot_all_compositionality_stats(simulation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strict, Non-shared context, O=3, Encoder input = <c, f(c)>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "simulation=\"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_0__nature_includes_function_0__num_objects_6__mini_batch_size_128__num_batches_5000\"\n",
    "plot_all_compositionality_stats(simulation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Untrained network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation=\"acl_ready/extremity_game_acl_untrained__object_size_3__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_6__mini_batch_size_128__num_batches_0\"\n",
    "plot_all_compositionality_stats(simulation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Message visualizations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "def visualize_messages(simulation_name, game_number=0):\n",
    "    simulation = simulations.load_simulation(simulation_name)\n",
    "    games=simulations.load_games(simulation_name)\n",
    "    message_size = simulation.message_sizes[0]\n",
    "    games[message_size][game_number]._plot_messages_information()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder input is <c, f>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strict, shared context, O=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "import utils\n",
    "plt.style.use('seaborn-whitegrid')\n",
    "\n",
    "simulation_name = \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\"\n",
    "visualize_messages(simulation_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strict, non-shared context, O=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_name = \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\"\n",
    "visualize_messages(simulation_name, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "### Non-strict, non-shared, O=5, C=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_name = \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_0__shared_context_0__nature_includes_function_1__num_objects_5__mini_batch_size_128__num_batches_5000\"\n",
    "visualize_messages(simulation_name, 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder input is <c, f(c)>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strict, shared context, O=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_name = \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_1__nature_includes_function_0__num_objects_10__mini_batch_size_128__num_batches_5000\"\n",
    "visualize_messages(simulation_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### O=5, Non-shared context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_name = \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_0__nature_includes_function_0__num_objects_10__mini_batch_size_128__num_batches_5000\"\n",
    "visualize_messages(simulation_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### O=3, Shared context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_name = \"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_1__nature_includes_function_0__num_objects_6__mini_batch_size_128__num_batches_5000\"\n",
    "visualize_messages(simulation_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### O=3, Non-shared context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_name = \"acl_ready/extremity_game_acl_o_3__object_size_3__strict_context_1__shared_context_0__nature_includes_function_0__num_objects_6__mini_batch_size_128__num_batches_5000\"\n",
    "visualize_messages(simulation_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder input is <c, f>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Non-shared context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_name = \"acl_ready/extremity_game_acl_o_5__object_size_5__strict_context_1__shared_context_0__nature_includes_function_1__num_objects_10__mini_batch_size_128__num_batches_5000\"\n",
    "visualize_messages(simulation_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Untrained network, O=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_name = \"acl_ready/extremity_game_acl_untrained__object_size_5__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_6__mini_batch_size_128__num_batches_0\"\n",
    "visualize_messages(simulation_name, 6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Untrained network, O=3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "simulation_name = \"acl_ready/extremity_game_acl_untrained__object_size_3__strict_context_1__shared_context_1__nature_includes_function_1__num_objects_6__mini_batch_size_128__num_batches_0\"\n",
    "visualize_messages(simulation_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}