t_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wandb\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "\n",
    "# plt.rcParams['text.usetex'] = True\n",
    "import numpy as np\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mpletctj6\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wandb.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "project_name = ['LDS_1', 'LDS_2', 'LDS_3']\n",
    "\n",
    "all_runs_data = []\n",
    "\n",
    "api = wandb.Api()\n",
    "for prj_ne in project_name:\n",
    "    runs = api.runs(prj_ne)\n",
    "    for run in runs:\n",
    "        metrics = run.history()\n",
    "        run_data = {\n",
    "            'run_id': run.id,\n",
    "            'name': run.name,\n",
    "            'config': run.config,\n",
    "            'seed' : run.name.split('-')[-1],\n",
    "            'metrics': metrics\n",
    "        }\n",
    "        all_runs_data.append(run_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lds\n",
    "Easy_lds_data = [ run_data for run_data in all_runs_data if 'Easy' in run_data['name']and 'lds' in run_data['name']]\n",
    "Ur_lds_data = [ run_data for run_data in all_runs_data if 'Ur' in run_data['name'] and 'lds' in run_data['name']]\n",
    "Hard_lds_data = [ run_data for run_data in all_runs_data if 'Hard' in run_data['name'] and 'lds' in run_data['name']]\n",
    "fetch_lds_data = [ run_data for run_data in all_runs_data if 'fetch' in run_data['name'] and 'lds' in run_data['name']]\n",
    "finger_lds_data = [ run_data for run_data in all_runs_data if 'finger' in run_data['name'] and 'lds' in run_data['name']]\n",
    "fetch_push_lds_data = [ run_data for run_data in all_runs_data if 'fetch_push' in run_data['name'] and 'lds' in run_data['name']]\n",
    "hopper_lds_data = [ run_data for run_data in all_runs_data if 'hopper' in run_data['name'] and 'lds' in run_data['name']]\n",
    "# diayn\n",
    "Easy_diayn_data = [ run_data for run_data in all_runs_data if 'Easy' in run_data['name']and 'diayn' in run_data['name']]\n",
    "Ur_diayn_data = [ run_data for run_data in all_runs_data if 'Ur' in run_data['name'] and 'diayn' in run_data['name']]\n",
    "Hard_diayn_data = [ run_data for run_data in all_runs_data if 'Hard' in run_data['name'] and 'diayn' in run_data['name']]\n",
    "fetch_diayn_data = [ run_data for run_data in all_runs_data if 'fetch' in run_data['name'] and 'diayn' in run_data['name']]\n",
    "finger_diayn_data = [ run_data for run_data in all_runs_data if 'finger' in run_data['name'] and 'diayn' in run_data['name']]\n",
    "fetch_push_diayn_data = [ run_data for run_data in all_runs_data if 'fetch_push' in run_data['name'] and 'diayn' in run_data['name']]\n",
    "hopper_diayn_data = [ run_data for run_data in all_runs_data if 'hopper' in run_data['name'] and 'diayn' in run_data['name']]\n",
    "# rnd \n",
    "Easy_rnd_data = [ run_data for run_data in all_runs_data if 'Easy' in run_data['name']and 'rnd' in run_data['name']]\n",
    "Ur_rnd_data = [ run_data for run_data in all_runs_data if 'Ur' in run_data['name'] and 'rnd' in run_data['name']]\n",
    "Hard_rnd_data = [ run_data for run_data in all_runs_data if 'Hard' in run_data['name'] and 'rnd' in run_data['name']]\n",
    "fetch_rnd_data = [ run_data for run_data in all_runs_data if 'fetch' in run_data['name'] and 'rnd' in run_data['name']]\n",
    "finger_rnd_data = [ run_data for run_data in all_runs_data if 'finger' in run_data['name'] and 'rnd' in run_data['name']]\n",
    "fetch_push_rnd_data = [ run_data for run_data in all_runs_data if 'fetch_push' in run_data['name'] and 'rnd' in run_data['name']]\n",
    "hopper_rnd_data = [ run_data for run_data in all_runs_data if 'hopper' in run_data['name'] and 'rnd' in run_data['name']]\n",
    "# ngu\n",
    "Easy_ngu_data = [ run_data for run_data in all_runs_data if 'Easy' in run_data['name']and 'ngu' in run_data['name']]\n",
    "Ur_ngu_data = [ run_data for run_data in all_runs_data if 'Ur' in run_data['name'] and 'ngu' in run_data['name']]\n",
    "Hard_ngu_data = [ run_data for run_data in all_runs_data if 'Hard' in run_data['name'] and 'ngu' in run_data['name']]\n",
    "fetch_ngu_data = [ run_data for run_data in all_runs_data if 'fetch' in run_data['name'] and 'ngu' in run_data['name']]\n",
    "finger_ngu_data = [ run_data for run_data in all_runs_data if 'finger' in run_data['name'] and 'ngu' in run_data['name']]\n",
    "fetch_push_ngu_data = [ run_data for run_data in all_runs_data if 'fetch_push' in run_data['name'] and 'ngu' in run_data['name']]\n",
    "hopper_ngu_data = [ run_data for run_data in all_runs_data if 'hopper' in run_data['name'] and 'ngu' in run_data['name']]\n",
    "# lsd\n",
    "Easy_lsd_data = [ run_data for run_data in all_runs_data if 'Easy' in run_data['name']and 'lsd' in run_data['name']]\n",
    "Ur_lsd_data = [ run_data for run_data in all_runs_data if 'Ur' in run_data['name'] and 'lsd' in run_data['name']]\n",
    "Hard_lsd_data = [ run_data for run_data in all_runs_data if 'Hard' in run_data['name'] and 'lsd' in run_data['name']]\n",
    "fetch_lsd_data = [ run_data for run_data in all_runs_data if 'fetch' in run_data['name'] and 'lsd' in run_data['name']]\n",
    "finger_lsd_data = [ run_data for run_data in all_runs_data if 'finger' in run_data['name'] and 'lsd' in run_data['name']]\n",
    "fetch_push_lsd_data = [ run_data for run_data in all_runs_data if 'fetch_push' in run_data['name'] and 'lsd' in run_data['name']]\n",
    "hopper_lsd_data = [ run_data for run_data in all_runs_data if 'hopper' in run_data['name'] and 'lsd' in run_data['name']]\n",
    "# csd \n",
    "Easy_csd_data = [ run_data for run_data in all_runs_data if 'Easy' in run_data['name']and 'csd' in run_data['name']]\n",
    "Ur_csd_data = [ run_data for run_data in all_runs_data if 'Ur' in run_data['name'] and 'csd' in run_data['name']]\n",
    "Hard_csd_data = [ run_data for run_data in all_runs_data if 'Hard' in run_data['name'] and 'csd' in run_data['name']]\n",
    "fetch_csd_data = [ run_data for run_data in all_runs_data if 'fetch' in run_data['name'] and 'csd' in run_data['name']]\n",
    "finger_csd_data = [ run_data for run_data in all_runs_data if 'finger' in run_data['name'] and 'csd' in run_data['name']]\n",
    "fetch_push_csd_data = [ run_data for run_data in all_runs_data if 'fetch_push' in run_data['name'] and 'csd' in run_data['name']]\n",
    "# metra\n",
    "Easy_metra_data = [ run_data for run_data in all_runs_data if 'Easy' in run_data['name']and 'metra' in run_data['name']]\n",
    "Ur_metra_data = [ run_data for run_data in all_runs_data if 'Ur' in run_data['name'] and 'metra' in run_data['name']]\n",
    "Hard_metra_data = [ run_data for run_data in all_runs_data if 'Hard' in run_data['name'] and 'metra' in run_data['name']]\n",
    "fetch_metra_data = [ run_data for run_data in all_runs_data if 'fetch' in run_data['name'] and 'metra' in run_data['name']]\n",
    "finger_metra_data = [ run_data for run_data in all_runs_data if 'finger' in run_data['name'] and 'metra' in run_data['name']]\n",
    "fetch_push_metra_data = [ run_data for run_data in all_runs_data if 'fetch_push' in run_data['name'] and 'metra' in run_data['name']]\n",
    "# smm\n",
    "Easy_smm_data = [ run_data for run_data in all_runs_data if 'Easy' in run_data['name']and 'smm' in run_data['name']]\n",
    "Ur_smm_data = [ run_data for run_data in all_runs_data if 'Ur' in run_data['name'] and 'smm' in run_data['name']]\n",
    "Hard_smm_data = [ run_data for run_data in all_runs_data if 'Hard' in run_data['name'] and 'smm' in run_data['name']]\n",
    "fetch_smm_data = [ run_data for run_data in all_runs_data if 'fetch' in run_data['name'] and 'smm' in run_data['name']]\n",
    "finger_smm_data = [ run_data for run_data in all_runs_data if 'finger' in run_data['name'] and 'smm' in run_data['name']]\n",
    "fetch_push_smm_data = [ run_data for run_data in all_runs_data if 'fetch_push' in run_data['name'] and 'smm' in run_data['name']]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fill_to_size(arr, size):\n",
    "    \"\"\" fill array to size with last element \"\"\"\n",
    "    arr = np.expand_dims(arr, axis=1)\n",
    "    while len(arr) < size:\n",
    "        arr = np.vstack([arr, arr[-1]])\n",
    "    return arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# env to do\n",
    "list_env = ['Easy', 'Ur', 'Hard', 'fetch', 'finger', 'fetch_push']\n",
    "list_alg = ['rnd', 'diayn', 'smm', 'ngu', 'lsd', 'csd', 'metra', 'lds']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lds\n",
    "Easy_lds_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Easy_lds_data]) if 'Easy' in list_env and 'lds' in list_alg else None\n",
    "Ur_lds_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Ur_lds_data]) if 'Ur' in list_env and 'lds' in list_alg else None\n",
    "Hard_lds_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Hard_lds_data]) if 'Hard' in list_env and 'lds' in list_alg else None\n",
    "fetch_lds_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_lds_data]) if 'fetch' in list_env and 'lds' in list_alg else None\n",
    "finger_lds_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in finger_lds_data]) if 'finger' in list_env and 'lds' in list_alg else None\n",
    "fetch_push_lds_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_push_lds_data]) if 'fetch_push' in list_env and 'lds' in list_alg else None\n",
    "hopper_lds_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in hopper_lds_data]) if 'hopper' in list_env and 'lds' in list_alg else None\n",
    "# diayn\n",
    "Easy_diayn_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Easy_diayn_data]) if 'Easy' in list_env and 'diayn' in list_alg else None\n",
    "Ur_diayn_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Ur_diayn_data]) if 'Ur' in list_env and 'diayn' in list_alg else None\n",
    "Hard_diayn_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Hard_diayn_data]) if 'Hard' in list_env and 'diayn' in list_alg else None\n",
    "fetch_diayn_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_diayn_data]) if 'fetch' in list_env and 'diayn' in list_alg else None\n",
    "finger_diayn_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in finger_diayn_data]) if 'finger' in list_env and 'diayn' in list_alg  else None\n",
    "fetch_push_diayn_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_push_diayn_data]) if 'fetch_push' in list_env and 'diayn' in list_alg  else None\n",
    "hopper_diayn_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in hopper_diayn_data]) if 'hopper' in list_env and 'diayn' in list_alg  else None\n",
    "# rnd\n",
    "Easy_rnd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Easy_rnd_data]) if 'Easy' in list_env and 'rnd' in list_alg  else None\n",
    "Ur_rnd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Ur_rnd_data]) if 'Ur' in list_env and 'rnd' in list_alg  else None\n",
    "Hard_rnd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Hard_rnd_data]) if 'Hard' in list_env and 'rnd' in list_alg  else None\n",
    "fetch_rnd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_rnd_data]) if 'fetch' in list_env and 'rnd' in list_alg  else None\n",
    "finger_rnd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in finger_rnd_data]) if 'finger' in list_env and 'rnd' in list_alg  else None\n",
    "fetch_push_rnd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_push_rnd_data]) if 'fetch_push' in list_env and 'rnd' in list_alg  else None\n",
    "hopper_rnd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in hopper_rnd_data]) if 'hopper' in list_env and 'rnd' in list_alg  else None\n",
    "# ngu \n",
    "Easy_ngu_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Easy_ngu_data]) if 'Easy' in list_env and 'ngu' in list_alg  else None\n",
    "Ur_ngu_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Ur_ngu_data]) if 'Ur' in list_env and 'ngu' in list_alg  else None\n",
    "Hard_ngu_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Hard_ngu_data]) if 'Hard' in list_env and 'ngu' in list_alg  else None\n",
    "fetch_ngu_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_ngu_data]) if 'fetch' in list_env and 'ngu' in list_alg  else None\n",
    "finger_ngu_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in finger_ngu_data]) if 'finger' in list_env and 'ngu' in list_alg  else None\n",
    "fetch_push_ngu_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_push_ngu_data]) if 'fetch_push' in list_env and 'ngu' in list_alg  else None\n",
    "hopper_ngu_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in hopper_ngu_data]) if 'hopper' in list_env and 'ngu' in list_alg  else None\n",
    "# lsd\n",
    "Easy_lsd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Easy_lsd_data]) if 'Easy' in list_env and 'lsd' in list_alg  else None\n",
    "Ur_lsd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Ur_lsd_data]) if 'Ur' in list_env and 'lsd' in list_alg  else None\n",
    "Hard_lsd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Hard_lsd_data]) if 'Hard' in list_env and 'lsd' in list_alg  else None\n",
    "fetch_lsd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_lsd_data]) if 'fetch' in list_env and 'lsd' in list_alg  else None\n",
    "finger_lsd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in finger_lsd_data]) if 'finger' in list_env and 'lsd' in list_alg  else None\n",
    "fetch_push_lsd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_push_lsd_data]) if 'fetch_push' in list_env and 'lsd' in list_alg  else None\n",
    "hopper_lsd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in hopper_lsd_data]) if 'hopper' in list_env and 'lsd' in list_alg  else None\n",
    "# csd\n",
    "Easy_csd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Easy_csd_data]) if 'Easy' in list_env and 'csd' in list_alg  else None\n",
    "Ur_csd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Ur_csd_data]) if 'Ur' in list_env and 'csd' in list_alg  else None\n",
    "Hard_csd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Hard_csd_data]) if 'Hard' in list_env and 'csd' in list_alg  else None\n",
    "fetch_csd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_csd_data]) if 'fetch' in list_env and 'csd' in list_alg  else None\n",
    "finger_csd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in finger_csd_data]) if 'finger' in list_env and 'csd' in list_alg  else None\n",
    "fetch_push_csd_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_push_csd_data]) if 'fetch_push' in list_env and 'csd' in list_alg  else None\n",
    "# metra\n",
    "Easy_metra_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Easy_metra_data]) if 'Easy' in list_env and 'metra' in list_alg  else None\n",
    "Ur_metra_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Ur_metra_data]) if 'Ur' in list_env and 'metra' in list_alg  else None\n",
    "Hard_metra_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Hard_metra_data]) if 'Hard' in list_env and 'metra' in list_alg  else None\n",
    "fetch_metra_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_metra_data]) if 'fetch' in list_env and 'metra' in list_alg  else None\n",
    "finger_metra_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in finger_metra_data]) if 'finger' in list_env and 'metra' in list_alg  else None\n",
    "fetch_push_metra_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_push_metra_data]) if 'fetch_push' in list_env and 'metra' in list_alg  else None\n",
    "# smm\n",
    "Easy_smm_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Easy_smm_data]) if 'Easy' in list_env and 'smm' in list_alg  else None\n",
    "Ur_smm_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Ur_smm_data]) if 'Ur' in list_env and 'smm' in list_alg  else None\n",
    "Hard_smm_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in Hard_smm_data]) if 'Hard' in list_env and 'smm' in list_alg  else None\n",
    "fetch_smm_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_smm_data]) if 'fetch' in list_env and 'smm' in list_alg  else None\n",
    "finger_smm_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in finger_smm_data]) if 'finger' in list_env and 'smm' in list_alg  else None\n",
    "fetch_push_smm_max_step = max([len(run_data['metrics']['coverage']) if 'coverage' in run_data['metrics'].keys() else None  for run_data in fetch_push_smm_data]) if 'fetch_push' in list_env and 'smm' in list_alg  else None\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def delete_none(liste):\n",
    "    return list(filter(None.__ne__, liste))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3300/1915478931.py:2: DeprecationWarning: NotImplemented should not be used in a boolean context\n",
      "  return list(filter(None.__ne__, liste))\n"
     ]
    }
   ],
   "source": [
    "\n",
    "Easy_lds_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Easy_lds_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Easy_lds_data]), axis=1) if 'Easy' in list_env and 'lds' in list_alg else None\n",
    "Ur_lds_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Ur_lds_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Ur_lds_data]), axis=1) if 'Ur' in list_env and 'lds' in list_alg else None\n",
    "Hard_lds_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Hard_lds_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Hard_lds_data]), axis=1) if 'Hard' in list_env and 'lds' in list_alg else None    \n",
    "fetch_lds_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_lds_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_lds_data]), axis=1) if 'fetch' in list_env and 'lds' in list_alg else None \n",
    "finger_lds_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, finger_lds_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in finger_lds_data]), axis=1) if 'finger' in list_env and 'lds' in list_alg else None\n",
    "fetch_push_lds_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_push_lds_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_push_lds_data]), axis=1) if 'fetch_push' in list_env and 'lds' in list_alg else None\n",
    "hopper_lds_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, hopper_lds_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in hopper_lds_data]), axis=1) if 'hopper' in list_env and 'lds' in list_alg else None\n",
    "# diayn\n",
    "Easy_diayn_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Easy_diayn_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Easy_diayn_data]), axis=1) if 'Easy' in list_env and 'diayn' in list_alg else None\n",
    "Ur_diayn_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Ur_diayn_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Ur_diayn_data]), axis=1) if 'Ur' in list_env and 'diayn' in list_alg else None\n",
    "Hard_diayn_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Hard_diayn_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Hard_diayn_data]), axis=1) if 'Hard' in list_env and 'diayn' in list_alg else None\n",
    "fetch_diayn_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_diayn_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_diayn_data]), axis=1) if 'fetch' in list_env and 'diayn' in list_alg else None\n",
    "finger_diayn_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, finger_diayn_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in finger_diayn_data]), axis=1) if 'finger' in list_env and 'diayn' in list_alg else None\n",
    "fetch_push_diayn_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_push_diayn_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_push_diayn_data]), axis=1) if 'fetch_push' in list_env and 'diayn' in list_alg else None\n",
    "hopper_diayn_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, hopper_diayn_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in hopper_diayn_data]), axis=1) if 'hopper' in list_env and 'diayn' in list_alg else None\n",
    "# rnd\n",
    "Easy_rnd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Easy_rnd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Easy_rnd_data]), axis=1) if 'Easy' in list_env and 'rnd' in list_alg else None\n",
    "Ur_rnd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Ur_rnd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Ur_rnd_data]), axis=1) if 'Ur' in list_env and 'rnd' in list_alg else None\n",
    "Hard_rnd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Hard_rnd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Hard_rnd_data]), axis=1) if 'Hard' in list_env and 'rnd' in list_alg else None\n",
    "fetch_rnd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_rnd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_rnd_data]), axis=1) if 'fetch' in list_env and 'rnd' in list_alg else None\n",
    "finger_rnd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, finger_rnd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in finger_rnd_data]), axis=1) if 'finger' in list_env and 'rnd' in list_alg else None\n",
    "fetch_push_rnd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_push_rnd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_push_rnd_data]), axis=1) if 'fetch_push' in list_env and 'rnd' in list_alg else None\n",
    "hopper_rnd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, hopper_rnd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in hopper_rnd_data]), axis=1) if 'hopper' in list_env and 'rnd' in list_alg else None\n",
    "# ngu\n",
    "Easy_ngu_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Easy_ngu_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Easy_ngu_data]), axis=1) if 'Easy' in list_env and 'ngu' in list_alg else None\n",
    "Ur_ngu_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Ur_ngu_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Ur_ngu_data]), axis=1) if 'Ur' in list_env and 'ngu' in list_alg else None\n",
    "Hard_ngu_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Hard_ngu_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Hard_ngu_data]), axis=1) if 'Hard' in list_env and 'ngu' in list_alg else None\n",
    "fetch_ngu_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_ngu_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_ngu_data]), axis=1) if 'fetch' in list_env and 'ngu' in list_alg else None\n",
    "finger_ngu_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, finger_ngu_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in finger_ngu_data]), axis=1) if 'finger' in list_env and 'ngu' in list_alg else None\n",
    "fetch_push_ngu_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_push_ngu_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_push_ngu_data]), axis=1) if 'fetch_push' in list_env and 'ngu' in list_alg else None\n",
    "hopper_ngu_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, hopper_ngu_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in hopper_ngu_data]), axis=1) if 'hopper' in list_env and 'ngu' in list_alg else None\n",
    "# lsd\n",
    "Easy_lsd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Easy_lsd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Easy_lsd_data]), axis=1) if 'Easy' in list_env and 'lsd' in list_alg else None\n",
    "Ur_lsd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Ur_lsd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Ur_lsd_data]), axis=1) if 'Ur' in list_env and 'lsd' in list_alg else None\n",
    "Hard_lsd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Hard_lsd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Hard_lsd_data]), axis=1) if 'Hard' in list_env and 'lsd' in list_alg else None\n",
    "fetch_lsd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_lsd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_lsd_data]), axis=1) if 'fetch' in list_env and 'lsd' in list_alg else None\n",
    "finger_lsd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, finger_lsd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in finger_lsd_data]), axis=1) if 'finger' in list_env and 'lsd' in list_alg else None\n",
    "fetch_push_lsd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_push_lsd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_push_lsd_data]), axis=1) if 'fetch_push' in list_env and 'lsd' in list_alg else None\n",
    "hopper_lsd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, hopper_lsd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in hopper_lsd_data]), axis=1) if 'hopper' in list_env and 'lsd' in list_alg else None\n",
    "# csd\n",
    "Easy_csd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Easy_csd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Easy_csd_data]), axis=1) if 'Easy' in list_env and 'csd' in list_alg else None\n",
    "Ur_csd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Ur_csd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Ur_csd_data]), axis=1) if 'Ur' in list_env and 'csd' in list_alg else None\n",
    "Hard_csd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Hard_csd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Hard_csd_data]), axis=1) if 'Hard' in list_env and 'csd' in list_alg else None\n",
    "fetch_csd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_csd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_csd_data]), axis=1) if 'fetch' in list_env and 'csd' in list_alg else None\n",
    "finger_csd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, finger_csd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in finger_csd_data]), axis=1) if 'finger' in list_env and 'csd' in list_alg else None\n",
    "fetch_push_csd_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_push_csd_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_push_csd_data]), axis=1) if 'fetch_push' in list_env and 'csd' in list_alg else None\n",
    "# metra\n",
    "Easy_metra_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Easy_metra_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Easy_metra_data]), axis=1) if 'Easy' in list_env and 'metra' in list_alg else None\n",
    "Ur_metra_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Ur_metra_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Ur_metra_data]), axis=1) if 'Ur' in list_env and 'metra' in list_alg else None\n",
    "Hard_metra_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Hard_metra_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Hard_metra_data]), axis=1) if 'Hard' in list_env and 'metra' in list_alg else None\n",
    "fetch_metra_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_metra_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_metra_data]), axis=1) if 'fetch' in list_env and 'metra' in list_alg else None\n",
    "finger_metra_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, finger_metra_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in finger_metra_data]), axis=1) if 'finger' in list_env and 'metra' in list_alg else None\n",
    "fetch_push_metra_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_push_metra_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_push_metra_data]), axis=1) if 'fetch_push' in list_env and 'metra' in list_alg else None\n",
    "# smm\n",
    "Easy_smm_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Easy_smm_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Easy_smm_data]), axis=1) if 'Easy' in list_env and 'smm' in list_alg else None\n",
    "Ur_smm_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Ur_smm_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Ur_smm_data]), axis=1) if 'Ur' in list_env and 'smm' in list_alg else None\n",
    "Hard_smm_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, Hard_smm_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in Hard_smm_data]), axis=1) if 'Hard' in list_env and 'smm' in list_alg else None\n",
    "fetch_smm_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_smm_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_smm_data]), axis=1) if 'fetch' in list_env and 'smm' in list_alg else None\n",
    "finger_smm_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, finger_smm_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in finger_smm_data]), axis=1) if 'finger' in list_env and 'smm' in list_alg else None\n",
    "fetch_push_smm_coverage_stack = np.concatenate(delete_none([fill_to_size(run_data['metrics']['coverage'].values, fetch_push_smm_max_step) if 'coverage' in run_data['metrics'].keys() else None for run_data in fetch_push_smm_data]), axis=1) if 'fetch_push' in list_env and 'smm' in list_alg else None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mean and std \n",
    "# lds\n",
    "Easy_lds_coverage_mean = np.mean(Easy_lds_coverage_stack, axis=1) if 'Easy' in list_env and 'lds' in list_alg else None\n",
    "Easy_lds_coverage_std = np.std(Easy_lds_coverage_stack, axis=1) if 'Easy' in list_env and 'lds' in list_alg else None\n",
    "Ur_lds_coverage_mean = np.mean(Ur_lds_coverage_stack, axis=1) if 'Ur' in list_env and 'lds' in list_alg else None\n",
    "Ur_lds_coverage_std = np.std(Ur_lds_coverage_stack, axis=1) if 'Ur' in list_env and 'lds' in list_alg else None\n",
    "Hard_lds_coverage_mean = np.mean(Hard_lds_coverage_stack, axis=1) if 'Hard' in list_env and 'lds' in list_alg else None\n",
    "Hard_lds_coverage_std = np.std(Hard_lds_coverage_stack, axis=1) if 'Hard' in list_env and 'lds' in list_alg else None\n",
    "fetch_lds_coverage_std = np.std(fetch_lds_coverage_stack, axis=1) if 'fetch' in list_env and 'lds' in list_alg else None\n",
    "fetch_lds_coverage_mean = np.mean(fetch_lds_coverage_stack, axis=1) if 'fetch' in list_env and 'lds' in list_alg else None\n",
    "finger_lds_coverage_mean = np.mean(finger_lds_coverage_stack, axis=1) if 'finger' in list_env and 'lds' in list_alg else None\n",
    "finger_lds_coverage_std = np.std(finger_lds_coverage_stack, axis=1) if 'finger' in list_env and 'lds' in list_alg else None\n",
    "fetch_push_lds_coverage_mean = np.mean(fetch_push_lds_coverage_stack, axis=1) if 'fetch_push' in list_env and 'lds' in list_alg else None\n",
    "fetch_push_lds_coverage_std = np.std(fetch_push_lds_coverage_stack, axis=1) if 'fetch_push' in list_env and 'lds' in list_alg else None\n",
    "hopper_lds_coverage_mean = np.mean(hopper_lds_coverage_stack, axis=1) if 'hopper' in list_env and 'lds' in list_alg else None\n",
    "hopper_lds_coverage_std = np.std(hopper_lds_coverage_stack, axis=1) if 'hopper' in list_env and 'lds' in list_alg else None\n",
    "# diayn \n",
    "Easy_diayn_coverage_mean = np.mean(Easy_diayn_coverage_stack, axis=1) if 'Easy' in list_env and 'diayn' in list_alg else None\n",
    "Easy_diayn_coverage_std = np.std(Easy_diayn_coverage_stack, axis=1) if 'Easy' in list_env and 'diayn' in list_alg else None\n",
    "Ur_diayn_coverage_mean = np.mean(Ur_diayn_coverage_stack, axis=1) if 'Ur' in list_env and 'diayn' in list_alg else None\n",
    "Ur_diayn_coverage_std = np.std(Ur_diayn_coverage_stack, axis=1) if 'Ur' in list_env and 'diayn' in list_alg else None\n",
    "Hard_diayn_coverage_mean = np.mean(Hard_diayn_coverage_stack, axis=1) if 'Hard' in list_env and 'diayn' in list_alg else None\n",
    "Hard_diayn_coverage_std = np.std(Hard_diayn_coverage_stack, axis=1) if 'Hard' in list_env and 'diayn' in list_alg else None\n",
    "fetch_diayn_coverage_std = np.std(fetch_diayn_coverage_stack, axis=1) if 'fetch' in list_env and 'diayn' in list_alg else None\n",
    "fetch_diayn_coverage_mean = np.mean(fetch_diayn_coverage_stack, axis=1) if 'fetch' in list_env and 'diayn' in list_alg else None\n",
    "finger_diayn_coverage_mean = np.mean(finger_diayn_coverage_stack, axis=1) if 'finger' in list_env and 'diayn' in list_alg else None\n",
    "finger_diayn_coverage_std = np.std(finger_diayn_coverage_stack, axis=1)   if 'finger' in list_env and 'diayn' in list_alg else None\n",
    "fetch_push_diayn_coverage_mean = np.mean(fetch_push_diayn_coverage_stack, axis=1) if 'fetch_push' in list_env and 'diayn' in list_alg else None\n",
    "fetch_push_diayn_coverage_std = np.std(fetch_push_diayn_coverage_stack, axis=1) if 'fetch_push' in list_env and 'diayn' in list_alg else None\n",
    "hopper_diayn_coverage_mean = np.mean(hopper_diayn_coverage_stack, axis=1) if 'hopper' in list_env and 'diayn' in list_alg else None\n",
    "hopper_diayn_coverage_std = np.std(hopper_diayn_coverage_stack, axis=1) if 'hopper' in list_env and 'diayn' in list_alg else None\n",
    "# rnd\n",
    "Easy_rnd_coverage_mean = np.mean(Easy_rnd_coverage_stack, axis=1)[:150] if 'Easy' in list_env and 'rnd' in list_alg else None\n",
    "Easy_rnd_coverage_std = np.std(Easy_rnd_coverage_stack, axis=1)[:150] if 'Easy' in list_env and 'rnd' in list_alg else None\n",
    "Ur_rnd_coverage_mean = np.mean(Ur_rnd_coverage_stack, axis=1)[:150] if 'Ur' in list_env and 'rnd' in list_alg else None\n",
    "Ur_rnd_coverage_std = np.std(Ur_rnd_coverage_stack, axis=1)[:150] if 'Ur' in list_env and 'rnd' in list_alg else None\n",
    "Hard_rnd_coverage_mean = np.mean(Hard_rnd_coverage_stack, axis=1)[:150] if 'Hard' in list_env and 'rnd' in list_alg else None\n",
    "Hard_rnd_coverage_std = np.std(Hard_rnd_coverage_stack, axis=1)[:150] if 'Hard' in list_env and 'rnd' in list_alg else None\n",
    "fetch_rnd_coverage_std = np.std(fetch_rnd_coverage_stack, axis=1) if 'fetch' in list_env and 'rnd' in list_alg else None\n",
    "fetch_rnd_coverage_mean = np.mean(fetch_rnd_coverage_stack, axis=1) if 'fetch' in list_env and 'rnd' in list_alg else None\n",
    "finger_rnd_coverage_mean = np.mean(finger_rnd_coverage_stack, axis=1) if 'finger' in list_env and 'rnd' in list_alg else None\n",
    "finger_rnd_coverage_std = np.std(finger_rnd_coverage_stack, axis=1) if 'finger' in list_env and 'rnd' in list_alg else None\n",
    "fetch_push_rnd_coverage_mean = np.mean(fetch_push_rnd_coverage_stack, axis=1) if 'fetch_push' in list_env and 'rnd' in list_alg else None\n",
    "fetch_push_rnd_coverage_std = np.std(fetch_push_rnd_coverage_stack, axis=1) if 'fetch_push' in list_env and 'rnd' in list_alg else None\n",
    "hopper_rnd_coverage_mean = np.mean(hopper_rnd_coverage_stack, axis=1) if 'hopper' in list_env and 'rnd' in list_alg else None\n",
    "hopper_rnd_coverage_std = np.std(hopper_rnd_coverage_stack, axis=1) if 'hopper' in list_env and 'rnd' in list_alg else None\n",
    "# ngu\n",
    "Easy_ngu_coverage_mean = np.mean(Easy_ngu_coverage_stack, axis=1) if 'Easy' in list_env and 'ngu' in list_alg else None\n",
    "Easy_ngu_coverage_std = np.std(Easy_ngu_coverage_stack, axis=1) if 'Easy' in list_env and 'ngu' in list_alg else None\n",
    "Ur_ngu_coverage_mean = np.mean(Ur_ngu_coverage_stack, axis=1) if 'Ur' in list_env and 'ngu' in list_alg else None\n",
    "Ur_ngu_coverage_std = np.std(Ur_ngu_coverage_stack, axis=1) if 'Ur' in list_env and 'ngu' in list_alg else None\n",
    "Hard_ngu_coverage_mean = np.mean(Hard_ngu_coverage_stack, axis=1) if 'Hard' in list_env and 'ngu' in list_alg else None\n",
    "Hard_ngu_coverage_std = np.std(Hard_ngu_coverage_stack, axis=1) if 'Hard' in list_env and 'ngu' in list_alg else None\n",
    "fetch_ngu_coverage_std = np.std(fetch_ngu_coverage_stack, axis=1) if 'fetch' in list_env and 'ngu' in list_alg else None\n",
    "fetch_ngu_coverage_mean = np.mean(fetch_ngu_coverage_stack, axis=1) if 'fetch' in list_env and 'ngu' in list_alg else None\n",
    "finger_ngu_coverage_mean = np.mean(finger_ngu_coverage_stack, axis=1) if 'finger' in list_env and 'ngu' in list_alg else None\n",
    "finger_ngu_coverage_std = np.std(finger_ngu_coverage_stack, axis=1) if 'finger' in list_env and 'ngu' in list_alg else None\n",
    "fetch_push_ngu_coverage_mean = np.mean(fetch_push_ngu_coverage_stack, axis=1) if 'fetch_push' in list_env and 'ngu' in list_alg else None\n",
    "fetch_push_ngu_coverage_std = np.std(fetch_push_ngu_coverage_stack, axis=1) if 'fetch_push' in list_env and 'ngu' in list_alg else None\n",
    "hopper_ngu_coverage_mean = np.mean(hopper_ngu_coverage_stack, axis=1) if 'hopper' in list_env and 'ngu' in list_alg else None\n",
    "hopper_ngu_coverage_std = np.std(hopper_ngu_coverage_stack, axis=1) if 'hopper' in list_env and 'ngu' in list_alg else None\n",
    "# lsd\n",
    "Easy_lsd_coverage_mean = np.mean(Easy_lsd_coverage_stack, axis=1) if 'Easy' in list_env and 'lsd' in list_alg else None\n",
    "Easy_lsd_coverage_std = np.std(Easy_lsd_coverage_stack, axis=1) if 'Easy' in list_env and 'lsd' in list_alg else None\n",
    "Ur_lsd_coverage_mean = np.mean(Ur_lsd_coverage_stack, axis=1) if 'Ur' in list_env and 'lsd' in list_alg else None\n",
    "Ur_lsd_coverage_std = np.std(Ur_lsd_coverage_stack, axis=1) if 'Ur' in list_env and 'lsd' in list_alg else None\n",
    "Hard_lsd_coverage_mean = np.mean(Hard_lsd_coverage_stack, axis=1) if 'Hard' in list_env and 'lsd' in list_alg else None\n",
    "Hard_lsd_coverage_std = np.std(Hard_lsd_coverage_stack, axis=1) if 'Hard' in list_env and 'lsd' in list_alg else None\n",
    "fetch_lsd_coverage_std = np.std(fetch_lsd_coverage_stack, axis=1) if 'fetch' in list_env and 'lsd' in list_alg else None\n",
    "fetch_lsd_coverage_mean = np.mean(fetch_lsd_coverage_stack, axis=1) if 'fetch' in list_env and 'lsd' in list_alg else None\n",
    "finger_lsd_coverage_mean = np.mean(finger_lsd_coverage_stack, axis=1) if 'finger' in list_env and 'lsd' in list_alg else None\n",
    "finger_lsd_coverage_std = np.std(finger_lsd_coverage_stack, axis=1) if 'finger' in list_env and 'lsd' in list_alg else None\n",
    "fetch_push_lsd_coverage_mean = np.mean(fetch_push_lsd_coverage_stack, axis=1) if 'fetch_push' in list_env and 'lsd' in list_alg else None\n",
    "fetch_push_lsd_coverage_std = np.std(fetch_push_lsd_coverage_stack, axis=1) if 'fetch_push' in list_env and 'lsd' in list_alg else None\n",
    "hopper_lsd_coverage_mean = np.mean(hopper_lsd_coverage_stack, axis=1) if 'hopper' in list_env and 'lsd' in list_alg else None\n",
    "hopper_lsd_coverage_std = np.std(hopper_lsd_coverage_stack, axis=1) if 'hopper' in list_env and 'lsd' in list_alg else None\n",
    "# csd\n",
    "Easy_csd_coverage_mean = np.mean(Easy_csd_coverage_stack, axis=1) if 'Easy' in list_env and 'csd' in list_alg else None\n",
    "Easy_csd_coverage_std = np.std(Easy_csd_coverage_stack, axis=1) if 'Easy' in list_env and 'csd' in list_alg else None\n",
    "Ur_csd_coverage_mean = np.mean(Ur_csd_coverage_stack, axis=1) if 'Ur' in list_env and 'csd' in list_alg else None\n",
    "Ur_csd_coverage_std = np.std(Ur_csd_coverage_stack, axis=1) if 'Ur' in list_env and 'csd' in list_alg else None\n",
    "Hard_csd_coverage_mean = np.mean(Hard_csd_coverage_stack, axis=1) if 'Hard' in list_env and 'csd' in list_alg else None\n",
    "Hard_csd_coverage_std = np.std(Hard_csd_coverage_stack, axis=1) if 'Hard' in list_env and 'csd' in list_alg else None\n",
    "fetch_csd_coverage_std = np.std(fetch_csd_coverage_stack, axis=1) if 'fetch' in list_env and 'csd' in list_alg else None\n",
    "fetch_csd_coverage_mean = np.mean(fetch_csd_coverage_stack, axis=1) if 'fetch' in list_env and 'csd' in list_alg else None\n",
    "finger_csd_coverage_mean = np.mean(finger_csd_coverage_stack, axis=1) if 'finger' in list_env and 'csd' in list_alg else None\n",
    "finger_csd_coverage_std = np.std(finger_csd_coverage_stack, axis=1) if 'finger' in list_env and 'csd' in list_alg else None\n",
    "fetch_push_csd_coverage_mean = np.mean(fetch_push_csd_coverage_stack, axis=1) if 'fetch_push' in list_env and 'csd' in list_alg else None\n",
    "fetch_push_csd_coverage_std = np.std(fetch_push_csd_coverage_stack, axis=1) if 'fetch_push' in list_env and 'csd' in list_alg else None\n",
    "# metra\n",
    "Easy_metra_coverage_mean = np.mean(Easy_metra_coverage_stack, axis=1) if 'Easy' in list_env and 'metra' in list_alg else None\n",
    "Easy_metra_coverage_std = np.std(Easy_metra_coverage_stack, axis=1) if 'Easy' in list_env and 'metra' in list_alg else None\n",
    "Ur_metra_coverage_mean = np.mean(Ur_metra_coverage_stack, axis=1) if 'Ur' in list_env and 'metra' in list_alg else None\n",
    "Ur_metra_coverage_std = np.std(Ur_metra_coverage_stack, axis=1) if 'Ur' in list_env and 'metra' in list_alg else None\n",
    "Hard_metra_coverage_mean = np.mean(Hard_metra_coverage_stack, axis=1) if 'Hard' in list_env and 'metra' in list_alg else None\n",
    "Hard_metra_coverage_std = np.std(Hard_metra_coverage_stack, axis=1) if 'Hard' in list_env and 'metra' in list_alg else None\n",
    "fetch_metra_coverage_std = np.std(fetch_metra_coverage_stack, axis=1) if 'fetch' in list_env and 'metra' in list_alg else None\n",
    "fetch_metra_coverage_mean = np.mean(fetch_metra_coverage_stack, axis=1) if 'fetch' in list_env and 'metra' in list_alg else None\n",
    "finger_metra_coverage_mean = np.mean(finger_metra_coverage_stack, axis=1) if 'finger' in list_env and 'metra' in list_alg else None\n",
    "finger_metra_coverage_std = np.std(finger_metra_coverage_stack, axis=1) if 'finger' in list_env and 'metra' in list_alg else None\n",
    "fetch_push_metra_coverage_mean = np.mean(fetch_push_metra_coverage_stack, axis=1) if 'fetch_push' in list_env and 'metra' in list_alg else None\n",
    "fetch_push_metra_coverage_std = np.std(fetch_push_metra_coverage_stack, axis=1) if 'fetch_push' in list_env and 'metra' in list_alg else None\n",
    "# smm\n",
    "Easy_smm_coverage_mean = np.mean(Easy_smm_coverage_stack, axis=1) if 'Easy' in list_env and 'smm' in list_alg else None\n",
    "Easy_smm_coverage_std = np.std(Easy_smm_coverage_stack, axis=1) if 'Easy' in list_env and 'smm' in list_alg else None\n",
    "Ur_smm_coverage_mean = np.mean(Ur_smm_coverage_stack, axis=1) if 'Ur' in list_env and 'smm' in list_alg else None\n",
    "Ur_smm_coverage_std = np.std(Ur_smm_coverage_stack, axis=1) if 'Ur' in list_env and 'smm' in list_alg else None\n",
    "Hard_smm_coverage_mean = np.mean(Hard_smm_coverage_stack, axis=1) if 'Hard' in list_env and 'smm' in list_alg else None\n",
    "Hard_smm_coverage_std = np.std(Hard_smm_coverage_stack, axis=1) if 'Hard' in list_env and 'smm' in list_alg else None\n",
    "fetch_smm_coverage_std = np.std(fetch_smm_coverage_stack, axis=1) if 'fetch' in list_env and 'smm' in list_alg else None\n",
    "fetch_smm_coverage_mean = np.mean(fetch_smm_coverage_stack, axis=1) if 'fetch' in list_env and 'smm' in list_alg else None\n",
    "finger_smm_coverage_mean = np.mean(finger_smm_coverage_stack, axis=1) if 'finger' in list_env and 'smm' in list_alg else None\n",
    "finger_smm_coverage_std = np.std(finger_smm_coverage_stack, axis=1) if 'finger' in list_env and 'smm' in list_alg else None\n",
    "fetch_push_smm_coverage_mean = np.mean(fetch_push_smm_coverage_stack, axis=1) if 'fetch_push' in list_env and 'smm' in list_alg else None\n",
    "fetch_push_smm_coverage_std = np.std(fetch_push_smm_coverage_stack, axis=1) if 'fetch_push' in list_env and 'smm' in list_alg else None\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize by the maximum coverage for each environment\n",
    "# max coverage\n",
    "max_coverage = {}\n",
    "std_max_coverage = {}\n",
    "for env in list_env:\n",
    "    max_coverage[env] = 0\n",
    "    std_max_coverage[env] = 0\n",
    "    for alg in list_alg:\n",
    "        alg_stack = globals()[f'{env}_{alg}_coverage_stack']\n",
    "        alg_std = globals()[f'{env}_{alg}_coverage_std']\n",
    "        if alg_stack is not None:\n",
    "            max_coverage[env] = max(max_coverage[env], np.max(alg_stack))\n",
    "            std_max_coverage[env] = max(std_max_coverage[env], alg_std[-1])\n",
    "# normalized coverage\n",
    "mean_normalized_coverage = {alg: {} for alg in list_alg}\n",
    "std_normalized_coverage = {alg: {} for alg in list_alg}\n",
    "for env in list_env:\n",
    "    for alg in list_alg:\n",
    "        alg_mean = globals()[f'{env}_{alg}_coverage_mean'][-1]\n",
    "        alg_std = globals()[f'{env}_{alg}_coverage_std'][-1]\n",
    "        if alg_mean is not None:\n",
    "            mean_normalized_coverage[alg][env] = alg_mean / max_coverage[env] *100\n",
    "            std_normalized_coverage[alg][env] = alg_std / max_coverage[env] *100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Standard t-test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Umaze LEADS/CSD :      |     0.00411    |\n",
      "| Umaze LEADS/LSD :      |     0.00264    |\n",
      "| Hard LEADS/CSD :      |     0.00099    |\n",
      "| Hard LEADS/LSD :      |     0.00133    |\n",
      "| Fetch Slide LEADS/CSD :      |     0.00007    |\n",
      "| Fetch Slide LEADS/LSD :      |     0.00001    |\n",
      "| Finger CSD/LEADS :      |     0.05521    |\n",
      "| Finger CSD/LSD :      |     0.01261    |\n",
      "| Fetch Reach LEADS/CSD :      |     0.09463    |\n",
      "| Fetch Reach LEADS/DIAYN :      |     0.1032    |\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy.stats import ttest_1samp, ttest_ind, f_oneway\n",
    "# T-test\n",
    "# U \n",
    "Ur_lds_values = Ur_lds_coverage_stack[-1,:5]\n",
    "Ur_csd_values = Ur_csd_coverage_stack[-1,:5]\n",
    "Ur_lsd_values = Ur_lsd_coverage_stack[-1,:5]\n",
    "stat_lds_csd, p_lds_csd = ttest_ind(Ur_lds_values, Ur_csd_values)\n",
    "stat_lds_lsd, p_lds_lsd = ttest_ind(Ur_lds_values, Ur_lsd_values)\n",
    "stat_anova, p_anova = f_oneway(Ur_lds_values, Ur_csd_values, Ur_lsd_values)\n",
    "print('| Umaze LEADS/CSD :      |'+'     '+str(round(p_lds_csd,5))+'    |')\n",
    "print('| Umaze LEADS/LSD :      |'+'     '+str(round(p_lds_lsd,5))+'    |')\n",
    "\n",
    "# Hard\n",
    "Hard_lds_values = Hard_lds_coverage_stack[-1,:5]\n",
    "Hard_csd_values = Hard_csd_coverage_stack[-1,:5]\n",
    "Hard_lsd_values = Hard_lsd_coverage_stack[-1,:5]\n",
    "state_lds_csd, p_lds_csd = ttest_ind(Hard_lds_values, Hard_csd_values)\n",
    "state_lds_lsd, p_lds_lsd = ttest_ind(Hard_lds_values, Hard_lsd_values)\n",
    "stat_anova, p_anova = f_oneway(Hard_lds_values, Hard_csd_values, Hard_lsd_values)\n",
    "print('| Hard LEADS/CSD :      |'+'     '+str(round(p_lds_csd,5))+'    |')\n",
    "print('| Hard LEADS/LSD :      |'+'     '+str(round(p_lds_lsd,5))+'    |')\n",
    "\n",
    "# fetch_push\n",
    "fetch_push_lds_values = fetch_push_lds_coverage_stack[-1,:5]\n",
    "fetch_push_csd_values = fetch_push_csd_coverage_stack[-1,:5]\n",
    "fetch_push_lsd_values = fetch_push_lsd_coverage_stack[-1,:5]\n",
    "state_lds_csd, p_lds_csd = ttest_ind(fetch_push_lds_values, fetch_push_csd_values)\n",
    "state_lds_lsd, p_lds_lsd = ttest_ind(fetch_push_lds_values, fetch_push_lsd_values)\n",
    "stat_anova, p_anova = f_oneway(fetch_push_lds_values, fetch_push_csd_values, fetch_push_lsd_values)\n",
    "print('| Fetch Slide LEADS/CSD :      |'+'     '+str(round(p_lds_csd,5))+'    |')\n",
    "print('| Fetch Slide LEADS/LSD :      |'+'     '+str(round(p_lds_lsd,5))+'    |')\n",
    "\n",
    "# Finger \n",
    "finger_lds_values = finger_lds_coverage_stack[-1,:5]\n",
    "finger_csd_values = finger_csd_coverage_stack[-1,:5]\n",
    "finger_lsd_values = finger_lsd_coverage_stack[-1,:5]\n",
    "finger_metra_values = finger_metra_coverage_stack[-1,:]\n",
    "state_csd_lds, p_csd_lds = ttest_ind(finger_csd_values, finger_lds_values)\n",
    "state_csd_lsd, p_csd_lsd = ttest_ind(finger_csd_values, finger_lsd_values)\n",
    "state_csd_metra, p_csd_metra = ttest_ind(finger_csd_values, finger_metra_values)\n",
    "print('| Finger CSD/LEADS :      |'+'     '+str(round(p_csd_lds,5))+'    |')\n",
    "print('| Finger CSD/LSD :      |'+'     '+str(round(p_csd_lsd,5))+'    |')\n",
    "\n",
    "\n",
    "# fetch reach \n",
    "fetch_lds_values = fetch_lds_coverage_stack[-1,:5]\n",
    "fetch_csd_values = fetch_csd_coverage_stack[-1,:5]\n",
    "fetch_diayn_values = fetch_diayn_coverage_stack[-1,:5]\n",
    "state_lds_csd, p_lds_csd = ttest_ind(fetch_lds_values, fetch_csd_values)\n",
    "state_lds_diayn, p_lds_diayn = ttest_ind(fetch_diayn_values, fetch_diayn_values)\n",
    "stat_anova, p_anova = f_oneway(fetch_lds_values, fetch_csd_values, fetch_diayn_values)\n",
    "print('| Fetch Reach LEADS/CSD :      |'+'     '+str(round(p_lds_csd,5))+'    |')\n",
    "print('| Fetch Reach LEADS/DIAYN :      |'+'     '+str(round(p_lds_diayn,5))+'    |')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### paired t-test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Umaze LEADS/CSD :      |     0.01522    |\n",
      "| Umaze LEADS/LSD :      |     0.01152    |\n",
      "| Hard LEADS/CSD :      |     0.00246    |\n",
      "| Hard LEADS/LSD :      |     0.00977    |\n",
      "| Fetch Slide LEADS/CSD :      |     0.00007    |\n",
      "| Fetch Slide LEADS/LSD :      |     0.00055    |\n",
      "| Finger CSD/LEADS :      |     0.07072    |\n",
      "| Finger CSD/LSD :      |     0.00070    |\n",
      "| Fetch Reach LEADS/CSD :      |     0.06354    |\n",
      "| Fetch Reach LEADS/DIAYN :      |     0.08307    |\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import ttest_rel\n",
    "# T-test\n",
    "# U \n",
    "Ur_lds_values = Ur_lds_coverage_stack[-1,:5]\n",
    "Ur_csd_values = Ur_csd_coverage_stack[-1,:5]\n",
    "Ur_lsd_values = Ur_lsd_coverage_stack[-1,:5]\n",
    "stat_lds_csd, p_lds_csd = ttest_rel(Ur_lds_values, Ur_csd_values)\n",
    "stat_lds_lsd, p_lds_lsd = ttest_rel(Ur_lds_values, Ur_lsd_values)\n",
    "stat_anova, p_anova = f_oneway(Ur_lds_values, Ur_csd_values, Ur_lsd_values)\n",
    "print('| Umaze LEADS/CSD :      |'+'     '+str(round(p_lds_csd,5))+'    |')\n",
    "print('| Umaze LEADS/LSD :      |'+'     '+str(round(p_lds_lsd,5))+'    |')\n",
    "\n",
    "# Hard\n",
    "Hard_lds_values = Hard_lds_coverage_stack[-1,:5]\n",
    "Hard_csd_values = Hard_csd_coverage_stack[-1,:5]\n",
    "Hard_lsd_values = Hard_lsd_coverage_stack[-1,:5]\n",
    "state_lds_csd, p_lds_csd = ttest_rel(Hard_lds_values, Hard_csd_values)\n",
    "state_lds_lsd, p_lds_lsd = ttest_rel(Hard_lds_values, Hard_lsd_values)\n",
    "stat_anova, p_anova = f_oneway(Hard_lds_values, Hard_csd_values, Hard_lsd_values)\n",
    "print('| Hard LEADS/CSD :      |'+'     '+str(round(p_lds_csd,5))+'    |')\n",
    "print('| Hard LEADS/LSD :      |'+'     '+str(round(p_lds_lsd,5))+'    |')\n",
    "\n",
    "# fetch_push\n",
    "fetch_push_lds_values = fetch_push_lds_coverage_stack[-1,:5]\n",
    "fetch_push_csd_values = fetch_push_csd_coverage_stack[-1,:5]\n",
    "fetch_push_lsd_values = fetch_push_lsd_coverage_stack[-1,:5]\n",
    "state_lds_csd, p_lds_csd = ttest_rel(fetch_push_lds_values, fetch_push_csd_values)\n",
    "state_lds_lsd, p_lds_lsd = ttest_rel(fetch_push_lds_values, fetch_push_lsd_values)\n",
    "stat_anova, p_anova = f_oneway(fetch_push_lds_values, fetch_push_csd_values, fetch_push_lsd_values)\n",
    "print('| Fetch Slide LEADS/CSD :      |'+'     '+str(round(p_lds_csd,5))+'    |')\n",
    "print('| Fetch Slide LEADS/LSD :      |'+'     '+str(round(p_lds_lsd,5))+'    |')\n",
    "\n",
    "# Finger \n",
    "finger_lds_values = finger_lds_coverage_stack[-1,:5]\n",
    "finger_csd_values = finger_csd_coverage_stack[-1,:5]\n",
    "finger_lsd_values = finger_lsd_coverage_stack[-1,:5]\n",
    "finger_metra_values = finger_metra_coverage_stack[-1,:]\n",
    "state_csd_lds, p_csd_lds = ttest_rel(finger_csd_values, finger_lds_values)\n",
    "state_csd_lsd, p_csd_lsd = ttest_rel(finger_csd_values, finger_lsd_values)\n",
    "state_csd_metra, p_csd_metra = ttest_rel(finger_csd_values, finger_metra_values)\n",
    "print('| Finger CSD/LEADS :      |'+'     '+str(round(p_csd_lds,5))+'    |')\n",
    "print('| Finger CSD/LSD :      |'+'     '+str(round(p_csd_lsd,5))+'    |')\n",
    "\n",
    "\n",
    "# fetch reach \n",
    "fetch_lds_values = fetch_lds_coverage_stack[-1,:5]\n",
    "fetch_csd_values = fetch_csd_coverage_stack[-1,:5]\n",
    "fetch_diayn_values = fetch_diayn_coverage_stack[-1,:5]\n",
    "state_lds_csd, p_lds_csd = ttest_rel(fetch_lds_values, fetch_csd_values)\n",
    "state_lds_diayn, p_lds_diayn = ttest_rel(fetch_diayn_values, fetch_diayn_values)\n",
    "stat_anova, p_anova = f_oneway(fetch_lds_values, fetch_csd_values, fetch_diayn_values)\n",
    "print('| Fetch Reach LEADS/CSD :      |'+'     '+str(round(p_lds_csd,5))+'    |')\n",
    "print('| Fetch Reach LEADS/DIAYN :      |'+'     '+str(round(p_lds_diayn,5))+'    |')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}