Lesson_1_Multi_Armed_Bandit_with_OpenAi_Gym.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lesson 1: Multi- Armed Bandit with OpenAi Gym.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "DzVfeL9n7W98",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 130
        },
        "outputId": "557f1fae-8bbc-4fd1-c3e6-b2d105f35432"
      },
      "cell_type": "code",
      "source": [
        "          _____                _____                    _____                    _____                    _____                    _____          \n",
        "         /\\    \\              /\\    \\                  /\\    \\                  /\\    \\                  /\\    \\                  /\\    \\         \n",
        "        /::\\    \\            /::\\    \\                /::\\    \\                /::\\    \\                /::\\    \\                /::\\    \\        \n",
        "       /::::\\    \\           \\:::\\    \\              /::::\\    \\              /::::\\    \\              /::::\\    \\               \\:::\\    \\       \n",
        "      /::::::\\    \\           \\:::\\    \\            /::::::\\    \\            /::::::\\    \\            /::::::\\    \\               \\:::\\    \\      \n",
        "     /:::/\\:::\\    \\           \\:::\\    \\          /:::/\\:::\\    \\          /:::/\\:::\\    \\          /:::/\\:::\\    \\               \\:::\\    \\     \n",
        "    /:::/__\\:::\\    \\           \\:::\\    \\        /:::/__\\:::\\    \\        /:::/__\\:::\\    \\        /:::/__\\:::\\    \\               \\:::\\    \\    \n",
        "    \\:::\\   \\:::\\    \\          /::::\\    \\      /::::\\   \\:::\\    \\      /::::\\   \\:::\\    \\      /::::\\   \\:::\\    \\              /::::\\    \\   \n",
        "  ___\\:::\\   \\:::\\    \\        /::::::\\    \\    /::::::\\   \\:::\\    \\    /::::::\\   \\:::\\    \\    /::::::\\   \\:::\\    \\    ____    /::::::\\    \\  \n",
        " /\\   \\:::\\   \\:::\\    \\      /:::/\\:::\\    \\  /:::/\\:::\\   \\:::\\    \\  /:::/\\:::\\   \\:::\\____\\  /:::/\\:::\\   \\:::\\    \\  /\\   \\  /:::/\\:::\\    \\ \n",
        "/::\\   \\:::\\   \\:::\\____\\    /:::/  \\:::\\____\\/:::/  \\:::\\   \\:::\\____\\/:::/  \\:::\\   \\:::|    |/:::/  \\:::\\   \\:::\\____\\/::\\   \\/:::/  \\:::\\____\\\n",
        "\\:::\\   \\:::\\   \\::/    /   /:::/    \\::/    /\\::/    \\:::\\  /:::/    /\\::/   |::::\\  /:::|____|\\::/    \\:::\\  /:::/    /\\:::\\  /:::/    \\::/    /\n",
        " \\:::\\   \\:::\\   \\/____/   /:::/    / \\/____/  \\/____/ \\:::\\/:::/    /  \\/____|:::::\\/:::/    /  \\/____/ \\:::\\/:::/    /  \\:::\\/:::/    / \\/____/ \n",
        "  \\:::\\   \\:::\\    \\      /:::/    /                    \\::::::/    /         |:::::::::/    /            \\::::::/    /    \\::::::/    /          \n",
        "   \\:::\\   \\:::\\____\\    /:::/    /                      \\::::/    /          |::|\\::::/    /              \\::::/    /      \\::::/____/           \n",
        "    \\:::\\  /:::/    /    \\::/    /                       /:::/    /           |::| \\::/____/               /:::/    /        \\:::\\    \\           \n",
        "     \\:::\\/:::/    /      \\/____/                       /:::/    /            |::|  ~|                    /:::/    /          \\:::\\    \\          \n",
        "      \\::::::/    /                                    /:::/    /             |::|   |                   /:::/    /            \\:::\\    \\         \n",
        "       \\::::/    /                                    /:::/    /              \\::|   |                  /:::/    /              \\:::\\____\\        \n",
        "        \\::/    /                                     \\::/    /                \\:|   |                  \\::/    /                \\::/    /        \n",
        "         \\/____/                                       \\/____/                  \\|___|                   \\/____/                  \\/____/         "
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "error",
          "ename": "SyntaxError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-1-09f36a0c4edc>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    _____                _____                    _____                    _____                    _____                    _____\u001b[0m\n\u001b[0m                             ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "excxXVzFXgDf",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Lesson 1, exercise 1: The Multi Armed Bandit Problem with OpenAi Gym\n",
        "\n",
        "the purpose of this notebook is:\n",
        "\n",
        "\n",
        "\n",
        "1.   To understand the Gym Environment,\n",
        "2.   Implement Epsilon-Greedy on the bandit problem as discussed in the first lesson\n",
        "3.   **Win at slot machines!!.**\n",
        "\n",
        "Lets get started!\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "VBwBIZPkZf_g",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "First we need to download & install the Gym Library so that it works in Colabs."
      ]
    },
    {
      "metadata": {
        "id": "MoRJN4PpXsGH",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip install gym > /dev/null 2>&1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "on5JDYmWaK-w",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Great!** now for our first bit of code\n"
      ]
    },
    {
      "metadata": {
        "id": "p8fBpXyfbk89",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Lets import the Gym class and walk through a basic example of Gym Code"
      ]
    },
    {
      "metadata": {
        "id": "5hK-jC9ceDuY",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import gym"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "WLePIoe7VJXS",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Gym's main purpose is to provide a large collection of \"environments\" that expose a common interface, using standardized inputs & outputs for Reinforcement Learning model testing purposes. You can find a listing of these environments below, as follows:"
      ]
    },
    {
      "metadata": {
        "id": "twrKYXVAVIp8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "f136c844-b53e-401a-9830-de52c0300b01"
      },
      "cell_type": "code",
      "source": [
        "from gym import envs\n",
        "print(envs.registry.all())"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dict_values([EnvSpec(Copy-v0), EnvSpec(RepeatCopy-v0), EnvSpec(ReversedAddition-v0), EnvSpec(ReversedAddition3-v0), EnvSpec(DuplicatedInput-v0), EnvSpec(Reverse-v0), EnvSpec(CartPole-v0), EnvSpec(CartPole-v1), EnvSpec(MountainCar-v0), EnvSpec(MountainCarContinuous-v0), EnvSpec(Pendulum-v0), EnvSpec(Acrobot-v1), EnvSpec(LunarLander-v2), EnvSpec(LunarLanderContinuous-v2), EnvSpec(BipedalWalker-v2), EnvSpec(BipedalWalkerHardcore-v2), EnvSpec(CarRacing-v0), EnvSpec(Blackjack-v0), EnvSpec(KellyCoinflip-v0), EnvSpec(KellyCoinflipGeneralized-v0), EnvSpec(FrozenLake-v0), EnvSpec(FrozenLake8x8-v0), EnvSpec(CliffWalking-v0), EnvSpec(NChain-v0), EnvSpec(Roulette-v0), EnvSpec(Taxi-v2), EnvSpec(GuessingGame-v0), EnvSpec(HotterColder-v0), EnvSpec(Reacher-v2), EnvSpec(Pusher-v2), EnvSpec(Thrower-v2), EnvSpec(Striker-v2), EnvSpec(InvertedPendulum-v2), EnvSpec(InvertedDoublePendulum-v2), EnvSpec(HalfCheetah-v2), EnvSpec(Hopper-v2), EnvSpec(Swimmer-v2), EnvSpec(Walker2d-v2), EnvSpec(Ant-v2), EnvSpec(Humanoid-v2), EnvSpec(HumanoidStandup-v2), EnvSpec(FetchSlide-v1), EnvSpec(FetchPickAndPlace-v1), EnvSpec(FetchReach-v1), EnvSpec(FetchPush-v1), EnvSpec(HandReach-v0), EnvSpec(HandManipulateBlockRotateZ-v0), EnvSpec(HandManipulateBlockRotateParallel-v0), EnvSpec(HandManipulateBlockRotateXYZ-v0), EnvSpec(HandManipulateBlockFull-v0), EnvSpec(HandManipulateBlock-v0), EnvSpec(HandManipulateEggRotate-v0), EnvSpec(HandManipulateEggFull-v0), EnvSpec(HandManipulateEgg-v0), EnvSpec(HandManipulatePenRotate-v0), EnvSpec(HandManipulatePenFull-v0), EnvSpec(HandManipulatePen-v0), EnvSpec(FetchSlideDense-v1), EnvSpec(FetchPickAndPlaceDense-v1), EnvSpec(FetchReachDense-v1), EnvSpec(FetchPushDense-v1), EnvSpec(HandReachDense-v0), EnvSpec(HandManipulateBlockRotateZDense-v0), EnvSpec(HandManipulateBlockRotateParallelDense-v0), EnvSpec(HandManipulateBlockRotateXYZDense-v0), EnvSpec(HandManipulateBlockFullDense-v0), EnvSpec(HandManipulateBlockDense-v0), EnvSpec(HandManipulateEggRotateDense-v0), EnvSpec(HandManipulateEggFullDense-v0), EnvSpec(HandManipulateEggDense-v0), EnvSpec(HandManipulatePenRotateDense-v0), EnvSpec(HandManipulatePenFullDense-v0), EnvSpec(HandManipulatePenDense-v0), EnvSpec(AirRaid-v0), EnvSpec(AirRaid-v4), EnvSpec(AirRaidDeterministic-v0), EnvSpec(AirRaidDeterministic-v4), EnvSpec(AirRaidNoFrameskip-v0), EnvSpec(AirRaidNoFrameskip-v4), EnvSpec(AirRaid-ram-v0), EnvSpec(AirRaid-ram-v4), EnvSpec(AirRaid-ramDeterministic-v0), EnvSpec(AirRaid-ramDeterministic-v4), EnvSpec(AirRaid-ramNoFrameskip-v0), EnvSpec(AirRaid-ramNoFrameskip-v4), EnvSpec(Alien-v0), EnvSpec(Alien-v4), EnvSpec(AlienDeterministic-v0), EnvSpec(AlienDeterministic-v4), EnvSpec(AlienNoFrameskip-v0), EnvSpec(AlienNoFrameskip-v4), EnvSpec(Alien-ram-v0), EnvSpec(Alien-ram-v4), EnvSpec(Alien-ramDeterministic-v0), EnvSpec(Alien-ramDeterministic-v4), EnvSpec(Alien-ramNoFrameskip-v0), EnvSpec(Alien-ramNoFrameskip-v4), EnvSpec(Amidar-v0), EnvSpec(Amidar-v4), EnvSpec(AmidarDeterministic-v0), EnvSpec(AmidarDeterministic-v4), EnvSpec(AmidarNoFrameskip-v0), EnvSpec(AmidarNoFrameskip-v4), EnvSpec(Amidar-ram-v0), EnvSpec(Amidar-ram-v4), EnvSpec(Amidar-ramDeterministic-v0), EnvSpec(Amidar-ramDeterministic-v4), EnvSpec(Amidar-ramNoFrameskip-v0), EnvSpec(Amidar-ramNoFrameskip-v4), EnvSpec(Assault-v0), EnvSpec(Assault-v4), EnvSpec(AssaultDeterministic-v0), EnvSpec(AssaultDeterministic-v4), EnvSpec(AssaultNoFrameskip-v0), EnvSpec(AssaultNoFrameskip-v4), EnvSpec(Assault-ram-v0), EnvSpec(Assault-ram-v4), EnvSpec(Assault-ramDeterministic-v0), EnvSpec(Assault-ramDeterministic-v4), EnvSpec(Assault-ramNoFrameskip-v0), EnvSpec(Assault-ramNoFrameskip-v4), EnvSpec(Asterix-v0), EnvSpec(Asterix-v4), EnvSpec(AsterixDeterministic-v0), EnvSpec(AsterixDeterministic-v4), EnvSpec(AsterixNoFrameskip-v0), EnvSpec(AsterixNoFrameskip-v4), EnvSpec(Asterix-ram-v0), EnvSpec(Asterix-ram-v4), EnvSpec(Asterix-ramDeterministic-v0), EnvSpec(Asterix-ramDeterministic-v4), EnvSpec(Asterix-ramNoFrameskip-v0), EnvSpec(Asterix-ramNoFrameskip-v4), EnvSpec(Asteroids-v0), EnvSpec(Asteroids-v4), EnvSpec(AsteroidsDeterministic-v0), EnvSpec(AsteroidsDeterministic-v4), EnvSpec(AsteroidsNoFrameskip-v0), EnvSpec(AsteroidsNoFrameskip-v4), EnvSpec(Asteroids-ram-v0), EnvSpec(Asteroids-ram-v4), EnvSpec(Asteroids-ramDeterministic-v0), EnvSpec(Asteroids-ramDeterministic-v4), EnvSpec(Asteroids-ramNoFrameskip-v0), EnvSpec(Asteroids-ramNoFrameskip-v4), EnvSpec(Atlantis-v0), EnvSpec(Atlantis-v4), EnvSpec(AtlantisDeterministic-v0), EnvSpec(AtlantisDeterministic-v4), EnvSpec(AtlantisNoFrameskip-v0), EnvSpec(AtlantisNoFrameskip-v4), EnvSpec(Atlantis-ram-v0), EnvSpec(Atlantis-ram-v4), EnvSpec(Atlantis-ramDeterministic-v0), EnvSpec(Atlantis-ramDeterministic-v4), EnvSpec(Atlantis-ramNoFrameskip-v0), EnvSpec(Atlantis-ramNoFrameskip-v4), EnvSpec(BankHeist-v0), EnvSpec(BankHeist-v4), EnvSpec(BankHeistDeterministic-v0), EnvSpec(BankHeistDeterministic-v4), EnvSpec(BankHeistNoFrameskip-v0), EnvSpec(BankHeistNoFrameskip-v4), EnvSpec(BankHeist-ram-v0), EnvSpec(BankHeist-ram-v4), EnvSpec(BankHeist-ramDeterministic-v0), EnvSpec(BankHeist-ramDeterministic-v4), EnvSpec(BankHeist-ramNoFrameskip-v0), EnvSpec(BankHeist-ramNoFrameskip-v4), EnvSpec(BattleZone-v0), EnvSpec(BattleZone-v4), EnvSpec(BattleZoneDeterministic-v0), EnvSpec(BattleZoneDeterministic-v4), EnvSpec(BattleZoneNoFrameskip-v0), EnvSpec(BattleZoneNoFrameskip-v4), EnvSpec(BattleZone-ram-v0), EnvSpec(BattleZone-ram-v4), EnvSpec(BattleZone-ramDeterministic-v0), EnvSpec(BattleZone-ramDeterministic-v4), EnvSpec(BattleZone-ramNoFrameskip-v0), EnvSpec(BattleZone-ramNoFrameskip-v4), EnvSpec(BeamRider-v0), EnvSpec(BeamRider-v4), EnvSpec(BeamRiderDeterministic-v0), EnvSpec(BeamRiderDeterministic-v4), EnvSpec(BeamRiderNoFrameskip-v0), EnvSpec(BeamRiderNoFrameskip-v4), EnvSpec(BeamRider-ram-v0), EnvSpec(BeamRider-ram-v4), EnvSpec(BeamRider-ramDeterministic-v0), EnvSpec(BeamRider-ramDeterministic-v4), EnvSpec(BeamRider-ramNoFrameskip-v0), EnvSpec(BeamRider-ramNoFrameskip-v4), EnvSpec(Berzerk-v0), EnvSpec(Berzerk-v4), EnvSpec(BerzerkDeterministic-v0), EnvSpec(BerzerkDeterministic-v4), EnvSpec(BerzerkNoFrameskip-v0), EnvSpec(BerzerkNoFrameskip-v4), EnvSpec(Berzerk-ram-v0), EnvSpec(Berzerk-ram-v4), EnvSpec(Berzerk-ramDeterministic-v0), EnvSpec(Berzerk-ramDeterministic-v4), EnvSpec(Berzerk-ramNoFrameskip-v0), EnvSpec(Berzerk-ramNoFrameskip-v4), EnvSpec(Bowling-v0), EnvSpec(Bowling-v4), EnvSpec(BowlingDeterministic-v0), EnvSpec(BowlingDeterministic-v4), EnvSpec(BowlingNoFrameskip-v0), EnvSpec(BowlingNoFrameskip-v4), EnvSpec(Bowling-ram-v0), EnvSpec(Bowling-ram-v4), EnvSpec(Bowling-ramDeterministic-v0), EnvSpec(Bowling-ramDeterministic-v4), EnvSpec(Bowling-ramNoFrameskip-v0), EnvSpec(Bowling-ramNoFrameskip-v4), EnvSpec(Boxing-v0), EnvSpec(Boxing-v4), EnvSpec(BoxingDeterministic-v0), EnvSpec(BoxingDeterministic-v4), EnvSpec(BoxingNoFrameskip-v0), EnvSpec(BoxingNoFrameskip-v4), EnvSpec(Boxing-ram-v0), EnvSpec(Boxing-ram-v4), EnvSpec(Boxing-ramDeterministic-v0), EnvSpec(Boxing-ramDeterministic-v4), EnvSpec(Boxing-ramNoFrameskip-v0), EnvSpec(Boxing-ramNoFrameskip-v4), EnvSpec(Breakout-v0), EnvSpec(Breakout-v4), EnvSpec(BreakoutDeterministic-v0), EnvSpec(BreakoutDeterministic-v4), EnvSpec(BreakoutNoFrameskip-v0), EnvSpec(BreakoutNoFrameskip-v4), EnvSpec(Breakout-ram-v0), EnvSpec(Breakout-ram-v4), EnvSpec(Breakout-ramDeterministic-v0), EnvSpec(Breakout-ramDeterministic-v4), EnvSpec(Breakout-ramNoFrameskip-v0), EnvSpec(Breakout-ramNoFrameskip-v4), EnvSpec(Carnival-v0), EnvSpec(Carnival-v4), EnvSpec(CarnivalDeterministic-v0), EnvSpec(CarnivalDeterministic-v4), EnvSpec(CarnivalNoFrameskip-v0), EnvSpec(CarnivalNoFrameskip-v4), EnvSpec(Carnival-ram-v0), EnvSpec(Carnival-ram-v4), EnvSpec(Carnival-ramDeterministic-v0), EnvSpec(Carnival-ramDeterministic-v4), EnvSpec(Carnival-ramNoFrameskip-v0), EnvSpec(Carnival-ramNoFrameskip-v4), EnvSpec(Centipede-v0), EnvSpec(Centipede-v4), EnvSpec(CentipedeDeterministic-v0), EnvSpec(CentipedeDeterministic-v4), EnvSpec(CentipedeNoFrameskip-v0), EnvSpec(CentipedeNoFrameskip-v4), EnvSpec(Centipede-ram-v0), EnvSpec(Centipede-ram-v4), EnvSpec(Centipede-ramDeterministic-v0), EnvSpec(Centipede-ramDeterministic-v4), EnvSpec(Centipede-ramNoFrameskip-v0), EnvSpec(Centipede-ramNoFrameskip-v4), EnvSpec(ChopperCommand-v0), EnvSpec(ChopperCommand-v4), EnvSpec(ChopperCommandDeterministic-v0), EnvSpec(ChopperCommandDeterministic-v4), EnvSpec(ChopperCommandNoFrameskip-v0), EnvSpec(ChopperCommandNoFrameskip-v4), EnvSpec(ChopperCommand-ram-v0), EnvSpec(ChopperCommand-ram-v4), EnvSpec(ChopperCommand-ramDeterministic-v0), EnvSpec(ChopperCommand-ramDeterministic-v4), EnvSpec(ChopperCommand-ramNoFrameskip-v0), EnvSpec(ChopperCommand-ramNoFrameskip-v4), EnvSpec(CrazyClimber-v0), EnvSpec(CrazyClimber-v4), EnvSpec(CrazyClimberDeterministic-v0), EnvSpec(CrazyClimberDeterministic-v4), EnvSpec(CrazyClimberNoFrameskip-v0), EnvSpec(CrazyClimberNoFrameskip-v4), EnvSpec(CrazyClimber-ram-v0), EnvSpec(CrazyClimber-ram-v4), EnvSpec(CrazyClimber-ramDeterministic-v0), EnvSpec(CrazyClimber-ramDeterministic-v4), EnvSpec(CrazyClimber-ramNoFrameskip-v0), EnvSpec(CrazyClimber-ramNoFrameskip-v4), EnvSpec(DemonAttack-v0), EnvSpec(DemonAttack-v4), EnvSpec(DemonAttackDeterministic-v0), EnvSpec(DemonAttackDeterministic-v4), EnvSpec(DemonAttackNoFrameskip-v0), EnvSpec(DemonAttackNoFrameskip-v4), EnvSpec(DemonAttack-ram-v0), EnvSpec(DemonAttack-ram-v4), EnvSpec(DemonAttack-ramDeterministic-v0), EnvSpec(DemonAttack-ramDeterministic-v4), EnvSpec(DemonAttack-ramNoFrameskip-v0), EnvSpec(DemonAttack-ramNoFrameskip-v4), EnvSpec(DoubleDunk-v0), EnvSpec(DoubleDunk-v4), EnvSpec(DoubleDunkDeterministic-v0), EnvSpec(DoubleDunkDeterministic-v4), EnvSpec(DoubleDunkNoFrameskip-v0), EnvSpec(DoubleDunkNoFrameskip-v4), EnvSpec(DoubleDunk-ram-v0), EnvSpec(DoubleDunk-ram-v4), EnvSpec(DoubleDunk-ramDeterministic-v0), EnvSpec(DoubleDunk-ramDeterministic-v4), EnvSpec(DoubleDunk-ramNoFrameskip-v0), EnvSpec(DoubleDunk-ramNoFrameskip-v4), EnvSpec(ElevatorAction-v0), EnvSpec(ElevatorAction-v4), EnvSpec(ElevatorActionDeterministic-v0), EnvSpec(ElevatorActionDeterministic-v4), EnvSpec(ElevatorActionNoFrameskip-v0), EnvSpec(ElevatorActionNoFrameskip-v4), EnvSpec(ElevatorAction-ram-v0), EnvSpec(ElevatorAction-ram-v4), EnvSpec(ElevatorAction-ramDeterministic-v0), EnvSpec(ElevatorAction-ramDeterministic-v4), EnvSpec(ElevatorAction-ramNoFrameskip-v0), EnvSpec(ElevatorAction-ramNoFrameskip-v4), EnvSpec(Enduro-v0), EnvSpec(Enduro-v4), EnvSpec(EnduroDeterministic-v0), EnvSpec(EnduroDeterministic-v4), EnvSpec(EnduroNoFrameskip-v0), EnvSpec(EnduroNoFrameskip-v4), EnvSpec(Enduro-ram-v0), EnvSpec(Enduro-ram-v4), EnvSpec(Enduro-ramDeterministic-v0), EnvSpec(Enduro-ramDeterministic-v4), EnvSpec(Enduro-ramNoFrameskip-v0), EnvSpec(Enduro-ramNoFrameskip-v4), EnvSpec(FishingDerby-v0), EnvSpec(FishingDerby-v4), EnvSpec(FishingDerbyDeterministic-v0), EnvSpec(FishingDerbyDeterministic-v4), EnvSpec(FishingDerbyNoFrameskip-v0), EnvSpec(FishingDerbyNoFrameskip-v4), EnvSpec(FishingDerby-ram-v0), EnvSpec(FishingDerby-ram-v4), EnvSpec(FishingDerby-ramDeterministic-v0), EnvSpec(FishingDerby-ramDeterministic-v4), EnvSpec(FishingDerby-ramNoFrameskip-v0), EnvSpec(FishingDerby-ramNoFrameskip-v4), EnvSpec(Freeway-v0), EnvSpec(Freeway-v4), EnvSpec(FreewayDeterministic-v0), EnvSpec(FreewayDeterministic-v4), EnvSpec(FreewayNoFrameskip-v0), EnvSpec(FreewayNoFrameskip-v4), EnvSpec(Freeway-ram-v0), EnvSpec(Freeway-ram-v4), EnvSpec(Freeway-ramDeterministic-v0), EnvSpec(Freeway-ramDeterministic-v4), EnvSpec(Freeway-ramNoFrameskip-v0), EnvSpec(Freeway-ramNoFrameskip-v4), EnvSpec(Frostbite-v0), EnvSpec(Frostbite-v4), EnvSpec(FrostbiteDeterministic-v0), EnvSpec(FrostbiteDeterministic-v4), EnvSpec(FrostbiteNoFrameskip-v0), EnvSpec(FrostbiteNoFrameskip-v4), EnvSpec(Frostbite-ram-v0), EnvSpec(Frostbite-ram-v4), EnvSpec(Frostbite-ramDeterministic-v0), EnvSpec(Frostbite-ramDeterministic-v4), EnvSpec(Frostbite-ramNoFrameskip-v0), EnvSpec(Frostbite-ramNoFrameskip-v4), EnvSpec(Gopher-v0), EnvSpec(Gopher-v4), EnvSpec(GopherDeterministic-v0), EnvSpec(GopherDeterministic-v4), EnvSpec(GopherNoFrameskip-v0), EnvSpec(GopherNoFrameskip-v4), EnvSpec(Gopher-ram-v0), EnvSpec(Gopher-ram-v4), EnvSpec(Gopher-ramDeterministic-v0), EnvSpec(Gopher-ramDeterministic-v4), EnvSpec(Gopher-ramNoFrameskip-v0), EnvSpec(Gopher-ramNoFrameskip-v4), EnvSpec(Gravitar-v0), EnvSpec(Gravitar-v4), EnvSpec(GravitarDeterministic-v0), EnvSpec(GravitarDeterministic-v4), EnvSpec(GravitarNoFrameskip-v0), EnvSpec(GravitarNoFrameskip-v4), EnvSpec(Gravitar-ram-v0), EnvSpec(Gravitar-ram-v4), EnvSpec(Gravitar-ramDeterministic-v0), EnvSpec(Gravitar-ramDeterministic-v4), EnvSpec(Gravitar-ramNoFrameskip-v0), EnvSpec(Gravitar-ramNoFrameskip-v4), EnvSpec(Hero-v0), EnvSpec(Hero-v4), EnvSpec(HeroDeterministic-v0), EnvSpec(HeroDeterministic-v4), EnvSpec(HeroNoFrameskip-v0), EnvSpec(HeroNoFrameskip-v4), EnvSpec(Hero-ram-v0), EnvSpec(Hero-ram-v4), EnvSpec(Hero-ramDeterministic-v0), EnvSpec(Hero-ramDeterministic-v4), EnvSpec(Hero-ramNoFrameskip-v0), EnvSpec(Hero-ramNoFrameskip-v4), EnvSpec(IceHockey-v0), EnvSpec(IceHockey-v4), EnvSpec(IceHockeyDeterministic-v0), EnvSpec(IceHockeyDeterministic-v4), EnvSpec(IceHockeyNoFrameskip-v0), EnvSpec(IceHockeyNoFrameskip-v4), EnvSpec(IceHockey-ram-v0), EnvSpec(IceHockey-ram-v4), EnvSpec(IceHockey-ramDeterministic-v0), EnvSpec(IceHockey-ramDeterministic-v4), EnvSpec(IceHockey-ramNoFrameskip-v0), EnvSpec(IceHockey-ramNoFrameskip-v4), EnvSpec(Jamesbond-v0), EnvSpec(Jamesbond-v4), EnvSpec(JamesbondDeterministic-v0), EnvSpec(JamesbondDeterministic-v4), EnvSpec(JamesbondNoFrameskip-v0), EnvSpec(JamesbondNoFrameskip-v4), EnvSpec(Jamesbond-ram-v0), EnvSpec(Jamesbond-ram-v4), EnvSpec(Jamesbond-ramDeterministic-v0), EnvSpec(Jamesbond-ramDeterministic-v4), EnvSpec(Jamesbond-ramNoFrameskip-v0), EnvSpec(Jamesbond-ramNoFrameskip-v4), EnvSpec(JourneyEscape-v0), EnvSpec(JourneyEscape-v4), EnvSpec(JourneyEscapeDeterministic-v0), EnvSpec(JourneyEscapeDeterministic-v4), EnvSpec(JourneyEscapeNoFrameskip-v0), EnvSpec(JourneyEscapeNoFrameskip-v4), EnvSpec(JourneyEscape-ram-v0), EnvSpec(JourneyEscape-ram-v4), EnvSpec(JourneyEscape-ramDeterministic-v0), EnvSpec(JourneyEscape-ramDeterministic-v4), EnvSpec(JourneyEscape-ramNoFrameskip-v0), EnvSpec(JourneyEscape-ramNoFrameskip-v4), EnvSpec(Kangaroo-v0), EnvSpec(Kangaroo-v4), EnvSpec(KangarooDeterministic-v0), EnvSpec(KangarooDeterministic-v4), EnvSpec(KangarooNoFrameskip-v0), EnvSpec(KangarooNoFrameskip-v4), EnvSpec(Kangaroo-ram-v0), EnvSpec(Kangaroo-ram-v4), EnvSpec(Kangaroo-ramDeterministic-v0), EnvSpec(Kangaroo-ramDeterministic-v4), EnvSpec(Kangaroo-ramNoFrameskip-v0), EnvSpec(Kangaroo-ramNoFrameskip-v4), EnvSpec(Krull-v0), EnvSpec(Krull-v4), EnvSpec(KrullDeterministic-v0), EnvSpec(KrullDeterministic-v4), EnvSpec(KrullNoFrameskip-v0), EnvSpec(KrullNoFrameskip-v4), EnvSpec(Krull-ram-v0), EnvSpec(Krull-ram-v4), EnvSpec(Krull-ramDeterministic-v0), EnvSpec(Krull-ramDeterministic-v4), EnvSpec(Krull-ramNoFrameskip-v0), EnvSpec(Krull-ramNoFrameskip-v4), EnvSpec(KungFuMaster-v0), EnvSpec(KungFuMaster-v4), EnvSpec(KungFuMasterDeterministic-v0), EnvSpec(KungFuMasterDeterministic-v4), EnvSpec(KungFuMasterNoFrameskip-v0), EnvSpec(KungFuMasterNoFrameskip-v4), EnvSpec(KungFuMaster-ram-v0), EnvSpec(KungFuMaster-ram-v4), EnvSpec(KungFuMaster-ramDeterministic-v0), EnvSpec(KungFuMaster-ramDeterministic-v4), EnvSpec(KungFuMaster-ramNoFrameskip-v0), EnvSpec(KungFuMaster-ramNoFrameskip-v4), EnvSpec(MontezumaRevenge-v0), EnvSpec(MontezumaRevenge-v4), EnvSpec(MontezumaRevengeDeterministic-v0), EnvSpec(MontezumaRevengeDeterministic-v4), EnvSpec(MontezumaRevengeNoFrameskip-v0), EnvSpec(MontezumaRevengeNoFrameskip-v4), EnvSpec(MontezumaRevenge-ram-v0), EnvSpec(MontezumaRevenge-ram-v4), EnvSpec(MontezumaRevenge-ramDeterministic-v0), EnvSpec(MontezumaRevenge-ramDeterministic-v4), EnvSpec(MontezumaRevenge-ramNoFrameskip-v0), EnvSpec(MontezumaRevenge-ramNoFrameskip-v4), EnvSpec(MsPacman-v0), EnvSpec(MsPacman-v4), EnvSpec(MsPacmanDeterministic-v0), EnvSpec(MsPacmanDeterministic-v4), EnvSpec(MsPacmanNoFrameskip-v0), EnvSpec(MsPacmanNoFrameskip-v4), EnvSpec(MsPacman-ram-v0), EnvSpec(MsPacman-ram-v4), EnvSpec(MsPacman-ramDeterministic-v0), EnvSpec(MsPacman-ramDeterministic-v4), EnvSpec(MsPacman-ramNoFrameskip-v0), EnvSpec(MsPacman-ramNoFrameskip-v4), EnvSpec(NameThisGame-v0), EnvSpec(NameThisGame-v4), EnvSpec(NameThisGameDeterministic-v0), EnvSpec(NameThisGameDeterministic-v4), EnvSpec(NameThisGameNoFrameskip-v0), EnvSpec(NameThisGameNoFrameskip-v4), EnvSpec(NameThisGame-ram-v0), EnvSpec(NameThisGame-ram-v4), EnvSpec(NameThisGame-ramDeterministic-v0), EnvSpec(NameThisGame-ramDeterministic-v4), EnvSpec(NameThisGame-ramNoFrameskip-v0), EnvSpec(NameThisGame-ramNoFrameskip-v4), EnvSpec(Phoenix-v0), EnvSpec(Phoenix-v4), EnvSpec(PhoenixDeterministic-v0), EnvSpec(PhoenixDeterministic-v4), EnvSpec(PhoenixNoFrameskip-v0), EnvSpec(PhoenixNoFrameskip-v4), EnvSpec(Phoenix-ram-v0), EnvSpec(Phoenix-ram-v4), EnvSpec(Phoenix-ramDeterministic-v0), EnvSpec(Phoenix-ramDeterministic-v4), EnvSpec(Phoenix-ramNoFrameskip-v0), EnvSpec(Phoenix-ramNoFrameskip-v4), EnvSpec(Pitfall-v0), EnvSpec(Pitfall-v4), EnvSpec(PitfallDeterministic-v0), EnvSpec(PitfallDeterministic-v4), EnvSpec(PitfallNoFrameskip-v0), EnvSpec(PitfallNoFrameskip-v4), EnvSpec(Pitfall-ram-v0), EnvSpec(Pitfall-ram-v4), EnvSpec(Pitfall-ramDeterministic-v0), EnvSpec(Pitfall-ramDeterministic-v4), EnvSpec(Pitfall-ramNoFrameskip-v0), EnvSpec(Pitfall-ramNoFrameskip-v4), EnvSpec(Pong-v0), EnvSpec(Pong-v4), EnvSpec(PongDeterministic-v0), EnvSpec(PongDeterministic-v4), EnvSpec(PongNoFrameskip-v0), EnvSpec(PongNoFrameskip-v4), EnvSpec(Pong-ram-v0), EnvSpec(Pong-ram-v4), EnvSpec(Pong-ramDeterministic-v0), EnvSpec(Pong-ramDeterministic-v4), EnvSpec(Pong-ramNoFrameskip-v0), EnvSpec(Pong-ramNoFrameskip-v4), EnvSpec(Pooyan-v0), EnvSpec(Pooyan-v4), EnvSpec(PooyanDeterministic-v0), EnvSpec(PooyanDeterministic-v4), EnvSpec(PooyanNoFrameskip-v0), EnvSpec(PooyanNoFrameskip-v4), EnvSpec(Pooyan-ram-v0), EnvSpec(Pooyan-ram-v4), EnvSpec(Pooyan-ramDeterministic-v0), EnvSpec(Pooyan-ramDeterministic-v4), EnvSpec(Pooyan-ramNoFrameskip-v0), EnvSpec(Pooyan-ramNoFrameskip-v4), EnvSpec(PrivateEye-v0), EnvSpec(PrivateEye-v4), EnvSpec(PrivateEyeDeterministic-v0), EnvSpec(PrivateEyeDeterministic-v4), EnvSpec(PrivateEyeNoFrameskip-v0), EnvSpec(PrivateEyeNoFrameskip-v4), EnvSpec(PrivateEye-ram-v0), EnvSpec(PrivateEye-ram-v4), EnvSpec(PrivateEye-ramDeterministic-v0), EnvSpec(PrivateEye-ramDeterministic-v4), EnvSpec(PrivateEye-ramNoFrameskip-v0), EnvSpec(PrivateEye-ramNoFrameskip-v4), EnvSpec(Qbert-v0), EnvSpec(Qbert-v4), EnvSpec(QbertDeterministic-v0), EnvSpec(QbertDeterministic-v4), EnvSpec(QbertNoFrameskip-v0), EnvSpec(QbertNoFrameskip-v4), EnvSpec(Qbert-ram-v0), EnvSpec(Qbert-ram-v4), EnvSpec(Qbert-ramDeterministic-v0), EnvSpec(Qbert-ramDeterministic-v4), EnvSpec(Qbert-ramNoFrameskip-v0), EnvSpec(Qbert-ramNoFrameskip-v4), EnvSpec(Riverraid-v0), EnvSpec(Riverraid-v4), EnvSpec(RiverraidDeterministic-v0), EnvSpec(RiverraidDeterministic-v4), EnvSpec(RiverraidNoFrameskip-v0), EnvSpec(RiverraidNoFrameskip-v4), EnvSpec(Riverraid-ram-v0), EnvSpec(Riverraid-ram-v4), EnvSpec(Riverraid-ramDeterministic-v0), EnvSpec(Riverraid-ramDeterministic-v4), EnvSpec(Riverraid-ramNoFrameskip-v0), EnvSpec(Riverraid-ramNoFrameskip-v4), EnvSpec(RoadRunner-v0), EnvSpec(RoadRunner-v4), EnvSpec(RoadRunnerDeterministic-v0), EnvSpec(RoadRunnerDeterministic-v4), EnvSpec(RoadRunnerNoFrameskip-v0), EnvSpec(RoadRunnerNoFrameskip-v4), EnvSpec(RoadRunner-ram-v0), EnvSpec(RoadRunner-ram-v4), EnvSpec(RoadRunner-ramDeterministic-v0), EnvSpec(RoadRunner-ramDeterministic-v4), EnvSpec(RoadRunner-ramNoFrameskip-v0), EnvSpec(RoadRunner-ramNoFrameskip-v4), EnvSpec(Robotank-v0), EnvSpec(Robotank-v4), EnvSpec(RobotankDeterministic-v0), EnvSpec(RobotankDeterministic-v4), EnvSpec(RobotankNoFrameskip-v0), EnvSpec(RobotankNoFrameskip-v4), EnvSpec(Robotank-ram-v0), EnvSpec(Robotank-ram-v4), EnvSpec(Robotank-ramDeterministic-v0), EnvSpec(Robotank-ramDeterministic-v4), EnvSpec(Robotank-ramNoFrameskip-v0), EnvSpec(Robotank-ramNoFrameskip-v4), EnvSpec(Seaquest-v0), EnvSpec(Seaquest-v4), EnvSpec(SeaquestDeterministic-v0), EnvSpec(SeaquestDeterministic-v4), EnvSpec(SeaquestNoFrameskip-v0), EnvSpec(SeaquestNoFrameskip-v4), EnvSpec(Seaquest-ram-v0), EnvSpec(Seaquest-ram-v4), EnvSpec(Seaquest-ramDeterministic-v0), EnvSpec(Seaquest-ramDeterministic-v4), EnvSpec(Seaquest-ramNoFrameskip-v0), EnvSpec(Seaquest-ramNoFrameskip-v4), EnvSpec(Skiing-v0), EnvSpec(Skiing-v4), EnvSpec(SkiingDeterministic-v0), EnvSpec(SkiingDeterministic-v4), EnvSpec(SkiingNoFrameskip-v0), EnvSpec(SkiingNoFrameskip-v4), EnvSpec(Skiing-ram-v0), EnvSpec(Skiing-ram-v4), EnvSpec(Skiing-ramDeterministic-v0), EnvSpec(Skiing-ramDeterministic-v4), EnvSpec(Skiing-ramNoFrameskip-v0), EnvSpec(Skiing-ramNoFrameskip-v4), EnvSpec(Solaris-v0), EnvSpec(Solaris-v4), EnvSpec(SolarisDeterministic-v0), EnvSpec(SolarisDeterministic-v4), EnvSpec(SolarisNoFrameskip-v0), EnvSpec(SolarisNoFrameskip-v4), EnvSpec(Solaris-ram-v0), EnvSpec(Solaris-ram-v4), EnvSpec(Solaris-ramDeterministic-v0), EnvSpec(Solaris-ramDeterministic-v4), EnvSpec(Solaris-ramNoFrameskip-v0), EnvSpec(Solaris-ramNoFrameskip-v4), EnvSpec(SpaceInvaders-v0), EnvSpec(SpaceInvaders-v4), EnvSpec(SpaceInvadersDeterministic-v0), EnvSpec(SpaceInvadersDeterministic-v4), EnvSpec(SpaceInvadersNoFrameskip-v0), EnvSpec(SpaceInvadersNoFrameskip-v4), EnvSpec(SpaceInvaders-ram-v0), EnvSpec(SpaceInvaders-ram-v4), EnvSpec(SpaceInvaders-ramDeterministic-v0), EnvSpec(SpaceInvaders-ramDeterministic-v4), EnvSpec(SpaceInvaders-ramNoFrameskip-v0), EnvSpec(SpaceInvaders-ramNoFrameskip-v4), EnvSpec(StarGunner-v0), EnvSpec(StarGunner-v4), EnvSpec(StarGunnerDeterministic-v0), EnvSpec(StarGunnerDeterministic-v4), EnvSpec(StarGunnerNoFrameskip-v0), EnvSpec(StarGunnerNoFrameskip-v4), EnvSpec(StarGunner-ram-v0), EnvSpec(StarGunner-ram-v4), EnvSpec(StarGunner-ramDeterministic-v0), EnvSpec(StarGunner-ramDeterministic-v4), EnvSpec(StarGunner-ramNoFrameskip-v0), EnvSpec(StarGunner-ramNoFrameskip-v4), EnvSpec(Tennis-v0), EnvSpec(Tennis-v4), EnvSpec(TennisDeterministic-v0), EnvSpec(TennisDeterministic-v4), EnvSpec(TennisNoFrameskip-v0), EnvSpec(TennisNoFrameskip-v4), EnvSpec(Tennis-ram-v0), EnvSpec(Tennis-ram-v4), EnvSpec(Tennis-ramDeterministic-v0), EnvSpec(Tennis-ramDeterministic-v4), EnvSpec(Tennis-ramNoFrameskip-v0), EnvSpec(Tennis-ramNoFrameskip-v4), EnvSpec(TimePilot-v0), EnvSpec(TimePilot-v4), EnvSpec(TimePilotDeterministic-v0), EnvSpec(TimePilotDeterministic-v4), EnvSpec(TimePilotNoFrameskip-v0), EnvSpec(TimePilotNoFrameskip-v4), EnvSpec(TimePilot-ram-v0), EnvSpec(TimePilot-ram-v4), EnvSpec(TimePilot-ramDeterministic-v0), EnvSpec(TimePilot-ramDeterministic-v4), EnvSpec(TimePilot-ramNoFrameskip-v0), EnvSpec(TimePilot-ramNoFrameskip-v4), EnvSpec(Tutankham-v0), EnvSpec(Tutankham-v4), EnvSpec(TutankhamDeterministic-v0), EnvSpec(TutankhamDeterministic-v4), EnvSpec(TutankhamNoFrameskip-v0), EnvSpec(TutankhamNoFrameskip-v4), EnvSpec(Tutankham-ram-v0), EnvSpec(Tutankham-ram-v4), EnvSpec(Tutankham-ramDeterministic-v0), EnvSpec(Tutankham-ramDeterministic-v4), EnvSpec(Tutankham-ramNoFrameskip-v0), EnvSpec(Tutankham-ramNoFrameskip-v4), EnvSpec(UpNDown-v0), EnvSpec(UpNDown-v4), EnvSpec(UpNDownDeterministic-v0), EnvSpec(UpNDownDeterministic-v4), EnvSpec(UpNDownNoFrameskip-v0), EnvSpec(UpNDownNoFrameskip-v4), EnvSpec(UpNDown-ram-v0), EnvSpec(UpNDown-ram-v4), EnvSpec(UpNDown-ramDeterministic-v0), EnvSpec(UpNDown-ramDeterministic-v4), EnvSpec(UpNDown-ramNoFrameskip-v0), EnvSpec(UpNDown-ramNoFrameskip-v4), EnvSpec(Venture-v0), EnvSpec(Venture-v4), EnvSpec(VentureDeterministic-v0), EnvSpec(VentureDeterministic-v4), EnvSpec(VentureNoFrameskip-v0), EnvSpec(VentureNoFrameskip-v4), EnvSpec(Venture-ram-v0), EnvSpec(Venture-ram-v4), EnvSpec(Venture-ramDeterministic-v0), EnvSpec(Venture-ramDeterministic-v4), EnvSpec(Venture-ramNoFrameskip-v0), EnvSpec(Venture-ramNoFrameskip-v4), EnvSpec(VideoPinball-v0), EnvSpec(VideoPinball-v4), EnvSpec(VideoPinballDeterministic-v0), EnvSpec(VideoPinballDeterministic-v4), EnvSpec(VideoPinballNoFrameskip-v0), EnvSpec(VideoPinballNoFrameskip-v4), EnvSpec(VideoPinball-ram-v0), EnvSpec(VideoPinball-ram-v4), EnvSpec(VideoPinball-ramDeterministic-v0), EnvSpec(VideoPinball-ramDeterministic-v4), EnvSpec(VideoPinball-ramNoFrameskip-v0), EnvSpec(VideoPinball-ramNoFrameskip-v4), EnvSpec(WizardOfWor-v0), EnvSpec(WizardOfWor-v4), EnvSpec(WizardOfWorDeterministic-v0), EnvSpec(WizardOfWorDeterministic-v4), EnvSpec(WizardOfWorNoFrameskip-v0), EnvSpec(WizardOfWorNoFrameskip-v4), EnvSpec(WizardOfWor-ram-v0), EnvSpec(WizardOfWor-ram-v4), EnvSpec(WizardOfWor-ramDeterministic-v0), EnvSpec(WizardOfWor-ramDeterministic-v4), EnvSpec(WizardOfWor-ramNoFrameskip-v0), EnvSpec(WizardOfWor-ramNoFrameskip-v4), EnvSpec(YarsRevenge-v0), EnvSpec(YarsRevenge-v4), EnvSpec(YarsRevengeDeterministic-v0), EnvSpec(YarsRevengeDeterministic-v4), EnvSpec(YarsRevengeNoFrameskip-v0), EnvSpec(YarsRevengeNoFrameskip-v4), EnvSpec(YarsRevenge-ram-v0), EnvSpec(YarsRevenge-ram-v4), EnvSpec(YarsRevenge-ramDeterministic-v0), EnvSpec(YarsRevenge-ramDeterministic-v4), EnvSpec(YarsRevenge-ramNoFrameskip-v0), EnvSpec(YarsRevenge-ramNoFrameskip-v4), EnvSpec(Zaxxon-v0), EnvSpec(Zaxxon-v4), EnvSpec(ZaxxonDeterministic-v0), EnvSpec(ZaxxonDeterministic-v4), EnvSpec(ZaxxonNoFrameskip-v0), EnvSpec(ZaxxonNoFrameskip-v4), EnvSpec(Zaxxon-ram-v0), EnvSpec(Zaxxon-ram-v4), EnvSpec(Zaxxon-ramDeterministic-v0), EnvSpec(Zaxxon-ramDeterministic-v4), EnvSpec(Zaxxon-ramNoFrameskip-v0), EnvSpec(Zaxxon-ramNoFrameskip-v4), EnvSpec(CubeCrash-v0), EnvSpec(CubeCrashSparse-v0), EnvSpec(CubeCrashScreenBecomesBlack-v0), EnvSpec(MemorizeDigits-v0)])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "QA-998XefX85",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Unfortunatley, Gym does **not provide a bandit** environment so we need to import it, lets install one with the command below:"
      ]
    },
    {
      "metadata": {
        "id": "sGDaa_u8fjO3",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/JKCooper2/gym-bandits.git > /dev/null 2>&1\n",
        "!pip install ~/gym-bandits/. > /dev/null 2>&1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UX5HfOW7U3jS",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip install /content/gym-bandits/. > /dev/null 2>&1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "kogFWTLPhcjx",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "And import the bandit library too"
      ]
    },
    {
      "metadata": {
        "id": "ecG0Xn0fhfd9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import gym_bandits"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jEkYlxG7cwjC",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Unlike in the lecture, where there was only 2 bandits & we were trying to figure out on average which one paid out the most, this time around we are going to be dealing with **TEN** (10) bandits!!!"
      ]
    },
    {
      "metadata": {
        "id": "nz924RhYdlif",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Each bandit will have a payout with a normal distribution (bell curve), but the average payout or, centre of the distribution, will be different for each bandit, like in the image below"
      ]
    },
    {
      "metadata": {
        "id": "0NJTjv1xmi29",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "![alt text](https://i.stack.imgur.com/SazYv.png)"
      ]
    },
    {
      "metadata": {
        "id": "rDhrNHQLU2xu",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "."
      ]
    },
    {
      "metadata": {
        "id": "2e5lDZlheZuZ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "  >** >>>> It will be our goal to try & determine which bandit, out of the 10, pays out the most!!!!!!! <<<<**"
      ]
    },
    {
      "metadata": {
        "id": "HIFfNSFiU4NU",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "."
      ]
    },
    {
      "metadata": {
        "id": "6GA0H5L3UNDm",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Lets make a a variable called \"env\" to hold our freshly created 10 arm multi bandit environment"
      ]
    },
    {
      "metadata": {
        "id": "WJIh1OV_UMMT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "41e64b75-4234-48d3-bc93-c1296ce2c0c5"
      },
      "cell_type": "code",
      "source": [
        "#gaussian distribution is just another name for \"normal distribution\" or bell curve (so many different names for the same thing!)\n",
        "env = gym.make('BanditTenArmedGaussian-v0')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[33mWARN: Environment '<class 'gym_bandits.bandit.BanditTenArmedGaussian'>' has deprecated methods '_step' and '_reset' rather than 'step' and 'reset'. Compatibility code invoked. Set _gym_disable_underscore_compat = True to disable this behavior.\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "8E8LePgXTWF6",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "."
      ]
    },
    {
      "metadata": {
        "id": "tHDprdS4BI1Z",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "We are now going to go over a basic example of how OpenAi Gym works."
      ]
    },
    {
      "metadata": {
        "id": "me9kN1MPWEEf",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "run the code below & then we will explain what is going on here piece by piece"
      ]
    },
    {
      "metadata": {
        "id": "63v0tSoHbuwY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 527
        },
        "outputId": "790fb209-5469-4362-d5b0-864b03e960ef"
      },
      "cell_type": "code",
      "source": [
        "observation = env.reset()\n",
        "\n",
        "for i_episode in range(5):\n",
        "  \n",
        "    print(\"episode Number is\", i_episode)   \n",
        "    \n",
        "    action = env.action_space.sample() # sampling the \"action\" array which in this case only contains 10 \"options\" because there is 10 bandits\n",
        "        \n",
        "    print(\"action is\", action)\n",
        "        \n",
        "        \n",
        "    # here we taking the next \"step\" in our environment by taking in our action variable randomly selected above\n",
        "    observation, reward, done, info = env.step(action) \n",
        "        \n",
        "    print(\"observation space is: \",observation)\n",
        "    print(\"reward variable is: \",reward)\n",
        "    print(\"done flag is: \",done)\n",
        "    print(\"info variable is: \",info)\n",
        "        \n",
        "      \n",
        "            \n",
        "env.close()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "episode Number is 0\n",
            "action is 5\n",
            "observation space is:  0\n",
            "reward variable is:  -0.701962335225276\n",
            "done flag is:  True\n",
            "info variable is:  {}\n",
            "episode Number is 1\n",
            "action is 0\n",
            "observation space is:  0\n",
            "reward variable is:  0.8728194075449056\n",
            "done flag is:  True\n",
            "info variable is:  {}\n",
            "episode Number is 2\n",
            "action is 3\n",
            "observation space is:  0\n",
            "reward variable is:  -0.6928738246710773\n",
            "done flag is:  True\n",
            "info variable is:  {}\n",
            "episode Number is 3\n",
            "action is 3\n",
            "observation space is:  0\n",
            "reward variable is:  0.047301603126609804\n",
            "done flag is:  True\n",
            "info variable is:  {}\n",
            "episode Number is 4\n",
            "action is 7\n",
            "observation space is:  0\n",
            "reward variable is:  2.705668657249583\n",
            "done flag is:  True\n",
            "info variable is:  {}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "EHSzWhEmh1ha",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "."
      ]
    },
    {
      "metadata": {
        "id": "MgJlSiCGbMBP",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Reinforcement Learning** is an **extremely broad machine learning \"framework\"**, that looks like this:"
      ]
    },
    {
      "metadata": {
        "id": "T3aynRR6ZcOI",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "."
      ]
    },
    {
      "metadata": {
        "id": "HkYTLOA3aphJ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "![alt text](https://keon.io/images/deep-q-learning/rl.png)"
      ]
    },
    {
      "metadata": {
        "id": "HJmgkP_PjNsT",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "explaining that picture above, The RL framework goes something like this:"
      ]
    },
    {
      "metadata": {
        "id": "PZnKxZw_jWlo",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "1.   You have an **AGENT** (machine learning algorithm), it the image above the agent is a human brain, lol\n",
        "2.   The agent takes an **ACTION**, in the image above available actions are using the joystick, up, down, left, right and the red button. So five actions available (5) in total.\n",
        "3.   a single ACTION is chosen (from our available five from the joystick) and fed to our **ENVIRONMENT**, which in our example above is the Atari game environment\n",
        "4.   at this point our ENVIRONMENT measures how good the action taken was and produces a **REWARD** signal. a Postive number is usually good & negative number is usually bad. \n",
        "5.   the environment then produces an **OBSERVATION**, again using our example above, think of the OBSERVATION as the next graphical \"frame\" of the game. In RL, we also call observations **STATES**  \n",
        "6.   The new OBSERVATION & REWARD signal (produced by the old observation-action pair) is then fed back to AGENT for it to decide what move to make next and so on and so forth\n"
      ]
    },
    {
      "metadata": {
        "id": "uH3nZ60oVUi8",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "However, lets now take the above framework and see how it is implements on our example \"toy problem\" of the multi armed bandit"
      ]
    },
    {
      "metadata": {
        "id": "xuDzvJasVurw",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Taking the code apart piece by piece"
      ]
    },
    {
      "metadata": {
        "id": "HjHOjKpsVymi",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "we have already created our enviroment with this line of code:\n",
        "\n",
        "\n",
        "```\n",
        "env = gym.make('BanditTenArmedGaussian-v0')\n",
        "```\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "rDCrgRjLVndy",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "but, we need to ask the environment to produce the first **OBSERVATION** (or as we also call it - state) so that we can feed it to our **AGENT** (the RL algorithm) to decide what to do next. \n"
      ]
    },
    {
      "metadata": {
        "id": "aKrvnNKkX1V6",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "So next up we get the *first* OBSERVATION by calling the following code:\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "observation = env.reset()\n",
        "\n",
        "```\n"
      ]
    },
    {
      "metadata": {
        "id": "oxIQ6ZDEYL4N",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Next up, we create a 'for loop' that looped 5 times. In the Multi armed banded scenario an \"episode\" is just a single \"play\" of the game. So think of the For loop then as playing the multi armed bandit game 5 times.\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "for i_episode in range(5):\n",
        "```\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "CceslAxIZBst",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "each **ENVIRONMENT** is different. That is to say, each environment gives us different **ACTIONS** that are available, different **OBSERVATIONS** that are available etc"
      ]
    },
    {
      "metadata": {
        "id": "GFzJTTmFaTeu",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "For the *10 armed bandit problem,* we should have *ten actions available *to us (as there should be 10 different slot machines, each with a different lever to pull)"
      ]
    },
    {
      "metadata": {
        "id": "IXRhRNz6aqxC",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "We can confirm this is the case by running the code below:"
      ]
    },
    {
      "metadata": {
        "id": "GMBt0CDuaqGs",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c718e326-1781-48ae-aaba-324b269c468a"
      },
      "cell_type": "code",
      "source": [
        "print(env.action_space)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Discrete(10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "VeGThbjxbje3",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "So taking this back to our example code previous, the next thing we do is take an action,* by randomly sampling from the* **ACTION SPACE**. Again, dont worry. The ACTION SPACE is just a number assigned to each our bandits. EG [0,1,2,3,4,5,6,7,8,9]. So for our example if we randomly sample the ACTION SPACE and get back the number 8 - all this means is that we will be pulling the \"lever\" on bandit 8."
      ]
    },
    {
      "metadata": {
        "id": "uM1J8_n4cbsT",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "It is also important to note that at this point - we have not implemented any machine learning yet. We are only choosing actions at random. So lets choose one then. This was done with this line of code:\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "action = env.action_space.sample()\n",
        "```\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "H8v64MX8dKcU",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "We have now completed step 2 of our Reinforcement Learning framework discussed earlier. Lets now do step 3, feeding the action into the environment, this was achieved with this line of code\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "observation, reward, done, info = env.step(action) \n",
        "```\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "r_j80CP2d3Va",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "The above line of code is actually doing a couple of things. We are feeding in our selected action with this line:\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "env.step(action) \n",
        "```\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "AwQ6JLupeHfY",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "And getting back 4 new variables in return , in this part of the code:\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "observation, reward, done, info = \n",
        "```\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "JeAZp6rMeUfP",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "For now, we do not have to worry about the DONE and INFO variables. All we care about, in this tutorial is the OBSERVATION & REWARD variables. Also note that in this single line of code we have achieved step 4 & 5 of our RL framework discussed earlier in one go! "
      ]
    },
    {
      "metadata": {
        "id": "nAC11E-Jfl7a",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Where we now breakaway from the RL framework, is that, we are not feeding the **REWARD** & **OBSERVATION** variables back to any **AGENT** (aka step 6 from the framework) to do anything intelligent yet. This is because we yet to create an **AGENT**!!\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "S1pGRQ0Zk5na",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## The final piece of the puzzle: the Reward-Averaging Sampling “learning rule”"
      ]
    },
    {
      "metadata": {
        "id": "Fcyht8exfgAU",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "In the lecture we learned about the epsilon-greedy agent. It is a synthesis of a purely exploratory agent and a completely greedy agent.\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "EHDa_E8TdLyd",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "\n",
        "In the multiarmed bandit problem a **purely exploratory agent** would sample all the different bandits (or options) available uniformily - building up a distribution of each. However this has the downside such that the agent never gets to use its knowledge of the best option discovered so far. **A purely greedy agent  ** however would choose a bandit and stick to it's choice for all eternity - it will not try other agents to see if they provide better long term reward & can be thought of as being very \"narrow-minded\".\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "tHEEPHKpfj9Z",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "To get the best of both worlds, the Epsilon-greedy agent is designed to ***explore*** at an ***Epsilon Chance*** (let say 10%) whilst the rest of the time (90%) it goes ***greedy*** on the best option it had discovered so far."
      ]
    },
    {
      "metadata": {
        "id": "q9EelHF8f0Fr",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "The idea here is that the Greedy mechanism helps the agent exploit the best option it has discovered so far, whilst the small amount of exploration leftover ensures that our agent keeps searching the other options available - to prove that there are not *even better options out there.*"
      ]
    },
    {
      "metadata": {
        "id": "ux6cmmhlh5js",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "The last remaining piece of the puzzle is, how do we assign the idea of \"Value\", in the epsilon-greedy algorithm, to each of our bandits."
      ]
    },
    {
      "metadata": {
        "id": "F1n6kNFrxyMN",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Given some state S1 and an action A1, we can define a mathematical function to define the long term *expected* reward for this state action pair (state-action pairs are called \"Q values\", but we will get to this in a sec)"
      ]
    },
    {
      "metadata": {
        "id": "djdNT6huyjCs",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "So the *expected* Q value (or average over time) for any one bandit (or choice), can be given by:"
      ]
    },
    {
      "metadata": {
        "id": "hl9CLLxP23NP",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "![alt text](https://image.ibb.co/evACsU/qforumula.png)"
      ]
    },
    {
      "metadata": {
        "id": "UpHON3EF3swc",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Where:\n",
        "\n",
        "\n",
        "\n",
        "*  **Q** stands for **Quality**, this is an arbitary naming convention\n",
        "*  **n** stands for ***the number of times that bandit (or state) has been visited***\n",
        "* **r** is reward from visiting a bandit, n many times\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "iOc9DRsV4rqf",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "So for example, lets say we visit bandit #1, 3 times. \n",
        "\n",
        "This means that we get three different rewards r1, r2 & r3. Because we visited bandit#1 3 times we also know that n = 3"
      ]
    },
    {
      "metadata": {
        "id": "BsoQUW0B5jXE",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Therefore: \n",
        "\n",
        "Q = (1/n)*(r1+r2+r3)"
      ]
    },
    {
      "metadata": {
        "id": "nbTr1MW66D2k",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Q = 1/3*(r1+r2+r3)"
      ]
    },
    {
      "metadata": {
        "id": "8-1VDX-l6WbC",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "This whole process also has another name in life - we also call it , drumroll, \"the average\". However how the idea of average is used in the context here- is that we are only working out the average **for a single bandit**. \n",
        "\n",
        "In order to work out the average for another bandit, we will have to keep track of all of its reward variables & number of times it has been visited seperately"
      ]
    },
    {
      "metadata": {
        "id": "y6eUyp198V4g",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "In a real life multi armed bandit scenario, we would have to keep track of many reward variables the longer our agent is run for. So in order to reduce the amount of computer memory that is in use, we can use some mathematical trickery to compress the above equation, such that we update our Q value with every visit to the bandit."
      ]
    },
    {
      "metadata": {
        "id": "LqZxc8wr85rs",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "the formula then becomes:"
      ]
    },
    {
      "metadata": {
        "id": "oofNiNaj_Hu4",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "![alt text](https://image.ibb.co/nCPdhU/qforumulaupdated.png)"
      ]
    },
    {
      "metadata": {
        "id": "HxPT7op3ci17",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "this then is the heart of the epsilon greedy algorithm, or the forumula we need in order to update our Q table is pseudocode, **you have to use this code up in the upcoming exercise to make the agent work.**\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "hxNb0VgPc8yX",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "```\n",
        "NewQvalue = OldQvalue + ((reward - OldQvalue)/numberOfTimesLeverHasBeenPulledForThisBandit)\n",
        "```\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "b-VohcmEjx2p",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Exercise 1: creating your first (very simple) Agent"
      ]
    },
    {
      "metadata": {
        "id": "YRLF3R_Rj--G",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "We are now ready to create your very first agent, the epsilon Greedy algorithm"
      ]
    },
    {
      "metadata": {
        "id": "2CsZVD-KkKmx",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "We will need to keep track of which agent is the best, we do this by creating a big table, with 10 cells, one for each bandit. In computer terms this table is known as an array. We are creating this array to keep track of which bandit is doing the best for every time we play a game. For this exercise we will be using a powerful python array library known as *numpy*, so lets import that."
      ]
    },
    {
      "metadata": {
        "id": "1YbALrYCktJz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "LuXsMBMTFosy",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "we also want to randomly initialise our enviroment, do this by running the code below:"
      ]
    },
    {
      "metadata": {
        "id": "5cc_prTsFyhP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "63935b4d-891d-492e-feb0-248617f8881a"
      },
      "cell_type": "code",
      "source": [
        "#rerun this part of the code if you would like to \"reset\" or reinitialize your bandit environement\n",
        "env.seed(34)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[34]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "metadata": {
        "id": "i10ZSUORnzus",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Let also make a variable that is the total number of bandits operating in our environment, **complete the code below:**"
      ]
    },
    {
      "metadata": {
        "id": "1ASjY3yNn8BG",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "numberOfBandits = 10"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hZTZsSATlJNw",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "we are going to call the array (remember this is just a table) which keeps track of which bandit is the best, a **Q TABLE**. "
      ]
    },
    {
      "metadata": {
        "id": "EFeQkqeTmW64",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Q in this case just stands for quality. The idea being that the number that is the highest in our table, is associated with the action we would like to take with the highest quality. but we are getting a little ahead of ourselves. If you did not get that dont worry. It will all become apparent in a couple of lines of reading!"
      ]
    },
    {
      "metadata": {
        "id": "iMBmGTs8nZ3M",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "we also want to initialize our table so that all of the values are zero at the start, so lets do that"
      ]
    },
    {
      "metadata": {
        "id": "Zl4fw0YRnm2a",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "q_table = np.zeros(numberOfBandits)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Wadn4RHooEz3",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Remember, we would like to keep track of the bandit with the *HIGHEST AVERAGE PAYOUT*. In order to do this we need another table to keep track of the number of times each bandit has been \"pulled\". We also want this table to be fulled with ones, this is to stop a divide by zero error later. **complete the code below:**"
      ]
    },
    {
      "metadata": {
        "id": "0VfVlrZEoWdi",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "n_table = np.ones(numberOfBandits)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "vgYR_56KpJ-_",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "In the lecture we talked a lot about epsilon. Lets create a new variable called \"epsilon\" and initialize it to 0.9. **complete the code below:**"
      ]
    },
    {
      "metadata": {
        "id": "yLi2bsWCpf65",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "epsilon = 0.9"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "T4cT-7cPtFQj",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Below is the pseudo code for the epsilon greedy algorithm. In this implementation we are not going to vary epsilon - it is going to be a fixed number. \n",
        "\n",
        "This means that we will keep our exploration rate fixed."
      ]
    },
    {
      "metadata": {
        "id": "7OaxMbsU37Sc",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "We will now give you the pseudo code (code recipe) on how to implement the epsilon greedy algorithm. Look below at the \"recipe\". If you need help, all the ingredients on how to implement each part are below in the pseudo code."
      ]
    },
    {
      "metadata": {
        "id": "BaKRJx3Ft8US",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "```\n",
        "create a for loop, to loop 1000 times\n",
        "\n",
        "      (IF STATEMENT, inside the loop,) generate a random number between 0 and 1 , if this number is less than \n",
        "      Epsilon enter \"exploitation mode\" aka use the best bandit we have discovered so far\n",
        "      \n",
        "            (inside the if statement) get the POSITION (index) in our array of the current max value within our\n",
        "            table, this index is the bandit that is giving the best payout so far.\n",
        "            \n",
        "            (inside the if statement) set your ACTION variable equal to the index we discovered in the last \n",
        "            statement\n",
        "            \n",
        "      (ELSE, otherwise..) if the number is greater than or equal to Epsilon, go into \"exploration mode\" and\n",
        "      choose a bandit at random \n",
        "      \n",
        "          \n",
        "            (inside the if statement) generate a random number between 0 and THE_NUMBER_OF_BANDITS\n",
        "            \n",
        "            (inside the if statement) set your ACTION variable to equal to the random number we just generated.\n",
        "            \n",
        "        \n",
        "            \n",
        "      (inside the loop) feed our ACTION variable into our environment by updating it with a step generated \n",
        "      by either of the steps above\n",
        "      \n",
        "      (inside the loop) now that we have gained some new information from our environment we want to update our \n",
        "      Q_table. We do this using the formula: Q_n+1 = Q_n + (R - Q_n)/n or in simpler english:\n",
        "      \n",
        "      NewQvalue = OldQvalue + ((reward - OldQvalue)/numberOfTimesLeverHasBeenPulledForThisBandit)\n",
        "      \n",
        "      Lets think about the intituition of what this forumula is doing. Implement the formula in code.\n",
        "      \n",
        "      \n",
        "      (inside the loop) now that we have updated our Q table, we also need to update the table that is keeping\n",
        "      track of how many times each bandit's lever has been pulled. Do this by adding +1 in the position\n",
        "      of our currently selected bandit in the N_TABLE array\n",
        "      \n",
        "      \n",
        "(OUTSIDE the loop) once everything is done, we would like to print the Bandit with the highest score! Using\n",
        "a print statement, and numpy's argmax function, using our Q table, print the bandit with the highest\n",
        "AVERAGE payout\n",
        "      \n",
        "   \n",
        "   \n",
        "  \n",
        "```\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "4hqS1uXaUgX7",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "."
      ]
    },
    {
      "metadata": {
        "id": "FnAfeNYWUBod",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Here are all the pieces required to build the above:**"
      ]
    },
    {
      "metadata": {
        "id": "vBbCYa3QUhrc",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "How to make a loop that loops a 10 times:"
      ]
    },
    {
      "metadata": {
        "id": "KKGvYZUmUl1S",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "32655493-0e16-42d2-b3d4-1374a9e4019f"
      },
      "cell_type": "code",
      "source": [
        "#notice the indentation of the print statement, this indicates that this function (the print statement), is \"inside\" the loop\n",
        "\n",
        "for k in range(10):\n",
        "  print(\"now in loop iteration number: \",k)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "now in loop iteration number:  0\n",
            "now in loop iteration number:  1\n",
            "now in loop iteration number:  2\n",
            "now in loop iteration number:  3\n",
            "now in loop iteration number:  4\n",
            "now in loop iteration number:  5\n",
            "now in loop iteration number:  6\n",
            "now in loop iteration number:  7\n",
            "now in loop iteration number:  8\n",
            "now in loop iteration number:  9\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "XAU8WC0bUzs-",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "How to generate a random number between 0 & 1 using numpy:"
      ]
    },
    {
      "metadata": {
        "id": "E2jW4cMFU9xt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "05be6c0a-4312-48b7-f544-ba73e89acf6a"
      },
      "cell_type": "code",
      "source": [
        "#rerun this multiple times to generate a different random number\n",
        "#remember np stands for \"numpy\" as declared above\n",
        "\n",
        "randomNumber = np.random.random(1)[0] \n",
        "\n",
        "print(\"random number is \", randomNumber)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "random number is  0.770958096023186\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "n28zjNfQVXj3",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "How to create an if statement to see if our random number is less than epsilon"
      ]
    },
    {
      "metadata": {
        "id": "elf3xvsVWcbW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ee4fe448-7f42-4c1b-e6b7-b3f6de9af0ad"
      },
      "cell_type": "code",
      "source": [
        "#epsilon should be set to 0.9 above...\n",
        "\n",
        "if(randomNumber < epsilon):\n",
        "  print(\"doing something less than epsilon\")"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "doing something less than epsilon\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "-ka0VSYXWu8l",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "How to create an if statement to see if our random number is less than epsilon and do something else if this is not the case"
      ]
    },
    {
      "metadata": {
        "id": "ab6kK3F0Ws3J",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b46493ce-a52b-4a55-9d77-3d4dbdcf3964"
      },
      "cell_type": "code",
      "source": [
        "if(randomNumber < epsilon):\n",
        "  print(\"doing something less than epsilon\")\n",
        "else:\n",
        "  print(\"doing something more than epsilon\")"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "doing something less than epsilon\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "oYAlpVatXhwL",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "How to lookup the best bandit's *location* (index) in our Q table, again remember there are 10 values to choose from (10 bandits) & we need to look up the index (location) in our Q_table array of the bandit with the highest score. Numpy's argmax() function allows us to do this.\n"
      ]
    },
    {
      "metadata": {
        "id": "rXuQ87hkXlpS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7db8745c-a883-4675-c9ca-190f4f825753"
      },
      "cell_type": "code",
      "source": [
        "best_bandit = np.argmax(q_table)\n",
        "\n",
        "print(\"best bandit is \",best_bandit)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "best bandit is  0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "hxbGaOG_YROy",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "how to assign the index of our best action so that OpenAi gym can understand it:"
      ]
    },
    {
      "metadata": {
        "id": "IzD6aQHRYatW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "action = best_bandit"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "BYE8HoMPYmHX",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "how to generate a random bandit ACTION number between 0 to 9 to take:"
      ]
    },
    {
      "metadata": {
        "id": "nds0qfLCY3d4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "outputId": "3672011c-2b6f-410c-ee43-926c4ecc601d"
      },
      "cell_type": "code",
      "source": [
        "#run this multiple times to see different results\n",
        "random_number = np.random.random_integers(numberOfBandits-1)\n",
        "\n",
        "print(random_number)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "9\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: DeprecationWarning: This function is deprecated. Please call randint(1, 9 + 1) instead\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "4A6eKeMfY7MG",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "how to update OpenAi gym one time step into the future"
      ]
    },
    {
      "metadata": {
        "id": "vRPq3EN-ZB08",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "observation, reward, done, info = env.step(action) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-PQn_PFOZP_Q",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "How to directly access a numpy array's value, in this example we are accessing the best bandit's so far payout"
      ]
    },
    {
      "metadata": {
        "id": "JQ8CQpAlZVX2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "185f3d80-81fa-465d-a9b9-49e54c2be712"
      },
      "cell_type": "code",
      "source": [
        "example_table = np.array([2,3,1,0,5,7,9,8,6,4])\n",
        "\n",
        "example_table[best_bandit]"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "metadata": {
        "id": "YvrXwE05fZMc",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "To print out all the values of a numpy array for testing purposes, use this code"
      ]
    },
    {
      "metadata": {
        "id": "eiyUZQggff5e",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b4312075-c373-4927-a2af-5cf0b9dd396d"
      },
      "cell_type": "code",
      "source": [
        "print(example_table[:])"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2 3 1 0 5 7 9 8 6 4]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "ie_RqC6LdCAQ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "after looping 1000 times we would like to present our result - the best bandit out of the 10. Using Numpys argmax function:"
      ]
    },
    {
      "metadata": {
        "id": "mvp8I-XBdfKv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "42a9560a-4911-4903-f87b-7aa8cda49ef7"
      },
      "cell_type": "code",
      "source": [
        "# think about the result after running this relative to our example table above:\n",
        "\n",
        "print('and the best bandit is....', np.argmax(example_table))"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "and the best bandit is.... 6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "g9KqH2ScdvwI",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Code up Epsilon-greedy below"
      ]
    },
    {
      "metadata": {
        "id": "kmH313czeI0u",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "We now have everything required to code up epsilon greedy. Using our **recipe**, and all the **blocks above,** code up an implementation of epsilon greedy below:"
      ]
    },
    {
      "metadata": {
        "id": "LAIuudWtc-Sx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "953b2eb4-1858-4e98-906f-9436f95aba28"
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "env.reset()\n",
        "env.seed(35)\n",
        "\n",
        "number_of_bandits = 10\n",
        "# Rewards given for each bandit. Keeps track of which bandit is\n",
        "# the best.\n",
        "q_table = np.zeros(number_of_bandits)\n",
        "# Values for the number of times a bandit has been visited.\n",
        "n_table = np.ones(number_of_bandits)\n",
        "\n",
        "EPSILON = 0.9\n",
        "\n",
        "## YOUR CODE GOES HERE\n",
        "for i in range(1000):\n",
        "    random_number = np.random.random(1)[0]\n",
        "    if random_number < EPSILON:\n",
        "        # Enter \"exploitation mode\" aka use the best bandit we have\n",
        "        # discovered so far\n",
        "        action = np.argmax(q_table)\n",
        "    else:\n",
        "        # Go into \"exploration mode\" and choose a bandit at random\n",
        "        # action = np.random.random_integers(number_of_bandits - 1)\n",
        "        action = np.random.randint(0, high=number_of_bandits)\n",
        "    \n",
        "    _observation, reward, _done, _info = env.step(action)\n",
        "    \n",
        "    old_q_value = q_table[action]\n",
        "    # NewQvalue = OldQvalue +\n",
        "    # ((reward - OldQvalue)/numberOfTimesLeverHasBeenPulledForThisBandit)\n",
        "    new_q_value = old_q_value + ((reward - old_q_value) / n_table[action])\n",
        "    # Update Q Table with new value\n",
        "    q_table[action] = new_q_value\n",
        "    # Increment number of times this bandit has been pulled\n",
        "    n_table[action] += 1\n",
        "    \n",
        "\n",
        "best_bandit = np.argmax(q_table)\n",
        "print('The best bandit is....', best_bandit)\n",
        "print('It has the highest average payout of ', q_table[best_bandit])\n",
        "print('This bandit has been pulled ', n_table[best_bandit], ' times')\n",
        "env.close()\n",
        "      "
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The best bandit is.... 7\n",
            "It has the highest average payout of  1.8400469054269197\n",
            "This bandit has been pulled  861.0  times\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}