DQN Results.ipynb

{
 "metadata": {
  "language": "lua",
  "name": "",
  "signature": "sha256:dafb4ff1aeafbbaa3831e97595d6197ab0213361fb75cb9f4476fcb02f5f698a"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "require \"dqn\";\n",
      "require 'cunn';\n",
      "plot = require 'itorch.Plot'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "if agent then\n",
      "    agent = nil\n",
      "end\n",
      "collectgarbage()\n",
      "collectgarbage()\n",
      "agent = torch.load('Itorch_DQN3_0_1_pong_FULL_Y.t7')\n",
      "--print(agent)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "function plot_results(field)\n",
      "    if not agent or not agent[field] then\n",
      "        print('Agent or field not found ', field)\n",
      "        print(agent)\n",
      "        return\n",
      "    end\n",
      "    local x = agent[field]\n",
      "    if #x == 1 then x[2] = x[1] end\n",
      "    plot():line(torch.range(1,#x), torch.Tensor(x)):title(field):draw();\n",
      "end\n",
      "plot_results('reward_history')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<script type=\"text/javascript\">\n",
        "$(function() {\n",
        "    if (typeof (window._bokeh_onload_callbacks) === \"undefined\"){\n",
        "  window._bokeh_onload_callbacks = [];\n",
        "    }\n",
        "    function load_lib(url, callback){\n",
        "  window._bokeh_onload_callbacks.push(callback);\n",
        "  if (window._bokeh_is_loading){\n",
        "      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", new Date());\n",
        "      return null;\n",
        "  }\n",
        "  console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", new Date());\n",
        "  window._bokeh_is_loading = true;\n",
        "  var s = document.createElement('script');\n",
        "  s.src = url;\n",
        "  s.async = true;\n",
        "  s.onreadystatechange = s.onload = function(){\n",
        "      Bokeh.embed.inject_css(\"http://cdn.pydata.org/bokeh-0.7.0.min.css\");\n",
        "      window._bokeh_onload_callbacks.forEach(function(callback){callback()});\n",
        "  };\n",
        "  s.onerror = function(){\n",
        "      console.warn(\"failed to load library \" + url);\n",
        "  };\n",
        "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
        "    }\n",
        "\n",
        "    bokehjs_url = \"http://cdn.pydata.org/bokeh-0.7.0.min.js\"\n",
        "\n",
        "    var elt = document.getElementById(\"005b3e91-330a-4b54-cbd5-09603e8fd2c5\");\n",
        "    if(elt==null) {\n",
        "  console.log(\"Bokeh: ERROR: autoload.js configured with elementid '005b3e91-330a-4b54-cbd5-09603e8fd2c5'\"\n",
        "        + \"but no matching script tag was found. \")\n",
        "  return false;\n",
        "    }\n",
        "\n",
        "    if(typeof(Bokeh) !== \"undefined\") {\n",
        "  console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
        "  var modelid = \"576cd5e0-b854-47c7-c7aa-6667e860a3bc\";\n",
        "  var modeltype = \"Plot\";\n",
        "  var all_models = [{\"id\":\"3fa8ca6a-0825-414d-c166-dadcf7638543\",\"type\":\"ColumnDataSource\",\"attributes\":{\"data\":{\"y\":[-21,-21],\"x\":[1,2]},\"column_names\":[\"y\",\"x\"],\"cont_ranges\":{},\"discrete_ranges\":{},\"selected\":[],\"id\":\"3fa8ca6a-0825-414d-c166-dadcf7638543\",\"doc\":null,\"tags\":[]}},{\"id\":\"e2d133b7-7f3c-409d-c09f-134ef0db5ace\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"red\"},\"line_color\":{\"value\":\"red\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"e2d133b7-7f3c-409d-c09f-134ef0db5ace\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"e6861527-5d16-4a3c-ca77-74299ba4e55f\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"red\"},\"line_color\":{\"value\":\"red\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"e6861527-5d16-4a3c-ca77-74299ba4e55f\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"8dcd2416-16f4-4dd1-cd0f-ec1a0a229ed9\",\"type\":\"GlyphRenderer\",\"attributes\":{\"name\":null,\"nonselection_glyph\":{\"type\":\"Line\",\"id\":\"e6861527-5d16-4a3c-ca77-74299ba4e55f\"},\"doc\":null,\"server_data_source\":null,\"data_source\":{\"type\":\"ColumnDataSource\",\"id\":\"3fa8ca6a-0825-414d-c166-dadcf7638543\"},\"glyph\":{\"type\":\"Line\",\"id\":\"e2d133b7-7f3c-409d-c09f-134ef0db5ace\"},\"selection_glyph\":null,\"id\":\"8dcd2416-16f4-4dd1-cd0f-ec1a0a229ed9\",\"tags\":[]}},{\"id\":\"c7e78af2-5429-4dab-cd62-44103d0384dd\",\"type\":\"DataRange1d\",\"attributes\":{\"sources\":[{\"columns\":[\"x\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"3fa8ca6a-0825-414d-c166-dadcf7638543\"}}],\"id\":\"c7e78af2-5429-4dab-cd62-44103d0384dd\",\"tags\":[],\"doc\":null}},{\"id\":\"cd4dc240-48a7-486b-c8d3-f41d24013f2d\",\"type\":\"DataRange1d\",\"attributes\":{\"sources\":[{\"columns\":[\"y\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"3fa8ca6a-0825-414d-c166-dadcf7638543\"}}],\"id\":\"cd4dc240-48a7-486b-c8d3-f41d24013f2d\",\"tags\":[],\"doc\":null}},{\"id\":\"39eadd4a-b76a-4a44-caec-ddddaeaf56de\",\"type\":\"ToolEvents\",\"attributes\":{\"tags\":[],\"id\":\"39eadd4a-b76a-4a44-caec-ddddaeaf56de\",\"geometries\":[],\"doc\":null}},{\"id\":\"d9280e83-33bc-4c07-cc82-d5f4e65c802c\",\"type\":\"BasicTickFormatter\",\"attributes\":{\"id\":\"d9280e83-33bc-4c07-cc82-d5f4e65c802c\",\"tags\":[],\"doc\":null}},{\"id\":\"3822fa51-ca21-41f1-c105-3bd1aaed4f4b\",\"type\":\"BasicTicker\",\"attributes\":{\"num_minor_ticks\":5,\"id\":\"3822fa51-ca21-41f1-c105-3bd1aaed4f4b\",\"tags\":[],\"doc\":null}},{\"id\":\"8f197c3f-2586-4ae7-ca67-5ee7f74f9199\",\"type\":\"LinearAxis\",\"attributes\":{\"formatter\":{\"type\":\"BasicTickFormatter\",\"id\":\"d9280e83-33bc-4c07-cc82-d5f4e65c802c\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"3822fa51-ca21-41f1-c105-3bd1aaed4f4b\"},\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"axis_label\":null,\"id\":\"8f197c3f-2586-4ae7-ca67-5ee7f74f9199\",\"doc\":null,\"tags\":[]}},{\"id\":\"ef639ed0-79d7-47b1-c7a9-72b14a17c566\",\"type\":\"Grid\",\"attributes\":{\"dimension\":0,\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"3822fa51-ca21-41f1-c105-3bd1aaed4f4b\"},\"id\":\"ef639ed0-79d7-47b1-c7a9-72b14a17c566\",\"doc\":null,\"tags\":[]}},{\"id\":\"ed206cdd-6216-481c-c8ed-8d5f08a3edea\",\"type\":\"BasicTickFormatter\",\"attributes\":{\"id\":\"ed206cdd-6216-481c-c8ed-8d5f08a3edea\",\"tags\":[],\"doc\":null}},{\"id\":\"2dde2aef-94cf-41bf-c1b4-dc9425309739\",\"type\":\"BasicTicker\",\"attributes\":{\"num_minor_ticks\":5,\"id\":\"2dde2aef-94cf-41bf-c1b4-dc9425309739\",\"tags\":[],\"doc\":null}},{\"id\":\"9a734023-b12c-4ddc-cd3c-b21bbce40eea\",\"type\":\"LinearAxis\",\"attributes\":{\"formatter\":{\"type\":\"BasicTickFormatter\",\"id\":\"ed206cdd-6216-481c-c8ed-8d5f08a3edea\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"2dde2aef-94cf-41bf-c1b4-dc9425309739\"},\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"axis_label\":null,\"id\":\"9a734023-b12c-4ddc-cd3c-b21bbce40eea\",\"doc\":null,\"tags\":[]}},{\"id\":\"c5d8de29-a066-4f43-cfdb-85bd3a586243\",\"type\":\"Grid\",\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"2dde2aef-94cf-41bf-c1b4-dc9425309739\"},\"id\":\"c5d8de29-a066-4f43-cfdb-85bd3a586243\",\"doc\":null,\"tags\":[]}},{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"PanTool\",\"attributes\":{\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"dimensions\":[\"width\",\"height\"],\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"doc\":null,\"tags\":[]}},{\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\",\"type\":\"WheelZoomTool\",\"attributes\":{\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"dimensions\":[\"width\",\"height\"],\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\",\"doc\":null,\"tags\":[]}},{\"id\":\"4db058ec-f32a-423f-c2e3-4150464d3f26\",\"type\":\"BoxZoomTool\",\"attributes\":{\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"4db058ec-f32a-423f-c2e3-4150464d3f26\",\"tags\":[],\"doc\":null}},{\"id\":\"9202e3a1-2ff5-46c3-c61f-42bfb64d2041\",\"type\":\"PreviewSaveTool\",\"attributes\":{\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"9202e3a1-2ff5-46c3-c61f-42bfb64d2041\",\"tags\":[],\"doc\":null}},{\"id\":\"4dcfb421-12de-42e9-c218-08dbaf70abd6\",\"type\":\"ResizeTool\",\"attributes\":{\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"4dcfb421-12de-42e9-c218-08dbaf70abd6\",\"tags\":[],\"doc\":null}},{\"id\":\"23560b0b-3267-4c1a-ccf3-960d3e751db7\",\"type\":\"ResetTool\",\"attributes\":{\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"23560b0b-3267-4c1a-ccf3-960d3e751db7\",\"tags\":[],\"doc\":null}},{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"attributes\":{\"x_range\":{\"type\":\"DataRange1d\",\"id\":\"c7e78af2-5429-4dab-cd62-44103d0384dd\"},\"tool_events\":{\"type\":\"ToolEvents\",\"id\":\"39eadd4a-b76a-4a44-caec-ddddaeaf56de\"},\"below\":[{\"type\":\"LinearAxis\",\"id\":\"8f197c3f-2586-4ae7-ca67-5ee7f74f9199\"}],\"renderers\":[{\"type\":\"GlyphRenderer\",\"id\":\"8dcd2416-16f4-4dd1-cd0f-ec1a0a229ed9\"},{\"type\":\"LinearAxis\",\"id\":\"8f197c3f-2586-4ae7-ca67-5ee7f74f9199\"},{\"type\":\"Grid\",\"id\":\"ef639ed0-79d7-47b1-c7a9-72b14a17c566\"},{\"type\":\"LinearAxis\",\"id\":\"9a734023-b12c-4ddc-cd3c-b21bbce40eea\"},{\"type\":\"Grid\",\"id\":\"c5d8de29-a066-4f43-cfdb-85bd3a586243\"}],\"above\":[],\"tools\":[{\"type\":\"PanTool\",\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\"},{\"type\":\"WheelZoomTool\",\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\"},{\"type\":\"BoxZoomTool\",\"id\":\"4db058ec-f32a-423f-c2e3-4150464d3f26\"},{\"type\":\"PreviewSaveTool\",\"id\":\"9202e3a1-2ff5-46c3-c61f-42bfb64d2041\"},{\"type\":\"ResizeTool\",\"id\":\"4dcfb421-12de-42e9-c218-08dbaf70abd6\"},{\"type\":\"ResetTool\",\"id\":\"23560b0b-3267-4c1a-ccf3-960d3e751db7\"}],\"doc\":null,\"right\":[],\"title\":\"reward_history\",\"extra_x_ranges\":{},\"left\":[{\"type\":\"LinearAxis\",\"id\":\"9a734023-b12c-4ddc-cd3c-b21bbce40eea\"}],\"y_range\":{\"type\":\"DataRange1d\",\"id\":\"cd4dc240-48a7-486b-c8d3-f41d24013f2d\"},\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"extra_y_ranges\":{},\"tags\":[]}}];\n",
        "  Bokeh.load_models(all_models);\n",
        "  var model = Bokeh.Collections(modeltype).get(modelid);\n",
        "  $(\"#005b3e91-330a-4b54-cbd5-09603e8fd2c5\").html(''); // clear any previous plot in window_id\n",
        "  var view = new model.default_view({model: model, el: \"#005b3e91-330a-4b54-cbd5-09603e8fd2c5\"});\n",
        "    } else {\n",
        "  load_lib(bokehjs_url, function() {\n",
        "      console.log(\"Bokeh: BokehJS plotting callback run at\", new Date())\n",
        "      var modelid = \"576cd5e0-b854-47c7-c7aa-6667e860a3bc\";\n",
        "      var modeltype = \"Plot\";\n",
        "      var all_models = [{\"id\":\"3fa8ca6a-0825-414d-c166-dadcf7638543\",\"type\":\"ColumnDataSource\",\"attributes\":{\"data\":{\"y\":[-21,-21],\"x\":[1,2]},\"column_names\":[\"y\",\"x\"],\"cont_ranges\":{},\"discrete_ranges\":{},\"selected\":[],\"id\":\"3fa8ca6a-0825-414d-c166-dadcf7638543\",\"doc\":null,\"tags\":[]}},{\"id\":\"e2d133b7-7f3c-409d-c09f-134ef0db5ace\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"red\"},\"line_color\":{\"value\":\"red\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"e2d133b7-7f3c-409d-c09f-134ef0db5ace\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"e6861527-5d16-4a3c-ca77-74299ba4e55f\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"red\"},\"line_color\":{\"value\":\"red\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"e6861527-5d16-4a3c-ca77-74299ba4e55f\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"8dcd2416-16f4-4dd1-cd0f-ec1a0a229ed9\",\"type\":\"GlyphRenderer\",\"attributes\":{\"name\":null,\"nonselection_glyph\":{\"type\":\"Line\",\"id\":\"e6861527-5d16-4a3c-ca77-74299ba4e55f\"},\"doc\":null,\"server_data_source\":null,\"data_source\":{\"type\":\"ColumnDataSource\",\"id\":\"3fa8ca6a-0825-414d-c166-dadcf7638543\"},\"glyph\":{\"type\":\"Line\",\"id\":\"e2d133b7-7f3c-409d-c09f-134ef0db5ace\"},\"selection_glyph\":null,\"id\":\"8dcd2416-16f4-4dd1-cd0f-ec1a0a229ed9\",\"tags\":[]}},{\"id\":\"c7e78af2-5429-4dab-cd62-44103d0384dd\",\"type\":\"DataRange1d\",\"attributes\":{\"sources\":[{\"columns\":[\"x\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"3fa8ca6a-0825-414d-c166-dadcf7638543\"}}],\"id\":\"c7e78af2-5429-4dab-cd62-44103d0384dd\",\"tags\":[],\"doc\":null}},{\"id\":\"cd4dc240-48a7-486b-c8d3-f41d24013f2d\",\"type\":\"DataRange1d\",\"attributes\":{\"sources\":[{\"columns\":[\"y\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"3fa8ca6a-0825-414d-c166-dadcf7638543\"}}],\"id\":\"cd4dc240-48a7-486b-c8d3-f41d24013f2d\",\"tags\":[],\"doc\":null}},{\"id\":\"39eadd4a-b76a-4a44-caec-ddddaeaf56de\",\"type\":\"ToolEvents\",\"attributes\":{\"tags\":[],\"id\":\"39eadd4a-b76a-4a44-caec-ddddaeaf56de\",\"geometries\":[],\"doc\":null}},{\"id\":\"d9280e83-33bc-4c07-cc82-d5f4e65c802c\",\"type\":\"BasicTickFormatter\",\"attributes\":{\"id\":\"d9280e83-33bc-4c07-cc82-d5f4e65c802c\",\"tags\":[],\"doc\":null}},{\"id\":\"3822fa51-ca21-41f1-c105-3bd1aaed4f4b\",\"type\":\"BasicTicker\",\"attributes\":{\"num_minor_ticks\":5,\"id\":\"3822fa51-ca21-41f1-c105-3bd1aaed4f4b\",\"tags\":[],\"doc\":null}},{\"id\":\"8f197c3f-2586-4ae7-ca67-5ee7f74f9199\",\"type\":\"LinearAxis\",\"attributes\":{\"formatter\":{\"type\":\"BasicTickFormatter\",\"id\":\"d9280e83-33bc-4c07-cc82-d5f4e65c802c\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"3822fa51-ca21-41f1-c105-3bd1aaed4f4b\"},\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"axis_label\":null,\"id\":\"8f197c3f-2586-4ae7-ca67-5ee7f74f9199\",\"doc\":null,\"tags\":[]}},{\"id\":\"ef639ed0-79d7-47b1-c7a9-72b14a17c566\",\"type\":\"Grid\",\"attributes\":{\"dimension\":0,\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"3822fa51-ca21-41f1-c105-3bd1aaed4f4b\"},\"id\":\"ef639ed0-79d7-47b1-c7a9-72b14a17c566\",\"doc\":null,\"tags\":[]}},{\"id\":\"ed206cdd-6216-481c-c8ed-8d5f08a3edea\",\"type\":\"BasicTickFormatter\",\"attributes\":{\"id\":\"ed206cdd-6216-481c-c8ed-8d5f08a3edea\",\"tags\":[],\"doc\":null}},{\"id\":\"2dde2aef-94cf-41bf-c1b4-dc9425309739\",\"type\":\"BasicTicker\",\"attributes\":{\"num_minor_ticks\":5,\"id\":\"2dde2aef-94cf-41bf-c1b4-dc9425309739\",\"tags\":[],\"doc\":null}},{\"id\":\"9a734023-b12c-4ddc-cd3c-b21bbce40eea\",\"type\":\"LinearAxis\",\"attributes\":{\"formatter\":{\"type\":\"BasicTickFormatter\",\"id\":\"ed206cdd-6216-481c-c8ed-8d5f08a3edea\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"2dde2aef-94cf-41bf-c1b4-dc9425309739\"},\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"axis_label\":null,\"id\":\"9a734023-b12c-4ddc-cd3c-b21bbce40eea\",\"doc\":null,\"tags\":[]}},{\"id\":\"c5d8de29-a066-4f43-cfdb-85bd3a586243\",\"type\":\"Grid\",\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"2dde2aef-94cf-41bf-c1b4-dc9425309739\"},\"id\":\"c5d8de29-a066-4f43-cfdb-85bd3a586243\",\"doc\":null,\"tags\":[]}},{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"PanTool\",\"attributes\":{\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"dimensions\":[\"width\",\"height\"],\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"doc\":null,\"tags\":[]}},{\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\",\"type\":\"WheelZoomTool\",\"attributes\":{\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"dimensions\":[\"width\",\"height\"],\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\",\"doc\":null,\"tags\":[]}},{\"id\":\"4db058ec-f32a-423f-c2e3-4150464d3f26\",\"type\":\"BoxZoomTool\",\"attributes\":{\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"4db058ec-f32a-423f-c2e3-4150464d3f26\",\"tags\":[],\"doc\":null}},{\"id\":\"9202e3a1-2ff5-46c3-c61f-42bfb64d2041\",\"type\":\"PreviewSaveTool\",\"attributes\":{\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"9202e3a1-2ff5-46c3-c61f-42bfb64d2041\",\"tags\":[],\"doc\":null}},{\"id\":\"4dcfb421-12de-42e9-c218-08dbaf70abd6\",\"type\":\"ResizeTool\",\"attributes\":{\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"4dcfb421-12de-42e9-c218-08dbaf70abd6\",\"tags\":[],\"doc\":null}},{\"id\":\"23560b0b-3267-4c1a-ccf3-960d3e751db7\",\"type\":\"ResetTool\",\"attributes\":{\"plot\":{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"23560b0b-3267-4c1a-ccf3-960d3e751db7\",\"tags\":[],\"doc\":null}},{\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"type\":\"Plot\",\"attributes\":{\"x_range\":{\"type\":\"DataRange1d\",\"id\":\"c7e78af2-5429-4dab-cd62-44103d0384dd\"},\"tool_events\":{\"type\":\"ToolEvents\",\"id\":\"39eadd4a-b76a-4a44-caec-ddddaeaf56de\"},\"below\":[{\"type\":\"LinearAxis\",\"id\":\"8f197c3f-2586-4ae7-ca67-5ee7f74f9199\"}],\"renderers\":[{\"type\":\"GlyphRenderer\",\"id\":\"8dcd2416-16f4-4dd1-cd0f-ec1a0a229ed9\"},{\"type\":\"LinearAxis\",\"id\":\"8f197c3f-2586-4ae7-ca67-5ee7f74f9199\"},{\"type\":\"Grid\",\"id\":\"ef639ed0-79d7-47b1-c7a9-72b14a17c566\"},{\"type\":\"LinearAxis\",\"id\":\"9a734023-b12c-4ddc-cd3c-b21bbce40eea\"},{\"type\":\"Grid\",\"id\":\"c5d8de29-a066-4f43-cfdb-85bd3a586243\"}],\"above\":[],\"tools\":[{\"type\":\"PanTool\",\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\"},{\"type\":\"WheelZoomTool\",\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\"},{\"type\":\"BoxZoomTool\",\"id\":\"4db058ec-f32a-423f-c2e3-4150464d3f26\"},{\"type\":\"PreviewSaveTool\",\"id\":\"9202e3a1-2ff5-46c3-c61f-42bfb64d2041\"},{\"type\":\"ResizeTool\",\"id\":\"4dcfb421-12de-42e9-c218-08dbaf70abd6\"},{\"type\":\"ResetTool\",\"id\":\"23560b0b-3267-4c1a-ccf3-960d3e751db7\"}],\"doc\":null,\"right\":[],\"title\":\"reward_history\",\"extra_x_ranges\":{},\"left\":[{\"type\":\"LinearAxis\",\"id\":\"9a734023-b12c-4ddc-cd3c-b21bbce40eea\"}],\"y_range\":{\"type\":\"DataRange1d\",\"id\":\"cd4dc240-48a7-486b-c8d3-f41d24013f2d\"},\"id\":\"576cd5e0-b854-47c7-c7aa-6667e860a3bc\",\"extra_y_ranges\":{},\"tags\":[]}}];\n",
        "      Bokeh.load_models(all_models);\n",
        "      var model = Bokeh.Collections(modeltype).get(modelid);\n",
        "      $(\"#005b3e91-330a-4b54-cbd5-09603e8fd2c5\").html(''); // clear any previous plot in window_id\n",
        "      var view = new model.default_view({model: model, el: \"#005b3e91-330a-4b54-cbd5-09603e8fd2c5\"});\n",
        "  });\n",
        "    }\n",
        "});\n",
        "</script>\n",
        "<div class=\"plotdiv\" id=\"005b3e91-330a-4b54-cbd5-09603e8fd2c5\"></div>\n"
       ],
       "metadata": {},
       "output_type": "display_data"
      }
     ],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "agent['reward_history']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 16,
       "text": [
        "{\n",
        "  1 : -21\n",
        "}\n"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}