Independent Solution Homogeneous.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using POMDPs, POMDPTools, QuickPOMDPs, MCTS, DiscreteValueIteration, POMDPSimulators, POMDPModels, POMDPPolicies\n",
    "using Distributions, Combinatorics, StaticArrays, D3Trees, Random\n",
    "using FileIO, JLD2, TickTock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "action=[(0,0), (0,1), (1,1)]\n",
    "\n",
    "Number_level = 10;\n",
    "#Transition_matrix = [0.86 0.14 0.0 0.0; 0.0 0.83 0.17 0.0;0.0 0.0 0.89 0.11;1.0 0.0 0.0 0.0];\n",
    "fullname = \"tm10\";\n",
    "# generate_transition_matrix(Number_level, fullname) \n",
    "Transition_matrix = load(fullname*\".jld2\",\"transition_matrix\");\n",
    "failure_penalty = -1000.0;\n",
    "maintenance_penalty = -200.0;\n",
    "setup_cost = -800.0;\n",
    "normal_operation = 0.0;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "reward (generic function with 3 methods)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function reward(s, a,s1=10, rng=23)       #s is a vector of number units at each level and a is the number of units we will repair\n",
    "       if s==Number_level\n",
    "            r=failure_penalty  + normal_operation  +  setup_cost/NumberUnits  ;\n",
    "        return r\n",
    "       end\n",
    "        r=0\n",
    "     if a==1\n",
    "        if s1>=s\n",
    "            crd = Categorical(Transition_matrix[s,:]);\n",
    "            s = rand(crd);\n",
    "            r = normal_operation ;\n",
    "         return r\n",
    "        else\n",
    "            return 0.0\n",
    "        end\n",
    "        end\n",
    "#     if a==2\n",
    "#          if s1>=s\n",
    "#             crd = Categorical(Transition_matrix[s,:]);\n",
    "#             s = rand(crd);\n",
    "#             r = normal_operation +  setup_cost/NumberUnits;\n",
    "#          return r\n",
    "#         else\n",
    "#             return 0.0\n",
    "#         end\n",
    "#         end\n",
    "    if a==2\n",
    "        s1=deepcopy(s)\n",
    "        s=1\n",
    "        r = maintenance_penalty \n",
    "         crd = Categorical(Transition_matrix[s,:]);\n",
    "            s = rand(crd);\n",
    "            r+= normal_operation  +  setup_cost/NumberUnits  ;\n",
    "#            r*=(s1/10);\n",
    "        return r\n",
    "    end\n",
    "    end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "T=zeros(Number_level,Number_level,3);\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1:0.1:1.0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((1:10)./10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "T[:,:,1].=Transition_matrix;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "T[:,:,2].=Transition_matrix;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "for j in 1:10\n",
    "        T[j,:,2].=Transition_matrix[1,:];\n",
    "        end\n",
    "# for j in 1:10\n",
    "#         T[j,:,3].=0.0\n",
    "#     T[j,1,3]=1.0\n",
    "#         end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10-element view(::Array{Float64, 3}, 10, :, 2) with eltype Float64:\n",
       " 0.2695279572108137\n",
       " 0.16860741623354286\n",
       " 0.1306613209439592\n",
       " 0.09857168678543618\n",
       " 0.09171796107138079\n",
       " 0.09134806335671229\n",
       " 0.09048875946531719\n",
       " 0.04832973229444604\n",
       " 0.00607725632113932\n",
       " 0.004669846317252394"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T[10,:,1].=Transition_matrix[1,:]\n",
    "T[10,:,2].=Transition_matrix[1,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "generative (generic function with 2 methods)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function explore()\n",
    "    c=[1/3,1/3,1/3]\n",
    "\n",
    "     crd = Categorical(c);\n",
    "     return rand(crd)\n",
    "    end \n",
    "function generative(s, a, rng=23)       #s is a vector of number units at each level and a is the number of units we will repair\n",
    "        if s==Number_level\n",
    "        s=1\n",
    "        crd = Categorical(Transition_matrix[s,:]);\n",
    "            s = rand(crd);\n",
    "            r=failure_penalty + normal_operation  ;\n",
    "        return (sp=s, r=r)\n",
    "       end\n",
    "        r=0\n",
    "     if a==1\n",
    "            crd = Categorical(Transition_matrix[s,:]);\n",
    "            s = rand(crd);\n",
    "            r = normal_operation ;\n",
    "         return (sp=s, r=r)\n",
    "        end\n",
    "    if a==2\n",
    "        if s==1\n",
    "            crd = Categorical(Transition_matrix[s,:]);\n",
    "            s = rand(crd);\n",
    "            r= normal_operation  ;\n",
    "            return (sp=s, r=r)\n",
    "        else\n",
    "        s1=deepcopy(s)\n",
    "        s=1\n",
    "        r = maintenance_penalty \n",
    "         crd = Categorical(Transition_matrix[s,:]);\n",
    "            s = rand(crd);\n",
    "            r+= normal_operation  ;\n",
    "#            r*=(s1/10);\n",
    "       return (sp=s, r=r)\n",
    "        end\n",
    "    end\n",
    "    end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ValueIteration (generic function with 3 methods)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function ValueIteration(S, T, γ, p1=0.01, d=100)\n",
    "    Q=zeros((Number_level,3))\n",
    "#     print(Q)\n",
    "    V= rand(Float64, (Number_level))*1000\n",
    "    diff=0\n",
    "    i=0\n",
    "    \n",
    "    \n",
    "\n",
    "    \n",
    "    p=[1 1 1 ]\n",
    "    while i<=2000\n",
    "#     while  i<=10000\n",
    "    for s in 1:10\n",
    "        Threads.@threads for a in 1:2\n",
    " \n",
    "                Q[s,a]= (reward(s,a)) + γ*( sum(T[s,s1,a]*(V[s1]) for (s1) in 1:10 ))\n",
    "    end\n",
    "            V[s]=findmax(Q[s,1:2])[1]\n",
    "        end\n",
    "        i+=1\n",
    "        end\n",
    "    return Q,V\n",
    "    end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "decison (generic function with 2 methods)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# function decison(Q, s)\n",
    "#    a_final=[]\n",
    "#    if Number_level in s\n",
    "# #         print(\"hi\")\n",
    "#         d=(collect(Iterators.product((Q[s[i],2:3] for i in 1:NumberUnits)...)))\n",
    "#         d1=(collect(Iterators.product((2:3 for i in 1:NumberUnits)...)))\n",
    "#         min=-1e+10\n",
    "#         opt=0\n",
    "#         for (i,j) in enumerate(d)\n",
    "#             if min < sum((j))\n",
    "#                 min=sum((j))\n",
    "#                 opt=i\n",
    "#             end\n",
    "#         end\n",
    "#         return d1[opt]\n",
    "#     end\n",
    "#     q0=0.0\n",
    "#     a_0=repeat(1:1,NumberUnits);\n",
    "#     for i in 1:NumberUnits\n",
    "#             q0+=Q[s[i],1]\n",
    "#     end\n",
    "#     d=(collect(Iterators.product((Q[s[i],2:3] for i in 1:NumberUnits)...)))\n",
    "#     d1=(collect(Iterators.product((2:3 for i in 1:NumberUnits)...)))\n",
    "# #     d[1]=a_0\n",
    "# #     print(\"here\")\n",
    "# #     d1[1]=(1,1,1,1,1,1,1,1,1,1)\n",
    "#     min=-1e+10\n",
    "#     opt=0\n",
    "#     for (i,j) in enumerate(d)\n",
    "#         if i==1\n",
    "#             if min < q0\n",
    "#             min=q0\n",
    "#             opt=i\n",
    "#         end\n",
    "#         elseif min < sum((j))\n",
    "#             min=sum((j))\n",
    "#             opt=i\n",
    "#         end\n",
    "#     end\n",
    "#     if opt==1\n",
    "#         return a_0\n",
    "#     else\n",
    "#         return d1[opt]\n",
    "#     end\n",
    "#     return opt\n",
    "# end\n",
    "\n",
    "\n",
    "function decison(Q, s,n=0.2)\n",
    "   a_final=[]\n",
    "   if true\n",
    "#         print(\"hi\")\n",
    "        for i in 1:NumberUnits\n",
    "            append!(a_final,argmax(a->Q[s[i],a], [1,2]))\n",
    "            end\n",
    "        return a_final\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60[0, -1000, -200, -1000]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:39:03.745\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60\n",
      "1\n",
      "-90774.76304335413\n",
      "3111.1234890108817\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          71.7338567s: 1 minute, 11 seconds, 733 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------\n",
      "60[0, -1200, -200, -1000]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:40:17.612\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60\n",
      "1\n",
      "-94597.57222443598\n",
      "3088.79939571102\n",
      "-------------------------"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          63.7269299s: 1 minute, 3 seconds, 726 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "60[0, -1000, -200, -1400]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:41:22.703\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60\n",
      "1\n",
      "-95912.54638507981\n",
      "3068.672281952535\n",
      "-------------------------\n",
      "60"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          57.8432223s: 57 seconds, 843 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, -1000, -200, -1600]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:42:21.347\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60\n",
      "1\n",
      "-97827.35252144559\n",
      "3433.2786992602446\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          60.8162633s: 1 minute, 816 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------\n",
      "50[0, -1000, -200, -1000]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:43:24.898\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50\n",
      "1\n",
      "-78787.69475250793\n",
      "2861.549415854776\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          55.9339157s: 55 seconds, 933 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------\n",
      "50[0, -1200, -200, -1000]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:44:23.333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50\n",
      "1\n",
      "-82585.84981845117\n",
      "2874.5937405708128\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          64.0301691s: 1 minute, 4 seconds, 30 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------\n",
      "50[0, -1000, -200, -1400]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:45:29.111\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50\n",
      "1\n",
      "-83131.2050236433\n",
      "2803.809906773197\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          57.2718235s: 57 seconds, 271 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------\n",
      "50[0, -1000, -200, -1600]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:46:28.322\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50\n",
      "1\n",
      "-84710.38286585212\n",
      "3138.2462372866985\n",
      "-------------------------"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m           57.767645s: 57 seconds, 767 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "40[0, -1000, -200, -1000]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:47:27.373\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n",
      "1\n",
      "-66770.06295135293\n",
      "2556.278147039365\n",
      "-------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          38.3897239s: 38 seconds, 389 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40[0, -1200, -200, -1000]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:48:06.703\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n",
      "1\n",
      "-70531.29593956064\n",
      "2580.4893271980304\n",
      "-------------------------\n",
      "40[0, -1000, -200, "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          42.4933509s: 42 seconds, 493 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1400]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:48:49.876\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n",
      "1\n",
      "-70264.75016289642\n",
      "2477.327107736805\n",
      "-------------------------\n",
      "40["
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          40.7828025s: 40 seconds, 782 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0, -1000, -200, -1600]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:49:31.097\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n",
      "1\n",
      "-71533.88593644125\n",
      "2796.670239619562\n",
      "-------------------------\n",
      "30"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          46.3905733s: 46 seconds, 390 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, -1000, -200, -1000]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:50:19.677\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\n",
      "1\n",
      "-54675.27909246378\n",
      "2238.8116890274327\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          36.7812181s: 36 seconds, 781 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------\n",
      "30[0, -1200, -200, -1000]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:50:59.014\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\n",
      "1\n",
      "-58443.39618802793\n",
      "2266.7672998508533\n",
      "-------------------------\n",
      "30"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          35.4556256s: 35 seconds, 455 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, -1000, -200, -1400]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:51:36.524\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\n",
      "1\n",
      "-57419.996995520974\n",
      "2154.369700443528\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m           33.691077s: 33 seconds, 691 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------\n",
      "30[0, -1000, -200, -1600]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:52:11.893\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\n",
      "1\n",
      "-58367.19182441046\n",
      "2420.208418861999\n",
      "-------------------------"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          35.9705246s: 35 seconds, 970 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "20[0, -1000, -200, -1000]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:52:49.922\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n",
      "1\n",
      "-42443.0374171177\n",
      "1899.9538632364242\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          23.3780841s: 23 seconds, 378 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------\n",
      "20[0, -1200, -200, -1000]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:53:14.217\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n",
      "1\n",
      "-46182.722175109986\n",
      "1959.949906623045\n",
      "-------------------------\n",
      "20["
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          18.8471849s: 18 seconds, 847 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0, -1000, -200, -1400]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:53:33.700\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n",
      "1\n",
      "-44530.45135466769\n",
      "1803.6560177925444\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          16.9950852s: 16 seconds, 995 milliseconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------\n",
      "20[0, -1000, -200, -1600]\n",
      "false\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m started timer at: 2023-04-01T23:53:51.256\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n",
      "1\n",
      "-45142.678151559034\n",
      "1989.1884683362812\n",
      "-------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[36m\u001b[1m[ \u001b[22m\u001b[39m\u001b[36m\u001b[1mInfo: \u001b[22m\u001b[39m          12.9726529s: 12 seconds, 972 milliseconds\n"
     ]
    }
   ],
   "source": [
    "cost=[ \n",
    "    [0 -1000 -200 -1000],\n",
    "    [0 -1200 -200 -1000],\n",
    "    [0 -1000 -200 -1400],\n",
    "    [0 -1000 -200 -1600]\n",
    "    \n",
    "    ]\n",
    "\n",
    "# print(Q)\n",
    "\n",
    "for units in [60,50,40,30,20]\n",
    "    for c in cost\n",
    "   print(units)  \n",
    "    global normal_operation,setup_cost,maintenance_penalty,failure_penalty=c\n",
    "     println([normal_operation,setup_cost,maintenance_penalty,failure_penalty])   \n",
    "     for lambda in 1:1:1\n",
    "        global  lamb=lambda\n",
    "    global NumberUnits=units\n",
    "     Q,V=ValueIteration(s,T,.95,3000)\n",
    "         println(isnan(Q[1,1]))\n",
    "       if sum(isnan(Q[1,1]))>0.0\n",
    "            continue\n",
    "        end\n",
    "tick()\n",
    "trials=10000;\n",
    "simsteps = 100;\n",
    "results=zeros(trials)*0.1\n",
    "Threads.@threads for h in 1:trials\n",
    "s=repeat(1:1,NumberUnits)\n",
    "r1=[]\n",
    "a1=[]   \n",
    "r=repeat(1:1,NumberUnits)*1.0\n",
    " for k in 1:simsteps\n",
    "    a =decison(Q,s)\n",
    "    r2=0.0\n",
    "    s1=repeat(1:1,NumberUnits)\n",
    "              \n",
    "            \n",
    "#     print(a)\n",
    "    for i in 1:NumberUnits\n",
    "#             print(generative(s[i],a[i],234))\n",
    "         s1[i],r[i]=generative(s[i],a[i],234);\n",
    "          r2+=r[i]\n",
    "            end \n",
    "#     println(r2)\n",
    "#     println(s)\n",
    "    if (Number_level in s) || (2 in a)\n",
    "            r2+=setup_cost\n",
    "                    end\n",
    "    s=s1;\n",
    "    append!(a1,a)\n",
    "    append!(r1,r2)\n",
    "end\n",
    "rk=0\n",
    "for i in 1:simsteps\n",
    "    rk=0.95*rk+r1[simsteps-i+1]\n",
    "end\n",
    "#     println(rk)\n",
    "    results[h]=rk\n",
    "#     println(results)\n",
    "end\n",
    "println(units)\n",
    "println(lambda)\n",
    "\n",
    "println(mean(results))\n",
    "\n",
    "println(std(results))\n",
    "pushfirst!(df,[units,normal_operation,setup_cost,maintenance_penalty,failure_penalty,lamb,mean(results),std(results)])\n",
    "        \n",
    "tock()\n",
    "end\n",
    "    println(\"-------------------------\")\n",
    "end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><div style = \"float: left;\"><span>20×8 DataFrame</span></div><div style = \"clear: both;\"></div></div><div class = \"data-frame\" style = \"overflow-x: scroll;\"><table class = \"data-frame\" style = \"margin-bottom: 6px;\"><thead><tr class = \"header\"><th class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">Row</th><th style = \"text-align: left;\">Units</th><th style = \"text-align: left;\">n</th><th style = \"text-align: left;\">s</th><th style = \"text-align: left;\">m</th><th style = \"text-align: left;\">f</th><th style = \"text-align: left;\">lambda</th><th style = \"text-align: left;\">mean</th><th style = \"text-align: left;\">std</th></tr><tr class = \"subheader headerLastRow\"><th class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\"></th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th><th title = \"Any\" style = \"text-align: left;\">Any</th></tr></thead><tbody><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">1</td><td style = \"text-align: left;\">20.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1600.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-45142.7</td><td style = \"text-align: left;\">1989.19</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">2</td><td style = \"text-align: left;\">20.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1400.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-44530.5</td><td style = \"text-align: left;\">1803.66</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">3</td><td style = \"text-align: left;\">20.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1200.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-46182.7</td><td style = \"text-align: left;\">1959.95</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">4</td><td style = \"text-align: left;\">20.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-42443.0</td><td style = \"text-align: left;\">1899.95</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">5</td><td style = \"text-align: left;\">30.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1600.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-58367.2</td><td style = \"text-align: left;\">2420.21</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">6</td><td style = \"text-align: left;\">30.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1400.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-57420.0</td><td style = \"text-align: left;\">2154.37</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">7</td><td style = \"text-align: left;\">30.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1200.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-58443.4</td><td style = \"text-align: left;\">2266.77</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">8</td><td style = \"text-align: left;\">30.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-54675.3</td><td style = \"text-align: left;\">2238.81</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">9</td><td style = \"text-align: left;\">40.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1600.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-71533.9</td><td style = \"text-align: left;\">2796.67</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">10</td><td style = \"text-align: left;\">40.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1400.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-70264.8</td><td style = \"text-align: left;\">2477.33</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">11</td><td style = \"text-align: left;\">40.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1200.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-70531.3</td><td style = \"text-align: left;\">2580.49</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">12</td><td style = \"text-align: left;\">40.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-66770.1</td><td style = \"text-align: left;\">2556.28</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">13</td><td style = \"text-align: left;\">50.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1600.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-84710.4</td><td style = \"text-align: left;\">3138.25</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">14</td><td style = \"text-align: left;\">50.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1400.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-83131.2</td><td style = \"text-align: left;\">2803.81</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">15</td><td style = \"text-align: left;\">50.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1200.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-82585.8</td><td style = \"text-align: left;\">2874.59</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">16</td><td style = \"text-align: left;\">50.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-78787.7</td><td style = \"text-align: left;\">2861.55</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">17</td><td style = \"text-align: left;\">60.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1600.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-97827.4</td><td style = \"text-align: left;\">3433.28</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">18</td><td style = \"text-align: left;\">60.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1400.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-95912.5</td><td style = \"text-align: left;\">3068.67</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">19</td><td style = \"text-align: left;\">60.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1200.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-94597.6</td><td style = \"text-align: left;\">3088.8</td></tr><tr><td class = \"rowNumber\" style = \"font-weight: bold; text-align: right;\">20</td><td style = \"text-align: left;\">60.0</td><td style = \"text-align: left;\">0.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">-200.0</td><td style = \"text-align: left;\">-1000.0</td><td style = \"text-align: left;\">1.0</td><td style = \"text-align: left;\">-90774.8</td><td style = \"text-align: left;\">3111.12</td></tr></tbody></table></div>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccccccc}\n",
       "\t& Units & n & s & m & f & lambda & mean & std\\\\\n",
       "\t\\hline\n",
       "\t& Any & Any & Any & Any & Any & Any & Any & Any\\\\\n",
       "\t\\hline\n",
       "\t1 & 20.0 & 0.0 & -1000.0 & -200.0 & -1600.0 & 1.0 & -45142.7 & 1989.19 \\\\\n",
       "\t2 & 20.0 & 0.0 & -1000.0 & -200.0 & -1400.0 & 1.0 & -44530.5 & 1803.66 \\\\\n",
       "\t3 & 20.0 & 0.0 & -1200.0 & -200.0 & -1000.0 & 1.0 & -46182.7 & 1959.95 \\\\\n",
       "\t4 & 20.0 & 0.0 & -1000.0 & -200.0 & -1000.0 & 1.0 & -42443.0 & 1899.95 \\\\\n",
       "\t5 & 30.0 & 0.0 & -1000.0 & -200.0 & -1600.0 & 1.0 & -58367.2 & 2420.21 \\\\\n",
       "\t6 & 30.0 & 0.0 & -1000.0 & -200.0 & -1400.0 & 1.0 & -57420.0 & 2154.37 \\\\\n",
       "\t7 & 30.0 & 0.0 & -1200.0 & -200.0 & -1000.0 & 1.0 & -58443.4 & 2266.77 \\\\\n",
       "\t8 & 30.0 & 0.0 & -1000.0 & -200.0 & -1000.0 & 1.0 & -54675.3 & 2238.81 \\\\\n",
       "\t9 & 40.0 & 0.0 & -1000.0 & -200.0 & -1600.0 & 1.0 & -71533.9 & 2796.67 \\\\\n",
       "\t10 & 40.0 & 0.0 & -1000.0 & -200.0 & -1400.0 & 1.0 & -70264.8 & 2477.33 \\\\\n",
       "\t11 & 40.0 & 0.0 & -1200.0 & -200.0 & -1000.0 & 1.0 & -70531.3 & 2580.49 \\\\\n",
       "\t12 & 40.0 & 0.0 & -1000.0 & -200.0 & -1000.0 & 1.0 & -66770.1 & 2556.28 \\\\\n",
       "\t13 & 50.0 & 0.0 & -1000.0 & -200.0 & -1600.0 & 1.0 & -84710.4 & 3138.25 \\\\\n",
       "\t14 & 50.0 & 0.0 & -1000.0 & -200.0 & -1400.0 & 1.0 & -83131.2 & 2803.81 \\\\\n",
       "\t15 & 50.0 & 0.0 & -1200.0 & -200.0 & -1000.0 & 1.0 & -82585.8 & 2874.59 \\\\\n",
       "\t16 & 50.0 & 0.0 & -1000.0 & -200.0 & -1000.0 & 1.0 & -78787.7 & 2861.55 \\\\\n",
       "\t17 & 60.0 & 0.0 & -1000.0 & -200.0 & -1600.0 & 1.0 & -97827.4 & 3433.28 \\\\\n",
       "\t18 & 60.0 & 0.0 & -1000.0 & -200.0 & -1400.0 & 1.0 & -95912.5 & 3068.67 \\\\\n",
       "\t19 & 60.0 & 0.0 & -1200.0 & -200.0 & -1000.0 & 1.0 & -94597.6 & 3088.8 \\\\\n",
       "\t20 & 60.0 & 0.0 & -1000.0 & -200.0 & -1000.0 & 1.0 & -90774.8 & 3111.12 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "\u001b[1m20×8 DataFrame\u001b[0m\n",
       "\u001b[1m Row \u001b[0m│\u001b[1m Units \u001b[0m\u001b[1m n   \u001b[0m\u001b[1m s       \u001b[0m\u001b[1m m      \u001b[0m\u001b[1m f       \u001b[0m\u001b[1m lambda \u001b[0m\u001b[1m mean     \u001b[0m\u001b[1m std     \u001b[0m\n",
       "     │\u001b[90m Any   \u001b[0m\u001b[90m Any \u001b[0m\u001b[90m Any     \u001b[0m\u001b[90m Any    \u001b[0m\u001b[90m Any     \u001b[0m\u001b[90m Any    \u001b[0m\u001b[90m Any      \u001b[0m\u001b[90m Any     \u001b[0m\n",
       "─────┼─────────────────────────────────────────────────────────────────\n",
       "   1 │ 20.0   0.0  -1000.0  -200.0  -1600.0  1.0     -45142.7  1989.19\n",
       "   2 │ 20.0   0.0  -1000.0  -200.0  -1400.0  1.0     -44530.5  1803.66\n",
       "   3 │ 20.0   0.0  -1200.0  -200.0  -1000.0  1.0     -46182.7  1959.95\n",
       "   4 │ 20.0   0.0  -1000.0  -200.0  -1000.0  1.0     -42443.0  1899.95\n",
       "   5 │ 30.0   0.0  -1000.0  -200.0  -1600.0  1.0     -58367.2  2420.21\n",
       "   6 │ 30.0   0.0  -1000.0  -200.0  -1400.0  1.0     -57420.0  2154.37\n",
       "   7 │ 30.0   0.0  -1200.0  -200.0  -1000.0  1.0     -58443.4  2266.77\n",
       "   8 │ 30.0   0.0  -1000.0  -200.0  -1000.0  1.0     -54675.3  2238.81\n",
       "   9 │ 40.0   0.0  -1000.0  -200.0  -1600.0  1.0     -71533.9  2796.67\n",
       "  10 │ 40.0   0.0  -1000.0  -200.0  -1400.0  1.0     -70264.8  2477.33\n",
       "  11 │ 40.0   0.0  -1200.0  -200.0  -1000.0  1.0     -70531.3  2580.49\n",
       "  12 │ 40.0   0.0  -1000.0  -200.0  -1000.0  1.0     -66770.1  2556.28\n",
       "  13 │ 50.0   0.0  -1000.0  -200.0  -1600.0  1.0     -84710.4  3138.25\n",
       "  14 │ 50.0   0.0  -1000.0  -200.0  -1400.0  1.0     -83131.2  2803.81\n",
       "  15 │ 50.0   0.0  -1200.0  -200.0  -1000.0  1.0     -82585.8  2874.59\n",
       "  16 │ 50.0   0.0  -1000.0  -200.0  -1000.0  1.0     -78787.7  2861.55\n",
       "  17 │ 60.0   0.0  -1000.0  -200.0  -1600.0  1.0     -97827.4  3433.28\n",
       "  18 │ 60.0   0.0  -1000.0  -200.0  -1400.0  1.0     -95912.5  3068.67\n",
       "  19 │ 60.0   0.0  -1200.0  -200.0  -1000.0  1.0     -94597.6  3088.8\n",
       "  20 │ 60.0   0.0  -1000.0  -200.0  -1000.0  1.0     -90774.8  3111.12"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Independent with setup.csv\""
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using CSV\n",
    "CSV.write(\"Independent with setup.csv\", df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "MethodError: no method matching getindex(::Tuple{Matrix{Float64}, Vector{Float64}}, ::Int64, ::Int64)\n\u001b[0mClosest candidates are:\n\u001b[0m  getindex(::Tuple, ::Int64) at tuple.jl:29\n\u001b[0m  getindex(::Tuple, ::Integer) at tuple.jl:30\n\u001b[0m  getindex(::Tuple, ::Real) at deprecated.jl:70\n\u001b[0m  ...",
     "output_type": "error",
     "traceback": [
      "MethodError: no method matching getindex(::Tuple{Matrix{Float64}, Vector{Float64}}, ::Int64, ::Int64)\n\u001b[0mClosest candidates are:\n\u001b[0m  getindex(::Tuple, ::Int64) at tuple.jl:29\n\u001b[0m  getindex(::Tuple, ::Integer) at tuple.jl:30\n\u001b[0m  getindex(::Tuple, ::Real) at deprecated.jl:70\n\u001b[0m  ...",
      "",
      "Stacktrace:",
      " [1] top-level scope",
      "   @ .\\In[40]:3"
     ]
    }
   ],
   "source": [
    " Q=ValueIteration(s,T,.95,3000)\n",
    "for s in 1:Number_level\n",
    "p=[exp(0.01*sum(Q[s,1])),exp(0.01*sum(Q[s,2])),exp(0.01*sum(Q[s,3]))]\n",
    "    p./=sum(p)\n",
    "    println(p)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "ArgumentError: Package Plots not found in current path.\n- Run `import Pkg; Pkg.add(\"Plots\")` to install the Plots package.",
     "output_type": "error",
     "traceback": [
      "ArgumentError: Package Plots not found in current path.\n- Run `import Pkg; Pkg.add(\"Plots\")` to install the Plots package.",
      "",
      "Stacktrace:",
      " [1] macro expansion",
      "   @ .\\loading.jl:1163 [inlined]",
      " [2] macro expansion",
      "   @ .\\lock.jl:223 [inlined]",
      " [3] require(into::Module, mod::Symbol)",
      "   @ Base .\\loading.jl:1144"
     ]
    }
   ],
   "source": [
    "using Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "UndefVarError: plot not defined",
     "output_type": "error",
     "traceback": [
      "UndefVarError: plot not defined",
      "",
      "Stacktrace:",
      " [1] top-level scope",
      "   @ .\\In[42]:2"
     ]
    }
   ],
   "source": [
    "for i in [3,5,10,15,20,25,30,40,50,60]\n",
    "   display( plot(df[df.Units.==i,:].lambda,df[df.Units.==i,:].mean, xtick=df[df.Units.==i,:].lambda, show = true,title=string(i),label=false, xaxis=:log10,rotation=40))\n",
    "    plot(show=true)\n",
    "    println(df[df.Units.==i,:][findmax(df[df.Units.==i,:].mean)[2],:])\n",
    "\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "UndefVarError: plot! not defined",
     "output_type": "error",
     "traceback": [
      "UndefVarError: plot! not defined",
      "",
      "Stacktrace:",
      " [1] top-level scope",
      "   @ In[43]:1"
     ]
    }
   ],
   "source": [
    "plot!()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.8.2",
   "language": "julia",
   "name": "julia-1.8"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}