REINFORCE.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3w04_bU_4H9g"
      },
      "outputs": [],
      "source": [
        "import gym\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.distributions import Categorical\n",
        "import matplotlib.pyplot as plt\n",
        "!pip install gym[classic_control]\n",
        "#pip install gym[box2d] #for lunarlander\n",
        "!apt update\n",
        "!apt install xvfb\n",
        "!pip install pyvirtualdisplay\n",
        "!pip install gym-notebook-wrapper\n",
        "import gnwrapper\n",
        "!nvidia-smi\n",
        "print(torch.cuda.is_available())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kWwb1Cd_gyJn",
        "outputId": "99763ef9-24e0-4866-e453-c481e90d1bd7"
      },
      "outputs": [],
      "source": [
        "class Agent(nn.Module):\n",
        "    def __init__(self, input_dim, output_dim, width):\n",
        "        super().__init__()\n",
        "        self.model = PolicyNetwork(input_dim, output_dim, width)\n",
        "        self.model.to(device)\n",
        "        self.model.train()\n",
        "        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0003)\n",
        "        self.trajectory = []\n",
        "        self.env = gym.make(\"CartPole-v1\")        \n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.model(x)\n",
        "        return x\n",
        "\n",
        "    def self_play(self, max_timestep=1000000):\n",
        "        game_score = 0\n",
        "        state = self.env.reset() # env start\n",
        "        for _ in range(max_timestep):\n",
        "            output = self.forward(torch.from_numpy(state).float().to(device)) # inference\n",
        "            prob_distribution = Categorical(output) \n",
        "            action = prob_distribution.sample() \n",
        "            state, r, done, _ = self.env.step(action.item())           \n",
        "            self.trajectory.append((prob_distribution.log_prob(action), r)) \n",
        "            game_score += r \n",
        "            if done:\n",
        "                break\n",
        "        return game_score\n",
        "\n",
        "    def update_weights(self):\n",
        "        self.optimizer.zero_grad()\n",
        "        G_t = 0\n",
        "        for gradient_policy_a_s, r in self.trajectory[::-1]: \n",
        "            G_t = 0.99 * G_t + r\n",
        "            loss = -1 * G_t * gradient_policy_a_s # -1 for gradient ascent\n",
        "            loss.backward()\n",
        "        self.optimizer.step()\n",
        "        self.trajectory.clear()\n",
        "        return\n",
        "\n",
        "class PolicyNetwork(nn.Module):  \n",
        "    def __init__(self, input_dim, output_dim, width):\n",
        "        super().__init__()\n",
        "        self.layer1 = torch.nn.Linear(input_dim, width)\n",
        "        self.layer2 = torch.nn.Linear(width, width) \n",
        "        self.layer3 = torch.nn.Linear(width, output_dim) \n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.layer1(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        x = self.layer2(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        x = self.layer3(x)\n",
        "        x = torch.nn.functional.softmax(x, dim=0)\n",
        "        return x\n",
        "\n",
        "device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')\n",
        "score_arr = []\n",
        "env = gym.make(\"CartPole-v1\") \n",
        "agent = Agent(env.observation_space.shape[0], env.action_space.n, 128) \n",
        "print(agent)\n",
        "env.close()\n",
        "\n",
        "#Self play, weight update\n",
        "episode_nums = 1000\n",
        "for i in range(episode_nums):\n",
        "    game_score = agent.self_play() \n",
        "    score_arr.append(game_score)    \n",
        "    agent.update_weights()       \n",
        "    if i%50==0 : print('episode', i)   \n",
        "torch.save(agent.state_dict(), 'weights.pt') \n",
        "agent.env.close()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "_1Cz8LkVy4du",
        "outputId": "2f954398-4377-4291-cbe8-8c0a8bcb964d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f6551096f90>"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd5wTx9nHfyNdg6MfRy8HppliwBzNGBdsjG0Sl7jENW4xcUviOCHGLS5xYuK8sWOnEBfcG66BAK4UY2xTDjC9HXAcd5Q7jmtclzTvH9qVVqvd1ay0K610z5cPn5Nm2+yu9jfPPvPMM4xzDoIgCCK1cCW6AgRBEIT1kLgTBEGkICTuBEEQKQiJO0EQRApC4k4QBJGCpCW6AgDQtWtXnpeXl+hqEARBJBUbNmw4zjnP1VrmCHHPy8tDQUFBoqtBEASRVDDGDuotI7cMQRBECkLiThAEkYKQuBMEQaQgjvC5a9HS0oKSkhI0NjYmuipxJSsrC3369EF6enqiq0IQRBLjWHEvKSlB+/btkZeXB8ZYoqsTFzjnqKioQElJCQYMGJDo6hAEkcQIuWUYY0WMsa2MsR8YYwVSWRfG2JeMsb3S385SOWOMPc8YK2SMbWGMnR5NxRobG5GTk9NqhB0AGGPIyclpdW8rBEFYjxmf+7mc8zGc83zp+xwAyzjngwEsk74DwEUABkv/ZwGYF23lWpOwy7TGcyYIwnpi6VC9FMDr0ufXAVymKH+D+1kDoBNjrGcMxyGIVssnm0qwrbQan28/ioU/lCa6OkmLx+vD++sPwetLbIrzT7cewYm65rgcS9TnzgF8wRjjAF7gnL8IoDvn/Ii0/CiA7tLn3gAOKbYtkcqOKMrAGJsFv2WPfv36RVd7gkhhNhZX4jcLNoeUDe3RHsN6dEhQjZKX174rwpNLdqLF58P1E/snpA7ltU248+2NGJ/XGR/ccYbtxxMV9zM556WMsW4AvmSM7VIu5JxzSfiFkRqIFwEgPz8/5WcM8Xg8SEtzbP814UDqmjxhZY0tvgTUJPmpkKzlqvqWhNWh2eu/dyWVDXE5npBbhnNeKv0tA/AJgAkAjsnuFulvmbR6KYC+is37SGVJR11dHWbOnInRo0dj5MiRWLBgAdavX48zzjgDo0ePxoQJE1BbW4vGxkbccsstGDVqFMaOHYsVK1YAAF577TVccsklmDZtGs477zzU1dXh1ltvxYQJEzB27FgsXLgwwWdIOBmtSdKoRyZ5ife9i2hKMsayAbg457XS5wsAPAFgEYCbAMyV/spKtQjAPYyx9wBMBFCtcN9ExeP/244dh2ti2UUYw3t1wKM/HmG4zmeffYZevXphyZIlAIDq6mqMHTsWCxYswPjx41FTU4M2bdrgueeeA2MMW7duxa5du3DBBRdgz549AICNGzdiy5Yt6NKlCx588EFMmzYNr7zyCqqqqjBhwgScf/75yM7OtvTciNSF+tuTl3i7J0Qs9+4AVjPGNgNYB2AJ5/wz+EV9OmNsL4Dzpe8AsBTAfgCFAF4CcJfltY4To0aNwpdffon7778f33zzDYqLi9GzZ0+MHz8eANChQwekpaVh9erVuOGGGwAAw4YNQ//+/QPiPn36dHTp0gUA8MUXX2Du3LkYM2YMzjnnHDQ2NqK4uDgxJ0ckJYxs96QnXncwouXOOd8PYLRGeQWA8zTKOYC7LamdRCQL2y6GDBmCjRs3YunSpXj44Ycxbdo00/tQWuWcc3z00UcYOnSoldUkWhFkuUdHa7xslFvGgMOHD6Nt27a44YYbMHv2bKxduxZHjhzB+vXrAQC1tbXweDyYOnUq3n77bQDAnj17UFxcrCngM2bMwD/+8Q9wyZm6adOm+J0MkXSkfJQBYSsUvmHA1q1bMXv2bLhcLqSnp2PevHngnOOXv/wlGhoa0KZNG3z11Ve46667cOedd2LUqFFIS0vDa6+9hszMzLD9PfLII7j33ntx2mmnwefzYcCAAVi8eHECzoxIVshyJ0QhcTdgxowZmDFjRlj5mjVrwspeffXVsLKbb74ZN998c+B7mzZt8MILL1haR6J1QT735IVrhT/ZCLllCCKJIMs9NuItsFrEK8UIiTtBJBEk7tHRGq+bo8XdCa1svGmN50yIQ26Z6GiNj5VjxT0rKwsVFRWtSuzkfO5ZWVmJrgrhAFrTb781EO/b6dgO1T59+qCkpATl5eWJrkpckWdiIggtfCT4UeEEtwyJu0R6ejrNRkS0arQ63kjbYyOR14/HeeSCY90yBNHa0XLLxFsgUgUn9FXEu2EhcSeIJIIs9+hwQqMYb5caiTtBJJDNh6rw6/c2wSc4QxCJe/LixKyQBEHYxKw3C7Dwh8Moq20SWt8JFmgy4gy3DFnuBEHokOApQJOeRF4+8rkTBKELxb4nL+SWIYhWiJa7RUsMSNpjI5HOGblDNV4x9yTuBJFAzPqCyXKPDae4ZZ5auhMznl1l6/EcO4iJIIhwSNvFKK6oR1aGC93a+1N5OGGEqjIU8oVV+20/Hok7QTgAUdEmbRfjrL+uAAAUzZ2Z4JoEoQ5VgmhFmLUoRePhCechizv53AmiFfHkkh3hhRo6TtKevFBuGYJohSzdejSsTEsMKCtkbCQ0cRi5ZQii9WD0hu7zaRSStictlFuGIAgAFOeeatAgJoIgAGhbeuSWiQ4HREIGxijEK88NiTtBJBCtCTlktHSctD15IZ87QRAA9CbrSB2aPF6c9fQKrNxdluiqxAVyyxAEAUBbDFLJLVNa2YDiE/V4/H8aYaAWsr/8JFocMD5Aa4zC7qO1th2PxJ0gHIqmkCdeoyxDdknZ2WAdrW7EtL99jXkr9wFIbD58rSNf/u9vbTseiTtBOBRtbU8ddZd7G+x8GalpbLFv5ybRygrZ7NGKd7UGEneCcCia0TL2aUHckUXOzgbL5YSMYTLSacbLsyYs7owxN2NsE2NssfR9AGNsLWOskDG2gDGWIZVnSt8LpeV59lSdIFofqWO3B4XXTrFLczlH3J3cofprADsV3/8C4FnO+SAAlQBuk8pvA1AplT8rrUcQhAZGhqWW5Z6K+dztPCW3g8TdkZN1MMb6AJgJ4GXpOwMwDcCH0iqvA7hM+nyp9B3S8vOYUTAvQRCaaImeA4I+LMfOBkst7pRbJpy/A/g9ANnjlwOginPukb6XAOgtfe4N4BAASMurpfUJgjCBtpCnnrqn3hlpE7DcFWV2nntEcWeM/QhAGed8g5UHZozNYowVMMYKysvLrdw1QSQNRu+0moOYUkgJeRw6GJ10uZzoc58C4BLGWBGA9+B3xzwHoBNjTJ7JqQ+AUulzKYC+ACAt7wigQr1TzvmLnPN8znl+bm5uTCdBEKlIYfnJsLJUcsvIlqyd0TJO6qOId10iijvn/AHOeR/OeR6AawAs55xfD2AFgCul1W4CsFD6vEj6Dmn5cu6kK0wQScILX4fPs5lKce4BcbfTcnfQ5XKqz12L+wHcxxgrhN+nPl8qnw8gRyq/D8Cc2KpIEKmL2QyBThKrWJFPJZ5vI4m8fPG+d6YmyOacrwSwUvq8H8AEjXUaAVxlQd0IolWRN2cJvvn9uejbpa3uOqmUWyb4Qp8652QETdZBEK2YtQdOJLoKccMXjw5V1b4TGZPtxA5VgiBswuwIkBQy3BUdqvah7qNIrFtGHsREk3UQRKsj0mOfWm4Z/187z8lJl0t78hUb8+rYtmeCICzHSWIVK3GJlrFv18L4fBwvrtqHumZvXI9rqkOVIAh7OVhRZ7jcCWJlFcFBTKl0VuEs2XoEf166C93aZ8b1uGS5E0QCUbthnl9eaLh+Krpl7O1QTfz1amjxW+zxzi1P4k4QyUTitcoy7OpQVQr69sM16oUWH00cOTooXhE7JO4E4VAGdWsXVpaaI1StPSfloKhfvrvJ0n3HRBKNUCUIwkYy3OGPZyrllrFrhKoTXDFa0CAmgmhFGAmb9mQdNlYmzvCAW8Y+y91JaI3HTWjKX4Ig7MOssKVSh6pdIuxU15XXF57P3U5I3AkigRhpteagF/uqEnd8Pnvi3A2vqbWHiop41YHEnbCFhmYv9h6rTXQ1HI+REGlZ6T6n+hyiIOCmsPiUnP52E68+ARJ3whbuensDpj+7Cs0eX+SVWzFGD7rWklS6nrIIey0SO5+Pw+vjljcWjS0xjixNUFtD4k7Ywrf7/JNvOd2KSjRGV0fr2jV54juE3U7k0/Na9DZyz7sbccqDSy39ze09Vothj3yG99cfsmyf8YLEnbCHVqbpnHN8tKEEDSbzhxjqkMayVLLcrW73l2496t9vhPXeX38IM55dJbTPw9WNAIC31h6MvmKqHlTyuRNEEvHdvgr89oPN+NPSHaa2M7IytS331BF3u97qeIRL9PuPtmC3YH+Q3MeR5rInxsXOF1tKHEbYglPD0eyiVsobUlbTZGo7k4Z7Soj7sZpGbD9cDZdNec2t/O21eP3XO80Vgx2sqk68PJVkuRO2Qi53Y8xGy6SCuF/5n+9w62sFtjX/Ri585SUViVpp8UqWu9u6hihehg+JO2ELJOqiGETLpKjP/dCJBgCAxxs8QdlCjhalUIuGGoqs5vH56+W2yS1jJyTuhC0Eh1q3FpWP7uE3O4gplaJllA2VUuijYdnOssBn0eAbEZ+/XMeYfO4JahdI3AlbCOQNaS3aHiWGPneNi5cKlrtMszfYUPk4x9Of7cJji7ZHta+qhmCudFHLXaQR8MgdqhpJ3IQhnzuRSmglSbKbxhYvnlq6E/XNnjgeVSa6MzWyHp3cofryN/uRN2dJVNdadnEoGyof5/j3yn147buiqOqjNI5F74SI5e7xWmC5qyBxJ1KCeKZffWvNQbywaj/mrdwXt2OqMRsAYrZD1aoBP7Eyf/UBAEBVvfnZhWSdDBV3S6oFwPgaKd2EIj/NZsldpOdzv/2NAny8scRcBeMEiTthC4Ep1OJ4TDmyoTnGzrl4Yph+QGOR18eRN2cJHv9fdO4Lq4nm/sohkM1epdDG9ktRNqpG4u5RLBPpD9Ky3A+dqA98/nLHMdz3/uYIlYt4GFsgcSdsJZ4+92TsvNWq8QMfb8H7BYfg48DFo3qELJOF69Vvi+yvnE0ExF1hub/xvfgI0BavD9e9tAbri07gv5tKMeejLSHLPQbi/sLX+wOfRd4W5CgetxTn/tGGEkx9egXW7K8Qrm+ioEFMhL0kQG9ZAkylaBsxre3eXXcI7647hK7tMgKiImNVki2riOZKa7llnvlyj/D2Byvq8N2+Chyt3oL9x+sAAPl5XQLLvT6xNzcRn3sgzl2qdMHBSgDAvvKTmDQwR6zCYR2qFOdOpADJaE3HgtmGxehB93FAPXbGyCpNBFG5ZSShjDa2vbbR34nbPitomx6pagh8Fr1GkdIUAME6Lig4hNrGlkDDYVc6AishcSdsxWGGpuOIFArpUomIqFVqN7FIW9DnHpu4t8tKQ1a6X8Jqm4JRO6v2lAvtJ5LlXtvYgmpFiOWqPccDDYf6jcqJOL+GRFJjl7Y3eby4cf5abCuttukI8cE4cRjgZmpxt7tGYsRyX7VCIc1wUhLy9pnpwY57xXWsaRALz4wk7qMe+wJvry0OfPdyHujz+N0Hm8XzvFNWSIIQZ8fhGnyz9zge+mRroqsSglk3lJG+VDe0hOU0cYrlHgvyy4hezH51fYuhcNZLaZXbZLgDZcrrKOruMfLezHqjIKyMcx7i8tl8qEroOJEGMR0/aS7ZnCgk7oSt2NV5xCSLNtm9Pj07ZhkuV2dOdEqcezRumRavD2+vPYjjJ5sB6Fvuo5/4Aj+bv053P1q/qZeluHtAfKCX0W/zix3Hwsp8nMOrCN9kFmW1/GzbUUv2oyaiuDPGshhj6xhjmxlj2xljj0vlAxhjaxljhYyxBYyxDKk8U/peKC3Ps6XmRFJglxTJj1Volj+bDmYCsx2qg7u3N1yuHjzjtA5VMzzz5R489Mm2wHcjn/u6ohO6y+T7XFbbqCnkoqNczV5Kny/0+ker7eq3O5syHwtZ7k0ApnHORwMYA+BCxtgkAH8B8CznfBCASgC3SevfBqBSKn9WWo9opdgluK6A5R5+ALseFjvgHOiQpR+RHCbuMSbYSiT7y0+GfK9rii5NhOwr/7Ywtlhzs5OF+DgP2caq37Zdee0jijv3I9+VdOk/BzANwIdS+esALpM+Xyp9h7T8PGbV+wuRdNgVCin/opLfBc0N08mqO1RjTY1rNUaujWaPz1DA5QlOtGifGd7geX0cL3y9L9ChGitmxZ1za96c1Ie1SxyFBjExxtwANgAYBOBfAPYBqOKcy1e5BEBv6XNvAIcAgHPuYYxVA8gBcFy1z1kAZgFAv379YjsLwrnYZGjKmpe8dqwfzo1zhatDIZ3mljHSx2te/B4bi6tQNHcmgHALVQ5p1ELLHFyy9Qie+nQXMmLJ0KjArOXt4zyQjsC/vTX3IpFuGXDOvZzzMQD6AJgAYFisB+acv8g5z+ec5+fm5sa6O8Kh2Odzl9wyTnC0I/rz5DB+LVcucjHnWe5GbCwOjSZRn2Z5rbkokSYpgsaq3EHm3TLq3DTRod7OLseGqfQDnPMqxtgKAJMBdGKMpUnWex8ApdJqpQD6AihhjKUB6AjA+YkYCFuwS3sDlrsztD2A6HO6ak85CopOgHNjt4xS+N0u5jifeyyTXFfUNesu09qr1b5p0x2qijh3PXYeqUGvTm3QsU268H7tcsuIRMvkMsY6SZ/bAJgOYCeAFQCulFa7CcBC6fMi6Tuk5cu5U8wrIoyKk014b11x5BWjxG6fe7KmN/jZK+vw/PLCyJa74rPbxRxnuZt5sk1FEmns1+pBoeZ97qHirrX5Rc99g+tfXmNqv4m03HsCeF3yu7sAvM85X8wY2wHgPcbYkwA2AZgvrT8fwJuMsUIAJwBcY0O9CYu4551N+H5/BSYOzMGArtmW7982yz3glrFn//HCx40nX1YKf5rLhcaWRExEoo+pyx+jhlltuevZnHrlPi42zmBbaU2E44Z+T1iHKud8C4CxGuX74fe/q8sbAVxlSe0I2ymXRsc5zSKMhFGHajKFZnHOwyJilCgXOXGSZjMv5WZqr3lf4+SWeV0nTt6nGqEa/Vtj6HZ2pamhEaqErdhlWLsCPnfla3JymvHqiBglLMRyd464y/Uy47fWEudTe3YQ3t7q09dzy+wrr9NZHyHRMtGm8g233B3QoUoQZrFPcCO7ZVq8PrgYs93inTJ3OfK6to1qW66RHEyJcolRIxBvgvdV7P5+s7cce4/VhpVnpGnbl1q/G6PrFA16YyT0DuPz8UBeG83lgj91daNiVygkiTsBwD5Xhu3RMgbrDH7oU0wa2AXvzZpsTyUkSqsaUKrIJ24GjvC0vkqUD76TLHcZ0ft7o06uGL1zio9bRrvyer59H+chA6jUW4vm/VGPVbCrQ5XcMq2cZHVlBHPLGKcfWLNfP0eJE/APYtJfrg6FdApm3DJGvzEz1rjVp69XLX1xD02ZoG4c9BoLtW9eHc6asFBIwvlU1jVHZTn6fFzXv2gVdrcd8u4Ly2oDWQbjOm9rjAfjXDwU0pGWu4BbxqgB0GuwtC6r9XHu2hXTdcuoOlT3HAvNlaP3U1CXq6dKJLcMocvkucvQ2OILDPPWgnPuFxLFw/TiN/t117cKu+LQAx5f7m/czn9mlS3HiYRauMw+qD7ODUVLeb+cZLnLiLRtRu4KvTBQrd9NvOLc9S6zuiH/4+Idgc9lNY2YPHe59naq7x5VZFrCEocRzqexJXIY42OLtmPgg0tDypSzGNllPSinKbMDDh6WSCqeaepitdy9PuMRqkrSHDi1m8hAIKN1zDRYdodCen0cf1y8A8dqtNMiGL2BLN9VptuIqU8/zCCIVNEocd6vhbCF178/mJDjXvLPb23Zr/zA+HyJHcgUax4vb4Q4dxdjuDq/D4DUtNzN+dytPv/Qem0uqcL81QewaPNhzbWNGimjqkVqABOaOIxITZI7E7MVg0msqEVsx27x+gxHqDIG9M/xjxyONGuTU1H7mJXoRQrJm9Q1eXDoRD0A60Mhr5j3fcj3TJ2wTBmjhtwoVj3yL4TcMoQFKN0IySztMnJfQuKOH9v2zR4f0gzCZVwMuOPsU/D6rRNw/vDusR3MBmSr9JH/bsNTn+7UXsdAFXU7IaW/d7y1AVOfXgGfj5uKlrnj7FPEV5bISncbLjc6D8OHKcKPxK4XMhL3VkaSRj6GEXDLcI0UqnFstsIGpJg89p5jJ5GuerqVE1Uw+AdhnT0k13LL1Qrk039zzUG88LV2B72RWyaSy+L7ff6Essfrmky9aZ6Saz5PUixX97mv9uoui/TIUZw7YQnKh0n5m0o20VdWN5Gx+upDR+OmSVdZ7h0U6WKV98iB2i50tlGJu1Tcs5PfFVVS2WDq/KPpn4jUf2J0b/VCkZ/5Yjf+sHC74X6pQ5WwBL2fpxlJKig6gee+2osmj/5QbC3qmjwoq200tU0kjtY0Rqx7i9eHksp6S48rE0s+cxm1zz07M+gecHq/iNfHkTdnieE6T3++W3eZnqDKQto23f8WY3a+VVFxtzs30fPLCyOuY1cQFMW5tzKUv1+mUx6JK//j74jadbQG824YJ7zdj/+5GvvL63D71AFom5GG30wfIn5QFcr6vrPWOB/9HxZux7vrijH/pnyc2rMDenVqE/VxAWDt/gr0z8lGj45ZlnTlqi33zLSguLtCLHfnCb06ZluLDzeU6C4TFVSPjyNCf2cIopE1zV5f4HpHstw/3XZUvAImsMuNSJZ7K0P5aqkUi2gs0GU7y0ytv18aDfvSNwfw3DJ9H6UIyvNYe8B4oq/lu44BAG57vQBTn14R03EB4KcvrsHFz3/jr4cFmZLVI0+VURvOk/NQKutDxzHIkS2i6A7Zl4qDE6Gb+32KinuTJ3gDIz0D++0azU0dqoQViA6RFsEpY2qaVIO41M+1UhdEkztF4oQ0RZzaDxuNFaaOllFGbSgbYCcK/R1vbQj5PvXpFVixqyyiRZ6d4T9HPRFWjkAG/Ja7mf4M0Tm0m02IuxlW7z0uvC6NUCUsR/mTiuaHncjoDWV1GyP4/u3sb7WirUhX+dyz0oOPpcvhHapabCutxv+2HDFc5/T+nQEA5wztprnc6+OhwqsVFmWAsFtGcQwrfyc3zF+LSoM5YpVQhyphCVZaJ4nMLx4i7hHSL1jZUabelyUdqqpXoExlvHWyKLoCHweOVhsnspPfSNwM+PmZAzTX+Whj0FevTpMbiWjE3cpnAwDqW8QCDmiEKiEM5xwvrtqHo9XhkSkhv1/Fjyoqy10S96r6Zvxz+V74fBwfbyzB9sPVEba0lqYID5GVD63arWPFrtWWu9Ln7sCMAxHxcY5I/az9uwQnN9ETN6WgH6tpRIsJgReNlmn2KsVdePdCtHjEOmRoJiZCmKKKevx56S4s2XoUC++eErIsVNuDP6qofO7SU/ngJ1uxdOtRjOnbGfe9vzmaKptG6X+taTQOk7PymVVbkFa8FaS5GYrmzsRji7bjte+KQqJl4jkgyyo456byqehFAaW5WOA+P7lkp7n0C4KXrUExs5LVlvvn28Wia2iEKiGMbF3WamRk1J/Z3fwPWxb3k03+B8SjN29ZAlCep9lICyPCLHf1CgIP6rKdx0K+y24Z+folu8/dx811XOudlrpP54jGm6jZfaopq23CttJq5M1Zgu2Ha4T3L8JTn+4SW5HyuROiyIKgJdjKEqX7JBr5S6TLIFJbpNQWSy131Sw60TSKt71eEPJdnkdU3ndotExwvWSx4r2cm4tK0jmteV/vC3NZCe9SsCV8bNH2QMroR/67LWz5neecgnkr90VVB1Eozp0wTVFFPYY98mlIWWVdM15atR+cc+w6GpywOFmn29OCQT3y0Lp9q99OrHgpkOPcW2RxT0ueEapa+CK4ZdQuGT1xO3A8+rhy0atWWtVg+5wDkbDLSCLLPQVR+oUbW3whFvrsD7ZgXdEJjMvrHLJNNALo5OYgxHK3tUPVCp+7vltG+dwnWuf3l59EutuFgxXGA5W4WbeMDedllGbYadjVgJO4pyDNql76X76zKfD5RL0/9jbcvWD+OGaErb7ZXG6QyMc2Xq60HOuazeXAMUIdsWFGQ8Y+8QWum9gvrFx2Pcj3JDRaRv/B/9E/vsHgbu3x7E/HiFciBqb97Wuh9arqm9FRkfxMTTzaKKNIlVunDMAr3x4Q2k882ggKhSSEUUd0eDU6F9U/qGgsUHkTkW2H/+Fz0/s3PHaE9warHkrOOb4tPB6wRL3e6MW9sr4F/1oR7r8N71DV9rmfOahryHbbSmvwyaZS8QrEifcLSvDSN/riqbZUjbRNPQm1KEZx8X/48XC8cesE3Hv+4Kj2bTUULUMIo07m1KIxUKNI5c+0wnfsFP/w88sL8dYa/WkFG1u8whktX/uuCNe/vBafSUmjlD73vDlL8MWO8HC3uiaPKbdEmtpyV7plFJc0p11mYMq9ZEb9KzH7s5l/U37EdVp0Au2fvGwkAOCsIbkY1qODuQPbBnWoEoK0qKzLZsV32Yqf/eGWkHWiyUOeSK9mJIv5T0u1ZwUCgGGPfIYpOjPVq1m5uxxAMFeJ2iL8j2qCihaPDyMe/RyX/etbPPG/HULHkN0ysssnKyQrZOiD78R5VKMhNDupuXPqkp0RMqGJFmq3o8wNk/oHPrfJMJ55KV6QW4YQRh3RobRi9ELR9cTS5+MhAz1Ct0mMvHt9HJf+K7aJt4+fFMv7EcT/BIbnFQ+9Bo3SW9LW0mq88u0BoWsUcMt4w90yauxKMhVP1KegdUqR3lAiXQaRMRdtBcRd5HLHeksotwwhjNpqUXaw6rkL9DTo4YXbcOofPtMcCGT1cG1R1B3G8UB2Z13+7+8M1zte2xTyXb7eRgOpjDpU1aSC5a621LXOSJ3jPmR9xiLmNWrWsdyVKN+Q9GgbYV5VIPYEepQVkhBG7W9UftcLEdOLS5YnwnBSYJlVz8Lr3xUJTTYBiIf27TgSOsrxte+K4PVxww4+2XJvkazNDGU+d9XJpoLlrlRzxqB5Q42uNkNka1fkvoqkrO7fNfJcrLEm0EuYW4Yx1pcxtmvneXYAACAASURBVIIxtoMxtp0x9mupvAtj7EvG2F7pb2epnDHGnmeMFTLGtjDGTren6oQeaiFRftezICNJl+Zo1ySKJdbi0UXb8XaEWZwCk0VEea5PLtmJjzaWGDYO6ZKYy+uo87srSYTlvnJ3GV5apT35tRVonZHRwCLGInfe9xaYbSvStWyb4cZUVYSSFrHekkSOUPUA+C3nfDiASQDuZowNBzAHwDLO+WAAy6TvAHARgMHS/1kA5llea0ITWWz1IgUA85a70XJ1Sbxkx2z6VyMq68V872bdWUpqGz0Bq1yLdEkdnrtmLK6d0A+jencMLFNf00SI+82vrjfsoDaL+gwmDugS8z7UXDCiBz68Y7Lh9Rrcrb3hPnY8cSE6Z2dErEusb1MJs9w550c45xulz7UAdgLoDeBSAK9Lq70O4DLp86UA3uB+1gDoxBjraXnNiTBk/VFHyyjRtSAjiJQsYiETTSfIcP/pC99bti9Rd4uuuIsex+CeyJb6gK7ZeOono+B2MZw3THsSCye7Ze465xSh9dSncMagrtj2+Ayckht0gRhdLwYmJIj5eV2QEeEt6OGZp2oue+LSEZEPIDGmbyfhdbVwRLQMYywPwFgAawF055zL060cBdBd+twbwCHFZiVSmXpfsxhjBYyxgvLycpPVbn34fJGTMcmWe4PBaFDdqJgIJui0/1uJt9YcxJl/WREou2hUD8Nt1PWyCiuz9xk1hIByoFb0x2AwfttI00iOJUfMqB/8yafkRF8RmzHqBFWi5YZol5kWMtm6UbQLY8DlY8MkRadOxsrZrYN2GuEfndZLaP8A8JcrThNeV4uEJw5jjLUD8BGAeznnIU8X9z+9pn7+nPMXOef5nPP83NxcM5u2Sn4y7zuc8uBSw3Vk/ThwXD/3R6QJifU4XN2Ih1VZ84xC9pT82+aserHgFUxTHEuuEg7jN4R0jZ49vXEHZw/JdexAprJa8ZS8Wuc3pHvQTSI3hnqzND1wkbbFrSZSg6OXI16Z3ycSIiGVRtg1F7HQbhlj6fAL+9uc84+l4mOyu0X6WyaVlwLoq9i8j1RGxMAPh6oiriM/MKVV+uKuZ0FG02EoatW+u8640zKRhCZZ82Lin7/Cit1lgTLZco51Ym2jfhCzr+VtM5yZEqpJdOYhgfOVw0KnDNbu0BSNUIkk7p3bavvUMwXCJGXU0ySaJWGWO/N3S88HsJNz/oxi0SIAN0mfbwKwUFH+MylqZhKAaoX7hrARWWSNhCjaaBntbXjIXwe7g3VRjgkoPlGPYzVN+NOS8M5DvcbvhMAkyAzAnI+36C5v0Jgm0OiBj8ekKAcr6kwne9MbFTr/pnxsfvSCwHeRn8mgbu0AAN3aZ4a5Vsz8ztLTjFfWG1NgpuPaHWXOeZlE+tynALgRwDTG2A/S/4sBzAUwnTG2F8D50ncAWApgP4BCAC8BuMv6ahNayPpjZGTquRdiSRwmkyyTSShRWu5GtY/Vcv+2sEJ3Wbf2mWFlRukglIO4unfIxMkmDxZtPhxT/dSc/deVuOmVdaa2afH6MFAjLvy8U7uHZImcf/N43X3I6z1w8TC8e/skjOjVEavvn2aqHkqMOlQBbdfiqtnnmjpGNIOYrldkB01YPnfO+Wro/+7P01ifA7g7xnoRURCwpA2EWm+RFfnc9QTJyRa9ls9d6/rFKu56bHj4fOS0Cxd3Ga0GUynuWelu/P7DzVi69SiG9Wgf4reOlfVFlcibs0R4/RYvx9JfT8WwRz4zXG9c/85YvEW7MVp49xSsPVCBzDR3oPO4u6rT04wREcktk6nhWzf7e9Wz8hnTf67S3S706piFw9WNoMRhrYBVe8pxrEa8U0qN18fx302lESNAtFBql8/H8ckm44E3QPgP18qGQ5QbJ/XHTZP7R15RhxDL3eAZs3ryZBk9YTc6nPL+ul0skBbXyjTHSroIxHr76+VDVrobrxpY5oBqAhLVsryu2fjp+PCc9yHbm9DCGSOMI7q0UhCYHXGaplh/4yPTA5+3PTZDdxvGgsdxRCgkYS8/e2UdfvyP1VFvv2jzYdy74AesLjxuelul1f3JplL8ZsFmvPxNpFGJ3OBbkJLKBtP10UOdxOyikT3w+KUjo95fyYkGhSXsf8q0zkMwS4HlaD34fboER18yBJOZqd+cOOfYX24+H7q6TRedx1TuC5D95XrEmhrazOa/Pm8wCh4+X3e51rmZdZMoGwPltlohrjJeHw+MWaDEYa2EMlXiKTNUmM50GET5QNdLHXxFEaZT+2L7MVz2r28DFuOnW+3vN//ZK2st3d+6ohN44OOtAIKiEZqO1k8slvsTi8VS/yoxOtxvpw8NWU+e4FmdUO3DDSWY9rev8a3Jxl7dYavcr9FApRaPv9KRXCHx9NK5XAxdjdxeGi1FLAPFGGOYPWMoHrhoGDLT3Hj/F5M11/P4eOD3Ztc8CCTuKUQsr+XKV/EOWf6umEhWX0VdM344VBUI83tv/SHD9dXkzVmCvDlLcPZfV0ReWWJ9UWVogQXPxRc7juKppTtR2xgeHSJfFa+PJySXjtbpKROL+TgPWO7yfXhzzUHkzVmC7/b5O3H3HqsN34kB6m6IxpZgQdd2mbqWbbN0fKU1bCZeXJRoOu5vPiNPfP+q3V8yWnxAE2PA3ecOwi/O9jeCE3RSK3i9Qcvdrt8ViXsKEYt1qdw0W4qjXnvghOBxoz4sAESccNkIKyJ0ahs9eGHVfvzti90AtB+2Z77cg482xm+4hujkKV7OA9dfjjN/47siAEBpVYO0L3OoLXdlqCYHsP+pmYbbpSsan5W/C488iTn/eRTbP3bJCBTN1a63GvXtf/7asbjidLGBY1pW/2f3TsXy354dUtbi8ymS0gnt2jQk7knOkIc/DXx+btneqPejFBOzYzKssjy+LTyuG0URD77Z63dfFFXUo0pKKKZ88H73wea41ymSkCl1+LqX1uKedzYGLPt1UuNs9vZohdHnSJ2qF430d1BueeyCkM5DAPjZ5DwAoSNue2iMAGWMxWXi6WjRmi/gnmmDwsq6ZGdg/UOh/nytt5phPTpgYG5oP4TXxwNmCVnuhCZWTVyh/H2Z7Ty0yvK4/uW1uOedTWHlXh/HZ9uOaj4EdkUayFPkGU2yYZaJA7qgaO5MZAsMVxd93tXXZPGWIyFuGyB2yx0ALhjRHQeeuhi9pFS6HbLS0UkRu140dyauzvcPTBftgI0Wu/a+YNYkTBjQRbNBStNQ7cw0F3JVYxRE3yQ9CrcMWe4pTqJzo/tCxN1cXeyo+4pdZcibswQllfV4+Zv9uOOtDVii0WFr14Mud1JaGQIpi665PRqfYbNGS6weuLPE5NuQ1kC3dLcrrONPr2G1Oy2x1Q36ddKAookDc/D+LyZrdghrRb5oVUO0bh6fTyHuZLmnNIl+TVUKtFlxt8PykPPRbCutxmHJd1weQySRWeT4dysHL8miIT/Mv7tgiO66oket0Eh/oLbcNxZX4er/fI/CspOoa/Jg1Z7QLKycc3yx/WjgLUXrnHt2DJ/8Qi/Kw47ojy9/c5byCLrrPXDRMPw0v6/uci3+fPmoiP540VGoopE2XkW0jF3i7swMRK2QRLsgQ9wyJn9sVo/ebPH6AuKa5gpajC9qzAYkIiQ3TOqHt9aYS14mH9/KB092V8iX6+Yp2hkPlUQ6Pa3qaeVLWVd0AnM/3YUTdU3YWFyF1fefiz6d2wLwj2u47/3NeOLSEejZsU2gv0HJAIHp5uxksODIWzlKxWpE30ZE27UeHbNwuMo/YNEuw44sd4eQaLeMskPVrJ9ZVAA7tU2PvBKAk42eQFjf+qJgxM6R6uhG7z5+SfggpyvHGUc/yGkJ7LDc5Xut5ccV5Q4DEft+n3Yem692HsPGYn920cYWL5bvOgbOeWBsxaET9bj9jQLM/jA8yZnoKFXTRGHlJyKdhai4i1ju/77+dDw8czimDPKnVxCZ7SkaSNwdgp6GHD/ZpCm276wtRtHxOluOb94tI7Z+pCROMiebPIEMgy+s2o/iE/qhkiIPuosBPVT5SW6YZJyyQB7ir5XJYWCuf8Yks8ghpvLlNRKMSJd0zkXDdGO365rDs0yqeeP7g7j1tQIs3Xo00MgYTSjSRjB3fzyQr9rfrhqNt26bGJdjiot75HUuHtUTWelu3H/hMHw9+xyh+V6jgcTdIWjFNZfVNCL/ya/w96/2hJT7fBwPfrIVl//7W+uOH4O4i66vlaRJi/pmb0jEhtEkEPKztPiXZ+qvwxj+e/cUTMgLDiiJZDXL56TVsE4ckINrJxjnP9GibaZfIJ+/ZixG9u4gZLkbrRGLBSuPPj5W0xgQLqM3tjYZ1krFSGmeWK0skqJcMa4PztTJ9241WuKudAmeMzQ3rCwSaW4X+ufY5+4in7tD0LLU5Nflr3aWhZTLlnKVwQzxZlFa32Z97qKri1ru9c2ekORYWiNH1YxUTCqtRY+OWZh8Sg7WSW4eo7wfAAIJ3LTeSqINj5Qt95mn9cTM02KfVjiaVLMylVJHrDI1wieb9Adpic66Jco14/siv39nYV+6ErWAvnjjOMx6c4NVVdMkkrvlPzeMQ1lN/Dr8RSDLPQlQSglXjEi00vWoPIZtbpkIs9vI1lFDizekDjUGjZgZfVPuM9J8FyWVDVi997jmtYh2yj3ZcrcKs9kLlWwtrQ4rq1E1ol3bBX3BVs/+xBiLStiB8N/9BREyP1pBpLesrHQ3+uW0tb0eZiBxdwhaeqElXD5uT+hULKGQomKnDtFTcuZflgeOe91La0PER2umoiDiAqf0KYuc4+8+2IxSjYyW8vX/65ViEyNfLE0knm1KICPXr7reujc3LZSTRJv1uT99xWkRU/9Gi5M7VJ0EibtDMBJstfDK61oZTxyLz1101rdMA7eMUVpgZeIqIxbMmoQvQuKhQ1FOzCESuXO0phG1TeEuIdktc5VAPPWMEd0D9ymaiZSN7vFuKSHYh3doZx7s0zm2jjrl705vOjoAyNOwWK8e3xfnDusW0/HN8NGdZ2DJr/T7XWLFrsyNdkLi7hC05FQeyqwUXp9Jt0y/LmKvinKD8c7aYry33lxMuGhjYGS5m0H5nCk/TxyYYzgTkWy5X53fB30Fr4sWWhE0/3fV6LCIHMDvzpBz0LfPEgsFBcT6MR67ZATuPOcUjOvfOWzZ45eMwJJfTcXQGGZmUjaqei6gVbPPxSKDzmw70BriP65/Z4zoZdzv0togcXcIonlTODdnWYsK6roDJ3DoRD0e/GRrYGYfUUTdRFa92io7t8zsUb5usYqAVofqleP64IpxvUPKsjPcuO3MAaht9LtPOgvG+SsxOr8xfTvh/guHgTGG808NtZK7d8hCxzbpmnlSRGny+BulqQYRKf1y2qKDiUbLCpLQiE4IJO4OQSkX8qzzgckjlAOMeDCvuIioRpo4QebTbUfxrxWFYpVVoVcPrVf5aRa8qkfbRsiWu2yF6uXajsT04d01y88eEnpu25+4ECN7d0RNg/9+mrLcTdbpH9eeHvJddgFpJQETZVz/zrh+Yj88Ldi30NpweiND4u4QuOIZHP6Hz1HX5NF8/VS6ZUQMeDMZ+rYfrhFeN7RO2uVj+nYKK2uxYL465XUx4wv1ekNHhr57+yTs+uOFwtt/eMdkLJg1CZeNDVroPRWW8YQBXTT31+iR3TLmI05ET69Nhhun9fG/kTx52ciAtT0yhrcUr4/jT5eP0swrQzgfinN3COpBTMrY7hCfu8+cW8bMEHf5jcEsevUZ2btj2IQf8sjTP10+Eh8UlOCHQ1Wmjxfic9dY/s/rxuJkowf9c7KxsTg4c9M5Q3OxoOBQQATdLga3S7yTMz8v3NJf+qupqFTkYtGKB3/hxnF4f32JbSMRZd68dSLKTzZiULegn/13M4bivFO7IzvTjZnPm5ufV6TDOBE4xWJ2Sj30IHF3COKhkOame0szmHnjuWvG4OwhuRjzxJcAgLqm0JDDXh2zcFggn4ueW6Zru0y0SXeHhDIGJlHObReYAMIskQaUKEP4Jp+SE/h80aie2PnEhWhjMmplYNdsPDTzVM1lnbMzIuYGGdajA/7w4+GmjhlNrqGObdPRUeXXT3e7onI/zbloGLIzSR6SGXLLOASjR1m5TOmWEcFoVqUMtwuZioFFdSrL3S3o0tEbsckYkNMuVPhmSANO+uW0jdry0YuWEUFE2D+/NzSc8pRu7XDeqdp+druYPtx/nU5RzeATL0RHEyeCZAxLTATOvYOtDNGIEx83N0LSqEOVMRYijmr3ipHVr66TFpyHRshwALedOQDbHp8Rkx83NFrG+gd9SPdQQU1Ews5rJ/TFtsdnIM/CVLtv/1w8yVa6RWGrdkDSLoZz72ArQ0tA5DLlKzrn3FRuE6PwQxcLtXzVdRANXdRrmLw+X1j+E8YY2kmv+znZmVqbRcROw+2qcX00LMP4q7vyOlmF3NdghDxHqtGAs0TjFMPdDsPCSpx7B1sZ6g5VxrQzRXo5N2VJGnWouhgLsYKVx7tlSp5wZ6xefTw+bpj/5OEfafuxZRbMmqRZrtyjlQ/6tsdnYO4V4WF/iZ4lyyrUfRV6fTqAdQPOiMRBPSZOQSUgnAeH9Yf63M25ZQwtd1eoUCpDot2MCVvuevXxeHlIA6HuJGyflY7uHTJxTCebnrpzMFhvay2mr+47G20z3LqWcopoO9pmuHHdxH6oaWjB4i1H0LNDeIe5HKkqOj4iETjdYnYKzr2DKcKhE/WByZaNUAsIl/6pF/oUuWVEMBJoprLclZMt+8MExeeD1KLF64sY2XL3uYP066fzEMeS6laLQd3aoZdBmGKiZ8myCsYY/nz5qEC6gu6KGP2sdBf+c8M41EijaXPbR+cyiwdOccs4HRJ3m5n69Apc/Z/vI66nFmwf1/fDmxEbt0GnqEvVoRqyzMVijkqorG82jNYBgEvH9NZd5mLA7y8cGlYeS7SMKM9dMwYjenUAkDqWu4zc4A7rEYyHX3j3mbhwZA/MnjEUV43rg3yNfDVEckHiHgd2HIk88lOt1z5f0LeuDoU0M8jTKJrR36Gqbx3HqpvHTzYbNi5A5DeLu84Jt+yZzdEygL/RuW/6EFv2nWjky+d2sUAqBTl3+/i8LvjrVaMtd31ZSaJq9srN+Xj/F8EMnE5/g4joc2eMvQLgRwDKOOcjpbIuABYAyANQBOBqznkl8z91zwG4GEA9gJs55xvtqXpqEeaW4fodqmbcMkZuEcNlFjzcnHPDxgUwdrHovlXE6aGSr0GKeGU0eebq0dh1tBY57ZzrhgkjQaI6bVj3pHLRiVjurwFQJ8yYA2AZ53wwgGXSdwC4CMBg6f8sAPOsqWZyYuaHoF6XIzhYKSwU0qIfmJHloQ6TNMtDF5+KuVecFtFvb2TY6zU+IXHuNj7o8q7tmBzFKbTPSsd4jbQKTiaRHarJNIAqouXOOV/FGMtTFV8K4Bzp8+sAVgK4Xyp/g/vVaA1jrBNjrCfn/IhVFU4mWrQSf+sQ5pbR8a3/7oMtmnnDdfdrsMzIco/VLXP7WQMjHkM+jh56S+wKhQw7ThI9yGZIzbOKP89cPSbRVTAkWp97d4VgHwUgj83uDeCQYr0SqSwMxtgsxlgBY6ygvLw8ymo4GzPpVsPFXWG5K8p/OFSFz7YfFd6v0uq8ZHSvkGV2u2UA1QhVjZbGeJCV9rJUFV1CDKfcfq1JUpxEzB2qkpVu+r2Vc/4i5zyfc56fm5sbazUcSYvHhOWuuoTFJ+qxSZHRMFqUUYq/UXUQGum324JoGXk/RhgdQz+SR7GOjXaovOcU9sokJQ7RdscT7SCmY7K7hTHWE0CZVF4KQJkntI9U1ippicFyv+XV9brLzKB07ah11khYrYiWAWKbfUmvevHyucpvDlod26kANVrRc4/B+AynEK24LwJwE4C50t+FivJ7GGPvAZgIoLq1+tuBYO5yEezqtFPuVu3mMNJdl4tZImlnnJKDlbv9bjezIqnfoRr8bK/P3f835UTQKX6NKEm0W65o7syotnt45qkoqqizuDb6RHTLMMbeBfA9gKGMsRLG2G3wi/p0xtheAOdL3wFgKYD9AAoBvATgLltq7SAOVzXoRsWYmXXIOOWvuLrcf+GwkO+XjzUaJGRkuVszMvP2qQOj3lYoWibqvYuTcuKe5CRr0/TzqQPx5GWj4nY8kWiZa3UWnaexLgdwd6yVShZ2HK7Bxc9/g8d+PBw3TxkQttyUuBsIiBlxGZ/n7+RJdzPs/dPFIcvUnaSG4m6R5R6LlaW7aZye7oDPPUXdMkRqQyNUY+DAcf8rlnoqORmPmVk1LBIQt8HAm3Cfu/5+lA3B364ajTdunRBz3cxawPqDmOIT545UdctIJOtpJblXKW6QuMeA7CfXs4DNWO5G7YC5FL/+W6q1ibojMtLoVfm47bPS0KuTeGy9VYj43JP3JT1xJPsVo6yQYpC4x0BA3HV6Js10qFplHUaanCPku8Hdd7OgWybd7Yo4GEnJL87W9rMbZV7UQu+IZuoSC4OkKe6cOlF0q4W0XQjK5x4DsiBr/dYKik7gnnc2Bb7vKz9pOB+mVX7doFsmfH9q/3fEQUzSPtLc4ul/ASBXJ0/JE5eOEN6HUf1YnNwy3TpkRR0ZkQykqruJ8EOWuwHri05gxa4y3eVBt0ywrLaxBU0eL257vQBHa4ITIZz3t68NjxUPy10thEa66HYFXTtpLnOWu15+97YZ5mwJkcRhZMS1PsjnLgZZ7gZcJeVh17PeZA1TumVGPfYFxvbrJHyMPcdq8a8VhbhVI9pGxkxIojzzkdYW4dOsGUXLuAINTnqk1I4qTPUjG6BXP3q4rSFZr2OSVjvukLjHwBvfFwEI7+DZVFyFDllil/ahT7ZifVElxvQVbxCMSHOLR8sYeVoyFIKeZnLKNasGZOnVLzRahh51gtCiVbtlPt9+FM98uSeqbQ9W1GFLSTUAbRGqaYw8tR4A5GT7/dN684gC5kLW0gx6Sc343NPdrkA/gNn0vz6LTHd9yz2+g5hSlWT1uVODLkarFvdfvLkBzy/bK7z+ySZPQLiaPaHzjUZLtw5+cS+rbYywphhGETDqZ8JI3NPcLkWHsbkkYmYm8DZC33K3ZPetlmTXxiSvftxo1eJuhsYWL0Y++jmeXLITQKiAxfKwtMv0u29OGlj6ZtwcmW637rJwn7v+ftLdwTh3xoDsDP39qrHKcteNllF+pie91UH3XAwSdxWcc7z5fRGqG1pCyuubvQCATzaVAACUCR/NviZeMe875M1Zgu8KjyMr3S+aDS1e3fX1ok+0yEgzmhBb9d3ABM5wu0LcQZ3aZuCze6cK1cEqy12P0Nwy9KQThBbUoapiY3ElHlm4HeuKQnOpqyNWlKNPjWYT0mLDQf++X159AGOljlS58dDCaJkarciWn5zeGw3NXo0Rqvr78btl/Ocsn96wHh0Cyz+68wxcMe87zW3VbdHlY3uj2WC07tlDctGtfSY+2FCiql/kxGFELCSn050adDFI3FU0NPtFqOJkaAenbD3LwtKk8LkHU8Oae1g45wEr18gtY0bc3S6Gu889BRcM7xEok6cDa1S9HRh3qDJ065CFXUdrkZkW7pI5rU9H3W3Vbplnf2o8Hdnrt05AY4tXQ9xD1+uSnYETdc0hfhnSefMkuzjSPReDxB1AfbMHh040YGiP9rrryJan/MNq8gSFUhZJPffJZWN64dt9FWHlHEEh3H2sNpqqh8EYw+wZwzSXifjc83LaoqiiHhluF5776Rh8ueMYBnULH1lr1DDkdc02V2md/andXSywrundE0SrIyV97pzzEPGNxF1vb8SMv69Ck8er6y+W88QcP9mMs55egaaWcMtdLwtku6w0TeFfubsczy8vFK5nrIhEywRGpbpd6JydgavHa+dVMdLXa3S2MVM3f/2066asd7KG8zkBunapTUqK+/sFhzD04c9w6ES90PryTEHNHh8adFwgysmui0/U47jCbWNkuae7GdJcrsCyL0xMbm014TMxhSuqXM9I/Qh6i/f+6aKo4pBFLHctKNe6eRLl1ljyqzPx6i3jY94PuWXESElxX7rVL6CFZSdNbdfs8QX80sofkMfrC/GxA0BRRbDhcLsYquqb8fHGUJ8xALTPSoeLsYBoznpzg6k6WYnICFXZTWQULw+ECu9ZQ/wTnGelu5BucjSrUV1EIOvTPIm6ZiN6dcS5Q7vFvJ9k7zOIFykp7nLESHVDCx78ZGtYWKMeTR6fZufloIc+xcznV4eUFZ8IzoXodjHc+dZGPLJwe9i27TLTkOZmONnkwbeFx82chjCrZp8rtF6YD1vLcpczQUZSdwXPXzPG9DaR6ma8bvAzaXv0kAWc2qRkh6psPb655iA2HKxEu8w0lNc2ISc7Aw//aLjuds0eH+qbxdIGFB0PWu4+H8feMu0O0XQ3Q2llAwDg+pfXip6CKdoJ5rFRo2Utd26bgWM1TYEcNSLIlmAsI3UB4MZJ/fHmmoPCxyNiI1mvIzVKYqSM5e71cTy5eAdKKusD4i6P7Hxx1X58sqkUL68+YLiPQ5X1gRGokShW+PO3H67BySbtRiHN5UJVQ7PQPqPFTJy9clUtP/ert4zHny8fha46Odm1kK39WMX9j5eN1Cy/feoA3fpYMYl3a6NnR/+sWv1zzEc1OQHSdjGSXtxLKusx+all+N/mw3h59QGc+ZcVKK3yW8pmBxfdOH9d4HNLhFmUlGK+uvA4Glu0B+m4XSwmd4UIZnZ/4KmZ6NelLQDttAY9O7bBdRP7mTq+TzUGwGoemjkcBQ+fj05t0wGENiIk7eY5d1g3vP3ziZh1lvaMWURqkPRumfcLSnCkuhHvrC0OlMkjQPWiQfYcq8XQ7vox7UD4gJ9ocbuY7a+RZkX1ndsnYuXucrTPSjd9rIFds7H/eF1IWYc2/v3cN32I6f2Z4c3bJmLFrjLdCckJx/DbxgAACSdJREFUcaYM6proKkQNZYUUI+nFvUkWYY37Xd8S7ir51XubsGTLEXx4x2TD/YqMCm2T7jbMCQP4xd3MXKrRYFbc+3Ruixsm9Q8rv2lyeJmaD+88I8QlBQBZ6e64TEfXu1Mb3DCpf4i4k1em9UHSLkbSu2VkC7vZE+4W2VZaE1a2ZMsRaVm14X5FwigjCTvgnxnJKK9KLOT37wwg9g6mwd3a4bxh3fD4pdo+byVdsjMsm1hEi9EGaQ20IXUnCC2S3nKXfd0/HKoytd2xWv3JMazErSHuPz9zQMTOXRHm3zweu4/WBjJLRsuX950dc12sYuE9Z5panyz31sOPTuuJxVuOULSMIEkv7pX10UWizFu5T3jdh2eeKhxFo8btYmhpDhX3ztkZhtsM6d4Oe46dRO9ObQKdw0r+e/cUFB2vQ8c26ZgwoEtU9UoVnKjtb/98omV9NkSQZ64eg8cuGUE+d0GSXtyttsAvHNEDvTu3wXyFZX1an+jdEG4XQxuVZa034bSctOv3M4bh/OHdsflQFW6cvzZkyr5rxvfFmL6dwlwjy357tmWTZIjy+b1nGeaPt4vfnD8Y/9t8GIAzLfdk7qx0MhlpLlMhuq2dpPe5lwjmjxHlqvw+uPf8wYHvv50+BOPzOmuum5fTNuL+3C6G568di45tgpEpZw/phh4dssLW7dTWb9G7JfEf3bcT/n39uMDy124ZrxsLfkpuOwyOEAFkNUN7tMeAKDJAxsrA3HYY0t2fqZJyyxCENkkt7t/tO46KOmsGCE3I87s3TjZ5kJ0RfKH55XmDQ14DP7/3rMDnlbPPRWcp9lpP5NyMoWfHNtj86AUomjsTRXNnYmiP9lj0yylYMGtSYL1zh+biz5ePwug+HQN1AYDO2f79P/rj4ThnaLeoc7ekGk/9ZBRO79cpIY0LQSQDSe2W2VRcBcb8Ps6TjR7dpFzd2meiLIL7Zv7N+fjnikLMGNEjMP2c0p0yfXh37Cs/iaE92uOqcX0CE0t8+uuz0OL1oVPbdJz+xy8Dg59kP71eytxu7bPQrX0WXr15PBpbvLhoVE8A4R2KI3p1xKrZ56JvlzYCV6T1MK5/F3x815REV4MgHAtzwvDt/Px8XlBQYHq7Jo8XPh/QRpq8efmuY3hnbTFe+lk+GGPIm7MEgD+8bnNJNV6/dQLSXQzXSTle3vn5RGwsrkR9sxe/vzB0got1B06gd+c26N3JL6qc84gdOc0eH1wM+Hz7MVw8qgd1/BAEYSuMsQ2c83zNZXaIO2PsQgDPAXADeJlzPtdo/WjFPRIfFBxCj45ZGJjbDu+tK8Z904eAMYZKyZUTKWqFIAjCycRV3BljbgB7AEwHUAJgPYBrOec79LaxS9wJgiBSGSNxt6N3bgKAQs75fs55M4D3AFxqw3EIgiAIHewQ994ADim+l0hlBEEQRJxIWFwdY2wWY6yAMVZQXl6eqGoQBEGkJHaIeykAZfxfH6ksBM75i5zzfM55fm5urg3VIAiCaL3YIe7rAQxmjA1gjGUAuAbAIhuOQxAEQehg+SAmzrmHMXYPgM/hD4V8hXMePnM0QRAEYRu2jFDlnC8FsNSOfRMEQRCRoUQlBEEQKYgj0g8wxsoBHIxy864AjltYnWSAzrl1QOfcOojlnPtzzjUjUhwh7rHAGCvQG6GVqtA5tw7onFsHdp0zuWUIgiBSEBJ3giCIFCQVxP3FRFcgAdA5tw7onFsHtpxz0vvcCYIgiHBSwXInCIIgVJC4EwRBpCBJLe6MsQsZY7sZY4WMsTmJro9VMMb6MsZWMMZ2MMa2M8Z+LZV3YYx9yRjbK/3tLJUzxtjz0nXYwhg7PbFnEB2MMTdjbBNjbLH0fQBjbK10XgukXEVgjGVK3wul5XmJrHe0MMY6McY+ZIztYoztZIxNbgX3+DfSb3obY+xdxlhWKt5nxtgrjLEyxtg2RZnpe8sYu0lafy9j7CYzdUhacZdmfPoXgIsADAdwLWNseGJrZRkeAL/lnA8HMAnA3dK5zQGwjHM+GMAy6TvgvwaDpf+zAMyLf5Ut4dcAdiq+/wXAs5zzQQAqAdwmld8GoFIqf1ZaLxl5DsBnnPNhAEbDf+4pe48ZY70B/ApAPud8JPy5p65Bat7n1wBcqCozdW8ZY10APApgIvyTID0qNwhCcM6T8j+AyQA+V3x/AMADia6XTee6EP5pC3cD6CmV9QSwW/r8AvxTGcrrB9ZLlv/wp4ZeBmAagMUAGPyj9tLU9xv+pHSTpc9p0nos0edg8nw7AjigrneK32N5Ip8u0n1bDGBGqt5nAHkAtkV7bwFcC+AFRXnIepH+J63ljlYy45P0KjoWwFoA3TnnR6RFRwF0lz6nwrX4O4DfA/BJ33MAVHHOPdJ35TkFzldaXi2tn0wMAFAO4FXJFfUyYywbKXyPOeelAP4PQDGAI/Dftw1I7fusxOy9jemeJ7O4pzyMsXYAPgJwL+e8RrmM+5vylIhjZYz9CEAZ53xDousSR9IAnA5gHud8LIA6BF/TAaTWPQYAyaVwKfwNWy8A2Qh3XbQK4nFvk1nchWZ8SlYYY+nwC/vbnPOPpeJjjLGe0vKeAMqk8mS/FlMAXMIYK4J/QvVp8PujOzHG5LTUynMKnK+0vCOAinhW2AJKAJRwztdK3z+EX+xT9R4DwPkADnDOyznnLQA+hv/ep/J9VmL23sZ0z5NZ3FN2xifGGAMwH8BOzvkzikWLAMg95jfB74uXy38m9bpPAlCteP1zPJzzBzjnfTjnefDfx+Wc8+sBrABwpbSa+nzl63CltH5SWbic86MADjHGhkpF5wHYgRS9xxLFACYxxtpKv3H5nFP2Pqswe28/B3ABY6yz9NZzgVQmRqI7HWLssLgYwB4A+wA8lOj6WHheZ8L/yrYFwA/S/4vh9zcuA7AXwFcAukjrM/gjh/YB2Ap/NELCzyPKcz8HwGLp80AA6wAUAvgAQKZUniV9L5SWD0x0vaM81zEACqT7/F8AnVP9HgN4HMAuANsAvAkgMxXvM4B34e9XaIH/Le22aO4tgFul8y8EcIuZOlD6AYIgiBQkmd0yBEEQhA4k7gRBECkIiTtBEEQKQuJOEASRgpC4EwRBpCAk7gRBECkIiTtBEEQK8v9OPI7eXo5Q5AAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Score\n",
        "plt.plot(score_arr, label ='score')\n",
        "plt.legend(loc='upper left')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 982
        },
        "id": "KTGyfRFEzUHG",
        "outputId": "e3093240-21a8-4cad-ae2c-1c11e792faa9"
      },
      "outputs": [],
      "source": [
        "# play video\n",
        "agent.load_state_dict(torch.load(\"weights.pt\"))\n",
        "env = gnwrapper.LoopAnimation(gym.make('CartPole-v1'))\n",
        "state = env.reset()\n",
        "for _ in range(200):\n",
        "    with torch.no_grad():\n",
        "        output = agent.forward(torch.from_numpy(state).float().to(device)) # inference\n",
        "        prob_distribution = Categorical(output)\n",
        "        action = prob_distribution.sample() \n",
        "    env.render()\n",
        "    state, rew, done, _ = env.step(action.item())\n",
        "    if done:\n",
        "        state = env.reset()\n",
        "env.display()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}