SAC_online_BipedalWalker.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of SAC_online_BipedalWalker.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "eP8e9dKYFuHd"
      },
      "source": [
        "!pip install box2d-py\n",
        "!pip install stable-baselines3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IitdeHDwF3rx"
      },
      "source": [
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "from google.colab import drive\n",
        "import pandas as pd\n",
        "import pickle as pkl\n",
        "import gym\n",
        "import os\n",
        "\n",
        "import torch as T\n",
        "import torch.nn as nn\n",
        "\n",
        "from stable_baselines3.common.utils import set_random_seed\n",
        "from stable_baselines3.common.monitor import Monitor, load_results\n",
        "from stable_baselines3.common.vec_env import VecFrameStack, VecNormalize, VecTransposeImage\n",
        "from stable_baselines3.common.callbacks import CheckpointCallback\n",
        "from stable_baselines3.common.type_aliases import ReplayBufferSamples\n",
        "from stable_baselines3.common.buffers import ReplayBuffer\n",
        "from stable_baselines3 import SAC"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OUG8MAapGDaM"
      },
      "source": [
        "DEVICE = T.device('cuda' if T.cuda.is_available() else 'cpu')\n",
        "ENV_NAME = 'BipedalWalker-v3'\n",
        "SAVE_DIR = 'gdrive/MyDrive/SQIL/Online/'\n",
        "ONLINE_STEPS = 5e5    # 500,000 online environment steps\n",
        "SEED = 1337"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BeYYE97iemC3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c7462226-b3a5-4860-ec3a-c92ce42cb2f0"
      },
      "source": [
        "drive.mount('/content/gdrive/')  # model checkpoints + training stats \n",
        "set_random_seed(SEED)  # torch, np and py seeds\n",
        "\n",
        "if not os.path.exists(SAVE_DIR):\n",
        "    os.makedirs(SAVE_DIR)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5IuamzwgsE3L",
        "outputId": "8b7beb2d-ff61-479f-f461-dec356d59f18"
      },
      "source": [
        "env = Monitor(gym.make(ENV_NAME), filename=SAVE_DIR+ENV_NAME)\n",
        "\n",
        "checkpoint_callback = CheckpointCallback(\n",
        "    save_freq=ONLINE_STEPS, \n",
        "    save_path=SAVE_DIR, \n",
        "    name_prefix=ENV_NAME\n",
        ")\n",
        "\n",
        "model = SAC(\n",
        "    env=env,\n",
        "    policy='MlpPolicy',\n",
        "    learning_starts=10000,\n",
        "    learning_rate=7e-4,\n",
        "    buffer_size=300000,\n",
        "    batch_size=256,\n",
        "    train_freq=64,\n",
        "    gradient_steps=64,\n",
        "    gamma=0.98,\n",
        "    use_sde=True,\n",
        "    tau=0.02,\n",
        "    verbose=1,\n",
        "    policy_kwargs=dict(log_std_init=-3, net_arch=[400, 300])\n",
        ")\n",
        "\n",
        "# hyperparams https://github.com/DLR-RM/rl-baselines3-zoo/blob/master/hyperparams/sac.yml"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
            "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Using cuda device\n",
            "Wrapping the env in a DummyVecEnv.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P7f1m5gnd8XY",
        "outputId": "a4973b6a-dc32-48d2-9ca0-7cbd3eb4b969"
      },
      "source": [
        "model.learn(ONLINE_STEPS, log_interval=10, callback=checkpoint_callback)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 526      |\n",
            "|    ep_rew_mean     | -100     |\n",
            "| time/              |          |\n",
            "|    episodes        | 10       |\n",
            "|    fps             | 1735     |\n",
            "|    time_elapsed    | 3        |\n",
            "|    total timesteps | 5255     |\n",
            "| train/             |          |\n",
            "|    std             | 0.0498   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 378      |\n",
            "|    ep_rew_mean     | -104     |\n",
            "| time/              |          |\n",
            "|    episodes        | 20       |\n",
            "|    fps             | 1746     |\n",
            "|    time_elapsed    | 4        |\n",
            "|    total timesteps | 7565     |\n",
            "| train/             |          |\n",
            "|    std             | 0.0498   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 350      |\n",
            "|    ep_rew_mean     | -107     |\n",
            "| time/              |          |\n",
            "|    episodes        | 30       |\n",
            "|    fps             | 852      |\n",
            "|    time_elapsed    | 12       |\n",
            "|    total timesteps | 10499    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -11.6    |\n",
            "|    critic_loss     | 18.4     |\n",
            "|    ent_coef        | 0.698    |\n",
            "|    ent_coef_loss   | -2.35    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 512      |\n",
            "|    std             | 0.05     |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 298      |\n",
            "|    ep_rew_mean     | -110     |\n",
            "| time/              |          |\n",
            "|    episodes        | 40       |\n",
            "|    fps             | 392      |\n",
            "|    time_elapsed    | 30       |\n",
            "|    total timesteps | 11922    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -23      |\n",
            "|    critic_loss     | 2.63     |\n",
            "|    ent_coef        | 0.263    |\n",
            "|    ent_coef_loss   | -8.3     |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 1920     |\n",
            "|    std             | 0.05     |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 265      |\n",
            "|    ep_rew_mean     | -112     |\n",
            "| time/              |          |\n",
            "|    episodes        | 50       |\n",
            "|    fps             | 281      |\n",
            "|    time_elapsed    | 47       |\n",
            "|    total timesteps | 13248    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -17.3    |\n",
            "|    critic_loss     | 2.4      |\n",
            "|    ent_coef        | 0.114    |\n",
            "|    ent_coef_loss   | -10.8    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 3200     |\n",
            "|    std             | 0.0498   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 315      |\n",
            "|    ep_rew_mean     | -113     |\n",
            "| time/              |          |\n",
            "|    episodes        | 60       |\n",
            "|    fps             | 157      |\n",
            "|    time_elapsed    | 120      |\n",
            "|    total timesteps | 18910    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 0.316    |\n",
            "|    critic_loss     | 0.224    |\n",
            "|    ent_coef        | 0.00477  |\n",
            "|    ent_coef_loss   | -8.76    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 8896     |\n",
            "|    std             | 0.048    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 340      |\n",
            "|    ep_rew_mean     | -111     |\n",
            "| time/              |          |\n",
            "|    episodes        | 70       |\n",
            "|    fps             | 129      |\n",
            "|    time_elapsed    | 183      |\n",
            "|    total timesteps | 23765    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 3.39     |\n",
            "|    critic_loss     | 0.71     |\n",
            "|    ent_coef        | 0.00161  |\n",
            "|    ent_coef_loss   | -3.27    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 13760    |\n",
            "|    std             | 0.046    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 469      |\n",
            "|    ep_rew_mean     | -106     |\n",
            "| time/              |          |\n",
            "|    episodes        | 80       |\n",
            "|    fps             | 103      |\n",
            "|    time_elapsed    | 361      |\n",
            "|    total timesteps | 37536    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 2.66     |\n",
            "|    critic_loss     | 0.793    |\n",
            "|    ent_coef        | 0.00148  |\n",
            "|    ent_coef_loss   | -0.422   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 27520    |\n",
            "|    std             | 0.0428   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 533      |\n",
            "|    ep_rew_mean     | -102     |\n",
            "| time/              |          |\n",
            "|    episodes        | 90       |\n",
            "|    fps             | 96       |\n",
            "|    time_elapsed    | 495      |\n",
            "|    total timesteps | 47940    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 2.48     |\n",
            "|    critic_loss     | 0.197    |\n",
            "|    ent_coef        | 0.00176  |\n",
            "|    ent_coef_loss   | 0.0725   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 37952    |\n",
            "|    std             | 0.0416   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 579      |\n",
            "|    ep_rew_mean     | -98.8    |\n",
            "| time/              |          |\n",
            "|    episodes        | 100      |\n",
            "|    fps             | 92       |\n",
            "|    time_elapsed    | 623      |\n",
            "|    total timesteps | 57855    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 2.74     |\n",
            "|    critic_loss     | 0.262    |\n",
            "|    ent_coef        | 0.00176  |\n",
            "|    ent_coef_loss   | -0.546   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 47808    |\n",
            "|    std             | 0.0406   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 658      |\n",
            "|    ep_rew_mean     | -93.5    |\n",
            "| time/              |          |\n",
            "|    episodes        | 110      |\n",
            "|    fps             | 89       |\n",
            "|    time_elapsed    | 793      |\n",
            "|    total timesteps | 71041    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 2.32     |\n",
            "|    critic_loss     | 0.599    |\n",
            "|    ent_coef        | 0.00137  |\n",
            "|    ent_coef_loss   | 0.909    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 61056    |\n",
            "|    std             | 0.0395   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 738      |\n",
            "|    ep_rew_mean     | -88.2    |\n",
            "| time/              |          |\n",
            "|    episodes        | 120      |\n",
            "|    fps             | 87       |\n",
            "|    time_elapsed    | 925      |\n",
            "|    total timesteps | 81336    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 2.03     |\n",
            "|    critic_loss     | 0.053    |\n",
            "|    ent_coef        | 0.00149  |\n",
            "|    ent_coef_loss   | -2.69    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 71296    |\n",
            "|    std             | 0.0387   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 815      |\n",
            "|    ep_rew_mean     | -82.4    |\n",
            "| time/              |          |\n",
            "|    episodes        | 130      |\n",
            "|    fps             | 86       |\n",
            "|    time_elapsed    | 1061     |\n",
            "|    total timesteps | 91999    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 2.02     |\n",
            "|    critic_loss     | 0.0635   |\n",
            "|    ent_coef        | 0.00151  |\n",
            "|    ent_coef_loss   | -0.602   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 81984    |\n",
            "|    std             | 0.0384   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 916      |\n",
            "|    ep_rew_mean     | -74      |\n",
            "| time/              |          |\n",
            "|    episodes        | 140      |\n",
            "|    fps             | 85       |\n",
            "|    time_elapsed    | 1211     |\n",
            "|    total timesteps | 103563   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 1.94     |\n",
            "|    critic_loss     | 0.625    |\n",
            "|    ent_coef        | 0.00157  |\n",
            "|    ent_coef_loss   | -0.36    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 93568    |\n",
            "|    std             | 0.0379   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1.02e+03 |\n",
            "|    ep_rew_mean     | -65.8    |\n",
            "| time/              |          |\n",
            "|    episodes        | 150      |\n",
            "|    fps             | 84       |\n",
            "|    time_elapsed    | 1365     |\n",
            "|    total timesteps | 115472   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 1.61     |\n",
            "|    critic_loss     | 0.0904   |\n",
            "|    ent_coef        | 0.00163  |\n",
            "|    ent_coef_loss   | -3.68    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 105472   |\n",
            "|    std             | 0.0375   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1.09e+03 |\n",
            "|    ep_rew_mean     | -58.1    |\n",
            "| time/              |          |\n",
            "|    episodes        | 160      |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 1526     |\n",
            "|    total timesteps | 127936   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 1.5      |\n",
            "|    critic_loss     | 0.108    |\n",
            "|    ent_coef        | 0.00165  |\n",
            "|    ent_coef_loss   | 0.0567   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 117888   |\n",
            "|    std             | 0.0373   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1.18e+03 |\n",
            "|    ep_rew_mean     | -49.5    |\n",
            "| time/              |          |\n",
            "|    episodes        | 170      |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 1703     |\n",
            "|    total timesteps | 141618   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 1.31     |\n",
            "|    critic_loss     | 0.0584   |\n",
            "|    ent_coef        | 0.0021   |\n",
            "|    ent_coef_loss   | 0.554    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 131584   |\n",
            "|    std             | 0.0381   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1.19e+03 |\n",
            "|    ep_rew_mean     | -44      |\n",
            "| time/              |          |\n",
            "|    episodes        | 180      |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 1891     |\n",
            "|    total timesteps | 156084   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 0.952    |\n",
            "|    critic_loss     | 0.0822   |\n",
            "|    ent_coef        | 0.00246  |\n",
            "|    ent_coef_loss   | -0.0445  |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 146048   |\n",
            "|    std             | 0.0378   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1.21e+03 |\n",
            "|    ep_rew_mean     | -38.8    |\n",
            "| time/              |          |\n",
            "|    episodes        | 190      |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 2058     |\n",
            "|    total timesteps | 168938   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 0.622    |\n",
            "|    critic_loss     | 0.0852   |\n",
            "|    ent_coef        | 0.00257  |\n",
            "|    ent_coef_loss   | 1.02     |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 158912   |\n",
            "|    std             | 0.0376   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1.26e+03 |\n",
            "|    ep_rew_mean     | -29.8    |\n",
            "| time/              |          |\n",
            "|    episodes        | 200      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 2247     |\n",
            "|    total timesteps | 183409   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 0.195    |\n",
            "|    critic_loss     | 0.0644   |\n",
            "|    ent_coef        | 0.0024   |\n",
            "|    ent_coef_loss   | -0.115   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 173376   |\n",
            "|    std             | 0.0377   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1.25e+03 |\n",
            "|    ep_rew_mean     | -9.37    |\n",
            "| time/              |          |\n",
            "|    episodes        | 210      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 2417     |\n",
            "|    total timesteps | 196340   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -0.515   |\n",
            "|    critic_loss     | 0.0868   |\n",
            "|    ent_coef        | 0.00281  |\n",
            "|    ent_coef_loss   | 0.889    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 186304   |\n",
            "|    std             | 0.0376   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1.25e+03 |\n",
            "|    ep_rew_mean     | 6.61     |\n",
            "| time/              |          |\n",
            "|    episodes        | 220      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2548     |\n",
            "|    total timesteps | 206396   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -0.872   |\n",
            "|    critic_loss     | 0.109    |\n",
            "|    ent_coef        | 0.00282  |\n",
            "|    ent_coef_loss   | 0.337    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 196352   |\n",
            "|    std             | 0.0378   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1.25e+03 |\n",
            "|    ep_rew_mean     | 35.9     |\n",
            "| time/              |          |\n",
            "|    episodes        | 230      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2684     |\n",
            "|    total timesteps | 216812   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -1.85    |\n",
            "|    critic_loss     | 0.102    |\n",
            "|    ent_coef        | 0.00341  |\n",
            "|    ent_coef_loss   | 0.635    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 206784   |\n",
            "|    std             | 0.0384   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1.22e+03 |\n",
            "|    ep_rew_mean     | 63.8     |\n",
            "| time/              |          |\n",
            "|    episodes        | 240      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2801     |\n",
            "|    total timesteps | 225688   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -2.81    |\n",
            "|    critic_loss     | 0.122    |\n",
            "|    ent_coef        | 0.00466  |\n",
            "|    ent_coef_loss   | 0.753    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 215680   |\n",
            "|    std             | 0.0389   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1.17e+03 |\n",
            "|    ep_rew_mean     | 88.2     |\n",
            "| time/              |          |\n",
            "|    episodes        | 250      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2892     |\n",
            "|    total timesteps | 232682   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -3.32    |\n",
            "|    critic_loss     | 0.131    |\n",
            "|    ent_coef        | 0.00526  |\n",
            "|    ent_coef_loss   | -0.655   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 222656   |\n",
            "|    std             | 0.0394   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1.13e+03 |\n",
            "|    ep_rew_mean     | 122      |\n",
            "| time/              |          |\n",
            "|    episodes        | 260      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 3004     |\n",
            "|    total timesteps | 241271   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -4.07    |\n",
            "|    critic_loss     | 0.33     |\n",
            "|    ent_coef        | 0.00607  |\n",
            "|    ent_coef_loss   | 0.202    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 231232   |\n",
            "|    std             | 0.0402   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1.08e+03 |\n",
            "|    ep_rew_mean     | 152      |\n",
            "| time/              |          |\n",
            "|    episodes        | 270      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 3109     |\n",
            "|    total timesteps | 249237   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -4.87    |\n",
            "|    critic_loss     | 0.561    |\n",
            "|    ent_coef        | 0.00603  |\n",
            "|    ent_coef_loss   | -0.536   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 239232   |\n",
            "|    std             | 0.0402   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1.01e+03 |\n",
            "|    ep_rew_mean     | 182      |\n",
            "| time/              |          |\n",
            "|    episodes        | 280      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 3212     |\n",
            "|    total timesteps | 257155   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -5.41    |\n",
            "|    critic_loss     | 0.38     |\n",
            "|    ent_coef        | 0.00528  |\n",
            "|    ent_coef_loss   | 0.815    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 247168   |\n",
            "|    std             | 0.0402   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 962      |\n",
            "|    ep_rew_mean     | 215      |\n",
            "| time/              |          |\n",
            "|    episodes        | 290      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 3315     |\n",
            "|    total timesteps | 265097   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -5.8     |\n",
            "|    critic_loss     | 0.151    |\n",
            "|    ent_coef        | 0.00538  |\n",
            "|    ent_coef_loss   | 0.416    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 255104   |\n",
            "|    std             | 0.0404   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 896      |\n",
            "|    ep_rew_mean     | 244      |\n",
            "| time/              |          |\n",
            "|    episodes        | 300      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 3419     |\n",
            "|    total timesteps | 272972   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -6.03    |\n",
            "|    critic_loss     | 0.315    |\n",
            "|    ent_coef        | 0.00602  |\n",
            "|    ent_coef_loss   | 0.0226   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 262976   |\n",
            "|    std             | 0.0408   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 834      |\n",
            "|    ep_rew_mean     | 252      |\n",
            "| time/              |          |\n",
            "|    episodes        | 310      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 3507     |\n",
            "|    total timesteps | 279761   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -6.42    |\n",
            "|    critic_loss     | 0.174    |\n",
            "|    ent_coef        | 0.0054   |\n",
            "|    ent_coef_loss   | -0.52    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 269760   |\n",
            "|    std             | 0.0407   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 812      |\n",
            "|    ep_rew_mean     | 273      |\n",
            "| time/              |          |\n",
            "|    episodes        | 320      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 3608     |\n",
            "|    total timesteps | 287562   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -6.83    |\n",
            "|    critic_loss     | 0.207    |\n",
            "|    ent_coef        | 0.0067   |\n",
            "|    ent_coef_loss   | -1.16    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 277568   |\n",
            "|    std             | 0.0409   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 783      |\n",
            "|    ep_rew_mean     | 277      |\n",
            "| time/              |          |\n",
            "|    episodes        | 330      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 3707     |\n",
            "|    total timesteps | 295153   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -6.96    |\n",
            "|    critic_loss     | 0.14     |\n",
            "|    ent_coef        | 0.00619  |\n",
            "|    ent_coef_loss   | 0.245    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 285120   |\n",
            "|    std             | 0.0411   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 765      |\n",
            "|    ep_rew_mean     | 277      |\n",
            "| time/              |          |\n",
            "|    episodes        | 340      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 3800     |\n",
            "|    total timesteps | 302182   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -7.27    |\n",
            "|    critic_loss     | 0.186    |\n",
            "|    ent_coef        | 0.00652  |\n",
            "|    ent_coef_loss   | -0.507   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 292160   |\n",
            "|    std             | 0.041    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 772      |\n",
            "|    ep_rew_mean     | 287      |\n",
            "| time/              |          |\n",
            "|    episodes        | 350      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 3900     |\n",
            "|    total timesteps | 309883   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -8.09    |\n",
            "|    critic_loss     | 0.287    |\n",
            "|    ent_coef        | 0.00612  |\n",
            "|    ent_coef_loss   | -0.324   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 299840   |\n",
            "|    std             | 0.0406   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 760      |\n",
            "|    ep_rew_mean     | 284      |\n",
            "| time/              |          |\n",
            "|    episodes        | 360      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 3998     |\n",
            "|    total timesteps | 317304   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -8.97    |\n",
            "|    critic_loss     | 0.304    |\n",
            "|    ent_coef        | 0.00638  |\n",
            "|    ent_coef_loss   | -0.311   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 307264   |\n",
            "|    std             | 0.0409   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 752      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 370      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 4093     |\n",
            "|    total timesteps | 324456   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -9.33    |\n",
            "|    critic_loss     | 0.341    |\n",
            "|    ent_coef        | 0.00525  |\n",
            "|    ent_coef_loss   | 0.443    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 314432   |\n",
            "|    std             | 0.0407   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 733      |\n",
            "|    ep_rew_mean     | 272      |\n",
            "| time/              |          |\n",
            "|    episodes        | 380      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 4170     |\n",
            "|    total timesteps | 330464   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -9.57    |\n",
            "|    critic_loss     | 0.282    |\n",
            "|    ent_coef        | 0.00568  |\n",
            "|    ent_coef_loss   | 0.625    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 320448   |\n",
            "|    std             | 0.0408   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 724      |\n",
            "|    ep_rew_mean     | 268      |\n",
            "| time/              |          |\n",
            "|    episodes        | 390      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 4262     |\n",
            "|    total timesteps | 337520   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -9.88    |\n",
            "|    critic_loss     | 0.327    |\n",
            "|    ent_coef        | 0.0062   |\n",
            "|    ent_coef_loss   | 0.192    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 327488   |\n",
            "|    std             | 0.0407   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 720      |\n",
            "|    ep_rew_mean     | 266      |\n",
            "| time/              |          |\n",
            "|    episodes        | 400      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 4359     |\n",
            "|    total timesteps | 344973   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -10.2    |\n",
            "|    critic_loss     | 0.258    |\n",
            "|    ent_coef        | 0.0058   |\n",
            "|    ent_coef_loss   | -0.0317  |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 334976   |\n",
            "|    std             | 0.0406   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 729      |\n",
            "|    ep_rew_mean     | 274      |\n",
            "| time/              |          |\n",
            "|    episodes        | 410      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 4459     |\n",
            "|    total timesteps | 352633   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -10.8    |\n",
            "|    critic_loss     | 0.193    |\n",
            "|    ent_coef        | 0.00555  |\n",
            "|    ent_coef_loss   | -0.699   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 342592   |\n",
            "|    std             | 0.0406   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 724      |\n",
            "|    ep_rew_mean     | 272      |\n",
            "| time/              |          |\n",
            "|    episodes        | 420      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 4554     |\n",
            "|    total timesteps | 359981   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -11.2    |\n",
            "|    critic_loss     | 0.308    |\n",
            "|    ent_coef        | 0.00565  |\n",
            "|    ent_coef_loss   | -0.37    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 349952   |\n",
            "|    std             | 0.0408   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 722      |\n",
            "|    ep_rew_mean     | 272      |\n",
            "| time/              |          |\n",
            "|    episodes        | 430      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 4651     |\n",
            "|    total timesteps | 367339   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -12      |\n",
            "|    critic_loss     | 0.402    |\n",
            "|    ent_coef        | 0.00621  |\n",
            "|    ent_coef_loss   | -0.133   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 357312   |\n",
            "|    std             | 0.0406   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 728      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 440      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 4749     |\n",
            "|    total timesteps | 374975   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -12.6    |\n",
            "|    critic_loss     | 0.217    |\n",
            "|    ent_coef        | 0.00702  |\n",
            "|    ent_coef_loss   | 0.177    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 364928   |\n",
            "|    std             | 0.0408   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 721      |\n",
            "|    ep_rew_mean     | 278      |\n",
            "| time/              |          |\n",
            "|    episodes        | 450      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 4841     |\n",
            "|    total timesteps | 382023   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -12.9    |\n",
            "|    critic_loss     | 0.242    |\n",
            "|    ent_coef        | 0.00556  |\n",
            "|    ent_coef_loss   | 0.348    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 372032   |\n",
            "|    std             | 0.0404   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 706      |\n",
            "|    ep_rew_mean     | 271      |\n",
            "| time/              |          |\n",
            "|    episodes        | 460      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 4918     |\n",
            "|    total timesteps | 387921   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -13      |\n",
            "|    critic_loss     | 0.269    |\n",
            "|    ent_coef        | 0.0056   |\n",
            "|    ent_coef_loss   | -0.165   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 377920   |\n",
            "|    std             | 0.0405   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 700      |\n",
            "|    ep_rew_mean     | 270      |\n",
            "| time/              |          |\n",
            "|    episodes        | 470      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 5002     |\n",
            "|    total timesteps | 394469   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -13.5    |\n",
            "|    critic_loss     | 0.3      |\n",
            "|    ent_coef        | 0.00633  |\n",
            "|    ent_coef_loss   | -0.603   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 384448   |\n",
            "|    std             | 0.0406   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 716      |\n",
            "|    ep_rew_mean     | 284      |\n",
            "| time/              |          |\n",
            "|    episodes        | 480      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 5102     |\n",
            "|    total timesteps | 402078   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -14.1    |\n",
            "|    critic_loss     | 0.352    |\n",
            "|    ent_coef        | 0.00601  |\n",
            "|    ent_coef_loss   | 0.714    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 392064   |\n",
            "|    std             | 0.0406   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 720      |\n",
            "|    ep_rew_mean     | 289      |\n",
            "| time/              |          |\n",
            "|    episodes        | 490      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 5200     |\n",
            "|    total timesteps | 409561   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -14.5    |\n",
            "|    critic_loss     | 0.505    |\n",
            "|    ent_coef        | 0.00569  |\n",
            "|    ent_coef_loss   | 0.611    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 399552   |\n",
            "|    std             | 0.0406   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 717      |\n",
            "|    ep_rew_mean     | 288      |\n",
            "| time/              |          |\n",
            "|    episodes        | 500      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 5292     |\n",
            "|    total timesteps | 416708   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -14.8    |\n",
            "|    critic_loss     | 0.419    |\n",
            "|    ent_coef        | 0.00584  |\n",
            "|    ent_coef_loss   | 0.134    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 406720   |\n",
            "|    std             | 0.0406   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 715      |\n",
            "|    ep_rew_mean     | 289      |\n",
            "| time/              |          |\n",
            "|    episodes        | 510      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 5388     |\n",
            "|    total timesteps | 424088   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -15.2    |\n",
            "|    critic_loss     | 0.301    |\n",
            "|    ent_coef        | 0.00598  |\n",
            "|    ent_coef_loss   | 0.583    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 414080   |\n",
            "|    std             | 0.0405   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 707      |\n",
            "|    ep_rew_mean     | 286      |\n",
            "| time/              |          |\n",
            "|    episodes        | 520      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 5475     |\n",
            "|    total timesteps | 430730   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -15.5    |\n",
            "|    critic_loss     | 0.445    |\n",
            "|    ent_coef        | 0.00597  |\n",
            "|    ent_coef_loss   | -0.858   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 420736   |\n",
            "|    std             | 0.0406   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 707      |\n",
            "|    ep_rew_mean     | 289      |\n",
            "| time/              |          |\n",
            "|    episodes        | 530      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 5570     |\n",
            "|    total timesteps | 438065   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -16.1    |\n",
            "|    critic_loss     | 0.442    |\n",
            "|    ent_coef        | 0.00542  |\n",
            "|    ent_coef_loss   | -0.681   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 428032   |\n",
            "|    std             | 0.0404   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 704      |\n",
            "|    ep_rew_mean     | 289      |\n",
            "| time/              |          |\n",
            "|    episodes        | 540      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 5666     |\n",
            "|    total timesteps | 445362   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -16.4    |\n",
            "|    critic_loss     | 0.516    |\n",
            "|    ent_coef        | 0.00541  |\n",
            "|    ent_coef_loss   | 0.257    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 435328   |\n",
            "|    std             | 0.0402   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 705      |\n",
            "|    ep_rew_mean     | 291      |\n",
            "| time/              |          |\n",
            "|    episodes        | 550      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 5758     |\n",
            "|    total timesteps | 452498   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -16.7    |\n",
            "|    critic_loss     | 0.625    |\n",
            "|    ent_coef        | 0.00556  |\n",
            "|    ent_coef_loss   | -0.353   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 442496   |\n",
            "|    std             | 0.0402   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 718      |\n",
            "|    ep_rew_mean     | 302      |\n",
            "| time/              |          |\n",
            "|    episodes        | 560      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 5853     |\n",
            "|    total timesteps | 459752   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -17.3    |\n",
            "|    critic_loss     | 0.581    |\n",
            "|    ent_coef        | 0.00537  |\n",
            "|    ent_coef_loss   | -0.183   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 449728   |\n",
            "|    std             | 0.0401   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 721      |\n",
            "|    ep_rew_mean     | 306      |\n",
            "| time/              |          |\n",
            "|    episodes        | 570      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 5943     |\n",
            "|    total timesteps | 466614   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -17.6    |\n",
            "|    critic_loss     | 0.549    |\n",
            "|    ent_coef        | 0.00542  |\n",
            "|    ent_coef_loss   | -0.0497  |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 456576   |\n",
            "|    std             | 0.0401   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 717      |\n",
            "|    ep_rew_mean     | 306      |\n",
            "| time/              |          |\n",
            "|    episodes        | 580      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 6036     |\n",
            "|    total timesteps | 473738   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -18.1    |\n",
            "|    critic_loss     | 0.603    |\n",
            "|    ent_coef        | 0.00566  |\n",
            "|    ent_coef_loss   | -0.918   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 463744   |\n",
            "|    std             | 0.04     |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 711      |\n",
            "|    ep_rew_mean     | 302      |\n",
            "| time/              |          |\n",
            "|    episodes        | 590      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 6127     |\n",
            "|    total timesteps | 480701   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -18.3    |\n",
            "|    critic_loss     | 0.402    |\n",
            "|    ent_coef        | 0.00539  |\n",
            "|    ent_coef_loss   | 0.777    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 470656   |\n",
            "|    std             | 0.0399   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 712      |\n",
            "|    ep_rew_mean     | 305      |\n",
            "| time/              |          |\n",
            "|    episodes        | 600      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 6220     |\n",
            "|    total timesteps | 487869   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -18.5    |\n",
            "|    critic_loss     | 0.721    |\n",
            "|    ent_coef        | 0.00536  |\n",
            "|    ent_coef_loss   | -0.123   |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 477824   |\n",
            "|    std             | 0.0399   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 707      |\n",
            "|    ep_rew_mean     | 303      |\n",
            "| time/              |          |\n",
            "|    episodes        | 610      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 6311     |\n",
            "|    total timesteps | 494769   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -18.8    |\n",
            "|    critic_loss     | 0.561    |\n",
            "|    ent_coef        | 0.00567  |\n",
            "|    ent_coef_loss   | 0.253    |\n",
            "|    learning_rate   | 0.0007   |\n",
            "|    n_updates       | 484736   |\n",
            "|    std             | 0.0397   |\n",
            "---------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<stable_baselines3.sac.sac.SAC at 0x7f5dff22c690>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8_iA9EOA4ZPc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 611
        },
        "outputId": "1be611dc-0fc5-4db9-8f20-bc8a6c945d13"
      },
      "source": [
        "def plot_log(df_path):\n",
        "    \n",
        "    with open(df_path, \"rt\") as f:\n",
        "        _ = f.readline()\n",
        "        df = pd.read_csv(f)\n",
        "        df.sort_values(\"t\", inplace=True)\n",
        "        df.reset_index(inplace=True)\n",
        "\n",
        "    # compute rolling avg and std\n",
        "    sma_y = df['r'].rolling(100).mean()\n",
        "    std_y = df['r'].rolling(100).std()\n",
        "    \n",
        "    # plot with seaborn\n",
        "    sns.set()\n",
        "    fig, ax = plt.subplots(figsize=(10,10))\n",
        "    ax.set_xlabel('Timesteps')\n",
        "    ax.set_ylabel('Reward')\n",
        "    clrs = sns.color_palette(\"husl\", 8)\n",
        "\n",
        "    true_x = (ONLINE_STEPS // max(df['index'])) * df['index']\n",
        "\n",
        "    # fill standard deviation\n",
        "    ax.plot(true_x, sma_y, c=clrs[0])\n",
        "    ax.fill_between(true_x, sma_y-std_y,  sma_y+std_y, \n",
        "                    alpha=0.3, facecolor=clrs[0]) \n",
        "    plt.savefig(SAVE_DIR+ENV_NAME+'.reward_history.png', \n",
        "                dpi=300, pad_inches=0)\n",
        "    plt.show()\n",
        "\n",
        "plot_log(SAVE_DIR+ENV_NAME+'.monitor.csv')"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnUAAAJSCAYAAAClXjPaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdaYxkV3k//u9zzr219D7d0zPT3bN5x44NthnbEH7KD7II0B+ziEUkSlBECJGiKLxIUKyQAMqC2BIRKSBC4mwKgTcojuAFDuDkF3ZswNjDYnvs2Xvft6q6955z/i9u9TbTS/VMVd17q74faTztqu6qp7t6qr51lueIc86BiIiIiDJNJV0AEREREV0/hjoiIiKiFsBQR0RERNQCGOqIiIiIWgBDHREREVELYKgjIiIiagEMdUREREQtwEu6gDSYm1uBtdlt1zcw0IWZmeWky6AG4mPc+vgYtz4+xq2v0Y+xUoIDBzp3vJ6hDoC1LtOhDkDm66e98TFufXyMWx8f49aX5GPM6VciIiKiFpDKUPe3f/u3uO222/Dss88CAJ588km8/vWvx6tf/Wq8853vxMzMzPrn7nYdERERUbtIXaj78Y9/jCeffBIjIyMAAGst3vve9+L9738/Hn30UZw6dQof//jH97yOiIiIqJ2kKtQFQYA/+7M/wwc/+MH1y06fPo18Po9Tp04BAN7+9rfjy1/+8p7XEREREbWTVIW6v/mbv8HrX/96HD16dP2ysbExDA8Pr/9/f38/rLWYn5/f9ToiIiKidpKa3a8//OEPcfr0afzhH/5h0+97YKCr6fdZb4OD3UmXQA3Gx7j18TFufXyMW1+Sj3FqQt3jjz+O559/Hr/0S78EABgfH8dv/dZv4Td+4zcwOjq6/nmzs7NQSqGvrw9DQ0M7XrcfMzPLmd5mPjjYjamppaTLoAbiY9z6+Bi3Pj7Gra/Rj7FSsutAVGqmX9/97nfjG9/4Bh577DE89thjOHLkCB5++GG8613vQrlcxhNPPAEA+PznP4/XvOY1AIA777xzx+uIiIiI2klqRup2opTCRz/6UXzgAx9ApVLByMgIPvaxj+15HREREVE7Eedcducd64TTr5R2fIxbHx/j1sfHuPVx+pWIiIiIrhtDHREREVELYKgjIiIiagEMdUREREQtgKGOiIiIqAUw1BERERG1AIY6IiIiohbAUEdERETUAhjqiIiIiFoAQx0RERFRC2CoIyIiImoBXtIFEBERUeOJAIBs+rjGr1k7Gr16VLzJ7lHpLY+hjoiIqIWJANoYqOVVIAg3rrA2/gNUg5tbD25A9cO166tXS183bG/35k+jFGGoIyIiyigtDjoI4wRWDVqyFs6MhauGMje/DFOuXPf9qcVlqN4uGNQ41EdNxVBHRETUREpdHYjWp0MdqkNkbtP/bwpq1cuUNRBj4JZWYBZXN76mwSNothLAW1qB53vxfdq4LlssIFS6sXdOe2KoIyKitrKxnmzr+rK1v80+Fo2JCLQ1UJGpjpY5QACTzwPOQRkDWRstW1pBrlwCllY3pj7Xcpq1W0LblnTmALclsDkYY+NA1WzWIRqbvupifWQA0sNp2aQx1BERUdvQcPAWV+BMBLEOrjpNicjAOQdxDvrwAKJ8Hl65DJPPAQAkMhDnINWRKbEmnt6MDNzCMkxkAFRHrkSgcvHLqzMW1sajaLanCDO/mtw330jLqxCutUscQx0REbUNb7WEaGJ612lKuTwJr5iDLQXwDvTAVQLYlRKcc3FA2yu5OAdbCbe9vFXZSggvjGCVglOKO2QTwlBHRERtwXMWdm5hz3VnLoxgwggAEE3NNnydWitwYQR7aRwA4PX3wfZ0tXKGTS2GOiIianm+NZDxaZjVfe4AZTCpmQviIGxm5uH7GraQR8QzDpqKP20iImo5Sgm0Emg45IIAGJ2EWSklXVZbcGEEc3kSKthmCpoaiiN1RETUUjw4qNFJuDACnIs3MRi79xdS/VgHiQyQS7qQ9sJQR0RELUGk2mJkaQXRUovuMs0QVwkgnR1cW9dEDHVERJR5HhzU/BKwWoKpBEmXQwBQCSAicdsYagqGOiIiyiylBP7ySvVkhZWky6FNXBBCnAV4pFjTMNQREVFm6TCEmZyBC03SpdAVnLHwVlbhKQWIbNlIbAp5GMewV28MdURElCkigK4esyVLK7AMdKnkIoPo8tRVl4unoY4dgfH8BKpqbQx1RESUCZ6zEGuhghB2ahbO2PgMVMoUZwyU5ePWCAx1RNSWPBefx2mUump3nojEOymdg0My56bTVhqAGpuCrQSIjG3pI7dangMkCIFcPulKWg5DHRG1HS0OanwKLgiRGzwAU+wATARlDCSMgCCAq4RwYQQ5PIAggRcfpYCdBjNE4rVIe+0qjD8tDqjO7f75sr68SaCtgS6XEXUU973uafN9AqgeZl8bpapfZx3ExsFN1v6slhGtlnjCQ4twlQDSzZ2x9cZQR0RtRQvgLS4jWl4FHGAvT0HlvI2pvCteZLyVVUi+0LQXn7Vea/7CIlzOB5QGXDXgmHgdGYIQEIHt6Yq/phqAYF2829A6OGsBY4CouvasWIDp7ow/thZSPZhenAUiE3++tUBk4UplRJUQ+uhhuGIBylo4peCqSU3cptC1fnvV24kMxFo46wBn4QYOIFQ6/tkrgRgDWAtl4nVwplAABBBjoKcWYMuVuFGwc3DVP7Z6Wwx0rcOVAyglMGbvNxqb/+lJje8x1t74CAAVGYTq+g/QEhH45XL8b8Y5wAGmWEAk6Tmci6GOiFqaUoAKDZQ1kEoAt7KKaGnTiI9zsJWdjzOyS6vwe7phlWr4k7eIILeyAigFM7MAZwyA9WG2qz9/fimusRrQdp2SXFyBmluAqwY+Ww2Bu1pchh9FsDMLEE9DPA23djrDWuiyduP+t6HKAfIH+wAAbn4JNggB6+IALYDuLMKZOIBGuzwO1FpcEELPzkOrrSltfRTaxssj4sS/+Quxcfn639j0+1f9XQwiwBg46yBKoIcPY5f8WBM/CmBHJ+N/A1Xe0EGY7q7UrAZgqCOizNqpsalSAnEWOoyA2QXY1XJ8VNQ1PPPaSgi5MAbxPeT7exF2djRsjZ0fVGDGp6ujXJtepHaw+cVlT3uE1+2YxRVgcaX6InltgcuWA9hLkzvUBBie/NCWXGRgJmebc2dK4JVKkFxuUyiMR5zjv6uXbbpuPU+uvVkyFm6ldNW/OTu/BL9YQORp2BS0aGGoI6JM0QLoUnUKJAgR9fXAVt+ta2uhy5X4yXe1BGvs/oLPDlx1WtGOTcM7fgSB36ADLVdW41GrtEjL8APR9bAOZnQqHrbf9Du99obQbgl169fWNN1vSxWoSxPQI4dgU9CihaGOiFJHiYODWn/SFQF0FC/edwvLMKvl+MlZAO3itVpwgFtYauwUnrWQcgDUGOo0HMQYOM+DRbXkLZsX3PplLjJwHLUiaoh4ir8xb5hsEEIbAzDUEREBSgCvEsD6Hpyn4c0tweV8RB1FiHPwlpZhZuYRBdHWL3SAmZpraq1uZRW6rwdubZMBBBYCVV3sLybeKCDGxGvIygGUp+H19QBawYUREEVAZCAdRUAJXBjBri7ve3qUiFKiHEAKhaSrYKgjomSJCPzlZUQTM1D5HJSnEC2XIUrg93XBVQJEy+XUTAXaUgV6YhoIQrgogngedM6HqwTx7k9jrto44CIDOz599Y1tPqu0t5ia75GI9qlchkhv0lUw1BFRcrQ46IUlmJl5wFjY1fL6dc4C0fRCgtVtz0UGZm5x44JKCKyUkiuIiBLnyiG8UgnOdiRaR3qaqxBRW1EK8GYXYCZm6rKZgYgoKTYI4SZmdu4Y3iQMdUTUdCICf2kF0Uz6RuKIiLKKoY6ImkpEkCutxj2quIaMiKhuuKaOiJpCw0EHISQM4ynXNPVjIyJqAQx1RNRwWhy86XlE80t7H2dFRETXhKGOiBrKg4OamkU0t5R0KURELY2hjogaRsNBTc3AzC8nXQoRUcvjRgkiagglgDe3wEBHRNQkDHVE1BBeqYxodnHvTyQiorpgqCOiuvNg4abnEm/ESUTUThjqiKiulALUwhJsqZJ0KUREbYUbJYiobpQA/uw8T4ogIkoAR+qIqG68UhnRzCJg2YeOiKjZGOqIqC64jo6IKFkMdUR03bQSrqMjIkoY19QR0XXRAnjTs2xfQkSUMI7UEdE1ExHohaU40PE8VyKiRDHUEdE186MAdnaBgY6IKAUY6ojommhxwOwCXBglXQoREYGhjoiugQigF5dhFniuKxFRWjDUEdG+aWvhZhcBzroSEaUGQx0R7YsIoFdLsEGYdClERLQJQx0R7YtygJ1j+xIiorRhqCOiffHKJdhykHQZRER0BYY6IqqZVgI3x550RERpxFBHRDXT5QrMKo8CIyJKI4Y6IqqJUgIsLQPWJl0KERFtg6GOiGqiwxB2cSXpMoiIaAcMdUS0JxGBLK3ARSbpUoiIaAcMdUS0Jz8K2MaEiCjlGOqIaFciAllc5igdEVHKMdQR0a60NbCLq0mXQUREe2CoI6Jd6VKZR4IREWUAQx0R7UgL4OaXki6DiIhqwFBHRDvSQQBTYrNhIqIsYKgjom2JCGR5hc2GiYgygqGOiLbFDRJERNnCUEdE29JlbpAgIsoShjoiuopSgFtYTroMIiLaB4Y6IrqKF4Swq+WkyyAion1gqCOiLUQArJTgDDdIEBFlCUMdEW2h4OCWVpIug4iI9omhjoi20JUAthIkXQYREe0TQx0RrYt7060C1iVdChER7ZOXdAFElA5KCZQ1sMvsTUdEVLNyBYWvfQviHOxQP+AXEyuFI3VEBADwV1ag5xZggyjpUoiIMsO7OAZVKsPceAzS3ZlsLYneOxGlglYCN7cEs1JKuhQiagOysAQJ15qbSw1f4QC3+W8Azm38vek619MF19VR54p3YAz0xTHY3m5E99wBKeaBcnLPowx1RAQdcHMEETWHLC6j+JVvNOz2ndYo/8or4LriUTM1Ow89OhmfY+1cvGbYWoiz6x/DOsj69fbqz3EO4Z23wowc2fg+VksoPPp1iDEI7ry1Yd/PfjDUEbU5EUBKFbjIJF0KEbUBPTYJAKi8/B44b1MMcdts0BLZ/v+v+rv6H2OQ/9YPkP+f7wL5HMzgAPTEFNTSCpxSgJL4yByRrf+/ftmm//c0bPVz1OwC/B8/BzN8eP0+9blLcaC7+3ZEJ4+mYj0bQx1Rm1MA7CKPBCOi5tBjU7C93VtGveqp8sDd8J8/D4Qh/DPn4stO3QVz8ug136Y+dwn5J55G7omn4TwNWAdvdALm8EFEN5+sT+F1kKpQ97u/+7u4dOkSlFLo6OjAn/7pn+L222/H2bNn8dBDD2F+fh59fX34yEc+gpMnTwLArtcR0d50EHLqlYjqyznkvv0DqCvX6QogC8uIbr2hYXdthwZRGRoErEXhv74OKQcwR68vQJpjQ7DPnoUenahObwic1ggb+H1cC3Fuu/HOZCwtLaG7uxsA8NWvfhWf/OQn8R//8R94xzvegTe/+c14wxvegP/8z//EF77wBfzrv/4rAOx6Xa1mZpZhM9yXa3CwG1NTS0mXQQ3UqMdYBPAXlmAmZup+27Q/vb1FLCxwo0ora6fHWOYXUfzqN2EG+uDyeWzZ4KAE4V23ra95a2gdi8uQIIQ9eKDh96XyPnpfcjOm5xr3GCslGBjo2vn6ht3zNVgLdACwvLwMEcHMzAx+8pOf4HWvex0A4HWvex1+8pOfYHZ2dtfriGhvCoDj1CsR1ZmejN8oBg/cjeDn70Xw8y9F8Irqn5ff25RAB8Q7YZsR6NIiVdOvAPC+970P3/zmN+Gcwz/8wz9gbGwMhw8fhtYaAKC1xqFDhzA2Ngbn3I7X9ff313yfu6XerBgc7N77kyjTGvEYu+VV2IIH5FL1/q5t9fYm17SUmqNdHuNofh6upxM9Q7W/FmeeH0eqJF+PUxfq/vIv/xIA8Mgjj+CjH/0o3vOe9zT8Pjn9SmnXiMd4fep1bqWut0vXpp2m5tpVWzzG1iL/39+BmluAueEYVlv9+91E5X30Ag19Pc7U9Otmb3zjG/Hd734XR44cwcTEBIyJ2y0YYzA5OYmhoSEMDQ3teB0R7U4BcEsMdERUP2pqFnpuAeb4MMLb0rWJoB2kJtStrKxgbGxs/f8fe+wx9Pb2YmBgALfffju+9KUvAQC+9KUv4fbbb0d/f/+u1xHR7nQYwnHXKxHVg7VAEMK7MArnaQQvvbNp6+ZoQ2qmX0ulEt7znvegVCpBKYXe3l58+tOfhojggx/8IB566CF86lOfQk9PDz7ykY+sf91u1xHR9tYbDhubdClElHXOofDVb0JVN11Fx4eB6lp3aq5UtTRJCtfUUdrV+zHWSqAvjfOs1xRpi/VWba5VH2NZXEbxv76O6MQIbH8vzPBhuGIh6bKaLg0tTVIzUkdEzaHhoEtlNhwmorrQE9MAgPCOW+A622N3b1ox1BG1GW9lFdH4TLwGhohoP8oVqNmF+KB75wDroC+OwXZ1MNClAEMdURtRUt3xykBHlDlSKiP/9ceBMELlFS+F6+tpbgFRFK+dK1euuiq85WRza6FtMdQRtREVRbBlTrsSZZH33DnI0gqgFPznziG478W1f3EYxdOktjrCVv0j1dG2zZetjcDJFZephSWocgWVB+6G7eoAlIrfKYriKF1KMNQRtREdBIjCKOkyiFqad+Y89MUxmKOHEd2ytVebzC1Az8xtBCnrAGchm4PVpuC1frm10GOTMCOH4XI+vHOX4XV1xLfv7b3T1H/6Z/BfuLjv78WJAAJAFCBAeOsNMMfYCzatGOqI2oRSArfcejvviFIlMvBPPwNEBmp+EdHJo4DvAwDcSgmF//4OZJvlDw5YH/WCSPVjgVu7TAlcZwfCF90EiMA7P4rcj5+Dy+dhbjy2a0myWoJ37hKiEyMIb7tx/bYhUg1tsuWyq/5QZjDUEbUJZQzcajnpMrLNWvg/fg5qYQmyWgIig/DFt8EcGYSsliGlMmS1BLVaBqxFeMfN7NfVTpyDvjwOiQyCu25D7ulnkHvqZ7A93YC1MHPzgHMo/fIr4ulKUVvD1D6U3vgrKDz6dXgXx3YPdc7B/9HPAMfdqe2AoY6oTagghOHU63VR03Pwn3kBtrsTtrsTaqWE/Hee3PHzXUcB0U0nmlghNZv37Fn4z7wARAYwBgLAdhQQ3XoD9OUJeGcvrX+uAxC96Kb6bHAQgTk2BO+nZ+LNE8YC1kKsWf8YxkKshQQhgjtvZaBrAwx1RG1ARIDVUrw2h/bPOejRCeixSTgRlH/x5wHfA8IQ3nPnAKXgOgpwxWL17wLy//0deM+dR3TjcU5htShZKcF/+hnY/l7Y/j5AazhPwx4aAERQ+b/3A2EUj8YphZ4DnVhdunrn6LWKTh6FmpwGwjDetOBpWJ2LP9Yq/r2sTttGt/Ic1nbAUEfUBhQsHE+PuGZqeg75b/8QAGAO9seBDgB8H9Edt2z7NdGtJ5H/7o+gL47CHB9pVqlUZ2pmPt51GoZAFEEiA4QRJIriwCZA8MDdcB3bjIJpvWX6XVR9j1t3nUVUXvXyut4mZRtDHVEb0EEEE4RJl5FZ3tmLcIg3AZrDAzV9jTk6BPPsOfhPPQMzfBjwtnm6tRaoBJAghNNr90BpIasl5L/5BADAdnYAvgebzwOehvM9wPNgDg1sH+iIEsBQR9QGpFyJ19nQ/liL/P/7HtTMHKIbj8McH4Y9UON6KBGEd9+Own9/B/7PXkB4561brtZnLyL3/dPrMS7K+cD/9ypurEgR79mzQGRQ/pX/A9fdmXQ5RHuq71gwEaWOUoBbXkm6jExSM3PQM3MwJ48ivPNW2IMH9hW67MABRMeH4T17FrK8unGFc/CfeQGupwvBPXcgvP1mIAihpmYb8F3QNXEOemwS9vAAAx1lBkfqiFqcCiPYSntOveqLY4BzMMeHr+3rx6fhRBC85EXrvcb2K7zrNujLE8j94DTM0SNAJYBaXoVaXkXlvhfDnBgBjIH/7Fno8SnYI4PXdD9Zpy+NQY9ObqxXi0z173gdW3jHzVc18m0kWV6BWikh4AYDyhCGOqIWp8Ow7U6RkFIZanIGucefApRC6dAAUMhf9Xn63CW4YgH28MGNC50DggBSqsQjNf191xzoAMAVCwhvvwm5089CT87El3kezMCBOOQBgNaQ4UHosUmEL7m9fXbLOhc/VvOLyH33R0DOhyvk4TwPyPmwnUXA86CmZ+G9cHE91MnyCtTiMmAMzOFBIHftj89OvHOXAQCmTUM2ZRNDHVELa6dTJNRsvEtRzczFzX8R9wuT1TIK33gCgIMZOozo5FE4TwNwyH3/NCCC6IZjcdPg5RXISmlLx//girVw1yK67UaY4UOA78Pl/G2ncNWJIbiL45D5RbgDvdd9n1mQ++6T8C6NAwBsRxHlX3nFtgHae+4ccj/6KWR5BTAWha99Mz4+C/GxVeGLX1TXutTMHLxnXkB0fBius6Out03USAx1RC1MrIVbbYNQF0XIffsHEGNhDg0guqUPtrcHtr8X/ulnocenAAi8nz0P7/nzcPlcHOScg+0owDt3Ca6rA7a7C27oUNxrrlCAK+Zh6xGwROB6unf/lJMjcN/4IYpf+xai48MI7n/J9d9vmgUh9OUJRCOHYY4Pwwwc2HFE1AwfAn70U+S/9QMgCAHfR/nn74X/1M+gJ6ZR78UF3pnzQM5HcM/P1fmWiRqLoY6ohemwPU6R8J95AapUQflVL4MdOLDluvDuOxAinrIrfPl/4w0JQQj/9DOwPV0o/8r/iT8x4SlPyedgB/uhp2bhXRhFeMfNcF0tukDfWnjnLkGcQ3TbTbD9uwdn19kRh/CFJaBYQHDbjbADB2CGDiF3+tl4qv37TyN8ye1x+5jtBCH8584ivPkkkM/tfGfGQI9NxofW+3yJpGzhbyxRixIBUKoAtsVPkTAG3vMXEA0fuirQbea6OuMjmnwPEkaQxaXUnfYQ3Pdi6LFJ5H74E+ixKUS3JBvqZHEZuSd/AhiL4GV3wxULdbnd3Hd/BO/yOGxXR80tYoKX3nnVZebwQeD0s8h/43GIdfDOnN8x1Pk/OQP/zDmgEiK8d+cROD0xDYkMoqNDNdVFlCYMdUQtSgTA5jYarSgy8H/2PCQI44C2hyt7xaWN6ygiuukEvDPnoUcnEN18IrnQGYbIf/P78UkKYQTvuXNXr11zLj7vNIg/Jz51wazvXnVdnXEbmE1ktQR9eRzR8WGEd9x8Xd+f6+tBeMMxSBAAAPTlCajJGdjB/i23K6sleC+ch/M0vBcuwLs0Fvdt1ArlX37FlubBenQSzvfi2yDKGIY6ohalwgi2+mLXqvwf/RT+2Ysw/X1bd7BmXHRiBLnTzyL3xNPrI4sIwniEsRqyzPAhhHff0bAa9OUJqJVVlH/hfnhnL8J7/gLU/GIc3oJwvQ7Z5TxhB6DyC/fDHhqAmp6DPn8ZamERABD+3C3XvwlBBGF1BE8Wl+FdnkDhf7+HykvvhLnh2Pqn+T85AwAov/Jl8M5eXP9a/8x56AujiF50U7VgBzUxBXPoYNzgkShjGOqIWpQKQ5jQJF1G4zgH7/I4oqNHEDxwd6qmUa9XdNuNUEsr0OcvA74H5/vVvz3YjiIkCOGfOQ87OABXyK2Pjtn+vrpNkerJGbicDzvYjzCfg1paASID5HOwXR1xTTkfbq2+nBe3IvGrfytB/pvfR+7xp1B+zS8g94PTkJUSXEcB0a031H1XqevpQvlVL0fu2z+AnpheD3WyvAJ97hKim0/GI3ubNj+o2QV4F0Zhhg5BjIEsr0KVKgiPtM4bBGovDHVELUhEIC2+61XNzEGCEGbkSEsFOgCACIL7Xgycumv77y2MUPzy/0P+2z/YcrE5NIDKL9x//ffvHNTkDMyhgXjnbm83yr/8in3fTHDvnSj87/eQ+96PoBaXrxpBqzc7EI/Y6vGpeGpYBN7zF+IRvduubiJsjg8h9+RPUfzKN9YvcyJt2wCaso+hjqgFKVjY5XLSZTSMGp9C/vGn4JTAtPKoyk5h1fdQ/r/3Qy0swXk+4GvoS+Pwz5yHrKxuHQVzLv5T43SilMrIffuHUOUKokMD11W+PTQAMzgA7/IEXM6HOXZtJ3vshxnsh3f+MvynnwGUgnfuUhz8txnBjG44BpfPAwI4rQGt4YqFuo12EjUbQx1RC9JBBBO1bisT/8fPwnk67uV2Hac9ZJnr6YbZ1PvOdRThnTmP/P98F1ACMRaIIsBYiHMI7roN0W037nm73rNnoeYWEJ0YQTRy5LrrrDzwEuiZOdjebsCr/dzca2UOH4TTGv6zZ+EAIOcjuvXk9p+sddy6hKhFMNQRtSAJgnh3XwuShSXouUUEL7m9pTZHXC/XUUT4c7dAzS3G4UlrOK3iY7YmZ+D/9AzsQB9gXbw7de2M1TA+W3Xt//WlcZijR+Lp33oo5OORsmYpFlB6/S/FHyvVelPzRLtgqCNqMfHRYK3bysQ7exFOBNHxxk/lZU10+83bXi6LSyj81zdQ+J/vbnu9E1nf4OC6OxHucDuZsc0xbETtgKGOqMUoY2DLlaTLaIwwjNdIHRva/VQA2sL1dKPyypdBgiAObv6mXaq+xxEtohbBUEfUYlTQmq1MZGkFuSeejrv933wy6XIy58omwETUethdkaiFxEeDlePdji3GP/0M1MIighe/aM+zQomI2hFDHVELUSJwKy3Yn65SgR6dRHTDMUS3Xt1vjIiIOP1K1FJUGMIFYdJl1JUsLiP/vSchziE6eTTpcoiIrqYE4icfqThSR9RCVBDARa21ns5/+hnISgmVU3fB9Xbv/QVERE2mfA9u+DDESzbYMdQRtQilWm/qVZZXoMcmEd10AoajdESUVlrDquR3kDPUEbUIsRZutbWOBvNeuAiIILrpeNKlEBHtSDydiv1pDHVELUKHIWwrtTKxFt75yzBDh3gWJxGlm+cx1BFR/UglAGzrHA2mx6cglQDm5EjSpRAR7c7TAJJPdQx1RC2gFdfT6dEJON+DOTKYdClERLvi9CsR1Y0yBq4cJF1G/TgHPT4Nc8PgAUgAACAASURBVHgwPsKKiCjFnE7H81Q6qiCi66KiCDaKki6jbmR+EVKuwAxxlI6IUk4kNW8+01EFEV2z+GiwCmBTMPZfD8bAO3cJToRTr0SUfkrgUtDOBOCJEkSZJyJAi6yn06MTyH/rB3BA3Jcun0u6JCKiXYkInKRjjIyhjijjVBTBBhleT1cJkHv6GchqCesrjbVCeBvPeCWilBBsu7lVPA198AAirZte0nYY6ogyTkURTIb70+V+cBp6dBIAIM4hvPEYwrvvSM0aFSKqP/E01IFurKcluXL6UrYPUjvOcm66wlqYmfktS1J0X1d15F82bn7tPkWqfzbf3MZ1rvq3LK4gmlu86vuIerthUrL8haGOKMNEAJQrSMVe+n2SxWXkv/cjqPlFBHfeCilX4J85DzN8mIGOqMWJp2H7+2Dq/NTlHOAJoBZXYCubZjB6uhEW8ls+b7/yHQa4MtTlvBR0p9vAUEeUYVldTyerJeT/93sQ5xDccweiG48DkYHt7YY9fDDp8oio0ZSChcA14A2pgSDXXdwIdVoBWl33e1/n6fid9OYbyuVS9Z6aoY4ow1RkYCth0mXsj3PIPfE0JIxQ/sWXw/V2x5f7HswNx5KtjYiaQrzGrUFzzsF1dUIbB1gLWyrDquu/P6cURCu4aGO5i/heQ4LptWKoI8owFYUwUXbW08lqCbnHn4aemkFwz89tBDoiai+6sScwhLkcMDgAz0TQkzOItLruU7yc1hCl4LDxnOv8dMWodFVDRDXL3Hq6cgWFr34TMBaVl94ZtywhovbkKTTyrNT4adHBeh50Pgd35bTptdym1vEIY1CdHVEqdet/GeqIMipr6+n02CQkCFF+1ctgBw4kXQ4RJanBI3VrrHNwHYW63JcDoDoLEKXinbJawXrpilHpqoaIahb3p8vOejo9MQ1bzMP29yVdChElTJrU1805wBULcHVoOWKtQ9i/9Q2pc66RA477xlBHlFGZ6k/nHPTEDMzIoW36URFRu3FNnLYM6xi6bEr60e0kXZPBRFSTeD1dkJn1dGp6FhKGMId5litR2xMBmnhWakaeJuuCoY4og+L1dKtJl1Ez7/xlOE/DDB1KuhQiSpoIkJKzUlsNf6pEGaSMyc56ushAXxqHOXoEaGBvKiLKBlEC18SRunbCNXVEGZSl9XR6fAoSGUTHR5IuhYiabZvwJkrFLUao7hjqiDIma/3p9OVxuJwPe5BtTIjaiXgaauggsHmna/Vpy2qdql2jrYKhjihjRARYzUh/OmOgxyZhjg6lrkknETWYEth8HtF26+cY6BqCz7JEGaOsgStnYz2dmpyJp15HjiRdChE1mQjXzjUbQx1Rxqgogs3Iea/e5XE4z4M9PJB0KUTUbNzl2nT8aRNljFRCwNqky9ibtdCjkzDDhzj1StSGRGvOsjYZn2mJMkQpgStVki6jJmpqFhKEMCOHky6FiJLAFkZNx1BHlCHiLFypnHQZNdGXJ+C05ikSRO1Kq6xs0m8ZDHVEGaLCCC4L6+mcgzc6AXNkkO/WidqVVuA21+ZiqCPKEBVlI9SpyRlIuRKfIkFE7UlrjtQ1GUMdUUaIZGc9nXfuEpzvx5skiKgtieYofbMx1BFlgLMOudIq3EoGmg4HAfTlCUQnhrd2kieitsIedc3HEyWIsiAIYMZn4MIo6Ur25F0Yg1iL6OTRpEshoqQI2KMuAfyJE2WAqwRwUfoDHQB45y7C9vXA9fUkXQoRJUUE4Ehd0zHUEaWcCIBKkIlNZLK8AjW/hOjESNKlEFGCdD4HJwx1zcbpV6KU0wDcakZ6001MAwDMEHvTEbUr8T244UMwngfYDLwbbSEMdUQppgVQoxOZ+ZeqJqZhO4twnR1Jl0JECRElsFrBMtA1HadfiVJMogi2nI2pV1gLPTkDc/hgdc6YiNqReBqOmyQSwZ86UYqpyMCZ9DcbBgA1MQOJDOwRTr0StbVcLukK2hZDHVFKiQAoV7IxSgfAuzwG53nxSB0RtS/fg+NREonIyEodovailMAvleBWM9BsGIinXi9PxidItHrDYRGovA/xNOBp2FIFrhImXRVRakjO4/FgCWGoI0ohZSzsxAxsRsKCmpyBhGFbnPWqch7c8WGY6rpBb7UEOzZVvVIgIoAIRCvYcgBnbILVEjWZCFyrv7FLMYY6ohRSUQQTZmMtHQB4l8bhPN0eU69KwSmBNfFQRFQswDt2BFYEUApWCSAKogTqwijMSjba0RBdFyXwDg8AvgfDNXWJYagjSiGJQsBmZITHOejRCZjhw60/9Yrqzr5NU0vWAYHnb1zgADgHcYAu5gGGOmoDohRMsYBIM1YkiRsliFJGRIBSJekyaqbmFiFBCNMuu1692tYLOeeAfL7x9RClgYpHqilZfASIUkYJYLMU6qZmAADm0EDClTSJp1HrlmTneTz/ktqCiMAx1CWOjwBRykgUwYVR0mXUTE/OwPZ0AYX2GJW6cvp1N9bTkDaYkiaC1jzrNQUY6ohSRhkDF2Vkk4S1UNNz7TNKB1RH6mrjPA8qxzVG1PpkH/8uqHEY6ohSRoIQWWnypGbmIcbADrZJqFNqX1NM1jmgq7OBBRGlRI1rTamx+BaSKEWUEriMradzAMxgf9KlNJzK+dC9nbD7ONPSOQDFfHWhJF/xqIXtY60pNQ5H6ohSRJyDK2WnBYaenIE90Avk/L0/OePE9+B6u+M+dPtgfB/K3/79s3gaqpCD7izE4Y9rkiijRCuO1KUAR+qIUkSiCM5kZD1dGELNzCO65WTSlTSFFHKIPD/uH7iPFy8rCl5XcdvTQVR/L0x/H5xzUB0+ZHopO+spiTbjmrpUSM1I3dzcHH77t38br371q/Hggw/i937v9zA7OwsAePLJJ/H6178er371q/HOd74TMzMz61+323VEWaMiAxdlo+mwnpiGOBef99oO8jkYY2H2ORrhnIPr6AC2G4QrxLdprQM8BWFLCMoCQbUvXdybTvT+1ppS46TmURARvOtd78Kjjz6KL37xizh27Bg+/vGPw1qL9773vXj/+9+PRx99FKdOncLHP/5xANj1OqJMCoLMbJLQo5NwOR924EDSpTSeCNwOU6i1sPkc5IqvF622npGp2dOOaqAVdFcxud2mIvCGDkIfG4I+PgR9/AjUsSOwPBosFVIT6vr6+vDAAw+s///dd9+N0dFRnD59Gvl8HqdOnQIAvP3tb8eXv/xlANj1OqKsUSpDJ0k4Bz0+BTN0qPXXgYlAdxdhr+P4I6s1VGdx6816GtbbuE1RwrYQtCddyMMdH4bq6kjk/kUr2EIBQS6HwN/4E207FE3Nlso1ddZafO5zn8Mv/uIvYmxsDMPDw+vX9ff3w1qL+fn5Xa/r6+ur+f4GBrrqWn8SBge7ky6BrpMzBnZGgN7ittf37nB5EtzMPKIgRP7kEIopqqshPA11w1HIdW4GsdrBydrUugCFHPRgz5bP6T7QBcdc19Ku99+xDPRB9XXAhj1wLoEm5b4HNdB91cgzbUjy9TiVj8qf//mfo6OjA7/+67+Or3zlKw2/v5mZ5XhNS0YNDnZjamop6TLoOvkmgptf3XahfG9vEQsLpQSq2p53dhQ5AMudXXApqqsRVD4Hu1iGcde3K1nDQff2wYnAIV6PFG36dzs42I2lcgjT4j/PdqFyPiDYskGmHv+OdU8vgqkl5CoWZrHU9C4iurOIaLGc6dfMRmr067FSsutAVOpC3Uc+8hGcP38en/70p6GUwtDQEEZHR9evn52dhVIKfX19u15HlDUSGdiM7HxUU7OwnR1wHS0+SgdAPBUff3Sdax0NBMbbfbRPvNQ9JdO1EIEcORh/fHF8f7872y1nqF4kWq+v7XTVI+iavls6n8vKst+2lKpnkL/+67/G6dOn8ZnPfAa56qLLO++8E+VyGU888QROnTqFz3/+83jNa16z53VEWSICoJKR9XTGQE/NwowcSbqSmohWcOY6dhQ3M2hxTV1LEK1gq4+laLU1eCmBd2gAqB4ft56PROL/2ZLpZMtfABB5HuDic4V1IQdbqlzf7/c+Sd6HY6pLrdSEuueeew5/93d/h5MnT+Ltb387AODo0aP45Cc/iY9+9KP4wAc+gEqlgpGREXzsYx8DACildryOKEtEBCgHSZdREz0xDQkjREfTH+pEK+jBA4gmZ6/5RAfxddNGJpxS1Rd3vmhm2UaoEyhv62ia8j1E3Z0w17qxoPqrYSGQY0NQF8dglmub0tXdHZDNGyxEto4MXlnS2nWbPsf6rd9oPMtSE+puueUWPPPMM9ted++99+KLX/zivq8jygpxFq6cjZE6fWE0bmVyKMXnvVZ7Z+nuDrieLsjMApy9tkXlTnto1sIlpxRECdx+m+G1CPF03KtvPUNI/LEIYCxscHUD5zQS34erHicnhfyWN2zSWYyPmrvO4O4cYIyDzueAGkOd9HYh6Ojge4YWlppQR9TOVGSaOoVyzcIQemwS0YmjQAqbjeoD3ZCuDjitYbVGpBUgCjrnwYTXFuqaPlKnFJCF34U6E09DHT0M63nxGkYB4lAX//EWl2BHp5IuszaFeN2Zcw5SzEOthdFCHsgX6jZ96Zyr+Yg+8TRsjuvhWh1DHVEKSBRlYpOEvjwBMRbmxPDen9xk4mmgrweVzZsRHCBw0B0FYKV8TVObTjVvnZvzPCit4LIxIFUXa6NzqqcDYT6/saty7WFyDoCDTvF6Q5XzAU/Hs5SiIMX8enALe7ohvd1wDtADXQinl+s6ve5qnA4V34v7IjLUtTSGOqKEiSAz6+m886OwXR2w/enbYa4KOUQ5H7hikMs5AD1d0MUiJAwRjU1f9XV2p5//2lFITeKAuB9eRn4f6kEPHURUKMCK7N4mQ6n4sUhbK43qTlfTUdia1ap1Xvk91XuTgdOqps1A0tN15T8NakHpmz8hajPxJon0r6eT1RLU1Ayi48OpPEVCerpgd3jVCpSHIJ+HLeQheuNpTzwNdah/o5GqEohWEF9Dch5UPtfUMy2dc0Deb5/GrlrBeV7c7mWPrGNVOs/GVTkPNp+DMQ7WbvxpFqfj1ibrmx5EtpzLCq2g8j5sF9fStYM2eeYgSi9xFq6S/pEZfXEMAsAcT9/UK5TUNA1lfQ8670MV8whnFuNdisUC1NAg4rlaBackXuSuBFYJLKRpU1bOAZLLQQ/0wkzNZWOd5XUQpWBrDWrVUOeQ0DIFJQCkOsW6sStUdXYgVCqxEUSrFLwjB6HX1iBu+gsQuGq9RmtOvbYBhjqihGVik4Rz8M5dgunvg+vqTLqaq4jW633BduNEQXUW4To7gLlliKcRQWDz+R2+oM6F1sD5HtBRBGYWWn7DRNz6w6spEDmtIZumwiXnQUTgjG14A1596ED8mADVTRwS/2qIIFIq0dMVrAOCnX5/N2OgawsMdUQJy8JJEmp8CmppBZX7Xpx0KdsST9cUDpxzcD1dsFpDeTqV3fFNPh938VCq5V+HJe/X/D06xIv9UQ6ginlg5BCc0lCz8zDT8w2rUXXkYXt7eGA9ZUL6FigQtZH4JIn0T736z56DLRZgjg0lXcrVBFDFfE2zX84BoefDiIoDQi593fGNdbCo7uZtdfv4+TsXh0DxNFRfNyKlETk0fP2hdHXApHANKdF2OFJHlKBMbJIII6jpWUS33pDK3nTewT6Ynu6aw4G1Lg7TnYWae3wlIucDK9d3+HtdKAXle7BhWNd1Y7qvC+isffG+cw6mtwfS041Ib+od2ODwK76futFcop0w1BElKD5JIt0jdWp6FuIcTFpPkCjkEe2zl5xzgOnrgWviJoj9cM4BfjpG6rxDB2D6eqAvjdd8HNWelAB9PQj83L6+bLvHudFHq7l2GDGllpG+t91EbSTeJJHu9XR6ahZOBHbgQNKlXE3kmluOGCepa3m2xjlAvOTfc6tCDqanG8a69Y0C9SBaw+r6fH9urX9dI6j4RBKirOBvK1GCJAuhbnIGdqCv4dNc10LUtYe6tHMpCHXie7Ai8SBYMV+3/oTie3UbAXMN7F8nWmpvuUKUAsk/axC1syBI5fTfGllehZpfRHDXbUmXsj2t4HT6wmY9OM+Lw881nllbF563PqtpPR+6Ix+PIm7X5HZtGnT9DwAH2OlN/faqx2ipjjyiOv3eO6Wg1lqM1JkoBasZ6ig7GOqIEqKUAKV0b5LQl8cBAObokYQr2d5689oUB+NrFXkaucMDcKulamBCPBUo8feNKEI01bhWHgCq6/riH67RCu7Y0LZL1zYuc1v+XymBH4aIZhcBAN7gAbiOIqzW9dt1vDb9uvbz2elYkWuhNaAV9jzugiglGOqIEiLOpf4kCX1pHOZAb9ysN4XE8+JmsC24PdE5IOgoAh1rP3u35TrPGsj8ckNH8mTTSJ1zgNlnuLHWwfV2QxZX4KyF6yjue3PEXtb61+lCDqLUeoCsB/G9VvzVohbGcWWihEgUNbwT/vWQlVXouYXUjtIB1ea1Lfyi61y1YbJz1Y+xaTrUg+osbEyB6vpvGHB1mHo0uRwk50N5HmwDeso5B6C/F+5gP1Co4WSFWgkgncVET4sg2i+O1BElREUGJsWbJPSldE+9ev29cVuSVk51u7DWwfUfgO7tqU49CqRURjQ+U587EAH22SpmO9Y56K4OoFyGUbruZ6Q65xDk4jCX87y6tTdRHQVEXR0tObVPrYuhjigBIpL6TRLx1GtPaqde0VnYd3+6VhNqHa/7qsoVXDxaV4fgFO8svv6RP+cAdHcAxXzDA7jzNESruoyAS2cRlkeDUcZw+pWoyUSA3PIy7MxC0qXsSFZK8dTrSDpH6aBVKlp+pI3VHqReu4GVqlu7mEB5CHL5hk+VWx2HunqQXPrOBSbaC0MdUZOJCNxCYxe4X6+NXa8pPOsVgGgF26KtTK6H83TdzozNYg9ApxRUPrfjz0B8r7azYpXwJAnKpGz9iyVqAcqYVAc6ANCXxmD7euC60jn1qnyPoW4b1gGytllgSy+5akuUGqliHnLkYLyzOEOsdTAjh6G6O7e9XvV2QXUU9rwd0Qq2zaf2KZs4f0HUZMoY2BRvkJDVEvTsAoI7b026lB1JofFTeVnknAMO9EB3d1ZDnGyEuZUSzNRcTbcjnUWE+UImN6EYY6G2G2VTAnR1AKvlPW9DlIb1WrP/IbU2hjqiZouijQ77KZT2Xa8AmrLoPqsC7QHbnKua6wIwuwDs9bsnADqKmf35xufmXh3qVM6HzeWga3hDJTkPTlRL9j+k1sZQR9REIgAy0HDY9nXDdW0/hZU00QrW95MuI3OMn4Mu5GGDML5AANk8kledahWtYHPZ/vlutx5OujsRAVDai6ejdzl5otX7H1LrYqgjaiIFB7e0mnQZO4qnXucR/FyKp179ahNbvujui3GADA1ujD6JwK6tmVufqo1DnrUu2z9fpbf2q1MK6CzCOcBqBeVr2Kh6PO02I5fO8zI7UkntjaGOqIl0OYApp3ekLvVnvfoepLcbFgrZTh3JiGSHDRNrP8oWCTJOSbx7t3qsmcr7MLkc4OK2J97QoepxsQI3Ngl75b/JBpx8QdQM3P1K1CRKAVhYSvULp740DtvbDbfD7sGkqf5eRG18igTVxilV/QcXUx2F9UbC1gGB7yP0fEQ5H5K/4ixaVZ+TNIiSwFBH1CReqQKztJJ0GTuS1RL0zDyilI7SQQAUcjyLk/bklIJsPg1jh401zjkgv3X9oChVlzNviZLA31yiJhARyMpq3c+9rCd9eQJAmhsOa/amo9ootXXjxw4ba5yLT47Y+rXZa7pMtIYLB4iaQMTBlSpJl7Ez5+Cdu5TqqVfxNKzHDRK0NwdADR5YH7WIdjlSznl6y3m5oqqnlaT4DRjRThjqiJpAGQsbpfcUCT02CbWwhMqpu5IuZVviaajuDpjNOxqJdmCtQ6VQrO1ztQddyMNZC4hAFXJIb2twot0x1BE1gTIGJsUNh73nzsN2FGGODyddyrZUfy/CA71cT0d1Z7SGO3okHq1D3PqFv2eUVVw4QNQMUbR3J/+klCtQUzMwJ4a37BhMDRGgmOcLLTWEcw7GAcY4GOP4e0aZlsJncKLWEp8iESZdxo68yxMQAFFaN0j41bV0RES0K4Y6ogYTkVQfDaYvj8N2dcL1dCVdyrZUPsdQR0RUA4Y6ogYT5+DSGuoqFajJmfgECdnuqIEU6Ciw2TARUQ0Y6ogaTIzZ9nzJNNDrU68pbTgMAPk8N7wSEdWAoY6owZQxcCadTRK8S+OwXR1wvd1Jl7KtuDcdGw4TEdWCoY6owSQy6WxkWgmgpmbjEyRSOvUqnobjejoiopow1BE1kAjggnSup9OXJyDOpXrqVQp5uJQGTiKitGGoI2ogEQHK6Qx13qWxVE+9AgByPjdJEBHViKGOqIHEObgghT3q1qZeR1K86xWA5DxukiAiqhFDHVEDSZTOTRJ6NP1TrxCB09wkQURUK4Y6ogZSKW1n4l0ah+0swvX1JF3KjkQrhjoion1gqCNqIDFR+na+BkG14XB6d70CAJTApfEsWiKilOIzJlGDiCCVJ0lkYdcrAIjiSB0R0X4w1BE1SCp3vgYh/J+ege3uTPXUKwBIPoeUjXESEaXarl09f+3Xfi1+YdrDZz/72boVRNQqxNnU7Xz1f3oGUqqg8sqXpXvqFQDyObYzISLah11D3Vvf+tb1jy9cuIAvfOELeNOb3oTh4WGMjo7ikUcewZvf/OaGF0mURfHO13RtktBjU7CHD8IO9CVdys5EADgg77OdCRHRPuwa6t70pjetf/y2t70NDz/8MG655Zb1yx588EH88R//MX7/93+/cRUSZZQyFiZFoU5WVqGWVxDcdDzpUnaluzsA5wAeD0ZEtC81P2s+//zzOH5864vB0aNH8cILL9S9KKJWICZCmoaa1MQ0AMAcPphwJXso5CGehuEmCSKifal5o8R9992Hhx56COfOnUO5XMbZs2fxvve9D6dOnWpkfUSZpJTALa4mXcYW3ugkbEcRrrsz6VJ2JXkftliAZagjItqXmkPdhz/8YQDA6173Otxzzz148MEH4ZzDhz70oYYVR5RVKgxhS+Wky9gQhlAT0zAjh1O5QUJ3dcA7PgTdWYTTGpHWsGnr70dElHI1Tb8aY/Av//Iv+PCHP4y/+qu/wuzsLPr7+6HYGJRoW7oSIAqjpMtYp0cnIc7BpLA3nfgeMNCHIJ9HrqcTRus0zVoTEWVGTalMa41///d/h+/7UErh4MGDDHREOxCR1DUd1pfH4ynN/hTtehXA6++FOnYYYbV9iSvk2XCYiOga1ZzM3vjGN+Jzn/tcI2shagkiANIU6sIIejx9U6+6pwvRwT4EeqN1SeT7SM9+YSKibKl59+tTTz2Ff/u3f8PDDz+MI0eObGlKzObDRBvS1nRYj09CrE3V1KvK+8BAL4zbGjK5jI6I6NrVHOre9ra34W1ve1sjayFqCWlrOqwvjcMW8rADB5IuBaIV1MEDsN2diJQCzwEjIqqfmkPd5kbERLQzZSyMTUmoiyLo8SlEJ4+mYupVcj5MXw+MdQx0RER1tq+W7dPT03jqqacwNze35UzGt7zlLXUvjCirxESpmUfU41MQY2FG0jH1KjmfWY6IqEFqDnVf/epX8d73vhcnTpzAmTNncPPNN+O5557Dvffey1BHVCUCuEqK1tNdmoDL52AH+5MuJZbztrwhJCKi+qk51H3iE5/Ahz70Ibz2ta/Ffffdh0ceeQRf+MIXcObMmUbWR5QpIgKkZZOEMdBjk4hODKdi6hUAxPfZg46IqEFqbmkyOjqK1772tVsue9Ob3oRHHnmk7kURZZUAqelRF0+9mtRMvQKA8/a14oOIiPah5lA3MDCA6en4QPCRkRH88Ic/xIULF2DTsiCcKAUkilKz81VfGofL+emZetUKYNNyIqKGqfkZ9q1vfSu+//3vAwB+8zd/E+94xzvwhje8Ab/6q7/asOKIskZZC5eGNzrVqVczcjg1QUqUgtXpqIWIqBXVPBfy7ne/e/3jN77xjbj//vtRKpVw0003NaQwokwyBkjBSJ2emIZEBlGKpl5FK1it2cqEiKhBan7b/LWvfQ2Li4vr/z88PMxAR3SllGyS0Jcn4Hwf9tBA0qWsk2IBLiUbNoiIWlHNI3X/+I//iD/4gz/AiRMncN999+H+++/HqVOn0N+fkvU6RAkTESAN7UyshR6dgBlOz9QrAEgxD5uS/n1ERK2o5lD32c9+FpVKBU8++SQef/xxfPazn8Uf/dEfYWRkBF/60pcaWSNRJogACJLf+aomZiBhhChFZ71CKbicn3QVREQtbV9v440xCMMQQRCgUqmgu7sbN954Y6NqI8oUsRYuMkmXAe/yOJznpWrqVXka1mc7EyKiRqr5WfYtb3kLpqamcO+99+L+++/HX/zFX+Dmm29uZG1EmaKMSb6dyfrU6yFA62Rr2UR8D0bp1ByfRkTUimoeqevu7kYURVhcXFz/E0VRI2sjyhRJQahTU7OQIIRJ09QrABTzPEmCiKjBah6p+6d/+idEUYQf//jHePzxx/GZz3wGTz/9NG655Rb88z//cwNLJMoGiQySTi56fApOKZjDBxOt40pSyPHMVyKiBtvXmrrl5WVMTU1hfHwco6OjWFxcRLlcblRtRJkhArgUbJLQU7OwA32pmnqFUjwejIioCWp+pn3wwQdx4cIF3HXXXTh16hQeeugh3HPPPSgWi42sjygTUtHOJAgh84swt6drrat41abDRETUUDWHuj/5kz/B3XffjXw+38h6iDJJnINLuPGwmp6DADBpOeu1SjwP1vO4SYKIqMFqnn594IEHsLq6ikceeQR///d/DwCYmJjA+Ph4w4ojyoo0bJLQU7NwSsXTrykihTzzHBFRE9Qc6r73ve/hNa95Db74xS/iU5/6FADg/Pnz+OAHP9io2ogyQ1kLZxPe+To9A9vfGj2jmgAAIABJREFUm671dOAmCSKiZqk51H3oQx/CJz7xCTz88MPwqoueX/KSl+Cpp55qWHFEmREZIMmRujCEmluEHUxPw2EAgAhPkiAiapKaQ93ly5fx8pe/HEB1UTgA3/dhTPId9ImSpJQAS8vJ1jCV1vV0mpskiIiapOZQd9NNN+HrX//6lsu+9a1v4dZbb617UURZ4oUB7HIp0Rr0+CSc1ulbT6fZzoSIqFlqfrZ96KGH8Du/8zt45StfiXK5jPe///147LHH1tfXEbUjDw6Y+f/bu/dgu8r6/uPv51lr7X32uefckpMESAHBICiYACIXNdaRiSgw2tKC1ipVq9WOdew0akdboB1jO7VM1WpHx5lWWx3HETTe4FdUBAVBLsovQCSEEHLu9/vee631/P7Y4fwIkPvZe+219uc1k5nkrJOzvjmLvc+H5/J9ppM989U5/P3DRP299beertCEVtOJiNTGUY/UnXvuuXz3u9/l9NNP561vfSvr16/ns5/9LF/+8perWZ9IXfOWloimE556HZvEFEtE6+rsaDAqmyRibX0VEamJI47ULS4u8qUvfYnHHnuMU045hQ996ENMTEywfft2/v3f/52rrrqqFnWK1J3KKRIJNxwGvP3DlaPB1vQmXcrBtElCRKSmjhjqbrjhBnbu3Mkll1zCnXfeya5du3jyySe5+uqrufHGG+nqqq+F2SK1Y6AUJluCc3gDQ5WzXoP6WrtWOUmivmoSEcmyI77j/vznP+fWW2+lu7ubd7zjHbz2ta/lv/7rvzj//PNrUZ9I3TIGSPi8Vzs5g11YonzWSxKt48WYICAOdJKEiEitHHFN3cLCAt3dld5Xa9asobm5WYFOBDBxnOwGCcDbP4QzhmhtX6J1vBjTUiBW02ERkZo54khdFEXcc889B3WEf/6fn+1fd6K2b9/Oj3/8Y/bv38/3vve95XYpe/bsYdu2bUxNTdHZ2cn27dvZsGHDEa+JVFPiR4M5h7d/mLi3C3K55Op4McZAoQllOhGR2jHuCOf3bNmy5fBfwBj+93//d0WKuf/++1m3bh3XXXcdX/ziF5dD3Z/8yZ/w1re+lSuvvJJbb72Vb3/72/znf/7nEa8drfHxuVTv0OvtbWN0dDbpMhqKtQa/WCR6erAm04sdHQWmpw/uhWemZyncfhel884iPO2UqtdwLEzOh5PWEtqj3mDf8PQ6zj494+yr9jO21tDd3XrI60ccqbvjjjtWtKDD2bx58ws+Nj4+zs6dO/nqV78KwBVXXMGNN97IxMQEzrlDXtMGDqkmf3ERNzqZ6Hox/5khHBCuXZ1YDYdim/KEvqf1dCIiNVT3W9MGBwdZvXo13oGmqp7n0dfXx+DgIM65Q15TqJOqKpWJF4uJluDtHyLu6YJCU6J1vBjT2pzq0W8RkTSq+1BXC4cbykyL3t62pEtoKHFpEddRqOk9O55zPzc5Qzgzh73oFTTVuI4jshbb20F7S53VlQJ6HWefnnH2JfmM6z7U9ff3Mzw8TBRFeJ5HFEWMjIzQ39+Pc+6Q146F1tTJsbDW4I/NEM3X7rzX56+pCx59Ch+Y6+6G6WTPnX0+mw+IF8pECwn38EsZvY6zT884+5JeU1f3q5i7u7vZuHEjO3bsAGDHjh1s3LiRrq6uw14TqRbjYlyYbGDxnhms7Hptyidax4sxvofTBgkRkZo74u7XWrrpppu47bbbGBsbY9WqVXR2dvL973+f3bt3s23bNmZmZmhvb2f79u2ceuqpAIe9drQ0UifHwo8i2DeAK9euR91zR+rqedcrgLeqnXJvN3X01pIKeh1nn55x9iU9UldXoS4pCnVyLHLlUs1amTzruaHO37WH3G8eY/FNr8PV4SYJb003pTatGzpWeh1nn55x9iUd6up+TZ1IvTAGgqUl3MhEoq06vJFx4raWugx0AARB0hWIiDQkLXwROUpBGBIPjhEvJXjeaxxjxyaI+rqTq+FwrAVPbysiIknQu6/I0SqXceVkN0jYiSlMGBHXaagzniU+0DdSRERqS6FO5GglHOgAvP0jOGPqdqTO5gJiX6FORCQJCnUiR8EYA6VyskU4VzlFYnVP/a5ba86jrVciIslQqBM5CsYAxWRDnZmawS4sEq6rv7NelzU1KdSJiCREoU7kKNRDw2F//3Bl6nVtfYY643u4QBvqRUSSolAnchRMFOMSbGPinp167emCfC6xOg7H+B6xr1AnIpIUhTqRIzAGbKmMi2p3gsTzud37sLPzROvrc5QOwBSacMYkXYaISMNSqBM5gqBcJh4aS6zhsB2fJPrZ/URdnYQnr02khqNhCvlUn8wiIpJ2misROZIwTLQ/nbdvCDxL8dLzoV7XrFmDy9XpjlwRkQahkTqRI0myP51zeANDmLV99RvoAON5ajosIpIwhTqRwzDGYJIKdc4R/PZx7MISdkP9TrtC5SQJp1AnIpIohTqRwzAGXEL96ezoBMGuPYQb1mFOPzmRGo6WCXxtkhARSZhCnchhGOcS60/nDY/hjKH0irMwtr5fqiYXqOmwiEjC6vsnhUjCTBThorj2N14q4g2PEnd31vVaumW5AKdUJyKSqBT8tBBJjg0johqHOjMzR+G2nwNQOuslNb338dJJEiIiydNIncgheAaYnIa4tqHOe2YQgKivm2jDupre+7hYW/klIiKJ0v9eizyPMWCNwZuaJppdqPn9/f3DRN2rKF52Qc3vfTyMZ9XORESkDijUiQAWh18qY0olXBhBqUw0O0+tV/+b+QXs9Cyll7+0pvc9EcuhTkvqREQSpVAnAnhhSDwwjCsnd74rgB0ZByBa05toHcfC5IJKOxNtlBARSZQWwogAJo5xYQK7XJ/HGxnHNeVxbS1Jl3L08tr5KiJSDxTqRKiEusRHmpzDGxkn6u2qLOxLCZPLJf6tExERhToRAFyS57seYKZnMcUSUV930qUcPWNwvjZJiIjUA4U6aXiJnu/6HP4zQzggWtuXdClHTTtfRUTqh0KdNDxj6mOkzts/RNzbBfl80qUcNeN7OF/7rURE6oFCnTQ8A7hSOdkaZuaws/NE69YkWsexMk35ys5XERFJnEKdSBRDnOxKf2//UKWUdasTreNYmUKeOOHvnYiIVCjUScMzcYSr8VFgz+fvHyLq6sQVmhKt45gYg8sFSVchIiIHKNRJw7OxwyU42mTmFrBTs0TrUzb1qk0SIiJ1RaFOJI7BJTdSl9qpV8/iFOpEROqGQp00PBNHiZ5b6u0fJu5sx7U0J1fEcTC+j7N6CxERqRd6R5aGVmlnkuB5r0tFvIkpwpSN0gEHjgdLuggREXmWQp00OIMJk+tR541OABCv7kmshuOW05mvIiL1RKFOGpox4BIMdXZ0HOd7xJ3tidVw3ALtfBURqScKddLQjAFXSnakLu7pgrStTbMWtElCRKSupOwnicgKi+PKrwSYufnKKRJ93Ync/0TYwCMKdDyYiEg9UaiThmbCCBclE+q8pwdxkLr+dACmpUBs9PYhIlJP9K4sDc2WQ1yYwO5X5/D3DRD3dOGaC7W//4lqKWiThIhInVGok4ZljIHFpWTuPTWDnZ0nPHltIvc/ESbwibVJQkSk7ijUScOyONz8YiL39vcO4KxJ5dSrzQXEWk8nIlJ3FOqkYZkoSmbqtVjC3zdAtKYPcikc8WptVtNhEZE6pFAnDcsksfPVOfK/fADKIeFLT63tvVeCNVDIK9SJiNQhhTppWCaOcXFt04mZnsUbm6R8zpnEXZ01vfdKsPkcURpHF0VEGoBCnTQsE8fUesjJf2YIB4Qn99f0vivFtDYTY5IuQ0REXoRCnTSuWq+ncw5v/xBxbzfk87W99wownoXWgqZeRUTqlEKdNKQkzny1Q6OVNianpK+NCYBtbSYMckmXISIih6BQJw3JGAPlGo7UOUew8wnilgJRCnvTYQx0tFHjJYgiInIMFOqkcZVrN1LnPfUM3uQ05bNeAjZ9LzvjWWLfS7oMERE5jPT9dBFZAcY5XFSjkbo4JvfILqLuVekcpaMS6pxCnYhIXVOok4ZUaWdSmx51dnQCUywRnrGhMo2ZQiYX4IzeLkRE6pnepaUh2SiCqDahzn9mCOd7RGt6a3K/qsjncNr2KiJS1xTqpDFFEa4Woc45vIHhypFgXnqnL00+p1YmIiJ1TqFOGpIp1WaThJmZwxRLRGt6anK/qrAG5/tJVyEiIkegUCcNxxiDK5Vqci9vZByg0nA4pWzg62gwEZEUUKiThmMMUKxNqLMj48QtzbiWQk3uVw2mpYBLYRsWEZFGo3dqaTgmjnG1OCIsivBGx4n60jtKB0BLgVhdh0VE6p5CnTQcG4Y1CXV2ZBwTRkRrV1f9XtViAp840NSriEgaKNRJwzFhWJOdr/7+YZzvE/d1Vf1e1WJzAU6hTkQkFRTqpKEYAyzVYD3ds61M+ntT3cqE1gKxepmIiKSCQp00FGMMLC5V/T52bAJTKhOtS+/UK9ZAoUn96UREUkKhThqKiWNcufo96rz9wzhrU32KhPV94kD96URE0kKhThqKiWpw5qtzePuHKg2HU9y01xTyxDbFU8ciIg1GoU4ainUxrsrtOezQKHaxSHTKuqrep9qMWpmIiKSKQp00liiGKo/U+U89g8vniPr7qnqfajKexeVzSZchIiLHQKFOGoqJI6jm4FO5jDc4SnhSP6T4FAaTC4jUykREJFXS+1NH5DhUu+mwNziKiWOi9Wuqep9qM60Fqt/JT0REVpJCnTQMYwymyjtfvf1DxE154u5VVb1PVVkDzQW1MhERSRmFOmkYxoALqxjq4hhveJy4v+9Al+N0soFPlNN6OhGRtFGok4ZhDLhy9aZf7eQMJgyJ+rqrdo9asK3NxEZvDSIiaaN3bmkccQxRFUPd6DgAUYrPesUYXEszTnOvIiKpo1AnDaPSeLh6YcUbHifuaIN8vmr3qDYb+MQ57XoVEUkjhTppGMY5qrb6v1jCjk2k+lgwANPcROzpFAkRkTRSqJOGYeLqHRHmDQxjnCPMQisTnSIhIpJKCnXSOGJX+VUF/jNDxC0FXGd7Vb5+LRjP4tRwWEQktRTqpGGYuEqbJEol7Mg40fr+VLcyMb5H5PtJlyEiIsdJoU4aQjXbmXj7MzL12pTHpfhoMxGRRqd3cGkQpjrtTJzDf2p/6qdeASjk1cpERCTFFOqkIRgDVOGIMO/pAbzxScIzTk311CuePRDqki5ERESOl0KdNARjgJU+IqxYIvfwo0TdnYSnnrSyX7vGvKY8US69/fVERAS0KloyL3ARdr5EHK7s9Gvwf38H5ZDSK89O9ygdYDpaidTKREQk1RTqJLOMAT+KMCPjhLMLK/vFncPfP0S0fg2uo21lv3aNGd8jzueSLkNERE6QQp1kjjGGoFyChUXiyRni0sqvpTPTs5hiiWh1z4p/7VozuYA4CEADdSIiqaZQJ5liDASLC8SDY7gVnm5dVirj790PQNzXXZ171JBpbaY652yIiEgtKdRJpnhxjBuZqF6gA/K/ehhvaJS4vRXXXKjafWrCGig0aderiEgGKNRJJnjGYcMYMzdPVCxX70bFInZ4jPCUdZRf9pLq3adGbOAT5XQ0mIhIFijUSep4xmGePcfVgA1DGJ8iXipV/TB6/5khjHOUX7Ih/aN0gG1rJjQWDdWJiKRfJkLdnj172LZtG1NTU3R2drJ9+3Y2bNiQdFlSBbmwhBscw8XxgSBiiKIY4iqvCnOO4JFd+I8/SdzRlvodrwAYg2tu1ikSIiIZkYnmw5/61Ke49tpr+fGPf8y1117LJz/5yaRLkirIhSFuYJR4qYQrhbhyhCuH1Q90gP/oEwSPP0m0YT1Ll12Q+r50ADbnE6mViYhIZqQ+1I2Pj7Nz506uuOIKAK644gp27tzJxMREwpXJSvJxuOEx4mqulzsU5/Cf3Ee0ppfSprMhI0HItLUQm9S/BYiIyAGpf0cfHBxk9erVeJ4HgOd59PX1MTg4mHBlslKMATs7R7ywlMj97dgEdqlIeMq6TIzQAZVdry0FTb2KiGRIJtbUnaju7takSzhhvb0ZWON1CC6KiacmoaP2GxPczBzhbx8H36PlzJMxQXIvmY6V/PcHPravM9F/j7xQll/HUqFnnH1JPuPUv6P39/czPDxMFEV4nkcURYyMjNDf33/UX2N8fK7quyarqbe3jdHR2aTLqJpcuUQ0PgtR7Vvk5u75Dd7MPKULXsHCQhlIYPqXSqCbnl5csa/n93RQml7SSF0dyfrrWPSMG0G1n7G15rADUamffu3u7mbjxo3s2LEDgB07drBx40a6uroSrkxOlDWVQMf8YiKBDufwRieI1vYRrV1d+/tXizW4Fu16FRHJmtSP1AH83d/9Hdu2beMLX/gC7e3tbN++PemS5AR5BvzpWaKxSVxCo6hmZq5yvmsGjgJ7LpvPVXa9KtOJiGRKJkLdaaedxre+9a2ky5AVEsQRZmKKcGou0aa43sgYAHFvtkKdaWsmJiMbPkREZFkmQp1kgzGGYHERNzJe3aO+jkYU4T+xl7izHdeS/pMjlhkDTTrrVUQki1K/pk6ywVpDbn6eeGAkmV50z+P/7ins/CKlc85MupQVZXyPWDteRUQySe/ukihrDf7CIpRKRGNTuCQ2RDy/polpgp2/I1y3mnh1T9LlrCib84l8v3JuroiIZIpCnSTGM+DPzRMNjSUb5soh3jOD2OlZ7MwcdnQCl89ReuXLkqupWpoLxJp7FRHJJIU6qTljICiHMD5JOLeQ+KhRsGsPwaNP4DyPuL2V8LSTKW88DfL5ROtacdYeOEUi6UJERKQaFOqkJowxB07YcgQLi0RD47hymHRZANiRMaJVHRS3XJSdY8BehG3KEeXUykREJKsU6qTqPAPe9AwsFQEIZxcgTn7tHADlMnZimvDMUzMd6ABsRyv1EaNFRKQaFOqkqqwBb2qGaHQy0Z5zh+KNTmCcy1yD4eczgUdc0NSriEiWqaWJVI2Pw5+eIRqrz0AHYEfGcZ4l7u5MupSqss0FIrUyERHJNL3LS1UEcYQZGSOaXblD6KvBGxkn7ukCz0u6lOoxQHsLsdqYiIhkmkbqZMX5gBmZqPtAZxaXsDNzDTD16hNnbSeviIi8gEKdrCjPOLyJSaLZ+aRLOSI7OAKQ+VBnWwrEWR6JFBERQNOvsgI8HObAblZvcoZwcjrhio5CGBI8+gTRqnZcZ3vS1VSPMdCmqVcRkUagUCcnxMdhR8aJF5YACOuk99yR+E/txy4WWbrg3Ey3MrE5Tb2KiDQKhTo5Zs82EvbLJRieIJqv77VzL8Z7Zoi4vZW4tyvpUqrKtLUQGlu3u49FRGTlKNTJMcstLuCm54gXi3VzKsQxWSpixyYIN56edCVVZTwLbS04BToRkYagUCdHLYgjbKlEPDJBXCwnXc5x858ZwgDhutVJl1JVtqWZMAh0LJiISINQqJOjkgtD3MAwYYrDHADO4e/eS7SqI/sbJDpa0f4IEZHGoZYmcli+i8mVijAynurRuWfZkXHs7DzhaScnXUpV2VxA1KQNEiIijUQjdXJIvouxw6NEc0vZWGjvHMGjTxA35YlO6k+6mqoynW3ExmjqVUSkgSjUyQv4OLylJdzkDNFc+na2HoodncAbm6R07lmZPhbM+B60NGUih4uIyNFTqJODeKbSdy6cmku6lBXn7xvA+R7h761PupSqss0FQl8bJEREGo1CnQBgDXhhiJ2Zy2SgI47x9g8T9fdlepQOwLQ0aYOEiEgDUqgTjDEEc3NEoxOE5SjpcqrCjk5gSmWi9WuSLqW6rMHlcklXISIiCVCoa3DGQLC4SDQygQuzGegAvP1DOM8jWt2bdClVZXyPONDLWkSkEamlSQOz1hCUSsRDY9kNdK6yRtDfO0DU3wt+tqdebT5HnPHpZREReXH6X/oG5bsYb2SKaGExnUd9HSX/0SfI7XwCIPtTrwCFJh0LJiLSoBTqGpAxBjs9Szg5k3QpVWVHxgke3U3Y30fc20W0NtvHggHQlFMrExGRBqVQ14C8MCSenE26jKqy41Pkf/FrXFsLpfNfDrkg6ZKqzvgeztdLWkSkUeknQIPxPIs3O0eY0SlXMzdP8PgevGcGcfkcxUvPb4hABwc2Sfi++tOJiDQohboGEkQRZmCYqFhKupSqsBPT5H96DxhDtLqb8is24gpNSZdVM7alQIhBqU5EpDEp1DUI38WY0TGi+ZQf+7W4VBl5c2CnZsAavIERvIFhTLFUGZ3bclFDhTkA41lce4s2SYiINDCFugbg47BjE0Sz6Q50dmyC/J2/qvwhdpgDH3dA3LMK4pjSK89uuEAHB0bpcjmIk65ERESSolCXcZ4Bb2wy9Ud/mfkF8r94ANfcTLS2ctRXvKodopi4uxPXXEi6xOQYAx1txAp0IiINTaEuo4wBL47x5hcJ077TdXGJ/N2/BucoXrwJ19aSdEV1xTbliAp5LaUTEWlwCnUZFZTLuIFRwnKZVDcuCyOafnovZqmoQHcItrPtwAYJERFpZAp1GWOtwQtD3NAYcZp3ucYx3tMDeCPj2PkFli67gLivO+mq6o7J+UTNhVTndhERWRkKdRlhDFgc/vgU8dwC8VJ6A52ZmSN332/wJqcBCNeuVqA7BK+9lZLnp3s0VkREVoRCXQa42JGbmYUwIhyfSu/aqlIZf/degkd3g+9RvPBcXHMTcXtr0pXVJeN7uDa1MRERkQqFupQzBtz0LOHIBMQp/uFeLNL0f36BXVwiXLea0nkvg6Z80lXVNa+9hXIul+7nLiIiK0ahLuWCcogbn0rnD/Y4xhsYxgUBwWO7McUSS6+5kLi3K+nK6p4JfOLODuI0PncREakKhboUMsZU1tDFMW54DIKU7XyMIvyn9uM//iR2odIQ2QGlC16uQHeUbGszYeCnM8yLiEhVKNSljDXgT0/DUglXDokXlqAjJY13wxB/zz78x/dgl4pEXR0Uz92ImZ3HtRSI1vcnXWFqmJaCRulEROQgCnUp4y8uEY1OkarjA8phZQPErj2YUpmot4ul819e2dFqUjbKWAeM7xHngqTLEBGROqNQlyK+i3GjE+kKdMUSTT+7FzszR7Sml/JLT6uc0yrHzeZzREGgqVcRETmIQl1KWAPe+BRhmvrPlcvk77ofM7fA0iWbidf0Jl1R+hmD6WonUqATEZHnsUkXIEcnWFwknErRGa5hRP7uX2OnZii96lwFuhViC3nCQlPSZYiISB1SqEsB38XEoylqWxLH5H/5AHZsktIFLydauzrpirLBWkzPKiKndYgiIvJCCnV1zlrwJqeJl4pJl3J04pjcvQ/hDY9R2nQ20Ulrk64oM/zONkI1ZBYRkUPQmro6ZgwEs/OEkzNJl3J0nCP360fw9w9TesVLiX7vpKQryo7AJ+poT81grYiI1J5CXR3xcJgowjhXWRDvYqKxyfqedi2XsRPT4Hn4u57EHxihdNbphC/5vaQryxTT1UmEl95zfUVEpOoU6uqEteCPThHNzFUOaDcGB7gwSrq0Q4tj8nf9Gm98EgDnWUovfynhSzYkW1fGeG3NmI5W3NRi0qWIiEgdU6irA8ZAMLdAODVT36NyzxVG5H71MN74JKVzzsS1NhP1dEE+l3RlmWLzAfHqHkygl6qIiByeflLUgSAsE41MpCfQPbu7dXiM0rkbCU/fkHRFmWXaWwitl3QZIiKSAgp1CbHWYAATxzA6gSuHSZd01IJHd+MNj1F85dlEp2ozRDWYnI8t5KG5uTIdLyIicgQKdQmwBvypadz0HDhHVCwnXdJRM4tL+Lv2EK5fo0BXLcbg9XYRtSjQiYjI0VOoqzFrIJidIxydStcZrnGMHZ8i+M1j4Bzls89MuqLMsk05wpZmIgegRsMiInJ0FOpqyMPhTc4QjqfodAgqo3O5Xz6INzGFM4bSRefhWpuTLiuzTHsrKYr7IiJSJxTqasQa8KZmK33n0pLnoghv/zDBY7sxC4sUN59DvLoHp7NHq8Z4FgpNaNZVRESOlUJdDSxPuY5NpSfQlcvk7/413tgkzvcpXryJuLc76aoyz+tso5wL0vPfiYiI1A2FuhqwUUQ0lqI1dOWQ/D0PYcenKF7wcqJ1a8BTW41q81oKRF2daZqZFxGROqJQJwexQ6Pk73kQE0YUN51NdPK6pEtqCCbwoa+bUBsjRETkOCnUyTI7Pkn+ngdxLc0UX3k2cXdn0iU1Bmuxa3ooB76mXUVE5Lgp1AlmfoHcrx/Bjk/impooXrJZmyFqyCvkiZoLOM27iojICVCoa3TlkPwvH8DMLxKevI7wrNMV6GrMdLQSKdCJiMgJUqhrZOUy+Z/fh5meo/jqTcT9vUlX1HBscxORQrSIiKwAhboGZSemCB7ZhZ2cofSq8xToas2z+L1dRK3NhMYmXY2IiGSAQl0D8vYNkr/3IRxQ2nQ20brVSZfUcLz2FsrtrWpfIiIiK0ahrsE8uyki6uqkePEmyOeSLqnh2KYcrOpQoBMRkRWlUNco4hh/zz78J/eBgdKFr2iYQOe1FjD5HOHkbOINoG0uwKzto+TppSciIitLP1kaQTkkf+9DeEOjuFxA6fyX41qak66qNjwL3asIC014xRLR3GJytViD7eui5KsfnYiIrDyFuqxbKtJ014Edrq88m+jUk5KuqKb8Ve2U8zlcHOO1NEOCoc7vbKPcUsCl5LQ4ERFJF4W6DDMLi+Tv/BVmsUjx4k3Eaxprh6st5Ik62///2rWmHBgDrsbDZMbg9XYSdbQlPfsrIiIZplCXRc7hP/4kwe+egjimeOn5xD2rkq6qdgxgLKank/Jz2oXEvo8JPFwprGk5XmuBqLODSFOuIiJSRQp1WeMcwUM7CXY/TdTXTenlL8V1tidd1cqyBmIHxmA8i4tjjOdVLjU3QWcbYAjzwUFr12Lfx2/KEdUo1Bnfq+x07VmlQCciIlWnUJdIydb3AAASoElEQVQhdmiUYNcevJFxymf8HuVzzqxMN2aI11qA7k6YX4RCEy7wsXFM7HngHLFnCTkwOve8IBXHDtPaAjMLVa/T+B52XR9RU5OOABMRkZpQqMuA+Jlh/IExgt8+DrmA0rkbCU87JXOBzgQ+9HZR8gNMvgn37No47+i/hsvnKqN7UXUXt9nmAmE+T6xAJyIiNaJQl2bOETz8GNETT5ED4o42ll53EfjHkHLqmM0HYC2uHGICH9vdSSmoTKm649zsEAUBfkcb4eT0yrUVMQavkCculgDwelfhmpvUXFhERGpKoS6twpDcvQ/jD45gzzqNhZ5u4o7WTAQ643t43Z1E7a04A145JMrlCJ074Y2rkQN6OvHCkGhmfkXq9Xo7iVZ14JXKmHKZckuLRuhERKTmFOrSJIrI3fcbvNEJCCOIIkrnnkXLppcSTyfYVHelWIPX3QktBUq5fGU0zkHkB6zksFfkDF5XJ2ZhCRdGR1GXxWvOEy0sgQOb84lL5crv8wFxRxtRXKnTBAFOgU5ERBKgUJcW5TL5Xz6IHRknOmUdLvCJ1q4m7utOurLjZjyL8Tyci3HlCK+jlairs7KxoMq95MJcgF/IE80eftOE195SOae1KYc/MQX5PFFzE/74FPFiEdvVTsnY5ancWrfAExEReZZCXQqYuQXyd9+PmVugtPkcog3rky7puNlCHtPWAvkczvdwvocNIxifwj0b6GogdlTqOEyoM74H3Z2U/ABiiFd1VgYMHcQ9XRgDYXziU8IiIiIrQaGuzpm5efI/vRcTxxQvO5+4N70jc15bgXhNH6ExBwch3+Kv6SWscThy+Rwm5x+yGbHX0Uo5l1ue+n1u3tSaORERqTcKdfXGOYhjsBb/yacrbUqsZek1F+I62pKu7rh5bc3Eq3sIMS+667TWgQ6g7AcE69fgBkeJF4sHXTOBT9zRpvAmIiKpoVBXD5zDf2w3/tMDmPkFTOxwvo8JQ6LVPZTOOwvX2pJ0lcfNFvKVQPecI7vqgQNKnk++p5N4/8j/H4ozlQ0bJT/QIjkREUkNhbqkFYvk730Yb2ScqK+bqL8PlwuwcwvEzQXCjaeluomwzQWYNb0HncFab8qFAn5PJywWoaUAgU+5uaBdrCIikioKdUlYXMIbm8SUy/iPP4lZLFLcdHZlA0SKA9zzGc9i1vRQ8uv7P7PYQdjZAasM7tleeNU9cEJERGTF1fdP24wxM3MEj+3G2zeIOTCt55ryFF97IXFXZ8LVrTBjsN2dhIWmFe0xVy2VXa31X6eIiMih1MWc2K233sqb3/xmzjrrLL72ta8ddG1xcZEPf/jDvOENb+Dyyy/nJz/5yVFdqyvFEv6DO2m6/S68gWHC005m8fWvZnHra1nc+trsBTpr8Ho6iTrbtdFARESkRupipG7jxo189rOf5T/+4z9ecO0rX/kKra2t3H777Tz11FNcd9113HbbbbS0tBz2Wr0w45M0/fd3MXMLhBvWUzr7DGjKJ11W9RiD19tF2NlGrClMERGRmqmLkbozzjiD008/HWtfWM4Pf/hDrrnmGgA2bNjA2WefzZ133nnEa/XCRDHR+jUsveESSpvPyWSgM4GH8Sy2kMdf3X1ghC7pqkRERBpLXYzUHc7AwADr1q1b/nN/fz9DQ0NHvHYsurtbT7zQQ+ltw52+lvip/ZXzWquko6NQta99WNZg+vsqpy805TB1vikizXp709unUI6OnnH26RlnX5LPuCY/ga+++moGBgZe9NovfvELPM+rRRmHND4+V9W1X34cwcwSrvziJxecqI6OAtPTi1X52odlDH5PB+UQ4nIEiwnU0CB6e9sYHZ1NugypIj3j7NMzzr5qP2NrzWEHomoS6r7zne8c999du3Yt+/fvp6urC4DBwUEuvPDCI16T6vN7OgifPQ9VREREElUXa+oO5/LLL+eb3/wmAE899RS//e1vufTSS494TarLFvJEHe1ECnQiIiJ1oS5C3Y4dO7jsssv40Y9+xM0338xll13GE088AcD111/PzMwMb3jDG3jf+97HDTfcQGtr6xGvSXWZ9laiF9nYIiIiIskwzqnjak3W1D09mJ01ddbgndxPKcjV7p4NTmtxsk/POPv0jLMv6TV1GmqRY2Z9n1i7XEVEROqKQp0cG2uwrQXihHcsi4iIyME03CJHx1r87nZcSzNhLqfjv0REROqMQp0cnjHYfIDp7aJcaDpw8H3SRYmIiMjzKdTJIRnP4q3uJmxtJsagLTUiIiL1S6FOXpwxeH1dlNtaNdUqIiKSAgp18kKexevqIFSgExERSQ2FOjmYNfh9XZTb2xToREREUkShTiqMweZ8bFeHplxFRERSSKFOAPC7O4hWdVAyFh0yIiIikj4KdYLX0Uq0qoMQg7a4ioiIpJNCXYPzV7UR9XRTnVNpRUREpFYU6hqUCTxsdydRW6sCnYiISAYo1DUgmw8wq7spNxW0fk5ERCQjFOoahbV47c3Q3kacCwitNkSIiIhkiUJdA7CFPKZ3FWGhiTg+8EHlORERkUxRqMsw43vYrg7i9lbKxkJ85L8jIiIi6aRQl1E2H2DW9FLO5zXNKiIi0gAU6rLGWrzWZujppOT56jsnIiLSIBTqMsQEPra/l6jQRKRjvkRERBqKQl2aHTiv1XR14HV0gDGV6VYFOhERkYajUJcm1mLzAbarAzyLM4Yol8P2tlMana18jvKciIhIQ1KoqwfGYKzB+B6mKQ85HxM74vlFjOeBBdPRhvN9oiCgjEm6YhEREakzCnU14rU1Qy4H1uDiGIpliGMIfGgu4DxL7Pu45abABtvtcKYS4GJNqYqIiMhhKNTVQOR5xL3dBwUz21EJa85xcMuR5c9xRM9+goiIiMgRKNTVwAuCGxp5ExERkZVlky5ARERERE6cQp2IiIhIBijUiYiIiGSAQp2IiIhIBijUiYiIiGSAQp2IiIhIBijUiYiIiGSAQp2IiIhIBijUiYiIiGSAQp2IiIhIBijUiYiIiGSAQp2IiIhIBijUiYiIiGSAQp2IiIhIBijUiYiIiGSAQp2IiIhIBijUiYiIiGSAQp2IiIhIBijUiYiIiGSAQp2IiIhIBijUiYiIiGSAn3QB9cBak3QJJywL/wY5PD3j7NMzzj494+yr5jM+0tc2zjlXtbuLiIiISE1o+lVEREQkAxTqRERERDJAoU5EREQkAxTqRERERDJAoU5EREQkAxTqRERERDJAoU5EREQkAxTqRERERDJAoU5EREQkAxTqErB9+3a2bNnCmWeeya5du5Y/vmfPHq655hre+MY3cs011/DUU08ldk1OzOTkJO95z3t44xvfyJvf/GY++MEPMjExAcBDDz3EW97yFt74xjfy7ne/m/Hx8eW/V+trcmI+8IEP8Ja3vIWrrrqKa6+9lkcffRTQazmLPve5zx30nq3XcXZs2bKFyy+/nCuvvJIrr7ySn//850BKn7GTmrvvvvvcwMCAe93rXucef/zx5Y+/4x3vcLfccotzzrlbbrnFveMd70jsmpyYyclJd8899yz/+dOf/rT72Mc+5qIocr//+7/v7rvvPuecc5///Ofdtm3bnHOu5tfkxM3MzCz//vbbb3dXXXWVc06v5ax55JFH3PXXX7/8nq3XcbY8/2exc7V/jiv1jBXqEvTc/5DGxsbcpk2bXBiGzjnnwjB0mzZtcuPj4zW/JivvRz/6kXvnO9/pHn74YfemN71p+ePj4+Pu3HPPdc65ml+TlfWd73zHXX311XotZ0yxWHR/+Id/6Pbt27f8nq3Xcba8WKhL6zP2j2OkUqpgcHCQ1atX43keAJ7n0dfXx+DgIM65ml7r6upK4DuQXXEc8z//8z9s2bKFwcFB1q5du3ytq6uLOI6Zmpqq+bXOzs4q/8sbwyc+8QnuvvtunHN8+ctf1ms5Y26++Wbe8pa3sH79+uWP6XWcPR/96EdxzrFp0yY+8pGPpPYZa02dSJXdeOONNDc38/a3vz3pUqQK/uEf/oGf/vSn/NVf/RWf+cxnki5HVtCDDz7II488wrXXXpt0KVJFX//61/nud7/Lt7/9bZxz3HDDDUmXdNwU6upEf38/w8PDRFEEQBRFjIyM0N/fX/NrsnK2b9/O3r17+dd//VestfT39zMwMLB8fWJiAmstnZ2dNb8mK+uqq67i3nvvZc2aNXotZ8R9993H7t27ef3rX8+WLVsYGhri+uuvZ+/evXodZ8izr5VcLse1117LAw88kNr3aoW6OtHd3c3GjRvZsWMHADt27GDjxo10dXXV/JqsjH/5l3/hkUce4fOf/zy5XA6As88+m6WlJe6//34AvvGNb3D55Zcnck1OzPz8PIODg8t/vuOOO+jo6NBrOUPe+973ctddd3HHHXdwxx13sGbNGr7yla/wZ3/2Z3odZ8TCwgKzs7MAOOf4wQ9+wMaNG1P7Xm2cc+64vhNy3G666SZuu+02xsbGWLVqFZ2dnXz/+99n9+7dbNu2jZmZGdrb29m+fTunnnoqQM2vyYn53e9+xxVXXMGGDRtoamoCYP369Xz+85/ngQce4FOf+hTFYpF169bxT//0T/T09ADU/Jocv7GxMT7wgQ+wuLiItZaOjg7+5m/+hpe97GV6LWfUli1b+OIXv8gZZ5yh13FG7Nu3jw996ENEUUQcx5x22mn87d/+LX19fal8xgp1IiIiIhmg6VcRERGRDFCoExEREckAhToRERGRDFCoExEREckAhToRERGRDFCoE5GG9aY3vYl777036TJERFaEzn4Vkcw677zzln+/uLhILpdbPiP17//+7/n+979f85rOPPNMbrvtNk455ZSa31tEsk2hTkQy68EHH1z+/ZYtW7jpppt49atfnWBFIiLVo+lXEWlYW7Zs4Re/+AUA//Zv/8Zf/uVf8tGPfpTzzjuPN7/5zezZs4cvfelLXHTRRbzmNa/hrrvuWv67s7OzfPzjH+eSSy7h0ksv5bOf/ezy+at79+7l7W9/O5s2beLCCy/kwx/+MADXXXcdAFdeeSXnnXceP/jBDwD4yU9+wpVXXsnmzZv5oz/6Ix577LGDavzSl77E1q1bOf/88/nYxz5GsVgEKudDvu9972Pz5s1ccMEFXHvttcRxXP1vnIjUJYU6EZEDng1X9913Hxs3buT6668njmPuvPNO/uIv/oJPfvKTy5+7bds2fN/ntttu45ZbbuHuu+/mW9/6FgA333wzF198Mffddx933nknb3/72wH4+te/DsCtt97Kgw8+yNatW9m5cycf//jHueGGG7j33nu55ppr+MAHPkCpVFq+1/e+9z2+8pWvcPvtt7Nnzx6+8IUvAPDVr36V1atX88tf/pK7776bj3zkIxhjavXtEpE6o1AnInLA5s2bufTSS/F9n8svv5zJyUne+973EgQBW7duZf/+/czMzDA2NsbPfvYzPv7xj9Pc3Ex3dzd/+qd/urxGz/d9BgYGGBkZIZ/Ps3nz5kPe85vf/CbXXHMNr3jFK/A8j6uvvpogCHjooYeWP+e6666jv7+fzs5O3v/+9x90n9HRUQYGBgiCgM2bNyvUiTQwrakTETmgu7t7+fdNTU2sWrVqeWNFU1MTAAsLC4yMjBCGIZdccsny58dxTH9/PwB//dd/zc0338zb3vY2Ojo6eNe73sXb3va2F73nwMAAt9xyC1/72teWP1YulxkZGVn+87NfF2Dt2rXL166//no+97nP8e53vxuAa665hve+970n9D0QkfRSqBMROUZr1qwhl8txzz334PsvfBvt7e3lpptuAuD+++/nXe96F+eff/6L7njt7+/nz//8z3n/+99/yPsNDg4u/35gYIC+vj4AWltb2bZtG9u2bWPXrl28853v5JxzzuGiiy460X+iiKSQpl9FRI5RX18fF198MZ/+9KeZm5sjjmOefvppfvWrXwHwwx/+kKGhIQA6OjowxmBt5e22p6eHffv2LX+tP/iDP+Ab3/gGDz/8MM45FhYW+OlPf8rc3Nzy5/z3f/83Q0NDTE1N8cUvfpGtW7cClTWAe/fuxTlHW1sbnudp+lWkgWmkTkTkOHzmM5/hn//5n9m6dSvz8/OcdNJJvOc97wHgt7/9Lf/4j//I3Nwc3d3dfOITn+Ckk04C4IMf/CDbtm1jaWmJG264ga1bt3LjjTdyww03sHfvXpqamnjlK1950Dq8K664gne/+92MjIzw+te/fnlUb+/evdx4441MTEzQ3t7OH//xH/OqV72q9t8MEakLxjnnki5CRERenPrricjR0vSriIiISAYo1ImIiIhkgKZfRURERDJAI3UiIiIiGaBQJyIiIpIBCnUiIiIiGaBQJyIiIpIBCnUiIiIiGaBQJyIiIpIB/w/BO8Y+6FCsIwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4iGv72FR9gML"
      },
      "source": [
        "def get_expert_buf(num=100000):\n",
        "    buf = {'observation':[], 'action':[], 'terminal':[], 'reward':[]}\n",
        "    obs = env.reset()\n",
        "    for _ in range(num):\n",
        "        action, _states = model.predict(obs)\n",
        "        buf['observation'].append(obs)\n",
        "        buf['action'].append(action)\n",
        "        obs, reward, done, info = env.step(action)\n",
        "        buf['terminal'].append(done)\n",
        "        buf['reward'].append(reward)\n",
        "        if done:\n",
        "            obs = env.reset()\n",
        "    buf['observation'] = np.stack(buf['observation'])\n",
        "    buf['action'] = np.stack(buf['action'])\n",
        "    buf['reward'] = np.stack(buf['reward'])\n",
        "    \n",
        "    return buf\n",
        "\n",
        "expert_data = get_expert_buf()\n",
        "with open(SAVE_DIR+ENV_NAME+'.expert_data.pkl', 'wb') as f:\n",
        "    pkl.dump(expert_data, f)"
      ],
      "execution_count": 8,
      "outputs": []
    }
  ]
}