SAC_online_LunarLander.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "SAC_online_LunarLander.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "eP8e9dKYFuHd"
      },
      "source": [
        "!pip install box2d-py\n",
        "!pip install stable-baselines3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IitdeHDwF3rx"
      },
      "source": [
        "import numpy as np\n",
        "from google.colab import drive\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import pickle as pkl\n",
        "import pandas as pd\n",
        "import gym\n",
        "import os\n",
        "\n",
        "import torch as T\n",
        "import torch.nn as nn\n",
        "\n",
        "from stable_baselines3.common.utils import set_random_seed\n",
        "from stable_baselines3.common.monitor import Monitor\n",
        "from stable_baselines3.common.vec_env import VecFrameStack, VecNormalize, VecTransposeImage\n",
        "from stable_baselines3.common.callbacks import CheckpointCallback\n",
        "from stable_baselines3.common.type_aliases import ReplayBufferSamples\n",
        "from stable_baselines3.common.buffers import ReplayBuffer\n",
        "from stable_baselines3 import SAC"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OUG8MAapGDaM"
      },
      "source": [
        "DEVICE = T.device('cuda' if T.cuda.is_available() else 'cpu')\n",
        "ENV_NAME = 'LunarLanderContinuous-v2'\n",
        "SAVE_DIR = 'gdrive/MyDrive/SQIL/Online/'\n",
        "ONLINE_STEPS = 5e5    # 500,000 online environment steps\n",
        "SEED = 1337"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BeYYE97iemC3",
        "outputId": "0b3f0510-8dfd-4849-b558-5871346a6721"
      },
      "source": [
        "drive.mount('/content/gdrive/')  # model checkpoints + training stats \n",
        "set_random_seed(SEED)  # torch, np and py seeds\n",
        "\n",
        "if not os.path.exists(SAVE_DIR):\n",
        "    os.makedirs(SAVE_DIR)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5IuamzwgsE3L",
        "outputId": "512638ca-cda1-467f-a1cb-68e1305857f1"
      },
      "source": [
        "env = Monitor(gym.make(ENV_NAME), filename=SAVE_DIR+ENV_NAME)\n",
        "\n",
        "checkpoint_callback = CheckpointCallback(\n",
        "    save_freq=ONLINE_STEPS, \n",
        "    save_path=SAVE_DIR, \n",
        "    name_prefix=ENV_NAME\n",
        ")\n",
        "\n",
        "model = SAC(\n",
        "    env=env,\n",
        "    policy='MlpPolicy',\n",
        "    learning_starts=1000,\n",
        "    batch_size=256,\n",
        "    verbose=1\n",
        ")"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using cuda device\n",
            "Wrapping the env in a DummyVecEnv.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P7f1m5gnd8XY",
        "outputId": "e9284db9-6af9-4bcd-8913-edd1e3c7f748"
      },
      "source": [
        "model.learn(ONLINE_STEPS, log_interval=10, callback=checkpoint_callback)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 113      |\n",
            "|    ep_rew_mean     | -181     |\n",
            "| time/              |          |\n",
            "|    episodes        | 10       |\n",
            "|    fps             | 496      |\n",
            "|    time_elapsed    | 2        |\n",
            "|    total timesteps | 1131     |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -1.21    |\n",
            "|    critic_loss     | 175      |\n",
            "|    ent_coef        | 0.962    |\n",
            "|    ent_coef_loss   | -0.122   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 130      |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 140      |\n",
            "|    ep_rew_mean     | -238     |\n",
            "| time/              |          |\n",
            "|    episodes        | 20       |\n",
            "|    fps             | 129      |\n",
            "|    time_elapsed    | 21       |\n",
            "|    total timesteps | 2808     |\n",
            "| train/             |          |\n",
            "|    actor_loss      | 4.14     |\n",
            "|    critic_loss     | 3.13     |\n",
            "|    ent_coef        | 0.603    |\n",
            "|    ent_coef_loss   | -1.22    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 1807     |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 371      |\n",
            "|    ep_rew_mean     | -172     |\n",
            "| time/              |          |\n",
            "|    episodes        | 30       |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 135      |\n",
            "|    total timesteps | 11133    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -9.33    |\n",
            "|    critic_loss     | 11.2     |\n",
            "|    ent_coef        | 0.0778   |\n",
            "|    ent_coef_loss   | -1.5     |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 10132    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 450      |\n",
            "|    ep_rew_mean     | -158     |\n",
            "| time/              |          |\n",
            "|    episodes        | 40       |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 226      |\n",
            "|    total timesteps | 17992    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -12      |\n",
            "|    critic_loss     | 49.5     |\n",
            "|    ent_coef        | 0.0696   |\n",
            "|    ent_coef_loss   | -0.577   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 16991    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 525      |\n",
            "|    ep_rew_mean     | -135     |\n",
            "| time/              |          |\n",
            "|    episodes        | 50       |\n",
            "|    fps             | 77       |\n",
            "|    time_elapsed    | 340      |\n",
            "|    total timesteps | 26270    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -18.2    |\n",
            "|    critic_loss     | 4.17     |\n",
            "|    ent_coef        | 0.0828   |\n",
            "|    ent_coef_loss   | 0.412    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 25269    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 590      |\n",
            "|    ep_rew_mean     | -116     |\n",
            "| time/              |          |\n",
            "|    episodes        | 60       |\n",
            "|    fps             | 75       |\n",
            "|    time_elapsed    | 466      |\n",
            "|    total timesteps | 35377    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -17.4    |\n",
            "|    critic_loss     | 14.4     |\n",
            "|    ent_coef        | 0.0609   |\n",
            "|    ent_coef_loss   | 0.435    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 34376    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 630      |\n",
            "|    ep_rew_mean     | -100     |\n",
            "| time/              |          |\n",
            "|    episodes        | 70       |\n",
            "|    fps             | 75       |\n",
            "|    time_elapsed    | 584      |\n",
            "|    total timesteps | 44076    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -11.2    |\n",
            "|    critic_loss     | 29.5     |\n",
            "|    ent_coef        | 0.0481   |\n",
            "|    ent_coef_loss   | 1.07     |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 43075    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 635      |\n",
            "|    ep_rew_mean     | -83.5    |\n",
            "| time/              |          |\n",
            "|    episodes        | 80       |\n",
            "|    fps             | 75       |\n",
            "|    time_elapsed    | 673      |\n",
            "|    total timesteps | 50813    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -16.7    |\n",
            "|    critic_loss     | 2.63     |\n",
            "|    ent_coef        | 0.0712   |\n",
            "|    ent_coef_loss   | 0.304    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 49812    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 594      |\n",
            "|    ep_rew_mean     | -72      |\n",
            "| time/              |          |\n",
            "|    episodes        | 90       |\n",
            "|    fps             | 75       |\n",
            "|    time_elapsed    | 705      |\n",
            "|    total timesteps | 53504    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -19.6    |\n",
            "|    critic_loss     | 6.92     |\n",
            "|    ent_coef        | 0.065    |\n",
            "|    ent_coef_loss   | -0.325   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 52503    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 599      |\n",
            "|    ep_rew_mean     | -56      |\n",
            "| time/              |          |\n",
            "|    episodes        | 100      |\n",
            "|    fps             | 76       |\n",
            "|    time_elapsed    | 786      |\n",
            "|    total timesteps | 59910    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -20.5    |\n",
            "|    critic_loss     | 3.37     |\n",
            "|    ent_coef        | 0.0845   |\n",
            "|    ent_coef_loss   | 0.498    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 58909    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 655      |\n",
            "|    ep_rew_mean     | -31.5    |\n",
            "| time/              |          |\n",
            "|    episodes        | 110      |\n",
            "|    fps             | 76       |\n",
            "|    time_elapsed    | 871      |\n",
            "|    total timesteps | 66603    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -18.8    |\n",
            "|    critic_loss     | 1.58     |\n",
            "|    ent_coef        | 0.0667   |\n",
            "|    ent_coef_loss   | -0.124   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 65602    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 702      |\n",
            "|    ep_rew_mean     | 8.31     |\n",
            "| time/              |          |\n",
            "|    episodes        | 120      |\n",
            "|    fps             | 76       |\n",
            "|    time_elapsed    | 952      |\n",
            "|    total timesteps | 72990    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -19.4    |\n",
            "|    critic_loss     | 1.51     |\n",
            "|    ent_coef        | 0.0586   |\n",
            "|    ent_coef_loss   | 0.385    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 71989    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 657      |\n",
            "|    ep_rew_mean     | 26.8     |\n",
            "| time/              |          |\n",
            "|    episodes        | 130      |\n",
            "|    fps             | 76       |\n",
            "|    time_elapsed    | 999      |\n",
            "|    total timesteps | 76837    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -17      |\n",
            "|    critic_loss     | 19       |\n",
            "|    ent_coef        | 0.0578   |\n",
            "|    ent_coef_loss   | 0.339    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 75836    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 627      |\n",
            "|    ep_rew_mean     | 57.4     |\n",
            "| time/              |          |\n",
            "|    episodes        | 140      |\n",
            "|    fps             | 77       |\n",
            "|    time_elapsed    | 1047     |\n",
            "|    total timesteps | 80676    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -20.9    |\n",
            "|    critic_loss     | 4.82     |\n",
            "|    ent_coef        | 0.0529   |\n",
            "|    ent_coef_loss   | 0.0285   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 79675    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 590      |\n",
            "|    ep_rew_mean     | 64.5     |\n",
            "| time/              |          |\n",
            "|    episodes        | 150      |\n",
            "|    fps             | 77       |\n",
            "|    time_elapsed    | 1106     |\n",
            "|    total timesteps | 85264    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -23.2    |\n",
            "|    critic_loss     | 4.7      |\n",
            "|    ent_coef        | 0.0531   |\n",
            "|    ent_coef_loss   | 0.972    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 84263    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 536      |\n",
            "|    ep_rew_mean     | 59.2     |\n",
            "| time/              |          |\n",
            "|    episodes        | 160      |\n",
            "|    fps             | 77       |\n",
            "|    time_elapsed    | 1153     |\n",
            "|    total timesteps | 88974    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -20.4    |\n",
            "|    critic_loss     | 7.8      |\n",
            "|    ent_coef        | 0.0607   |\n",
            "|    ent_coef_loss   | -0.395   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 87973    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 510      |\n",
            "|    ep_rew_mean     | 65.3     |\n",
            "| time/              |          |\n",
            "|    episodes        | 170      |\n",
            "|    fps             | 77       |\n",
            "|    time_elapsed    | 1229     |\n",
            "|    total timesteps | 95055    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -19.5    |\n",
            "|    critic_loss     | 5.55     |\n",
            "|    ent_coef        | 0.0597   |\n",
            "|    ent_coef_loss   | 0.0685   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 94054    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 500      |\n",
            "|    ep_rew_mean     | 76.1     |\n",
            "| time/              |          |\n",
            "|    episodes        | 180      |\n",
            "|    fps             | 77       |\n",
            "|    time_elapsed    | 1304     |\n",
            "|    total timesteps | 100791   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -20.4    |\n",
            "|    critic_loss     | 3.09     |\n",
            "|    ent_coef        | 0.0571   |\n",
            "|    ent_coef_loss   | -0.764   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 99790    |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 515      |\n",
            "|    ep_rew_mean     | 98.2     |\n",
            "| time/              |          |\n",
            "|    episodes        | 190      |\n",
            "|    fps             | 77       |\n",
            "|    time_elapsed    | 1355     |\n",
            "|    total timesteps | 104977   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -20.4    |\n",
            "|    critic_loss     | 5.81     |\n",
            "|    ent_coef        | 0.0547   |\n",
            "|    ent_coef_loss   | 0.148    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 103976   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 486      |\n",
            "|    ep_rew_mean     | 115      |\n",
            "| time/              |          |\n",
            "|    episodes        | 200      |\n",
            "|    fps             | 77       |\n",
            "|    time_elapsed    | 1398     |\n",
            "|    total timesteps | 108524   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -26.6    |\n",
            "|    critic_loss     | 2.43     |\n",
            "|    ent_coef        | 0.0721   |\n",
            "|    ent_coef_loss   | 0.156    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 107523   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 465      |\n",
            "|    ep_rew_mean     | 131      |\n",
            "| time/              |          |\n",
            "|    episodes        | 210      |\n",
            "|    fps             | 77       |\n",
            "|    time_elapsed    | 1454     |\n",
            "|    total timesteps | 113072   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -30      |\n",
            "|    critic_loss     | 6.27     |\n",
            "|    ent_coef        | 0.0851   |\n",
            "|    ent_coef_loss   | 0.553    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 112071   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 445      |\n",
            "|    ep_rew_mean     | 145      |\n",
            "| time/              |          |\n",
            "|    episodes        | 220      |\n",
            "|    fps             | 77       |\n",
            "|    time_elapsed    | 1508     |\n",
            "|    total timesteps | 117508   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -35.4    |\n",
            "|    critic_loss     | 4.38     |\n",
            "|    ent_coef        | 0.0903   |\n",
            "|    ent_coef_loss   | -0.33    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 116507   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 448      |\n",
            "|    ep_rew_mean     | 155      |\n",
            "| time/              |          |\n",
            "|    episodes        | 230      |\n",
            "|    fps             | 77       |\n",
            "|    time_elapsed    | 1560     |\n",
            "|    total timesteps | 121653   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -39      |\n",
            "|    critic_loss     | 6.8      |\n",
            "|    ent_coef        | 0.085    |\n",
            "|    ent_coef_loss   | 0.428    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 120652   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 444      |\n",
            "|    ep_rew_mean     | 161      |\n",
            "| time/              |          |\n",
            "|    episodes        | 240      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 1601     |\n",
            "|    total timesteps | 125122   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -41.5    |\n",
            "|    critic_loss     | 3.32     |\n",
            "|    ent_coef        | 0.0774   |\n",
            "|    ent_coef_loss   | 0.0851   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 124121   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 428      |\n",
            "|    ep_rew_mean     | 180      |\n",
            "| time/              |          |\n",
            "|    episodes        | 250      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 1636     |\n",
            "|    total timesteps | 128086   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -44.1    |\n",
            "|    critic_loss     | 11.6     |\n",
            "|    ent_coef        | 0.0759   |\n",
            "|    ent_coef_loss   | 0.276    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 127085   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 419      |\n",
            "|    ep_rew_mean     | 207      |\n",
            "| time/              |          |\n",
            "|    episodes        | 260      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 1668     |\n",
            "|    total timesteps | 130841   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -41.8    |\n",
            "|    critic_loss     | 4.52     |\n",
            "|    ent_coef        | 0.0706   |\n",
            "|    ent_coef_loss   | -0.334   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 129840   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 385      |\n",
            "|    ep_rew_mean     | 223      |\n",
            "| time/              |          |\n",
            "|    episodes        | 270      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 1701     |\n",
            "|    total timesteps | 133552   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -41.7    |\n",
            "|    critic_loss     | 13.6     |\n",
            "|    ent_coef        | 0.0633   |\n",
            "|    ent_coef_loss   | 0.538    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 132551   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 354      |\n",
            "|    ep_rew_mean     | 225      |\n",
            "| time/              |          |\n",
            "|    episodes        | 280      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 1732     |\n",
            "|    total timesteps | 136199   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -44.7    |\n",
            "|    critic_loss     | 25.8     |\n",
            "|    ent_coef        | 0.0681   |\n",
            "|    ent_coef_loss   | 0.593    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 135198   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 338      |\n",
            "|    ep_rew_mean     | 220      |\n",
            "| time/              |          |\n",
            "|    episodes        | 290      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 1762     |\n",
            "|    total timesteps | 138760   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -41.2    |\n",
            "|    critic_loss     | 12.6     |\n",
            "|    ent_coef        | 0.0766   |\n",
            "|    ent_coef_loss   | 0.122    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 137759   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 327      |\n",
            "|    ep_rew_mean     | 209      |\n",
            "| time/              |          |\n",
            "|    episodes        | 300      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 1791     |\n",
            "|    total timesteps | 141227   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -39.8    |\n",
            "|    critic_loss     | 17.9     |\n",
            "|    ent_coef        | 0.0797   |\n",
            "|    ent_coef_loss   | 0.156    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 140226   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 308      |\n",
            "|    ep_rew_mean     | 208      |\n",
            "| time/              |          |\n",
            "|    episodes        | 310      |\n",
            "|    fps             | 78       |\n",
            "|    time_elapsed    | 1822     |\n",
            "|    total timesteps | 143858   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -43.3    |\n",
            "|    critic_loss     | 5.26     |\n",
            "|    ent_coef        | 0.0815   |\n",
            "|    ent_coef_loss   | 0.286    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 142857   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 286      |\n",
            "|    ep_rew_mean     | 204      |\n",
            "| time/              |          |\n",
            "|    episodes        | 320      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 1849     |\n",
            "|    total timesteps | 146134   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -42.7    |\n",
            "|    critic_loss     | 7.75     |\n",
            "|    ent_coef        | 0.0839   |\n",
            "|    ent_coef_loss   | -0.0156  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 145133   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 274      |\n",
            "|    ep_rew_mean     | 197      |\n",
            "| time/              |          |\n",
            "|    episodes        | 330      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 1883     |\n",
            "|    total timesteps | 149008   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -48      |\n",
            "|    critic_loss     | 8.05     |\n",
            "|    ent_coef        | 0.084    |\n",
            "|    ent_coef_loss   | 0.583    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 148007   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 275      |\n",
            "|    ep_rew_mean     | 198      |\n",
            "| time/              |          |\n",
            "|    episodes        | 340      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 1927     |\n",
            "|    total timesteps | 152648   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -49.5    |\n",
            "|    critic_loss     | 13.3     |\n",
            "|    ent_coef        | 0.0787   |\n",
            "|    ent_coef_loss   | -0.409   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 151647   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 271      |\n",
            "|    ep_rew_mean     | 198      |\n",
            "| time/              |          |\n",
            "|    episodes        | 350      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 1956     |\n",
            "|    total timesteps | 155152   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -49.3    |\n",
            "|    critic_loss     | 3.32     |\n",
            "|    ent_coef        | 0.0833   |\n",
            "|    ent_coef_loss   | -0.152   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 154151   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 275      |\n",
            "|    ep_rew_mean     | 202      |\n",
            "| time/              |          |\n",
            "|    episodes        | 360      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 1994     |\n",
            "|    total timesteps | 158312   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -44.5    |\n",
            "|    critic_loss     | 2.52     |\n",
            "|    ent_coef        | 0.0793   |\n",
            "|    ent_coef_loss   | 0.99     |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 157311   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 281      |\n",
            "|    ep_rew_mean     | 205      |\n",
            "| time/              |          |\n",
            "|    episodes        | 370      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 2034     |\n",
            "|    total timesteps | 161647   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -48.4    |\n",
            "|    critic_loss     | 5.2      |\n",
            "|    ent_coef        | 0.0787   |\n",
            "|    ent_coef_loss   | -0.0949  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 160646   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 284      |\n",
            "|    ep_rew_mean     | 213      |\n",
            "| time/              |          |\n",
            "|    episodes        | 380      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 2070     |\n",
            "|    total timesteps | 164577   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -44.1    |\n",
            "|    critic_loss     | 8.48     |\n",
            "|    ent_coef        | 0.0759   |\n",
            "|    ent_coef_loss   | -0.00493 |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 163576   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 286      |\n",
            "|    ep_rew_mean     | 220      |\n",
            "| time/              |          |\n",
            "|    episodes        | 390      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 2103     |\n",
            "|    total timesteps | 167361   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -44.4    |\n",
            "|    critic_loss     | 2.93     |\n",
            "|    ent_coef        | 0.0773   |\n",
            "|    ent_coef_loss   | -0.298   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 166360   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 286      |\n",
            "|    ep_rew_mean     | 232      |\n",
            "| time/              |          |\n",
            "|    episodes        | 400      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 2132     |\n",
            "|    total timesteps | 169830   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -45.3    |\n",
            "|    critic_loss     | 2.81     |\n",
            "|    ent_coef        | 0.0753   |\n",
            "|    ent_coef_loss   | -0.609   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 168829   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 291      |\n",
            "|    ep_rew_mean     | 234      |\n",
            "| time/              |          |\n",
            "|    episodes        | 410      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 2168     |\n",
            "|    total timesteps | 172921   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -48.9    |\n",
            "|    critic_loss     | 6.81     |\n",
            "|    ent_coef        | 0.0677   |\n",
            "|    ent_coef_loss   | -0.0672  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 171920   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 293      |\n",
            "|    ep_rew_mean     | 240      |\n",
            "| time/              |          |\n",
            "|    episodes        | 420      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 2198     |\n",
            "|    total timesteps | 175412   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -50.9    |\n",
            "|    critic_loss     | 2.38     |\n",
            "|    ent_coef        | 0.0676   |\n",
            "|    ent_coef_loss   | -0.0968  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 174411   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 292      |\n",
            "|    ep_rew_mean     | 249      |\n",
            "| time/              |          |\n",
            "|    episodes        | 430      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 2231     |\n",
            "|    total timesteps | 178218   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -48.8    |\n",
            "|    critic_loss     | 6.96     |\n",
            "|    ent_coef        | 0.0681   |\n",
            "|    ent_coef_loss   | -0.0364  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 177217   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 282      |\n",
            "|    ep_rew_mean     | 240      |\n",
            "| time/              |          |\n",
            "|    episodes        | 440      |\n",
            "|    fps             | 79       |\n",
            "|    time_elapsed    | 2261     |\n",
            "|    total timesteps | 180852   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -50.9    |\n",
            "|    critic_loss     | 11.1     |\n",
            "|    ent_coef        | 0.0709   |\n",
            "|    ent_coef_loss   | 0.811    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 179851   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 283      |\n",
            "|    ep_rew_mean     | 243      |\n",
            "| time/              |          |\n",
            "|    episodes        | 450      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2292     |\n",
            "|    total timesteps | 183417   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -53.8    |\n",
            "|    critic_loss     | 3.44     |\n",
            "|    ent_coef        | 0.0721   |\n",
            "|    ent_coef_loss   | -0.763   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 182416   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 282      |\n",
            "|    ep_rew_mean     | 243      |\n",
            "| time/              |          |\n",
            "|    episodes        | 460      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2330     |\n",
            "|    total timesteps | 186549   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -48.2    |\n",
            "|    critic_loss     | 5.28     |\n",
            "|    ent_coef        | 0.0735   |\n",
            "|    ent_coef_loss   | 0.773    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 185548   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 275      |\n",
            "|    ep_rew_mean     | 238      |\n",
            "| time/              |          |\n",
            "|    episodes        | 470      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2360     |\n",
            "|    total timesteps | 189106   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -50      |\n",
            "|    critic_loss     | 11.2     |\n",
            "|    ent_coef        | 0.0667   |\n",
            "|    ent_coef_loss   | 0.614    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 188105   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 274      |\n",
            "|    ep_rew_mean     | 236      |\n",
            "| time/              |          |\n",
            "|    episodes        | 480      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2394     |\n",
            "|    total timesteps | 191983   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -46.2    |\n",
            "|    critic_loss     | 7.38     |\n",
            "|    ent_coef        | 0.0677   |\n",
            "|    ent_coef_loss   | 0.118    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 190982   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 281      |\n",
            "|    ep_rew_mean     | 233      |\n",
            "| time/              |          |\n",
            "|    episodes        | 490      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2436     |\n",
            "|    total timesteps | 195467   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -51.6    |\n",
            "|    critic_loss     | 2.17     |\n",
            "|    ent_coef        | 0.0686   |\n",
            "|    ent_coef_loss   | -0.669   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 194466   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 289      |\n",
            "|    ep_rew_mean     | 233      |\n",
            "| time/              |          |\n",
            "|    episodes        | 500      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2474     |\n",
            "|    total timesteps | 198690   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -50.7    |\n",
            "|    critic_loss     | 21.2     |\n",
            "|    ent_coef        | 0.0631   |\n",
            "|    ent_coef_loss   | -0.116   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 197689   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 283      |\n",
            "|    ep_rew_mean     | 236      |\n",
            "| time/              |          |\n",
            "|    episodes        | 510      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2504     |\n",
            "|    total timesteps | 201232   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -50.9    |\n",
            "|    critic_loss     | 8.95     |\n",
            "|    ent_coef        | 0.069    |\n",
            "|    ent_coef_loss   | -0.134   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 200231   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 278      |\n",
            "|    ep_rew_mean     | 233      |\n",
            "| time/              |          |\n",
            "|    episodes        | 520      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2528     |\n",
            "|    total timesteps | 203253   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -53.2    |\n",
            "|    critic_loss     | 3.47     |\n",
            "|    ent_coef        | 0.0677   |\n",
            "|    ent_coef_loss   | -0.605   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 202252   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 280      |\n",
            "|    ep_rew_mean     | 235      |\n",
            "| time/              |          |\n",
            "|    episodes        | 530      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2563     |\n",
            "|    total timesteps | 206256   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -52      |\n",
            "|    critic_loss     | 3.5      |\n",
            "|    ent_coef        | 0.0728   |\n",
            "|    ent_coef_loss   | 0.29     |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 205255   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 276      |\n",
            "|    ep_rew_mean     | 238      |\n",
            "| time/              |          |\n",
            "|    episodes        | 540      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2588     |\n",
            "|    total timesteps | 208464   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -52.5    |\n",
            "|    critic_loss     | 1.98     |\n",
            "|    ent_coef        | 0.067    |\n",
            "|    ent_coef_loss   | -0.29    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 207463   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 272      |\n",
            "|    ep_rew_mean     | 238      |\n",
            "| time/              |          |\n",
            "|    episodes        | 550      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2614     |\n",
            "|    total timesteps | 210661   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -52.3    |\n",
            "|    critic_loss     | 2.9      |\n",
            "|    ent_coef        | 0.0648   |\n",
            "|    ent_coef_loss   | -0.271   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 209660   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 264      |\n",
            "|    ep_rew_mean     | 241      |\n",
            "| time/              |          |\n",
            "|    episodes        | 560      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2641     |\n",
            "|    total timesteps | 212951   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -55.4    |\n",
            "|    critic_loss     | 3.46     |\n",
            "|    ent_coef        | 0.0703   |\n",
            "|    ent_coef_loss   | -0.115   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 211950   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 263      |\n",
            "|    ep_rew_mean     | 248      |\n",
            "| time/              |          |\n",
            "|    episodes        | 570      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2670     |\n",
            "|    total timesteps | 215383   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -54.7    |\n",
            "|    critic_loss     | 9.67     |\n",
            "|    ent_coef        | 0.07     |\n",
            "|    ent_coef_loss   | -0.219   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 214382   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 256      |\n",
            "|    ep_rew_mean     | 253      |\n",
            "| time/              |          |\n",
            "|    episodes        | 580      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2695     |\n",
            "|    total timesteps | 217597   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -54.3    |\n",
            "|    critic_loss     | 11.4     |\n",
            "|    ent_coef        | 0.0683   |\n",
            "|    ent_coef_loss   | 0.606    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 216596   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 244      |\n",
            "|    ep_rew_mean     | 255      |\n",
            "| time/              |          |\n",
            "|    episodes        | 590      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2721     |\n",
            "|    total timesteps | 219819   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -58.2    |\n",
            "|    critic_loss     | 4.49     |\n",
            "|    ent_coef        | 0.0721   |\n",
            "|    ent_coef_loss   | -0.523   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 218818   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 233      |\n",
            "|    ep_rew_mean     | 257      |\n",
            "| time/              |          |\n",
            "|    episodes        | 600      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2746     |\n",
            "|    total timesteps | 221941   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -54      |\n",
            "|    critic_loss     | 3.2      |\n",
            "|    ent_coef        | 0.069    |\n",
            "|    ent_coef_loss   | -0.154   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 220940   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 236      |\n",
            "|    ep_rew_mean     | 258      |\n",
            "| time/              |          |\n",
            "|    episodes        | 610      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2781     |\n",
            "|    total timesteps | 224872   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -54.5    |\n",
            "|    critic_loss     | 3.76     |\n",
            "|    ent_coef        | 0.0723   |\n",
            "|    ent_coef_loss   | 0.553    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 223871   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 239      |\n",
            "|    ep_rew_mean     | 258      |\n",
            "| time/              |          |\n",
            "|    episodes        | 620      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2807     |\n",
            "|    total timesteps | 227154   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -55.5    |\n",
            "|    critic_loss     | 1.9      |\n",
            "|    ent_coef        | 0.0661   |\n",
            "|    ent_coef_loss   | -0.673   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 226153   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 237      |\n",
            "|    ep_rew_mean     | 259      |\n",
            "| time/              |          |\n",
            "|    episodes        | 630      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2840     |\n",
            "|    total timesteps | 229939   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -58.1    |\n",
            "|    critic_loss     | 4.04     |\n",
            "|    ent_coef        | 0.069    |\n",
            "|    ent_coef_loss   | -0.0348  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 228938   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 236      |\n",
            "|    ep_rew_mean     | 262      |\n",
            "| time/              |          |\n",
            "|    episodes        | 640      |\n",
            "|    fps             | 80       |\n",
            "|    time_elapsed    | 2865     |\n",
            "|    total timesteps | 232041   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -57.1    |\n",
            "|    critic_loss     | 2.97     |\n",
            "|    ent_coef        | 0.0731   |\n",
            "|    ent_coef_loss   | -0.619   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 231040   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 237      |\n",
            "|    ep_rew_mean     | 265      |\n",
            "| time/              |          |\n",
            "|    episodes        | 650      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 2892     |\n",
            "|    total timesteps | 234387   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -60.2    |\n",
            "|    critic_loss     | 2.21     |\n",
            "|    ent_coef        | 0.0748   |\n",
            "|    ent_coef_loss   | -0.269   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 233386   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 236      |\n",
            "|    ep_rew_mean     | 264      |\n",
            "| time/              |          |\n",
            "|    episodes        | 660      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 2918     |\n",
            "|    total timesteps | 236599   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -61.1    |\n",
            "|    critic_loss     | 6.09     |\n",
            "|    ent_coef        | 0.0739   |\n",
            "|    ent_coef_loss   | 0.508    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 235598   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 234      |\n",
            "|    ep_rew_mean     | 264      |\n",
            "| time/              |          |\n",
            "|    episodes        | 670      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 2944     |\n",
            "|    total timesteps | 238782   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -60.2    |\n",
            "|    critic_loss     | 2.71     |\n",
            "|    ent_coef        | 0.0783   |\n",
            "|    ent_coef_loss   | -0.0388  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 237781   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 243      |\n",
            "|    ep_rew_mean     | 259      |\n",
            "| time/              |          |\n",
            "|    episodes        | 680      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 2982     |\n",
            "|    total timesteps | 241940   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -60.6    |\n",
            "|    critic_loss     | 52.8     |\n",
            "|    ent_coef        | 0.0798   |\n",
            "|    ent_coef_loss   | 0.147    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 240939   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 254      |\n",
            "|    ep_rew_mean     | 252      |\n",
            "| time/              |          |\n",
            "|    episodes        | 690      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3021     |\n",
            "|    total timesteps | 245176   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -59.8    |\n",
            "|    critic_loss     | 2.88     |\n",
            "|    ent_coef        | 0.0812   |\n",
            "|    ent_coef_loss   | 0.411    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 244175   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 271      |\n",
            "|    ep_rew_mean     | 248      |\n",
            "| time/              |          |\n",
            "|    episodes        | 700      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3069     |\n",
            "|    total timesteps | 249015   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -63.4    |\n",
            "|    critic_loss     | 12.5     |\n",
            "|    ent_coef        | 0.0864   |\n",
            "|    ent_coef_loss   | -0.236   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 248014   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 277      |\n",
            "|    ep_rew_mean     | 241      |\n",
            "| time/              |          |\n",
            "|    episodes        | 710      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3112     |\n",
            "|    total timesteps | 252568   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -61.3    |\n",
            "|    critic_loss     | 2.37     |\n",
            "|    ent_coef        | 0.0856   |\n",
            "|    ent_coef_loss   | 0.204    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 251567   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 276      |\n",
            "|    ep_rew_mean     | 237      |\n",
            "| time/              |          |\n",
            "|    episodes        | 720      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3138     |\n",
            "|    total timesteps | 254767   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -59.1    |\n",
            "|    critic_loss     | 6.53     |\n",
            "|    ent_coef        | 0.0817   |\n",
            "|    ent_coef_loss   | 1.25     |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 253766   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 267      |\n",
            "|    ep_rew_mean     | 229      |\n",
            "| time/              |          |\n",
            "|    episodes        | 730      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3159     |\n",
            "|    total timesteps | 256622   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -63.3    |\n",
            "|    critic_loss     | 5.97     |\n",
            "|    ent_coef        | 0.0808   |\n",
            "|    ent_coef_loss   | -0.833   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 255621   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 269      |\n",
            "|    ep_rew_mean     | 232      |\n",
            "| time/              |          |\n",
            "|    episodes        | 740      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3187     |\n",
            "|    total timesteps | 258977   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -60.3    |\n",
            "|    critic_loss     | 3.04     |\n",
            "|    ent_coef        | 0.0804   |\n",
            "|    ent_coef_loss   | 0.167    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 257976   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 266      |\n",
            "|    ep_rew_mean     | 232      |\n",
            "| time/              |          |\n",
            "|    episodes        | 750      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3211     |\n",
            "|    total timesteps | 261029   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -59.5    |\n",
            "|    critic_loss     | 2.63     |\n",
            "|    ent_coef        | 0.0752   |\n",
            "|    ent_coef_loss   | 0.128    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 260028   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 273      |\n",
            "|    ep_rew_mean     | 229      |\n",
            "| time/              |          |\n",
            "|    episodes        | 760      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3245     |\n",
            "|    total timesteps | 263869   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -55.7    |\n",
            "|    critic_loss     | 7.63     |\n",
            "|    ent_coef        | 0.0721   |\n",
            "|    ent_coef_loss   | 0.15     |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 262868   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 277      |\n",
            "|    ep_rew_mean     | 224      |\n",
            "| time/              |          |\n",
            "|    episodes        | 770      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3275     |\n",
            "|    total timesteps | 266456   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -52.8    |\n",
            "|    critic_loss     | 5.41     |\n",
            "|    ent_coef        | 0.0723   |\n",
            "|    ent_coef_loss   | 1.33     |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 265455   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 267      |\n",
            "|    ep_rew_mean     | 228      |\n",
            "| time/              |          |\n",
            "|    episodes        | 780      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3301     |\n",
            "|    total timesteps | 268681   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -57.7    |\n",
            "|    critic_loss     | 5.87     |\n",
            "|    ent_coef        | 0.0732   |\n",
            "|    ent_coef_loss   | -0.146   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 267680   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 257      |\n",
            "|    ep_rew_mean     | 232      |\n",
            "| time/              |          |\n",
            "|    episodes        | 790      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3327     |\n",
            "|    total timesteps | 270911   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -58      |\n",
            "|    critic_loss     | 12.3     |\n",
            "|    ent_coef        | 0.0742   |\n",
            "|    ent_coef_loss   | 0.237    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 269910   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 244      |\n",
            "|    ep_rew_mean     | 228      |\n",
            "| time/              |          |\n",
            "|    episodes        | 800      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3357     |\n",
            "|    total timesteps | 273441   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -57.4    |\n",
            "|    critic_loss     | 27       |\n",
            "|    ent_coef        | 0.0715   |\n",
            "|    ent_coef_loss   | -0.494   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 272440   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 231      |\n",
            "|    ep_rew_mean     | 235      |\n",
            "| time/              |          |\n",
            "|    episodes        | 810      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3383     |\n",
            "|    total timesteps | 275714   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -59.5    |\n",
            "|    critic_loss     | 3.76     |\n",
            "|    ent_coef        | 0.0726   |\n",
            "|    ent_coef_loss   | 0.212    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 274713   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 239      |\n",
            "|    ep_rew_mean     | 235      |\n",
            "| time/              |          |\n",
            "|    episodes        | 820      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3419     |\n",
            "|    total timesteps | 278697   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -56.2    |\n",
            "|    critic_loss     | 7.98     |\n",
            "|    ent_coef        | 0.0691   |\n",
            "|    ent_coef_loss   | -0.598   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 277696   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 250      |\n",
            "|    ep_rew_mean     | 237      |\n",
            "| time/              |          |\n",
            "|    episodes        | 830      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3454     |\n",
            "|    total timesteps | 281647   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -57.6    |\n",
            "|    critic_loss     | 3.17     |\n",
            "|    ent_coef        | 0.0679   |\n",
            "|    ent_coef_loss   | 0.8      |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 280646   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 250      |\n",
            "|    ep_rew_mean     | 233      |\n",
            "| time/              |          |\n",
            "|    episodes        | 840      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3482     |\n",
            "|    total timesteps | 284005   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -59      |\n",
            "|    critic_loss     | 6        |\n",
            "|    ent_coef        | 0.0639   |\n",
            "|    ent_coef_loss   | -0.242   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 283004   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 261      |\n",
            "|    ep_rew_mean     | 233      |\n",
            "| time/              |          |\n",
            "|    episodes        | 850      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3519     |\n",
            "|    total timesteps | 287145   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -59.3    |\n",
            "|    critic_loss     | 6.3      |\n",
            "|    ent_coef        | 0.0632   |\n",
            "|    ent_coef_loss   | 0.518    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 286144   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 257      |\n",
            "|    ep_rew_mean     | 237      |\n",
            "| time/              |          |\n",
            "|    episodes        | 860      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3548     |\n",
            "|    total timesteps | 289584   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -57.6    |\n",
            "|    critic_loss     | 5.4      |\n",
            "|    ent_coef        | 0.0619   |\n",
            "|    ent_coef_loss   | 0.0189   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 288583   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 267      |\n",
            "|    ep_rew_mean     | 241      |\n",
            "| time/              |          |\n",
            "|    episodes        | 870      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3590     |\n",
            "|    total timesteps | 293130   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -63.9    |\n",
            "|    critic_loss     | 8.41     |\n",
            "|    ent_coef        | 0.0628   |\n",
            "|    ent_coef_loss   | -0.893   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 292129   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 269      |\n",
            "|    ep_rew_mean     | 240      |\n",
            "| time/              |          |\n",
            "|    episodes        | 880      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3619     |\n",
            "|    total timesteps | 295612   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -59.2    |\n",
            "|    critic_loss     | 2.39     |\n",
            "|    ent_coef        | 0.0636   |\n",
            "|    ent_coef_loss   | -0.501   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 294611   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 277      |\n",
            "|    ep_rew_mean     | 243      |\n",
            "| time/              |          |\n",
            "|    episodes        | 890      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3654     |\n",
            "|    total timesteps | 298582   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -57.7    |\n",
            "|    critic_loss     | 9.9      |\n",
            "|    ent_coef        | 0.0695   |\n",
            "|    ent_coef_loss   | 0.24     |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 297581   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 275      |\n",
            "|    ep_rew_mean     | 251      |\n",
            "| time/              |          |\n",
            "|    episodes        | 900      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3682     |\n",
            "|    total timesteps | 300984   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -59      |\n",
            "|    critic_loss     | 2        |\n",
            "|    ent_coef        | 0.0673   |\n",
            "|    ent_coef_loss   | 0.151    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 299983   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 286      |\n",
            "|    ep_rew_mean     | 251      |\n",
            "| time/              |          |\n",
            "|    episodes        | 910      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3722     |\n",
            "|    total timesteps | 304321   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -58.9    |\n",
            "|    critic_loss     | 6.39     |\n",
            "|    ent_coef        | 0.0643   |\n",
            "|    ent_coef_loss   | 0.159    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 303320   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 281      |\n",
            "|    ep_rew_mean     | 256      |\n",
            "| time/              |          |\n",
            "|    episodes        | 920      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3752     |\n",
            "|    total timesteps | 306834   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -59.4    |\n",
            "|    critic_loss     | 2.6      |\n",
            "|    ent_coef        | 0.0662   |\n",
            "|    ent_coef_loss   | 0.244    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 305833   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 276      |\n",
            "|    ep_rew_mean     | 259      |\n",
            "| time/              |          |\n",
            "|    episodes        | 930      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3780     |\n",
            "|    total timesteps | 309264   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -59.7    |\n",
            "|    critic_loss     | 14.3     |\n",
            "|    ent_coef        | 0.06     |\n",
            "|    ent_coef_loss   | 0.274    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 308263   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 273      |\n",
            "|    ep_rew_mean     | 264      |\n",
            "| time/              |          |\n",
            "|    episodes        | 940      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3804     |\n",
            "|    total timesteps | 311293   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -58.8    |\n",
            "|    critic_loss     | 3.06     |\n",
            "|    ent_coef        | 0.0605   |\n",
            "|    ent_coef_loss   | -0.234   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 310292   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 263      |\n",
            "|    ep_rew_mean     | 264      |\n",
            "| time/              |          |\n",
            "|    episodes        | 950      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3830     |\n",
            "|    total timesteps | 313480   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -57.3    |\n",
            "|    critic_loss     | 2.14     |\n",
            "|    ent_coef        | 0.0612   |\n",
            "|    ent_coef_loss   | -0.126   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 312479   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 259      |\n",
            "|    ep_rew_mean     | 265      |\n",
            "| time/              |          |\n",
            "|    episodes        | 960      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3854     |\n",
            "|    total timesteps | 315518   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -60.2    |\n",
            "|    critic_loss     | 1.84     |\n",
            "|    ent_coef        | 0.0638   |\n",
            "|    ent_coef_loss   | 0.132    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 314517   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 243      |\n",
            "|    ep_rew_mean     | 268      |\n",
            "| time/              |          |\n",
            "|    episodes        | 970      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3876     |\n",
            "|    total timesteps | 317469   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -64.6    |\n",
            "|    critic_loss     | 3.77     |\n",
            "|    ent_coef        | 0.0621   |\n",
            "|    ent_coef_loss   | 0.106    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 316468   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 238      |\n",
            "|    ep_rew_mean     | 271      |\n",
            "| time/              |          |\n",
            "|    episodes        | 980      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3899     |\n",
            "|    total timesteps | 319408   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -63      |\n",
            "|    critic_loss     | 11.2     |\n",
            "|    ent_coef        | 0.0601   |\n",
            "|    ent_coef_loss   | 0.372    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 318407   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 228      |\n",
            "|    ep_rew_mean     | 272      |\n",
            "| time/              |          |\n",
            "|    episodes        | 990      |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3923     |\n",
            "|    total timesteps | 321409   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -60.9    |\n",
            "|    critic_loss     | 5.62     |\n",
            "|    ent_coef        | 0.0602   |\n",
            "|    ent_coef_loss   | -0.326   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 320408   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 224      |\n",
            "|    ep_rew_mean     | 272      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1000     |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3946     |\n",
            "|    total timesteps | 323404   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -62.6    |\n",
            "|    critic_loss     | 3.59     |\n",
            "|    ent_coef        | 0.063    |\n",
            "|    ent_coef_loss   | 0.196    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 322403   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 211      |\n",
            "|    ep_rew_mean     | 274      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1010     |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3970     |\n",
            "|    total timesteps | 325392   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -64.4    |\n",
            "|    critic_loss     | 11       |\n",
            "|    ent_coef        | 0.0598   |\n",
            "|    ent_coef_loss   | -0.287   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 324391   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 205      |\n",
            "|    ep_rew_mean     | 274      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1020     |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 3992     |\n",
            "|    total timesteps | 327317   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -60.5    |\n",
            "|    critic_loss     | 3.9      |\n",
            "|    ent_coef        | 0.0601   |\n",
            "|    ent_coef_loss   | -0.0322  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 326316   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 199      |\n",
            "|    ep_rew_mean     | 277      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1030     |\n",
            "|    fps             | 81       |\n",
            "|    time_elapsed    | 4014     |\n",
            "|    total timesteps | 329196   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -64.1    |\n",
            "|    critic_loss     | 3.58     |\n",
            "|    ent_coef        | 0.0619   |\n",
            "|    ent_coef_loss   | -0.0708  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 328195   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 198      |\n",
            "|    ep_rew_mean     | 278      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1040     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4036     |\n",
            "|    total timesteps | 331075   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -62      |\n",
            "|    critic_loss     | 9.06     |\n",
            "|    ent_coef        | 0.0587   |\n",
            "|    ent_coef_loss   | 0.209    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 330074   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 194      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1050     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4058     |\n",
            "|    total timesteps | 332922   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -64.5    |\n",
            "|    critic_loss     | 17.2     |\n",
            "|    ent_coef        | 0.061    |\n",
            "|    ent_coef_loss   | -0.28    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 331921   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 193      |\n",
            "|    ep_rew_mean     | 280      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1060     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4081     |\n",
            "|    total timesteps | 334863   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -64.3    |\n",
            "|    critic_loss     | 6.86     |\n",
            "|    ent_coef        | 0.0622   |\n",
            "|    ent_coef_loss   | 0.178    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 333862   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 193      |\n",
            "|    ep_rew_mean     | 278      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1070     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4102     |\n",
            "|    total timesteps | 336722   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -65.8    |\n",
            "|    critic_loss     | 1.68     |\n",
            "|    ent_coef        | 0.0609   |\n",
            "|    ent_coef_loss   | 0.218    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 335721   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 194      |\n",
            "|    ep_rew_mean     | 278      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1080     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4127     |\n",
            "|    total timesteps | 338851   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -65.9    |\n",
            "|    critic_loss     | 58.5     |\n",
            "|    ent_coef        | 0.0597   |\n",
            "|    ent_coef_loss   | 0.42     |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 337850   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 193      |\n",
            "|    ep_rew_mean     | 280      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1090     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4150     |\n",
            "|    total timesteps | 340746   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -68.5    |\n",
            "|    critic_loss     | 6.62     |\n",
            "|    ent_coef        | 0.0596   |\n",
            "|    ent_coef_loss   | 0.193    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 339745   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 192      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1100     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4172     |\n",
            "|    total timesteps | 342650   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -64.6    |\n",
            "|    critic_loss     | 7.44     |\n",
            "|    ent_coef        | 0.0608   |\n",
            "|    ent_coef_loss   | 0.0881   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 341649   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 191      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1110     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4193     |\n",
            "|    total timesteps | 344533   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -68.2    |\n",
            "|    critic_loss     | 5.13     |\n",
            "|    ent_coef        | 0.061    |\n",
            "|    ent_coef_loss   | -0.0382  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 343532   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 192      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1120     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4216     |\n",
            "|    total timesteps | 346504   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -66.4    |\n",
            "|    critic_loss     | 26       |\n",
            "|    ent_coef        | 0.0606   |\n",
            "|    ent_coef_loss   | 0.662    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 345503   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 193      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1130     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4239     |\n",
            "|    total timesteps | 348455   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -64.9    |\n",
            "|    critic_loss     | 7.44     |\n",
            "|    ent_coef        | 0.0579   |\n",
            "|    ent_coef_loss   | 0.656    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 347454   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 194      |\n",
            "|    ep_rew_mean     | 277      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1140     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4263     |\n",
            "|    total timesteps | 350475   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -68.2    |\n",
            "|    critic_loss     | 4.29     |\n",
            "|    ent_coef        | 0.0583   |\n",
            "|    ent_coef_loss   | 0.776    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 349474   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 197      |\n",
            "|    ep_rew_mean     | 276      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1150     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4288     |\n",
            "|    total timesteps | 352624   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -66.4    |\n",
            "|    critic_loss     | 3.03     |\n",
            "|    ent_coef        | 0.0603   |\n",
            "|    ent_coef_loss   | 0.118    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 351623   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 198      |\n",
            "|    ep_rew_mean     | 275      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1160     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4311     |\n",
            "|    total timesteps | 354670   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -69.2    |\n",
            "|    critic_loss     | 2.45     |\n",
            "|    ent_coef        | 0.0603   |\n",
            "|    ent_coef_loss   | 0.877    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 353669   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 200      |\n",
            "|    ep_rew_mean     | 276      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1170     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4336     |\n",
            "|    total timesteps | 356746   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -70.2    |\n",
            "|    critic_loss     | 7.16     |\n",
            "|    ent_coef        | 0.0626   |\n",
            "|    ent_coef_loss   | -0.0595  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 355745   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 199      |\n",
            "|    ep_rew_mean     | 276      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1180     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4359     |\n",
            "|    total timesteps | 358714   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -69.7    |\n",
            "|    critic_loss     | 3.5      |\n",
            "|    ent_coef        | 0.0602   |\n",
            "|    ent_coef_loss   | 0.993    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 357713   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 199      |\n",
            "|    ep_rew_mean     | 273      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1190     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4381     |\n",
            "|    total timesteps | 360678   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -69.2    |\n",
            "|    critic_loss     | 14.9     |\n",
            "|    ent_coef        | 0.0598   |\n",
            "|    ent_coef_loss   | 0.105    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 359677   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 200      |\n",
            "|    ep_rew_mean     | 275      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1200     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4404     |\n",
            "|    total timesteps | 362621   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -69.9    |\n",
            "|    critic_loss     | 17.8     |\n",
            "|    ent_coef        | 0.0595   |\n",
            "|    ent_coef_loss   | -0.0955  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 361620   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 201      |\n",
            "|    ep_rew_mean     | 275      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1210     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4427     |\n",
            "|    total timesteps | 364606   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -68.2    |\n",
            "|    critic_loss     | 8.78     |\n",
            "|    ent_coef        | 0.0589   |\n",
            "|    ent_coef_loss   | 0.366    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 363605   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 201      |\n",
            "|    ep_rew_mean     | 275      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1220     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4451     |\n",
            "|    total timesteps | 366643   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -68      |\n",
            "|    critic_loss     | 6.48     |\n",
            "|    ent_coef        | 0.057    |\n",
            "|    ent_coef_loss   | 0.433    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 365642   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 205      |\n",
            "|    ep_rew_mean     | 276      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1230     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4478     |\n",
            "|    total timesteps | 368966   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -72.3    |\n",
            "|    critic_loss     | 3.5      |\n",
            "|    ent_coef        | 0.0597   |\n",
            "|    ent_coef_loss   | -0.309   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 367965   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 205      |\n",
            "|    ep_rew_mean     | 278      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1240     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4500     |\n",
            "|    total timesteps | 370933   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -71.1    |\n",
            "|    critic_loss     | 9.63     |\n",
            "|    ent_coef        | 0.0578   |\n",
            "|    ent_coef_loss   | 0.47     |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 369932   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 202      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1250     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4523     |\n",
            "|    total timesteps | 372844   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -72.9    |\n",
            "|    critic_loss     | 46.8     |\n",
            "|    ent_coef        | 0.057    |\n",
            "|    ent_coef_loss   | -0.196   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 371843   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 200      |\n",
            "|    ep_rew_mean     | 280      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1260     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4544     |\n",
            "|    total timesteps | 374700   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -72.5    |\n",
            "|    critic_loss     | 6.77     |\n",
            "|    ent_coef        | 0.057    |\n",
            "|    ent_coef_loss   | -0.156   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 373699   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 200      |\n",
            "|    ep_rew_mean     | 280      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1270     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4567     |\n",
            "|    total timesteps | 376715   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -73.5    |\n",
            "|    critic_loss     | 2.13     |\n",
            "|    ent_coef        | 0.0556   |\n",
            "|    ent_coef_loss   | 0.0769   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 375714   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 200      |\n",
            "|    ep_rew_mean     | 281      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1280     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4591     |\n",
            "|    total timesteps | 378712   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -70.2    |\n",
            "|    critic_loss     | 4.14     |\n",
            "|    ent_coef        | 0.0562   |\n",
            "|    ent_coef_loss   | -0.0204  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 377711   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 200      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1290     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4614     |\n",
            "|    total timesteps | 380704   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -71.7    |\n",
            "|    critic_loss     | 9.88     |\n",
            "|    ent_coef        | 0.0549   |\n",
            "|    ent_coef_loss   | -0.041   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 379703   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 201      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1300     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4637     |\n",
            "|    total timesteps | 382695   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -72.9    |\n",
            "|    critic_loss     | 5.12     |\n",
            "|    ent_coef        | 0.0581   |\n",
            "|    ent_coef_loss   | -1.21    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 381694   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 200      |\n",
            "|    ep_rew_mean     | 283      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1310     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4659     |\n",
            "|    total timesteps | 384580   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -71.7    |\n",
            "|    critic_loss     | 1.82     |\n",
            "|    ent_coef        | 0.0568   |\n",
            "|    ent_coef_loss   | 0.206    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 383579   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 201      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1320     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4684     |\n",
            "|    total timesteps | 386727   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -71.2    |\n",
            "|    critic_loss     | 6.47     |\n",
            "|    ent_coef        | 0.055    |\n",
            "|    ent_coef_loss   | 0.0704   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 385726   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 198      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1330     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4708     |\n",
            "|    total timesteps | 388752   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -70.7    |\n",
            "|    critic_loss     | 27.8     |\n",
            "|    ent_coef        | 0.0543   |\n",
            "|    ent_coef_loss   | 0.318    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 387751   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 198      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1340     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4731     |\n",
            "|    total timesteps | 390727   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -74.4    |\n",
            "|    critic_loss     | 4.44     |\n",
            "|    ent_coef        | 0.0571   |\n",
            "|    ent_coef_loss   | 0.454    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 389726   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 197      |\n",
            "|    ep_rew_mean     | 281      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1350     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4752     |\n",
            "|    total timesteps | 392589   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -75.1    |\n",
            "|    critic_loss     | 14.5     |\n",
            "|    ent_coef        | 0.0556   |\n",
            "|    ent_coef_loss   | 0.464    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 391588   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 200      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1360     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4777     |\n",
            "|    total timesteps | 394699   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -72.1    |\n",
            "|    critic_loss     | 11.8     |\n",
            "|    ent_coef        | 0.0534   |\n",
            "|    ent_coef_loss   | -0.107   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 393698   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 199      |\n",
            "|    ep_rew_mean     | 280      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1370     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4799     |\n",
            "|    total timesteps | 396596   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -72.1    |\n",
            "|    critic_loss     | 13.2     |\n",
            "|    ent_coef        | 0.0553   |\n",
            "|    ent_coef_loss   | 0.101    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 395595   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 200      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1380     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4823     |\n",
            "|    total timesteps | 398676   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -72.4    |\n",
            "|    critic_loss     | 2        |\n",
            "|    ent_coef        | 0.0572   |\n",
            "|    ent_coef_loss   | -0.112   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 397675   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 200      |\n",
            "|    ep_rew_mean     | 280      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1390     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4846     |\n",
            "|    total timesteps | 400664   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -71.5    |\n",
            "|    critic_loss     | 1.6      |\n",
            "|    ent_coef        | 0.0556   |\n",
            "|    ent_coef_loss   | 0.278    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 399663   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 200      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1400     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4869     |\n",
            "|    total timesteps | 402664   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -73.5    |\n",
            "|    critic_loss     | 4.57     |\n",
            "|    ent_coef        | 0.0553   |\n",
            "|    ent_coef_loss   | 0.0995   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 401663   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 200      |\n",
            "|    ep_rew_mean     | 280      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1410     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4892     |\n",
            "|    total timesteps | 404570   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -74.6    |\n",
            "|    critic_loss     | 2        |\n",
            "|    ent_coef        | 0.0543   |\n",
            "|    ent_coef_loss   | 0.0487   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 403569   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 198      |\n",
            "|    ep_rew_mean     | 280      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1420     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4914     |\n",
            "|    total timesteps | 406528   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -73.1    |\n",
            "|    critic_loss     | 16.8     |\n",
            "|    ent_coef        | 0.0522   |\n",
            "|    ent_coef_loss   | 0.119    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 405527   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 199      |\n",
            "|    ep_rew_mean     | 281      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1430     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4939     |\n",
            "|    total timesteps | 408629   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -73      |\n",
            "|    critic_loss     | 5.72     |\n",
            "|    ent_coef        | 0.0531   |\n",
            "|    ent_coef_loss   | 0.478    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 407628   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 199      |\n",
            "|    ep_rew_mean     | 281      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1440     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4962     |\n",
            "|    total timesteps | 410602   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -76.1    |\n",
            "|    critic_loss     | 1.55     |\n",
            "|    ent_coef        | 0.0503   |\n",
            "|    ent_coef_loss   | -0.738   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 409601   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 199      |\n",
            "|    ep_rew_mean     | 281      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1450     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 4984     |\n",
            "|    total timesteps | 412533   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -73.9    |\n",
            "|    critic_loss     | 1.24     |\n",
            "|    ent_coef        | 0.0502   |\n",
            "|    ent_coef_loss   | 0.0991   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 411532   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 198      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1460     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5007     |\n",
            "|    total timesteps | 414480   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -74.6    |\n",
            "|    critic_loss     | 3.82     |\n",
            "|    ent_coef        | 0.053    |\n",
            "|    ent_coef_loss   | -0.0758  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 413479   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 199      |\n",
            "|    ep_rew_mean     | 283      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1470     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5030     |\n",
            "|    total timesteps | 416452   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -78.3    |\n",
            "|    critic_loss     | 4.43     |\n",
            "|    ent_coef        | 0.0527   |\n",
            "|    ent_coef_loss   | -0.821   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 415451   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 198      |\n",
            "|    ep_rew_mean     | 283      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1480     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5053     |\n",
            "|    total timesteps | 418480   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -74.8    |\n",
            "|    critic_loss     | 4.35     |\n",
            "|    ent_coef        | 0.0509   |\n",
            "|    ent_coef_loss   | -0.0989  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 417479   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 198      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1490     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5076     |\n",
            "|    total timesteps | 420462   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -72.2    |\n",
            "|    critic_loss     | 2.51     |\n",
            "|    ent_coef        | 0.0572   |\n",
            "|    ent_coef_loss   | 0.262    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 419461   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 197      |\n",
            "|    ep_rew_mean     | 283      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1500     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5099     |\n",
            "|    total timesteps | 422406   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -75.8    |\n",
            "|    critic_loss     | 1.12     |\n",
            "|    ent_coef        | 0.0523   |\n",
            "|    ent_coef_loss   | -0.0703  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 421405   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 198      |\n",
            "|    ep_rew_mean     | 283      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1510     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5121     |\n",
            "|    total timesteps | 424325   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -72.3    |\n",
            "|    critic_loss     | 6.2      |\n",
            "|    ent_coef        | 0.0516   |\n",
            "|    ent_coef_loss   | 0.46     |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 423324   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 198      |\n",
            "|    ep_rew_mean     | 283      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1520     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5144     |\n",
            "|    total timesteps | 426333   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -71.8    |\n",
            "|    critic_loss     | 4.8      |\n",
            "|    ent_coef        | 0.0518   |\n",
            "|    ent_coef_loss   | 0.769    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 425332   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 197      |\n",
            "|    ep_rew_mean     | 283      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1530     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5168     |\n",
            "|    total timesteps | 428354   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -75.6    |\n",
            "|    critic_loss     | 12.9     |\n",
            "|    ent_coef        | 0.0506   |\n",
            "|    ent_coef_loss   | 0.209    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 427353   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 197      |\n",
            "|    ep_rew_mean     | 283      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1540     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5191     |\n",
            "|    total timesteps | 430342   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -78.7    |\n",
            "|    critic_loss     | 6.38     |\n",
            "|    ent_coef        | 0.0534   |\n",
            "|    ent_coef_loss   | -0.401   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 429341   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 198      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1550     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5214     |\n",
            "|    total timesteps | 432326   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -74.8    |\n",
            "|    critic_loss     | 12.3     |\n",
            "|    ent_coef        | 0.0553   |\n",
            "|    ent_coef_loss   | -0.269   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 431325   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 198      |\n",
            "|    ep_rew_mean     | 280      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1560     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5237     |\n",
            "|    total timesteps | 434301   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -76.4    |\n",
            "|    critic_loss     | 2.34     |\n",
            "|    ent_coef        | 0.0546   |\n",
            "|    ent_coef_loss   | -0.435   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 433300   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 197      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1570     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5259     |\n",
            "|    total timesteps | 436190   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -75      |\n",
            "|    critic_loss     | 4.15     |\n",
            "|    ent_coef        | 0.0537   |\n",
            "|    ent_coef_loss   | 0.489    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 435189   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 195      |\n",
            "|    ep_rew_mean     | 280      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1580     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5279     |\n",
            "|    total timesteps | 437988   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -77.7    |\n",
            "|    critic_loss     | 3.88     |\n",
            "|    ent_coef        | 0.0521   |\n",
            "|    ent_coef_loss   | -0.135   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 436987   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 195      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1590     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5302     |\n",
            "|    total timesteps | 439949   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -78.8    |\n",
            "|    critic_loss     | 2.14     |\n",
            "|    ent_coef        | 0.0493   |\n",
            "|    ent_coef_loss   | -0.463   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 438948   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 194      |\n",
            "|    ep_rew_mean     | 281      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1600     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5324     |\n",
            "|    total timesteps | 441822   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -75.9    |\n",
            "|    critic_loss     | 2.33     |\n",
            "|    ent_coef        | 0.0488   |\n",
            "|    ent_coef_loss   | -0.151   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 440821   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 199      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1610     |\n",
            "|    fps             | 82       |\n",
            "|    time_elapsed    | 5352     |\n",
            "|    total timesteps | 444229   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -78.7    |\n",
            "|    critic_loss     | 1.83     |\n",
            "|    ent_coef        | 0.0495   |\n",
            "|    ent_coef_loss   | 0.751    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 443228   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 198      |\n",
            "|    ep_rew_mean     | 281      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1620     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5375     |\n",
            "|    total timesteps | 446181   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -76.6    |\n",
            "|    critic_loss     | 31.2     |\n",
            "|    ent_coef        | 0.0501   |\n",
            "|    ent_coef_loss   | 0.287    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 445180   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 197      |\n",
            "|    ep_rew_mean     | 283      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1630     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5397     |\n",
            "|    total timesteps | 448089   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -76.8    |\n",
            "|    critic_loss     | 1.25     |\n",
            "|    ent_coef        | 0.0504   |\n",
            "|    ent_coef_loss   | -0.791   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 447088   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 196      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1640     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5418     |\n",
            "|    total timesteps | 449909   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -79.8    |\n",
            "|    critic_loss     | 10.6     |\n",
            "|    ent_coef        | 0.0529   |\n",
            "|    ent_coef_loss   | -0.205   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 448908   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 194      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1650     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5440     |\n",
            "|    total timesteps | 451765   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -80.9    |\n",
            "|    critic_loss     | 14.8     |\n",
            "|    ent_coef        | 0.0491   |\n",
            "|    ent_coef_loss   | -0.0655  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 450764   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 194      |\n",
            "|    ep_rew_mean     | 283      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1660     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5463     |\n",
            "|    total timesteps | 453741   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -80.5    |\n",
            "|    critic_loss     | 3.66     |\n",
            "|    ent_coef        | 0.0503   |\n",
            "|    ent_coef_loss   | -0.217   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 452740   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 194      |\n",
            "|    ep_rew_mean     | 283      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1670     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5484     |\n",
            "|    total timesteps | 455617   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -80.7    |\n",
            "|    critic_loss     | 4.28     |\n",
            "|    ent_coef        | 0.048    |\n",
            "|    ent_coef_loss   | -0.756   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 454616   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 195      |\n",
            "|    ep_rew_mean     | 284      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1680     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5505     |\n",
            "|    total timesteps | 457448   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -74.5    |\n",
            "|    critic_loss     | 10.7     |\n",
            "|    ent_coef        | 0.0494   |\n",
            "|    ent_coef_loss   | 0.41     |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 456447   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 195      |\n",
            "|    ep_rew_mean     | 283      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1690     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5528     |\n",
            "|    total timesteps | 459424   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -80.2    |\n",
            "|    critic_loss     | 1.91     |\n",
            "|    ent_coef        | 0.0506   |\n",
            "|    ent_coef_loss   | 0.0362   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 458423   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 193      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1700     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5548     |\n",
            "|    total timesteps | 461151   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -81.7    |\n",
            "|    critic_loss     | 6.31     |\n",
            "|    ent_coef        | 0.0479   |\n",
            "|    ent_coef_loss   | 0.403    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 460150   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 188      |\n",
            "|    ep_rew_mean     | 281      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1710     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5570     |\n",
            "|    total timesteps | 463013   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -78.5    |\n",
            "|    critic_loss     | 6.95     |\n",
            "|    ent_coef        | 0.0486   |\n",
            "|    ent_coef_loss   | 0.259    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 462012   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 186      |\n",
            "|    ep_rew_mean     | 281      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1720     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5591     |\n",
            "|    total timesteps | 464815   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -81      |\n",
            "|    critic_loss     | 4.23     |\n",
            "|    ent_coef        | 0.0519   |\n",
            "|    ent_coef_loss   | -0.835   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 463814   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 186      |\n",
            "|    ep_rew_mean     | 280      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1730     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5612     |\n",
            "|    total timesteps | 466706   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -78.6    |\n",
            "|    critic_loss     | 4.25     |\n",
            "|    ent_coef        | 0.0487   |\n",
            "|    ent_coef_loss   | -0.575   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 465705   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 186      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1740     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5634     |\n",
            "|    total timesteps | 468524   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -77      |\n",
            "|    critic_loss     | 1.66     |\n",
            "|    ent_coef        | 0.0464   |\n",
            "|    ent_coef_loss   | 0.15     |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 467523   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 186      |\n",
            "|    ep_rew_mean     | 282      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1750     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5655     |\n",
            "|    total timesteps | 470367   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -76.5    |\n",
            "|    critic_loss     | 9.93     |\n",
            "|    ent_coef        | 0.0482   |\n",
            "|    ent_coef_loss   | 0.956    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 469366   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 185      |\n",
            "|    ep_rew_mean     | 281      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1760     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5677     |\n",
            "|    total timesteps | 472261   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -82.4    |\n",
            "|    critic_loss     | 2.17     |\n",
            "|    ent_coef        | 0.0477   |\n",
            "|    ent_coef_loss   | -0.759   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 471260   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 185      |\n",
            "|    ep_rew_mean     | 280      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1770     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5698     |\n",
            "|    total timesteps | 474137   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -79.9    |\n",
            "|    critic_loss     | 3.91     |\n",
            "|    ent_coef        | 0.0502   |\n",
            "|    ent_coef_loss   | 0.375    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 473136   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 185      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1780     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5719     |\n",
            "|    total timesteps | 475948   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -83.8    |\n",
            "|    critic_loss     | 4.41     |\n",
            "|    ent_coef        | 0.0479   |\n",
            "|    ent_coef_loss   | -0.595   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 474947   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 184      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1790     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5741     |\n",
            "|    total timesteps | 477839   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -80      |\n",
            "|    critic_loss     | 1.21     |\n",
            "|    ent_coef        | 0.0462   |\n",
            "|    ent_coef_loss   | -0.448   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 476838   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 186      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1800     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5763     |\n",
            "|    total timesteps | 479724   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -78.7    |\n",
            "|    critic_loss     | 1.87     |\n",
            "|    ent_coef        | 0.0477   |\n",
            "|    ent_coef_loss   | 0.255    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 478723   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 187      |\n",
            "|    ep_rew_mean     | 280      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1810     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5786     |\n",
            "|    total timesteps | 481711   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -80.3    |\n",
            "|    critic_loss     | 5.36     |\n",
            "|    ent_coef        | 0.0476   |\n",
            "|    ent_coef_loss   | -0.00499 |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 480710   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 190      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1820     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5811     |\n",
            "|    total timesteps | 483824   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -80.4    |\n",
            "|    critic_loss     | 3.64     |\n",
            "|    ent_coef        | 0.0499   |\n",
            "|    ent_coef_loss   | 0.0973   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 482823   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 190      |\n",
            "|    ep_rew_mean     | 280      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1830     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5833     |\n",
            "|    total timesteps | 485709   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -80.5    |\n",
            "|    critic_loss     | 46.2     |\n",
            "|    ent_coef        | 0.0473   |\n",
            "|    ent_coef_loss   | -0.432   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 484708   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 194      |\n",
            "|    ep_rew_mean     | 278      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1840     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5859     |\n",
            "|    total timesteps | 487969   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -79.5    |\n",
            "|    critic_loss     | 14.3     |\n",
            "|    ent_coef        | 0.0485   |\n",
            "|    ent_coef_loss   | -0.118   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 486968   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 195      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1850     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5881     |\n",
            "|    total timesteps | 489872   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -80.9    |\n",
            "|    critic_loss     | 1.51     |\n",
            "|    ent_coef        | 0.0439   |\n",
            "|    ent_coef_loss   | -0.101   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 488871   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 194      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1860     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5902     |\n",
            "|    total timesteps | 491677   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -81.3    |\n",
            "|    critic_loss     | 3.34     |\n",
            "|    ent_coef        | 0.0494   |\n",
            "|    ent_coef_loss   | -0.0784  |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 490676   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 194      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1870     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5923     |\n",
            "|    total timesteps | 493490   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -82.4    |\n",
            "|    critic_loss     | 5.46     |\n",
            "|    ent_coef        | 0.0495   |\n",
            "|    ent_coef_loss   | -0.259   |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 492489   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 194      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1880     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5945     |\n",
            "|    total timesteps | 495369   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -82.9    |\n",
            "|    critic_loss     | 2.97     |\n",
            "|    ent_coef        | 0.0472   |\n",
            "|    ent_coef_loss   | 0.409    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 494368   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 199      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1890     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5972     |\n",
            "|    total timesteps | 497698   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -79.5    |\n",
            "|    critic_loss     | 4.74     |\n",
            "|    ent_coef        | 0.0482   |\n",
            "|    ent_coef_loss   | 1.49     |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 496697   |\n",
            "---------------------------------\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 198      |\n",
            "|    ep_rew_mean     | 279      |\n",
            "| time/              |          |\n",
            "|    episodes        | 1900     |\n",
            "|    fps             | 83       |\n",
            "|    time_elapsed    | 5994     |\n",
            "|    total timesteps | 499534   |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -81.8    |\n",
            "|    critic_loss     | 6.57     |\n",
            "|    ent_coef        | 0.0486   |\n",
            "|    ent_coef_loss   | 0.184    |\n",
            "|    learning_rate   | 0.0003   |\n",
            "|    n_updates       | 498533   |\n",
            "---------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<stable_baselines3.sac.sac.SAC at 0x7f82c9194150>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 611
        },
        "id": "2g50_7DFyN3G",
        "outputId": "c6eb3812-b19f-46d8-d1da-435014f1502f"
      },
      "source": [
        "def plot_log(df_path):\n",
        "\n",
        "    with open(df_path, \"rt\") as f:\n",
        "        _ = f.readline()\n",
        "        df = pd.read_csv(f)\n",
        "        df.sort_values(\"t\", inplace=True)\n",
        "        df.reset_index(inplace=True)\n",
        "\n",
        "    # compute rolling avg and std\n",
        "    sma_y = df['r'].rolling(100, min_periods=1).mean()\n",
        "    std_y = df['r'].rolling(100, min_periods=1).std()\n",
        "    \n",
        "    # plot with seaborn\n",
        "    sns.set()\n",
        "    fig, ax = plt.subplots(figsize=(10,10))\n",
        "    ax.set_xlabel('Timesteps')\n",
        "    ax.set_ylabel('Reward')\n",
        "    clrs = sns.color_palette(\"husl\", 8)\n",
        "\n",
        "    true_x = (ONLINE_STEPS // max(df['index'])) * df['index']\n",
        "\n",
        "    # fill standard deviation\n",
        "    ax.plot(true_x, sma_y, c=clrs[0])\n",
        "    ax.fill_between(true_x, sma_y-std_y,  sma_y+std_y, \n",
        "                    alpha=0.3, facecolor=clrs[0]) \n",
        "    plt.savefig(SAVE_DIR+ENV_NAME+'.reward_history.png', \n",
        "                dpi=300, pad_inches=0)\n",
        "    plt.show()\n",
        "\n",
        "plot_log(SAVE_DIR+ENV_NAME+'.monitor.csv')"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnUAAAJSCAYAAAClXjPaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdSZBk13nY+/93zrk3p5qnnjGDAAiCJCBIIinJAy372c96z1q8BUOhlUIrvYVWthRymIqwwgvKCu8UYUc4wkt74/cYIl+YtCx6oChRHCSAAEmAAAh0o9FjzVU53Lz3nPMWNyurqruquqq7qjKr6vtFgOyqHO7JvJWZX37nnO+TGGNEKaWUUkqdaGbQA1BKKaWUUo9OgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVPADXoAw2BpqUkIR1eub3p6hIWF9SO7f/Vw9LwMHz0nw0nPy/DRczKcjvq8GCNMTjZ2vVyDOiCEeKRB3cYx1PDR8zJ89JwMJz0vw0fPyXAa5HnR6VellFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzqllFJKqVNAgzql1CMzBkQGPQqllDrbNKhTasgNe7BkjJCsrJN2OthBD0Yppc4wN+gBKKV2JyIkVugWYdBD2ZXrdgmLK4TCY0frmLlpgjUIYLs5ReLwwzt8pZQ6NTSoU2rYeQ8McbouBEJeQIz4lXVMXuAaNcgLfKuNmxqH8TFCjMQ46MEqpdTppUGdUkNKBJwzSJaDGeKXaoxsjdZCq0Nodfo/F3cWcVkXRhvk1SpBAzullDoSuqZOqSNkjFAJBcYIxtyfbRMpryNS/gdgrSGVSNpsYZaWIffHPewDkQel30KkWFoj3F7AdfPjGZRSSp1BQ/z1X6mTyRGJIhgrmPUWcXEFl6ZIjMTxEWKaYPKCmCZIXkCrU369MpbQqGGWV4hrTYpWBxDcxdlBP6RdicB+F8yFLMf6ApLkaAellFJnlAZ1Sh0y22wRFlcwaULIckLWhWZvOnJlHUksPkRMmuCLgtgtejc0GGspioLNOcpyatMYIQzhvGWa54TltX1fX1oZrlqlGOY1gkopdUJpUKfUIRKBWHhCp0vodO+/Qoz9IM4X90yr+kDYIesVV5skMRAnxoFI8IEoBiuQB4gD2n0gAnG9uW393IMUiyvYVpvKxVl8khAiQxmsKqXUSaRBnVKH7ZCDLL/eglYHu9YCMbhKQsgLTCWBiYlDO87Bs4ECBy21EiO+nWFvzuNGa2UislYjJI5cdImvUko9Cg3qlDpsR5E5CwG/3gbAb8x2ToyW2bJHPJwIpFkXfEHeaOw7sHPO7Hs93b18q4PfyPCZZexIHXdhlhgiRsCEQLBWs3hKKXUAGtQpdajkwbtBD8shHcfESLizgFQSaDT2dZtK9MjCKkVn/1OvuwoRv9bCJUuETovEprCwjJ0YpVupaG07pZTaJw3qlDpkx7fG7XCOIzESvUfC/t4OjACLq+QLK4dyfABipFhYgaJGyMo1iaaT4S6fJxpBnEPygsIawg7JQREZ2NpCpZQaFhrUKXXYjiu2OKypyRjLMe932rXbpTjAjteD2thgErIct7AMVhBnIYAZHyXssPYuDQUUHgpPqFfJo+6uVUqdPRrUqTNBZCObc7SZNBEgHlOj08N6HLH3P/sdd7ll9XCO/QAbwaMdrWMSVx7XbgZ1G+eVtTb+9iLRB2yjSmV6klCtEASIkYD0n64khnLNnrMQom7QUEqdGhrUqVPNCdjgIcuh2YbEUYyN4g85rrNGtpSWO1lr6oTY7/QluyS4tu6MlRiOLxvZ49daBGswE2P98UgMuPUWVFNi7om9TRu+2YH2LdxoHZOmkDowFt+oYbs5rKwR1tuINcjMJFQqh7LhRCmlBk2DOnXiWCtIiHju/yC2BmwI5ToxEczKGmG1RcgLYuGR1CEjDTjE7EwiEbOyBmmCb9QPb1r0AR41CDGmzF4ZwMde9m2H+6z4HNYzSBw4B+1D2BzxEKIPyPIq6dw0ZnGZaCzF7QXsSI2YF9uvHCLFShNobqTzMJUEX3hiUfQfp8sLKs5CXuDrNXwvsNVdt0qpk0iDOjV0rDVYXxCdI7+nDpqTiJtfJmRdzPQERaVCCLEsy+ELWGsT11qQOqTw5K3O9rIbkV5K6tHHKSIkwSO35ynW29hGDYvgO9mj3/l+xNjPrFkj+D0CEWtNf9rZCogPyOp6uesh65alRKzBFTnOB7ppSozgDLC4Xm5iEEESu9kBYwCKpTUSgXx5vX9e/Vpr7xv10pChff958fNLlFPPlOdvtAHWkqXpEYxeKaWOlgZ1aui4bhdZWSP6gJubpujFKqkBs7xKPr8MgAmRZHqc2KgRIsQbC/hWVmacmrvc+cY8Y8/DTrtZI7j1dcLiKr4XLPhmu2wHdkzrzTY2ODgr2HaHUKnet17QGMEFj5lfJjpLbLaRkQbiC4rFVeKWrhah00Vu3MVUU9LpCWKMyOIafmNTxJZuGAMTI/nC6uHd3ZYMX7GyDqtN3PQYUqnoblql1ImjQZ0aOrHwvcwQuNQRpyax7Q6yukaxuhmthWabmOe4sRHEWYpm58ERWoxlpsoKYWEZI/bh1tfFQFhY2d4KLESOe7GZy/MyAA4Bmav21sUJxoDNcqTbJa6skfcKFwOwS2Yr+kBsZ4SsizTbEMpSJ8e9fm6gYiyn6XWNnVLqBNKgTg0d2YgiIhQLK9gsLwO4e3ulArFblJk7s79P4Rgjst7Chki0EcbGeai52I1p3AEKWU68dpNYeEyakGQZwdpyWrXVIXS7xPz+5+zBdzwEGblBKny5o3aHaFbr4SmlhpkGdWr4bP3Q9AG/sv7g2+x3YXuI5LcXy3/PjELg4dbXHX9SbocxxH6gG7o54cNbiBHCwwRyqi9kOW5tDVepEK2li+CI2CwDH8hH9t9KTSmljpMGderYWFO2pMqj3De9lRoo6JXNOK4PzBh7WcGDR3VCJA48qruHD0SN5x5ZzAuKG3cxaYKZnqDiHHF5lWKtha1XYGR/rdSUUuq4aVCnjo3xHhMiNklwnWzbDkPp5ljnwJhj3GhAfyesSBna7Tue3OjCoE6n2Jvevr1QnubeTtsY4q61/JRSatC0lLo6NqaTQYy4LIMs2/bhGGPErqySZMdUDqQ8KkYEaw1JkZN0OlizewFedfZEH+4piaPBvFJqeGmmTh05kbIgcFhcxcxMwtIKVBLK3NjGpohIfncJ28iQ9Jj+LENEFpZwxuBXm5hqhaTZIo6OENKUGCPBmLLG2Y4f5PrpfuYMwQYZpZTajQZ16siICIkVpJNBXlC0Otgso2hnGOe2T3n2/vPrDygke5hi3NaY3ucFHmBpDVNJyyze+Cikyf3FaDVjc0Zt/LFqOlcpNXw0qFNHJgkeWW0S11uELAfAr6yXOzZDwAHS6dCt1jbLmAyDsKX7QKuDuzBzXzHanQteqFNvGHY9K6XULnRNnToSIsBak+L2An693a/cv1GsVwRkfZ04v4zzxVBPacXVJubeV8o9nSnU2RCJSIwYI7r2Uik1dDRTp45E4iyS57tmNfx6G9bLgsKu3eH+qGl4xKLo7cjd/BQXIse0R1cNEx+wWRfXySgmxznDJZqVUkNoqIK63/qt3+L69esYY6jX6/yLf/EveOGFF3j//ff53d/9XZaXl5mYmOBLX/oSTzzxBMCel6njZ6TsRWruLFDs0Wh9a3eI2GwhQ1z7K4aI+ADG9n8nne72XZHqTIg+UHx0B+MsjI+CDO+XEXV6yYDTxBt1RvfbYcWYcrwxRp3gOGJD9Y70pS99iT/5kz/hy1/+Mr/xG7/B7/3e7wHw+7//+/zar/0aX//61/m1X/s1vvjFL/Zvs9dl6vi5bhe5vUixvLatWfpeQqsDzWPcIHFAsfDYVgcXyyDOGNmxZZk6I0IghrCtnqLIYKdjrUQSIlZ2/sQUKf9urTVld4xdrneSlD2OBWvKjh87nQMRwdrhmCe3thyLI2AN2N45cUTS4ElMWaD9vtsZIQ2eSqtF4j1p3iVdWiKsrOEIpBKpNJtUOm1c8Bgj/f82iJRrnJPoSYhsXCQCTiKOcF+guPn89q5joJJ1SLMO6eoalW6XdHmFJHisRCpZRtrNcBuPKwacQNrNSOYXcbfvknQ6OIHEF6RZhttlKmdYljZsHcdegfTGuRwGQ5WpGx0d7f97fX0dEWFhYYEf/ehH/If/8B8A+JVf+RX+4A/+gMXFRWKMu142NTU1kMdwllkiLK9SLO+jrdcWMfcHvs2xipHi1jzu/BTWujId2c0HPSo1QDFEXFFgs24/bVE0avh97Io1RrCFJySOEMIjZS6sgOt0iCvrxKyLq6SY2Um8sf0yPCKQZhlxZQ3yghgitl7FNWrgHMFaopQVuAPDn00RARcCZmWdmBfErIsAZqQOMSBJAs4Cvddpu42ZmSYkdqPWeO+52f44t34oH6QNnDGCCYGid+6tgIRANIZA2UXHBo9ZbRGa7TLFlTgoPFhL7OZEEYyzGGdxow18rYbEgGm2oZOVG8zyAkkcMQSiD8Sii7RysIaiVW7sMmlCOt4gut79pwmxWkHynHBnsXzg1pCMjRDrVSTLCEtrIELaqBImxohiECJmeRWMEFbKz2KslMfpzVCINUQfkKVVjBGKXntCN1LFpilhrQki+G7er+ou623EWmIIhMJj61Uq56YJzmGCJ4hgOl0IgW6jgSVisi5YU+45D4GQJARjykL23S6hkhDE9KsN7ZY5tEawRVH+WcjmrIsjQggUxvYDNxsDpiiQThc/2sDkOaw1YaROcK5s0dj7G7PdHG7PI2lCnBgdeG/ooQrqAP75P//nfOtb3yLGyL//9/+emzdvcu7cOawtT4K1lrm5OW7evEmMcdfLNKg7fjbP8avNQQ/jaMRIcXOh/Pe9Pc7U2RMC/qM7xBDBCCZNkFp11xRDmSkJYA2yvEJca2KNwc1MEiopvtcdb6/pLCvlHW1c6rIMVtbKL0S924ROF5N1cWMNqFYIlRSTB/zNu8TuZuY8tDNYWMGkCRhBADtSJ+QFMlonNOpI1sWnCT4OJgOxEWSFfhcPwcaAbbUJ88sUvU1Xfa3O9p+3bFE3WY6rV/vFpN1YvXwujQUjxDRFFlfKIDBNoF4lVCuEXutCYwSXZUgI5fpfYwgIptslrjch96RzU+UBF5aJWV4GaBNj5e7/Vgf/gJmLjbyvrKxjqhUwUKx3tr3XbJv9CLFfVaD/q25OuLu8/WlwtnzcW+4ntLP7uveEVgfT7GArCQDF0hp7iRtdVvJiW76tWG0BO8+8xMJvm+XwzTby4S0kcYTe+Ip2hjhLeh5otilW1pHemusQArZew6SO2M4oshxTTctsrLHluRtrEKwliCmztzFgujnS6hCWVpFKQmVilFivIZ0ucWGJmBekF+eQXmUG1pr4VofoA2ZxmeADsfDI0mo/nWqMwdSr5XPe6kCrUxbYnxvf83k7akMX1P2rf/WvAPjyl7/MH/7hH/Lbv/3bR37M6emRIz/G7Ozog690woWbd4mj1UEP40DGx2uDHoK6x4k8J9ZgRqtIrXLfRTFGaHcIN+fBWui0IenNszXXoS1IpYJMjRNX1pCpcaSyvS5i7GTl7QGpOLCOuLpSfgCN7fCayzrQzcqMUIhQS8r/9pJnm2PqlrUlqVWRqTFi1j3y97AYy4zJRnAWltfKrFtiy+CsyMugZr0FFQuVA/6d+F4AZIHmli+fRsrzshEwdT0UXYQRSB04B1mHuLpWZqmM9NZS9sYbKT9JV3rBlESo9j5a13uBUT0BHvD872Snc7vFkbxWQu95OPbXYe/LQ9o77lrvubvvMyWW5zI1kG59vfny3C3lUEnBGCR1xDyHZrv3OnDl7ddWe3/jefn3YB2srpTXCaEcSuOe2qQ78b3bbzxXSXneB/l5P3RB3YZf/dVf5Ytf/CLnz5/n9u3beO+x1uK9586dO1y4cIEY466XHcTCwvqB0u0HNTs7yt27e3/rOemsEczNxX7JkpNgfLzGykp70MNQW5zYc2IMLl0A76Fa2Zz6qqRIq0NcXMHfm0kCYPOxys1FYl7g8kDRqONanfLDpdUhrDYJg5jyX2rCrUXGp0ZoVmoUI3VCLFNgG1NVG9krk+f4JDnw1KUlQhEwyyuE9VY/kxMHvRFpqRf4DWlRyhP7Whkah//cmUrCOBzp570xsmciamiCumazyerqaj8g+8Y3vsH4+DjT09O88MILfPWrX+Wf/JN/wle/+lVeeOGF/vTqXpep4yOFH/ybsFKDEgLFrYXtvzOCqaTlmqJ9vDb6U2utDJflFPPLwzHNH2LZEWbxLrZawU2OleMKgdjqlBmuNCF2MpLpCbojIztOIW9MLRsjxFiuwbV3F4hZmYErhnWd6hCcAqX2S+KgV/X1zM/P81u/9Vu0222MMYyPj/M7v/M7vPjii7z33nv87u/+Lqurq4yNjfGlL32Jp556CmDPy/ZLM3WPrpJ3Ka7dZJcmqUNJv+kOHz0nw2m/50WcxVw+R56mxNib2SyKcrF7bzqXbgZikLwgn1/SoOkh6Wtl+JhKwvinnmF+6ejOy4MydUMT1A2SBnWPxlnBLS2T314a9FAORN8Uh4+ek+F0kPNiKgnm3DTROWRlDb/a3MxCGjlRX/yGmb5Whs8wBHVDM/2qTiZrBNtsU8yvDHooSqkhELKccP12Od1677SzBnRKHSkN6tQjcXlOuLuo6+mUUptCJOq8qlLHbqg6SqiTRQQoihO141UppZQ6rTSoUw/NhkBcWH7wFZVSSil15HT6VT00UxT4djboYSil1MkWApJ1idXKwRufdvNyA4obvo9zM7+EWVgC5yiuXCg7PqgjNXx/BerEkKLQhc9KqcGIEWm1SX74DjFxxLFR6HaRwiPrTezdRWKtSv7C0/hL58vaer1+nrFRK9tkbRRQbrXLfqnHEXQUBfb2POQF7toNpNnGNMu2WmFilPz5pwHBzC8SRxuYhWXsjdvEkTp+dgazuIxZb+KffYzKnSXsnQWiMfiL50DKxxLmZggTo/jzs2W3jG6OdDLiSL18Huxm79OyIe4+ehbfnsfevIOdXyRMjhOtI1ZT/LkZkh+9W95/o0aYnkTWW9j5RczKZtWH9G9+SPel5/CXzmGWV8sgNk3xV/ZoFrDRZcSYzRZuBw16zxgtaYKWNHkYIpAsLuPnT+70q5YEGD56To5YXiDNFnFi7EA3G6rz0s1JX/8x9votxPsHX38XoVohjo2UAUY3JyaOMDlO8dQV/OWDdSUCIM9JX/sxZmUNf2GO/Lkney3Gutj5JexHt5BWGzu/vfRTqFUR7wmNOma9iezQIzb2MnHSzQn1GrFawS5uvvf66QnM6vp9tw0jdYonr5D85H0kK9c+h0aN7DMvY1bWSH74E6Sd9d7QHflzTxHGR3E//RCztILESEwc+IBpHez8R2fxczP4K+eh8LhrN7F3F+67XhhtEKsVimcex968i6ytE2amKB67SOUv/hrTbBEpG3v487Pkzz+NdDJCo0YcHy0Dvn0y80uEkXrZ9eUgvC+7xFT2bh02DCVNNKhDg7qHYY1grt8kNHdqfXQyDNUHlQL0nBxYN8f99Bru3auYTkaYGKP7yecJc9P9q8h6C/vhDczKGu76LQDyZ5/AX7mALK8R5qaQrItZWMYsLFE88zgxSZCiIMyU3XmO/bx0c0gc5u4i9votzOo6WIO02sh6C4kRf34Wf24Gf2GunH4EYuLK4MXaMvMWA5Xv/AAzv4g/N0uYHMMsrmDWm0RrkU6GaXfwk2PEeh17Zx7JizLgG6kTa1XC6Ajuxm3IuoTJ8X77N7O8illeJaYJZq1JNKZsmbZFmBjDLK/2f46mbDxfPHkZf36WMD66PcAoCtwH1zFLqxSPX0JabfyFWahUyinadkasl71Qx4qM9ZU2YWK0DB5jBO+xdxYI9RpmeZXkzZ+UTeaBMDYCMWLWmtvG469cLLOHC8tIZ3M5TXSWWEmJ1SrSahEmxvFPXMLPTmNWVolJUraHW10r27FenCv/1m7P4+em7//i4D3u7Z9i1ppldq6bY+eXcB9c3/6cNWqYZvm3FkUonrpSPj7vce9dQ7aELLGSEsZHidUKYWyEWKuCs/hzM/3nxCyu4H56DXvrbpkddJY4Uqe4eL78m8oL8mce334esi7ug+uEiTGkKEi//RpYQ/a5nyFWe63/GnXiaANZWUPaHUgSuDirQd0w0KDu4FzwcO3mZlHRE0gDiOGj52R/zO15knc+wN66C0Co18ogqDfdVVw6T/HUFcL4KNX/9i1MJyuLARcHe712P/UC/uIcoxMNVrvhyKa+ZL2JubOAWV3H3rhzX1Yo9jJJsV7FT0/hL58nzB6gHeReU4xFsbkerfC4dz/A/fQaZkuv3ugcsVpB1pvcey+xWiGKEEcbFE8/jr84h715h+TNnyBZtwxwatVyWvUQp3f381qRdgezsISfm+kfW9ab2Gs3oFKheOLS5lRsNy8f+4c36b70POHi3KGNdU9Zhl1Yxl6/Rf7CM8TRBvaD65jVdYonrxBHG5uPZ3UN98FHZfDezZFmGzu/CIXfFlBHkW3BH/QygrUq0mpj1lvbLouVlOLKxfJLQ1Fg7izcf56t3ZYZjoB/8jLu/c2gNEyNU/3tX2ehc3QlvjSo2wcN6g4uzXP8hzf31dNyWGkAMXz0nOxMlldJv/8GZrVJmJ7A3imnsWKa0H31pXI9FSDNNrX/8j+23TYaQ/dnPlFmR0KEGLHXb/an68L4CPbDW4SZSfKPPUnlr15DWp2yd+2WrE6YGKP74rOEqYnNaagQIIRyPCHiL5/f92MyC8vYj8os3EZwCuWHZ0xcmXHpTbcdeLrsEMhaE3vjNv7iuTI7Zm35eGMs3/eSXiA4oDVe+lrZIkZkdR3Juti7i0izhb1+EwmxzFzPTJZZ0V4AK6trmPkl4kiDmDgqf/XatkAvpgndn/kE5tY8sVGjePwSGMF98BGSF8RKir1+C7uwhJ+ZpPjYk9irH2Hnl0j/7y+wWGnsNtJHpkHdPmhQdzAikCyv4u8sDnooj0TfFIePnpNNsrRSTkOJkP7VaxBCucA/REgc2c99ijg2cv+aIu/Labzrt7Ef3iD/xHOEmcmDHTyWwZ97/0PMwjKu2SQuryPelxmKi3OQJP0prQ35U1fIP/3x/phkrYm7+hGyto5ZaxLGRspA6fws7sadzSHPTpG/9BxhtFFOY6kH0tfKIeqW6x7D+Ciy3iqXLzwoWC887oMPKa5c7H/JMalj/NPPapswdbIYEWi2HnxFpdTBFQXpd35QruPq8ZNjdD/7CrFee/DtrQVrKZ5+jOLpxx5uDCIgQvH04/D049TGa6zcXcVd/Qj37tV+QFZOW10oS3HESPLuVeziCsVjF8EYkjfeLjcBVCuYTlaujQPcjTuEeo3uz75ErNfLTJjualSDkqb9bHds1Pd3G2cpnnli+++G4G9Ygzp1YC7LKNraRUKpR2E/vEH6+luEeg1/5Tz+/CzSbJO89V5/h2Rx6TxhqtyROfAMVppQPPsExTOPY5ZWiaZcR7a1PEaYHCf9mx+S/uCt8ufRBtknnyecn4XCY1bXCKMj5dTlEHwAKnXaaFCnDi7rlmtLlFIHEyPJaz/GLK9iF8rAzXaysjzF62/1r9Z9+eNllmwYiRCmxne8yD9+ifb5WcziCjhLmJ7YnB5OHGH6gNPASqkD0aBOHYiIlOUGlFIHlrzxNsl7V/s/dz/+DMWzT5QLsNudstxCrYp/8soAR/mIKinhwuygR6HUmaRBnToQEYgHLIuglIL0e2/gPriOvzBL9pmXy1/2pi6LZ58Y3MCUUqfG/ksxK0VZ1Tt2Nag7E/IcM79U7oRUDy8E3I/fxX1wneKxi2SffaW/mUEppQ6TZurUgUjwJ7o2nXqAGDG37mLvLpalKLIu3U+/QEyScldYCITJscEv2h9GvTIg0skwy6tl5f3lVdLvvF52LZiZovvyiwdqa6SUUgehQZ06EFN4wiP0W1TDzdxZoPqt7wNlxXSA9LUfb7tO8fgluj/7yWMe2XCTlTUq3/3BtpZQW+UvPE3+8Wd1x+dxE9FM84CJs8RCPzOOiwZ16kCkKIiaqTuV3DsfkL5eBnCdX3yVMDuF++A67ifvE6YmymrtdxZwVz8iTE9QPPWQNdBOmxip/Pl3kSwve4aOjhBHauAD0snIn396Wy9WdfjEWWIImMQRQ0SMYCbGkBDIF5bLgs0iiJGyOfxGr9i8DDZMrYJYg18/5mK+xiACppoiU+MQI2FhhVgUmEpK9IHQLvuxijWb771bvhyIs5hqim92ykzxsARRxmAbNWR6vOybu7RKyLqb3xbVkdCgTh1MPgRvFurQyXqrH9AVl8+XdcWA4unHt5XWqPzP72DvLpD+9Q/LArNO30LIC0w7o/vJ5yk+9uSgR3OmiLOYehVmp6AoiEmCxIjESJGmWO9x1ZS4sILMTBCtQ2IgbnS86OaQF2Uz+BBxlWXCepuQPeQOfyNlUJX7PTOEklhMmiKTY5AmeGvxvUDNVirYGPGJw3YyTDcHayBNYXmN6AxSrSCpxcZlZGyE0Kjj1tbLbiKNGhQBgieutvC9gs8PylpuCxr3eHxufASSBL+wjFhL9B6MwY3WCUlSdhipJMR6jWAt3hikUsE06rjVNYqFFQgRsQY7Wic0O4S82HlsArZWLfvWBg8+4pvaRWMv+o6s9k0EotdNEqeBuXmH6re+T3FhbtvOy+wzL+/ZvzN/6WOYb/wlAtS++g06n/8scWz06Ac8ZMz8EvbaDeyN2/02WXFkn5Xo1cEYg5sYIXa6xN7yDxHBTY0RGnV8kuAjkPY2nmwksXzAI33qtqsAACAASURBVEitTno+IU+S+9tB1hxSF2KMZa/bmWnceI65ebdcP5r7flJMnN092BPBJA4zOYofG8W1Wvg7S8R8+/ulOFse5/wMvlolsCWW6f1/YSwFQABfqUCl0r+OmZ3qLd2MjM2O4l1atqMNERqNzbit91Qk1SrOCn69jZ2ZKMtRhUCxsg4hYioJZrROFAMjdWRtnbBetqaTsUbZ59QHYidD0gSmJghpQgBcvYp3DtNbjhNqVfIiYIwQQyBseVwxRoIY4uQ4rlopT5IzRJcQpzyu1SasrIEPmJE60Qg0O0jq8OdmCAgi4IoCd3sBrMG3Og+3aU8AMae21qoGdWrfREQzdSdNJ6P+1W8AEEbqZQNr53DXbwLgbt7BLizRfeXF8jqjezeiDlMTdP7R36byze9i1lvYm3cp6rWzlbGLker/+Pa2XxWPX8KfPyG12YzALr2uTZoQYyTmxWCn8YxgnANnMHPTFJUUEMR7bFGAQDdJy4fxgOm8GCOZdbs+5q3tz0OI5M5hL58ve992u0RrIcbyddNuE5vtMkMVI8YaQquDOEc8N02OECKERoPkUgKr61BJkRDKzUapIyAUzu02nHvGtv3ne4NS77f/fO/1czEk05PI5Dh5b3OTEcrAqigIY2PkzvYDRTM1iZ0Yx3hPN0nLbnGUG+QiQsHmtK9Pyn6nuN7Gn95nw71j2jbeAL5S3TJgwFjC2Ciu19c4TxJiBDtRZlT9lpPsxWIvzGGk97ewuExotstzGGL592oEEdmWdZS0fH8ySYJMjYEIcWGF0Mn2vZzIVBJCNx/66eMz9E6sHpW1UqbA1VCT9RYEj7t6g+Ttn27+vp0h3byccgLa/+CXcO9fJ3nnfezNuwDE9MG7WmOjTufv/yL1//e/kr7xNvbuItkvvno0D2bI2GsfYW/NA5A/+wT5p16Adgdq1Qfc8mhJ4jBpglRTwlqr/PC5V29q0M5OEtda+NUmkjii94gx5bqy2SliDMjCCkyMEu4sEjrdMnNzhEGeJA472iiLbFUq5XSjteA93TQlBoBYZlg2gokj+nCNkTJ4EaGobD+v0mggI5vN1EUEM54TjMWzGVSFCFmSYmaniTEiItsDsmMMDHIx5fPZO34A/OjI5pi2jCtECGLKQC3GLUHi0e7YDiHSFVtmGHvjKdj5y4eP4GP5t2BnpmA6IiFgQkC8LwM2Y5FORvlEC6FRQ0LAJwmh96DM5Ro2yxAfCLcX7n/NbKzB9AE71oBzM9ibd/HrLcRZxJqHn6Y/QhrUqX0z7Qzf0Z6vw0qWVsodmBtraHq6Lz3XWxcXce9dI33jbfLnnyaOjRBGyylDd/UjQq0KlXR/B7OWaAQJEXNn/pAfyZDqZFS+8wMAYrWyOW09oIDOVBJAMGMN/FgD38twuMlx3MJSb61SjliLPTcNiSNaQ2EdtlLBpQ5GGmXQYW2Z/bLlR4I7P0uBkFycJVltItNj2JlIXFmHVhvfyranhYxgKmkZAN6TLpLUYZxDqimxmxN9QIzp3SYhGAO1Knm12s8Y9W/be0zD4t7xQcSL3TVI2wjk4jA9CHZ6HCdTmRQUMLb8z21+KZXRjX/3gtMtgW1524hPUiSF5MIMdmGZ2JtuFmMwMxPlWr5uTqhXyRHcuWncWB0qFULiMNdvE7Muplp+ETFm8LvbNahT+1cUWnh4GHW7pK/9GHftBgB+cpwwN42fmy57b26ZGi0eu4i0O+TPlQv648QY0Vr8+VnyTz1/oBpqnb//i1T/+7eJlbT8lDjl5TrKb/7g56bLzOQx15sTa7BzU8RWB6kkMFZmjArryimq3rRXVwzJzBRxBtzyCiQJ+UhjM1MUwVuHmZ4qp1q3xygA5L0fujbBTk9ipkfI7q5hpiYw0xPYG3fxa81yTdbsFISAH6nj2h3i4mo5rdXbjWouncM7RxDBxIiJAW8MiCES8b0AI+6QlTkFcYcakP0GrTFCN63gLsyBCDbrEq0h702RS5r2/w4LYykavUxthHRmEtPJKKYmkI0Xjwy2DqUGdWpfRCBqz9ehZO8s9AO67ic+RvH807tfuVYl//TH+z+GqQnav/r3Hyogi6Mj+EvncO9fx167gX/80oHvY+h1MtwH17E37kBSvl3mLzxzfAHdxvo3EezMJMXYaH9jSj9I2yEYyntrn8LUZJmV2eE6920a2IXfOj0XYrlIfnYKNzECxtBNy+xuDFBUqriLVaQoMCGANXStKz8UY3lbMOUcIAFdzKGGRYFAhCLdPr2/V2yYV6tIrULYun7PalCnTgQBzdINJWl3AOj87Z8nzEw+xB08fIYt/9iT5bq8H7+Lf+ziqcrW2eu3SL/zOnLPLrnYqB3L8cVZ7Plp4uIKMjZCMTqyLcDaj/0GbgdVGLO53uyeQxQRsK6/A3PYF5Yr9bDuy3QPAQ3q1L6IAPnZzdTZD2+S/PAnSCcjTE3Q/dlPEge8OB7ALCyRvv4WMUnKgO6Yg6o4OoKfnsQuLFH5X9+h++pLZTuxU8Beu4GEQPa5V5BmG4j4S+eJ9aMP6iRx2NlJwkgDSRMKl7DHpkKllAI0qFP7JDAcVcoHIQQqf/Va/0d7ZwF77QbFc08NdEzJaz8i+emHQNmGalBZsuxzr1D5i7/G3l2k+l+/Sf7xZymeunKi+8PK8iruxm38uRn8xXPHe+zEYc7PkNdrBB/BJprtUkrti3aWVvvi8vzMtger/K/vAOV6tdb/9Y+IIqRvvD3QMZmlFZKffkisVuj8ws8MtpNBJSX7Oz9P+5d/gVivkb7xNu69a4MbzyGofvO7AISJsWM/tpkcpajXjmzqVCl1emlQpx5IBGi176uOfibEiFlcIdRrmyUsNmq5ZVm5nq23gUTWW9S+8mfI0srRj6t3zOxzrxAuzB398R5EhDgxRufvfQ4A+9HtAQ/o4cnyKpJ18dOT5B9/5liOWZYn6R3fOQ3olFIPRadf1QOJCHEIiywei6xbNgV/9omyGCqQP/cU6Q/eovaVb/Trq3c//gzJj99DYsRdv0U+OX6kw9ooILyfYsHHyjnCaAO7tFK24Tnmsh+PyiwsU/3vfwlA8ezj/XN+VNzkKFQrUEkxa82yDMNZ6s6hlDpU+u6hHkhi2fvvLDLrLQDClsX/G4HU1hVs6Y/eve/yIxnP7Xmkk5H85P0jP9bDyj/2JJXvv0nyxk8onrpCfEDrsUExdxep/NVrRMBfOk/+6ReQ9SYAxZULR9/2yxqYHKfrytZcZqayeZlm6pRSD0GDOvVAUgRicQanXgH3zgcA2wKTmO7ddUGOapo6xv5aL4AwNjKUmxHieFlHLXnnfeztu3T+wS8NeEQ7c+9dRToZApj3rpK8d7V/WfeVF482YyYgxuCRfpFUnXJVSj0qDerUA0kMOxYvPQsky8pG3FuCunBumvz5pwn1KvRKiSQ/fIdQr5L+6F2St95D1lvEkTr5i88+0q5UWV0r16elCa6XnQuNGt2f/RRhYnQo68KFqQmyz7xM8qN3MKvryFpzuLJ1MULhMStrhLERss++jHvnKu7qdWRjM9ARBnRlZ4hpSCyFM7qzVSl1aDSoUw9kvKc4g0Gde+s97PwSxb2dEqwl/8THtv2q++pLAIgPuJ9+iLt+EwA/O0U4N3P/nW80lb55lzA9gbQ7xLGRbWvQZHWd2n/98+03EyH7xZ8driBpB/7yeSBS+fZrVL/xF2SfeXnn5+EoZF0qf/HXZTeDV1+6r65c8jc/7JeCKR6/RBwdIX/lRfKXPw5ZF7PWPNJg2c5OUoyP4n3QgE4pdag0qFN7EhFotc9eE0bvSd78SdnI+YU92m7dI3/pOfKXnoN2h/r/99+pfvO7FE9c7gd9AIRA/f/5+n23LR67SPfnPlXuuL11l2TLOr0N7f/j723uvh1y/tJ5slc+QeWv3yT9mx/S+d/+VnnBIQZM0myRvP4WxROXMcsr+MvnSX7yPnZhCYDkzZ+Uz+mGGDdr+z3z+PZSMCJQrRCqFY6ECHa8gR9plAGdUkodMg3q1J5EOHM7X83CEsnrbyFA9xPPEUceIiu2JTBwH1zHLC4T6zWKxy7ifrpZw83PTBLGRnDXb2Gv3aDSybB3Fu67u2gtYXrixAR0AIjgn7pCvrpG8u5Vav/5a2BNWZz42ScOZWds8uP3cDdu4270SqhsCYTD+Cju2g3M8irdTz5PHBshee3HQLmDOX/puUc+/kGY1BHmpini8E2ZK6VOBw3q1J4kxrNRny5GZG0ds7hC5XtvAODnpgmzUw93fyJ0Pv9ZzOIK5s48knUxt+ep3Lrbv0r7l3+B2CtuG0capD94qx/Q+elJCKEsDQK0//Hf7TeUP2mKZx4nefdquVvYB9I33iZ9421iJcVPT+IvzuEvXwC3S/mQGDG354n1arlJZSNgjhHbm+a+V/a5V8pOIN9+DbO6TvXPv9e/zM9MlSVqjpu1RGPQfl9KqaNyMj8l1LER789EJwl79aN+MAdllif7Wz/3SPcZpiYIUxPwzOPlL/K83DjQyQiNej+gAyiefqwMWqwlnJ/tT1G6H79LHBs9WRm6e8SRBn52Cnt3EShLniQ/eR/Juv0sm3/vKtkv/Sxwf19Ve/0mlb96vf/zxjS1LK8iW1rXRcoyM/7CLP7CHIjQ+tU5zOIyZm0daWeE8dFyvd9xbjAxBlNNMVNjGs8ppY6UBnVqTyYEfDjCoC5G7AfXCdOT5UaBAdlYgwXQ/eTz+ItH0KUhSQjTk7sMwJbZqnsULxxPR4Ojlv3Cz1D/8p8CkH/yefJPPo+02hAC7uoN3FvvUf+TP8N/5lNw6cK2oEt6tQI3uGs3IC+wd+aJIsRaFdNq0/3cK8Q0LVt7bdzeWcLcNGFu+ngeqBHEWUyjBoUHa5CROr5Wpyuc2V3kSqnjoUGd2lVqBcnjkRZClbUmle+/CUD2mU/vGNgcB2m2gTLbUzz92JF3EjhznCN/6jHC3OZ09sau1PzFZ7HXbyJrTcK3X6cydZXs858FIHntRyTvXr3/7m7eIVYrZD/3KWKtir1xu5+dGxRJLPbCLDFJ8NYSe2OJkbIWncZzSqkjpkGduo+IkBQ5khVIs3WkO1+3ZmEq336Nzt+t7p7NOioxYlbWKK5cKHepakB3JPJXXtz1sjA6UpYSAeziMoSAWVy5L6DLP/Yk+XNPYe/Ml9ndXmBYPPfU0Q18P4zBXpilW61tBnBnbce4UmrgNKhT97EGuLOA73SJW9YsHQXT3D61Zq/dKNehHWPGRToZknXL3aUa0A1E99WX8POXqBEIf/Ea5vY81W99v7zspefwl8+X52hyvNxVe+XigEe8nRghONfvDqGUUoOgQZ26j+1kxxLQQVlnLIqQ/Z2fx71zleS9a2As+aeeP/Jjb7AfleUwwuT4sR1T3SNN8BfPIc01gH5AF62heOwi1KrELf13h5GGc0qpQXv0QlHqVBGB2O4cS0AHZaYujo0Qpifpvvzx8nfzi8dy7P4Y5pcItWqZIVQDJRdm6Xz+s4RaFYDsF16F3r+HmshQtmxTSp0tmqlT2zhrtpWJOGrSbBM2ivv26pZJs4W9+hH+wiyk6ZEdO7x7jcqPfopZWC7r0emH8sCJCGFqgs4//ruQ55Cc3FIuSil13DRTp/qMEeziMqHVOZ4Dxlg2vt8yrRarFUwno/LdH1D7kz87usXmMeK/+X3s3UXiSJ38+f23AlPH5CQFdIJ+KVBKDZxm6lSf8x6/sk7odI/leNLJkBCII5sFZ4unH0NabezSSvk52e7c15D9UBQFhEj3k89v7/+p1EMQhKBBnVJqwDRTpzblOeEY+7xKb+dr2JKpC3PTZH/vc3R63RzMwvLRHLtbPs54gjs1qCFipMzWKaXUAGmmTmEFrC+QojjW2lqyWtYl22lXY5iZJIzUce9dw185/ILEZmm1/MdJmuJTQ8VUEsxkr3uFNXiN6pRSA6ZBncIWBfGj24Rj3CAB4G7cJiYJsbHD9KoxhJkp7K27h35cabZJv/cDEClbSil1QGIN5sIseZr2S5loiTql1KBpUHfGVYocVtbxxzjtukHWm/hz02B2XgUQGjVcJwPvD60osLTa1P7L/wDA/u+/dDTr9dTpJ0Iw5ig76Cml1IHpmrozzBmg06VYXDn2Y8t6C2m29ywouxFwSat9aMetfPO7m2M4P3No96vOGIGo061KqSGjmbozzDbbhOXV4z9wUVD72v8s//n47u2eNqZla1//Jt2XX4Q8x314k+KJyxTPPvFQh5b1FmFijO6nXmBUdyuqh6YbI5RSw0eDujNKBGKeE9rZ8R97vdz16memiGOju14vTE9SPHkZ9/510r/5Yf/36es/hhAO3sTdeyRG8svny2LD6oHEWWII6DzjPbQunVJqCOn065klcMwbIzaYXimT7oP6u4rQ/ZmX8DsEYOkbb+97ZbpZXIFOhlku+4qS6HeZPQmYaoodb2CvnMeONgY9oqEjIkQN6pRSQ2ZoPt2Wlpb4Z//sn3Ht2jXSNOXxxx/nX/7Lf8nU1BSvvfYaX/ziF8myjEuXLvGv//W/Znp6GmDPy9TuRCgL8A7i2M1yjdx+G7RnvZp19vot/KVzuLfeI/3Ru2V7r5nJvY/ValP9xl9s+53WptubrdeIF2cpTLk5xVn97reVWINYU2brNIGplBoiQ/NuLSL85m/+Jl//+tf5yle+wpUrV/ijP/ojQgj803/6T/niF7/I17/+dV599VX+6I/+CGDPy9TeBIjdQQV1LWLiYL/BVa9Zur9yAYzBXzoPgL29d7kTabVJv/N6/+f8hWfIn38af372ocd+JjiLN5YQIjFG0KCuTxKHvTAD52cIw/P2qZRSwBAFdRMTE/z8z/98/+dPf/rT3LhxgzfffJNKpcKrr74KwBe+8AW+9rWvAex5mXqAEMr/BsB9cJ1YrTz07eP4KKFe66/N6/MeWVopp2VjpPq1/4mdXwKg9SufJ3/xWfJPfEwLDj+IMf2Z7RhBXEIyO4m7MIu7NIubHh/s+AZJwNdqdG1SBrxKKTVEhmb6dasQAv/xP/5HPv/5z3Pz5k0uXtzcITk1NUUIgeXl5T0vm5iYGMTQTwwJgTiAxe/SbCE+EP2jBZRxpN5vM7Yh/fZruJt3yssrKbL18T1CEHnmWMPWecXQqIERCgQRIZHm4MY2YIKUwZyup1NKDaGhDOr+4A/+gHq9zq//+q/zp3/6p0d+vOnpkSM/xuzs7rs8ByE2W4SRCoTjzVqF9TU8kLz6cSrjD1/410+NEd56n5H33sd8+gX8n32b2AvoACTr9v9tXn6B8V2OtdvvzyQjUK8jIzXM9B67kk0gHuHzNtTnJHGYyQZyBjfbDNt7mNJzMqwGeV6G7p3pS1/6ElevXuXf/tt/izGGCxcucOPGjf7li4uLGGOYmJjY87KDWFhYJxxh1mp2dpS7d9eO7P4fRppl+OXmsS/0tneXqQDrjRHiysMXFTZzs1Tfep/wN2+RX7uNXSinWfMXnkaW13A375B99mX8xXNlVmWHY42P11h5hDGcNqaSIOdHKQKEPf5eK1lGcUTP27CfE1NJCMutM9fndRjfw846PSfD6ajPizGyZyJqaNbUAfybf/NvePPNN/njP/5j0jQF4BOf+ASdTofvfe97APyn//Sf+If/8B8+8DK1O2MEybsD2blnWm0iEGvVR7qfMDdN9+WPA2AXlgiNGq3/85fJX/wY3c++TPsf/Z1yQ4VOk+2fCN7HB37BOfOdFM74w1dKDa+hydS98847/Lt/9+944okn+MIXvgDA5cuX+eM//mP+8A//kN///d/fVrYEwBiz62VqdwLElcGsizKLK2VAt0u/14MonnoMPz2JvbNQ7ozd2E1rTL8bxWFys5NQTaGbE5bXCVumeE8Nkf3V/zvT5Tw0qlNKDaehCeqeffZZ3n777R0ve+WVV/jKV75y4MvULkIg+gEUHo4Rc2eB4onLh3N/IsSJMYqJscO5vz2PBbFepZtWsI06tlvACQ/qJLGItYRusbkTej/xighnNqrTzK9SaogNTVCnjs/Adr622kiMhMkTWBKjVysPyvjHnqAPd0kcbnwE32wjRgh5gW3UiJNj4Byu1aG4cWffAYsYwY3UKNZaD77yabTfbKZSSh0zDerOoDKoO/4adf1OEiP76yQxTLa3hYpghzeoE2vKHFqImGqKmRrDj40SJwqCtbhOBgJdm0CEJHH9oHU/sUpME2RyDM5sUDfoASil1M40qDuDJETCADING8WC99sebJi4yTEKt+XlIkO1x6jPjo/A9DhS+DIIdZbcJYQilGMOEessErcH9QfpT98NkG5kLs9axuoEZWiVUmePBnVnjEivhtsApl9Ns0UUIdYfbefrsTNCHG1Q9FI0MZa/G0YyPkJmE8Qlm/HWPec6GkHilqC0P7V8gMckcjaX1e0zm6mUUoMwnOkGdYSEOKAF/tJslbtST1i2Q0S4d7JahqwfqiQONzFC7O0q3jPwMIa4JdPYv+oBTksUwbgz9J2wF8SfsD9dpdQZM1yfTOrIiQB5PphjtzrE+hB3C9jNtvV0PZXKUAV2JnGEi3Pk6YPbofkIxU4lZQ4QsQTnMBdnMbXT3X5NnMXNTeEun0NSh5ylQFYpdeLoO9QZI0AsBlDOBJAsIzROYE/eLTtfNwxiTeKeTFk4eD9N5kNkeypv4/EdIKjzCLFSwVn7EIM9OezsJMXoCIjgZiahWtHpV6XU0NKg7owR7wey8xVAOl1i9QRmdnbaRTBs83CHUMz5oR7TEO8CPhQi+F4QHEZGyqBZozql1JAanvkjdSxsURD9AIK6oigDykp6/Md+RLLT9Gt5wfEPZjeHsXHjYe5imJ6Do7Blin0/WVCllBokDerOmm4OAwjqpFNuzjiRmTq4L+AZuo93MQ+fQNrYyXrAqC5Ghra0y2E5831ulVInyul+R1bbiAx2PR1wIjN1ZdAz5C+VR8jURQRh/8WHtzntMc9pz0QqpU4VXVN3pggMKqhrdQCItRNWo27DfZ/tMlwBjXmEonECdqROrFcOfh+HsZZvSEnqhrYeoVJK7USDujNEBPDFYI690U3iBLYIw+wwtfkQ05VHQZzF1Kow0nik6dcwPUF+4ExdHIan4GgYgz0/g9cSJkqpE0Tfsc6aAWXqTLNVTr2ewA9JGdJsjUkTzKU5ikpKCPGhE3U+liVKHub24izJzAT5/PLDHXxIiRG8cxTDPu2ulFJbnLxPWPXQJMbB7Hyl7CYRTmKWDjDD2tbMCEWa4v1gtm3ECNRrA+tQcqQOWLdPKaWGgX4NPUMkhMHVqGtnJ3Y9XUySMhOm7pOZ0/m9UDSoU0qdQBrUnSESwn3N3Y/t2Fl2Mne+ws6L5WVINkoMxRiGYRCHR5zF1CvEIZ12V0qp3WhQd4YMLFOXF0henNigbmhrlQ1LMDUs4zgkJk2IF+YI+vaolDph9F3rLIlxIJk6e/MOAGFy/NiP/ch2ahHGltpuavgKMT8qEYqwvz66Sik1TDSoOyNEBCkGU87ErDWJQJibHsjxH80ua6t2iedMJcEe18aKhykWfBROWaZOa9MppU4qDerOCBGgVyvuuJjFZczdBeyHN4j1Klh7rMd/ZEZws5PEdOfNACZNcBdnkY3+oCLYsRFM/XhaoQ1VLDVMY3lUGtQppU6o07l1Td1HgJDlx3fALKP6jb/s/+hnp47v2IdEjCGMNsh3+u4jIIkljDaQxRUQjzhLmJ2C67ePa4THdJwHkd5/w5A2PARDFS0rpdT+aVB3Roj3x7qervLn39/2sz+JU68ie4cp1oIRzPQ4RAjLa0RA5Jie52GJPU5ZTIfs0EFEKaVOAA3qzgiT5/j8mNbUxYhZXiVMjNL5/OeQVpvYOHmFh0WEsGtHASmn6cRArUrMuog1hBiPcQPFsER1cFKjuo2p821FuYfpaVVKqQPQNXVnhITAsaQf8pz6f/4aEiPF45fLvqkjjZM3pWVMr6H7zhcHoJgYJwLR2rKx/Za1dcdiSJ7SmCS48ZFBD+Oh2Lkp7MwkdqTeX0snJ+1vVSmlejRTd1YcU89Xs9bcPOQTl/Z/Q2tgQC3MduJmJwijjV6tsvuD4RihAIwPCII1BpMm+HL+9XgGOSTBRyFC4k7o90Nj8GN1ZGIU1+lCN+tt6Dl5WUellNKg7gwQ4f9n796j5Kjr/P+/Pp+q7p57kplMkklAWEH5ZZdVkABfb6zG5SQqtxyRYMBdhV1Y/YrHG8eIHlSWZcFdFlkVV9eseJasejx+iRKUywqKimBYUXCjYpSAMpN7wsxkbt1Vn98fPdOZIXPpma6qrul+Ps7JSU/XTNVnumbSr3wu749cUqFu935J0uAbXiVlMuV9TS4rb2GL8rsPxNm02fF95Y03Y+/m2PZhLpORbWtNtrZZOjJdkZ2noc5IhbH/S2SzMrmcMp5RMZ0T7ADML4S6umAS66nz9h+UJLmWWcyhs6Y4hJkGxshm/SNDqWUKnRQaK+OUYNhKR6pzrrhSeF56QW+nc04jBcIcgPmJUFcHjJGU1CKJfEFBx6Kye+mk0WiSgtpgJuPLa2qQW7pYgVRBR019zamTJGV8Gd9LrEc4Ki4lQ9gAEIV5+t9rzIZxTi6hUGdGRuQaZ1l815ti14aEee1tUlODAmNUSWdNYt/KlCtzk1doapLXsVC2MZeusDmjedVYAJhWet4VEBsThHJh/IsQvGe7i1uC5bKz+0LrpSLUFVewehXOpHKJ9v6kpZ5aEITKL2xTeOwy+csWJ7arRkVMOv4zAQBRYfi1DpgwiD3U2X0Hlf2fX8plfBVecvwsvzgl/7cwRvJTMrevHCnLI8VFI0aurVV+IZAGhqvXGGtkMxmF+fykRbe9hS3FHuz5dL8BYAaEujpgQldapRkXf+cfJUlDb3hVsS7dbKQo1Dk/gl+JOMOWkfzWZjlrZBrS2RvmnEv2no7Nxxz3M25zWZnjlss/2KvC6IrsZbXKvwAAIABJREFU8cyCVuUbGmL/vQCAJBHq6oGLufBwEMjf+UcFixfNPtBJqZlTJ2uUt1au0jf6OL8Xa6XFi1TIZOScki2hUibnJJNgD1imc5EUhsrvPVR6znieCk7KZCZphzVyMgQ6ADUnJV0kiJOJeejV9A9IqmB/V2tV2narilxke37G930YYxSoONSZxkBXkssmFuyc7x01j9E0FOd1usl6DE31f9YAIA6EuhqXROFhOxrqwmWdc/p6Y21xWLFrsWy2/FIokYpy0nxcecGY4l6laRmunkaQ5CIEY+VkZDK+bDYjk/GlthaFoZPxPHmtTTJjPXbGFMvWWObSAag9DL/WuIxnY59Pbw6PhrrZFBweb2xT9ZYWmUP90kg+qqaVz0bTe1PsPIvhFTdG/qJWqaVZeWvTv9mBMckt5Bi9b97SDikMFe4/pMJoMet8Niu7dLGsC2XyBTljFGazxdCZ9tcQAGaJUFfDjJFUCBIZfnUZX8rOspTJKGdscRN1a6RMdX4kjTFyUXXUebY09BhFL6nfsUBqaZJGRjTSkEtNGZPpOGtkjYk2NxmjzIJmuVxOhT0Higl6tOfSeLb42Eh2UZtCa6XRBUKhscWafrlxP1vz4DUEgNlK/zgO5ixjJJPPSzGHuszvn5Wb60pMo1KvjjNGpqlBNleFIVhrIhuSC1qaZV7UJe/YZdEMJ2czKjQ2StabF4FOkmSsbC4r2zC3oH/0+Yxsxi8OPTc3FEOcJOt7CnPZ4v0bHfJ1TQ0KWAQBoA4R6mqYGR6RGRhU0D8Y3zV6+4sP5hocR3vonJ9REIQKWptlF7VF18Bym+F5ke0/W3BS3vOVz2aL87si4JyrWi/mXITOKVi6WGZpRyRz6/y2ZnktjcXi0J5/5JzGKDRG0mios7ZYzBoA6tD8eZfArLnQqbDnYKzX8HbtlSSNnPKnczvBaO/KiJPknEJV538aprVZYRzdYF5l343NZkoLI1w2M2+GDZ2TCjLKWFscVg8qa7gZnTfnfE/yrGzGUxiGsk0NCqViT6+1Kvh+8eLz5HUCgCgR6mqYcQlsDbZrr1wuq3Dp4jl9fXEumznquaQZP6ahzal6/6yR15CTC8Ji8AuLW7m5kYl79NqGrORZhaHTSNq2kCiDM6Z4jys90dhq2lxWzhjZpYvlBYEKjY0KAid/9DjDrgDqGaGuRllrpMPxDbtKkoZH5O3Zr8Ixy+ZeZmN0Lt3E56oQXirsUZuMc5LxTDEwvmDBhPV9maUdCr1iz5OT5A8OqfDsroknMZoXJUymZK1sQ1bBwJAUOhlvtLjzbBO0LQ6xBp6nQiGU/Ezxz1iI87xiTToyHYA6RqirYS7m0iCZ3+6UJIXtC+d+kipvqm6yvoy1kc2nm8hJDQ3yljeq8NweKZjYc1rwfQUy0ujT3thQ5fjepnlSl24qgbXS8qXy+/pV6Nknr3OR1JCTnu+TGx6RC51cIZALghkC2fQ/JyPZ7PxZRAIAMSHU1ShjVBzai5E91CtJKrzk+Dmfw5Qmuo8/cUIhz1rZFUsVZHyFspFvpeacNNLUJM+F8he2SnIK+wblnJPXlCuV3Sh9vjHF4rm+J5PNSLmcJJdcvbcYFOfWjc4NNEbyPA1nsvKWLJYZfb1tIS8znFew98DUJ7JHFkZMdR0AqHeEuhplnCb2+ETMHjgkb9deBYsWVNbTNtZTV4035dGh38AZxdUA55wKMgoXt0uS7MKCrJxC6x11e0Lfl12+RC6bUWH0mLVGNl/QvGdtcVeH0XA2Ye6bl5FpzijrWamv7+gvbcimY29gAEg5Ql2N8vJ5BTHWp7MHi710+ZevrOxEY0WHx6+OTPANPKksObZ5fDi+3MYLupcCGQWeP+G1cE6yNRBowkxG3jT/AXBOkufJrFgi3z+owr7nx7552a5OBQ052QPPE+4AYBrzd7IOpuUKgVyMPTymf0DOWoUdFcynU3Hf16OGzhJ74zaaF2ObNbD5fMFJ8r3pQ7RnZdta5NpaSiVMjDEKJeULoYK2lqOH6gEAJYS6GmXCyrenmvb8hw/LtTRVHsD8o38EExuJnSeZbp40cma5jMw0iz7caHHl0Kn0c+W1NRVXtapY946KJQAwNUJdrYp7kUT/gMKW5orPY3x/kknuNRJiolIjv6VBLif5U68yzo8NO4/OszSelRYtLJZ9AQDMqEbeLjBeceVrjD11zhWHX1uaKj9Xc2NxC6wXSmwENu0B0hU3o68BYeimLR1T+jEYvwWYYl3vAwA1pTbeLfACRirEuEhi7wGZMFS4sLXicx1VeFgqliQb3Q6q3jlXnE9WC5xTWTX3nIq7UGiS3UYAAFNj9WsNMkZSjD11Y/XpgqWdlZ3ITDGpzffkrVgqhaGC7r1H7cYQHSM3D4Z6J+3JnIecc0eGWKdjJZvLyAUBCyMAYBYIdTXIhqHccHy7Sdjn++R8X8pmKjvRFJluxPqSlfwwiHd4dN4slKgd5eTTUFbBsk55+bycrVINQwCYhxjfqkEmCOQK8ZQzsd175D/znArHdlUcuIy10w6vOWtLpS1QP8YKNg9nsqOFoQEA5SDU1SAThsVN02OQ+c3v5CTlT35pReexuYz8tubp91y1VibulY9kBgBAjSDU1aLQSS6GyfXOyR7sVeHE46VctrJzWSu3sK244fsUgtDJLGip7DozItUBAGoDoa4WuTCWeUhmcKjYC9haeX06SSpkszPOsXKZTHy5a2zfWQAAagChrgaZmFZLmv4BSVIYQX0649nyVnXaeIKX8T2Z9gXMwQcA1IxUhbqbbrpJq1ev1kknnaSnnnqq9PzTTz+t9evXa82aNVq/fr127txZ1rF6lAsKUkwlQMZCXSRFh8sMas6M1iyLmG3MyXYsVEhlWwBAjUhVqHvDG96gzZs3a8WKFROe//jHP64NGzbo3nvv1YYNG3TttdeWdawuDY9Ig0OxnNrbs08ul5Vraqz8ZGWuanXWllWwdrZMpeVYAABImVSFulWrVqmrq2vCc/v379f27dt1zjnnSJLOOeccbd++XQcOHJj2WD2y1khhqMLzh+M5f2+/go5F0QyHlnsOY6KdU2eM/CWLpNgXYAAAkKzUFx/u6enR0qVL5Y2WtvA8T0uWLFFPT4+cc1Mea29vL/saHR3xv8F3dla+pdZMXBAoHOiXFkTQkzaJ/NCw/GOWqDGC85vWRtkyXndXCBQeapTyEdXds0a2q0Omufg9JHFfMDvck3TivqQP9ySdqnlfUh/qkrB/f3+sc6s6O1u1d29fbOcf44eBtOugXFQBaLx8QU0jeQ15GRWeH6z4dJ6XUX5f/4yLJTxrZPuHFQ6NVHxNqbhAw/QOKj9QSOy+oHzck3TivqQP9ySd4r4v1pppO6JSH+q6urq0e/duBUEgz/MUBIH27Nmjrq4uOeemPFaPjHOx7RNqDz4vSXKNDZGcr9zFD06SXdAqmX6Fg8ORXBsAgFqUqjl1k+no6NDKlSu1detWSdLWrVu1cuVKtbe3T3usHsW5k4T/252SpLAtmqFqZ43KKaYXhk6FRQtkIwqT1KYDANSqVPXUXX/99brvvvu0b98+vfOd79TChQt199136xOf+IQ2btyo2267TW1tbbrppptKXzPdsbrjnBTGsJOERveTzWXlFrZFdcayPzMMQznPyvieXATlWihiAgCoRcbFNV43j9TKnLrc4IAKf9wTy7kbvvOgwsXtGjnj5ZGcz1vSrvzCthl3lBiTGx4qbn/W2y9npGCOK3yN70kv6lLB85mTkkLck3TivqQP9ySdmFOHyETRizWpIJAdGFIhiqLDY6wpO9BJ0khDozK2WF/ODI9Icy3bYsTwKwCgJqV+Th3KY4ykIKadJIaKCxSiWiQhSWaWBYWdcyo4KZ/xiwWJ5xzMCHQAgNpEqKsZRirEM59OI/ni37lsdOf0Zv+jF4ZOYSi5XEb+8s5YdpoAAGC+4l2xRhgjqRBDfTqpONwpyWWjC3XOzP1HL288BbmsTJlbjU3A8CsAoEYR6mqEkeIpOizJHjgkSXIN0YQ6k/GLCxYqOgmlSQAAGI9QVyNMEMjFVM4ku32HJMk1R7NQwnhWQTZT0TmctTJzGMI1MnKEQQBADSLU1QDPM/KGh+NZ/Tq6+KKwfEl0PWPWVlwrzlkj21Tewg3jWdnGXLGHMFNhDyEAAClFSZN5zhjJC0O5g73FOm4R83r2SpKCF62I7JzGs7MqZzIZ5yTnlRfQTDYju7RDgefJOVfsqav76owAgFpDqJv3jExvn4LhaDa8P+rsvcUiikFXZ3Qn9Sv/sXNO5a+gdU6FXE7BWOgl0AEAahChbh4zxshaIzc0IpePp0ZdZvuOYl24MnvFyuJZRbGRyYy17ozkNTfKZjOKqSwzAACpQaibx3wrmaAQW9FhBYGMpLDCRQ1Hiaq+3GQ9dXZ0aNU5Gc+TujrlhoYrHu4FACDtWCgxj7lCIDMwGMtcOkky/QOSpPzL/r9oz+t7kYQs53lHLd7wly2W1zq6StcayVgpm4mkZxAAgDSjp24+C53CfYcUxlSfrlR0uCEX3UmtkYuo5895nvxFrSoc6C2eOpeVclkZSZnRawSelYwvxbTZBgAAaUGom8esQoWFIL6eutEdKlwmuh8TY21xjl4E8tZTdkGrdKhPNuPLHLtUBetJ2az8wSGFjTkVAifPs1JMNfwAAEgLQt08ZoZG5II5hpUgkLdrn1xDTmHHwsk/Z6wHMMJQJxNt8d/Q8+QvXSz5ViPWLw6zOiebyyqQkXNOhQJDrwCA2keom6eM0dwDnaTML36lzO//IEkauPCNk18jn5cUbU9d1HuvFoxV0NpcnKM3bt5cXia2HkwAANKIUDefhXNf9eodeP7I4+7dCpYvHT1nqMz2HTLP98nv2VMs6ZaJbvWrMUahjbb4L2sgAABg9eu8Za2V0dx7vFwuW3rs73hmwuPMr38nv2ePpGLHWmQlSKRivbsozwcAACQR6uYtz4UyFdSnM8PDCpZ1qnDcCpm+wzL9h+U9+5yyT/xaklQ4tkuSFHS2R9Le0nUzPj1rAADEgOHXeSosBAr7Bub89ebwkIL2RVLGlx0cUvbx7fJ275MkOWM0csbLNXL6yyKd/yZJItQBABALQt08ZZybe0HdfF4mn5drbizVjBsLdJKUf/nKYpiLOtBJkudRCBgAgBgw/DpPVRLqzOFBSZJrblLwouWlodYxwfIlFbdvymv7Ee4hCwAASuipm4c8I5l8Yc4lO+zh4rBt2NwoeZ5GzjxFI6e/TObwoFxDNtLVrkeZbL9WAABQMULdPOSNjCjcd3DOtTzG99SVWCvX2hxF86bFwCsAAPGg22Q+CkOFQyNz/nJzeKBYUDiiPVhnd/EY5ukBAABC3XxkKlxoYJ/vV9jaElFrZoE8BwBAbAh181GF21+Z/sNybfEPtU5yZZHsAACIB6FuPgrnvuerJJmRvFw2O/MnRo1MBwBAbAh184wxkqsk1AWBTBiW6tMBAIDaQKibjyoIdWYkL0lVCnV01QEAEBdC3bxjpEIFoW5gqPigIRdRe2ZxbSM5Mh0AALEg1M0zxkgqFOb+9f2HJUlhAjXpAABAcgh184wxkgvm3lNnn++Ts2Zi4eGkxLWfLAAAINTNO6GTKgl1vf1yrS2SrcKtN5Ij1AEAEAtC3XwTujlvDyZJ5vk+hQtaI2zQrK5epesCAFD7CHXzjDWSm2uoc05mcEiuuTHaRpXJGCNn+JEDACAOvMMmxFpT8XQyY4zswKBcEMztBMMjMpJcLvmVr5KKHXX8xAEAEAveYhPiWyPPm/vLba1R9nC/gt3757xNmBkekSS5hirsJiGNLpJgCBYAgDgQ6pJSwTw4SfLzIwp2H5ArzLGXTpI9PFBsSmNDRW2ZM2OK48cAACByhLqkVLK4wRSLBlcS6CTJ9BdDXbVq1BnfqzTbAgCAKRDqkjLHIdMxldSmG2P6Dhe3B8tWafjV9wl1AADEhFCXEOPCygJNBGnI9h+u7k4SvpVEqgMAIA6EuqRU1FNnpDCinrqW6oU649FTBwBAXAh1ialsTl2lw7cqFGSHhqvbU5fxqndtAABqHKEuKZX2ULnKeupM3+HiaaoY6lw1tiYDAKBO8C6blArGHX1VXt3N9hdDXVit4Vdq1AEAECtCXcoZU9yvtdA3UNl5+g7LSXItTdE0bNYNEJkOAIAYEeoSNfveukyhoGD/oYrn1JnBYSmXlbzqzGsr7vtKqgMAIC6EuhSz1kiHeuXyhYrPZUZG5HJVqk8njXY5EuoAAIiLX+0G1JVZzquzQaCwv7JhV0nydv5R/nO7Kz5PRYyRY/wVAIDY0FOXoNlGGjuSl8tXtjWYJOUee1JSdVef2lxGzqekCQAAcSHUJWmW0+JMPl/5ThLjvn7oDa+q7FyV8H3m1AEAECNCXUoZYyKZS6ehYUlS/qV/IregtfLzzZXvybGdBAAAsSHUJcVpVj11xkgayVd8Wft8nyQp6Oqs+FwVqdKqWwAA6gWhLlHlpzrjnFyEoS5c0FbxuebWgOKQq/E99n0FACBG065+3bBhg0wZ86A2b94cWYNqmlHZuc4WCnKFyhdJ2Of7FDY2SNlMxeea9bWzGXkLW4oLNDz+/wAAQJymDXVvfetbS4+fffZZffOb39S6deu0fPlydXd3a8uWLXrLW94SeyNrxmyGX/N5hRGEOjMwJNfcWPF55sJb1KqgY6GUD2TDyr8XAAAwtWlD3bp160qPL7roIm3atEkveclLSs+de+65uuaaa/Te9743vhbWjLFJdTP3fBojaXgkkqua4WGFbckvkDCeVdjUqHw+lGQkS0lEAADiVPaY2O9+9zu96EUvmvDcMccco9///veRN2q2nn76aa1fv15r1qzR+vXrtXPnzmo3aXJl9tQZY6ILdUMjcg3J7iRhMr685Z0KPYIcAABJKTvUnX766dq4caN27typoaEhPf300/roRz+qVatWxdm+snz84x/Xhg0bdO+992rDhg269tprq92kihhJbrjyRRIKQ5l8PvHtwYy1KjQ2qkBdOgAAElN2qLvxxhslSeecc45OPfVUnXvuuXLO6YYbboitceXYv3+/tm/frnPOOafUvu3bt+vAgQNVbddRZrPyMwylsPKlomasty+Xq/hcs2JNadUrAABIRlnjY0EQ6Ctf+YpuvPFG3XzzzTpw4IDa29tlq7jt1Jienh4tXbpU3mgdNM/ztGTJEvX09Ki9vb2sc3R0tMTZRElSS2uDTENGpoyA5QaHFbZkpaCy4UuXH1ZBUmN7i+yCBBdLtDTKa4//NY1CZ2cVCzJjUtyTdOK+pA/3JJ2qeV/KSg2e5+m//uu/dNVVV8laq8WLF8fdrkTt39+vMIKesal0draqv29QwWBeBW/muXLZkREFBwcq3iLM7u9Vg6TDgVH4/GBF55oNv7lFI/v6U7+DRGdnq/bu7at2MzAO9ySduC/pwz1Jp7jvi7Vm2o6osrvaLrjgAn31q1+NpFFR6urq0u7duxUExZIZQRBoz5496urqqnLLKhAGle/5qiPDr0kvlJA1qQ90AADUmrLH95544gndcccd2rRpk5YtWzahKHE1iw93dHRo5cqV2rp1q84//3xt3bpVK1euLHvoNUnlzjIzQRjN9Ub3fXWJz6mr/rA8AAD1puxQd9FFF+miiy6Ksy1z9olPfEIbN27Ubbfdpra2Nt10003VbtLkyui9MkZyUYW64ZHibg5+svuuurLjKwAAiErZoW58IeK0OeGEE/SNb3yj2s2IhO9ZmYiGLs3QsFxDbrSacUKMyu+SBAAAkZnV8sp9+/bpiSee0MGDByfMmbrwwgsjb1gtKifr2MGh6ELd8EjiNepIdQAAVEfZoe6///u/dfXVV+u4447Tjh07dOKJJ+q3v/2tXvGKVxDqyjVDWCvuJDEsBdHsk2r6DytctDCSc5V/UZHpAACogrJD3ac//WndcMMNeuMb36jTTz9dW7Zs0Te/+U3t2LEjzvbVheKiE6eMZ+T6BxUMDFV+0kJB9vCgCscfU/m5ymWN/LYWhT7bgwEAkLSylyl2d3frjW9844Tn1q1bpy1btkTeqJo1RUddxpN835MZHpYbyUeyUML29hcv2ZZcEWCvY6GCJR3KG1a/AgCQtLLffTs6OrRv3z5J0ooVK/T444/r2WefVRhGs1KzrhWC4lqGwWG5iIogm4FiseGwuSmS85V1zYasCpSnAwCgKsoOdW9961v1P//zP5Kkd7zjHfqrv/ornX/++Xrb294WW+Nqz+SJxwShlC9IwyNy+UIkVyoVHk5ooYTJ+gqTrocHAABKyp78dMUVV5QeX3DBBTrjjDM0ODioE044IZaG1aQperGcczJDI3LD+eiuNRrqlFSos1ah50kxbrcGAACmVnZP3fe+9z319vaWPl6+fDmBLiphWFzxGuHWWmYkL5fxE9vdwfjeVJkVAAAkoOyeuv/4j//QBz/4QR133HE6/fTTdcYZZ2jVqlWp3I4rlZybutRHGBZrykVYCiTxGnWZTHLXAgAARyk71G3evFnDw8P6+c9/rm3btmnz5s368Ic/rBUrVmjr1q1xtrFmTJXZTCFQ/mBvpEOXiYc6304oSA0AAJI1q7G5IAiUz+c1MjKi4eFhtba26sUvfnFcbas9k4QeYyRXCKQgjHb4dXBILsGFC8b3o2w+AACYpbJ76i688ELt3btXr3jFK3TGGWfo+uuv14knnhhn22rLFInH2uj2ei3JF2T6Dis8dnm0552G873ErgUAAI5Wdk9da2urCoWCent7S38KhWjKb9SNSbKbZ9zojhLRMcPDMpJcc2Ok552KzWZkEt9jFgAAjFd2T92Xv/xlFQoF/e///q+2bdumL37xi3ryySf1kpe8RLfffnuMTawhk/XIFQLJRVvA2YwUS6O4bEKLF6xRwfMpZwIAQBXNak5df3+/9u7dq127dqm7u1u9vb0aGopgn9I6ZvKFyMOQ3X+o+CChFam2uZFyJgAAVFnZPXXnnnuunn32Wf35n/+5Vq1apY0bN+rUU09VY2MyQ3w1YbKeujBUcDjaYJzZvqN4uYaEFko0NSqklw4AgKoqO9R97GMf0ymnnKIcW0HN2aQlP8JQ4djuDxEJW5tkD/bKtSSz76uLeE4gAACYvbKHX88880wNDAxoy5Yt+vd//3dJ0u7du7Vr167YGldTnCbfJiyGHi4zPKLgmGWRn3fyi5niHwAAUFVlh7qf/vSnWrt2re666y7ddtttkqRnnnlGn/jEJ+JqW+1x7qj848JoF0nIuWKNusaGaM87BduQlbxktiIDAABTK/vd+IYbbtCnP/1pbdq0Sb5fHLV9+ctfrieeeCK2xtWcF6xyNUaRFhyWJI2MyIQuuVDX2qwCW4QBAFB1ZYe65557Tq985SslqVRXLZPJKAiCeFpWa5yTAqejNguLePjVDA4XL5dQqFPGZ5EEAAApUHaoO+GEE/TDH/5wwnMPP/ywXvrSl0beqJoVhkdPP4u4p84MFFfSJhHqbGMusQLHAABgemWvft24caOuvPJKve51r9PQ0JCuvfZaPfDAA6X5dSjDUfPnzCTPVcYOjYW6+Fcpm6yvgrHRDyEDAIBZK7un7pRTTtG3v/1tnXjiiXrLW96iY445Rrfccou+9KUvxdm+2hIEEwZfjZEURLybxMBQcaFtEjXqPJ88BwBASszYUzc4OKgvfOEL+vWvf63jjjtOV111lQ4cOKCbbrpJn//853XBBRck0c6aEI4UZPJ5yRv3sheinZNohoaKvXQ2gRWpGU+T12kBAABJmzHUXXfdddq+fbte85rX6KGHHtJTTz2l3//+91q3bp3+/u//Xu3t7Um0szY4N7GnTpKLeKGJGRiSa0hmkYTx6akDACAtZgx1P/zhD/Wtb31LHR0devvb367Xve51+s///E+dfvrpSbSvdjh35E/puTD6hRKDQwrbWiI951Sc7yVyHQAAMLMZx+gGBgbU0dEhSVq2bJmampoIdHN0VHyLoZfLDA4nU87Es8kM8QIAgLLM2FMXBIEeeeSRCfuWvvDjsfp1mIGTzLjXzTg3+X6wc5UvyBQKyZQzyWYU0lMHAEBqzBjqOjo6dM0115Q+Xrhw4YSPjTH63ve+F0/ras4LApxzkfbWebv2Fk+bRDmThpwC68Wydy0AAJi9GUPdAw88kEQ76oPTxBDnpChTnb9jp5yksH1hZOecjPE9mYZstL2MAACgImUXH0YUJi6UMAqjWycxPCJv/yHlT3iRXEtzRCedhDGyXZ3KNzaw8hUAgBRhpntC3GRDrU6RrX71du+TJIUdMffSZTyF2QyjrgAApAyhLmnuyA4SRtENvpr+w3KSghXLIjrjFNfxPDmfDl4AANKGUJcwM6FOXXQLJWz/QHHVqxfvilSTzbCHBAAAKUSoS9pR45bRRCTTPxDvXLqx67Q0KWTsFQCA1CHUJcyF4bgPFGFP3WGFLU3RnGzKixiGXgEASClCXdJCJ1PaADaiRDeSlxnJy8Uc6oy1ch4/MgAApBHv0HELArlDfVIhKH7sXtBTF0Gws8/3SpLCBa0Vn2v6Cxk5tgYDACCVeIeOmf+b32v4us/L7nim+ETgVFz3OiqCzjp7aDTULWyr/GTT8BqycjEvxAAAAHNDqIvb6Fhr5r8fLn78wtWvlXJOXs9ehQ05qSHm7cFamhVScRgAgFQi1MUsOG6FJClsX1B8IgxLc+qMmeKLZsEc6pW3Z3/pOrExksv47CIBAEBKEepi5poapbYWucXto0+Mn1NXeULKbXtCklQ4/piKzzUdYy3z6QAASDHepZNgJGNHu+XCCIdfg0C2t1/B4kWxr3yVZ5lPBwBAilF0LAnGyI2GOjc+1FVSxHdwSA0/3CZptJduLmO51pTdBmOMwijGiwEAQCzoqUvKWHgaN6eukp46b+8B2d7+4mnmuJNEpr1NXnNDeZ9sTDSPzdm5AAAgAElEQVSTAAEAQCwIdUkwLyhhMhrwXAWhzgwMSpIKf3KMwo6FczqHy2ZlmxrLvKAp9uwBAIBUItQlxY174EZ3lagg1NkDh+Qacho57c/n3oNmjFxzY1lhzXiWla8AAKQYoS4BppjgJBV758xYOqok1B3sVbCko8KGSc5amTJWtZpcllAHAECKEeqS5nQkzM01JTknOzhULJdSSVOMUSHjy2ubYU6eNVJLU0XDxQAAIF6EuiQYMzHIueJWYWaOq1/9X+0onqqxwh0kjFUYSmqcfrGE19SoIJut7FoAACBWhLqkjRt+nUvPl+3Zo+z2YqgrHLt87u2wRxY+OH+G+nONWQWGHxUAANKMd+okHJlSJ+ck4zTnhRLe3gNHPshm5t4kz1NYKiY8TbkSa2UaGxl6BQAg5Qh1SRi3UGJ0Ul3x4RyGX81IPpomjd8hwoz+mYTNeCo0VDjMCwAAYkeoS1qFCyXG6tMN/59TKmqGacjJjfbOuekKC1N0GACAeYFQlwRjxnXUuSNhbi49dYcHVTi2S8ExXZW1qSF7ZEjVmKk66iRrp+zFAwAA6UGoS8yRADfnOnXOyQwMVlzKREZS9kjdOSdN2RtnjKE+HQAA8wChLgnje+okKXSjYWl2ackMDsk4p7C50lBnJ+4iMTbEOlkRYoZeAQCYF1IR6r71rW/p3HPP1Z/+6Z/qjjvumHBscHBQ73vf+3T22Wdr7dq1evDBB8s6lmphWPx7tqHucHE+XaU9dcaoNJ9OkmQ9+QtalOlcKNvwgnp0HqEOAID5wK92AyRp5cqVuuWWW/TFL37xqGObNm1SS0uL7r//fu3cuVOXXHKJ7rvvPjU3N097LFXGFx+W5IJQRm72oW50kYRrboqmTaMKkkz7AtlCIPUelm3MyeayCvoOF3v1AABA6qXiHfulL32pTjzxRNlJhv+++93vav369ZKk448/XieffLIeeuihGY+l2lhP3SzZ5/uKi2ebpt8BYkbG6IVxMu+MCtmsjGfltTXLdS2Wt7xTNuMzpw4AgHkgFT110+nu7taKFStKH3d1dWnXrl0zHkuVF85LC8LigtLZpCXn5P2xR+HSxZI3ww4Q5bRnkrlyzjmZbEautVn5wMk2NsgPQ+moCAgAANImkVC3bt06dXd3T3rs4YcflldpSKlQR0dLrOcfNka+Z9WwYHQuXFNGdkGjwtYGqaG8WxD27FUwMCTvjD8/cp658j3ZhU0yuaP3cw0LwzIdrTKj98QtapHxUtGhG4vOztZqNwEvwD1JJ+5L+nBP0qma9yWRUHfnnXfO+WuXL1+u5557Tu3t7ZKknp4enXnmmTMem439+/sVzqFmXLnajBQUAh1+vjgnzitIahmU6x1SWOYOEdknd8jzPfUtXCSNnmeuTNaXnh9UwQxPfN4YZWU1fGCgovPPF52drdq7t6/azcA43JN04r6kD/ckneK+L9aaaTuiUt8Fs3btWn3961+XJO3cuVNPPvmkXvva1854LHUmLJQIJBfKlTusOTwi74+7VDj+WMmvvFfTGDNx9WupiU4j3tz3kwUAANWTilC3detWnXXWWbrnnnt066236qyzztKOHTskSZdffrl6e3t19tln68orr9R1112nlpaWGY+lygsDlHMyQVD2VDVv734Z5xQcsyya5jQ2yE1Wk06ade08AACQDqlYKHHOOefonHPOmfRYU1OT/vVf/3XWx1JnXFZyQSjlC2UvlPC698hlMwrbF0TSFNPSGOtwMwAASF4qeupqnjEan+pcOLtQZ/fsV7Csc/IdH+bQFlflhSkAACB6hLqkvHCbsEJQ3uirczLDI3KNFdamG2WsmXLoFQAAzF+8uydhsv1Tg6C8nrpCQcY5uWxECxiskWOXCAAAak4q5tTVh4kBLn+gt6yvMoPFsiNukppyc2KMZNnPFQCAWkOXTRKMmfOmDF73bklS2LEooqYYOUIdAAA1h1CXhOKeYHP6Um/PfoVtLXKtzdG0xfOiWXABAABShXf3RMyxp65QkN13QMHSzuhakssqDCM7HQAASAlCXRLmONrp7dkvEzoFyxZH15aMpzmPBQMAgNQi1CViYp26ctld++Q8T+HiaObTSZLxvHLL4wEAgHmEUJeEuWU6eXv3K+xsL86Di0oEe8cCAID0IdSl1Uhetu+wgo6FkZ3SZDyFmYjq3QEAgFShTl0SjCl7S7Ax9sAhSVLYHk2osw1ZmVxGBd9nSh0AADWIUJeAuayTsAeel5MUti+o+Po2l5HtbJfK3ZoMAADMOwy/JmEOxYdtb59cS5MUwXCpWdhW3JHCGhZJAABQowh1STCzXylh+w4rbG2p/NrWSI05hRJFhwEAqGG8yydhtpnOOZm+w5HsImGsVWit5HlyjbmKzwcAANKJOXWJmF2qM4cHZMJQYRRbgxkjWatC6GSMnVU7AADA/EFPXRJm2VNneg9Lklxb5cOvxho5W1yq4ZhQBwBAzSLUJWJ2619tX78kRddTZ7jNAADUOt7tk2Ck2XTV2b7DxdWq2WyE1wcAALWMOXUJCHc8K1sIpJG8lJ25RInp649m5askWUsZEwAA6gA9dUkoBJIke/D5mT/XOdnewwrbIhh6lWQM3XQAANQDQl2SyglYwyMy+Xwk5UzKviYAAJj3CHUpYw8PSJLC5ohCnSXUAQBQDwh1KWMGhyRJrqkhohMS6gAAqAeEuiSVEbDMAKEOAADMHqEuZczgkJznSZmZV8mWd0LD6lcAAOoAoS5JZXSamYGhYi9dVD1s9NQBAFAXCHWJKmP4dXBQrjGioVdJ8ma5RxkAAJiXCHWJmiFcFQryDjwf3Xw6SfI8hl8BAKgDhLokzZCucj/6n+KnRbU9mCTjeZGdCwAApBehLkkz9Jh5+w5Ef0mPWwwAQD3gHT9J0/XUheGRhwtbo7meMZLhFgMAUA94x0/SNKGu6f/dK0kqdC1RcOzyaK5njRw7SgAAUBcIdUkqY8VCsGJpZGVIjDFylDQBAKAuEOoSZKYKdeOGXl1zU3QXtEay3GIAAOoB7/hJmirUDY8c+ZSW6EKdMQy/AgBQLwh1SZoi1Jnxoa4hF931WCgBAEDd4B0/AZlL3lx8MEmms/sOqvG/fyxJGvnTl0S6rZfxPPaSAACgThDqEmCOXVZ8MElPXfaxJ0uPg+NWRHthhl4BAKgbhLokjA2BThLqxubQBR2L5Jobo70uiyQAAKgbvOsnYazHbLJQl8vKZXwNv/7/xHBdy76vAADUCUJdAoyduqdOI3m5poh76ErXZfgVAIB6QahLwmi4MuHRoc6M5OWymVguWyxnQlcdAAD1gFCXhMaG4t/5/FGHzMiIXDYbz3WtF895AQBA6hDqEmCyGTnfkxmeLNTF11NnPObUAQBQLwh1SWlskBkZmficc9JIXoop1Mnj9gIAUC9410+Iy2akfGHCc2ZgUMa5+ObURVjIGAAApBuhLim+J3vgkFQ4Euxy33+0+CCOenKWLcIAAKgnvOsnJQhlh4aV3fZE6Sk7OFR8MMkCikoZa+UYfgUAoG7wrp8QE4SSJG//odJzweJFxb9ftDz6C1ojx44SAADUDd71kxIWQ934oOUyGYULWuVamiO/nDFmtE4dAACoB4S6pARB8e9xQ6ImX5DL+PFcz1rm1AEAUEd4109KYTTUjeupM4W8FFOo89qaJFa/AgBQNwh1CTFDw5IkN7a7hCTlC3KZGMqZGMk1NCoYnccHAABqH6EuIYU3niVJE1akxjb8ai3z6QAAqDOEuoSEJ/2JgkVtpVWwcq5YjNiPPtQZYxQy9AoAQF0h1CXJ94/UpAvC4m4SsfTUmXgKGgMAgNRKxTv/Jz/5Sa1du1bnnXeeLr74Yj355JOlY/v27dNll12mNWvW6LzzztMvfvGLso6lkWtulD08WPxgbGeJOHrqrKVGHQAAdSYV7/xnnXWW7rrrLn3729/WlVdeqfe///2lYzfffLNWrVqle++9V9dee62uvvpqOedmPJZGrrGxuGDCOZnR1bDO9yK/jsll2PcVAIA6k4pQ9/rXv16Z0VWgp5xyinbt2qVwtFjvPffco4svvliStGrVKmWz2VJP3nTHUseYIwGuEMTaUyfPV4qzLQAAiEEqQt14mzdv1ute9zpZa3Xw4EE559Te3l463tXVpV27dk17LLXGQl0QyATx9dTJM5JIdQAA1JOYtjOYaN26deru7p702MMPPyzPKwabu+++W3fddZc2b96cRLNKOjpaYr9GS0uDgpZGBZLamjJyBU+BpOYFTbILGiO9llnQLLu4NdJz1qrOTl6ntOGepBP3JX24J+lUzfuSSKi78847Z/yc+++/X7fccotuv/12LV68WJK0aFFxw/sDBw6UeuR6enq0bNmyaY/N1v79/QrD+Hq2Ojtb1d8/JDcSKCep7+Bh2f4B5ST1Dwdyzw9Gej2/Ja/hvX2RnrMWdXa2ai+vU6pwT9KJ+5I+3JN0ivu+WGum7YhKxfDrgw8+qH/8x3/Upk2bdMwxx0w4tnbtWn3ta1+TJD322GMaGhrSySefPOOxNBobajVBcGTbsIiHX41npYZspOcEAADpl0hP3Uw+8pGPKJPJ6L3vfW/pudtvv12LFi3SBz/4QV199dXasmWLcrmcPvWpT8mOluuY7lgqeUcWSpRWv3oRz6kzRgHT6QAAqDupCHWPPPLIlMc6Ozt1++23z/pYKo1tERaG8a1+Nab4BwAA1JUUd2vVnrGCwCYMSz11UQ+/yohQBwBAHSLUJWlsaDgIpSAoDr1GHMCMMRQeBgCgDhHqkvTC4dc4dpPI+PTUAQBQhwh1SbJHQp0pBJEXHjaelToXKYz0rAAAYD4g1CVo/Jw6FYLIF0nY5iYVMlm2CAMAoA4R6pI0oaeuEHk5E9PWpBhrKAMAgBQj1CXJG7dQohBEOqfOZH0FDQ2RnQ8AAMwvhLokje+pCwK5CIdfbUNOYdSFjAEAwLxBqEvSaLkRE7rIV7+a5sZY968FAADpRqhLmu9JhUK0q189K8d+rwAA1DVCXcJcNiMzko909avNZhRkCHUAANQzQl3CXDYrMzwiEwRSRHPgTGsztekAAKhzhLqkZTMyA4OSFMnwq8n4ci1N1KYDAKDOEeqSMrp1V9jaLNt3uPhchaHOZHz57W0KMplKWwcAAOa5aLc0wIzCBa2lx3MtaeI1NchkPMlYhQsXsOoVAAAQ6pLmxoW6ufbU2ebG4rlyGY0Q6AAAgBh+TZwbN1Q6123CXNaXrNhBAgAAlBDqkmbHveRz6KkzWV+uuVkul1Ng2UECAAAUEeoS5qw58ngOPXW2sUGBtXKNDXIseQUAAKMIdUnzxvfUzX5Ko2luVBCEKsjM/MkAAKBuEOqSVsnwq7Vy2eKcPFa8AgCA8Qh1SRsX6mZbfNj6nsKIthYDAAC1hVCXNDNu2HSWc+pM1lcY0dZiAACgthDqkjY+1NlZvvxNLI4AAACTI9TNJw059ngFAACTItTNE8b35rytGAAAqH2khCooHNsl19I8q68x3ugiCXrqAADAJAh1VTBy5imz/hrTkJUzRoy/AgCAyTD8mpCKs1hDlkUSAABgSoS6+cAaKcciCQAAMDVCXVIq2NXL5rIKG3PRtQUAANQcQt08YBobFDj2egUAAFMj1M0Dhvl0AABgBoS6tDNGLsMiZQAAMD1CXcoZz7LfKwAAmBGhLuWM58n5hDoAADA9Ql3KmVxGznCbAADA9EgLaZdjkQQAAJgZoS7lTC5L0WEAADAjQl2aWcN8OgAAUBZCXYoZz2PlKwAAKAuhLjGz3xHCeFaOUAcAAMpAqEsxk/XlDNuDAQCAmRHq0izLylcAAFAeQl2KmWyGla8AAKAshLq0MpLz2fMVAACUh1CXUsZaOY/bAwAAykNqSCtWvgIAgFkg1KVUsaeOUAcAAMpDqEspk82INRIAAKBchLqkzLbcXDZDORMAAFA2Ql1KUc4EAADMBqEupShnAgAAZoNQl0bWFv8AAACUieSQQsazCqlRBwAAZoHkkELGGgoPAwCAWSE5JGUWix6M78kZbg0AACgfySGNMplqtwAAAMwzhLo0yvrUqAMAALOSilD3+c9/Xueee64uuOACnX/++frOd75TOjY4OKj3ve99Ovvss7V27Vo9+OCDZR2bz0yGGnUAAGB2UlEM7dJLL9W73vUuSdLu3bv1xje+Ua9+9au1YMECbdq0SS0tLbr//vu1c+dOXXLJJbrvvvvU3Nw87bH5zPns+QoAAGYnFT11ra2tpccDAwMyxigMQ0nSd7/7Xa1fv16SdPzxx+vkk0/WQw89NOOxVCpjqzCbzVCjDgAAzFoqeuok6atf/aq+8pWvaNeuXbrhhhu0aNEiSVJ3d7dWrFhR+ryuri7t2rVrxmOz0dHRUmHrZ9ba1iDX1jjzJy5olW1vkcllY28TpM7O1pk/CYninqQT9yV9uCfpVM37kkioW7dunbq7uyc99vDDD8vzPL3tbW/T2972Nv3mN7/Rhz70Ib3yla8sBbu47d/frzCMbxJbZ2er+nqHFPQOzljaxG9ukTs8opHe4djag6LOzlbt3dtX7WZgHO5JOnFf0od7kk5x3xdrzbQdUYmEujvvvLPszz3ppJO0ZMkS/fSnP9WaNWu0fPlyPffcc2pvb5ck9fT06Mwzz5SkaY/NW76vfIwBEwAA1KZUTN7asWNH6fEf/vAH/epXv9KJJ54oSVq7dq2+/vWvS5J27typJ598Uq997WtnPDbvGCmzqFXOGla+AgCAWUvFnLrPfOYz2rFjh3zfl+d5+tjHPqYTTjhBknT55Zdr48aNOvvss2Wt1XXXXaeWlpYZj803xlrJ8xSySAIAAMyBcVS5TWRO3fN/3KfgDz1TzqkzvievY6GC1mYV2CIsEcxJSR/uSTpxX9KHe5JO1Z5TR3pIC2sk35PzqFEHAABmj1CXlBlq1BlrpWymrFp2AAAAL0SoSwnjWbmMzyIJAAAwJ4S6tMhkJM8j1AEAgDkh1CXEGCObyUz9Cbms8oET61YAAMBcEOoSZBum2PrLWqkpR6ADAABzRqhLAWONQsqYAACACpAkkmKMjGcn7a2zDTmJUiYAAKAChLqkjJYq8Zoajj7UmJXzuBUAAGDuSBKJGU11k2wD5qyNdUcLAABQ+wh1SRkrKmyPri5sKGUCAAAqRKhLiimGOTfZggiGXgEAQIVIE0mzxQUTJWaKoAcAADALpImEuNGeOmUyMv64la7GTDokCwAAMBuEusSMDr/mMhOKDBtjjgQ+AACAOSLUJc1aGZkJH7tJVsQCAADMBmkiKUbFoVZjND7TmbHnAAAAKkCoS5IdC3BHQpzX0sjqVwAAUDHSRFJGw5x7QU+dy2UVip46AABQGUJdkuzku0qwmwQAAKgUoS5Jxhy9cwRDrwAAIAIkigQVS5e44uKI0nPcAgAAUDkSRWLMkWHXsUxH4WEAABARQl1SjKSMP+4DHSlxAgAAUCFCXZLGQt1Yh50RhYcBAEAkSBRJGQ1wYeiO9M5ZyxZhAAAgEoS6JJUWRZgjfxHqAABABAh1SSplOjP6l5FjoQQAAIgAoS4xRjK2WKfOjF8owS0AAACVI1EkqBCEE5+wVuwlAQAAokCoS4oxcmPbSYwNv7KbBAAAiAipogpKayM8r6rtAAAAtYNQl5TSeggn+b5sQ1bGs0fvBQsAADAHhLqE5PPBkQBnrbzmRjnPk5hVBwAAIkCoS8hYoHNOUmNOLgzpqQMAAJEh1FWBa8zJBU7ymVMHAACiQairgkIhlCz7vgIAgOiQKqrEGMMWYQAAIDKEumqxVo5QBwAAIkKoqxJjjJwIdQAAIBqEuipwTpI1xT8AAAARINRVEQslAABAVEgV1cJCCQAAECFCXZU4Y1goAQAAIkOoqxaGXgEAQIRIFlXhWCQBAAAiRairFsu+rwAAIDqEumqxVhKpDgAARINQVy3G0FMHAAAiQ6irFo+XHgAARIdkUSUUHgYAAFEiWVSFoacOAABEimRRBc45FcJqtwIAANQSQl2VhCGrJAAAQHQIdQAAADWAUAcAAFADCHUAAAA1gFAHAABQA1IV6h599FGtXLlSd9xxR+m5ffv26bLLLtOaNWt03nnn6Re/+EVZxwAAAOpJakJdf3+//vmf/1lnnXXWhOdvvvlmrVq1Svfee6+uvfZaXX311XKj+2tNdwwAAKCepCbU3Xjjjbr88su1aNGiCc/fc889uvjiiyVJq1atUjab1ZNPPjnjMQAAgHriV7sBkvSDH/xAfX19Wrt2rb7//e+Xnj948KCcc2pvby8919XVpV27dunYY4+d8tjLXvayWV2/o6Ol4u9hJp2drbFfA7PHfUkf7kk6cV/Sh3uSTtW8L4mEunXr1qm7u3vSY/fcc49uvvlmffnLX06iKZPav78/1mLAnZ2t2ru3L7bzY264L+nDPUkn7kv6cE/SKe77Yq2ZtiMqkVB35513Tnnsscce0969e/XWt75VUrF37sEHH9ShQ4f0nve8R5J04MCBUo9cT0+Pli1bVhqmnewYAABAvan6nLpVq1bpJz/5iR544AE98MADWrNmja666qpSoFu7dq2+9rWvSSoGwKGhIZ188skzHgMAAKgnqZhTN50PfvCDuvrqq7Vlyxblcjl96lOfkrV2xmMAAAD1xDhqgDCnrk5xX9KHe5JO3Jf04Z6kU7Xn1NGtBQAAUAMIdQAAADWAUAcAAFADCHUAAAA1gFAHAABQAwh1AAAANYBQBwAAUAMIdQAAADUg9TtKJMFaUxPXwOxxX9KHe5JO3Jf04Z6kU5z3ZaZzs6MEAABADWD4FQAAoAYQ6gAAAGoAoQ4AAKAGEOoAAABqAKEOAACgBhDqAAAAagChDgAAoAYQ6gAAAGoAoQ4AAKAGEOpi9vTTT2v9+vVas2aN1q9fr507d1a7SfPWTTfdpNWrV+ukk07SU089VXp+utc46WP15uDBg/rbv/1brVmzRueee67e85736MCBA5Kkn//85zrvvPO0Zs0aXXbZZdq/f3/p65I+Vo/e/e5367zzztMFF1ygDRs26Fe/+pUkfl/S4LOf/eyEf8f4Xame1atXa+3atTr//PN1/vnn64c//KGkeXxPHGL19re/3W3ZssU559yWLVvc29/+9iq3aP7atm2b6+7udq9//evdb37zm9Lz073GSR+rNwcPHnSPPPJI6eMbb7zRfeQjH3FBELi//Mu/dNu2bXPOOfe5z33Obdy40TnnEj9Wr3p7e0uP77//fnfBBRc45/h9qbZf/vKX7vLLLy/9O8bvSnW98P3EueRf9yjvCaEuRvv27XOnnXaaKxQKzjnnCoWCO+2009z+/fur3LL5bfwv4XSvcdLH4Nw999zj/vqv/9r94he/cG9+85tLz+/fv9+dcsopzjmX+DE4d+edd7p169bx+1Jlw8PD7qKLLnJ/+MMfSv+O8btSXZOFuvl8T/y59e+hHD09PVq6dKk8z5MkeZ6nJUuWqKenR+3t7VVuXW2Y7jV2ziV6rN7vaRiG+upXv6rVq1erp6dHy5cvLx1rb29XGIY6dOhQ4scWLlwY83eeXh/96Ef14x//WM45felLX+L3pcpuvfVWnXfeeTrmmGNKz/G7Un0f+tCH5JzTaaedpg984APz+p4wpw5AJP7+7/9eTU1NuvTSS6vdFIz6h3/4B33/+9/X+9//fn3qU5+qdnPq2uOPP65f/vKX2rBhQ7WbgnE2b96sb3/72/rmN78p55yuu+66ajepIoS6GHV1dWn37t0KgkCSFASB9uzZo66uriq3rHZM9xonfaye3XTTTXrmmWf06U9/WtZadXV1qbu7u3T8wIEDstZq4cKFiR+DdMEFF+jRRx/VsmXL+H2pkm3btul3v/ud3vCGN2j16tXatWuXLr/8cj3zzDP8rlTR2M9iNpvVhg0b9LOf/Wxe//tFqItRR0eHVq5cqa1bt0qStm7dqpUrV9blsENcpnuNkz5Wr/7lX/5Fv/zlL/W5z31O2WxWknTyySdraGhIjz32mCTpa1/7mtauXVuVY/Xo8OHD6unpKX38wAMPaMGCBfy+VNEVV1yhH/3oR3rggQf0wAMPaNmyZdq0aZP+5m/+ht+VKhkYGFBfX58kyTmn73znO1q5cuW8/vfLOOfcnL4SZfnd736njRs3qre3V21tbbrpppv04he/uNrNmpeuv/563Xfffdq3b58WLVqkhQsX6u677572NU76WL357W9/q3POOUfHH3+8GhoaJEnHHHOMPve5z+lnP/uZPv7xj2t4eFgrVqzQP/3TP2nx4sWSlPixerNv3z69+93v1uDgoKy1WrBggT784Q/rz/7sz/h9SYnVq1fr3/7t3/TSl76U35Uq+cMf/qCrrrpKQRAoDEOdcMIJ+tjHPqYlS5bM23tCqAMAAKgBDL8CAADUAEIdAABADSDUAQAA1ABCHQAAQA0g1AEAANQAQh2AuvXmN79Zjz76aLWbAQCRYO9XADXr1FNPLT0eHBxUNpst7UP6yU9+UnfffXfibTrppJN033336bjjjkv82gBqG6EOQM16/PHHS49Xr16t66+/Xq961auq2CIAiA/DrwDq1urVq/Xwww9Lkj7zmc/ove99rz70oQ/p1FNP1bnnnqunn35aX/jCF/TKV75Sf/EXf6Ef/ehHpa/t6+vTNddco9e85jV67Wtfq1tuuaW0x+kzzzyjSy+9VKeddprOPPNMve9975MkXXLJJZKk888/X6eeeqq+853vSJIefPBBnX/++Vq1apUuvvhi/frXv57Qxi984Qt605vepNNPP10f+chHNDw8LKm4R+SVV16pVatW6YwzztCGDRsUhmH8LxyAVCLUAcCosXC1bds2rVy5UpdffrnCMNRDDz2k//t//6+uvfba0udu3LhRvu/rvvvu05YtW/TjH/9Y3/jGNyRJt956q1796ldr27Zteuihh3TppfuwVzEAAANaSURBVJdKkjZv3ixJ+ta3vqXHH39cb3rTm7R9+3Zdc801uu666/Too49q/fr1eve7362RkZHSte666y5t2rRJ999/v55++mnddtttkqQvf/nLWrp0qX7yk5/oxz/+sT7wgQ/IGJPUywUgZQh1ADBq1apVeu1rXyvf97V27VodPHhQV1xxhTKZjN70pjfpueeeU29vr/bt26cf/OAHuuaaa9TU1KSOjg694x3vKM3R831f3d3d2rNnj3K5nFatWjXlNb/+9a9r/fr1evnLXy7P87Ru3TplMhn9/Oc/L33OJZdcoq6uLi1cuFDvete7Jlxn79696u7uViaT0apVqwh1QB1jTh0AjOro6Cg9bmho0KJFi0oLKxoaGiRJAwMD2rNnjwqFgl7zmteUPj8MQ3V1dUmSrr76at1666268MILtWDBAr3zne/UhRdeOOk1u7u7tWXLFt1xxx2l5/L5vPbs2VP6eOy8krR8+fLSscsvv1yf/exnddlll0mS1q9fryuuuKKi1wDA/EWoA4BZWrZsmbLZrB555BH5/tH/jHZ2dur666+XJD322GN65zvfqdNPP33SFa9dXV36u7/7O73rXe+a8no9PT2lx93d3VqyZIkkqaWlRRs3btTGjRv11FNP/f/t3D2qYjEYxvFHbWzUQrE7S7CxUiztgq2FlSDYuQDB7hwQERfgCsQd2FrbiSuINgcLsYhiZW4hCBemmA+YmZv7/9UhJCnCw/uGqN/vq1arqdls/ukWAXxBtF8B4BdVq1W1Wi3NZjM55/R8PnU8HrXb7SRJm81GaZpKkkqlkjKZjLLZ13VbqVR0Op3ec3W7Xa3Xa+33e3nvdb/ftd1u5Zx7j1mtVkrTVNfrVcvlUsYYSa83gNZaee9VKBSUy+VovwLfGJU6APgN8/lci8VCxhjdbjdFUaThcChJOhwOmk6ncs6pXC5rMpkoiiJJ0mg00ng81uPxUBzHMsYoSRLFcSxrrfL5vOr1+qd3eJ1OR4PBQOfzWe12+13Vs9YqSRJdLhcVi0X1ej01Go2/fxgA/gsZ773/14sAAPwY/+sB+Fm0XwEAAAJAqAMAAAgA7VcAAIAAUKkDAAAIAKEOAAAgAIQ6AACAABDqAAAAAkCoAwAACAChDgAAIAAf520pc1ylN0QAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CPW71pXE86zC"
      },
      "source": [
        "def get_expert_buf(num=100000):\n",
        "    buf = {'observation':[], 'action':[], 'terminal':[], 'reward':[]}\n",
        "    obs = env.reset()\n",
        "    for _ in range(num):\n",
        "        action, _states = model.predict(obs)\n",
        "        buf['observation'].append(obs)\n",
        "        buf['action'].append(action)\n",
        "        obs, reward, done, info = env.step(action)\n",
        "        buf['terminal'].append(done)\n",
        "        buf['reward'].append(reward)\n",
        "        if done:\n",
        "            obs = env.reset()\n",
        "    buf['observation'] = np.stack(buf['observation'])\n",
        "    buf['action'] = np.stack(buf['action'])\n",
        "    buf['reward'] = np.stack(buf['reward'])\n",
        "    \n",
        "    return buf\n",
        "\n",
        "expert_data = get_expert_buf()\n",
        "with open(SAVE_DIR+ENV_NAME+'.expert_data.pkl', 'wb') as f:\n",
        "    pkl.dump(expert_data, f)"
      ],
      "execution_count": 8,
      "outputs": []
    }
  ]
}