6.td3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.distributions import Normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Use CUDA</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "device   = torch.device(\"cuda\" if use_cuda else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Replay Buffer</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer:\n",
    "    def __init__(self, capacity):\n",
    "        self.capacity = capacity\n",
    "        self.buffer = []\n",
    "        self.position = 0\n",
    "    \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        if len(self.buffer) < self.capacity:\n",
    "            self.buffer.append(None)\n",
    "        self.buffer[self.position] = (state, action, reward, next_state, done)\n",
    "        self.position = (self.position + 1) % self.capacity\n",
    "    \n",
    "    def sample(self, batch_size):\n",
    "        batch = random.sample(self.buffer, batch_size)\n",
    "        state, action, reward, next_state, done = map(np.stack, zip(*batch))\n",
    "        return state, action, reward, next_state, done\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.buffer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NormalizedActions(gym.ActionWrapper):\n",
    "    def _action(self, action):\n",
    "        low  = self.action_space.low\n",
    "        high = self.action_space.high\n",
    "        \n",
    "        action = low + (action + 1.0) * 0.5 * (high - low)\n",
    "        action = np.clip(action, low, high)\n",
    "        \n",
    "        return action\n",
    "\n",
    "    def _reverse_action(self, action):\n",
    "        low  = self.action_space.low\n",
    "        high = self.action_space.high\n",
    "        \n",
    "        action = 2 * (action - low) / (high - low) - 1\n",
    "        action = np.clip(action, low, high)\n",
    "        \n",
    "        return actions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Adding Gaussian Noise</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GaussianExploration(object):\n",
    "    def __init__(self, action_space, max_sigma=1.0, min_sigma=1.0, decay_period=1000000):\n",
    "        self.low  = action_space.low\n",
    "        self.high = action_space.high\n",
    "        self.max_sigma = max_sigma\n",
    "        self.min_sigma = min_sigma\n",
    "        self.decay_period = decay_period\n",
    "    \n",
    "    def get_action(self, action, t=0):\n",
    "        sigma  = self.max_sigma - (self.max_sigma - self.min_sigma) * min(1.0, t / self.decay_period)\n",
    "        action = action + np.random.normal(size=len(action)) * sigma\n",
    "        return np.clip(action, self.low, self.high)\n",
    "    \n",
    "#https://github.com/vitchyr/rlkit/blob/master/rlkit/exploration_strategies/gaussian_strategy.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def soft_update(net, target_net, soft_tau=1e-2):\n",
    "    for target_param, param in zip(target_net.parameters(), net.parameters()):\n",
    "            target_param.data.copy_(\n",
    "                target_param.data * (1.0 - soft_tau) + param.data * soft_tau\n",
    "            )\n",
    "            \n",
    "def plot(frame_idx, rewards):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, rewards[-1]))\n",
    "    plt.plot(rewards)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Addressing Function Approximation Error in Actor-Critic Methods</h1>\n",
    "<h2><a href=\"https://arxiv.org/abs/1802.09477\">Arxiv</a></h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ValueNetwork(nn.Module):\n",
    "    def __init__(self, num_inputs, num_actions, hidden_size, init_w=3e-3):\n",
    "        super(ValueNetwork, self).__init__()\n",
    "        \n",
    "        self.linear1 = nn.Linear(num_inputs + num_actions, hidden_size)\n",
    "        self.linear2 = nn.Linear(hidden_size, hidden_size)\n",
    "        self.linear3 = nn.Linear(hidden_size, 1)\n",
    "        \n",
    "        self.linear3.weight.data.uniform_(-init_w, init_w)\n",
    "        self.linear3.bias.data.uniform_(-init_w, init_w)\n",
    "        \n",
    "    def forward(self, state, action):\n",
    "        x = torch.cat([state, action], 1)\n",
    "        x = F.relu(self.linear1(x))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        x = self.linear3(x)\n",
    "        return x\n",
    "    \n",
    "\n",
    "class PolicyNetwork(nn.Module):\n",
    "    def __init__(self, num_inputs, num_actions, hidden_size, init_w=3e-3):\n",
    "        super(PolicyNetwork, self).__init__()\n",
    "        \n",
    "        self.linear1 = nn.Linear(num_inputs, hidden_size)\n",
    "        self.linear2 = nn.Linear(hidden_size, hidden_size)\n",
    "        self.linear3 = nn.Linear(hidden_size, num_actions)\n",
    "        \n",
    "        self.linear3.weight.data.uniform_(-init_w, init_w)\n",
    "        self.linear3.bias.data.uniform_(-init_w, init_w)\n",
    "        \n",
    "    def forward(self, state):\n",
    "        x = F.relu(self.linear1(state))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        x = F.tanh(self.linear3(x))\n",
    "        return x\n",
    "    \n",
    "    def get_action(self, state):\n",
    "        state  = torch.FloatTensor(state).unsqueeze(0).to(device)\n",
    "        action = self.forward(state)\n",
    "        return action.detach().cpu().numpy()[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Twin Dueling DDPG Update</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def td3_update(step,\n",
    "           batch_size,\n",
    "           gamma = 0.99,\n",
    "           soft_tau=1e-2,\n",
    "           noise_std = 0.2,\n",
    "           noise_clip=0.5,\n",
    "           policy_update=2,\n",
    "          ):\n",
    "\n",
    "    state, action, reward, next_state, done = replay_buffer.sample(batch_size)\n",
    "\n",
    "    state      = torch.FloatTensor(state).to(device)\n",
    "    next_state = torch.FloatTensor(next_state).to(device)\n",
    "    action     = torch.FloatTensor(action).to(device)\n",
    "    reward     = torch.FloatTensor(reward).unsqueeze(1).to(device)\n",
    "    done       = torch.FloatTensor(np.float32(done)).unsqueeze(1).to(device)\n",
    "\n",
    "    next_action = target_policy_net(next_state)\n",
    "    noise = torch.normal(torch.zeros(next_action.size()), noise_std).to(device)\n",
    "    noise = torch.clamp(noise, -noise_clip, noise_clip)\n",
    "    next_action += noise\n",
    "\n",
    "    target_q_value1  = target_value_net1(next_state, next_action)\n",
    "    target_q_value2  = target_value_net2(next_state, next_action)\n",
    "    target_q_value   = torch.min(target_q_value1, target_q_value2)\n",
    "    expected_q_value = reward + (1.0 - done) * gamma * target_q_value\n",
    "\n",
    "    q_value1 = value_net1(state, action)\n",
    "    q_value2 = value_net2(state, action)\n",
    "\n",
    "    value_loss1 = value_criterion(q_value1, expected_q_value.detach())\n",
    "    value_loss2 = value_criterion(q_value2, expected_q_value.detach())\n",
    "\n",
    "    value_optimizer1.zero_grad()\n",
    "    value_loss1.backward()\n",
    "    value_optimizer1.step()\n",
    "\n",
    "    value_optimizer2.zero_grad()\n",
    "    value_loss2.backward()\n",
    "    value_optimizer2.step()\n",
    "\n",
    "    if step % policy_update == 0:\n",
    "        policy_loss = value_net1(state, policy_net(state))\n",
    "        policy_loss = -policy_loss.mean()\n",
    "\n",
    "        policy_optimizer.zero_grad()\n",
    "        policy_loss.backward()\n",
    "        policy_optimizer.step()\n",
    "\n",
    "        soft_update(value_net1, target_value_net1, soft_tau=soft_tau)\n",
    "        soft_update(value_net2, target_value_net2, soft_tau=soft_tau)\n",
    "        soft_update(policy_net, target_policy_net, soft_tau=soft_tau)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "env = NormalizedActions(gym.make('Pendulum-v0'))\n",
    "noise = GaussianExploration(env.action_space)\n",
    "\n",
    "state_dim  = env.observation_space.shape[0]\n",
    "action_dim = env.action_space.shape[0]\n",
    "hidden_dim = 256\n",
    "\n",
    "value_net1 = ValueNetwork(state_dim, action_dim, hidden_dim).to(device)\n",
    "value_net2 = ValueNetwork(state_dim, action_dim, hidden_dim).to(device)\n",
    "policy_net = PolicyNetwork(state_dim, action_dim, hidden_dim).to(device)\n",
    "\n",
    "target_value_net1 = ValueNetwork(state_dim, action_dim, hidden_dim).to(device)\n",
    "target_value_net2 = ValueNetwork(state_dim, action_dim, hidden_dim).to(device)\n",
    "target_policy_net = PolicyNetwork(state_dim, action_dim, hidden_dim).to(device)\n",
    "\n",
    "soft_update(value_net1, target_value_net1, soft_tau=1.0)\n",
    "soft_update(value_net2, target_value_net2, soft_tau=1.0)\n",
    "soft_update(policy_net, target_policy_net, soft_tau=1.0)\n",
    "\n",
    "\n",
    "value_criterion = nn.MSELoss()\n",
    "\n",
    "policy_lr = 1e-3\n",
    "value_lr  = 1e-3\n",
    "\n",
    "value_optimizer1 = optim.Adam(value_net1.parameters(), lr=value_lr)\n",
    "value_optimizer2 = optim.Adam(value_net2.parameters(), lr=value_lr)\n",
    "policy_optimizer = optim.Adam(policy_net.parameters(), lr=policy_lr)\n",
    "\n",
    "\n",
    "replay_buffer_size = 1000000\n",
    "replay_buffer = ReplayBuffer(replay_buffer_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_frames  = 10000\n",
    "max_steps   = 500\n",
    "frame_idx   = 0\n",
    "rewards     = []\n",
    "batch_size  = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAE/CAYAAACkbK8cAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsvXeYXGd5sH8/03Zme9EWNatZcpEbWLZMsLGxhQsEHNNiQoAEEge+OOVLpSSkECAhyS+BhJDPISSQhBBqMOBubGwDLpLBluSibqttX+3u7O709/fHOWf27Oz0sjMrPfd17bUz5z3lnfY+5+lijEFRFEU5s/HUewKKoihK/VFhoCiKoqgwUBRFUVQYKIqiKKgwUBRFUVBhoCiKoqDCoGRE5BwR+amITIvIb9Z7PkptEZEjIrKj3vNQlFqjwqB0/gB4yBjTZoz5TL0nk4mI3CEiL4pISkR+Kcv4/xWRQRGZEpEviEiTa2y9iDwkIrMi8kLmIljJsWciIhIQka/bAsWIyDUZ478vInvsG4vDIvL7GeNHRGRORML23315rrVaRL4tIuMickxE3p8x/kb7WmER+ZGInO8a+2fXNcIiEhWRadd4OOMvKSL/4Bp/u4g8b7+O50Tk51xjTSLydyJyQkQmROSfRMTvGj9PRL4vIpMickBEbsnx+j5qv4c7XNvebr+WWRF5OMsx14rI0/b39ZCI3OYae62I7BaRUyIyJiLfEpHVrvFPichR+9iXROTDrrEt9ns9Yr/f94rIOa7xW+3f4KSIDIvIF0WkPddn1zAYY/SvhD/gAeBX8ox76zy/XweuA3YCv5QxdgMwBGwFuoCHgb90jf8Y+P+AEPAW4BTQW+mxJc7fV6f3Let1gSPAjjLPGQB+G7gSOAlckzH+B8ArAR9wDvAScGs51wYeAv4e8AMXA+PAa+2xzcCUPQ8f8CHgQJ7X/O/AF3KMtQJh4DX289VADLgJEOANwCzQZ4//CfAo0A30Ao8Df+a858A+4HcAL3AtMANsybjmJmA3cML9fgA7gLcDHwUezjjGD0wCv2bP6zJ73hfb4/3AKvtxE/Ap4E7X8ecALa7XuBd4s/38cuB99mvyAx8DXnAduxZY4Xq//gv4TD2+1yV9X+s9geX0B3wfSAIR+4u1xf7hfA64y/4i77B/ED+xf4BHgT91nWM9YIBftscmgPfbX9ZnsRbRf8y47nuB5+197wXWFTHXx1gsDL4MfML1/Dpg0H68BYgCba7xR4H3V3psEXM9Avyh/fqj9iKxCvgGMAIcBn7T3jcIzLl+bB8BEkC7/fxjwN/bj4v5HN4HvAw8Ym9/F9aiPGaf+whlCoOM13iMDGGQZZ/PAP+Q8b4UvLa94Bhcwhe4A/gP+/HtwPdcYx77Pbwuy7lagGng6hzXeg9wCBD7+XZgOGOfEeBV9uOdwNtcY78AHLUfX4D1OxLX+H3AxzLOdw/w+lzvB/ArLBYG/fZ70uza9hTwjizHNwGfBJ7L8ZpXYwmjP8gx3m1fqyfHZ/Ml4K5Kv0O1/lMzUQkYY67FWuRuN8a0GmP22UO/AHwcaMNahGeAdwOdWAvSB9yqs812rDu2n8e6o/sIliDZCrxdRK4GEJGbgQ8Db8a6s3oU+O8yX8JW4BnX82eAfhHpsccOGWOmM8a3VuHYYngH1nvVCaSA79jnWI0leH5bRG4wxkSwftRX28ddjbV4v9r1/Af242I+h6uB84AbbNPJ57AEwiqgB1jj7CgiV4rIqRJeU9GIiABXYd2Buvkv2xxxn4hcnOvwjP/O4wuy7OM8zhx3eAvWYv5Ijmu9B/iSsVc6rMX+eRF5k4h47fc3iiXYc117jYh05Hkt6XmJyNuAqDHmrhz7Z8UYM4T1O/lle16vAtZh/T6dc59lf55zwO9haQe4xj8oImEsQd6CdUOUjddg3RiNuY69UkQmsQTrW7B+4w2NCoPq8G1jzA+NMSljTMQY87AxZrf9/FmsL+XVGcd8zN73PqxF67+NMcPGmONYC/4r7P3eD3zSGPO8MSYBfAK4RETWlTHPVizV2cF53JZlzBlvq8KxxfAZY8xRY8wclpbUa4z5c2NMzBhzCPgX4FZ73x8AV4uID7gI6476ahEJ2sc+AlDk5/CnxpgZ+7pvBb5rjHnEGBMF/hhLMGGf7zFjTGcJr6kU/hTr9/hvrm3vxNJg1mGZge4VkUXXt4XwD4E/FpGgiLwSawFqtnd5AOv9uUZEAlg3FwHXuJvMxT6N/Z27Gvii69pJrDvfL2MJgS8Dv2aMmbF3uQf4LRHpFZEBwAm6aAZeBIaB3xcRv4hcb5+/2b5eG9b3/beyvWFF8N9YJqQo1m/qI8aYo665v2x/niuAPwJecB9sjPlLrO/wK4H/YPF3HBFZA3wWy9TlPvYxY0wH1s3EX2NpNQ2NCoPqcNT9RES2i+VMHbHvDt6P9YVzM+R6PJfleav9eB3wadvRdQrLFixYd8ylEgbcjizn8XSWMWfcuduv5NhicL+H64BVzmu2X/eHsVR/sITBNVg/0t3A/ViLyBXAAecOrcjPwX3dVe7n9oI2RhHYd5lpJ2tRr3j+2NuxNJg32ELIuf4PjTFzxphZY8wnsUyIV+U4zTuBDfb8Pwf8J9YdLcaYF7AW+X/E8l2sAJ5zxt2vAet9/VKOa7wLeMwYc9h1zA6sO+prsATM1cDnReQSe5ePY5nqfgr8CPhfIA4MGWPiwM9haW2DwO8CX3XN60+xTF1HcswnJyJyLvAVrPc1gKWl/oGIvCFzX2PMOJaA+7Z9g+EeM8aYn2D9Jv8s4xq9WGatfzLGZNXW7Zu7e+y5NDQqDKpD5l3Ul4E7gbX23cE/s1BVLoWjWHdana6/kDHmR2Wcay+Wc9HhYqwf5Zg9ttG+G3OP763CscXgfg+PAoczXnObMeb19viPsBx8twA/MMY8B5yFZVf+ges8xXwO7uuexHL+ASAizVimosKTt+4yW52/Yo6xr/Fe4INY9vtjBXY35PgeGWNeMsb8rDGm1xizHWvBf9I1/nVjzAXGmB4sp+56LHObm3cBP7Q1sWy8G5dWYHMJlr9lp62BPQU8gWXyxBZmtxtjVhtjNmIJ113GmJQ9/qwx5mpjTI8x5gZgo2ve1wG/KVYE2yDWZ/NVEfnDAu8TWKamfcaYe+15vQh8D8vRnQ0f0Mfimxr3+CbniYh0YQmCO40xHy8wlwXHNizlOBrO5D+sKJpfcT3/d+AvMvYZBt5jP77cfv6f9vP1WD9qn2v/Bc5FrLu6P7If3wLsAbbazztwOeSyzC+A5WT9IfCr9mOPPXYj1h3Y+Vh29O+zMCLoceBv7GNuYWE0UdnHFvGeHmFhlIgXeBrLqRyyn18AXOba50dYjuGr7Odfs5+7nZWlfg5bsbScK+338W+wnNNlO5CxnJNB+zO+3n7sOF/fab+n52U57iwsP4jzef4+li1/kZPS3v88LJNGAPhFYJSFDuVL7fexF+vu+8tZzvEi8N4c5/8ZLHNmW8b2q+1rXWI/fwXWgn+9/Xw1lsYlWJrbUWfMHr/Ifn3NWHb7w0CTPdYDDLj+jgJvA1pd35Mglsb3iP3Yb49tsj/La+1rb8KKoLrNHn8z1g2Fx/WePG2PebCikLrsYy/HulFwghjasQTWP+Z4r94JnGU/Xod1g/LNpVynyvqu1nsCy+2P4oTBW7GcmtPAd7HU87KEgf38XVjmECcqJmvYn2t+JuPPfe7fwTJJTWHZp5tcY+vt4+ewFoYdGecu61j7x7E3z5yPZLnWKiyb7yBWFNXjGef8pH0tZ+G43X6t/eV+Dvb292BFFy2KJsIy0YRL/L4cyfJ5rLfHDmOZTMKuv3+2x7ZiOWEdU9WDwLZc7ylWCOuIvf9j7n3t8cfs92Ec+H/YYZOu8VeRZbF3jf8/7OikLGO3Yy2001iRRr/rGnuN/R7M2t+Ld2Yc+9f25xsG7gbOLvZ7AvxSlvf2313jb8e6kZrG+o39FfM3Rr9hv/8z9nfsK9hReljC4B77vQpjhb9+mHkh/h77WjMZn50jAD5uX2/G/n8HOYR4I/05L05RFEU5g1GfgaIoiqLCQFEURVFhoCiKoqDCQFEURUGFgaIoioKVDLGsWbFihVm/fn29p6EoitJw7Nq1a9QY01vMvsteGKxfv56dO3fWexqKoigNh4i8VOy+aiZSFEVRVBgoiqIoKgwURVEUVBgoiqIoqDBQFEVRUGGgKIqi0IDCQERuFJEXReSAiHyw3vNRFEU5E2goYSAiXqx+ojdhNVF5h92kXFEURakhDSUMsDoKHTDGHDLGxLAaTtxc5zkpiqKc9jSaMFjNwgblxyiv8btSJSLxJE8cKqonvKIoy5hGEwZFISK3ichOEdk5MjJS7+mc1tz50xPc+i+PMxaO1nsqiqLUkEYTBseBta7na+xtCzDG3GGM2WaM2dbbW1QNJqVMpiJxjIGpSKLeU1EUpYY0mjB4CtgsIhtEJADcCtxZ5zmd0UTiSQBmoioMFOV0pqGqlhpjEiJyO3Av4AW+YIzZW+dpndFEEykA5myhoCjK6UlDCQMAY8xdwF31nodi4QiD2ZgKA0U5nWk0M5HSYDhmolk1EynKaY0KAyUv0bhqBopyJqDCQMlLJGFrBjHVDBTldEaFgZIX1QwU5cxAhYGSl2haM1BhoCinMyoMlLxE0pqBmokU5XRGhYGSF9UMFOXMQIWBkpd00pkKA0U5rVFhoOQlXY5CzUSKclqjwkDJi2YgK8qZgQoDJS8qDBTlzECFgZKXdDkKFQaKclqjwkDJy7wDWX0GinI6o8JAyYkxhpgtDGZUM1CU0xoVBkpOHK0ANLS0EuLJlCbtLWOMMSXtH0+m+PITL5NIpgrvXIBvPn2Mv7z7BZKp0uZQDioMlJw4dYlaAl5mY4mSfxSKxT88uJ+f++wP6z0NpUw+9M3d3PalnUXv/9iBUT78rd38+NBYxdf+7ydf5pF9I3g9UvG5CqHCQMmJk33c1RIgZRZqCkrxvDw+y6GRGRWmy5DZWIL//elxnjl2quhjBicjAJw8Fano2sPTEXa+NMENWwcqOk+xqDBQcuLUJepuCQAaUVQuM7EkiZRhWhsELTsefnGESDzFyHS0aLPP0JQlBAanKhMG9z83hDFw4wUqDJQ642gGnc2WMJjRxawsnPdtYiZW55kopXLX7pMApAyMFfn5VUsY3LNnkA0rWtjS31rReYpFhYGSk7Rm0OwHYC6umkE5OMJgXIXBsiIST/L9F4ZZ2REE5s0/hRiailr/i9w/G5OzcX58cIwbtg4gUnt/AagwUPLg9hmAmonKxQnLnZhVYbCc+MG+EWZjSd71qnXA/B1/IZz9TlYgDB58YYhEyiyZiQhUGCh5cBzGXbaZaFbNRGUxrxnE6zwTpRTu3n2SrmY/N1+yGihdGBS7fzbu2TPIyo4gF63uKPscpaLCQMmJU4pCNYPKUJ/B8iOaSPLA88Ncf/4AA+1BPDJv/slHPJliNBwj4PMwNhNLa9elMBtL8Mj+EW7YOoBnCUJKHWomDETkr0XkBRF5VkS+JSKdrrEPicgBEXlRRG5wbb/R3nZARD5Yq7kpxeFoBt2OZqA+g5IxxqTNRONqJlo2PLZ/lHA0wU0XDuD1CL1tTUU5hEemLYFx/sp2AIaLECCZPLLPimBaqpBSh1pqBvcDFxhjLgL2AR8CEJHzgVuBrcCNwD+JiFdEvMBngZuA84F32PsqdSLtM7AdyGomKp1oIpXOHlXNYPlw1+5B2oM+fmbTCgAG2oNFmX2cfS5Za937lhNRdM+eQbqa/Vy2vqvkYyuhZsLAGHOfMcZZPR4H1tiPbwa+YoyJGmMOAweAy+2/A8aYQ8aYGPAVe1+lTjjRRGomKh93OG6xoYlKfYklUtz/3CCvO3+AgM9aIvtKFAYXr7Vs/cVGILmv/eDzw7zu/H583qW14i/V1d4L3G0/Xg0cdY0ds7fl2q7UiahtFppPOlPNoFRmovMCVDWD5cGPDo4yFUnw+gvnzTSWZlDY5OPsc9GaTvt5acLgRwdHmY4mljSKyMFXycEi8gCQbdYfMcZ8297nI0AC+K9KrpVx3duA2wDOOuusap1WycDxGbQFfXg9oppBGTjtQv1eUZ/BMuHu3YO0Nvm4cvOK9Lb+9iYm5+JE4kmCfm/OY4emIvi9woaeFoJ+T8nhpffuta7tmKeWkoqEgTFmR75xEfkl4GeB68x8YZbjwFrXbmvsbeTZnnndO4A7ALZt26YFX2qEYyZq8nlpDnhVGJSBYyZa1RlSzWAZEE+muPe5QXac10eTb37R72+3Es+GpiKs62nJefzgVIS+tiAejzDQHizJZ5BMGe7bO8Rrz+3LK3BqRS2jiW4E/gB4kzFm1jV0J3CriDSJyAZgM/Ak8BSwWUQ2iEgAy8l8Z63mpxQmmkji9wpej9jCQM1EpeJEEq3taubUXHxJShEr5fPEoXFOzca56cKVC7bPC4P8pqLhqSh97U0ADHQES8pC3nlknLGZGDcucRSRQy19Bv8ItAH3i8hPReSfAYwxe4GvAs8B9wC/boxJ2s7m24F7geeBr9r7KnUimkil745aAj7VDMrA0QzWdIUwBibnNPGskblrz0maA16u3tK7YPuAU5KiwJ3+0FSE/jZr31I1g3v2DhLwebjmnN7CO9eAisxE+TDGnJ1n7OPAx7Nsvwu4q1ZzUkrDso9a9wuhgFcb3JSBIwzWdjcDVn0ixyGvNBbJlOHePYNcm8VM4yzwwwUW98GpCK8+27L393dYEUiplCmYPGaMde3XbO6lpalmy3JeNANZyYlbM2gOeNPOUKV43JoBaH2iRubJw5aZ5vUZJiKA9pCPoN+TN1R0NpZgOpJIm4lWtgeJJ01RgQO7j09yYjJSlygiBxUGSk4sYWB9RZoDPtUMysDxGazpmtcMakEyZbR5ToXcveckQX92M42I0N8eZGg6t8/AyTZOm4lKqHZ6795BvB5hx3l95Uy9KtRHH1GWBZF4kib/vGZw/JQKg1KZiSbweSS9MBQTUTQVifOOOx4n4POwvqeFdT3N6f9ndTczORdn31CYA8PT7BsKs384zMGRMFdv6eVf3r2tZq/lq08dZXVXKG0GOZ1IpQz37Bnkmi19NAeyL4v97fkdwo5/wHE2uyOQLihQcO77L4xw+frudO+QeqDCQMmJWzNoJJ9BJJ7knZ9/gj96w3m84qzapewbYyquJT8TTdAc8KbrOxVjMtg3OM3eE1OcO9DGk4fH+d+fHifXTf/qzhCb+1tpCXj5/gvDTM7G6bDLh1STeDLFn9y5l019LXz3N66q+vmrzb6haT7+vef5zK2vKOr9ePrlCYano9x0YW4zTX97kGfztL90EswGOuajiaBwKetEMsWB4Wnee+WGgvOsJSoMlJxE4sm0MLCiiRrDZzA4GWHXSxM8/OJIzYTB4dEZbvz7R7jz9is5Z6Ct7PPMxJK0NvkIBbyE/N6iNAMnfPHvb72EcwfaicSTHJuY5cjoLC+Pz9Ie8rOlv5VNva1pZ+OulyZ4y+d+xMP7htMll6vJcyemmIsn2XN8imMTs2mzV6PypR8f4Qf7Rvju7hO8c/u6gvvfvWeQgNfDtefmNtMMtDdx32Qk502CYybqszWC3tYmu9ppfmFwdGKOeNJwdu/SdDTLhfoMlJxEE6l0VIXlQG4MzcDpuHZkbKZm1zg8GiaaSPGTlycqOs9MNEGzvWB3twSK6mngmBsG7EUl6Pdydl8bO87v571XbuCtl67hojWdC6JOLlnbSU9LgAefH65ovrl46sh4+vH9zw3V5BrVIpFMcdfuQQDu/OmJgvsbY5mIrty8grZgbi2ivz1INJFiai77TdHQVISQ30ub/bn4vB6r2mkBzeDAcBiATX0qDJQGJerSDJoDPmKuCpz1xMl3ODxaO2Hg/OArvUY4mkgv2l0t/qKiiYanIgR8HjpCxZt7vB7htef28fCLw0U3bi+FXS9NsKYrxOa+Vu7b29jC4EcHxxifiXHRmg6ePDJecDHefXyS46fmCkbyOD6AXLkDg1MRBjqCC7SGYnINHGFwtgoDpVGJJlILHMjQGMXqHN/F4ZGZmkXQTEWsO/iDI5UJg9lYktYm673rag4UFU00OBVhoD1Ysr9ix3l9TEUS7HypMm0mE2MMTx2Z4LL13Vy/tZ8nj4w3dGmN7zxzgrYmH59660UYA999Nr92cPceK5Lndef1593P7RDOxvBUlL62pkXHFKMZ9LU10Z5HK1kKVBgoOYnGkwRdDmRojDLWjkCajiZqVhZ6OuJoBuGKzmM5kOfNRMVoBkNTEfrbmwrul8mVm3sJeD08+Hx179xfHp9lNBzl0nVd3LB1gGTK8P0XamOOqpRoIsk9ewe5fusA5w60c8Hqdu58JrcwcExEr9rYky7VnouBAprB0HQkLTAcVnYUoRmMhOuuFYAKAyUPlmZgO5CbGkcYzLk6rtXKVDRll414eXy2IrPLTCxBq2MmKlIzGJqKLlpUiqG1ycf2jd1V9xs8dcTSNC5b382FqzsYaA9y33ODVb1GtXhk3yjTkQRvvNhKHHvTxat49thkzu/Ji0PTVrBAEcleTjJZtixkYwyDk4uFeH9HkOlIYkFfi8zjDg2rMFAaHCuayBICIb+1oOX6Ui8lboFUM2FgawbxpOHYxFzZ55mJJtMmtu6WANORBPE8wsUYY2sGpQsDgB3n9XNodIZDI5VpNG52HhmnPehjc18rIsL1W/v5wb6Rhgk1dvOdZ07Q1exP50L87EWr0tuzcffuQUTg+q35TURgOfI7m/1Z7/Sn5hJEE6lFn1shbWJ4Osp0NMGmOkcSgQoDJQ9WNNFCzWCuAfogL40wmI/6qeQaM1GXZmCbIfKZiqajCWZjybLMREA6NLKaZpydL01w6bqudH2d688fIBJP8ej+kapdoxrMxhLc/9wQN124Er/dJWxVZ4jL1ndx5zMnsvqX7t07yGXruulrK0749rdlb3IzNL0w4czByTXIlazWKM5jUGGg5CCRTJFImQW1iaBBzEQxp/hbiMMVOnhzMTUXZ12PFUt/sMy77EQyRTSRSkcTOYlnE3nCS4ensi8qxbK2u5lz+tt4oEp+g4mZGAeGw2xb353etn1jN+1BH/c1WIjp918YZi6e5GcvWlhb6E0Xr+LAcJjnT04v2H54dIYXBqdLqgfkFJ/LxHESl6oZqDBQGh6ny1m6aqltJpptEDORzyOc099Ws1yD6UiCs7qb6Qj5y9YMnJaXjiDtarGiRfL5DZy7znKFAcB15/Xx1JEJJmcrL5e9y45M2rZuPrnP7/Vw3Xn9PPj8UE3CWMvlO8+coLetie0behZsf/2FK/F6ZJEj+e49JwFKEwZtTVmFwVBGbohDodLXB4bDtDX5FkUh1QMVBkpWHGGQ7mfQQA7k2ViSUMDL+p4WDo/OkKpB7sNUJE57yM/G3pbyhYGtwThmop4W6wefz0zk3GFmLiqlcN15/SRThh9UwYzz1Evj+L3CxWs7F2y//vx+JmbjVQ9jLZfpSJyHXhzhDfbC76antYlXn72C72SYiu7ZM8jFaztZ1Rkq+joDHUFGpqOLhODwtJN9vHBRbw74aAv6coaXHhwJs8n2xdQbFQZKVqIJa9FvWhRaWn/NYC6WJOT3sqG3hWgixckSm44Xw3QkQXvQz4YVLRwq0xTlONubXUlnQN5w2Fy251KYz0au3Iyz68gEF6zuWFTf/zVbegn4PNy7tzGiiu5/bohYIsUbL16VdfxNF6/i+Kk5nn7Zqi10bGKWZ49NclOJJaP72oOkzOLPcHAyQkfIn7Vd5cqO3LkGB4bDDeE8BhUGSg6c/sfz5ShsM1EDaAZzcStCZ4Pdi/ZIDZzIU3Nx2oM+NvW2MjgVKSuKyinf4U46g/yVS4cmI7QHfWnhWw5ej3DNOX08/OJIRWacSDzJs8cmuczlL3BoafJx1dkruG/vUEOUzv7OMydY3RnilWd1Zh2/YWs/AZ8nHVV0zx5LiJXaYjLtA8hY3IfsRMFs9Ldn9zNMReIMT0cbwl8AKgyUHCzSDPyNZibysaHXEgaHqiwMookk0USK9pClGUB5dZDSmoEtSP1eD21BX0GfQSVagcOO8/qYnIunbf7lsPv4JLFkaoG/wM31W/s5fmqO505OlX2NajAxE+PR/aP87MUrc5pb2oJ+rj2nj+8+e5JEMsU9ewY5d6CN9StyN7fPhhPllbm4D01HF5mIHHKVpGgk5zGoMFByELU1AyfpzOsRgn5PY5iJ4lZZ6P62ICG/t+qagZN93Bb0pYVBOaaicHShzwAKZyE79W0q5aotdjZyBSGmO+1ks0tzCIMd5/XjEepeq+ievYMkUoY3XpTdROTwpktWMRqOcuczJ9j18gQ3XbC4o1khnLv/zCY3w3lyQ3L5GVQYKMuCiJ1PEPTNmyusMtaNoRk0B7x4PMK6nuaq5xo42ceOzwDKyzVwBKe7umihLOThqUjRMe/5cLKRKwkx3XlknI29LfS0Zr/j7WltYtu67rqHmH7nmRNsXNHC1lXtefe79tw+WgJe/uw7z2EMeXsX5KKntQmvRxbkDSRThuHpaM7ckIEOy88wEl4oQA6OhAl4PaztKt6BXUtUGChZSUcT+ee/Io3S4MZxIAMVRfvkwtEMrL63XlZ3hsq6RtgOLW1x2f/zaQYpe1FxmqNUynXn9nFoZKasuadShl0vT+Q0ETlcv7Wf509OcXR8ttxpVsTwVIQfHxrjZy9eVTAiJ+j3cv3WASbn4mzsbWFzGXfkXo/Q27owvHRsJkoyZXL6DHL5GQ4Oh1m/ohmftzGW4caYhdJwOJpBk0szsHoa1N9M5GgGABtWtHB0fDZviYdScbKPndr2VkRR6YlnTk5GpmaQK+lsbCZGImWq4jMAK8QUKCuq6OBImFOz8QXJZtl43fnWNaoVVVSsMzoST/LQi8N89Nt7MQbeeFFxJp832dFGN10wUHY4Z3970wIfQGZTm8X7ZxcGBxqkJpGDdjpTspKZdAaWI7RRzEQh2ym7vqeFRMqqH7ShRGdgLpxeBk5J4Y29LXzrJ8dLboPpOJBDfrdm4M9pJhqqMPs4E3c28vuu3FDS3J38gWyRRG7W9bRw7kCPPzI/AAAgAElEQVQb9z03xK9ctbGi+caTKa7924eZjSbZ0t/GOQNt9v9WNve3MRaO8fCLwzz84giPHxpLl0t51xXr2NxfXDe612zp5cOvP5c3v3JN2fPsbw/y0ti8JpQr+9hhZZbEs0g8ycvjs2nh1AjUXBiIyO8CfwP0GmNGxfpGfhp4PTAL/JIx5ml73/cAf2Qf+hfGmC/Wen5KdjKTzsDSDBpBGMzFEmnNYGPvfHhptYTBdFozsH4eG1a0MB1JMBqO0VtCpuhMLEmL7dtw6GoJMBdPWqaujPDRagsDsLJv/+6BfbzrX5/kgzedW7Axu8NTR8bpaQmwvqdwe8trzunjXx87lPU1lcJPXj7F0fE5XrOll+lInK/tPJq1u97GFS38wvazuOacPrZv6M4a258Lr0e47TWbyp4jWJ/PE4fnO785uSG5zETdLQECXs8CYXBkbIaUqX93Mzc1FQYisha4HnjZtfkmYLP9tx34HLBdRLqBPwG2AQbYJSJ3GmMaI8XxDGPeTLRQM5iYLb+CZzUwxjAbnzcTre+ZDy99bZWu4ZiJ2kPzZiKwnMglCQNXlzOHdH2i2RihwELH4XwpiuqVJvg/r91Ee8jHZx7czxv/8TFuuWQ1v3P9loI9jHcemWDb+q6itIntG7v55x8c5OmXJ9LVQsvhkX0jeD3CP7zjFXSE/BhjOH5qjn1D0+wbCtMS8HL1lj7OKkJA1ZKBjiCTc3Ei8SRBv5ehqSgisKI1ez8EEaGvvWmB0/ngsOXHaSQzUa19Bn8H/AHW4u5wM/AlY/E40CkiK4EbgPuNMeO2ALgfuLHG81NyMO9AXqgZzNXZZxBNpDBmPiO6uyVAe9BXcRMaN9ORBB6Zd/w6GaKlXiOcRRg4lUuzmYoGpyKIWI3Uq4Xf6+GXX72Bh3//tbz/6k18b/dJrv3bH/DJu55nci6772J4KsLL47NsW5ffROSwbV0XHoEnDo1VNNdH9o9wydrOdLtPEWFNVzPXntvP+6/exLtetb7uggBI1xFyNLmhyQgrWpvyOoIH2oOcdAmDA8NhRGDjijNAGIjIzcBxY8wzGUOrgaOu58fsbbm2K3UgM+kMHAdyfc1ETjSTY4cXETb0tnJktHrRLFNzcdqC/vRd8arOEAGfp+Rcg9lYMl3TyaE7Txnr4anCi0q5dIT8/OGN5/LQ713DGy9axR2PHuI1n3qITz+wn1MZc3H8BdvW548kcmgL+tm6qmOB6aRUxmdi7D4+yWs295Z9jqUiXZba1uSGpnNnH7uPcUcgHRgJs7ozVJFZrdpU9K0TkQdEZE+Wv5uBDwMfrc40F133NhHZKSI7R0Yaq6b66YJTjiLTTFTv0NLZ+MJKoAAbqpxrMBVJ0B6av6P3eoT1Pc0lZzqHXS0vHZySFLk0g0oK1BXDqs4Qf/v2i/neb1zFZeu7+LsH9vHqv/w+n7zreYZt2/fOIxME/R62rirOvwCwfUM3Pzl6Km1eLJXHDoxiDLxmS/lmpqWiP6MstZU1nl+bc7KQnWipRoskggqFgTFmhzHmgsw/4BCwAXhGRI4Aa4CnRWQAOA6sdZ1mjb0t1/Zs173DGLPNGLOtt7fx7ySWI9FEkiafZ4HN2HIgJ+pai8YxU4Vci+yGFa0cPzVX9kKUyXQkTlvTwubkG1aUns8w62p56ZDWDLIIg2IWlWpx/qp2Pv+ey7jnt69ix/n9/Mujh7jyrx7ij/53N4/uH+HiNZ0EfMUvD9s39hBLpHjm6Kmy5vPIvhE6Qn4uWpO9tlAj4QiD4bQwiOQMK3UY6AgSiaeYmkuQTBkOjYQ5u0EK1DnUxExkjNltjOkzxqw3xqzHMvm80hgzCNwJvFssrgAmjTEngXuB60WkS0S6sBzP99ZifkphovHUAq0ALDt9ysz7E+qBE83U7PJlODWK3OF+lTA1t1AzAEvgvDQ2U1LhN3fLS4eOkB8RGM/Sa6CSdpflcu5AO5++9RV8/3ev4S2vXM3/PHWU/cPhok1EDpev70aEskxFxhge3T/ClWevWFR+uhFpD/oI+j0MTkaIJpKMz8ToL5A17piWTk7NceLUHNFEquE0g3rkGdyFFVZ6ACu09JcBjDHjIvIx4Cl7vz83xpRvhFQqIppILgrZa3F1OyslnK+apIXBAjORE+0T5pyB4uLN8zEVibO2e6GjcmNvC/GkFd2yrqe4EFZ3y0sHr0foDPkXaQbpRWWJhYHD+hUtfPLNF/Gb123mzp+e4JZXluau62j2c05/G0+WIQz2DYUZmoouCxMRWH6qgfYgQ9NRRuwaRYWyxt1ZyI5i3UhhpbBEwsDWDpzHBvj1HPt9AfjCUsxJyU80nlpQigLmq2/ORBNpc8dSk3Ygu4TB+hXWwn24Sk5kp5eBm40r5kNYSxEGmdFEYEUUjWc4bZ0s1lr7DAqxsiPEr11dXhz+FRt7+J+njhJPptI9iIvhkX2W3++qZeA8duhrDzI0GUk7kQuZiRwhPzQVSSc1nhFmImX5E02kFiScATTbkTFzVbLNl8O8ZjC/yLYF/axobapaeOnUXDyLmai06qWplEknnWXS3RxYpBk4zttcZZCXA9s3dDNn90AohUf2j3B2X2tJHcfqjaUZROYTBQuYiRxhcHIywoHhMD0tgXSYcaOgwkDJipVQk6kZ1L+ngVMJNNMWv7EMB282UilDOJZI1yVy6G4J2P2QixM4jsDMqRks6pTlmBvqqxlUwuUbrLyEJw4Xn28QiSd54vD4sggpddPf3sTgZGS+TWmBzy3g87CiNcDQVIQDdqvLRkOFgZKVbJpByG93Oyuj61e1cBbZzPhsK9qncjPRdDSBMZaT0I2IlNQCM7PlpZvu5sWVS4u9w2xkelqbOLuvtSS/wROHx4klUly1TPwFDv3tQaKJFPuHw/i9Qlezv6hjBm3NoNGcx6DCQMmBE1rqxkmgqq9msNiBDJYDdDQcTdcVKhd3L4NMStE+MlteuulqsSqXukN0h6YjBHweOotYVBqZ7Ru62Xlkouioq0f2jRDwebhiQ0+NZ1ZdHLPPs8dO0dcWLKpsx0B7kOdOTjE5F2+YvsduVBgoWYnEU4sihtJmogbwGQR9izUDoOJMZHcvg0w29rZwcjJSVLe3zJaXbrpb/MSSqQXZ3EOTEfrbm8ouq9wobN/YQziaKLoV5qP7R7h8fXdDZeIWg2MWenFwuujckP6OYNrhrJqBsmzIphk4iV71NBM5vgxPRjz6xnQ/5MqcyOkidVk0gw0rnBpFhbWDbC0vHdJZyOF5U9HQVHRZm4gctjt+g0OFTUUnJ+fYNxTmqs3Ly0QE8+a8RMoU7edZ6Yo4UmGgLBsiWZLOWhrEgZztbvus7mZEymtP6Wa+/3EWM1Fv8S0ws7W8dHDCct3hpUNTEfqXsfPYob89yPqe5qKSzx7dNwpYPQaWG+6or2LblDqfb3PAy6oG/KxVGChZyZZ0FkoLg/ppBrOulpdugn4vqzpCHKlQGKR9BlnMRE657MNFOJGztbx06MpSkmJoKnJaaAYA2zf08NSRcVKp/GVLHtk/Qm9bE+dWIVFwqQn6vWn/TrGJgk4Oyabe1oY0B6owULJiRRMt/HoEvB58HqmrZjAXW1ziwaGc+kGZZLa8dBOy7+iKKViXreWlQ3dGsbrpSJyZWLJqvY/rzfaN3UzOxXlhcDrnPsmU4bEDo1y1eUVDLozF4CzuxX5uTsezTb3VacJUbVQYKFmJxJMLehmAFV4ZqnO3s9kCwuDQ6EzWQnqxRKqoQnbzZqLsyfkbe1uLEgaOz6Ali0mrK6OM9XxTm9NDMygm32D38UlOzca5ehmaiBycrONiNbqVnSH8XuH8Ve21nFbZqDBQFmGMsfrLZqla6VQurRf5Wis67SndCV2plOFrO49y5V99n1/6tycLnn9qLk5zwJuznMKGFS0cHgkXrNzqCMzMfgZg5TD4PJKeZy3aXdaTNV3NrO4M5c03cEpQXFlBZ7R6M2D7DQqVonBobfJx5+1X8u5Xra/hrMqnHoXqlAYnnjQYwyLNAKw73bpqBvFEToeduz1lT2sTTxwa42Pfe449x6cIeD3sGyocaTQdSeTUCpxrTEUSjM3EWJGnI9lMNEGTz5O1UY2IWLkGs6enMADLVPSDF0cwxmQ1Az26f4QLVrfTU8WubkvNQIdVPqOUsuPnrWxMrQBUM1CyEMnS5cwhFPDWtcHNbAHNAOCR/aN84D938fN3PM54OManb72ED1yzifGZGLEC5benIvGsYaUOxUYUZWt56aa7eb4kxWBaGCzfhTGTKzb0MDYT48DwYgF8dHyWp18+texKUGTyi9vP4p/e+cqs/qXliGoGyiKi8cX9jx1aAj5m6mwmas5RPntNVwifR/jMg/sJ+b38zuu28KtXbSQU8PLlJ14GYDQczVsQbSoSz6sZOD1rD42EuWx97h7B2Vpeuulq8TMxYzmrh6eitAV9WUNmlyuO3+Dxw+Ns7reihY5NzPLZhw7y9V1H8YrwhotW1nOKFdPXHuT1Fy7v1+Dm9Pn2KVUjW/9jh1DAu6hn7lKSz4Hs83p4+2VrSSRT/M7rzlmQDOQ0MR+ezi8MpiP5y3Ov7grhETg2MZd3nuFoIqvz2KG7JZA2Ww0tQbvLpWZdTzP97U08eXica7b08tmHDvD1XcfwiPDzl63lA9eczeplVKX0TECFgbIIp/9xtgY2zQEvx0/VMbQ0niSYp3TBJ265MOt2J0lo2NWUPBtTc/F0PkE2vB5hRWtTuv9ALmZj+c1EXa4y1oN16HBWa0SE7Rt6uHfvIHfvPonHI7xz+1m8/5pNrOxQIdCIqDBQFpFPM2gO+OrmM0imDLFEimZ/6V/bXlszGAnnX8QLOZCdczn9B3IRjiYXVT510207kFMpw/BUlO0bGzP2vBJuvGCA+54b5BevWMf7r960rMtznwmoMFAW4fQ4zi4M6hdamquXQTGsaG1ChLx39MYYy4Ecyu8Q7GtrYni6gGYQTeQtOdDVHCBl4NRc/LQ0EwG8/sKVp5VN/XRHo4mURTjJWbnMRDN10gyytbwsFr/XQ3dzIO8iHomniCdNQc2gry1YUBjkannp4PglDo6ESaTMaWcmUpYfKgyUReTXDHzEEqmi69VXk1y9DIqlt62JkTzmnek8FUvd9LU3MRaOksxTe8dyIOeLJrKEwfN2qWcVBkq9UWGgLCIdWurLrhlAfXoaVCoM+trz39Gny1cXYSZKGRibyX4uY4wdWpo/zwDg+ZNW/Z7TKcdAWZ6oMFAW4TiQM3sgAzTbsfP1cCLPxS2fQajMePy+tiZG8gqD/HWJHHrtDOhc/odoIkUiZfJHE7VYAsfRDNS5qtQbFQbKIvIlnTXXsadBdcxE0ZyllfO1vHTjhKnmEizpukR55un4DF4cnEaEvKUtFGUpUGGgLCJvOQo7rHOmDt3OnEU2Wz+DYuhrayKRMoua0Ts4mkG+kFDnPEDO8NKZPOWrHUJ+L00+D3PxJCtam3IWxlOUpaKm30AR+Q0ReUFE9orIp1zbPyQiB0TkRRG5wbX9RnvbARH5YC3npuQmmifpzCmxMFcHn8FcpT4Dx7yT445+ukifgXMXn8tMFC5CGIhIWjtQf4HSCNQsz0BEXgvcDFxsjImKSJ+9/XzgVmArsAp4QES22Id9FngdcAx4SkTuNMY8V6s5KtnJn3TWCGaiMn0G7fMlKc7LEv4+NedoBvmFQdDvpSPkzylU8rW8dNPVHODk5OmZY6AsP2qZdPYB4C+NMVEAY8ywvf1m4Cv29sMicgC43B47YIw5BCAiX7H3VWGwxETiKTwCPs/i0sPOQjxbBzORo41UYiaC3Lb+6Ugcn0eyOs6znSuXmShfy0s3jmZQbD18RakltTQTbQGuEpEnROQHInKZvX01cNS13zF7W67tixCR20Rkp4jsHBkZqcHUz2yc/sfZ6tDXUzOYiznRROU7kCG3rd/JPi6mDWNfe+7IpHwtL904uQanS+9jZXlTkWYgIg8AA1mGPmKfuxu4ArgM+KqIbKzkeg7GmDuAOwC2bduWv+WUUjLZ+h87hNLCoD4OZJ9HCOSYWyGaAz5am3w5bf1Tc4XrEjn0tQV56kj2Tl75Wl666bYbqp8uvY+V5U1FwsAYsyPXmIh8APimsfoDPikiKWAFcBxY69p1jb2NPNuVJSQST2Z1HsP8Alcvn0G5WoFDvlyD6QKNbTLPMzwdzdrJK1/LSzddaiZSGohamon+F3gtgO0gDgCjwJ3ArSLSJCIbgM3Ak8BTwGYR2SAiASwn8501nJ+Sg7yagb+eZqLcvQyKJV/F0akiKpa6zxNLpNJOZzfFRBPBvM9AHchKI1BLB/IXgC+IyB4gBrzH1hL2ishXsRzDCeDXjTFJABG5HbgX8AJfMMbsreH8lBxE46mspSgAPLaDtS5moniy4m5gfe1Bnj12KuvYdCROr93JrJjzgOV/6GheqE3MRBN4PZJToDpcs6WPt2+bZFNvcddUlFpSM2FgjIkBv5hj7OPAx7Nsvwu4q1ZzUoojkkjmjahpCfiqqhl8/tFDfPPp49z1W1fl3W8ulig7ksghn5loai5Be6hIzaB1PkzVaevoMBtL0hLI7oB3c1ZPM59668VFXU9Rao2mPSqLyKcZgOVErlZtomTK8IXHDvPcyamCzerztbwslr62JmZjybQpx43V/7hIn0F77sikcIHy1YrSiKgwUBYRTSRpKqAZzFTJTPTjg2OcmLQW1Em7NlAuquFAToeXZrS/TCRTzMaSJTmQrfMs1jIKtbxUlEZEhYGyiEgRmkG1zERf3zWfWnIqR80gh2o4kHOVpJgusmKpQ2uTj5Dfm9XkFI7mL1+tKI2ICgNlEYU0g+YqCYOpSJx79g6ycYXV/3diNr9mMBdPVu4zcJWkcOMIg0J1iRxEhL727O0vZwo0tlGURkSFgbKIfKGlYCVvVUMY3PXsSSLxFO+9cgNQWDOwzEQVRhPlKEmRbmxTpGbgnCubz6BQy0tFaURUGCiLiMRTOZPOwNIM5qrgM/j6rmOc3dfK1Vt6AThVSDOIJSo2E3WE/AS8nkWLuNPLoFgHMuTuhTwTU81AWX6oMFAWEU0kC2gGXmYq1AwOj86w86UJ3nrpmnQmbq4+A2C3koxX7jMQEavJzVSmZuCYiYq/o892HoBZ9RkoyxAVBsoiCoWWNgd8FYeWfmPXMTwCt7xiNS0BL36v5PUZRBMpjCm/SJ2b3rbFtv55M1HxmkFvWxPT0cSi90JDS5XliAoDZQGplCGWTOVNOrMcyAmshPLSSaYM33j6GK/Z0kt/exARobM5wORcbs0g3cugQgcyZLf1px3IJZmJFucaJJIpoolUwSJ1itJoqDBQFhBL2v2P82kGTV5SxrpbL4cfHxzj5GSEt166Jr2tq9nPxExuzcApf1FpOQogaxSQ4zNoLcWB3L44THWmyCJ1itJoqDBQFhCxG8jk1QwqLFb39V1H6Qj52XFef3pbZyiQ12fgmGKqYSbqawtyajae7ugGlpmotcmHN0tDn9znWRyZVEz/Y0VpRFQYKAtw7vYL+QxgfuErBSe34E0Xr1oQsdTZ7M8bTTRbYf9jN84iPhqeFz7TkURJYaXu87izmYtteakojYYKA2UB0bgjDPJoBrYJxGlDWQrfs3ML3CYisPoBnyrCZ1AtBzIsXMSn5oqvS+TQ1RzA55EFZqJiW14qSqOhwkBZQCThmIny5xlAeWair+86xua+Vi5a07Fge2ezn4nZeE6n9Fy8ij6DLCUprJaXpZ3b45FFkUnFtrxUlEZDhYGygKI0A6fbWYlmokMjYXbZuQWZ5Z07mwPEEqmc2sZczJpXpeUoIHtJCstMVJpmAPMdzxyKbXmpKI2GCgNlAY5TtVBtIihdM/jG0/O5BZl02Q1icvkN5qOJKhcGPS0BRBY6fq3y1aUv4L1tTRk+A40mUpYnKgyUBURszaAoM1GJPoMfHhhj2/rurD1/O5vzZyE7GkM1fAY+r4eeliZGXPkB05FE0UXq3PS2BRcIFUczaFUzkbLMUGGgLCCtGdTATDQajrK6M5R1rLOgZlC9aCKwzTt2KQljjO1ALn0B72trYmwmRtzOz3AirJpVGCjLDBUGygKKCy0t3UxkjGE0HGVFayDreFcBzcC5VjDPvErB7fidiSVJmdKyjx0c/8OYHaY6U8VMaUVZSlQYKAsoJukslBYGxWsGM7EkkXiKFXbv4EwK+Qyc/seeEpLC8uEuSTHt1CUqw0w0H5lknWsmalVWrdY8FWWpUGGgLKAYzSDg9eDzSEmawah9F55LGHSkhUFuzaBaJiKw7uhHwzGSKcPUXGldzhacJ6P9pba8VJYrKgyUBUTjhX0GIlJy68uxGWux7MlhJmryeWkOeHNWLp2rQv9jN31tQZIpw8RsbF4zqMBM5JicwtGkOo+VZYkKA2UBkUThaCKw4uhLMRONTFt3/Lk0A7CzkPM4kKuqGbju6J3y1eVoBs7ryTQTKcpyo2bCQEQuEZHHReSnIrJTRC63t4uIfEZEDojIsyLyStcx7xGR/fbfe2o1NyU3TtJZII9mAKX3QR4NW3fOTimIbFj1iXKYieKVt7x0M39HH0mbicrxGfi9HrpbAvPOaO1loCxTaqkZfAr4M2PMJcBH7ecANwGb7b/bgM8BiEg38CfAduBy4E9EpKuG81OyEEkk8XulYPXOUs1EjjDobsluJgKnJEV2YRCJJasaodPbOl+SohIzESwMU9WWl8pypZbCwADt9uMO4IT9+GbgS8bicaBTRFYCNwD3G2PGjTETwP3AjTWcn5KFaDxVVPhmqWaisXCMzmY/fm/ur1xnPjNRPFFdn0H7fPlpp+VlOWYisNtf2mYibXmpLFdq+a39beBeEfkbLKHzM/b21cBR137H7G25titLSDSRzFuKwiEU8OY06WTDyjHIbSICK7z01Fxun0E1hUHQ76Ut6GNkOkqT30PA5ynoJ8lFX1uQg8NhwG55qXWJlGVIRd9aEXkAGMgy9BHgOuD/GmO+ISJvB/4V2FHJ9VzXvQ3LxMRZZ51VjVMqNpEC/Y8dWpq8HD9VmpkoV8KZQ2cowKnZGKmUWRSnP1dlMxHM5xp0hAIl9zJYcJ72JkbCUYwxzMZUM1CWJxV9a40xORd3EfkS8Fv2068Bn7cfHwfWunZdY287DlyTsf3hHNe9A7gDYNu2beU14lWyUrRm4PctagSfj9FwjK2r2vPu09nsJ2WsOkFO3oFDtaOJwLqjH56KIiJl+wus8zQRTxrGZ2LMxBK0apE6ZRlSS5/BCeBq+/G1wH778Z3Au+2ooiuASWPMSeBe4HoR6bIdx9fb25QlJJooTjOwoomK9xmMThdjJspdksLKM6juHbfTC3k6kqCtjEii9HnsLOSXxmcxRusSKcuTWn5rfxX4tIj4gAi2WQe4C3g9cACYBX4ZwBgzLiIfA56y9/tzY8x4DeenZCEST+YtReHQ3ORN1+Ep5pzT0URBM1FXi52FnOE3SCRTxJKpqmsGva2Wmai7pXIzEcDhkRlAG9soy5OafWuNMY8Bl2bZboBfz3HMF4Av1GpOSmEszaAIYeD3EUukSCRT+PJECAGMzRROOAPoCGXXDJxS2VU3E7U3EYmnOHFqjsvWd5d9nl77dR0Zs4WBhpYqyxDNQFYWUIqZCIrraVCoLpFDV476RHNV7H/sxt3+stywUpjXDA6NqmagLF9UGCgLiJZgJgKKciI7CWe56hI5OD6DzFyDavcycOhzZUOXk33s0Bzw0drk44gtDLQ2kbIcUWGgLKBkzaAIYeDU+i+kGbSH/IiwqFid46gO+avvQHZoq3AB72trSgsDrU2kLEdUGCgLiMaTxfkM7MiemSK6nY0UUZcIwOuxQjwzzUSRKra8dNPbNt9+sxLNwDpXU9qhrpqBshxRYaAsIJJIFZWJ69z9zhXjMwhHaW3yFXXermZ/Fs2gNmai9qAvXZCvPVShZuDq66yhpcpyRIWBsoDiNYPizUSj4VhBf4GDVZ8oI5rIcSBXOQNZRNJ+g7amyjQDt/+hVctRKMsQFQbKAqKJVFEZyB22WWVipnB9orEi6hI5WGWsF2oGczXSDGB+Ea/UTOQWBs2agawsQ1QYKGkSyRSJlCmqaunqzmYAjo7PFty3mLpEDl3NgcV5BmlhUP07bie8tJLQUpj3hwR8nryVWRWlUdFvrZIm3f+4yKqlvW1NHJ0oRhjEStIMJnNFE9VCM2ivlmZgCRV1HivLFRUGSppIuv9xcYvu2q4QR8fn8u6TSKaYmI3RU6wwCAWYjiaIJ1PpbbU0E63sCOGRebNXuThCRcNKleWK3sYoaaLp/sfF3SOs7W5m10sTefcZn41hDPQWayZy6hPNxtOml9m41X2tFuaXX9h+Fheu7qj4jt7xGahmoCxXVDNQ0qTNREVrBs2cnIyQcN3FZzI6XVzCmUNnOgt53m8wF0tWPZLIoSPk58rNK6pynoDPo5qBsmxRYaCkccxExWsGIZIpw8nJSM59nFIUKwoknDmk6xO5KpfOxhI1cR5XExGht7VJ6xIpy5YzUhhE4kk+9M1nuf+5oXpPpaEoRzMAeDlPRFG6LlFLkXkGTuVSV8jqXLz65atrwS2vWM2O8/rrPQ1FKYsz8jamyefhG08fpy3o53Xn64/XIZp2IBfvM4D84aXpukRFagadzfM+A4e5WKLs/sRLye/dcE69p6AoZXNGagYiwqqOICdO5Y+EOdOIpENLi1t4V3YE8Xokb3jpaDhKwOcpuhBcl61BnJqb1wxq0fJSUZSFnJHCAKyQQhUGCylVM/B5PazqDOYNLx0JR+ltbUJEcu7jpiXgxeeRBfWJZmPJmuQYKIoyz5krDDqDeR2fZyKlhpaC5TfIrxkUX5cILK0tsz7RnGoGilJzzlhhsLozxNBU/rDIM41Sk87AFgZ5NINS6hI5dDX7mZhxaQbxxo8mUpTlzhkrDAU+SvoAABqhSURBVFZ2hEgZGLJbMiqllaNwWNsdYjQcTZeMyKSUukQOXc2BBT6DOTUTKUrNOXOFQadVS+ak+g3SlBpaCvMRRccmFr+PqZRhrIS6RA4dGZVLZ2NJmpdBNJGiLGfOWGGwujMEwAn1G6QpNekM8oeXTs7FSaRM0XWJHKwGN5ZmYIxhLq4+A0WpNWesMFjZoZpBJo5mECihBpCTeJZNGKSzj8swE03MxjHGEImnMAZC6jNQlJpSkTAQkbeJyF4RSYnItoyxD4nIARF5UURucG2/0d52QEQ+6Nq+QUSesLf/j4iUtoKUSFvQT1uTT8NLXUQTVpezYsNAwVroQ34vR7OYiUbthLPeMsxEsUSKSDyV9kWoZqAotaVSzWAP8GbgEfdGETkfuBXYCtwI/JOIeEXEC3wWuAk4H3iHvS/AXwF/Z4w5G5gA3lfh3AqysjPYUGaiTz+wn49+e0/drh+NF9f/2I2IsKYrlF8zKDL72KHLLlY3MRtL91hWB7Ki1JaKhIEx5nljzItZhm4GvmKMiRpjDgMHgMvtvwPGmEPGmBjwFeBmsW5FrwW+bh//ReDnKplbMazqDHFysnE0g0f2j3DX7pN1u76jGZTK2u7mHJpBaXWJHJxidROzsXQvg1pVLVUUxaJWPoPVwFHX82P2tlzbe4BTxphExvaaYmUhN45mMBaOMhqOLWoIv1SUoxmA0+RmFmPMgu2j4Shej6Tv9IvFKWM9ORt3tbxUYaAotaSgMBCRB0RkT5a/m5digjnmdJuI7BSRnSMjI2WfZ1VHkPGZWDqKpt44Rd0OjszU5fqRCjSDcDSxqJH9WDhGd0sAj6d4HwTMF6ubcAkDNRMpSm0pGKJhjNlRxnmPA2tdz9fY28ixfQzoFBGfrR249882pzuAOwC2bdtmcu1XiFV2eOnJyQgbVrSUe5qqEE0kmY5aitHBkTCXruta+jnEUyUlnDmkw0snZtOF5sBJOCvNXwALfQahgDUfzUBWlNpSKzPRncCtItIkIhuAzcCTwFPAZjtyKIDlZL7TWPaFh4C32se/B/h2jeaWxkk8a4SIonFX/f6DI+G6zCGSSBIsIeHMYT68dOH7OBKOlRxWCu4y1jE1EynKElFpaOktInIMeBXwPRG5F8AYsxf4KvAccA/w68aYpH3XfztwL/A88FV7X4A/BH5HRA5g+RD+tZK5FcOqDjvxrAGEgWMiAjg4XB8zUfmagfU+ZhasG50uTzNo8nlpDng55TYTqQNZUWpKRbq3MeZbwLdyjH0c+HiW7XcBd2XZfggr2mjJGHASzxogvHTM1gz62po4VCfNIJpI0R7yl3xcW9BPZ7N/QXipMYaxmdLrEjl0hvxMzMbT0USqGShKbTljM5ABgn4vK1oDDRFeOmaHYV62oZuXxmeJJZa+mmokniypFIUbq5T1/Ps4E0sSiafK0gyAdBnreTOR+gwUpZac0cIArPDS4w0QXuqYibZv6CaZMrw8vvSmomgiVVKROjdruxcmno3a1WBLrUvk0NXi59RcnLlYApHS6iUpilI6Z/wvbGVHsCHqE43ORAl4PVy8phOAA3XwG5SbdAZWRNHxiTlSKSu4q9y6RA6doUA6Aznk95ZUIkNRlNI544XBqk6r/WVmwtRSM27H5G/qawXqE1EUKTPpDCwzUSyZYmja0rKcukTlm4n8aQeyOo8VpfaoMOgMMhNLMhXJ3pxlqRibsdpDtjb5GGgP1kUYVKoZwHx4qaMZ9JZYl8ihy+Uz0IQzRak9Z7wwWNnhJJ7V11Q0Fo6m7eub+lqWPAvZGGP7DMp1INvhpbbfwBEG3SXWJXLobPZbneimIhpJpChLwBkvDNJZyHV2Io/NxFhhL5ybels5NBxeUtNVLGn1DWgq0ySzuiuEyHyuwWg4SmezH38JvRHcOPWJTk5GtJeBoiwBKgzsLOTjdXYiO3V8wBIG09EEI0vYn3m+5WV5X4kmn5f+tmDaTFROu0s3TuXS46fmtOWloiwBZ7ww6GsL4vVIXc1Es7EEc/HkvJmo13IiH1hCv0E0bguDChZed3ipVZeo/P5EjmYQS6TUTKQoS8AZLwy8HmGgPVhXM5GTY9BjL56b+qyieUvpN0j3Py5TMwCnr4EjDGJl5xjAfH0i0IqlirIUnPHCAKxcgxN11AycUhROE5iB9iDNAS8Hh5dQM0hUQTPoamZwKkI0kWR0Olpyu0s37h4IqhkoSu1RYQCs7KxvkxunFIVzJy0ilhN5dOk0g2iiOpqBMXBkdJbpaKIiM1FHyI+TZ6alKBSl9qgwwHIiD05G0tmzS03aTOQKw9zU27KkmkGkGj4DO7z0p0cngPITzsAy37UHLVORmokUpfaoMMAqZR1LptLmmqUmbSZqdQuDVo6fmktX7aw1jmZQbjQRzCeePf3SKaD8ukQOjt9Ao4kUpfaoMMDyGUD9+hqMhaOE/N4F5hCnLMWh0aXRDpxoonLLUQD0twfxe4WfpDWD8s1EMB9RpJqBotQeFQa421/WSRjYpSjcOOGlSxVRVA3NwOsR1nQ1s982b1ViJoL5XAMVBopSe1QYMC8M6uVEHnWVonBY19OMR1gyv0GlSWcOa7pCOInT5dYlcui0G+1oNJGi1B4VBlh3oE0+T900g/GZ2ALnMVjmmjVdzUtWsC6dZ1Chfd7xG7Q2+So+V9pM5NdoIkWpNSoMsEI5V9UxvHQsvFgYgB1RtGRmoupoBmu7LGGQafYqByfXQDUDRak9KgxsVnXWJ/HM6RWcLfJmU28rh0bCSxLyWo1yFGCVpIDK/QVgdTsDFQaKshSoMLBZ2RGqS0mK6WiCeNJkjbzZ1NdKNJFakiJ61ShHAfOaQaWRRDBf/ro1qGYiRak1KgxsVnUEGZqOEE8ubSN6J+EsW93/+Yii2vsNookUXo/gK7PktIPjM6iGZrDjvH4+9daLOKe/reJzKYqSHxUGNqs6rSiYoaml1Q4yS1G42dRbXsG6wckIV3ziQf73J8eLPqaSLmduupr9/MymHrZv7Kn4XEG/l7dvW6v9jxVlCajo1y8ibxORvSKSEpFtru2vE5FdIrLb/n+ta+xSe/sBEfmM2L90EekWkftFZL/9v6uSuZXKynSuwRILg5nFpSgculsCdDb7S9YM/vy7exmcivBvPzxc9DGV9D92IyJ8+Vev4E0Xr6r4XIqiLB2V3gruAd4MPJKxfRR4ozHmQuA9wH+4xj4H/Cqw2f670d7+QeBBY8xm4EH7+ZKxqk5ZyJnlq904BetKyTV46IVh7to9yOa+Vp45Nsm+oemijpuJJqqiGSiKsjyp6NdvjHneGPNilu0/McacsJ/uBUIi0iQiK4F2Y8zjxurp+CXg5+z9bga+aD/+omv7krCyTolnYwV6BZcSXjoXS/LH397D2X2t/Mf7tuPzCF/bebTgcbOxBA88P8QrzuosfuKKopxWLMWt4FuAp40xUWA1cMw1dszeBtBvjDlpPx4E+pdgbmlam3y0B31Lnng2NhOjLeijyZfdRLOpt5XRcJTJ2XjBc/3D9/dzbGKOj//cBQx0BLnuvD6+9ZPjBZ3i33j6OFORBO999YayXoOiKMufgsJARB4QkT1Z/m4u4titwF8Bv1bKpGytIWdwvYjcJiI7RWTnyMhIKafOSz0Sz8Zm8vcKTkcUFShYt29omjseOcRbL12Tdt6+7dK1jIZjPPTCcM7jUinDv/3wMBet6eDSdUvqplEUpYEoKAyMMTuMMRdk+ft2vuNEZA3wLeDdxpiD9ubjwBrXbmvsbQBDthkJ+3/OFcwYc4cxZpsxZltvb2+hl1A0KzuCdfAZRHOaiGC+emk+v0EqZfjIt3bTGvTx4defl95+zTm9rGht4mu7juU89pH9IxwameG9r96gUTuKcgZTEzORiHQC3wM+aIz5obPdNgNNicgVdhTRuwFHqNyJ5WzG/p9X2NSCVZ2hpTcT5ShF4bC2K4TfK3n9Bl/fdYynjkzw4ZvOWyBYfF4Pb3nlah56YZhR2zeRyRd+eIS+tiZef+HK8l+EoijLnkpDS28RkWPAq4Dvici99tDtwNnAR0Xkp/Zfnz32f4DPAweAg8Dd9va/BF4nIvuBHfbzJWVVZ4iJ2fiSNZQBp3x1bjORz+thfU9LzvDS8ZkYn7j7eS5f381bL12zaPxt29aQSJmsOQf7h6Z5ZN8I77piHQGNJFKUM5qK8vyNMd/CMgVlbv8L4C9yHLMTuCDL9jHgukrmUynpJjeTc2lbfS1JpQzjM9G8mgFYfoMXBqcYnorg93rw+zz4vYLf4+ETdz1POJLgL265AI9nsZnn7L42Llnbydd2HuN9Vy40Bf3bj44Q8Hn4he1nVf21KYqyvNCiLy7STW5ORZZEGJyai5MyhSt8bulv5Z69g1z+iQezjn/gmk1syVOy4W3b1vCRb+1h9/FJLlpjhY+emo3xzaePccslqytuT6koyvJHhYGLVR12rsES+Q3ylaJw874rN3JWTwvRRJJ4IkU8aYglU8QSKdqCPn7xinV5j3/jxav48+88x9d2HksLg/9+8iiReIpfvnJ9VV6LoijLGxUGLvo7rEV5qaqXOqUoVhQwE3U0+7P6A4qlPejnpgsG+PZPj/ORN5yH1yP/f3t3HxxXdd5x/PtIq7eVbWStbTC2Xgw4GJNQpxYUOwaM81JoM+Gl2E2mTGAmKbSTdtJJE5K0f4S0Q6bpHyRtkklDSgqZkCYkrRsmccJbRE0ouNgYiI2twTaSsWpbL7axZVmSJT39416JtdjVrna1q9Xu7zOj0d5z7l2dI1/r2XvOPc/l+8+3s/biGCsumJfx+4pI8dCsYZyqSDkL51bl7fbS8Yyl05DuOZWNLQ2cHBjmideO8qtdRzj81oAWmYnIOF0ZTHBhXU1enh8A0Hs6HCaqzf2Y/ZqLYiypq+En29+kb3CYpliUDSsWpT5QREqCrgwmaKyPcvBYf15+Vm/fEGZB2udcKysz/mj1Up59vYedB09w59rmhHcfiUhpUjCYoKk+SueJM3l5yE3v6UHqaiqyfqBMujaG8w5zqyJsbGnIy88UkdlBw0QTNMWijIw6ncfP0LygNqc/q7dv8gVn062hPsqda5tprI8yp0r/9CLyNv1FmKApFgSAjmP9+QkGKe4kmm73fuTyvP48EZkdNEw0QVMseIZvR+/UHjWZid7Tg9PyrGARkWwpGEywaG4V1RVldPTmfhK59/TQpBlLRUTyRcFgAjOjqb4258Hg7MgoJ/rPpkxFISKSDwoGCTTGojkfJjreP/bsYw0TicjMUzBIoDkWrDUYHU36sLWsja0+TpWKQkQkHxQMEgiSwo3SdSrxA2Gmw3gqCgUDESkACgYJNId3FLXncKhoPBWFholEpAAoGCTQVB+sLziYw0nk8WEiTSCLSAFQMEjgwrpqImWW8yuD8jJjXnXu8xKJiKSiYJBApLyMpfNr6MhhwrrevmCNgZLFiUghUDBIojFWm/Uw0amBs0nrek/nPxWFiEgyCgZJNNVHae89jXtmt5e+eayf1X//FJt3HkpY39unVBQiUjgUDJJoikU5NTDMif7kn+4n89SeowyNjPLNX+9LuF5BqShEpJAoGCQRn700E61t3UTKjP3dp3l6b9c76oP01QoGIlIYFAySyCZ76ZmhEV440MvtVzexdH4N3/nv/efUD5wdoW9wWMNEIlIwsgoGZrbRzHab2aiZtSSobzSzPjP7bFzZDWbWZmb7zOwLceXLzGxbWP5jM5vRj82N9WPBYOpXBs8f6GFoeJT3X7aIT65bxvaO42xvPzZef+y0Vh+LSGHJ9spgF3ArsDVJ/f3AL8c2zKwc+BZwI7AS+JiZrQyrvwp8zd0vAY4Dn8iybVmprijngnnVGQWD1r3d1FSUc9WyejZd2UBdtILvbD0wXj+24Ex3E4lIocgqGLj7HndvS1RnZjcDbwC744qvAva5+wF3HwJ+BNxkZgZsAH4a7vcwcHM2bZsOmWQvdXda27p43yULqIqUE62M8PE1zTz52lH2dfUBSkUhIoUnJ3MGZjYH+Dzw5QlVS4A347YPhWUx4IS7D08oT/b+d5nZdjPb3t3dPX0Nn6A5Fp3yBPL+7j4OHT/D9SsWjpfdsaaJqkgZ3w2vDpSKQkQKTcpgYGZPmdmuBF83TXLYvQRDPn3T1tI47v6Au7e4e8vChQtTH5Chplgt3acG6R8aTr1zqHVvEJzWX7povCw2p4pNLQ1s3tnJ0ZMD41cGmjMQkUIRSbWDu38gg/f9PeA2M/tHoA4YNbMBYAfQELffUqAT6AXqzCwSXh2Mlc+o+EnkyxbPS+uY1rYuLj1/Lkvqas4p/+Q1y3hkWwf/9lw7jlMZKWNOVcpfv4hIXuRkmMjdr3H3ZndvBr4OfMXdvwm8CCwP7xyqBD4KPObBMt9W4LbwLe4AfpaLtk1F89hagzQnkfsGh3mx/RjrV7zzaqUpVsuN71nMIy900NHTz4LaSoKpEhGRmZftraW3mNkhYA3wCzN7fLL9w0/9fwE8DuwBHnX3sQnmzwOfMbN9BHMID2bTtunQOMW1Bs/t6+HsiHN93BBRvLuvvYhTg8M88doR6jVfICIFJKtxCnffDGxOsc+9E7a3AFsS7HeA4G6jgnFeTQXzoxVpTyI/09bF3KoIq5vmJ6y/Ymkday+O8T/7e4nV6k4iESkcWoGcQrrZS92d1r3drFu+gIry5L/Wu6+7GECpKESkoCgYpDCWvTSVvUdOceTkQNIhojHXLl/Are9dwoYVk+8nIpJPup0lheZYlJ+/+n8MDY9SGUkeO1vbgmR01106+a2uZsb9f7xqWtsoIpItXRmk0BirZdSh88SZSfd7Zm83l184j/PnVeepZSIi00fBIIWx7KWTDRW9deYsOw4eTzlEJCJSqBQMUhgLBpNNIv/m9R5GRv2cFBQiIrOJgkEKC+dUEa0sn3ThWWtbF3XRClY1JL6lVESk0CkYpGBmNNYnz146Ouo809bNNcsXUl6mFcUiMjspGKShaZLspa92vkVP3yDXp7iLSESkkCkYpKEpVsvBY/3veLD9yYGzfO4nrzCvOqLJYxGZ1RQM0tBYH2VoeJQjJwfGy86OjPKpR17ijZ7T/Mvtq5mvdNQiMospGKRhYvZSd+dLj+3m2dd7+Mot72HtJQtmsnkiIllTMEjD+O2lx4JJ5Ad/8wY/3HaQP19/MZuubJjsUBGRWUHBIA2Lz6smUma09/bzxO4j3LdlDze++wI+96FLZ7ppIiLTQrmJ0hApL6OhPsozbd089Fw7Vyw5j/s3raJMt5KKSJHQlUGaGuuj7Dl8kvraSr57Rws1leUz3SQRkWmjYJCmyxbPY05VhAfvbGHRXCWjE5HiomGiNH3mg+/i7msv0i2kIlKUFAzSVBkpozKiQCAixUnDRCIiomAgIiIKBiIigoKBiIiQZTAws41mttvMRs2sZULdFWb2fFj/WzOrDstXh9v7zOyfzczC8noze9LMXg+/60kxIiJ5ku2VwS7gVmBrfKGZRYAfAH/m7pcD64GzYfW3gT8FlodfN4TlXwCedvflwNPhtoiI5EFWwcDd97h7W4KqDwGvuvsr4X697j5iZouBee7+grs78H3g5vCYm4CHw9cPx5WLiEiO5WrO4F2Am9njZvaSmd0Tli8BDsXtdygsAzjf3Q+Hr48A5+eobSIiMkHKRWdm9hRwQYKqv3X3n03yvuuAK4F+4Gkz2wG8lU6j3N3NzJPVm9ldwF0AjY2N6byliIhMImUwcPcPZPC+h4Ct7t4DYGZbgN8lmEdYGrffUqAzfH3UzBa7++FwOKlrkjY9ADwA0NLSkjRoiIhIenKVjuJx4B4ziwJDwHXA18I/9CfN7GpgG/Bx4BvhMY8BdwD/EH5PdtVxjh07dvSYWUeG7VwA9GR47GymfpcW9bv0jPW9Kd0DLJjHzYyZ3ULwx3whcAJ42d1/P6y7Hfgi4MAWd78nLG8BHgJqgF8CfxkOC8WAR4FGoAPY5O7HMm5ceu3f7u4tqfcsLup3aVG/S08mfc/qysDdNwObk9T9gGBYaGL5duDdCcp7gfdn0x4REcmMViCLiEjJB4MHZroBM0T9Li3qd+mZct+zmjMQEZHiUOpXBiIiQokGAzO7wczawmR5RZ0Dycy+Z2ZdZrYrrqzokwKaWYOZtZrZa2GyxE+H5UXddzOrNrP/NbNXwn5/OSxfZmbbwnP+x2ZWlI/tM7NyM9tpZj8Pt4u+32bWHib/fNnMtodlUz7PSy4YmFk58C3gRmAl8DEzWzmzrcqph3g7GeCYUkgKOAz8tbuvBK4GPhX+Oxd73weBDe7+O8Aq4IZwXc9XCdb6XAIcBz4xg23MpU8De+K2S6Xf17v7qrjbSad8npdcMACuAva5+wF3HwJ+RJAkryi5+1Zg4nqNok8K6O6H3f2l8PUpgj8QSyjyvnugL9ysCL8c2AD8NCwvun4DmNlS4A+Bfw23jRLodxJTPs9LMRgsAd6M245PllcqSiopoJk1A+8lWPVe9H0Ph0peJkjp8iSwHzjh7sPhLsV6zn8duAcYDbdjlEa/HXjCzHaEedsgg/M8V+koZJZIlRRwtjOzOcB/AH/l7ifDZykBxdt3dx8BVplZHcGi0BUz3KScM7MPA13uvsPM1s90e/Jsnbt3mtki4Ekz2xtfme55XopXBp1AQ9x2fLK8UnE0TAZIqqSAs5mZVRAEgkfc/T/D4pLoO4C7nwBagTVAXfjQKSjOc/59wEfMrJ1g6HcD8E8Uf79x987wexdB8L+KDM7zUgwGLwLLw7sMKoGPEiTJKyVjSQFhCkkBZ5NwvPhBYI+73x9XVdR9N7OF4RUBZlYDfJBgvqQVuC3crej67e5fdPel7t5M8H/61+7+JxR5v82s1szmjr0meLDYLjI4z0ty0ZmZ/QHB+GI58D13v2+Gm5QzZvbvBI8dXQAcBb4E/Bd5TgqYb2a2DngW+C1vjyH/DcG8QdH23cyuIJgwLCf4sPeou/+dmV1E8Im5HtgJ3O7ugzPX0twJh4k+6+4fLvZ+h/0byw8XAX7o7vdlkvizJIOBiIicqxSHiUREZAIFAxERUTAQEREFAxERQcFARERQMBARERQMREQEBQMREQH+HwqUNefJtAbgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "while frame_idx < max_frames:\n",
    "    state = env.reset()\n",
    "    episode_reward = 0\n",
    "    \n",
    "    for step in range(max_steps):\n",
    "        action = policy_net.get_action(state)\n",
    "        action = noise.get_action(action, step)\n",
    "        next_state, reward, done, _ = env.step(action)\n",
    "        \n",
    "        replay_buffer.push(state, action, reward, next_state, done)\n",
    "        if len(replay_buffer) > batch_size:\n",
    "            td3_update(step, batch_size)\n",
    "        \n",
    "        state = next_state\n",
    "        episode_reward += reward\n",
    "        frame_idx += 1\n",
    "        \n",
    "        if frame_idx % 1000 == 0:\n",
    "            plot(frame_idx, rewards)\n",
    "        \n",
    "        if done:\n",
    "            break\n",
    "        \n",
    "    rewards.append(episode_reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch4]",
   "language": "python",
   "name": "conda-env-pytorch4-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}