Muesli_cartpole.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3w04_bU_4H9g"
      },
      "outputs": [],
      "source": [
        "!pip install tensorboardX\n",
        "from tensorboardX import SummaryWriter\n",
        "import gym\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "from torch.distributions import Categorical\n",
        "\n",
        "\n",
        "!pip install gym[classic_control]\n",
        "!pip install gym[box2d] #for lunarlander\n",
        "!apt update\n",
        "!apt install xvfb\n",
        "!pip install pyvirtualdisplay\n",
        "!pip install gym-notebook-wrapper\n",
        "import gnwrapper\n",
        "!nvidia-smi\n",
        "print(torch.cuda.is_available())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WgMBnokKfDY7"
      },
      "outputs": [],
      "source": [
        "## For linear lr decay\n",
        "## https://github.com/cmpark0126/pytorch-polynomial-lr-decay\n",
        "! pip install git+https://github.com/cmpark0126/pytorch-polynomial-lr-decay.git\n",
        "from torch_poly_lr_decay import PolynomialLRDecay"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n_n1jjuq_TCF"
      },
      "outputs": [],
      "source": [
        "class Representation(nn.Module): \n",
        "    \"\"\"Representation Network\n",
        "\n",
        "    Representation network produces hidden state from observations.\n",
        "    Hidden state scaled within the bounds of [-1,1]. \n",
        "    Simple mlp network used with 1 skip connection.\n",
        "\n",
        "    input : raw input\n",
        "    output : hs(hidden state) \n",
        "    \"\"\"\n",
        "    def __init__(self, input_dim, output_dim, width):\n",
        "        super().__init__()\n",
        "        self.skip = torch.nn.Linear(input_dim, output_dim)  \n",
        "        self.layer1 = torch.nn.Linear(input_dim, width)\n",
        "        self.layer2 = torch.nn.Linear(width, width)\n",
        "        self.layer3 = torch.nn.Linear(width, width) \n",
        "        self.layer4 = torch.nn.Linear(width, width)  \n",
        "        self.layer5 = torch.nn.Linear(width, output_dim)     \n",
        "        \n",
        "    def forward(self, x):\n",
        "        s = self.skip(x)\n",
        "        x = self.layer1(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        x = self.layer2(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        x = self.layer3(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        x = self.layer4(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        x = self.layer5(x)    \n",
        "        x = torch.nn.functional.relu(x+s)\n",
        "        x = 2*(x - x.min(-1,keepdim=True)[0])/(x.max(-1,keepdim=True)[0] - x.min(-1,keepdim=True)[0])-1 \n",
        "        return x\n",
        "\n",
        "\n",
        "class Dynamics(nn.Module): \n",
        "    \"\"\"Dynamics Network\n",
        "\n",
        "    Dynamics network transits (hidden state + action) to next hidden state and inferences reward model.\n",
        "    Hidden state scaled within the bounds of [-1,1]. Action encoded to one-hot representation. \n",
        "    Zeros tensor is used for action -1.\n",
        "    \n",
        "    Output of the reward head is categorical representation, instaed of scalar value.\n",
        "    Categorical output will be converted to scalar value with 'to_scalar()',and when \n",
        "    traning target value will be converted to categorical target with 'to_cr()'.\n",
        "    \n",
        "    input : hs, action\n",
        "    output : next_hs, reward \n",
        "    \"\"\"\n",
        "    def __init__(self, input_dim, output_dim, width, action_space):\n",
        "        super().__init__()\n",
        "        self.layer1 = torch.nn.Linear(input_dim + action_space, width)\n",
        "        self.layer2 = torch.nn.Linear(width, width) \n",
        "        self.hs_head = torch.nn.Linear(width, output_dim)\n",
        "        self.reward_head = nn.Sequential(\n",
        "            nn.Linear(width,width),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(width,width),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(width,support_size*2+1)           \n",
        "        ) \n",
        "        self.one_hot_act = torch.cat((F.one_hot(torch.arange(0, action_space) % action_space, num_classes=action_space),\n",
        "                                      torch.zeros(action_space).unsqueeze(0)),\n",
        "                                      dim=0).to(device)        \n",
        "       \n",
        "    def forward(self, x, action):\n",
        "        action = self.one_hot_act[action.squeeze(1)]\n",
        "        x = torch.cat((x,action.to(device)), dim=1)\n",
        "        x = self.layer1(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        x = self.layer2(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        hs = self.hs_head(x)\n",
        "        hs = torch.nn.functional.relu(hs)\n",
        "        reward = self.reward_head(x)    \n",
        "        hs = 2*(hs - hs.min(-1,keepdim=True)[0])/(hs.max(-1,keepdim=True)[0] - hs.min(-1,keepdim=True)[0])-1\n",
        "        return hs, reward\n",
        "\n",
        "\n",
        "class Prediction(nn.Module): \n",
        "    \"\"\"Prediction Network\n",
        "\n",
        "    Prediction network inferences probability distribution of policy and value model from hidden state. \n",
        "\n",
        "    Output of the value head is categorical representation, instaed of scalar value.\n",
        "    Categorical output will be converted to scalar value with 'to_scalar()',and when \n",
        "    traning target value will be converted to categorical target with 'to_cr()'.\n",
        "        \n",
        "    input : hs\n",
        "    output : P, V \n",
        "    \"\"\"\n",
        "    def __init__(self, input_dim, output_dim, width):\n",
        "        super().__init__()\n",
        "        self.layer1 = torch.nn.Linear(input_dim, width)\n",
        "        self.layer2 = torch.nn.Linear(width, width) \n",
        "        self.policy_head = nn.Sequential(\n",
        "            nn.Linear(width,width),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(width,width),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(width,output_dim)           \n",
        "        ) \n",
        "        self.value_head = nn.Sequential(\n",
        "            nn.Linear(width,width),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(width,width),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(width,support_size*2+1)           \n",
        "        ) \n",
        "   \n",
        "    def forward(self, x):\n",
        "        x = self.layer1(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        x = self.layer2(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        P = self.policy_head(x)\n",
        "        P = torch.nn.functional.softmax(P, dim=-1) \n",
        "        V = self.value_head(x)      \n",
        "        return P, V\n",
        "\n",
        "\n",
        "\n",
        "\"\"\"\n",
        "For categorical representation\n",
        "reference : https://github.com/werner-duvaud/muzero-general\n",
        "In my opinion, support size have to cover the range of maximum absolute value of \n",
        "reward and value of entire trajectories. Support_size 30 can cover almost [-900,900].\n",
        "\"\"\"\n",
        "support_size = 30\n",
        "eps = 0.001\n",
        "\n",
        "def to_scalar(x):\n",
        "    x = torch.softmax(x, dim=-1)\n",
        "    probabilities = x\n",
        "    support = (torch.tensor([x for x in range(-support_size, support_size + 1)]).expand(probabilities.shape).float().to(device))\n",
        "    x = torch.sum(support * probabilities, dim=1, keepdim=True)\n",
        "    scalar = torch.sign(x) * (((torch.sqrt(1 + 4 * eps * (torch.abs(x) + 1 + eps)) - 1) / (2 * eps))** 2 - 1)\n",
        "    return scalar\n",
        "\n",
        "def to_scalar_no_soft(x): ## test purpose\n",
        "    probabilities = x \n",
        "    support = (torch.tensor([x for x in range(-support_size, support_size + 1)]).expand(probabilities.shape).float().to(device))\n",
        "    x = torch.sum(support * probabilities, dim=1, keepdim=True)\n",
        "    scalar = torch.sign(x) * (((torch.sqrt(1 + 4 * eps * (torch.abs(x) + 1 + eps)) - 1) / (2 * eps))** 2 - 1)\n",
        "    return scalar\n",
        "\n",
        "def to_cr(x):\n",
        "    x = x.squeeze(-1).unsqueeze(0)\n",
        "    x = torch.sign(x) * (torch.sqrt(torch.abs(x) + 1) - 1) + eps * x\n",
        "    x = torch.clip(x, -support_size, support_size)\n",
        "    floor = x.floor()\n",
        "    under = x - floor\n",
        "    floor_prob = (1 - under)\n",
        "    under_prob = under\n",
        "    floor_index = floor + support_size\n",
        "    under_index = floor + support_size + 1\n",
        "    logits = torch.zeros(x.shape[0], x.shape[1], 2 * support_size + 1).type(torch.float32).to(device)\n",
        "    logits.scatter_(2, floor_index.long().unsqueeze(-1), floor_prob.unsqueeze(-1))\n",
        "    under_prob = under_prob.masked_fill_(2 * support_size < under_index, 0.0)\n",
        "    under_index = under_index.masked_fill_(2 * support_size < under_index, 0.0)\n",
        "    logits.scatter_(2, under_index.long().unsqueeze(-1), under_prob.unsqueeze(-1))\n",
        "    return logits.squeeze(0)\n",
        "\n",
        "\n",
        "\n",
        "class Target(nn.Module):\n",
        "    \"\"\"Target Network\n",
        "    \n",
        "    Target network is used to approximate v_pi_prior, q_pi_prior, pi_prior.\n",
        "    It contains older network parameters, (exponential moving average update)\n",
        "    \"\"\"\n",
        "    def __init__(self, state_dim, action_dim, width):\n",
        "        super().__init__()\n",
        "        self.representation_network = Representation(state_dim*8, state_dim*4, width) \n",
        "        self.dynamics_network = Dynamics(state_dim*4, state_dim*4, width, action_dim)\n",
        "        self.prediction_network = Prediction(state_dim*4, action_dim, width) \n",
        "        self.to(device)\n",
        "\n",
        "\n",
        "\n",
        "class Agent(nn.Module):\n",
        "    \"\"\"Agent Class\"\"\"\n",
        "\n",
        "    def __init__(self, state_dim, action_dim, width):\n",
        "        super().__init__()\n",
        "        self.representation_network = Representation(state_dim*8, state_dim*4, width) \n",
        "        self.dynamics_network = Dynamics(state_dim*4, state_dim*4, width, action_dim)\n",
        "        self.prediction_network = Prediction(state_dim*4, action_dim, width) \n",
        "        self.optimizer = torch.optim.AdamW(self.parameters(), lr=0.0003, weight_decay=0)\n",
        "        self.scheduler = PolynomialLRDecay(self.optimizer, max_decay_steps=1000, end_learning_rate=0.0000)   \n",
        "        self.to(device)\n",
        "\n",
        "        self.state_traj = []\n",
        "        self.action_traj = []\n",
        "        self.P_traj = []\n",
        "        self.r_traj = []      \n",
        "\n",
        "        ## replay buffer\n",
        "        self.state_replay = []\n",
        "        self.action_replay = []\n",
        "        self.P_replay = []\n",
        "        self.r_replay = []   \n",
        "\n",
        "        self.action_space = action_dim\n",
        "        self.env = gym.make(game_name)     \n",
        "\n",
        "        ## variance variables for normalized advantage\n",
        "        self.var = 0\n",
        "        self.beta_product = 1.0\n",
        "\n",
        "        self.var_m = [0 for _ in range(5)]\n",
        "        self.beta_product_m = [1.0 for _ in range(5)] \n",
        "\n",
        "\n",
        "    def self_play_mu(self, max_timestep=10000):     \n",
        "        \"\"\"Self-play and save trajectory to replay buffer\n",
        "\n",
        "        (Originally target network used to inference policy, but i used agent network instead)\n",
        "\n",
        "        Eight previous observations stacked -> representation network -> prediction network \n",
        "        -> sampling action follow policy -> next env step\n",
        "        \"\"\"   \n",
        "\n",
        "        game_score = 0\n",
        "        state = self.env.reset()\n",
        "        state_dim = len(state)\n",
        "        for i in range(max_timestep):   \n",
        "            start_state = state\n",
        "            if i == 0:\n",
        "                stacked_state = np.concatenate((state, state, state, state, state, state, state, state), axis=0)\n",
        "            else:\n",
        "                stacked_state = np.roll(stacked_state,-state_dim,axis=0)                \n",
        "                stacked_state[-state_dim:]=state\n",
        "\n",
        "            with torch.no_grad():\n",
        "                hs = self.representation_network(torch.from_numpy(stacked_state).float().to(device))\n",
        "                P, v = self.prediction_network(hs)    \n",
        "            action = np.random.choice(np.arange(self.action_space), p=P.detach().cpu().numpy())   \n",
        "            state, r, done, info = self.env.step(action)                    \n",
        "            \n",
        "            if i == 0:\n",
        "                for _ in range(8):\n",
        "                    self.state_traj.append(start_state)\n",
        "            else:\n",
        "                self.state_traj.append(start_state)\n",
        "            self.action_traj.append(action)\n",
        "            self.P_traj.append(P.cpu().numpy())\n",
        "            self.r_traj.append(r)\n",
        "\n",
        "            game_score += r\n",
        "\n",
        "            if done:\n",
        "                last_frame = i\n",
        "                break\n",
        "\n",
        "        #print('self_play: score, r, done, info, lastframe', int(game_score), r, done, info, i)\n",
        "\n",
        "\n",
        "        # for update inference over trajectory length\n",
        "        for _ in range(5):\n",
        "            self.state_traj.append(np.zeros_like(state))\n",
        "\n",
        "        for _ in range(6):\n",
        "            self.r_traj.append(0.0)\n",
        "            self.action_traj.append(-1)  \n",
        "\n",
        "        # traj append to replay\n",
        "        self.state_replay.append(self.state_traj)\n",
        "        self.action_replay.append(self.action_traj)\n",
        "        self.P_replay.append(self.P_traj)\n",
        "        self.r_replay.append(self.r_traj)  \n",
        "\n",
        "        writer.add_scalars('Selfplay',\n",
        "                           {'lastreward': r,\n",
        "                            'lastframe': last_frame+1\n",
        "                           },global_i)\n",
        "\n",
        "        return game_score , r, last_frame\n",
        "\n",
        "\n",
        "    def update_weights_mu(self, target):\n",
        "        \"\"\"Optimize network weights\n",
        "\n",
        "        Minibatch size: 16 (4 replay, 4 seqeuences in one replay)\n",
        "        Iteration: 20\n",
        "        Replay: uniform replay without on-policy data\n",
        "        Discount: 0.997\n",
        "        Unroll: 5 step\n",
        "        L_m: 5 step\n",
        "        \"\"\"\n",
        "\n",
        "        for _ in range(20): \n",
        "            state_traj = []\n",
        "            action_traj = []\n",
        "            P_traj = []\n",
        "            r_traj = []      \n",
        "            G_arr_mb = []\n",
        "\n",
        "            for epi_sel in range(4):\n",
        "                if(epi_sel>-1):## replay proportion\n",
        "                    sel = np.random.randint(0,len(self.state_replay)) \n",
        "                else:\n",
        "                    sel = -1\n",
        "\n",
        "                ## multi step return G (orignally retrace used)\n",
        "                G = 0\n",
        "                G_arr = []\n",
        "                for r in self.r_replay[sel][::-1]:\n",
        "                    G = 0.997 * G + r\n",
        "                    G_arr.append(G)\n",
        "                G_arr.reverse()\n",
        "                \n",
        "                for i in np.random.randint(len(self.state_replay[sel])-5-7,size=4):\n",
        "                    state_traj.append(self.state_replay[sel][i:i+13])\n",
        "                    action_traj.append(self.action_replay[sel][i:i+5])\n",
        "                    r_traj.append(self.r_replay[sel][i:i+5])\n",
        "                    G_arr_mb.append(G_arr[i:i+6])                        \n",
        "                    P_traj.append(self.P_replay[sel][i])\n",
        "\n",
        "\n",
        "            state_traj = torch.from_numpy(np.array(state_traj)).to(device)\n",
        "            action_traj = torch.from_numpy(np.array(action_traj)).unsqueeze(2).to(device)\n",
        "            P_traj = torch.from_numpy(np.array(P_traj)).to(device)\n",
        "            G_arr_mb = torch.from_numpy(np.array(G_arr_mb)).unsqueeze(2).float().to(device)\n",
        "            r_traj = torch.from_numpy(np.array(r_traj)).unsqueeze(2).float().to(device)\n",
        "            inferenced_P_arr = []\n",
        "\n",
        "            ## stacking 8 frame\n",
        "            stacked_state_0 = torch.cat((state_traj[:,0], state_traj[:,1], state_traj[:,2], state_traj[:,3],\n",
        "                                         state_traj[:,4], state_traj[:,5], state_traj[:,6], state_traj[:,7]), dim=1)\n",
        "\n",
        "\n",
        "            ## agent network inference (5 step unroll)\n",
        "            first_hs = self.representation_network(stacked_state_0)\n",
        "            first_P, first_v_logits = self.prediction_network(first_hs)      \n",
        "            inferenced_P_arr.append(first_P)\n",
        "\n",
        "            second_hs, r_logits = self.dynamics_network(first_hs, action_traj[:,0])    \n",
        "            second_P, second_v_logits = self.prediction_network(second_hs)\n",
        "            inferenced_P_arr.append(second_P)\n",
        "\n",
        "            third_hs, r2_logits = self.dynamics_network(second_hs, action_traj[:,1])    \n",
        "            third_P, third_v_logits = self.prediction_network(third_hs)\n",
        "            inferenced_P_arr.append(third_P)\n",
        "\n",
        "            fourth_hs, r3_logits = self.dynamics_network(third_hs, action_traj[:,2])    \n",
        "            fourth_P, fourth_v_logits = self.prediction_network(fourth_hs)\n",
        "            inferenced_P_arr.append(fourth_P)\n",
        "\n",
        "            fifth_hs, r4_logits = self.dynamics_network(fourth_hs, action_traj[:,3])    \n",
        "            fifth_P, fifth_v_logits = self.prediction_network(fifth_hs)\n",
        "            inferenced_P_arr.append(fifth_P)\n",
        "\n",
        "            sixth_hs, r5_logits = self.dynamics_network(fifth_hs, action_traj[:,4])    \n",
        "            sixth_P, sixth_v_logits = self.prediction_network(sixth_hs)\n",
        "            inferenced_P_arr.append(sixth_P)\n",
        "\n",
        "\n",
        "            ## target network inference\n",
        "            with torch.no_grad():\n",
        "                t_first_hs = target.representation_network(stacked_state_0)\n",
        "                t_first_P, t_first_v_logits = target.prediction_network(t_first_hs)  \n",
        "\n",
        "\n",
        "            ## normalized advantage\n",
        "            beta_var = 0.99\n",
        "            self.var = beta_var*self.var + (1-beta_var)*(torch.sum((G_arr_mb[:,0] - to_scalar(t_first_v_logits))**2)/16)\n",
        "            self.beta_product *= beta_var\n",
        "            var_hat = self.var/(1-self.beta_product)\n",
        "            under = torch.sqrt(var_hat + 1e-12)\n",
        "\n",
        "\n",
        "            ## L_pg_cmpo first term (eq.10)\n",
        "            importance_weight = torch.clip(first_P.gather(1,action_traj[:,0])\n",
        "                                        /(P_traj.gather(1,action_traj[:,0])),\n",
        "                                        0, 1\n",
        "            )\n",
        "            first_term = -1 * importance_weight * (G_arr_mb[:,0] - to_scalar(t_first_v_logits))/under\n",
        "\n",
        "\n",
        "            ##lookahead inferences (one step look-ahead to some actions to estimate q_prior, from target network)\n",
        "            with torch.no_grad():                                \n",
        "                r1_arr = []\n",
        "                v1_arr = []\n",
        "                a1_arr = []\n",
        "                for _ in range(self.action_space): #sample <= N(action space), now N    \n",
        "                    action1_stack = []\n",
        "                    for p in t_first_P:             \n",
        "                        action1_stack.append(np.random.choice(np.arange(self.action_space), p=p.detach().cpu().numpy()))    \n",
        "                    hs, r1 = target.dynamics_network(t_first_hs, torch.unsqueeze(torch.tensor(action1_stack),1))\n",
        "                    _, v1 = target.prediction_network(hs)\n",
        "\n",
        "                    r1_arr.append(to_scalar(r1))\n",
        "                    v1_arr.append(to_scalar(v1) )\n",
        "                    a1_arr.append(torch.tensor(action1_stack))               \n",
        "\n",
        "            ## z_cmpo_arr (eq.12)\n",
        "            with torch.no_grad():   \n",
        "                exp_clip_adv_arr = [torch.exp(torch.clip((r1_arr[k] + 0.997 * v1_arr[k] - to_scalar(t_first_v_logits))/under, -1, 1))\n",
        "                                    .tolist() for k in range(self.action_space)]\n",
        "                exp_clip_adv_arr = torch.tensor(exp_clip_adv_arr).to(device)\n",
        "                z_cmpo_arr = []\n",
        "                for k in range(self.action_space):\n",
        "                    z_cmpo = (1 + torch.sum(exp_clip_adv_arr[k],dim=0) - exp_clip_adv_arr[k]) / self.action_space \n",
        "                    z_cmpo_arr.append(z_cmpo.tolist())\n",
        "            z_cmpo_arr = torch.tensor(z_cmpo_arr).to(device)\n",
        "\n",
        "\n",
        "            ## L_pg_cmpo second term (eq.11)\n",
        "            second_term = 0\n",
        "            for k in range(self.action_space):\n",
        "                second_term += exp_clip_adv_arr[k]/z_cmpo_arr[k] * torch.log(first_P.gather(1, torch.unsqueeze(a1_arr[k],1).to(device)))\n",
        "            regularizer_multiplier = 5 \n",
        "            second_term *= -1 * regularizer_multiplier / self.action_space\n",
        "\n",
        "\n",
        "            ## L_pg_cmpo               \n",
        "            L_pg_cmpo = first_term + second_term\n",
        "\n",
        "\n",
        "            ## L_m\n",
        "            L_m  = 0\n",
        "            for i in range(5):\n",
        "                stacked_state = torch.cat(( state_traj[:,i+1], state_traj[:,i+2], state_traj[:,i+3], state_traj[:,i+4],\n",
        "                                            state_traj[:,i+5], state_traj[:,i+6], state_traj[:,i+7], state_traj[:,i+8]), dim=1)\n",
        "                with torch.no_grad():\n",
        "                    t_hs = target.representation_network(stacked_state)\n",
        "                    t_P, t_v_logits = target.prediction_network(t_hs) \n",
        "\n",
        "                beta_var = 0.99\n",
        "                self.var_m[i] = beta_var*self.var_m[i] + (1-beta_var)*(torch.sum((G_arr_mb[:,i+1] - to_scalar(t_v_logits))**2)/16)\n",
        "                self.beta_product_m[i]  *= beta_var\n",
        "                var_hat = self.var_m[i] /(1-self.beta_product_m[i])\n",
        "                under = torch.sqrt(var_hat + 1e-12)\n",
        "\n",
        "                with torch.no_grad():                                \n",
        "                    r1_arr = []\n",
        "                    v1_arr = []\n",
        "                    a1_arr = []\n",
        "                    for j in range(self.action_space):  \n",
        "                        action1_stack = []\n",
        "                        for _ in t_P:             \n",
        "                            action1_stack.append(j)    \n",
        "                        hs, r1 = target.dynamics_network(t_hs, torch.unsqueeze(torch.tensor(action1_stack),1))\n",
        "                        _, v1 = target.prediction_network(hs)\n",
        "\n",
        "                        r1_arr.append(to_scalar(r1))\n",
        "                        v1_arr.append(to_scalar(v1) )\n",
        "                        a1_arr.append(torch.tensor(action1_stack))    \n",
        "\n",
        "                with torch.no_grad():   \n",
        "                    exp_clip_adv_arr = [torch.exp(torch.clip((r1_arr[k] + 0.997 * v1_arr[k] - to_scalar(t_v_logits))/under,-1, 1))\n",
        "                                        .tolist() for k in range(self.action_space)]\n",
        "                    exp_clip_adv_arr = torch.tensor(exp_clip_adv_arr).to(device)\n",
        "\n",
        "                ## Paper appendix F.2 : Prior policy\n",
        "                t_P = 0.967*t_P + 0.03*P_traj + 0.003*torch.tensor([[0.5,0.5] for _ in range(16)]).to(device) \n",
        "\n",
        "                pi_cmpo_all = [(t_P.gather(1, torch.unsqueeze(a1_arr[k],1).to(device)) \n",
        "                                * exp_clip_adv_arr[k])\n",
        "                                .squeeze(-1).tolist() for k in range(self.action_space)]                \n",
        "        \n",
        "                pi_cmpo_all = torch.tensor(pi_cmpo_all).transpose(0,1).to(device)\n",
        "                pi_cmpo_all = pi_cmpo_all/torch.sum(pi_cmpo_all,dim=1).unsqueeze(-1)\n",
        "                kl_loss = torch.nn.KLDivLoss(reduction=\"batchmean\")\n",
        "                L_m += kl_loss(torch.log(inferenced_P_arr[i+1]), pi_cmpo_all) \n",
        "            \n",
        "            L_m/=5\n",
        "\n",
        "\n",
        "            ## L_v\n",
        "            ls = nn.LogSoftmax(dim=-1)\n",
        "\n",
        "            L_v = -1 * (\n",
        "                (to_cr(G_arr_mb[:,0])*ls(first_v_logits)).sum(-1, keepdim=True)\n",
        "                +  (to_cr(G_arr_mb[:,1])*ls(second_v_logits)).sum(-1, keepdim=True)\n",
        "                +  (to_cr(G_arr_mb[:,2])*ls(third_v_logits)).sum(-1, keepdim=True)\n",
        "                +  (to_cr(G_arr_mb[:,3])*ls(fourth_v_logits)).sum(-1, keepdim=True)\n",
        "                +  (to_cr(G_arr_mb[:,4])*ls(fifth_v_logits)).sum(-1, keepdim=True)\n",
        "                +  (to_cr(G_arr_mb[:,5])*ls(sixth_v_logits)).sum(-1, keepdim=True)\n",
        "            )\n",
        "\n",
        "            ## L_r     \n",
        "            L_r = -1 * (\n",
        "                (to_cr(r_traj[:,0])*ls(r_logits)).sum(-1, keepdim=True)\n",
        "                + (to_cr(r_traj[:,1])*ls(r2_logits)).sum(-1, keepdim=True)\n",
        "                + (to_cr(r_traj[:,2])*ls(r3_logits)).sum(-1, keepdim=True)\n",
        "                + (to_cr(r_traj[:,3])*ls(r4_logits)).sum(-1, keepdim=True)\n",
        "                + (to_cr(r_traj[:,4])*ls(r5_logits)).sum(-1, keepdim=True)\n",
        "            )\n",
        "\n",
        "\n",
        "            ## start of dynamics network gradient *0.5\n",
        "            first_hs.register_hook(lambda grad: grad * 0.5)\n",
        "            second_hs.register_hook(lambda grad: grad * 0.5) \n",
        "            third_hs.register_hook(lambda grad: grad * 0.5)    \n",
        "            fourth_hs.register_hook(lambda grad: grad * 0.5)   \n",
        "            fifth_hs.register_hook(lambda grad: grad * 0.5)  \n",
        "            sixth_hs.register_hook(lambda grad: grad * 0.5)  \n",
        "\n",
        "\n",
        "            ## total loss\n",
        "            L_total = L_pg_cmpo + L_v/6/4 + L_r/5/1 + L_m   \n",
        "          \n",
        "            \n",
        "\n",
        "            ## optimize\n",
        "            self.optimizer.zero_grad()\n",
        "            L_total.mean().backward()\n",
        "            nn.utils.clip_grad_value_(self.parameters(), clip_value=1.0)\n",
        "            self.optimizer.step()\n",
        "            \n",
        "\n",
        "            ## target network(prior parameters) moving average update\n",
        "            alpha_target = 0.01 \n",
        "            params1 = self.named_parameters()\n",
        "            params2 = target.named_parameters()\n",
        "            dict_params2 = dict(params2)\n",
        "            for name1, param1 in params1:\n",
        "                if name1 in dict_params2:\n",
        "                    dict_params2[name1].data.copy_(alpha_target*param1.data + (1-alpha_target)*dict_params2[name1].data)\n",
        "            target.load_state_dict(dict_params2)          \n",
        "\n",
        "\n",
        "        self.scheduler.step()\n",
        "\n",
        "        self.state_traj.clear()\n",
        "        self.action_traj.clear()\n",
        "        self.P_traj.clear()\n",
        "        self.r_traj.clear()\n",
        "\n",
        "        writer.add_scalars('Loss',{'L_total': L_total.mean(),\n",
        "                                  'L_pg_cmpo': L_pg_cmpo.mean(),\n",
        "                                  'L_v': (L_v/6/4).mean(),\n",
        "                                  'L_r': (L_r/5/1).mean(),\n",
        "                                  'L_m': (L_m).mean()\n",
        "                                  },global_i)\n",
        "        \n",
        "        writer.add_scalars('vars',{'self.var':self.var,\n",
        "                                   'self.var_m':self.var_m[0]\n",
        "                                  },global_i)\n",
        "        \n",
        "        return\n",
        "\n",
        "\n",
        "\n",
        "%rm -rf scalar/\n",
        "%load_ext tensorboard\n",
        "%tensorboard --logdir scalar --port=6009\n",
        "\n",
        "device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')\n",
        "score_arr = []\n",
        "game_name = 'CartPole-v1' \n",
        "env = gym.make(game_name) \n",
        "target = Target(env.observation_space.shape[0], env.action_space.n, 128)\n",
        "agent = Agent(env.observation_space.shape[0], env.action_space.n, 128)  \n",
        "print(agent)\n",
        "env.close()\n",
        "\n",
        "## initialization\n",
        "target.load_state_dict(agent.state_dict())\n",
        "\n",
        "## Self play & Weight update loop\n",
        "episode_nums = 1000\n",
        "for i in range(episode_nums):\n",
        "    writer = SummaryWriter(logdir='scalar/')\n",
        "    global_i = i    \n",
        "    game_score , last_r, frame= agent.self_play_mu()       \n",
        "    writer.add_scalar('score', game_score, global_i)\n",
        "    \n",
        "    score_arr.append(game_score)  \n",
        "    if i%10==0:\n",
        "        mean_score = np.mean(np.array(score_arr[i-30:i]))\n",
        "        #print('episode, avg, score, last_r, len\\n', i, mean_score, int(game_score), last_r, frame)\n",
        "\n",
        "    if i%100==0:\n",
        "        torch.save(agent.state_dict(), 'weights.pt') \n",
        "\n",
        "    if mean_score > 475 and np.mean(np.array(score_arr[-7:])) > 475:\n",
        "        torch.save(agent.state_dict(), 'weights.pt')\n",
        "        print('Done')\n",
        "        break\n",
        "\n",
        "    agent.update_weights_mu(target) \n",
        "    writer.close()\n",
        "\n",
        "torch.save(agent.state_dict(), 'weights.pt') \n",
        "agent.env.close()\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Earned score per episode \n",
        "window = 30\n",
        "mean_arr = []\n",
        "for i in range(len(score_arr) - window + 1):\n",
        "    mean_arr.append(np.mean(np.array(score_arr[i:i+window])))\n",
        "for i in range(window - 1):\n",
        "    mean_arr.insert(0, np.nan)\n",
        "\n",
        "plt.plot(score_arr, label ='score')\n",
        "plt.plot(mean_arr, label ='mean')\n",
        "plt.legend(loc='upper left')"
      ],
      "metadata": {
        "id": "F2MbdWmPFoEK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "outputId": "5b8a34b5-3a2e-48f9-858f-4bc40271d92a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f86b5a5ac10>"
            ]
          },
          "metadata": {},
          "execution_count": 6
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD4CAYAAAANbUbJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydeZwcZZ3/309Vd0/PTGYyk4PJMbmAHBxJCJBwhGAMIiIoqIAiqHihgqyuq6L+ZMVjvdb13kXZlUNEF4RFbuRGrkAC5CaEECbJ5E4mmXu6u6qe3x9VT3VVX9M905PMZJ7365XXdFdXVVcPw6e+/fkej5BSotFoNJrDG+NQX4BGo9FoBh4t9hqNRjMM0GKv0Wg0wwAt9hqNRjMM0GKv0Wg0w4DIob4AgDFjxsipU6ce6svQaDSaIcUrr7yyV0o5tph9B4XYT506leXLlx/qy9BoNJohhRBic7H7ahtHo9FohgFa7DUajWYYoMVeo9FohgGDwrPPRSqVorm5mZ6enkN9KQedeDxOY2Mj0Wj0UF+KRqM5TBi0Yt/c3ExNTQ1Tp05FCHGoL+egIaVk3759NDc3M23atEN9ORqN5jChKBtHCNEkhFgthFghhFjubRslhHhMCPGm97Pe2y6EEL8WQmwUQqwSQpzYlwvr6elh9OjRw0roAYQQjB49elh+o9FoNANHKZ79O6WUJ0gpT/aefwN4Qko5HXjCew5wLjDd+3clcENfL264Cb1iuH5ujUYzcPTHxrkAWOw9vhV4GrjW2/5H6c5OXiqEqBNCjJdS7ujPhWo0msOfR9bsZN32Vs6bM4HW7hTPvbkHgPnTRjF1dDVv7+3kzBljae1K8Y8393D2sQ3cv3I7F53UmDdIemTNDtZtb/OfL551BCdOrueNne08uGo7ANMbanjf3Al0JCyeeH0X7z52HLe80ER30hrwz3zWMQ3MnVQ34O9TrNhL4FEhhAR+L6W8EWgICPhOoMF7PBHYGji22dsWEnshxJW4kT+TJ0/u29VrNJrDimvvXkVrd4ptB3rY0tLJsqb9AMxsqGHh0WP424ptvHrd2Xz+T6/w4qZ9fP+C47ju3rXMnVTHjIaanOf81j1raOlMIgRICSuaW/njpxZw4z82cferzQDEIgbvmzuBh1fv4Gt3reKnH5rDTx5ZD8BAf9E+ojY+qMT+DCnlNiHEEcBjQoj1wRellNK7ERSNd8O4EeDkk08+7FdQsSyLSGTQ5sM1mkGB47hSYDsOKVuyaPoYauNR1u9sI2U7pGwHgDd3twOQst39k5aT95wp2+FTC6fxr+87lkt+9yIpb9+U7TB1dBUXzpvILx9/E8eR9KRsAFq7UwA8cM0ZHD9x5MB82INMUZ69lHKb93M3cA+wANglhBgP4P3c7e2+DZgUOLzR2zbk6Ozs5LzzzmPu3Lkcf/zx3HHHHSxbtozTTz+duXPnsmDBAtrb2+np6eGTn/wks2fPZt68eTz11FMA3HLLLbz//e9nyZIlnHXWWXR2dvKpT32KBQsWMG/ePO69995D/Ak1msFFynGF2JFuZZohhB+RO1K6HgPQkbC8/dwNlpM/XpQSDC86Nw2B5b2H7UgMQxA1XRlM2g5J7+bRlXRFX712ONBrqCmEqAYMKWW79/jdwPeA+4BPAD/2firlug/4ohDif4FTgNb++vXfvX9tyHMrB8dOqOU77zuu4D6PPPIIEyZM4MEHHwSgtbWVefPmcccddzB//nza2tqorKzkV7/6FUIIVq9ezfr163n3u9/Nhg0bAHj11VdZtWoVo0aN4lvf+hZLlizhpptu4sCBAyxYsIB3vetdVFdXl/WzaTRDFdsTbUdKHE+kDSH850rce1KuYFuBbwL5cKQr6gARU9BjqWMkEUNQEXEFPWU7/jeEbi/Cj5gD5OFYCbCT7mOzAiKxgXmfAMX4Cg3APV7yIwL8WUr5iBBiGXCnEOLTwGbgEm//h4D3AhuBLuCTZb/qg8Ts2bP5l3/5F6699lrOP/986urqGD9+PPPnzwegtrYWgOeee45rrrkGgFmzZjFlyhRf7M8++2xGjRoFwKOPPsp9993Hz372M8AtL92yZQvHHHPMwf5oGs2gRIm3iuQNITBEOtLPDOAtW0Xp+c/pSOn77hFDYHnRu+2dP+aJfdJK20TKzokaAxDZ73kDfrcI7IT7/Lyfw/xPl/99MuhV7KWUm4C5ObbvA87KsV0CV5fl6jx6i8AHihkzZvDqq6/y0EMP8e1vf5slS5aUfI5g1C6l5O6772bmzJnlvEyN5rDAdiRe4O5H8kKIQGQv/cheoTx7q2Bk7347ADANI/BtQBIxBbGAjaPEvjs5gJH9sv8BJJz9PRAGTFpQ/vfIweFjSA0A27dvp6qqissvv5yvfe1rvPTSS+zYsYNly5YB0N7ejmVZLFq0iNtvvx2ADRs2sGXLlpyCfs455/Cb3/wG6f3Bvvbaawfvw2g0g5ygYDtSep69K/hupO9b9lnHFNB6HEf6nn3UFIFvAxIzI7IfcBsn2QUr74BjL4CFX4LTr4Fxs8v7HnnQ5SEFWL16NV/72tcwDINoNMoNN9yAlJJrrrmG7u5uKisrefzxx7nqqqv4whe+wOzZs4lEItxyyy1UVFRkne+6667jy1/+MnPmzMFxHKZNm8YDDzxwCD6ZRjP4sAMejZNl40j/BhDEKiqyl4HIXvjvYzsS08gQe+9G4Cdoy23jvHILJFrh5IG3bTLRYl+Ac845h3POOSdr+9KlS7O23XzzzVnbrrjiCq644gr/eWVlJb///e/Leo0azeGCsmQg7c8bRjpBq6L7XMfYBapxlB0EbnVNKlCNYxppGycR8Oy7U261TzTSD7Hv2AOv3xv+2vHsf8C0M2HKaX0/bx/RYq/RaAYFuSJ7IQSGkX4uM6L7YBllLtS+wdJLO3CDMITwBT0ZrMZRnr3RDxvn+V/Ci78NbxMmnPWdvp+zH2ix12g0g4JMzx4vsep69qr0Mm2xQO+Rvdps+JG9SCdopSQeMagwg9U47mvdXmlnv+rsm56FyafBh29Pb4tUQMWIvp+zH+gErUajGRRYdi7PHr/0UlXitPWkAsd4kb3MJ/bZkX2wGiez9DIZKL0Uwt2/T/S0ws7VrmVTPTr97xAJPWix12g0g4RgdO579oHSS2XJqFEGQEi4c6HEXnn2EcMIVeNEAgnaVIaN06/k7NaXQTowZWHfz1FmtNhrNJpBgRXy7KXfDGWo0kvP5WntSou9SqgGvxUEkRk2TiQjss+sxkknaO3+lV02PQtGFBrn9/0cZUaLvUajGRRYgTZYx1EzbdzZOMGGqgPByD7QDZsLdYyy3k0zh9ibuRO0/UrOvvUkTDoFYlV9P0eZ0WKv0WgGBbkieyMY2Xsvh22cwtU4WQnaoI0jw5F9sPQyaTv+9pLp2O369UeX3nE/kGix12g0g4KwZ5/dVKU8+7buoI1TeOplpmdvGsJN9joynaA1gwna9HkiffXs33rS/XlU1jSZQ4oW+wI0NTUxa9YsrrjiCmbMmMFll13G448/zsKFC5k+fTovv/xy3rHFTU1NLFq0iBNPPJETTzyRF154AYCnn36axYsXc9FFFzFr1iwuu+yyrK5AjWY4krLDpZe5ZuNAWOzT4xLyePbeKYPjEtzjZFaCNjguAfoxKmH9A1A9FsbN6dvxA8TQqLN/+Bvu16JyMm42nPvjXnfbuHEjf/3rX7npppuYP38+f/7zn3nuuee47777+OEPf8ixxx6bc2zxEUccwWOPPUY8HufNN9/k0ksvZfny5YA7E2ft2rVMmDCBhQsX8vzzz3PGGWeU9/NpNEMMO8PGCc7GcQI2TiqwX2+Rve2XXqYHoan3UvPsY4GmquANp0819h174I2H4ZTPu+2/g4ihIfaHkGnTpjF7tjuo6LjjjuOss85CCMHs2bNpamqiubk559jiCRMm8MUvfpEVK1ZgmqY/8hhgwYIFNDY2AnDCCSfQ1NSkxV4z7FHCHTUFEgKll6oU05t1HxB25b/ni+wz6+xVZJ9ynHRkb2ZX40Afu2dX/gUcC078eOnHDjBDQ+yLiMAHiuBAM8Mw/OeGYWBZFqZp5hxbfP3119PQ0MDKlStxHId4PJ7znKZpYlkDv6ixRjPYUZF91DQymqqEN8/e3S8YxavHpXj2ALYt/QRtxDQwRLjOHiDSl8j+jYdg/FwYO/jGmA+u7xlDkHxji1tbWxk/fjyGYXDbbbdh23ah02g0wx7lv8cihhvJO95snIzSSzuHjZNvpaqsOntPwJVnr8Q/FjGyIvtoqZ59sgual8O0d5R23EFCi30/ue6660ilUsyZM4fjjjuO6667DoCrrrqKW2+9lblz57J+/Xq99KBG0wuWHYzsZaDO3i29DI4mTh9TeKWqTBtHWTOWZ+OY3k0gZhokMhK0JXv2zS+Dk3JHJAxChoaNc4iYOnUqa9as8Z/fcsstOV/LNbZ4+vTprFq1yn/+k5/8BIDFixezePFif/tvf/vbzEM1mmGJsmJipoHjhG0cyL24ePoGkFvtM+vsfbG30wlagFjE9BYc74dn3/ScO9Vy0imlHXeQ0JG9RqMpSHtPivf+6lne2Nk+oO+jhDsWMdILjhvCj8pzLS7uz6bP10HrKM/efR7JUXoJEDNFaOollBjZW0lYdae7xGC8tvjjDiJa7DUaTUF2tvawbkcb63e2Dej7KM8+agq/qUoI/Og7HcUHjuml9DJ7No4qvXTcBccDnn13yg5ZRCXV2b/2RziwGRb9S/HHHGQGtdgP12aj4fq5NYMTpX8D/WeZ37MPvx6K7NW2PIPQfM/eUzoVyafsjMg+YtCZCFfFldRB+8qtMOFEOPpdxR9zkBm0Yh+Px9m3b9+wEz4pJfv27QuVamo0hxK/vn2A/1/MtnEkgnRUbucos7R6s3GymqrCnr2foM0h9kVX4/S0wa41MP3stF80CBm0CdrGxkaam5vZs2fPob6Ug048HvebrjSaQ01a7Af2fVK+jWNkzcaBwGiE4LKEJa9Ule6WhXRHbcw06EiEy6OLrrNv9mbXTz7468qWwqAV+2g0yrRp0w71ZWg0wx7p2zgHJ7Kv8CJ7CaFqnFyll/48+17XoA1H9gnL9p67+8UiBns7kqFjo8VW42x+0a3CGUSz63MxaG0cjUYzOFCR9MH07G3PsxdenT3kXpVKbcs3LiE9G8d9rpKuCSsjso+YOWycIuXx7X+4XbOHcMnBYtBir9FoCqJEfqA9+2A1juNPq0zbOE4Ozz6Xjx9EnUdkVOMkUkrs3ddjpqAjM0FbjGe/c41r4xx3Ye/7HmIGrY2j0WgGBwfLs/ebqiKmL/xBG6fQerPFDkJLR/a2tz2doE1Y4casXiP7tffAS7+HSBzmfazwvoMAHdlrNJqCOGWO7HtSNt+9fy1tPanQdjsw9VIJerCpKpdnryi+zj5s46SbqrKlsGAHbbIL7vk8NC+DUz4HVaPy7ztI0GKv0WgKIn3Pvjxiv3Z7Kzc/38TyppbQdjWnPmoYvngLQZZnn0vY81fjZNbZp8cZA6FBaIoK73HBapymZ8HqgY/eCWd/L/9+gwgt9hqNpiDpyL4851ONUKmMRijbcYgYAsMQfpRveCtVua+nk7GZUXdvYu979nkTtGkprIqZQC919m8+CtEqmLKw0EcdVGjPXqPRFKTcTVVKmFMZoyotb+SwIdLRu+vZh4+zHEnEFDnn2mdfO955Mm2cjNJL0/SPqYpF2N+Vyt1Bu+Hv8MT3Yc96t4kqOnSaH4uO7IUQphDiNSHEA97zaUKIl4QQG4UQdwghYt72Cu/5Ru/1qQNz6RqN5mBQ7gStXxufEdlbthuxGyLg2eeI7N0xB2Hpyj/PPnPEcWY1TnZkX13hRfaRjMi+qwX+96NgJ9yVqN5xbZGfeHBQio3zJeD1wPOfAL+QUh4N7Ac+7W3/NLDf2/4Lbz+NRjNEKXdTVb7I3nakv2qUqsYRQkBGB63tRfahY/NcWmZkb2bZOO7rscD5KmOu4RHNjOx3rHSXHDz3p3D+z2HCCcV83EFDUWIvhGgEzgP+x3sugCXAXd4utwKq0PQC7zne62cJMYgHRmg0moKUu6kq3yLhKdv17NUC4xAuvQxWBWV79vnm2YdHHEczbBx17lHVMf+Yqqgb2WfV2e9Y4f4cP7fXzzgYKTay/yXwdUD9RkcDB6SUqguhGZjoPZ4IbAXwXm/19tdoNEOQcpdeqgjdyhHZm0batoFwU1X6+Fw2TuEErZk1LkGVXrrnOeuYBv8YZeNkVePsWAl1U4ZEmWUuehV7IcT5wG4p5SvlfGMhxJVCiOVCiOXDcdiZRjNUKLdnn7ZxMjx7RxL1bBxFMLIPHm8WWY3j19kb4TVoMztoJ42q8o9J2ziB97At2L5iyEb1UFxkvxB4vxCiCfhfXPvmV0CdEEJV8zQC27zH24BJAN7rI4F9mSeVUt4opTxZSnny2LFj+/UhNBrNwCHLXI2TtnEyqnFsx43sAyIrAvPsFbYjs8oi81Xj2E5mgjazGictgefPGQ/AyEpX1vzIvmMP/HQa7H97yPn0QXoVeynlN6WUjVLKqcBHgCellJcBTwEXebt9ArjXe3yf9xzv9SflcBtKr9EcRihNLtf/xsq+yRXZRwxBUMaD1TiKXJF9b+MSMuvskxkJWoBffWQeT391MSMro0Cgzr7pWUi0wcIvw0mfLPJTDj7601R1LfAVIcRGXE/+D972PwCjve1fAb7Rv0vUaDSHEiWj5bJxrILVOOkpl5DbxrEcJ0vsS12WMLOpyn0smDqm2p+J48/G2fqS20C15NtD1q+HEpuqpJRPA097jzcBC3Ls0wNcXIZr02g0g4ByN1VZeersU7bENDI9++wEre2ktyuN73VcgncO03BtId/GyVEoqETer/jZ8iJMPAnMaNGfcTCiO2g1Gk1BZJkTtH5k72RG9g5RM2zbBGfjBPcTQmAaAqfElarAFXEV2edqkq10OrnKvJfpry+F3VXuGONFXyntQw5CtNhrNJqCpBccL1fppZegzeHZq3EJinyll4Z/E+hN7MN19uBaOaoaJ1TCaafAjDJn+52cHL0D1njbzQqYfk5pH3IQosVeo9EUZOBsnMxqHOk3VSkMI9uzdxx3bdqgBZNvwfHMZQlBRfaB2ThdLXDP59xxxZ95ghk77uMlZxbOxx/ktKO8FqHDoC9UT73UaDQFSUf25Tlf2sbJnHrpNktlNVVlqJQtZWhAGmR/S1DktHFMEU7QPv0jeOspcGy45Txqu7Zwl30m0YihfKS+ftRBhRZ7jUZTkLJ79mrEsZU59dIhYoZtm+AatArbkQgRrscvNkELrsD7pZdCwMbH4aglcMmtMKKBXaNP4UH71MLz7Icgh9en0Wg0ZSc9G6fMnr2Tx7M3gpF9rtJLmbU9n42j3kLkSdDGO5uhZRMc9U5X8D/3DC+ccRNdxAuvVDUE0WKv0WgKoopmyu3ZZ82zt10bR/SSoJUSvxpHkX9cQnZk79o4rmdfs+1Zd+NRS/zX6yrdoWiquepwQYu9RqMpSLln4+SrxrGd9Dx7hSA7sgeyPPteB6EZuSP7mqZHYOQkGDPDf/0dM8Zy/xfPCM3LORzQYq/RaAqiAvqyT73MqLNPOQ5mTs8++xwiY4xCXrH33iKcoDWQEo5gP1Vbn4E5Hw4lYQ1DMLtxZKkfa9CjxV6j0RSk3PPsrbxr0EqiWSOOC0X26e2ZNw7/nDnr7N0nHzCfQ0gHTvho3z7IEEOLvUajKUi559nnn3rpjksQmaWXOcU+/A0gj9bnrrP3Bpy9y3yFZMNcGH1Unz7HUEOLvUajKYik3AuO55t66XiefXqb21SVfQ4jo/QyX2Sfq87eNAyq6OEE8RbJye/o46cYemix12g0BUlH9uU5n2qmyjf1MjwbJ7vO3t0eFnBH5i4Nzaqz79hDPe0sMNYTFTbWlEX9/ThDBj0uQaPRFESW2bO37dzVOClbZkf2OUov1fbcq1XJUDNUqM5eSrjtA/x672bWmo0kZBSn8ZSyfKahgI7sNRpNQdTCIOVrqspdZ+8uSpLp2edP0GZu3tHaw7Hf+TuvbtnvbwvV2e9eB7tWEyPJycYb3Gyfg1lxeJVXFkJH9hqNpiADl6DN4dmbuRYcz52gzZxFv6uth6TlsG1/NydOrnev2QkkaNf8HwiDb0++jfvf6CRBjMsPsy7ZQujIXqPRFGSgFhzPN/UyXGefew5ZpmcP6dWngjX3oQTt+gdh6iJkdQMJ3C7ZXIuXHK5osddoNAUpd1NVKscatFJKdw1aM8fUy5wJ2nA1DqTXlbVCYu/V2afaYc96mHI6NfG0oZHp+x/OaLHXaDQFKXtTlT8IzcnaFjVE9mycHCoVHJcQM8PrytqB86prjuxZC0iYMI9aLfYajUaTTbk9+/SC4+nzqcqc7Mg+X4I2XY0Tiyixt7POq645snOlu2H8CdTEo4Hz9PvjDBm02Gs0moKUfRBajqmXaj3aqBmO5EWB0ktVtRP1OmKTBTx7c+dKqBkPNQ0hGydXDf/hihZ7jUZTkPTiJeXqoM2us/cj+xyzcfI1VXkaT9SzcZJ2fs/e2LECxp8AEIrshxNa7DUaTUHKveB4ys6eeqmi/YhZymwcFdl7Nk4q27N3HEk13bDvTZjgiv2I+PCsONdir9FoCjJQCdqULf0biBqhEDWL7aDFr8ZRnn3uyB6OFZsRXnIWCNk4wwkt9hqNpiBln2cfsG8ya+4zFxzPVU8P4ZtAlmefkaCdbbztPvFsnFot9hqNRpNN2Rccz1FymfKrcbIj+5w5VJEum/RtHK8aJxjZSyX2XnIWtGev0Wg0OSm3Zx+M7JMZ/n3EMHAXI3RxRxwX59nnq8aZbWzyo3rQNo5Go9HkZKDWoIW08FsFIvveVqqKZYh98PwRq4Mj2eEnZwEqo2Z5PsgQQ4u9RqMpSNmbqmzHF1wrI6kazRqEVmDxEtVBG8nfQTumYwOGSCdnYXjV1gfRYq/RaApSfs9eUhlzxT6VK0Gb0VRFDm0OLjieKfbByH5c53r3QcDGGa5osddoNAVJl16Wb1xC3BNoK2MoWsQURdbZB0ovMxO0gZxAQ+d6dsl6Pzk7nNFir9FoClJOG0dKie1I4iqyz1h8PFrCbJwsGyeVHdmP73yddRzZ7+s+HOhV7IUQcSHEy0KIlUKItUKI73rbpwkhXhJCbBRC3CGEiHnbK7znG73Xpw7sR9BoNANJOZuqlBDHI55n74l8eFxCen9B/qaqzEFoPV5k73v2a+5mTM9mlonj+n/hhwHFRPYJYImUci5wAvAeIcSpwE+AX0gpjwb2A5/29v80sN/b/gtvP41GM0QpZ1OVEnXfs7fCi49nRvYi72yctN3ji70X2UsrAX+6CO7+DFuqj+dOcW7W8X//8pnc/YXT+v15hhK9ir106fCeRr1/ElgC3OVtvxW40Ht8gfcc7/WzxHBNf2s0hwHlLL1U0y2r/ARt2HpxPfv0/oaRO7J3B6Hl9uynH3gONj7G9hmXc/u0H+MY2U1UM8fVcNKUUf3/QEOIojx7IYQphFgB7AYeA94CDkgpLW+XZmCi93gisBXAe70VGJ3jnFcKIZYLIZbv2bOnf59Co9EMGOVM0KpRBhXKxrHDkX3muIRiPPuKDM/+xP2PsFPW89CEf6LDrBtWM+sLUZTYSyltKeUJQCOwAJjV3zeWUt4opTxZSnny2LFj+3s6jUYzQKQTtP0/l4rklY3j19nnHXFcRDVOoPRygXidExLL+Zu9kB7bvWZtLLiUVI0jpTwAPAWcBtQJIVTfcSOwzXu8DZgE4L0+EthXlqvVaDQHnXJ69rafoPUE2nb4f/esZu32NiC7gzbfguPBm4CycUantnFb7EdsN8Zzk3UuCctBSqkje49iqnHGCiHqvMeVwNnA67iif5G32yeAe73H93nP8V5/UparQFej0Rx0ytlUlZmg3dOe4PaXtvDk+l2Am6Atps5eCJH27L0bx5mpF6kQFl+JXsdu6klYDo6UOY8fjhQzEWg8cKsQwsS9OdwppXxACLEO+F8hxA+A14A/ePv/AbhNCLERaAE+MgDXrdFoDhLl9OyVN6/GJXQm3LRfR8JNrmaWXhaeZ+8+jkUMDBzOlMtYI6fSZI8GkiRSNo7M7fkPR3oVeynlKmBeju2bcP37zO09wMVluTqNRnPIKWdTlW/jZIi9+lnKguNq+8S2Fayv+DQxYfML60N0J90bh4rstda7DM9ZnxqNpmj80kunlx2LQHXMVle4Yt/uiXx3yhXozEFoIs88++BN4LimW+mmghXOJO52FtHlpMXe3VerPWix12g0vaAC+nIk3lTHbHWFKz1t3Vbo9YhhhOvsRVrwg18shBCYhqBR7GbCrqf5rX0B/2FdEjpXwrK9Jq0yXPhhgJ6No9FoClJOz141T1XHXLHvSITFPnvEsQj9VKgqnbONVxBI7rDfmfVeiZSjPfsAWuw1Gk1ByunZq2oc1UHb3pPyXzMNdwRCcMRxWuzD5zG8apxTjPV0VU9iO9m9OtqzD6PFXqPRFKSc4xIybZz2nnRkHzGyo3j1MLMxSi1qssB4nQNj5xMxs6UsYdk4jvQHpg13tNhrNEMUKSVOuVYU6eV9oLzVOLGIQcQQtHWnI3u1lmxm6WXmNnDFf2xPE6NEB60N8/0bRRBdZx9Gi71GM0T509LNHPmth9jXkRjQ91FVOOUccWwagljECHn2EdMVZZFReun+zIzsBafs/AsArQ2n5RZ7z7PX4xJctNhrNEOU21/aAsCutgEW+zJG9uqbiCkEFRGjVxsnX4J2SstzzNlzH7+1LsCqmeR/KwiSsGw9LiGAFnuNZoiS9LpR1biAgWIgmqpMQ1ARMcORvZFt46Q9+/B5JrUsJWXE+aX1odBCJkFcG0dX4yi02Gs0QxQ10nfgtax8TVXqhmEagopoWH6UjaPEObhwSaZgj+l4nT3V07GIYBgiT2SvPPv+X/fhgBZ7jWaIoiJ7e4CTtOr05ayzNw3hT6tUKMEWOXz68FKFDqPbN7B3xCx/v5yRvTcbR3v2LlrsNZohStIKz4IfKPymqjKcS92YDJEjss/w7HNV5QBMFruJ2Z3srZnp76e+FR065aAAACAASURBVATRI47DaLHXaIYoahk+qxz+SgHK6dmHbBxvtSpFxC+9zK7KCT4+TjQBsK/Gi+wNkbMax3IkScvRnr2HFnuNZojiR/YDbOOUc5695zwRMYS/nKAiaqa9esiM7NOPZxtvY4sIrdVHea8JP7mbSY8Wex8t9hrNEEWJ78B79mVcg9b7FmLkEHsVnef27NOP5xkb2V87C6JxwC3jjOawcQB6krYel+ChxV6jGeIMuGfvRePljOxNIbJKRjNtnFwJWhObOWIT++rm+taOyFN6Ce7oZB3Zu2ix12iGOAPv2ZdxXIJ3DsMgy7OP5ii9VChhnym2UiUS7KufgwrmTUPknI0DnthrlQO02Gs0Q55SPfuNu9vZuLuj6P39BcfLENqrc0QMI4eNE26qCsbjhgERLC4wnwdgf/1cjED1Tq4ELUB3Ukf2Ci32Gs0Qxy7Rxvnu/ev47v1ri94/7dmX9DY5sYLjEvKUXvqNVEbYs78mcg+fizzIc/ZxdFVN9EXcNCgc2WuxB7TYazRDnlIj+86ERZe3TmsxDMRsHMOAmJlZehmur89M0C401vKaM53LU9/CMIxQiWa+yN52dJ29Qou9RjMESdlpn75Uz952JJZd/DHpOvuS3ib3exccl5CZoE2/ZmJznGhiFUcD7gInKpg3C4h98HzDHS32Gs0QJBiZl1p6aTnSX/i7GKTfQVu+QWiGyFFnb2QmaNMiPVVuo1IkWYdbWy8Qobk5mbNxgtU5elyCixZ7jWYI0pVMT4sstfTSdmRJN4iyRvZ+gjbdQRv3IvxI1myc9HEznY0AvG4c5e/zjhljufLMI5lYX5lVejnCWwkr8zzDGS32Gs0QpD+Rfcp2SJVg/aiIvjxNVeHFSwBq41Egu/TSt19si3eknqdDxmkWE/zXGmrjfOu9x3illyJ0jrDYa7UHLfYazaDkK3eu4L+e3pj39a5EWuxLEW7oQ2RfxqYqtQC4CNg4tZWu2GeWXhpCQPcBuOMyFljL+ZX1QQwvqZsp4MqzV98WauIBsdcqB0Ck9100Gs3B5pXN+0OCnknQxumLZ19KkF7WpipHYgolzK4KK2HONc+eB74MGx/nPyu/wH/3LKLByE7eusca/jk7EuHIXnv2Lvqep9EMQixb+vPqcxG0cfri2adKqMZRGi9l/60c25G+v14RVVG4snFyzLPf/ALMvpiHq84D0tF/poCryF5ZQyPi2sbJRIu9RjMISdpOQUHubzVOaQna9L79De5DYu979l5kH2iqEgLGyBbo2AXj56Z9fE+xsiJ7wwidU91Acu07XNFir9EMQizb8UcY56IzYOOU6tlbvdxIMgmKfX+tHFumbZxYhjAHu2ANIZgp33afjJvjR/JpXz+s4Cox61f4RAzfHtKRvYsWe41mEGLZha2W7mBkX6KNU2pkH9T3/iZpHUf6YxBUFD6yMko8avgRPrjR+NHOJvfJuNnpqZdGOMJXpK0h94VoxGCkl/jVWu+iE7QazSAkaTsFPXu1ShWUPi7BdmRJx5QzsrccmVU5UxUz+dvVC5lUX+XvJ4RghrMJ6qdBvDY9B0clb8nw7L1vBbHAz7qqKM37u/1jhju9RvZCiElCiKeEEOuEEGuFEF/yto8SQjwmhHjT+1nvbRdCiF8LITYKIVYJIU4c6A+h0RxuWI4kZeUX1qBY98Wz78u4hHLgyOzIviJiMGtcLdUZjVAz7I0w4QT/OaQj+Ez9jmRG9qagrjLmHavFHoqzcSzgX6SUxwKnAlcLIY4FvgE8IaWcDjzhPQc4F5ju/bsSuKHsV63RHMY4ns1SyMZRFTgRQ/Spzt6RxY8sLjayt2yn15tIrtLLzEVMABrEfhrkHmicD6SrbzLLMxWRDM8+aqZtHF1n79Lrr0FKuUNK+ar3uB14HZgIXADc6u12K3Ch9/gC4I/SZSlQJ4QYX/Yr12gOU5R4F7JxVGQfixglefZSpv36Yq2cYj37/3fPGq66/dWC57KdgL/uCXMusT9BvOU+aFwAZE/CzN9UpSJ7g5FVyrPXkT2UmKAVQkwF5gEvAQ1Syh3eSzuBBu/xRGBr4LBmb1vmua4UQiwXQizfs2dPiZet0Ry+qKi9UDWOZTtEDHfaYyn+e3DfYqdlFhvZb27ppHl/t/98R2s3f1+7M+tcSuwn1MW5+KRGFh41Jutcc8UGUkRg/BzAFXchgqIf3j+z9DIWTNAW8yGHAUWLvRBiBHA38GUpZVvwNel2WpTk7Ekpb5RSniylPHns2LGlHKrRHNYo+6aQjWM7kojpLsdXimdvh8S+dBtHFrg/9KTCSeW/vLSFq29/NdSIZQXq7COmwb9fPJepY6rDJ2rdxjtZzqbIURCpAFyxV/8gO1rPLL10Pfuof12aIsVeCBHFFfrbpZT/523epewZ7+dub/s2YFLg8EZvm0ajySBpOVx641Je3bLf36bGDxcaQ5yyJRHDwDRESfPsQ5F9kfZP2MbJf0xPyg7doHosJ6vM0ylmMZE/f5hx7OPuqov9TSqqzzXrHsBUkX00YON4Yt/ek+rlDYcHxVTjCOAPwOtSyp8HXroP+IT3+BPAvYHtH/eqck4FWgN2j0ajCbCvM8GLm/axYssBf5tVhGdvOw6mIYgaoqRxCVYfFj1xihT77pQdsp7U48zKocxxxCF6WmHXam4SH2RZ/HR/s2vjCN+TMYzcCVpVehn1Si8B2rTYA8XV2S8EPgasFkKs8LZ9C/gxcKcQ4tPAZuAS77WHgPcCG4Eu4JNlvWKN5jAikcoWdlVymbQcpJQ5E4yWI4maAtMUJdk4fYvsg559/v26k3boZqCi/KTtEPfm4NhS+lF4TnauAWCDmJaxLGH6n3oeRCVo1fu4nr1betnWbaEpQuyllM+RP8dxVo79JXB1P69LoxkWJLzoNxHwlYOllErUM7FsN0KOGAapvnr2RYq9I6WfCC40CK07FZ7SqcQ++D5uZF/gzXauAmCDcSQjAh8707PPlCTVVDVldBXfeu8szj6mgR2tPQC0J3RkD7qDVqM5pKhO2KSde4plynayltwD1YnqevZ2Xz37Emwc0xP7QveVnpQdsmhUziFoHQXr7HOyczVUj2V/sp6awH7CE/pcq1hBeknDqGlw5Znualbq5qMjexct9hrNIURVioQi+0xLJ5Z9nOU4bjVOiZ59sCa/lGqciCFIkHsd2j3tCQzhinvwlEk726IKdtDmZOcqGDcHo1mEBN0QKkmbu84+PTMnvV0naMNosddoDiHpyD632CdsG4hmHubPmImU6NmHLKISqnGUmOZ6q6tuf8UXVrUKlmkIUlZuGyeST+ytJOxeD6edhdgmMjx7kU7SkmvqpbeObeDcVTHXv3/P8eOK+pyHO1rsNZqDzNaWLt77q2f52xcX+hF9OLKXOR8HcZuqDMz+ePYlNFUpMc01YmFXW4KWzmTgmh1Mw8zZL2A5Mv+smj3rwUnB+Dne0oXplwwjnKDNPIW6GQVtJCEEr3z7XaHZ9sMZPTVCoznIbG3poj1hsWVfl5+gDUb2VsjGyS3IKnqOlOrZF3EjycQJVNDkys92Jix2tyf85+oz5eoXcAqVXnrJWcbNyUjGBjx773m+2TiZ+YDRIypyjmMYjujfgkZzkEl4Yp6wHN/GCY4sDkbq+bpoVZVOyZ59H6ZlOjJtj+Sqs+9MWrT3BBZTyfDqg5/BloXEfjVEq2HUkZ5HH7ZxROAGkFm9qcYlmDkqlzQuWuw1moOMitYTlp2O7K3c0Xy+xiq/9LJfnn1x3whkQKAzxd6ynaxxBEkrLPJBu6hgZL9jFTQcB4bpRfbpl5SFk8+zn9EwgkXTx3Dc+NqiPtNwRIu9RnOQ8atULIdESkX2uTtb8w1Dc6tx+ufZF3ucUyBB25VRWw/Zs32SVrgCKGfppZ1yI3tv8Fmw8gaCCVr1PHx4XVWM2z59CkfUxov6TMMRLfYazUEmFbJx0o/TrxeToJX+1Mu+evbFHqdKL4GspqrORHYNuy/2lhqlHK6zz1l62fQsJNvhqCUAWZF9ejaOv6Woa9ek0WKv0RxkkgHrxq+zD4l97sdBLEcSMY1+efYvbWrhgVXbez2mUOllLrFPZNo4wQStzBPZr7vX9esDYh/07I8cU820sdV5B6FpekeLvUZzkEkGKnD8OvugjRMQx7yeveP0qc4+GGX/97Ob+Pe/v1FwfxXJK7HPbKrqTOSycWTo2pOZHbSZSdREB7z+AMx4N0QrATj72AbOnJEeff7FJdO5/TOnBmwcrfalouvsNZoikFLyu2c28eH5kxhVnaOltQSSnhgmUkEbxw68XoRn7yVoTcMobfGSUJQdru/PhTq1Km3MdH5yRfZZCdrM2TiZQv3IN6BrHyz4nL/pq+fMzHk9+RK0mt7Rkb1GUwRbW7r5ySPreXzdrn6fKx3Z23ki+95tHDtYetnH2TjuNfQm9l5kL3JX43QmCyVoZeg55Ci93LMBXrsNTr8GppzW6/WnFy/pdVdNBlrsNZoiUIPKEkWWKxbCT9CmnHQHbagap7g6e9NwPftS1qDNtHwKLX0IAbH3E7Th1wtG9hkRPrjfDEJR+Zq7AAGnXlXU9ftNVdq0Lxkt9hpNESQDM+b7f66gZ6+EP7eNk7LyVOM4DlHPsy9tDdrcNfH5UOKumpayI/scYp/VVJVnNo6UsPoumLYIascXdf355tlrekeLvUZTBLm6QftKKlhnn2MQWtDjzvdNIu3Zlyb2WZG97RScUZ8Z2WeJfaHSyxxNVVaw9HL9g9DyFsz5cNHXn2/qpaZ3tNhrNEUQFOj+EqytDz5WolvMbBxLLThuGEV3wrrnzhb2fL69lJLn3twLBBK0WTZOtmeftBzswOz7lC1J2Q4vbdrnzdkBkl3wyDfhiGNLEnuhPfs+o6txNJoiSFnlE/vgjaPHs2+kTM+7SdpFePbe1Eu3qapvyxIqkpZDRcTM2r6saT9X3vYKEPTssyP7qpiJIyVJy8GR7jVn9gr824Ovc8sLTe65hIDnfg6tW+CKh8AsfiqlLr3sOzqy1wxpUrbDK5tbBvx9ymnjJAPllsHErHps2Y4/qbFwgtZdg7Y0Gyf7fPluYLvaevzHkXxNVUmLqliEmniUWm+mfdJysqZ4vvDWXv95nbUXnv+VG9FPXVj0tUNgxHFJR2lAi71miPPImp186IYX2dHaPaDvkwzYLf0+V3BcQqDOXb2H5UgqvYWzn9u4l6Wb9mWdI1x62c/IPs8N5UB3eoWn/J69zYgKk5qKCLXe3PikLTOGuUn2daTn3U9vXwp2Ehb9S9HXrdCefd/RYq8Z0uzrcOeoB0fsDgSZXaG52H6gm60tXUWcKztBC+FVq6KmQcw0WLqphY/cuDTrHG6C1iBiGNi9LASeeVwm+RqrDgQWJClUelldEaEmHqG2MuJ/rmAFjmU77Auca0rHCqgeC2NmFHXNQbSN03e02GuGNB1eNUg5vPRCpId75X+f03/8JIt++pT//P6V2/nETS9n7ZfMSNAq3UoGbJyoKYgWmM1uOY4f2UPxs+lLiez3d6UjeyWuWZ590qI6FuHyU6fw8dOmArk9+yCT21bAlNP7lGX1E7RauUpG/8o0g4Y3drbnLOUrRLsS+zJ46YXINeelN5Y1tfDMhj1ZS/mpBGzSE/uaCjciDq7wFDFFzvHB4M6EV2OH1ZyZYq0c5dkH12rNd6M80J2OxvN69gmb6gqTi0+exCUnT/IWHg979ilbEsVigXidxcYKRiZ3wJTSvHqFoSP7PqOrcTSDgoRlc84v/8GSWUdw0xXziz6u8yBF9skSqnE6EhYjKiK0eZ53V8qmOmaStN2ql2RgdapEyqa2MkpbjxWaKRM1jZxLAEJa2NWI4+C23lD7xaOm/60oXx7iQFfQs8/TVJWwmDy6yn8eNQ3Pxkmfc19LCzdHf8IZ5loAeswa4jPeU9T1ZqIWJtRNVaWjI3vNoEAtWP3alv0lHdfRc5BtnICIvb23k9ZAElOhfHuVR+hKWDy9YQ/zvvcYrd2pkP+fsBw/sak8e8uWRDPX3QugLBt3xLG73yubi/u9qbVrgxZR3si+K1dkHxb71u4UdZXp0slYxGB3e4KNuzuYwF7+OXIXX93yBU4z1vH91GV8IfklbllwP9RPKep6M9GRfd/RYq8ZFKhqjRHx0r5slsOzf3tvZ68DzoILjiguvXEpNzz9Vta+SuzbetwbQWfSpmlvJ11Jm5bOZNqz96Ze1sQzbRyHaCQsZkGvPBWwYlSz0yduermoEtRUYFqmIm81TjCyN7MTtI4j2d+VDE0BjZkG97y2jS/9eTm/j/2ca8x7MO1uPpb6Jn+wz+Nh5xTsWE2v15kP3VTVd7SNoxkUqGqNmoriG2wgIPb98Ozf+bOnAWj68Xl590kFfHZwxXdvR8KvBgqyJSOy70xYdCXT1o26cajXVX36+h3tzG2sI+VIP2JXKAsI8AefRbxxCQYOR4ntdOfoZs3E9ubgFxXZB7615Irs23ssHOkuCaiIGw5XmI9wgfkCs40mrk7+Ew86p4bOm3cN2iLQpZd9R4u9ZlDQ0umK5sGO7IstWcxcfSlhOViODCVRKyIGCcuheb9b8+979kk77Y+nHP9au71jVWT/vQfW8fLbLX41TpDupO2LvYrsTdOgJrmbW6I/4UxzNU0r34bpPyoY9vrNWL0kaB1H+jZOFIvzmn/B3OhOjlk6Bl7phFO/QEudK+Kjqr0b9NIbeDj1A2qjHax1pvBL64M86JySde5IP8RefTQt9aWjxV4zKFA2Tm2pYt9Pz35rS7oZy7LdRbxzkcqoxlGRerf303Gkf0PY4ts4XmSftOgKJEMzE6LKswdYva2VCXVxoqbBrZ9awH0rtnP3q810JW3qvDyo8uzHta5k8ctXYhsWz9rHs2jtDdCzwR0X3HgSVNZnfQ41dTJqGpws1hMXKZL23Kz9VNQOMN9Yzyl77mKKUQ9botgVFubdn6Ht/Y8AXmS/ZwM8+m02mbP4Rfd5POPMISjJ8ajhL8HYn6hce/Z9R4u9ZlCgbJxcM1oK0ZHInhpZCi83pX3uHsuhUoicNkPmbBxVBdSVzK5oeXN3O7Yj/Wi+K2H71xm0cRTBG9y0MdV0JCwqYwbvmDGW1u6UL/YAdy7f6ucJjnn7VpIizjnJ77BNjuHeuWuYs/EGuP1D7slO/DjMeh+YEWicDxU1WJbF57iLxV2vMqviTZLS5PEDpwKNoWsKll3OExsBeHfip7RRzQ9PivHRFZcz9uWfAB9idMyB+78E0SquN7/Kis6Ahx9xq3Nq4lF6Uu63t3LYOFrrS0eLvaZsfOWOFSDg55ecUPKxyvsuVbQ7Eq5V0tfIPlj987un3+K3T21k9fXvpiYezh1krr6k5riryF5ZMlNGV7F5Xxdv7Gz3j3U9+4CNk/EZG0bG/cc18Qj7u5JEPUGs8sYmqPf5+l2rAKinjQm7niZ50mc5y57PrS9uZu3ky5hzwT9D03Ow8TF46ffw6h/dE486Eo44lq++tZxRzk7WmMfyG+tCPm4+yimvXguNP8DqbqNr7SPUTp7N/sbL/Gs60XiTvZXTaOupBmCLOQnmXU7DK7dyjDiZGU/8FzS/CB/8bzb9XxWQ7pWoipmu2FdE2NPu/jfu18IjwhV6odW+ZHqtxhFC3CSE2C2EWBPYNkoI8ZgQ4k3vZ723XQghfi2E2CiEWCWEOHEgL14zuNi0t5O393b26VhVelmKaFu241sDfY3sd7amh33d/WozAE+/sSdrP7+pyo/sXfFVEbeaXrlk1hEAPLJ2p39sZ9Lyl+9LWA4pywlFt5NHpevUOxKWW3rp2UlVMZN62ujpbA1cjeSayN8wpEV8/sf457PdsQPdSRuicZj+Ljj3J/ClFfCZJ+DDfwIrAdteZUvlLL4X/RLfrPt3/sO6hK+nPkdVz064/SIi//cp5PoH4LHrqFj3VwAaamLMM95kR83x/ru396TglC9gOBYPV3yTip3L4cIbYM7FvnWlUDermnjEt2D649kbQmgLp48UE9nfAvwW+GNg2zeAJ6SUPxZCfMN7fi1wLjDd+3cKcIP3UzMMSFhOnxNneztKF/vgLPW+RvZ7AzNbpo2pZkdrD4+s2cn75k4I7adWqkpk2TjhyP6ESXWMGRHj72vSYt+VtP39E5ZN0nYYURHxa/Snjq4OfCaLlOO4JZV2ipkvfZPlFX/FvrsOTryUp2N3M1HsJSpsmo66jKkNxxH33rvHyqjGqZ/q/gOYdT5Iyc13rmTl1gPUmwIh4O/OfG495SI+NGYr1965nGecuaxs/BlTV/yUf43M5zijnVGig6VVx/mnbe+xYMzR/H3a11m/cRNfuvprMDb3AuGVMVfsR8QjRE03gZ214HgJzGgYweyJI/t8/HCm18heSvkPILOA9wLgVu/xrcCFge1/lC5LgTohRHHrjWmGPO7I3t7L/3KhIvtSjm9PpEsD+yr2+zoSjPRKH1Vd+ZPrd2ddR2ZTlbJllMgrmyUeNTn1yNG8sSts4yix70m5Q8JGVKTjrIl1lfz5s6ew8OjRdCZcT3+ssxf+8hFGv3knf7LfRSo6Arn0Bt6W4/iDfS4/Sl3Km/O+DbhVQEJAT47Fv32EAMPwq3EihvDHNHTLGH/ZdzRPOidiY/LM9G/SZY7kEvNppjub+Jt9OqtrFvmnautJIaXkqerz+XP8I4g8Qg9Q7b1HTUXU/7bSHxvnA/Ma+dvVfRu1MNzpq2ffIKXc4T3eCTR4jycCWwP7NXvbdpCBEOJK4EqAyZMn9/EyNIOJRMrpc+KsFM++J+U2J4Ui+z7YOKpW/qixI2jtTrHPK//sTtnsaU/QWJ+2VzJXqkrbOOmIHaAyajJ74kgeWJX+k+9K2v43AJVjqAkkZQ1DcPpRY7jrlWaa9nYh7CRXb/kKOC20LP4x33lkMiMXHckJYw0++d8b/eNOjbjnEEIQj5j0FHHDSy96YlBdEaE75X7TaO9MUV8VxXIkL3ZP4Lmj/8ija3dy8YmT+N0zb3GxUwO4VlJ7j8Xs6x+lI2Exs6Fwg5Qa1exG9u4fR56CJ80A0+9fu3QLlYsfqJ0+7kYp5clSypPHjh3b38vQDAIyF+Molp6U7XvaxUTot77QxLm/etb1jj36Etl3JW16Ug6TPFEPzlwPdo8Gz69EXyVoe1IOjiPpTrrbK2Mmx06oDR0bjOxVqaiK7Csi6f8F66I2nYkU70w8wZjkNrjkVuyTPgVAu6zk7VS4lDJihksbuwtF9h62Ws7QFFRGTWLeLJv9nSnqq2PMbKjhjZ3t7GrrYdzIONWeDdMd6CfY25HwK41GVoYT2cH8AwQi+3jEL2s1C4yC0AwcfY3sdwkhxkspd3g2zW5v+zZgUmC/Rm+bZhjQ18i+IzDpshjR3t2eoLU7xe72dPdqX24yStwb6yuB8DCxzJk36puD5UgcR4a+VXSnbF8M4xGTaWPSHvyo6pjr2Xs3B5XAVM1jo6pjsP4hePjr/GvrVi5xJjFR7GVXzXE0TH83Vcl0Injr/vCs/GCStzJq+kniQlhenX1l1PQqfzyx70oyqirGzHE13L9yOxPqKplYV0mVJ9Y9gZn3qmkM4PWdbaHzP/ylRSQth3nff8z9fURdYa+piBBTYq8TrIeEvt5i7wM+4T3+BHBvYPvHvaqcU4HWgN2jOczJ1TBUDEGRKkbs1f7bD6RFpy/LBe7xrKOJntgDjPfKIDMj++D5r7ztFZ5av9t/7n5D8GycmMGYERX+a+Nq47T1pHyxVCMSJhl7OVFs4CpxF/zvpRCv4+UpV9JNBc84c1h+0k9BCN8G6UraNO0Ni31wpEI8aoai73yoQWjfOHcWP/rgHL8OvqUzSV1VjFnjamjrsdi4u4NxI+NceMIEZk8cyWcWTQudQ3HqkaND56+uiFAfmJWjVkGsiUe1jXOI6TWyF0L8BVgMjBFCNAPfAX4M3CmE+DSwGbjE2/0h4L3ARqAL+OQAXLNmEGI70o9+pZQl1UGrG0RNRaRIz97dZ4dXNqmsiGJo6UwSNQU18aifJwh68xPqKtnR2hNqKgJCKy89/np4aFp3MhDZR8NNYSMro+xpTzBF7OQ84yV2dV3Ch4x/8P23fwcVQA9w3AfhA79n3UvbuP6NxQDcOPZowPXz41GDrqTF46/v4sgx1WzyylvDNo4Zir7zYdnu3J0jx44AvKYn2+FAV4o5jVEWTHPF23Ik40fGGT2igvuvOYMt+7JX4LrxYydxxvQxBd9P5TJGBGwcXTp5aOhV7KWUl+Z56awc+0rg6v5elGboERTbhOVkiV4hVFTsznXPHhmcb/9mz9YYPSJWtNh/+tZlRA2DOz53qt+1O2lU75F9ofN3pSz/mtTn/tnFc3l7w2qO3Xkrsm0bi2LLGCm6SDbdg4jabKg8gX9rPYe5M4/kKx+6BLyEqSI4SbIqFuEfG/aypaWL6993LNffvw4I16u74wiKsXGc0DcCdaNs6UpSXx1jRsMIxoyoYG9HgnEj07+Xiqjy24Uf2R8zvpaqWGEJUTegmnjEv95IgRW4NAOH7qDVlIXwWqqlin1aEPZmTJFUSdhgR6sStc37uogYglHVsaKrcd7e28mBrhRPv7GHvZ7nP7EuLWqjqmPEo0aWZ1/IJuoO2jje575oSjc881Xstu3skKN4xZnBzfZ7uKj2dQ50dNHwzuuZtdfkU4uPAk98g+WYQSukMmryxq52qmMmHzqpMSD2adGujBXv2cejabGNRUwOdLtjl+urYgghOP2o0dy3crt/4wNoqI3zzNcWs3TTPq69ezXg3mTzce/VC1nZfIB7Xtvmf7ZYREf2hxIt9pqykAhF9jZQ/KjiRCCyT9pOyAb65ztWEDEMfvexk/z9lWWypaWLuqoYFZHibJyelO1H7D979A1OmlJPTTzCiAq3u9ORbhRdVxkLLdwBhUs7u5O2X40TT+yFv37RHVcQG8F/Tv8ffr46LZrNoEQ1ogAAIABJREFUkdN42+rkDzVj+eaChtB5QpF9VTCyd28gZ0wfE7rphWyciJn1bSQXljfPXhGLGH4XsXrPxTPHcv+q7VmVNVNGV/P6jjb/mgpF9XMn1TF3Uh1/fmkLoDz79LcDzcFHi72mLASjykQR3nHoWO9bQW08ipRu9KmSedsP9GStjqTeqytp01hf6ScZe0PNZjntyNG8uGkf2w50M7amwq1Tj5p0Jd3lA+uqojkTtDHTCIn+xGgbu1NVdHme/ZLIaszffwkS7bDk2zD7Elqf7wTe9o9RCdpojiylEntDhEsald208OiwPx6ycYqI7KWUbGnp4viJ6dLQCtNgU5v7e1HfJj4wbyJzGkcyKUPsIf0Nq1BUH8TPxwTq7HVkf2jQYq8pC4kMz74UlI1TW+n+OSYtxxfDrqSVlXgMPq+vihGLmLR2p2jtTvHVv67k3y48niNq42Syu92NYD975jT2dSbY2tLN18+ZBbhWSVfSpjJmMrIyGlq4Q13TaRWbODP5LCY208ROzjBWsyJ2NJENl3H+5mc5PvIEVB0DH78PGo4FoLpig38OIdK2VCySLfbKxqmvioW6TFV38elHhStfQjZOIEF75/Kt3LW8me6UzTfOneXfJLa2dNPaneL4wLiBWMTwF22vr4p61yk4+ojczVKqGSxYcVQIdQOqipn+f9Ni18vVlBct9pqyEIzmSx2Z4CdovagxaTlUe1rSlbSz/PNgBFtfFcNyJCnLYVXzAR5bt4sLT5jIeXOyp3Ts9iLYcbWV/Pmzp2LZknGeL61yDNUVEeqqom6Zo5WESAyk5NzUE1wv/4seM0qSKLtkPXc57+ADxrPEVlxPe2QUdxjv5cNX3gTRdA4gOMelvirmC3fuyN69hqBfH+Qor4JGYZrZCdqVWw/w9btWMbOhhv1dSb57/1o+Mn8yZ0wfw5u7OgCYM7HOPy5408n3vkH8yL66OLGf6FU3VcfSCVqrH6uKafqOFntNv/j+A+tY1tQSihaLKQEMko7s1cLb6eO7km5XbrcXdbv7B8S+OkZrd5Kk7fg2TWaS97Ut+5k6uppdbW5k31BbweiMyFSduyoqOMlezUfbbocfLIPKUZDs4HqZ5NXICVze8U90kf7W8F/i/Xx58WSeaRnNa82tfDgg9JCegglu5KzEPrPzFNKRfdCvB7dRqSNhZZWzRjOaqrpTNs9t3AvAnz97Cks3tXD1n1/lew+s493HNnDk2BFETcGMcembRixw06mvKkbsVWRfnI3z+4+dxPLN+6mvjvk3uL70RGj6z7AX+56Uje3IUHJMUzx3Lt9Ke4/FW7s7/G2lRvYJ37NP2zjgesxq9sz+riRv7UmypaUr1DxUXxWlO2mRtNJir+rnD3QlGVER4QP/9QJTR1dx7uzxRE0RFrVEO6y7j/c4b/OoGMPif/yQkfvX0C4rkad+np6uTiprR/G9f7Tyxtj309WRHokMsFmOY3tsGt3WAb8SJ4hpCK5+51H851NvMW5knLf2uDXyo3JE0epvMPO1Y8bXZu2rzq2Iex20z2/cy6xxNYweUcG5x4/j8+84it//4y3W72ynK2kzc1xNaIEYVVIpRO4bUCY18QhCwNia4iL70SMqOOe4cQBEI0rstY1zKBj2CveDB9exYWcHd37+tEN9KUMO25HpRbWT4dLLUsiM7JN2eva7snf3dyX53TNvsXTTvtA3h1HVMfZ2JEJiv7czSWtXitN//CRf8Wa9N+3rYndbgiNq4mk/PNEON78Xdq7iq8A1sQiiu44nZ36Hq1ZOZeRrtexqS/C3qxdy85PPs6iyBrcLCo6fWMtnzjiSr9y5ghVbD7Bxd0desfzaObP40lkz+PIdrwH5hTVqGsQiRlF2ChBaQjEeNXEkvLhpH59a6Ha7Gl6nbH1VlB89vJ6drT189JTw0EFlv508pb6oKpmKiMkNl53EvMl1ve6bSVwtmK49+0PCsBf7LS3dfnOOpjQ6MhaqUJRcjZMKR/bqZtEVuIEc6Eqx/YCbYAwm+NwErVslo8Yf7G1PsHZ7K11Jm6Wb9vn7btjV7kakLW/DmrvhtT/BgS1wyW3c//ADHNW6FN73J3Z31tGzcjUjcSteHl69AylhREU6In7gGnfk75fvWMFj69yO2qmjs6tXFLGI4UfUdZXRvML65XdNZ/7UUUX93iIZkT2AlNkjDOZNdgeoJW2HD544MfTapadMZmJ9pX9TLIb3HD+u6H2DfOu9s6iMGZw7u2/Ha/rHYSP2u9t6+OFDr/PDD87utasvSHtPKjSIS5NmT3uCHzy4jh9+YHZOmytft2vJCVrLDolh5jqv4Eb2O1p7siyA+uooMdMM2zidSdZ59eBrt6cHdb2+bR/fb1wG//U/YHVD4wI472dw9Lu4f/kEHt1zAY+Nmcb7jq5EAheeMJErbn6ZRz0xr/b+rnKttHTakaM5eWr2At9B1ITLXBaO4qrFRxc8R5BgnX3QQpo7Kby4x+yJIzENwZFjqrMW/njHjLG8Y8bBmTo7ekQFP7hw9kF5L002h43Yv7hpH39bsZ3LT53CyUVGRuBGp11Ju+R5LsOBZU0t3LtiOx/L8ztVVTLjauPsbEt72aXaOImUQzxi+JUhrd0pLNsJefP7OpJ+gjWIH9lnePZK7He09tAodnOx+Q8+ZP6Dxr174ch3wvt/DXVpS8NP0FZEqK6IcOkC97Uls47gRw+vB9KeumpyAvj6e2YydXQ1753d+xo9xYh9MUysq2Tbge6MQWjpx0fUhMtOK2Mm//yu6Rw7oVb/jQ9jDhuxV6NjM8v0eqMjYWE5suQW/+GAsmna89g1bd7vurG+sk9iv6yphZMm19OTsolHTV8Mr7h5GV9859GcdUy6kuWNXe0Erd4RFRE6EhajqrNtnH0dSVq2buA7kbsYQTcXmM8TFQ7tE86AM6+CmeeSOYtZRcZVGX8D75g51hd7dX3BbzmlROIV3rmLqXopxF1fOI1XNu/PGnEM4Tk/Qb64ZHq/3lMz9Dl8xN4TnpLFvie9lqgW+zCq2aY9j82lbJzG+kqWb97vb08EInLHm4aZ+btdumkfH7lxKV87Z6Yv9sGa79XbWkNNROu2h+emN9RW0LHHor46Rr21l6NFMxu7Gpke3cul9kNc3vYY0jRIYfKwcyrv+8rvqa2fRD7U9VVVhK8zuBKT6hpVUX+pxMsU2Y8fWcn5c8KirvIY0/M0Q2k0w1rsHUfS4ZX2dXpR4uGElJLvPbCOi05q5LgJpS/SrG6E+RKxbd3u9uA8eAhH9n96aTO/fXIjS795VqgrVM2eeXjNDhrrqohHjZDYb23pClX4KFtGcfmpU6iP2dQ+9W0++dLv+GQFLHNmcJLxJlLCnfZiHj3iUzy13aShtoILCgg9uNH2iMACGwohBHMbR7KyuZXR1RW8/r33hCyTUlCfr7aIEsdSUZH9O2fqVd80uTl8xL6ndLHvTFqosStqJaHDib0dSW5+vonKqNk3sffWS+1I5P6dpiN7twolYgjXEgtE9ut3trO7PUF7j8XIqrTI7fdmz7yxs52xIyrcyD4gtM37u/0EbVXMDFXmnCTe4MKVv6F+36tgJ1jfeDFPN3XzafNhXhp1If+8fQk7Gc2/nXI8T92zpqjW/k+eMZVzjm/I6WnPnzqKlc2tbG7p8r39vpD0kstV/ThHPs465gj++vnTOHlK4SSxZvhy2KwZo6LMUsQ+WIXTWURFzqrmAzhDqEZYdZJuC6zoVAodicKRfWt3CkOkZ8Ar3z0Y2asxwi1eJC+lZOXWA37jU8qWvLm7g4pIOLJP2g5N+9wGpCmj3WX+TjfWcG3kL9wS+ylVHU2w4LPw8ft4dfa/8mProxybuJlNp3yXnYzmyLHV/k2oGLGvjUeZNS5389InTp9KVczknOMacr5eLOpvrLqEarFiEUIwf+oonYDV5GVYR/ZBEQuuKZqLjbs7eP9vn+e/P34yZx/bv//pDxZK7LeXIPa723p4ZsMeLjqp0U/Mbm7p4vzfPMuvPzLPX+EIXOusJh71G4QqIgaGCNs4Kmna0plg2phqHn99N5/943JOmZau7mne3820MdUhsa+khzd3HOAs4xX+bdx+Nu17idONtThSsEmOp+d9/8fxxxwDQGx/MwCOEeWD8xpZuqmFr757hv9toNihXfmYNKqKdd97T7/OAfjdwJl5AY3mYHDYiL0S+bYSxL69hMi+yVsKbiAasNZub2XWuFo27elg0qiqsiWK02KfXbKYj188/iZ/eXkL9VUx/3ey7O0Wtrf2sHRTS1jseyxqKyP+cKyKiIFhiFCdvSqHbOl0/7us3d4KwGtbDjC2psJ/vdKE2hd+wg8jrzHD2MYJYiPtm2qoj7UiN1bR0DCF1DHf4oQnZtJpR3i4Lt0cpER0yawjqIyZ/ObSed57ut8mxtQMjlxMt3fzGYjIXqPpjcPIxik9sg+WFPbWWLWj1Y2O93YksLwFNvrD7rYe3v/b51jW1ML5v3mOW15o4rxfP8eflm7u13mD7G13xW5nW0/RkwaVUP/8sQ1+OesOr6xya8aNrq07xcjKqD+auELZOKn0bBsl5vs94d3ozdBJ2g5TRlX5g7U+2PZH4i/+nHPNZUSwuNE+n9XWZP5in4W4djPi6qVEl1xLPF4NiFAT0QRv+bxPLpwaur76qiiXnzrZn81yqPn6e2bx/rkTBs31aIYXh02I0Zc6+6CNE0wA5mK7t5rPlpZuTvrB4/zog7OLaqTJxyub97OquZUHV7mt+Le92ETSdmje3zd/PRcqsrcdya72RGj5vXtXbENKuHBeuH1eNS6t+//tnXd0XOWVwH93RjOjURtVq1vFsiVXZLljMNjEDWdjEgiBsIEAIQnlbLJZ9qwTNglhd5PdZEOAPWQ5JHEWWGJKCpgE1pSYYnDvVbZsybKK1a2u0ZRv/3hvRsWSm2zkGX+/c3TmzfeePt377ujO9+67797aNqLtfY/gg5Eh05/Wbg9xkbZgaeLBYZx2tze43TTI2QMUOlrIjyojzlPKsuYX8RffwfRNKwDjPkBtaw8up43bI/pW5nFOG02dvQOufm6YOIYd3198WjaViFxWT2xmxDt5yrzq0Gg+bcLC2Sulgk7+/G7Qevpte1FK4fOrAQWmAtSace/tFc20dns4WNs2ImcfWCUHwhoVTcb7hkHleUdC/7lqTnWTGe/kh6/vIycpmt9tqcRutdDe42Fn5Ske/1IxYMTPE6JstHR5BqQ+wunOvq3HQ35yjHFz1Woh29JAYe8+xjT5YOcBWiMnEEMXuXISqahn5ysvs7ipgr+zlRNND9dV7jEmssHh2DlMWPE4bHoPgKK0WGpbe07LXAnUz+mf/igiYZc2q9FcbMLC2Xf1GmWKrRa54DBOV6+X5zce54dr93PXvBwe/dxkAA7XdVCYFhtc2QdeA+GJC6XSdJwHa9sHjDeOcN4Bc3X0EuuIoN3tDd6kXbu7huQYB+WNnbicNr7/+n4AHlxUQF5SNDWnurluQgrvHqw/bb4T5lXHsYYOMhOctHUbMXvx+5jrKOcXLf9GjOqALuB1yAb2OASLKKgw5phuhUo1Bis+Nmbfx27rFNaXNnJVyWK+Z4skNjKCe+bnoZRifWkDg3NLAjnq+gE4jeb8CAtnH8jESXdFUtXSHXwi82wE4vQxjgg63T62NxtPgT638TgPLixgd1Ur9z2/jXXfXhCM2QcY3CDjfDnR3D1AhvOd950Ddfzqo2M8tnLysCmDje1upmS62HisieNNXXT3+mjp8gRz3Js7e7FZBY9P8ccdVXxlbi4en6IkJ2FIZ9/c2cuJ5i6WPfERq5YX0dLlZk7PBnjydp73V3PKksgjSU9RL0n86raJ7NnwBu9v3clhyafDF8Fefx5erEwel8MnR5t4bNJkujp62XzoCNNtRsbM3keXBvWDvi/XAHGRNkT6ShdoNJpzIzycvZljn50QRVVLN23dnjM6+/q2HqP3Zo+XaLuV2MgIOt3eYF43GGGWPVWnANhX3crJQU7nfFb2PR4f2ypaSIi2MSk9jsN1HQNudgYaWdusQmOHEdv2+Pw8++ExitJiuWHiwFRPpRQ/W3eIw3Ud3PzLT3j/HxfS4/Gd1iC6scPN1EwX3R4fr+2sHjLsFKgi+drOGq4vNGrRTEyPw2mz4vG4mW05RI7Uca1lL4VygqRnetke0UXzh5nMsniZWlYBqVN5Nfp2GtIWUNPs5GSbG5LGsT15JY97xzM5I479NW1MzXSxqGgMKbEOPjnaRFK0I7hyH1xBc3B1xgBxzggiI6w6n1yjOU/CwtkHQjfZiU42HoNlT37E6w/OJzsxil6vnwd/t4Nlk9O4eUYWAHeu3kJGvJOUGAcxkUaVw85eL+WNncwvSOLjsiYqm7soPWmEWDYda8LjU2QnOs0VuWJZyxrYtA/sUWCPZrtvPJtborimIJlpWQMbO/znulJ+vaEcEbhn3lhe+2QvSdLGNOllr8qjYEwcq5YWsOdgKU9ubg3K/M6Bk+S4bCyc8Bks1r4vr4/Lmjhc18E3rxvHMx8c5ZZnPuF4UxevfnMexdnx2CyCv7cbR2c1y9o/5N6ELt6rPUnl62/yoLUFl3Riw4sbG25sxEVHc6A9iiM7TnGv9V3mfPBfvB1xHLH0kiVGm7s6SWG7N4/DksRJn59Cdw0WvJSVPELBiu/wRavxUfrxmwf5uKwJt9dHfbubCIuQlxzN/po2FhWN4e8XT2DXCeNLNDc5Cr959zdQPiFAatzQufGLJ6XqEI5GcwGEtLN/e/9JXt1exa0zjbongRzw5s5e3j1Yx93z83hrXy3vHKjjvYN1pLkiKUyL5dDJdo41GI49xixpW9ncRXuPl2sKUvi4rIkTzd2U1hnO/oPDDYDRqPlEczcOPNzv+1/4vz5ZrsJCtD8T74Y4GJcDvZ3g9+KVCK4rb+MWl5fknkoSt7fx/ci+tM0d/gLS2t1kvFLLAr+XexwOLD+P5RddHTgje7G6/fh+7ITsmZB7DSTksWFjB/dGHeFhy0aWJGzD2V5NtL2bmOc8uP09WKUXC34+sgPHQVkiyI0Ae7WXRTboUg68WLHjwY4Xi1uBDdjzLNjA75tGtSMfd2cbL7i+xrqmFK6dPYfXdtfQ3mqGncyX96++Hqx9H6Pp2fE86/NzsLado/UdjE2KCj7UFGivV5wdz8bvLiLd5aTFzL8/1TVwZS8i/PKOEuKjBtaRWVSUyqKi0HioTaO5nAhpZ9/W4+WdA3UkmA5hxdR0xqXE8J1XdrG3qhWlFKs3lJOXHE2H28uaLZV8dloGYOR5f1zWRHF2PBFWYeMxY7VZlBbLmFgHpXXtwZuo9e1urBZh2ZQ0/rK3Fq/FTmHP/7Dl4bm4IjyUVpzg/d8/zURHE1G9zfiaj2ONjAGLjaaWU8SpdrJcCeyPupYXG2w0qzgalYur4rv5bOcfaIssIGP6rRzojGHTls3MckWxub2bz0zLZe3+Fqa7vExp2EFCxU8AWBU4AVvsTIrL5ZBk0WOPYUOjl8joWKo7BbszlqouK3f97T0UTLyKB57byoaDJ1AI9y2chNNu5WfrSgHFXx6Yw09eeANn5wlSC4r513tW8rs1O1m7u4bFY1KpaKzjSwlRfHZaBmu2VAbPv9UipxVBKzbb1e2sbOFAbRvF2fHBkr6TM/ruLaSbufEzcxO4bkIKq5YXnWbfkWQ7aTSagYS0sw902Hl1exV5ydFkJTjJToxiVm4ie6tb2VDWyO6qVv5l5WS2VrSwubyJ1LhIHBEWIixCZ6+PhxYV8MKm48Fc8tzkaMYmRrH+UD1KwYTUGA7XdXD1uCQKxhhXDkVpRgx6R5OVkrHJvNXYw5O+O3jib4q586VdPDm/mNrWHpSC1R+Xk5cZzSvfmEfWqW56Nh3nofl5/GZDORGxDq7+80LunzeOohuK6D7ewmOfFDLRHccRfzt33LSUI749vHCsCUfEHTR7Ovh6kYe9paX89BtfIClzPJEWK8UYYZD1G8q5e34e3355Fx8cbiA3KYrHiqYBRmu6dw/Wkxxj5+GlhQA888FR2nu8ZCa7SJswgz/sSOXtFQsASDPr3QTq3iTH2Jmdl8iaLZUUmVdHmfFObIPSVNNdTtLiIvnwcANVLd18ec5YVkxNxx5hISvh9FrrkTYrz90z++J+MDQazWmEtLNPiXUwLcvFnqpWlk9JC960m5rp4q+H6nnsjQNkxju5dZYR5lm7u4Y/76nhqux45uUn4fX7WTAhhT/trAaMlWrgCyNQn31lcSY/W1fKsilp5CVHs6hoDAvGJ7O/5gB3/3Zr8ItjenZ88Ebqt17aNUDOX95RAhgdhv5pmbGCXbW8KJhxkmKGOQKvB80VsdNuZf64JN7YXWPOZOWJ/VZunLqUpOyBK+H4KDvfWWI48VtmZPHB4QaWTUkPnpPibGPFHVhRg9FhyufvxuW08fCSQlYWZzDerN+eGmc4+YDTT4qxUzI2nme/MgN7hIWv/nYrOcP0XJ2Rk8Bf9tYCMCk9jpykaO6/ftzwhtRoNJeckHb2AAsLx5jOvu+SP5DJcaS+g59/8SocEVZmmYW36trc/MPiwuAXAMCXZmXj9SumZMRhs1qIMTsRrSzO4M55ObR2e1hZnEmkzcrqr84a8BToF0oy6XD7uHVmFjGOCOKjbJzq8rDmvrm09Xg40dw1bAPpwKo54FD713BZVGRkxswvSA6OBUoI31ySdcZzsmRyKt+4Lp+7rs4Jjk3LciH9KlSCUYfeIoKIkOaKDMoBUJztIiHKxk3FmbR1e5mXn4yIsGRyGvXtRmbS2MShnf3d83MHOHuNRjP6hLyzv/faPIrSYpma1ZeqNzs/kQUTUvj89Aw+P91wjBPGxJIcY6cwLZYvzhzoLOfmJzE3v68r0k3TMylv7ORHn5tMbKSN7904ccDxKbF9mSI/+cK0Afte/Nocejw+ZuScvQ/u5Iw4nv5yCZ8xrwii7BHcXJJFblIUDyw02t1lJ0aRleCk1+tnTn4SW8qbWHCWBtGOCCvfXT5Q5thIG1+ePXZAU+x/XjGR7t6ha+bMyElk5w+WAJwWT0+JcbBiajpLhqnxMjM3kQxXJDWtPQPOlUajGT1kpAW9hpxUZBnwJGAFfq2U+vczHT9z5ky1bdu2iy7HYKpaukiIsg/oIXohKKV46r0ylk5JHfaBpovJW3tr8fgV1xem0OX2DViBX650uL20dXvIiB+6J6pGoxk5IrJdKTXznI692M5eRKzAYWAxUAVsBW5XSh0Y7nc+LWev0Wg04cT5OPtL8cz5bKBMKXVMKdULvASsvAR/R6PRaDTnyKVw9pnAiX7vq8wxjUaj0YwSo1ZNSkS+LiLbRGRbQ0PDaImh0Wg0VwSXwtlXY1S3DZBljg1AKfWsUmqmUmpmSsqZs0s0Go1GMzIuhbPfCowXkTwRsQO3AWsvwd/RaDQazTly0fPslVJeEXkIWIeRerlaKbX/Yv8djUaj0Zw7l+ShKqXUm8Cbl2JujUaj0Zw/ut2PRqPRXAFckidoz1sIkQbg+AX+ejLQeBHFuRwIN53CTR8IP53CTR8IP52G0idHKXVOGS6XhbMfCSKy7VyfIAsVwk2ncNMHwk+ncNMHwk+nkeqjwzgajUZzBaCdvUaj0VwBhIOzf3a0BbgEhJtO4aYPhJ9O4aYPhJ9OI9In5GP2Go1Gozk74bCy12g0Gs1Z0M5eo9ForgBC2tmLyDIRKRWRMhFZNdryXAgiUiEie0Vkl4hsM8cSReQdETliviacbZ7RRERWi0i9iOzrNzakDmLwlGmzPSJSMnqSD80w+jwqItWmnXaJyI399n3X1KdURJaOjtRnRkSyRWS9iBwQkf0i8i1zPCTtdAZ9QtZOIhIpIltEZLep04/M8TwR2WzK/rJZcwwRcZjvy8z9uWf8A0qpkPzBqLtzFMgH7MBuYNJoy3UBelQAyYPGfgqsMrdXAf8x2nKeRYcFQAmw72w6ADcCbwECzAU2j7b856jPo8DDQxw7yfzsOYA88zNpHW0dhpAzHSgxt2MxuslNClU7nUGfkLWTea5jzG0bsNk8968At5njzwD3m9sPAM+Y27cBL59p/lBe2YdzR6yVwHPm9nPATaMoy1lRSn0INA8aHk6HlcDzymATEC8i6Z+OpOfGMPoMx0rgJaWUWylVDpRhfDYvK5RStUqpHeZ2O3AQo6lQSNrpDPoMx2VvJ/Ncd5hvbeaPAhYBvzfHB9soYLvfAzeIiAw3fyg7+3DpiKWAt0Vku4h83RxLVUrVmtsngdTREW1EDKdDKNvtITOksbpfaC3k9DEv96djrBxD3k6D9IEQtpOIWEVkF1APvINxBXJKKeU1D+kvd1Anc38rkDTc3KHs7MOFa5RSJcBy4EERWdB/pzKu0UI6PzYcdAD+GxgHFAO1wM9HV5wLQ0RigD8A31ZKtfXfF4p2GkKfkLaTUsqnlCrGaPo0Gyi6WHOHsrM/p45YlztKqWrztR74E4aB6wKXzOZr/ehJeMEMp0NI2k0pVWf+I/qBX9EXAggZfUTEhuEYX1RK/dEcDlk7DaVPONgJQCl1ClgPzMMIoQXK0feXO6iTud8FNA03Zyg7+5DviCUi0SISG9gGlgD7MPS4yzzsLuD10ZFwRAynw1rgTjPbYy7Q2i+McNkyKF79eQw7gaHPbWZmRB4wHtjyact3NsxY7m+Ag0qpx/vtCkk7DadPKNtJRFJEJN7cdgKLMe5FrAduMQ8bbKOA7W4B/mpenQ3NaN+BHuHd6xsx7sIfBR4ZbXkuQP58jAyB3cD+gA4Ycbf3gCPAu0DiaMt6Fj3WYFwyezBiivcOpwNGxsHTps32AjNHW/5z1OcFU9495j9Zer/jHzH1KQWWj7b8w+h0DUaIZg+wy/y5MVTtdAZ9QtZOwDRgpyn7PuAH5ng+xhdTGfAq4DDHI833Zeb+/DPNr8slaDQazRVAKIdxNBqNRnOpqyqIAAAANklEQVSOaGev0Wg0VwDa2Ws0Gs0VgHb2Go1GcwWgnb1Go9FcAWhnr9FoNFcA2tlrNBrNFcD/A3u+RahcnZxmAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KTGyfRFEzUHG"
      },
      "outputs": [],
      "source": [
        "## game play video\n",
        "agent.load_state_dict(torch.load(\"weights.pt\"))\n",
        "env = gnwrapper.LoopAnimation(gym.make(game_name)) \n",
        "state = env.reset()\n",
        "state_dim = len(state)\n",
        "game_score = 0\n",
        "score_arr2 = []\n",
        "state_arr = []\n",
        "for i in range(1000):\n",
        "    if i == 0:\n",
        "        stacked_state = np.concatenate((state, state, state, state, state, state, state, state), axis=0)\n",
        "    else:\n",
        "        stacked_state = np.roll(stacked_state,-state_dim,axis=0)        \n",
        "        stacked_state[-state_dim:]=state\n",
        "    with torch.no_grad():\n",
        "        hs = agent.representation_network(torch.from_numpy(stacked_state).float().to(device))\n",
        "        P, v = agent.prediction_network(hs)\n",
        "        action = np.random.choice(np.arange(agent.action_space), p=P.detach().numpy())   \n",
        "    if i %5==0:\n",
        "        env.render()\n",
        "    state, r, done, info = env.step(action.item())\n",
        "    print(r, done, info)\n",
        "    state_arr.append(state[0])\n",
        "    game_score += r \n",
        "    score_arr2.append(game_score)\n",
        "    if i %10==0: \n",
        "        print(game_score)\n",
        "    if done:\n",
        "        print('last frame number : ',i)\n",
        "        print('score :', game_score)\n",
        "        print(state_arr)\n",
        "       \n",
        "        break\n",
        "env.reset()\n",
        "env.display()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "er3f1i6AI8nD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "outputId": "9e2d45da-55af-415a-afc2-c4f3a72a9d8c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f86b9958970>"
            ]
          },
          "metadata": {},
          "execution_count": 8
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxM9/7H8dc3hAixxBoiEsSSVUhiV0vtalfdlFK63GrvbX+1u6W6aHVze3vbq6W4t7e0EktRVUW1RSklm5CEIBFiSSKRRJb5/v7IkUcQJJFkkpnP8/HIIzPfOWfm8x2TtzPnnPmM0lojhBDCstiYuwAhhBClT8JdCCEskIS7EEJYIAl3IYSwQBLuQghhgaqauwCABg0aaFdXV3OXIYQQlcqhQ4cuaa0bFnZbhQh3V1dX/vjjD3OXIYQQlYpS6vSdbpPdMkIIYYEk3IUQwgJJuAshhAWqEPvcC5OdnU1cXByZmZnmLkUIcQd2dnY4Oztja2tr7lLELSpsuMfFxeHg4ICrqytKKXOXI4S4hdaay5cvExcXh5ubm7nLEbco0m4ZpVSsUipUKXVEKfWHMeaolPpRKRVl/K5njCul1D+UUtFKqRClVMeSFJaZmUn9+vUl2IWooJRS1K9fX95dV1DF2efeR2vdQWvtb1yfBfyktXYHfjKuAwwG3I2facCnJS1Ogl2Iik3+Riuu+zmgOgJYZVxeBYwsML5a59kP1FVKOd3H4wghhMXJyMrl7e+PEZeUXib3X9Rw18B2pdQhpdQ0Y6yx1jrBuHweaGxcbgacLbBunDF2E6XUNKXUH0qpPy5evFiC0q3bggULeO+99+66zIYNG4iIiCj2fdeqVaukZZWbIUOGkJycbO4yKqTdu3czbNiw+76fy5cv06dPH2rVqsULL7xQCpWJG/bGXGLgR3v4988n2XW8bPKvqAdUe2it45VSjYAflVKRBW/UWmulVLG+9UNrvQxYBuDv7y/fGFIGNmzYwLBhw/Dw8DB3KbfJycmhatWSH8/funVrKVYjCmNnZ8eiRYsICwsjLCzM3OVYhKuZ2by99RhfHziLa3171kzrQpeW9cvksYq05a61jjd+JwLrgUDgwo3dLcbvRGPxeKB5gdWdjbFKZ+TIkXTq1AlPT0+WLVuWP75t2zY6duyIr68v/fr1AyAtLY2nnnoKb29vfHx8CAoKAm7eCl63bh2TJk0CYNKkSTz33HN06dKFli1bsnv3biZPnkz79u3zl7nb+gV9/vnnBAQE4Ovry5gxY0hPT2fv3r1s2rSJV199lQ4dOhATE0NMTAyDBg2iU6dO9OzZk8jIvP+jT506RdeuXfH29mbevHmFPhfXrl1j6NCh+Pr64uXlxdq1awE4ePAg3bp1w9fXl8DAQFJTU8nMzMx/Lvz8/Ni1axcAK1euZPjw4fTt25d+/fpx7do1Jk+eTGBgIH5+fmzcuBGA8PBwAgMD6dChAz4+PkRFRd1Wj6urK5cuXSI2Npb27dszdepUPD09GTBgABkZGbctHxsbS9++ffHx8aFfv36cOXMm/9/hxRdfpFu3brRs2ZJ169blr7NkyRICAgLw8fHhtddeK/R5Wb58OW3atCEwMJCpU6fmb+F+9913dO7cGT8/Px588EEuXLgA5L3jmjhxIj179qRFixYEBwczY8YMvL29GTRoENnZ2QAcOnSIBx54gE6dOjFw4EASEhJue+xJkybx7LPP4u/vT5s2bdi8efNtyxw4cICuXbvi5+dHt27dOH78OAC9evXiyJEj+cv16NGDo0eP3rRuzZo16dGjB3Z2doXOXRTPjxEX6P/Bz6w9eJZnHmjJtr/2KrNghyJsuSulagI2WutU4/IA4HVgEzARWGz83missgl4QSm1BugMpBTYfVMiC78LJ+Lc1fu5i9t4NK3Naw953nWZFStW4OjoSEZGBgEBAYwZMwaTycTUqVPZs2cPbm5uXLlyBYBFixZRp04dQkNDAUhKSrpnDUlJSezbt49NmzYxfPhwfvvtN7744gsCAgI4cuQIHTp0KNJcRo8ezdSpUwGYN28ey5cvZ/r06QwfPpxhw4YxduxYAPr168dnn32Gu7s7v//+O88//zw7d+7kpZde4rnnnuPJJ5/kk08+KfQxtm3bRtOmTdmyZQsAKSkpZGVlMX78eNauXUtAQABXr16lRo0aLF26FKUUoaGhREZGMmDAAE6cOAHA4cOHCQkJwdHRkTlz5tC3b19WrFhBcnIygYGBPPjgg3z22We89NJLPP7442RlZZGbm3vX+UdFRfH111/z+eef8/DDDxMUFMQTTzxx0zLTp09n4sSJTJw4kRUrVvDiiy+yYcMGABISEvj111+JjIxk+PDhjB07lu3btxMVFcWBAwfQWjN8+HD27NlDr1698u/z3LlzLFq0iMOHD+Pg4EDfvn3x9fUF8sJy//79KKX44osvePfdd3n//fcBiImJYdeuXURERNC1a1eCgoJ49913GTVqFFu2bGHo0KFMnz6djRs30rBhQ9auXcvcuXNZsWLFbXOPjY3lwIEDxMTE0KdPH6Kjo2+6vV27dvzyyy9UrVqVHTt2MGfOHIKCgpgyZQorV67ko48+4sSJE2RmZubXLkrXpbTrLNgUzuaQBNo1ceDzJ/3xca5b5o9blPfFjYH1xlHxqsD/tNbblFIHgW+UUlOA08DDxvJbgSFANJAOPFXqVZeTf/zjH6xfvx6As2fPEhUVxcWLF+nVq1f+eb2Ojo4A7NixgzVr1uSvW69evXve/0MPPYRSCm9vbxo3boy3tzcAnp6exMbGFjncw8LCmDdvHsnJyaSlpTFw4MDblklLS2Pv3r2MGzcuf+z69esA/Pbbb/nvNCZMmMDMmTNvW9/b25tXXnmFmTNnMmzYMHr27EloaChOTk4EBAQAULt2bQB+/fVXpk+fDuSFS4sWLfLDvX///vnP2fbt29m0aVP+sYPMzEzOnDlD165defPNN4mLi2P06NG4u7vfdf5ubm75z1WnTp2IjY29bZl9+/YRHBycP8cZM2bk3zZy5EhsbGzw8PDI38Levn0727dvx8/PL//5i4qKuincDxw4wAMPPJA/n3HjxuXPMy4ujvHjx5OQkEBWVtZN54EPHjwYW1tbvL29yc3NZdCgQfnPcWxsLMePHycsLIz+/fsDkJubi5NT4eckPPzww9jY2ODu7k7Lli3z343dkJKSwsSJE4mKikIplf/OYNy4cSxatIglS5awYsWKQt8RivujtWbjkXMs/C6ca9dzeaV/G555oBXVqpZPY4B7hrvW+iRw23/pWuvLQL9CxjXwl1KpznCvLeyysHv3bnbs2MG+ffuwt7end+/eJTqft+CpYreuX716dQBsbGzyL9+4npOTc8/1b5g0aRIbNmzA19eXlStXsnv37tuWMZlM1K1b96a34neqszBt2rTh8OHDbN26lXnz5tGvXz9GjRp113UKU7NmzfzLWmuCgoJo27btTcu0b9+ezp07s2XLFoYMGcK///1v+vbte8f7LPjcValSpdDdMndTcP0bXxivtWb27Nk888wzxbqvG6ZPn87LL7/M8OHD2b17NwsWLLjt8WxsbLC1tc1/7m/8u2ut8fT0ZN++ffd8nFv/3W69Pn/+fPr06cP69euJjY2ld+/eANjb29O/f382btzIN998w6FDh0o0T1G4c8kZzNsQxs7IRPxc6vLuGB/cGzuUaw3SW+YOUlJSqFevHvb29kRGRrJ//34AunTpwp49ezh16hRA/m6Z/v3737RL48ZumcaNG3Ps2DFMJlP+u4DiKMr6qampODk5kZ2dzVdffZU/7uDgQGpqKpC3Ve3m5sa3334L5IXXjX2s3bt3z3/XUXD9gs6dO4e9vT1PPPEEr776KocPH6Zt27YkJCRw8ODB/DpycnLo2bNn/v2cOHGCM2fO3BbgAAMHDuTjjz/OD9Q///wTgJMnT9KyZUtefPFFRowYQUhISPGetEJ069btpjn27NnzrssPHDiQFStWkJaWBkB8fDyJiYk3LRMQEMDPP/9MUlISOTk5+e9+IO/106xZ3kliq1atojjatm3LxYsX88M9Ozub8PDwQpf99ttvMZlMxMTEcPLkydue54J1rFy58qbbnn76aV588UUCAgKK9E5T3JvJpPnv/tMM+HAP+2Iu8/dhHqx7tlu5BztIuN/RoEGDyMnJoX379syaNYsuXboA0LBhQ5YtW8bo0aPx9fVl/PjxQN6+7qSkJLy8vPD19c0/iLh48WKGDRtGt27d7vjW+m6Ksv6iRYvo3Lkz3bt3p127dvnjjzzyCEuWLMHPz4+YmBi++uorli9fjq+vL56envkHMJcuXconn3yCt7c38fGFH/sODQ3NP8i5cOFC5s2bR7Vq1Vi7di3Tp0/H19eX/v37k5mZyfPPP4/JZMLb25vx48ezcuXKm7aOb5g/fz7Z2dn4+Pjg6enJ/PnzAfjmm2/w8vKiQ4cOhIWF8eSTTxb7ebvVxx9/zJdffomPjw//+c9/WLp06V2XHzBgAI899lj+geaxY8fm/0d5Q7NmzZgzZw6BgYF0794dV1dX6tSpA+QdOB03bhydOnWiQYMGxaq1WrVqrFu3jpkzZ+Lr60uHDh3Yu3dvocu6uLgQGBjI4MGD+eyzz247+Dljxgxmz56Nn59f/rvBGzp16kTt2rV56qk77zl1dXXl5ZdfZuXKlTg7O5fo1FprcerSNR75fD/zNoTRoXldtv+tF5N7uFHFxjwf9FI3tprMyd/fX9/6ZR3Hjh2jffv2ZqpIiKJJS0ujVq1a5OTkMGrUKCZPnlyi3VUlMWnSpJsOmBfXuXPn6N27N5GRkdjYlHw7z9r/VnNyTXzx6yk+/PEE1araMH+oB+P8ncvl07tKqUMFugbcpMI2DhOiMliwYAE7duwgMzOTAQMGMHLkyHuvVAGsXr2auXPn8sEHH9xXsFu7iHNXmRkUQmh8CgM8GrNopBeNa1eMU0dly10IcV+s8W/1ek4u/9wZzae7Y6hrb8vC4V4M8W5S7r12Ku2Wu9ZaGhMJUYFVhI3D8nbodBIzg0KITkxjdMdmzB/qQb2a1cxd1m0qbLjb2dlx+fJlafsrRAV1o5+7tXyCNT0rhyU/HGfl3lia1qnByqcC6N22kbnLuqMKG+7Ozs7ExcUhTcWEqLhufBOTpfs16hKzgkOIS8rgya4tmDGoHbWqV9j4BCpwuNva2sq3uwghzColPZs3t0bwzR9xtGxQk2+e6Uqgm6O5yyqSChvuQghhTtvCzjN/YxhXrmXxXO9WvNTPHTvbKuYuq8gk3IUQooCLqXmNvraEJuDhVJsvJwXg1ayOucsqNgl3IYQg7wBx8OF4Xt8cQUZWLq8ObMu0Xi2xrVI5Pwcg4S6EsHrxyRnMCQ7l5xMX6dSiHu+M8aF1o4r/jWR3I+EuhLBaJpPmv7+f5p3vI9HAwuGeTOjSAhsz9YMpTRLuQgirFHMxjVlBIRyMTaKnewPeGuVNc0d7c5dVaiTchRBWJTvXxOe/nOSjHVHUsK3Ce+N8GdOxmcV9WFLCXQhhNcLiU5gZFEL4uasM9mrCwhGeNHKwzE/YSrgLISxeZnYuH++M4rOfT1LPvhqfPt6Rwd7F/36FykTCXQhh0f6IvcKMoBBOXrzGuE7OzB3anrr2Fa/RV2mTcBdCWKS06zks2RbJ6v2naVqnBqsnB9KrTUNzl1VuJNyFEBbn5xMXmRMcyrmUDCZ2deXVgW2pWcEbfZU265qtEMKiJadnsWjzMYIOx9GqYU2+faYr/q6Vo9FXaZNwF0JYhO9DE5i/MZyk9Cxe6NOaF/q2rlSNvkqbhLsQolJLvJrJ3zeGsy38PF7NarNqcgCeTStfo6/SJuEuhKiUtNasOxTHos0RZOaYmDmoHVN7ulG1kjb6Km0S7kKISufslXTmrA/ll6hLBLo6sniMNy0bVu5GX6VNwl0IUWnkmjSr98Wy5IfjKGDRCE8e72wZjb5Km4S7EKJSiE5MZWZQKIdOJ/FAm4a8NdqbZnVrmLusCkvCXQhRoWXnmvj3zzH846do7KtX4YOHfRnlZ3mNvkqbhLsQosIKjUthRlAIxxKuMtTHiQUPedLQobq5y6oUJNyFEBVOZnYuH+2I4vNfTlK/ZjX+PaETAz2bmLusSqXI4a6UqgL8AcRrrYcppdyANUB94BAwQWudpZSqDqwGOgGXgfFa69hSr1wIYZF+P3mZWcGhnLp0jfH+zZkztD11atiau6xKpzgnhL4EHCtw/R3gQ611ayAJmGKMTwGSjPEPjeWEEOKuUjOzmb8hjPHL9pNjMvHV0515Z6yPBHsJFSnclVLOwFDgC+O6AvoC64xFVgEjjcsjjOsYt/dTcuRDCHEXu44nMvDDPfz399NM7u7GD3/tRffWDcxdVqVW1N0yHwEzAAfjen0gWWudY1yPA5oZl5sBZwG01jlKqRRj+UsF71ApNQ2YBuDi4lLS+oUQlVjStSwWbY4g+M943BvVIui5bnR0qWfusizCPcNdKTUMSNRaH1JK9S6tB9ZaLwOWAfj7++vSul8hRMWntWZLaAKvbQwnJSObF/u585c+rahe1XobfZW2omy5dweGK6WGAHZAbWApUFcpVdXYencG4o3l44HmQJxSqipQh7wDq0IIwYWrmczbEMaPERfwca7Df5/uTHun2uYuy+Lcc5+71nq21tpZa+0KPALs1Fo/DuwCxhqLTQQ2Gpc3Gdcxbt+ptZYtcyGsnNaatQfP8OAHP7PnxEXmDGlH8HPdJNjLyP2c5z4TWKOUegP4E1hujC8H/qOUigaukPcfghDCip25nM6s4BD2xlyms5sj74zxwbVBTXOXZdGKFe5a693AbuPySSCwkGUygXGlUJsQopLLNWlW7o3lvR+OU8VG8eYoLx4NcJFGX+VAPqEqhCgTJy6kMmNdCEfOJtO3XSPeHOWFUx1p9FVeJNyFEKUqK8fEp7tj+OeuKGpVr8rSRzow3LepNPoqZxLuQohSc/RsMjODQog8n8pw36a89pAH9WtJoy9zkHAXQty3jKxcPtxxgi9+OUkjBzu+eNKfBz0am7ssqybhLoS4L/tiLjM7OITYy+k8GujC7CHtqG0n/WDMTcJdCFEiVzOzWfx9JP/7/Qwt6tvzv6md6dZK+sFUFBLuQohi++nYBeauDyMxNZOpPd14uX9balST1gEViYS7EKLILqddZ+F3EWw6eo62jR34bEInOjSva+6yRCEk3IUQ96S1ZtPRcyz8LoLUzGz+9mAbnuvdimpVi/OVEKI8SbgLIe4qISWDeevD+CkyEd/mdXl3jA9tmzjce0VhVhLuQohCmUyaNQfP8vbWY2SbTMwb2p6nurtRRVoHVAoS7kKI28Reusas4BD2n7xC15b1WTzGmxb1pdFXZSLhLoTIl5Nr4svfYnn/x+PY2tiweLQ34wOaS+uASkjCXQgBQOT5q8xcF8LRuBQebN+YN0Z60aSOnbnLEiUk4S6Elbuek8snu2L4165o6tSw5eNH/Rjm4yRb65WchLsQVuzPM0nMDArhxIU0Rvk1Y/4wDxxrVjN3WaIUSLgLYYXSs3J4f/sJVvx2iia17VgxyZ++7aTRlyWRcBfCyuyNvsSs4FDOXEnniS4uzBzUDgdp9GVxJNyFsBIpGdm8vfUYaw6exa1BTdZM60KXlvXNXZYoIxLuQliB7eHnmbchjEtp13nmgZb87cE22NlKoy9LJuEuhAW7lHadBZvC2RySQLsmDnwx0R8fZ2n0ZQ0k3IWwQFprNhyJZ+F3EaRfz+WV/m14tncrbKtIoy9rIeEuhIU5l5zB3PWh7Dp+ET+XvEZf7o2l0Ze1kXAXwkKYTJqvDpzhne8jyTVp/j7Mg4ndXKXRl5WScBfCApy8mMasoFAOxF6hR+sGvD3am+aO9uYuS5iRhLsQlVhOrokvfj3Fhz+eoHpVG94d68O4Ts7SOkBIuAtRWUWcu8qMoKOExV9loGdjFo3wolFtafQl8ki4C1HJXM/J5Z87o/l0dwx17W351+MdGezVRLbWxU0k3IWoRA6dvsLMoFCiE9MY3bEZ84d6UE8afYlCSLgLUQlcu57Dkh+Os2pfLE3r1GDlUwH0btvI3GWJCuye4a6UsgP2ANWN5ddprV9TSrkBa4D6wCFggtY6SylVHVgNdAIuA+O11rFlVL8QFu+XqIvMDg4lLimDiV1b8OqgdtSqLttl4u6K8nG160BfrbUv0AEYpJTqArwDfKi1bg0kAVOM5acAScb4h8ZyQohiSknP5tVvjzJh+QGqVbXh22e7snCElwS7KJJ7vkq01hpIM67aGj8a6As8ZoyvAhYAnwIjjMsA64B/KqWUcT9CiCLYFnae+RvDuHIti+d7t+LFfu7S6EsUS5E2AZRSVcjb9dIa+ASIAZK11jnGInFAM+NyM+AsgNY6RymVQt6um0u33Oc0YBqAi4vL/c1CCAuRmJrJgk3hbA09j4dTbb6cFIBXszrmLktUQkUKd611LtBBKVUXWA+0u98H1lovA5YB+Pv7y1a9sGpaa4IOx7NocwQZ2bm8OrAt03q1lEZfosSKtfNOa52slNoFdAXqKqWqGlvvzkC8sVg80ByIU0pVBeqQd2BVCFGIuKR05qwPY8+Ji/i3qMfiMT60blTL3GWJSq4oZ8s0BLKNYK8B9CfvIOkuYCx5Z8xMBDYaq2wyru8zbt8p+9uFuJ3JpPnP/tO8sy0SgIXDPZnQpQU20uhLlIKibLk7AauM/e42wDda681KqQhgjVLqDeBPYLmx/HLgP0qpaOAK8EgZ1C1EpRZzMY2Z60L443QSvdo05K1RXjjXk0ZfovQU5WyZEMCvkPGTQGAh45nAuFKpTggLk51rYtmekyz9KYoatlV4b5wvYzo2k9YBotTJCbNClJOw+BRmrAshIuEqQ7ybsGC4J40cpNGXKBsS7kKUsczsXJb+FMWyPSepZ1+Nz57oyCAvJ3OXJSychLsQZehg7BVmrgvh5KVrjOvkzLyhHtSxtzV3WcIKSLgLUQbSrufw7rZIVu87jXO9GvxnSiA93RuauyxhRSTchShlP5+4yJzgUM6lZDCpmyuvDmxLTekHI8qZvOKEKCXJ6Vm8vjmC4MPxtGpYk3XPdqVTC0dzlyWslIS7EPdJa833Yef5+8YwktOzeaFPa17o21oafQmzknAX4j4kXs1k/sYwfgi/gFez2qyaHIhnU2n0JcxPwl2IEtBa8+2hON7YHMH1HBOzBrfj6R5uVJVGX6KCkHAXopjOXklndnAov0ZfItDVkcVjvGnZUBp9iYpFwl2IIso1aVbvi+XdbcexUbBopBePB7pIoy9RIUm4C1EE0YmpzFgXwuEzyfRu25A3R3nTrG4Nc5clxB1JuAtxF9m5Jj7bHcPHO6Oxr16FD8f7MrKDNPoSFZ+EuxB3EBqXwqvrjhJ5PpVhPk4sGO5Jg1rVzV2WEEUi4S7ELTKzc/lwxwk+33OSBrWqs2xCJwZ4NjF3WUIUi4S7EAX8fvIys4JDOXXpGo8ENGf2kPbUqSGNvkTlI+EuBJCamc072yL57/4zNHeswVdPd6Z76wbmLkuIEpNwF1ZvV2Qic9aHcv5qJlN6uPHKgDbYV5M/DVG5yStYWK0r17J4/btwNhw5h3ujWgQ9142OLvXMXZYQpULCXVgdrTWbQxJYsCmclIxsXuznzl/6tKJ6VWn0JSyHhLuwKheuZjJ3fRg7jl3Ax7kOX03tTLsmtc1dlhClTsJdWAWtNWsPnuXNrcfIyjExd0h7nuruKo2+hMWScBcW78zldGYFh7A35jKd3Rx5Z4wPrg1qmrssIcqUhLuwWLkmzZe/neK97cepamPDW6O8eSSguTT6ElZBwl1YpOPnU5kRFMLRs8n0bdeIN0d54VRHGn0J6yHhLixKVo6Jf+2O5pNd0TjY2bL0kQ4M920qjb6E1ZFwFxbj6NlkZqwL4fiFVEZ0aMrfh3lQXxp9CSsl4S4qvYysXD748TjLfz1FIwc7vnjSnwc9Gpu7LCHMSsJdVGp7Yy4xOziU05fTeayzC7MGt6O2nTT6EkLCXVRKVzOzeXtrJF8fOEOL+vb8b2pnurWSRl9C3CDhLiqdHREXmLshlIup15nWqyV/e7ANNapJ6wAhCrpnuCulmgOrgcaABpZprZcqpRyBtYArEAs8rLVOUnmnJSwFhgDpwCSt9eGyKV9Yk8tp11n4XQSbjp6jXRMHlk3wx7d5XXOXJUSFVJQt9xzgFa31YaWUA3BIKfUjMAn4SWu9WCk1C5gFzAQGA+7GT2fgU+O3ECWitWbT0XMs2BRO2vUc/vZgG57r3YpqVaV1gBB3cs9w11onAAnG5VSl1DGgGTAC6G0stgrYTV64jwBWa601sF8pVVcp5WTcjxDFkpCSwbz1YfwUmUiH5nV5d6wPbRo7mLssISq8Yu1zV0q5An7A70DjAoF9nrzdNpAX/GcLrBZnjN0U7kqpacA0ABcXl2KWLSydyaT5+uAZ3t4aSY7JxLyh7XmquxtVpHWAEEVS5HBXStUCgoC/aq2vFvzEn9ZaK6V0cR5Ya70MWAbg7+9frHWFZTt16RqzgkL4/dQVurWqz+LRPrjUtzd3WUJUKkUKd6WULXnB/pXWOtgYvnBjd4tSyglINMbjgeYFVnc2xoS4q5xcEyt+O8X7209QraoN74zx5mH/5tI6QIgSKMrZMgpYDhzTWn9Q4KZNwERgsfF7Y4HxF5RSa8g7kJoi+9vFvRxLuMrMoBBC4lLo79GYN0Z60bi2nbnLEqLSKsqWe3dgAhCqlDpijM0hL9S/UUpNAU4DDxu3bSXvNMho8k6FfKpUKxYW5XpOLp/siuFfu6KpU8OWfz7mx1BvJ9laF+I+FeVsmV+BO/2l9StkeQ385T7rElbg8JkkZq4LISoxjVF+zfj7MA/q1axm7rKEsAjyCVVR7tKzcnjvhxN8ufcUTWrb8eWkAPq0a2TusoSwKBLuolz9Fn2JWcEhnL2SwRNdXJg5qB0O0uhLiFIn4S7KRUpGNm9tOcbaP87i1qAma6d1oXPL+uYuSwiLJeEuytz28PPM2xDG5WtZPPtAK/76oDt2ttLoS4iyJOEuyszF1Oss+C6cLQaLqFIAAA3wSURBVCEJtHeqzfKJAXg71zF3WUJYBQl3Ueq01qz/M57XN0eQfj2X/xvQhmceaIVtFWn0JUR5kXAXpSo+OYO560PZffwiHV3yGn21biSNvoQobxLuolSYTJqvfj/N4u8jMWl47SEPnuzqKo2+hDATCXdx305eTGNWUCgHYq/Q070Bb43yprmjNPoSwpwk3EWJ5eSa+PyXU3y44wR2VW1YMtaHsZ2cpXWAEBWAhLsokYhzV5kRdJSw+KsM9GzMohFeNJJGX0JUGBLuolgys3P5585oPvs5hrr21fj08Y4M9nYyd1lCiFtIuIsiO3T6CjPWhRBz8RpjOjozf1h76tpLoy8hKiIJd3FP167nsOSH46zaF0vTOjVYNTmQB9o0NHdZQoi7kHAXd7XnxEVmB4dyLiWDJ7u04NVB7ahVXV42QlR08lcqCpWSns2iLRGsOxRHy4Y1+eaZrgS4Opq7LCFEEUm4i9tsC0tg/sZwrlzL4vnerXixnzT6EqKykXAX+RJTM3ltYzjfh53Hw6k2X04KwKuZNPoSojKScBdorVl3KI43thwjIzuXVwe2ZVqvltLoS4hKTMLdyp29ks6c9aH8EnUJ/xb1WDzGh9aNapm7LCHEfZJwt1Imk2b1vlje/eE4Cnh9hCdPdG6BjTT6EsIiSLhboejENGYFhfDH6SR6tWnIW6O8cK4njb6EsCQS7lYkO9fEsj0nWbojihrVqvD+OF9Gd2wmjb6EsEAS7lYiLD6FGetCiEi4yhDvJiwc7kVDh+rmLksIUUYk3C1cZnYuS3+KYtmekzjWrMZnT3RkkJc0+hLC0km4W7CDsVeYuS6Ek5eu8bC/M3OHeFDH3tbcZQkhyoGEuwVKu57Du9siWb3vNM71avDfKZ3p4d7A3GUJIcqRhLuF2XU8kbnBoSRczeSp7q7834C21JRGX0JYHfmrtxBJ17JYtDmC4D/jad2oFuue7UanFvXMXZYQwkwk3Cs5rTVbQ8/z2qYwktOzmd63NS/0bU31qtLoSwhrJuFeiSVezWTehjC2R1zAu1kdVk/ujEfT2uYuSwhRAdwz3JVSK4BhQKLW2ssYcwTWAq5ALPCw1jpJ5X0aZikwBEgHJmmtD5dN6dZLa823f8SxaEsEWTkmZg9ux5QeblSVRl9CCENR0mAlMOiWsVnAT1prd+An4zrAYMDd+JkGfFo6ZYobzl5JZ8LyA8wICqG9U22+f6knzzzQSoJdCHGTe265a633KKVcbxkeAfQ2Lq8CdgMzjfHVWmsN7FdK1VVKOWmtE0qrYGuVa9Ks2hvLkh+OU8VG8cZILx4LdJFGX0KIQpV0n3vjAoF9HmhsXG4GnC2wXJwxdlu4K6Wmkbd1j4uLSwnLsA5RF1KZERTCn2eS6d22IW+N8qZp3RrmLksIUYHd9wFVrbVWSukSrLcMWAbg7+9f7PWtQVaOic9+juGfO6OpWb0KH43vwIgOTaXRlxDinkoa7hdu7G5RSjkBicZ4PNC8wHLOxpgoppC4ZGasCyHyfCoP+TbltYc8aFBLGn0JIYqmpEfhNgETjcsTgY0Fxp9UeboAKbK/vXgys3N5e+sxRn7yG0npWXz+pD8fP+onwS6EKJainAr5NXkHTxsopeKA14DFwDdKqSnAaeBhY/Gt5J0GGU3eqZBPlUHNFmv/ycvMCgoh9nI6jwY2Z9bg9tSpIY2+hBDFV5SzZR69w039CllWA3+536KsTWpmNou/j+Sr38/g4mjP/57uTLfW0uhLCFFy8glVM9sZeYG568O4cDWTp3u48fKANthXk38WIcT9kRQxkyvXsnj9u3A2HDmHe6Na/Ou5bvi5SKMvIUTpkHAvZ1prvgtJYMGmcFIzs3mpnzvP92kljb6EEKVKwr0cnU/Ja/S149gFfJ3r8M7YzrRrIo2+hBClT8K9HGitWXPwLG9tOUa2ycTcIe2Z3MONKtI6QAhRRiTcy9jpy9eYFRTKvpOX6dLSkcWjfXBtUNPcZQkhLJyEexnJNWm+/O0U720/jq2NDW+N8uaRgObS6EsIUS4k3MvA8fN5jb6Onk2mX7tGvDHKC6c60uhLCFF+JNxLUVaOiX/tjuaTXdE42Nnyj0f9eMjHSRp9CSHKnYR7KTlyNpmZ60I4fiGVER2a8tpDnjjWrGbusoQQVkrC/T5lZOXy/vbjrPjtFI0c7Fg+0Z9+7Rvfe0UhhChDEu73YW/MJWYFhXLmSjqPdXZh1uB21LaTRl9CCPOTcC+Bq5nZvL31GF8fOEuL+vZ8PbULXVvVN3dZQgiRT8K9mHZEXGDuhlAupl5nWq+W/O3BNtSoJq0DhBAVi4R7EV1Ou86C7yL47ug52jVxYNkEf3yb1zV3WUIIUSgJ93vQWrPxyDkWfhdO2vUcXu7fhmcfaEW1qiX9EishhCh7Eu53cS45g3kbwtgZmUiH5nV5d6wPbRo7mLssIYS4Jwn3QphMmv8dOMPi7yPJNWnmD/NgUjdXafQlhKg0JNxvcerSNWYFhfD7qSt0b12ft0f54FLf3txlCSFEsUi4G3JyTSz/9RQf/HiCalVteGeMNw/7N5fWAUKISknCHTiWcJWZQSGExKXQ36Mxb4z0onFtO3OXJYQQJWbV4X49J5dPdkbzr90x1LW35ZPHOjLEu4lsrQshKj2rDfdDp5OYGRRCdGIao/2aMX+YB/Wk0ZcQwkJYXbinZ+Ww5IfjrNwbi1NtO758KoA+bRuZuywhhChVVhXuv0ZdYlZwCHFJGUzo0oIZg9riII2+hBAWyCrCPSUjmze3RPDNH3G4NajJ2mld6NxSGn0JISyXxYf7D+Hnmb8hjMvXsniudyte6ueOna00+hJCWDaLDfeLqddZsCmcLaEJtHeqzfKJAXg71zF3WUIIUS4sLty11gQfjuf1zRFkZOXy6sC2TOvVEtsq0uhLCGE9LCrc45MzmBMcys8nLtLRJa/RV+tG0uhLCGF9yiTclVKDgKVAFeALrfXisnicG0wmzX9/P80730eigQUPeTChqzT6EkJYr1IPd6VUFeAToD8QBxxUSm3SWkeU9mMBxFxMY1ZQCAdjk+jp3oC3RnnT3FEafQkhrFtZbLkHAtFa65MASqk1wAig1MP9m4NnmbcxDLuqNiwZ68PYTs7SOkAIISibcG8GnC1wPQ7ofOtCSqlpwDQAFxeXEj2QW8Oa9GvXiIUjPGnkII2+hBDiBrMdUNVaLwOWAfj7++uS3EeAqyMBro6lWpcQQliCsjg/MB5oXuC6szEmhBCinJRFuB8E3JVSbkqpasAjwKYyeBwhhBB3UOq7ZbTWOUqpF4AfyDsVcoXWOry0H0cIIcSdlck+d631VmBrWdy3EEKIe5PP5AshhAWScBdCCAsk4S6EEBZIwl0IISyQ0rpEnx8q3SKUugicLuHqDYBLpVhOZSBztg4yZ+twP3NuobVuWNgNFSLc74dS6g+ttb+56yhPMmfrIHO2DmU1Z9ktI4QQFkjCXQghLJAlhPsycxdgBjJn6yBztg5lMudKv89dCCHE7Sxhy10IIcQtJNyFEMICVepwV0oNUkodV0pFK6Vmmbue0qKUWqGUSlRKhRUYc1RK/aiUijJ+1zPGlVLqH8ZzEKKU6mi+yktOKdVcKbVLKRWhlApXSr1kjFvsvJVSdkqpA0qpo8acFxrjbkqp3425rTVaZ6OUqm5cjzZudzVn/SWllKqilPpTKbXZuG7R8wVQSsUqpUKVUkeUUn8YY2X62q604V7gi7gHAx7Ao0opD/NWVWpWAoNuGZsF/KS1dgd+Mq5D3vzdjZ9pwKflVGNpywFe0Vp7AF2Avxj/npY87+tAX621L9ABGKSU6gK8A3yotW4NJAFTjOWnAEnG+IfGcpXRS8CxAtctfb439NFadyhwTnvZvra11pXyB+gK/FDg+mxgtrnrKsX5uQJhBa4fB5yMy07AcePyv4FHC1uuMv8AG4H+1jJvwB44TN73DV8Cqhrj+a9z8r4joatxuaqxnDJ37cWcp7MRZH2BzYCy5PkWmHcs0OCWsTJ9bVfaLXcK/yLuZmaqpTw01lonGJfPA42Nyxb3PBhvv/2A37HweRu7KI4AicCPQAyQrLXOMRYpOK/8ORu3pwD1y7fi+/YRMAMwGdfrY9nzvUED25VSh5RS04yxMn1tm+0LskXJaa21Usoiz2FVStUCgoC/aq2vKqXyb7PEeWutc4EOSqm6wHqgnZlLKjNKqWFAotb6kFKqt7nrKWc9tNbxSqlGwI9KqciCN5bFa7syb7lb2xdxX1BKOQEYvxONcYt5HpRStuQF+1da62Bj2OLnDaC1TgZ2kbdboq5S6saGV8F55c/ZuL0OcLmcS70f3YHhSqlYYA15u2aWYrnzzae1jjd+J5L3n3ggZfzarszhbm1fxL0JmGhcnkjePukb408aR9i7ACkF3upVGipvE305cExr/UGBmyx23kqphsYWO0qpGuQdYzhGXsiPNRa7dc43nouxwE5t7JStDLTWs7XWzlprV/L+XndqrR/HQud7g1KqplLK4cZlYAAQRlm/ts19oOE+D1IMAU6Qt59yrrnrKcV5fQ0kANnk7W+bQt6+xp+AKGAH4Ggsq8g7aygGCAX8zV1/Cefcg7z9kiHAEeNniCXPG/AB/jTmHAb83RhvCRwAooFvgerGuJ1xPdq4vaW553Afc+8NbLaG+RrzO2r8hN/IqrJ+bUv7ASGEsECVebeMEEKIO5BwF0IICyThLoQQFkjCXQghLJCEuxBCWCAJdyGEsEAS7kIIYYH+H8xfD99rW4RkAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "plt.plot(score_arr2, label ='accumulated scores in one game play 1')\n",
        "plt.legend(loc='upper left')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hpeJyIbOeY2q"
      },
      "outputs": [],
      "source": [
        "## game play video 2\n",
        "agent.load_state_dict(torch.load(\"weights.pt\"))\n",
        "env = gnwrapper.LoopAnimation(gym.make(game_name)) \n",
        "state = env.reset()\n",
        "state_dim = len(state)\n",
        "game_score = 0\n",
        "score_arr2 = []\n",
        "state_arr = []\n",
        "for i in range(1000):\n",
        "    if i == 0:\n",
        "        stacked_state = np.concatenate((state, state, state, state, state, state, state, state), axis=0)\n",
        "    else:\n",
        "        stacked_state = np.roll(stacked_state,-state_dim,axis=0)        \n",
        "        stacked_state[-state_dim:]=state\n",
        "    with torch.no_grad():\n",
        "        hs = agent.representation_network(torch.from_numpy(stacked_state).float().to(device))\n",
        "        P, v = agent.prediction_network(hs)\n",
        "        action = np.random.choice(np.arange(agent.action_space), p=P.detach().numpy())   \n",
        "    if i %5==0:\n",
        "        env.render()\n",
        "    state, r, done, info = env.step(action.item())\n",
        "    print(r, done, info)\n",
        "    state_arr.append(state[0])\n",
        "    game_score += r \n",
        "    score_arr2.append(game_score)\n",
        "    if i %10==0: \n",
        "        print(game_score)\n",
        "    if done:\n",
        "        print('last frame number : ',i)\n",
        "        print('score :', game_score)\n",
        "        print(state_arr)\n",
        "       \n",
        "        break\n",
        "env.reset()\n",
        "env.display()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ArdvgJqDediI",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "outputId": "94985350-7a89-4efc-b9a4-9059eef9720f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f86ae73b8b0>"
            ]
          },
          "metadata": {},
          "execution_count": 10
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deVyVdfr/8dcHxQX3PQQR94VdcE8zzVzLPauZUluspm2m+SWaOlm2WDY1tkyNjaVWM5bgnpWaS1NamqVsioKigguIgCI75/r9wZEvKiogcOCc6/l48OA+n3Pf51yfw+HNfe5zcx0jIiillLIvTrYuQCmlVPnTcFdKKTuk4a6UUnZIw10ppeyQhrtSStmhmrYuAKB58+bi6elp6zKUUqpa2bt371kRaVHcdVUi3D09Pfn1119tXYZSSlUrxphj17pOD8sopZQd0nBXSik7pOGulFJ2qEoccy9Obm4u8fHxZGVl2boUpdQ11KlTB3d3d5ydnW1dirpClQ33+Ph4GjRogKenJ8YYW5ejlLqCiJCcnEx8fDzt2rWzdTnqCiU6LGOMiTPGhBtj9hljfrWONTXGbDbGHLZ+b2IdN8aYd40xMcaYMGNMj7IUlpWVRbNmzTTYlaqijDE0a9ZMX11XUaU55n67iPiLSJD18kzgexHpBHxvvQwwAuhk/ZoOfFjW4jTYlara9He06rqZN1THAMusy8uAsUXGl0uBn4HGxhjXm7gfpZSyO5k5+bz+zQHiUzIq5PZLGu4CbDLG7DXGTLeOtRKRU9bl00Ar67IbcKLItvHWscsYY6YbY341xvyalJRUhtId27x583jrrbeuu86aNWuIiooq9W3Xr1+/rGVVmpEjR5KammrrMqqk7du3M3r06Ju+nc2bNxMYGIiPjw+BgYFs3bq1HKpTADtjzzLsHz/wrx1H2BZdMflX0jdUbxWRBGNMS2CzMeZg0StFRIwxpfrUDxFZDCwGCAoK0k8MqQBr1qxh9OjRdO/e3dalXCUvL4+aNcv+fv7GjRvLsRpVnObNm7N+/Xpat25NREQEw4YNIyEhwdZlVWvns3J5feMB/rv7BJ7NXFgxvQ992jerkPsq0Z67iCRYvycCq4FewJlLh1us3xOtqycAbYps7m4dq3bGjh1LYGAgXl5eLF68uHD822+/pUePHvj5+TFkyBAA0tPTmTZtGj4+Pvj6+hIaGgpcvhccEhLC1KlTAZg6dSpPPPEEffr0oX379mzfvp2HHnqIbt26Fa5zve2L+vjjj+nZsyd+fn5MmDCBjIwMdu7cybp163j++efx9/cnNjaW2NhYhg8fTmBgIAMGDODgwYK/0UePHqVv3774+PgwZ86cYh+LixcvMmrUKPz8/PD29ubLL78EYM+ePfTr1w8/Pz969erFhQsXyMrKKnwsAgIC2LZtGwBLly7l7rvvZvDgwQwZMoSLFy/y0EMP0atXLwICAli7di0AkZGR9OrVC39/f3x9fTl8+PBV9Xh6enL27Fni4uLo1q0bjz76KF5eXtx5551kZmZetX5cXByDBw/G19eXIUOGcPz48cKfwzPPPEO/fv1o3749ISEhhdssXLiQnj174uvry4svvljs47JkyRI6d+5Mr169ePTRR3nqqacAWL9+Pb179yYgIIA77riDM2fOAAWvuKZMmcKAAQNo27Ytq1atYsaMGfj4+DB8+HByc3MB2Lt3L7fddhuBgYEMGzaMU6dOXXXfU6dO5fHHHycoKIjOnTuzYcOGq9bZvXs3ffv2JSAggH79+hEdHQ3AwIED2bdvX+F6t956K/v3779s24CAAFq3bg2Al5cXmZmZZGdnF/s4qBvbHHWGoW/v4Ms9J3jstvZ8++eBFRbsQMHpTNf7AuoBDYos7wSGAwuBmdbxmcCb1uVRwDeAAfoAu290H4GBgXKlqKiowuV56yLkno92luvXvHURV93nlZKTk0VEJCMjQ7y8vOTs2bOSmJgo7u7ucuTIkcvWmTFjhjz77LOF2547d05EROrVq1c4tnLlSpkyZYqIiEyZMkUmT54sFotF1qxZIw0aNJCwsDDJz8+XHj16yO+//37d7V988UVZuHChiIicPXu2cJ3Zs2fLu+++W3gfK1euLLxu8ODBcujQIRER+fnnn+X2228XEZG77rpLli1bJiIi77///mX3eUlISIg88sgjhZdTU1MlOztb2rVrJ7t37xYRkbS0NMnNzZW33npLpk2bJiIiBw4ckDZt2khmZqZ8+umn4ubmVviYzZo1Sz777DMREUlJSZFOnTpJenq6PPXUU/L555+LiEh2drZkZGRcVU/btm0lKSlJjh49KjVq1Ch8vCZNmlR4m0WNHj1ali5dKiIiS5YskTFjxhQ+RhMnTpT8/HyJjIyUDh06iIjId999J48++qhYLBbJz8+XUaNGyY4dOy67zYSEBGnbtq0kJydLTk6O3HrrrfLkk0+KSMHP32KxiIjIxx9/LM8991zhz61///6Sk5Mj+/btk7p168rGjRtFRGTs2LGyevVqycnJkb59+0piYqKIiKxYsaLw8SxqypQpMmzYMMnPz5dDhw6Jm5ubZGZmyrZt22TUqFGX/UxERDZv3izjx48XEZGlS5cWPl+jo6OluN/BolauXClDhgwp9rqiv6vqakkXsuTJL/ZK2+ANMuydHbL/REq53Tbwq1wjV0vyurgVsNr6rnhN4D8i8q0xZg/wlTHmYeAYcI91/Y3ASCAGyACm3dyfH9t59913Wb16NQAnTpzg8OHDJCUlMXDgwMLzeps2bQrAli1bWLFiReG2TZo0ueHt33XXXRhj8PHxoVWrVvj4+AAFe0lxcXH4+/uXqM6IiAjmzJlDamoq6enpDBs27Kp10tPT2blzJ5MmTSocu7QX9tNPPxW+0njggQcIDg6+ansfHx/++te/EhwczOjRoxkwYADh4eG4urrSs2dPABo2bAjAjz/+yNNPPw1A165dadu2LYcOHQJg6NChhY/Zpk2bWLduXeF7B1lZWRw/fpy+ffvy6quvEh8fz/jx4+nUqdN159+uXbvCxyowMJC4uLir1tm1axerVq0qnOOMGTMKrxs7dixOTk507969cA9706ZNbNq0iYCAgMLH7/DhwwwcOLBwu927d3PbbbcVzmfSpEmF84yPj2fy5MmcOnWKnJycy84DHzFiBM7Ozvj4+JCfn8/w4cMLH+O4uDiio6OJiIhg6NChAOTn5+PqWvw5Cffccw9OTk506tSJ9u3bF74auyQtLY0pU6Zw+PBhjDGFrwwmTZrE/PnzWbhwIZ988kmxrwgviYyMJDg4mE2bNl1zHXU1EWHtvpO8tD6Si9n5/HVoZx67rQO1alZOY4AbhruIHAH8ihlPBoYUMy7Ak+VSndWLd3mV582VyPbt29myZQu7du3CxcWFQYMGlel83qKnil25fe3atQFwcnIqXL50OS8v74bbXzJ16lTWrFmDn58fS5cuZfv27VetY7FYaNy48WUvxa9VZ3E6d+7Mb7/9xsaNG5kzZw5Dhgxh3Lhx192mOPXq1StcFhFCQ0Pp0qXLZet069aN3r178/XXXzNy5Ej+9a9/MXjw4GveZtHHrkaNGsUelrmeotuL9QPjRYRZs2bx2GOPleq2Lnn66ad57rnnuPvuu9m+fTvz5s276v6cnJxwdnYufOwv/dxFBC8vL3bt2nXD+7ny53bl5blz53L77bezevVq4uLiGDRoEAAuLi4MHTqUtWvX8tVXX7F3795ibz8+Pp5x48axfPlyOnToUNLpO7yTqZnMWRPB1oOJBHg05s0JvnRq1aBSa9DeMteQlpZGkyZNcHFx4eDBg/z8888A9OnThx9++IGjR48CcO7cOaBgj/SDDz4o3D4lJQWAVq1aceDAASwWS+GrgNIoyfYXLlzA1dWV3Nxcvvjii8LxBg0acOHCBaBgr7pdu3asXLkSKAivS8dY+/fvX/iqo+j2RZ08eRIXFxf++Mc/8vzzz/Pbb7/RpUsXTp06xZ49ewrryMvLY8CAAYW3c+jQIY4fP35VgAMMGzaM9957rzBQf//9dwCOHDlC+/bteeaZZxgzZgxhYWGle9CK0a9fv8vmOGDAgOuuP2zYMD755BPS09MBSEhIIDEx8bJ1evbsyY4dO0hJSSEvL6/w1Q8UPH/c3ApOElu2bBml0aVLF5KSkgrDPTc3l8jIyGLXXblyJRaLhdjYWI4cOXLV41y0jqVLl1523SOPPMIzzzxDz549i32lmZqayqhRo1iwYAH9+/cv1RwclcUifP7zMe585wd2xSbzt9HdCXm8X6UHO2i4X9Pw4cPJy8ujW7duzJw5kz59+gDQokULFi9ezPjx4/Hz82Py5MkAzJkzh5SUFLy9vfHz8yt8E3HBggWMHj2afv36XfOl9fWUZPv58+fTu3dv+vfvT9euXQvH7733XhYuXEhAQACxsbF88cUXLFmyBD8/P7y8vArfwFy0aBEffPABPj4+1zwbIjw8vPBNzpdeeok5c+ZQq1YtvvzyS55++mn8/PwYOnQoWVlZ/OlPf8JiseDj48PkyZNZunTpZXvHl8ydO5fc3Fx8fX3x8vJi7ty5AHz11Vd4e3vj7+9PREQEDz74YKkftyu99957fPrpp/j6+vLZZ5+xaNGi665/5513cv/99xe+0Txx4sTCP5SXuLm58cILL9CrVy/69++Pp6cnjRo1AgreOJ00aRKBgYE0b968VLXWqlWLkJAQgoOD8fPzw9/fn507dxa7roeHB7169WLEiBF89NFH1KlT57LrZ8yYwaxZswgICCh8NXhJYGAgDRs2ZNq04o+cvv/++8TExPDyyy/j7++Pv7//VX/g1P85evYi9378M3PWRODfpjGb/jKQh25tRw0n2/yjl7m012RLQUFBcuWHdRw4cIBu3brZqCKlSiY9PZ369euTl5fHuHHjeOihh8p0uKospk6dyujRo5k4cWKZtj958iSDBg3i4MGDODmVfT/P0X9X8/It/PvHo7yz+RC1ajoxd1R3JgW5V8p/7xpj9sr/dQ24TJVtHKZUdTBv3jy2bNlCVlYWd955J2PHjr3xRlXA8uXLmT17Nm+//fZNBbujizp5nuDQMMIT0rizeyvmj/WmVcM6N96wEuieu1Lqpjji72p2Xj7vb43hw+2xNHZx5qW7vRnpc0ul99qptnvuIqKNiZSqwqrCzmFl23ssheDQMGIS0xnfw425o7rTpF4tW5d1lSob7nXq1CE5OVnb/ipVRYm1n/uVb+Laq4ycPBZ+F83SnXG0blSXpdN6MqhLS1uXdU1VNtzd3d2Jj49Hm4opVXVd+iQme/fj4bPMXBVGfEomD/Zty4zhXalfu8rGJ1CFw93Z2Vk/3UUpZVNpGbm8ujGKr36Np33zenz1WF96tWtq67JKpMqGu1JK2dK3EaeZuzaCcxdzeGJQB54d0ok6zjVsXVaJabgrpVQRSReymbcukq/DT9HdtSGfTu2Jt1sjW5dVahruSilFwRvEq35L4OUNUWTm5PP8sC5MH9ge5xrV8/8ANNyVUg4vITWTF1aFs+NQEoFtm/DGBF86tqz6n0h2PRruSimHZbEIn/9yjDe+OYgAL93txQN92uJko34w5UnDXSnlkGKT0pkZGsaeuBQGdGrOa+N8aNPUxdZllRsNd6WUQ8nNt/Dx/47wjy2Hqetcg7cm+TGhh5vd/bOkhrtSymFEJKQRHBpG5MnzjPC+hZfGeNGygX3+h62Gu1LK7mXl5vPe1sN8tOMITVxq8eEfejDCp/Sfr1CdaLgrpezar3HnmBEaxpGki0wKdGf2qG40dql6jb7Km4a7UsoupWfnsfDbgyz/+RitG9Vl+UO9GNi5ha3LqjQa7kopu7PjUBIvrArnZFomU/p68vywLtSr4o2+yptjzVYpZddSM3KYv+EAob/F06FFPVY+1pcgz+rR6Ku8abgrpezCN+GnmLs2kpSMHJ66vSNPDe5YrRp9lTcNd6VUtZZ4Pou/rY3k28jTeLs1ZNlDPfFqXf0afZU3DXelVLUkIoTsjWf+hiiy8iwED+/KowPaUbOaNvoqbxruSqlq58S5DF5YHc7/Dp+ll2dTFkzwoX2L6t3oq7xpuCulqo18i7B8VxwLv4vGAPPHePGH3vbR6Ku8abgrpaqFmMQLBIeGs/dYCrd1bsFr431wa1zX1mVVWRruSqkqLTffwr92xPLu9zG41K7B2/f4MS7A/hp9lTcNd6VUlRUen8aM0DAOnDrPKF9X5t3lRYsGtW1dVrWg4a6UqnKycvP5x5bDfPy/IzSrV4t/PRDIMK9bbF1WtVLicDfG1AB+BRJEZLQxph2wAmgG7AUeEJEcY0xtYDkQCCQDk0UkrtwrV0rZpV+OJDNzVThHz15kclAbXhjVjUZ1nW1dVrVTmhNCnwUOFLn8BvCOiHQEUoCHreMPAynW8Xes6yml1HVdyMpl7poIJi/+mTyLhS8e6c0bE3012MuoROFujHEHRgH/tl42wGAgxLrKMmCsdXmM9TLW64cYfedDKXUd26ITGfbOD3z+yzEe6t+O7/48kP4dm9u6rGqtpIdl/gHMABpYLzcDUkUkz3o5HnCzLrsBJwBEJM8Yk2Zd/2zRGzTGTAemA3h4eJS1fqVUNZZyMYf5G6JY9XsCnVrWJ/SJfvTwaGLrsuzCDcPdGDMaSBSRvcaYQeV1xyKyGFgMEBQUJOV1u0qpqk9E+Dr8FC+ujSQtM5dnhnTiyds7ULum4zb6Km8l2XPvD9xtjBkJ1AEaAouAxsaYmta9d3cgwbp+AtAGiDfG1AQaUfDGqlJKceZ8FnPWRLA56gy+7o34/JHedHNtaOuy7M4Nj7mLyCwRcRcRT+BeYKuI/AHYBky0rjYFWGtdXme9jPX6rSKie+ZKOTgR4cs9x7nj7R38cCiJF0Z2ZdUT/TTYK8jNnOceDKwwxrwC/A4ssY4vAT4zxsQA5yj4g6CUcmDHkzOYuSqMnbHJ9G7XlDcm+OLZvJ6ty7JrpQp3EdkObLcuHwF6FbNOFjCpHGpTSlVz+RZh6c443voumhpOhlfHeXNfTw9t9FUJ9D9UlVIV4tCZC8wICWPfiVQGd23Jq+O8cW2kjb4qi4a7Uqpc5eRZ+HB7LO9vO0z92jVZdK8/d/u11kZflUzDXSlVbvafSCU4NIyDpy9wt19rXryrO83qa6MvW9BwV0rdtMycfN7Zcoh//+8ILRvU4d8PBnFH91a2LsuhabgrpW7KrthkZq0KIy45g/t6eTBrZFca1tF+MLam4a6UKpPzWbks+OYg//nlOG2bufCfR3vTr4P2g6kqNNyVUqX2/YEzzF4dQeKFLB4d0I7nhnahbi1tHVCVaLgrpUosOT2bl9ZHsW7/Sbq0asBHDwTi36axrctSxdBwV0rdkIiwbv9JXlofxYWsXP5yR2eeGNSBWjVL85EQqjJpuCulrutUWiZzVkfw/cFE/No05s0JvnS5pcGNN1Q2peGulCqWxSKs2HOC1zceINdiYc6obkzr344a2jqgWtBwV0pdJe7sRWauCuPnI+fo274ZCyb40LaZNvqqTjTclVKF8vItfPpTHH/fHI2zkxMLxvswuWcbbR1QDWm4K6UAOHj6PMEhYeyPT+OObq14Zaw3tzSqY+uyVBlpuCvl4LLz8vlgWyz/3BZDo7rOvHdfAKN9XXVvvZrTcFfKgf1+PIXg0DAOnUlnXIAbc0d3p2m9WrYuS5UDDXelHFBGTh5/33SIT346yi0N6/DJ1CAGd9VGX/ZEw10pB7Mz5iwzV4Vz/FwGf+zjQfDwrjTQRl92R8NdKQeRlpnL6xsPsGLPCdo1r8eK6X3o076ZrctSFUTDXSkHsCnyNHPWRHA2PZvHbmvPX+7oTB1nbfRlzzTclbJjZ9Ozmbcukg1hp+h6SwP+PSUIX3dt9OUINNyVskMiwpp9Cby0PoqM7Hz+OrQzjw/qgHMNbfTlKDTclbIzJ1Mzmb06nG3RSQR4FDT66tRKG305Gg13peyExSJ8sfs4b3xzkHyL8LfR3ZnSz1MbfTkoDXel7MCRpHRmhoazO+4ct3ZszuvjfWjT1MXWZSkb0nBXqhrLy7fw7x+P8s7mQ9Su6cSbE32ZFOiurQOUhrtS1VXUyfPMCN1PRMJ5hnm1Yv4Yb1o21EZfqoCGu1LVTHZePu9vjeHD7bE0dnHmn3/owQjvW3RvXV1Gw12pamTvsXMEh4YTk5jO+B5uzB3VnSba6EsVQ8NdqWrgYnYeC7+LZtmuOFo3qsvSaT0Z1KWlrctSVdgNw90YUwf4AahtXT9ERF40xrQDVgDNgL3AAyKSY4ypDSwHAoFkYLKIxFVQ/UrZvf8dTmLWqnDiUzKZ0rctzw/vSv3aul+mrq8k/66WDQwWET/AHxhujOkDvAG8IyIdgRTgYev6DwMp1vF3rOsppUopLSOX51fu54Elu6lV04mVj/flpTHeGuyqRG74LBERAdKtF52tXwIMBu63ji8D5gEfAmOsywAhwPvGGGO9HaVUCXwbcZq5ayM4dzGHPw3qwDNDOmmjL1UqJdoFMMbUoODQS0fgAyAWSBWRPOsq8YCbddkNOAEgInnGmDQKDt2cveI2pwPTATw8PG5uFkrZicQLWcxbF8nG8NN0d23Ip1N74u3WyNZlqWqoROEuIvmAvzGmMbAa6Hqzdywii4HFAEFBQbpXrxyaiBD6WwLzN0SRmZvP88O6MH1ge230pcqsVAfvRCTVGLMN6As0NsbUtO69uwMJ1tUSgDZAvDGmJtCIgjdWlVLFiE/J4IXVEfxwKImgtk1YMMGXji3r27osVc2V5GyZFkCuNdjrAkMpeJN0GzCRgjNmpgBrrZuss17eZb1+qx5vV+pqFovw2c/HeOPbgwC8dLcXD/Rpi5M2+lLloCR77q7AMutxdyfgKxHZYIyJAlYYY14BfgeWWNdfAnxmjIkBzgH3VkDdSlVrsUnpBIeE8euxFAZ2bsFr47xxb6KNvlT5KcnZMmFAQDHjR4BexYxnAZPKpTql7ExuvoXFPxxh0feHqetcg7cm+TGhh5u2DlDlTk+YVaqSRCSkMSMkjKhT5xnpcwvz7vaiZQNt9KUqhoa7UhUsKzefRd8fZvEPR2jiUouP/tiD4d6uti5L2TkNd6Uq0J64cwSHhHHk7EUmBbozZ1R3Grk427os5QA03JWqAOnZebz57UGW7zqGe5O6fPZwLwZ0amHrspQD0XBXqpztOJTEC6vCOZmWydR+njw/rAv1tB+MqmT6jFOqnKRm5PDyhihW/ZZAhxb1CHm8L4Ftm9q6LOWgNNyVukkiwjcRp/nb2ghSM3J56vaOPDW4ozb6Ujal4a7UTUg8n8XctRF8F3kGb7eGLHuoF16ttdGXsj0Nd6XKQERYuTeeVzZEkZ1nYeaIrjxyaztqaqMvVUVouCtVSifOZTBrVTg/xpyll2dTFkzwoX0LbfSlqhYNd6VKKN8iLN8Vx5vfRuNkYP5Yb/7Qy0MbfakqScNdqRKISbzAjJAwfjueyqAuLXh1nA9ujevauiylrknDXanryM238NH2WN7bGoNL7Rq8M9mPsf7a6EtVfRruSl1DeHwaz4fs5+DpC4z2dWXe3V40r1/b1mUpVSIa7kpdISs3n3e2HOLjH47QvH5tFj8QyJ1et9i6LKVKRcNdqSJ+OZLMzFXhHD17kXt7tmHWyG40qquNvlT1o+GuFHAhK5c3vj3I5z8fp03TunzxSG/6d2xu67KUKjMNd+Xwth1M5IXV4Zw+n8XDt7bjr3d2xqWW/mqo6k2fwcphnbuYw8vrI1mz7ySdWtYn9Il+9PBoYuuylCoXGu7K4YgIG8JOMW9dJGmZuTwzpBNP3t6B2jW10ZeyHxruyqGcOZ/F7NURbDlwBl/3RnzxaG+63tLQ1mUpVe403JVDEBG+3HOCVzceICfPwuyR3ZjW31MbfSm7peGu7N7x5AxmrgpjZ2wyvds15Y0Jvng2r2frspSqUBruym7lW4RPfzrKW5uiqenkxGvjfLi3Zxtt9KUcgoa7skvRpy8wIzSM/SdSGdy1Ja+O88a1kTb6Uo5Dw13ZlZw8C//cHsMH22JoUMeZRff6c7dfa230pRyOhruyG/tPpDIjJIzoMxcY49+av43uTjNt9KUclIa7qvYyc/J5e3M0S348SssGdfj3g0Hc0b2VrctSyqY03FW1tjP2LLNWhXMsOYP7e3swc0RXGtbRRl9Kabiraul8Vi6vbzzIf3cfp20zF/7zaG/6ddBGX0pdouGuqp0tUWeYvSacpAvZTB/Ynr/c0Zm6tbR1gFJF3TDcjTFtgOVAK0CAxSKyyBjTFPgS8ATigHtEJMUUnJawCBgJZABTReS3iilfOZLk9GxeWh/Fuv0n6XpLAxY/EIRfm8a2LkupKqkke+55wF9F5DdjTANgrzFmMzAV+F5EFhhjZgIzgWBgBNDJ+tUb+ND6XakyERHW7T/JvHWRpGfn8Zc7OvPEoA7UqqmtA5S6lhuGu4icAk5Zly8YYw4AbsAYYJB1tWXAdgrCfQywXEQE+NkY09gY42q9HaVK5VRaJnNWR/D9wUT82zTmzYm+dG7VwNZlKVXlleqYuzHGEwgAfgFaFQns0xQctoGC4D9RZLN469hl4W6MmQ5MB/Dw8Chl2creWSzCf/cc5/WNB8mzWJgzqhvT+rejhrYOUKpEShzuxpj6QCjwZxE5X/Q//kREjDFSmjsWkcXAYoCgoKBSbavs29GzF5kZGsYvR8/Rr0MzFoz3xaOZi63LUqpaKVG4G2OcKQj2L0RklXX4zKXDLcYYVyDROp4AtCmyubt1TKnrysu38MlPR/n7pkPUqunEGxN8uCeojbYOUKoMSnK2jAGWAAdE5O0iV60DpgALrN/XFhl/yhizgoI3UtP0eLu6kQOnzhMcGkZYfBpDu7filbHetGpYx9ZlKVVtlWTPvT/wABBujNlnHXuBglD/yhjzMHAMuMd63UYKToOMoeBUyGnlWrGyK9l5+XywLZZ/bouhUV1n3r8/gFE+rrq3rtRNKsnZMj8C1/pNG1LM+gI8eZN1KQfw2/EUgkPCOJyYzrgAN/42ujtN6tWydVlK2QX9D1VV6TJy8njru0N8uvMotzSsw+azzDwAAA5lSURBVKdTe3J715a2Lkspu6LhrirVTzFnmbkqjBPnMvljHw+Ch3elgTb6UqrcabirSpGWmctrXx/gy19P0K55Pb6c3ofe7ZvZuiyl7JaGu6pwmyJPM2dNBMkXc3j8tg78+Y5O1HHWRl9KVSQNd1Vhki5kM299JF+HnaKba0OWTOmJj3sjW5ellEPQcFflTkRY/XsCL2+IIiM7n/93Z2ceu60DzjW00ZdSlUXDXZWrhNRMZq8OZ3t0Ej08Chp9dWypjb6Uqmwa7qpcWCzCF78cY8E3B7EIvHhXdx7s66mNvpSyEQ13ddOOJKUzMzSc3XHnGNCpOa+N86FNU230pZQtabirMsvLt/Dx/47yzpZD1KnpxMKJvkwMdNfWAUpVARruqkyiTp5nRuh+IhLOM8yrFfPHeNNSG30pVWVouKtSycrN5/2tMXy0I5bGLrX48A89GOHjauuylFJX0HBXJbb32DlmhIQRm3SRCT3cmTu6G41dtNGXUlWRhru6oYvZeSz8Lpplu+Jo3aguyx7qxW2dW9i6LKXUdWi4q+v64VASs1aFczItkwf7tOX54V2pX1ufNkpVdfpbqoqVlpHL/K+jCNkbT/sW9fjqsb709Gxq67KUUiWk4a6u8m3EKeaujeTcxRz+NKgDzwzRRl9KVTca7qpQ4oUsXlwbyTcRp+nu2pBPp/bE200bfSlVHWm4K0SEkL3xvPL1ATJz83l+WBemD2yvjb6UqsY03B3ciXMZvLA6nP8dPktQ2yYsmOBLx5b1bV2WUuomabg7KItFWL4rjje/i8YAL4/x4o+92+Kkjb6Usgsa7g4oJjGdmaFh/HoshYGdW/DaOG/cm2ijL6XsiYa7A8nNt7D4hyMs2nKYurVq8PdJfozv4aaNvpSyQxruDiIiIY0ZIWFEnTrPSJ9beOlub1o0qG3rspRSFUTD3c5l5eaz6PvDLP7hCE3r1eKjP/ZguLc2+lLK3mm427E9cecIDgnjyNmL3BPkzuyR3Wnk4mzrspRSlUDD3Q6lZ+fx5rcHWb7rGO5N6vL5w725tVNzW5ellKpEGu52Zlt0IrNXhXPqfBbT+nvy/+7sQj1t9KWUw9HfejuRcjGH+RuiWPV7Ah1b1ifk8X4Etm1i67KUUjai4V7NiQgbw0/z4roIUjNyeXpwR54a3JHaNbXRl1KOTMO9Gks8n8WcNRFsijqDj1sjlj/Um+6tG9q6LKVUFXDDcDfGfAKMBhJFxNs61hT4EvAE4oB7RCTFFPw3zCJgJJABTBWR3yqmdMclIqz8NZ75X0eRk2dh1oiuPHxrO2pqoy+llFVJ0mApMPyKsZnA9yLSCfjeehlgBNDJ+jUd+LB8ylSXnDiXwQNLdjMjNIxurg355tkBPHZbBw12pdRlbrjnLiI/GGM8rxgeAwyyLi8DtgPB1vHlIiLAz8aYxsYYVxE5VV4FO6p8i7BsZxwLv4umhpPhlbHe3N/LQxt9KaWKVdZj7q2KBPZpoJV12Q04UWS9eOvYVeFujJlOwd49Hh4eZSzDMRw+c4EZoWH8fjyVQV1a8No4H1o3rmvrspRSVdhNv6EqImKMkTJstxhYDBAUFFTq7R1BTp6Fj3bE8v7WGOrVrsE/Jvszxr+1NvpSSt1QWcP9zKXDLcYYVyDROp4AtCmynrt1TJVSWHwqM0LCOHj6Anf5tebFu7rTvL42+lJKlUxZ34VbB0yxLk8B1hYZf9AU6AOk6fH20snKzef1jQcY+8FPpGTk8PGDQbx3X4AGu1KqVEpyKuR/KXjztLkxJh54EVgAfGWMeRg4BtxjXX0jBadBxlBwKuS0CqjZbv18JJmZoWHEJWdwX682zBzRjUZ1tdGXUqr0SnK2zH3XuGpIMesK8OTNFuVoLmTlsuCbg3zxy3E8mrrwn0d606+jNvpSSpWd/oeqjW09eIbZqyM4cz6LR25tx3N3dsallv5YlFI3R1PERs5dzOHl9ZGs2XeSTi3r888n+hHgoY2+lFLlQ8O9kokI68NOMW9dJBeycnl2SCf+dHsHbfSllCpXGu6V6HRaQaOvLQfO4OfeiDcm9qbrLdroSylV/jTcK4GIsGLPCV77+gC5FguzR3bjoVvbUUNbByilKoiGewU7lnyRmaHh7DqSTJ/2TVkw3hfP5vVsXZZSys5puFeQfIvw6U9HeWtTNM5OTrw2zod7e7bRRl9KqUqh4V4Bok8XNPrafyKVIV1b8so4b1wbaaMvpVTl0XAvRzl5Fv65PYYPtsXQoI4z794XwF2+rtroSylV6TTcy8m+E6kEh4QRfeYCY/xb8+JdXjStV8vWZSmlHJSG+03KzMnn75ui+eSno7RsUIclU4IY0q3VjTdUSqkKpOF+E3bGnmVmaDjHz2Vwf28PZo7oSsM62uhLKWV7Gu5lcD4rl9c3HuC/u0/QtpkL/320D307NLN1WUopVUjDvZS2RJ1h9ppwki5kM31ge/5yR2fq1tLWAUqpqkXDvYSS07OZtz6K9ftP0vWWBix+IAi/No1tXZZSShVLw/0GRIS1+07y0vpI0rPzeG5oZx6/rQO1apb1Q6yUUqriabhfx8nUTOasiWDrwUT82zTmzYm+dG7VwNZlKaXUDWm4F8NiEf6z+zgLvjlIvkWYO7o7U/t5aqMvpVS1oeF+haNnLzIzNIxfjp6jf8dmvD7OF49mLrYuSymlSkXD3Sov38KSH4/y9uZD1KrpxBsTfLgnqI22DlBKVUsa7sCBU+cJDg0jLD6Nod1b8cpYb1o1rGPrspRSqswcOtyz8/L5YGsM/9weS2MXZz64vwcjfW7RvXWlVLXnsOG+91gKwaFhxCSmMz7Ajbmju9NEG30ppeyEw4V7Rk4eC7+LZunOOFwb1uHTaT25vUtLW5ellFLlyqHC/cfDZ5m5Koz4lEwe6NOWGcO70EAbfSml7JBDhHtaZi6vfh3FV7/G0655Pb6c3ofe7bXRl1LKftl9uH8XeZq5ayJIvpjDE4M68OyQTtRx1kZfSin7ZrfhnnQhm3nrIvk6/BTdXBuyZEpPfNwb2bospZSqFHYX7iLCqt8SeHlDFJk5+Tw/rAvTB7bHuYY2+lJKOQ67CveE1ExeWBXOjkNJ9PAoaPTVsaU2+lJKOZ4KCXdjzHBgEVAD+LeILKiI+7nEYhE+/+UYb3xzEAHm3dWdB/pqoy+llOMq93A3xtQAPgCGAvHAHmPMOhGJKu/7AohNSmdmaBh74lIY0Kk5r43zoU1TbfSllHJsFbHn3guIEZEjAMaYFcAYoNzD/as9J5izNoI6NZ1YONGXiYHu2jpAKaWomHB3A04UuRwP9L5yJWPMdGA6gIeHR5nuqF2Legzp2pKXxnjRsoE2+lJKqUts9oaqiCwGFgMEBQVJWW6jp2dTeno2Lde6lFLKHlTE+YEJQJsil92tY0oppSpJRYT7HqCTMaadMaYWcC+wrgLuRyml1DWU+2EZEckzxjwFfEfBqZCfiEhked+PUkqpa6uQY+4ishHYWBG3rZRS6sb0f/KVUsoOabgrpZQd0nBXSik7pOGulFJ2yIiU6f+HyrcIY5KAY2XcvDlwthzLqQ50zo5B5+wYbmbObUWkRXFXVIlwvxnGmF9FJMjWdVQmnbNj0Dk7hoqasx6WUUopO6ThrpRSdsgewn2xrQuwAZ2zY9A5O4YKmXO1P+aulFLqavaw566UUuoKGu5KKWWHqnW4G2OGG2OijTExxpiZtq6nvBhjPjHGJBpjIoqMNTXGbDbGHLZ+b2IdN8aYd62PQZgxpoftKi87Y0wbY8w2Y0yUMSbSGPOsddxu522MqWOM2W2M2W+d80vW8XbGmF+sc/vS2jobY0xt6+UY6/Wetqy/rIwxNYwxvxtjNlgv2/V8AYwxccaYcGPMPmPMr9axCn1uV9twL/JB3COA7sB9xpjutq2q3CwFhl8xNhP4XkQ6Ad9bL0PB/DtZv6YDH1ZSjeUtD/iriHQH+gBPWn+e9jzvbGCwiPgB/sBwY0wf4A3gHRHpCKQAD1vXfxhIsY6/Y12vOnoWOFDksr3P95LbRcS/yDntFfvcFpFq+QX0Bb4rcnkWMMvWdZXj/DyBiCKXowFX67IrEG1d/hdwX3HrVecvYC0w1FHmDbgAv1HwecNngZrW8cLnOQWfkdDXulzTup6xde2lnKe7NcgGAxsAY8/zLTLvOKD5FWMV+tyutnvuFP9B3G42qqUytBKRU9bl00Ar67LdPQ7Wl98BwC/Y+bythyj2AYnAZiAWSBWRPOsqRedVOGfr9WlAs8qt+Kb9A5gBWKyXm2Hf871EgE3GmL3GmOnWsQp9btvsA7JV2YmIGGPs8hxWY0x9IBT4s4icN8YUXmeP8xaRfMDfGNMYWA10tXFJFcYYMxpIFJG9xphBtq6nkt0qIgnGmJbAZmPMwaJXVsRzuzrvuTvaB3GfMca4Ali/J1rH7eZxMMY4UxDsX4jIKuuw3c8bQERSgW0UHJZobIy5tONVdF6Fc7Ze3whIruRSb0Z/4G5jTBywgoJDM4uw3/kWEpEE6/dECv6I96KCn9vVOdwd7YO41wFTrMtTKDgmfWn8Qes77H2AtCIv9aoNU7CLvgQ4ICJvF7nKbudtjGlh3WPHGFOXgvcYDlAQ8hOtq10550uPxURgq1gPylYHIjJLRNxFxJOC39etIvIH7HS+lxhj6hljGlxaBu4EIqjo57at32i4yTcpRgKHKDhOOdvW9ZTjvP4LnAJyKTje9jAFxxq/Bw4DW4Cm1nUNBWcNxQLhQJCt6y/jnG+l4LhkGLDP+jXSnucN+AK/W+ccAfzNOt4e2A3EACuB2tbxOtbLMdbr29t6Djcx90HABkeYr3V++61fkZeyqqKf29p+QCml7FB1PiyjlFLqGjTclVLKDmm4K6WUHdJwV0opO6ThrpRSdkjDXSml7JCGu1JK2aH/D1xG9WKoUf4MAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "plt.plot(score_arr2, label ='accumulated scores in one game play 2')\n",
        "plt.legend(loc='upper left')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aFotlh87SfLB"
      },
      "outputs": [],
      "source": [
        "## game play video 3\n",
        "agent.load_state_dict(torch.load(\"weights.pt\"))\n",
        "env = gnwrapper.LoopAnimation(gym.make(game_name)) \n",
        "state = env.reset()\n",
        "state_dim = len(state)\n",
        "game_score = 0\n",
        "score_arr2 = []\n",
        "state_arr = []\n",
        "for i in range(1000):\n",
        "    if i == 0:\n",
        "        stacked_state = np.concatenate((state, state, state, state, state, state, state, state), axis=0)\n",
        "    else:\n",
        "        stacked_state = np.roll(stacked_state,-state_dim,axis=0)        \n",
        "        stacked_state[-state_dim:]=state\n",
        "    with torch.no_grad():\n",
        "        hs = agent.representation_network(torch.from_numpy(stacked_state).float().to(device))\n",
        "        P, v = agent.prediction_network(hs)\n",
        "        action = np.random.choice(np.arange(agent.action_space), p=P.detach().numpy())   \n",
        "    if i %5==0:\n",
        "        env.render()\n",
        "    state, r, done, info = env.step(action.item())\n",
        "    print(r, done, info)\n",
        "    state_arr.append(state[0])\n",
        "    game_score += r \n",
        "    score_arr2.append(game_score)\n",
        "    if i %10==0: \n",
        "        print(game_score)\n",
        "    if done:\n",
        "        print('last frame number : ',i)\n",
        "        print('score :', game_score)\n",
        "        print(state_arr)\n",
        "       \n",
        "        break\n",
        "env.reset()\n",
        "env.display()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WlLErFQhShIo",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "outputId": "98d9b346-2709-43b4-a6bd-d99af7b03279"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f86ae6cbca0>"
            ]
          },
          "metadata": {},
          "execution_count": 12
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deVyU9fr/8dcHxQX3XQIRd2VHcU8zzdxzz+occ6msTts5nW8uqSc7tlh26th+7Fjaco4l4JJZmaV2Ss2llB0VRQVREBFFdub6/cHID5UUERiYuZ6Pxzy45zP3PXPdw/Dmnvu+5xojIiillLIvTrYuQCmlVPnTcFdKKTuk4a6UUnZIw10ppeyQhrtSStmhmrYuAKB58+bi6elp6zKUUqpa2bdv3xkRaVHSbVUi3D09Pdm7d6+ty1BKqWrFGHPs927T3TJKKWWHNNyVUsoOabgrpZQdqhL73EuSl5dHQkIC2dnZti5FKfU76tSpg7u7O87OzrYuRV2hyoZ7QkICDRo0wNPTE2OMrctRSl1BREhNTSUhIYF27drZuhx1hVLtljHGxBtjwo0x+40xe61jTY0x3xljDll/NrGOG2PMm8aYw8aYMGNM97IUlp2dTbNmzTTYlaqijDE0a9ZM311XUTeyz/12EQkQkSDr9bnA9yLSCfjeeh1gBNDJepkFvFfW4jTYlara9G+06rqZA6pjgVXW6VXAuGLjH0uhXUBjY4zrTTyOUkrZnazcAl7+OpqEtMwKuf/ShrsAm40x+4wxs6xjrUQkyTp9CmhlnXYDThRbNsE6dhljzCxjzF5jzN6UlJQylO7YFi1axGuvvXbNedatW0dUVNQN33f9+vXLWlalGTlyJOfOnbN1GVXStm3bGD169E3fz+7duwkICCAgIAB/f3/Wrl1bDtUpgB1xZxj2zx/51/YjbI2tmPwr7QHVW0Uk0RjTEvjOGBNT/EYREWPMDX3rh4gsB5YDBAUF6TeGVIB169YxevRovLy8bF3KVfLz86lZs+zH8zdt2lSO1aiS+Pj4sHfvXmrWrElSUhL+/v6MGTPmpn5vju58dh4vb4rmv7tP4NnMhdWz+tCnfbMKeaxSbbmLSKL1ZzKwFugFnL60u8X6M9k6eyLQptji7taxamfcuHH06NEDb29vli9fXjT+zTff0L17d/z9/RkyZAgAGRkZzJgxA19fX/z8/AgJCQEu3woODg5m+vTpAEyfPp1HH32UPn360L59e7Zt28bMmTPp1q1b0TzXWr64Dz74gJ49e+Lv78/EiRPJzMxkx44dbNiwgWeeeYaAgADi4uKIi4tj+PDh9OjRgwEDBhATU/g/+ujRo/Tt2xdfX18WLFhQ4nNx8eJFRo0ahb+/Pz4+Pnz++ecA7Nmzh379+uHv70+vXr24cOEC2dnZRc9FYGAgW7duBWDlypXcddddDB48mCFDhnDx4kVmzpxJr169CAwMZP369QBERkbSq1cvAgIC8PPz49ChQ1fV4+npyZkzZ4iPj6dbt2489NBDeHt7c+edd5KVlXXV/PHx8QwePBg/Pz+GDBnC8ePHi34PTz75JP369aN9+/YEBwcXLbN06VJ69uyJn58fzz33XInPy4oVK+jcuTO9evXioYce4vHHHwfgyy+/pHfv3gQGBnLHHXdw+vRpoPAd17Rp0xgwYABt27YlNDSU2bNn4+vry/Dhw8nLywNg37593HbbbfTo0YNhw4aRlJR01WNPnz6dRx55hKCgIDp37szGjRuvmmf37t307duXwMBA+vXrR2xsLAADBw5k//79RfPdeuutHDhw4LJlXVxcioI8Oztb96/fpO+iTjP09e18vucED9/Wnm/+PLDCgh0oPJ3pWhegHtCg2PQOYDiwFJhrHZ8LvGqdHgV8DRigD7D7eo/Ro0cPuVJUVFTR9KINEXL3+zvK9bJoQ8RVj3ml1NRUERHJzMwUb29vOXPmjCQnJ4u7u7scOXLksnlmz54tTz31VNGyZ8+eFRGRevXqFY2tWbNGpk2bJiIi06ZNkylTpojFYpF169ZJgwYNJCwsTAoKCqR79+7y22+/XXP55557TpYuXSoiImfOnCmaZ/78+fLmm28WPcaaNWuKbhs8eLAcPHhQRER27dolt99+u4iIjBkzRlatWiUiIm+//fZlj3lJcHCwPPjgg0XXz507Jzk5OdKuXTvZvXu3iIikp6dLXl6evPbaazJjxgwREYmOjpY2bdpIVlaWfPTRR+Lm5lb0nM2bN08++eQTERFJS0uTTp06SUZGhjz++OPy6aefiohITk6OZGZmXlVP27ZtJSUlRY4ePSo1atQoer4mT55cdJ/FjR49WlauXCkiIitWrJCxY8cWPUeTJk2SgoICiYyMlA4dOoiIyLfffisPPfSQWCwWKSgokFGjRsn27dsvu8/ExERp27atpKamSm5urtx6663y2GOPiUjh799isYiIyAcffCBPP/100e+tf//+kpubK/v375e6devKpk2bRERk3LhxsnbtWsnNzZW+fftKcnKyiIisXr266Pksbtq0aTJs2DApKCiQgwcPipubm2RlZcnWrVtl1KhRl/1ORES+++47mTBhgoiIrFy5suj1GhsbKyX9DYoUvk68vLykXr16EhoaWuI8xf9W1dVSLmTLY5/tk7ZzNsqwN7bLgRNp5XbfwF75nVwtzfurVsBa63/tmsB/ROQbY8we4AtjzAPAMeBu6/ybgJHAYSATmHFz/35s58033yzaz3jixAkOHTpESkoKAwcOLDqvt2nTpgBs2bKF1atXFy3bpEmT697/mDFjMMbg6+tLq1at8PX1BcDb25v4+HgCAgJKVWdERAQLFizg3LlzZGRkMGzYsKvmycjIYMeOHUyePLloLCcnB4Cff/656J3G1KlTmTNnzlXL+/r68te//pU5c+YwevRoBgwYQHh4OK6urvTs2ROAhg0bAvDTTz/xxBNPANC1a1fatm3LwYMHARg6dGjRc7Z582Y2bNhQdOwgOzub48eP07dvX1588UUSEhKYMGECnTp1uub6t2vXrui56tGjB/Hx8VfNs3PnTkJDQ4vWcfbs2UW3jRs3DicnJ7y8vIq2sDdv3szmzZsJDAwsev4OHTrEwIEDi5bbvXs3t912W9H6TJ48uWg9ExISmDJlCklJSeTm5l52HviIESNwdnbG19eXgoIChg8fXvQcx8fHExsbS0REBEOHDgWgoKAAV9eSz0m4++67cXJyolOnTrRv377o3dgl6enpTJs2jUOHDmGMKXpnMHnyZBYvXszSpUv58MMPS3xHCNC7d28iIyOJjo5m2rRpjBgxgjp16pQ4r7qciLB+/0me/zKSizkF/HVoZx6+rQO1alZOY4DrhruIHAH8SxhPBYaUMC7AY+VSndVzY7zL8+5KZdu2bWzZsoWdO3fi4uLCoEGDynQ+b/G3slcuX7t2bQCcnJyKpi9dz8/Pv+7yl0yfPp1169bh7+/PypUr2bZt21XzWCwWGjdufNlb8d+rsySdO3fm119/ZdOmTSxYsIAhQ4Ywfvz4ay5Tknr16hVNiwghISF06dLlsnm6detG7969+eqrrxg5ciT/+te/GDx48O/eZ/HnrkaNGiXulrmW4suL9QvjRYR58+bx8MMP39B9XfLEE0/w9NNPc9ddd7Ft2zYWLVp01eM5OTnh7Oxc9Nxf+r2LCN7e3uzcufO6j3Pl7+3K6wsXLuT2229n7dq1xMfHM2jQIKBwl8vQoUNZv349X3zxBfv27bvm43Tr1o369esTERFBUFDQNedVcPJcFgvWRfBDTDKBHo15daIfnVo1qNQatLfM70hPT6dJkya4uLgQExPDrl27AOjTpw8//vgjR48eBeDs2bNA4RbpO++8U7R8WloaAK1atSI6OhqLxVKmsw1Ks/yFCxdwdXUlLy+Pzz77rGi8QYMGXLhwASjcqm7Xrh1r1qwBCsPr0j7W/v37F73rKL58cSdPnsTFxYU//vGPPPPMM/z666906dKFpKQk9uzZU1RHfn4+AwYMKLqfgwcPcvz48asCHGDYsGG89dZbRYH622+/AXDkyBHat2/Pk08+ydixYwkLC7uxJ60E/fr1u2wdBwwYcM35hw0bxocffkhGRgYAiYmJJCcnXzZPz5492b59O2lpaeTn5xe9+4HC14+bW+FJYqtWreJGdOnShZSUlKJwz8vLIzIyssR516xZg8ViIS4ujiNHjlz1PBevY+XKlZfd9uCDD/Lkk0/Ss2fPEt9pHj16tGgj49ixY8TExKDfu3BtFovw6a5j3PnGj+yMS+Vvo70IfqRfpQc7aLj/ruHDh5Ofn0+3bt2YO3cuffr0AaBFixYsX76cCRMm4O/vz5QpUwBYsGABaWlp+Pj44O/vX3QQccmSJYwePZp+/fr97lvraynN8osXL6Z3797079+frl27Fo3fc889LF26lMDAQOLi4vjss89YsWIF/v7+eHt7Fx3AXLZsGe+88w6+vr4kJpZ87Ds8PLzoIOfzzz/PggULqFWrFp9//jlPPPEE/v7+DB06lOzsbP70pz9hsVjw9fVlypQprFy58rKt40sWLlxIXl4efn5+eHt7s3DhQgC++OILfHx8CAgIICIigvvvv/+Gn7crvfXWW3z00Uf4+fnxySefsGzZsmvOf+edd3LfffcVHWieNGlS0T/KS9zc3Hj22Wfp1asX/fv3x9PTk0aNGgGFB04nT55Mjx49aN68+Q3VWqtWLYKDg5kzZw7+/v4EBASwY8eOEuf18PCgV69ejBgxgvfff/+qXSazZ89m3rx5BAYGFgX1JT169KBhw4bMmFHyntOffvqp6PHHjx/Pu+++e8Pr4kiOnrnIPR/sYsG6CALaNGbzXwYy89Z21HCyzYFoc2mryZaCgoLkyi/riI6Oplu3bjaqSKnSycjIoH79+uTn5zN+/HhmzpxZpt1VZTF9+nRGjx7NpEmTyrT8yZMnGTRoEDExMTg5lX07z9H/VvMLLPz7p6O88d1BatV0YuEoLyYHuVfK2UXGmH3y/7sGXEZPWFXqJixatIgtW7aQnZ3NnXfeybhx466/UBXw8ccfM3/+fF5//fWbCnZHF3XyPHNCwghPTOdOr1YsHudDq4ZV44CzbrkrpW6KI/6t5uQX8PYPh3lvWxyNXZx5/i4fRvq2rvTPAlTbLXcR0Q9OKFWFVYWNw8q271gac0LCOJycwYTubiwc5UWTerVsXdZVqmy416lTh9TUVG37q1QVJdZ+7o5y3ntmbj5Lv41l5Y54bmlUl5UzejKoS0tbl/W7qmy4u7u7k5CQgDYVU6rquvRNTPbup0NnmBsaRkJaFvf3bcvs4V2pX7vKxidQhcPd2dlZv91FKWVT6Zl5vLgpii/2JtC+eT2+eLgvvdo1tXVZpVJlw10ppWzpm4hTLFwfwdmLuTw6qANPDelEHecati6r1DTclVKqmJQLOSzaEMlX4Ul4uTbko+k98XFrZOuybpiGu1JKUXiAOPTXRP6+MYqs3AKeGdaFWQPb41yjen4OQMNdKeXwEs9l8WxoONsPptCjbRNemehHx5ZV/xvJrkXDXSnlsCwW4dNfjvHK1zEI8Pxd3kzt0xYnG/WDKU8a7kophxSXksHckDD2xKcxoFNzXhrvS5umLrYuq9xouCulHEpegYUP/neEf245RF3nGrw22Z+J3d3s7sOSGu5KKYcRkZjOnJAwIk+eZ4RPa54f603LBvb5CVsNd6WU3cvOK+CtHw7x/vYjNHGpxXt/6M4I3xv/foXqRMNdKWXX9safZXZIGEdSLjK5hzvzR3WjsUvVa/RV3jTclVJ2KSMnn6XfxPDxrmPc0qguH8/sxcDOLWxdVqXRcFdK2Z3tB1N4NjSck+lZTOvryTPDulCvijf6Km+OtbZKKbt2LjOXxRujCfk1gQ4t6rHm4b4EeVaPRl/lTcNdKWUXvg5PYuH6SNIyc3n89o48PrhjtWr0Vd403JVS1Vry+Wz+tj6SbyJP4ePWkFUze+J9S/Vr9FXeNNyVUtWSiBC8L4HFG6PIzrcwZ3hXHhrQjprVtNFXedNwV0pVOyfOZvLs2nD+d+gMvTybsmSiL+1bVO9GX+VNw10pVW0UWISPd8az9NtYDLB4rDd/6G0fjb7Km4a7UqpaOJx8gTkh4ew7lsZtnVvw0gRf3BrXtXVZVZaGu1KqSssrsPCv7XG8+f1hXGrX4PW7/RkfaH+NvsqbhrtSqsoKT0hndkgY0UnnGeXnyqIx3rRoUNvWZVULGu5KqSonO6+Af245xAf/O0KzerX419QeDPNubeuyqpVSh7sxpgawF0gUkdHGmHbAaqAZsA+YKiK5xpjawMdADyAVmCIi8eVeuVLKLv1yJJW5oeEcPXORKUFteHZUNxrVdbZ1WdXOjZwQ+hQQXez6K8AbItIRSAMesI4/AKRZx9+wzqeUUtd0ITuPhesimLJ8F/kWC5892JtXJvlpsJdRqcLdGOMOjAL+bb1ugMFAsHWWVcA46/RY63Wstw8xeuRDKXUNW2OTGfbGj3z6yzFm9m/Ht38eSP+OzW1dVrVW2t0y/wRmAw2s15sB50Qk33o9AXCzTrsBJwBEJN8Yk26d/0zxOzTGzAJmAXh4eJS1fqVUNZZ2MZfFG6MI/S2RTi3rE/JoP7p7NLF1WXbhuuFujBkNJIvIPmPMoPJ6YBFZDiwHCAoKkvK6X6VU1ScifBWexHPrI0nPyuPJIZ147PYO1K7puI2+yltpttz7A3cZY0YCdYCGwDKgsTGmpnXr3R1ItM6fCLQBEowxNYFGFB5YVUopTp/PZsG6CL6LOo2feyM+fbA33Vwb2rosu3Pdfe4iMk9E3EXEE7gH+EFE/gBsBSZZZ5sGrLdOb7Bex3r7DyKiW+ZKOTgR4fM9x7nj9e38eDCFZ0d2JfTRfhrsFeRmznOfA6w2xrwA/AassI6vAD4xxhwGzlL4D0Ep5cCOp2YyNzSMHXGp9G7XlFcm+uHZvJ6ty7JrNxTuIrIN2GadPgL0KmGebGByOdSmlKrmCizCyh3xvPZtLDWcDC+O9+Henh7a6KsS6CdUlVIV4uDpC8wODmP/iXMM7tqSF8f74NpIG31VFg13pVS5ys238N62ON7eeoj6tWuy7J4A7vK/RRt9VTINd6VUuTlw4hxzQsKIOXWBu/xv4bkxXjSrr42+bEHDXSl107JyC3hjy0H+/b8jtGxQh3/fH8QdXq1sXZZD03BXSt2UnXGpzAsNIz41k3t7eTBvZFca1tF+MLam4a6UKpPz2Xks+TqG//xynLbNXPjPQ73p10H7wVQVGu5KqRv2ffRp5q+NIPlCNg8NaMfTQ7tQt5a2DqhKNNyVUqWWmpHD819GseHASbq0asD7U3sQ0KaxrctSJdBwV0pdl4iw4cBJnv8yigvZefzljs48OqgDtWreyFdCqMqk4a6Uuqak9CwWrI3g+5hk/Ns05tWJfnRp3eD6Cyqb0nBXSpXIYhFW7znBy5uiybNYWDCqGzP6t6OGtg6oFjTclVJXiT9zkbmhYew6cpa+7ZuxZKIvbZtpo6/qRMNdKVUkv8DCRz/H84/vYnF2cmLJBF+m9GyjrQOqIQ13pRQAMafOMyc4jAMJ6dzRrRUvjPOhdaM6ti5LlZGGu1IOLie/gHe2xvHu1sM0quvMW/cGMtrPVbfWqzkNd6Uc2G/H05gTEsbB0xmMD3Rj4WgvmtarZeuyVDnQcFfKAWXm5vOPzQf58OejtG5Yhw+nBzG4qzb6sica7ko5mB2HzzA3NJzjZzP5Yx8P5gzvSgNt9GV3NNyVchDpWXm8vCma1XtO0K55PVbP6kOf9s1sXZaqIBruSjmAzZGnWLAugjMZOTx8W3v+ckdn6jhroy97puGulB07k5HDog2RbAxLomvrBvx7WhB+7troyxFouCtlh0SEdfsTef7LKDJzCvjr0M48MqgDzjW00Zej0HBXys6cPJfF/LXhbI1NIdCjsNFXp1ba6MvRaLgrZScsFuGz3cd55esYCizC30Z7Ma2fpzb6clAa7krZgSMpGcwNCWd3/Flu7diclyf40qapi63LUjak4a5UNZZfYOHfPx3lje8OUrumE69O8mNyD3dtHaA03JWqrqJOnmd2yAEiEs8zzLsVi8f60LKhNvpShTTclapmcvILePuHw7y3LY7GLs68+4fujPBprVvr6jIa7kpVI/uOnWVOSDiHkzOY0N2NhaO8aKKNvlQJNNyVqgYu5uSz9NtYVu2M55ZGdVk5oyeDurS0dVmqCrtuuBtj6gA/ArWt8weLyHPGmHbAaqAZsA+YKiK5xpjawMdADyAVmCIi8RVUv1J273+HUpgXGk5CWhbT+rblmeFdqV9bt8vUtZXm42o5wGAR8QcCgOHGmD7AK8AbItIRSAMesM7/AJBmHX/DOp9S6galZ+bxzJoDTF2xm1o1nVjzSF+eH+ujwa5K5bqvEhERIMN61dl6EWAwcJ91fBWwCHgPGGudBggG3jbGGOv9KKVK4ZuIUyxcH8HZi7n8aVAHnhzSSRt9qRtSqk0AY0wNCne9dATeAeKAcyKSb50lAXCzTrsBJwBEJN8Yk07hrpszV9znLGAWgIeHx82thVJ2IvlCNos2RLIp/BRerg35aHpPfNwa2bosVQ2VKtxFpAAIMMY0BtYCXW/2gUVkObAcICgoSLfqlUMTEUJ+TWTxxiiy8gp4ZlgXZg1sr42+VJnd0M47ETlnjNkK9AUaG2NqWrfe3YFE62yJQBsgwRhTE2hE4YFVpVQJEtIyeXZtBD8eTCGobROWTPSjY8v6ti5LVXOlOVumBZBnDfa6wFAKD5JuBSZReMbMNGC9dZEN1us7rbf/oPvblbqaxSJ8susYr3wTA8Dzd3kztU9bnLTRlyoHpdlydwVWWfe7OwFfiMhGY0wUsNoY8wLwG7DCOv8K4BNjzGHgLHBPBdStVLUWl5LBnOAw9h5LY2DnFrw03gf3JtroS5Wf0pwtEwYEljB+BOhVwng2MLlcqlPKzuQVWFj+4xGWfX+Ius41eG2yPxO7u2nrAFXu9IRZpSpJRGI6s4PDiEo6z0jf1iy6y5uWDbTRl6oYGu5KVbDsvAKWfX+I5T8eoYlLLd7/Y3eG+7jauixl5zTclapAe+LPMic4jCNnLjK5hzsLRnnRyMXZ1mUpB6DhrlQFyMjJ59VvYvh45zHcm9Tlkwd6MaBTC1uXpRyIhrtS5Wz7wRSeDQ3nZHoW0/t58sywLtTTfjCqkukrTqlyci4zl79vjCL010Q6tKhH8CN96dG2qa3LUg5Kw12pmyQifB1xir+tj+BcZh6P396Rxwd31EZfyqY03JW6Ccnns1m4PoJvI0/j49aQVTN74X2LNvpStqfhrlQZiAhr9iXwwsYocvItzB3RlQdvbUdNbfSlqggNd6Vu0ImzmcwLDeenw2fo5dmUJRN9ad9CG32pqkXDXalSKrAIH++M59VvYnEysHicD3/o5aGNvlSVpOGuVCkcTr7A7OAwfj1+jkFdWvDieF/cGte1dVlK/S4Nd6WuIa/Awvvb4njrh8O41K7BG1P8GRegjb5U1afhrtTvCE9I55ngA8ScusBoP1cW3eVN8/q1bV2WUqWi4a7UFbLzCnhjy0E++PEIzevXZvnUHtzp3drWZSl1QzTclSrmlyOpzA0N5+iZi9zTsw3zRnajUV1t9KWqHw13pYAL2Xm88k0Mn+46Tpumdfnswd7079jc1mUpVWYa7srhbY1J5tm14Zw6n80Dt7bjr3d2xqWW/mmo6k1fwcphnb2Yy9+/jGTd/pN0almfkEf70d2jia3LUqpcaLgrhyMibAxLYtGGSNKz8nhySCceu70DtWtqoy9lPzTclUM5fT6b+Wsj2BJ9Gj/3Rnz2UG+6tm5o67KUKnca7sohiAif7znBi5uiyc23MH9kN2b099RGX8puabgru3c8NZO5oWHsiEuld7umvDLRD8/m9WxdllIVSsNd2a0Ci/DRz0d5bXMsNZ2ceGm8L/f0bKONvpRD0HBXdin21AVmh4Rx4MQ5BndtyYvjfXBtpI2+lOPQcFd2JTffwrvbDvPO1sM0qOPMsnsCuMv/Fm30pRyOhruyGwdOnGN2cBixpy8wNuAW/jbai2ba6Es5KA13Ve1l5Rbw+nexrPjpKC0b1OHf9wdxh1crW5ellE1puKtqbUfcGeaFhnMsNZP7enswd0RXGtbRRl9Kabiraul8dh4vb4rhv7uP07aZC/95qDf9OmijL6Uu0XBX1c6WqNPMXxdOyoUcZg1sz1/u6EzdWto6QKnirhvuxpg2wMdAK0CA5SKyzBjTFPgc8ATigbtFJM0UnpawDBgJZALTReTXiilfOZLUjBye/zKKDQdO0rV1A5ZPDcK/TWNbl6VUlVSaLfd84K8i8qsxpgGwzxjzHTAd+F5Elhhj5gJzgTnACKCT9dIbeM/6U6kyERE2HDjJog2RZOTk85c7OvPooA7UqqmtA5T6PdcNdxFJApKs0xeMMdGAGzAWGGSdbRWwjcJwHwt8LCIC7DLGNDbGuFrvR6kbkpSexYK1EXwfk0xAm8a8OsmPzq0a2Lospaq8G9rnbozxBAKBX4BWxQL7FIW7baAw+E8UWyzBOnZZuBtjZgGzADw8PG6wbGXvLBbhv3uO8/KmGPItFhaM6saM/u2ooa0DlCqVUoe7MaY+EAL8WUTOF//En4iIMUZu5IFFZDmwHCAoKOiGllX27eiZi8wNCeOXo2fp16EZSyb44dHMxdZlKVWtlCrcjTHOFAb7ZyISah0+fWl3izHGFUi2jicCbYot7m4dU+qa8gssfPjzUf6x+SC1ajrxykRf7g5qo60DlCqD0pwtY4AVQLSIvF7spg3ANGCJ9ef6YuOPG2NWU3ggNV33t6vriU46z5yQMMIS0hnq1YoXxvnQqmEdW5elVLVVmi33/sBUINwYs9869iyFof6FMeYB4Bhwt/W2TRSeBnmYwlMhZ5Rrxcqu5OQX8M7WON7dephGdZ15+75ARvm66ta6UjepNGfL/AT83l/akBLmF+Cxm6xLOYBfj6cxJziMQ8kZjA9042+jvWhSr5aty1LKLugnVFWly8zN57VvD/LRjqO0bliHj6b35PauLW1dlgZyu/EAAA5eSURBVFJ2RcNdVaqfD59hbmgYJ85m8cc+HswZ3pUG2uhLqXKn4a4qRXpWHi99Fc3ne0/Qrnk9Pp/Vh97tm9m6LKXsloa7qnCbI0+xYF0EqRdzeeS2Dvz5jk7UcdZGX0pVJA13VWFSLuSw6MtIvgpLoptrQ1ZM64mveyNbl6WUQ9BwV+VORFj7WyJ/3xhFZk4B/3dnZx6+rQPONbTRl1KVRcNdlavEc1nMXxvOttgUunsUNvrq2FIbfSlV2TTcVbmwWITPfjnGkq9jsAg8N8aL+/t6aqMvpWxEw13dtCMpGcwNCWd3/FkGdGrOS+N9adNUG30pZUsa7qrM8gssfPC/o7yx5SB1ajqxdJIfk3q4a+sApaoADXdVJlEnzzM75AARiecZ5t2KxWN9aKmNvpSqMjTc1Q3Jzivg7R8O8/72OBq71OK9P3RnhK+rrctSSl1Bw12V2r5jZ5kdHEZcykUmdndn4ehuNHbRRl9KVUUa7uq6Lubks/TbWFbtjOeWRnVZNbMXt3VuYeuylFLXoOGurunHgynMCw3nZHoW9/dpyzPDu1K/tr5slKrq9K9UlSg9M4/FX0URvC+B9i3q8cXDfenp2dTWZSmlSknDXV3lm4gkFq6P5OzFXP40qANPDtFGX0pVNxruqkjyhWyeWx/J1xGn8HJtyEfTe+Ljpo2+lKqONNwVIkLwvgRe+CqarLwCnhnWhVkD22ujL6WqMQ13B3fibCbPrg3nf4fOENS2CUsm+tGxZX1bl6WUukka7g7KYhE+3hnPq9/GYoC/j/Xmj73b4qSNvpSyCxruDuhwcgZzQ8LYeyyNgZ1b8NJ4H9ybaKMvpeyJhrsDySuwsPzHIyzbcoi6tWrwj8n+TOjupo2+lLJDGu4OIiIxndnBYUQlnWekb2uev8uHFg1q27ospVQF0XC3c9l5BSz7/hDLfzxC03q1eP+P3Rnuo42+lLJ3Gu52bE/8WeYEh3HkzEXuDnJn/kgvGrk427ospVQl0HC3Qxk5+bz6TQwf7zyGe5O6fPpAb27t1NzWZSmlKpGGu53ZGpvM/NBwks5nM6O/J/93ZxfqaaMvpRyO/tXbibSLuSzeGEXob4l0bFmf4Ef60aNtE1uXpZSyEQ33ak5E2BR+iuc2RHAuM48nBnfk8cEdqV1TG30p5cg03Kux5PPZLFgXweao0/i6NeLjmb3xuqWhrctSSlUB1w13Y8yHwGggWUR8rGNNgc8BTyAeuFtE0kzhp2GWASOBTGC6iPxaMaU7LhFhzd4EFn8VRW6+hXkjuvLAre2oqY2+lFJWpUmDlcDwK8bmAt+LSCfge+t1gBFAJ+tlFvBe+ZSpLjlxNpOpK3YzOySMbq4N+fqpATx8WwcNdqXUZa675S4iPxpjPK8YHgsMsk6vArYBc6zjH4uIALuMMY2NMa4iklReBTuqAouwakc8S7+NpYaT4YVxPtzXy0MbfSmlSlTWfe6tigX2KaCVddoNOFFsvgTr2FXhboyZReHWPR4eHmUswzEcOn2B2SFh/Hb8HIO6tOCl8b7c0riurctSSlVhN31AVUTEGCNlWG45sBwgKCjohpd3BLn5Ft7fHsfbPxymXu0a/HNKAGMDbtFGX0qp6ypruJ++tLvFGOMKJFvHE4E2xeZzt46pGxSWcI7ZwWHEnLrAGP9beG6MF83ra6MvpVTplPUo3AZgmnV6GrC+2Pj9plAfIF33t9+Y7LwCXt4Uzbh3fiYtM5cP7g/irXsDNdiVUjekNKdC/pfCg6fNjTEJwHPAEuALY8wDwDHgbuvsmyg8DfIwhadCzqiAmu3WriOpzA0JIz41k3t7tWHuiG40qquNvpRSN640Z8vc+zs3DSlhXgEeu9miHM2F7DyWfB3DZ78cx6OpC/95sDf9OmqjL6VU2eknVG3sh5jTzF8bwenz2Tx4azuevrMzLrX016KUujmaIjZy9mIuf/8yknX7T9KpZX3efbQfgR7a6EspVT403CuZiPBlWBKLNkRyITuPp4Z04k+3d9BGX0qpcqXhXolOpRc2+toSfRp/90a8Mqk3XVtroy+lVPnTcK8EIsLqPSd46ato8iwW5o/sxsxb21FDWwcopSqIhnsFO5Z6kbkh4ew8kkqf9k1ZMsEPz+b1bF2WUsrOabhXkAKL8NHPR3ltcyzOTk68NN6Xe3q20UZfSqlKoeFeAWJPFTb6OnDiHEO6tuSF8T64NtJGX0qpyqPhXo5y8y28u+0w72w9TIM6zrx5byBj/Fy10ZdSqtJpuJeT/SfOMSc4jNjTFxgbcAvPjfGmab1ati5LKeWgNNxvUlZuAf/YHMuHPx+lZYM6rJgWxJBura6/oFJKVSAN95uwI+4Mc0PCOX42k/t6ezB3RFca1tFGX0op29NwL4Pz2Xm8vCma/+4+QdtmLvz3oT707dDM1mUppVQRDfcbtCXqNPPXhZNyIYdZA9vzlzs6U7eWtg5QSlUtGu6llJqRw6Ivo/jywEm6tm7A8qlB+LdpbOuylFKqRBru1yEirN9/kue/jCQjJ5+nh3bmkds6UKtmWb/ESimlKp6G+zWcPJfFgnUR/BCTTECbxrw6yY/OrRrYuiyllLouDfcSWCzCf3YfZ8nXMRRYhIWjvZjez1MbfSmlqg0N9yscPXORuSFh/HL0LP07NuPl8X54NHOxdVlKKXVDNNyt8gssrPjpKK9/d5BaNZ14ZaIvdwe10dYBSqlqScMdiE46z5yQMMIS0hnq1YoXxvnQqmEdW5ellFJl5tDhnpNfwDs/HObdbXE0dnHmnfu6M9K3tW6tK6WqPYcN933H0pgTEsbh5AwmBLqxcLQXTbTRl1LKTjhcuGfm5rP021hW7ojHtWEdPprRk9u7tLR1WUopVa4cKtx/OnSGuaFhJKRlMbVPW2YP70IDbfSllLJDDhHu6Vl5vPhVFF/sTaBd83p8PqsPvdtroy+llP2y+3D/NvIUC9dFkHoxl0cHdeCpIZ2o46yNvpRS9s1uwz3lQg6LNkTyVXgS3VwbsmJaT3zdG9m6LKWUqhR2F+4iQuivifx9YxRZuQU8M6wLswa2x7mGNvpSSjkOuwr3xHNZPBsazvaDKXT3KGz01bGlNvpSSjmeCgl3Y8xwYBlQA/i3iCypiMe5xGIRPv3lGK98HYMAi8Z4MbWvNvpSSjmucg93Y0wN4B1gKJAA7DHGbBCRqPJ+LIC4lAzmhoSxJz6NAZ2a89J4X9o01UZfSinHVhFb7r2AwyJyBMAYsxoYC5R7uH+x5wQL1kdQp6YTSyf5MamHu7YOUEopKibc3YATxa4nAL2vnMkYMwuYBeDh4VGmB2rXoh5Durbk+bHetGygjb6UUuoSmx1QFZHlwHKAoKAgKct99PRsSk/PpuVal1JK2YOKOD8wEWhT7Lq7dUwppVQlqYhw3wN0Msa0M8bUAu4BNlTA4yillPod5b5bRkTyjTGPA99SeCrkhyISWd6Po5RS6vdVyD53EdkEbKqI+1ZKKXV9+pl8pZSyQxruSillhzTclVLKDmm4K6WUHTIiZfr8UPkWYUwKcKyMizcHzpRjOdWBrrNj0HV2DDezzm1FpEVJN1SJcL8Zxpi9IhJk6zoqk66zY9B1dgwVtc66W0YppeyQhrtSStkhewj35bYuwAZ0nR2DrrNjqJB1rvb73JVSSl3NHrbclVJKXUHDXSml7FC1DndjzHBjTKwx5rAxZq6t6ykvxpgPjTHJxpiIYmNNjTHfGWMOWX82sY4bY8yb1ucgzBjT3XaVl50xpo0xZqsxJsoYE2mMeco6brfrbYypY4zZbYw5YF3n563j7Ywxv1jX7XNr62yMMbWt1w9bb/e0Zf1lZYypYYz5zRiz0XrdrtcXwBgTb4wJN8bsN8bstY5V6Gu72oZ7sS/iHgF4AfcaY7xsW1W5WQkMv2JsLvC9iHQCvrdeh8L172S9zALeq6Qay1s+8FcR8QL6AI9Zf5/2vN45wGAR8QcCgOHGmD7AK8AbItIRSAMesM7/AJBmHX/DOl919BQQXey6va/vJbeLSECxc9or9rUtItXyAvQFvi12fR4wz9Z1leP6eQIRxa7HAq7WaVcg1jr9L+DekuarzhdgPTDUUdYbcAF+pfD7hs8ANa3jRa9zCr8joa91uqZ1PmPr2m9wPd2tQTYY2AgYe17fYusdDzS/YqxCX9vVdsudkr+I281GtVSGViKSZJ0+BbSyTtvd82B9+x0I/IKdr7d1F8V+IBn4DogDzolIvnWW4utVtM7W29OBZpVb8U37JzAbsFivN8O+1/cSATYbY/YZY2ZZxyr0tW2zL8hWZSciYoyxy3NYjTH1gRDgzyJy3hhTdJs9rreIFAABxpjGwFqgq41LqjDGmNFAsojsM8YMsnU9lexWEUk0xrQEvjPGxBS/sSJe29V5y93Rvoj7tDHGFcD6M9k6bjfPgzHGmcJg/0xEQq3Ddr/eACJyDthK4W6JxsaYSxtexderaJ2ttzcCUiu51JvRH7jLGBMPrKZw18wy7Hd9i4hIovVnMoX/xHtRwa/t6hzujvZF3BuAadbpaRTuk740fr/1CHsfIL3YW71qwxRuoq8AokXk9WI32e16G2NaWLfYMcbUpfAYQzSFIT/JOtuV63zpuZgE/CDWnbLVgYjMExF3EfGk8O/1BxH5A3a6vpcYY+oZYxpcmgbuBCKo6Ne2rQ803ORBipHAQQr3U863dT3luF7/BZKAPAr3tz1A4b7G74FDwBagqXVeQ+FZQ3FAOBBk6/rLuM63UrhfMgzYb72MtOf1BvyA36zrHAH8zTreHtgNHAbWALWt43Ws1w9bb29v63W4iXUfBGx0hPW1rt8B6yXyUlZV9Gtb2w8opZQdqs67ZZRSSv0ODXellLJDGu5KKWWHNNyVUsoOabgrpZQd0nBXSik7pOGulFJ26P8BXRXtNDcNboQAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "plt.plot(score_arr2, label ='accumulated scores in one game play 3')\n",
        "plt.legend(loc='upper left')"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}