train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training workflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats\n",
    "import os\n",
    "\n",
    "from IPython.display import Video\n",
    "import features.config as fconf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "PREDICTIONS_DIR = \"predictions\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "SAVE_MODEL = False\n",
    "IS_SHORT_TERM = True\n",
    "\n",
    "USE_DEV_SET = False\n",
    "USE_TRAIN_SET = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "RANDOM_SEED = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = \"m_75\" if IS_SHORT_TERM else \"part_2_scores\"\n",
    "feature = \"glove\"\n",
    "model_type = \"gru\"\n",
    "aggregate_with = np.median"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = f\"{feature}_{model_type}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_parameters = {\n",
    "    \"random_seed\": RANDOM_SEED,\n",
    "    \"use_dev_set\": USE_DEV_SET,\n",
    "}\n",
    "\n",
    "if model_type == \"gru\":\n",
    "    model_parameters[\"hidden_dim\"] = 64\n",
    "    model_parameters[\"learning_rate\"] = 1e-3\n",
    "    model_parameters[\"num_epochs\"] = 150\n",
    "    model_parameters[\"batch_size\"] = 64\n",
    "    model_parameters[\"gru_units\"] = 64\n",
    "    model_parameters[\"gru_dropout\"] = 0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "NOTES = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(RANDOM_SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data\n",
    "\n",
    "Extracted features, scores, metadata, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_set_data = pd.read_csv(\"testing_set/test_urls.csv\").set_index(\"video_id\")\n",
    "\n",
    "if USE_TRAIN_SET:\n",
    "    training_set_data = pd.read_csv(\"training_set/scores_v2.csv\").set_index(\"video_id\")\n",
    "\n",
    "if USE_DEV_SET:\n",
    "    development_set_data = pd.read_csv(\"development_set/dev_scores.csv\").set_index(\"video_id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/tony/anaconda3/envs/video-mem/lib/python3.8/site-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "non-resource variables are not supported in the long term\n"
     ]
    }
   ],
   "source": [
    "from features.video import load_C3D_features\n",
    "from features.image import load_ResNet152_features, load_LBP_features, load_HOG_features\n",
    "from features.audio import load_VGGish_features\n",
    "from features.text import load_GloVe_features\n",
    "from features.emotion import load_Emotion_features, extract_emotions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_features_to_df(dfs, set_names, label, feature_dir, load_func):\n",
    "    for df, set_name in zip(dfs, set_names):\n",
    "        df[label] = load_func(df.index, fconf.set_dataset(set_name, feature_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = [testing_set_data]\n",
    "set_names = [\"testing_set\"]\n",
    "\n",
    "if USE_TRAIN_SET:\n",
    "    dfs.append(training_set_data)\n",
    "    set_names.append(\"training_set\")\n",
    "\n",
    "if USE_DEV_SET:\n",
    "    dfs.append(development_set_data)\n",
    "    set_names.append(\"development_set\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Emotion\n",
    "if feature == \"emotion\":\n",
    "    add_features_to_df(dfs, set_names, \"emotion\",\n",
    "                      fconf.EMOTION_FEATURE_DIR, load_Emotion_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LBP\n",
    "if feature == \"lbp\":\n",
    "    add_features_to_df(dfs, set_names, \"lbp\",\n",
    "                      fconf.LBP_FEATURE_DIR, load_LBP_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# HOG\n",
    "if feature == \"hog\":\n",
    "    add_features_to_df(dfs, set_names, \"hog\",\n",
    "                      fconf.HOG_FEATURE_DIR, load_HOG_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ResNet152\n",
    "if feature == \"resnet152\":\n",
    "    add_features_to_df(dfs, set_names, \"resnet152\",\n",
    "                      fconf.RESNET152_FEATURE_DIR, load_ResNet152_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# C3D\n",
    "if feature == \"c3d\":\n",
    "    add_features_to_df(dfs, set_names, \"c3d\",\n",
    "                      fconf.C3D_FEATURE_DIR, load_C3D_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# VGGish\n",
    "if feature == \"vggish\":\n",
    "    add_features_to_df(dfs, set_names, \"vggish\",\n",
    "                      fconf.VGGISH_FEATURE_DIR, load_VGGish_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GLoVE\n",
    "if feature == \"glove\":\n",
    "    add_features_to_df(dfs, set_names, \"glove\",\n",
    "                      fconf.GLOVE_FEATURE_DIR, load_GloVe_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = testing_set_data\n",
    "\n",
    "if USE_DEV_SET and USE_TRAIN_SET:\n",
    "    train_data = pd.concat([training_set_data, development_set_data])\n",
    "elif USE_DEV_SET and not USE_TRAIN_SET:\n",
    "    train_data = development_set_data\n",
    "elif USE_TRAIN_SET and not USE_DEV_SET:\n",
    "    train_data = training_set_data\n",
    "else:\n",
    "    raise ValueError(\"train_data is empty\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Target Prep\n",
    "\n",
    "Khosla points out that the memorability score used in Isola's paper did not take into account the memory retention duration.\n",
    "Cohendet utilized the same idea as Khosla, which involved a decay rate\n",
    "\n",
    "$$\\alpha \\leftarrow \n",
    "\\frac{\\sum^N_{i=1}\\frac{1}{n^{(i)}} \\sum^{n^{(i)}}_{j=1} \\log(\\frac{t^{(i)}_j}{T})[x^{(i)}_j - m^{(i)}_T] }\n",
    "{\\sum^N_{i=1}\\frac{1}{n^{(i)}} \\sum^{n^{(i)}}_{j=1}[ \\log(\\frac{t^{(i)}_j}{T})]^2}\n",
    "$$\n",
    "\n",
    "to calculate memorability \n",
    "$$\n",
    "m_T^{(i)} \\leftarrow\n",
    "\\frac{1}{n^{(i)}} \\sum^{n^{(i)}}_{j=1}[x^{(i)}_j - \\alpha \\log(\\frac{t_j^{(i)}}{T})]\n",
    "$$\n",
    "\n",
    "where we have $n^{(i)}$ observations for image $i$ given by $x^{(i)} \\in {0,1}$ and $t^{(i)}_j$ where $x_j=1$  implies that the image repeat was correctly detected when it shown after time $t_j$\n",
    "\n",
    "IDEA: potentially explore calculating $\\alpha$ per user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from target_augmentation import add_position_delta, calculate_alpha_and_memorability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "if USE_DEV_SET and USE_TRAIN_SET:\n",
    "    annotations = add_position_delta(pd.concat([\n",
    "        pd.read_csv(\"training_set/short_term_annotations_v2.csv\"),\n",
    "        pd.read_csv(\"development_set/dev_short_term_annotations.csv\")\n",
    "    ]))\n",
    "elif USE_TRAIN_SET and not USE_DEV_SET:\n",
    "    annotations = add_position_delta(pd.read_csv(\"training_set/short_term_annotations_v2.csv\"))\n",
    "elif USE_DEV_SET and not USE_TRAIN_SET:\n",
    "    annotations = add_position_delta(pd.read_csv(\"development_set/dev_short_term_annotations.csv\"))\n",
    "else:\n",
    "    raise ValueError(\"annotations is empty\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average t: 74.96191840735993\n",
      "Calculating adjusted value for m_75\n",
      "Alpha: -0.026433626378523905\n",
      "Avg diff: 0.00040057981850080194\n",
      "Max diff: 0.007422429973521383\n",
      "Min diff: -0.006598279844217569\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEGCAYAAACHGfl5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAl6ElEQVR4nO3dfbxVZZn/8c+X4yFBUTTRKQQ1B03N5zOiYWUZ6Uw+4KQZyTRZo+OUY0kymlryK52YcTS1ZjJx1CmNfEhPNFnqaEg5QECAgFqiJnAsxZTBEBMP1++PtQ5uNvthnYd19t5nf9+v13mdve+91t7XWcq69rrvdV+3IgIzM2tug2odgJmZ1Z6TgZmZORmYmZmTgZmZ4WRgZmbANrUOoCd22WWX2HPPPWsdhplZQ1m4cOGLETGi1GsNmQz23HNPFixYUOswzMwaiqRny73mbiIzM3MyMDMzJwMzM8PJwMzMcDIwMzMa9G4iM7Nmc2n7UmbMW0VnBC0SE8eO4vIJB/bZ++d6ZSDpJkkvSFpW5nVJuk7SCkmPSjosz3jMzBrRpe1LuXXuSjrTKtOdEdw6dyWXti/ts8/Iu5voFuD4Cq//JTAm/Tkb+FbO8ZiZNZxb567sVntP5JoMImI28FKFTU4GvhOJucBwSW/LMyYzM9tarQeQRwKrCp6vTtu2IulsSQskLVizZk2/BGdm1ixqnQxUoq3k0msRcUNEtEVE24gRJUtrmJlZD9U6GawGRhU83x14rkaxmJk1rVong5nAJ9K7io4E/i8iflfjmMzM6so2pfpQKrT3RN63ls4A5gD7Slot6dOSzpF0TrrJvcDTwApgOvCZPOMxM2tEK7724a1O/Nsoae8riijZRV/X2trawiWszcy6R9LCiGgr9Vqtu4nMzKwOOBmYmZmTgZmZORmYmRkZqpZK2hY4AXgP8HZgA7AM+HFELM83PDMz6w8Vk4GkqcCJwCxgHvACsC2wDzAtTRRfiIhH8w3TzMzyVO3KYH5ETC3z2tWSdgVG921IZmaNq31RB1fe92ueW7uBtw8fwpTj9mXCoSVLrtWViskgIn5c5fUXSK4WzMyaRrkTfvuiDqbcuYSNm5L5Wx1rNzDlziUAdZ8QKg4gSzqo4HGrpEslzZT0z5KG5h+emVl9aV/UwRfuXELH2g0EyQn/C3cuoX1RB1NnLt+cCLps3BRMnVn/w6vV7ia6peDxNODPgauAIcD1OcVkZla3LrlnKZ1FJ/zOTcEl9yxl7YaNJfcp115Pqo0ZFFbDOBb4i4jYKGk2sCS/sMzM6tP61zu71d4oqiWDHSWdQnIF8ZaI2AgQESGp8YoamZll1JOB4J2GtvLyq1tfBew0tDWvMPtMtWTwMHBS+niupN0i4nlJfwa8mG9oZma10b6og8/fvnjz8461G7Z4Xs5lJx7AlLuWsLHzze/KrS3ishMPyCHKvlXtbqIzy7T/nqTbyMxswCl34q+WELquHAbcraWSTgLui4g/9VM8ZmYNbcKhIxvi5F+sWjfR7cB6ST8BZpAkhsYeJTEzSzXqBLE8VEsGTwAfAE4FvgDcLOkeYEZEPJx3cGZmeenpuECLoLPE7TMtfbgEZS1Um2cQEfFyREyPiGOBg4HHSOoSrco/PDOzfPR0XOCqjx7SrfZG0Z15Bl0Dx9cB10naI7eozMzqVCMPEldSLRmcX+6FiHi2j2MxM2sIjTpIXEnFbqKImJXlTSTN6ZNozMzqwFu2KX1qLNc+EPTVX7ZtH72PmVnN/ctHDmJQ0YDwICXtA1XVlc4ycmkKM6tL46+exZMvrN/8fMyu2/HA5GMY3CJeL3Fb0OAWDdhxgUr6KhmYmdWd4kQA8OQL6xl/9Sz+9dSDOf/2xVt8kxXwr6ceDAzMcYFK+ioZNPgdtmY2EBUngsL2Zvz2X0m3koGkHQr3iYiX0od/05dBmZl1R09nEjfbt/9KMiUDSX8PfAXYwJvjAwG8AyAiluUSnZlZFT2dSWxbynplcAFwQES4bLWZ1ZWeziS2LWW9tfQp4NU8AzEzs9rJemXwReB/Jc0DNpezjojzconKzMz6VdZk8G3gIWApsCm/cMzMSht7xQM8/8rrm5/vNmww8y4ZX3GfcXvvzCNPvVSy3baUNRm8ERGTc43EzKyM4kQA8PwrrzP2igcq7nfbWUdxxvQ5WySEcXvvzG1nHZVLnI0sazL4maSzgR+xZTfR1inXzKyPFSeCau2FfOLPJmsy+Hj6+4sFbZtvLTUz6wteeax2MiWDiNgr70DMrLl5vkBtVby1VNLRVV7fQdK7+jYkM2tGPZ0vcM3ph3Sr3UqrdmXwEUn/CvwUWAisISlX/efA+4E9SNZGLkvS8cC1QAtwY0RMK3p9J+AmYG/gNeBTntFsZlm5xlDfqJgMIuL89GR9KnAa8DaSkhSPA9+OiF9U2l9SC/DvwHhgNTBf0syIeKxgs4uBxRFxiqR3ptsf29M/yMyaj2sM9V7VMYOIeBmYnv501xHAioh4GkDS94GTgcJksD/wtfSznpC0p6TdIuL5HnyemdW5ngwS7zZscMk7h3YbNjivMJtO3mu4jQRWFTxfnbYVWgL8NYCkI0i6nnYvfiNJZ0taIGnBmjVrcgrXzPLUvqiDyXcspmPtBoJkkHjyHYtpX9RRcb95l4zf6sSfZdKZZZf34jal1jkoXlpoGnCtpMUkM5wXAW9stVPEDcANAG1tbV5ZzawBXXz3o2wq+te7KZL2anziz1feyWA1MKrg+e7Ac4UbRMQ64EwASQKeSX/MbIB5dWPpajbl2q3/ZOomkjRU0pckTU+fj5F0QoZd5wNjJO0laTDwMWBm0XsPT18D+DtgdpogzMysn2S9MriZ5NbSrnndq4E7gf+utFNEvCHpXOA+kltLb4qI5ZLOSV+/HtgP+I6kTpKB5U93+68ws7rSk3pAk44cza1zV5Zst/xlTQZ7R8TpkiYCRMSGtEunqoi4F7i3qO36gsdzgDEZ4zCzOlecCAAeeeolzpg+p+J+l084EIAZ81bRGUGLxMSxoza3W76yJoPXJQ0hHfyVtDcFBevMzLqUKhldqb3Q5RMO9Mm/RrImg8tIZiGPknQbMA74ZF5BmZlZ/6qaDCQNAnYimQtwJMntop/zeshmZgNH1buJImITcG5E/CEifhwR/+1EYGY94aJy9StrN9EDki4AbgfWdzV6cRuz5tWTshIuKle/siaDT6W/P1vQ5sVtzJpUb9YecFG5+uTFbcys23q69oDVr0zJQFIr8A/Ae9OmWSQlrDfmFJeZ1QEvJt88snYTfQtoBf4jff43advf5RGUmdVeTyePWWPKmgz+IiIOLnj+kKQleQRkZvWhp5PHXFaiMWVdz6AznXUMgKR3AJ35hGRmjezyCQcy6cjRtKQVa1okJh052jOL61zWK4MpwM8kPU0y6WwP0rLTZmbFXFai8WS9m+hBSWOAfUmSwRMR4dpEZgNAT+YL2MCTdT2DzwJDIuLRiFgCDJX0mXxDM7O8tS/q4Pzbt1yG8vzbqy9D6ZnEA0/WMYOzImJt15OIeBk4K5eIzKzfXHDnkq3WoY20vZIJh47kmtMPYeTwIQgYOXwI15x+iK8oGljWMYNBkhQRXSWsW4DBVfYxszr3RvGCxFXaC3km8cCSNRncB9wh6XqSLw7nkJS0NrM65zEByyJrN9GFwIMks5A/mz7+p7yCMrO+0VVDqHBM4PMZxgTA4wLNJlMyiIhNEXF9RJxKMlYwJyI8z8Cszk0uUyuoXHshjws0l6y1iWYBJ6XbLwbWSHo4IibnF5qZ9dambrYX87hA88jaTbRjRKwjWe3s5og4HPhgfmGZmVl/yjqAvI2ktwEfBS7JMR4z64GxVzzA86+8vvn5bsMGM++S8TWMyBpN1iuDr5DcUbQiIuantYmezC8sM8uqOBEAPP/K64y94oGq+47be+dutdvAlXUA+c6IOCgiPpM+fzoiPpJvaGaWRXEiqNZe6LazjtrqxO81C5pT1m4iMxugfOI3yN5NZGY11r6og3HTHmKvi37MuGkPZZorUG4NAa8tYMWy3lra4nkFZrXTvqiDybcv3nxLaMfaDZnmCnSVkZ4xbxWdEbRITBw7yuWlbStKyw1V3kh6BriL5LbSx3KPqoq2trZYsGBBrcMw6zf7feknbNi49eyAIa2DSrZ3+e20D+cZljUYSQsjoq3Ua1m7iQ4CfgPcKGmupLMl7dBnEZpZReVO+JUSgVl3ZF3c5hVgOjBd0nuBGcDXJd0FfDUiVuQYo1nTcFE5q5XMYwbAh0mWutwTuAq4DXgPcC+wT07xmTWNrqJyXbqKypn1h6y3lj4J/Ay4MiL+t6D9rvRKwcx6qdyJv1pCGLf3zjzy1Esl282yyjpm8ImI+HRhIpA0DiAizsslMjPLxBPHrC9kvTK4DjisqO0bJdrMrAZ84rfeqpgMJB0FvBsYIamwXPUOQEuegZmZWf+p1k00GNieJGkMK/hZB5yab2hm1sWrjlneKl4ZRMTDwMOSbomIZ3vyAZKOB64luZK4MSKmFb2+I3ArMDqN598i4uaefJZZIzjosp+y7k9vTujf4S0tPPr/jq+4T9ftpb7t1PJScQaypGsi4vOSfgRstWFEnFTxzZNbUn8DjAdWA/OBiYWzmCVdTLJ4zoWSRgC/Bv4sIsqWXPQMZGtUxYmgyw5vaSnZ3sUzia0vVJqBXG0A+bvp73/r4WcfQbIGwtNpIN8HTgYKS1oEMEySSLqkXgLe6OHnmdW1cif8SonArD9U6yZamP5+uIfvPxJYVfB8NTC2aJtvAjOB50jGI06PCM+xt6bj+QJWS9XuJlpKie6hLhFxUJX3V6ndip4fBywGPgDsDTwg6efpmsuFsZwNnA0werTL71p9O2P6nC1O7Fnu+7/trKN6tJ9ZX6jWTXRCL99/NTCq4PnuJFcAhc4EpkUyeLEirZD6TuCXhRtFxA3ADZCMGfQyLrPcFJ/QAR556iXOmD6n6r4+8VutVOsm6tEdRAXmA2Mk7QV0AB8DPl60zUrgWODnknYD9gWe7uXnmtVMqa6eSu1m9aDiPANJv0h/vyJpXfHvam8eEW8A5wL3AY8Dd0TEcknnSDon3eyrwLvTLqkHgQsj4sXe/FFm9WrMrtt1q92sv1S7Mjg6/T2spx8QEfeSVDYtbLu+4PFzwId6+v5mtdKT/v0HJh/D+Ktn8eQL6ze3jdl1Ox6YfExeYZplkrU2EZIOA44mGQD+RUQsyi0qszrXm3EBn/itHmWqWirpy8B/AW8FdgFukXRpnoGZ1TOPC9hAk/XKYCJwaES8BiBpGvAr4PK8AjNrVK2DoNRqlK1ZC8ab1UDWZPBbYFvgtfT5W4Cn8gjIrJ5c2r6UGfNW0RlBi8TEsaO4fMKBFfe58rRDSi5Ic+Vph+QTpFkfqDbp7BskYwR/ApZLeiB9Ph74Rf7hmdXOpe1LuXXuys3POyO2eF6Oi8pZI6p2ZdBVDW4hcE9B+6xcojGrI+VO/FkTgk/+1kiq3Vr6X/0ViJmZ1U6mMQNJY4CvAfuTjB0AEBHvyCkus7o2fEgrazdsLNlu1oiyDiDfDFwGfB14P0k9oVJF6MwaTk8GiaeedABT7lzCxk1vlslqHSSmnnRA3uGa5SJrMhgSEQ9KUlqvaKqkn5MkCLOG5UFis0TWZPCapEHAk5LOJSk6t2t+YZn1Dw8SmyWyToP5PDAUOA84HJgE/G1OMZnVBReVs2ZSNRmk6xh/NCL+GBGrI+LMiPhIRMzth/jMauaBycdsdeJ3UTkbqKp2E0VEp6TD0/ECLypjDal9UUeP+vd94rdmkXXMYBHwQ0l3Aptr70bE3blEZdaH2hd1bFEeomPthpLlIsyaWdZksDPwB5J1irsE4GRgde8LdyzuVrtZM8qUDCLizLwDMctLZ5nOzc5IFqQpVXZ63N475xyVWX3Jup7B7pLukfSCpOcl/UDS7nkHZ5a32846aqsTf5YVy8wGmu7MQP4ecFr6fFLaNj6PoMx6oqeDxD7xm2VPBiMi4uaC57dI+nwO8Zj1SPuiDibfsZiu6hAdazcw2WMCZpllnXT2oqRJklrSn0kkA8pmdeHiux9lU9HYwKZI2s2suqzJ4FPAR4HfA78DTk3bzOrCq6XWmUzbrzn9kJKvlWs3a0ZZ7yZaCZyUcyxmVfVkXMBF5cyqy7qewV7APwJ7Fu4TEU4Q1m96My7gonJmlWUdQG4H/hP4EVD6etwsZx4XMMtP5hLWEXFdrpGYVVFpXMDMeidrMrhW0mXA/cCfuhoj4le5RGVNq6dzBXYbNpjnX3m9ZLuZVZc1GRwI/A1JbaKur2HBlrWKzHqlUkG5aglh3iXjGXvFA1skhN2GDWbeJZ4XaZZF1mRwCvCOiNj6q5dZH6lUUC7L1YFP/GY9l3WewRJgeI5xmFUsKGdm+cp6ZbAb8ISk+Ww5ZuBbS63bxl89iydf2LwsRubVw1xh1Cw/WZPBZblGYU2jOBEAPPnCesZfPavqvreddRRnTJ+zRUJwhVGzvpF1BvLDkvYAxkTE/0gaCrTkG5oNRMWJoFp7MZ/4zfKRdT2Ds4C7gG+nTSNJJqKZmdkAkHUA+bPAOGAdQEQ8CeyaV1DWnCYdObpb7WbWd7KOGfwpIl6XBICkbUjmGZj1mcsnHAjAjHmr6IygRWLi2FGb280sP1mTwcOSLgaGSBoPfIakTpFZST2dSXz5hAN98jergazJ4CLg08BS4O+Be4Ebs+wo6XjgWpIB5xsjYlrR61OAMwri2Y9kZbWt7yG0htC+qIPJty/ePFW9Y+0GJhfMLDaz+qOI/Hp7JLUAvyFZK3k1MB+YGBGPldn+ROD8iKhY5qKtrS0WLFjQ1+FaH9nvSz9hQ4nicUNaB/HaG5so9b+cBM987cP9EJ1Z85K0MCLaSr2W9W6iEyQtkvSSpHWSXpG0LsOuRwArIuLptJTF94GTK2w/EZiRJSarX6USQVf7GWNLDwaXazez/pH1bqJrgL8F3hoRO0TEsIjYIcN+I4FVBc9Xp21bSecuHA/8oMzrZ0taIGnBmjVrMoZt9ebyCQcy6cjRtKQ3I7RITDpytMcJzGos65jBKmBZdL9PSSXayr3HicAj5cYKIuIG4AZIuom6GYfl4NL2pT2688eDxGb1J2sy+CfgXkkPs2Vtoqur7LcaGFXwfHfguTLbfgx3ETWMS9uXcuvclZufd0Zs8dzMGkvWZHAF8EdgW6A7q4XMB8akayh3kJzwP168kaQdgfcBk7rx3tYPyt0ieluZE3+5djOrb1mTwc4R8aHuvnlEvCHpXOA+kltLb4qI5ZLOSV+/Pt30FOD+iMhWoMb6RaVbRMv107n/zqwxZU0G/yPpQxFxf3c/ICLuJZmXUNh2fdHzW4Bbuvvelq8v3v0oxfcFbUrbK3GpabPG053aRD+VtKGbt5ZaA6t0i2glt5111FYnfpeaNqtvWUtYD8s7EGss2w1uYf3rnSXbwaWmzRpN1isDsy1cccqBtAza8s7hlkHiilN8y6hZI8o6ZmC2ha6icz0pRmdm9cfJwHpswqEjffI3GyAqJgNJFW//cGXRgW34kFbWbthYst3MBpZqVwYLSW4dFzAaeDl9PBxYCeyVZ3DWP8qVlZh60gFMuXMJGze9OXugdZCYetIBNYzWzPJQMRlExF4Akq4HZqZzBpD0l8AH8w/P8laprERX/SCPC5gNfJnWM0hrYB9e1LagXF3svHk9g76z10U/LjlrWMAz07y+gNlAUmk9g6wDyC9KuhS4laTbaBLwhz6Kz2rIZSXMDLLPM5gIjADuSX9GpG1mZjYAZJ2B/BLwOUnbR8Qfc47JzMz6WaZkIOndwI3A9sBoSQcDfx8Rn8kzOOueM6bP2aJAXJZ6QJOOHF1yHYJJR3oZSrNmkrWb6OvAcaTjBBGxBHhvXkFZ9xUnAoBHnnqJM6bPqbifl6E0M+jGDOSIWCVtUYtm6yplVjOlSkZXai/kZSjNLPMayGlXUUgaDJwHPJ5fWGZm1p+ydhOdQ7KmwUiSdY0PATxeYGY2QGS9Mtg3Is4obJA0Dnik70OySsqVjjAz642syeAbwGEZ2ixHlUpHmJn1RrWqpUcB7wZGSJpc8NIOJAvcWz+6bV7pE3+5djOzrKqNGQwmmVuwDTCs4GcdcGq+oVmxcmWkImDk8CElXyvXbmZWqFrV0oeBhyXdEhHPAkgaBGwfEev6I8Bm1L6oo9uVQqccty9fvHspGza+ecfvkNYWphy3b97hmtkAkHXM4GuSziGZW7AQ2FHS1RFxZX6hNaf2RR1brCHQsXYDU+5cUnU/L0NpZr2RNRnsHxHrJJ0B3AtcSJIUnAz62NSZy7dYTAZg46Zg6szlVff1MpRm1lNZ5xm0SmoFJgA/jIiNuMpxLkotM9nV7nEBM8tL1mTwbeC3wHbAbEl7kAwiWz+acty+DGnd8iYujwuYWV/IWsL6OuC6gqZnJb0/n5CsHI8LmFlespaw/nKZl77Sh7E0lXIziXca2srLr27dVbTT0FbA4wJmlo+sA8jrCx5vC5yAC9X1WKWZxJedeABT7lrCxs43h2RaW8RlJx7Q73GaWfPI2k10VeFzSf8GzMwloiZQroTErXNXbq4z5K4gM+tPmdczKDIUeEdfBmJvcleQmfW3rGMGS3nzVtIWYAQeL6jKFUbNrFFkvTI4oeDxG8DzEfFGDvEMGK4wamaNpFrV0h3SGkSvFL20g6QA1kWEl78sodK4wJDWQWzYuGmr14a0Zp32YWbWt6qdfb6X/l4ILEh/d/38Cvi9pH/OL7yB6Wt/fdBWB35Q2m5mVgvVqpaekP7eq9TrklqAZcDFfR/awOXJY2ZWb6p1E1VcySwifgXsV+U9jgeuJRl4vjEippXY5hjgGqAVeDEi3lfpPQcC3zFkZvWk2gBy1/yCbYE2YAkg4CBgHnB0pZ3TK4d/B8YDq4H5kmZGxGMF2wwH/gM4PiJWStq1B39HzZS7Y2i7wS2sf33r4ZTtBnuBODOrPxXHDCLi/RHxfuBZ4LCIaIuIw4FDgRUZ3v8IYEVEPB0RrwPfB04u2ubjwN0RsTL9zBe6+0fUStcdQ53pEmRddwxd2r6UK045kJZB2mL7lkHiilN8a6mZ1Z+st6+8MyKWdj2JiGXAIRn2GwmsKni+Om0rtA+wk6RZkhZK+kSpN5J0tqQFkhasWbMmY9j5mjFvVdn2CYeO5KrTDmbk8CGIpMz0Vacd7K4hM6tLWecZPC7pRuBWkslnk8hWm0gl2orXQdgGOBw4FhgCzJE0NyJ+s8VOETcANwC0tbXVxVoKnWUWJe5q97iAmTWKrMngTOAfgM+lz2eT9PNXsxoYVfB8d+C5Etu8GBHrgfWSZgMHA7+hzrVIJRNCi0rlQDOz+pWpmygiXouIr0fEKRFxCjAfuDrDrvOBMZL2kjQY+BhbF7j7IfAeSdtIGgqMpY4qorYv6mDctIfY66IfM27aQ7Qv6tj82sSxo0ruU67dzKxeZS5UJ+kQYCJwOvAMcHe1fSLiDUnnAveR3Fp6U0Qsl3RO+vr1EfG4pJ8CjwKbSG4/XdbtvyQH7Ys6+OLdS9mwMbkrqGPtBr54dzJ0MuHQkZvrDLn+kJk1OkWZfm8ASfuQfJufCPwBuB24ICL26J/wSmtra4sFCxbk/jnjpj1Ex9oNW7WPHD6ERy76QO6fb2bWlyQtjIi2Uq9VuzJ4Avg5cGJErEjf7Pw+jq9uPVciEVRqNzNrVNXGDD4C/B74maTpko6l9B1CA9Lbhw/pVruZWaOqNunsnog4HXgnMAs4H9hN0rckfagf4usX5QaJpxy3L0Nat5wxPKS1hSnH7VuLMM3McpN12cv1wG3AbZJ2Bk4DLgLuzzG2flFtkBhcUM7MBr6KA8j1qicDyO2LOkqe1D1IbGbNojcDyANCpW//HiQ2M8tem6ihXXnfrzcngi4bNnZy5X2/9iCxmRlNkgwqffv3ILGZWZN0E+04pJW1GzaWbPcgsZlZkySDcnXjutpdXdTMml1TdBOtfXXrq4JK7WZmzaYpkoEHic3MKmuKZOBBYjOzyppizMCDxGZmlTVFMgAPEpuZVdIU3URmZlaZk4GZmTkZmJmZk4GZmeFkYGZmNOh6BpLWAM/WOo4iuwAv1jqIOuTjUp6PTXk+NuX15tjsEREjSr3QkMmgHklaUG7RiGbm41Kej015Pjbl5XVs3E1kZmZOBmZm5mTQl26odQB1yselPB+b8nxsysvl2HjMwMzMfGVgZmZOBmZmhpNBt0g6XtKvJa2QdFGZbY6RtFjSckkP93eMtVLt2Eiakh6XxZKWSeqUtHMtYu1vGY7NjpJ+JGlJ+v/NmbWIsxYyHJudJN0j6VFJv5T0rlrE2d8k3STpBUnLyrwuSdelx+1RSYf1+kMjwj8ZfoAW4CngHcBgYAmwf9E2w4HHgNHp811rHXe9HJui7U8EHqp13PVybICLgX9JH48AXgIG1zr2Ojk2VwKXpY/fCTxY67j76di8FzgMWFbm9b8CfgIIOBKY19vP9JVBdkcAKyLi6Yh4Hfg+cHLRNh8H7o6IlQAR8UI/x1grWY5NoYnAjH6JrPayHJsAhkkSsD1JMnijf8OsiSzHZn/gQYCIeALYU9Ju/Rtm/4uI2ST/H5RzMvCdSMwFhkt6W28+08kgu5HAqoLnq9O2QvsAO0maJWmhpE/0W3S1leXYACBpKHA88IN+iKseZDk23wT2A54DlgKfi4hN/RNeTWU5NkuAvwaQdASwB7B7v0RX3zL/m8vKySA7lWgrvi93G+Bw4MPAccCXJO2Td2B1IMux6XIi8EhEVPrWM5BkOTbHAYuBtwOHAN+UtEO+YdWFLMdmGskXrMXAPwKLaI6rpmq6828uk6ZZ9rIPrAZGFTzfneSbXPE2L0bEemC9pNnAwcBv+ifEmslybLp8jObpIoJsx+ZMYFokncErJD1D0j/+y/4JsWaqHpuIWEdyfEi70Z5Jf5pdd/7NZeIrg+zmA2Mk7SVpMMlJbWbRNj8E3iNpm7Q7ZCzweD/HWQtZjg2SdgTeR3KcmkWWY7MSOBYg7Q/fF3i6X6OsjarHRtLw9DWAvwNmpwmi2c0EPpHeVXQk8H8R8bvevKGvDDKKiDcknQvcR3IXxE0RsVzSOenr10fE45J+CjwKbAJujIiSt4YNJFmOTbrpKcD96ZVTU8h4bL4K3CJpKcnl/4URMeDLN2c8NvsB35HUSXKn3qdrFnA/kjQDOAbYRdJq4DKgFTYfl3tJ7ihaAbxKevXUq89Mb1MyM7Mm5m4iMzNzMjAzMycDMzPDycDMzHAyMDMznAyslyTtLumHkp6U9JSkawvuCy/e9u2S7srwnvdKGt7DeKZKuqAn+9YDSX8s0/4VSR9MH8+S1JY+vje9F3+4pM/0Z6w2sDgZWI+lM0LvBtojYgxJbabtgStKbLtNRDwXEadWe9+I+KuIWNvX8dYDSS092S8ivhwR/1OivetYDQdqmgwked5SA3MysN74APBaRNwMEBGdwPnApyQNlfRJSXdK+hFwv6Q9u+qzp6/fkdZiv13SvIJvu7+VtEu6/eOSpqd1/u+XNCTd5ixJ85WsAfCDdMZ3WZJukfQtST+T9LSk96U14x+XdEvBdh+SNEfSr9LYty+I6Z/T1xZIOkzSfenV0DnpNpJ0pZL1GpZKOj1tPyb93O+RFKJDUruSYobLJZ1dFOtV6ec/KGlEQfxbJdKuY0VSw2dvJetFXCnpu5JOLtjuNkknFe37Nkmz9eYaE+9J249PP3+JpAfTtp3TmB+VNFfSQWn7VEk3SLqfZHLYiPS/x/z0Z1yl/y5WR2pdt9s/jfsDnAd8vUT7IuAg4JMkNVR2Ttv3JK3PDlwAfDt9/C6S4mNt6fPfAruk278BHJK23wFMSh+/teDzLgf+MX08FbigREy3kJRIFkn533XAgSRfiBaSFIjbBZgNbJfucyHw5YKY/iF9/HWSWebDSNYfeCFt/wjwAMls2t1Iyky8jWQm6Xpgr4J4uo7JEGBZ199DUmzsjPTxl4FvFsR/avp4Vpljtazg/d9HcsUGsCNJPZ9tio7JF4BL0sctBX/Pqq5YC+L8Bm+uK/ABYHHB8V4IDEmffw84On08Gni81v+f+ifbjy/rrDdE6UqJhe0PROkKpUcD1wJExDJJj5b5jGciYnH6eCHJSQ/gXZIuJ+ke2Z6kpEE1P4qIUFL24fmI6PqWvjx9391J6uc/kvSAMRiYU7B/V92cpcD2EfEK8Iqk19IxjqOBGZFcIT2vZKW7vyBJPL+MiMICa+dJOiV9PAoYA/yBpIzJ7Wn7rSTdcN0WEQ9L+ndJu5KUgP5BRBRX+5wP3CSplSRxLJZ0DEn9n2fS9+n6b3c0SbIjIh6S9FYltaYAZkbEhvTxB4H90+MHsIOkYemxsjrmZGC9sZz0BNFFSenlUSQrWB1O8o24lFIleEv5U8HjTpJv0pB8U54QEUskfZLk23fW99pU9L6bSP4tdJIkr4k93L/S37T5OKQn3A8CR0XEq5JmAduW2a839WK+C5xBUgDuU1u9ccRsSe8lKbn+XUlXAmvLfGalksmF/40HkfxdG0psb3XMYwbWGw8CQ5Uu4pMOjl4F3BIRr1bZ9xfAR9P99ifpsumOYcDv0m+1Z3Rz33LmAuMk/Xka11B1bz2K2cDpklrSvv73UroM9Y7Ay2kieCfJsoVdBgFdYwMfJzlOWbxCckwK3QJ8HiAilhfvIGkPki6u6cB/kiyzOAd4n6S90m261qmeTXqc02T2YpSuHno/cG7BZxySMX6rMScD67GICJJKpKdJepJk3YbXSNb0reY/gBFp99CFJH3w/9eNj/8SMI+kj/6J7sRdTkSsIRnnmJHGNZdkXYGs7iH5O5YADwH/FBG/L7HdT4Ft0s/4avo5XdYDB0haSNI3/5WMsf+BpHtrWfoNn4h4nqSE+s1ldjsGWCxpEckV3rXpMTgbuFvSEt7sspoKtKUxTwP+tsx7nte1naTHgHOyxG+156qlVhPpVURrRLwmaW+Sq4x9IlkL1/pAeofVUuCwiOhOorUm5DEDq5WhwM/Sbh6R3KnjRNBHlExQuwm42onAsvCVgZmZeczAzMycDMzMDCcDMzPDycDMzHAyMDMz4P8DcPEnY9R8f24AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Average t:\", np.mean(annotations[\"t\"]))\n",
    "\n",
    "# We use approximately the average_t to calculate T as the memorability in question\n",
    "big_t = int(np.around(np.mean(annotations[\"t\"])))\n",
    "label = f\"m_{big_t}\"\n",
    "print(f\"Calculating adjusted value for {label}\")\n",
    "alpha, adjusted_score = calculate_alpha_and_memorability(annotations, T = big_t)\n",
    "train_data[label] = adjusted_score\n",
    "test_data[label] = np.nan\n",
    "print(f\"Alpha: {alpha}\")\n",
    "target_diff = train_data[\"part_1_scores\"] - train_data[label]\n",
    "print(\"Avg diff:\", np.mean(target_diff))\n",
    "print(\"Max diff:\", np.max(target_diff))\n",
    "print(\"Min diff:\", np.min(target_diff))\n",
    "plt.scatter(train_data[\"part_1_scores\"], train_data[label])\n",
    "plt.xlabel(\"Original memorability score\")\n",
    "plt.ylabel(f\"Adjusted memorability score ({label})\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Prep\n",
    "\n",
    "Building datasets\n",
    "\n",
    "potentially weighting samples based on annotations?\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from train import split_training, build_matrixes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: 472\n",
      "validation: 118\n"
     ]
    }
   ],
   "source": [
    "training_data, validation_data = split_training(train_data)\n",
    "print(\"training:\",len(training_data))\n",
    "print(\"validation:\", len(validation_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Pick Features and Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num videos in training set: 472\n",
      "num videos in validation set: 118\n",
      "num videos in test set: 500\n",
      "features_train shape: (1761, 86, 300)\n",
      "targets_train shape: (1761, 1)\n",
      "features_valid shape: (430, 86, 300)\n",
      "targets_valid shape: (430, 1)\n",
      "Total features: 2191\n",
      "Total videos: 590\n",
      "Avg features per video: 3.7135593220338983\n",
      "Min features per video: 2\n",
      "Max features per video: 5\n"
     ]
    }
   ],
   "source": [
    "features_train, targets_train, video_ids_train = build_matrixes(training_data, target_name = target, feature_name = feature)\n",
    "features_valid, targets_valid, video_ids_valid = build_matrixes(validation_data, target_name = target, feature_name = feature)\n",
    "features_test, targets_test, video_ids_test = build_matrixes(test_data, target_name = target, feature_name = feature, is_test = True)\n",
    "\n",
    "\n",
    "print(\"num videos in training set:\", len(training_data))\n",
    "print(\"num videos in validation set:\", len(validation_data))\n",
    "print(\"num videos in test set:\", len(test_data))\n",
    "print(\"features_train shape:\", features_train.shape)\n",
    "print(\"targets_train shape:\", targets_train.shape)\n",
    "print(\"features_valid shape:\", features_valid.shape)\n",
    "print(\"targets_valid shape:\", targets_valid.shape)\n",
    "total_features = len(features_train) + len(features_valid)\n",
    "unique, count = np.unique(np.concatenate([video_ids_valid, video_ids_train]), return_counts=True)\n",
    "print(\"Total features:\", total_features)\n",
    "print(\"Total videos:\", len(unique))\n",
    "print(\"Avg features per video:\", total_features / len(unique))\n",
    "print(\"Min features per video:\", min(count))\n",
    "print(\"Max features per video:\", max(count))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from train import train_svr, train_gru, train_bayesian_ridge\n",
    "\n",
    "def train_model(model_type, features_train, targets_train, features_valid, targets_valid, model_parameters):\n",
    "    \n",
    "    np.random.seed(model_parameters[\"random_seed\"])\n",
    "\n",
    "    if \"svr\" == model_type:\n",
    "        model = train_svr(features_train, targets_train)\n",
    "        \n",
    "    elif \"gru\" == model_type:\n",
    "        model, train_losses, valid_losses = train_gru(\n",
    "            features_train, targets_train, features_valid, targets_valid,\n",
    "            hidden_dim = model_parameters[\"hidden_dim\"],\n",
    "            num_epochs = model_parameters[\"num_epochs\"],\n",
    "            gru_units = model_parameters[\"gru_units\"],\n",
    "            batch_size = model_parameters[\"batch_size\"],\n",
    "            learning_rate = model_parameters[\"learning_rate\"],\n",
    "            verbose = True,\n",
    "            gru_dropout = model_parameters[\"gru_dropout\"],\n",
    "            \n",
    "        )\n",
    "        plt.plot(train_losses, label=\"Training Loss\")\n",
    "        plt.plot(valid_losses, label=\"Validation Loss\")\n",
    "        plt.legend()\n",
    "        plt.show()\n",
    "        \n",
    "    elif \"bayesian_ridge\" == model_type:\n",
    "        model = train_bayesian_ridge(features_train, targets_train)\n",
    "        \n",
    "    else:\n",
    "        raise ValueError(f\"'{model_type}' is not a valid model type\")\n",
    "    return model\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input dimensions: (86, 300)\n",
      "hidden dimension: 64\n",
      "Train on 1761 samples, validate on 430 samples\n",
      "Epoch 1/150\n",
      "1760/1761 [============================>.] - ETA: 0s - loss: 0.0146 - mean_squared_error: 0.0146 - mean_absolute_error: 0.0926 - mean_absolute_percentage_error: 11.6453WARNING:tensorflow:From /home/tony/anaconda3/envs/video-mem/lib/python3.8/site-packages/tensorflow/python/keras/engine/training_v1.py:2048: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16-11-2020:16:52:30,278 WARNING  [deprecation.py:317] From /home/tony/anaconda3/envs/video-mem/lib/python3.8/site-packages/tensorflow/python/keras/engine/training_v1.py:2048: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0146 - mean_squared_error: 0.0146 - mean_absolute_error: 0.0925 - mean_absolute_percentage_error: 11.6411 - val_loss: 0.0480 - val_mean_squared_error: 0.0480 - val_mean_absolute_error: 0.1988 - val_mean_absolute_percentage_error: 22.6072\n",
      "Epoch 2/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0106 - mean_squared_error: 0.0106 - mean_absolute_error: 0.0818 - mean_absolute_percentage_error: 10.2357 - val_loss: 0.0459 - val_mean_squared_error: 0.0459 - val_mean_absolute_error: 0.1938 - val_mean_absolute_percentage_error: 22.0225\n",
      "Epoch 3/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0105 - mean_squared_error: 0.0105 - mean_absolute_error: 0.0817 - mean_absolute_percentage_error: 10.2292 - val_loss: 0.0469 - val_mean_squared_error: 0.0469 - val_mean_absolute_error: 0.1961 - val_mean_absolute_percentage_error: 22.2950\n",
      "Epoch 4/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0108 - mean_squared_error: 0.0108 - mean_absolute_error: 0.0836 - mean_absolute_percentage_error: 10.4310 - val_loss: 0.0366 - val_mean_squared_error: 0.0366 - val_mean_absolute_error: 0.1715 - val_mean_absolute_percentage_error: 19.5089\n",
      "Epoch 5/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0102 - mean_squared_error: 0.0102 - mean_absolute_error: 0.0803 - mean_absolute_percentage_error: 10.0355 - val_loss: 0.0361 - val_mean_squared_error: 0.0361 - val_mean_absolute_error: 0.1697 - val_mean_absolute_percentage_error: 19.3092\n",
      "Epoch 6/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0101 - mean_squared_error: 0.0101 - mean_absolute_error: 0.0797 - mean_absolute_percentage_error: 9.9710 - val_loss: 0.0319 - val_mean_squared_error: 0.0319 - val_mean_absolute_error: 0.1591 - val_mean_absolute_percentage_error: 18.1032\n",
      "Epoch 7/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0100 - mean_squared_error: 0.0100 - mean_absolute_error: 0.0805 - mean_absolute_percentage_error: 10.0619 - val_loss: 0.0336 - val_mean_squared_error: 0.0336 - val_mean_absolute_error: 0.1638 - val_mean_absolute_percentage_error: 18.6555\n",
      "Epoch 8/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0099 - mean_squared_error: 0.0099 - mean_absolute_error: 0.0788 - mean_absolute_percentage_error: 9.8451 - val_loss: 0.0290 - val_mean_squared_error: 0.0290 - val_mean_absolute_error: 0.1511 - val_mean_absolute_percentage_error: 17.2512\n",
      "Epoch 9/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0096 - mean_squared_error: 0.0096 - mean_absolute_error: 0.0779 - mean_absolute_percentage_error: 9.7510 - val_loss: 0.0297 - val_mean_squared_error: 0.0297 - val_mean_absolute_error: 0.1531 - val_mean_absolute_percentage_error: 17.4657\n",
      "Epoch 10/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0095 - mean_squared_error: 0.0095 - mean_absolute_error: 0.0772 - mean_absolute_percentage_error: 9.6562 - val_loss: 0.0262 - val_mean_squared_error: 0.0262 - val_mean_absolute_error: 0.1431 - val_mean_absolute_percentage_error: 16.3375\n",
      "Epoch 11/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0097 - mean_squared_error: 0.0097 - mean_absolute_error: 0.0786 - mean_absolute_percentage_error: 9.8178 - val_loss: 0.0258 - val_mean_squared_error: 0.0258 - val_mean_absolute_error: 0.1423 - val_mean_absolute_percentage_error: 16.2683\n",
      "Epoch 12/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0093 - mean_squared_error: 0.0093 - mean_absolute_error: 0.0763 - mean_absolute_percentage_error: 9.5432 - val_loss: 0.0254 - val_mean_squared_error: 0.0254 - val_mean_absolute_error: 0.1411 - val_mean_absolute_percentage_error: 16.1792\n",
      "Epoch 13/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0092 - mean_squared_error: 0.0092 - mean_absolute_error: 0.0764 - mean_absolute_percentage_error: 9.5584 - val_loss: 0.0250 - val_mean_squared_error: 0.0250 - val_mean_absolute_error: 0.1396 - val_mean_absolute_percentage_error: 16.0206\n",
      "Epoch 14/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0093 - mean_squared_error: 0.0093 - mean_absolute_error: 0.0764 - mean_absolute_percentage_error: 9.5478 - val_loss: 0.0254 - val_mean_squared_error: 0.0254 - val_mean_absolute_error: 0.1411 - val_mean_absolute_percentage_error: 16.1831\n",
      "Epoch 15/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0092 - mean_squared_error: 0.0092 - mean_absolute_error: 0.0764 - mean_absolute_percentage_error: 9.5427 - val_loss: 0.0217 - val_mean_squared_error: 0.0217 - val_mean_absolute_error: 0.1300 - val_mean_absolute_percentage_error: 14.9703\n",
      "Epoch 16/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0090 - mean_squared_error: 0.0090 - mean_absolute_error: 0.0763 - mean_absolute_percentage_error: 9.5378 - val_loss: 0.0212 - val_mean_squared_error: 0.0212 - val_mean_absolute_error: 0.1286 - val_mean_absolute_percentage_error: 14.8192\n",
      "Epoch 17/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0091 - mean_squared_error: 0.0091 - mean_absolute_error: 0.0761 - mean_absolute_percentage_error: 9.5073 - val_loss: 0.0214 - val_mean_squared_error: 0.0214 - val_mean_absolute_error: 0.1289 - val_mean_absolute_percentage_error: 14.8402\n",
      "Epoch 18/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0093 - mean_squared_error: 0.0093 - mean_absolute_error: 0.0772 - mean_absolute_percentage_error: 9.6238 - val_loss: 0.0207 - val_mean_squared_error: 0.0207 - val_mean_absolute_error: 0.1267 - val_mean_absolute_percentage_error: 14.6216\n",
      "Epoch 19/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0089 - mean_squared_error: 0.0089 - mean_absolute_error: 0.0755 - mean_absolute_percentage_error: 9.4253 - val_loss: 0.0186 - val_mean_squared_error: 0.0186 - val_mean_absolute_error: 0.1196 - val_mean_absolute_percentage_error: 13.8655\n",
      "Epoch 20/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0088 - mean_squared_error: 0.0088 - mean_absolute_error: 0.0756 - mean_absolute_percentage_error: 9.4465 - val_loss: 0.0183 - val_mean_squared_error: 0.0183 - val_mean_absolute_error: 0.1189 - val_mean_absolute_percentage_error: 13.8051\n",
      "Epoch 21/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0086 - mean_squared_error: 0.0086 - mean_absolute_error: 0.0735 - mean_absolute_percentage_error: 9.1994 - val_loss: 0.0184 - val_mean_squared_error: 0.0184 - val_mean_absolute_error: 0.1193 - val_mean_absolute_percentage_error: 13.8509\n",
      "Epoch 22/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0086 - mean_squared_error: 0.0086 - mean_absolute_error: 0.0742 - mean_absolute_percentage_error: 9.2723 - val_loss: 0.0169 - val_mean_squared_error: 0.0169 - val_mean_absolute_error: 0.1143 - val_mean_absolute_percentage_error: 13.3343\n",
      "Epoch 23/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0089 - mean_squared_error: 0.0089 - mean_absolute_error: 0.0759 - mean_absolute_percentage_error: 9.4812 - val_loss: 0.0162 - val_mean_squared_error: 0.0162 - val_mean_absolute_error: 0.1114 - val_mean_absolute_percentage_error: 13.0237\n",
      "Epoch 24/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0086 - mean_squared_error: 0.0086 - mean_absolute_error: 0.0742 - mean_absolute_percentage_error: 9.2739 - val_loss: 0.0165 - val_mean_squared_error: 0.0165 - val_mean_absolute_error: 0.1129 - val_mean_absolute_percentage_error: 13.1897\n",
      "Epoch 25/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0086 - mean_squared_error: 0.0086 - mean_absolute_error: 0.0741 - mean_absolute_percentage_error: 9.2514 - val_loss: 0.0148 - val_mean_squared_error: 0.0148 - val_mean_absolute_error: 0.1060 - val_mean_absolute_percentage_error: 12.4851\n",
      "Epoch 26/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0086 - mean_squared_error: 0.0086 - mean_absolute_error: 0.0741 - mean_absolute_percentage_error: 9.2771 - val_loss: 0.0156 - val_mean_squared_error: 0.0156 - val_mean_absolute_error: 0.1096 - val_mean_absolute_percentage_error: 12.8401\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 27/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0086 - mean_squared_error: 0.0086 - mean_absolute_error: 0.0741 - mean_absolute_percentage_error: 9.2663 - val_loss: 0.0145 - val_mean_squared_error: 0.0145 - val_mean_absolute_error: 0.1055 - val_mean_absolute_percentage_error: 12.4334\n",
      "Epoch 28/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0087 - mean_squared_error: 0.0087 - mean_absolute_error: 0.0746 - mean_absolute_percentage_error: 9.3116 - val_loss: 0.0127 - val_mean_squared_error: 0.0127 - val_mean_absolute_error: 0.0981 - val_mean_absolute_percentage_error: 11.7101\n",
      "Epoch 29/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0083 - mean_squared_error: 0.0083 - mean_absolute_error: 0.0730 - mean_absolute_percentage_error: 9.1099 - val_loss: 0.0124 - val_mean_squared_error: 0.0124 - val_mean_absolute_error: 0.0968 - val_mean_absolute_percentage_error: 11.5772\n",
      "Epoch 30/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0081 - mean_squared_error: 0.0081 - mean_absolute_error: 0.0721 - mean_absolute_percentage_error: 9.0248 - val_loss: 0.0123 - val_mean_squared_error: 0.0123 - val_mean_absolute_error: 0.0962 - val_mean_absolute_percentage_error: 11.5315\n",
      "Epoch 31/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0083 - mean_squared_error: 0.0083 - mean_absolute_error: 0.0729 - mean_absolute_percentage_error: 9.1160 - val_loss: 0.0114 - val_mean_squared_error: 0.0114 - val_mean_absolute_error: 0.0917 - val_mean_absolute_percentage_error: 11.1351\n",
      "Epoch 32/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0081 - mean_squared_error: 0.0081 - mean_absolute_error: 0.0725 - mean_absolute_percentage_error: 9.0562 - val_loss: 0.0123 - val_mean_squared_error: 0.0123 - val_mean_absolute_error: 0.0964 - val_mean_absolute_percentage_error: 11.5491\n",
      "Epoch 33/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0082 - mean_squared_error: 0.0082 - mean_absolute_error: 0.0721 - mean_absolute_percentage_error: 9.0328 - val_loss: 0.0116 - val_mean_squared_error: 0.0116 - val_mean_absolute_error: 0.0929 - val_mean_absolute_percentage_error: 11.2286\n",
      "Epoch 34/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0082 - mean_squared_error: 0.0082 - mean_absolute_error: 0.0723 - mean_absolute_percentage_error: 9.0379 - val_loss: 0.0117 - val_mean_squared_error: 0.0117 - val_mean_absolute_error: 0.0936 - val_mean_absolute_percentage_error: 11.2959\n",
      "Epoch 35/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0082 - mean_squared_error: 0.0082 - mean_absolute_error: 0.0725 - mean_absolute_percentage_error: 9.0644 - val_loss: 0.0112 - val_mean_squared_error: 0.0112 - val_mean_absolute_error: 0.0912 - val_mean_absolute_percentage_error: 11.0801\n",
      "Epoch 36/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0081 - mean_squared_error: 0.0081 - mean_absolute_error: 0.0718 - mean_absolute_percentage_error: 8.9969 - val_loss: 0.0113 - val_mean_squared_error: 0.0113 - val_mean_absolute_error: 0.0919 - val_mean_absolute_percentage_error: 11.1370\n",
      "Epoch 37/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0082 - mean_squared_error: 0.0082 - mean_absolute_error: 0.0728 - mean_absolute_percentage_error: 9.0845 - val_loss: 0.0110 - val_mean_squared_error: 0.0110 - val_mean_absolute_error: 0.0903 - val_mean_absolute_percentage_error: 10.9988\n",
      "Epoch 38/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0082 - mean_squared_error: 0.0082 - mean_absolute_error: 0.0727 - mean_absolute_percentage_error: 9.0871 - val_loss: 0.0109 - val_mean_squared_error: 0.0109 - val_mean_absolute_error: 0.0898 - val_mean_absolute_percentage_error: 10.9528\n",
      "Epoch 39/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0080 - mean_squared_error: 0.0080 - mean_absolute_error: 0.0717 - mean_absolute_percentage_error: 8.9686 - val_loss: 0.0111 - val_mean_squared_error: 0.0111 - val_mean_absolute_error: 0.0907 - val_mean_absolute_percentage_error: 11.0336\n",
      "Epoch 40/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0081 - mean_squared_error: 0.0081 - mean_absolute_error: 0.0719 - mean_absolute_percentage_error: 8.9899 - val_loss: 0.0111 - val_mean_squared_error: 0.0111 - val_mean_absolute_error: 0.0906 - val_mean_absolute_percentage_error: 11.0387\n",
      "Epoch 41/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0079 - mean_squared_error: 0.0079 - mean_absolute_error: 0.0714 - mean_absolute_percentage_error: 8.9293 - val_loss: 0.0112 - val_mean_squared_error: 0.0112 - val_mean_absolute_error: 0.0910 - val_mean_absolute_percentage_error: 11.0669\n",
      "Epoch 42/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0081 - mean_squared_error: 0.0081 - mean_absolute_error: 0.0713 - mean_absolute_percentage_error: 8.9373 - val_loss: 0.0111 - val_mean_squared_error: 0.0111 - val_mean_absolute_error: 0.0909 - val_mean_absolute_percentage_error: 11.0564\n",
      "Epoch 43/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0081 - mean_squared_error: 0.0081 - mean_absolute_error: 0.0718 - mean_absolute_percentage_error: 8.9821 - val_loss: 0.0111 - val_mean_squared_error: 0.0111 - val_mean_absolute_error: 0.0903 - val_mean_absolute_percentage_error: 11.0251\n",
      "Epoch 44/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0081 - mean_squared_error: 0.0081 - mean_absolute_error: 0.0722 - mean_absolute_percentage_error: 9.0154 - val_loss: 0.0110 - val_mean_squared_error: 0.0110 - val_mean_absolute_error: 0.0901 - val_mean_absolute_percentage_error: 10.9822\n",
      "Epoch 45/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0080 - mean_squared_error: 0.0080 - mean_absolute_error: 0.0714 - mean_absolute_percentage_error: 8.9345 - val_loss: 0.0111 - val_mean_squared_error: 0.0111 - val_mean_absolute_error: 0.0907 - val_mean_absolute_percentage_error: 11.0446\n",
      "Epoch 46/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0080 - mean_squared_error: 0.0080 - mean_absolute_error: 0.0718 - mean_absolute_percentage_error: 8.9779 - val_loss: 0.0112 - val_mean_squared_error: 0.0112 - val_mean_absolute_error: 0.0907 - val_mean_absolute_percentage_error: 11.0460\n",
      "Epoch 47/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0081 - mean_squared_error: 0.0081 - mean_absolute_error: 0.0721 - mean_absolute_percentage_error: 9.0187 - val_loss: 0.0112 - val_mean_squared_error: 0.0112 - val_mean_absolute_error: 0.0910 - val_mean_absolute_percentage_error: 11.0768\n",
      "Epoch 48/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0081 - mean_squared_error: 0.0081 - mean_absolute_error: 0.0720 - mean_absolute_percentage_error: 9.0116 - val_loss: 0.0110 - val_mean_squared_error: 0.0110 - val_mean_absolute_error: 0.0898 - val_mean_absolute_percentage_error: 10.9705\n",
      "Epoch 49/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0079 - mean_squared_error: 0.0079 - mean_absolute_error: 0.0712 - mean_absolute_percentage_error: 8.9074 - val_loss: 0.0115 - val_mean_squared_error: 0.0115 - val_mean_absolute_error: 0.0923 - val_mean_absolute_percentage_error: 11.1951\n",
      "Epoch 50/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0079 - mean_squared_error: 0.0079 - mean_absolute_error: 0.0712 - mean_absolute_percentage_error: 8.9263 - val_loss: 0.0110 - val_mean_squared_error: 0.0110 - val_mean_absolute_error: 0.0899 - val_mean_absolute_percentage_error: 10.9620\n",
      "Epoch 51/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0078 - mean_squared_error: 0.0078 - mean_absolute_error: 0.0704 - mean_absolute_percentage_error: 8.8034 - val_loss: 0.0114 - val_mean_squared_error: 0.0114 - val_mean_absolute_error: 0.0920 - val_mean_absolute_percentage_error: 11.1578\n",
      "Epoch 52/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0080 - mean_squared_error: 0.0080 - mean_absolute_error: 0.0717 - mean_absolute_percentage_error: 8.9665 - val_loss: 0.0113 - val_mean_squared_error: 0.0113 - val_mean_absolute_error: 0.0915 - val_mean_absolute_percentage_error: 11.1230\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 53/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0080 - mean_squared_error: 0.0080 - mean_absolute_error: 0.0715 - mean_absolute_percentage_error: 8.9445 - val_loss: 0.0110 - val_mean_squared_error: 0.0110 - val_mean_absolute_error: 0.0899 - val_mean_absolute_percentage_error: 10.9719\n",
      "Epoch 54/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0080 - mean_squared_error: 0.0080 - mean_absolute_error: 0.0715 - mean_absolute_percentage_error: 8.9497 - val_loss: 0.0109 - val_mean_squared_error: 0.0109 - val_mean_absolute_error: 0.0895 - val_mean_absolute_percentage_error: 10.9359\n",
      "Epoch 55/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0078 - mean_squared_error: 0.0078 - mean_absolute_error: 0.0707 - mean_absolute_percentage_error: 8.8476 - val_loss: 0.0109 - val_mean_squared_error: 0.0109 - val_mean_absolute_error: 0.0897 - val_mean_absolute_percentage_error: 10.9377\n",
      "Epoch 56/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0079 - mean_squared_error: 0.0079 - mean_absolute_error: 0.0710 - mean_absolute_percentage_error: 8.8857 - val_loss: 0.0110 - val_mean_squared_error: 0.0110 - val_mean_absolute_error: 0.0903 - val_mean_absolute_percentage_error: 10.9923\n",
      "Epoch 57/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0079 - mean_squared_error: 0.0079 - mean_absolute_error: 0.0713 - mean_absolute_percentage_error: 8.9154 - val_loss: 0.0111 - val_mean_squared_error: 0.0111 - val_mean_absolute_error: 0.0904 - val_mean_absolute_percentage_error: 11.0157\n",
      "Epoch 58/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0079 - mean_squared_error: 0.0079 - mean_absolute_error: 0.0717 - mean_absolute_percentage_error: 8.9539 - val_loss: 0.0113 - val_mean_squared_error: 0.0113 - val_mean_absolute_error: 0.0912 - val_mean_absolute_percentage_error: 11.0866\n",
      "Epoch 59/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0078 - mean_squared_error: 0.0078 - mean_absolute_error: 0.0703 - mean_absolute_percentage_error: 8.8175 - val_loss: 0.0109 - val_mean_squared_error: 0.0109 - val_mean_absolute_error: 0.0898 - val_mean_absolute_percentage_error: 10.9591\n",
      "Epoch 60/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0078 - mean_squared_error: 0.0078 - mean_absolute_error: 0.0705 - mean_absolute_percentage_error: 8.8054 - val_loss: 0.0110 - val_mean_squared_error: 0.0110 - val_mean_absolute_error: 0.0901 - val_mean_absolute_percentage_error: 10.9795\n",
      "Epoch 61/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0078 - mean_squared_error: 0.0078 - mean_absolute_error: 0.0710 - mean_absolute_percentage_error: 8.8689 - val_loss: 0.0109 - val_mean_squared_error: 0.0109 - val_mean_absolute_error: 0.0895 - val_mean_absolute_percentage_error: 10.9304\n",
      "Epoch 62/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0077 - mean_squared_error: 0.0077 - mean_absolute_error: 0.0698 - mean_absolute_percentage_error: 8.7348 - val_loss: 0.0108 - val_mean_squared_error: 0.0108 - val_mean_absolute_error: 0.0890 - val_mean_absolute_percentage_error: 10.8904\n",
      "Epoch 63/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0077 - mean_squared_error: 0.0077 - mean_absolute_error: 0.0698 - mean_absolute_percentage_error: 8.7330 - val_loss: 0.0109 - val_mean_squared_error: 0.0109 - val_mean_absolute_error: 0.0896 - val_mean_absolute_percentage_error: 10.9512\n",
      "Epoch 64/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0078 - mean_squared_error: 0.0078 - mean_absolute_error: 0.0703 - mean_absolute_percentage_error: 8.8008 - val_loss: 0.0112 - val_mean_squared_error: 0.0112 - val_mean_absolute_error: 0.0906 - val_mean_absolute_percentage_error: 11.0213\n",
      "Epoch 65/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0076 - mean_squared_error: 0.0076 - mean_absolute_error: 0.0693 - mean_absolute_percentage_error: 8.6885 - val_loss: 0.0114 - val_mean_squared_error: 0.0114 - val_mean_absolute_error: 0.0922 - val_mean_absolute_percentage_error: 11.1661\n",
      "Epoch 66/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0077 - mean_squared_error: 0.0077 - mean_absolute_error: 0.0701 - mean_absolute_percentage_error: 8.7720 - val_loss: 0.0110 - val_mean_squared_error: 0.0110 - val_mean_absolute_error: 0.0893 - val_mean_absolute_percentage_error: 10.9146\n",
      "Epoch 67/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0076 - mean_squared_error: 0.0076 - mean_absolute_error: 0.0694 - mean_absolute_percentage_error: 8.6757 - val_loss: 0.0111 - val_mean_squared_error: 0.0111 - val_mean_absolute_error: 0.0901 - val_mean_absolute_percentage_error: 10.9847\n",
      "Epoch 68/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0076 - mean_squared_error: 0.0076 - mean_absolute_error: 0.0701 - mean_absolute_percentage_error: 8.7530 - val_loss: 0.0112 - val_mean_squared_error: 0.0112 - val_mean_absolute_error: 0.0907 - val_mean_absolute_percentage_error: 11.0280\n",
      "Epoch 69/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0077 - mean_squared_error: 0.0077 - mean_absolute_error: 0.0702 - mean_absolute_percentage_error: 8.7829 - val_loss: 0.0108 - val_mean_squared_error: 0.0108 - val_mean_absolute_error: 0.0890 - val_mean_absolute_percentage_error: 10.8850\n",
      "Epoch 70/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0076 - mean_squared_error: 0.0076 - mean_absolute_error: 0.0696 - mean_absolute_percentage_error: 8.7198 - val_loss: 0.0111 - val_mean_squared_error: 0.0111 - val_mean_absolute_error: 0.0904 - val_mean_absolute_percentage_error: 10.9976\n",
      "Epoch 71/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0077 - mean_squared_error: 0.0077 - mean_absolute_error: 0.0699 - mean_absolute_percentage_error: 8.7539 - val_loss: 0.0109 - val_mean_squared_error: 0.0109 - val_mean_absolute_error: 0.0897 - val_mean_absolute_percentage_error: 10.9390\n",
      "Epoch 72/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0075 - mean_squared_error: 0.0075 - mean_absolute_error: 0.0698 - mean_absolute_percentage_error: 8.7287 - val_loss: 0.0112 - val_mean_squared_error: 0.0112 - val_mean_absolute_error: 0.0908 - val_mean_absolute_percentage_error: 11.0452\n",
      "Epoch 73/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0076 - mean_squared_error: 0.0076 - mean_absolute_error: 0.0702 - mean_absolute_percentage_error: 8.7730 - val_loss: 0.0107 - val_mean_squared_error: 0.0107 - val_mean_absolute_error: 0.0888 - val_mean_absolute_percentage_error: 10.8556\n",
      "Epoch 74/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0076 - mean_squared_error: 0.0076 - mean_absolute_error: 0.0701 - mean_absolute_percentage_error: 8.7562 - val_loss: 0.0112 - val_mean_squared_error: 0.0112 - val_mean_absolute_error: 0.0911 - val_mean_absolute_percentage_error: 11.0473\n",
      "Epoch 75/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0074 - mean_squared_error: 0.0074 - mean_absolute_error: 0.0689 - mean_absolute_percentage_error: 8.6199 - val_loss: 0.0111 - val_mean_squared_error: 0.0111 - val_mean_absolute_error: 0.0899 - val_mean_absolute_percentage_error: 10.9520\n",
      "Epoch 76/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0075 - mean_squared_error: 0.0075 - mean_absolute_error: 0.0694 - mean_absolute_percentage_error: 8.6781 - val_loss: 0.0109 - val_mean_squared_error: 0.0109 - val_mean_absolute_error: 0.0895 - val_mean_absolute_percentage_error: 10.9201\n",
      "Epoch 77/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0075 - mean_squared_error: 0.0075 - mean_absolute_error: 0.0688 - mean_absolute_percentage_error: 8.6013 - val_loss: 0.0116 - val_mean_squared_error: 0.0116 - val_mean_absolute_error: 0.0924 - val_mean_absolute_percentage_error: 11.1850\n",
      "Epoch 78/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0076 - mean_squared_error: 0.0076 - mean_absolute_error: 0.0693 - mean_absolute_percentage_error: 8.6758 - val_loss: 0.0111 - val_mean_squared_error: 0.0111 - val_mean_absolute_error: 0.0901 - val_mean_absolute_percentage_error: 10.9638\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 79/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0074 - mean_squared_error: 0.0074 - mean_absolute_error: 0.0690 - mean_absolute_percentage_error: 8.6567 - val_loss: 0.0111 - val_mean_squared_error: 0.0111 - val_mean_absolute_error: 0.0905 - val_mean_absolute_percentage_error: 11.0212\n",
      "Epoch 80/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0075 - mean_squared_error: 0.0075 - mean_absolute_error: 0.0695 - mean_absolute_percentage_error: 8.6780 - val_loss: 0.0114 - val_mean_squared_error: 0.0114 - val_mean_absolute_error: 0.0920 - val_mean_absolute_percentage_error: 11.1492\n",
      "Epoch 81/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0075 - mean_squared_error: 0.0075 - mean_absolute_error: 0.0696 - mean_absolute_percentage_error: 8.6963 - val_loss: 0.0112 - val_mean_squared_error: 0.0112 - val_mean_absolute_error: 0.0906 - val_mean_absolute_percentage_error: 11.0314\n",
      "Epoch 82/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0075 - mean_squared_error: 0.0075 - mean_absolute_error: 0.0689 - mean_absolute_percentage_error: 8.6226 - val_loss: 0.0110 - val_mean_squared_error: 0.0110 - val_mean_absolute_error: 0.0894 - val_mean_absolute_percentage_error: 10.9224\n",
      "Epoch 83/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0074 - mean_squared_error: 0.0074 - mean_absolute_error: 0.0690 - mean_absolute_percentage_error: 8.6508 - val_loss: 0.0110 - val_mean_squared_error: 0.0110 - val_mean_absolute_error: 0.0902 - val_mean_absolute_percentage_error: 10.9817\n",
      "Epoch 84/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0075 - mean_squared_error: 0.0075 - mean_absolute_error: 0.0696 - mean_absolute_percentage_error: 8.6992 - val_loss: 0.0108 - val_mean_squared_error: 0.0108 - val_mean_absolute_error: 0.0891 - val_mean_absolute_percentage_error: 10.8902\n",
      "Epoch 85/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0074 - mean_squared_error: 0.0074 - mean_absolute_error: 0.0694 - mean_absolute_percentage_error: 8.6611 - val_loss: 0.0113 - val_mean_squared_error: 0.0113 - val_mean_absolute_error: 0.0912 - val_mean_absolute_percentage_error: 11.0815\n",
      "Epoch 86/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0074 - mean_squared_error: 0.0074 - mean_absolute_error: 0.0690 - mean_absolute_percentage_error: 8.6431 - val_loss: 0.0106 - val_mean_squared_error: 0.0106 - val_mean_absolute_error: 0.0879 - val_mean_absolute_percentage_error: 10.7549\n",
      "Epoch 87/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0072 - mean_squared_error: 0.0072 - mean_absolute_error: 0.0676 - mean_absolute_percentage_error: 8.4648 - val_loss: 0.0110 - val_mean_squared_error: 0.0110 - val_mean_absolute_error: 0.0899 - val_mean_absolute_percentage_error: 10.9355\n",
      "Epoch 88/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0075 - mean_squared_error: 0.0075 - mean_absolute_error: 0.0700 - mean_absolute_percentage_error: 8.7539 - val_loss: 0.0112 - val_mean_squared_error: 0.0112 - val_mean_absolute_error: 0.0906 - val_mean_absolute_percentage_error: 10.9959\n",
      "Epoch 89/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0072 - mean_squared_error: 0.0072 - mean_absolute_error: 0.0679 - mean_absolute_percentage_error: 8.4896 - val_loss: 0.0110 - val_mean_squared_error: 0.0110 - val_mean_absolute_error: 0.0894 - val_mean_absolute_percentage_error: 10.9010\n",
      "Epoch 90/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0074 - mean_squared_error: 0.0074 - mean_absolute_error: 0.0685 - mean_absolute_percentage_error: 8.5772 - val_loss: 0.0111 - val_mean_squared_error: 0.0111 - val_mean_absolute_error: 0.0901 - val_mean_absolute_percentage_error: 10.9827\n",
      "Epoch 91/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0074 - mean_squared_error: 0.0074 - mean_absolute_error: 0.0693 - mean_absolute_percentage_error: 8.6710 - val_loss: 0.0114 - val_mean_squared_error: 0.0114 - val_mean_absolute_error: 0.0917 - val_mean_absolute_percentage_error: 11.1154\n",
      "Epoch 92/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0073 - mean_squared_error: 0.0073 - mean_absolute_error: 0.0684 - mean_absolute_percentage_error: 8.5570 - val_loss: 0.0114 - val_mean_squared_error: 0.0114 - val_mean_absolute_error: 0.0915 - val_mean_absolute_percentage_error: 11.1194\n",
      "Epoch 93/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0073 - mean_squared_error: 0.0073 - mean_absolute_error: 0.0674 - mean_absolute_percentage_error: 8.4560 - val_loss: 0.0114 - val_mean_squared_error: 0.0114 - val_mean_absolute_error: 0.0916 - val_mean_absolute_percentage_error: 11.1026\n",
      "Epoch 94/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0071 - mean_squared_error: 0.0071 - mean_absolute_error: 0.0677 - mean_absolute_percentage_error: 8.4633 - val_loss: 0.0111 - val_mean_squared_error: 0.0111 - val_mean_absolute_error: 0.0902 - val_mean_absolute_percentage_error: 10.9920\n",
      "Epoch 95/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0073 - mean_squared_error: 0.0073 - mean_absolute_error: 0.0680 - mean_absolute_percentage_error: 8.5047 - val_loss: 0.0116 - val_mean_squared_error: 0.0116 - val_mean_absolute_error: 0.0916 - val_mean_absolute_percentage_error: 11.1283\n",
      "Epoch 96/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0071 - mean_squared_error: 0.0071 - mean_absolute_error: 0.0675 - mean_absolute_percentage_error: 8.4596 - val_loss: 0.0114 - val_mean_squared_error: 0.0114 - val_mean_absolute_error: 0.0918 - val_mean_absolute_percentage_error: 11.1274\n",
      "Epoch 97/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0072 - mean_squared_error: 0.0072 - mean_absolute_error: 0.0683 - mean_absolute_percentage_error: 8.5294 - val_loss: 0.0110 - val_mean_squared_error: 0.0110 - val_mean_absolute_error: 0.0885 - val_mean_absolute_percentage_error: 10.8367\n",
      "Epoch 98/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0073 - mean_squared_error: 0.0073 - mean_absolute_error: 0.0684 - mean_absolute_percentage_error: 8.5543 - val_loss: 0.0113 - val_mean_squared_error: 0.0113 - val_mean_absolute_error: 0.0906 - val_mean_absolute_percentage_error: 11.0225\n",
      "Epoch 99/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0073 - mean_squared_error: 0.0073 - mean_absolute_error: 0.0685 - mean_absolute_percentage_error: 8.5739 - val_loss: 0.0113 - val_mean_squared_error: 0.0113 - val_mean_absolute_error: 0.0903 - val_mean_absolute_percentage_error: 10.9922\n",
      "Epoch 100/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0071 - mean_squared_error: 0.0071 - mean_absolute_error: 0.0678 - mean_absolute_percentage_error: 8.4858 - val_loss: 0.0112 - val_mean_squared_error: 0.0112 - val_mean_absolute_error: 0.0902 - val_mean_absolute_percentage_error: 10.9665\n",
      "Epoch 101/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0073 - mean_squared_error: 0.0073 - mean_absolute_error: 0.0681 - mean_absolute_percentage_error: 8.5343 - val_loss: 0.0114 - val_mean_squared_error: 0.0114 - val_mean_absolute_error: 0.0913 - val_mean_absolute_percentage_error: 11.0924\n",
      "Epoch 102/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0071 - mean_squared_error: 0.0071 - mean_absolute_error: 0.0675 - mean_absolute_percentage_error: 8.4400 - val_loss: 0.0119 - val_mean_squared_error: 0.0119 - val_mean_absolute_error: 0.0929 - val_mean_absolute_percentage_error: 11.2396\n",
      "Epoch 103/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0071 - mean_squared_error: 0.0071 - mean_absolute_error: 0.0666 - mean_absolute_percentage_error: 8.3270 - val_loss: 0.0114 - val_mean_squared_error: 0.0114 - val_mean_absolute_error: 0.0909 - val_mean_absolute_percentage_error: 11.0651\n",
      "Epoch 104/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0072 - mean_squared_error: 0.0072 - mean_absolute_error: 0.0672 - mean_absolute_percentage_error: 8.4123 - val_loss: 0.0115 - val_mean_squared_error: 0.0115 - val_mean_absolute_error: 0.0920 - val_mean_absolute_percentage_error: 11.1579\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 105/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0070 - mean_squared_error: 0.0070 - mean_absolute_error: 0.0667 - mean_absolute_percentage_error: 8.3467 - val_loss: 0.0117 - val_mean_squared_error: 0.0117 - val_mean_absolute_error: 0.0921 - val_mean_absolute_percentage_error: 11.1690\n",
      "Epoch 106/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0070 - mean_squared_error: 0.0070 - mean_absolute_error: 0.0666 - mean_absolute_percentage_error: 8.3411 - val_loss: 0.0120 - val_mean_squared_error: 0.0120 - val_mean_absolute_error: 0.0938 - val_mean_absolute_percentage_error: 11.3270\n",
      "Epoch 107/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0070 - mean_squared_error: 0.0070 - mean_absolute_error: 0.0666 - mean_absolute_percentage_error: 8.3439 - val_loss: 0.0116 - val_mean_squared_error: 0.0116 - val_mean_absolute_error: 0.0917 - val_mean_absolute_percentage_error: 11.1355\n",
      "Epoch 108/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0070 - mean_squared_error: 0.0070 - mean_absolute_error: 0.0668 - mean_absolute_percentage_error: 8.3600 - val_loss: 0.0114 - val_mean_squared_error: 0.0114 - val_mean_absolute_error: 0.0909 - val_mean_absolute_percentage_error: 11.0613\n",
      "Epoch 109/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0071 - mean_squared_error: 0.0071 - mean_absolute_error: 0.0677 - mean_absolute_percentage_error: 8.4393 - val_loss: 0.0109 - val_mean_squared_error: 0.0109 - val_mean_absolute_error: 0.0888 - val_mean_absolute_percentage_error: 10.8711\n",
      "Epoch 110/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0071 - mean_squared_error: 0.0071 - mean_absolute_error: 0.0674 - mean_absolute_percentage_error: 8.4385 - val_loss: 0.0121 - val_mean_squared_error: 0.0121 - val_mean_absolute_error: 0.0936 - val_mean_absolute_percentage_error: 11.3357\n",
      "Epoch 111/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0069 - mean_squared_error: 0.0069 - mean_absolute_error: 0.0663 - mean_absolute_percentage_error: 8.2770 - val_loss: 0.0114 - val_mean_squared_error: 0.0114 - val_mean_absolute_error: 0.0903 - val_mean_absolute_percentage_error: 11.0179\n",
      "Epoch 112/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0069 - mean_squared_error: 0.0069 - mean_absolute_error: 0.0658 - mean_absolute_percentage_error: 8.2379 - val_loss: 0.0114 - val_mean_squared_error: 0.0114 - val_mean_absolute_error: 0.0906 - val_mean_absolute_percentage_error: 11.0429\n",
      "Epoch 113/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0068 - mean_squared_error: 0.0068 - mean_absolute_error: 0.0662 - mean_absolute_percentage_error: 8.2803 - val_loss: 0.0117 - val_mean_squared_error: 0.0117 - val_mean_absolute_error: 0.0927 - val_mean_absolute_percentage_error: 11.2211\n",
      "Epoch 114/150\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0069 - mean_squared_error: 0.0069 - mean_absolute_error: 0.0665 - mean_absolute_percentage_error: 8.3224 - val_loss: 0.0117 - val_mean_squared_error: 0.0117 - val_mean_absolute_error: 0.0918 - val_mean_absolute_percentage_error: 11.1695\n",
      "Epoch 115/150\n",
      "1761/1761 [==============================] - 4s 2ms/sample - loss: 0.0070 - mean_squared_error: 0.0070 - mean_absolute_error: 0.0672 - mean_absolute_percentage_error: 8.4134 - val_loss: 0.0116 - val_mean_squared_error: 0.0116 - val_mean_absolute_error: 0.0918 - val_mean_absolute_percentage_error: 11.1458\n",
      "Epoch 116/150\n",
      "1760/1761 [============================>.] - ETA: 0s - loss: 0.0068 - mean_squared_error: 0.0068 - mean_absolute_error: 0.0657 - mean_absolute_percentage_error: 8.2291Restoring model weights from the end of the best epoch.\n",
      "1761/1761 [==============================] - 3s 2ms/sample - loss: 0.0068 - mean_squared_error: 0.0068 - mean_absolute_error: 0.0657 - mean_absolute_percentage_error: 8.2262 - val_loss: 0.0118 - val_mean_squared_error: 0.0118 - val_mean_absolute_error: 0.0924 - val_mean_absolute_percentage_error: 11.1978\n",
      "Epoch 00116: early stopping\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD8CAYAAAB6paOMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAyw0lEQVR4nO3deXxU9b3/8ddnZrKvZCMkYYvsOxiR4gZuRaTiWrFaRe/Vaq22etuq3bTX69Xr9ddaWpfrXjeo1mpxrxtipSirCLJDgBAg+77M9v398Z2EEAIMIWGSmc/z8chjMmfOzHy+Wc77fL/nnO+IMQallFKRyxHqApRSSoWWBoFSSkU4DQKllIpwGgRKKRXhNAiUUirCaRAopVSECyoIRGSGiGwUkS0icmcHj4uIzAs8vkZEJrV5rFBEvhaR1SKyvCuLV0opdexcR1pBRJzAI8A5QBGwTEQWGmO+abPaecDQwNfJwGOB2xbTjTFlXVa1UkqpLhNMj2AysMUYs80Y4wYWALPbrTMbeN5YS4FUEenXxbUqpZTqBkfsEQC5wK4294s4cG//UOvkAnsAA/xDRAzwf8aYJzp6ExG5AbgBICEh4cQRI0YE1QCllFKwYsWKMmNMZmeeG0wQSAfL2s9Lcbh1TjHGFItIFvCBiGwwxiw+aGUbEE8AFBQUmOXL9XCCUkoFS0R2dPa5wQwNFQH929zPA4qDXccY03JbAryOHWpSSinVQwQTBMuAoSIyWESigTnAwnbrLASuDpw9NAWoNsbsEZEEEUkCEJEE4FxgbRfWr5RS6hgdcWjIGOMVkR8B7wNO4BljzDoRuTHw+OPAO8BMYAvQAFwbeHpf4HURaXmvl40x73V5K5RSSnWa9MRpqPUYgVLHh8fjoaioiKamplCXooIUGxtLXl4eUVFRBywXkRXGmILOvGYwB4uVUmGqqKiIpKQkBg0aRKDnrnowYwzl5eUUFRUxePDgLntdnWJCqQjW1NREenq6hkAvISKkp6d3eQ9Og0CpCKch0Lt0x+8rfILA0wSf/wG2fhLqSpRSqlcJnyBwRsPn8+Cr+aGuRCkVpPLyciZMmMCECRPIzs4mNze39b7b7T7sc5cvX86tt956xPeYOnVql9S6aNEiZs2a1SWv1dOEz8FihwMGnw7bPgVjQLu7SvV46enprF69GoB77rmHxMREfvrTn7Y+7vV6cbk63kwVFBRQUHDkk2SWLFnSJbWGs/DpEQDkT4O6vVC6MdSVKKU6ae7cudx+++1Mnz6dO+64gy+//JKpU6cyceJEpk6dysaN9v+77R76Pffcw3XXXce0adPIz89n3rx5ra+XmJjYuv60adO49NJLGTFiBFdeeSUtp8+/8847jBgxglNPPZVbb731qPb858+fz9ixYxkzZgx33HEHAD6fj7lz5zJmzBjGjh3L73//ewDmzZvHqFGjGDduHHPmzDn2H1YXCZ8eAUD+GfZ2+6eQpZPWKXU0fvvmOr4prunS1xyVk8zd3xl91M/btGkTH374IU6nk5qaGhYvXozL5eLDDz/kF7/4Ba+99tpBz9mwYQOffPIJtbW1DB8+nJtuuumgc+1XrVrFunXryMnJ4ZRTTuHzzz+noKCAH/zgByxevJjBgwdzxRVXBF1ncXExd9xxBytWrKBPnz6ce+65vPHGG/Tv35/du3ezdq2dSKGqqgqABx54gO3btxMTE9O6rCcIrx5Bn0H2a9ui/cv2fQM7tGuoVG9y2WWX4XQ6Aaiuruayyy5jzJgx3Hbbbaxbt67D55x//vnExMSQkZFBVlYW+/btO2idyZMnk5eXh8PhYMKECRQWFrJhwwby8/Nbz8s/miBYtmwZ06ZNIzMzE5fLxZVXXsnixYvJz89n27Zt3HLLLbz33nskJycDMG7cOK688kpefPHFQw55hULPqaSrDD4D1r0OPi8YP8yfA64Y+NGyUFemVI/WmT337pKQkND6/a9//WumT5/O66+/TmFhIdOmTevwOTExMa3fO51OvF5vUOscy+wKh3punz59+Oqrr3j//fd55JFHeOWVV3jmmWd4++23Wbx4MQsXLuTee+9l3bp1PSIQwqtHAPY4QXMNFK+ClX+Gqh3QVB3qqpRSnVRdXU1ubi4Azz33XJe//ogRI9i2bRuFhYUA/OUvfwn6uSeffDKffvopZWVl+Hw+5s+fzxlnnEFZWRl+v59LLrmEe++9l5UrV+L3+9m1axfTp0/nwQcfpKqqirq6ui5vT2eEPoq62uDT7e3Gd2D1S/b75trQ1aOUOiY///nPueaaa/jd737HmWee2eWvHxcXx6OPPsqMGTPIyMhg8uRDz5T/0UcfkZeX13r/1Vdf5f7772f69OkYY5g5cyazZ8/mq6++4tprr8Xv9wNw//334/P5uOqqq6iursYYw2233UZqamqXt6czwnPSucdPhZL14PfCCWfC1o/hNxXgcHZdkUqFgfXr1zNy5MhQlxFydXV1JCYmYozh5ptvZujQodx2222hLuuQOvq9Hcukc+E3NAR2eMjvhWEzYMg5dpn2CpRSh/Dkk08yYcIERo8eTXV1NT/4wQ9CXdJxFX5DQwAjZ8OqF+Gs38DuFXZZcy3EpYa0LKVUz3Tbbbf16B5AdwvPIOh/EtxRaL8v22RvtUeglFIdCs+hobZikuytu2ccnVdKqZ4mAoLAXshBc9deMamUUuEi/IMg2s4zokNDSinVsfAPgpahIQ0CpXqcadOm8f777x+w7OGHH+aHP/zhYZ/Tcnr5zJkzO5yz55577uGhhx467Hu/8cYbfPPNN633f/Ob3/Dhhx8eRfUd643TVUdQEOgxAqV6miuuuIIFCxYcsGzBggVBz/fzzjvvdPqirPZB8J//+Z+cffbZnXqt3i6CgkB7BEr1NJdeeilvvfUWzc3NABQWFlJcXMypp57KTTfdREFBAaNHj+buu+/u8PmDBg2irKwMgPvuu4/hw4dz9tlnt05VDfYagZNOOonx48dzySWX0NDQwJIlS1i4cCE/+9nPmDBhAlu3bmXu3Ln89a9/BewVxBMnTmTs2LFcd911rfUNGjSIu+++m0mTJjF27Fg2bNgQdFt78nTV4Xn6aFsOJ0TF68FipY7k3Tth79dd+5rZY+G8Bw75cHp6OpMnT+a9995j9uzZLFiwgMsvvxwR4b777iMtLQ2fz8dZZ53FmjVrGDduXIevs2LFChYsWMCqVavwer1MmjSJE088EYCLL76Y66+/HoBf/epXPP3009xyyy1ccMEFzJo1i0svvfSA12pqamLu3Ll89NFHDBs2jKuvvprHHnuMn/zkJwBkZGSwcuVKHn30UR566CGeeuqpI/4Yevp01eHfIwDbK9AegVI9UtvhobbDQq+88gqTJk1i4sSJrFu37oBhnPY+++wzLrroIuLj40lOTuaCCy5ofWzt2rWcdtppjB07lpdeeumQ01i32LhxI4MHD2bYsGEAXHPNNSxevLj18YsvvhiAE088sXWiuiPp6dNVh3+PAGwQ6HUESh3eYfbcu9OFF17I7bffzsqVK2lsbGTSpEls376dhx56iGXLltGnTx/mzp1LU1PTYV9HDvHxtHPnzuWNN95g/PjxPPfccyxatOiwr3Ok+ddaprI+1FTXR/OaPWW6au0RKKVCKjExkWnTpnHddde19gZqampISEggJSWFffv28e677x72NU4//XRef/11Ghsbqa2t5c0332x9rLa2ln79+uHxeHjppZdalyclJVFbe/B2YcSIERQWFrJlyxYAXnjhBc4444xjamNPn646MnoE0YkaBEr1YFdccQUXX3xx6xDR+PHjmThxIqNHjyY/P59TTjnlsM+fNGkSl19+ORMmTGDgwIGcdtpprY/de++9nHzyyQwcOJCxY8e2bvznzJnD9ddfz7x581oPEgPExsby7LPPctlll+H1ejnppJO48cYbj6o9vW266vCchrq9+d+Dqp1w0z+77jWVCgM6DXXvpNNQd0ZMkp41pJRShxAhQaBDQ0opdSgREgR6sFipQ+mJw8Pq0Lrj9xU5QeD3gLc51JUo1aPExsZSXl6uYdBLGGMoLy8nNja2S183Ms4aap2KuhZcMaGtRakeJC8vj6KiIkpLS0NdigpSbGzsAWckdYXICILWqahrICEjtLUo1YNERUUxePDgUJehQixyhoZAjxMopVQHIiwIdJoJpZRqL8KCQHsESinVngaBUkpFuMgKArcGgVJKtRdUEIjIDBHZKCJbROTODh4XEZkXeHyNiExq97hTRFaJyFtdVfhR0R6BUkod0hGDQEScwCPAecAo4AoRGdVutfOAoYGvG4DH2j3+Y2D9MVfbWVHxIA4NAqWU6kAwPYLJwBZjzDZjjBtYAMxut85s4HljLQVSRaQfgIjkAecDR/48t+4iAtE6zYRSSnUkmCDIBXa1uV8UWBbsOg8DPwf8h3sTEblBRJaLyPJuucoxJklPH1VKqQ4EEwQdff5b+4lJOlxHRGYBJcaYFUd6E2PME8aYAmNMQWZmZhBlHSWdiloppToUTBAUAf3b3M8DioNc5xTgAhEpxA4pnSkiL3a62mOhM5AqpVSHggmCZcBQERksItHAHGBhu3UWAlcHzh6aAlQbY/YYY+4yxuQZYwYFnvexMeaqrmxA0PQzCZRSqkNHnHTOGOMVkR8B7wNO4BljzDoRuTHw+OPAO8BMYAvQAFzbfSV3UkwS1LTvyCillApq9lFjzDvYjX3bZY+3+d4ANx/hNRYBi466wq6iQ0NKKdWhyLiyGPT0UaWUOoTICYKWHoF+EpNSSh0gsoIAA+76UFeilFI9SoQFATo8pJRS7WgQKKVUhIu8INCpqJVS6gCRFwTaI1BKqQNoECilVISLnCCITrS3GgRKKXWAyAmCmGR7q1NRK6XUASIoCJLAGQ2b3gWvO9TVKKVUjxE5QeCKhhkPwNaP4dW5GgZKKRUQOUEAcNK/wcyHYOPb8Np1Ot2EUkoR5OyjYWXy9VBfBp8+AKUbIGtkqCtSSqmQiqweQYuh59jbysKQlqGUUj1BZAZBn0H2VoNAKaUiNAji0+11BRoESikVoUEgYnsFGgRKKRWhQQAaBEopFRC5QZA60AaBnkKqlIpwkRsEfQaBtwnq9oW6EqWUCqnIDgKAyh0hLUMppUJNg0CPEyilIlzkBkHqAHurQaCUinCRGwRRsZCUsz8I/H5Y+jjU7AlpWUopdbxFbhDAgaeQFi2D9+6Ar+aHsiKllDruNAhaguCbv9vb6qJQVaOUUiER4UEwEGqLwdMI6xfaZdW7QluTUkodZxEeBIPs7Td/twHgjIEqDQKlVGTRIAD4fB44XDD6IhsIerWxUiqCaBAAlKyD/GmQPRbcddBUFcKilFLq+IrsIEjsC65Y+/2o2ZDa336vw0NKqQgS2UHQMh21OGH4+ZCSZ5frAWOlVASJvM8sbu+EM6HfBEhIB+O3y7RHoJSKIBoEM+7f/31CBrjitEeglIookT001J6IHR7SIFBKRRANgvZS8nRoSCkVUTQI2kvtrz0CpVRE0SBoL2UA1JfaaSeUUioCBBUEIjJDRDaKyBYRubODx0VE5gUeXyMikwLLY0XkSxH5SkTWichvu7oBXa7lWoLq3aGtQymljpMjBoGIOIFHgPOAUcAVIjKq3WrnAUMDXzcAjwWWNwNnGmPGAxOAGSIypWtK7yat1xLsDG0dSil1nATTI5gMbDHGbDPGuIEFwOx268wGnjfWUiBVRPoF7tcF1okKfPXsiXxS9OpipVRkCSYIcoG2W8WiwLKg1hERp4isBkqAD4wxX3T0JiJyg4gsF5HlpaWlQZbfDZJzQBz6uQRKqYgRTBBIB8va79Ufch1jjM8YMwHIAyaLyJiO3sQY84QxpsAYU5CZmRlEWd3EGWU/wrJlFtL3fmE/wlIppcJUMEFQBPRvcz8PKD7adYwxVcAiYMbRFnnctVxL8PnDsPQRWPViqCtSSqluE0wQLAOGishgEYkG5gAL262zELg6cPbQFKDaGLNHRDJFJBVAROKAs4ENXVd+N0ntD8Ur4cPf2iknyjfbD7dXSqkwdMQgMMZ4gR8B7wPrgVeMMetE5EYRuTGw2jvANmAL8CTww8DyfsAnIrIGGygfGGPe6uI2dL2U/uBpgOwxcPbd4G3Si8yUUmErqEnnjDHvYDf2bZc93uZ7A9zcwfPWABOPscbjb9CpsOl9mPPy/usJyjbZzzhWSqkwo7OPdmTIWfYLICrB3pZtgqHnhK4mpZTqJjrFxJEkpENcmg0CpZQKQxoEwcgcDmWbQ12FUkp1Cw2CYGQM1R6BUipsaRAEI2OYnZG0oSLUlSilVJfTIAhGxjB7q8NDSqkwpEEQjIyh9laHh5RSYUiDIBipA8EZo0GglApLGgTBcDghfYgODSmlwpIGQbAyhkLZxlBXoZRSXU6DIFgZw6CyELzNoa5EKaW6lAZBsDKGgfFDxbZQV6KUUl1KgyBYmYFTSFc+D35faGtRSqkupEEQrL5jYcKVsPRReOFCqN0X6oqUUqpLaBAEy+GA2Y/Yr13L4PFTYPMHoa5KKaWOmQbB0RCBiVfBDZ9AQha8dCm8eyd4mg5cz+vW6SiUUr2GBkFnZI2E6z+GyT+ALx6Dp86CkvX2sZL18Pip8NhUMCa0dSqlVBD0g2k6KyoWZj5oP8Dm7zfD/51hewurX7YfbYmB2r2Q3C/UlSql1GFpj+BYDfs23LQE8qfB8qchrwAuCnyKZ/mWkJamlFLB0B5BV0jMgu/9BfZ8BdljoabYLi/fAoNPC21tSil1BBoEXUUEcibY75NzwRWrPQKlVK+gQ0PdweGAtBM0CJRSvYIGQXfJGKJBoJTqFTQIukv6EDtJnc8T6kqUUuqwNAi6S/oQ8HuhckeoK1FKqcPSIOgu6UPsrQ4PKaV6OA2C7qJBoJTqJTQIukt8GsSlaRAopXo8DYLulK5nDimlej4Ngu6kQaCU6gU0CLpTxhCo3QPNdaGuRCmlDkmDoDvpAWOlVC+gQdCdNAiUUr2ABkF3SssHBMo2hboSpZQ6JA2C7hQVBzkTYcmfYOsnoa5GKaU6pNNQd7cr5sMLF8PL34XvzAMMbP4HRMXDBX8EhzPUFSqlIpwGQXdLyoZr34aXvgtv3GiXxadDQ7l97KzfhLY+pVTE0yA4HuL6wNVvwPq3IHM4ZI+Dt34Mn/0/6H+y/bhLpZQKET1GcLxEJ8D4y+2nmDkccN6D9mMt/3aDzlCqlAqpoIJARGaIyEYR2SIid3bwuIjIvMDja0RkUmB5fxH5RETWi8g6EflxVzeg14qKg+++AD43/PP3oa5GKRXBjhgEIuIEHgHOA0YBV4jIqHarnQcMDXzdADwWWO4F/sMYMxKYAtzcwXMjV9pgyJ8G2/SMIqVU6ATTI5gMbDHGbDPGuIEFwOx268wGnjfWUiBVRPoZY/YYY1YCGGNqgfVAbhfW3/vlT7OfZFaxPdSVKKUiVDBBkAvsanO/iIM35kdcR0QGAROBL466ynCWP93eblsU0jKUUpErmCCQDpaZo1lHRBKB14CfGGNqOnwTkRtEZLmILC8tLQ2irDCRMRSScjQIlFIhE0wQFAH929zPA4qDXUdEorAh8JIx5m+HehNjzBPGmAJjTEFmZmYwtYcHETs8tP1T8PtCXY1SKgIFEwTLgKEiMlhEooE5wMJ26ywErg6cPTQFqDbG7BERAZ4G1htjftellYeTE6ZDYyXsXRPqSpRSEeiIQWCM8QI/At7HHux9xRizTkRuFJHApbK8A2wDtgBPAj8MLD8F+D5wpoisDnzN7OpG9HqDz7C3Oh+RUioExJj2w/2hV1BQYJYvXx7qMo6vR6dCQgZc076zpZRSRyYiK4wxBZ15rl5Z3FPkT4OdS6GhItSVKKUijAZBTzH6QvB74I8nwpdP2mMGu1fA2tfs90op1U10aKgn2bMG3v8FFH524PKTrofzHwpNTUqpXuFYhoZ09tGepN84uOZN2PQelKy31xisegm+fhW+fR+4YkJdoVIqDGkQ9DQiMPw8+wV2crpN79pwGNV+Zg+llDp2eoygp8ufDkn9YPXLoa5EKRWmNAh6OocTxs+BzR9A7b5QV6OUCkMaBL3B+O+B8cHXr4S6EqVUGNIg6A0yh0HeSXZ4qAee5aWU6t00CHqLiVdByTfw1fxQV6KUCjMaBL3FhKtg0Gnw1u2wb51d5mmE1fOhsSqkpSmlejcNgt7C6YJLnobYZHjlavhmITw6Bd64ET5/ONTVKaV6MQ2C3iSprw2Dim3wyvfBEQVZo2DD26GuTCnVi2kQ9DaDT4OLnoBz/wtu+hxOvBbKNkHpplBXppTqpTQIeqNxl8HUW+yUEyPOt8s2vBnampRSvZYGQW+Xkgs5k2D9W/a+MfCPX8Hn80Jbl1Kq19AgCAcjZ0HxSqjeDatfgiV/hI9+a48lKKXUEWgQhIMR37G3//oTvHuHvfjMGQ2f/Hdo61JK9QoaBOEgcxhkDIOlj9q5iS57Dk6+Eb7+K+xdG+rqlFI9XNgEgc9vmP/lTpYVRuhHPY4M9ApmPQwpeXDKrfaag4/vDWlZSqmeL2yCwCHw32+v5+01e0JdSmic8mO46m8w5mJ7P66PXbbpPSiKwE97U0oFLWyCQETonxbPjvL6UJcSGrEpMOSsA5dN/gE4Y+znHiul1CGETRAADEyPZ0dFQ6jL6DliEmHw6bDxXZ21VCl1SGEVBAPS4ymqaMTn141eq+EzoHI7lG0OdSVKqR4qrIJgYFoCbp+fvTVNoS6l5xg2w95ueje0dSileqywCoIBafEA7CzX4aFWKXnQdyxsfC/UlSileqiwCoKB6YEgqIjQA8aHMnwG7FoKDRF6aq1S6rDCKgj6pcTicgg7tEdwoGHngfHDlg9DXYlSqgcKqyBwOR3k9YnTM4fay5kICVn27CGllGonrIIAYEB6gh4jaM/hgGHn2iBY8kdw69CZUmq/8AuCtLjIvajscE7/OfSfbKeofngcvHeXnYuoaufB6zZWgd933EtUSoVG2AXBwLQEapq8VDd4Ql1Kz9JnIFyzEK57H3InwfJn4LV/s6HwzcL96zVUwB9PhJcu1TBQ6mj10gs3wy4IBgTOHNqhZw51bMAUuPJVuKsIfrAY+o6xU1c319rHP/pPaCiDrR/DZ78Lba1KdZf6suA+r6OuBBY9ALtXHnndFc/B78cc28WbPm/nn3sMwi4IWk4h1TOHjsAZBf3Gw6zfQW0xfPo/ULza/jFP+SGMvQwW/TcU/hP8ftj3DVQWhrhopbrAri/h0Snw6FTY+knH6zTV2AD4wwRYdD88fQ58+uChN9SeJvv5HzVF8PLlHZ+q3VAB69+0Q7KrXoLtn4Gn0fYidi6FV6+17xOCXoXruL9jN2u9qEzPHApO/8kw8fuw9DHY/AEkZMC0O0EcULwKFnzPft9YaT/s5tJn9k953ZPsWQPJObb+nszvB7/Hft70cX9vH7x+I8QkwXkPgrOH/fv7/eBrBlcsiBz8+Po34e2fQvoQGDXbfiX1Pbr3+GoBLLwFknMhIdNutOe8DCecCSXfwLZFsOUD2LEEfG77Hqf8GP71KHxyH2x8B0663n4qYGxKm9d9Ger2wfRfweIH4ZWr7WzArmgbHiuetVPCN1UfWI8zGpL6QdUO+3qTrrbve5z/PsT0wDGtgoICs3x556dOLvivDzlzRCYPXjq+C6sKY/Vl9rhAUxXMfhQmXmmX710Lb/0EMofDgKn2j3n3SrjwMRh9Eez7GqqLYMg5EB0fuvpXvRj4586Da/4Oafl2qOvvP4Jtn0DaCbYNk66GgVOP/vWNsb2hlP6d33h63fD1K/DP39uD8Ze/0LlajsWiB+zeLdjf38VP2p5hix3/skODInYDOPICSO535NetKwV3rf25H6196+zGvWiZDUgAcdpp1FMHwEn/bnunq56Hd34GWaPA74XSDRCVYH+OLbPuNlbBmr/YD2eKS7M7Bcm5kJgFm/8BXz4FO5fAoNPgu8/b5zw/275WdCI0BvbiM0fA0HNg9MX2eFqLNa/ajXnVDjurb8F1cO5/2cf+dKJ9z+s/hq9fhb9dbzfwCRn2b7GyEAafAdPugvh0+3dUttn2uEs3wPDzYNzlEJ1w9D/DABFZYYwp6NRzwzEILnlsCVFOYcEN3+rCqsLcxndtN3nGA/Z0044018GCK2D7YrvX5g3M6ZSQCVNvtf8YMYl2mTH2cxB2/NN+Lw67N1S1E2p22w1QbCrEp9l/mKRs+0+bkmdfr7IQ9q21/yw1xfarvtT2TDyNMOJ8mHKTfY/37oCBp0DJersndcGf4INfQ+lGuxGp2wt7v7Zd82/dDGf+GqJiO26jzwsVW+1GpbnG7hmufc3+82eOgHPvg6FnB/9zbaiAlc/Dl0/aYYO+Y8HbCJU74IJ5MG6OXV62yW4U962DmGQ4/Wf793bry2wdeSfZDbPXbfdAlz1te3Mn37C/9k/+yw7j5Uy0G7GBp9jfydZP4IWLYPwc6Dvanj02/HwYd5ntKWx63wZVcq59/9L1gNg95RPnwpCz7c+9ucb2vFr2WLcvhlfn2qGRf/8Q+o7a3/amatj2qT3eJAJjLoUB37LfV263P5clf7R7wuOvsD0VZ5Q9vbmxEnZ+ASXr7IazodzOm3Xps3ano2Q9vHa9rfPCx+2yt263v+tD6TMIJt9gv1oCsLES3g30gAefZkMitf+hX6Pl73rlc3YHZNh5MGKm3RG5/MX9veXVL9veRVON3cM/8RobrB31dLqIBkE7t/9lNUu3lbPkrrOOvLI6Op4mu1fp90Jegd1oLJln/+gdLrvHljXKTmnR/piCOO2GPqW/3ftrrLL/4A1lh34/V5zd+CXlQGKm3Sj4vbD2b3ajBDBilh2yKt9q9/DqS+zG5bI/wwnT7TruevjgN7DsKUgdaDeGsakQl2r3PqMT7D/41o9tz6htzfnT7HTeK/9sDzBmj7Ubz/oy+ylwOROh3wS7oUnOsf/sRcth579gw9s2MAedZocYhpxtX/+Vq+1G1Bljh0NaJOXYn4czBk7/KVTvshucltDtNx7qy214JGTacJz2C5h6C/z1Oju5YFq+/dkbv32dIWfZcfGEDLvHGp0ASx+3AdrCGW3D/LTb7eOlG20ArnrRBndbsam2R5GUbcfN00+wG7yoWLj+E/s3sfh/4bOH7O8qOsnW4qm37fPU7x8imXAVnHuv3SFozxjY8pH9CNb0IfDt/z6wR9ZUDQuuhMLP7P2+Y+E7f4CUXBvA9SVQvRtq90D2OPuzP9ROTmd8+aTtpWAgYzj8cGnXvv5R0iBo5+EPN/GHjzaz4d4ZxLic1Dd7eWX5Lv6ybBdzTurP3FMGd2G1CrAbmo3v2uMK+9bZPcOx37Vd3qj4wEYpuuOhFa/bjq/WFNsNXO0+OyyQPQZSBnT8z9Vca/e6Girs3nPL65ZtgSV/gKk/howhBz9vy4d2L7S+zG5IGivBXWcfS+xrNxaDTrMb2dhkO6yUkL6/zmVPwoZ3bIDEp9vX2bP64I0lQGK2nedp8g02eNryeWyANlbajVz6EBug8Wk20N69w45VO6LsXvzYy2D3Crvn7oq27cufZvdEv3rZbmBr98DM/4XJ19ve2+4V9ney/k3b1n//ELJG7K+haqddz+GyIdHRxtjntT+zvV/bn0dUnA2w9W/Zns2IWXaosHQjPDcT+p9sf9c7Prc9gILr7HEon9s+Z8Ob9ufWb4I9gy1r5MHveTQ8Tbb3l5Rtg6ztUNfx8M1COwR5wTwYfeHxfe92uj0IRGQG8AfACTxljHmg3eMSeHwm0ADMNcasDDz2DDALKDHGjAmmqGMNgtdXFXHbX74iNzWOvskxbC2tp7rRQ1ZSDCW1zfzPJWO5/KQBnX59FWa8btu7iEvr/B5dfbnde6/ZbTd6uSfank9nhwKMsQGTmH34cXq/H/7xS3u210WP27H9jl7L22Q34l2ludYO2/WbsP9ntvIFWPgjO3Y/63c2wCKB3x/SnkCLbg0CEXECm4BzgCJgGXCFMeabNuvMBG7BBsHJwB+MMScHHjsdqAOeP15BUNPk4fFFWymuaqS0rpm0hBjmTh3E2NwUrn9+OYs3l/Lw5RP4zrgcHI7uG7NT6rjxeY7/3nBHNrxt9/I7c+BYHZPuDoJvAfcYY74duH8XgDHm/jbr/B+wyBgzP3B/IzDNGLMncH8Q8NbxCoLDaXT7uOaZL/mysAKXQ+ibHMuI7CSmj8jizBFZ9EuJRbrxgI5SSnWHYwmCYM6FywV2tblfhN3rP9I6ucCezhTVneKinTxz7Un8ffVuiiob2VPVyIqdlXy0oQSAKKeQFBtFXp84fjlzJCfnp4e4YqWU6l7BBEFHu8ftuxHBrHP4NxG5AbgBYMCA7h2/T4xxceXJA1vvG2PYWlrH4k1llNY1U9vk4dNNpVz+xFKu+dZAbp4+hNT4aERgWWEFH68voa7Zy3+cO5zMpBBcGKSUUl0omCAoAtqeWJsHFHdincMyxjwBPAF2aOhonnusRIQhWUkMyUpqXdbg9vLgext5bkkhf/7XDgCcDsHnN0Q7HSDwycYSHvneJAoGdXC2RZD8fqPHKZRSIRVMECwDhorIYGA3MAf4Xrt1FgI/EpEF2GGj6pbjA71VfLSLey4YzUUTc1m9q4qaRg8NHh/j81I5dWgGO8sbuOmlFcx5Yinnje1HXp84MhNjaHB7Ka93YwyckJXI0KxERvZLJiXuwAN5VQ1unvm8kOc+387UEzL438vGkRTbAw72KaUiTrCnj84EHsaePvqMMeY+EbkRwBjzeOD00T8BM7Cnj15rjFkeeO58YBqQAewD7jbGPH249+vOg8VdqbrRw2/fXMeX2yvYV9OEx2d/lokxLvzG0ODeP41zfmYCI/sl4/MZ6pq9rNpZSb3bx5T8NJYVVjIoPZ7/+34BQ7ISQ9UcpVQvpheU9QB+v6Gq0UNCjJMYlxNjDHuqm9i0r5Z1xTWs3lXFpn21RDsdJMa6yM9I5IbT8xmencSSrWXc8vIqapu9DEqPJyc1jnG5KVw5ZSB9kw+cCqG2ycNLX+zE7fUzZ3J/spIOMVWCUiqiaBCEgeKqRp7+53Z2VjRQXNXI+j01OB3CrHE5TBqQSlJsFLurGnnqs21UNngQgSiHg1nj+zGqXzJx0U5iXU5cTiHK6aC60cPuykaKqxtpdPto9vpJjY/i1jOHMiij8xNbKaV6Jg2CMLSjvJ5nPy/k1eW7qG8zxHTGsExuP2cYKXFRPPv5dl5ZXkSjp+NPEnM6hL5JMSTGuohxOdlWWofHZ7jh9Hx+OP0E4qP3HyLy+vx4fIa4aGe3t00p1fU0CMKYx+enutFDTaMHh8hBe/Nen58Gj49Gt/3y+g1ev5/EGBfZybG4nPsvfS+paeL+dzfw+qrdJMa4OG9MNmeOyOKL7RW8tWYPNU0eLpyQw3WnDibW5WTptnLWFdcQH+MkPSGauCgn9W4fDc1e+qXGMTY3hQHp8WzYU8vqXZWU17tJiHaRFOtidE4KEwekEuU8/KX3xhi8foPH57eTlIoNsBjXkQPJ7fUDEO0K/eX9SoWaBoE6Kit3VjL/i5288/Ue6t0+ol0Ozh6ZRUpcFK+v2k2Tx9+6blKMiyavr/VA+OFEOx24ffufmxDtZExuCo0eHxX1bpJjo5g8OI0J/VPZtK+WxZtLWVdc0+EHMuX1iWNMTgpDshJJjHURH+2ktLaZjXtr2VJaR2ltM7VNXuKjncyekMtVUwaQkxLH7qpGqhs9jMlJISXenoVVWe/m002l7K1poq7JS5PHR3JcFKnxUQzJTGRKfnrrKbz1zV7W7q6m0ePD7bWBekJWIllJMXrFuerRNAhUpzS6fazaVcmY3BSSY/dvNF9ftZsol4Nv5adzQqbtgdQ1e2ny2A1jjMvBrsoG1hRVs7OigeF9k5gwIJWMxJjWHszywkr+uaWUb4prSIqNok98FCW1zazcWUmTx4/LIUwa0IcTB/UhIdqJy+lAsFchur3+1oPsheX1rUHhEBickcDQrCSyU2JJS4hmZ0UDb35VTLPXf0DbHAIT+qcSG+Xki+0V+Pz2RWxvw3HAGV05KbHMGp/DjvJ6Fm0sPei1wJ4JNjgjgRMyExiQFk98jA2nBrePvdVNVDa4GZ2TzOnDMhneN6k1NKobPazbXc328nrG56UyOic5qEBp8vgoq2smPtpFn/goDSF1RBoEqtdwe/1sLqllQFp8UNdN+P2GJq+PumYvybFRxEYdPGRU3eDhzTXFuL1+clLjSIhxsmx7BZ9uLqPR7eXskX359uhshvVNIjbKgYjg8fmpavCwdFs5f11RxGebS8lMimHG6GymDc8iOS6KGJc96L61tI6tJXVsK6tnW2k9u6saD3j/xBgXybEuiqvtZwbERzuJcjoQgaoGzwHrZiXFMDw7icoGN2W1bvokRDMyO4kB6fHsqmhkc0ktOysaDnhetNNBanwUXr+h2eMjMymGs0f25cyRWTS6fazfU8P2sgbqm73Uu73kpsYxc2w/pp6QjtMhVNS7qWzwkBIXRUpcFBX1btbvqWFbWT1DshI5cWAfEmPs8aJGt49Gjw+v34/fb4MzyinERjlbf/Yen58d5fXsrW5mXP/9OxGHYozBGPTCyW6mQaDUMapr9hIf5QxqY+X3Gxo9PhrcPmKjHK2Btqe6kc82lbF+rx3u8htD3+RYeywlLZ4VOyr5eGMJuyoayEiMIS0hmtLaZjbsrWFfTTNZSTEM7ZvI4IwEspNjyUyKocHtY19NM5X1bqJc9tjJ1tI6lmwpP2AYrl9KLEmxLuKiXWwtqaOu2UtSrAuf/8DrWTridAjZybFUNbgPODGhvRiXg+S4KCrr3XgDPSyXQzg5P40TMhMprmpiX00TI7KTuGhiLqNzUvjL8p08+3khzV4/V00ZyPenDKSmycM/N5examclJbXNlNY2ExftZER2EsOzkxmYFk+/1NigdxZalNXZocPaJg8Nbh+p8VGcPjSz9ThZZb2bVbsqOXFg2kEXeIYDDQKlerlmry+oA+Qtaps8LN1WQZ/4KIZnJx2wwWzy+Ph0UymfbCghPtpF/7Q40hKiqWnyUlXvtgfzc1MYmB7Ppr11fLG9nF0VDaQlxJCeGE1ijAunQ3A6xJ584PPT4PZR0+ihqsFDWmI0Q7MSSU+M4V9by/lo/T721jSRmxpHRmIMq3dVUdfsRcR+FMKU/DQSY1ytEzu2bHL6pcSSkxpHRmI09c22Z1Ne725thwiMzE5mSn46fmNYtauKzftq6d8nnrF5KeSkxlFa28y+miY27q09qKcGkJ0cy3dP6s/O8nreWbsXt9dPtMvBOaP6ctaILLKSYkmNj2JXRQNfFVWzt7qRU4ZkcM6ovqTGRx/xd7ZhTy0b9taQHBtF/7R4BmUktPaujjcNAqVUj9Hk8fHh+n18XVTNd8bnMCY3BYBtpXX8beVuslNiOW1oBgPTD76epayuuXVW4E37bEit2FGJQ4SxeSmMyE5iV4U9PlVe7yY9IZq+ybGckJXIuNwURuUk0yc+mvhoJxv31fLi0h18trmMpBgXF07M5cwRWSzaWMLCr4qpbDds53IIqfFRlNW5cTmEnNQ4Gtw+GtxeopwOEqLt8JjPGHx+Q0lN8wG9MrC9qyn5aXx7dDaD0hNo9Pjw+Pytx7ZcDmHD3lqWbitn7e5qvtlTw+6qRs4akcX3vzWQSQP6dPp4kAaBUipsub1+HMIBp0K3nHZ8pNOTwZ42bc8827+n7vb62VnRQEW9m4r6ZvqlxDE8O4kYl4Ovd1fz3tq97K5qJCHGRVyUE6/PT73bR5PHZ3tLImQmxzA+L5VR/ZKpa/bagNpdzT/W7WVraf1BdUQ7HcRGOahp8gLQNzmGkf2SSU+I4R/r9lLb7GV0TjJ/++HUo+odttAgUEqpHmRraR2V9W7iop04RNhSUsfa3dXUNHkoGJjGt05IJyd1/0eHNri9/H11MVtL6vjVrFGdek8NAqWUinDHEgR6SaZSSkU4DQKllIpwGgRKKRXhNAiUUirCaRAopVSE0yBQSqkIp0GglFIRToNAKaUiXI+8oExESoEdnXx6BlDWheX0BNqm3kHb1POFW3tgf5sGGmMyO/MCPTIIjoWILO/s1XU9lbapd9A29Xzh1h7omjbp0JBSSkU4DQKllIpw4RgET4S6gG6gbeodtE09X7i1B7qgTWF3jEAppdTRCccegVJKqaOgQaCUUhEubIJARGaIyEYR2SIid4a6ns4Qkf4i8omIrBeRdSLy48DyNBH5QEQ2B277hLrWoyUiThFZJSJvBe736jaJSKqI/FVENgR+X98KgzbdFvi7Wysi80Uktre1SUSeEZESEVnbZtkh2yAidwW2GRtF5NuhqfrwDtGm/w387a0RkddFJLXNY0fdprAIAhFxAo8A5wGjgCtEpHOf9xZaXuA/jDEjgSnAzYF23Al8ZIwZCnwUuN/b/BhY3+Z+b2/TH4D3jDEjgPHYtvXaNolILnArUGCMGQM4gTn0vjY9B8xot6zDNgT+t+YAowPPeTSwLelpnuPgNn0AjDHGjAM2AXdB59sUFkEATAa2GGO2GWPcwAJgdohrOmrGmD3GmJWB72uxG5dcbFv+HFjtz8CFISmwk0QkDzgfeKrN4l7bJhFJBk4HngYwxriNMVX04jYFuIA4EXEB8UAxvaxNxpjFQEW7xYdqw2xggTGm2RizHdiC3Zb0KB21yRjzD2OMN3B3KZAX+L5TbQqXIMgFdrW5XxRY1muJyCBgIvAF0NcYswdsWABZISytMx4Gfg742yzrzW3KB0qBZwPDXU+JSAK9uE3GmN3AQ8BOYA9QbYz5B724TW0cqg3hst24Dng38H2n2hQuQSAdLOu158WKSCLwGvATY0xNqOs5FiIyCygxxqwIdS1dyAVMAh4zxkwE6un5QyaHFRg3nw0MBnKABBG5KrRVdbtev90QkV9ih5RfalnUwWpHbFO4BEER0L/N/Txst7bXEZEobAi8ZIz5W2DxPhHpF3i8H1ASqvo64RTgAhEpxA7ZnSkiL9K721QEFBljvgjc/ys2GHpzm84GthtjSo0xHuBvwFR6d5taHKoNvXq7ISLXALOAK83+C8I61aZwCYJlwFARGSwi0diDJQtDXNNRExHBjjuvN8b8rs1DC4FrAt9fA/z9eNfWWcaYu4wxecaYQdjfy8fGmKvo3W3aC+wSkeGBRWcB39CL24QdEpoiIvGBv8OzsMeoenObWhyqDQuBOSISIyKDgaHAlyGo76iJyAzgDuACY0xDm4c61yZjTFh8ATOxR8+3Ar8MdT2dbMOp2G7cGmB14GsmkI4922Fz4DYt1LV2sn3TgLcC3/fqNgETgOWB39UbQJ8waNNvgQ3AWuAFIKa3tQmYjz3G4cHuHf/b4doA/DKwzdgInBfq+o+iTVuwxwJathOPH0ubdIoJpZSKcOEyNKSUUqqTNAiUUirCaRAopVSE0yBQSqkIp0GglFIRToNAKaUinAaBUkpFuP8Pf43So7mZ6hoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = train_model(model_type, features_train, targets_train, features_valid, targets_valid, model_parameters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test\n",
    "\n",
    "Spearman's rank correlation, ROC curves, etc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from train import get_predictions\n",
    "\n",
    "predictions, actuals, _ = get_predictions(\n",
    "    model_type, model, features_valid, targets_valid, video_ids_valid, aggregate_with=aggregate_with\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPEARMAN RANK: 0.19194066153188785\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfgAAAHwCAYAAABKe30SAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAABE20lEQVR4nO3de5gU9Zn3//fNQRlBGQ3EyAwCKkHODI5AIApqBIyrIiFRNBE1ifEUNZtF0ez+9Hmyu5IH10OixsRs1DVGE0+oUReNKBglxkFgDSCGAOIMruKBMwoD9++P6ml6Znpmema6uqurP6/r6gu6qrr7W1U9fdf3dJe5OyIiIhIvHfJdABEREck+BXgREZEYUoAXERGJIQV4ERGRGFKAFxERiSEFeBERkRhSgBcJgZm5mR2V73LEiZndYGa/Sfz/cDPbZmYdc/C568zsK2F/jki2KcCLUHg/4ikBLvXhZvbDxPoJZra3wfoZ+S53trj7enfv5u57mtsucRyqc1WutkpzLveY2c8S6/omzm3q+n/Jd5kl+jrluwAi0nruvh7oVvfczPoBq4FHUzbb4O7luS5bJsysk7vX5rscUeHuqeeyK/A+8HCDzUp1zKQ1VIOXWDGzWWb2dzPbamYrzOzMBuu/a2YrU9aPNLP7gcOBpxK1o6vT1fxSa/lmNsrMFpnZJjN7z8xuN7P9MizjS2b2r2b2auLznjKzz5nZA2a2xcxeN7O+rdz184CF7r6ula+rO2aPNFh2m5n9NPH/881sTeKYrTWzc5t4nxvM7BEz+11i2zfMbHjK+nVmdo2Z/Q+w3cw6mdmYxHHYZGbLzGxCyvb9zGxB4r2eB3qkrKur1XZKPD/EzO4xsw1m9omZzU0EymeBXik1315m1iHle/KRmf3ezA5Jee9vmdk7iXU/auHY3Wtmd5rZs4n3f8XMvmBmtybK8ZaZVWR+NgCYBnwAvNzK14nU5+566BGbB/B1oBfBxetZwHbgsJR1NcCxgAFHAX0S69YBX0l5nwlAdYP3Tm4DHAOMIWgF6wusBK5K2daBo5oo40sEte0jge7ACuBt4CuJ9/sv4J5W7vffgfMblH8XQU1wLXAL0LWJ1/YBdgAHJZ53BN5L7F9XYAswILHuMGBwE+9zA7CbIEB1Bv4p8dmdU47fUqA3UAKUAR8BX02cr5MTz3smtl8E3AzsDxwPbAV+k1jXN3GMOyWePw38Djg48dnjmzmPVwF/BsoT7/0L4MHEukHAtsTn7Z/4/NrU70aD97oX+DDxfegCzE/s83mJ4/ivwIutPJfzgRtSntftaw1QDdwD9Mj335oe0X/kvQB66BHmIxFQzkj8fx5wZRPbraMVAT7N668CHk953lKA/1HK8/8Ank15fhqwtBX7eFwiKHVLWfaFRLDqAPQDFgK/aOY9/gScl/j/ycDfE//vCmwCvgaUtFCOG4A/pzzvQHChcFzK8bswZf01wP0N3mMeMIOgRaWWlIsS4LekCfAEFx17gYPTlCndeVwJnJTy/DCCC5NOwP8HPJSyrivBhVJzAf7ulOffB1amPB8KbGrFuTwc2AP0S1nWDahMlO9Q4BFgXq7/lvQovIea6CVWzOw8M1uaaPLdBAxhX9Nub4KabjY+54tm9gcz+18z2wL8e8rnZOL9lP/vTPO8G5mbATzq7tvqFrj7/7r7Cnff6+5rgasJatZN+S0wPfH/cxLPcfftBC0hFwPvmdnTZnZ0M+/zbkoZ9hLUOHulW0/QcvD1unOVOF9fJgi4vYBPEp9f550mPrM38LG7f9JMuVL1AR5P+cyVBEH10MTnpu7DdoJWheZk81yeB/wpcc7qyrDN3avcvdbd3wcuByaa2UGteF8pQgrwEhtm1ge4m+AH8HPuXgr8laA5HoIf7iObeHnD2ypuBw5Iee+OQM+U9T8H3gL6u/tBwHUpn5MzZlZC0PVwXwubOs2X72FggpmVA2eSCPAA7j7P3U8mCLxvERzjpvROKVsHgmbwDQ3KUeddghp8acqjq7vPJqj5H5zoR69zeBOf+S5wiJmVplmX7naZ7wKnNPjcLu5ek/jc1H04APhck3ubfeeR2bmEPHzfpLAowEucdCX48dsIYGYXENTg6/wK+CczO8YCRyUuCiCodR2Rsu3bQBczO9XMOgP/TNAnW+dAgr7pbYka7SWh7FHLziRoQn8xdWFikODhif3sDcwGnmjqTdx9I0HXwT3AWndfmXifQ83s9ESg/YygK6C5qWnHmNnUxOC3qxKv+XMT2/4GOM3MJplZRzPrkih3ubu/A1QB/8fM9jOzLxN0XaQr+3sEg+nuNLODzayzmR2fWP0+8Dkz657ykruAf6s792bW08zOSKx7BPgHM/uyBYMm/y85+p00s7EE4xIebrB8tJkNSAwO/BzwU+Ald9+ci3JJ4VKAl9hw9xUE/dmLCH7YhwKvpKx/GPg3gtrpVmAuUDd6+kbgnxPNtv+U+PG8lOCioIagRp86qv6fCJqytxLUaH8X2o41bwbwX+7esKY6kuA4bAdeJWjJuKKF9/otwUC/36Ys6wD8kKAW/jEwnuC4NOUJgib9T4BvAVPdfXe6Dd39XeAMgtaPjQQ165ns+106Bxid+NzrCQYfNuVbBP3obxGMQL8q8RlvAQ8CaxLnthdwG/Ak8JyZbSW4ABmd2H45cFniGLyX2I9czaOfATzm7lsbLD8C+G+C79pfCS6apiPSAmv8uyAi0npmdgPBwMJv5rssIqIavIiISCwpk51IRJnZcQR9y414SuYziT4zO5wg30E6gzzITCiSVWqiFxERiSE10YuIiMRQqAHezCab2SozW21ms9Ks725BHu5lZrY8Ma2pbl2pBXmt37Igd/iXwiyriIhInITWRJ9IDPI2QdrLauB1YHpiKlPdNtcB3d39GjPrCawCvuDuu8zsPuBld/9VYj7qAe6+qbnP7NGjh/ft2zeU/REREYmaxYsXf+juPdOtC3OQ3ShgtbuvATCzhwjmvKYONHHgQDMzgnSOHwO1iRSMxwPnA7j7LoJ80M3q27cvVVVV2dwHERGRyDKzplI4h9pEX0b9vNPViWWpbgcGEiTReJPgRiB7CRI7bATuMbMlZvarBikrk8zsIjOrMrOqjRs3Zn0nREREClGYAT5dnuSG/QGTCO721QsYAdyeqL13IsjE9XN3ryDIxtWoDx/A3X/p7pXuXtmzZ9pWChERkaITZoCvJuWmDTS+6QTABQSpGd3dVxPcR/noxGur3f21xHaPEAR8ERERyUCYffCvA/3NrB9BLu+zCXJLp1oPnAS8bGaHAgOANe7+oZm9a2YD3H1VYpumkkQ0a/fu3VRXV/Ppp5+2eUdERHKhS5culJeX07lz53wXRWIgtADv7rVmdjkwD+gI/Nrdl5vZxYn1dwE/Bu41szcJmvSvcfcPE2/xfeCBxAj6NQS1/Varrq7mwAMPpG/fvgRj+UREosfd+eijj6iurqZfv375Lo7EQKipat39GeCZBsvuSvn/BmBiE69dClS2twyffvqpgruIRJ6Z8bnPfQ4NFpZsKYpMdgruIlII9Fsl2VQUAT7f/u3f/o3BgwczbNgwRowYwWuvvdbyi4rADTfcwE033ZTvYgDQrVvz927ZtGkTd955Z/L5hg0bmDZtWtjFSuvGG2/kqKOOYsCAAcybNy/tNg8//DCDBw+mQ4cO9XJD7Nq1iwsuuIChQ4cyfPhwXnrppeS6H/3oR/Tu3bvRsXjnnXc46aSTGDZsGBMmTKC6uv7t0bds2UJZWRmXX355ctnatWsZPXo0/fv356yzzmLXrn1pLF566SVGjBjB4MGDGT9+fHL5pk2bmDZtGkcffTQDBw5k0aJFze7LunXrKCkpYcSIEYwYMYKLL744uW7y5MkMHz6cwYMHc/HFF7Nnzx4AFi5cyMiRI+nUqROPPPJIvf24+uqrGTx4MAMHDuSKK66gLgnYCy+8wMiRIxkxYgRf/vKXWb16NQBz5sxJfvaQIUPo2LEjH3/8MQC33HILgwcPZsiQIUyfPj05BuiGG26grKws+bpnnqnXwCmSXe4em8cxxxzjDa1YsaLRslx69dVXfcyYMf7pp5+6u/vGjRu9pqYmtM/bvXt3aO/dlL179/qePXta/brrr7/e58yZE0KJGpeppTJ27dq12fdbu3atDx48OGvla6vly5f7sGHD/NNPP/U1a9b4EUcc4bW1tY22W7Fihb/11ls+fvx4f/3115PLb7/9dj///PPd3f3999/3kSNHJo/LokWLfMOGDY2OxbRp0/zee+91d/cXXnjBv/nNb9Zbf8UVV/j06dP9sssuSy77+te/7g8++KC7u3/ve9/zO++8093dP/nkEx84cKC/8847yTLUOe+88/zuu+92d/fPPvvMP/nkk2b3pblzsnnzZncPzvvUqVOTZVm7dq0vW7bMv/Wtb/nDDz+c3P6VV17xsWPHem1trdfW1vqYMWP8xRdfdHf3/v37J39H7rjjDp8xY0ajz3vyySf9hBNOcHf36upq79u3r+/YsSN5LO655x53z+w7n+/fLCksQJU3ERNVg29g7pIaxs2eT79ZTzNu9nzmLqlp1/u999579OjRg/333x+AHj160KtXLyDIvHfNNdcwatQoRo0alawZbNy4ka997Wsce+yxHHvssbzyyisA/OUvf2Hs2LFUVFQwduxYVq1aBcC9997L17/+dU477TQmTpzIvffey5QpUzjttNPo168ft99+OzfffDMVFRWMGTMmWcu4++67OfbYYxk+fDhf+9rX2LFjBwDnn38+V1xxBWPHjuWII45oVNOBoPY0cOBALr30UkaOHMm7777LJZdcQmVlJYMHD+b6669Pbtu3b1+uv/56Ro4cydChQ3nrrbcavd/dd9/NKaecws6dO+stf//99znzzDMZPnw4w4cP59VXXwXg5ptvZsiQIQwZMoRbb701bZlefvnlRmWcM2cOxx57LMOGDatXxjrbtm3jpJNOSpb1iSeeAGDWrFn8/e9/Z8SIEcycOZN169YxZMgQIBjnUVcrrqio4MUXX0yel6lTpzJ58mT69+/P1Vdf3fyXJQNPPPEEZ599Nvvvvz/9+vXjqKOO4i9/+Uuj7QYOHMiAAQMaLV+xYgUnnXQSAJ///OcpLS1N1orHjBnDYYcd1uxrTjjhhOQxAVi8eDHvv/8+EyfuG0rj7syfPz/ZwjFjxgzmzp0LwG9/+1umTp3K4YcfniwDBK0ACxcu5Nvf/jYA++23H6Wlpc3uS3MOOuggAGpra9m1a1ey6btv374MGzaMDh3q//SZGZ9++im7du3is88+Y/fu3Rx66KHJdVu2bAFg8+bNyb/fVA8++CDTp09PPq+trWXnzp3U1tayY8eOtK8RCV1Tkb8QH+2twT/+RrUf/c/Pep9r/pB8HP3Pz/rjb1Rn/B4Nbd261YcPH+79+/f3Sy65xF966aXkuj59+vi//uu/urv7fffd56eeeqq7u0+fPt1ffvlld3d/5513/Oijj3b3oFZSV0N//vnnferUqe7ufs8993hZWZl/9NFHyedHHnmkb9myxT/44AM/6KCD/Oc//7m7u1911VV+yy23uLv7hx9+mCzLj370I//pT3/q7u4zZszwadOm+Z49e3z58uV+5JFHNtqvtWvXupn5okWLksvqPr+2ttbHjx/vy5YtS+5n3Xvfcccd/u1vf9vd99Vmfvazn/lpp52WbOVI9Y1vfCNZ3traWt+0aZNXVVX5kCFDfNu2bb5161YfNGiQv/HGG43K1PD5vHnz/Lvf/W6yNn/qqaf6ggUL3H1fDX737t3J2t/GjRv9yCOP9L179zaqLaY+v+mmm5K14pUrV3rv3r19586dfs8993i/fv1806ZNvnPnTj/88MN9/fr1jfbxqquu8uHDhzd63HjjjY22veyyy/z+++9PPr/wwgvr1UQbaljr/cUvfuHTpk3z3bt3+5o1a7x79+7+yCOP1HtNwxr89OnT/dZbb3V390cffdQB//DDD33Pnj0+fvx4X79+vd9zzz3JGnzdcauzfv365LG68sor/dJLL/Xx48f7yJEj/b777nN39yVLlvixxx7rM2bM8BEjRvi3v/1t37ZtW7P7snbtWj/ggAN8xIgRfvzxx/vChQvrbT9x4kQvLS316dOnN2rlmDFjRqPj9sMf/tC7d+/uBx10kF933XXJ5QsXLvRDDjnEy8rKfODAgcnvR53t27f7wQcfnPz+u7vfeuut3rVrV+/Ro4efc845yeXXX3+99+nTx4cOHeoXXHCBf/zxx96QavDSGqgGn5k581axc/eeest27t7DnHmr2vye3bp1Y/Hixfzyl7+kZ8+enHXWWdx7773J9XVX/dOnT0/2Of7xj3/k8ssvZ8SIEZx++uls2bKFrVu3snnzZr7+9a8zZMgQfvCDH7B8+fLk+5x88skccsghyecnnHACBx54ID179qR79+6cdtppAAwdOpR169YB8Ne//pXjjjuOoUOH8sADD9R7vylTptChQwcGDRrE+++/n3bf+vTpw5gxY5LPf//73zNy5EgqKipYvnw5K1bsS10wdepUAI455pjk5wPcf//9PPvsszz66KPJVo5U8+fP55JLLgGgY8eOdO/enT/96U+ceeaZdO3alW7dujF16lRefvnltGVKff7cc8/x3HPPUVFRwciRI3nrrbf429/+Vu/z3J3rrruOYcOG8ZWvfIWampom97/On/70J771rW8BcPTRR9OnTx/efvttAE466SS6d+9Oly5dGDRoEO+80zht9C233MLSpUsbPWbNapy80dPcHKo1A7MuvPBCysvLqays5KqrrmLs2LF06tT8ZJqbbrqJBQsWUFFRwYIFCygrK6NTp07ceeedfPWrX6V37971tm+ujLW1tSxevJinn36aefPm8eMf/5i3336b2tpa3njjDS655BKWLFlC165dmT17drPlOuyww1i/fj1Llizh5ptv5pxzzknWtAHmzZvHe++9x2effcb8+fObfa/Vq1ezcuVKqqurqampYf78+SxcuBAIzs8zzzxDdXU1F1xwAf/4j/9Y77VPPfUU48aNS/79ffLJJzzxxBOsXbuWDRs2sH37dn7zm98AcMkll/D3v/+dpUuXcthhh/HDH/6w2XKJtEeo0+QKzYZNO1u1PFMdO3ZkwoQJTJgwgaFDh3Lfffdx/vnnA/V/nOv+v3fvXhYtWkRJSUm99/n+97/PCSecwOOPP866deuYMGFCcl3XrvVT9acGyw4dOiSfd+jQgdraWiBoip87dy7Dhw/n3nvvrTfgKvX16X6wG37m2rVruemmm3j99dc5+OCDOf/88+slF6p7v44dOyY/H2DIkCEsXbq0VXN/mypPwzI1fO7uXHvttXzve99r8vUPPPAAGzduZPHixXTu3Jm+ffu2mCSpufKkHseG+17nBz/4QbJZP9XZZ5/dKMiXl5fz7rv7bvFQXV3dqubfTp06ccsttySfjx07lv79+zf7ml69evHYY48BQRfGo48+Svfu3Vm0aBEvv/wyd955J9u2bWPXrl1069aNG2+8kU2bNlFbW0unTp3qlbG8vJwePXrQtWtXunbtyvHHH8+yZcs47rjjKC8vZ/To0QBMmzatxQC///77J4/vMcccw5FHHsnbb79NZeW+2bVdunTh9NNP54knnuDkk09u8r0ef/xxxowZkxxgeMopp/DnP/+ZgQMHsmzZsmS5zjrrLCZPnlzvtQ899FC95vk//vGP9OvXj7rU2VOnTuXVV1/lm9/8ZrLZH+C73/0u//AP/9DsPoq0h2rwKXqVlrRqeSZWrVpVr5a4dOlS+vTpk3z+u9/9Lvnvl74U3PJ+4sSJ3H777fVeA0H/X1lZcL+e1FaAttq6dSuHHXYYu3fv5oEHHmjXe23ZsoWuXbvSvXt33n//fZ599tmMXldRUcEvfvELTj/9dDZsaJjJOKgB//znPwdgz549bNmyheOPP565c+eyY8cOtm/fzuOPP85xxx3X4mdNmjSJX//612zbtg2AmpoaPvjgg3rbbN68mc9//vN07tyZF198MVnjPvDAA9m6dWva9z3++OOTx+/tt99m/fr1reozbk0N/vTTT+ehhx7is88+Y+3atfztb39j1KhRGX9W3TEDeP755+nUqRODBg1q9jUffvghe/fuBYIR/BdeeCEQXAytX7+edevWcdNNN3Heeecxe/ZszIwTTjghOXbjvvvu44wzzgDgjDPO4OWXX072Tb/22msMHDiQL3zhC/Tu3Ts5ruSFF15osVwbN25Mjo5fs2YNf/vb3zjiiCPYtm0b7733HhC0GDzzzDMcffTRzb7X4YcfzoIFC6itrWX37t0sWLCAgQMHcvDBB7N58+Zki8zzzz/PwIEDk6/bvHkzCxYsSO5f3Xv9+c9/ZseOHbg7L7zwQvI1deWC4KKibhyHSBgU4FPMnDSAks4d6y0r6dyRmZNaN8An1bZt25gxYwaDBg1i2LBhrFixghtuuCG5/rPPPmP06NHcdtttyZrVT3/6U6qqqhg2bBiDBg3irruC3EBXX3011157LePGjUv+sLXHj3/8Y0aPHs3JJ5/c4g9gS4YPH05FRQWDBw/mwgsvZNy4cRm/9stf/jI33XQTp556Kh9++GG9dbfddhsvvvgiQ4cO5ZhjjmH58uWMHDmS888/n1GjRjF69Gi+853vUFFR0eLnTJw4kXPOOYcvfelLDB06lGnTpjUK2ueeey5VVVVUVlbywAMPJI/L5z73OcaNG8eQIUOYOXNmvddceuml7Nmzh6FDhya7YNJ1N2TD4MGD+cY3vsGgQYOYPHkyd9xxBx07Bt/Z73znO8kBc48//jjl5eUsWrSIU089lUmTJgHwwQcfMHLkSAYOHMhPfvIT7r///uR7X3311ZSXl7Njxw7Ky8uT39OXXnqJAQMG8MUvfpH333+fH/3oRy2W8yc/+Qk333wzRx11FB999FFy8NzAgQOZPHkyw4YNY9SoUXznO99JBrmf/exnnHvuuQwbNoylS5dy3XXXNbsvCxcuZNiwYQwfPpxp06Zx1113ccghh7B9+3ZOP/305LrPf/7zySl0r7/+OuXl5Tz88MN873vfY/DgwUDQYnDkkUcmpw8OHz6c0047jU6dOnH33Xfzta99jeHDh3P//fczZ86c5H4+/vjjTJw4sV5L0ejRo5k2bVpyoObevXu56KKLksd46NChDBs2jBdffLFea4pItllzzYuFprKy0hveD37lypX1rrhbMndJDXPmrWLDpp30Ki1h5qQBTKloeJfb7Ki7f32PHj1CeX8RKTyt/c0qZrn8vY4qM1vs7mmzvqoPvoEpFWVF9wURESk0c5fUcO1jbyYHRtds2sm1j70JoN/wBDXR59G6detUexcRaYMwZj3FjQK8iIgUnLBmPcVJUQT4OI0zEJH40m9V5sKY9RQ3sQ/wXbp04aOPPtIfjohEmntwP/guXbrkuygFIYxZT3ET+0F25eXlVFdX6x7LIhJ5Xbp0oby8PN/FKAh1A+mKfRR9c2I/TU5ERCSumpsmF/smehERkWKkAC8iIhJDCvAiIiIxpAAvIiISQwrwIiIiMaQALyIiEkMK8CIiIjGkAC8iIhJDCvAiIiIxpAAvIiISQwrwIiIiMaQALyIiEkMK8CIiIjGkAC8iIhJDCvAiIiIxpAAvIiISQwrwIiIiMaQALyIiEkMK8CIiIjGkAC8iIhJDCvAiIiIxpAAvIiISQwrwIiIiMaQALyIiEkMK8CIiIjGkAC8iIhJDnfJdABERiZa5S2qYM28VGzbtpFdpCTMnDWBKRVm+iyWtpAAvIiJJc5fUcO1jb7Jz9x4Aajbt5NrH3gRQkC8waqIXEZGkOfNWJYN7nZ279zBn3qo8lUjaSgFeRESSNmza2arlEl0K8CIiktSrtKRVyyW6FOBFRCRp5qQBlHTuWG9ZSeeOzJw0IE8lkrbSIDsREUmqG0inUfSFTwFeRETqmVJRpoAeA2qiFxERiSHV4EVEJFaUqCegAC8iIrGhRD37qIleRERiQ4l69lGAFxGR2FCinn3URC8ikaM+VGmrXqUl1KQJ5sWYqEc1eBGJlLo+1JpNO3H29aHOXVKT76JJAVCinn0U4EUkUtSHKu0xpaKMG6cOpay0BAPKSku4cerQomwBUhO9iESK+lClvZSoJ6AavIhEim52IpIdCvAiEinqQxXJDjXRi0ik6GYnItmhAC85oWlP0hrqQxVpPwV4CZ1SR4qI5J764CV0mvYkIpJ7CvASOk17EhHJPQV4CZ2mPYmI5J4CvIRO055ERHJPg+wkdJr2JCKSewrwkhOa9iQikltqohcREYkh1eBFRKSgklEVUlnzSQFeRKTIFVIyqkIqa76piV5EpMgVUjKqQiprvoUa4M1sspmtMrPVZjYrzfruZvaUmS0zs+VmdkGD9R3NbImZ/SHMcoqIFLNCSkZVSGXNt9ACvJl1BO4ATgEGAdPNbFCDzS4DVrj7cGAC8B9mtl/K+iuBlWGVUURECisZVSGVNd/CrMGPAla7+xp33wU8BJzRYBsHDjQzA7oBHwO1AGZWDpwK/CrEMoqIFL1CSkZVSGXNtzAH2ZUB76Y8rwZGN9jmduBJYANwIHCWu+9NrLsVuDqxXEREQlJIyagKqaz5FmaAtzTLvMHzScBS4ETgSOB5M3sZOB74wN0Xm9mEZj/E7CLgIoDDDz+8fSUWESlShZSMqpDKmk9hNtFXA71TnpcT1NRTXQA85oHVwFrgaGAccLqZrSNo2j/RzH6T7kPc/ZfuXunulT179sz2PoiIiBSkMAP860B/M+uXGDh3NkFzfKr1wEkAZnYoMABY4+7Xunu5u/dNvG6+u38zxLKKiIjESmhN9O5ea2aXA/OAjsCv3X25mV2cWH8X8GPgXjN7k6BJ/xp3/zCsMomIZErZ0qTQmXvDbvHCVVlZ6VVVVfkuhogUuIbZ0iAYqX3j1KEK8hIpZrbY3SvTrVMmOxGRBpQtTeJAAV5EpAFlS5M4UIAXEWlA2dIkDhTgRUQaULY0iQPdLlZEpAFlS5M4UIAXEUlD2dKk0KmJXkREJIYU4EVERGJITfQiIgVK2fakOQrwIiIFqGG2vZpNO7n2sTcBFOQFUBO9iEhBUrY9aYkCvIhIAVK2PWmJAryISAFStj1piQK8iEgBUrY9aYkG2YmIFCBl25OWKMCLiBSoTLPtaTpdcVKAFxGJMU2nK17qgxcRiTFNpyteCvAiIjGm6XTFSwFeRCTGNJ2ueCnAi4gUiLlLahg3ez79Zj3NuNnzmbukpsXXaDpd8dIgOxGRAtDWwXKaTle8FOBFRApAc4PlWgrWmU6nk3hRE72ISAHQYDlpLdXgRaRdlEQlN3qVllCTJpi3Z7Cczl28KcCLSJspiUr46oJwzaadGOAp69ozWE7nLv7URC8ibaYkKuGqC8J1NXcHLLGurLSEG6cObXMw1rmLP9Xgi5ia56S91C8crnRB2AmC+yuzTmzXe+vcxZ9q8EUqtWbg7Guey2RerUgdJVEJV5hBWOcu/hTgi5Sa5yQblEQlXN1LOqddno0grHMXf2qiL1JqnpNsUBKV8MxdUsP2XbWNlnfuYFkJwlE9d+o6zB4F+CIVxpQbKU5KohKOOfNWsXuPN1rerUunrB3v1py7XARejezPLjXRFyk1z4lEW1OtaZt27M5xSXI3Zkddh9mlGnyRimrzXBSpyVDyIUqtbO1Jk9sa2eg61N/rPgrwRUxNqy1Tk6Hky8xJA+p99yB/rWy5GrPT3osa/b3WpyZ6kWaoyVDyZUpFGTdOHUpZaQlG+xPbtEeuptS1t+tQf6/1qQYv0gzNNpB8ikorW65aE9rbdRjVv9d8dRsowIs0I0r9oCL5kssxO+25qIni32s+uw0U4EWaEaV+0CjTwKbClsn5i0prQnOi+PeaqwGK6SjAizRDsw1apoFNhS1O5y+Kf6/57DZQgBdpQSHUXPIpnzUUab+4nb+o/b3ms9tAo+hFpF2iOrApCuYuqWHc7Pn0m/U042bPj+TNnHT+wpXPpGIK8CLSLrorWXqFcsdGnb9w5XO6owK8iLSL0h6n11TT91W/Wxqp2nwUz18htHy0xpSKMmZOGkCv0hI2bNrJnHmrcrJP6oMXkXaJ4sCmKGiuiTtKA9midv7iNOivTr72ydwb362oUFVWVnpVVVW+iyESK5oC1zbjZs9PO7gqVVlpCa/MOjFHJSoMTR23Qj5WYe6TmS1298p069RELyJNKpR+5ChK1/TdkAayNdbSoL9CbL7P10BGBXgRaZJye7dd6uCqpmggW2PNDfor1AvOfA1kVIAXkSZpClX7TKko45VZJ3LrWSMiN5Atqpob9FeoF5z5GsioQXYi0qQo5vYuRM0NZNMYh/qaO1Y/+N3StK+J+gVnvgYyapCdiDSp4ehfCGoe+bptadzo+LZOHAfgtZcG2YlIm0TpnuRxVKhNzvkyc9IAOnewess6dzB1dTRBTfQi0qyo5faOE41xaANr4bkkqQYvIpInShPbOnPmrWL3nvrdyrv3uFo8mqAALyKSJ1FMExtlavFoHTXRi4jkSbZHV8d9RL5mdbSOAryISB5la4xDHHO4NzRz0oC0sw7U4pGeAnyOxP3KupjoXEoUNTciPy7fz6jdGCfqFOBzoBiurIuFzqXkQlsuIpvrn47TRalmdWROg+xyQHNd40PnUsLW1nzrTfVDlx7QuSDzt0v7KcC3UyZ3NtLIz/jQuZSwtfUisqkR+e7oorRIKcC3Q6ZX2prrGh86lxK2tl5ENpV1cPPO3W16Pyl8CvDtkOmVtua6xofOpYStPReRdXevWzv7VF6ZdSJTKsp0UVrEFODbIdMrbeXzjg+dSwlbti8idVFavDSKvh1ak3RBIz/jQ+dSwpTtqWCaWla8dLvYdtCtHkWkTpymoknhaO52sarBt4OujEUKTxiBOFv5EXSRINmkGryIFI2wWt3GzZ6ftruurLSEV2admNeytZUuNgpDczV4DbITkaIRVqKibORHiFISpbYm25FoUYAXkaIRVqKibExFi1ISpShdbEjbKcCLSNEIa054NqaiRWm+epQuNqTtFOBFpGiENSc8G/kRojRfPUoXG9J2GkUvIkUjzJkv7c2PEKVZOYVw33UNAmyZRtGLiEgjUQ2gc5fUcMOTy9nUIMd+seYgaW4UvQK8iIgUhHRTCRsqi9DFSC7kbZqcmU02s1VmttrMZqVZ393MnjKzZWa23MwuSCzvbWYvmtnKxPIrwyyniIhEX7rR/Q1pSt8+oQV4M+sI3AGcAgwCppvZoAabXQascPfhwATgP8xsP6AW+KG7DwTGAJelea2IiBSRTEfxa0pfIMwa/ChgtbuvcfddwEPAGQ22ceBAMzOgG/AxUOvu77n7GwDuvhVYCRRHe4uIiKRV0jnzkKUpfeEG+DLg3ZTn1TQO0rcDA4ENwJvAle6+N3UDM+sLVACvhVZSERGJtLlLatixe2/LGyZoSl+4Ad7SLGs4om8SsBToBYwAbjezg5JvYNYNeBS4yt23pP0Qs4vMrMrMqjZu3JiNcouISMS0psk9alP68iXMAF8N9E55Xk5QU091AfCYB1YDa4GjAcysM0Fwf8DdH2vqQ9z9l+5e6e6VPXv2zOoOiBSquUtqGDd7Pv1mPc242fM14EgKXnNN7qUlnduVZCiuwkx08zrQ38z6ATXA2cA5DbZZD5wEvGxmhwIDgDWJPvn/BFa6+80hllEkdrJ161KRKOlVWpL2jn0G3HD6YH230witBu/utcDlwDyCQXK/d/flZnaxmV2c2OzHwFgzexN4AbjG3T8ExgHfAk40s6WJx1fDKqtInOhGIRJH6VL5GnDumMMV3JsQaqpad38GeKbBsrtS/r8BmJjmdX8ifR++iLRANwqROGprKt+oZuTLBeWiF4mZppoyNapYCl1r8/0Xe3eV7iYnEjNRuiuZSD7UDTK96ndLi7q7SjV4kZiJ0l3JRHItk3z1xdJdpQAvEkPtvXWpSKHKJF99sXRXqYleRERio6XaeTF1V6kGLxJDxTxyWIpbU4NMofhuJasALxIzxT5yOMp04RW+mZMGNOqDL+ncsSiz26mJXiRmlOgmmuouvGo27cTRfcvDMqWijBunDlXqWlSDlwhTbadtlOgmmpq78NL3Ors0yDSgGrxEkmo7bdfUCOFiGTkcVbrwklxTgJdIilIzc6Hdma2lRDeFtj9xoQsvyTUFeImkqNR2CrElobk+yELcn7hQhkHJNfXBSyRFJZ96ofabNtUHWaj7Ewe5yjCosStSRwFeIqmpqS65ru1EpSUhW+K2P4Um7MFfmiIpqdREL5EUlakuces3jdv+SH1RGrsi+acavERWFKa6RKUlIVvitj9SXzZbaNTUX/gyqsGb2ZVmdpAF/tPM3jCziWEXTiTfotKSkC1x2x+pL1stNBqMGQ/m7i1vZLbM3Yeb2STgMuBfgHvcfWTYBWyNyspKr6qqyncxRKRIRK2Wm+5WqalpWjMt77jZ89MOci0rLeGVWSeGug/SOma22N0r063LtIneEv9+lSCwLzMza+4FIiJxFsUBbc2N1G9NeTUYMx4yDfCLzew5oB9wrZkdCOwNr1gi0RG1WppEQ1SnHGZjimRUpqlK+2Q6iv7bwCzgWHffAewHXBBaqUQiQn2R0pRCq+W2prxKyhMPGQV4d98LvA8MMrPjgcFAaYjlEokETTuSphTalMPWlFeDMeMhoyZ6M/sJcBawAqj7tXNgYUjlEomEQqulSe4U2pTD1pY3CtNUpX0y7YOfAgxw989CLItI5KgvUpqSq9Sz2VJo5ZX2yzTArwE6AwrwUlQKrZYmuVVotdxCK6+0T6YBfgew1MxeICXIu/sVoZQqzzRqWuqo1iMihSrTAP9k4hF7UZzbKvmlWo8UOlVailNGAd7d7zOz/YAvJhatcvfd4RUrf6I6t1VE8q8QA2U2Ky2FuP/FLNNR9BOA+4B1BFntepvZDHeP3Sh6jZoWkXQKtXUvW5WWqO6/Ljqalmmim/8AJrr7eHc/HpgE3BJesfKn0Oa2ikhuFGpOhGxVWqK4/0pE1bxMA3xnd0+eRXd/m2BUfewog5OIpFOorXvZqrREcf+jeNERJZkG+KrEbWInJB53A4vDLFi+KIOTiKSTjUA5d0kN42bPp9+spxk3e35OaprZqrREsXUzihcdUZLpKPpLCG4TewVBH/xC4M6wCpVvGjUtIg21NydCvvqwszXVM4o5IZSIqnkZ3Q++UOh+8CISpvYM6IrDPdYz2f9cDnpreNEEwUVHMbW6tvl+8Gb2e3f/hpm9SZB7vh53H5alMoqIRF57Wvfi0Jzc0v7nupVCiaia11IT/ZWJf/8h7IKIZJOmzjRNxyY/iqE5OR95RNSl2rRmB9m5+3uJ/17q7u+kPoBLwy+eSOtp6kzTdGzypxhm6MShlSJOMh1Ff3KaZadksyAi2aKpM01r67HJx+jvuCmGGTpRHGlfzFrqg7+EoKZ+pJn9T8qqA4FXwyyYSFupFtG0thybqGYwK0Rxb06O4kj7YtZSDf63wGnAE4l/6x7HuPu5IZdNpE1Ui2haW46NWkQkU8XQSlFImq3Bu/tmYLOZ3QZ87O5bAczsQDMb7e6v5aKQIq2hWkTT2nJs1CIirRH3VopCkmkf/M+BbSnPtyeWiUSOahFNa8uxUYuISGHKNJOdeUpGHHffa2aZvlYk51SLaFprj01cWkQ0PVCKTaY1+DVmdoWZdU48rgTWhFkwEYmGOLSIaHqgFKOMUtWa2eeBnwInEmS0ewG4yt0/CLd4raNUtSKSThzSxIqk0+ZUtXUSgfzsrJZKRCRHNFBQilFL8+Cvdvf/Z2Y/I30u+itCK5mISJYUQ5pYkYZaqsGvTPyrdm8RKVhxGSgo0hotzYN/KvHvfbkpjohI9umuY/GjWREta6mJ/inSNM3XcffTs14ikQKgH5fCo6mTLSuU77XSJ2empSb6mxL/TgW+APwm8Xw6sC6kMolEmn5cJI4K6Xudj9vSFqKWbhe7wN0XABXufpa7P5V4nAN8OTdFFIkW5WaXOCqk77VmRWQm00Q3Pc3siLonZtYP6BlOkUSiTT8uEkeF9L1W+uTMZBrgfwC8ZGYvmdlLwIvAVWEVSqShKN2PXD8uEkeF9L2eOWkAJZ071lumWRGNZRTg3f2/gf7AlYnHAHefF2bBROpELc2oflwkjgrpex2H9Mm5kFEmOzM7APhHoI+7f9fM+pvZAHf/Q7jFE4negBpNuYqnQhlBHpZC+15rVkTLMr0j3D3AYuBLiefVwMOAAryELl0GMshv36B+XAJxCYqFNII8TPpex0umffBHuvv/A3YDuPtOwEIrlUjC3CU1TX7Rotg3WEyi1nXSHoU0glwkU5kG+F1mVkIi6Y2ZHQl8FlqpRBLmzFuVNtOSQST7BotJnIJiIY0gF8lUpgH+euC/gd5m9gDB7WKvDq1UIglN/cA6xdV0GkVxCoqFNIJcJFMtBngz6wAcTJDN7nzgQaDS3V8KtWQiNP0DW6Yf3ryLU1AspBHkIplqMcC7+17gcnf/yN2fdvc/uPuHOSibiH54IyxO50bTriSOMh1F/7yZ/RPwO2B73UJ3/ziUUokkFNrUnWISt3OjEeQSN+be5M3i9m1ktpY0d5Vz9yPSbJ43lZWVXlWlW9eLiIQlLlMj48LMFrt7Zbp1mdbgBwGXEtxgxoGXgbuyUzwpBvpRECl8yhdQWDIdRX8fMBD4KfCzxP/vC6tQEi9xmi8tUsziNDWyGGRagx/g7sNTnr9oZsvCKJDET9RSzYrkS6G3ZMVpamQxyLQGv8TMxtQ9MbPRwCvhFEniRj8KIvFoyYrT1MhikGmAHw28ambrzGwdsAgYb2Zvmtn/hFY6iQX9KORWlG6tK/s01ZL1f55anqcStV6cpkYWg0yb6CeHWgqJtZmTBtQbmAP6UQiLBkFFV1MtVp/s2M3cJTUFcX7iNjUy7jIK8O7+TtgFkfjSj0LuaLxDdPUqLWnyzoiFdH6UL6BwZFqDF2kX/SjkhsY7RNfMSQO46ndL067T+ZEwZNoHLyIFQOMdomtKRRmlJZ3TrtP5kTAowIvEiAZBRdsNpw/W+ZGcURO9SIxovEO06fxILmWUi77Nb242GbgN6Aj8yt1nN1jfHfgNcDjBxcZN7n5PJq9NR7noRUSkmDSXiz60Jnoz6wjcAZxCkMt+upkNarDZZcCKRJa8CcB/mNl+Gb5WREREmhBmE/0oYLW7rwEws4eAM4AVKds4cKCZGdAN+BioJUis09JrRUTqyVYq2EJPKSsC4Qb4MuDdlOfVBIE71e3Ak8AG4EDgLHffa2aZvFZEJClbSX6ULEjiIsxR9JZmWcMO/0nAUqAXMAK43cwOyvC1wYeYXWRmVWZWtXHjxraXVkQKWrbudKY7pklchBngq4HeKc/LCWrqqS4AHvPAamAtcHSGrwXA3X/p7pXuXtmzZ8+sFV5ECku2kvwoWZDERZgB/nWgv5n1M7P9gLMJmuNTrQdOAjCzQ4EBwJoMXysikpStJD9KFiRxEVqAd/da4HJgHrAS+L27Lzezi83s4sRmPwbGmtmbwAvANe7+YVOvDausIlL4spXkR8mCJC5CnQefa5oHL1Lc4jiKPkplkehpbh68AryISEQ1HNEPQWvCjVOHKsgL0HyAV6pakSxSbSu+8nFudftfaQ8FeJEs0fzp+MrXudWIfmkP3U1OJEs0fzq+8nVuNaJf2kMBXiRLVNuKr3ydW43ol/ZQgBfJEtW24itf53ZKRRk3Th1KWWkJBpSVlmiAnWRMffAiWTJz0oC0I55V2yp8+Ty3UyrKFNClTRTgRbKk7kdYo+jTK+QZBjq3Uog0D15EQqf53CLhaG4evPrgRSR0mmEgknsK8CISOs0wEMk99cGLSOh6lZZQkyaYa4aBtEchj+vIBdXgRSR0ms8t2VY3rqNm006cfdkF5y6pyXfRIkMBXkRCp/nckm0a19EyNdGLSE7EdT63monzQ+M6WqYALyKNKGhlRjcYyh+N62iZmuhFpB71bWZOzcT5o3EdLVMNXkTqifo9yKPUuqBm4vxRdsGWKcCLSD1RDlpRaxJXM3F+xXVcR7aoiV5E6onyXfGi1iSuZmKJMgV4EaknykEraq0Lmv4nUaYmehGpJ599my31r0exSVzNxBJVCvAi0kg+glYm/ev5vC+7SKFRgBeJkSiNMG+tTEbvF+rI6UI+L1K4FOBFYiJqI8xbK9P+9XStC1EOoIV+XqRwaZCdSExEbYR5a7V19H7UE/MU+nmJgrlLahg3ez79Zj3NuNnzI3Nuo04BXiQmojbCvLXaOno/6gG00M9LvqW7gJv58DIq/u9zCvgtUBN9REW5yVGiKYojzFujrf3ruQqgbf2bLPTzkm/pLuB273U+2bEbUJdHcxTgI0h9dtIWcRhh3pbR+7kIoO35m4zDecmnTC7UopRKOUrURB9BUW9ylGgq1qQruUjM056/yWI9L9mS6YWaujwaUw0+gtRnJ21VjElXcjF1rr1/k8V4XrIlXQtIOuryaEwBPoLUZyfSOmEHUP1N5k/DC7juJZ3ZvquW3Xs8uY26PNJTE30ERTkXuEgx0t9kfk2pKOOVWSeydvapLL1+InOmDVeXRwZUg4+gQs3WJRJX+puMFnV5ZMbcveWtCkRlZaVXVVXluxgiIiI5YWaL3b0y3To10YuIiMSQmuhFJCeUvEkktxTgRSR0St4kkntqoheR0Cl5k0juKcCLSOiUvEkk99REL1IgCrkPW4liRHJPNXiRAhD1e563RIliRHJPAV6kABR6H7ZuuCKSe2qiFykAcejDVvYxkdxSgBcpAOrDlrgr5DEmUaUmepECoD5sibNCH2MSVQrwIgVAfdgSZ4U+xiSq1EQfIWqikuaoD1viKg5jTKJINfiIUBOViBSrpsaSaIxJ+yjAR4SaqESkWGmMSTjURB8RaqISaZ66sOKr7jzq/GaXAnxEaBqUSNN0N7r40xiT7FMTfUSoiUqkaerCEmk91eAjQk1UIk1TF5ZI6ynAR4iaqETSUxeWSOupiV5EIk9dWCKtpxq8iESeurBaRzMOBBTgRaRAqAsrM5pxIHXURC8iEiOacSB1VIMXkYKnJul9NONA6qgGLyIFTfdxqE953aWOAryIFDQ1SdenGQdSR030IlLQ1CRdn2YcSB0FeBEpaEqC05hmHAioiV5ECpyapEXSUw1eRAqamqRF0lOAF5GCpyZpkcYU4EUkNjQfXmQfBXgRiYViS9FaDBczxbCPYdIgOxGJhWKaD18MyX2KYR/DpgAvIrFQTPPhi+Fiphj2MWwK8CISC8WUorUYLmaKYR/DpgAvIrFQTPPhi+Fiphj2MWwK8CISC1Mqyrhx6lDKSkswoKy0hBunDo3loKxiuJhpaR/nLqlh3Oz59Jv1NONmz1fffBoaRS8isVEs8+GLIblPc/tYbDMm2srcPbw3N5sM3AZ0BH7l7rMbrJ8JnJt42gkYCPR094/N7AfAdwAH3gQucPdPm/u8yspKr6qqyvJeiIhIlIybPT/t/QfKSkt4ZdaJeShR/pjZYnevTLcutCZ6M+sI3AGcAgwCppvZoNRt3H2Ou49w9xHAtcCCRHAvA64AKt19CMEFwtlhlVVERAqHBuBlJsw++FHAandf4+67gIeAM5rZfjrwYMrzTkCJmXUCDgA2hFZSEREpGBqAl5kwA3wZ8G7K8+rEskbM7ABgMvAogLvXADcB64H3gM3u/lyIZZUiosE5IoWtGAYZZkOYAd7SLGuqw/804BV3/xjAzA4mqO33A3oBXc3sm2k/xOwiM6sys6qNGzdmodgSZ8qOJVL4imnGRHuEOYq+Guid8rycppvZz6Z+8/xXgLXuvhHAzB4DxgK/afhCd/8l8EsIBtm1v9gSZ81lx9KPg0jhKJYZE+0RZg3+daC/mfUzs/0IgviTDTcys+7AeOCJlMXrgTFmdoCZGXASsDLEskqR0OAcESkWoQV4d68FLgfmEQTn37v7cjO72MwuTtn0TOA5d9+e8trXgEeANwimyHUgUUsXaQ8NzhGRYhHqPPhc0zx4aUnDBBkQDM5R/52IFKLm5sErk50UlWLIACYiAgrwUoQ0OEdEioFuNiMiIhJDCvAiIiIxpAAvIiISQwrwIiIiMaQALyIiEkMK8CIiIjGkAC8iIhJDCvAiIiIxpAAvIiISQwrwIiIiMaQALyIiEkMK8CIiIjGkAC8iIhJDCvAiIiIxpAAvIiISQwrwIiIiMaQALyIiEkMK8CIiIjGkAC8iIhJDCvAiIiIxpAAvIiISQwrwIiIiMaQALyIiEkOd8l0AkWIwd0kNc+atYsOmnfQqLWHmpAFMqSjLd7FEJMYU4EVCNndJDdc+9iY7d+8BoGbTTq597E0ABXkRCY2a6EVCNmfeqmRwr7Nz9x7mzFuVpxKJSDFQgBcJ2YZNO1u1XEQkGxTgRULWq7SkVctFRLJBAV4kZDMnDaCkc8d6y0o6d2TmpAF5KpGIFAMNshMJWd1AOo2iF5FcUoAXyYEpFWUK6CKSU2qiFxERiSEFeBERkRhSgBcREYkhBXgREZEYUoAXERGJIQV4ERGRGFKAFxERiSEFeBERkRhSgBcREYkhBXgREZEYUqpaEZE8m7ukRvcqkKxTgBcRyaO5S2q49rE32bl7DwA1m3Zy7WNvAhRckNeFSrSoiV5EJI/mzFuVDO51du7ew5x5q/JUorapu1Cp2bQTZ9+FytwlNfkuWtFSgBcRyaMNm3a2anlUxeVCJU4U4EVE8qhXaUmrlkdVXC5U4kQBXkQkj2ZOGkBJ5471lpV07sjMSQPyVKK2icuFSpwowIuI5NGUijJunDqUstISDCgrLeHGqUMLbnBaXC5U4kSj6EVE8mxKRVnBBfSG6sqvUfTRoQAvIiJZEYcLlThRE72IiEgMKcCLiIjEkAK8iIhIDKkPXkREWk1paaNPAV5ERFolzPz5unDIHjXRi4hIq4SVllb57LNLAV5ERFolrLS0ymefXQrwIiLSKmGlpVU+++xSgBcRkVYJKy2t8tlnlwK8iIi0Slj585XPPrs0il5ERFotjLS0ymefXQrwIiISGcpnnz1qohcREYkhBXgREZEYUoAXERGJIQV4ERGRGNIgOxHJCeUYF8ktBXgRCV2YNycRkfTURC8ioVOOcZHcU4AXkdApx7hI7inAi0jolGNcJPcU4EUkdMoxLpJ7GmQnIqFTjnGR3As1wJvZZOA2oCPwK3ef3WD9TODclLIMBHq6+8dmVgr8ChgCOHChuy8Ks7wiEh7lGBfJrdCa6M2sI3AHcAowCJhuZoNSt3H3Oe4+wt1HANcCC9z948Tq24D/dvejgeHAyrDKKiIiEjdh9sGPAla7+xp33wU8BJzRzPbTgQcBzOwg4HjgPwHcfZe7bwqxrCIiIrESZoAvA95NeV6dWNaImR0ATAYeTSw6AtgI3GNmS8zsV2bWNcSyioiIxEqYAd7SLPMmtj0NeCWleb4TMBL4ubtXANuBWWk/xOwiM6sys6qNGze2t8wiIiKxEGaArwZ6pzwvBzY0se3ZJJrnU15b7e6vJZ4/QhDwG3H3X7p7pbtX9uzZs51FFhERiYcwA/zrQH8z62dm+xEE8ScbbmRm3YHxwBN1y9z9f4F3zaxukuxJwIoQyyoiIhIroU2Tc/daM7scmEcwTe7X7r7czC5OrL8rsemZwHPuvr3BW3wfeCBxcbAGuCCssoqIiMSNuTfVLV54KisrvaqqKt/FEBERyQkzW+zulenWKVWtiIhIDCnAi4iIxJACvIiISAwpwIuIiMSQAryIiEgMKcCLiIjEkAK8iIhIDMVqHryZbQTeyXc5MtAD+DDfhShiOv75peOfXzr++ZXt49/H3dPmaY9VgC8UZlbVVGICCZ+Of37p+OeXjn9+5fL4q4leREQkhhTgRUREYkgBPj9+me8CFDkd//zS8c8vHf/8ytnxVx+8iIhIDKkGLyIiEkMK8CEys8lmtsrMVpvZrCa2mWBmS81suZktyHUZ46yl429mMxPHfqmZ/dXM9pjZIfkoaxxlcPy7m9lTZrYs8f2/IB/ljKsMjv/BZva4mf2Pmf3FzIbko5xxZGa/NrMPzOyvTaw3M/tp4tz8j5mNDKUg7q5HCA+gI/B34AhgP2AZMKjBNqXACuDwxPPP57vccXlkcvwbbH8aMD/f5Y7LI8Pv/3XATxL/7wl8DOyX77LH4ZHh8Z8DXJ/4/9HAC/kud1wewPHASOCvTaz/KvAsYMAY4LUwyqEafHhGAavdfY277wIeAs5osM05wGPuvh7A3T/IcRnjLJPjn2o68GBOSlYcMjn+DhxoZgZ0IwjwtbktZmxlcvwHAS8AuPtbQF8zOzS3xYwnd19I8H1uyhnAf3ngz0CpmR2W7XIowIenDHg35Xl1YlmqLwIHm9lLZrbYzM7LWeniL5PjD4CZHQBMBh7NQbmKRSbH/3ZgILABeBO40t335qZ4sZfJ8V8GTAUws1FAH6A8J6WTjH+f2kMBPjyWZlnDKQudgGOAU4FJwL+Y2RfDLliRyOT41zkNeMXdm7viltbJ5PhPApYCvYARwO1mdlC4xSoamRz/2QQVjKXA94ElqAUlV1rz+9RmnbL9hpJUDfROeV5OUFNpuM2H7r4d2G5mC4HhwNu5KWKsZXL865yNmuezLZPjfwEw24NOydVmtpagL/gvuSlirLV4/N19C8E5INFNsjbxkPC15vepzVSDD8/rQH8z62dm+xEEkScbbPMEcJyZdUo0E48GVua4nHGVyfHHzLoD4wnOhWRPJsd/PXASQKLvdwCwJqeljK8Wj7+ZlSbWAXwHWJgI+hK+J4HzEqPpxwCb3f29bH+IavAhcfdaM7scmEcwovXX7r7czC5OrL/L3Vea2X8D/wPsBX7l7mmnVUjrZHL8E5ueCTyXaEWRLMnw+P8YuNfM3iRosrzG3XWXsyzI8PgPBP7LzPYQzOb5dt4KHDNm9iAwAehhZtXA9UBnSB77ZwhG0q8GdpBoScl6ORJD9kVERCRG1EQvIiISQwrwIiIiMaQALyIiEkMK8CIiIjGkAC8iIhJDCvAi0mqJuyCObed7bMtWeUSkMQV4EWmLCUC7AryIhEsBXkSSzGxu4sZHy83sosSyyWb2RuK+7S+YWV/gYuAHZrbUzI4zs3vNbFrK+2xL/Nst8Zo3zOxNM2vujn4ikkVKdCMiSWZ2iLt/bGYlBOlOTwKqgOPdfW3K+huAbe5+U+J19wJ/cPdHEs+3uXs3M+sEHODuW8ysB/BnoL+7e902edhNkaKgVLUikuoKMzsz8f/ewEUEOcrXArThjnsG/LuZHU+QjrkMOBT43yyVV0SaoAAvIkAwcA74CvAld99hZi8R3DN8QAYvryXR5Ze4M1ndTUzOBXoCx7j7bjNbB3TJasFFJC31wYtIne7AJ4ngfjQwBtgfGG9m/SBowk9suxU4MOW164BjEv8/g8SNNRLv+UEiuJ8A9Al3F0SkjvrgRQQAM9sfmEvQjL6KoOZ9A1AC/DtBheADdz/ZzL4IPELQ7P594G2CW+52AF4Avp/og+8BPEUQ8JcC44BT3H2d+uBFwqUALyIiEkNqohcREYkhBXgREZEYUoAXERGJIQV4ERGRGFKAFxERiSEFeBERkRhSgBcREYkhBXgREZEY+v8BCwQgTK7wQ1gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "spearman_rank, _ = stats.spearmanr(actuals, predictions)\n",
    "print(\"SPEARMAN RANK:\",spearman_rank)\n",
    "\n",
    "fig, ax = plt.subplots(1, figsize=(8,8))\n",
    "min_mem = min(np.min(actuals), np.min(predictions))\n",
    "max_mem = max(np.max(actuals), np.max(predictions))\n",
    "plt.scatter(actuals, predictions, label = f\"Spearman rank correlation = {spearman_rank}\")\n",
    "# plt.plot([min_mem, max_mem], [min_mem, max_mem], label=\"1 to 1\")\n",
    "plt.title(f\"actual {target} vs predicted {target}\")\n",
    "plt.legend()\n",
    "plt.ylabel(\"predictions\")\n",
    "plt.xlabel(\"actual\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving Predictions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_train, actual_train, vid_train = get_predictions(model_type, model, features_train, targets_train, video_ids_train, aggregate_with=aggregate_with)\n",
    "pred_valid, actual_valid, vid_valid = get_predictions(model_type, model, features_valid, targets_valid, video_ids_valid, aggregate_with=aggregate_with)\n",
    "pred_test, actual_test, vid_test = get_predictions(model_type, model, features_test, targets_test, video_ids_test, aggregate_with=aggregate_with)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation spearman rank: 0.19194066153188785\n",
      "Training spearman rank: 0.6394903773885922\n"
     ]
    }
   ],
   "source": [
    "print(\"Validation spearman rank:\", stats.spearmanr(actual_valid, pred_valid)[0])\n",
    "print(\"Training spearman rank:\", stats.spearmanr(actual_train, pred_train)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = np.concatenate([pred_train, pred_valid, pred_test])\n",
    "actuals = np.concatenate([actual_train, actual_valid, actual_test])\n",
    "video_ids = np.concatenate([vid_train, vid_valid, vid_test])\n",
    "in_training_set = np.array(np.concatenate([np.ones(len(pred_train)), np.zeros(len(pred_valid + pred_test))]), dtype=bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "default_prediction = np.mean(predictions)\n",
    "for vid in train_data.index:\n",
    "    if vid not in video_ids:\n",
    "        print(f\"Adding vid {vid} with default prediction {default_prediction}\")\n",
    "        video_ids = np.append(video_ids, vid)\n",
    "        predictions = np.append(predictions, default_prediction)\n",
    "        in_training_set = np.append(in_training_set, vid in training_data.index)\n",
    "        actuals = np.append(actuals, train_data.loc[vid][target])\n",
    "        \n",
    "for vid in test_data.index:\n",
    "    if vid not in video_ids:\n",
    "        video_ids = np.append(video_ids, vid)\n",
    "        predictions = np.append(predictions, default_prediction)\n",
    "        in_training_set = np.append(in_training_set, False)\n",
    "        actuals = np.append(actuals, np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(predictions) == len(actuals) == len(video_ids) == len(in_training_set) == len(train_data) + len(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_predictions(model_name, video_ids, actuals, predictions, in_training_set, model_parameters, predictions_dir = PREDICTIONS_DIR):\n",
    "\n",
    "    if not os.path.exists(predictions_dir):\n",
    "        os.mkdir(predictions_dir)\n",
    "        \n",
    "    model_data_dir = f\"{predictions_dir}/model_data.csv\"\n",
    "    if not os.path.exists(model_data_dir):\n",
    "        model_data = pd.DataFrame(columns=[\"name\", \"validation_spearman_rank\", \"type\", \"feature\", \"parameters\", \"predictions\", \"is_short_term\", \"seed\", \"notes\"])\n",
    "    else:\n",
    "        model_data = pd.read_csv(model_data_dir)\n",
    "        \n",
    "    model_dir = f\"{predictions_dir}/{model_name}\"\n",
    "    if not os.path.exists(model_dir):\n",
    "        os.mkdir(model_dir)\n",
    "        \n",
    "    pred_filename = f\"{model_dir}/{'st' if IS_SHORT_TERM else 'lt'}-{RANDOM_SEED}.csv\"\n",
    "    \n",
    "    valid_spearman_rank, _ = stats.spearmanr(actuals[in_training_set], predictions[in_training_set])\n",
    "    \n",
    "    pred_data = pd.DataFrame({\n",
    "        \"video_id\": video_ids,\n",
    "        \"prediction\": predictions,\n",
    "        \"actual\": actuals,\n",
    "        \"in_training_set\": in_training_set\n",
    "    }).sort_values(\"video_id\")\n",
    "    \n",
    "    pred_data.to_csv(pred_filename, index = False)\n",
    "    \n",
    "    model_info = {\n",
    "        \"name\": model_name,\n",
    "        \"type\": model_type,\n",
    "        \"feature\": feature,\n",
    "        \"validation_spearman_rank\": np.around(spearman_rank,4),\n",
    "        \"parameters\": model_parameters,\n",
    "        \"predictions\": pred_filename,\n",
    "        \"notes\": NOTES,\n",
    "        \"seed\": RANDOM_SEED,\n",
    "        \"is_short_term\": IS_SHORT_TERM\n",
    "    }\n",
    "    \n",
    "    model_data.append(model_info, ignore_index = True).to_csv(model_data_dir, index = False)\n",
    "    \n",
    "    print(\"Saved model info and predictions: \", model_info)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "ename": "AssertionError",
     "evalue": "Model was not saved.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-dffe1252d0a3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32massert\u001b[0m \u001b[0mSAVE_MODEL\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Model was not saved.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m: Model was not saved."
     ]
    }
   ],
   "source": [
    "assert SAVE_MODEL, \"Model was not saved.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_predictions(model_name, video_ids, actuals, predictions, in_training_set, model_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "video-mem",
   "language": "python",
   "name": "video-mem"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}