model training and evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "supreme-delhi",
   "metadata": {},
   "source": [
    "### GCN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "prescription-satin",
   "metadata": {},
   "outputs": [],
   "source": [
    "# libraries\n",
    "import os, math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm.notebook import tqdm\n",
    "import torch\n",
    "from torch_geometric.data import DataLoader\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import roc_auc_score, classification_report, confusion_matrix\n",
    "\n",
    "# pytorch\n",
    "import torch\n",
    "import torch.nn.functional as F \n",
    "from torch.nn import Linear\n",
    "from torch_geometric.nn import GCNConv, GATConv\n",
    "from torch_geometric.nn import global_mean_pool as gap\n",
    "from torch_geometric.nn import global_max_pool as gmp\n",
    "\n",
    "# from local files\n",
    "from custom_dataset import MoleculeDataset\n",
    "from model import GCN, GAT\n",
    "from trainer import Trainer\n",
    "from utils import optimal_cutoff, draw_roc_curve, plot_confusion_matrix, summerize_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "laden-christmas",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device:  cpu\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"Device: \", device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "thorough-legislature",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "friendly-cliff",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = MoleculeDataset(root=\"Dataset/\", filename=\"solubility-dataset-train.csv\")\n",
    "test_dataset = MoleculeDataset(root=\"Dataset/\", filename=\"solubility-dataset-test.csv\", test=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "static-brunswick",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = DataLoader(train_dataset, batch_size=batch_size)\n",
    "test_loader = DataLoader(test_dataset, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "adolescent-space",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8847, 986)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_loader.dataset.length, test_loader.dataset.length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "selected-mixture",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_true_test_labels(test_loader):\n",
    "    y_true_list = []\n",
    "    for data in test_loader:\n",
    "        ys = data.y.flatten()\n",
    "        y_true_list.extend(list(ys.cpu().detach().numpy()))\n",
    "    return np.array(y_true_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "threatened-breach",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train and evaluate GCN model\n",
    "def train_evaluate(params):\n",
    "    \n",
    "    hidden_channels = params[\"hidden_channels\"]\n",
    "    dropout = params[\"dropout\"]\n",
    "    lr = params[\"lr\"]\n",
    "    #momentum = params.get(\"momentum\", 0.9)\n",
    "    weight_decay = params[\"weight_decay\"]\n",
    "    epochs = params[\"n_epochs\"]\n",
    "    \n",
    "    model = GCN(n_features=30, hidden_channels=hidden_channels, dropout=dropout)\n",
    "    model.to(device)\n",
    "    \n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)\n",
    "    trainer = Trainer(model=model, optimizer=optimizer, train_loader=train_loader, valid_loader=test_loader)\n",
    "    \n",
    "    # training\n",
    "    (train_losses, train_scores), (valid_losses, valid_scores) = trainer.run(n_epochs=epochs)\n",
    "    \n",
    "    # prediction on test/valid loader\n",
    "    y_proba = trainer.predict(test_loader)\n",
    "    y_true = get_true_test_labels(test_loader=test_loader)\n",
    "    test_auc = roc_auc_score(y_true, y_proba)\n",
    "    return (train_losses, train_scores), (valid_losses, valid_scores), (y_true, y_proba)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "scenic-economy",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21aab3cf14c94052a8a3957c3f6a7e31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed74d57ecfbc4ec09b888fb27bcedf22",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "079f02c09fbf423dbf8e8f48b730b178",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d905297c1a7c44379e70301b6d175055",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66c201665f254025b543d4aae078502c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb43db613f9346cd82ffd26cef370cb0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf25b1e4721d42b5b0b31e0c2438672b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6861b97ec1df462497bba412a161eee0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41b58b8902524fefbe2fd96fc406f11b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35959359a93f4ef6aa4bcae75ef7b7c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47e8c75c9d474e5f9ea9de820214a9c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2b610393c89491c9c97a2df898fedaf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2490b4f1620045a0b4840622ef8c06a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66f8b4b624b44a5ea1ae52aaca18b550",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06675e7dd5b7470d96cdd15ff3655c8c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "188d6c83704d47269b6b0f44244911d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a1d66652ec14489b32f2167929c6c8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8c1199022a948bd85c2ed660ce26d9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3eca14c918f42108f2265d84d4852a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a5a6e5fef45e4fd39ab3ec4dd09a6c6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3397506eda84a72982e88cd75734e71",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce8cd9f6efd14a6189151830560f9072",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ded23e775014dc98ad84f9b550a87ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9687a20ce92a408184a66eece0d330bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d9b102ecee946c8be73c4ff12e47816",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1311847ae8354e44a6c4e7387c129862",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a0c260e3fbe4d51b3fad29868849c8d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a2a8a34a7104c09b84443c26dfc3b48",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a51b9fbf9de04a1e9327fa625e5a7a81",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4378d8ff9f84b80ac4a6c25e8dc0912",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "795ed03cd7184f109e645adf66f38286",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9427eea1884048dd93a82c9fb301c442",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f8e2c5ae23b4578af9b1d1e98cec3b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c806f08578cb447eb47732a977752a58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "525f1fd676b04fb8a4baf0855dfe3ff4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5da38507f50b47a8aaac7ac60afb6d44",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e0cdb558ee74a139642caf7a673113c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1918b6d1ffe44cca247b9e28942272c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cdf8d346245f4851bc8a6a4e611dcee2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac0d2417f98046f2881c021aab40b1b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2bda176ab9d844819defe2ee982e0e9e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce3dc8b7724d452bb75620139198de46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa315cf762064d9f90b09de7977ee657",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e2a5e994ba548388f037082910f42f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3e3cbcdc98e47a4b4d3523b6073529f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ba63cbd601b4bb99d4cd575fcecedd0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "926250bdccdc418f9a0469c9d43dee73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6e51075eb58437789b49bfe0e9872a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7573f638de84585868061ece50a0d97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14f8e1dee76343b498d2fb42bd9a2c7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/139 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54afaafd137b4ae5b68cc09cea173f77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/16 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# training params\n",
    "params = {\n",
    "          \"hidden_channels\": 128, \n",
    "          \"dropout\": 0.4, \n",
    "          \"lr\": 0.01, \n",
    "          \"weight_decay\": 7e-5, \n",
    "          \"n_epochs\": 50\n",
    "         }\n",
    "\n",
    "# training and evaulation\n",
    "(train_losses, train_scores), (valid_losses, valid_scores), (y_true, y_proba) = train_evaluate(params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "sensitive-welcome",
   "metadata": {},
   "outputs": [],
   "source": [
    "N_EPOCHS = params[\"n_epochs\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bored-desire",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAxVUlEQVR4nO3de3xcdZ3/8dcnk6RJb+mF0ktS2nIRWmhpoZRK0a3W5X6piFgUdx/dXVkU5bIuUvytLKK7VFEXWVgRlZ/6E8QulFIQqdxRFGyhpfdKKaVNAvSaXpPmMp/fH2eSTibnpDNtJpNk3s/HI49kvuecme83bc7nfO/m7oiIiKQqyHUGRESka1KAEBGRUAoQIiISSgFCRERCKUCIiEiowlxnoCMdddRRPnr06FxnQ0Sk23j99de3ufuQsGM9KkCMHj2aJUuW5DobIiLdhpm9G3VMTUwiIhJKAUJEREIpQIiISKge1QcRpqGhgcrKSurq6nKdlawrKSmhoqKCoqKiXGdFRHqAHh8gKisr6devH6NHj8bMcp2drHF3tm/fTmVlJWPGjMl1dkSkB+jxAaKurq7HBwcAM2Pw4MFs3bo111kRkU6yYGkVdy5aR3VNLSMGlHLTuScyc1J5h71/jw8QQI8PDs3ypZwi+SYsEADcMn8FtQ1NAFTV1HLL/BUAHRYk8iJAiIh0VwuWVrUJBHMeXU6vooKWtGa1DU3cuWidAkR3UVNTw0MPPcSXvvSljK674IILeOihhxgwYEB2MiYiGYlqzsk0PdPP+O7Ta9sEgrrGOHWN8dD3qK6p7bAyW0/aMGjy5MmeOpN6zZo1jB07Nkc5go0bN3LRRRexcuXKVulNTU3EYrEO/7xcl1ekK+momzS0bs4BKC2K8anTy3n09aq00++4bHzk56fWFAAKDOIZ3qLLB5TyypyPp32+mb3u7pPDjqkGkWVz5szh7bffZuLEiRQVFdG3b1+GDx/OsmXLWL16NTNnzmTz5s3U1dVx/fXXc/XVVwMHlw3Zu3cv559/PmeffTZ/+tOfKC8v5/HHH6e0tDTHJRPp2sKaZprb6IHIp/7Ua25+dDnFMQttznnw1U2k3r/bS//u02tDaxdfnH4c31u0rs1nxB0M2rwXwIDSIg40xtsEoeaA1hGyWoMws/OAHwIx4KfuPjfl+EDgAeA4oA74B3dfaWYjgV8Cw4A4cL+7//BQn3eoGsQ3n1jF6urdR1yuZONG9OffLz458nhyDeLFF1/kwgsvZOXKlS1DUXfs2MGgQYOora3ljDPO4KWXXmLw4MGtAsTxxx/PkiVLmDhxIldccQWXXHIJV111VejnqQYhPVkmNYJpc5+nKqS5pU9xjIYmp77pYBNNYYFx2qiBLNtU0yo9G6aOGcQbGX5OaVEstDYC4YEuEzmpQZhZDLgX+FugElhsZgvdfXXSaV8Hlrn7J83spMT5M4BG4Kvu/oaZ9QNeN7NnUq7tlqZMmdJqnsLdd9/NY489BsDmzZt56623GDx4cKtrxowZw8SJEwE4/fTT2bhxY2dlVyRtHdkWH/X+mYzaCQsOAPvqm9qkNcadJRt3ZNycU0DwBJsqZkZTyMN3SVEBr72zI7RGEPVe5YnfWdTvsCOHtabKZhPTFGC9u28AMLOHgUuB5Jv8OOAOAHdfa2ajzWyou78HvJdI32Nma4DylGsz1t6Tfmfp06dPy88vvvgizz77LH/+85/p3bs306dPD53x3atXr5afY7EYtbUd1wkl0hGibt5L3t3Rqi0+nWaeKHeGNME0N9skv9ew/iWMG9E/smkmintwMw4LLFHNOYfTB3HDb5aFfn6c8JpC8+8lm4EgSjbXYioHNie9rkykJXsTuAzAzKYAo4CK5BPMbDQwCXgt7EPM7GozW2JmS7riJLF+/fqxZ8+e0GO7du1i4MCB9O7dm7Vr1/Lqq692cu5Eoi1YWsW0uc8zZs5vmTb3eRYsrYo8N+rm/eCrm0LT/23BSm5+dDlVNbU4BwNHe58RVSOo3lXHV+e92fJe7+2u47m1Wxh1VG96Fba+xZUWxRjYO3wpmuYgVVoUa3PNbZeczB2Xjad8QClGEEjuuGw83545PqP0mZPKKR8Q3n/YfE7YNbmSzRpE2Kyt1IA+F/ihmS0DVgBLCZqXgjcw6ws8Ctzg7qGdB+5+P3A/BH0QR57tjjV48GCmTZvGKaecQmlpKUOHDm05dt5553HfffcxYcIETjzxRKZOnZrDnIoclG4H7/CyEj45qTzy5h31B7n3QGObtKgx/PG4873fr4vMq0Foc05Do/OdT01Ie0RScg0mk+acqKf7qPSbzj2x3c/PZUBIlbVOajP7MHCbu5+beH0LgLvfEXG+Ae8AE9x9t5kVAU8Ci9z9B+l8Zlcc5trZ8q28cmSi+geiOnjLSgqpb/I2tQKzoIkmVVRbfHsW/59P8Mr6bdy5aB1VNbWUFBZQ1xjnw8cOZummna3G/6c2ybTKE/DO3AtDj2V7iYpDyfXnJ8vVMNfFwAlmNgaoAmYBn03J2ABgv7vXA/8EvJwIDgb8DFiTbnAQkcw6iqHtUg03P7qcF9dtiawR7Kpr++QP0L9X28DRXlt8SVEBO/c3hL7XWXOfI+7QlOgxrmuMU1RgXDG5gs+cMbJNOZoDSaoREU05EP1031ly/fnpylqAcPdGM/sysIhgmOsD7r7KzK5JHL8PGAv80syaCDqg/zFx+TTg88CKRPMTwNfd/als5VekS1o+D567HXZVQlkFzLgVJlyR0do8UR3FJSFLNRxojLNgWTWFBUZjBkN6dtc18l+fmRganCaPGpR2M8+Nf3sCdz37FvtTRho1xJ3v/f6vvDLn46E31qgmGzkymkndw+RbebuTjJsVls+j8fGvUNh0cGRbY6yEpafezt8tHtXqhlhUYJhBfVPbv+dMR/MA3PWZiaE33agn/0xn70L072PMnN+G5rcrNxl1Z5pJLZJjC5ZW8cfH/oeH/DccXVzD9v1l3PXYLCBYoyvs5rb/d7fSu6mOHd6HTT6UKh9CdeNg3nptPWVNJYy0Wjb6MOopoqGdp/1Mg0P5gNLIzlrouKf1qGaWERFDTdttMoq9wsxet0NJJfSqgNitwBUZ56lDRdT+uhMFCJH2dNAf+etP/oRT4uu4svEbbKOMKQVrOctW8sxjP+e5+GktHa/NK3WufX83tvsjTC9YzpmxtfRhM1t9AOt8JH9qGkcfO8Aw28H0gjc5teBtPmSbubthJk/42W0+O6qjeEBpEZ9oeokbeJgRto1qP4q7mMXZ5wZBq7128mw+rbc3yifU8nnwxHXQkAgquzYHr6Fjb8hR/xfC0uHw8pTp/7csByE1MfUw+VbeSB3xh5N64wEoKoWL747sB5g5qZzFC3/MyDfu5GjfSjVD+VXFN1jwtrOLPny64EWOLXifB5tm8JaPjPzok20j9xX9F8NtO2/Ej2eY7WS47aDImnAPRg2lquYoZjTdm3ZH8S/PeJdJb97apgmr8NL/jr7xddITcEZNRv91SnADTlU2Mshzujf15rKlc8OH4P/CqZ+FNx9qnR4rBotBY0hHf9lIuHFlZp9x8d3Bz5mcn8G/U3tNTAoQXUzfvn3Zu3cv1dXVXHfddTzyyCNtzpk+fTrf+973mDy57b9pdytvVhzixh55Teof4HO3R954FkxfFPqU+80xq7j43bkU0cgTTVP5k5/CsfYeHytYxodsMwVJN/Zt3o/fNU3hnsaZ7KIvdfRiBNu4uOAVbiyazwHrRS+vo8QOjhyq9UJKrDFikpHx+KWr0l/u4sVzw8vXezCc/S/w/LegMWlm/2HcfLIuHofbB0Yfj/WCpgMHX0fd1AtL4BPfhKZ6eOHb0Jh0TawYCoqgYV/H5HnyP8KyB1v/bgtLgq+6mrbn9+oflCE5TwVFYAWty9asOQilSQGiG90wmwNEexQgDiHTJ0poG1AsBh4+vh5gWsljnLLrBUYXbGGP9+Z9BrE8PoaxtpkzY2uYVrCSCbaBmDlxD7qJC0Lu6k0YjR7M9t1Of4azEzN4xypYc+6vef6p3ySagLZT7YO5i1nc3udRete+F16+qBtDagCcPgcevzayfJEO96m8oyR/Rr9hwc1zW/Qkui7nEP+vOuhD4Laa9M9WJ3Xu3HzzzYwaNaplw6DbbrsNM+Pll19m586dNDQ08O1vf5tLL7201XXJq8DW1tYye/ZsVq9ezdixY/NvLaZMmgOmfjE8OADs2kzjgmspjNe3vG5a8CViBbHWT3NwyD/iH++/kV/YOTzQdAEn2btcFHuVOYW/5kMFwVIR7/kg3vWhjOF9CsyJe6L5Jqk5p8mKMG+ilwWfNYKdLceG9mpkzNQJ1PcazGcWzWj15N87dmrbgAZw/Izo319qe/jjX44uXN+hsPeD8GO7NgeBpeng75AnroNNr7Z+Kk9uc4eOCRyp5djzXvA1+iNQubhtbSf195MNUTf80kFBE1NqLfaiu+Cxf85unsoqDn1OmvIrQPxuDry/4tDnZWLYeDh/buThWbNmccMNN7QEiHnz5vH0009z44030r9/f7Zt28bUqVO55JJLIveU/tGPfkTv3r1Zvnw5y5cv57TTTuvYMnRl7XVAQttji76OE73OS0twSIjFG/B4Q+j50PqmfsAL+V//GFsLhvBJnuPO4vuZ6z8hZgdr4U1uxMwZbjtaZWKLDWHYpf/Z6kYZm3ErzP9C6Of2rgtu0OEdxYmba/N79R8RPEm//nPY/R58sAp2Vx28GT/7zZCbpUNxP/DGtjexc74d3bwGB4NDs4ZaWPIAbcZLNdTCb7/aunkk3cAR9lDwzK3hN/2dG+GS/06/iTDqpl428mAeU0Xd8MOaq4pK4fzvRJfv+W9n9hmFpVC7I/3zm2vFHSC/AkQOTJo0iS1btlBdXc3WrVsZOHAgw4cP58Ybb+Tll1+moKCAqqoqPvjgA4YNGxb6Hi+//DLXXRf8UU2YMIEJEyZ0ZhFy67nb294UGmrhqZsO/pxiN33YEy/hXxq+xCY/mn5WS19qGW7bmVawkqkFa+hn+4kRZxB7wqMJsL90OLfu+xRf8AUs92P5QeOneY/BHH90X/6m5i+M5oNWwQGgwJxaL6bUDt5Ea72YzaffxLAJV7R9co7s5zjEU2DqezU1wIOXw1uLDqbt2hw8rXrEvgP1e+Gy+6Nv0mH9OJFP5RFN1QdCllBrqIWF1wU36dSaSLPUwD//6ujP2FXZ9vcR9j7NZYi6qbfX8dveDf+YqdG/w7A8zbg1s884nDx1kPwKEO086WfT5ZdfziOPPML777/PrFmzePDBB9m6dSuvv/46RUVFjB49OnSZ72RRtYtO05Hty5m8V9RTbFhnXkI8DhfU38EBirmo4M9s9zJW+mhqKWZN0yi+33gFO+gPwFB2cGbBGqYXLOPjsWWUsY/NfjR/jo/lJ7svZX3TMB7hrFbvv/9AI6cNPAC72n62lY1k5XFfSYxi2sYWO4rNp9/EGZdENCtE3SwyfQqMFcH2t9ume5zIqXJlFdE31gkptZTDfSqPEjbCp6E2EQgIyW9zvTCiHGGiynCom3rUNcnvmfo5mfwttJevqM84nPM7QH4FiByZNWsWX/jCF9i2bRsvvfQS8+bN4+ijj6aoqIgXXniBd999t93rP/rRj/Lggw/ysY99jJUrV7J8+fJOynnC4Y4zz3R8OCQ1m5TDmLODkRphT8D9y4PV4fZUt0re6yVc1fB/2E1fAB6N/03LsbMLVvGfhT+ht9Xzng/iD03jeS4+id/Fz2RhfBoFjXEGsLcleER5b1cdXBh9Yz9jwhWQCAjDEl+RDnWzyMSuyogD3vbpP50g1BFP5VHNI5HaGzRzGOVoLwBG/Y4zveEfjsMJKjkYPaYA0QlOPvlk9uzZQ3l5OcOHD+dzn/scF198MZMnT2bixImcdNJJ7V7/xS9+kdmzZzNhwgQmTpzIlClTOinnCVHNPM/dntlEoYVfSQwXDHmvJ28Mmkmah+3troQ3H4Z+w4MbTPIQv6JS+MRtbT6j1ov5x8avscZHURwraLWlY2lRjL6nX8mtbxg3eDAq6KyCVfylYDwN9cGfQZyCQwYHSMzonXDhwd/Nkd7YO+qPv6wi8/kAh5NXSP+pHMIDSlTgaK8foCPLIWnRMNceJivlvW0AkU92F3wfnvm3tjeAWK92m4HS1r+Cxcdf17rJ5rSDTTbNk9LK4jXMbryZ1+JjuWvWJNyjN6VPdzXQqF3Ecr2JS6TDmf/RWfnKdFJYVyxHD6VhrnJk+o8IRsWEeeqrbdMaatvpzMyM765KLEz3w5a00sUx7hgZ5OeWlGNFMcM9801cwia93XZJsEVtt1kEriObqzo6X5m2qx/qmHQKBQhpX/1+KO7TNr2oFD56Mzx3W2bvVzqIxvr9bZd3KO4d2uTwAUeFbln5zSdW4U6bYw1NHrorWXsOZxexLitHbdWHJdf9AHJIeREg3D33o4A6QUbNhemMJGqsh3l/B9vegilXw7rftT1/yc8yGtO9eOwcfrNkU+vZwfFZTCkfxMXvzm0ZHlrrxfwxfgpzG2aFZj9qsxmA6ojNbtrTXTZwEelMPT5AlJSUsH37dgYPHtyjg4S7s337dkpKSg598qEmnzUPZWweMXLx3XD638MFd7Z9rwzHdN/w1FFU1Ve0GTr66F/hbr7LKQXvsMv7sMRPoqGd/55D+/UC4IM9bdeiaW9ZaBFJX48PEBUVFVRWVrJ169ZcZyXrSkpKqKhIY5p91KikJ2+EeMPBEUMNtVBQGNzwo2QwptvdqXoofFNAByo5msr40W2Ope47XFoU45YLgo547SQmkj09PkAUFRUxZsyYXGeja4mafFYfskhgvPHgcNYIC5qmceeBu6muq2VESSk3NZ3ITFov2Xx0v14M6lsc+R5RexaUJ400iuoo7jadyCLdTI8f5iopmhrhjorwmayRoleHXLC0KvQpPmwPAoDTRw1gVdXulg1y2ju/Sw8pFekh2hvmWtDZmZEsWD4vWOL6tgHB9+Xzws+LN8GCLwbBoaCo9bGi0qBjOUxZBQuWVjFt7vOMmfNbps19ngVLg2Gm3316begoo1+9uqlNOsD7uw4w91MTKB9QihHUEO64bDzfnjmeOy4b3yZdwUEkd1SD6O4ONTmqZbTSZijqE2x68vFvwIBj0p68tHj8NxNzEQ7e8AsLjJOG9WNldchibO1ob+N5Eel8OZsoZ2bnAT8EYsBP3X1uyvGBwAPAcUAd8A/uvjKdayUhqsP52W8GPyff8Bv2BZ3OA47JaPLSDU8dRW3KZzTGnbXv72nTgdwsqk9BI4xEuo+sNTGZWQy4FzgfGAdcaWbjUk77OrDM3ScAf0cQENK9Nv+ENSVFdTjvroQFX2obPJo7naNMuCLYley2GrhxJT7+06HLUAA0xZ07LhtPaVGsVXppUYwrzxwZmq4RRiLdRzZrEFOA9e6+AcDMHgYuBVYnnTMOuAPA3dea2WgzGwocm8a1+SVs7sKCa6LPL+4H9XvCj0Wu+tl25FFZ76LIc0cMKG13FvLkUYM0wkikG8tmgCgHkh9vK4EzU855E7gM+KOZTQFGARVpXguAmV0NXA1wzDHHdEjGu6SwpqR4U7AontF2tdOLftDuZjRhi9ZB63kFH+w5wAd7DjBl9ECWV+5qM/Ko+ZpM1z0Ske4hm6OYonZ9TDYXGGhmy4CvAEuBxjSvDRLd73f3ye4+eciQIUeQ3S4u6qm/qR4uuSexTLIF35s7qGfc2naSW1Epi4/7CrfMX0FVTS0OVNXUMufR5Xz9sRWh/QlVNXWhI4908xfp2bJZg6gERia9rgBa7e7i7ruB2QAWrIPxTuKr96GuzTt9jw7fSP4wdgUL63ROrh2kqq6pVW1AJA9lM0AsBk4wszFAFTAL+GzyCWY2ANjv7vXAPwEvu/tuMzvktXnlnT9A3W7abLl4mLuCVT3024w+XiOPRPJT1gKEuzea2ZeBRQRDVR9w91Vmdk3i+H3AWOCXZtZE0AH9j+1dm628djnJK632HgR1u2Dw8XD6bPjzPRmtkZ/c1zCsfwknDusXeW7UBjkaeSSSnzRRrqsJm/hmBXDhD2Dy7IzeKmwZDIDjhvShamdtm07nOy4bD2htI5F8oh3lupOw0Uoehz98P+MAceeidaGdznUNceZ+akLP2CBHRLJGASKXUjft+Zuboye+tTN3IUrUBDd1OotIOhQgciVs4tvCL0efX5bGPg9JVlbtosAgHtKCqE5nEUmHVnPNlbCmJAhmQIfMXTjkaKUkr7+7gyt/8iplpUX0Kmz9T6xOZxFJl2oQudLepj2X3X/o/aKTJI9UGtynmN0HGhlRVsKDX5jK4nd2qNNZRA6LAkRnSO5r6F8OHzonGJnkIZPT2pv4FiJ1pNK2ffUYMHvaGMoHlFKuvgYROUxqYsq25r6GXZsBD1ZZXfIA9D4KCnu1PjfDpiQIH6nkwP0vbziyfItI3lOAyLaovoZYcfQaShlob6SSiMiRUBNTNrm3s19DVUZNSdC6r2F4WQlTxkRsEYpGKonIkVOA6Cip/QzjL4d3Xo4+P8Nhq6l9DdW76liwrJryASVs31sfuRS3iMjhUhNTRwjrZ3jlLti5EU77eygsaX1+B/U1QFBJ0VLcIpINqkF0hKh+hqLecMndMPrsjIathonqa3hvV51mRYtIVihAdISoZTB2VwXfj6CvYcSAUi6bVE7MjKaQhRXV1yAi2aImpo4Q1Z+QYT8DHOxrSN7t7b9fWE/vXjHNihaRTqUA0RFm3AoFKZWxw+hngOi+hr69CvmO+hpEpBOpiakjjP80PPMN2L8DmhoOu58BoucvvK++BhHpZAoQHaFyCex5Hy75bzjt7w77bZriTt9ehew50NjmmPoaRKSzKUB0hKX/LxixdPInM7osuTN6aP8SykqL2HOgsU2HtPoaRCQXFCCOVP1+WDkfxs2EXtH7PadKnfj2/u463t9dx2cmj2TqsYP43u//qhVYRSSnFCCO1JonoH4PTPpcRpdFdUb/cf02vnP5BD55WuYjoEREOpJGMR2pZb+CgaNh1LSMLovqjNYieyLSVWQ1QJjZeWa2zszWm9mckONlZvaEmb1pZqvMbHbSsRsTaSvN7NdmVpJ6fc7t3BistzTxc2CW0aVD+4cXR53RItJVZC1AmFkMuBc4HxgHXGlm41JOuxZY7e6nAtOB75tZsZmVA9cBk939FCAGzMpWXg/bsl8DBqdemdFljU1xehfH2qSrM1pEupJs1iCmAOvdfYO71wMPA5emnONAPzMzoC+wA2ge41kIlJpZIdAbqM5iXjMXj8Oyh+DY6TBgZEaX/uCZv7Jh2z6uOvMYTXwTkS4rm53U5UDyZgiVwJkp59wDLCS4+fcDPuPucaDKzL4HbAJqgd+7+++zmNfMbfwD7NoEn/j3jC57cd0W/ufFt5l1xki+/cnxWcqciMiRy2YNIqxRPnW1uXOBZcAIYCJwj5n1N7OBBLWNMYljfczsqtAPMbvazJaY2ZKtW7d2VN6jLZ8H/3UK/PISwKCxLu1L39tVy7/Me5OThvXjtktOzl4eRUQ6QDZrEJVActtLBW2biWYDc93dgfVm9g5wEjAKeMfdtwKY2XzgLOBXqR/i7vcD9wNMnjy57XKnHal534eWpb0dnvrXYPvQdpbVWLC0iu8+vZbqXXUY8M8fPZaSorZ9ECIiXUk2axCLgRPMbIyZFRN0Mi9MOWcTMAPAzIYCJwIbEulTzax3on9iBrAmi3lNT9i+Dw21QXqE5glx1buCmoYDdz37FguWVmUxoyIiRy5rAcLdG4EvA4sIbu7z3H2VmV1jZtckTvsWcJaZrQCeA252923u/hrwCPAGsCKRz/uzlde0Re37EJVO+IS42oYm7ly0riNzJiLS4bI6k9rdnwKeSkm7L+nnauCciGv/HcisBzjbyioS24qGpEeI2glOE+JEpKvTTOpMHMa+D/1KwmOwJsSJSFenAJGJUy6HXv2hsAQwKBsJF98d2UG9fste9tc3EUuZZa0JcSLSHWixvkxs/APU7oBP/QzGX97uqfG48/X5K+hTHASD+17aoNVZRaRbUYDIxJu/hl5lcNKFhzx13pLN/GXjDr7zqfF85oxj+PyHR2c/fyIiHUhNTOk6sBdWL4STZwb9Du3YuucA//nUGs4cM4grJme2DIeISFehGkS61iyEhn0w8bOhh5N3hyspilHfFOc/LxuPZbjKq4hIV6EaRLqWPQQDx8DI1OWkDk6Gq6qpxQnmORiwonJXp2dTRKSjKECko2ZT0EF96pWh+z6ETYZrjLsmw4lIt6YAkY43fxN8PzV8SwrtDiciPZECxKG4B6OXRp0NA0eFnhI16U2T4USkO1OAOJTNf4Edb8PE6F3j/vWcD1GQ0vKkyXAi0t0pQBzKmw9BUW8Yl7oZ3kGlxTHiDmWlRdodTkR6DA1zjbJ8Hjx7G+yuCgLEut+FLqlRW9/Et55cw0nD+vHkV86mMKaYKyI9gwJEmNSNgRr2B6+hTZD4nxfXU1VTy7x//rCCg4j0KLqjhUlzY6CN2/bx45c2MHPiCKaMGdSJGRQRyT4FiDBpbAzk7nzziVUUFxbw9QvGdlLGREQ6j5qYwrSzMVDzkhrNGwFdOnEER/cv6eQMiohkn2oQYWbcCpbyqykqZfFxX2lZUqPZopXva39pEemRFCDCnHRhsHNccV+SNwa6YfUJbZbUqGuMa0kNEemR1MQUZv2z0FQPVz0KYz7aklz90G9DT9eSGiLSE6kGEWb1QigdBMec1SpZS2qISD5JK0CY2VQz65f0up+ZtV33uidoPAB/XRQ0M8VaV7BuOvdEilPmOmhJDRHpqdKtQfwI2Jv0el8irV1mdp6ZrTOz9WY2J+R4mZk9YWZvmtkqM5uddGyAmT1iZmvNbI2ZfTjNvB6ZDS9C/Z7QpTVmTirnIx86quW1ltQQkZ4s3T4Ic3dvfuHucTNr91oziwH3An8LVAKLzWyhu69OOu1aYLW7X2xmQ4B1Zvagu9cDPwSedvfLzawY6J1BuQ7f6oXQqz+M+ZvQw9v2HOCM0QP532vOCj0uItJTpFuD2GBm15lZUeLremDDIa6ZAqx39w2JG/7DQOpjuQP9LNiXsy+wA2g0s/7AR4GfAbh7vbvXpJnXw9fUAOt+Cx86DwqL2xzeVdvAiqpdnHXcUSEXi4j0LOkGiGuAs4AqgtrAmcDVh7imHEiebVaZSEt2DzAWqAZWANe7exw4FtgK/F8zW2pmPzWzPmEfYmZXm9kSM1uydevWNIsT4d1XoHYnjLsk9PBrG7YTdzjruMFH9jkiIt1AWgHC3be4+yx3P9rdh7r7Z919yyEua7s3Z1BjSHYusAwYAUwE7knUHgqB04Afufskgj6PNn0Yibzd7+6T3X3ykCFD0ilOtNULg5Vbj5sRevhPb2+npKiASccMPLLPERHpBtLqgzCz/0vbmzvu/g/tXFYJjEx6XUFQU0g2G5ib6N9Yb2bvACcBm4BKd38tcd4jRASIDhOPw9on4fhPQHF4d8ef3t7GGaMHUVyo0cEi0vOle6d7Evht4us5oD+tRzWFWQycYGZjEp3Ms4CFKedsAmYAmNlQ4ERgg7u/D2w2s+bxozOA1WTT5tdg7weRGwNt2VPHXz/Yy7Tj1f8gIvkhrRqEuz+a/NrMfg08e4hrGs3sy8AiIAY84O6rzOyaxPH7gG8BPzezFQRNUje7+7bEW3wFeDARXDYQ1DayZ80TECuGE84JPfznt7cDME0d1CKSJw53qY0TgGMOdZK7PwU8lZJ2X9LP1UDoHdndlwGTDzN/mXEPAsSxH4OS/qGn/Gn9dvqXFDJuRPhxEZGeJt0+iD0c7INw4APga9nKVKdaPg8W/Rvs+yCYILd8XujWoq+8vY0PHzeYWEFY37uISM+TbhNTPzMbRFBzaN78oE2ndbeTurVo7c7QrUU379hP5c5avvCRY3OQSRGR3Eh3LaZ/Al4CngZuS/revaW5tegr64NukWnHa/6DiOSPdEcxXQ+cAbzr7h8DJhFMZOve0thaFOCVt7dzdL9eHDekbydkSkSka0g3QNS5ex2AmfVy97UEQ1K7t7KKQ6a7O39+extnHTeYYEUQEZH8kG6AqDSzAcAC4Bkze5y2k966nxm3QlHKXg5FpUF6wl8/2Mu2vfWcpfkPIpJn0u2k/mTix9vM7AWgjKAfontr7oh+7vagWamsIggOSR3Uf3o76H/Q+ksikm8yngfh7i9lIyM5M+GK0GGtzV5Zv51Rg3tTMbBzVhsXEekqtKhQOxqb4ry2YbtqDyKSlxQgIixYWsWH73iePQcaeXrl+yxYWpXrLImIdKrDXWqjR1uwtIpb5q+gtqEJgJ37G7hl/goAbS8qInlDNYgQdy5a1xIcmtU2NHHnonU5ypGISOdTgAhRXVObUbqISE+kABFixIDSjNJFRHoiBYgQN517IqVFsVZppUUxbjq3+08eFxFJlzqpQ8ycVE59Y5yvPbocgPIBpdx07onqoBaRvKIAEeEjHwqW1rjjsvFcOeWQeyOJiPQ4amKKsHNfAwADexflOCciIrmhABFh5/56AAb0Ls5xTkREckMBIkJzgBioACEieUoBIsLO/WpiEpH8ltUAYWbnmdk6M1tvZnNCjpeZ2RNm9qaZrTKz2SnHY2a21MyezGY+w9TsUxOTiOS3rAUIM4sB9wLnA+OAK81sXMpp1wKr3f1UYDrwfTNLviNfD6zJVh7bs3N/A317FVJcqEqWiOSnbN79pgDr3X2Du9cDDwOXppzjQD8L9vLsC+wAGgHMrAK4EPhpFvMYqWZ/PQPUvCQieSybAaIc2Jz0ujKRluweYCzB9qUrgOvdPZ44dhfwNSBOO8zsajNbYmZLtm7d2hH5BoJOanVQi0g+y2aAsJA0T3l9LrAMGAFMBO4xs/5mdhGwxd1fP9SHuPv97j7Z3ScPGTLkCLN80M79DapBiEhey2aAqARGJr2uIKgpJJsNzPfAeuAd4CRgGnCJmW0kaJr6uJn9Kot5baNGNQgRyXPZDBCLgRPMbEyi43kWsDDlnE3ADAAzGwqcCGxw91vcvcLdRyeue97dr8piXtvYsa9eQ1xFJK9lbS0md280sy8Di4AY8IC7rzKzaxLH7wO+BfzczFYQNEnd7O7bspWndDU2xdld16ghriKS17K6WJ+7PwU8lZJ2X9LP1cA5h3iPF4EXs5C9SLtqNUlORESD/EO0zKLuoxqEiOQvBYgQNVqHSUREASLMwXWYFCBEJH8pQIQ4uNS3+iBEJH8pQIRoaWJSH4SI5DEFiBA79zdQFDP6FMdynRURkZxRgAixc189A3oXE6whKCKSnxQgQgQL9an/QUTymwJEiGChPvU/iEh+U4AIUaMahIiIAkSYnfsbNAdCRPKeAkQKdw9qEBriKiJ5TgEixb76JhqaXE1MIpL3FCBS7NzXPItaNQgRyW8KEClqtA6TiAigANHGzpaVXNXEJCL5TQEixcGF+lSDEJH8pgCRorkPQjUIEcl3ChApmveCKCtVgBCR/KYAkaJmfz39SwopjOlXIyL5TXfBFDv3N2iSnIgIWQ4QZnaema0zs/VmNifkeJmZPWFmb5rZKjObnUgfaWYvmNmaRPr12cxnsmAlVwUIEZGsBQgziwH3AucD44ArzWxcymnXAqvd/VRgOvB9MysGGoGvuvtYYCpwbci1WVGzv0Ed1CIiZLcGMQVY7+4b3L0eeBi4NOUcB/pZsDNPX2AH0Oju77n7GwDuvgdYA5RnMa8tVIMQEQlkM0CUA5uTXlfS9iZ/DzAWqAZWANe7ezz5BDMbDUwCXgv7EDO72syWmNmSrVu3HnGma7QXhIgIkN0AEbZfp6e8PhdYBowAJgL3mFn/ljcw6ws8Ctzg7rvDPsTd73f3ye4+eciQIUeU4frGOHsPNKqJSUSE7AaISmBk0usKgppCstnAfA+sB94BTgIwsyKC4PCgu8/PYj5b1DTPotYoJhGRrAaIxcAJZjYm0fE8C1iYcs4mYAaAmQ0FTgQ2JPokfgascfcfZDGPrexsWahPNQgRkawFCHdvBL4MLCLoZJ7n7qvM7BozuyZx2reAs8xsBfAccLO7bwOmAZ8HPm5myxJfF2Qrr80OLtSnGoSISGE239zdnwKeSkm7L+nnauCckOv+SHgfRla1NDGpBiEiopnUyXZqLwgRkRYKEEnUxCQicpACRJKa/Q2UFBVQWhzLdVZERHJOASLJzn2aRS0i0kwBIsnO/fWaRS0ikqAAkWSnFuoTEWmhAJFEC/WJiBykAJEkWKhPNQgREVCAaBGPOzWqQYiItFCASNhT10jcNYtaRKSZAkSCJsmJiLSmAJHQHCAGaalvERFAAaJFTWIdJjUxiYgEFCAS1MQkItKaAkTCjn0KECIiyRQgEmr2N1Bg0K8kq1tkiIh0GwoQCc3rMBUUdPo+RSIiXZICRIJmUYuItKYAkaB1mEREWlOASNBKriIirSlAJNRoLwgRkVayGiDM7DwzW2dm681sTsjxMjN7wszeNLNVZjY73Ws7WtDEpBqEiEizrAUIM4sB9wLnA+OAK81sXMpp1wKr3f1UYDrwfTMrTvPaDlNb30RdQ5yBWmZDRKRFNmsQU4D17r7B3euBh4FLU85xoJ+ZGdAX2AE0pnlth9EsahGRtrIZIMqBzUmvKxNpye4BxgLVwArgenePp3ktAGZ2tZktMbMlW7duPayMHgwQamISEWmWzQARNuPMU16fCywDRgATgXvMrH+a1waJ7ve7+2R3nzxkyJDDyujBhfpUgxARaZbNAFEJjEx6XUFQU0g2G5jvgfXAO8BJaV7bYdTEJCLSVjYDxGLgBDMbY2bFwCxgYco5m4AZAGY2FDgR2JDmtR1mZ6IGoSYmEZGDsrYynbs3mtmXgUVADHjA3VeZ2TWJ4/cB3wJ+bmYrCJqVbnb3bQBh12YrrzWJlVzVxCQiclBWly5196eAp1LS7kv6uRo4J91rs2Xn/gb6FMcoLtS8QRGRZrojolnUIiJhFCBIzKLuo/4HEZFkeR8gFiyt4o/rt7GyajfT5j7PgqVVuc6SiEiXkNcBYsHSKm6Zv4KGpmCKRVVNLbfMX6EgISJCngeIOxeto7ahqVVabUMTdy5al6MciYh0HXkdIKprajNKFxHJJ3kdIEYMKM0oXUQkn+R1gLjp3BMpLYq1SistinHTuSfmKEciIl1HVifKdXUzJwULxN65aB3VNbWMGFDKTeee2JIuIpLP8jpAQBAkFBBERNrK6yYmERGJpgAhIiKhFCBERCSUAoSIiIRSgBARkVDmHrrVc7dkZluBdw9x2lHAtk7ITlejcucXlTu/HEm5R7n7kLADPSpApMPMlrj75Fzno7Op3PlF5c4v2Sq3mphERCSUAoSIiITKxwBxf64zkCMqd35RufNLVsqdd30QIiKSnnysQYiISBoUIEREJFTeBAgzO8/M1pnZejObk+v8ZIuZPWBmW8xsZVLaIDN7xszeSnwfmMs8ZoOZjTSzF8xsjZmtMrPrE+k9uuxmVmJmfzGzNxPl/mYivUeXu5mZxcxsqZk9mXidL+XeaGYrzGyZmS1JpHV42fMiQJhZDLgXOB8YB1xpZuNym6us+TlwXkraHOA5dz8BeC7xuqdpBL7q7mOBqcC1iX/jnl72A8DH3f1UYCJwnplNpeeXu9n1wJqk1/lSboCPufvEpPkPHV72vAgQwBRgvbtvcPd64GHg0hznKSvc/WVgR0rypcAvEj//ApjZmXnqDO7+nru/kfh5D8FNo5weXnYP7E28LEp8OT283ABmVgFcCPw0KbnHl7sdHV72fAkQ5cDmpNeVibR8MdTd34PgRgocneP8ZJWZjQYmAa+RB2VPNLMsA7YAz7h7XpQbuAv4GhBPSsuHckPwEPB7M3vdzK5OpHV42fNlRzkLSdP43h7IzPoCjwI3uPtus7B/+p7F3ZuAiWY2AHjMzE7JcZayzswuAra4++tmNj3H2cmFae5ebWZHA8+Y2dpsfEi+1CAqgZFJryuA6hzlJRc+MLPhAInvW3Kcn6wwsyKC4PCgu89PJOdF2QHcvQZ4kaAPqqeXexpwiZltJGgy/riZ/YqeX24A3L068X0L8BhBM3qHlz1fAsRi4AQzG2NmxcAsYGGO89SZFgJ/n/j574HHc5iXrLCgqvAzYI27/yDpUI8uu5kNSdQcMLNS4BPAWnp4ud39FnevcPfRBH/Pz7v7VfTwcgOYWR8z69f8M3AOsJIslD1vZlKb2QUEbZYx4AF3/4/c5ig7zOzXwHSC5X8/AP4dWADMA44BNgGfdvfUjuxuzczOBv4ArOBgm/TXCfohemzZzWwCQYdkjOCBb567325mg+nB5U6WaGL6V3e/KB/KbWbHEtQaIOgmeMjd/yMbZc+bACEiIpnJlyYmERHJkAKEiIiEUoAQEZFQChAiIhJKAUJEREIpQIh0AWY2vXlFUpGuQgFCRERCKUCIZMDMrkrsv7DMzH6cWChvr5l938zeMLPnzGxI4tyJZvaqmS03s8ea1+c3s+PN7NnEHg5vmNlxibfva2aPmNlaM3vQ8mEhKenSFCBE0mRmY4HPECyUNhFoAj4H9AHecPfTgJcIZq8D/BK42d0nEMzwbk5/ELg3sYfDWcB7ifRJwA0Ee5YcS7DekEjO5MtqriIdYQZwOrA48XBfSrAgWhz4TeKcXwHzzawMGODuLyXSfwH8b2INnXJ3fwzA3esAEu/3F3evTLxeBowG/pj1UolEUIAQSZ8Bv3D3W1olmn0j5bz21q9pr9noQNLPTejvU3JMTUwi6XsOuDyxBn/zHsCjCP6OLk+c81ngj+6+C9hpZh9JpH8eeMnddwOVZjYz8R69zKx3ZxZCJF16QhFJk7uvNrN/I9jJqwBoAK4F9gEnm9nrwC6CfgoIlly+LxEANgCzE+mfB35sZrcn3uPTnVgMkbRpNVeRI2Rme929b67zIdLR1MQkIiKhVIMQEZFQqkGIiEgoBQgREQmlACEiIqEUIEREJJQChIiIhPr/ApgdADBf7YcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "epoch_list = np.arange(1, N_EPOCHS+1, 1)\n",
    "\n",
    "plt.plot(epoch_list, train_scores, label=\"train\")\n",
    "plt.plot(epoch_list, valid_scores, label=\"valid\");\n",
    "plt.scatter(epoch_list, train_scores)\n",
    "plt.scatter(epoch_list, valid_scores);\n",
    "plt.legend();\n",
    "plt.ylabel(\"auc\")\n",
    "plt.xlabel(\"epoch\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "magnetic-campaign",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA43UlEQVR4nO3deXhU5dn48e89kxVICEuCEFYRUdEIGJD3VSuuuAOKitalttWq1VbaouCvtVRbRbFqfeuGy1vfFkVcAMUFFbUUl5awCIgiqChJkD2sk23m/v1xJmEyOTOZgZms9+e6uCbznOec85wAc895lvuIqmKMMcbEytPUDTDGGNOyWOAwxhgTFwscxhhj4mKBwxhjTFwscBhjjIlLSlM3oDF07dpV+/bt29TNMMaYFmXJkiVbVTU3vLxNBI6+fftSVFTU1M0wxpgWRUS+dSu3ripjjDFxscBhjDEmLhY4jDHGxKVNjHEYY0y8qqqqKC4upry8vKmbknQZGRn07NmT1NTUmOpb4DDGGBfFxcVkZWXRt29fRKSpm5M0qsq2bdsoLi6mX79+Me1jgSOCOctKmDZ/DaVlPnrkZDJx1EDGDMlv6mYZYxpJeXl5qw8aACJCly5d2LJlS8z7WOBwMWdZCZNfWYmvyg9ASZmPya+sBLDgYUwb0tqDRo14r9MGx11Mm7+mNmjU8FX5mTZ/TRO1yBhjmg8LHC5Ky3xxlRtjTKKVlZXx6KOPxr3fOeecQ1lZWeIbFMICh4seOZlxlRtjTKJFChx+v9+l9n5vvPEGOTk5SWqVwwKHi4mjBpKZ6q1TlpnqZeKogU3UImNMWzNp0iS++uorBg8ezLBhwzjllFO4/PLLOeaYYwAYM2YMxx13HIMGDWL69Om1+/Xt25etW7eyfv16jjzySK699loGDRrEmWeeic+XmF4TGxx3UTMAfte81WzbW0nXDmn89tyjbGDcmDbqD699xurSXQk95lE9svn9+YMibp86dSqrVq1i+fLlfPDBB5x77rmsWrWqdsrsM888Q+fOnfH5fAwbNoyLLrqILl261DnG2rVref7553nyySe55JJLePnll7niiisOuu1JveMQkbNEZI2IrBORSS7bR4rIThFZHvxzR0P7ikhnEXlHRNYGXzslo+1jhuQz49rjAZhywSALGsaYJjV8+PA66ywefvhhjj32WEaMGMGGDRtYu3ZtvX369evH4MGDATjuuONYv359QtqStDsOEfECjwBnAMXAYhF5VVVXh1X9l6qeF8e+k4AFqjo1GFAmAbcl4xrysjIA2LyrIhmHN8a0ENHuDBpL+/bta3/+4IMPePfdd/n4449p164dI0eOdF3hnp6eXvuz1+tNWFdVMu84hgPrVPVrVa0EZgKjE7DvaODZ4M/PAmMS1+S6OrVLJdUrbN5tgcMY07iysrLYvXu367adO3fSqVMn2rVrxxdffMEnn3zSqG1L5hhHPrAh5H0xcLxLvf8SkU+BUuA3qvpZA/t2U9WNAKq6UUTy3E4uItcB1wH07t37gC5ARMjtkM7m3a0/V40xpnnp0qULJ5xwAkcffTSZmZl069atdttZZ53F448/TkFBAQMHDmTEiBGN2rZkBg63pYga9n4p0EdV94jIOcAcYECM+0alqtOB6QCFhYVx7RsqNyudLXbHYYxpAs8995xreXp6Om+++abrtppxjK5du7Jq1ara8t/85jcJa1cyu6qKgV4h73vi3FXUUtVdqron+PMbQKqIdG1g300i0h0g+Lo5Oc135GZl2BiHMcaESGbgWAwMEJF+IpIGjAdeDa0gIodIMEmKiAwPtmdbA/u+Clwd/PlqYG4Sr4G8bOuqMsaYUEnrqlLVahG5CZgPeIFnVPUzEbk+uP1xYBxwg4hUAz5gvKoq4Lpv8NBTgVki8hPgO+DiZF0DQF5WOjv2VVFZHSAtxdZLGmNMUhcABruf3ggrezzk578Cf41132D5NuC0xLY0spopuVv3VFjKEWOMwVKONCgvy5kHbVNyjTHGYYGjAXnZwcCxy8Y5jDEGLHA0qHb1uN1xGGOasQ4dOgBQWlrKuHHjXOuMHDmSoqKigz6XBY4GdO2QhogFDmNMy9CjRw9eeumlpJ7DsuM2IMXroUv7NLbYlFxjTCO67bbb6NOnDzfeeCMAU6ZMQURYuHAhO3bsoKqqij/+8Y+MHl03k9P69es577zzWLVqFT6fj2uuuYbVq1dz5JFHWlr1xtS1Q7otAjSmLXtzEny/MrHHPOQYOHtqxM3jx4/nlltuqQ0cs2bN4q233mLChAlkZ2ezdetWRowYwQUXXBDxmeGPPfYY7dq1Y8WKFaxYsYKhQ4cmpOkWOGKQl51hXVXGmEY1ZMgQNm/eTGlpKVu2bKFTp050796dCRMmsHDhQjweDyUlJWzatIlDDjnE9RgLFy7kF7/4BQAFBQUUFBQkpG0WOGKQl5XOmu8T+xAXY0wLEuXOIJnGjRvHSy+9xPfff8/48eOZMWMGW7ZsYcmSJaSmptK3b1/XdOqhIt2NHAwbHI9BXlY6W/dU4g8ccK5EY4yJ2/jx45k5cyYvvfQS48aNY+fOneTl5ZGamsr777/Pt99+G3X/H/zgB8yYMQOAVatWsWLFioS0ywJHDPKy0vEHlB37Kpu6KcaYNmTQoEHs3r2b/Px8unfvzg9/+EOKioooLCxkxowZHHHEEVH3v+GGG9izZw8FBQXcd999DB8+PCHtsq6qGORl738SYNcO6Q3UNsaYxFm5cv+gfNeuXfn4449d6+3ZsweAvn371qZTz8zMZObMmQlvk91xxGB/2hGbkmuMMRY4YmCrx40xZj8LHDGoyVdlTwI0pm1xnvLQ+sV7nRY4YpCR6iUrI8USHRrThmRkZLBt27ZWHzxUlW3btpGRkRHzPjY4HqO8rHTrqjKmDenZsyfFxcVs2bKlqZuSdBkZGfTs2TPm+hY4YpRrgcOYNiU1NZV+/fo1dTOaJeuqilFeVobNqjLGGCxwxCwvy0l02Nr7O40xpiFJDRwicpaIrBGRdSIyKUq9YSLiF5FxwfcDRWR5yJ9dInJLcNsUESkJ2XZOMq+hRl52OhXVAXZXVDfG6YwxptlK2hiHiHiBR4AzgGJgsYi8qqqrXerdC8yvKVPVNcDgkO0lwOyQ3R5U1fuT1XY3tWs5dlWQnZHamKc2xphmJZl3HMOBdar6tapWAjOB0S71bgZeBjZHOM5pwFeqGj2bV5LZ6nFjjHEkM3DkAxtC3hcHy2qJSD4wFng8ynHGA8+Hld0kIitE5BkR6eS2k4hcJyJFIlKUiOl0tgjQGGMcyQwcbkngw0eWHwJuU1W/6wFE0oALgBdDih8D+uN0ZW0E/uy2r6pOV9VCVS3Mzc2Nr+UuckO6qowxpi1L5jqOYqBXyPueQGlYnUJgZvBBI12Bc0SkWlXnBLefDSxV1U01O4T+LCJPAvMS3/T6sjNSSE/xWFeVMabNS2bgWAwMEJF+OIPb44HLQyuoau3qGhH5GzAvJGgAXEZYN5WIdFfVjcG3Y4FVCW+5CxEhL9sWARpjTNICh6pWi8hNOLOlvMAzqvqZiFwf3B5tXAMRaYczI+tnYZvuE5HBON1e6122J01uh3TrqjLGtHlJTTmiqm8Ab4SVuQYMVf1R2Pt9QBeXelcmsIlxycvKYO3m3U11emOMaRZs5XgcrKvKGGMscMQlLyud3eXVlFe5TgIzxpg2wQJHHGpWj9taDmNMW2aBIw652bZ63BhjLHDEoTbtiM2sMsa0YRY44lCb6NC6qowxbZgFjjh0aZ+G1yPWVWWMadMscMTB4xG6dkizripjTJtmgSNOziNkLXAYY9ouCxxxys2yRYDGmLbNAkec8rLS2WJjHMaYNswCR5zystLZtreSan+gqZtijDFNwgJHnHKzM1CFbXsrm7opxhjTJCxwxMkWARpj2joLHHGqDRw2zmGMaaMscMQpL9tWjxtj2jYLHHHK7WBdVcaYts0CR5zSUjx0apdqXVXGmDbLAscBsNXjxpi2LKmBQ0TOEpE1IrJORCZFqTdMRPwiMi6kbL2IrBSR5SJSFFLeWUTeEZG1wddOybwGN/YIWWNMW5a0wCEiXuAR4GzgKOAyETkqQr17gfkuhzlFVQeramFI2SRggaoOABYE3zeaOctKKFq/g083lHHC1PeYs6ykMU9vjDFNLpl3HMOBdar6tapWAjOB0S71bgZeBjbHeNzRwLPBn58FxhxkO2M2Z1kJk19ZiS/4zPGSMh+TX1lpwcMY06YkM3DkAxtC3hcHy2qJSD4wFnjcZX8F3haRJSJyXUh5N1XdCBB8zXM7uYhcJyJFIlK0ZcuWg7iM/abNX1MbNGr4qvxMm78mIcc3xpiWIJmBQ1zKNOz9Q8Btqup3qXuCqg7F6er6uYj8IJ6Tq+p0VS1U1cLc3Nx4do2otMwXV7kxxrRGyQwcxUCvkPc9gdKwOoXATBFZD4wDHhWRMQCqWhp83QzMxun6AtgkIt0Bgq+xdnEdtB45mXGVG2NMa5TMwLEYGCAi/UQkDRgPvBpaQVX7qWpfVe0LvATcqKpzRKS9iGQBiEh74ExgVXC3V4Grgz9fDcxN4jXUMXHUQDJTvXXKMlO9TBw1sLGaYIwxTS4lWQdW1WoRuQlntpQXeEZVPxOR64Pb3cY1anQDZotITRufU9W3gtumArNE5CfAd8DFybqGcGOGOEM00+avoSTYPXXrWQNry40xpi0Q1fBhh9ansLBQi4qKGq4Yhy837ebMBxcy9cJjGD+8d0KPbYwxzYGILAlbDgHYyvEDNiCvA4dkZ/DPLxMzY8sYY1oKCxwHSEQ4+fBcFq3dSpU9DdAY04ZY4DgIJw/MZXdFNcs3lDV1U4wxptFY4DgIJxzWFa9H+Oca664yxrQdFjgOQsfMVIb0yrFxDmNMm2KB4yCdfHguK0t2snWPZcs1xrQNFjgiWTELHjwapuQ4rytmuVY7eaCTzmTR2q2N2DhjjGk6FjjcrJgFr/0Cdm4A1Hl97ReuwePoHh3p3D7NuquMMW2GBQ43C+6EqrDEhVU+pzyMxyOcNKArC7/cQiDQ+hdTGmOMBQ43O4vjKj/58Fy27a3ks9JdSWyUMcY0DxY43HTsGVf5SQOccY5/ftloiXqNMabJWOBwc9odkBqWKt3jdcpd5Galc3R+Ngu/tAFyY0zrZ4HDTcElcP7D0LEXIJDWAQIB6Nw/4i4nH57Lku92sKu8qvHaaYwxTcACRyQFl8CEVTClDH71OWQdAq9PgIDbwwrh5MPz8AeUj9bZXYcxpnWzwBGLjGwYdTds/BQWP+1aZUjvHNJTPPxq1qf0m/Q6J0x9jznLShq5ocYYk3wWOGI1aCwcegq8dxfs/r7e5tdXbKTKH2BfpR8FSsp8TH5lpQUPY0yrY4EjViJw7p+hah88PKTeivJp89cQvozDV+Vn2vw1jd9WY4xJoqQ9OrZVKlnivFbtc15rVpQDpWXtXXcpLfO5lhtjTEtldxzxWHAnBKrrlgVXlPfIyXTdJVK5Mca0VEkNHCJyloisEZF1IjIpSr1hIuIXkXHB971E5H0R+VxEPhORX4bUnSIiJSKyPPjnnGReQx1RVpRPHDWQzFRvneL0FA8TRw1shIYZY0zjSVpXlYh4gUeAM4BiYLGIvKqqq13q3QvMDymuBn6tqktFJAtYIiLvhOz7oKren6y2R9SxZzDxYf3yMUPyAWeso6TMh0ec53WcOahbIzfSGGOSK5l3HMOBdar6tapWAjOB0S71bgZeBmrzdajqRlVdGvx5N/A5kJ/EtsbGbUV5SnrtivIxQ/L5cNKprJ96Lv/4yfFs2VPBna+tdjmQMca0XMkMHPlA6NfzYsI+/EUkHxgLPB7pICLSFxgC/Duk+CYRWSEiz4hIpwj7XSciRSJStGVLglKe11lRDiCQP8wpD/Pfh3XlhpP7M3PxBl5fsTEx5zfGmGYgmYFDXMrC844/BNymqq7LsUWkA87dyC2qWpN69jGgPzAY2Aj82W1fVZ2uqoWqWpibmxt/6yOpXVG+E477EZQUwb7trlUnnHE4g3vl8OsXl3P83e/awkBjTKsQU+AQkV+KSLY4nhaRpSJyZgO7FQO9Qt73BErD6hQCM0VkPTAOeFRExgTPmYoTNGao6is1O6jqJlX1q2oAeBKnS6xpDL8Wqsth2d9dN6d6PZxX0J3yqgCbdlXYwkBjTKsQ6x3Hj4Pf+M8EcoFrgKkN7LMYGCAi/UQkDRgPvBpaQVX7qWpfVe0LvATcqKpzRESAp4HPVfWB0H1EpHvI27HAqhivIfG6DYI+J8LipyLmsPrfD9fXK7OFgcaYlizWwFHT7XQO8L+q+inuXVG1VLUauAlnttTnwCxV/UxErheR6xs43wnAlcCpLtNu7xORlSKyAjgFmBDjNSTH8Guh7DtY+7br5kgLAG1hoDGmpYp1Ou4SEXkb6AdMDk6RDTS0k6q+AbwRVuY6EK6qPwr5eRERApOqXhljmxvHEedCVg/4z3QYeHa9zT1yMilxCRK2MNAY01LFesfxE2ASMExV9wGpON1VxpsKhT+Gr96DrWvrbXZbGJhhCwONMS1YrIHjv4A1qlomIlcAvwV2Jq9ZLcxxV4Mn1RnrCDNmSD73XHgM+SF3GKcf1a12waAxxrQ0sQaOx4B9InIscCvwLfB/SWtVS9Mhz0m7vvw5qNhdb3PowsCRA3P5cN1W9lZUuxzIGGOav1gDR7WqKs7K77+o6l+ArOQ1qwUafh1U7HJSrYelXA/1i9MGsGNfFX//5NvGb6MxxiRArIFjt4hMxpnp9Howv1Rq8prVAm3/2nlmR3kZoPtTrocFj6G9O3HSgK48ufBr9lXaXYcxpuWJNXBcClTgrOf4Hid1yLSktaoleu8u0LCF8cGU6+FuOX0A2/ZW8g+76zDGtEAxBY5gsJgBdBSR84ByVbUxjlBRUq6HO65PZ048rCvTF36Nr9J94aAxxjRXsaYcuQT4D3AxcAnw75pnZ5igjj3jKv/l6QPYuqeSEXcvsBxWxpgWJdauqv+Hs4bjalW9Cic/1O+S16wWyC3lempmbcr1cCU7nGd27CyvshxWxpgWJdbA4VHVzSHvt8Wxb9sQnnJdvHDeQ64p18F54FMgbEjEclgZY1qCWFOOvCUi84Hng+8vJSyViMEJEgWXwOq5MOsqyMiJWNVyWBljWqpYB8cnAtOBAuBYYLqq3pbMhrVoA8+BDodA0TMRq0TKVWU5rIwxzV3M3U2q+rKq/kpVJ6jq7GQ2qsXzpsLQq5yMuTvcp9y65bBK8YjlsDLGNHtRA4eI7BaRXS5/dovIrmj7tnlDr3IWBC591nVzaA4rATJTvfhV6dnJ7jiMMc2baPiitVaosLBQi4qKGv/Ez42HkiUw4TNISYtadXd5Fec+vAh/QHnjlyfRMdMW5htjmpaILFHVwvBymxmVTMN+Ans3wxfzGqyalZHKX8YPZtOucm6fvZK2ENCNMS1TrLOqzIHofyrk9HYGyY++sMHqQ3p3YsIZhzNt/ho+XLuVnb4qeuRkMnHUQEvDboxpNuyOI5k8XjjuGlj/L9jyZUy7dM/OwCNQ5rOFgcaY5skCR7INucJZDPjkKVHTrdf48ztf2sJAY0yzltTAISJnicgaEVknIpOi1BsmIv7Q/FeR9hWRziLyjoisDb52SuY1HLSvP3BeK/cQLd16DVsYaIxp7pIWOILP7HgEOBs4CrhMRI6KUO9eYH6M+04CFqjqAGBB8H3zteBO0LAMuBHSrUPkBYB5WemJbpkxxhyQZN5xDAfWqerXqloJzMR5gmC4m4GXgc0x7jsaqFkc8SwwJgltT5w40q2D+8JAcLqrPt9oS2eMMU0vmbOq8oENIe+LgeNDK4hIPjAWOBUYFuO+3VR1I4CqbhSRPLeTi8h1wHUAvXv3PvCrOFgdezrdU27lLmpmT02bv4bSMh89cjK5ckQf/vbRei589CPap3vZtqfSZlsZY5pMMgOHuJSFL054CLhNVf0idarHsm9UqjodJ78WhYWFTbco4rQ7nDGNqpAxCvHAyMkRdxkzJL9eQEhL8XDXvNX4qpxur5rZVjX1jTGmsSSzq6oY6BXyvidQGlanEJgpIuuBccCjIjKmgX03iUh3gOBraBdX81Mn3bpAZhfQgDNFN45Ffk8v+qZe5LTZVsaYppDMO47FwAAR6QeUAOOBy0MrqGq/mp9F5G/APFWdIyIpUfZ9FbgamBp8nZvEa0iMmnTrNT6YCh/cA7lHwIm3xHSISLOqSmy2lTGmkSUtcKhqtYjchDNbygs8o6qficj1we2Px7tvcPNUYJaI/AT4Dudxti3LybfBli/g3Snw0f/Avm3OmMdpd0R88FOPnEzXICECb67cSEV1oM64iI1/GGOSxZIcNpWlf4fXbq7bXZWa6XRruQSPOctKmPzKytoxDoD0FA/dsjP4bvs+vCL4Q46VmerlnguPseBhjDlgluSwufnnvfXHOKKs7whPw56fk8m9FxWw4Ncn0yE9pU7QABv/MMYkjyU5bCpxru8A99lWAHsrql3r22pzY0wy2B1HU4mwjoPM+DOoRFpt3r1jRtzHMsaYhljgaCqn3eGMaYQSD/i2w8s/dZIhxpAUESKvNs/LzmDW4g2cMPU9+k16nROmvmdZdo0xB826qppKzQD4gjud7qmOPZ1FgYufgpUv7q9XkxSxRmj94Cys8NXm3TtmcFyfTsxbsZFPi8tqh1Js0aAxJhFsVlVz88Ag2OUyzpHeEfwVUF2+vyzKLCyAoXe9w/a9lfXK83My+XDSqYlqsTGmlbJZVS3FrghdSRU76wYNiDoLC2CHS9AAGzQ3xhwcCxzNTaRB80iizMKKNGgeqdwYY2JhgaO5cRs0T82EzM7u9aMEGrdBcwFuGNn/IBtpjGnLLHA0N+FJETv2ct6ffW/9gAJw9EURDxW+aLBrhzRSvMKLS4ojrv0wxpiG2OB4S7Ji1v5ZVdk9nLIqH/zsn5AT2zNH3lm9iev/sYTD8jqw21fFxp3lltvKGOMq0uC4BY6WbOs6ePIU6HIY/PgtSInt8bKTX17B84vrPlzKclsZY8LZrKrWqOthMOZRKF0KMy52XzS4Yla98oVrt9Y7lOW2MsbEyhYAtnRHng8DRsHa+fvLahYNfvcJfPrc/qcPBssL915DCSfWO5RN0zXGxMLuOFqDTZ/VL6vyQdHTdR9ZGyyfnPZi/fpAt+wM1zsUY4wJZYGjNYi0aDCCbmxhXNpHLEr7BV+nX87UlOl48VNdvofVs6c6dyYo7NxA9dybLXgYY+qwwfHW4MGjgx/2YcQL6q9fDgTEiydk23IGcGvFT+nEbu5I+T8Geb+r3bYvsztvn7nAnjBoTBsTaXDcxjhag9PucMY0QrulUjPh2MvrjnEApGSAKh5/RZ1DDGYtb6ffVqcsoIIfIcO3kQ9n/5UXmEWP9K2U7uvKQ7PHAzc2ffAInaLcwON3jTGJYV1VrUGkRYPnPVC//IL/Ab97DitV8Gkav6u6mnEVd/Cc/1T2kokHmCx/Z4325AX/KXSW3dwp01n++vTGvMr6VsxyAmZI1xqv/cK61oxJsqR2VYnIWcBfAC/wlKpODds+GrgLCADVwC2qukhEBgIvhFQ9FLhDVR8SkSnAtcCW4LbbVfWNaO1o9V1V8YrQtVVGFmlaQQZVrNbefKuH8E2gG+0op8C7nsGyjhQJMKv6B1zg/YitmkPPO79qggsIitRF17EXTFjV+O0xppVp9HUcIuIFHgHOBo4CLhORo8KqLQCOVdXBwI+BpwBUdY2qDg6WHwfsA2aH7PdgzfaGgoZxESEf1tqhv+MOvY5S7cJR8h3Hyles9/bmR6nvUOj5khQJ8G0gj0tSFvJxYBB5sr1p2l/jAB6/a4w5eMkc4xgOrFPVrwFEZCYwGlhdU0FV94TUbw+43f6cBnylqt8msa1ti9tDpE67g2EFl1DSq4RL559WZxC8/O2XaefbCEAfz2ZWBfpwivdTlukAjq304Ulromy7HXtGuOOIM8OwMSYuyQwc+UDo/+pi4PjwSiIyFrgHyAPOdTnOeOD5sLKbROQqoAj4taruSEiL25KCS1wHkccMya8/4O29k+q5N5Pid54HcrTnWxYGCviBZwVfP3w2/Ub+CPnX/Y0/QH3qb2H2z+qWpWY65zfGJE0yB8fFpazeHYWqzlbVI4AxOOMd+w8gkgZcAISuWHsM6A8MBjYCf3Y9uch1IlIkIkVbtmxxq2JiVXAJKaP/p84g+0kX3czcQ6fQa/cKtrz2e9e1H3OWlST3eec1qeYzcpzX9I5Rn4hojEmMpA2Oi8h/AVNUdVTw/WQAVb0nyj7fAMNUdWvw/Wjg56p6ZoT6fYF5qnp0tLbY4HhyqCpP/f5qLpe38OMhW/ZP+92R2o3/Ln8YX9X+tSIJT6T4whVOWpUJq+Hp0yElE34yv+H9jDExaYp1HIuBASLSDyjB6XK6PKxRh+GMX6iIDAXSgG0hVS4jrJtKRLqr6sbg27GATZ9pIiLCj+Q1Hqy+iB+lvEM2Pv7pP4Zn/WfyfUUXfGGLD0MTKR70YsI9m2HNmzDiRkhJc/J1/et+2Lcd2kV46JUxJiGSFjhUtVpEbgLm40zHfUZVPxOR64PbHwcuAq4SkSrAB1yqwVsgEWkHnAGEdWJzn4gMxun2Wu+y3TSiTdqZCSmv8KT/bA6TjfzAs4KTPKtYFDia+6suxivKl9qTvWSSSjUlZT5ufWkFlf4AACVlPia/shIgvuDx6fMQqIYhVzrvDx8FC++DdQug4OJEX6YxJoSlHDEHZcoff8+tVY/STvYvKqxSDx7AQwARKNcUFgUKmOcfweuB46kitd5x8nMy+XDSqbGdVBX+Wgjtc53nkAAE/HD/AOh/Glz0ZAKuzBhjz+MwSTH43Ou4Q6+jONCVgArFga5M1hvZndoZCU6PyJBqTvcu5aG0R3kndaLrcUrKfLEPpn/3MWxbB0Ov2l/m8cJhZ8C6d50gYoxJGstVZQ6K0710Y721HzlzH3Gt39u7mQv8i7g1ZRapVDGtejyvBE4igIdfv/gp/oBzBxzehTVnWUntuMij7aZzekoHUo8aXffgh58JK2ZCcRH0rjfz2xiTIBY4zEFzXfvxgfviPA/wYOpjeMUJEPenPcENgblc65/EN/5cMqhkvPd9DmE7H1QXcPcbKVRU+Zny2mp8VX6y2ctI/0e8XH0yGZ+VMWZI+/0H73+qkxF47dsWOIxJIgscJjncMvamZACCt7ruw6X6e77nRf4fmzydecJ/AX/3n4Efr7NxdwW3Be88AC7wfkSmVPKPypPZ4TJDa27nwXRdOx9O+12yr9CYNssGx03yuKU8f+U63DLLKPtXjPpV+J5OlGgun2s/fl+1fyzjtbTb8aCcW3k3IKR5PbUztABuTpvHrz3P8daZ73PXwjJ7fogxB8EGx03jK7jEyVI7pcx5LbgkpjxSXlHyZTvDPWu4yvsW+TmZXOBZxL/TbuQYz3rKtD2H4CRYDA0aAG9XFQDw4VvPU1LmQ9k/XpLwlevGtFEWOEzjipCZVzLdF+0JwitZ07gv9Um6ecoAOMG7mvfTf8058nG9+mu0FyXahRN1aZ3y0MWHxpiDY4HDNK5ID506+976ASUlHfr8N922fESGVNXZlCmV3JE+0+UEwvv+wZzoWUkadfcpLfO51DfGxMsGx03ji5CZF3B/DOyUjq5Vu7GVzFRvvXxYi1MKuUIXMMzzBR8Gjqnd1iMnSvp3ewStMTGzwGGaj0gBpWMv16m90rEn94w8pl7eK291fyrmPcCpnuW1gSPFI0wcNdD9vDWPoK2ZAVbzCNqaNhlj6rDAYZo/t6m9wedujClwWUMCbFp8PGdsWc4fq68kM82Lr9JPt+wM9+MvuLPuscF5v+BOKLikzuJDm6FljAUO0xJEeGJhtLuBbt17wuZFfJNxOfuyD+U8351MeGE5b/7yJDq1T6tbOcojaOcsK2HyKytru8NCV7RDArL8Npam7IqzbsBWx9ZxmNZnxSx49SaorqgtWuUZyIXld3DyEYcw/crjENn/nDHf1AFklm+ud5hdnmwu0L+QXv49vWQLh0op3dnOwsAxLE0/nkq/Jvd5I4kS3hUHzh1bYzz0KtHntiDUqCKt47DAYVqfB492HRN5OuUy7tpzPh0zU9nlq6JHTia/Pbkzw+aPobOW8WHgKN4NFPJFoBcneFZxQ8prbCeLW6uuZV2gJ5vpRFXwJt2Dn0DN6vYQ+TmZfHjO1ub14Rbh90HHXs76mpZy7qYMgG1UUzzIyZimEaHr6Zqq5/mTnM9OnzNNd1/ZJvq/eTOZso+LK+9giQ6kHeX0l1Je8Z/E/MBwXki7k2fTpgFQqR6+0h58rn35t/8IFgSGUk4a5aRxo3cu3wS64dkFvPa/iRtoT8Q37ChdcUmXyHNHG4uq2R7P76k13L000TVY4DCtT0f3BIuIcJfnSR7wX0wVKfw9bSq9ZRPXVN3KEnVmXO0jg5V6KAD5HTNpX15eu3uaBDhSijmSYsZ4F1GuqVSRym+qrucJ//lc5Z3PjSmvuX+4vfsH5+d4/pMnYraXvxrSs6BiV/1tMaziP2jtusC+rfXLs3vEfyy3v9Oa8jk3OA/2qnnf0O+puc6kiycQNOE1WOAwrY9rgsV0NlW244ep7zM65WN2azu6yQ72aDq57HBdDzJx1EDkA/epwB6gnVQBVUxPfQBF8AQz/vpVarP/1tpVTGD29XhqHqe7cwPVc292/gNGW9MSZbaXq9APnqxDILWdEzTEC2GP8mXAGe7HSJTVr4JvB4g4D98KldYeKvdBWrvYjlX2HXhS9geHcOHlDf2eDuR3m2zxBoKGriGJdyO2cty0Pm6r0y/4K+JxxiQ6SDndPdvxiJLtKef2tBe558JjyM/JRHDGKWoHud1SpKTUndYrAh5R1gZ6MKbiDwyqeJqX/SfVqaOwP2jUHMZfzr4374h8HRG/YUfo5qn54Nm5wTnj7o2w/SsY9lMY+/j+30d2PuQeCUv+BqteiXz+SFbMcsYupuQ4rytm1S+/91B48WroWQjnPVT372L4dbB1LbxwBSyb0fCx7h8Ij58E3lTwhs2IC/+7ieX3FG3bzg2xXV9oeaI01BXn1tZI5W/dDq/evP/fQk0QSlCbbXDctBk6JQdxzcwryJSyyDu6fXNbcKfrf9x1ms/tlT/mP3okx8kacmUn7Sink+yms+ziCNnASZ5VpInzDTmA4HE7d9l38JdjQQP1t0UaWI5nILpyL/xjHGz4Nxz/M/j8tQPrHgHnw/vYy+HT5+qWiwfO+wscd1X94yz9uzPzTTx1rzHSsQBO+73Tvhj/LqIOwE87DPZucd8WfneWkg4Dz4M18+rM1KsdmIfEfLOPkCEBgLHT4b27nHNk58OR58N/pte/i3QuALcM1EDckxJsVpUFDtMIM3x+ufca/AqHeHayLDCA7XRgo3ZBAB/pAPSWTcxLu51s8bE50JG8O7+re2xfGTwzCsq+dT5UQz+svKkw+lH3D6YpObh/YIiToThcxW7nm/yOb+qWR/tAfPcPsMvt23qED6tov9upfaDcpV14gDgCptvfBcDJk+CUyfXrb10LT/wgWD+kzTV3M/5K9/a6Sclw/o5C9zmQgPL9KnjiJPcvClA/wAJkdnK+AISf+9wHnDEf9wO5/1uIVLsp0qqLyFkiskZE1onIJJfto0VkhYgsF5EiETkxZNt6EVlZsy2kvLOIvCMia4OvnZJ5DaYViZCZl9OidBdFEiFZY1H2GczTE3nKfy5L9HC+0R6Uk84obxGfpv2Epek/Y4is46yKe9muHejo2QffLNx/3OpKpwtn21dw2QssLriL78kloFBJClWkQv/T3NvUPte9PNIgeHqW+4dklQ/mTXDuCEK7Omb/LELQgIjfcKN1F5XvjLAhwodnpGOF/11k5zu/i6XPwp6w9Tn7tsNzlzhjP6Purvv3N/oR8Fe5nSGy6vL6v8MqH8z7VexdRRs/hWfPg/Rs5+4mVEqG01a3gJLa3mlzeMLQwZcHy1wkaEJE0u44RMQLfAmcARQDi4HLVHV1SJ0OwF5VVREpAGap6hHBbeuBQlXdGnbc+4Dtqjo1GIw6qept0dpidxymVpKnL4avNAdnoP2i4/IpXzqTW5hJN7bzq+ob+SRwJLOzppHvL0EyOjqzj1LbQdU+GPsEcwIn1TnW4bKB19Nup7TPBfT58d/qnnhXKTwyIjh7KuT/dEPrHCLepUQS4c7CbfAdot9xRLoDjPeOw833K+Gp06HnMLhyDnhTnKD8jwud7rmrX4PeI2JvU6Tri1fNNdT+O9wAiHP3cO0CKC6K+eFnUe8eErTmpSnWcQwH1qnq18EGzARGA7WBQ1X3hNRvT2z/gkcDI4M/Pwt8AEQNHMbUipaZNwFqVo27pSKZ06czl84/jdIyH907ZnBEXgfOXzuJeem/pcferYgAVfvwSwqohzuDz1mv8aX24in/Odzw3Wz+9c5YJi3JorTMR9+OXl7O/COd1c/KIyeQ+/nfydOtbJaubDhmIsOiXW+kqcsRqfMBFMsYR0N3c5FykB3IscIdcowzKD/neri3L1Tudr6hV+11xgvcgsaBtCklE3zbY2/Xzg3wzNlQsjjk7kaddhUXuf/7jDiGE+Xu4QDS9MQjmYEjHwi92mLg+PBKIjIWuAfIA84N2aTA2yKiwBOqOj1Y3k1VNwKo6kYRyXM7uYhcB1wH0Lt374O8FGNiN2aIe+LF8HJV5U9/eIGzy+9hbtpv6SebqFIPM6pP4elZPrYH2tc7xl+qL+Q8zyd0XzSZzRX3oHi5ce8jdK5YyYy+d/PHVf3xVe3/gpi52Ms9vUoip0GJ9EEZ6QOxY6+QAemwD6TeI+L7oIr24Rbvsdx4vM6fyt3O+6q9zpTekHQzB90miO/3l5IJGz6p3/VUXRF5KnCUJJ9RJfFLUjK7qi4GRqnqT4PvrwSGq+rNEer/ALhDVU8Pvu+hqqXBwPAOcLOqLhSRMlXNCdlvh6pGHeewrirTXAWm5PBo9QU8Wj2aH3vf4Dn/aWynI0NkLd9mDmL7vvpjECM9y/hb2jQeqLqIdlLB9SnzeLL6bO6uvtL1lj0/J5MPJ50asQ2LX32CXkun7b9LGTqRYX07tfz0Ho2VasWt+xMi//5euTbCgRroemqCFeJN0VVVDISO0PQESiNVDgaF/iLSVVW3qmppsHyziMzG6fpaCGwSke7Bu43uQP3sdMa0EKWBLtyUMpcMKrm3+jJO9yzhpylv0I0dFJ2/0HW8JNu/l+2BDvwiZTaCsjhwOHP9/0079rGX+gvqoj35cM6yEiYv7oOv6i/7z7HYyz29jmHM+Q+37JQcjZVqJd4Hkx1o11Mz+t0nM3AsBgaISD+gBBgPXB5aQUQOA74KDo4PBdKAbSLSHvCo6u7gz2cCNatgXgWuBqYGX+cm8RqMSaqn0q7g1qpH+WnKm1ztfZtU8bNP07gv9UamRBgvGTH3Jjqzf3hwmHzJvPTfUaJdOaHi4XrnSE/18PS/vuaZD9fXG3eZ+uYXdQIThDyffdQJTKt4mNJyHz0yMpnoH8iYKNfSGM8tiesckcZvGiPVCkT+sD/QrqdmJKnrOETkHOAhwAs8o6p/EpHrAVT1cRG5DbgKqAJ8wERVXSQihwKzg4dJAZ5T1T8Fj9kFmAX0Br4DLlbVqKNT1lVlmqs5y0pYNPtRbmEmPWQbpdqFhxjPiWNvjPiBGG0h41H+mXUCQYrHqekP1K2f4hG6ZWdQEuVuJMUjVIfsFy1tfKTZZIlMMx/3OZpzNt0WkmDRFgBa4DDNVNzf1KP03c8ZOb/esaa++QXf7yqvVz0txUNGiodd5RHyP7nIDx4z/Bz3vfUFpTvrn6Oh8ZV4nDD1PddAF/UcLeQDurmywGGBw7QWcX6T7jfp9UirAHjw0sGu3+LDu69CpXk9VPr3zwoKvzMJP8c3U8913RavvpNeT/o5TF1NsnLcGJMEEVatR/om3SPHPRFgj5xMxgzJd03wmB9hH6BO0AAiBo1o547X2599f0DnmLOshBOmvke/Sa9zwtT3mLOsJCHtaessrboxLVEcs2wmjhroelcxcZTzDJJI607ivRNx2z68X+eobYvUTRda3jEzlZ3lVfTu3I7Nu8opr64buEYPdn+2R7TnxTe7x/u2MHbHYUwrF+muItqHZ7x3IqHbBejRMYOCnh2ZvayE2cvcp7/WfLCXlPlQ9n+w/3bOyjrlZb4qBPj5Kf2ZelFB7Tm6d8ygW3Y6Lyze4DrleNr8NZFnjJmDYmMcxpiYxTOzqbzKz4//tphPvt5Gp3ZpbN9bWXtXccGxPTj+7gVs2VMRfoqIScHdBsHXbd7D6L8uYkC3LF742QjSU/Y/B97GRA6ePXPcGHPQouXiCpeR6uWCY3vw8dfb2LbXWQFfUubj17M+5bdzVrGnwn02V6Svsm53FYfldeD+i4/lhhlLGXrnO+yr9NMtO4PDu3WIeA152ekRt0HjrEdp6SxwGGPiEmlMxM3/vLeu3lNj/ar4A0qndqns2Fc/jblXBL9LT0ikQfCK6gApHmFvpXMX9P2ucr7fVc5hee0p3u6rNyZSUR1g3eY9HJZXP7hEGxeB2ALmwWgpQcvGOIwxSRMp3Ul5lZ/fnz+IzFRvnfLMVC+XHd/LtbxmMD/ctPlrXGd2+SoDdcZE8nMy+c2Zh5PiES554mMeXrC23oyrSCvpfz93let4TCJnaUUa82mOM8HsjsMYkzQ9cjJdF+3VTAUG92/xhX06x/zNO1JwKi3zud4dnVvQg4se+4gH3vmytqykzMevZi0n0szinS6LJGsG2hN1R9DQYH5zuhOxwGGMSZoDnQocT3dYtODkpl/X9qR563e2BDTq07pdlZT5YppSHP5hH77t0sJeEdO/lJT5mPjSp1T5tfZ9U08rtllVxpikSna//YHkyYq0mr5m3/BjZaR6XMdjoH6wyUjxcOHQfGYvK3VtE9RfI+N2nIYkMp1LJJZyxAKHMa1WvMEpWt4rt1xcUP/DPiPFgwj4qiI8I91FmtfZp6K6/j4dM1Ko9Gtciy4funRwUoOyBQ4LHMaYoAO5S3ELThNeWB7XXUI0NbnDws8xbf6aiN1Y8WQwPqA22ToOY4xxxLMeJXSf8O2RPtQjTSmuWXkfbcJALOlf0lM8qCqV/rrnCB2wT2YXoQUOY0ybFM8AfCSRBv8vOi6fl5eURJwUEG3CgFs7oX6Qm/DCctf6JWU+Jr74Ka9+WlrbJZboAXULHMYYc4AOZkpxMu92XlxSPz9YIqcP2xiHMca0QJHGae4eezQTZn3quk+8ebrseRzGGNOKRMpgPHZoz4hZjBP1fBTrqjLGmBYq0jhNQwsvD1ZS7zhE5CwRWSMi60Rkksv20SKyQkSWi0iRiJwYLO8lIu+LyOci8pmI/DJknykiUhLcZ7mInJPMazDGmJbmQJ7BEo+kjXGIiBf4EjgDKAYWA5ep6uqQOh2AvaqqIlIAzFLVI0SkO9BdVZeKSBawBBijqqtFZAqwR1Xvj7UtNsZhjDHxa4oxjuHAOlX9WlUrgZnA6NAKqrpH90eu9gRX3KvqRlVdGvx5N/A50PxyCxtjTBuUzMCRD2wIeV+My4e/iIwVkS+A14Efu2zvCwwB/h1SfFOwi+sZEenkdnIRuS7Y/VW0ZcuWg7gMY4wxoZIZOMSlrF6/mKrOVtUjgDHAXXUO4HRlvQzcoqq7gsWPAf2BwcBG4M9uJ1fV6apaqKqFubm5B3oNxhhjwiQzcBQDvULe9wRKI1VW1YVAfxHpCiAiqThBY4aqvhJSb5Oq+lU1ADyJ0yVmjDGmkSQzcCwGBohIPxFJA8YDr4ZWEJHDRESCPw8F0oBtwbKngc9V9YGwfbqHvB0LrEriNRhjjAmTtHUcqlotIjcB8wEv8IyqfiYi1we3Pw5cBFwlIlWAD7g0OMPqROBKYKWILA8e8nZVfQO4T0QG43R7rQd+1lBblixZslVEvm2gWldga5yX2RrYdbctdt1tz8Fcex+3wjaRciQWIlLkNu2stbPrblvsutueZFy7pRwxxhgTFwscxhhj4mKBY7/pTd2AJmLX3bbYdbc9Cb92G+MwxhgTF7vjMMYYExcLHMYYY+LS5gNHQ6nfW5Ngbq/NIrIqpKyziLwjImuDr665v1qySGn6W/u1i0iGiPxHRD4NXvcfguWt+rrByc4tIstEZF7wfau/ZgARWS8iK2seVREsS/i1t+nAEUz9/ghwNnAUcJmIHNW0rUqqvwFnhZVNAhao6gBgQfB9a1MN/FpVjwRGAD8P/j239muvAE5V1WNxcrudJSIjaP3XDfBLnKzaNdrCNdc4RVUHh6zdSPi1t+nAQQyp31uTYD6w7WHFo4Fngz8/i5NsslWJkqa/VV+7OvYE36YG/yit/LpFpCdwLvBUSHGrvuYGJPza23rgiCn1eyvXTVU3gvMBC+Q1cXuSKixNf6u/9mCXzXJgM/COqraF634IuBUIhJS19muuocDbIrJERK4LliX82tv6M8djSv1uWofwNP3B/Jqtmqr6gcEikgPMFpGjm7hJSSUi5wGbVXWJiIxs4uY0hRNUtVRE8oB3gs86Sri2fscRV+r3VmpTTcbh4OvmJm5PUkRI098mrh1AVcuAD3DGuFrzdZ8AXCAi63G6nk8VkX/Quq+5lqqWBl83A7NxuuMTfu1tPXA0mPq9DXgVuDr489XA3CZsS1JESdPfqq9dRHKDdxqISCZwOvAFrfi6VXWyqvZU1b44/5/fU9UraMXXXENE2otIVs3PwJk4j51I+LW3+ZXjInIOTp9oTer3PzVti5JHRJ4HRuKkWd4E/B6YA8wCegPfAReravgAeosWTNP/L2Al+/u9b8cZ52i11y4iBTiDoV6cL4mzVPVOEelCK77uGsGuqt+o6nlt4ZpF5FCcuwxwhiGeU9U/JePa23zgMMYYE5+23lVljDEmThY4jDHGxMUChzHGmLhY4DDGGBMXCxzGGGPiYoHDmGZOREbWZHk1pjmwwGGMMSYuFjiMSRARuSL4/IvlIvJEMMHgHhH5s4gsFZEFIpIbrDtYRD4RkRUiMrvmGQkicpiIvBt8hsZSEekfPHwHEXlJRL4QkRnSFhJtmWbLAocxCSAiRwKX4iSZGwz4gR8C7YGlqjoU+CfOan2A/wNuU9UCnBXtNeUzgEeCz9D4b2BjsHwIcAvOc2MOxcnJZEyTaOvZcY1JlNOA44DFwZuBTJxkcgHghWCdfwCviEhHIEdV/xksfxZ4MZhnKF9VZwOoajlA8Hj/UdXi4PvlQF9gUdKvyhgXFjiMSQwBnlXVyXUKRX4XVi9ajp9o3U8VIT/7sf+7pglZV5UxibEAGBd8DkLNc5774PwfGxesczmwSFV3AjtE5KRg+ZXAP1V1F1AsImOCx0gXkXaNeRHGxMK+tRiTAKq6WkR+i/P0NQ9QBfwc2AsMEpElwE6ccRBw0ls/HgwMXwPXBMuvBJ4QkTuDx7i4ES/DmJhYdlxjkkhE9qhqh6ZuhzGJZF1Vxhhj4mJ3HMYYY+JidxzGGGPiYoHDGGNMXCxwGGOMiYsFDmOMMXGxwGGMMSYu/x/5wV8q3kKglgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "epoch_list = np.arange(1, N_EPOCHS+1, 1)\n",
    "plt.plot(epoch_list, train_losses, label=\"train\")\n",
    "plt.plot(epoch_list, valid_losses, label=\"valid\");\n",
    "plt.scatter(epoch_list, train_losses)\n",
    "plt.scatter(epoch_list, valid_losses);\n",
    "plt.legend();\n",
    "plt.ylabel(\"loss\")\n",
    "plt.xlabel(\"epoch\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "selective-wednesday",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEHCAYAAABBW1qbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaaElEQVR4nO3de5SU9Z3n8fcHQRuFTVRaF+jGxoisiFGThlzciUZ3FDkmmDni4BgFZWM4gUmyRo+XPZnoTshhz1En2XXEIWpAoxJidCXGdYJXTCRi46CI6EqUSwEHW7JesxppvvtHPTyW0NBVTT116fq8zunTVb/6PU99f1yeTz2X+j2KCMzMzAD6VbsAMzOrHQ4FMzNLORTMzCzlUDAzs5RDwczMUv2rXcC+GDJkSLS1tVW7DDOzurJixYo3IqK5u9fqOhTa2tro6OiodhlmZnVF0vo9vebDR2ZmlnIomJlZyqFgZmapuj6n0J0PP/yQXC7H+++/X+1SyqKpqYmWlhYGDBhQ7VLMrAH0uVDI5XIMHjyYtrY2JFW7nH0SEWzbto1cLsfIkSOrXY6ZNYA+d/jo/fff59BDD637QACQxKGHHtpn9nrMrPb1uVAA+kQg7NSXxmJmta9PhoKZmfWOQ2EXp5xySo9fiJs2bRr33HPPbu0dHR18+9vfBmD+/PnMmjULgJtvvpnbb789bd+8eXOZqzYzK48+d6K5mtrb22lvb9+tfcaMGenj+fPnM3bsWIYNG1bJ0sysxrWNaGH9xk1F9z+idTjrNuTKXkdDhMJ7773HueeeSy6Xo6uri+9///sMGTKEyy67jO3btzNu3Djmzp3LAQcc8LHlBg0axLvvvgvAPffcwwMPPMD8+fMBePjhh/nJT37C1q1bueGGGzjrrLN4/PHHue6663jggQc+tp5rrrmGQYMGpdNynH/++QwcOJDZs2dzyy23cN999wGwZMkS5s6dy7333pv9H4qZ1ZT1GzcRj/6o6P469epM6miIw0cPPfQQw4YN47nnnuOFF15gwoQJTJs2jV/84hesWrWK7du3M3fu3JLWuW7dOp544gl+85vfMGPGjKKuEDrnnHNob2/nzjvvZOXKlUycOJE1a9bQ2dkJwM9+9jMuuuiiXo3RzKwcGiIUjjvuOB5++GGuuOIKnnzySdatW8fIkSM5+uijAZg6dSpLly4taZ3nnnsu/fr1Y9SoURx55JG89NJLJdcliQsuuICf//znvPnmmyxbtowzzzyz5PWYmZVLQxw+Ovroo1mxYgUPPvggV111FaeffnpRyxVeDrrrnsCul4r29tLRiy66iK985Ss0NTUxefJk+vdviL8SM6tRDbGnsHnzZg488EC+/vWvc9lll/HUU0+xbt061q5dC8Add9zBySefvNtyhx9+OGvWrGHHjh3pcf+dfvnLX7Jjxw7++Mc/8uqrrzJ69Oiiahk8eDDvvPNO+nzYsGEMGzaMH/7wh0ybNq33gzQzK4OG+Fi6atUqLr/8cvr168eAAQOYO3cub731FpMnT05PNBdeIbTTnDlzOOuss2htbWXs2LHpSWeA0aNHc/LJJ7N161Zuvvlmmpqaiqpl2rRpzJgxg4EDB7Js2TIGDhzI+eefT2dnJ2PGjCnbmM3MekMRUe0aeq29vT12/U7BmjVrOOaYY6pUUe/MmjWLE088kenTp3f7ej2OycxKI6nkq496u/2WtCIidr9+ngbZU6hln/3sZznooIO4/vrrq12KmZlDodpWrFhR7RLMzFINcaLZzMyK41AwM7NUZqEgqUnScknPSVot6dqk/RBJSyS9kvw+uGCZqyStlfSypDOyqs3MzLqX5Z7CB8CpEXE8cAIwQdLngSuBRyJiFPBI8hxJY4ApwLHABOAmSftlWJ+Zme0is1CIvJ0X9g9IfgKYBCxI2hcAZyePJwELI+KDiHgNWAuMz6q+1hFHIKlsP60jjujxPR966CFGjx7NUUcdxZw5c7IamplZr2V69VHySX8FcBTwzxHxtKTDI2ILQERskXRY0n048IeCxXNJWyZyGzdww29fLtv6Lj19799o7urqYubMmSxZsoSWlhbGjRvHV7/6VX9hzcxqSqYnmiOiKyJOAFqA8ZLG7qV7d5MH7fbNDEmXSOqQ1LFzdtF6sHz5co466iiOPPJI9t9/f6ZMmcL9999f7bLMzD6mIlcfRcSbwOPkzxVslTQUIPn9etItB7QWLNYC7HaLsoiYFxHtEdHe3NycZdlltWnTJlpbPxpeS0sLmzYVf0MNM7NKyPLqo2ZJn0weDwT+E/ASsBiYmnSbCuz8uLwYmCLpAEkjgVHA8qzqq7Tuvo7e25lVzcyykuU5haHAguS8Qj9gUUQ8IGkZsEjSdGADMBkgIlZLWgS8CGwHZkZEV4b1VVRLSwsbN25Mn+dyOd+S08xqTmahEBHPAyd2074NOG0Py8wGZmdVUzWNGzeOV155hddee43hw4ezcOFC7rrrrmqXZWb2MQ0791FL64gerxgqdX17079/f2688UbOOOMMurq6uPjiizn22GPL9v5mZuXQsKGwccP6ir/nxIkTmThxYsXf18ysWJ77yMzMUg4FMzNLORTMzCzlUDAzs5RDwczMUg4FMzNLNWwotI1oKevU2W0jWnp8z4svvpjDDjuMsWP3Ni+gmVn1NOz3FNZv3EQ8+qOyrU+nXt1jn2nTpjFr1iwuvPDCsr2vmVk5NeyeQjV86Utf4pBDDql2GWZme+RQMDOzlEPBzMxSDgUzM0s5FMzMLNWwVx8d0Tq8qCuGSllfT8477zwef/xx3njjDVpaWrj22muZPn162WowM9tXDRsK6zbkKv6ed999d8Xf08ysFD58ZGZmKYeCmZml+mQoRES1SyibvjQWM6t9fS4Umpqa2LZtW5/YmEYE27Zto6mpqdqlmFmD6HMnmltaWsjlcnR2dla7lLJoamqipaXnyfbMzMqhz4XCgAEDGDlyZLXLMDOrS5kdPpLUKukxSWskrZb0naT9GkmbJK1MfiYWLHOVpLWSXpZ0Rla1mZlZ97LcU9gOfC8inpU0GFghaUny2j9FxHWFnSWNAaYAxwLDgIclHR0RXRnWaGZmBTLbU4iILRHxbPL4HWANsLev/U4CFkbEBxHxGrAWGJ9VfWZmtruKXH0kqQ04EXg6aZol6XlJt0k6OGkbDmwsWCzH3kPEzMzKLPNQkDQI+BXw3Yh4G5gLfAo4AdgCXL+zazeL73ZdqaRLJHVI6ugrVxiZmdWKTENB0gDygXBnRNwLEBFbI6IrInYAP+WjQ0Q5oLVg8RZg867rjIh5EdEeEe3Nzc1Zlm9m1nCyvPpIwK3Amoi4oaB9aEG3rwEvJI8XA1MkHSBpJDAKWJ5VfWZmtrssrz46CbgAWCVpZdJ2NXCepBPIHxpaB3wTICJWS1oEvEj+yqWZvvLIzKyyMguFiPgd3Z8neHAvy8wGZmdVk5mZ7V2fm/vIzMx6z6FgZmYph4KZmaUcCmZmlnIomJlZyqFgZmYph4KZmaUcCmZmlnIomJlZyqFgZmYph4KZmaUcCmZmlnIomJlZyqFgZmYph4KZmaUcCmZmlnIomJlZyqFgZmYph4KZmaUcCmZmlnIomJlZyqFgZmYph4KZmaUyCwVJrZIek7RG0mpJ30naD5G0RNIrye+DC5a5StJaSS9LOiOr2szMrHtZ7ilsB74XEccAnwdmShoDXAk8EhGjgEeS5ySvTQGOBSYAN0naL8P6zMxsF5mFQkRsiYhnk8fvAGuA4cAkYEHSbQFwdvJ4ErAwIj6IiNeAtcD4rOozM7PdVeScgqQ24ETgaeDwiNgC+eAADku6DQc2FiyWS9p2XdclkjokdXR2dmZat5lZo8k8FCQNAn4FfDci3t5b127aYreGiHkR0R4R7c3NzeUq08zMyDgUJA0gHwh3RsS9SfNWSUOT14cCryftOaC1YPEWYHOW9ZmZ2cdlefWRgFuBNRFxQ8FLi4GpyeOpwP0F7VMkHSBpJDAKWJ5VfWZmtrv+Ga77JOACYJWklUnb1cAcYJGk6cAGYDJARKyWtAh4kfyVSzMjoivD+szMbBeZhUJE/I7uzxMAnLaHZWYDs7OqyczM9s7faDYzs1RRoSDppGLazMysvhW7p/A/i2wzM7M6ttdzCpK+AHwRaJZ0acFL/w7wFBRmZn1MTyea9wcGJf0GF7S/DZyTVVFmZlYdew2FiHgCeELS/IhYX6GazMysSoq9JPUASfOAtsJlIuLULIoyM7PqKDYUfgncDNwC+AtlZmZ9VLGhsD0i5mZaiZmZVV2xl6T+WtK3JA1N7px2iKRDMq3MzMwqrtg9hZ0T2F1e0BbAkeUtx8zMqqmoUIiIkVkXYmZm1VdUKEi6sLv2iLi9vOWYmVk1FXv4aFzB4ybys5w+CzgUzMz6kGIPH/194XNJnwDuyKQiMzOrmt5Onf1n8ndGMzOzPqTYcwq/Jn+1EeQnwjsGWJRVUWZmVh3FnlO4ruDxdmB9ROQyqMfMzKqoqMNHycR4L5GfKfVg4C9ZFmVmZtVR7J3XzgWWA5OBc4GnJXnqbDOzPqbYw0f/FRgXEa8DSGoGHgbuyaowMzOrvGKvPuq3MxAS20pY1szM6kSxewoPSfpX4O7k+d8CD2ZTkpmZVcteP+1LOkrSSRFxOfAvwKeB44FlwLwelr1N0uuSXihou0bSJkkrk5+JBa9dJWmtpJclnbFPozIzs17p6RDQj4F3ACLi3oi4NCL+C/m9hB/3sOx8YEI37f8UESckPw8CSBoDTAGOTZa5SdJ+xQ7CzMzKo6dQaIuI53dtjIgO8rfm3KOIWAr8qcg6JgELI+KDiHgNWAuML3JZMzMrk55CoWkvrw3s5XvOkvR8cnjp4KRtOLCxoE8uaduNpEskdUjq6Ozs7GUJZmbWnZ5C4RlJ39i1UdJ0YEUv3m8u8CngBGALcP3OVXbTN7ppIyLmRUR7RLQ3Nzf3ogQzM9uTnq4++i5wn6Tz+SgE2oH9ga+V+mYRsXXnY0k/BR5InuaA1oKuLcDmUtdvZmb7Zq+hkGzEvyjpy8DYpPk3EfFob95M0tCI2JI8/Rqw88qkxcBdkm4AhpGfgXV5b97DzMx6r9j7KTwGPFbKiiXdDZwCDJGUA34AnCLpBPKHhtYB30zWv1rSIuBF8hPuzYyIrlLez8zM9l2xX14rWUSc103zrXvpPxuYnVU9ZmbWM09VYWZmKYeCmZmlHApmZpZyKJiZWcqhYGZmKYeCmZmlHApmZpZyKJiZWcqhYGZmKYeCmZmlHApmZpZyKJiZWcqhYGZmKYeCmZmlHApmZpZyKJiZWcqhYGZmKYeCmZmlHApmZpZyKJiZWcqhYGZmKYeCmZmlHApmZpbKLBQk3SbpdUkvFLQdImmJpFeS3wcXvHaVpLWSXpZ0RlZ1mZnZnmW5pzAfmLBL25XAIxExCngkeY6kMcAU4NhkmZsk7ZdhbWZm1o3MQiEilgJ/2qV5ErAgebwAOLugfWFEfBARrwFrgfFZ1WZmZt2r9DmFwyNiC0Dy+7CkfTiwsaBfLmnbjaRLJHVI6ujs7My0WDOzRlMrJ5rVTVt01zEi5kVEe0S0Nzc3Z1yWmVljqXQobJU0FCD5/XrSngNaC/q1AJsrXJuZWcOrdCgsBqYmj6cC9xe0T5F0gKSRwChgeYVrMzNreP2zWrGku4FTgCGScsAPgDnAIknTgQ3AZICIWC1pEfAisB2YGRFdWdVmZmbdyywUIuK8Pbx02h76zwZmZ1WPmZn1rFZONJuZWQ1wKJiZWcqhYGZmKYeCmZmlHApmZpZyKJiZWcqhYGZmKYeCmZmlHApmZpZyKJiZWcqhYGaWgbYRLUgq+qdWZDb3kZlZI1u/cRPx6I+K7q9Tr86wmuJ5T8HMzFIOBTMzSzkUzMws5VAwM7OUQ8HMzFIOBTMzSzkUzMws5VAwM7OUQ8HMzFIOBTMzSzkUzMwsVZVQkLRO0ipJKyV1JG2HSFoi6ZXk98HVqM3MbFelTm5XSxPclaqaE+J9OSLeKHh+JfBIRMyRdGXy/IrqlGZm9pFSJ7eD2pngrlS1dPhoErAgebwAOLt6pZiZNaZqhUIAv5W0QtIlSdvhEbEFIPl9WHcLSrpEUoekjs7OzgqVa2bWGKoVCidFxGeAM4GZkr5U7IIRMS8i2iOivbm5eZ+KaB1xREnHCFtHHLFP72dmVuuqck4hIjYnv1+XdB8wHtgqaWhEbJE0FHg96zpyGzdww29fLrr/paePzrAaM7Pqq/iegqSDJA3e+Rg4HXgBWAxMTbpNBe6vdG1mZo2uGnsKhwP3JZds9QfuioiHJD0DLJI0HdgATK5CbWZmDa3ioRARrwLHd9O+DTit0vWYmdlHaumSVDMzqzKHgpmZpRwKZmaWciiYWcMpdS6jRlLNuY/MzKqi1LmM6nUeo97wnoKZmaUcCmZmlnIomJlZyqFgZmYph4KZ1T1fTVQ+vvrIzOqeryYqH+8pZMj3azDLK/WTfNuIlmqX3LC8p5Ah36/BLM+f5OuH9xTMzCzlULC9KvUQWP8B+/uQmVkd8+GjUqhfXV+50DriCHIbN5S8XKmHwHzIzKx+ORRKETvqeoNX6jkOqL0xWGPYT9T1B7B65lCwqivlP/8RrcNZtyGXYTWlK3UPrKV1BBs3rC+6f9uIFtZv3FR0/1r8MypVV+AT01XiUKglJR6e2q//ALq2f5hhQZVR7//5e3OVWamfgrP8Myo1dAAOGLAfH3zYVdIyVh8cCrWkF4en6vlwVl9y7bXXltS/loJwfW5Lyct8sD1qagxWPg4Fqy8l7k2VeqimtyfjfzD1tKL7XvrUwtI2kirtIsHeHI+/Yd6tJfW/9JvfyHQMmVO/+q4/Qw4Fq65S/3NS4tVQZxxT8gbyEyf9XUn93/r9XSX1J3aUtBHuzQY4Hv1h8d178ym+AmPItD+lBeGll0wvad29UiNB5VCw6ip141Lqf85erL+UT/0Al5YaCqXKegNcCb34e8i6f83J+v9CkWouFCRNAH4C7AfcEhFzqlxSTSv1WHYl1GJNDaVGNi5Wn2oqFCTtB/wz8NdADnhG0uKIeLG6lfVe1hvIko5l//6uimywS62pVA4dq7heHJ6q1/MQNRUKwHhgbUS8CiBpITAJyCwUSt3AlNq/1jbavTk0Umsb4axDx2w3Je59QY0exiuCIqLaNaQknQNMiIj/nDy/APhcRMwq6HMJcEnydDRQ2ld0PzIEeGMfyq1HHnNj8Jgbw76M+YiIaO7uhVrbU+juMpGPpVZEzAPm7fMbSR0R0b6v66knHnNj8JgbQ1ZjrrWDXjmgteB5C7C5SrWYmTWcWguFZ4BRkkZK2h+YAiyuck1mZg2jpg4fRcR2SbOAfyV/SeptEbE6o7fb50NQdchjbgwec2PIZMw1daLZzMyqq9YOH5mZWRU5FMzMLNXnQ0HSBEkvS1or6cpuXpek/5G8/rykz1SjznIqYsznJ2N9XtJTko6vRp3l1NOYC/qNk9SVfCemrhUzZkmnSFopabWkJypdY7kV8W/7E5J+Lem5ZMwXVaPOcpF0m6TXJb2wh9fLv/2KiD77Q/5k9R+BI4H9geeAMbv0mQj8b/Lfkfg88HS1667AmL8IHJw8PrMRxlzQ71HgQeCcatddgb/nT5KfDWBE8vywatddgTFfDfz35HEz8Cdg/2rXvg9j/hLwGeCFPbxe9u1XX99TSKfNiIi/ADunzSg0Cbg98v4AfFLS0EoXWkY9jjkinoqI/5s8/QP574PUs2L+ngH+HvgV8Holi8tIMWP+O+DeiNgAEBH1Pu5ixhzAYOXnSx9EPhS2V7bM8omIpeTHsCdl33719VAYDmwseJ5L2krtU09KHc908p806lmPY5Y0HPgacHMF68pSMX/PRwMHS3pc0gpJF1asumwUM+YbgWPIf+l1FfCdiNhRmfKqouzbr5r6nkIGepw2o8g+9aTo8Uj6MvlQ+I+ZVpS9Ysb8Y+CKiOgq9aY7NaqYMfcHPgucBgwElkn6Q0T8n6yLy0gxYz4DWAmcCnwKWCLpyYh4O+PaqqXs26++HgrFTJvR16bWKGo8kj4N3AKcGRHbKlRbVooZczuwMAmEIcBESdsj4n9VpMLyK/bf9hsR8R7wnqSlwPFAvYZCMWO+CJgT+QPuayW9BvwHYHllSqy4sm+/+vrho2KmzVgMXJicxf888FZElH4n89rR45gljQDuBS6o40+NhXocc0SMjIi2iGgD7gG+VceBAMX9274f+CtJ/SUdCHwOWFPhOsupmDFvIL9nhKTDyc+k/GpFq6yssm+/+vSeQuxh2gxJM5LXbyZ/JcpEYC3wZ/KfNOpWkWP+B+BQ4Kbkk/P2qOMZJoscc59SzJgjYo2kh4DngR3k72TY7aWN9aDIv+d/BOZLWkX+0MoVEVG3U2pLuhs4BRgiKQf8ABgA2W2/PM2FmZml+vrhIzMzK4FDwczMUg4FMzNLORTMzCzlUDAzs5RDwczMUg4FswqT1LanqZDNqs2hYJYRSftVuwazUjkUzBKS/lHSdwqez5b07W76nSJpqaT7JL0o6WZJ/ZLX3pX03yQ9DXxB0qWSXkh+vluwmv6SFiQ3RrknmYYCSf8g6Zmk/zz1kdn7rH44FMw+ciswFSDZyE8B7txD3/HA94DjyM/G+TdJ+0Hkb4jyOeD/kZ924HPkb4DyDUknJv1GA/Mi4tPA28C3kvYbI2JcRIwlP7PpWeUbnlnPHApmiYhYB2xLNtynA/+2lxlklyc3e+kC7uaj6ce7yN/Ih6Ttvoh4LyLeJT8J4V8lr22MiN8nj39esPyXJT2dzN1zKnBsmYZnVpQ+PSGeWS/cAkwD/j1w21767Tpp2M7n7ydBAd3Pdb/H5SU1ATcB7RGxUdI1QFMxRZuVi/cUzD7uPmACMI78bJx7Mj6Zwrkf8LfA77rpsxQ4W9KBkg4if+e3J5PXRkj6QvL4vGT5nQHwhqRBwDn7NhSz0nlPwaxARPxF0mPAmwWf+LuzDJhD/pzCUvJhsuu6npU0n49u8HJLRPybpDby9zWYKulfgFeAuRHxZ0k/JX8byXXk7x9gVlGeOtusQPLJ/1lgckS8soc+pwCXRYRPAluf48NHZglJY8jfrOSRPQWCWV/nPQWzPZB0HHDHLs0fJJebmvVJDgUzM0v58JGZmaUcCmZmlnIomJlZyqFgZmap/w+huAtY0ngDDQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "tmp = pd.DataFrame(data={\"solubility\":  y_true, \"y_proba\": y_proba})\n",
    "sns.histplot(x=\"y_proba\", hue=\"solubility\", data=tmp, bins=25);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "announced-population",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.69"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cutoff = optimal_cutoff(y_true, y_proba)\n",
    "cutoff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "included-height",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = (y_proba > cutoff-0.2).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "english-identity",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASgAAAEGCAYAAADWoQhfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAveElEQVR4nO3dd3wUdf7H8dcnDUgCCSUCBkIHEYWAQaxIORXEju0sd1bO3vgpeHrIWVGxd+wVbKicIKDeKR69KiXAIRpIlBJKIED65/fHbDDEkGySnczu5vN8PPJgZ3Z25p2Q/eQ7s9/5fkVVMcaYYBThdQBjjDkYK1DGmKBlBcoYE7SsQBljgpYVKGNM0IryOkB1tWjRQtu3b+91DGNMLSxevDhbVZOq2s61AiUirwOnA1tU9YgKnhfgaeA0YC9wuaouqWq/7du3Z9GiRYGOa4ypQyKS4c92bp7ivQkMqeT5oUAX39cI4EUXsxhjQpBrLShVnSUi7SvZ5CzgbXV6is4TkUQRaa2qv7mVyRhTS5s3w4YNfm+uqkhMDKSm1uhwXl6DSgY2llnO9K2zAmXCzw8/QFaWO/v+5RdYsAAaNqxyU504kZKYGDQiskaHityWjRQX+7VtfgtYdQ90eq85TRZk1+h4XhYoqWBdhffdiMgInNNAUlJS3MxkzB/t2gWlt4T973+walXVrykpgbffhmbNIC8Ppk51NyOwq1FjiqIqf0tHFoMW5DGt2wk1OkZ2pwSWHdoNlYrevmWOE1XIWX9+gQYN99L47ttoUqOjeVugMoG2ZZbbAL9WtKGqTgAmAKSlpdnNg8ZdmZmQnw8vvgiPP17r3f2UlEJks2TeTj2NxcmHBSDgH2XHNSUr4RBX9l1ThRt28XNOF94YcWWN9+FlgZoC3Cgik4B+QI5dfzIBt2YNOmwYxVHRUMVffYCo1el/WLchoSVvHXWGs6DKhqatWZ3Uvsp9FUZEsalJi+omDmm9khZQWBLDqm2pfJdZ2Wdk/nGzm8FEYADQQkQygXuBaABVfQmYhtPFYB1ON4Mr3MpiQpAq3Hff79dtfvkF/fZbSuLj0Uj/f22jsrciOL/oU7sdX/ULuh1P0327mXzEIIoiIvl3p77sahhfo2+hvumfPJO/9niONTuOYNW2XlR8Fad63PwU789VPK/ADW4d3wSRnTudgrN9O8yadWBLJj8f3nsPmjffv6okI4OIpUv3L2c3aU5ESQmNi0v4d4tubI1r6v+x28CKlp2YlFr7v+bmYJQzOn7A8K7vsnxrH55bdheBKE4Qgj3JTYjYswfGjIEnnvBr88KISNYnOR+A5EY3ZEG/89gb3YD3ep/G9tgEN5OaWhCKubT7BAa3m8rsrIG8vuIWijVwZcUKlAms7dth2DCYN2//qg0JLXkj7UwAdjWIZ37KgTcW5EfGsDW+Gq0iEzSUCBpE5TFt/bl8tPZyNMB9v61AmcD68sv9xemd3qfxWP+/2DWcMBQblUts9B6y97XkteW3BLwwlbICZQJjwwa47DLnGhNwwrWvkZnQ0uNQxg2JDbIZmXYvEVLCP2Y/R4nWrNOnP6xAmep76y30wQfR5i32X/COmDvn96f7DCOzSXD1yTGB0TpuI/+XNobY6FyeXXq3q8UJrEAZf61YAZdcQnFODpEZGQiQuWUXvyS2dp5vl8qapHY8OOgqVGyYsXDUKTGd2/rcR7FG8vD8cWzY3cn1Y1qBMgdXVOR8CrdsGUycCEAkMKV7f17tezY/tu7qaTxTl5ThXd4htzCexxfdz9Z9rerkqFagzIGefRYmT4boaPjqq/2rs+Ob8kbv03n+2Av86pFtwkeEFFOikbywbBQiyu6CxDo7thUo87s1a+DmmwHY0LQ1+S1SyEhoyeihN5Ndnc6RJkw4HTC7NVvJk4vHkFtY9/3RrEDVF3v3wiOPQG7uwbfxdaq869QbmWg9r+s1oZhLD3+ZwSnTmJ01kED1DK8uK1DhbulS9JhjkIKC/av2NmhU4abRkVHMTunFJ0cMrqt0JghFRxTwt57jSWs1h6nrh/PR2suxAmUCJysLrrnGuVY0bRoCZDVO4pMjBvH8cReSHxXjdUITxK4+8knSWs3h/fRrmJlxlqdZrECFi4ICWLkSHfE3ZNHC/atXtO7CDy07cfepN3oYzoSSf/10AYs3H8uCTf29jmIFKmR99x3FV12NFhUBEJXxC/B7Q/y1tLN4wPokGT+1jtvIUS3n8sX6C8jM7UBmbgevIwFWoELPlCnw0UewahWRP61jSvf+FEZEQo8OlEgk07sdy/ft+1AQFe11UhMiynbA/C7zlDrtRlAVK1Ch5tlnyf/2O7Y2acGG9qncevpISmo4AL4xvZIWcH3qI+zIa8bji+4PquIEVqBCS1YWfP01K5MP49xLx3udxoS4E5K/5ooez5CxqxNPLrk36IoTWIEKHXv3wmHOgPvrmrXxOIwJB3lFDVmR3YcXfhhFfnHFXU+8ZgUq2G3Zgp54IrJ2LQC5MY24c+gtHocyoUoopmPC//gp5zAWbT6BRZuPx6s+Tv6wj3iCVX4+3HortGy5vzg9efzFHHXTe3YvnKmR6IgCbkh9hLv6jaJVXKZvbXD/LlkLKhh9+imce+7+xReOOY+njr/EPpkzNRYblcvNfR7gsGYreD/9GjbtCY3LBFaggpFvFIHPu5/EA4OuYmt8M48DmVBWOgJm67gsXlx2B/M3neR1JL9ZgQpGeXnkxDbhljPv8DqJCQPHtJ5Fi0ZbeGLxWFZtS/U6TrVYgQoG27fD7t3OAHEDBkBmJhEN47xOZUJclBRSpNFM/+UcFm0+nux9oTdGvF0k90pJCdx6KyVtU5xJK9u3h86dIdO5eHnrsNu9zWdCWmrSfMb1H0HL2CxAQrI4gbWgvKEK110HEyYQAcxu15O5KT3ZHN+c/KhoZnY5hrzohl6nNCGqdAryjN2d2FsU2i1xK1B1JScHbr8dXn/9gNWDr36Rn5q39SiUCS9/nII8WDtg+ssKlNu++w7uvx9mz4a8PAB+SWzNvzv15YNep1hxMgFzUpsZDO/6ritTkHsl9L+DYLV9O9xyC7z7LgAL2xzO7phYRg67jR2xdT+2swl/c38dQKQU85+NQ12b6beuWYFyScnQoUQsWADA272HMeaU6zxOZMJRbFQu53V9i4/WXs6+ojj+vXGY15ECygpUoKjC99/DvfdC8+ZIejo7G8Zz9A3vWA9w44qyHTAXbz6Oldt6ex0p4KxABcJ//wsnnnjAqp+S2vHacWdYcTKuKDsF+eOLxpK+PdXrSK6wAlVLJRdcQMRHHwHwc9PWjB5yM/NTjvQ4lQlnHRLWMPKoeynWqDqbgtwrrhYoERkCPI0zY/arqjqu3PMJwLtAii/LeFV9w81MATV+/P7iNPK02/jkSJuuybgvJ78ZG3d34PUVt9TZFOReca1AiUgk8DxwMpAJLBSRKaq6qsxmNwCrVPUMEUkC1ojIe6paUMEug8/cuQAM++tTrGzV2eMwJtwd3nwZ6duOZHteEo8sfNjrOHXCzc8ijwbWqep6X8GZBJSfZEuBxiIiQDywHShyMVPgLF0KkyezLqmdFSfjMuWMjpO4s+89DGg7w+swdcrNU7xkYGOZ5UygX7ltngOmAL8CjYELVbWk/I5EZAQwAiAlJcWVsNWyYwf06QPA1C7HeRzGhDOhmEu7T2Bwu6nMzhrIrMxTvI5Up9xsQVU0VJ+WWz4VWAYcCqQCz4lIkz+8SHWCqqapalpSUlKgc1bPr79CM2d8plWHdODJEy72No8JW6UjYA5uN5Wp64fzyvLbw6J3eHW4WaAygbL3cbTBaSmVdQUwWR3rgJ+Bw1zMVHs33AA4Y4NfePE4G37XuObQ+A0cmbSY99Ov4aO1VxDsw/O6wc1yvBDoIiIdgCzgIqB8c2MDMBj4XkRaAt2A9S5mqrnsbPSSS2H+PHIbxJJ680SKbT4644KYyDwKihuSsaszd856hZz8+juiqmstKFUtAm4EZgDpwIequlJErhWRa32b3Q8cJyLLgW+AUaqa7VamWlm5Epk5gxUNWzDupMutOBlXtI7byEMnXM/xh34DUK+LE7jcD0pVpwHTyq17qczjX4HQuOq3dCkADw28irntenocxoSjslOQb9zd3us4QaF+XXGria1b4fPP4bbbANhlQ/EaF5SfgjzcO2D6ywpUZTZsgG7d9o/j9GK/81jZMnxvKzDeaBmbxc29HyBjdyeeXBycU5B7xQrUweTnQ7t2+xf73vAOW+ObehjIhKvNe5N5ZfntLN3SL+RHwAy08BjVyg2+VtPsdj3peMfnVpxMQAnFXHTYK3RKTAdg3m8DrDhVwApUWQUF8M478Ne/QmIiAF937keJfWJnAqi0A+aQ9p/To/kyr+MENTvFK1VYiB5yCJKTs3/V/DY9mNI9dGZhNcHvwCnIr2ZmxtleRwpqVqBKjRqF5OSQHxnNqVc+x29NksiPivE6lQkjcdG7GH3032kdlxlyU5B7xQoUOBfE33sPgOOue4NtcYne5jFhaW9hHBm7OjJx9dUhNwW5V6xAAXzyCWzZwoamraw4mYDrlLCabXkt2JnfgleX24zR1WEXyfftg0suAeCqc/7hcRgTblKT5nPn0XdzafeXvY4Skup9C6pkyFAigK2xifzUvI3XcUwYKTsF+VurbvA6TkiqvwXqq68oPv8CInN2AnDaFc9adwITIOE3BblX6meB+uYbePllInN28maf03k/dYh1xDQBExOZz9Gtvw+rKci9Uv9+ci+8sH/QuZxGjXlo4FU2d50JiOiIAkApKG7IuAXj2FsYFzZTkHvF7wIlInGqusfNMK7atw+GDIFZswC4+tx/8H37VCtOJiBKO2DuLmjC88vuYk9hY68jhYUqy7uIHCciq3AGnUNEeonIC64nC7RFi/YXp2vPvouvu/QjP7qBx6FMOEhskM1d/UbROXE1izYdT30cmtct/rSgnsSZ3GAKgKr+ICL9XU3lBnXma/jzRQ8yt10vj8OYcFFfpiD3il+neKq6UQ6cHKDYnTjGhI4IKeaWPvcTFVEY9lOQe8WfArVRRI4DVERigJvxne4ZU5+VaCQv/ziS3IIEGwHTJf58xHAtzhTlyThTSaUC17uYyR2FhV4nMGHixOSZnNFxEgA/53Sz4uQif1pQ3VT1krIrROR4YLY7kVywdCn86U8ANhuLqYUDO2BGSDElar9PbvKnBfWsn+uClvomPJjfpgdLD+3mcRoTioRiLuv+EsO7vsucrIE8teQfVpzqwEFbUCJyLHAckCQiZW/BbgKE1v9MQSHLWnflwkse8TqJCUnKtb3G06/190xbfy4frb3cOmDWkcpO8WKAeN82ZXud7QLOczNUwImQG2P3QpmaEpZn9+Gnnd1sBMw6dtACparfAd+JyJuqmlGHmQLr11+RObPB+j6ZakpskE1y/AZWbuvDf7NO9jpOveTPRfK9IvIY0ANoWLpSVQe5liqQXnwRgMyElh4HMaGktANmVEQhd8x6lYLihlW/yAScPyfS7wGrgQ7AP4FfgIUuZgqsoiIARg+5yeMgJlR0Skzn7n53EhVRyOOL/mnFyUP+FKjmqvoaUKiq36nqlcAxLucKqMKoaBC7P8pUrVfSAu7sew+5hfE8MG+89Q73mD+neKU9HH8TkWHAr4ANPWnCUo/my8jKTbEpyIOEPwXqARFJAEbi9H9qAtzqZihj6pbSOCaH3QWJTFx9FdGRhXZaFySqPMVT1S9UNUdVV6jqQFU9CtheB9mMcV1pB8yxx95K4+gclEgrTkGkso6akcAFOPfgTVfVFSJyOvB3oBHQu24i1lBhIcVduhCZkUGxjftkKhAdUcDfeo4nrdUcpq4fTq4NMhd0KjvFew1oCywAnhGRDOBYYLSqflYH2WonL4/IjAxmte/NBz1P8TqNCTIHTkF+DTMzzvI6kqlAZQUqDeipqiUi0hDIBjqr6iZ/dy4iQ4CncW6NeVVVx1WwzQDgKSAayFbVgM4HPatDb6Z2PzGQuzRh4Lyub9E5cbVNQR7kKitQBapaAqCqeSKytprFKRJ4HjgZZ5iWhSIyRVVXldkmEXgBGKKqG0TkkJp8E8ZU10drL2febyexdscRXkcxlajsIvlhIvKj72t5meXlIvKjH/s+GlinqutVtQCYBJRvR18MTFbVDQCquqUm34Qx/uiUmM4tfe4jOiKffUVxVpxCQGUtqO613HcysLHMcibQr9w2XYFoEfkW54bkp1X17fI7EpERwAiAlJSUWsYy9VGvpAVcn/oIO/Ka0ThmF9vzkryOZPxQ2c3Ctb1BuKKu21rB8Y8CBuN8MjhXROap6tpyWSYAEwDS0tLK78OYSpWdgtw6YIYWNyfuzMT5FLBUG5xe6OW3yfbNt7dHRGYBvYC1GBMAg9pO5S89XrQpyEOUm6NuLQS6iEgH32QLF+GbuqqMz4ETRSRKRGJxTgFtQgYTMCu3pfJVxhk8tWSMFacQ5FeBEpFGIlKtsXJVtQi4EZiBU3Q+VNWVInKtiFzr2yYdmA78iNPf6lVVXVGd4xhTXnREASe1mQ4om/cm81763yhWN08WjFuq/F8TkTOA8TgjbHYQkVTgPlU9s6rXquo0YFq5dS+VW34MeKwamY05qLIdMLNy27FuZ20/6zFe8qcFNRany8BOAFVdBrR3K5AxNeVMQT56fwdMK06hz592b5Gq5oiNp2SCWOu4jYxMG0NcdC5PLB7Lqm2pXkcyAeBPgVohIhcDkSLSBWdm4TnuxjKmepo13IqgjFvwMBm7OnsdxwSIP6d4N+GMR54PvA/kYONBmSDRrKFz88HKbX0Y/f0EK05hxp8C1U1V71bVvr6ve1Q1z/VkxlShf/JMHu0/giNaLAagsCTG40Qm0Pw5xXtCRFoDHwGTVHWly5mMqcKBU5D/b8fhXgcyLqmyQKnqQBFphTN43QQRaQJ8oKoPuJ7OmHKEYi7tPoHB7aYyJ2sgr624mWKN9jqWcYlfHTVVdZOqPgNcCywDxrgZypiD6X3IAga3m8q09efyyvLbrDiFOX86anYHLsSZ7nwbzrApI13OVXslJV4nMAGlgLBkyzE8PP9h1uw40utApg7404J6A9gBnKKqJ6nqi6EwblPJRRcBUBwR6XESU1uJDbK56+hRtIn/GRArTvWIP9egQmqSzlL6qzNwwmeHD/A2iKmV0inIY6NziY/Z7XUcU8cqm9XlQ1W9wDeaZtkxmARQVe3perraEOGrzv3YEZvgdRJTQ50S07mtz30UayQPzx9ns/zWQ5W1oG7x/Xt6XQQxpqwOTdZyZ9972JHXjMcX3c/Wfa28jmQ8cNBrUKr6m+/h9aqaUfYLuL5u4tXQV18R+cMPXqcwtbBhdwf+s3EID85/zIpTPebPRfKTK1g3NNBBAupf/wJg6mEneBzEVI8yOOUL4qNzKNZoJq2+xobnrecquwZ1HU5LqWO5WVwaA7PdDlZbu2Mb81mPgV7HMH4q2wEzNmoP/1p/odeRTBCo7BrU+8CXwMPA6DLrd6vqdldTmXql/BTkX6w/3+tIJkhUVqBUVX8RkRvKPyEizaxImUCwKchNZapqQZ0OLKa0G+/vFOjoYi5TT0RFFNI4JsemIDcVqmxevNN9/3aouzimvmjRaBM78lqwq6ApY2Y/a5MamApV+SmeiBwvInG+x5eKyBMiErzT++bnw+TJRBYXeZ3EHESnxHTGHnsbF3R7A8CKkzkof7oZvAjsFZFewJ1ABvCOq6lqY8YMyMqiUOwevGDUK2kBd/a9h9zCeL7OOMPrOCbI+VOgilRVgbOAp1X1aZyuBsEpPx+Av5z/T4+DmPJOTJ7Jzb0fICs3xTpgGr/407beLSJ3AZfhzAIcCQT9IDx7oxt6HcGU0ThmJxd3f4VV21JtCnLjN38K1IXAxcCVqrrJd/3JJto0fnI+AN5dkMiD8x7ltz1t7ZqT8VuVp3iqugl4D0gQkdOBPFV92/VkJuRFRxRwY+rDnNzucwAycztYcTLV4s+neBcAC4DzccYlny8i57kdzIS22KhcRqaNIa3VHGzKV1NT/vw5uxvoWzqKpogkAV8DH7sZzISuxAbZjEwbS+u4TOuAaWrFnwIVUW6I3234OdmCqX9iIvO4u98o4mN28fiisaRvT/U6kglh/hSo6SIyA5joW74QmOZeJBPKCoob8sX68/k5p4uNgGlqzZ8xye8QkXOBE3Dux5ugqp+6nsyElF5JCygojiF9eyrfZQ7xOo4JE5WNB9UFGA90ApYD/6eqWXUVzISO/skz+WuP51iz4wjSt/cCuyxuAqSya0mvA18Aw3FGNHi2ThKZEKKc0XESVx75DKu2pfL0kn9gxckEUmUFqrGqvqKqa1R1PNC+ujsXkSEiskZE1onI6Eq26ysixdZ9IXQIxVzW/SWGd32X2VkDeWrJGOsdbgKusmtQDUWkN7//SWxUdllVl1S2Y98tMc/jjGmeCSwUkSmquqqC7R4BZtTsWzBeUCJoELWPqeuH89Hay7GWk3FDZQXqN+CJMsubyiwrMKiKfR8NrFPV9QAiMgnnhuNV5ba7CfgE6OtnZuOh2KhcYqP3kL2vJa8tvxW1HifGRZUNWFfbGQeSgY1lljOBfmU3EJFk4BycYnfQAiUiI4ARACkpwTsUVbhzOmDeSwTKP+Y8S4nakDbGXW7++auoza/llp8CRqlqcWU7UtUJqpqmqmlJSUmBymeqoXXcRv5xzB20aLSFd9NHWHEydcLNOzczgbZlltsAv5bbJg2YJCIALYDTRKRIVT9zMZepJpuC3HjFzQK1EOgiIh2ALOAinGFb9is73rmIvAl8YcUp2CjndXmb3MJ4m4Lc1LkqC5Q4zZtLgI6qep9vPKhWqrqgstepapGI3Ijz6Vwk8LqqrhSRa33Pv1T7+MZNEVJMiUbywrLRIGqz/Jo6508L6gWgBOdC9n3Abvz81E1Vp1Huvr2DFSZVvdyPLKZOKGd0/IBuzVby5OIx7C5M8DqQqaf8uUjeT1VvAPIAVHUHEONqKuOZsh0wd+Uneh3H1HP+tKAKfZ0pFfaPB1XiairjieiIAkb0HE/fVnOYtv5cPlp7ufVzMp7yp0A9A3wKHCIiDwLnAfe4msp44qojn6Jvqzm8n341MzPO9jqOMX4Nt/KeiCwGBuP0bTpbVdNdT1YThYXoZZfZTRc19MVP57Nk8zEs2NTf6yjGAP6NSZ4C7AX+BUwB9gTtzMLbtiH5+eyLasAvzQ71Ok1IaB23kWEdPwScSQ2sOJlg4s8p3lRK5w6ChkAHYA3Qw8VctfLAoKsojAz6qfs8V7YD5qzMU6wbgQk6/pziHVl2WUT6AH9zLZGpE6lJ87ku9VF25DXj8UX3W3EyQanaPclVdYmI2MgDIeyE5K+4osezZOzuxJOL77XiZIKWPz3Jby+zGAH0Aba6lsi4Lq+oESu29eaFZaNtkDkT1PxpQTUu87gI55rUJ+7EMW4RiumY8D9+yjmMRZtPYNHm47FB5kywq7RA+TpoxqvqHXWUx7ggOqKAv/UcT+oh87n7vy+weW8yVpxMKKhsVpco3w2/feoykAms2Khcbu7zAIc1W8H76df4ipMxoaGyFtQCnOtNy0RkCvARsKf0SVWd7HI2U0ulI2C2jsuyKchNSPLnGlQznOnOB/F7fygFrEAFuWNbf0eLRltsCnITsiorUIf4PsFbwe+FqVT5oXtNEImSQoo0mi9/OZeFm48n2waZMyGqsltdIoF431fjMo9Lv0wQ6pW0gHH9/0bL2CxArDiZkFbptFOqel+dJTG1VjoFecauTuwttL8hJvRVVqDsc+iQ4YyAObzruyzf2ofnlt1lHTBNWKisQA2usxSmVk5qM2P/FOSvr7iFYnVzLgxj6k5lE3dur8sgpubm/jqAqIgi/r3hNBsB04QV+20OUbFRufzl8OdpFLWHgpKGfLPhdCtOJuyE12/0U095naBOJDbI5q5+o+nf5is6Jqz1Oo4xrgmrixVFH3xIFLAk+TCvo7imddxGRqaNIS46lycWj2XVtlSvIxnjmrAqUERE8NnhJ5F+SEevk7iiQ5O1jEy716YgN/VG+JziZWcTtf4nr1O4KqcgkY272/PAvPFWnEy9ED4F6t57AdjZsHEVG4ae7s2WIZSwPe8QHln4MFutd7ipJ8KnQO3bB8CDg67yOEggKWd0nMSoo+9hQNvpXocxps6F1TWozQmHhM1sLkIxl3afwOB2U5mdNZBZmad4HcmYOhdWBSpclI6AmWZTkJt6zgpUEDo0fgNHJi22KchNvWcFKojEROZRUNyQjF2duXPWK+TkN/M6kjGesvOGINE6biMPnXAdxx36DYAVJ2OwFlRQKDsFeebu9l7HMSZouNqCEpEhIrJGRNaJyOgKnr9ERH70fc0RkV5u5glGqUnzubPvPeQWxlsHTGPKca0F5ZtT73ngZCATWCgiU1R1VZnNfgZOUtUdIjIUmAD0cytTsGkZm8VNvR8kY1cnnlxiU5AbU56bp3hHA+tUdT2AiEwCzgL2FyhVnVNm+3lAGxfzBJ3Ne5N5dfltLNlyjI2AaUwF3DzFSwY2llnO9K07mKuALyt6QkRGiMgiEVm0devWAEase0IxFx32Cp0SVgMw97eBVpyMOQg3W1AVjWle4XRVIjIQp0CdUNHzqjoB5/SPtLS0kJ3yqmwHzL2F8fyUE77DwhgTCG4WqEygbZnlNsCv5TcSkZ7Aq8BQVd3mYh5PlZ+CfGbGWV5HMibouVmgFgJdRKQDkAVcBFxcdgMRScGZofgyVQ3boSHjo3MYdfTfbQpyY6rJtQKlqkUiciMwA2cS0NdVdaWIXOt7/iVgDNAceEFEAIpUNc2tTF7ZUxjPhl0dmbj6GhsB05hqcLWjpqpOA6aVW/dSmcdXA1e7mcFLHRPWsD2vOTvzW/DK8pFexzEm5NitLi7plbSAUUf/ncsOf6nqjY0xFbJbXVywfwry3Z14c+WNXscxJmRZgQoo5YyOHzK86zs2BbkxAWAFKoBiIvM5uvUs5mQN5LUVN1Os4TG6pzFesQIVANERBQAUFDdk3IJx7C2MsxEwjQkAexfVUmxULiPTxjCi5+OAsqewsRUnYwIkPN5Je/fC5MlEaHGdHtaZgnwUnRNXs2jTcVR8d48xpqbC4xRvwgTIyWFf09Z1dsjWcRv5v7QxxNoU5Ma4JvQL1NatcNttAFx8wf11csgIKeaWPvcTFVFoU5Ab46LQL1A5OQB8fMRgMhPrZsbdEo1kwo8j2V2QYLP8GuOi0C9QPv9tn+r6Mfonz6RJg518sf4C1ud0c/14xtR34XGR3HXOFORXHvkM3ZquIELq9mK8MfVV2LSg3FJ+CvLXV9xCiUZ6HcuYesEKVKWUa3uNp1/r75m6fjgfr/2r9XEypg5ZgaqUsCK7Nz/tPMxGwDTGA1agKpDYIJvk+I2s3Nab77NO8TqOMfWWna+U0zpuI/cccwcjej5OTGSe13GMqdesBVVG2SnIH1/0TwqKG3odybigSYMIburXlHaJ0YjdnuQaRcnYWUhJcVGN92EFyic1aT7XpT7KjrxmPL7ofuuAGcZu6teUPp0OJSq2Mb6x8I0LVJXmzXdRkJMNrRJrtA87xfM5vPkysnJTeHD+Y1acwly7xGgrTnVARIiKbUJJYUGN91HPW1BK45gcdhckMnH11URHFtppXT0giBWnOiIiaMXz9fql3raghGIu6/4SY4+9lcbROSiRVpyMCTL1sgUVHVHAiJ7j6dtqDtPWn0tuYWOvIxkPnfnc7IDub8qNx/u13TdffsHtIy7js//Mp0PnrgAsnPtf3nr5WZ5784P92/3jtuvp/6dTOXnYWRQWFvL8+Af5Ztq/iI6JoWGjWK4fOZoTBp580OMU5Odz963Xkb58GQlNm/HoC6+T3DblD9tNnzKZV599nOKSEvoPOpnb7r4PgMXzZvPoP//O/9JX8sjzr3HysAP7BObu3sXZA49h0JBh/P2Bx/z63v1V71pQpSNg9m01h/fTr+bDtVda73DjielTPqF332OYPmWy3695fvyDZG/ZzCdfz2HyN3N59o2J7MnNrfQ1n056hyaJCXzx3yVcevV1PPXQ2D9ss3PHdp58cAwTJn3Op9/MZVv2Vub/9zsAWiW35f4nnmfo2ecdJNNDpB1znN/fQ3XUu3fmeV3fonPial5cdgczM872Oo6pp/buyWXpwvmMHf+s3wVq3769TH7/bUbf9wgxDRoA0DzpEE4945xKX/efmV9y5nl/BuDkYWexYPZ3qB54XSgz4xfadexMs+YtAOh3wkl8PW0KAMltU+ja/Qgi5I/lYtWPy9i2dQvH9h/k1/dQXfWuQH209nIeXfgA8zed5HUUU4/9e8ZUjh8wmPYdO5OQmEj68h+qfM3GX9bTKrkN8Y2bVPj82DtuZuUPS/+wfsumX2l1aDIAUVFRxDduws4d2w/YJqV9R35e9z+yNm6gqKiI/8yYxqbfsirNU1JSwuP338Pt99xXZfaaqhcFqlNiOjf3vp/oiHz2FcWxdscRXkcy9dz0zz9hyJnnAnDqmcP58vOPAQ7acdSfTx3HPvYMPXr1/sP6ij5DK7+/JomJ3P3QeO68/kquGH4ah7ZJITKy8kvUH7z9KicMOplWh7apMltNhf1F8l5JC7g+9RF25DWjccwutucleR3J1HM7d2xnwezvWbcmHRGhuLgYEeG2u+8joWlTdufsPGD7nJ07SGzanLbtO7IpK5M9ubuJi/f/g52WrQ5l069ZtGydTFFREbm7d5GQ2PQP2w04eSgDTh4KwMfvvUlkZOXtlx8XL2TJgrl8+PZr7N2zh8LCQmLj4rj1rrF+Z6tK6Legfv75oE/1T57Jzb0f2N8B04qTCQZfTf2c08+7kOnzlvPl3B+ZuWAlyW3bsXTBXNp16MSWzZtY/781APyauYG16Svo1uMIGjWK5ZyLLmXcmNEUFjidH7du3sQXkz+o7HAMOHkIUz6euP/YRx/fv8IW2bbsrQDs2rmTD99+jXP+/JdK9/vws68wY/4Kvpz7I7ffcz+nD78woMUJwqAFVXLxJUQAuTGxB6wf1HYqf+nxok1Bbqrkb7eAQJn++Sdcef2tB6wbfNqZTPvsY/r0O46Hnn6ZMSNvpCA/j6ioaO599BkaN0kA4MY77uG5xx7gnEHHENOgAY1i47h+5F2Acw3q/Euv+MNp3jkXXcbdt17L6Sf0oUliUx59/rX9z11w6ol8OON7AB69dzRr01cCMOKWO2jfsTMAK5Yt4bZrLmNXzk6++3o6Lzwxjk+/mevKz6Y8KX81P9ilpaXpokWL9i8XJyWxIqopZ//lcbTMpwwtY7M4ud2/mLj6KpuC3BzglTNb0zKlo9cx6o+dWfQ8oscBq0RksaqmVfXSkD/F04hIVrbshEoE0REFnNRmOqBs3pvMu+nXWnEyJoS5WqBEZIiIrBGRdSIyuoLnRUSe8T3/o4j0qemxSjtgXnHEc3RKXF274MaYoOBagRKRSOB5YChwOPBnETm83GZDgS6+rxHAizU5VkRCHnf1G72/A+ZPO7vXIrkJd4r+oaOicYeq1mrMLTdbUEcD61R1vaoWAJOA8gN7nwW8rY55QKKIVGv+8r3JRbS4ZR4tGm3micVjrQOmqVLGzkKK9u6yIuUyVaVo7y4iomNqvA83P8VLBjaWWc4E+vmxTTLwm78HKWhRQlSUMHf3G/Rofxg92tcwrak3MvYph2bvJLHBThtP00UK7MyHPt071Xgfbhaoiv7vy//J8mcbRGQEzikgKSkH3oXd4pqXObFFEgOOG1CzlMaYoOVmgcoE2pZZbgP8WoNtUNUJwARwuhkc8OT552PTaBoTnty8BrUQ6CIiHUQkBrgImFJumynAX3yf5h0D5Kiq36d3xpjw5loLSlWLRORGYAYQCbyuqitF5Frf8y8B04DTgHXAXuAKt/IYY0JPyPUkF5GtQEa51S2AbA/i1Fao5obQzR6quSF0s1eUu52qVnlzbMgVqIqIyCJ/us0Hm1DNDaGbPVRzQ+hmr03ukL/VxRgTvqxAGWOCVrgUqAleB6ihUM0NoZs9VHND6Gavce6wuAZljAlP4dKCMsaEIStQxpigFTIFqi7Hlgo0P7Jf4sv8o4jMEZFeXuQsr6rcZbbrKyLFIlLxzI4e8Ce7iAwQkWUislJEvqvrjBXx43clQUT+JSI/+HIHRedmEXldRLaIyIqDPF+z96eqBv0XTk/0n4COQAzwA3B4uW1OA77EuQH5GGC+17mrkf04oKnv8dBgyO5P7jLb/RvnroDzvM5djZ95IrAKSPEtHxIiuf8OPOJ7nARsB2KCIHt/oA+w4iDP1+j9GSotqDoZW8olVWZX1TmqusO3OA/npmmv+fMzB7gJ+ATYUpfhquBP9ouByaq6AUBVgyG/P7kVaCzOtCzxOAWqqG5j/pGqzvJlOZgavT9DpUAdbNyo6m7jhermugrnL43XqswtIsnAOcBLdZjLH/78zLsCTUXkWxFZLCKVz7FUN/zJ/RzQHWfUj+XALapaUjfxaqVG789QmXYqYGNLecDvXCIyEKdAneBqIv/4k/spYJSqFvsz820d8id7FHAUMBhoBMwVkXmqutbtcJXwJ/epwDJgENAJ+EpEvlfVXS5nq60avT9DpUAFbGwpD/iVS0R6Aq8CQ1V1Wx1lq4w/udOASb7i1AI4TUSKVPWzOkl4cP7+vmSr6h5gj4jMAnoBXhYof3JfAYxT58LOOhH5GTgMWFA3EWusZu9Pry+u+XkBLgpYD3Tg94uHPcptM4wDL8It8Dp3NbKn4Aw5c5zXeauTu9z2bxI8F8n9+Zl3B77xbRsLrACOCIHcLwJjfY9bAllAC69/5r487Tn4RfIavT9DogWlITy2lJ/ZxwDNgRd8rZEi9fiudT9zByV/sqtquohMB34ESoBXVbXCj8jrip8/8/uBN0VkOc6bfZSqej4Ei4hMBAYALUQkE7gXiIbavT/tVhdjTNAKlU/xjDH1kBUoY0zQsgJljAlaVqCMMUHLCpQxJmhZgQpzvlEGlpX5al/JtrkBON6bIvKz71hLROTYGuzjVRE53Pf47+Wem1PbjL79lP5cVvhGB0isYvtUETktEMc2/rNuBmFORHJVNT7Q21ayjzeBL1T1YxE5BRivqj1rsb9aZ6pqvyLyFrBWVR+sZPvLgTRVvTHQWczBWQuqnhGReBH5xte6WS4ifxihQERai8isMi2ME33rTxGRub7XfiQiVRWOWUBn32tv9+1rhYjc6lsXJyJTfWMbrRCRC33rvxWRNBEZBzTy5XjP91yu798PyrZofC234SISKSKPichC37hDf/PjxzIX342rInK0OGNyLfX9202cmbHvAy70ZbnQl/1133GWVvRzNAHgdfd4+3L3CyjGubl0GfApzu0UTXzPtcDp2Vvaks71/TsSuNv3OBJo7Nt2FhDnWz8KGFPB8d7Ed8sLcD4wH+em3OVAHM4QISuB3sBw4JUyr03w/fstTmtlf6Yy25RmPAd4y/c4BudO+UbACOAe3/oGwCKgQwU5c8t8fx8BQ3zLTYAo3+M/AZ/4Hl8OPFfm9Q8Bl/oeJ+Lcwxfn9f93uH2FxK0uplb2qWpq6YKIRAMPiUh/nFs8knHu6dpU5jULgdd9236mqstE5CTgcGC273acGJyWR0UeE5F7gK04ozMMBj5V58ZcRGQycCIwHRgvIo/gnBZ+X43v60vgGRFpAAwBZqnqPt9pZU/5fXTPBKAL8HO51zcSkWU4948tBr4qs/1bItIF52776IMc/xTgTBH5P99yQ5x7KtOr8T2YKliBqn8uwRmJ8ShVLRSRX3DeXPup6ixfARsGvCMijwE7gK9U9c9+HOMOVf24dEFE/lTRRqq6VkSOwrlH62ERmamq9/nzTahqnoh8izP8yIXAxNLDATep6owqdrFPVVNFJAH4ArgBeAbnXrf/qOo5vg8Uvj3I6wUYrqpr/MlrasauQdU/CcAWX3EaCLQrv4GItPNt8wrwGs5QrvOA40Wk9JpSrIh09fOYs4Czfa+Jwzk9+15EDgX2quq7wHjfccor9LXkKjIJ56bTE3FusMX373WlrxGRrr5jVkhVc4Cbgf/zvSYBZ4QAcE7rSu3GOdUtNQO4SXzNSRHpfbBjmJqzAlX/vAekicginNbU6gq2GQAsE5GlONeJnlbVrThv2Iki8iNOwTrMnwOq6hKca1MLcK5JvaqqS4EjgQW+U627gQcqePkE4MfSi+TlzMQZC/trdYbIBWdMrVXAEnEG8H+ZKs4UfFl+AC4CHsVpzc3GuT5V6j/A4aUXyXFaWtG+bCt8yybArJuBMSZoWQvKGBO0rEAZY4KWFShjTNCyAmWMCVpWoIwxQcsKlDEmaFmBMsYErf8HQDLnH0j7xX0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_roc_curve(y_true, y_proba)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "happy-capital",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQoAAAEcCAYAAAA/RPpPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAVxElEQVR4nO3deZgU1bnH8e/bM8O+iiCLgopBDRojeN1xjag3akSJiHoTiMQtxqhRE6Pi/hijeZIYN+LF7RITjUQUFxABFVGRRaNERCHKZtgF2ZnlvX9UD7TjMKcmTE0XPb/P8/RD9anq7rdh5sep03VOm7sjIlKTTL4LEJH0U1CISJCCQkSCFBQiEqSgEJEgBYWIBCkoJDFmtruZuZkdVIvHPGpmz9fBa7uZ9d/e55GIgqLAZH9Baro9mu8aZcdTnO8CpM51ytk+BXioStuG3IPNrMTdS+ujMNlxqUdRYNx9ceUNWJXbBjQBVpnZQDObYGYbgAvNbJCZrc19HjM7JtsD2Tmn7XAze83M1pvZIjN7wMxaxa3NzIrMbLiZfWpmG8zsEzO7xsy+9nNoZteb2RIzW2tmj5hZ05x9ln3c3OzzfGBm59X6L0tiU1A0THcA9wPfBEbFeYCZ7Q+8DDwHHACcAXwbeLgWr5sBFgFnAfsC1wG/AgZXOe7o7GscD5wJ9AXuzNl/G3A+8JPse7gDGGZm361FLVILOvVomP7o7k9X3jGzOI+5GnjS3X+b87iLgXfNrIO7Lw09QfYUZ2hO02dm1gsYCAzPaS8HBrv7WmCmmf0CGG5m12b3Xwn0dfdJ2fufmtnBRMHxQpw3I7WjoGiYpv0Hj+kN7GVmA3LaKhOmOxAMCgAzuwgYAnQDmgIlwLwqh72fDYlKbwGNsq/TmOgUaoyZ5c5oLAE+i/VOpNYUFA3Tuir3K9j6S1+ppMr9DPC/wO+qeb5FcV40GzK/B64C3gS+JOoF9Ivz+Jw6AE4F5lfZp0HZhCgoBGAZ0MzMWrn7l9m2b1c5ZgbQ093nbMfrHAlMcfd7KxvMrHs1x+1vZs3dvTLQDgU2A3OJgmIT0M3dJ2xHLVILCgoBmELUy7jDzH5HNJB4SZVj7gTeNrMHgWHAGmAf4FR3vzDm63wMDDKzk4E5wNlEA5dfVDmuGHjYzG4BOgO/Bh6qDA4zuxu426LBldeBFkRhUuHuf4r/tiUufeohuPtK4FzgBOAD4ALghirHvA8cBewOvAb8g+jThiW1eKlhwFPAE8DU7HP9tprjXgP+CUwEngEmANfk7L8BuInoFOafwDiiT0c+rUUtUgumFa5EJEQ9ChEJUlCISJCCQkSCFBQiEqSgEJEgBUU9MLOuZjbazNaZ2XIzu8fMGuW7Lqk7ZnaGmY01s2XZWbfH5LumuqSgSJiZFRFNVGoJ9CGaANWf6q8fkB1Xc6LL0q/MdyFJ0HUUCctehfgC0SXHC7Jt5xHNm+iQc8m0FIDs+h3LgGPd/dU8l1Nn1KNI3mHArMqQyBpLNAuyd35KEqkdBUXyOvL1y5yXE6250LH+yxGpPQVF/djW+Z3O+2SHoKBI3mK+3nPYGSiidhOqRPJGQZG8t4B9zWzXnLYTiNZUmJ6fkkRqR+tRJO9loqnQj5vZz4F2wF1E6yvoE48CYWY7AV2BNtmmvcxsFVC5AvoOTT2KhLl7OfBdYD0wGXgS+DvRWgpSOE4D3iVaQwOi71N5F7gobxXVIV1HISJB6lGISJCCQkSCFBQiEqSgEJEgBYWIBCkoRCRIQSEiQQoKEQlSUIhIkIJCRIIUFCISpKAQkSAFhYgEKShEJEhBISJBCgoRCVJQiEiQgkJEghQUIhKkoBCRIAWFiATtEN/r0bptO+/YZbd8lyG10KLxDvGjJTlmzJi+3N3bV7dvh/jX7NhlN4aNHJ/vMqQWDu3eLt8lSC01LbF529qnUw8RCVJQiEiQgkJEghQUIhKkoBCRIAWFiAQpKEQkSEEhIkEKChEJUlCISJCCQkSCFBQiEqSgEJEgBYWIBCkoRCRIQSEiQQoKEQlSUIhIkIJCRIIUFCISpKAQkSAFhYgEKShEJEhBISJBCgoRCVJQiEiQgkJEghQUIhKkoBCRIAWFiAQpKEQkSEEhIkEKChEJUlCISJCCQkSCFBQiEqSgEJEgBYWIBCkoRCSouL5eyMy6AR0BB5a4+7z6em0R2T6J9yjM7AozWwD8C3gLeBv4l5ktMLPLk359Edl+ifYozOwG4GrgTmAssAQwoAPQF7jZzFq4+21J1iEi2yfpU48LgB+6+zNV2ucD08zsI+CPgIJCJMWSPvVoB3xUw/7ZQNuEaxCR7ZR0ULwDDDWzRlV3ZNuuyx4jIimW9KnHpcA4YKmZTSIao3CiTz/6AOuBExKuQUS2U6I9CnefCfQArgVWArsBXbPbvwT2cfd/JlmDiGy/xK+jcPc1wAPZm4jsgOrtgquGaOm/F3HHLy5h5fKlWCbDKWf9gP4/uJAvV33BLVcOYfGi+XTs0pUbfzeclq3bUFZayl3XX84nH75PeXkZfb83gHMvvDzfb6NB23uv3WnZoiVFRUUUFxczeco0br7xBp5/7lkymQztO3TgT8MfpXPnzvkuNVG6hDtBRUVFXPyLW3jsxbe4/69jePbPw/lszmyeeOgP9Dr0KEaMnUqvQ4/iiYf+AMCrY56ltHQTD4+exLCR4xn95GMsXjg/z+9CxrwykSnT32PylGkAXPHzq5n67vtMmf4eJ//3Kdxx2y15rjB5CooEtevQkR49DwCgWYuWdO3eg+VL/s2b41/ixNMHAHDi6QOY/MqLAJgZG9evp7ysjE0bN1JSUkKzFi3zVr9Ur1WrVlu2169fh5nlsZr6oVOPerJ44XzmzPqAfQ/ozcoVy2jXoSMQhckXK5cDcPSJpzF5wkuc2acnmzZu4JJf3kqrNrrMJJ/MjFNP7ouZcf6PL+T8H18AwI03XMefRzxO69atGTNuYp6rTJ56FPVgw7q1DL1sED+59naa19BDmPXBDDKZIp5+fSZPvDKdvz1yP58v+Kz+CpWvmfDaZN6aOoNRz7/EsAfu441JrwNw8623M+fTBZw98FwevP/ePFeZPAVFwspKSxl62WC+c2p/jup7CgA7tWvPiqWLAVixdDFtd9oZgPHPj+TgPsdTXFJC23bt6dnrEGbPfC9fpQtsGaTs0KEDp53ej6lTv3p94Flnn8OoZ0bmo7R6paBIkLvzm+t/RrfuPThr8CVb2g8/7iTGjnoSgLGjnuTw408GYJdOu/Lu25NwdzasX8esf0yj657fyEvtAuvWrWPNmjVbtl8Z9zI9e+7HnE8+2XLMC6Ofo8fe++SrxHqjMYoEzZwxhXHPPsWePb7JkNOPAWDIFdcx8Mc/4+YrzufFkSPo0GlXbvr9wwCcfs6PuPNXlzH41CPBnZPOGEj3vXvm8R00bEuXLGFA/34AlJWXMeDsc+h74kmcfdaZfPLxbDKWoWu3btxz34N5rjR55u75riFo7/2+7cNGjs93GVILh3Zvl+8SpJaalth0dz+oun3b7FGYWa+4L+DuM/6TwkRkx1DTqcc0oglcoQ+JHSiqs4pEJHVqCoo96q0KEUm1bQaFFr8VkUqxPx41s/3N7F4ze8nMOmXbTjezA5MrT0TSIFZQmFlfYCrQBTgOaJrd1R24MZnSRCQt4vYobgWudPd+wOac9leBg+u6KBFJl7hB0RN4sZr2lcBOdVeOiKRR3KD4gui0o6pewMK6K0dE0ihuUDwB3GVmuxJdN1FsZkcDdwOPJ1WciKRD3KC4HvgUmAe0AD4EJgBvALcnU5qIpEWsSWHuXgqca2ZDgQOJAuZdd/+k5keKSCGo1exRd59rZkuy22uTKUlE0qY2F1xdbmbzgdXA6uy3kV9hDWHBQJEGLlaPwsx+Q/SFw3cBb2WbDwOGAp2AaxKpTkRSIe6pxxBgiLs/ndM2wcxmA8NQUIgUtNoshff+Ntq0nJ5IgYv7S/448JNq2i8G/q/uyhGRNKpphat7qhx3npmdCLydbTsE6Az8ObnyRCQNahqj2L/K/enZP7tl/1ycvRX+EsQiDVxNC9ccW5+FiEh6aSBSRIJiX5lpZscCA4GuQKPcfe5+XB3XJSIpEneFq0HAS0BL4BhgGdCWaJr5hwnVJiIpEffU4yrgUncfCJQC17r7gcAIQHM+RApc3KDYE3glu72JaKo5wL3AoDquSURSJm5QrCA67QBYBOyX3W7H1oV2RaRAxR3MnAT0BT4AngLuMbMTgOOBcQnVJiIpETcoLgWaZLfvAMqAI4hC47YE6hKRFIm7wtXKnO0K4M7EKhKR1KlprkfsZfhzg0RECk9NPYrlRCtu18TQt5mLFLyagkJzPUQEqHlS2Gv1WYiIpJcmhYlIkIJCRIIUFCISpKAQkSAFhYgE1XTB1cNxn8Tdf1Q35YhIGtV0HUX7KvePAiqIJoZBNIM0A7yeQF0ikiI1XUdxauW2mV0LbAAGu/u6bFtzYDhbg0NEClTcMYrLgJsqQwIgu30r8NMkChOR9IgbFC2Ivuynqk5As7orR0TSKG5QjAQeMbOzzWz37O1solOPvydXnoikQdyFay4Gfgs8CpRk28qIguKqui9LRNIk7sI1G4BLzOxqoDvR9PI5uWMWIlK4anvBVdPs7SOFhEjDEfcLgFqa2d+ApcCbQJds+4NmdlNy5YlIGsQdo7iT6FOPXsAbOe3PA7cDN9VtWV/VpKSIfTu1SvIlpI61/a9L812C1KG4QXEa0M/d3zOz3OXxZhF9OZCIFLC4YxRtib4EqKqWQHndlSMiaRQ3KKYS9SoqVfYqLiQasxCRAhb31ONXwFgz65l9zJXZ7YOJJouJSAGL1aNw9zeBw4FGwFyirxL8HDjM3WckV56IpEHcHgXu/gHwwwRrEZGUinsdRbmZdaimvZ2ZaTBTpMDFHcy0bbQ3BjbXUS0iklI1nnqY2ZXZTQcuMrO1ObuLgD7ARwnVJiIpERqjqFyUxoAhfPWaic3AZ8BFdV+WiKRJjUHh7nsAmNlE4Ax3/6JeqhKRVIn7qcdJVDNOYWZNgAp31ziFSAGLO5j5FHBJNe0XZfeJSAGLGxRHAC9X0z6O6EIsESlgcYOiGdHSd1VVEE0ME5ECFjco3gcGVtN+DjCz7soRkTSKO5h5KzDKzPYCJmTbjge+D/RLojARSY+4k8JeAE4FugH3ZG9dgdPc/fnkyhORNKjNpLAxwJgEaxGRlKrtKtwi0gBts0dhZl8Ce7r7cjNbw9ZVrb7G3bXyrUgBq+nU46fAmuy2llQWacC2GRTu/lh12yLS8GiMQkSCahqjqKCGcYlc7l5UZxWJSOrUNEZxFluDYhfgFuAZ4K1s22HA6cCNSRUnIulQ0xjF05XbZvYccK27P5RzyMNm9g5RWNyfWIUikndxxyiOAyZW0z4ROKbOqhGRVIobFMuB/tW09weW1V05IpJGcS/hHgo8YmbHsnWM4lDgO8D5SRQmIukRKyjc/XEzmw1cRvQdpAZ8CBzh7lMSrE9EUqA2k8KmAOcmWIuIpFTsC67MbBczu8rM7jeznbNtR5jZHsmVJyJpEPcrBXsDs4l6FEOAyklgJwC3J1OaiKRF3B7F3cAf3P1AYFNO+1iihXdFpIDFDYreQHUTw/5NdNWmiBSwuEGxAWhbTfs+wNK6K0dE0ihuUDwL3GhmjbP33cx2B+4ERiZRmIikR9yguArYiegqzGbAG8AcYBVwfSKViUhqxL2OooxoTsdRQC+igJnh7q8kVJeIpEgwKMysCFgNHODuE9j6vR4i0kAETz3cvRyYBzRKvhwRSaO4YxS3Ar+uvCJTRBqWuGMUVwF7AIvMbCGwLnenu3+rrgsTkfSIGxQjibl+pogUnrjTzG9KuA4RSbEaxyjMrJmZ3Wdmi8xsqZk9oXEKkYYnNJh5MzAIeAH4K9Fs0QcSrklEUiZ06nEGcL67/xXAzEYAk82sKPuxqYg0AKEexW7ApMo77v4O0VWanZMsSkTSJdSjKAI2V2kri/E4qcZDD/yREY8Nx90574fnc8Ell3HBoHOYO+djAFavXk3r1q0Z/8a0PFfasHnZJkoXTMQ3rgCMkq7HUb5qLhVffgZWhDVuRclux2PFjXEvp3T+RHzDMnCnaKe9Kd6ld77fQp0L/cIbMMLMcheraQI8ZGbrKxvc/bQkiisksz6cyYjHhvPShDdp1KgRA884he+ceDJ/evSJLcfceN01tGrVqoZnkfpQumgSmVZdKd7jJLyiHCrKyLTcjeLOh2GWofTzNylbOp2SzodTsWoueDmN9xmIV5SyadZfyLT5BpnGhfXvGDr1eAz4HFiRcxsBLKjSJgGfzP6I3gcdQrNmzSguLuawI/vw4uhnt+x3d0Y/8zT9+g/IY5Xi5ZvxdZ9TtNO+AFimCCtuTFGrrphFvy6ZZh3x0rVbH1RRhnsFVJRjmQxWVHizHWrsUbj74PoqpNDt882e/PrWoaxcuYImTZoy/uUxHHDg1i7q22++wc7tO7Bn92/ksUrxTauhuCml8yfgG5eTadqe4i59sKKSLceUr5xFUZu9AMi06U756k/ZNPMR8DKKOx+JFTfJV/mJ0VhDPemx975cevnVDPjeyTRv0YKe+32L4uKtf/3PPP2kehOp4Pj6ZZR06UOmeUdKF06ibOkMSjodAkDZ4mlgRqZtj+jodUvBjMb7DYKyTWye8wyZlruSadw6j++h7sVerl+23zk/GMy4Se8w6qUJtGnblj32jP5XKisr48XRo/jeGd/Pc4ViJc2hpAWZ5h0BKGrTPRqoBMpXfkT5l59R0u0EzCxqW/UxRS27YVaElTQj07wjvr7wVodUUNSjZcuiH6CFC+bz4uhRW3oQr786nr167E3nLrvmszwhCgpr1IKKjV8AUL5mIda4LeVfzqNsyQwa7fldLFOSc3xLKtYuxN3x8lIq1i3BmlS3vOyOTace9WjI/wxg5coVlJSUcMfd99CmbfQDNWrkU/Q7U6cdaVHSpQ+l88aBV2CNWlHS9Tg2f/w33CvYPCcagM4070jJbsdQtPN+lM6fwObZfwGHonb7kGlaeLMczD1/k0LNrBjo7O7zazrugAN7+8uvvV1PVUld2P3oK/JdgtTSxvfum+7uB1W3L9+nHj2BT/Ncg4gE5DsoRGQHkOgYhZn9K3BI4V2ZIlKAkh7M7AQ8Dny8jf1dgJ8lXIOIbKekg2Im8L6731fdTjM7AAWFSOolPUYxGehRw/61wOsJ1yAi2ynRHoW7Xx7YPxc4NskaRGT76VMPEQlSUIhIkIJCRIIUFCISpKAQkSAFhYgEKShEJEhBISJBCgoRCVJQiEiQgkJEghQUIhKkoBCRIAWFiAQpKEQkSEEhIkEKChEJUlCISJCCQkSCFBQiEqSgEJEgBYWIBCkoRCRIQSEiQQoKEQlSUIhIkIJCRIIUFCISpKAQkSAFhYgEKShEJEhBISJBCgoRCVJQiEiQgkJEghQUIhKkoBCRIAWFiAQpKEQkSEEhIkEKChEJUlCISJC5e75rCDKzZcC8fNchUuC6uXv76nbsEEEhIvmlUw8RCVJQiEiQgkJEghQUIhKkoBCRIAWFiAQpKEQkSEEhIkEKChEJ+n8noVlcLnjtygAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_confusion_matrix(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "prepared-approval",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=========================\n",
      "        RESULTS\n",
      "=========================\n",
      "Accuracy:  0.85\n",
      "-------------------------\n",
      "sensitivity:  0.92\n",
      "specificity:  0.68\n",
      "-------------------------\n",
      "positive predictive value:  0.87\n",
      "negative predictive value:  0.8\n",
      "-------------------------\n",
      "precision:  0.87\n",
      "recall:  0.92\n"
     ]
    }
   ],
   "source": [
    "summerize_results(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ignored-colonial",
   "metadata": {},
   "source": [
    "### Lastly,\n",
    "- Also, tested the GAT model, but it did rather perform slightly worse than GCN model.\n",
    "- Unexpectedly, Graph Neural Net models performed even worse than molecular descriptors based model, see the LightGBM model with RDKit Descriptors (123 different molecular properties) in separate Notebook.\n",
    "    - This paper: https://jcheminf.biomedcentral.com/articles/10.1186/s13321-020-00479-8 shows that graph based models perform at best as good as descriptor based models for molecular representation of drug molecules.\n",
    "\n",
    "\n",
    "- TODOs:\n",
    "    - Implement conformal oversampling for molecules: https://jcheminf.biomedcentral.com/articles/10.1186/s13321-020-00420-z\n",
    "    - Exhaustive hyper-parameters tuning for GCN model [it's computationally expensive to carry-out more wider search with longer epochs, it took > 12hrs in Dell XPS 15 for 50 runs each with only 20 epochs.]\n",
    "    - Regression model with GCN for aqueous solubility prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "governing-ideal",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}