Adam_MNIST_all_plots.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## Tuning Adam on MNIST and plotting the results"
      ],
      "metadata": {
        "id": "jsUlLqgDkcK3"
      },
      "id": "jsUlLqgDkcK3"
    },
    {
      "cell_type": "markdown",
      "id": "9e359ffb-71a2-43db-87a3-4de42e63aa55",
      "metadata": {
        "id": "9e359ffb-71a2-43db-87a3-4de42e63aa55"
      },
      "source": [
        "# 1.Imports & environment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "id": "2a167aec-3ddd-4067-8b5f-77af4fe2854f",
      "metadata": {
        "id": "2a167aec-3ddd-4067-8b5f-77af4fe2854f"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from torch import nn\n",
        "import torch.nn.functional as F\n",
        "import matplotlib.pyplot as plt\n",
        "import os\n",
        "import gzip\n",
        "import numpy as np\n",
        "import sys\n",
        "# os.environ[\"KMP_DUPLICATE_LIB_OK\"]=\"TRUE\"\n",
        "\n",
        "# Setup predictable randomization\n",
        "seed = 10\n",
        "np.random.seed(seed)\n",
        "\n",
        "# Setup CUda\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "8c42f77e-92d5-4cb3-9099-b699d7c7858d",
      "metadata": {
        "id": "8c42f77e-92d5-4cb3-9099-b699d7c7858d"
      },
      "source": [
        "# 2. Loading and preparing the data\n",
        "As a basis for comparison we will be using the MNIST dataset. If we manage to do all the work we want, we will then use other datasets for comparison."
      ]
    },
    {
      "cell_type": "markdown",
      "id": "3668867e-acbe-4676-84dc-1911b88c0729",
      "metadata": {
        "id": "3668867e-acbe-4676-84dc-1911b88c0729"
      },
      "source": [
        "### 2.1. Definition of methods to extract data and labels"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        " Source: Image Analysis and Pattern Recognition (EE-451) lab 3"
      ],
      "metadata": {
        "id": "Bj25NDEMlLgs"
      },
      "id": "Bj25NDEMlLgs"
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "id": "febd210c-2240-45d1-8860-14be25f58c40",
      "metadata": {
        "id": "febd210c-2240-45d1-8860-14be25f58c40"
      },
      "outputs": [],
      "source": [
        "def extract_data(filename, image_shape, image_number):\n",
        "    with gzip.open(filename) as bytestream:\n",
        "        bytestream.read(16)\n",
        "        buf = bytestream.read(np.prod(image_shape) * image_number)\n",
        "        data = np.frombuffer(buf, dtype=np.uint8).astype(np.float32)\n",
        "        data = data.reshape(image_number, image_shape[0], image_shape[1])\n",
        "    return data\n",
        "\n",
        "\n",
        "def extract_labels(filename, image_number):\n",
        "    with gzip.open(filename) as bytestream:\n",
        "        bytestream.read(8)\n",
        "        buf = bytestream.read(1 * image_number)\n",
        "        labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64)\n",
        "    return labels"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "a048581f-bdb6-4a22-9b0c-9f371c1303b3",
      "metadata": {
        "id": "a048581f-bdb6-4a22-9b0c-9f371c1303b3"
      },
      "source": [
        "### 2.2. Load data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "id": "c7c71149-8c1e-4fbd-b2b9-2e332fcf8e1b",
      "metadata": {
        "id": "c7c71149-8c1e-4fbd-b2b9-2e332fcf8e1b"
      },
      "outputs": [],
      "source": [
        "image_shape = (28, 28)\n",
        "train_set_size = 60000\n",
        "test_set_size = 10000\n",
        "data_folder = 'drive/MyDrive/mnist_data'\n",
        "\n",
        "train_images_path = os.path.join(data_folder, 'train-images-idx3-ubyte.gz')\n",
        "train_labels_path = os.path.join(data_folder, 'train-labels-idx1-ubyte.gz')\n",
        "test_images_path = os.path.join(data_folder, 't10k-images-idx3-ubyte.gz')\n",
        "test_labels_path = os.path.join(data_folder, 't10k-labels-idx1-ubyte.gz')\n",
        "\n",
        "train_images = extract_data(train_images_path, image_shape, train_set_size)\n",
        "test_images = extract_data(test_images_path, image_shape, test_set_size)\n",
        "train_labels = extract_labels(train_labels_path, train_set_size)\n",
        "test_labels = extract_labels(test_labels_path, test_set_size)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "eff8b6f8-0472-4936-a88d-cde07e2359b4",
      "metadata": {
        "id": "eff8b6f8-0472-4936-a88d-cde07e2359b4"
      },
      "source": [
        "### 2.3. Convert data from numpy arrays to torch tensors"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "id": "b3b5bfe7-685e-4adc-8a48-fc1a25a39381",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b3b5bfe7-685e-4adc-8a48-fc1a25a39381",
        "outputId": "dec151c2-5b5b-4df3-8546-dde6bb938070"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training features: torch.Size([60000, 28, 28]) \n",
            "Testing features: torch.Size([10000, 28, 28])\n",
            "Training labels: torch.Size([60000]) \n",
            "Testing labels: torch.Size([10000])\n"
          ]
        }
      ],
      "source": [
        "features_train=torch.from_numpy(train_images).to(device)\n",
        "features_test=torch.from_numpy(test_images).to(device)\n",
        "print('Training features:', features_train.shape, '\\n'\n",
        "'Testing features:', features_test.shape)\n",
        "\n",
        "labels_train=torch.from_numpy(train_labels).to(device)\n",
        "labels_test=torch.from_numpy(test_labels).to(device)\n",
        "print('Training labels:', labels_train.shape, '\\n'\n",
        "'Testing labels:', labels_test.shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "072269a3-e3f6-470c-8ca9-58a0555d9851",
      "metadata": {
        "id": "072269a3-e3f6-470c-8ca9-58a0555d9851"
      },
      "source": [
        "### 2.4. Normalize data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "id": "d24f2cbd-0279-44d6-89d2-1dca355248df",
      "metadata": {
        "id": "d24f2cbd-0279-44d6-89d2-1dca355248df"
      },
      "outputs": [],
      "source": [
        "mean, std = features_train.float().mean(), features_train.float().std()\n",
        "\n",
        "features_train = features_train.float().sub_(mean).div_(std)\n",
        "features_test = features_test.float().sub_(mean).div_(std)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#reshape to make the 1st channel be a batch size\n",
        "\n",
        "features_train = features_train.reshape(-1, 1, 28, 28)\n",
        "features_test = features_test.reshape(-1, 1, 28, 28)"
      ],
      "metadata": {
        "id": "gO2vsS-k6GY9"
      },
      "id": "gO2vsS-k6GY9",
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "id": "bdc94097-21c0-4308-ba10-af4dc6d04f30",
      "metadata": {
        "id": "bdc94097-21c0-4308-ba10-af4dc6d04f30"
      },
      "source": [
        "# 3. Setting up networks and evaluation methods"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "93eeb387-7739-466e-972b-3f10a9a358a7",
      "metadata": {
        "id": "93eeb387-7739-466e-972b-3f10a9a358a7"
      },
      "source": [
        "### 3.1. Multilayer perceptron (MLP)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "id": "13013745-d51c-4348-9f26-211b04d3cc1d",
      "metadata": {
        "id": "13013745-d51c-4348-9f26-211b04d3cc1d"
      },
      "outputs": [],
      "source": [
        "class MLP(nn.Module):\n",
        "    \n",
        "    def __init__(self, hidden_size_1=512, hidden_size_2=100, hidden_size_3=10):\n",
        "        super(MLP, self).__init__()\n",
        "        \n",
        "        self.layers = nn.Sequential(\n",
        "            nn.Flatten(),\n",
        "            nn.Linear(784, hidden_size_1),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.Linear(hidden_size_1, hidden_size_2),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.Linear(hidden_size_2, hidden_size_3))\n",
        "    \n",
        "    def forward(self, x):\n",
        "        return self.layers(x)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###3.2. Convolutional Neural Network (CNN)"
      ],
      "metadata": {
        "id": "g3hZg487lszk"
      },
      "id": "g3hZg487lszk"
    },
    {
      "cell_type": "code",
      "source": [
        "class CNN(nn.Module):\n",
        "   def __init__(self):\n",
        "        super(CNN, self).__init__()\n",
        "        self.conv1 = torch.nn.Conv2d(1, 32, kernel_size=5, stride=1, padding='same')\n",
        "        self.conv2 = torch.nn.Conv2d(32, 64, kernel_size=5, stride=1, padding='same')\n",
        "        self.fc1 = nn.Linear(64*7*7, 1024)\n",
        "        self.fc2 = nn.Linear(1024, 10)\n",
        "\n",
        "   def forward(self, input:torch.Tensor) -> torch.Tensor:\n",
        "        pool1 = torch.max_pool2d(F.relu(self.conv1(input)), kernel_size=2, stride=2)\n",
        "        pool2 = torch.max_pool2d(F.relu(self.conv2(pool1)), kernel_size=2, stride=2)\n",
        "        res = pool2.reshape(-1, 64*7*7)\n",
        "        hidden = F.relu(self.fc1(res))\n",
        "        output = self.fc2(hidden)\n",
        "\n",
        "        return output\n"
      ],
      "metadata": {
        "id": "yUu42pH8l5md"
      },
      "id": "yUu42pH8l5md",
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "id": "e119d2d9-562a-4bdf-8ce7-eea9d59272fa",
      "metadata": {
        "id": "e119d2d9-562a-4bdf-8ce7-eea9d59272fa"
      },
      "source": [
        "### 3.3. Implementation of method for training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "id": "8e5ca3dd-313e-4c15-99da-10937d42595c",
      "metadata": {
        "id": "8e5ca3dd-313e-4c15-99da-10937d42595c"
      },
      "outputs": [],
      "source": [
        "def run_nn(x_train, y_train, x_test, y_test, model, optimizer, criterion, num_epoch, size_minibatch):\n",
        "\n",
        "    loss_train_ret = 0\n",
        "    loss_test_ret = 0\n",
        "    loss_train = 0\n",
        "            \n",
        "    for epoch in range(num_epoch):\n",
        "        for b in range(0, x_train.size(0), size_minibatch):\n",
        "            y = model(x_train[b:b+size_minibatch])\n",
        "            loss_train = criterion(y, y_train[b:b+size_minibatch])\n",
        "        \n",
        "            optimizer.zero_grad()\n",
        "            loss_train.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "        if epoch == num_epoch - 1:\n",
        "\n",
        "            y_test_obt = model(x_test)\n",
        "            loss_test = criterion(y_test_obt, y_test)\n",
        "            \n",
        "            loss_train_ret = loss_train\n",
        "            loss_test_ret = loss_test\n",
        "            \n",
        "            print('Final, Train Loss: %.4f, Test Loss: %.4f' %(loss_train, loss_test))\n",
        "\n",
        "    return loss_train_ret, loss_test_ret"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "93565277-6e32-47ca-8c17-55098b57d7bb",
      "metadata": {
        "id": "93565277-6e32-47ca-8c17-55098b57d7bb"
      },
      "source": [
        "# 4. Metrics of our tuning protocol\n",
        "At this stage, we want to select the hyperparameter search space for each optimizer. This way, we can first tune the hyperparameters of each optimizer separately and then select the trial that achieved lowest final train and test error.\n",
        "We then comapre the optimizers' performance by looking at the train and test errors as suggested in the paper \"On empirical comparisons of optimizers for deep learning\".\n",
        "\n",
        "No regularization or weight decay is used."
      ]
    },
    {
      "cell_type": "markdown",
      "id": "124b4d1e-5dbb-4a8a-a365-eb2ce1ecafc7",
      "metadata": {
        "id": "124b4d1e-5dbb-4a8a-a365-eb2ce1ecafc7"
      },
      "source": [
        "### 4.1. Tuning protocol using bootstrap\n",
        "To estimate means and uncertainties of our tuning protocol we will use bootstrapping starting from an initial search space suggested by the paper \"On Empirical Comparisons of Optimizers for Deep Learning\".\n",
        "We run N trials by randomly picking values in the search space of the algorithm at every trial.\n",
        "Then we sample these trials with replacement and compute our statistic on the first K trials of this sample. We repeat this process 50 or 100 times and compute the 5th percentile and 95th percentile of the bootstrap distribution.\n",
        "\n",
        "This allows us to plot the error bars to show the results."
      ]
    },
    {
      "cell_type": "markdown",
      "id": "92a304d9-db74-4bc5-a12c-b97b971097bd",
      "metadata": {
        "id": "92a304d9-db74-4bc5-a12c-b97b971097bd"
      },
      "source": [
        "### 4.2. Tuning Adam for the MLP on MNIST\n",
        "The hyperparameters we are tuning are alpha_0/epsilon, 1 - beta_1, 1 - beta_2, epsilon.\n",
        "The initial search spaces are suggested based on the experience of the writers of the same paper, \"On empirical comparisons of optimizers for deep learning\"."
      ]
    },
    {
      "cell_type": "markdown",
      "id": "465c3fd8-c425-4ce5-991a-0ad3cd2ba426",
      "metadata": {
        "id": "465c3fd8-c425-4ce5-991a-0ad3cd2ba426"
      },
      "source": [
        "##### Set up model for training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "id": "52d84470-824e-4386-a781-c9df9bb78f10",
      "metadata": {
        "id": "52d84470-824e-4386-a781-c9df9bb78f10"
      },
      "outputs": [],
      "source": [
        "# Model fixed parameters\n",
        "model = MLP()\n",
        "model = model.to(device)\n",
        "criterion = nn.CrossEntropyLoss().to(device) # good loss function for classification tasks\n",
        "num_epoch = 50\n",
        "size_minibatch = 128\n",
        "\n",
        "x_train = features_train\n",
        "y_train = labels_train\n",
        "x_test = features_test\n",
        "y_test = labels_test"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "25ca30bc-c245-45ab-8d7e-ec9c37d62ab3",
      "metadata": {
        "tags": [],
        "id": "25ca30bc-c245-45ab-8d7e-ec9c37d62ab3"
      },
      "source": [
        "##### Tune to find best parameter\n",
        "We perform trials until we have K of them, then we pick the best based on our statistic of interest"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### 4.2.1. Initial search for best hyperparameters for **Adam** optimizer on **MLP**. K= 100. \n",
        "Interrupted because of nan loss."
      ],
      "metadata": {
        "id": "mGYqlhu0eJwA"
      },
      "id": "mGYqlhu0eJwA"
    },
    {
      "cell_type": "markdown",
      "source": [
        "##### Set up parameters and search space for the initial trial"
      ],
      "metadata": {
        "id": "0pqeK5opwRvk"
      },
      "id": "0pqeK5opwRvk"
    },
    {
      "cell_type": "code",
      "source": [
        "N = 200\n",
        "K = 100 # Number of trials being kept for the statistic\n",
        "\n",
        "# Initial search spaces for parameters\n",
        "alpha_0 = np.linspace(10**(-2), 10**(4), N)\n",
        "beta_1 = np.linspace(10**(-3), 1, N)\n",
        "beta_2 = np.linspace(10**(-4), 1, N)\n",
        "eps = np.linspace(10**(-10), 10**(10), N)"
      ],
      "metadata": {
        "id": "FhGg_EqiwWKJ"
      },
      "id": "FhGg_EqiwWKJ",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Perform search"
      ],
      "metadata": {
        "id": "W09RkUufwxAx"
      },
      "id": "W09RkUufwxAx"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5c167327-ee1a-42e9-b628-14f61022aef4",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "5c167327-ee1a-42e9-b628-14f61022aef4",
        "outputId": "6d2265bf-ca19-4229-b80d-212451de5ecd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: 0.4954, Test Loss: 0.4477\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-12-8bc99d7bba9e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0;31m# Run\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m     \u001b[0mtrain_error\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_error\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun_nn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_epoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize_minibatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m     \u001b[0;31m# Concatenate hyperparameters with results\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-9-799746aece03>\u001b[0m in \u001b[0;36mrun_nn\u001b[0;34m(x_train, y_train, x_test, y_test, model, optimizer, criterion, num_epoch, size_minibatch)\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m             \u001b[0mloss_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    361\u001b[0m                 \u001b[0mcreate_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    362\u001b[0m                 inputs=inputs)\n\u001b[0;32m--> 363\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    364\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    365\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    164\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m     \u001b[0mgrad_tensors_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_tensor_or_tensors_to_tuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 166\u001b[0;31m     \u001b[0mgrad_tensors_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_make_grads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_grads_batched\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    167\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mretain_graph\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m         \u001b[0mretain_graph\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36m_make_grads\u001b[0;34m(outputs, grads, is_grads_batched)\u001b[0m\n\u001b[1;32m     66\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m                     \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"grad can be implicitly created only for scalar outputs\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m                 \u001b[0mnew_grads\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mones_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmemory_format\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreserve_format\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m                 \u001b[0mnew_grads\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "nb_hyperamaters_to_tune = 4\n",
        "nb_exported_statistics  = 2\n",
        "\n",
        "lowest_test_error = [sys.maxsize] * (nb_hyperamaters_to_tune + nb_exported_statistics)\n",
        "\n",
        "for _ in range(K):\n",
        "    # Pick random values from the intervals given for the different parameters\n",
        "    alpha_0_pick  = float(np.random.choice(alpha_0, 1)) # np.random.choice samples uniformely with replacement\n",
        "    beta_1_pick   = float(-np.random.choice(beta_1, 1) + 1)\n",
        "    beta_2_pick   = float(-np.random.choice(beta_2, 1) + 1)\n",
        "    eps_pick      = float(np.random.choice(eps, 1))\n",
        "    learning_rate = alpha_0_pick * eps_pick\n",
        "    \n",
        "    # Build optimizer from parameters\n",
        "    model=MLP()\n",
        "    model=model.to(device)\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(beta_1_pick, beta_2_pick), eps=eps_pick)\n",
        "    \n",
        "    # Run\n",
        "    train_error, test_error = run_nn(x_train,y_train, x_test, y_test, model, optimizer, criterion, num_epoch, size_minibatch)\n",
        "    \n",
        "    # Concatenate hyperparameters with results\n",
        "    vector = [beta_1_pick, beta_2_pick, eps_pick, learning_rate, train_error, test_error]\n",
        "    \n",
        "    # Check wether we have the smallest test error and store parameters in case we find it\n",
        "    if test_error < lowest_test_error[len(lowest_test_error) - 1]:\n",
        "        lowest_test_error = vector"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### 4.2.2. Final search for best hyperparameters for **Adam** optimizer on **MLP**. K = 50"
      ],
      "metadata": {
        "id": "6yOLXHgin-Ve"
      },
      "id": "6yOLXHgin-Ve"
    },
    {
      "cell_type": "markdown",
      "source": [
        "##### Set up parameters and search space for the final trial"
      ],
      "metadata": {
        "id": "WYEeVzSvw-fz"
      },
      "id": "WYEeVzSvw-fz"
    },
    {
      "cell_type": "code",
      "source": [
        "N = 200\n",
        "K = 50 # Number of trials being kept for the statistic\n",
        "\n",
        "# Final search spaces for parameters\n",
        "alpha_0 = np.linspace(10**(-1), 10, N)\n",
        "beta_1 = np.linspace(10**(-3), 1, N)\n",
        "beta_2 = np.linspace(10**(-4), 1, N)\n",
        "eps = np.linspace(10**(-6), 10**(-2), N)"
      ],
      "metadata": {
        "id": "4jUNhWkVv96z"
      },
      "id": "4jUNhWkVv96z",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Perform search"
      ],
      "metadata": {
        "id": "4gXEoY3nxDaO"
      },
      "id": "4gXEoY3nxDaO"
    },
    {
      "cell_type": "code",
      "source": [
        "nb_hyperamaters_to_tune = 4\n",
        "nb_exported_statistics  = 2\n",
        "\n",
        "lowest_test_error = [sys.maxsize] * (nb_hyperamaters_to_tune + nb_exported_statistics)\n",
        "\n",
        "\n",
        "for _ in range(K):\n",
        "    # Pick random values from the intervals given for the different parameters\n",
        "    alpha_0_pick  = float(np.random.choice(alpha_0, 1)) # np.random.choice samples uniformely with replacement\n",
        "    beta_1_pick   = float(-np.random.choice(beta_1, 1) + 1)\n",
        "    beta_2_pick   = float(-np.random.choice(beta_2, 1) + 1)\n",
        "    eps_pick      = float(np.random.choice(eps, 1))\n",
        "    learning_rate = alpha_0_pick * eps_pick\n",
        "    \n",
        "    # Build optimizer from parameters\n",
        "    model = MLP()\n",
        "    model = model.to(device)\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(beta_1_pick, beta_2_pick), eps=eps_pick)\n",
        "    \n",
        "    # Run\n",
        "    train_error, test_error = run_nn(x_train,y_train, x_test, y_test, model, optimizer, criterion, num_epoch, size_minibatch)\n",
        "    \n",
        "    # Concatenate hyperparameters with results\n",
        "    vector = [beta_1_pick, beta_2_pick, eps_pick, learning_rate, train_error, test_error]\n",
        "    \n",
        "    # Check wether we have the smallest test error and store parameters in case we find it\n",
        "    if test_error < lowest_test_error[len(lowest_test_error) - 1]:\n",
        "        lowest_test_error = vector"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RA5uFjwh2-V8",
        "outputId": "b4d877f0-78da-4fbe-bb34-91a380f2921d"
      },
      "id": "RA5uFjwh2-V8",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Final, Train Loss: 0.1809, Test Loss: 0.6990\n",
            "Final, Train Loss: 0.0476, Test Loss: 0.4258\n",
            "Final, Train Loss: 2.1889, Test Loss: 2.1555\n",
            "Final, Train Loss: 1.0889, Test Loss: 0.5313\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1779\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0964\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1362\n",
            "Final, Train Loss: 0.0006, Test Loss: 0.0769\n",
            "Final, Train Loss: 0.0003, Test Loss: 0.3194\n",
            "Final, Train Loss: 0.1132, Test Loss: 0.4826\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.3511\n",
            "Final, Train Loss: 0.1975, Test Loss: 0.1361\n",
            "Final, Train Loss: 0.0444, Test Loss: 0.5672\n",
            "Final, Train Loss: 0.0002, Test Loss: 0.0817\n",
            "Final, Train Loss: 0.0937, Test Loss: 0.7164\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.2495\n",
            "Final, Train Loss: 1.8274, Test Loss: 1.9169\n",
            "Final, Train Loss: 2.3033, Test Loss: 2.3022\n",
            "Final, Train Loss: 1.6200, Test Loss: 2.3105\n",
            "Final, Train Loss: 2.3062, Test Loss: 2.3021\n",
            "Final, Train Loss: 2.3056, Test Loss: 2.3013\n",
            "Final, Train Loss: 2.1334, Test Loss: 2.0490\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.2111\n",
            "Final, Train Loss: 0.0831, Test Loss: 0.4959\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1397\n",
            "Final, Train Loss: 0.1140, Test Loss: 0.5212\n",
            "Final, Train Loss: 1.3208, Test Loss: 1.7032\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.3030\n",
            "Final, Train Loss: 0.0327, Test Loss: 0.7529\n",
            "Final, Train Loss: 0.0501, Test Loss: 0.7031\n",
            "Final, Train Loss: 2.0190, Test Loss: 2.2513\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0993\n",
            "Final, Train Loss: 0.0624, Test Loss: 0.8515\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1384\n",
            "Final, Train Loss: 2.3058, Test Loss: 2.3016\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.2738\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1229\n",
            "Final, Train Loss: 0.8551, Test Loss: 0.6336\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1204\n",
            "Final, Train Loss: 0.0344, Test Loss: 0.5174\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0985\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.2784\n",
            "Final, Train Loss: 0.1176, Test Loss: 0.7488\n",
            "Final, Train Loss: 2.3047, Test Loss: 2.3015\n",
            "Final, Train Loss: 2.1130, Test Loss: 2.0974\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1038\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1082\n",
            "Final, Train Loss: 2.2091, Test Loss: 1.9663\n",
            "Final, Train Loss: 0.0274, Test Loss: 0.2662\n",
            "Final, Train Loss: 0.0479, Test Loss: 0.4492\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f69a983b-b65e-4185-b56c-b6fabd784c4e",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f69a983b-b65e-4185-b56c-b6fabd784c4e",
        "outputId": "9b3ae363-9684-4ebd-d4e2-df7866b987d5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Beta 1: 0.21\n",
            "Beta 2: 0.69\n",
            "Epsilon: 4.27e-03\n",
            "Learning rate: 0.000427\n",
            "Train error: 0.000572\n",
            "Test error: 0.0769\n"
          ]
        }
      ],
      "source": [
        "# Print best parameters\n",
        "\n",
        "print('Beta 1: %.2f' % lowest_test_error[0])\n",
        "print('Beta 2: %.2f' % lowest_test_error[1])\n",
        "print('Epsilon: %.2e' % lowest_test_error[2])\n",
        "print('Learning rate: %.6f' % lowest_test_error[3])\n",
        "print('Train error: %.6f' % lowest_test_error[4])\n",
        "print('Test error: %.4f' % lowest_test_error[5])"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Run the model again with the best parameters to plot the convergence later"
      ],
      "metadata": {
        "id": "LOmqSuZhLuAx"
      },
      "id": "LOmqSuZhLuAx"
    },
    {
      "cell_type": "code",
      "source": [
        "model = MLP()\n",
        "model = model.to(device)\n",
        "learning_rate = lowest_test_error[3]\n",
        "beta_1 = lowest_test_error[0]\n",
        "beta_2 = lowest_test_error[1]\n",
        "eps = lowest_test_error[2]\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(beta_1, beta_2), eps=eps)\n",
        "    \n",
        "loss_all_train_mlp, loss_all_test_mlp = [], []\n",
        "            \n",
        "for epoch in range(num_epoch):\n",
        "     for b in range(0, x_train.size(0), size_minibatch):\n",
        "            y = model(x_train[b:b+size_minibatch])\n",
        "            loss_train = criterion(y, y_train[b:b+size_minibatch])\n",
        "        \n",
        "            optimizer.zero_grad()\n",
        "            loss_train.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "     loss_train = loss_train.to('cpu').detach().numpy()\n",
        "     loss_all_train_mlp.append(loss_train)\n",
        "\n",
        "     y_test_obt = model(x_test)\n",
        "     loss_test = criterion(y_test_obt, y_test)\n",
        "     loss_test = loss_test.to('cpu').detach().numpy()\n",
        "     loss_all_test_mlp.append(loss_test)\n",
        "     if epoch == num_epoch - 1 :\n",
        "        print('Final, Train Loss: %.4f, Test Loss: %.4f' %(loss_train, loss_test))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kYAHZYifJ9U4",
        "outputId": "e3b53705-9f1e-4b4f-a2e9-b3e3d4597d09"
      },
      "id": "kYAHZYifJ9U4",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Final, Train Loss: 0.0006, Test Loss: 0.0760\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "id": "3c5b6adf-dac0-4e2e-a3f4-27186b406140",
      "metadata": {
        "tags": [],
        "id": "3c5b6adf-dac0-4e2e-a3f4-27186b406140"
      },
      "source": [
        "##### 4.2.3. Estimating trial outcomes via bootstrap\n",
        "At this stage we want to estimate means and uncertainties of our tuning protocol"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "db030739-278f-4fe8-9932-b4af662923ff",
      "metadata": {
        "id": "db030739-278f-4fe8-9932-b4af662923ff"
      },
      "source": [
        "###### Run N trials"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2ad08fad-3e95-4d86-8d4d-38c2f43d6aa3",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2ad08fad-3e95-4d86-8d4d-38c2f43d6aa3",
        "outputId": "4283a241-342e-42b3-d705-f89ea8ad8ac9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Final, Train Loss: 0.0001, Test Loss: 0.3323\n",
            "Final, Train Loss: 1.8190, Test Loss: 1.9930\n",
            "Final, Train Loss: 2.3077, Test Loss: 2.3014\n",
            "Final, Train Loss: 1.6570, Test Loss: 2.6781\n",
            "Final, Train Loss: 2.3139, Test Loss: 2.2030\n",
            "Final, Train Loss: 1.9307, Test Loss: 2.3562\n",
            "Final, Train Loss: 2.3055, Test Loss: 2.3017\n",
            "Final, Train Loss: 0.0277, Test Loss: 0.6947\n",
            "Final, Train Loss: 2.3085, Test Loss: 2.3019\n",
            "Final, Train Loss: 1.5564, Test Loss: 0.9790\n",
            "Final, Train Loss: 0.0688, Test Loss: 0.5211\n",
            "Final, Train Loss: 2.1636, Test Loss: 2.0797\n",
            "Final, Train Loss: 1.1641, Test Loss: 1.3472\n",
            "Final, Train Loss: 1.3095, Test Loss: 0.7236\n",
            "Final, Train Loss: 0.0616, Test Loss: 0.5079\n",
            "Final, Train Loss: 2.0921, Test Loss: 1.6448\n",
            "Final, Train Loss: 2.3069, Test Loss: 2.3022\n",
            "Final, Train Loss: 2.0068, Test Loss: 2.3230\n",
            "Final, Train Loss: 2.3067, Test Loss: 2.3015\n",
            "Final, Train Loss: 0.0003, Test Loss: 0.0792\n",
            "Final, Train Loss: 0.0012, Test Loss: 0.0760\n",
            "Final, Train Loss: 0.0294, Test Loss: 0.2719\n",
            "Final, Train Loss: 0.0582, Test Loss: 0.3365\n",
            "Final, Train Loss: 1.1182, Test Loss: 1.3806\n",
            "Final, Train Loss: 2.3069, Test Loss: 2.3020\n",
            "Final, Train Loss: 0.3465, Test Loss: 0.7455\n",
            "Final, Train Loss: 2.3064, Test Loss: 2.3016\n",
            "Final, Train Loss: 0.3457, Test Loss: 0.6752\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.5160\n",
            "Final, Train Loss: 0.3972, Test Loss: 0.6288\n",
            "Final, Train Loss: 2.3082, Test Loss: 2.3027\n",
            "Final, Train Loss: 0.3386, Test Loss: 0.7945\n",
            "Final, Train Loss: 2.3065, Test Loss: 2.3015\n",
            "Final, Train Loss: 0.0622, Test Loss: 0.7603\n",
            "Final, Train Loss: 2.3095, Test Loss: 2.3023\n",
            "Final, Train Loss: 0.0890, Test Loss: 0.6952\n",
            "Final, Train Loss: 2.3063, Test Loss: 2.3014\n",
            "Final, Train Loss: 2.3046, Test Loss: 2.3017\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.0842\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1960\n",
            "Final, Train Loss: 0.0716, Test Loss: 0.5401\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1763\n",
            "Final, Train Loss: 1.8487, Test Loss: 1.7735\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1010\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1202\n",
            "Final, Train Loss: 0.0442, Test Loss: 0.6209\n",
            "Final, Train Loss: 0.0011, Test Loss: 0.5853\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.0903\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.0895\n",
            "Final, Train Loss: 0.4413, Test Loss: 0.5144\n",
            "Final, Train Loss: 1.3403, Test Loss: 1.0217\n",
            "Final, Train Loss: 2.3057, Test Loss: 2.3022\n",
            "Final, Train Loss: 2.3058, Test Loss: 2.3030\n",
            "Final, Train Loss: 0.0264, Test Loss: 0.4658\n",
            "Final, Train Loss: 2.3007, Test Loss: 2.3039\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0909\n",
            "Final, Train Loss: 2.3085, Test Loss: 2.3019\n",
            "Final, Train Loss: 0.0017, Test Loss: 0.4732\n",
            "Final, Train Loss: 2.3053, Test Loss: 2.3017\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1383\n",
            "Final, Train Loss: 2.3073, Test Loss: 2.3016\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.4298\n",
            "Final, Train Loss: 0.0296, Test Loss: 0.7828\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1224\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1376\n",
            "Final, Train Loss: 0.0893, Test Loss: 0.3909\n",
            "Final, Train Loss: 0.0005, Test Loss: 0.0798\n",
            "Final, Train Loss: 0.0733, Test Loss: 0.5107\n",
            "Final, Train Loss: 2.3058, Test Loss: 2.3018\n",
            "Final, Train Loss: 0.0572, Test Loss: 0.6542\n",
            "Final, Train Loss: 0.7034, Test Loss: 0.9884\n",
            "Final, Train Loss: 2.3052, Test Loss: 2.3004\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.0921\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.2087\n",
            "Final, Train Loss: 0.0296, Test Loss: 0.6244\n",
            "Final, Train Loss: 2.3812, Test Loss: 1.3389\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.1170\n",
            "Final, Train Loss: 0.0667, Test Loss: 0.7081\n",
            "Final, Train Loss: 0.0224, Test Loss: 0.4350\n",
            "Final, Train Loss: 0.0456, Test Loss: 0.4815\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.2157\n",
            "Final, Train Loss: 1.8498, Test Loss: 1.8797\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1468\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.0821\n",
            "Final, Train Loss: 0.1970, Test Loss: 0.5915\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1264\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1019\n",
            "Final, Train Loss: 0.0083, Test Loss: 0.2613\n",
            "Final, Train Loss: 2.1776, Test Loss: 2.1056\n",
            "Final, Train Loss: 0.0718, Test Loss: 0.5389\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0863\n",
            "Final, Train Loss: 2.3066, Test Loss: 2.3014\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1763\n",
            "Final, Train Loss: 0.0007, Test Loss: 0.3708\n",
            "Final, Train Loss: 0.1511, Test Loss: 0.5912\n",
            "Final, Train Loss: 2.3061, Test Loss: 2.3016\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.5070\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.0779\n",
            "Final, Train Loss: 0.0219, Test Loss: 0.5972\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.2407\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1919\n",
            "Final, Train Loss: 2.1987, Test Loss: 3.6654\n",
            "Final, Train Loss: 0.0482, Test Loss: 0.5809\n",
            "Final, Train Loss: 0.0637, Test Loss: 0.6450\n",
            "Final, Train Loss: 0.1016, Test Loss: 0.4565\n",
            "Final, Train Loss: 2.3182, Test Loss: 2.2277\n",
            "Final, Train Loss: 0.1255, Test Loss: 0.9728\n",
            "Final, Train Loss: 0.7614, Test Loss: 2.0057\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.1207\n",
            "Final, Train Loss: 3.2574, Test Loss: 1.5328\n",
            "Final, Train Loss: 2.2692, Test Loss: 2.1618\n",
            "Final, Train Loss: 0.0332, Test Loss: 0.5677\n",
            "Final, Train Loss: 1.5492, Test Loss: 1.6509\n",
            "Final, Train Loss: 2.3072, Test Loss: 2.3016\n",
            "Final, Train Loss: 0.2438, Test Loss: 0.6538\n",
            "Final, Train Loss: 0.0590, Test Loss: 0.7003\n",
            "Final, Train Loss: 0.6451, Test Loss: 2.0763\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.2545\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1029\n",
            "Final, Train Loss: 1.9813, Test Loss: 2.5686\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1612\n",
            "Final, Train Loss: 0.0784, Test Loss: 0.7375\n",
            "Final, Train Loss: 0.1009, Test Loss: 0.8878\n",
            "Final, Train Loss: 2.0958, Test Loss: 8.3748\n",
            "Final, Train Loss: 2.3040, Test Loss: 2.3011\n",
            "Final, Train Loss: 2.3096, Test Loss: 2.3026\n",
            "Final, Train Loss: 0.0455, Test Loss: 0.4822\n",
            "Final, Train Loss: 0.1259, Test Loss: 0.7752\n",
            "Final, Train Loss: 0.0344, Test Loss: 0.7780\n",
            "Final, Train Loss: 2.2376, Test Loss: 1.9129\n",
            "Final, Train Loss: 1.7767, Test Loss: 1.8556\n",
            "Final, Train Loss: 0.0024, Test Loss: 0.2691\n",
            "Final, Train Loss: 0.0610, Test Loss: 0.7122\n",
            "Final, Train Loss: 0.9780, Test Loss: 1.5027\n",
            "Final, Train Loss: 0.0003, Test Loss: 0.0812\n",
            "Final, Train Loss: 2.3053, Test Loss: 2.3017\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.3424\n",
            "Final, Train Loss: 0.0695, Test Loss: 1.2145\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.0903\n",
            "Final, Train Loss: 0.0852, Test Loss: 0.6646\n",
            "Final, Train Loss: 0.0539, Test Loss: 0.6897\n",
            "Final, Train Loss: 0.0519, Test Loss: 0.8233\n",
            "Final, Train Loss: 0.0177, Test Loss: 0.5054\n",
            "Final, Train Loss: 2.3066, Test Loss: 2.3014\n",
            "Final, Train Loss: 2.1793, Test Loss: 1.9572\n",
            "Final, Train Loss: 0.0229, Test Loss: 0.2419\n",
            "Final, Train Loss: 1.9150, Test Loss: 1.8055\n",
            "Final, Train Loss: 0.0448, Test Loss: 0.5584\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1684\n",
            "Final, Train Loss: 0.1751, Test Loss: 0.7452\n",
            "Final, Train Loss: 2.2340, Test Loss: 2.5444\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1668\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0947\n",
            "Final, Train Loss: 0.0048, Test Loss: 0.3613\n",
            "Final, Train Loss: 0.0342, Test Loss: 0.4002\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.4060\n",
            "Final, Train Loss: 2.3048, Test Loss: 2.3016\n",
            "Final, Train Loss: 2.3047, Test Loss: 2.3011\n",
            "Final, Train Loss: 0.0284, Test Loss: 0.5373\n",
            "Final, Train Loss: 2.3052, Test Loss: 2.3036\n",
            "Final, Train Loss: 0.0378, Test Loss: 0.5642\n",
            "Final, Train Loss: 2.1666, Test Loss: 1.9663\n",
            "Final, Train Loss: 0.0586, Test Loss: 0.5746\n",
            "Final, Train Loss: 2.3089, Test Loss: 2.3020\n",
            "Final, Train Loss: 1.2322, Test Loss: 1.0717\n",
            "Final, Train Loss: 0.3482, Test Loss: 0.9416\n",
            "Final, Train Loss: 0.0469, Test Loss: 1.3146\n",
            "Final, Train Loss: 1.9858, Test Loss: 2.3714\n",
            "Final, Train Loss: 0.3354, Test Loss: 0.9037\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1177\n",
            "Final, Train Loss: 2.5075, Test Loss: 1.1668\n",
            "Final, Train Loss: 7.3731, Test Loss: 2.1247\n",
            "Final, Train Loss: 0.0003, Test Loss: 0.0800\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1480\n",
            "Final, Train Loss: 3.9607, Test Loss: 1.6224\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1033\n",
            "Final, Train Loss: 0.0478, Test Loss: 0.8397\n",
            "Final, Train Loss: 0.0401, Test Loss: 0.4872\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1732\n",
            "Final, Train Loss: 2.1006, Test Loss: 1.9854\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1311\n",
            "Final, Train Loss: 0.0647, Test Loss: 0.4739\n",
            "Final, Train Loss: 0.0003, Test Loss: 0.0769\n",
            "Final, Train Loss: 1.6127, Test Loss: 2.2761\n",
            "Final, Train Loss: 2.3036, Test Loss: 2.3027\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.1007\n",
            "Final, Train Loss: 0.0698, Test Loss: 0.5877\n",
            "Final, Train Loss: 2.3073, Test Loss: 2.3015\n",
            "Final, Train Loss: 0.0297, Test Loss: 0.4074\n",
            "Final, Train Loss: 2.3051, Test Loss: 2.3020\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1179\n",
            "Final, Train Loss: 0.0743, Test Loss: 0.4198\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.1007\n",
            "Final, Train Loss: 0.0093, Test Loss: 0.4644\n",
            "Final, Train Loss: 2.0617, Test Loss: 1.9476\n",
            "Final, Train Loss: 2.3037, Test Loss: 2.3024\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1819\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.3140\n",
            "Final, Train Loss: 0.0744, Test Loss: 0.6812\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.2877\n"
          ]
        }
      ],
      "source": [
        "# We first run and store N trials\n",
        "N=200\n",
        "N_trials = []\n",
        "\n",
        "\n",
        "for _ in range(N):\n",
        "    # Pick random values from the intervals given for the different parameters\n",
        "    alpha_0_pick  = float(np.random.choice(alpha_0, 1)) # np.random.choice samples uniformely with replacement\n",
        "    beta_1_pick   = float(-np.random.choice(beta_1, 1) + 1)\n",
        "    beta_2_pick   = float(-np.random.choice(beta_2, 1) + 1)\n",
        "    eps_pick      = float(np.random.choice(eps, 1))\n",
        "    learning_rate = alpha_0_pick * eps_pick\n",
        "    \n",
        "    # Build optimizer from parameters\n",
        "    model = MLP()\n",
        "    model = model.to(device)\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(beta_1_pick, beta_2_pick), eps=eps_pick)\n",
        "    \n",
        "    # Run\n",
        "    train_error, test_error = run_nn(x_train,y_train, x_test, y_test, model, optimizer, criterion, num_epoch, size_minibatch)\n",
        "    \n",
        "    # Store parameters, train and test error\n",
        "    N_trials.append([beta_1_pick, beta_2_pick, eps_pick, learning_rate, train_error.item(), test_error.item()])"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# save the results\n",
        "\n",
        "import pickle\n",
        "with open(\"N_trials_adam_mlp.pth\", \"wb\") as fp:\n",
        "  pickle.dump(N_trials, fp)"
      ],
      "metadata": {
        "id": "031_uIdTsEhC"
      },
      "id": "031_uIdTsEhC",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 4.3. Tuning Adam for the CNN on MNIST"
      ],
      "metadata": {
        "id": "DBRfSh49xasS"
      },
      "id": "DBRfSh49xasS"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VuStHJoqzl5Y"
      },
      "source": [
        "##### Set up model for training"
      ],
      "id": "VuStHJoqzl5Y"
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "FfnUAu5xzl5Z"
      },
      "outputs": [],
      "source": [
        "# Model fixed parameters\n",
        "model = CNN()\n",
        "model = model.to(device)\n",
        "criterion = nn.CrossEntropyLoss().to(device) # good loss function for classification tasks\n",
        "num_epoch = 50\n",
        "size_minibatch = 128\n",
        "\n",
        "x_train = features_train\n",
        "y_train = labels_train\n",
        "x_test = features_test\n",
        "y_test = labels_test"
      ],
      "id": "FfnUAu5xzl5Z"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "t1eNEdgdzl5a"
      },
      "source": [
        "##### Tune to find best parameter\n",
        "We perform trials until we have K of them, then we pick the best based on our statistic of interest"
      ],
      "id": "t1eNEdgdzl5a"
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### 4.3.1. Initial search for best hyperparameters for **Adam** optimizer on **CNN**. K= 100. \n",
        "Interrupted because of nan loss."
      ],
      "metadata": {
        "id": "MSu9ppsnzl5b"
      },
      "id": "MSu9ppsnzl5b"
    },
    {
      "cell_type": "markdown",
      "source": [
        "##### Set up parameters and search space for the initial trial"
      ],
      "metadata": {
        "id": "oyss_Dmczl5b"
      },
      "id": "oyss_Dmczl5b"
    },
    {
      "cell_type": "code",
      "source": [
        "N = 200\n",
        "K = 100 # Number of trials being kept for the statistic\n",
        "\n",
        "# Initial search spaces for parameters\n",
        "alpha_0 = np.linspace(10**(-2), 10**(4), N)\n",
        "beta_1 = np.linspace(10**(-3), 1, N)\n",
        "beta_2 = np.linspace(10**(-4), 1, N)\n",
        "eps = np.linspace(10**(-10), 10**(10), N)"
      ],
      "metadata": {
        "id": "f6umyX4jzl5c"
      },
      "execution_count": null,
      "outputs": [],
      "id": "f6umyX4jzl5c"
    },
    {
      "cell_type": "markdown",
      "source": [
        "Perform search"
      ],
      "metadata": {
        "id": "4yWE5lNhzl5d"
      },
      "id": "4yWE5lNhzl5d"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aMz4BHWrzl5d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 571
        },
        "outputId": "66ddbe16-196a-40ad-cf4b-4cb26983613f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: 0.2574, Test Loss: 0.1889\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n",
            "Final, Train Loss: nan, Test Loss: nan\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-13-ae126e59159c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0;31m# Run\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m     \u001b[0mtrain_error\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_error\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun_nn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_epoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize_minibatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-10-a138ffa15448>\u001b[0m in \u001b[0;36mrun_nn\u001b[0;34m(x_train, y_train, x_test, y_test, model, optimizer, criterion, num_epoch, size_minibatch)\u001b[0m\n\u001b[1;32m     16\u001b[0m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m             \u001b[0mloss_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mnum_epoch\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     86\u001b[0m                 \u001b[0mprofile_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"Optimizer.step#{}.step\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecord_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprofile_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m                     \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    151\u001b[0m                    \u001b[0mweight_decay\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'weight_decay'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m                    \u001b[0meps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 153\u001b[0;31m                    maximize=group['maximize'])\n\u001b[0m\u001b[1;32m    154\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/optim/_functional.py\u001b[0m in \u001b[0;36madam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, amsgrad, beta1, beta2, lr, weight_decay, eps, maximize)\u001b[0m\n\u001b[1;32m    108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m         \u001b[0mstep_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlr\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mbias_correction1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 110\u001b[0;31m         \u001b[0mparam\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddcdiv_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexp_avg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdenom\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mstep_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    111\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m def adamw(params: List[Tensor],\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "nb_hyperamaters_to_tune = 4\n",
        "nb_exported_statistics  = 2\n",
        "\n",
        "lowest_test_error = [sys.maxsize] * (nb_hyperamaters_to_tune + nb_exported_statistics)\n",
        "\n",
        "for _ in range(K):\n",
        "    # Pick random values from the intervals given for the different parameters\n",
        "    alpha_0_pick  = float(np.random.choice(alpha_0, 1)) # np.random.choice samples uniformely with replacement\n",
        "    beta_1_pick   = float(-np.random.choice(beta_1, 1) + 1)\n",
        "    beta_2_pick   = float(-np.random.choice(beta_2, 1) + 1)\n",
        "    eps_pick      = float(np.random.choice(eps, 1))\n",
        "    learning_rate = alpha_0_pick * eps_pick\n",
        "    \n",
        "    # Build optimizer from parameters\n",
        "    model=CNN()\n",
        "    model=model.to(device)\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(beta_1_pick, beta_2_pick), eps=eps_pick)\n",
        "    \n",
        "    # Run\n",
        "    train_error, test_error = run_nn(x_train,y_train, x_test, y_test, model, optimizer, criterion, num_epoch, size_minibatch)\n",
        "    \n",
        "    \n",
        "    # Concatenate hyperparameters with results\n",
        "    vector = [beta_1_pick, beta_2_pick, eps_pick, learning_rate, train_error, test_error]\n",
        "    \n",
        "    # Check wether we have the smallest test error and store parameters in case we find it\n",
        "    if test_error < lowest_test_error[len(lowest_test_error) - 1]:\n",
        "        lowest_test_error = vector"
      ],
      "id": "aMz4BHWrzl5d"
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### 4.3.2. Final search for best hyperparameters for **Adam** optimizer on **CNN**. K = 50"
      ],
      "metadata": {
        "id": "G97NRkQ70yBX"
      },
      "id": "G97NRkQ70yBX"
    },
    {
      "cell_type": "markdown",
      "source": [
        "##### Set up parameters and search space for the final trial"
      ],
      "metadata": {
        "id": "BcWL5tIe0yBY"
      },
      "id": "BcWL5tIe0yBY"
    },
    {
      "cell_type": "code",
      "source": [
        "N = 200\n",
        "K = 50 # Number of trials being kept for the statistic\n",
        "\n",
        "# Final search spaces for parameters\n",
        "alpha_0 = np.linspace(10**(-1), 10, N)\n",
        "beta_1 = np.linspace(10**(-3), 1, N)\n",
        "beta_2 = np.linspace(10**(-4), 1, N)\n",
        "eps = np.linspace(10**(-6), 10**(-2), N)"
      ],
      "metadata": {
        "id": "2LHRX7mT0yBY"
      },
      "execution_count": null,
      "outputs": [],
      "id": "2LHRX7mT0yBY"
    },
    {
      "cell_type": "markdown",
      "source": [
        "Perform search"
      ],
      "metadata": {
        "id": "VSruqZFS0yBZ"
      },
      "id": "VSruqZFS0yBZ"
    },
    {
      "cell_type": "code",
      "source": [
        "nb_hyperamaters_to_tune = 4\n",
        "nb_exported_statistics  = 2\n",
        "\n",
        "lowest_test_error = [sys.maxsize] * (nb_hyperamaters_to_tune + nb_exported_statistics)\n",
        "\n",
        "\n",
        "for _ in range(K):\n",
        "    # Pick random values from the intervals given for the different parameters\n",
        "    alpha_0_pick  = float(np.random.choice(alpha_0, 1)) # np.random.choice samples uniformely with replacement\n",
        "    beta_1_pick   = float(-np.random.choice(beta_1, 1) + 1)\n",
        "    beta_2_pick   = float(-np.random.choice(beta_2, 1) + 1)\n",
        "    eps_pick      = float(np.random.choice(eps, 1))\n",
        "    learning_rate = alpha_0_pick * eps_pick\n",
        "    \n",
        "    # Build optimizer from parameters\n",
        "    model=CNN()\n",
        "    model=model.to(device)\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(beta_1_pick, beta_2_pick), eps=eps_pick)\n",
        "    \n",
        "    # Run\n",
        "    train_error, test_error = run_nn(x_train,y_train, x_test, y_test, model, optimizer, criterion, num_epoch, size_minibatch)\n",
        "    \n",
        "    # Concatenate hyperparameters with results\n",
        "    vector = [beta_1_pick, beta_2_pick, eps_pick, learning_rate, train_error, test_error]\n",
        "    \n",
        "    # Check wether we have the smallest test error and store parameters in case we find it\n",
        "    if test_error < lowest_test_error[len(lowest_test_error) - 1]:\n",
        "        lowest_test_error = vector"
      ],
      "metadata": {
        "id": "Mormnp9dw7Z9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bf993af6-1b85-4218-8df6-6e25351bbe4b"
      },
      "id": "Mormnp9dw7Z9",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Final, Train Loss: 0.0000, Test Loss: 0.0360\n",
            "Final, Train Loss: 2.3075, Test Loss: 2.3020\n",
            "Final, Train Loss: 0.1967, Test Loss: 0.0366\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0545\n",
            "Final, Train Loss: 2.3060, Test Loss: 2.3016\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.3814\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.5991\n",
            "Final, Train Loss: 2.3062, Test Loss: 2.3016\n",
            "Final, Train Loss: 2.3066, Test Loss: 2.3016\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0849\n",
            "Final, Train Loss: 0.0390, Test Loss: 0.3894\n",
            "Final, Train Loss: 0.1512, Test Loss: 0.1367\n",
            "Final, Train Loss: 2.3085, Test Loss: 2.3019\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.5518\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0458\n",
            "Final, Train Loss: 2.3084, Test Loss: 2.3019\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0604\n",
            "Final, Train Loss: 2.3063, Test Loss: 2.3020\n",
            "Final, Train Loss: 2.3055, Test Loss: 2.3019\n",
            "Final, Train Loss: 0.1624, Test Loss: 0.2472\n",
            "Final, Train Loss: 2.3083, Test Loss: 2.3020\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0397\n",
            "Final, Train Loss: 2.3056, Test Loss: 2.3016\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.2339\n",
            "Final, Train Loss: 2.3095, Test Loss: 2.3018\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0411\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1845\n",
            "Final, Train Loss: 0.0000, Test Loss: 1.2006\n",
            "Final, Train Loss: 0.0909, Test Loss: 0.7276\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0390\n",
            "Final, Train Loss: 0.4880, Test Loss: 1.1956\n",
            "Final, Train Loss: 2.3074, Test Loss: 2.3019\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.9494\n",
            "Final, Train Loss: 2.3058, Test Loss: 2.3021\n",
            "Final, Train Loss: 0.2185, Test Loss: 1.2426\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.3324\n",
            "Final, Train Loss: 0.0003, Test Loss: 0.2520\n",
            "Final, Train Loss: 2.3093, Test Loss: 2.3019\n",
            "Final, Train Loss: 1.0637, Test Loss: 0.6397\n",
            "Final, Train Loss: 0.3435, Test Loss: 1.0673\n",
            "Final, Train Loss: 2.3070, Test Loss: 2.3021\n",
            "Final, Train Loss: 2.3083, Test Loss: 2.3019\n",
            "Final, Train Loss: 2.3035, Test Loss: 2.3022\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0486\n",
            "Final, Train Loss: 0.0003, Test Loss: 0.0426\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0373\n",
            "Final, Train Loss: 2.3074, Test Loss: 2.3020\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1977\n",
            "Final, Train Loss: 2.3076, Test Loss: 2.3018\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.3468\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Print best parameters\n",
        "\n",
        "print('Beta 1: %.2f' % lowest_test_error[0])\n",
        "print('Beta 2: %.2f' % lowest_test_error[1])\n",
        "print('Epsilon: %.2e' % lowest_test_error[2])\n",
        "print('Learning rate: %.2e' % lowest_test_error[3])\n",
        "print('Train error: %.6f' % lowest_test_error[4])\n",
        "print('Test error: %.4f' % lowest_test_error[5])"
      ],
      "metadata": {
        "id": "52xLPeV8w3ZX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "06b75963-e5a4-435c-e756-11e7e7ce3cd4"
      },
      "id": "52xLPeV8w3ZX",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Beta 1: 0.37\n",
            "Beta 2: 0.92\n",
            "Epsilon: 3.22e-03\n",
            "Learning rate: 1.76e-03\n",
            "Train error: 0.000005\n",
            "Test error: 0.0360\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Run the model again with the best parameters to plot the convergence later."
      ],
      "metadata": {
        "id": "4QMdt95e81Uz"
      },
      "id": "4QMdt95e81Uz"
    },
    {
      "cell_type": "code",
      "source": [
        "model = CNN()\n",
        "model = model.to(device)\n",
        "learning_rate =  lowest_test_error[3]\n",
        "beta_1 = lowest_test_error[0]\n",
        "beta_2 = lowest_test_error[1]\n",
        "eps = lowest_test_error[2]\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(beta_1, beta_2), eps=eps)\n",
        "    \n",
        "loss_all_train_cnn, loss_all_test_cnn = [], []\n",
        "            \n",
        "for epoch in range(num_epoch):\n",
        "     for b in range(0, x_train.size(0), size_minibatch):\n",
        "            y = model(x_train[b:b+size_minibatch])\n",
        "            loss_train = criterion(y, y_train[b:b+size_minibatch])\n",
        "        \n",
        "            optimizer.zero_grad()\n",
        "            loss_train.backward()\n",
        "            optimizer.step()\n",
        "      \n",
        "     loss_train = loss_train.to('cpu').detach().numpy()\n",
        "     loss_all_train_cnn.append(loss_train)\n",
        "\n",
        "     y_test_obt = model(x_test)\n",
        "     loss_test = criterion(y_test_obt, y_test)\n",
        "     loss_test = loss_test.to('cpu').detach().numpy()\n",
        "     loss_all_test_cnn.append(loss_test)\n",
        "     if epoch == num_epoch - 1 :\n",
        "        print('Final, Train Loss: %.4f, Test Loss: %.4f' %(loss_train, loss_test))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "65UAk5e2OvQP",
        "outputId": "74901f56-7de5-45ed-9e16-037bd715ab47"
      },
      "id": "65UAk5e2OvQP",
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Final, Train Loss: 0.0000, Test Loss: 0.0367\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "mbJtvIFv0yBd"
      },
      "source": [
        "##### 4.3.3. Estimating trial outcomes via bootstrap\n",
        "At this stage we want to estimate means and uncertainties of our tuning protocol"
      ],
      "id": "mbJtvIFv0yBd"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IohNC8cu0yBd"
      },
      "source": [
        "###### Run N trials"
      ],
      "id": "IohNC8cu0yBd"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QJjhEOxk0yBf"
      },
      "source": [
        "###### Perform bootstrapping"
      ],
      "id": "QJjhEOxk0yBf"
    },
    {
      "cell_type": "code",
      "source": [
        "N = 50\n",
        "K = 25\n",
        "\n",
        "# Final search spaces for parameters\n",
        "alpha_0 = np.linspace(10**(-1), 10, N)\n",
        "beta_1 = np.linspace(10**(-3), 1, N)\n",
        "beta_2 = np.linspace(10**(-4), 1, N)\n",
        "eps = np.linspace(10**(-6), 10**(-2), N)"
      ],
      "metadata": {
        "id": "LsDLOazX-c16"
      },
      "id": "LsDLOazX-c16",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# We first run and store N trials\n",
        "N_trials = []\n",
        "\n",
        "for _ in range(N):\n",
        "    # Pick random values from the intervals given for the different parameters\n",
        "    alpha_0_pick  = float(np.random.choice(alpha_0, 1)) # np.random.choice samples uniformely with replacement\n",
        "    beta_1_pick   = float(-np.random.choice(beta_1, 1) + 1)\n",
        "    beta_2_pick   = float(-np.random.choice(beta_2, 1) + 1)\n",
        "    eps_pick      = float(np.random.choice(eps, 1))\n",
        "    learning_rate = alpha_0_pick * eps_pick\n",
        "    \n",
        "    # Build optimizer from parameters\n",
        "    model = CNN()\n",
        "    model = model.to(device)\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(beta_1_pick, beta_2_pick), eps=eps_pick)\n",
        "    \n",
        "    # Run\n",
        "    train_error, test_error = run_nn(x_train,y_train, x_test, y_test, model, optimizer, criterion, num_epoch, size_minibatch)\n",
        "    \n",
        "    # Store parameters, train and test error\n",
        "\n",
        "    N_trials.append([beta_1_pick, beta_2_pick, eps_pick, learning_rate, train_error.item(), test_error.item()])"
      ],
      "metadata": {
        "id": "eDtMp9RuxGUJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "59887ce9-6b41-4af7-af2c-011d2fe2c5de"
      },
      "id": "eDtMp9RuxGUJ",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Final, Train Loss: 0.2178, Test Loss: 0.0870\n",
            "Final, Train Loss: 2.3087, Test Loss: 2.3022\n",
            "Final, Train Loss: 2.3063, Test Loss: 2.3026\n",
            "Final, Train Loss: 2.3045, Test Loss: 2.3010\n",
            "Final, Train Loss: 2.3067, Test Loss: 2.3030\n",
            "Final, Train Loss: 0.0909, Test Loss: 0.8008\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.3051\n",
            "Final, Train Loss: 1.0839, Test Loss: 0.4106\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.5906\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0711\n",
            "Final, Train Loss: 0.0246, Test Loss: 1.0186\n",
            "Final, Train Loss: 2.3055, Test Loss: 2.3017\n",
            "Final, Train Loss: 0.2376, Test Loss: 0.4517\n",
            "Final, Train Loss: 0.2236, Test Loss: 0.4607\n",
            "Final, Train Loss: 0.0002, Test Loss: 0.3177\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.4322\n",
            "Final, Train Loss: 2.3071, Test Loss: 2.3018\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.2491\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.1825\n",
            "Final, Train Loss: 2.3059, Test Loss: 2.3015\n",
            "Final, Train Loss: 0.1232, Test Loss: 0.7462\n",
            "Final, Train Loss: 2.3083, Test Loss: 2.3026\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.6467\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.8323\n",
            "Final, Train Loss: 2.3110, Test Loss: 2.3018\n",
            "Final, Train Loss: 2.3076, Test Loss: 2.3023\n",
            "Final, Train Loss: 0.0881, Test Loss: 0.0380\n",
            "Final, Train Loss: 2.3054, Test Loss: 2.3016\n",
            "Final, Train Loss: 2.3056, Test Loss: 2.3020\n",
            "Final, Train Loss: 2.3080, Test Loss: 2.3016\n",
            "Final, Train Loss: 2.3062, Test Loss: 2.3016\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.2778\n",
            "Final, Train Loss: 2.3053, Test Loss: 2.3016\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.0352\n",
            "Final, Train Loss: 0.0001, Test Loss: 0.0358\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0359\n",
            "Final, Train Loss: 2.3060, Test Loss: 2.3022\n",
            "Final, Train Loss: 2.3065, Test Loss: 2.3019\n",
            "Final, Train Loss: 2.3059, Test Loss: 2.3019\n",
            "Final, Train Loss: 2.3069, Test Loss: 2.3017\n",
            "Final, Train Loss: 0.0636, Test Loss: 0.4245\n",
            "Final, Train Loss: 2.3101, Test Loss: 2.3019\n",
            "Final, Train Loss: 0.0198, Test Loss: 0.5476\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.3023\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.7853\n",
            "Final, Train Loss: 2.3054, Test Loss: 2.3018\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0454\n",
            "Final, Train Loss: 2.3055, Test Loss: 2.3016\n",
            "Final, Train Loss: 0.0000, Test Loss: 0.0473\n",
            "Final, Train Loss: 0.3896, Test Loss: 0.3432\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#save the results\n",
        "\n",
        "import pickle\n",
        "with open(\"N_trials_adam_cnn.pth\", \"wb\") as fp:\n",
        "  pickle.dump(N_trials, fp)"
      ],
      "metadata": {
        "id": "Nn7B5TH97VQ0"
      },
      "id": "Nn7B5TH97VQ0",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Plots of convergence on CNN"
      ],
      "metadata": {
        "id": "6uP9nBFu9D7D"
      },
      "id": "6uP9nBFu9D7D"
    },
    {
      "cell_type": "code",
      "source": [
        "# load the results\n",
        "\n",
        "file = open(\"plot_nosadam_cnn_train.pth\", \"rb\")\n",
        "loss_all_train_cnn_nos = pickle.load(file)"
      ],
      "metadata": {
        "id": "NNYrMOB03jWx"
      },
      "id": "NNYrMOB03jWx",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# load the results\n",
        "\n",
        "file = open(\"plot_nosadam_cnn_test.pth\", \"rb\")\n",
        "loss_all_test_cnn_nos = pickle.load(file)"
      ],
      "metadata": {
        "id": "uupLJ1V433uK"
      },
      "id": "uupLJ1V433uK",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.core.pylabtools import figsize\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "plt.style.use('seaborn-whitegrid')\n",
        "\n",
        "epochs = np.arange(0, 50, 1)\n",
        "\n",
        "plt.plot(epochs, loss_all_train_cnn, label='Train Loss Adam')\n",
        "plt.plot(epochs, loss_all_test_cnn, label='Test Loss Adam')\n",
        "plt.plot(epochs, loss_all_train_cnn_nos, label='Train Loss NosAdam')\n",
        "plt.plot(epochs, loss_all_test_cnn_nos, label='Test Loss NosAdam')\n",
        "\n",
        "plt.legend()\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.savefig('cnn_conv.pdf')\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276
        },
        "id": "ZRJn7SEN9Hd-",
        "outputId": "0954623b-fe16-47a5-e9a4-e0b247981ddc"
      },
      "id": "ZRJn7SEN9Hd-",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEDCAYAAAA2k7/eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3wUZf7A8c9sy6ZuekIJLVJC6CAch4oiCPpDReAEjnYWbCAcygnHqYjSbHgqdrCBIoooUgQbeBaaNEkICEhJqAkhZdO2ze+P2WwSEiAJ2QDZ7/vlvqY/88wGv/PsMzPfUVRVVRFCCFHn6S51BYQQQtQOCfhCCOEjJOALIYSPkIAvhBA+QgK+EEL4CAn4QgjhIwyXugLns3Xr1ktdBSGEuCJ17ty53LzLOuBDxZWujJSUFBISEmq4Npc/OW7fIsftWyp73OdqLEuXjhBC+AgJ+EII4SMk4AshhI+QgC+EED5CAr4QQvgIrwb8P/74g969e7No0aJyy3799VcGDx7MkCFDeO2117xZDSGEEHgx4Ofn5/PMM8/QvXv3CpfPmDGDV199lcWLF/PLL7+wf/9+b1VFCCEEXgz4JpOJd955h+jo6HLLUlNTsVgs1KtXD51OR8+ePdmwYUON7fuRT3fw0c4zNVaeEKJic+bMYeTIkfTr14+ePXsycuRIxo0bV6ltJ06cSGFh4QXXS0tLY+DAgRdb1fMqKiqiS5cuvP/++xUuz8vLo1evXl6tQ23w2oNXBoMBg6Hi4tPT0wkPD/dMh4eHk5qaWmP7PnDKisFlq7HyhBAVmzJlCgDLli1j3759TJ48udLbvvTSS96qVpWtX7+eyMhIVq9ezT/+8Y9LXR2vueyftE1JSanyNjqnjdwiR7W2vdIVFhbKcfuQy+W4jx07xunTpz11efnllzEYDOTm5jJ+/Hjmzp1LYWEhRUVFjBkzhhYtWjBmzBheeeUV3n77bcLDwzlw4AAZGRlMnDiR+Ph4T9knT54sd5yFhYV8/fXXvP322yiKgr+/PxMmTECn0/H8889jt9txOBzcd999xMbGlptXunyAjz/+mEGDBvHee++xfv16YmJiyM/P59lnn8Vms9G6dWtsNhspKSn8+OOPrFq1Cp1OR1xcHGPHjuX7778nOTmZnJwcUlNTGT58OD/99BOpqak88sgjtGjRoka+54v9e1+SgB8dHU1GRoZn+uTJkxV2/QDVeny6/rYCdh7OkEevfYgcN3y+NY1Pf6u5X8oAd3aJY1DnhpWqR15enqcuoaGhREZGMmnSJA4ePMhdd91F79692bBhAx9//DG33347JpOJli1bEhoaSkhICEuWLGHx4sXs3LmT/v37e8oODg7GbDaX+fumpKSwePFinnrqKdq3b8+CBQvYuHEjrVq1Ij4+nlmzZpGamsrBgwfJzMwsN690WVarlb179/LWW2+RmZnJ3r17uf766/noo4/o0KEDU6dOZfXq1WzcuJGEhAR+//13PvroI0JCQhg+fDg6nY769evzyy+/8PHHH/PZZ5+xcOFCvvzyS5YtW0ZSUhK33357jfw9rsjUCg0bNsRqtZKWlobD4WDdunX06NGjxsoP8TeQZ3PVWHlCiKpr164dAJGRkaxdu5Zhw4bxwgsvkJWVVW7dLl26ABAbG4vVaq1U+QcOHKB9+/YAdOvWjd27d9OhQwd27NjBk08+yeHDh7nuuusqnFfa2rVrueaaazCbzfTv35+VK1d6yu/YsSMAXbt29axvsVh46KGHGDFiBAcOHPAcT5s2bVAUhaioKFq2bIlerycyMrLSx1MbvNbCT0pK4tlnn+Xo0aMYDAbWrl1Lr169aNiwIX369OGpp57i0UcfBeCWW26hadOmNbbvELMRqwR84WMGdW5YqdZ4bTEajQB88MEHxMTE8Pzzz7Nr1y6ee+65cuvq9XrPuKqqVd6X3W5Hp9MRHR3N8uXL2bRpE4sXL2bHjh2MGzeuwnnFVq5cyZEjRzyt8EOHDrF//35UVUWn09rELpcWT2w2G08//TTLly8nKiqK+++/31NO6WuWpcerczze4rWA36ZNGxYuXHjO5VdffTVLlizxyr5D/I3YXSqFdidmo/7CGwghvObMmTO0bNkSgO+++w673V4j5TZv3pzt27fTsWNHtmzZQps2bfj111+x2+307NmTq666iqeeeqrCecXS09PZv38/69at8wTpefPmsXLlSpo2bUpSUhJ9+/Zl06ZNgHa3jl6vJyoqiuPHj5OUlFRjx1MbLvuLttUR4q+1LHIK7RLwhbjEbr/9diZPnsyaNWsYPnw4K1eu5PPPP69SGQcPHmTkyJGe6cGDB/P4448zffp0FEXBYrEwe/ZssrKy+Ne//sX8+fNRFIXx48cTGxtbbl6x1atX079//zIt8jvuuIO7776bzz77jLFjxzJ69GhPmvawsDB69OjBoEGDaNWqFffeey+zZ89m9OjRF/kt1Q5FvZx+b5xl69at1cqHv3zHUSZ8soPvHunJVdFBXqjZ5UsuXvoWOW7fUpWLthXFzjqZS6e4hZ9dcOX81BJCCG+rmwHfXNKlI4QQQlMnA77FX+uPy5EWvhBCeNTJgF/Swndc4poIIcTlo24G/OK7dKSFL4QQHnUy4JuNeow6RfrwhRCilDp5Hz5AkElHToF06QjhTXPmzCE5OZn09HQKCgpo1KgRFouFefPmVWr7PXv24OfnV+5J+169erFixQoCAwO9UW0A7rnnHvz8/Hj99dcrXD5w4EBeeeUVGja8fJ5evlh1NuAHmnTSwhfCyy4mPTLAt99+S5s2bWo0tUplnD59mgMHDlBYWEhubi7BwcG1uv9LpW4HfOnDF6LWOZ1OnnjiCVJTU3E4HIwfP57u3bvz5ZdfsmjRIoxGI61atWLo0KF88sknhIeHExER4Um2di65ublMmTKFnJwccnNzmTlzJomJicyYMYOkpCScTifDhg1j4MCBFc4rbfXq1dxwww3k5OTwzTffMGjQIEB7E9/27dtp2rSpJ2XCnj17mD59OgaDAZ1Ox8svv4zVauWxxx6jUaNGbN++nWHDhrF371527tzJ8OHDGT58uHe+3ItUZwN+kAR84Wt2LIbt5d8ffVE6joAOw6q0yYoVK4iKimLWrFlkZmYyevRoVqxYwYIFC3j77bepV68en3/+OY0bN+baa6+lb9++Fwz2oCVha9++Pffddx8rVqxg9uzZzJs3j/Xr13ty9HzxxRdkZWWVm3e2lStX8q9//Yvc3FwWLVrEoEGD2L9/P9u2bWPp0qWcPHmSPn36ANqvgSeeeILWrVvz8ssvs2LFCm644QZSUlJ47bXXyM7Opn///nz//fcUFRXx8MMPS8CvbYEmHalW6cMXorZt376drVu3sm3bNkB7faDNZqN///6MHTuW2267jf79+2M2m6tUblJSEg8++CAAV111FYcPHyY0NJQmTZrw4IMP0q9fPwYMGIDJZCo3r7TU1FROnjxJ586dcTgcPP7442RmZrJ//37at2+PTqejXr16xMXFARAREcELL7xAYWEhp06d4tZbbwWgUaNGhIWFYTKZCA8PJyYmhry8PHJzcy/2K/SaOhvwtRZ+0aWuhhC1p8OwKrfGvcFoNPLAAw+UeYkJwP3338+tt97K2rVrGT16NIsWVe3XiKIoZVINF6csnj9/PsnJyaxcuZLly5fz7rvvVjiv2MqVKykqKvKcCBwOB19//TXh4eGedMily585cyZjxozhuuuuY8GCBeTn5wNlUzqf63Wul5s6eVsmQJBJT06h/bLKRS2EL2jfvj3ff/89oHWHzJ07F5fLxUsvvURUVBR33XUXHTp04NixYyiKgtPprFS5bdu29aQp3rt3L82bNyctLY0PP/yQxMREJk+eTFZWVoXzSlu1ahXvv/8+y5cvZ/ny5cybN49Vq1bRtGlTkpOTUVWVo0ePcvToUQCysrJo1KgRNpuNH3/88YpKh3y2K+O0VA2BJh12p0qh3YW/SVIkC1Fbbr75ZjZu3MjQoUNxOp2MGzcOnU5HYGAgQ4YMITg4mLi4OBISEujSpQszZswgMDCQ7t27lylnzJgxnlZ0//79GTVqFFOnTmXUqFFYrVaeffZZoqOj2b59O6tXr8ZoNDJo0KAK5xXbs2eP59WKxbp06cLp06exWCy0aNGCIUOG0KRJE1q1agXAiBEjGDt2LHFxcYwcOZKnn36aW265pRa+yZpXJ9Mjc/oAb/6wlzlbVTZNvZGYkKr1FV7JJG2sb5Hj9i2SHrkiX9zPjSfeASRFshBCFKubAV/RE+zIBCSfjhBCFKubAd8/FLNTe1O8PG0rhBCauhnwzaH4OdwBX/LpCCEEUFcDvn8oRof28IO08IUQQlM3A745FIMjDx0u6cMXQgi3OhrwLQBEGwvkrVdCeNGcOXMYOXIk/fr1o2fPnowcOZJx48ZVatuJEydSWFh4wfXS0tLKJT+rSZs2baJjx46kp6d75r366queh7yq6p577uGhhx465/KBAweSlpZWrbIvVt188Mo/FID6fkXSwhfCiy4mPfJLL73krWpVWcOGDZk3bx7Tp0+/qHIu97TLdTPgm7WAX8+vSO7DF+ISmDJlCkajkaysLGbPns2jjz5Kfn4+hYWFPPHEE7Rr187zkpNnnnmG6OhokpOTOXbsGC+88AKJiYkX3MfevXt5+umnPU/xzpkzB71ezz//+U9sNhs2m40nn3ySRo0alZt3dvk33XQTv/zyCwcPHiyXm/+5555j27ZtOJ1Ohg8fzoABA8qlep42bRrg/bTLnTp1uqi/S90M+O4WfrSxgD/koq3wEV8d+Iov9pVPBXwx7mh+B7fF31atbS0WC8888wwHDx7kb3/7G71792bDhg288847vPrqq2XWtdlsLFiwgMWLF/Pll19WKuDPnDmTxx57jPbt27NgwQI+/PBDWrVqRUxMDLNmzSI1NZWDBw9y9OjRcvMqMnHiRObOnVumblu2bGHfvn188skn5Ofnc9ttt9G7d+9yqZ4LCwsxm81eT7t8sQG/jvbhawE/0lAot2UKcYkU57iPjIxk7dq1DBs2jBdeeKFcMjPQ8tkAxMbGYrVaK1X+gQMHaN++PQDdunVj9+7ddOjQgR07dvDkk09y+PBhrrvuugrnVaRbt27YbDZ27NjhmZeUlMTVV18NQEBAgCctc3Gq5/fff5+ePXtiNpvLpF2+5ppr2LNnzwXTLs+dO5cRI0awatUqz/dSnHY5KirKk3Y5IiKiRtIu1+kWfqQ+nxyrtPCFb7gt/rZqt8a9wWg0AtqLS2JiYnj++efZtWsXzz33XLl1S6cark56L7vdjk6nIzo6muXLl7Np0yYWL17Mjh07GDduXIXzKvLII48wY8YMunbtCmgpmSvaT0Wpnq+EtMt1uoUfpsuXi7ZCXGJnzpyhUaNGAJ63UNWE5s2bs337dkDremnTpg2//vorv/76K9dccw1PPPEESUlJFc47l5YtW9KgQQPWrVsHQJs2bTx36+Tl5XHkyBEaN25cYarnKyHtsldb+LNmzWLnzp0oisLUqVPLvMbso48+4quvvkKn09GmTRv+85//1NyOjf6oOgMWJY+cQgeqqpY7Uwshasftt9/O5MmTWbNmDcOHD2flypV8/vnnVSrj4MGDjBw50jM9ePBgHn/8caZPn46iKFgsFmbPnk1WVhb/+te/mD9/PoqiMH78eGJjY8vNO58JEybQt29fQOtqatOmDcOHD8fhcPDoo48SEBBQLtWzXq+/MtIuq16yadMm9b777lNVVVX379+v3nnnnZ5lubm56g033KDa7XZVVVX1rrvuUrdv316ujN9++63a+7fPaqzufnO02njyStVaaK92OVea3bt3X+oqXBJy3L5Fjvv8zhU7vdals2HDBnr37g1AfHw82dnZnosxRqMRo9FIfn4+DoeDgoICLBZLje7faQohUM0DJL2CEEKAF/vwMzIyCAsL80yHh4d7nmTz8/Nj7Nix9O7dmxtuuIH27duXu/f1YjlNwfi7tBOM3IsvhBC1eJeOWurKu9Vq5a233mLNmjUEBQUxevRo9uzZ4+nbKi0lJaVa+6uvD0SXnwHArj37Uc/4V6/iV5jCwsJqf2dXMjlu3yLHXT1eC/jR0dFkZGR4pk+dOkVUVBSg3T8bFxdHeHg4oF3cSEpKqjDgV/c1ZtkbLAQVHAcgLLoBCQkx1SrnSiOvfvMtcty+pSqvOKyI17p0evTowdq1awFITk4mOjqaoKAgABo0aODJNwHaww1NmjSp0f07TSEYbDmA9OELIQR4sYXfqVMnEhMTGTp0KIqiMG3aNJYtW0ZwcDB9+vThnnvuYdSoUej1ejp27Oh50q6muEzB6IqyUSRFshBCAF7uw580aVKZ6dJdNkOHDmXo0KFe27fTGISCSjCSIlkIb5kzZw7Jycmkp6dTUFBAo0aNsFgszJs3r1Lb79mzBz8/v3I3bRQnVgsMDPRGtWnZsiVvvPEGvXr1ArQUyZs3b+bhhx+ucllvvfUW7733Hj///HOFT8Y+++yzNG/e3KspniurbqZWQOvSAYg2FUoLXwgvuZj0yADffvstbdq0qfG79C6kSZMmzJs3j549e5ZJZVAdK1euJDQ0lF9//fWceXouF3U24LtMWh7q+qYi6cMXohY5nU6eeOIJUlNTcTgcjB8/nu7du5dLKTx06FA++eQTwsPDiYiIKPMkfkVyc3OZMmUKOTk55ObmMnPmTBITE5kxYwZJSUk4nU6GDRvGwIEDK5xXWnR0NG3btuWLL75g8ODBZZatXr2a999/H71eT2JiIo8//ji7d+9m+vTpmEwmTCYTL730EiEhIezduxeXy8Xdd9/NqlWrPAF/+fLlzJ8/n5iYGMxmM82bN8dqtVaYJrp3797ceeedrFmzhsaNG5OYmOgZf/HFF2v0b1NnA77THfBj/QrkPnzhE7K+/JLsz5fVaJmWQQMJdScDq6wVK1YQFRXFrFmzyMzMZPTo0axYsaJcSuHGjRtz7bXX0rdv3wsGe9CSsLVv35777ruPFStWMHv2bObNm8f69es9OXq++OILsrKyys2ryP3338+IESPo37+/Z15eXh4vvfQSX375JYGBgTzwwANs3LiR7777jmHDhjFgwAA2bNhAeno6ISEhrFy5kltuuYWbbrqJuXPnUlRU5DkhfP7554SEhHhONunp6RWmiXa5XLRu3ZoxY8Zw/fXXc9NNN7F06VKuv/56cnJyCAkJqdL3fz51N+Ab3V06xkJSJUWyELVm+/btbN26lW3btgFQVFSEzWbzpBS+7bbb6N+/P2azuUrlJiUl8eCDDwJ40hSHhobSpEkTHnzwQfr168eAAQMwmUzl5lXEYrFw++238+GHH3rSLB86dIjGjRt7rh107dqVlJQUbrzxRp566ikOHTrELbfcQnx8PKqqsmrVKt577z1CQ0Pp0KEDP/74I126dCEwMJCIiAgATw77yMhIXn/9dRYsWIDNZiMgIMBTl3bt2qEoChEREbRu3RrQHlbNzc2VgF8ZxS38SH2BdOkInxA6YECVW+PeYDQaeeCBB8q0nIEKUwpXhaIoZR7gLE4zPH/+fJKTk1m5ciXLly/n3XffrXBeRUaOHMngwYM9t4WfvQ+73Y6fnx/du3dn6dKlrFu3jilTpvDYY49hNBo5ffq0Jxlbbm4uq1atokuXLmXSIReXd7400aWvI1xsqujzqZvpkSnpww/X50vAF6IWtW/fnu+//x7Q3uo0d+5cXC5XhSmFFUXB6XRWqty2bdt6UhXv3buX5s2bk5aWxocffkhiYiKTJ08mKyurwnnn4ufnx1133cWbb74JaBdzDx8+7Mn7tXnzZtq0acOiRYvIysritttuY/To0aSkpLBy5UomTZrkSYe8cuVKtmzZgslkIjc3l5ycHOx2u+eXjrfSRFdFnW3huwwBoOgJVfLkrVdC1KKbb76ZjRs3MnToUJxOJ+PGjfO8d7Z0SuGEhAS6dOnCjBkzCAwMpHv37mXKGTNmjKe1279/f0aNGsXUqVMZNWoUVquVZ599lujoaLZv387q1asxGo0MGjSownnnM2DAAN577z1Ae6vVY489xr333otOp6Nz58506dKF/Px8JkyYQHBwMCaTidmzZzNo0KAyqZYDAgK4/vrr+eGHHxg3bhwjRoygQYMGNG/eHKiZNNEXS1Fr+jdDDdq6dSudO3eu1rYpKSkkfHULO0JuYOCRQeyfeQs6Xd3PiS+PnPsWOW7fUpXUChXFzjrbpQOAfyhBqhWXCnk2aeULIXxb3Q745lACVUmRLIQQUNcDvn8o/g7tTe/Sjy+E8HV1O+CbQ/FzSMZMIYSAuh7w/UMx2t0BX7p0hBA+rm4HfHMo+qJsQJWMmUIIn1fHA74FRXUSQJG08IUQPq9uB3z/UAAs5EkfvhDC59XtgG/WAn49v0K5S0cI4fPqdsD3Lwn4ch++EMLX1e2A727hRxskY6YQQtTtgO9u4UcZ5TWHQghRtwO+u4UfoZcXmQshRN0O+H4hgEKYLk9a+EIIn1e3A75OB+YQLIq8BEUIIep2wAcwhxKCFWuRA5frsk39L4QQXlf3A75/KIEuK6oKuUXSjy+E8F11P+CbQwlwFadIlm4dIYTvqvsB3z8Uszsnvjx8JYTwZXU/4JtDMRW/BEUu3AohfFjdD/j+oRhsxTnxpQ9fCOG76n7AN4eicxbhh01a+EIIn+YDAd8CuFMkSx++EMKHGbxZ+KxZs9i5cyeKojB16lTatWvnWXb8+HEeeeQR7HY7rVu35umnn/ZOJdz5dEJ1eZJeQQjh07zWwt+8eTOHDx9myZIlzJw5k5kzZ5ZZPmfOHO6++26WLl2KXq/n2LFj3qlIcU58kyRQE0L4Nq8F/A0bNtC7d28A4uPjyc7Oxmq1AuByudi6dSu9evUCYNq0adSvX987FXG38GMl4AshfJzXunQyMjJITEz0TIeHh5Oenk5QUBCZmZkEBgYye/ZskpOT6dKlC48++miF5aSkpFRr/4WFhaSkpGDMPc1VQCi57Eg/U+3yrhTFx+1r5Lh9ixx39Xi1D780VVXLjJ88eZJRo0bRoEED7rvvPtavX8/1119fbruEhIRq7S8lJUXbNj8GVkM9s53tBnO1y7tSeI7bx8hx+xY57vPbunVrhfO91qUTHR1NRkaGZ/rUqVNERUUBEBYWRv369WnUqBF6vZ7u3buzb98+71TELwSAcL289UoI4du8FvB79OjB2rVrAUhOTiY6OpqgoCAADAYDcXFxHDp0yLO8adOm3qmI3gCmYEIVuS1TCOHbvNal06lTJxITExk6dCiKojBt2jSWLVtGcHAwffr0YerUqUyZMgVVVWnRooXnAq5X+Idq9+HLbZlCCB/m1T78SZMmlZlu1aqVZ7xx48YsXrzYm7svYQ4l2K7lxHc4XRj0df95MyGEOJtvRD7/UAJc2i2hudLKF0L4KN8I+GYL/k4t4MuFWyGEr/KNgO8fip9DMmYKIXybbwR8cyhGuzvgSwtfCOGjfCPg+4eidxRgxCG3ZgohfFalAn5KSgo///wzAK+99hoPPfTQOZ/kuiy5E6hpt2ZKwBdC+KZKBfzp06fTpEkTfvnlF/bs2cO0adN49dVXvV23mlMc8BWr9OELIXxWpQK+yWSiYcOGfPvttwwbNoyYmBhcLpe361Zz/IsDfr608IUQPqtSAd9oNPL444/z22+/0a1bN/73v//hcFxBLeXinPh+RWRLH74QwkdVKuC//PLL9OzZk/feew+9Xo/RaOT555/3dt1qjruFH2MskIu2QgifVamAn5qair+/P1FRUbz22mssXLiQEydOeLtuNcfdwo8yFkg+HSGEz/KNi7buFn6ETlr4Qgjf5RsXbfVGMAYSppOLtkII31Wli7Zbtmy5Mi/aApgthOvzOJ5ViNOlXnh9IYSoY6p00faDDz64Mi/aAviHEmuykVvkYO+J3EtdGyGEqHWVyofvcrnYs2cPX3zxBTqdjjZt2tCuXTtv161mmUOJcOYDsOVQJq3rh1ziCgkhRO2qVAt/8uTJBAUFMXbsWO699150Oh3//ve/vV23muXOmFnPYmbzocxLXRshhKh1lWrh5+Xlcdddd3mmO3TowD/+8Q9v1ck7zKEohbu4ukk4G/88jaqqKIpyqWslhBC1plItfJfLxa5duzzTO3fuvLLu0gHt1syCLK5uGs6p3CKOZOZf6hoJIUStqlQL/8knn2TmzJkcOHAAgBYtWjB+/HivVqzGmUPBlkvXRlrf/eaDmTSOCLzElRJCiNpTqYDfokULPvjggzLzRo0axYcffuiVSnmF2QJA8xAnFn8jWw5l8rcucZe4UkIIUXuq/QIUVb3C7mV3P22rK8rm6iZhbDl05hJXSAghale1A/4Vd8HTnU+HwiyubhLOwYw8TuUWXto6CSFELTpvl86gQYMqDOyqqnLo0CFv1ck73C187cJtMwB+O3SGW9rWu4SVEkKI2nPegP/KK6/UVj28r1QLv00TC2ajjs0HMyXgCyF8xnkDfoMGDWqrHt5XqoVvMujoGBfGFnkASwjhQ6rdh3/FKdXCB+jaNJyU4znkSvZMIYSP8J2AbzSDwQwFJQHfpcLWw3K3jhDCN/hOwAftXvzCbAA6NgrFoFOkW0cI4TN8LOCHerp0AkwGEhtY2HJQWvhCCN/g1YA/a9YshgwZwtChQ/n9998rXOfFF19k5MiR3qxGCXc+nWJdm4SxIzWLQruzdvYvhBCXkNcC/ubNmzl8+DBLlixh5syZzJw5s9w6+/fvZ8uWLd6qQnmlWvgAVzcJx+Z08Xtadu3VQQghLhGvBfwNGzbQu3dvAOLj48nOzsZqtZZZZ86cOUycONFbVSjPPwzyMjyTVzcJB5B+fCGET/BawM/IyCAsLMwzHR4eTnp6umd62bJldO3atXbv9W/QCXKOQuafAIQFmmgeHcTmgxLwhRB1X6WyZdaE0snWsrKyWLZsGe+99x4nT54873YpKSnV2l9hYWG5bY1KE64CTvz8EWeaDwageajC+oMZJCXvRq+7wvIDVaCi4/YFcty+RY67erwW8KOjo8nIKOk+OXXqFFFRUQBs3LiRzBiZVfIAACAASURBVMxMhg8fjs1m48iRI8yaNYupU6eWKychIaFa+09JSSm/rdoKfm1MbN5uYt3L+haFsPqPHRDagIQGlmrt63JS4XH7ADlu3yLHfX5bt26tcL7XunR69OjB2rVrAUhOTiY6OpqgoCAA+vXrx+rVq/n000+ZN28eiYmJFQb7GqcocFVv+PNHcNgA6ccXQvgOrwX8Tp06kZiYyNChQ5kxYwbTpk1j2bJlfPvtt97aZeVc1RvseZC6EYD6of40CPWXgC+EqPO82oc/adKkMtOtWrUqt07Dhg1ZuHChN6tRVtNrQWeE/d9B0+sALc3CT/sy5MXmQog6zbeetAXwC4ZGf4H933tm/TU+ggxrET/vzzjPhkIIcWXzvYAPWrfOySTIOQ7AbR3q0yDUn2fX7MHlusJe3SiEEJXkuwEf4IDWyvcz6Hn0phYkHc1h1a7jl7BiQgjhPb4Z8GMSIShW68d3u71DA1rFBvPCN3uxOVyXsHJCCOEdvhnwFQWuuhEOrAOnAwC9TmFyv1YcPp3Pki1HLnEFhRCi5vlmwAct4BdmwbFtnlnXt4yia9NwXv5+H3lFjktYOSGEqHm+G/Cb3QCKrszdOoqiMOXmVmRYbSz4+eAlrJwQQtQ83w34AeHQoHOZfnyATo3C6JsYw1s/HuC0tegSVU4IIWqe7wZ80O7WOboV8ss+Zfuvvq0osDuZt27/JaqYEELUPAn4qHDgh7Kzo4O4s0scizYeJjUz/9LUTQghaphvB/z6HbWXopTqxy/2z94t0CkKc7/94xJUTAghap5vB3ydXrt4u/87cJW99z7WYuauHk35csdRNv15+hJVUAghao5vB3zQunXyTmmpFs7y4PXxNI0I5N4PfuP3tKwKNhZCiCuHBPyrbtSGZ92tA2DxN/LRmG5YAoyMencze0/k1nLlhBCi5kjAD46FmLYV9uMD1LP48/G9f8Fs0DN8/ib+TLdWuJ4QQlzuJOCD1spP3QjW9AoXN4oIYNG93VBVleHzN8mdO0KIK5IEfICOI7ThuhnnXOWq6CAW3duNfJuT4fM3cSK7sJYqJ4QQNUMCPkBkc+h6H2z9AI7/fs7VEuqF8MHdXcnMszF8/kZO5UrQF0JcOSTgF+v5mHZP/pp/g3rul6B0iAvl3X9czdGsAvr99yeW7ziKep71hRDiciEBv5h/GPT6Dxz+GVK+Ou+qXZuG89W4a2gUHsCET3Zw9/tbOJZVUEsVFUKI6pGAX1qnf0B0InzzONjP313TIiaYzx/8K0/0b83GPzPpM/dHFm44JK9IFEJctiTgl6Y3QL/ZkHUENsy78Oo6hXuuaco3E6+jY6MwnliezJC3N5ByPKcWKiuEEFUjAf9szXpCq/7w01zPS84vJC48gIX3dOX5we3YeyKXm1/+iTvf2sDyHUcpcji9XGEhhKicOhnw/7v1v3xw+AMcrmq+teqmGeCyw/fTK72Joij8rUsc6/91A/++uRUnsguZ8MkO/jr7B+Z8vYcjp+XefSHEpVUnA36EfwSrTq5i4vqJFDiqcTE1vCl0Hws7F0Pa1qptGmji/p7xrJ90PR/e3ZXOjcN4+38H6PnCOv7+zkbe+d+f7DmRI3f2CCFqneFSV8AbRrYeyelTp3n38Lvc/+39vNrrVSx+lqoVcu2jsONjWDMZ7vlWe/F5Feh0Cte1iOK6FlEczy7gk82prN51nJmrU2A1RAf7cW3zKK5rEUmPqyKJDPKrWv2EEKKK6mTAB+gb05fWTVoz5acp/GPNP3iz95vEBMZUvgC/YLjxSVg+Fn75L1wzsdp1qWfxZ2KfFkzs04Lj2QX89EcG/9uXzvd7TvL5tjQAGoT607aBhbYNLbRpYKFN/RAi5CQghKhBdTbgA9zU5CYsfhYmrJvAyK9H8mafN2lmaVb5Atr/XXsb1ndPQUhDaPe3i65TPYs/d14dx51Xx+F0qSQdzWbDn6dJOppN0tFs1iSf8Kxb32ImPjqIppGBZT4NQv0x6Otkb5wQwovqdMAH6FavG+/1fY8Hv3uQUV+P4vUbX6ddVLvKbazTwYA3wHoKvnwQgqK1u3hqiF6n0D4ulPZxoZ552QV2dh/LIeloNsnHsvkzI48vth0lt6jkArRRrxBrMVMvxJ8Yi5l6FjMxIdowP7OQgOg8IoP8CPSr839eIUQV+ERESIhIYOHNC7n/u/sZ+/1YVg1cRYgppHIbG/xgyCJ4tx8sGQF3fQ2xbbxWV4u/ke7xEXSPj/DMU1WV03k2DmbkcTA9j4On8ziWVcCJ7EJ+T8vim+RCihyl3tj19TEA/I16IoNNRAb5ERnkR1iAkbBAE2EBJm08wERYoAmLv5EQsxGLvxGzUYdSxesVQogrg1cD/qxZs9i5cyeKojB16lTatStpWW/cuJG5c+ei0+lo2rQpM2fORKfzXjdFXEgcL/Z8kSErh7Bg1wImdq5Cn7x/KIxYCvP7wEd/g3u/BUtDr9X1bIqieIL21U3Cyy1XVZWsfDvHswvZtnsf5rAYMqxFZOQWaUOrjdTMfHal2cnMt2FzuCrYi8aoVzzBP8hsINBkINDPQLDZQKCfniA/I0F+egJM2nTxMNBkIMBkwN+kx9+kJ8CoDf0McgIR4nLhtYC/efNmDh8+zJIlSzhw4ABTp05lyZIlnuVPPvkkH374IbGxsYwfP56ffvqJnj1rrrukIgkRCfRv1p9FuxcxtOVQ6gXVq/zGloZa0H+3HywaDHev0U4ElwFFUbSWe6AJJTuAhIRzn4xUVaXA7iQzz0ZWvp3MPBs5hXZyChxkF9jJLrCTU6gNrYUO8oocpJ3Jx1qkjVuLHNidlb+lVKdovzT8TXrMRj3+xlJD9wlB++jxM5aMm406zEZtudnonvaso803udc1GXScsjqIzC3CpNdhNCiY9Dr0OkVONkKU4rWAv2HDBnr37g1AfHw82dnZWK1WgoKCAFi2bJlnPDw8nDNnznirKmWM6ziOtYfW8ur2V5l17ayqbRyTqHXvLBoEnwyHkcu0Lp8riKIoBLhb4w3DqleGzeEi3+Ygz+Ykv6jssMDupMDmoMDmJN/u1IY2J4V2J4V2F4X24nWc5BTYKXK4KHI4KbKXDAsdziqdVEocOetYwajX4afXTg7FH6Neh6nUPD+DNm3Ul1puUDDotHGjXsGgL55W0HuGCgadgkGvw6BTMBl0nnWM7vIM7nFTqfGSbXXodGDQ6dArCnq9gl5RPPN0CnLCEjXKawE/IyODxMREz3R4eDjp6emeIF88PHXqFL/88gsTJkzwVlXKqB9Un+Gth/N+0vuMbD2ShIiEqhXQrKd2IXfZvfDxnfC397VMmz5EC5QmQgO8tw+nS6XIUXKSKHKUDIvsTmxOFzaHiyKHNjyUmkZEdCx2hwu7e5nd6aLIPe75OMuPW4scnvW1oYrdqU07nCo29/ilyIunU7SL+3qddjLwjOu0k4zL6cDsdwKDTkHnPgEVr6NTFM/22rji/tVz9jw842j/oVO09YrHdbqSk1FxOcXb6BTtuRNFoWRaOffy0uUXL1dK7Q/3OsXL9e79lz6W48esHHIcdz8eU7bs0sdYsm3Jcep1Jd9F6e+puL6KQsn27hOz/qzv1nCF/nqstYu2FT1Zevr0aR544AGmTZtGWFjFQTMlJaVa+yssLDzntteZrmOpYSnP/O8ZHm/5eNX/cMY2WLo+Tr3f5mB7/TrSrn0BW3CjatWzpp3vuOsCP/fHw6B9GseZMJvPlb5CR008VO5SVVwqOFwqTpc27XCpOFzaCap43OEZL1lmP2u506WVVVyOU8Uzr3g/TlXF5aJk3L3M6SqZttkVFJ0Op0t1z3N5lqvubW2ebUvKVovHKTWuFh+nNlSL1y117KXLcLlARdW2P6sc1b2dd53y+h7ORztJoJ1UPCejUkP3SebsoXYSw30SLX2SLBlvEmZiVMfy1+su9v9vrwX86OhoMjIyPNOnTp0iKirKM221WhkzZgz//Oc/ueaaa85ZTkJCFVvgbikpKefd9iH9Qzy75VkyLZlc0+Dc+z9PxSCxB35LRhD/wxi480Nodn216lqTLnTcdZUc9+VHVdUyJwFXmWkVFVCLT07F06p2EqHUicbp0rYrPqmpqsq+/X/StFnTMicdKHWydKmebV3uE6d2AlVLTo7uodN9hiter7huzuJ13Sdrp8ulnbSdJeU6XFo5jgrWdbrA6XJhP2udkuWltlHR6uPUyo1SAir8u1b27711a8UpYbwW8Hv06MGrr77K0KFDSU5OJjo62tONAzBnzhxGjx7Ndddd560qnNeQlkP4eM/HvPjbi3Sv1x29Tl/1Qhr/Fcb8AB8PhYUD4Zbn4Op7a76yQlyBirtpdNR814f9tIlWsZW8tVp4eC3gd+rUicTERIYOHYqiKEybNo1ly5YRHBzMNddcw5dffsnhw4dZunQpAP3792fIkCHeqk45Rr2RCZ0mMOnHSXx14CvuaH5H9QoKawL3fAOf3wurHoVTe6DfHC23vhBCXEa8GpUmTZpUZrpVq1ae8aSkJG/uulJuanwT7SLbMW/7PPo26UuAsZpXIc0hMGwxfDcNfn0VTibB7a9BRHzNVlgIIS6CTydkURSFR7s8yqmCUyzcvfDiCtPptTz6d7wNJ3fDG3+FX14BZzVz8gshRA3z6YAP0CmmE73ievFu0rv8ceaPiy+w/RAYuwnib4Rvn4AFfbQTgBBCXGI+H/ABJnedTJAxiDHfjOHP7D8vvsCQejD0Ixj8rvZ+3Leug/VzwGG7+LKFEKKaJOCjPYz1Tt93ABizdgypOakXX6iiQJtBMHYzJA6A9bPhrWthz2o895AJIUQtkoDv1szSjHdueociVxH3fHMPx6zHaqbgwAgYNB+GLQGXAz4ZBgtugoM/1Uz5QghRSRLwS2kR1oK3+7yN1Wbl3m/u5WTeyZorvGU/eGgT3PoKZKfBB/21e/eP7ai5fQghxHlIwD9L64jWvNHnDU4XnGbMt2PIKMi48Eal/HHmj3NfB9AboPNoGL9Nu6Pn2HZ4uyd8OgqOVu1l6UIIUVUS8CvQPqo9r934Gsetx7nv2/vILsqu1Ha5tlzuXXsvD3//MC713DnnMfrDXx+GCTuh52TY/wO800vLt5+0TG7lFEJ4hQT8c+gS24VXer3Cn1l/8vK2lyu1zfxd8zlTdIYjuUf4X9r/LryBOQRumAqP7IZ+z0JeOiy9C15uBz+/BPmZF3kUQghRQgL+eXSv351hrYax9I+l7Erfdd51j1qPsmj3Im5uejMxATEs2r2o8jsyh8BfHoCHt8KwT7QndL97Cua2hs/HwL7vpNUvhLhoEvAvYGyHsUT6R/LMxmdwupznXO+Vba+gKAqPdH6EYa2GsenEJvZm7q3aznR6aHkzjF4BD/4K7YfCvm/go0EwNwHW/Fu7yCu3dQohqkEC/gUEmYKY1GUSKZkpfPbHZxWusyt9F6sPrmZU61HEBsYyuMVg/A3+fJTyUfV3HJMIt/4XJv2hvWWrUTfYMl+7yPtaN1g3C45u05KSCyFEJUjAr4Sbm95M19iuvLL9FU4XnC6zTFVVXvjtBcLN4dzT9h4ALH4Wbou/jVV/riq3fpUZ/CDhVi3oT/oDbn0ZAiPhf8/DOzdoLf+vHtYe6LKd6wUgQgghAb9SFEXhP93+Q4GjgLlb55ZZ9v2R79l2ahtjO4wl0Bjomf/3hL9jc9n49I9Pa64i/mHQ+R9w12qYtB8GvKm1/JO+0B7oeq4pcT/+U7vgm7ZV+v2FEGVI0vZKahbajFGtR/Fu0rsMaj6ITjGdsDvtvLT1JeIt8QxsPrDs+pZmXNPgGpbsWcI9be7BpDfVbIUCI6DDMO3jsMHhX+CPNRhT1mgXfAFMwdC4OzS5Fpr0gJi2YKjhegghrhgS8Kvg/nb3s/rgamZsmsGn/T9lyd4lHMk9wus3vo5BV/6rHJkwkvu/u581h9ZwW/xt3quYwQTxN0D8DfzZ5B8kxEXAoZ+09A2HftYu/ALo/aBeO2jQueQT3gyuwJcxCyGqTgJ+FQQYA5h89WQmrp/Imzvf5JO9n9C9XvdzvhO3e/3uxFviWbR7Ebc2u7X23nIfFK0lbmszSJvOOQ6pG7WneY9ug20fwqY3tWVmi9byj0ks+UQngCnw3OULIa5IEvCr6MZGN9KjQQ/e+v0tFLQXqJwrkCuKwojWI5i+YTpbT26lS2yXWq6tW0g9SLxD+4DWt5+xF9J+09I7nEyGHR+BzVpccwhvCpEtIfIqiGgOEVdBZHMIjJJfBEJcoSTgV5GiKEztOpVBXw3i/5r9Hy3DW553/f7N+vPytpdZlLLo0gX8s+kNJa35zqO1eS4XZB3Wgv/JZDiVDBn74MAP4Cwq2dZs0bqBQhtr7/P1fBqDJQ70xktwQEKIypCAXw2NQhqxZtAawsxhF1zXbDDztxZ/Y/6u+aTmphIXHFcLNSwrLTeNH478QKvwViRGJpa5m8hDp9Na9eFNIaF/yXyXE7JTIWM/nN6nnQTOHIQTv8OeVeCylypEgaAYsDSAkAZgaegeNoCgWAiO0Yamar47WAhxUSTgV1OEf0SF81X3U7Clu3mGtBzCe0nv8XHKx0zuOrlW6le6Pk/++iRbTmzR6oVCfGg8bSLb0DayLV1iutAstNm5C9DpS1rxzXuXXeZyQs4x7ZfBmUPa272yj0JOGpxKgf3fgb2CZwP8LO7gH6NdbwiM0p4tCCwej4KAcAiI0H5RSBeSEDVCAn4NsqUd5eiECegtITR84w10fn4AxATG0LdpX5b+sZT/a/Z/tIlsU2t12nBsA1tObGFCpwm0Cm/Froxd7ErfxY+pP/Ll/i/RKTrm3zSfq2OvrnrhOj2ExmmfJhVcuFZVKDijnRSsJyD3JOQeB+tJyD2hDY9th7wMKMo5xz4M2vMHARHaxz8M/EPBHFoy7h9GYHouBOdrJwizRctPZPCr+jEJUYfVyYB/+t33YOtW8u+5G/+OHWvl7pj8LVtIGz8B1WbDlZfHsSlTaPDiiyg67dm2SV0msePUDsZ+P5aPbvmIhsENvV4nl+riv9v+S4OgBoxqPQqT3uS5o0hVVdJy0xi9ZjRv7XyregH/QhTF3VIPBy5wkrMXatlCiz/5mZB/Ggrcw/zT2rzMP6EgSzuROAo8mzcC+PGsMg1m8AvRgr9fsDZePDSHgCkI/ILcw+Cy06ZA9ycIjAHayUN+aYgrXJ0M+IrRCL/+yuHvv8d0VTyhgwdjuf12DGEX7nOvjjNLPuXEM89giouj4euvYf1hHaeef55TMbHETNG6cCL9I3n9xtcZ+fVIHvzuQRbdsgiLn8Ur9Sn2zaFvSMlMYW74fVgXf0Zwrxsw1q8PaF1OcSFxjE4czQu/vcCOUzvoEN3Bq/U5L6O55NdCZdkLKUrezsmX5pGfmka9YX0JuboZii0XCrOgMFv7FFmhKFf7FZGXro0X5oAtF8733oLSFL12AjAGaO8zKDfu7/4ElB0a/LVjKzP0105GRrM29Hz8tKG+Tv5vKS4DiqpevqkXt27dSufOnau1bcq2bdT780/OfPYZhTt/RzEaCe7Th7ARIwjo1LFG6qfa7ZycPYczH39M4LXX0mDui+iDg1FVlZOzZnNm4UJi/j2F8NGjPdv8duI37vv2PtpFtePtPm/X+BO4KSkpJCQkYHfZGf9KP27+IZvmf+RpCxWFgG7dsAy4nZA+fdAFBpJvz6fv531pG9mW13u/XqN18SZnVhbpr87jzCefoAsMxBUaCkeOYG7dmuh/TSKwe/cLF6KqYC/QbkctynUPrdp1B5sVbHmlPlYtV5G9+FOgzbfna/MdBdq84mWOwuofnKIvewIw+JV89GXHs/OLsIRFag/fFS/Tm0qGnnGjtlxvLJmvN5aaNoLOWHa+Z9qgDXVGrRvvMvilU/zv3NdU9rjPFTvrbsAv9cUU7v2DrM8+I/urr3Dl5GAZMIDoyY9dVIvfceYMR/85kfxNmwi/+26iH30ERa/3LFedTo7+cyK5331Hg5fmEtKvn2fZ6j9XM/mnydxarw+PnupI3k8/YwiPwFi/Psb69TDWq4exfn0M9ep5rgNU5bibOJz8PmcqIVv34bIEEXvfgwT1vI6cNWvJXr4ce2oqSkAAIX36YBlwOx/7/84rO+expP8SWke0rvZ3UhtUh4Mznywh/dVXceXmEjZ0CJEPP8y+Y8eof+AA6f99GfuxYwRecw3Rkx7F3KrVpamoy+U+CRRqJwFHofuEUKDNdxS5x4u0ZcXLnTb3dFGpYZF7G5t2i6zDvY7TRlF+Dn463POLwGnXxl3eyqOklJwkdIZSJwaDe2gsOTHojdo6pT969zKdseJpncE9bSi1vf6scvQcP5lOvfoNz9rGvZ6iP888Xdnlir6C9fRllym6s8Yv3QlPAv45VPTFuAoKyHjzLU4vWIA+MJDoxx7DMvCOSvfxqzYbeZu3YP3hB3K++QZXdjaxTz9N6B0DKlzfVVjIkbvupjA5mUbvLiCgi3Yfvv3YMdb/dzKh3/xGUCGYGjfGVVCAIz29bK57RcHUrBn+bdvi374d5rbtMLdojmIq+VWgulw4MzOxnziJ48Rx0j5cCJs3Y/VX2NKrAfdP/wJ9UFDJ+qpKwbZtZH/5JTlfr8FltaILD+eHplas17Rn0ph3UQzV71JQVRV7WhqFybsp3K19bEeOlBzXWcenM/uh+AegM5vR+fujBPijc08rfn4ofiZ0fn4oJj/Q68hauhTb/gMEdP8LMVP+jbllC6Dk7+0qKuLMRx+T8dZbuHJyCO7Xl4AOHTA1aYKpSROMDRpc1PFdKqqqaicRVQVV1aZVlb0pKbRs2RJcLvdX617P6QCnHbX4hGEvAqcN1e4+KThs7pODzTOuFk+7ty3+qE6Hdvut0w4Oh7aOy4lavK7Loc13ObQyXE73x73M5dKGTgeqZ5mzZJnTAarTvcxVskx1QvE/GxTPeMmXctZoueXKeZad/f0C59lHmdmq4g78Om0b90lALR7nrOVl1im9nXsZxSeRkvmmZvEEPfpBuX8HEvDP4XxfTNG+fRx/ajoFW7cS0KULsdOfwi8+vsJ1nVlZWH/6idzvfyDvp59w5eWh+PsT+Ne/Enn/ffi3a3feejjOnOHwsL/jOHOGek9NI+ebb8j95ltQVVI7N+DNVscYNPDfdI3tijU/i7xjqdiOHcVx7DiGE6dpcsKFIykF52ktzbJiMuGX0ApFp8d+8gSO9Aywl7oXPiCAw7d24MkGm3jz9g/pFNPpnHVzFRZiXb+enLVryfrhO/RFDgizENqnL8F9+qAPDsJpzcNlzcVlteK0WnFZ81CLClFtdlSHQ/vY7agOO46TpyjcvRtXbq62A4MBv+bN8WvaBEoF2eITrOpSUQsLcRUUuD/5qPnauFpUhMtmQy0sLJPz3xgXR8yUyQT16lXmRH3239uZnc3pd94h67OlOLNLvZPYaMQUF4cpLg7FZER1qZ4gisuFigpOlzugucDpRHWVGrpcqKrLvY573KWed9wToN3bl5k+a1yrA2Wmhe8xRQYQ//PWcvMl4J/Dhb4Y1eUie9kyTj7/Aq78fEIHDgSdgjPjNI7Tp3GczsB5OhOXVUs3oI+KJPj6GwjqdQOB3bujM5srXRdbWhqHhg7DmZGBLjiY0L/9jfDhf0epF8PDPzzMz0d/Pue2DYMaMu/GeTTKM1OwaxcFv++icNcu0OsxxkRjiI7BEBuDMSYGQ0wMSQVnmHBgCp2iOzHvxnmVruOZMyf4z4s383+HI4hPPoOaf+7c+orRqF0Ydw8VgwHFaEQfFoa5dQLm1q0xt07Er0VzdKaLv0ahOhzaCaCoCL3FUqbrrNj5/t6OM2ewHTyE7dAhbAcPasO0NHA6tZaVooBOQSluden12t1V5xq6P55xRSk1X0FRSuaXmdYp7pOUtg9td6XWVdDKUUpvW9xC1KY9+3LPS09PJzo2pqS1qNO5F7m3Uc4uu1R5Clp5FawLpcsou025+cX1Krd+8T+YipZVVBbll7m3L7MtcPjwYZo0bVJ227OP7exjKLOPUstK74Nz7PvscZRSo+dY11OJ85RXeliqvjp//zK/5ItdbMC/8n7b1hBFpyN08GCCbriBU889R9bSpeiDgtBHRmKIiMA/MRF9eASG6GgCu3XF3Lat5xbLqjI1bEjjD96nYPt2Qm6+GV1gyZOuc6+fy4+pP6JTdAQZgwgyBXmGqbmpPLL+EUZ+PZKXrn+Jbv36lbkWUJHl3zyO1Wbl4Y4PV6mOYWGxXDVgJFN3f8DyFz8jcu8pQEUXGIQuKBB9UBC6oCB0gYEVBtyakl2UzfvJ7/OXen+hW71uANoJxWAo871VhSEsDENYWI1drL+cpKekEOGDFy/xN+Pvi8d9kXw24BczRERQ/9lnqTd7drUDemX4xcdX2G3kb/CnX9OKg3h0QDQf3fIR474fxwPfPsCT3Z/kjuZ3nHMfJ/NOsvrE6krl+KnIqMRRfLznY97dt4inr326yttfrHVH1vH0xqfJKMhg/q75/L3V3/ln53/ib/Cv9boIURd59Y1Xs2bNYsiQIQwdOpTff/+9zLJff/2VwYMHM2TIEF577TVvVqNSvBnsL0bD4IYsvGUhV8dezZO/Psl/t/4XV6l7x1VVZW/mXj5I/oDx68bjwsVDHR6q1r4i/SMZ1HwQKw6s4Jj1WE0dwgVlF2Xz75/+zfh14wk3h7Pw5oUMTxjOx3s+5s4Vd5KUkVRrdRGiLvNaC3/z5s0cPnyYJUuWcODAAaZOncqSJUs8y2fMmMGCBQuIiYlhxIgR9O3bl6uuuspb1bmiBZuCea33a8zeNJsFSQs4knuE6xpex8bjG9l4bCOnC7ULuk0tTbm38b0XlaDtrjZ38ekfn/Ju0rs8/pfHL7h+ZmEmB7IOcCDrAA6Xg5bhLWkR1qLSD5UVt+qzCrN4sP2DjGk7BqPeSIfoDvRsylglgQAACVVJREFU2JMnfnmCEatHcF+7+xjTbgxGnWTjFKK6vBbwN2zYQO/eWrKt+Ph4srOzsVqtBAUFkZqaisVioV69egD07NmTDRs2SMA/D6POyBN/eYLGIY158bcX+fbwt4Sbw/lLvb/QvX53/lLvL8QGxpKSknJR+4kNjOX2+Nv5Yt8XtArX7mF3uBwlH9XBcetxDmQf4M+sPzlTdKbCcuoH1qdFeAtahbeiUXAjXKoLu8uOzWnD7rJjd9lJOZ3CN4e/oWVYS97o/YZnf8W61+/OstuXMXvTbN7Y+Qb/S/sfwxOGe5a7VO2uGlVVOZZ+jD2GPeXqoSjaxVidov2C0yk6dIrOfYG24u9AOdeCy9DRzKMcOXTkUlej1tX1424U3IiEiJq/RuG1gJ+RkUFiYqJnOjw8nPT0dIKCgkhPTyc8PLzMstTU1ArLqW4AKywsvOjgdznqquvK822ex4WLRv6NtEBmhzNHznCGMzVy3Nf7X89X6ldM3zC9wuWB+kAa+jekU0gnGvo3JM4/job+DdGh41D+Ic9nX/o+fkz9UbvVsQJGxcjg+oMZWH8g6kmVlJMV13tU5Cha6FrwzqF3mPrz1HNX/GCVD7Vu2H+pK3CJ1OHjDjOG8VbHt8rNv9j/v2vtom117/6s7uPTdfnR6wTOfVw1cdwJJPBdwncUOAowKAb0Oj1GnRGDzoBe0eOn9zvnw2rdKZvSIN+ez8n8kxh0Bkw6E0a90TM06oyelndl6nRn1ztJz09HURR06Nx3BGqt9QP7D3h+IZY+wRT/CkAFFy5UVfUMK3IZ36VcoT///JNmzc6T3rqOquvHHekfSag5tNz8qtyWWRGvBfzo6GgyMjI806dOnSIqKqrCZSdPniQ6OtpbVRHVEG4Ov/BKlRBgDKCppWmNlBVoDCTQUvGtmdl+2dQLqlcj+7mS2APsXBXme12hvnrcF8trt6b06NGDtWvXApCcnEx0dDRB7kf8GzZsiNVqJS0tDYfDwbp16+jRo4e3qiKEEAIvtvA7depEYmIiQ4cORVEUpk2bxrJlywgODqZPnz489dRTPProowDccsstNG1aM61AIYT4//buLiSqfY3j+NecRIzaW6xEbfdCFE1MOxEMlJJEKJK6CaQXLLyojGHIG7OV2lX4LqQNUZJJYYqGUXRREEFSVApKSZJR9kImE5QUGTriTJ0LcY5yrMNJPbrX+n3u/sub/8+XZ5YPa/0fmdi09vCzs7PHrdeMObkwPj5+3GOaIiIyvWbn20YiIjLlVPBFRCxCBV9ExCJU8EVELGLWn4cvIiL/u3/cABQREZk6aumIiFiECr6IiEWYbuJVYWEhHR0dBAUFkZuby9//Zcj4P92LFy9wOp1kZGSQnp6Ox+MhJycHv9/PokWLKCsrI2QK5srONqWlpbS3t+Pz+cjMzGTdunWmzz04OIhhGPT19TE0NITT6WTNmjWmzz3K6/Wyfft2nE4nCQkJps/d2tpKVlYWq1atAmD16tUcOHBgUrlNdYc/duhKQUEBBQUFM72laTUwMMDJkydJSPj3CZWnT59m79691NfXs2zZMpqammZwh9OjpaWFly9f0tjYSHV1NYWFhZbIfffuXRwOB5cvX6aiooLi4mJL5B519uxZ/vhjZLCOVXJv2LCB2tpaamtrOXHixKRzm6rg/2zoilmFhIRw/vz5cSeNtra2kpKSAkBycjKPHj2aqe1Nm/j4eCorKwFYsGABg4ODlsidmprKwYMHAfB4PERGRloiN8CrV6/o7u5m8+bNgDV+zycy2dymKvifPn0iPDw8sB4dumJWNpuN0NDQcdcGBwcD/+JFRESYMn9wcDBhYWEANDU1kZSUZInco3bv3k12dja5ubmWyV1SUoJhGIG1VXJ3d3dz+PBh9uzZw4MHDyad23Q9/LGs/sSp2fPfuXOHpqYmampq2LJlS+C62XM3NDTQ1dXF0aNHx2U1a+7r168TGxvLX39NPKvZrLmXL1+Oy+Vi27Zt9PT0sH//fvx+f+Drv5PbVAX/V0NXrCIsLAyv10toaKipB8vcv3+fc+fOUV1dzfz58y2Ru7Ozk4iICKKiorDb7fj9fubNm2f63M3NzfT09NDc3MyHDx8ICQmxxM87MjKS1NRUAJYuXcrChQt5+vTppHKbqqXzq6ErVpGYmBj4Hty+fZtNmzbN8I6mXn9/P6WlpVRVVfHnnyNj4KyQu62tjZqaGmCkfTkwMGCJ3BUVFVy9epUrV66QlpaG0+m0RO4bN25w4cIFAD5+/EhfXx87d+6cVG7TvWlbXl5OW1tbYOjK2DP4zaazs5OSkhJ6e3ux2WxERkZSXl6OYRgMDQ0RHR1NUVERc+fOnemtTqnGxkbcbve4oTnFxcXk5+ebOrfX6yUvLw+Px4PX68XlcuFwODh27Jipc4/ldruJiYlh48aNps/97ds3srOz+fr1K8PDw7hcLux2+6Rym67gi4jIxEzV0hERkZ9TwRcRsQgVfBERi1DBFxGxCBV8ERGLMNWLVyK/4/379+zYsQOHwzHuutvtDjzn/zvcbjfh4eGkp6dPdosiU0IFXwRYsWIFtbW1M70NkWmlgi/yE4ZhEBYWxuvXr/n8+TNFRUWsXbuWS5cucfPmTQBSUlI4dOgQvb29GIaB3+8nOjqakpISYGReQWZmJm/fviUvL4+kpKSZjCQWpx6+yC/4fD4uXrxIVlYWZ86coaenh2vXrlFXV0ddXR23bt3i3bt3nDp1ioyMDOrr61m8eDGdnZ0AfPnyhaqqKvLz82loaJjhNGJ1usMXAd68ecO+ffsC69FjGxITEwGIjY2lvLycrq4u1q9fj8028qcTFxfH8+fPefbsGXl5eQDk5OQAcO/ePeLi4oCRg7D6+/v/b3lEJqKCL8LEPXzDMPj+/XtgHRQURFBQ0LhjaYeHh5kzZw7BwcETHlc7+sEgMhuopSPyC+3t7QA8fvyYlStXYrfbefLkCT6fD5/PR0dHB3a7HYfDQUtLCwCVlZU8fPhwJrctMiHdfojwny0dgNDQUGw2G5mZmXg8HsrKyliyZAm7du0iPT2dHz9+kJaWRkxMDEeOHOH48ePU19cTFRWFy+UKfFiIzBY6LVPkJwzDYOvWrSQnJ8/0VkSmhFo6IiIWoTt8ERGL0B2+iIhFqOCLiFiECr6IiEWo4IuIWIQKvoiIRajgi4hYxL8A287Qg8uaP0UAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Plots of convergence on MLP"
      ],
      "metadata": {
        "id": "SzYhA1Bo2jOU"
      },
      "id": "SzYhA1Bo2jOU"
    },
    {
      "cell_type": "code",
      "source": [
        "# load the results\n",
        "\n",
        "file = open(\"plot_nosadam_mlp_train.pth\", \"rb\")\n",
        "loss_all_train_mlp_nos = pickle.load(file)"
      ],
      "metadata": {
        "id": "5yl2JJ-I4BL5"
      },
      "id": "5yl2JJ-I4BL5",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# load the results\n",
        "\n",
        "file = open(\"plot_nosadam_mlp_test.pth\", \"rb\")\n",
        "loss_all_test_mlp_nos = pickle.load(file)"
      ],
      "metadata": {
        "id": "Zo8vpEvs4EcN"
      },
      "id": "Zo8vpEvs4EcN",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.core.pylabtools import figsize\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "plt.style.use('seaborn-whitegrid')\n",
        "\n",
        "epochs = np.arange(0, 50, 1)\n",
        "\n",
        "plt.plot(epochs, loss_all_train_mlp, label='Train Loss Adam')\n",
        "plt.plot(epochs, loss_all_test_mlp, label='Test Loss Adam')\n",
        "plt.plot(epochs, loss_all_train_mlp_nos, label='Train Loss NosAdam')\n",
        "plt.plot(epochs, loss_all_test_mlp_nos, label='Test Loss NosAdam')\n",
        "\n",
        "plt.legend()\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.savefig('mlp_conv.pdf')\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276
        },
        "id": "Sajp8TAcozOm",
        "outputId": "0aad4156-253d-4472-d57e-006ef30a5fc7"
      },
      "id": "Sajp8TAcozOm",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEDCAYAAAA4FgP0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeUBUVfvA8e/MsMkqyAwugCigICbmmoHggmvummIqaOWOpmlq/nKrwCXT17Tecs0txQVFwdRcs0Q0XEEwQUQUZVNWZef3B6+TBCgCA+qczz8x98459zlI88y999znSAoLCwsRBEEQ1Ja0pgMQBEEQapZIBIIgCGpOJAJBEAQ1JxKBIAiCmhOJQBAEQc2JRCAIgqDmNFTZuY+PD1euXEEikTB37lxatGhR4j3ffvstly9fZuvWreVqExISosqQBUEQ3litW7cudbvKEsH58+eJiYnB19eXqKgo5s6di6+vb7H3REZGcuHCBTQ1NcvdBsoeTHmEh4djb29f4favKzFu9SLGrV7KM+7nfYlW2aWhoKAg3NzcALC2tiY1NZWMjIxi71myZAnTp09/qTaCIAhC1VJZIkhKSsLY2Fj52sTEhMTEROVrPz8/2rVrR4MGDcrdRhAEQah6Kr1H8KxnK1mkpKTg5+fHpk2biI+PL1ebZ4WHh1c4jqysrEq1f12JcasXMW71UtlxqywRKBQKkpKSlK8TEhKQy+UAnDt3jocPHzJixAhycnK4c+cOPj4+z23zrMpcAxTXENWLGLd6EeMuW43cI3BycuLIkSMAhIWFoVAo0NfXB6Bnz54cOnSIXbt2sWbNGhwcHJg7d+5z2wiCIAiqobIzglatWuHg4IC7uzsSiYQFCxbg5+eHgYEB3bp1K3cbQRAEQbVUeo9g5syZxV7b2dmVeI+5ubnyGYLS2giCIAiqpTZPFj/KekT3Pd258/hOTYciCG+8JUuWMGrUKHr27ImrqyujRo3Cy8urXG2nT59OVlbWC9939+5dBg0aVNlQnys7O5s2bdrw888/l7o/MzOTLl26qDSG6lBts4ZqWm5BLvcz7xOWFkYPetR0OILwRpszZw5QNE385s2bzJ49u9xtV65cqaqwXtqpU6cwNTXl0KFDjB49uqbDURm1SQTyWnL0NfW5l3WvpkMRBLU1Z84cNDU1SUlJYfHixcyYMYPHjx+TlZXFvHnzaNGiBV26dOHgwYN89dVXKBQKwsLCiIuLY/ny5Tg4OLzwGDdu3ODLL79EKpWip6fHkiVLkMlkTJs2jZycHHJycpg/fz6WlpYltv27/4CAAKZOncrSpUuJjY3FwsKCjIwMpkyZQnZ2drEqBwcOHGDbtm1IpVJsbW356quv8PPz48KFCzx69IibN28yffp0AgICiIqKYvny5Tg6Olb577gi1CYRSCQSGhs15t4TkQgE9bI35C67/oqt0j6HtrFgcGvzCrU1MjLiq6++Ijo6mvfffx83NzeCgoJYt24dq1evLvbenJwcNmzYwI4dO9i/f3+5EoG3tzezZs3C0dGRDRs2sGXLFuzs7DAzM8PHx4fY2Fiio6O5d+9eiW3PysjI4MKFC3zzzTdcu3aNQ4cOMX78ePz9/bG1tWXu3LkcOnSIwMBAAJ48ecL69esxNDRkxIgR3LhxA4Dbt2/zyy+/sHv3bn766Sf279+Pn58fAQEBr0wiUJt7BACNjBqJMwJBqGFPC0mamppy5MgRhg8fzvLly0lJSSnx3jZt2gBQt27dcpebiYqKUn7Atm/fnuvXr9OyZUsuX77M/PnziYmJwcXFpdRtzzpy5AjOzs7o6OjQp08fAgIClP2//fbbALRr1075fiMjIyZNmsTIkSOJiopSjqd58+ZIJBLkcjlNmzZFJpNhamr6SpXPUZszAoDGtRvjH+VPWk4ahlqGNR2OIFSLwa3NK/ztXRWeFpncvHkzZmZmym/cy5YtK/FemUym/LmsSgPPk5ubi1QqRaFQ4O/vT3BwMDt27ODy5ct4eXmVuu2pgIAA7ty5Q//+/YGib/aRkZEUFhYilRZ9hy4oKACKzly+/PJL/P39kcvljB8/XtmPhoZGqT9XZDyqolaJoJFhIwBupdyipaJlDUcjCOrt0aNHNG3aFIBjx46Rm5tbJf3a2tpy6dIl3n77bS5cuEDz5s05e/Ysubm5uLq6YmNjw8KFC0vd9lRiYiKRkZGcPHlS+eG9Zs0aAgICaNSoEaGhofTo0YPg4GCgaPaQTCZDLpdz//59QkNDq2w81UGtEkHj2o0BiE6NFolAEGpY//79mT17NocPH2bEiBEEBASwd+/el+ojOjqaUaNGKV8PGTKEL774gkWLFiGRSDAyMmLx4sWkpKTw2WefsX79eiQSCVOnTqVu3boltj116NAh+vTpU+wb/MCBA/nwww/ZvXs3kydPxtPTU3mz2NjYGCcnJwYPHoydnR0ff/wxixcvxtPTs5K/peohKXyVzk/KISQkpMLrEeQV5NF2W1tGNhvJjDYzqjiyV5uowaJexLjVS3lrDZX12alWN4s1pBrU06lHdGr0i98sCIKgJtQqEQA0qNWAW6m3ajoMQRCEV4baJQJzHXPuZdwjOz+7pkMRBEF4JahdImhQqwEFhQXcTr1d06EIgiC8EtQyEQDiPoEgCML/qF0iqK9THwkScZ9AEAThf9TqOQIALakW9fXrizMCQVChJUuWEBYWRmJiIk+ePMHS0hIjIyPWrFlTrvYRERFoa2vTqFGjYtufFqTT09NTRdgAfPTRR2hra/PDDz+Uun/QoEF89913mJu/Ok9rV5baJQKAxkaNxRmBIKhQZcpQA/z22280b968RCJQteTkZKKiosjKyiI9PR0DA4NqPX5NUWki8PHx4cqVK0gkEubOnassNgWwa9cu9uzZg1Qqxc7OjgULFnD+/Hk++eQTbG1tAWjSpAnz5s2r8rgaGzUm+H4w+QX5yKSyFzcQBKHS8vPzmTdvHrGxseTl5TF16lQ6dOjA/v372bZtG5qamtjZ2eHu7s7OnTsxMTGhTp06xT43SpOens6cOXNIS0sjPT0db29vHBwc+PrrrwkNDSU/P5/hw4czaNCgUrc969ChQ3Tu3Jm0tDSOHj3K4MGDAfj666+5dOkSjRo1UpaOiIiIYNGiRWhoaCCVSlm1ahUZGRnMmjULS0tLLl26xPDhw7lx4wZXrlxhxIgRjBgxQjW/3EpSWSI4f/48MTEx+Pr6EhUVxdy5c/H19QWKyrUGBgayfft2NDU18fDw4NKlS0BRNb/vvvtOVWEBRaUmcgpyiMuIw8LQQqXHEoQad3kHXNpWtX2+PRJaDn+pJgcPHkQul+Pj48PDhw/x9PTk4MGDbNiwgbVr11KvXj327t1Lw4YN6dixIz169HhhEoCi4nWOjo6MGzeOgwcPsnjxYtasWcOpU6eUNYz27dtHSkpKiW3/FhAQwGeffUZ6ejrbtm1j8ODBREZGcvHiRfbs2UN8fLxyzfXk5GTmzZtHs2bNWLVqFQcPHqRz586Eh4fz/fffk5qaSp8+fTh+/DjZ2dlMmTJF/RJBUFAQbm5uAFhbW5OamkpGRgb6+vrUqlWLzZs3A0VJISMjA7lcTlxcnKrCKaaxUVHNoVupt0QiEIRqcunSJUJCQrh48SJQtAxkTk4Offr0YfLkyfTr148+ffqgo6PzUv2GhoYyceJEAGxsbIiJiaF27dpYWVkxceJEevbsyYABA9DS0iqx7VmxsbHEx8fTunVr8vLy+OKLL3j48CGRkZE4OjoilUqpV68eFhZFnxl16tRh+fLlZGVlkZCQQN++fQGwtLTE2NgYLS0tTExMMDMzIzMzk/T09Mr+ClVGZYkgKSmp2CISJiYmJCYmoq+vr9y2du1atmzZgoeHBxYWFsTFxREZGcmECRNITU3Fy8sLJyenEn2Hh4dXOK6srCzy4vMAOHfzHIoMRYX7ep1kZWVV6vf2uhLjBrRbwjsqKLJYjt9rXFwcycnJhIeHk5GRQd++fYvV/Y+KisLFxQV7e3vOnj2Lu7s73t7epKSkEBsbW+LfLicnhxs3blCrVi3ltszMTKKjo9HS0iIrK4ucnBzCw8OZMWMGUVFR/P7772zfvp1FixaVuu2p3bt38/jxY3r27AkUJapNmzZhZGRESkqKMpasrCwiIyNZvXo1gwYNolWrVuzfv19ZsTQ3N5fw8HCePHlCQUGB8uencalCZf/Oq+1mcWm17caNG4eHhwdjx46ldevWWFlZ4eXlRa9evYiNjcXDw4OjR4+ipaVVrF1liko9Lc5U53odMrQz1KZAlSjGpV5elXGHh4eTmZmJvb09nTp14sSJE4wfP57k5GQ2b97MtGnTWLVqFV5eXri4uJCeno6+vj7GxsbUr1+/xBi0tLRo2rRpsVlDHTp0ID4+ngEDBrB//37s7e0xMDDgxIkTeHh40KdPHwYNGlTqtmf7P3/+PFu3blWWxr5w4QIrV65k/vz5HDt2DDs7O+Li4khMTMTGxoacnBycnJyoX78+4eHhtGzZEhsbG3R0dLC3tyczMxMtLa0SP6vq91yeonNlUVkiUCgUJCUlKV8nJCQgl8sBSElJ4ebNm7Rt2xYdHR1cXFy4ePEirVu3pnfv3kDR6ZWpqSnx8fHKU7Gq1MiokZhCKgjVqFevXpw7dw53d3fy8/Px8vJSris8bNgwDAwMsLCwwN7enjZt2vD111+jp6dHhw4divUzduxY5YI1ffr0wcPDg7lz5+Lh4UFGRgZLly5FoVBw6dIlDh06hKamJoMHDy5121MRERHKJPNUmzZtSE5OxsjIiCZNmjBs2DCsrKyws7MDYOTIkUyePBkLCwtGjRrFl19+qfz8eu0UqkhISEjh6NGjCwsLCwtDQ0ML3d3dlfsSExMLu3TpUpiRkVFYWFhYOGXKlMLffvut0N/fv3D9+vWFhYWFhQkJCYWdOnUqzM7OLtbvX3/9Vam4rl+/XlhYWFj45dkvCzv80qGwoKCgUv29Lp6OW92IcasXMe6yPe+zU2VnBK1atcLBwQF3d3ckEgkLFizAz88PAwMDunXrxuTJk/Hw8EBDQ4OmTZvStWtXMjMzmTlzJsePHyc3N5eFCxeWuCxUVRrXbkx6TjrJWcmY1jJVyTEEQRBeByq9RzBz5sxir5+eUkHR03n/nsOrr6/Pjz/+qMqQlBoZ/bNspUgEgiCoM7WrNfTUs1NIBUEQ1JnaJgIzXTP0NPVEIhAEQe2pbSKQSCQ0MhQzhwRBENQmERQWFnLmZiIFzzzP0Li2KD4nCIKgNongTnwaJ6fP568bicptjYwakfA4gYycjBqMTBDePEuWLGHUqFH07NkTV1dXRo0ahZeXV7naTp8+naysrBe+7+7duyUmnFSl4OBg3n77bRIT//nMWL16NcHBwRXq76OPPmLSpEll7h80aBB3796tUN+VpTZlqI0zH/J+5CnOBjeGga7APzOHolOjeUv+Vk2GJwhvlMqUoV65cqWqwnpp5ubmrFmzplgpiop41ctbq00iMGhoQa5MA2lsrHLbszOHRCIQBNWbM2cOmpqapKSksHjxYmbMmMHjx4/Jyspi3rx5tGjRQrn4zFdffYVCoSAsLIy4uDiWL19erH5ZWW7cuMGXX36pfGp5yZIlyGQypk2bRk5ODjk5OcyfPx9LS8sS2/7df/fu3fnzzz+Jjo4usTbCsmXLuHjxIvn5+YwYMUJZ4uLZktoLFiwAVF/eulWrVpX6d1GbRCDR0OBRnXroxd9TbrMwsEBDqiHuEwhvtANRB9h3s2TJ5coYaDuQftb9KtTWyMiIr776iujoaN5//33c3NwICgpi3bp1rF69uth7c3Jy2LBhAzt27GD//v3lSgTe3t7MmjULR0dHNmzYwJYtW7Czs8PMzAwfHx9iY2OJjo7m3r17JbaVZvr06axYsaJYbBcuXODmzZvs3LmTx48f069fP9zc3EqU1M7KykJHR0fl5a0rmwjU5h4BQI65FYqH98nLLwBAQ6pBQ4OGIhEIQjV6usaAqakpR44cYfjw4SxfvpyUlJQS723Tpg0AdevWJSOjfPfyoqKicHR0BKB9+/Zcv36dli1bcvnyZebPn09MTAwuLi6lbitN+/btycnJ4fLly8ptoaGhtG3bFgBdXV1l+eunJbV//vlnXF1d0dHRKVbe2tnZmYiIiBeWt16xYgUjR44kMDBQ+Xt5Wt5aLpcry1vXqVOnSspbq80ZAYCWjTWKi2eIjk3C1qqo/HTj2o25+ehmDUcmCKrTz7pfhb+9q4KmpiZQtKCMmZkZ33zzDdeuXWPZsmUl3vu0uByUXsH4RXJzc5FKpSgUCvz9/QkODmbHjh1cvnwZLy+vUreV5tNPP+Xrr7+mXbt2QNH089KOM378ePr27cuRI0fw9PRk27ZtBAQEkJ2drVz/IC8vj19//RUTExOk0n++ixcUFH1B9fb2ZuzYsbi4uLBhwwYeP35c4nehoVG1H91qdUZQx6GoTGvspVDltkZGjYhNjyU3P7emwhIEtfTo0SMsLS0BlKuGVQVbW1vliocXLlygefPmnD17lrNnz+Ls7My8efMIDQ0tdVtZmjZtSoMGDTh58iQAzZs3V84eyszM5M6dOzRs2JCVK1cil8sZM2YMLVu2JC4ujsDAQH7++Wf8/f3x9/dnzZo1BAYG0qhRI8LCwigsLOTevXvcu1d02TolJQVLS0tycnI4ffp0lf1enketzggsWzXnHvAw7AYM7AIU3TDOL8wnJi0GG2Obmg1QENRI//79mT17NocPH2bEiBEEBASwd+/el+ojOjqaUaNGKV8PGTKEL774gkWLFiGRSDAyMmLx4sWkpKTw2WefsX79eiQSCVOnTqVu3boltj3PJ598Qo8ePYCiS1bNmzdnxIgR5OXlMWPGDHR1dUuU1JbJZK9FeWtJYUXOt2pQSEgIrVu3rlDbwvx8rrZ4m+vtujN803IAridfZ1jAML51/ZbuVt2rMtRXyquyUEl1E+NWL2LcZXveZ6daXRqSyGQkG5uhffe2cpuVoRUSJNxMEfcJBEFQT2qVCAAy6zbANOmecuaQrqYuzU2bc/be2RqOTBAEoWaoXSLAwgL5kxRuxyYoN7mYu3At6RpJT5Ke01AQBOHNpNJE4OPjw7Bhw3B3d+fq1avF9u3atYuhQ4fi7u7OwoULlVPDntemKtRq3BCAOxevK7e5mrtSSCF/3Pujyo8nCILwqlNZIjh//jwxMTH4+vri7e2Nt7e3ct+TJ08IDAxk+/bt7Ny5k1u3bnHp0qXntqkqpk2LHhN/FBah3GZnYoeiloLf7/5e5ccTBEF41aksEQQFBeHm5gaAtbU1qampyicDa9WqxebNm9HU1OTJkydkZGQgl8uf26aqaNc3I0emSU5UlHKbRCKho3lHzsadFc8TCIKgdlSWCJKSkjA2Nla+NjExKVbOFWDt2rV069aNnj17YmFhUa42lSaT8ci0Ptr3bhfb7GruSmZuJiEJIVV7PEFQQ5UpQw1FhddKq/3TpUsXMjMzqzLUYpo2bcqJEyeUr4ODg0vUPyqvn376iXfeeYe8vLxS9y9duhQ/P78K9V3Vqu2BstIeVxg3bhweHh6MHTu21PmtZT3iEB4eXuE4srKySJfXwzQqnNCw68ikRY+K186vjaZEk/1X9mOUYlTh/l9VWVlZlfq9va7EuGtG//796d+/P8ePH+fOnTuMGTMGKP//uzt27MDGxqbEugQ5OTncuHGDWrVqldqusuOuX78+y5cvR6FQIJPJiImJITExsUJ97t27F11dXXx9fUstCpecnIyenl6V/DtVdtwqSwQKhYKkpH9m4SQkJCCXy4GiR6hv3rxJ27Zt0dHRwcXFhYsXLz63zbMq88BIeHg4Bg4OyEPPoaVrgnWjusp97R60IzQ99I18IEU8aKNeXpVxh4eHk5mZib29Pfn5+cybN4/Y2Fjy8vKYOnUqHTp0KFG62d3dnePHj3Pp0iVatmypLFIHKJ/S1dPTU25LT09nzpw5pKWlkZ6ejre3Nw4ODnz99deEhoaSn5/P8OHDGTRoUKnbnmVubs5bb71FREQEQ4YMIS0tjfj4eOzt7Tl06BA///wzMpkMBwcHvvjiC65fv86iRYvQ0tJCS0uLlStXYmhoyI0bN9DU1OTjjz8mJCSEESNGAODv78/69esxMzNDR0eH+vXrY2FhUWo5bjc3N4YOHcrhw4dp2LAhDg4Oyp+//fbbEr/n8jxQVhaVJQInJydWr16Nu7s7YWFhKBQK9PX1gaKiS3PmzOHAgQPo6elx7do1+vXrh4mJSZltqpKJQ9Hj3jEXw4olAldzV3yCfbidehsrI6sqP64g1ISU/ftJ3Vu1lyCMBg+i9v+KqJXXwYMHkcvl+Pj48PDhQzw9PTl48GCJ0s0NGzakY8eO9OjRo1gSKMvmzZtxdHRk3LhxHDx4kMWLF7NmzRpOnTqlrGG0b98+UlJSSmwrzfjx4xk5ciR9+vRRbsvMzGTlypXs378fPT09JkyYwLlz5zh27BjDhw9nwIABBAUFkZiYiKGhIQEBAfTu3Zvu3buzYsUKsrOzlYli7969GBoaKpNQYmJiqeW4CwoKaNasGWPHjqVTp050796dPXv20KlTJ9LS0jA0NHyp3//zqCwRtGrVCgcHB9zd3ZFIJCxYsAA/Pz8MDAzo1q0bkydPxsPDAw0NDZo2bUrXrl2RSCQl2qhCw9bNiQNSrkfA4K7K7S7mLvgE+3D67mmRCAShil26dImQkBAuXrwIQHZ2Njk5OcrSzf369aNPnz7o6Oi8VL+hoaFMnDgRQFkOunbt2lhZWTFx4kR69uzJgAED0NLSKrGtNEZGRvTv358tW7Yoy1nfvn2bhg0bKs9E2rVrR3h4OF27dmXhwoXcvn2b3r17Y21tTWFhIYGBgWzatInatWvTsmVLTp8+TZs2bdDT06NOnToAystFpqam/PDDD2zYsIGcnBx0dXWVsbRo0QKJREKdOnVo1qwZUHTvND09/fVIBAAzZ84s9vppUSUoWp+ztPVG/91GFQwbNSRaQ4ucyKhi2xvoN8Cmtg1n7p7B08FT5XEIQnWoPWDAS397VwVNTU0mTJhQ7Js2UGrp5pchkUiK3U98Ws55/fr1hIWFERAQgL+/Pxs3bix1W2lGjRrFkCFDsLKyKvUYubm5aGtr06FDB/bs2cPJkyeZM2cOs2bNQlNTk+TkZGURu/T0dAIDA2nTpk2xstNP+3teOe5nS09XtiT386jfk8WARCrlkWl9dP41cwiKzgpC4kNIz6n8Yg+CIPzD0dGR48ePA0U3SlesWEFBQUGppZslEgn5+fnl6vett95SloS+ceMGtra23L17ly1btuDg4MDs2bNJSUkpdVtZtLW1GTNmDD/++CMAVlZWxMTEKKeznz9/nubNm7Nt2zZSUlLo168fnp6ehIeHExAQwMyZM5VlpwMCArhw4QJaWlqkp6eTlpZGbm6u8sxIVeW4X4ZalaF+Vo65FaahF8kvKFTOHIKi+wQbQzdyNu4sPax61GCEgvBm6dWrF+fOncPd3Z38/Hy8vLyU6wo/W7rZ3t6eNm3a8PXXX6Onp0eHDh2K9TN27Fjlt+M+ffrg4eHB3Llz8fDwICMjg6VLl6JQKLh06RKHDh1CU1OTwYMHl7rteQYMGMCmTZuAolXIZs2axccff4xUKqV169a0adOGx48f88knn2BgYICWlhaLFy9m8ODBxUpa6+rq0qlTJ06cOIGXlxcjR46kQYMG2NraAlVTjruy1KoMNfxzd/3kwm+pu3M92r+epPEzN4zzCvJw9XWlk0UnvJ2r/snmmvKqzCKpbmLc6kWMu2yiDHUp6jQvul8Rc/Fase0aUg2cGzjzx70/yC8o36mpIAjC60xtE4Hl2w4ApITdKLHP1dyVh1kPCU0ue+k6QRCEN4XaJgKjRpZka2iRGxVZYp9TAydkEhmnY0/XQGSCIAjVS20TgUQq5aFpA7Tv3Smxz0jbCEe5I2funamByARBEKqX2iYCgBzzhpgm3SO/oOT9clcLVyIeRvAg80ENRCYIglB91DoRaNnYYJKVxp3bJT/sXc1dAcQaBYIgvPHUOhH8M3Oo5E3hxkaNaaDfgKO3j1Z3WIIgCNVKrRNBw9ZvAfDoekSJfRKJhGFNhxH8IJjLCZerOzRBEIRqo9aJwMjKgiwNbXL/VXPoqWFNh2Gsbcx/r/y3miMTBEGoPmqdCCQSCQ/lDdC5F1Pqfl1NXcY0H8PZuLPirEAQhDeWWicC+F/NoTJmDkHRWYGJjgk/XP6hmiMTBEGoHuqTCDKT4Ft7tFOKXwbStrHBODud2NtxpTbT1dRljMMYgu4HcSnhUnVEKgiCUK3UJxEU5EF6HLr/WpxeuVpZSFiZTYc2HSrOCgRBeGOpTyLQNwNtQ7TSi98PaNi6OQCJV6+X2VRXU5cPm3/IufvnuBh/UaVhCoIgVDeVJgIfHx+GDRuGu7s7V69eLbbv3LlzDB06FHd3dz7//HMKCgoIDg7mnXfeYdSoUYwaNYqvvvqq6oKRSMC0CdppxRNBbSsLMnUNSL3wF1m5ZVcbVZ4VXBFnBYIgvFlUlgjOnz9PTEwMvr6+eHt74+1dvLb//Pnz+e6779i5cyeZmZmcOVNU16ddu3Zs3bqVrVu3Mm/evKoNyrQJWmm3i22SSCRovuvMW/euszv4dqnNAGpp1OLD5h8SfD+YkPiQMt8nCILwulFZIggKCsLNzQ0Aa2trUlNTlcu8Afj5+VG3btGCMCYmJjx69EhVofzD1BbNrCTISi222apvDwxyn3BizzFy8wvKbD606VDq6NQRzxUIgvBGUdlSlUlJSTg4OChfm5iYkJiYiL6+PoDyvwkJCfz555988skn/P3330RGRjJhwgRSU1Px8vLCycmpRN/h4eEVikk/Ww8LIPqv38iq809syOUUyGRYR17ix1//ws3aoMw+3lO8x5Y7W/A774e9weuzElJWVlaFf2+vMzFu9SLGXTHVtmZxaStiJicnM2HCBBYsWICxsTFWVlZ4eXnRq1cvYmNj8fDw4OjRo2hpaRVrV+Gl6OQa8Ac00s+Bf/UR0749HcMi+PLGYyb3bov0mXWMn2Vla+AhEL4AACAASURBVEXg3kAOPjzIwLYDkUhKf9+rRizhp17EuNVLeZeqLIvKLg0pFAqSkpKUrxMSEpDL5crXGRkZjB07lmnTpuHs7AyAmZkZvXv3RiKRYGlpiampKfHx8VUXlLEVhVINSPq7xC6Dzp1RpMaTFX2bo9fLLj1dS6MWk1pO4q/4v9gevr3qYhMEQaghKksETk5OHDlyBICwsDAUCoXychDAkiVL8PT0xMXFRbntwIEDbNiwAYDExESSk5MxMzOruqBkmuTom0NiyUSg37kzAD3TIvn+ZFSpZzBPvd/kfTqZd2JFyArCk9XvNFQQhDeLyi4NtWrVCgcHB9zd3ZFIJCxYsAA/Pz8MDAxwdnZm//79xMTEsGfPHgD69OnDe++9x8yZMzl+/Di5ubksXLiwxGWhyso2aIh2KWcEWuYN0La1pWf6TTbc68CZm0m4NJGX0kPRTKMvnb5kyIEhfPb7Z+zqswtdTd0qjVMQBKG6qPQewcyZM4u9trOzU/4cGlr6wvA//vijKkMix9AK7v8J+bkg0yy2T79TJ7I3bqRxywK+PxlZZiIAMNYxZonLEj468hHewd54O3uX+V5BEIRXmfo8Wfw/2YZWReUmHt4qsU+/c2fIz+cTg2SCox/y1+2Hz+2rbd22jHccz4GoAwTcClBRxIIgCKqldokgx9Cq6IdSLg/VcmyBzNiYlnevYaKnxQ+nSl+n4FnjW4ynlaIVXwV9xZ20O1UcrSAIguqpXyIwsCz6IfFGiX0SmQx9V1ey/vyTj96x4EREAmFxqSXe9ywNqQZLXZaiIdVg1u+zyM3PVUXYgiAIKqN2iaBAUw8MG0DSzVL363fqREFqKu/XeoS+tgYzdl0hJOb5l4jq6tXlS6cvCUsOY9XFVaoIWxAEQWXULhEAYGpb6qUhAD1nJ9DUpPDsH6wY6sijxzkM/m8QE7aGEJ2UWWaXXS27MqzpMDZf38yOiB2qilwQBKHKqWkiaFJ0RlDKswIyfX302rYl4+RJujvU5eTMTszo1oQzNxPptuI0C/xDSc7ILrXbWW1n0dmiMz7BPmwO26zqUQiCIFQJ9U0EOemQfr/U3fqdO5MTHU3O7dvoamkwpastpz7rzLC2FmwLvoPrN6f4KuA6W4NucyTsAZdjU7if+gQJGnzb6Vu6N+zO8r+Ws/bq2uodlyAIQgVUW62hV4ppk6L/Jt4Aw/oldut37kS8tzfpp05RZ/RoAOQG2ngPfIsxTo1YejiCn8/eLrHOsUQC9Qx16NhkJG3qFLD60mqy87Pxaun12tQkEgRB/ahnIpAXLU9J0k2w7lxit5a5Odq2NmSc/CcRPGWj0GedRxvyCwpJzswmIS2b+LQs4v/335sJ6QReTSAjuwt6DdJYe3UtoXEPWew6BxN97WoYnCAIwstRz0Twv2Ury7phDKDfqTPJmzaRn5aGzNCwxH6ZVILCQAeFgQ7NGxgV25edl8+5Ww85GmbJobj/cpY9vLs+DjfFeMa5NKaFee0qH5IgCEJFqec9gv8tW0lSyWcJntLv3Bny8sj844+X7l5bQ4ZrEzneAx35a8IP9LIYhqbxWU6nfEO/H44z7Kcgjl2Pp6Cg7MJ2giAI1UU9EwH8M3OoDE+fMk4/drxSh5HJpCzt/H/MajsLmX4E5g4/EZMeycdb/sJt5Wl2nL/z3LWSBUEQVE2NE4Ft0ayhrLRSd0tkMgx79SL92DHyKrmMpkQiYVSzUWzquRFtzXzy6q7Cs3sCuloyPve7RqdvTrH1XAzZeSIhCIJQ/dQ3ETx7w7gMxsPdKczJIdXPr0oO+bbibXb13UVLeUv8YlfQpvVxfv7wbcyNazFvfyhdlp9m5/k7z103WRAEoaqVKxGEh4fzx/+ulX///fdMmjTpucuevRaeTiF9zg1jbVtbdNu04dGOnRQWVM2Hc51adfip20+MfWsse2/u5Ycb01j+QX22fNgOUwNt5vhdo+u3p9kbcpc8kRAEQagG5UoEixYtwsrKij///JOIiAgWLFjA6tWrVR2bahlbgVTzuTeMAYw/GE7u3bsVumlcFplUxtRWU1ndZTV3M+4y+MBgrj3exc5xrdjg2QYDHQ1m7L5C1xWn2Rp0myc54pKRIAiqU65EoKWlhbm5Ob/99hvDhw/HzMyMgnJ8Q/bx8WHYsGG4u7tz9erVYvvOnTvH0KFDcXd35/PPP1f297w2VUqmCSaNn3tpCMDAzQ2ZqSmPfqn6+kGdLDrh398ft4Zu/HjlRwYeGIhMP5yAKc78NKo1xrpazPMP490lx1lx9AaJ6aWXthAEQaiMciUCTU1NvvjiC/766y/at2/P77//Tl5e3nPbnD9/npiYGHx9ffH29sbbu/gKXvPnz+e7775j586dZGZmcubMmRe2qXLPKT73lERLi9rvDyHj9Gly7t6r8hDkunKWuixlY4+N6Mh0mHJiClNOTKGZZS77Jr3LngkdaGtlwuqTkTgtPcGcvVeJTMio8jgEQVBf5UoEq1atwtXVlU2bNiGTydDU1OSbb755bpugoCDc3NwAsLa2JjU1lYyMfz7A/Pz8qFu3LgAmJiY8evTohW2qnLxp0UplL1hDwHjoUJBISPH1VVkobeu2ZXe/3cxsM5MLDy4wYP8Avr/8PfYNtFnr0Ybjn7ryfmtz9l26R7eVp5m0PYTrcaXPeBIEQXgZ5UoEsbGx1KpVC7lczvfff8/WrVt58ODBc9skJSVhbGysfG1iYkJiYqLytb6+PgAJCQn8+eefuLq6vrBNlTNt8r9lK6Of+zbNevXQ79KZlD17KMjJUVk4mlJNPB08OTDgAF0tu/LT1Z/o7debnRE7sahTVOvo7JwuTO5kw5m/k+j93Rk+3vwXV2JTVBaTIAhvvnKVmFi0aBHLly8vdrN49uzZ/Pzzz+U+UGEpJZ+Tk5OZMGECCxYsKJYAntcGimYxVVRWVpayvU6GFo2Au5ePk27+ghuyTk5w7Dg3Nm4EV9cKH7+8xijG4KzrzLbYbXgHe7Px8kaGWwynvXF73rOU4Fq3Af7haey/nsix8Hha16/FcEdjHBQ6pfb37LjViRi3ehHjrphyJYKnN4vXr19f7pvFCoWCpKQk5euEhATkcrnydUZGBmPHjmXatGk4OzuXq81T9vb25Qm7VOHh4f+0zzaH38Bc5zG8oM/Cpk259fNmZL+fwWrChAof/2XYY0/fNn35/e7v/Ofif1gRuYIW8hZ82vpT2pq1pq0jzM7KZeu5GNafiWbmr3F0tDVlmlsTWjcsnliLjVuNiHGrFzHusj1vyv9L3Sy+cOFCuW8WOzk5ceTIEQDCwsJQKBTKy0EAS5YswdPTExcXl3K3qXLaBs9dtvJZEqmU2sPdeXLxIlkREaqL6d/HlUhwtXBlT989fPnulzzIeMDow6OZcnwKUSlRGOhoMqmTDX/M7szc3nZcj0tj8H/P4rnxPJfFJSNBEMqhXGcEq1atIigoiGnTppX7ZnGrVq1wcHDA3d0diUTCggUL8PPzw8DAAGdnZ/bv309MTAx79uwBoE+fPgwbNqxEG5UztS11IfvS1B44kMT/rOLRjp3UW7RQtXH9i0wqY6DtQHo26sn28O1suLaBQQcGMdBmIJNaTkKhq2CcizUj32nIlqAYfjodxYDv/6SLnYLpbk3UtMysIAjlUa7Ph4KCAiIiIti3bx9SqZTmzZvTokWLF7abOXNmsdd2dnbKn0NDQ8vVRuVMm8DlHUXLVr5g8RiZkRGG7/Um9eBBFDNnIDMwqKYg/1FLoxYfv/Uxg20Hs/bqWnbe2EngrUBGNRvFh80/RF9LnwmuRQlh89nbrP39Fn3X/IGTpS6L6lhgo1DhGZYgCK+lcl0amj17Nvr6+kyePJmPP/4YqVTK559/rurYqscLlq38N+PhH1D4+DGp/gdUHNgL4tAxZna72RwYcIDOFp1Zd20dvfx68eOVH0nNTkVfW4PJnW04M7sz09xsuRj3hO4rTzN7z1XiUp7UaOyCILxaypUIMjMzGTNmDA4ODrRs2ZJx48aRlvaGzGEvR82hZ9Vq7oBOixY82r6dwvyaL/1gYWDBMtdl7HxvJ2+ZvsX3l7+n255uLD2/lPsZ9zHU0WSaWxM2DrZk9LuN2HfpHp2Wn8I78DqPMlU3FVYQhNdHuRJBQUEB165dU76+cuVKuUpMvBaeViFNLF8iAKjz4YfkREfX+FnBsxxMHfjB7Qf29tuLm6UbOyN20suvF5+f+Zy/H/1NbR0Z8/s24/gMV/q2qM/6P6JxWXaSb4/eID4tq6bDFwShBpUrEcyfP5/ly5fj7OyMs7Mzq1atYurUqaqOrXrom4GeAmKDy93EoEd3dFq0IPG77yjIerU+RJsYN8Gnow+HBh1iuN1wjt85zuADg1n29zLCk8OxMNHl26GOHP7EhQ7WdVhzMhKnJSeYuuMSl+5Ubt0FQRBeT+W6WdykSRM2b95cbJuHhwdbtmxRSVDVSiIB6y5w8ygU5INUVo4mEhQzZnDH05NH23+hzkcfVkOgL6eefj1mt5vNBMcJ/BLxC5uvbWZowFA6W3RmouNE7Ovas9ajDbeTMtkSFMOuv2I5cCWOlha1GeNkRe+36qEpU9/lKgRBnVT4//Synvp9Ldl0hScP4f7lcjfRa98OPVcXktauJT81VYXBVY6RthETHSeyxnENk1pO4q8HfzE0YChTT0wlPDkcK1M95vdtxrm5XVnYtxmpT3L5ZOdl3l1ygm+ORBD78HFND0EQBBWrcCKQvGCq5WvFugsggcgTL9VM8emnFKSlkbxunWriqkJ6GnpMdJzI4SGHS00I+toajHZqxPFPXdk0ui0tGhjx31NRuHxzEs+N5zkc+kCsnCYIb6jnXhoaPHhwqR/4hYWF3L59W1UxVT89U6jfEiKPgetn5W6m07QpRv3783DLVoxHjECzXj0VBlk1DLUMmeg4kRH2I9h+fTtbr29laOxQulh0YWLLidiZ2NHZTkFnOwVxKU/wvRCL74VYJmwLQWGgzeDW5vRwqEuLBkZIpW/QlwFBUGPPTQTfffdddcVR86y7wh8r4UkK1Kpd7mbyqVNIO3SIxNVrqO+j4vUTqpChliETW05kRLN/EsKJgyfoatmViY4TaWrSlPq1azG9WxOmdLHh5I1Edpy/w0+no/jvqShM9bXpYienq70Zzjam6GmLZ5cF4XX13P97GzRoUF1x1DwbNzizHKJPQ7P+5W6mWb8+xiNG8HDzZkxGe6LTpIkKg6x6zyaEbde3sfX6Vo7fOU5ni854OnjSStEKDZmUbs3M6NbMjEeZOZz6O4Hj4Qn8GvqAXX/dRUtDSofGdejZvC7dm5lRR1+7poclCMJLENNCnjJvC9pGRZeHXpLp+HFI9fRIXLFSBYFVD0MtQya1nMThwYeZ6DiRiwkXGX14NMMDh3Po1iFyC4oW7zHW02Lg2+as+aAVF+d145ex7Rn1TkOikzL53O8abb2PMXztObYE3RbPJwjCa0IkgqdkGtDYBSKPF9UdepmmtWtTZ9xYMk6d4vGFCyoKsHoYaRsxqeUkfhvyG/PemUdmbiazz8ym195ebAzdSGr2PzOkNGVS3rU2ZV6fZpz+rBOBU52Z3NmGxIxs5vuH8c7i4wz571nWn7klZh8JwitMJIJn2bhB2r1yVyN9lsmoUWiYmZGw/Ns3YmptLY1aDG06FP8B/nzf9XusDK1YGbISt91ufH7mc4LvB1NQ+M8sIolEgkN9I2Z0b8qxT135bboL092akJmTz9eB4XRcdpI+q8+w5sRNIhPSa3BkgiD8m7jD9yzrrkX/jTwGCrvnv/dfpDo6yKdO5f7//R8pO3diPHy4CgKsflKJFBdzF1zMXYh4GIHvDV8ORx8m4FYA9fXq08+mH/2s+2FhYFGsna2ZAbZmBkztaktMciZHwh5wOPQBy4/+zfKjf2Mt12NYWws8Oliho/nih/gEQVAdcUbwrNoWYNq0QvcJAIwGDUSvY0filywlOyqqioOreXYmdizosICTQ0+ytONSGho25KcrResqjzk8Br+bfqTllCxG2LCOHuNcrPGb5MS5z7vyZX8H6uhr43Mogk7fnGLH+TvkiWcUBKHGiETwbzZuEHMWcl7+mrZEIqG+jzdSXV3uzfxMpQvd1yQdDR16N+7N2u5rOTrkKFPenkLik0QWnF1AZ9/OfHrqU47fOU5Ofsnx1zXSwaODFbvGd8B33Ds0MK7F537X6L7ydwKv3qeg4PW/rCYIrxuRCP7NpgvkZ0PMnxVqriGXU8/7a7LDw0lctaqKg3v11NWry7gW4zg44CC/9P6F95u+T0h8CNNOTqPzrs4sClrEhQcXyC8oWbK7feM67JnQgfUebdCUSZn8y0X6ff8Hf9xMKuVIgiCoikrvEfj4+HDlyhUkEglz584ttqpZdnY28+fP5+bNm/j5+QEQHBzMJ598gq2tLVBU7G7evHmqDLGkhk6goVN0eci2W4W6MOjShdrDhvFw4yb0O3ZE7513qjjIV49EIuEt+Vu8JX+LmW1mcu7+OQJvBRJ4K5A9f+9BXktOd6vu9LTqSQt5C6QSqbKdWzMzOtsp8L98jxW//c3IDcF0b2bGvD7NsDDRreGRCcKbT2WJ4Pz588TExODr60tUVBRz587F19dXuX/ZsmXY29tz82bxhePbtWtXs080a9YCK+eiaaSVYDZ7Fo+Dg4mbPYfG/vuR1S7/08qvOw2pBs4NnHFu4Mzj3Mf8fu93DkcfZveN3WwP305dvbr0aNiDXo170cykGRKJBJlUwqBW5rzXoh4b/ohm9fFI3FacZoKrNRM7WYsbyoKgQiq7NBQUFISbmxsA1tbWpKamkpGRodw/ffp05f5XjnVXSL4Jj25XuAupri71ly8nLzmZ+wsXvRFTSitCV1OXnlY9+U/n/3B62Gl8nH1oYtyE7RHbcQ9wp+/+vvxw+QeiU6MB0NaQMamTDSdmutLdoS6rjt/EbcVpDoc+UNvfoSComsrOCJKSknBwcFC+NjExITExEX39osXT9fX1SUlJKdEuMjKSCRMmkJqaipeXF05OTiXeEx4eXuG4srKyXtheS9oIa+D+H9tJsRlU4WMhk4K7O+nbthFhawtdOle8r0oqz7irgw02eNX3YrRiNMEPg/kj+Q9+vPIj/73yXxrpNsK5jjPv1nmXOlp1mNRSB+e69fhvcBITtoXQun4tPnWWY1Kr/H+2r8q4q5sYt3qp7Lir7TmC8nybs7KywsvLi169ehEbG4uHhwdHjx5FS0ur2Pvs7e0rHEd4ePiL2xfawVkL6mVep579/1X4WACFn8/hTkQEWRs2YNW7F9qNGlWqv4oq17irWVva4oUXCY8TOBx9mF+jf2Vr7Fa2xW6jXd129LHuw3tObgx2acnWczEsPRzBjMMJ/DSqNY4W5bvU9iqOuzqIcauX8ow7JCSkzH0quzSkUChISvpn9kdCQgJyufy5bczMzOjduzcSiQRLS0tMTU2Jj49XVYhlk0iKFqu5dRryKjcFVCKTUX/ZUiTa2sSOHUdeYmIVBfnmUOgq8HDwYEefHQQMDGCi40TiMuOY9+c8Ou3qxNw/5mDdMJZd49ujIZPw/k9B7Am5W9NhC8IbQ2WJwMnJiSNHjgAQFhaGQqFQXhYqy4EDB9iwYQMAiYmJJCcnY2ZmpqoQn8/GDXLS4e75SnelWb8+Fj/9SN7Dh9wZO478dFFioSwNDRsyseVEAgcGsq33NgbYDCDofhCTj09myp8D6eJ0lmZWj5i5+zILD4SJxXIEoQqo7NJQq1atcHBwwN3dHYlEwoIFC/Dz88PAwIBu3boxdepUHjx4QHR0NKNGjWLo0KF06dKFmTNncvz4cXJzc1m4cGGJy0LVppELSGRFs4esnCvdXa0WLTBftYrYiRO5O9kLi3VrkWqLcs1lkUgkOModcZQ7MrvtbP6M+5ODUQcJjN5HjmYO9Rzq8svfDlyN78y64T1F6WtBqARJ4Ws2FSMkJITWrVtXuP1LXUPc3BeSb8HUS6BRNQkp9eBB4j6bhUH37jRYuQKJrHqmRb4p107Tc9I5FnOMwFuBnH9wnkIKkeY0xKNFPz5w6Es9/eKrxL0p435ZYtzqpbz3CMr67BRPFj9PhymQdheu7aqyLo369kUxZzbpR4/y4KuvxJTIl2SgZcBA24Gs77Ge34b8xgc2kygkj58jVtN9b3c+CPyATaGbuJsu7iEIQnmJRPA8tt2gbgs4swJKKZFQUXVGj6bOxx+RstOXpB9+qLJ+1Y2ZnhmfO03Er/8etO7PRZbyHo9zclkRsoJefr0YFjCMvff2EvEwQiRcQXgOUYb6eSQS6DgDdnvC9f3QfHCVdS2fMYO8pGSSVq9Bpq+PiadnlfWtbmwU+uz6qC/D18qJDe3KDyMticw8y28xv+F7zxffe74odBVF5bQbuNC+Xnt0NUXpCkF4SiSCF7HvB6ZNis4KHAYVJYcqIJFIqPfVlxRkZhK/eAm5CQkoZsxAIhUnaRVhLddnx7h3GL72HNO23eGXsUMY03wMQVeCeKD7gDP3zvBr9K/s+XsPWlIt3m3wLgOsB+Bi7oKmTLOmwxeEGiU+dV5EKgXnTyE+FP4+XKVdSzQ1afCflRh/MJyHGzYS99msN7Z0dXWwluuzc9w7aMgkDF93jogHadTWqs1A24Gs6LSCM8POsK77OoY2HUpYUhjTTk2jy+4uLDm/hPBk9XsaVRCeEomgPN4aArUt4fflL72e8YtIZDLM5s1DPuNT0gIDiR07jvy0kou7COXTWK7PznEd0JJJ+WBdMNGP/kmsmjJN3qn3DrPbzebokKP80PUH2tdrz64buxgaMJTBBwazJWwLD7Me1uAIBKH6iURQHjJNcPoE7v0F0aervHuJRILp2LHUX7aUxxcvEjNiJLkPHlT5cdRFI1M9do57By2ZlM+PxnEzvuQDfBpSDTqad2S563JODj3JF+2/QEuqxTd/fUPXXV355MQnnLxzktyC3BoYgSBUL5EIyqvlSNCvW3RWoCJG/fphufYncuPiuD3Mnawbf6vsWG86K1M9fhnbHplEwvB1wUQmZJT5XiNtI4bZDWNHnx3s67ePkc1GciXxClNPTsVttxvLLyzn5qObZbYXhNedSATlpakD73rB7TMQW/myE2XR69CBhtu3QWEht4cN49FOXzH1sYIay/VZ0qMeUMgH684RnZT5wjY2xjbMaDOD397/jdVdVvO24m22h29n0IFBDDowiHVX1xGbFqv64AWhGolE8DJaj4FaxnDmW5UeRsfODqs9u9Ft1YoHCxdyd7IXeQ/FdeuKsDDS4pex75BXUJQM7iSXby1qTakmnSw68Z/O/+H40OPMaTcHfU19vrv0Hb339cY9wJ2fQ3/mfsZ9FY9AEFRPJIKXoa0P70wqmj304JpKD6WpUGCxfh1mn88h88wZbvXvT8aZP1R6zDdVEzMDtn3Unie5+Qxfd467j8qXDJ4y0TFhhP0ItvTawtHBR5nZZiYA34Z8S/e93Rl1aBTbrm8jPrMGKuUKQhUQieBltRsLWgYqvVfwlEQqxcTTE6s9u9GoXZvYsWOJX7yYguxslR/7TdOsviHbPmpPelYuw9edIy7lSYX6qadfD08HT3b22UngwECmvD2FJ3lPWHphKW573PD81ZNfwn8h8bEoNy68PkQieFm1jKHDpKInjcP2VcshdZo2xWr3boxHjeLh5i1EDx5M5rlz1XLsN0nzBkZs/ag9KZm5jN50nsc5eZXqz9LQknEtxrGn3x78B/gzueVk0nLSWHx+MV13d2Xc0XEcvn2YnHzxbIjwahOJoCI6zgTztuA/BZKjquWQUh0d6v7fXCzW/kThkyzujB7D3SlTybkriqu9DEeL2vwwshU3EzL4Yl9old2Ib2zUmAmOE9jXfx/7++9nvON4bqfd5rPTn9F1d1eWXVhGVEr1/K0IwssSiaAiNLRgyCaQacAuD8it2GWGitB3caFxYADyaZ+Q8ccf3Or9Hgkr/0NB5otnxAhFOtrKmda1CX6X7rHjfNXPALKubc3klpP5ddCv/Oj2I23rtmVHxA4G+A9g5KGR7Lu5jyd51fc3IwgvIhJBRdW2gEHrikpP/DqrWg8t1dHBdMIErA//ikGPHiT/9BNRPXuRsn8/hflVVyX1TTaliw0uTeQsPBDGtbupKjmGTCrDqYETKzqt4NiQY8xsM5O0nDTmn51P111d8Qn2Ec8nCK8ElSYCHx8fhg0bhru7O1evXi22Lzs7m9mzZzNo0KByt3nl2HYrqk56cQtc3lHth9c0M6PBN8to+MsvaJiZcX/O59zq249Uf38K8yp3/ftNJ5VK+M+wlpjqazFxewipj1X7BHGdWnXwdPDEv78/m3psoqN5R/b8vYdBBwbh8asHB6MOkpWXpdIYBKEsKksE58+fJyYmBl9fX7y9vfH29i62f9myZSVW1HlRm1dSp7lg1RECpkP89RoJQbfV21jt8qXBim+RaGgQN3sOUb1682j3bgpFEbsymehpsWZEK+LTsvh012UKClT/4J5EIqFN3TYsdVnK8fePM6P1DB5mPWTuH3PpursrX5/7mmuJ18RDhEK1UlkiCAoKws3NDQBra2tSU1PJyPjnMf/p06cr95e3zStJpgGD14O2QdG6Bdk1E69EKsWwd28a7d+H+fdrkBkZ8WDefCJ79OThtu2QJb5tlqaVpTH/19ue4xEJ/Ph79d7MNdYxZnTz0RwYcIB13dfh3MCZ/ZH7+eDQBwzwH8DG0I0kPE6o1pgE9aSyRJCUlISxsbHytYmJCYmJ/8yt1tfXf+k2ryyDujBkAyRHwsFPqrxC6cuQSKUYdO2K1e5dWKxbi2bdusR//TV8+BH35y/gydWr4tvmv3i+a8V7Leqx/MgNgqKSq/34UomUd+q9w1KXpZwcepIFHRZgqGXIypCVdNvTjQnHJnD09lFy80UBPEE1qm1hmop8mv7UFwAAIABJREFU+JTVJjy84rXjs7KyKtW+bHLqNB+H4tqPpGQ85n6bz4uqltYkU1OY9wWER5B/5Agp+/eTsmsXWFqCmxu4uoKhQc3GqGLl/ff+sLk2V25rMmnrBdb0bYCJbs2t2eSAAw6NHIirG8fvyb9zOvE0M+7NwFDDEFdTV7rIu9CgVoPn9qG6v/NXmxh3xajsr12hUJCUlKR8nZCQgFwur5I2/7638DLCw8Mr1f657JaAaR1qn/SmNukwbGvRA2g1rVkzwpvZ08TcnLTAQ6Ts2UPWxo1Itm5Fv5MrBj16ot+pEzJ9vZqOtMq9zL/3RjNL+q/5k1UXMtg+tj2aspqdVGePPV3pSn5BPmfjzuJ3049fY3/l4IODtFK0YpDtIHpY9UBHQ6dEW5X+nb/CxLjLFhISUuY+lf2lOzk5ceTIEQDCwsJQKBSlXg6qbJtXikQCrrOKppXGBsP6bvAwuqajUpIZGGDsPoxGe3bTyH8/tYe78+TyFeJmzuTmu+8SO9mL1IMHyX/V78uoSBMzAxYPeovztx/yzZEbNR2Okkwqo6N5R1Z2Xsn/t3fe0VVVaeN+zrklvZByE1JJCJAQBAySgQTRUEeKKIigYhud0UFGdGT8MpRP1qeCbWwMMzgDDixHhlA1OiKgwk9KREFAgQSkhRBCSGjpueWc3x/n5uYmBKXd3JC7n7X22uXsfc773rLfXc7ee8P4Dfyxzx85W3eWmVtnMmTlEN7a+ZbY/E5wTbisR5CWlkZqaioTJ05EkiReeOEFVq9eTUBAAEOHDuXpp5/m1KlTHD16lAcffJB7772X0aNHX1TmhqTnvRAYDTkPwMLBcN8yiE13t1RN8O7Wjcjp04nIzqZ21y4qPl9H5fr1VH35JZLBgG9Gf/xvHYj/rQMwxse7W9xW466bo9lReJZ/fH2EtLgO/LpHpLtFakKYTxiP9niUR1IfYUfpDv5T8B8W71vM4n2LyYrN4oGUB7gl4hZ3iym4wZDUG2zmcOfOnfTp0+eqy7dq17H8EHx4D1SchLHvQerdrfPcFrgcvVVFoXb3HirXfU7lpk1YCo8DYIiLw3/AAPwG3opfejqyr29riHxduJrvu95q494FeRwpq+aTPwygU1jbHjIrqSoh50AOq35axfn68yQFJ5EVnMXjGY/ja7hxvqvrgRgaujQ/V3cKQ+Bqqs/Asvu0oaI+j8KgWeAX2nrPt3M1epsLC6nasoXqrzdT/e23qLW1YDDg06MHvrf0wfeWW/BJS0MX0HYnnK/2+z5xroaR724hKtiHNZMz8DboXCDd9aXOWsfao2tZWrCUgrMF+Bv8GZM0hgndJpAQlOBu8VoFYQguzc/VnbrZs2fPdoFcLqOkpISoqKirLl9eXv6Lk9bXFaMv3DQeLDWw433Y+S8w+EHHXiC33mTk1eitCw7Gp2dPgkaPIuTRR/BL74s+NATLyZNUrF9PRW4uZxYtovLLL6k/dBhbRQWytzdyYCCSJLlIkyvjar/vQB8DyZEBLNxylNKKOoaltq0hopbQy3pSQlMY33U8Hc0dkX1lcg/n8mH+h+w6vQs/gx9xAXHIUvvdWabV/99thMvR++fqTve9I+dJGLzh13Mh7WFtX6K1f9IMwh2vQsJAd0t3WcheXvhlZOCXkQGAUltL7Z4fqNmxg5odOzi/YgXnPvgAAF1QEN49e+Jz001497wJ75Tu6E3hbcY4XC5ZySb+MCiJeV8d4pZOHZjQN87dIl0WkiSRHJDM3el3U15bzqqDq1hxcAXPbHyGSL9IxnUZx9guYzH5mtwtqqCNIAxBa2JKhoc+hoJPYd10WDIauo+BrBkQ3s3d0l0Rso8Pfv1+hV+/XwGgWq3UHzpE7Z4fqP3xB+p++JHyrVtBUQDQdeiAV3I3vLsla35yMsbERGSj0Z1q/CLPDOnKruPnmfXxPlKjgugRHeRuka6IMJ8wnuj1BI/d9Bibijax/MBy5u+ez4I9C8iKzWJ8t/H069ivXfcSBL+MMAStjSRBymhIGgLb5sHmN2H/x9r5BjdPgtSx4B3obimvGEmvxzs5Ge/kZDpMuBcApbqauv37qcsvoO5AAfUFBzi3dGnj/keyjCEmBq+EBIx255WYgLFTJ3RhYW2iB6GTJd6Z2JtR87bw+w938umUWwnydfNCwatAL+sZEj+EIfFDOF5xnJUHV7Lm0Bq+OP4FsQGxjOsyjlGJo4jwi3C3qAI3ICaL3U3VadizDHZ/CGUFoPfRegk3PwDxA67bPEJb0Vu1WjEfO0ZdwQHMRw5Tf+Qo5qNHMR87hup0BKfs54cxPh5jp3gM8fFaOC4eQ8dI9OHhSPrLa8NcL713HT/Hve/lMSApjEUP90WW3W+kfo7L0dtsM7OhcAMrDq5gZ+lOJCTSO6YzOnE0Q+KH4Gdo229LtURb+Z23Ntc6WSx6BO7G3wSZT0PGH6D4e9j9b/hxFfywDPzCIa4fxGVofmRPbZO7GxhJr8crKQmvpKQm6aqiYDlZohmFo0cxFxZiLiyk9se9VHy+zjHEBIAsow8PRx8ZgSGyI4bICPQRkZqRiIjU4uHhSIbr13K/Oa4D/zuqO7M+3sdfNx7i6cFdrtu93YVRZ2Rk4khGJo6ksKKQT498yqeHP2Xm1pm89M1LZMVlMTpxNP2j+qOXb+zf3TWhqqBYwWYGm0VzisUpbm4Wbp7H2kIes1Mei3Z/xeqUZi/TEG7I128ydB123VX04G+3jSFJENNHc8PnQMF/4dAXULgN8j/R8hj8ILavNowUmgQhiZrzDdXK38BIsowxJhpjTDTcOqDJNdVsxnyiGEvRcSynSrGcKsF6qhRr6SnqDx6k6uuvtVdbm9xQQh8WBgEBHI+NQR8ahj6swYWiCw1DHxqCLiQEXVAQku6XXw+d1C+e74+f560vDtIrNpjburaft1PiA+N5qvdTTO41md1lu/n08Kd8fuxz1h5di8nHxJikMdyddDexgbHX76Gq2lgxNlR0zpWktd6eVm8Pmxv9JhVrYwUcVnoSTgReXFk7l3O+70WVs7VpmmKvpF2GpO1JJhtA1msNPdmgpTWkO4cV12w8KIaGbgQqTsLxPDj+DRTmwel9oDq1kL2CICRBc8Hx2ulpwfEQHAdBsWD0vTH1vkxUVUWprMRy6hTWU6fsfimWU6e4cOwY3nV1WMvLsZ49Cy0d2CPL6IKDNcPQIQRdcHCj69BB84OC0AX4Y/byZfJHBzheL/GfZwYRGxYIiq3lFp+1Hqx1jc5i9xUrSPLFTtbZ//ReoPdqDOvslYSsA0nnFJYBVestqTbtN6HYQLVx6KcDJHWKb9oqVZxanoqi+aqtaWvUSQ+ztZavK4+yuvIgW2tLUFDpawzlbq9ohhpC8bbZmlXSdWB1qqSbt6KbV/guqGBVJCSdEXTGxgrUEbZ/lnove5rd6Y32CtfYLH8LFbGueT6nsHNendFesTfP2+wZ8vVZnyIWlF0h7aJCtNbD+eNw9sjF7sIJ7U/mjG8odYZgvENitOEmv3DwC9N8n2Aw+oHR38n319Y/GHzbRk/DagZzlbYWQ7FqlZ3iVIE1qcTqm7QmT544TlSkCVQbqtWKraoa2/lKrOcrsJ2rxHqhEltFNdaKGmyVtVir6rFVm7HVWLHVWOEX/h2STkU2KOiMCjqDimxU0BkUZKOKzqDYr6vIOhVZr+WV7WlSQ5rdSTq1TXzcLXFKbyA3IIA1/j6c0OsIUFTuMKvcadHTU/JC0nmB3tvJgBmdKk/jJSpIY2MLuEk5p7INlbbey24gjU6+8aL75R84SEr37u7+uFodMUfgiei9IKyL5pqjKFBVqhmK88fhguZbTh3B21ILxTuhuhzMlZf3LINfo1Ew+oHBp7Fl1bzlJemcWrg0hpWG1qpTxa3YnMZGG8ZB7WFrvVbx11dpfnPDdgU4L5+R0H7wesDL8VnKYPKGKHtFpjPaKzR/VJ0Xis2AzaLHVi+jWHXYrDKKGcrPWzhYXEm4l5EYXy9stVaUWgu2WjPmmnqUs/UodWZU65WdIS0Z9E2dXods0CHZnRbW2325WX4Dst2vqK0hKDQMSW/Q5koMRnvYiOTlhWT0QjIakLy87WEvJINWwUp6L9AbtMpWb0Ty8iXMaOQxg4Hf6HV8f2YPq498RO7xr1huq6NTYARjksYwKnEUkX6tu/BOVVX7GL6CalPAbNY2TbRaUW027fO3WVEb5pga2r1Ovqoo9mEqG6qigqpoZ38rCqrFqpW32VAtVlSbFRQVRwvBqR2tKop2D5vN7iv2svb7o8nqkFkFFLuMdl+1WcGmoKqK9hxF0eRRtDJBo0fh7QJDJwxBe0OWIbCj5uJ+5Ug+0bzFYKnVDEJ9BZirnSpee9hcrbXAHX4NWKo1v6Gyrq9s2hJ3/MAV+9CVPSzJjcMZsl5zks7eGnQaE23oouu9tF6Jl39jD8XLXzNGDQZH1jW9p6Prb2xsNeqMHDp6jKQu3ZyGVezGqqHC/5nJdwnQ2V1zgoFVnxfw902HeXtCb+66ueXzAVSLBaW2VnPVNSi1Nag1NVq8pgalxu7X1qDW1qKazShmM6rZjGq2aH59PYq5HrWuHqWuDqW2HvVcHUp9vT1fo3OumFx1Jl0A8LAk8bAkoaKCeghV/Qtn+AtnJUCSkHR6JL1em3vR6TRflu1DUkqTCvyiuHNl2UBDV6nBdyrbnIMu0rtVkWWQZe0VaqewMS5WGALBdcTgo80ltHMsZRaX6TltWDd2HDvLrI/20jchhOhgn4vySAYDOoMBXaDr14aoqqq1hM1mDuzfT5fExMYWsdWqtTgtFs01GA+L2R632Fue9pZoQ9imaGUbylktjrAzFeZKfjp3kAPnDlJVX4EXEokBsSQFJGDyCtUqbJsCOhlJkrUKXZaRZAloqOwkreKTtIqPhmEy1aFgo+/IL2v3lLXeZ1l5OaaoKO31Yr3ObpAaeqoNxsTuNcRlnXYvWW4a1tnLGxoMmv1eDeP6ze8jSVoZvV4r32AIZVnLLNnzOrkmhtLJYLb2GhphCASCq0QnS7x5b29+/fbXPLd8N0sf7+fW9QWSJIHBPhTk748+tPU2NzQBScBwVeH70u/JPZzL64XrqbYcIdo/mlGJo7iz853EBbp2m46y/HxCb/Q5QDcg1pULBNdAbIgvL9yZyjdHzrJwyxF3i+N2ZEnmlshb+L/M/2PjvRuZe+tc4gLi+McP/2DkmpFM+mwSOQU5nK87725RBU4IQyAQXCPj+8QwPDWCN9YdJL+kwt3itBl89D6MShzFP4b9g/X3rOfZPs9Sbanmpe0vkbUii6lfTeWLwi8wX8PLAILrgzAEAsE1IkkSc+6+iUAfA8/m7KbOcmVvCnkCkX6R/KbHb1h952pWjF7B/cn380P5Dzy76VmylmfxYt6L7D69mxvsbfZ2g0vnCObMmcOePXuQJInp06fTs2dPx7Vt27bx5ptvotPpGDhwIE899RTbt29n6tSpdOmivRbZtWtXZs2a5UoRBYLrQqi/F6/f05NHF3/HX9YfYMZIz3uX/XKQJInkkGSSQ5J5ts+zbC/ZTu7hXHIP57L84HLiAuIY3Xk0oxJHERMQ425xPQaXGYJvv/2WwsJCcnJyOHz4MNOnTycnJ8dx/aWXXmLRokVEREQwadIkhg8fDkB6ejrvvvuuq8QSCFxGVrKJB34Vx8ItR8lKNpHROczdIrVp9LKezOhMMqMzqbZUs6FwA58c/oT5u+czf/d8+kT0YVyXcQzrNAwvndcv31Bw1bhsaCgvL48hQ4YA0LlzZy5cuEBVVRUARUVFBAUF0bFjR2RZ5rbbbiMvL89VoggErcaMkSl0CvVj2vI9XKh1zb4w7RE/gx93Jd3FouGLWDduHU/f/DTlteVM3zKdISuG8ObONymqLHK3mO0Wl/UIysvLSU1NdcRDQkIoKyvD39+fsrIyQkJCmlwrKiqia9euHDp0iCeffJILFy4wZcoUMjMzL7p3fn7+VctVV1d3TeVvVITercfU9CD+uPYkf1iyleyBJrecq3Cjf98DDAPI6JbB3oq9rD+9niV7l7B472J6BfVimGkYacFpLR6mc6PrfbVcq96tto7gciaBOnXqxJQpU7jjjjsoKirioYceYv369RibnWJ1LXsFtYu9hq4CoXfrkZICxbYAXl93gKG9/XiwX3yrPh/az/edSioTmEBpdSmrflrFyoMree2n1zD5mBjdeTR3Jd1Fp6BOjvztRe8r5XL3GroULhsaMplMlJeXO+KnT592HK7c/FppaSkmk4mIiAhGjBiBJEnExcURFhZGaWmpq0QUCFzG72/rzO3dwnnxk/38eOKCu8W54Ynwi2By78msu2cdb93+Fsmhyfxr378Y/dFoHvzsQVYdXEWVucrdYt6wuMwQZGZmsm7dOgD27duHyWTC398fgJiYGKqqqjhx4gRWq5WNGzeSmZlJbm4uixYtAqCsrIwzZ84QESGOzhPceMj2Vceh/kaeWvq9mC+4ThhkA0PihzB/8Hy+uOcLnu3zLBfMF5idN5tBKwbxtyN/I/+M5w0NXSsuGxpKS0sjNTWViRMnIkkSL7zwAqtXryYgIIChQ4cye/ZsnnvuOQBGjBhBQkIC4eHhTJs2jS+//BKLxcLs2bMvGhYSCG4UQvyM/PX+m5nw3jc8v3IPCyb1aRPnMLcXwn3D+U2P3/Bo6qP8UP4DHx36iE8OfcKmTzeRZkrjgZQHGBQ3yLNPV7tMXPoJTZs2rUk8OTnZEe7bt2+T10kB/P39WbBggStFEghalT7xIWTfkcxL/83n/a3HeGxAgrtFandIkkSv8F70Cu/FSP+R7JP38Z+C//Dc/3uOSL9IJnSbwLgu4+jg3cHdorZZxMpigcDFPDYggaHdI5j7WT7fHz/nbnHaNX56Px5OfZj/3v1f3s16l/jAeN75/h2GrBjCnzf/mV2nd4nVyy0gDIFA4GIkSeKNe3oRGeTNH5bu4nyN2FvH1ehkHVlxWSwctpA1d65hbJexbCzayENrH2Js7liW5i+lwiz2hWpAGAKBoBUI8jUw//40TlfW8UzObmyKaJW2FkkdkpjRbwZfjf+K2f1n46XzYu63cxm8fDCzts7i+9LvPb6XIAyBQNBK9IoNZvadqWw6UMacz8SbLa2Nr8GXcV3HsWzUMpaNWsbIxJGsO7aOhz9/mJFrRrJgzwJOVp10t5huQUynCwStyAO/iuen0ioWbTlKksmf+9Jde1CLoGVSQ1NJzUjl+b7Ps6FwA7mHcx17HKVHpjMmaQyD4wbjZ/Bzt6itgjAEAkErM3NkCkfKq5n10V7iQ33F5nRuxNfgy5ikMYxJGkNxVbG2E+qhXGZsmcGLuhcZGDOQEQkjGBAzoF1vfCeGhgSCVkavk/nr/TeTEObH7//9PUfLq90tkgCI9o/m971+z2djP2PJr5cwJmkMO0p38MymZ7g953ZmbJnBluItWJT2tzhQ9AgEAjcQ6G1g0cN9uetvW3ls8XesmZxJkK/B3WIJ0N7ySotIIy0ijez0bL4t+Za1x9byZeGX5B7Oxd/gT/+o/twafSuZ0ZmYfE3uFvmaEYZAIHATcaG+LJjUhwcWfsPkpTtZ/Gg6Bp3opLcl9LKejOgMMqIzmNVvFluKt/D1ia/ZXLyZDYUbAEgOSWZA9AAGRA+gZ3hPDPKNZ9CFIRAI3Eh6Qghz7r6JP638gf/9eB8v39UDWRbbULRFjDojg+IGMShuEKqqcvDcQbYUb2Fz8Wb+tfdfLPxxIb56X9Ij0+kX1Y+MqAw6BXa6IbYVEYZAIHAz42+J5Uh5NX/fdJiKOgt/Gd8Lb4PO3WIJfgZJkugW0o1uId147KbHqDBX8F3Jd2w7uY28kjw2ndgEaGc1p0em0z20O8khyXTr0A1/o797hW8BYQgEgjbA88O7EeRj4NXPCyg+V8s/H7qF8ID2+5ZKeyPQGMjg+MEMjh8MQFFlEXkn88g7mceW4i3kHs515I3xjyElNIWuHbrSObgziUGJxAXEYdC5b0hJGAKBoA0gSRJP3taZTqF+PJuzm7vmb2XRI7eQHBnobtEEV0FsQCyx3WK5t9u9qKpKWW0ZBWcLOHD2AAVnCyg4W+CYYwDQSTpiA2JJCEogMSiR2IBYovyjiPaPpqNfR5cbCWEIBII2xK97RBLToT+PLfmOcX/bxl/vTyMr+cZ/K8WTkSQJk68Jk6+JgTEDHek1lhqOVhzl6IWjHDl/RPMvHGHzic1YVWtjeSTCfcOJ9o/mdz1/x4DoAdddRmEIBII2Ro/oID5+agCPLfmOx5Z8x4yR3Xm4fzx68UZRu8LX4KutcA5NbZJuVaycrjlNcVUxJ6tOcrLqJMVVxZRUl1BnrXOJLMIQCARtkMggb1Y82Z9nlu3mxU/3M++rnxjUzcTQ7hEM7BqOn5f467ZX9LKeKP8oovyjWu+Zrrz5nDlz2LNnD5IkMX36dHr27Om4tm3bNt588010Oh0DBw7kqaee+sUyAoEn4WvUs2BSHzbkl7Ju3ym+KjjN6l3FGPUymZ1DGdI9gm4RAUQEemMK9MJLL940ElwdLjME3377LYWFheTk5HD48GGmT5/e5ESyl156iUWLFhEREcGkSZMYPnw4Z8+e/dkyAoGnIcsSw1MjGZ4aidWmsKPwHBv2l7Jhfykb1+xtkjfEz4gpwIvIIG8w12DaZ8bHoMPboMPLoMPbIONj0OFn1ONj1OHnpcPHoLf7Ogw6GYNexqCTMOpkLa6T0cuSWNvQznGZIcjLy2PIkCEAdO7cmQsXLlBVVYW/vz9FRUUEBQXRsWNHAG677Tby8vI4e/bsJcsIBJ6OXifTLzGUfomhzByZwtHyao6freF0RT2nKuo4VVHHabt/pqKe/DNl1FkU6iw26q3KNT1blkAvy+h1EjpZQi9L6GTZ7ktN0mVJC+ucw5KELONI01yzuIwjrJMlJEmLS+C4Dlq5hvKS070kCc6dPUvosfwm5SQJJLS8Wtg5DceCrxav2eNc8l72jDQt03g/mtybJmlS0+v2iPMznfPJkkRmUijBvtf/HHeXGYLy8nJSUxsnQUJCQigrK8Pf35+ysjJCQkKaXCsqKuLcuXOXLCMQCBqRJInEcH8Sw1v+b+Tn55OSkuKIK4pKvVWh1mKjxmylxmyzOys19TZqLTYsNgWLTcFsU7FYFUfcqqjYFBWLTcWmaHGrTcWqqCiKar/emG5TtXSbqpVTGnwFrDZFu65qMjVcV1Xs6VpZRcVxeE/DdcVeTlVVVHu6ojS9ZlMUkCrBnqY6lW8PPJXVmT8NT/7ljFdIq804Xc0JQJcqk59/9Yd61NXVXVP5GxWht2dxOXp72V0HCbjsRqZEY5u27VFXV4e3t3eL1xoMSEO10hBW7TFH2O43zWu/7nRNy6c67oXjXs73dY6rzeKNYUc6Tes9tVnemEClxe/1Wn/nLjMEJpOJ8vJyR/z06dOEh4e3eK20tBSTyYTBYLhkGWecWzpXSvOWkqcg9PYshN6exeXovXPnzktec9mLyZmZmaxbtw6Affv2YTKZHEM8MTExVFVVceLECaxWKxs3biQzM/NnywgEAoHANbisR5CWlkZqaioTJ05EkiReeOEFVq9eTUBAAEOHDmX27Nk899xzAIwYMYKEhAQSEhIuKiMQCAQC1+LSOYJp06Y1iScnN05y9O3bt8VXQ5uXEQgEAoFrEWvWBQKBwMMRhkAgEAg8HGEIBAKBwMMRhkAgEAg8HEm9mpVebuTn3oUVCAQCwaXp06dPi+k3nCEQCAQCwfVFDA0JBAKBhyMMgUAgEHg4HnPMkacdeHPw4EEmT57MI488wqRJkygpKeH555/HZrMRHh7O66+/jtF4/bezdTevvfYaO3fuxGq18sQTT3DTTTe1e71ra2vJzs7mzJkz1NfXM3nyZJKTk9u93g3U1dUxatQoJk+eTP/+/du93tu3b2fq1Kl06dIFgK5du/L4449fk94e0SNwPiTn5Zdf5uWXX3a3SC6lpqaGF198kf79+zvS3n33Xe6//36WLl1KfHw8K1eudKOEruGbb77hp59+Iicnh4ULFzJnzhyP0Hvjxo306NGDf//737z99tu88sorHqF3A3//+98JCgoCPON3DpCens4HH3zABx98wKxZs65Zb48wBJc6JKe9YjQa+ec//4nJZHKkbd++ncGDBwOQlZVFXl6eu8RzGX379uWdd94BIDAwkNraWo/Qe8SIEfz2t78FoKSkhIiICI/QG+Dw4cMcOnSI22+/HfCM33lLXKveHmEIysvL6dChgyPecOBNe0Wv11+0J3ttba2jqxgaGtou9dfpdPj6+gKwcuVKBg4c6BF6NzBx4kSmTZvG9OnTPUbvV199lezsbEfcU/Q+dOgQTz75JPfddx9bt269Zr09Zo7AGU9/Y7a96//FF1+wcuVK3n//fYYNG+ZIb+96L1u2jPz8fP70pz81Pdykner90Ucf0bt3b2JjY1u83l717tSpE1OmTOGOO+6gqKiIhx56CJvN5rh+NXp7hCH4uUNyPAVfX1/H6U0NBwG1RzZv3syCBQtYuHAhAQEBHqH33r17CQ0NpWPHjqSkpGCz2fDz82v3em/atImioiI2bdrEqVOnMBqNHvF9R0REMGLECADi4uIICwvjxx9/vCa9PWJoSBx4AxkZGY7PYP369dx6661uluj6U1lZyWuvvcZ7771HcHAw4Bl679ixg/fffx/QhkFramo8Qu+3336bVatWsXz5csaPH8/kyZM9Qu/c3FwWLVoEQFlZGWfOnGHs2LHXpLfHrCx+44032LFjh+PAG+ezEdobe/fu5dVXX6W4uBi9Xk9ERARvvPEG2dnZ1NfXExUVxdy5czEYDO4W9bqSk5PDvHnzSEhIcKS98sorzJw5s13rXVdXx4wZMygpKaGuro4pU6bQo0cP/ud//qdd6+3MvHnziI6OZsCAAe1e76qqKqZNm0YJrC3TAAACeklEQVRFRQUWi4UpU6aQkpJyTXp7jCEQCAQCQct4xNCQQCAQCC6NMAQCgUDg4QhDIBAIBB6OMAQCgUDg4QhDIBAIBB6ORywoEwiulBMnTjB69Gh69OjRJH3evHmONQpXw7x58+jQoQOTJk26VhEFguuGMAQCwSVISEjggw8+cLcYAoHLEYZAILgCsrOz8fX15ciRI5w7d465c+fSvXt3lixZwmeffQbA4MGD+d3vfkdxcTHZ2dnYbDaioqJ49dVXAe2siCeeeIJjx44xY8YMBg4c6E6VBAIxRyAQXClWq5XFixczdepU5s+fT1FREWvWrOHDDz/kww8/ZO3atRw/fpy33nqLRx55hKVLl2Iymdi7dy8A58+f57333mPmzJksW7bMzdoIBKJHIBBckqNHj/Lggw864g1bV2RkZADQu3dv3njjDfLz8+nVqxd6vfZ3SktLo6CggP379zNjxgwAnn/+eQC+/vpr0tLSAG3zsMrKylbTRyC4FMIQCASXoKU5guzsbBRFccQlSUKSpCZb/1osFmRZRqfTtbglcIPBEAjaCmJoSCC4Qnbu3AnArl276Ny5MykpKezevRur1YrVamXPnj2kpKTQo0cPvvnmGwDeeecdtm3b5k6xBYJLIpomAsElaD40BODt7Y1er+eJJ56gpKSE119/nZiYGCZMmMCkSZNQVZXx48cTHR3N008/zZ///GeWLl1Kx44dmTJlisOICARtCbH7qEBwBWRnZzN8+HCysrLcLYpAcN0QQ0MCgUDg4YgegUAgEHg4okcgEAgEHo4wBAKBQODhCEMgEAgEHo4wBAKBQODhCEMgEAgEHo4wBAKBQODh/H/9jwoYQP9xLQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Bootstrapping"
      ],
      "metadata": {
        "id": "ccYWmTuHfcWc"
      },
      "id": "ccYWmTuHfcWc"
    },
    {
      "cell_type": "code",
      "source": [
        "# load the results\n",
        "\n",
        "import pickle\n",
        "file = open(\"N_trials_adam_mlp.pth\", \"rb\")\n",
        "N_trials_adam_mlp = pickle.load(file)"
      ],
      "metadata": {
        "id": "RUKyieMoZPTN"
      },
      "id": "RUKyieMoZPTN",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# load the results\n",
        "\n",
        "import pickle\n",
        "file = open(\"N_trials_nosadam_mlp.pth\", \"rb\")\n",
        "N_trials_nosadam_mlp = pickle.load(file)"
      ],
      "metadata": {
        "id": "jQW3ZjvfZW6v"
      },
      "id": "jQW3ZjvfZW6v",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# load the results\n",
        "\n",
        "import pickle\n",
        "file = open(\"N_trials_adam_cnn.pth\", \"rb\")\n",
        "N_trials_adam_cnn = pickle.load(file)"
      ],
      "metadata": {
        "id": "VwjGffDgWjWc"
      },
      "id": "VwjGffDgWjWc",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# load the results\n",
        "\n",
        "import pickle\n",
        "file = open(\"N_trials_nosadam_cnn.pth\", \"rb\")\n",
        "N_trials_nosadam_cnn = pickle.load(file)"
      ],
      "metadata": {
        "id": "WMaRuO82XkA2"
      },
      "id": "WMaRuO82XkA2",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Bootstrapping for Adam on MLP"
      ],
      "metadata": {
        "id": "I_C5t4Kwu_iv"
      },
      "id": "I_C5t4Kwu_iv"
    },
    {
      "cell_type": "code",
      "source": [
        "N=50\n",
        "K=25\n",
        "N_trials = N_trials_adam_mlp\n",
        "\n",
        "means_train = []\n",
        "means_test  = []\n",
        "# Do the following 50 times :\n",
        "for _ in range(50):\n",
        "    # Resample N samples from the N-trials with replacement\n",
        "    N_sampled_indices = np.random.choice(list(range(len(N_trials))), N) # choose random indices in the list of N trials\n",
        "    \n",
        "    # Recover the lists associated to the indices and keep only intersting information, i.e. test and train errors\n",
        "    N_sampled_train_error = np.array([N_trials[i][4].cpu().detach().numpy() for i in N_sampled_indices])\n",
        "    N_sampled_test_error = np.array([N_trials[i][5].cpu().detach().numpy() for i in N_sampled_indices])\n",
        "    \n",
        "    # Compute statistic on the first K trials of the resampled dataset\n",
        "    means_train.append(N_sampled_train_error[:K].mean())\n",
        "    means_test.append(N_sampled_test_error[:K].mean())\n",
        "    \n",
        "# 5th percentile, 95 percentile of bootrap distribution\n",
        "fifth_percentile_train_adam_mlp = np.percentile(means_train, 5)\n",
        "fifth_percentile_test_adam_mlp = np.percentile(means_test, 5)\n",
        "\n",
        "ninety_fifth_percentile_train_adam_mlp = np.percentile(means_train, 95)\n",
        "ninety_fifth_percentile_test_adam_mlp = np.percentile(means_test, 95)\n",
        "\n",
        "# For plotting purposes only\n",
        "mean_all_train_adam_mlp = np.array(means_train).mean()\n",
        "mean_all_test_adam_mlp = np.array(means_test).mean()"
      ],
      "metadata": {
        "id": "hbTqEYOYZc7P"
      },
      "id": "hbTqEYOYZc7P",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Bootstrapping for NosAdam on MLP"
      ],
      "metadata": {
        "id": "a0_uxiIOvGkY"
      },
      "id": "a0_uxiIOvGkY"
    },
    {
      "cell_type": "code",
      "source": [
        "N=50\n",
        "K=25\n",
        "N_trials = N_trials_nosadam_mlp\n",
        "\n",
        "means_train = []\n",
        "means_test  = []\n",
        "# Do the following 50 times :\n",
        "for _ in range(50):\n",
        "    # Resample N samples from the N-trials with replacement\n",
        "    N_sampled_indices = np.random.choice(list(range(len(N_trials))), N) # choose random indices in the list of N trials\n",
        "    \n",
        "    # Recover the lists associated to the indices and keep only intersting information, i.e. test and train errors\n",
        "    N_sampled_train_error = np.array([N_trials[i][4].cpu().detach().numpy() for i in N_sampled_indices])\n",
        "    N_sampled_test_error = np.array([N_trials[i][5].cpu().detach().numpy() for i in N_sampled_indices])\n",
        "    \n",
        "    # Compute statistic on the first K trials of the resampled dataset\n",
        "    means_train.append(N_sampled_train_error[:K].mean())\n",
        "    means_test.append(N_sampled_test_error[:K].mean())\n",
        "    \n",
        "# 5th percentile, 95 percentile of bootrap distribution\n",
        "fifth_percentile_train_nosadam_mlp = np.percentile(means_train, 5)\n",
        "fifth_percentile_test_nosadam_mlp = np.percentile(means_test, 5)\n",
        "\n",
        "ninety_fifth_percentile_train_nosadam_mlp = np.percentile(means_train, 95)\n",
        "ninety_fifth_percentile_test_nosadam_mlp = np.percentile(means_test, 95)\n",
        "\n",
        "# For plotting purposes only\n",
        "mean_all_train_nosadam_mlp = np.array(means_train).mean()\n",
        "mean_all_test_nosadam_mlp = np.array(means_test).mean()"
      ],
      "metadata": {
        "id": "SmWtInc5ZqrB"
      },
      "id": "SmWtInc5ZqrB",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Bootstrapping for Adam on CNN"
      ],
      "metadata": {
        "id": "xan6zk5WvJQe"
      },
      "id": "xan6zk5WvJQe"
    },
    {
      "cell_type": "code",
      "source": [
        "N=50\n",
        "K=25\n",
        "N_trials = N_trials_adam_cnn\n",
        "\n",
        "means_train = []\n",
        "means_test  = []\n",
        "# Do the following 50 times :\n",
        "for _ in range(50):\n",
        "    # Resample N samples from the N-trials with replacement\n",
        "    N_sampled_indices = np.random.choice(list(range(len(N_trials))), N) # choose random indices in the list of N trials\n",
        "    \n",
        "    # Recover the lists associated to the indices and keep only intersting information, i.e. test and train errors\n",
        "    N_sampled_train_error = np.array([N_trials[i][4] for i in N_sampled_indices])\n",
        "    N_sampled_test_error = np.array([N_trials[i][5] for i in N_sampled_indices])\n",
        "    \n",
        "    # Compute statistic on the first K trials of the resampled dataset\n",
        "    means_train.append(N_sampled_train_error[:K].mean())\n",
        "    means_test.append(N_sampled_test_error[:K].mean())\n",
        "    \n",
        "# 5th percentile, 95 percentile of bootrap distribution\n",
        "fifth_percentile_train_adam_cnn = np.percentile(means_train, 5)\n",
        "fifth_percentile_test_adam_cnn = np.percentile(means_test, 5)\n",
        "\n",
        "ninety_fifth_percentile_train_adam_cnn = np.percentile(means_train, 95)\n",
        "ninety_fifth_percentile_test_adam_cnn = np.percentile(means_test, 95)\n",
        "\n",
        "# For plotting purposes only\n",
        "mean_all_train_adam_cnn = np.array(means_train).mean()\n",
        "mean_all_test_adam_cnn = np.array(means_test).mean()"
      ],
      "metadata": {
        "id": "ivHbliZQXrlj"
      },
      "id": "ivHbliZQXrlj",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Bootstrapping for NosAdam on CNN"
      ],
      "metadata": {
        "id": "oMR5w874vNFc"
      },
      "id": "oMR5w874vNFc"
    },
    {
      "cell_type": "code",
      "source": [
        "N=50\n",
        "K=25\n",
        "N_trials = N_trials_nosadam_cnn\n",
        "\n",
        "means_train = []\n",
        "means_test  = []\n",
        "# Do the following 50 times :\n",
        "for _ in range(50):\n",
        "    # Resample N samples from the N-trials with replacement\n",
        "    N_sampled_indices = np.random.choice(list(range(len(N_trials))), N) # choose random indices in the list of N trials\n",
        "    \n",
        "    # Recover the lists associated to the indices and keep only intersting information, i.e. test and train errors\n",
        "    N_sampled_train_error = np.array([N_trials[i][4] for i in N_sampled_indices])\n",
        "    N_sampled_test_error = np.array([N_trials[i][5] for i in N_sampled_indices])\n",
        "    \n",
        "    # Compute statistic on the first K trials of the resampled dataset\n",
        "    means_train.append(N_sampled_train_error[:K].mean())\n",
        "    means_test.append(N_sampled_test_error[:K].mean())\n",
        "    \n",
        "# 5th percentile, 95 percentile of bootrap distribution\n",
        "fifth_percentile_train_nosadam_cnn = np.percentile(means_train, 5)\n",
        "fifth_percentile_test_nosadam_cnn = np.percentile(means_test, 5)\n",
        "\n",
        "ninety_fifth_percentile_train_nosadam_cnn = np.percentile(means_train, 95)\n",
        "ninety_fifth_percentile_test_nosadam_cnn = np.percentile(means_test, 95)\n",
        "\n",
        "# For plotting purposes only\n",
        "mean_all_train_nosadam_cnn = np.array(means_train).mean()\n",
        "mean_all_test_nosadam_cnn = np.array(means_test).mean()"
      ],
      "metadata": {
        "id": "Pb2oaSAbYx24"
      },
      "id": "Pb2oaSAbYx24",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Plot for train error"
      ],
      "metadata": {
        "id": "r0Ns3h0NvSu1"
      },
      "id": "r0Ns3h0NvSu1"
    },
    {
      "cell_type": "code",
      "source": [
        "fig = plt.figure(figsize=(6, 6))\n",
        "plt.scatter(x=[0], y=[mean_all_train_adam_mlp], label='Adam on MLP')\n",
        "plt.errorbar(x=[0], y=[mean_all_train_adam_mlp], yerr=[[fifth_percentile_train_adam_mlp],[ninety_fifth_percentile_train_adam_mlp]], ecolor='black', capsize=2, elinewidth=1)\n",
        "plt.scatter(x=[1], y=[mean_all_train_nosadam_mlp], label='NosAdam on MLP')\n",
        "plt.errorbar(x=[1], y=[mean_all_train_nosadam_mlp], yerr=[[fifth_percentile_train_nosadam_mlp],[ninety_fifth_percentile_train_nosadam_mlp]], ecolor='black', capsize=2, elinewidth=1)\n",
        "\n",
        "plt.scatter(x=[2], y=[mean_all_train_adam_cnn], label = 'Adam on CNN')\n",
        "plt.errorbar(x=[2], y=[mean_all_train_adam_cnn], yerr=[[fifth_percentile_train_adam_cnn],[ninety_fifth_percentile_train_adam_cnn]], ecolor='black', capsize=2, elinewidth=1)\n",
        "plt.scatter(x=[3], y=[mean_all_train_nosadam_cnn], label = 'NosAdam on CNN')\n",
        "plt.errorbar(x=[3], y=[mean_all_train_nosadam_cnn], yerr=[[fifth_percentile_train_nosadam_cnn],[ninety_fifth_percentile_train_nosadam_cnn]], ecolor='black', capsize=2, elinewidth=1)\n",
        "plt.ylabel('Train Loss', fontsize = 10)\n",
        "plt.legend(loc = 'upper left')\n",
        "plt.savefig('bootsrapping.pdf')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        },
        "id": "aeP8L3xHYt81",
        "outputId": "feafa52d-64c3-4a80-e49e-fc0f0d3ecc41"
      },
      "id": "aeP8L3xHYt81",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAFlCAYAAADmu++zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3RU9b338feXkBIEgaOGilwMKgrkIoQsqQLWx1jlVLygPohaaniqnEX1CD4UW7TVyKqn7YIloHXJESmID0YBAaVoOVrxghVsghhAvKBELqJGLLFIkNv3+WOGKQkTMrnMTJL9ea01i5nf/u09382G+cy+zG+buyMiIsHVKtkFiIhIcikIREQCTkEgIhJwCgIRkYBTEIiIBJyCQEQk4Fonu4C6OuWUUzwjIyPZZYiINCslJSVfuXt6tGnNLggyMjIoLi5OdhkiIs2KmX1a0zQdGhIRCTgFgYhIwCkIREQCrtmdI4jmwIEDbN++nX379iW7FImztLQ0unXrRmpqarJLEWkxWkQQbN++nRNPPJGMjAzMLNnlSJy4O7t27WL79u307Nkz2eWItBgt4tDQvn37OPnkkxUCLZyZcfLJJ2vPT6SRtYggABQCAaHtLNL4WkwQNAVLly7FzHj//fdr7HPRRRc1ud9BFBYWYmZs3rw50jZ9+nTMLFJrRkYGX331VZX55s6dS3p6Ov369aNv377MmjUroXWLSONQEDSioqIiBg8eTFFRUbJLqbPs7GyefvrpyOuFCxeSmZlZ63zXX38969at49VXX+Xuu+/miy++iGeZIhIHgQyCpe/sYNDvX6Hnr5Yz6PevsPSdHQ1e5p49e1i1ahWzZ8+u8oFaWVnJyJEj6dOnD8OHD6eysjIybezYseTl5ZGZmcl9990Xac/IyGDSpEn069ePvLw81q5dy2WXXcaZZ57JzJkzo77/gw8+SFZWFllZWUyfPh2AsrIy+vTpw6233kpmZiaXXnpplfc/2tVXX81zzz0HwMcff0zHjh055ZRTYl7/zp07c+aZZ/LppzX+eFFEmqjABcHSd3YwafF6duyuxIEduyuZtHh9g8PgueeeY+jQoZx99tmcfPLJlJSUAPDoo49ywgknsGnTJu6///5IO8ADDzxAcXExpaWlvPbaa5SWlkam9ejRg3Xr1jFkyBAKCgpYtGgRq1evrhIYR5SUlDBnzhzWrFnD6tWrmTVrFu+88w4AH330EbfddhsbN26kU6dOPPvss1Hr79ChA927d2fDhg08/fTTXH/99XVa/08++YRPPvmEs846q07ziUjyBS4Ipqz4gMoDh6q0VR44xJQVHzRouUVFRYwcORKAkSNHRg4Pvf766/zkJz8BICcnh5ycnMg8CxYsIDc3l/79+7Nx40bee++9yLQrr7wSCB2yGThwICeeeCLp6em0adOG3bt3V3nvVatWMXz4cNq1a0f79u255ppreOONNwDo2bMn/fr1A2DAgAGUlZXVuA4jR47k6aefZunSpQwfPjym9X7mmWfo168fN9xwA//93//NSSedFNN8ItJ0BC4IPtsd/dBITe2x+Prrr3nllVe45ZZbyMjIYMqUKSxYsAB3r3GeLVu2MHXqVP76179SWlrK5ZdfXuWyyDZt2gDQqlWryPMjrw8ePBhzbUfPm5KSctx5hw0bxpNPPkmPHj3o0KFDTMs/co5gzZo1MYeHiNTNkQs6jjwKCwsbdfmBC4LTOrWtU3ssFi1axKhRo/j0008pKytj27Zt9OzZkzfeeIMLL7yQp556CoANGzZEDv988803tGvXjo4dO/LFF1/w4osv1vv9hwwZwtKlS9m7dy/ffvstS5YsYciQIXVezgknnMAf/vAH7rnnnnrXIiKNr7CwMPLF0t0bPQhaxC+L62LiZecwafH6KoeH2qamMPGyc+q9zKKiIn75y19Wabv22mspKiriwQcfZPTo0fTp04c+ffowYMAAAM4991z69+9P79696d69O4MGDar3++fm5lJQUMB5550HwC233EL//v2PexioJkcOb0WTk5NDq1ah7w4jRoyocphLRJovO97hi6YoLy/Pq1+Hv2nTJvr06RPzMpa+s4MpKz7gs92VnNapLRMvO4er+3dt7FIlTuq6vUVaCjM77iHnWuYtcfe8aNMCt0cAcHX/rvrgFxEJC9w5AhERqUpBICIScHELAjNLM7O3zexdM9toZvdH6VNgZuVmti78uCVe9YiISHTxPEfwHXCxu+8xs1RglZm96O6rq/V7xt1vj2MdIiJyHHELAg+d2t4TfpkafjSvS5RERAIgrucIzCzFzNYBXwIvufuaKN2uNbNSM1tkZt1rWM4YMys2s+Ly8vJ4llxvZsaECRMir6dOndqgH32MHz+erl27cvjw4Rr7tG/fvt7Lj5eLLrqIHj16VLnE7eqrr47UWlZWRlZW1jHzFRQURIbDyM3N5a233kpYzSJBF9cgcPdD7t4P6AacZ2bVPwGWARnungO8BDxRw3Iec/c8d89LT0+PZ8n11qZNGxYvXnzMmP31cfjwYZYsWUL37t157bXXGqG6xOrUqRNvvvkmALt372bnzp0xzTdlyhTWrVvH73//e/7jP/4jniWKyFESctWQu+8GVgJDq7Xvcvfvwi8fBwYkoh5KF8C0LCjsFPqzdEGDF9m6dWvGjBnDtGnTjplWVlbGxRdfTE5ODvn5+WzduhUIjfmflZXFueeey4UXXhjp/+qrr5KZmcnYsWOr3Ntgy5YtnH/++WRnZ/PrX/860r5nzx7y8/PJzc0lOzs7Mpx0WVkZvXv3pqCggLPPPpubbrqJl19+mUGDBtGrVy/efvvtY2rdt28fo0ePJjs7m/79+7Ny5UogdBOaa665hqFDh9KrVy/uuuuuGv8ujgxeB7B48WKuueaauvxVcuGFF1a5SY6IxJm7x+UBpAOdws/bAm8Aw6r16XLU8+HA6tqWO2DAAK/uvffeO6atRu8+4/7b77vf1+Ffj99+P9TeAO3atfOKigo//fTTfffu3T5lyhS/77773N192LBhPnfuXHd3nz17tl911VXu7p6VleXbt293d/d//OMfkWXdcsstPm/ePK+oqPDTTjvN9+/f7+7uV1xxhT/xxBPu7v7HP/7R27Vr5+7uBw4c8IqKCnd3Ly8v9zPPPNMPHz7sW7Zs8ZSUFC8tLfVDhw55bm6ujx492g8fPuxLly6N1HG0qVOn+ujRo93dfdOmTd69e3evrKz0OXPmeM+ePX337t1eWVnpPXr08K1btx4z/w9/+ENfvXq1Z2dn+8GDB/1HP/qRb9myJVLrli1bPDMz85j5br75Zl+4cKG7uy9YsMDPO++8Gv+u67S9RVqQ0Ed2vect9ho+V+O5R9AFWGlmpcDfCZ0j+LOZTTazK8N97ghfWvoucAdQEMd6Qv46GQ5UG2n0QGWovYE6dOjAT3/6Ux566KEq7W+99RY33ngjAKNGjWLVqlUADBo0iIKCAmbNmsWhQ6Gxj/bv388LL7zA1VdfTYcOHRg4cCArVqwA4M033+SGG26ILOcId+fuu+8mJyeHSy65hB07dkTuFNazZ0+ys7Np1aoVmZmZ5OfnY2ZkZ2dHHYto1apVkWGze/fuzemnn86HH34IQH5+Ph07diQtLY2+ffvWeBOalJQUBg8ezNNPP01lZSUZGRkx/f1NnDiRfv368dhjjzF79uyY5hGRhovnVUOlQP8o7fce9XwSMCleNURVsb1u7XU0fvx4cnNzGT16dK19Z86cyZo1a1i+fDkDBgygpKSEv/3tb+zevZvs7GwA9u7dS9u2bRk2bBgQ/ebt8+fPp7y8nJKSElJTU8nIyIgMaV19COujh7euy3DW1ZdV25DWI0eOZPjw4XU6YT5lyhSuu+66OtUkIg0XvF8Wd+xWt/Y6OumkkxgxYkSVb7QXXHBB5Jj5/PnzI0NEf/zxxwwcOJDJkyeTnp7Otm3bKCoq4vHHH6esrIyysjK2bNnCSy+9xN69exk0aFCV5RxRUVFB586dSU1NZeXKlQ26XeSQIUMiy/7www/ZunUr55xT95FZhwwZwqRJkyJ7MCLSdAUvCPLvhdRq9x5IbRtqbyQTJkyocvXQww8/zJw5c8jJyeHJJ59kxowZQOhQSHZ2NllZWVxwwQX06tWLv/zlL1x++eWRedu1a8fgwYNZtmwZM2bM4JFHHiE7O5sdO/51a82bbrqJ4uJisrOzmTdvHr1796537T//+c85fPgw2dnZXH/99cydO7fKnkCszIxf/OIXUe97/MEHH9CtW7fIY+HChfWuV0QaLpDDUFO6IHROoGJ7aE8g/17IGdHIlUq8aBhqCSoNQ92Yckbog19EJCx4h4ZERKQKBYGISMApCEREAk5BICIScAoCEZGAUxA0oqVLl2JmvP/++zX2ueiii6h++WtTMG/ePLKysiKDzU2dOhUIDQ/dtWtXvvsuNDbgV199FRkyoqysDDPj4Ycfjizn9ttvZ+7cuYkuX0QaQEHQiIqKihg8eHCVEUObgxdffJHp06fzP//zP6xfv57Vq1fTsWPHyPSUlBT+9Kc/RZ23c+fOzJgxg/379yeqXBFpZIEMguWfLOfSRZeS80QOly66lOWfLG/wMvfs2cOqVauYPXt2ZBgIgMrKSkaOHEmfPn0YPnw4lZX/GvBu7Nix5OXlkZmZyX333Rdpz8jIYNKkSfTr14+8vDzWrl3LZZddxplnnsnMmTOjvv+DDz5IVlYWWVlZTJ8+HQh9Y+/Tpw+33normZmZXHrppVXe/4jf/e53TJ06ldNOOw0IjSl06623RqaPHz+eadOmRR1bKD09nfz8fJ54IuqtJESkGQhcECz/ZDmFfytk57c7cZyd3+6k8G+FDQ6D5557jqFDh3L22Wdz8sknU1JSAsCjjz7KCSecwKZNm7j//vsj7QAPPPAAxcXFlJaW8tprr1FaWhqZ1qNHD9atW8eQIUMoKChg0aJFrF69ukpgHFFSUsKcOXNYs2YNq1evZtasWbzzzjsAfPTRR9x2221s3LiRTp068eyzzx4z/4YNGxgwoOZbQfTo0YPBgwfz5JNPRp3+y1/+kqlTp0ZGUBWR5iVwQTBj7Qz2HdpXpW3foX3MWDujQcstKipi5MiRQGjkzSOHh15//fXIsM45OTnk5ORE5lmwYAG5ubn079+fjRs38t5770WmXXllaKTu7OxsBg4cyIknnkh6ejpt2rRh9+7dVd571apVDB8+nHbt2tG+fXuuueYa3njjDYDI7R8BBgwYEHXo6VhMmjSJKVOmRL115hlnnMHAgQN56qmn6rVsEUmuwA0x8fm3n9epPRZff/01r7zyCuvXr8fMOHToEGbGlClTapxny5YtTJ06lb///e/827/9GwUFBZGho4Eqw0VXH0q6LsNHVx86OtqhoczMTEpKSrj44otrXE6vXr3o168fCxZEv5vb3XffzXXXXccPf/jDmGsTkaYhcHsEp7Y7tU7tsVi0aBGjRo3i008/paysjG3bttGzZ0/eeOMNLrzwwsg35Q0bNkQO/3zzzTe0a9eOjh078sUXX/Diiy/W+/2HDBnC0qVL2bt3L99++y1LliyJDHUdi0mTJjFx4kQ+/zwUhvv37+fxxx8/pt8999wTuZqout69e9O3b1+WLVtWv5UQkaQJXBCMyx1HWkpalba0lDTG5Y6r9zKLiooYPnx4lbZrr72WoqIixo4dy549e+jTpw/33ntv5Fj8ueeeS//+/enduzc33ngjgwYNqvf75+bmUlBQwHnnncfAgQO55ZZb6N//mHsC1ejHP/4xt99+O5dccgmZmZnk5ubyzTffHNPvyLSa3HPPPWzf3jg3+BGRxAnkMNTLP1nOjLUz+Pzbzzm13amMyx3H5WdcXvuM0iRoGGoJKg1D3YguP+NyffCLiIQF7tCQiIhUpSAQEQk4BYGISMApCEREAk5BICIScAqCRmJmTJgwIfJ66tSpFBYW1nt548ePp2vXrlGHdDiiffv29V5+vBw4cIBf/epX9OrVi9zcXM4///zIj+UyMjK49tprI30XLVpEQUEBAHPnzqVVq1ZVxlvKysqq95AYIhI7BUEjadOmDYsXL+arr75q8LIOHz7MkiVL6N69O6+99lojVJc4v/nNb9i5cycbNmxg7dq1LF26lH/+85+R6SUlJVXGVDpat27deOCBBxJVqoiEBTIIKpYt46OL89nUpy8fXZxPRSMMi9C6dWvGjBnDtGnTjplWVlbGxRdfTE5ODvn5+WzduhWAhQsXkpWVxbnnnsuFF14Y6f/qq6+SmZnJ2LFjq9zbYMuWLZx//vlkZ2fz61//OtK+Z88e8vPzyc3NJTs7m+eeey7yvr1796agoICzzz6bm266iZdffplBgwbRq1cv3n777WNq3bdvH6NHj47coGblypVA6Bv7Nddcw9ChQ+nVqxd33XXXMfPu3buXWbNm8fDDD0fGOPr+97/PiBEjIn0mTJhQ44f9sGHD2LhxIx988EHNf9Ei0ugCFwQVy5ax8zf3cvCzz8Cdg599xs7f3NsoYXDbbbcxf/58KioqqrT/53/+JzfffDOlpaXcdNNN3HHHHQBMnjyZFStW8O677/L8889H+hcVFXHDDTcwfPhwli9fzoEDBwAYN24cY8eOZf369XTp0iXSPy0tjSVLlrB27VpWrlzJhAkTIr8+3Lx5MxMmTOD999/n/fff56mnnmLVqlVMnTqV//qv/zpmHR555BHMjPXr11NUVMTNN98cGQxv3bp1PPPMM6xfv55nnnmGbdu2VZl38+bN9OjRgw4dOtT4dzRixAjWrl3L5s2bj5nWqlUr7rrrrqh1iUj8BC4Ivpw2Hd9XdRhq37ePL6dNb/CyO3TowE9/+lMeeuihKu1vvfUWN954IwCjRo1i1apVAAwaNIiCggJmzZoVGct///79vPDCC1x99dV06NCBgQMHsmLFCgDefPNNbrjhhshyIvW7c/fdd5OTk8Mll1zCjh07+OKLL4DQMNTZ2dm0atWKzMxM8vPzMTOys7OjHn9ftWpVZNjs3r17c/rpp/Phhx8CkJ+fT8eOHUlLS6Nv3758+umndf47SklJYeLEifzud7+LOv3GG29k9erVbNmypc7LFpH6CVwQHNy5s07tdTV+/Hhmz57Nt99+W2vfmTNn8tvf/pZt27YxYMAAdu3axYoVK9i9ezfZ2dlkZGSwatWqKoeHzOyY5cyfP5/y8nJKSkpYt24d3//+9yPf4qsPYX308NZ1Gc66+rJSUlKOmf+ss85i69atUQesO9qoUaN4/fXXj9mjgNAhtgkTJvCHP/yhTrWJSP0FLghaH3VIJZb2ujrppJMYMWIEs2fPjrRdcMEFkdtXzp8/PzJE9Mcff8zAgQOZPHky6enpbNu2jaKiIh5//HHKysooKytjy5YtvPTSS+zdu5dBgwZVWc4RFRUVdO7cmdTUVFauXFmvb+pHDBkyJLLsDz/8kK1bt3LOOefENO8JJ5zAz372M8aNGxe5h3F5eTkLFy6s0i81NZU777wz6vkUgIKCAl5++WXKy8vrvR4iErvABUHnO8djaVWHoba0NDrfOb7R3mPChAlVrh56+OGHmTNnDjk5OTz55JPMmBG6G9rEiRPJzs4mKyuLCy64gF69evGXv/yFyy//14B47dq1Y/DgwSxbtowZM2bwyCOPkJ2dzY4dOyJ9brrpJoqLi8nOzmbevHn07t273rX//Oc/5/Dhw2RnZ3P99dczd+7cKnsCtfntb39Leno6ffv2JSsri2HDhkU9Z/Czn/2sxj2S733ve9xxxx18+eWX9V4PEYldIIehrli2jC+nTefgzp207tKFzneOp+MVVzR2qRInGoZagkrDUDeijldcoQ9+EZGwwB0aEhGRquIWBGaWZmZvm9m7ZrbRzO6P0qeNmT1jZpvNbI2ZZcSrHhERiS6eewTfARe7+7lAP2Comf2gWp+fAf9w97OAaUC9rxlsbuc6pH60nUUaX9yCwEP2hF+mhh/V/xdfBTwRfr4IyLdoF8rXIi0tjV27dulDooVzd3bt2kVatau+RKRh4nqy2MxSgBLgLOARd19TrUtXYBuAux80swrgZOCrassZA4wB6NGjxzHv061bN7Zv367rzgMgLS2Nbt26JbsMkRYlrkHg7oeAfmbWCVhiZlnuvqEey3kMeAxCl49Wn56amkrPnj0bXK+ISBAl5Kohd98NrASGVpu0A+gOYGatgY7ArkTUJCIiIfG8aig9vCeAmbUFfgS8X63b88DN4efXAa+4DvSLiCRUPA8NdQGeCJ8naAUscPc/m9lkoNjdnwdmA0+a2Wbga2BkHOsREZEo4hYE7l4K9I/Sfu9Rz/cB/zteNYiISO30y2IRkYBTEIiIBJyCQEQk4BQEIiIBpyAQEQk4BYGISMApCEREAk5BICIScAoCEZGAUxCIiAScgkBEJOAUBCIiAacgEBEJOAWBiEjAKQhERAJOQSAiEnAKAhGRgFMQSNIUFhZiZpFHYWFhsksSCSRrbveKz8vL8+Li4mSXIY3IzGhu/w5FkqEh/1fMrMTd86JN0x6BiEjAKQhERAJOQSAiEnAKAhGRgFMQiIgEnIJARCTgFAQiIgGnIBARCTgFgYhE6NfewaRfFkvS6ZfFTY+2SdOkXxaLiEhcKAhERAJOQSAiEnAKAhGRgFMQiIgEnIJARCTg4hYEZtbdzFaa2XtmttHMxkXpc5GZVZjZuvDj3njVIyIi0bWO47IPAhPcfa2ZnQiUmNlL7v5etX5vuPuwONYhIiLHEbc9Anff6e5rw8//CWwCusbr/UREpH4Sco7AzDKA/sCaKJPPN7N3zexFM8usYf4xZlZsZsXl5eVxrFREJHjiHgRm1h54Fhjv7t9Um7wWON3dzwUeBpZGW4a7P+buee6el56eHt+CRUQCJq5BYGaphEJgvrsvrj7d3b9x9z3h5y8AqWZ2SjxrEhGRquJ51ZABs4FN7v5gDX1ODffDzM4L17MrXjWJiMix4nnV0CBgFLDezNaF2+4GegC4+0zgOmCsmR0EKoGRriEPRUQSKm5B4O6rAKulzx+BP8arBhERqZ1+WSwiEnAKAhGRgFMQiIgEnIJARCTgFAQiIgGnIBARCTgFgYhIwCkIREQCTkEgIhJwCgIRkYBTEIiIBJyCQEQk4BQEIiIBpyAQEQk4BYGISMApCEREAk5BICIScAoCEZGAUxCIiAScgkBEJOAUBCIiAacgEBEJuFqDwMzONLM24ecXmdkdZtYp/qWJiEgixLJH8CxwyMzOAh4DugNPxbUqERFJmFiC4LC7HwSGAw+7+0SgS3zLEhGRRIklCA6Y2Q3AzcCfw22p8StJREQSKZYgGA2cDzzg7lvMrCfwZHzLEhGRRGldWwd3fw+4A8DM/g040d3/EO/CREQkMWK5auhVM+tgZicBa4FZZvZg/EsTEZFEiOXQUEd3/wa4Bpjn7gOBS+JbloiIJEosQdDazLoAI/jXyWIREWkhYgmCycAK4GN3/7uZnQF8FN+yREQkUWI5WbwQWHjU60+Aa+NZlIiIJE4sJ4u7mdkSM/sy/HjWzLolojgREYm/WA4NzQGeB04LP5aF20REpAWIJQjS3X2Oux8MP+YC6bXNZGbdzWylmb1nZhvNbFyUPmZmD5nZZjMrNbPceqyDiIg0QCxBsMvMfmJmKeHHT4BdMcx3EJjg7n2BHwC3mVnfan3+HegVfowBHq1D7SIi0ghiCYL/Q+jS0c+BncB1QEFtM7n7TndfG37+T2AT0LVat6sI/TbB3X010Cl8qaqIiCRIrUHg7p+6+5Xunu7und39auCYwzzHY2YZQH9gTbVJXYFtR73ezrFhISIicVTfO5SNiLWjmbUndE+D8eFfKNeZmY0xs2IzKy4vL6/PIkREpAb1DQKLqZNZKqEQmO/ui6N02UHoRjdHdAu3VeHuj7l7nrvnpafXep5aRETqoMYflIUHmYs6iRiCwMwMmA1scveaBql7HrjdzJ4GBgIV7r6ztmWLiEjjOd4vi0sAJ/qH/v4Ylj0IGAWsN7N14ba7gR4A7j4TeAH4MbAZ2Evo3gciIpJANQaBu/dsyILdfRW17Dm4uwO3NeR9RESkYep7jkBERFoIBYGISMApCEREAi6mIAgPLXGamfU48oh3YY2psLAQM4s8CgsLk12SiEiTYaHztcfpYPafwH3AF8DhcLO7e06ca4sqLy/Pi4uL6zWvmVHb+kriabs0PdomTVNDtouZlbh7XrRptd6YhtBwEue4eywDzYmISDMTy6GhbUBFvAsREZHkiGWP4BPgVTNbDnx3pPE4vxYWEZFmJJYg2Bp+fC/8EBGRFiSWm9ffn4hCREQkOY436Nx0dx9vZssIjTlUhbtfGdfKREQkIY63R/Bk+M+piShERESS43iDzpWE/3wtceWIiEii1XqOwMx6Ab8D+gJpR9rd/Yw41iUiIgkSy+8I5gCPAgeB/wXMA/5fPIsSEZHEiSUI2rr7XwkNR/GpuxcCl8e3LBERSZRYfkfwnZm1Aj4ys9sJ3VO4fXzLEhGRRIllj2AccAJwBzAA+AlwczyLEhGRxDnuHoGZpQDXu/svgD3onsIiIi1OjXsEZtba3Q8BgxNYj4iIJNjx9gjeBnKBd8zseWAh8O2Rie6+OM61iYhIAsRyjiAN2AVcDAwDrgj/KSIiCVCxbBkfXZwPwEcX51OxbFmjLv94ewSdzez/AhsIjTVkR03TrYtERBKgYtkydv7mXnzfPgAOfvYZO39zLwAdr7iiUd7jeHsEKYQuE20PnHjU8yMPERGJsy+nTY+EwBG+bx9fTpveaO9xvD2Cne4+udHeSURE6uzgzp11aq+P4+0R2HGmiYhIArTu0qVO7fVxvCDIb7R3ERGReul853gsLa1Km6Wl0fnO8Y32HscbhvrrRnsXERGplyMnhL+cNh0+eJ/Wp51G5zvHN9qJYggNJNdoC0uEvLw8Ly4urte8ZkZzW98g0HZperRNmqaGbBczK3H3vGjTYvkdgYiItGAKAhGRgFMQiIgEnIJARCTgFAQiIgGnIBARCbi4BYGZ/cnMvjSzDTVMv8jMKsxsXfhxb7xqERGRmsVyz+L6mgv8EZh3nAGAbNwAAAqlSURBVD5vuLuGtBYRSaK47RG4++uAfp0sItLEJfscwflm9q6ZvWhmmUmuRUQkkOJ5aKg2a4HT3X2Pmf0YWAr0itbRzMYAYwB69OiRuApFRAIgaXsE7v6Nu+8JP38BSDWzU2ro+5i757l7Xnp6ekLrFBFp6ZIWBGZ2qplZ+Pl54Vp2JaseEZGgituhITMrAi4CTjGz7cB9QCqAu88ErgPGmtlBoBIY6RruUEQk4eIWBO5+Qy3T/0jo8lIREUmiZF81JCIiSaYgEBEJOAWBiEjAKQhERAJOQSAiEnAKAhGRgFMQiIgEnIJARCTgFAQiIgGnIBARCTgFgYhIwCkIREQCTkEgIhJwCgIRkYBTEIiIBJyCQEQk4BQEkjylC2BaVuj5tKzQaxFJuLjdoUzkuEoXwLI74EBl6HXFttBrgJwRyatLJIC0RyDJ8dfJ/wqBIw5UhtpFJKEUBJIcFdvr1i4icaMgkOTo2K1u7SISNwoCSY78eyG1bdW21LahdhFJKAWBJEfOCLjiIejYPfS6Y/fQa50oFkk4c/dk11AneXl5XlxcXK95zYzmtr5BoO3S9GibNE0N2S5mVuLuedGmaY9ARCTgFAQiErH8k+VcuuhSAC5ddCnLP1me5IokEfSDMhEBQiFQ+LdC9h3aB8DOb3dS+LdCAC4/4/IkVibxpj0CEQFgxtoZkRA4Yt+hfcxYOyNJFUmiKAhEBIDPv/28Tu3ScigIRASAU9udWqd2aTkUBCICwLjccaSlpFVpS0tJY1zuuCRVJImik8UiAvzrhPCMtTPYwAa6tOvCuNxxOlEcAPpBmSSdtkvTo23SNOkHZSIiEhcKAhGRgItbEJjZn8zsSzPbUMN0M7OHzGyzmZWaWW68ahERkZrFc49gLjD0ONP/HegVfowBHo1jLSIiUoO4BYG7vw58fZwuVwHzPGQ10MnMusSrHhERiS6Z5wi6AtuOer093CYiIgnULE4Wm9kYMys2s+Ly8vJklyMi0qIkMwh2AN2Pet0t3HYMd3/M3fPcPS89PT0hxYmIBEUyg+B54Kfhq4d+AFS4+84k1iMiEkhxG2LCzIqAi4BTzGw7cB+QCuDuM4EXgB8Dm4G9wOh41SIiIjWLWxC4+w21THfgtni9v4iIxKZZnCwWEZH4URCIiAScgkBEJOAUBCIiAacgEBEJOAWBiEjAKQhERAJOQSAiEnAKAhGRgAtEECx9ZweDfv8KAIN+/wpL34k6tp2ISCDFbYiJpmLpOzuYtHg9lQcOAbBjdyWTFq8H4Or+uv2BiEiL3yOYsuKDSAgcUXngEFNWfJCkikREmpYWHwSf7a6sU7uISNC0+CA4rVPbOrWLiARNiw+CiZedQ9vUlCptbVNTmHjZOUmqSESkaWnxJ4uPnBCesuIDPgW6dmrLxMvO0YliEZEwC90fpvnIy8vz4uLies1rZjS39Q0CbZemR9ukaWrIdjGzEnfPizatxR8aEhGR41MQiIgEnIJARCTgFAQiIgGnIBARCTgFgYhIwCkIREQCTkEgIhJwCgIRkYBTEIiIBJyCQEQk4BQEIiIBpyAQEQk4BYGISMApCEREAk5BICIScAoCEZGAUxCIiARcXIPAzIaa2QdmttnMfhVleoGZlZvZuvDjlnjWIyIix4rbzevNLAV4BPgRsB34u5k97+7vVev6jLvfHq86RETk+OK5R3AesNndP3H3/cDTwFVxfD8REamHeAZBV2DbUa+3h9uqu9bMSs1skZl1j2M9IiISRbJPFi8DMtw9B3gJeCJaJzMbY2bFZlZcXl6e0AJFRFq6eAbBDuDob/jdwm0R7r7L3b8Lv3wcGBBtQe7+mLvnuXteenp6XIoVEQmqeAbB34FeZtbTzL4HjASeP7qDmXU56uWVwKY41iMiIlHE7aohdz9oZrcDK4AU4E/uvtHMJgPF7v48cIeZXQkcBL4GCuJVj4iIRGfunuwa6iQvL8+Li4vrNa+Z0dzWNwi0XZoebZOmqSHbxcxK3D0v2rRknywWEZEkUxCIiAScgkBEJOAUBCIiAacgEBEJOAWBiEjAKQhERAJOQSAiEnAKAhGRgFMQSNIUFhZiZkDoF5OFhYXJLUgkoDTEhIgcQ/9XmiYNMSEicae9tGDSHoGISDOhPQIREYkLBYGISMApCEREmrh4n7vROQIRkQAI/DkCXQkhIlKzQO0RiIgEVeD3CEREpGYKAhGRgFMQiIgEnIJARCTgFAQiIgGnIBARCTgFgYhIwCkIREQCTkEgIhJwCgIRkYBTEIiIBJyCQEQk4BQEIiIB1+xGHzWzcuDTes5+CvBVI5aTTFqXpqmlrEtLWQ/QuhxxurunR5vQ7IKgIcysuKZhWJsbrUvT1FLWpaWsB2hdYqFDQyIiAacgEBEJuKAFwWPJLqARaV2appayLi1lPUDrUqtAnSMQEZFjBW2PQEREqmmRQWBmQ83sAzPbbGa/ijK9jZk9E56+xswyEl9lbGJYlwIzKzezdeHHLcmoszZm9icz+9LMNtQw3czsofB6lppZbqJrjFUM63KRmVUctU3uTXSNsTCz7ma20szeM7ONZjYuSp9msV1iXJfmsl3SzOxtM3s3vC73R+nTuJ9h7t6iHkAK8DFwBvA94F2gb7U+Pwdmhp+PBJ5Jdt0NWJcC4I/JrjWGdbkQyAU21DD9x8CLgAE/ANYku+YGrMtFwJ+TXWcM69EFyA0/PxH4MMq/r2axXWJcl+ayXQxoH36eCqwBflCtT6N+hrXEPYLzgM3u/om77weeBq6q1ucq4Inw80VAvplZAmuMVSzr0iy4++vA18fpchUwz0NWA53MrEtiqqubGNalWXD3ne6+Nvz8n8AmoGu1bs1iu8S4Ls1C+O96T/hlavhR/WRuo36GtcQg6ApsO+r1do79BxHp4+4HgQrg5IRUVzexrAvAteHd9kVm1j0xpTW6WNe1uTg/vGv/opllJruY2oQPLfQn9O3zaM1uuxxnXaCZbBczSzGzdcCXwEvuXuN2aYzPsJYYBEGzDMhw9xzgJf71LUGSZy2hn/OfCzwMLE1yPcdlZu2BZ4Hx7v5NsutpiFrWpdlsF3c/5O79gG7AeWaWFc/3a4lBsAM4+ltxt3Bb1D5m1hroCOxKSHV1U+u6uPsud/8u/PJxYECCamtssWy3ZsHdvzmya+/uLwCpZnZKksuKysxSCX1wznf3xVG6NJvtUtu6NKftcoS77wZWAkOrTWrUz7CWGAR/B3qZWU8z+x6hEynPV+vzPHBz+Pl1wCsePuvSxNS6LtWO115J6Nhoc/Q88NPwVSo/ACrcfWeyi6oPMzv1yPFaMzuP0P+zJvdFI1zjbGCTuz9YQ7dmsV1iWZdmtF3SzaxT+Hlb4EfA+9W6NepnWOv6zthUuftBM7sdWEHoqps/uftGM5sMFLv784T+wTxpZpsJnfQbmbyKaxbjutxhZlcCBwmtS0HSCj4OMysidNXGKWa2HbiP0Ekw3H0m8AKhK1Q2A3uB0cmptHYxrMt1wFgzOwhUAiOb6BeNQcAoYH34eDTA3UAPaHbbJZZ1aS7bpQvwhJmlEAqrBe7+53h+humXxSIiAdcSDw2JiEgdKAhERAJOQSAiEnAKAhGRgFMQiIgEnIJARCTgFAQiIgGnIBARCbj/D0FtBPrRk8AXAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Plot for test error"
      ],
      "metadata": {
        "id": "5Zme-i8EvcfK"
      },
      "id": "5Zme-i8EvcfK"
    },
    {
      "cell_type": "code",
      "source": [
        "# Train plot, each index in x will be a different optimizer and y its values\n",
        "fig = plt.figure(figsize=(6, 6))\n",
        "plt.scatter(x=[0], y=[mean_all_test_adam_mlp], label='Adam on MLP')\n",
        "plt.errorbar(x=[0], y=[mean_all_test_adam_mlp], yerr=[[fifth_percentile_test_adam_mlp],[ninety_fifth_percentile_test_adam_mlp]], ecolor='black', capsize=2, elinewidth=1)\n",
        "plt.scatter(x=[1], y=[mean_all_test_nosadam_mlp], label='NosAdam on MLP')\n",
        "plt.errorbar(x=[1], y=[mean_all_test_nosadam_mlp], yerr=[[fifth_percentile_test_nosadam_mlp],[ninety_fifth_percentile_test_nosadam_mlp]], ecolor='black', capsize=2, elinewidth=1)\n",
        "\n",
        "plt.scatter(x=[2], y=[mean_all_test_adam_cnn], label = 'Adam on CNN')\n",
        "plt.errorbar(x=[2], y=[mean_all_test_adam_cnn], yerr=[[fifth_percentile_test_adam_cnn],[ninety_fifth_percentile_test_adam_cnn]], ecolor='black', capsize=2, elinewidth=1)\n",
        "plt.scatter(x=[3], y=[mean_all_test_nosadam_cnn], label = 'NosAdam on CNN')\n",
        "plt.errorbar(x=[3], y=[mean_all_test_nosadam_cnn], yerr=[[fifth_percentile_test_nosadam_cnn],[ninety_fifth_percentile_test_nosadam_cnn]], ecolor='black', capsize=2, elinewidth=1)\n",
        "plt.ylabel('Test Loss', fontsize = 10)\n",
        "plt.legend(loc = 'upper left')\n",
        "plt.savefig('bootsrapping_test.pdf')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        },
        "id": "QxRPgmvCfm-D",
        "outputId": "3e56999f-0be7-4ad5-e219-93d2a1f8e3e2"
      },
      "id": "QxRPgmvCfm-D",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAFlCAYAAADmu++zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3RU9bn/8fdDSImCwFGDIhdDFQWSCRCyoApYf8Yqp6KCehC1aFhezqJawcPBilaNLj2tB5aA6JKKKOLBKCCgFC3VipeoYBPEAIKKErmIilhigSC35/fHDCMJCUxCJpNkf15rzWL2d9+ena3zmX2Z7zZ3R0REgqtJogsQEZHEUhCIiAScgkBEJOAUBCIiAacgEBEJOAWBiEjANU10AdV14oknelpaWqLLEBFpUIqKir5z99TKxjW4IEhLS6OwsDDRZYiINChm9mVV43RqSEQk4BQEIiIBpyAQEQm4BneNoDJ79uxh48aN7Nq1K9GlSJylpKTQvn17kpOTE12KSKPRKIJg48aNHHfccaSlpWFmiS5H4sTd2bp1Kxs3bqRTp06JLkek0WgUp4Z27drFCSecoBBo5MyME044QUd+IrWsUQQBoBAICO1nkdrXaIKgPpg/fz5mxpo1a6qc5txzz613v4PIy8vDzFi7dm20beLEiZhZtNa0tDS+++67cvNNnz6d1NRUevToQbdu3Zg6dWqd1i0itUNBUIvy8/Pp168f+fn5iS6l2kKhEM8//3x0ePbs2aSnpx9xviuvvJLly5fz5ptvcuedd/LNN9/Es0wRiYNABsH8DzfR909v0OmOhfT90xvM/3DTUS9z+/btFBQUMG3atHIfqGVlZQwdOpSuXbsyePBgysrKouNGjBhBdnY26enp3HvvvdH2tLQ0xo4dS48ePcjOzmbZsmVceOGFnHbaaUyZMqXS9T/88MNkZGSQkZHBxIkTASgpKaFr167ceOONpKenc8EFF5Rb/8EGDRrESy+9BMDnn39Oq1atOPHEE2Pe/jZt2nDaaafx5ZdV/nhRROqpwAXB/A83MXbuCjZtK8OBTdvKGDt3xVGHwUsvvcSAAQM444wzOOGEEygqKgLg8ccf59hjj2X16tXcd9990XaABx98kMLCQoqLi3nrrbcoLi6OjuvYsSPLly+nf//+5ObmMmfOHJYsWVIuMA4oKiri6aefZunSpSxZsoSpU6fy4YcfAvDZZ59x8803s2rVKlq3bs2LL75Yaf0tW7akQ4cOrFy5kueff54rr7yyWtv/xRdf8MUXX3D66adXaz4RSbzABcG4RZ9QtmdfubayPfsYt+iTo1pufn4+Q4cOBWDo0KHR00Nvv/02v/nNbwDIzMwkMzMzOs+sWbPIysqiZ8+erFq1io8//jg67pJLLgHCp2z69OnDcccdR2pqKs2aNWPbtm3l1l1QUMDgwYNp3rw5LVq04LLLLuOdd94BoFOnTvTo0QOAXr16UVJSUuU2DB06lOeff5758+czePDgmLb7hRdeoEePHlx11VX8+c9/5vjjj49pPhGpPwIXBF9tq/zUSFXtsfj+++954403uOGGG0hLS2PcuHHMmjULd69ynnXr1jF+/Hj+/ve/U1xczEUXXVTutshmzZoB0KRJk+j7A8N79+6NubaD501KSjrsvAMHDuTZZ5+lY8eOtGzZMqblH7hGsHTp0pjDQ0Sq58ANHQdeeXl5tbr8wAXBKa2PqVZ7LObMmcOwYcP48ssvKSkpYcOGDXTq1Il33nmHc845h+eeew6AlStXRk///PDDDzRv3pxWrVrxzTff8Oqrr9Z4/f3792f+/Pns3LmTHTt2MG/ePPr371/t5Rx77LE89NBD3HXXXTWuRURqX15eXvSLpbvXehA0il8WV8eYC89k7NwV5U4PHZOcxJgLz6zxMvPz8/n9739fru3yyy8nPz+fhx9+mOHDh9O1a1e6du1Kr169AOjevTs9e/akS5cudOjQgb59+9Z4/VlZWeTm5tK7d28AbrjhBnr27HnY00BVOXB6qzKZmZk0aRL+7jBkyJByp7lEpOGyw52+qI+ys7O94n34q1evpmvXrjEvY/6Hmxi36BO+2lbGKa2PYcyFZzKoZ7vaLlXipLr7W6SxMLPDnnI+wrxF7p5d2bjAHREADOrZTh/8IiIRgbtGICIi5SkIREQCTkEgIhJwCgIRkYBTEIiIBFzcgsDMUszsAzP7yMxWmdl9lUyTa2ZbzGx55HVDvOqJNzNj9OjR0eHx48cf1Y8+Ro0aRbt27di/f3+V07Ro0aLGy4+Xc889l44dO5a7xW3QoEHRWktKSsjIyDhkvtzc3Gh3GFlZWbz//vt1VrNI0MXziOBH4Dx37w70AAaY2S8qme4Fd+8ReT0Zx3riqlmzZsydO/eQPvtrYv/+/cybN48OHTrw1ltv1UJ1dat169a8++67AGzbto3NmzfHNN+4ceNYvnw5f/rTn/jP//zPeJYoIgeJWxB42PbIYHLkVT9+vVY8CyZkQF7r8L/Fs456kU2bNuWmm25iwoQJh4wrKSnhvPPOIzMzk5ycHNavXw+E+/zPyMige/funHPOOdHp33zzTdLT0xkxYkS5ZxusW7eOs846i1AoxB/+8Ido+/bt28nJySErK4tQKBTtTrqkpIQuXbqQm5vLGWecwTXXXMPrr79O37596dy5Mx988MEhte7atYvhw4cTCoXo2bMnixcvBsIPobnssssYMGAAnTt35vbbb6/yb3Gg8zqAuXPnctlll1XnT8k555xT7iE5IhJn7h63F5AELAe2Aw9VMj4X2AwUA3OADlUs5yagECjs2LGjV/Txxx8f0lalj15wf+Ak93tb/vR64KRw+1Fo3ry5l5aW+qmnnurbtm3zcePG+b333uvu7gMHDvTp06e7u/u0adP80ksvdXf3jIwM37hxo7u7//Of/4wu64YbbvAZM2Z4aWmpn3LKKb579253d7/44ov9mWeecXf3Rx991Js3b+7u7nv27PHS0lJ3d9+yZYufdtppvn//fl+3bp0nJSV5cXGx79u3z7Oysnz48OG+f/9+nz9/frSOg40fP96HDx/u7u6rV6/2Dh06eFlZmT/99NPeqVMn37Ztm5eVlXnHjh19/fr1h8z/y1/+0pcsWeKhUMj37t3rv/rVr3zdunXRWtetW+fp6emHzHfdddf57Nmz3d191qxZ3rt37yr/1tXa3yKNCNHv2DWat9Cr+KyO68Vid9/n7j2A9kBvM6t4cngBkObumcBrwDNVLOcJd8929+zU1NSjK+rv98OeCj2N7ikLtx+lli1bcu211/LII4+Ua3///fe5+uqrARg2bBgFBQUA9O3bl9zcXKZOncq+feG+j3bv3s0rr7zCoEGDaNmyJX369GHRokUAvPvuu1x11VXR5Rzg7tx5551kZmZy/vnns2nTpuiTwjp16kQoFKJJkyakp6eTk5ODmREKhSrti6igoCDabXaXLl049dRT+fTTTwHIycmhVatWpKSk0K1btyofQpOUlES/fv14/vnnKSsrIy0tLaa/35gxY+jRowdPPPEE06ZNi2keETl6ddLFhLtvM7PFwABg5UHtWw+a7Engf+NeTOnG6rVX06hRo8jKymL48OFHnHbKlCksXbqUhQsX0qtXL4qKinjvvffYtm0boVAIgJ07d3LMMccwcOBAoPKHt8+cOZMtW7ZQVFREcnIyaWlp0S6tK3ZhfXD31tXpzrriso7UpfXQoUMZPHhwtS6Yjxs3jiuuuKJaNYnI0YvnXUOpZtY68v4Y4FfAmgrTtD1o8BJgdbzqiWrVvnrt1XT88cczZMiQct9ozz777Og585kzZ0a7iP7888/p06cP999/P6mpqWzYsIH8/HyefPJJSkpKKCkpYd26dbz22mvs3LmTvn37llvOAaWlpbRp04bk5GQWL158VI+L7N+/f3TZn376KevXr+fMM6vfM2v//v0ZO3Zs9AhGROqveJ4aagssNrNi4B/Aa+7+FzO738wuiUxza+TW0o+AWwlfM4ivnHsgucKzB5KPCbfXktGjR5e7e2jy5Mk8/fTTZGZm8uyzzzJp0iQgfCokFAqRkZHB2WefTefOnfnrX//KRRddFJ23efPm9OvXjwULFjBp0iQee+wxQqEQmzb99GjNa665hsLCQkKhEDNmzKBLly41rv23v/0t+/fvJxQKceWVVzJ9+vRyRwKxMjP++7//u9LnHn/yySe0b98++po9e3aN6xWRoxfIbqgpnhW+JlC6MXwkkHMPZA6p5UolXtQNtQSVuqGuTZlD9MEvIhKhLiZERAJOQSAiEnAKAhGRgFMQiIgEnIJARCTgFAS1aP78+ZgZa9asqXKac889l4q3v9YHM2bMICMjI9rZ3Pjx44Fw99Dt2rXjxx9/BOC7776LdhlRUlKCmTF58uTocm655RamT59e1+WLyFFQENSi/Px8+vXrV67H0Ibg1VdfZeLEifztb39jxYoVLFmyhFatWkXHJyUl8dRTT1U6b5s2bZg0aRK7d++uq3JFpJYFMggWfrGQC+ZcQOYzmVww5wIWfrHwqJe5fft2CgoKmDZtWrQbCICysjKGDh1K165dGTx4MGVlP3V4N2LECLKzs0lPT+fee++NtqelpTF27Fh69OhBdnY2y5Yt48ILL+S0005jypQpla7/4YcfJiMjg4yMDCZOnAiEv7F37dqVG2+8kfT0dC644IJy6z/gj3/8I+PHj+eUU04Bwn0K3XjjjdHxo0aNYsKECZX2LZSamkpOTg7PPFNpf4Ei0gAELggWfrGQvPfy2LxjM46zecdm8t7LO+oweOmllxgwYABnnHEGJ5xwAkVFRQA8/vjjHHvssaxevZr77rsv2g7w4IMPUlhYSHFxMW+99RbFxcXRcR07dmT58uX079+f3Nxc5syZw5IlS8oFxgFFRUU8/fTTLF26lCVLljB16lQ+/PBDAD777DNuvvlmVq1aRevWrXnxxRcPmX/lypX06tWrym3r2LEj/fr149lnn610/O9//3vGjx8f7UFVRBqWwAXBpGWT2LVvV7m2Xft2MWnZpKNabn5+PkOHDgXCPW8eOD309ttvR7t1zszMJDMzMzrPrFmzyMrKomfPnqxatYqPP/44Ou6SS8LdMYVCIfr06cNxxx1HamoqzZo1Y9u2beXWXVBQwODBg2nevDktWrTgsssu45133gGIPv4RoFevXpV2PR2LsWPHMm7cuEofnfnzn/+cPn368Nxzz9Vo2SKSWIHrYuLrHV9Xqz0W33//PW+88QYrVqzAzNi3bx9mxrhx46qcZ926dYwfP55//OMf/Nu//Ru5ubnRrqOBct1FV+xKujrdR1fsOrqyU0Pp6ekUFRVx3nnnVbmczp0706NHD2bNqvxpbnfeeSdXXHEFv/zlL2OuTUTqh8AdEZzc/ORqtcdizpw5DBs2jC+//JKSkhI2bNhAp06deOeddzjnnHOi35RXrlwZPf3zww8/0Lx5c1q1asU333zDq6++WuP19+/fn/nz57Nz50527NjBvHnzol1dx2Ls2LGMGTOGr78Oh+Hu3bt58slDHx991113Re8mqqhLly5069aNBQsW1GwjRCRhAhcEI7NGkpKUUq4tJSmFkVkja7zM/Px8Bg8eXK7t8ssvJz8/nxEjRrB9+3a6du3KPffcEz0X3717d3r27EmXLl24+uqr6du3b43Xn5WVRW5uLr1796ZPnz7ccMMN9OzZM+b5f/3rX3PLLbdw/vnnk56eTlZWFj/88MMh0x0YV5W77rqLjRtr5wE/IlJ3AtkN9cIvFjJp2SS+3vE1Jzc/mZFZI7no5xcdeUapF9QNtQSVuqGuRRf9/CJ98IuIRATu1JCIiJSnIBARCTgFgYhIwCkIREQCTkEgIhJwCoJaYmaMHj06Ojx+/Hjy8vJqvLxRo0bRrl27Srt0OKBFixY1Xn687NmzhzvuuIPOnTuTlZXFWWedFf2xXFpaGpdffnl02jlz5pCbmwvA9OnTadKkSbn+ljIyMmrcJYaIxE5BUEuaNWvG3Llz+e677456Wfv372fevHl06NCBt956qxaqqzt33303mzdvZuXKlSxbtoz58+fzr3/9Kzq+qKioXJ9KB2vfvj0PPvhgXZUqIhGBDILSBQv47LwcVnftxmfn5VBaC90iNG3alJtuuokJEyYcMq6kpITzzjuPzMxMcnJyWL9+PQCzZ88mIyOD7t27c84550Snf/PNN0lPT2fEiBHlnm2wbt06zjrrLEKhEH/4wx+i7du3bycnJ4esrCxCoRAvvfRSdL1dunQhNzeXM844g2uuuYbXX3+dvn370rlzZz744INDat21axfDhw+PPqBm8eLFQPgb+2WXXcaAAQPo3Lkzt99++yHz7ty5k6lTpzJ58uRoH0cnnXQSQ4YMiU4zevToKj/sBw4cyKpVq/jkk0+q/kOLSK0LXBCULljA5rvvYe9XX4E7e7/6is1331MrYXDzzTczc+ZMSktLy7X/7ne/47rrrqO4uJhrrrmGW2+9FYD777+fRYsW8dFHH/Hyyy9Hp8/Pz+eqq65i8ODBLFy4kD179gAwcuRIRowYwYoVK2jbtm10+pSUFObNm8eyZctYvHgxo0ePjv76cO3atYwePZo1a9awZs0annvuOQoKChg/fjz/8z//c8g2PPbYY5gZK1asID8/n+uuuy7aGd7y5ct54YUXWLFiBS+88AIbNmwoN+/atWvp2LEjLVu2rPJvNGTIEJYtW8batWsPGdekSRNuv/32SusSkfgJXBB8O2Eivqt8N9S+axffTph41Mtu2bIl1157LY888ki59vfff5+rr74agGHDhlFQUABA3759yc3NZerUqdG+/Hfv3s0rr7zCoEGDaNmyJX369GHRokUAvPvuu1x11VXR5UTrd+fOO+8kMzOT888/n02bNvHNN98A4W6oQ6EQTZo0IT09nZycHMyMUChU6fn3goKCaLfZXbp04dRTT+XTTz8FICcnh1atWpGSkkK3bt348ssvq/03SkpKYsyYMfzxj3+sdPzVV1/NkiVLWLduXbWXLSI1E7gg2Lt5c7Xaq2vUqFFMmzaNHTt2HHHaKVOm8MADD7BhwwZ69erF1q1bWbRoEdu2bSMUCpGWlkZBQUG500NmdshyZs6cyZYtWygqKmL58uWcdNJJ0W/xFbuwPrh76+p0Z11xWUlJSYfMf/rpp7N+/fpKO6w72LBhw3j77bcPOaKA8Cm20aNH89BDD1WrNhGpucAFQdODTqnE0l5dxx9/PEOGDGHatGnRtrPPPjv6+MqZM2dGu4j+/PPP6dOnD/fffz+pqals2LCB/Px8nnzySUpKSigpKWHdunW89tpr7Ny5k759+5ZbzgGlpaW0adOG5ORkFi9eXKNv6gf0798/uuxPP/2U9evXc+aZZ8Y077HHHsv111/PyJEjo88w3rJlC7Nnzy43XXJyMrfddlul11MAcnNzef3119myZUuNt0NEYhe4IGhz2ygspXw31JaSQpvbRtXaOkaPHl3u7qHJkyfz9NNPk5mZybPPPsukSeGnoY0ZM4ZQKERGRgZnn302nTt35q9//SsXXfRTh3jNmzenX79+LFiwgEmTJvHYY48RCoXYtGlTdJprrrmGwsJCQqEQM2bMoEuXLjWu/be//S379+8nFApx5ZVXMn369HJHAkfywAMPkJqaSrdu3cjIyGDgwIGVXjO4/vrrqzwi+dnPfsatt97Kt99+W+PtEJHYBbIb6tIFC/h2wkT2bt5M07ZtaXPbKFpdfHFtlypxom6oJajUDXUtanXxxfrgFxGJCNypIRERKU9BICIScI0mCBratQ6pGe1nkdoXtyAwsxQz+8DMPjKzVWZ2XyXTNDOzF8xsrZktNbO0mqwrJSWFrVu36kOikXN3tm7dSkqFu75E5OjE82Lxj8B57r7dzJKBAjN71d2XHDTN9cA/3f10MxsKPARcWd0VtW/fno0bN+q+8wBISUmhffv2iS5DpFGJWxB4+Ov59shgcuRV8Sv7pUBe5P0c4FEzM6/mV/vk5GQ6dep0FNWKiARXXK8RmFmSmS0HvgVec/elFSZpB2wAcPe9QClwQjxrEhGR8uIaBO6+z917AO2B3maWUZPlmNlNZlZoZoU6/SMiUrvq5K4hd98GLAYGVBi1CegAYGZNgVbA1krmf8Lds909OzU1Nd7liogESjzvGko1s9aR98cAvwLWVJjsZeC6yPsrgDeqe31ARESOTjzvGmoLPGNmSYQDZ5a7/8XM7gcK3f1lYBrwrJmtBb4HhsaxHhERqUQ87xoqBnpW0n7PQe93Af8RrxpEROTIGs0vi0VEpGYUBCIiAacgEBEJOAWBiEjAKQhERAJOQSAiEnAKAhGRgFMQSMLk5eVhZtFXXl5eoksSCSRraD06ZGdne2FhYaLLkFpkZnqokEgMjub/FTMrcvfsysbpiEBEJOAUBCISpdN1waRTQ5JwOjVU/2if1E86NSQiInGhIBARCTgFgYhIwCkIREQCTkEgIhJwCgIRkYBTEIiIBJyCQEQk4BQEIiIBpyAQEQk4BYGISMApCEREAk5BICIScAoCEZGAUxCIiAScgkBEJOAUBCIiAacgEBEJOAWBiEjAKQhERAJOQSAiEnAKAhGRgFMQiIgEXNyCwMw6mNliM/vYzFaZ2chKpjnXzErNbHnkdU+86hERkco1jeOy9wKj3X2ZmR0HFJnZa+7+cYXp3nH3gXGsQ0REDiNuRwTuvtndl0Xe/wtYDbSL1/pERKRm6uQagZmlAT2BpZWMPsvMPjKzV80svYr5bzKzQjMr3LJlSxwrFREJnrgHgZm1AF4ERrn7DxVGLwNOdffuwGRgfmXLcPcn3D3b3bNTU1PjW7CISMDENQjMLJlwCMx097kVx7v7D+6+PfL+FSDZzE6MZ00iIlJePO8aMmAasNrdH65impMj02FmvSP1bI1XTSIicqh43jXUFxgGrDCz5ZG2O4GOAO4+BbgCGGFme4EyYKi7exxrEhGRCuJ511CBu5u7Z7p7j8jrFXefEgkB3P1Rd0939+7u/gt3fy8eteTl5WFm0VdeXl48ViMi0iBZQ/sCnp2d7YWFhTWa18xoaNsbBNov9Y/2Sf10NPvFzIrcPbuycepiQkQk4BQEIiIBpyAQEQk4BYGISMApCEREAk5BICIScAoCEZGAUxCIiAScgkBEJOAUBCIiAacgEBEJOAWBiEjAKQhERAJOQSAiEnAKAhGRgFMQiIgEnIJARCTgFAQiIgGnIBARCTgFgYhIwCkIREQCTkEgIhJwRwwCM/tfM2tpZslm9ncz22Jmv6mL4kREJP5iOSK4wN1/AAYCJcDpwJh4FiUiInUnliBoGvn3ImC2u5fGsR4REaljTY88CX8xszVAGTDCzFKBXfEtS0RE6soRjwjc/Q7gbCDb3fcAO4BL412YiIjUjVguFv8HsMfd95nZH4D/A06Je2UiIlInYrlGcLe7/8vM+gHnA9OAx+NbloiI1JVYgmBf5N+LgCfcfSHws/iVJCIidSmWINhkZn8GrgReMbNmMc4nIiINQCwf6EOARcCF7r4NOB79jkBEpNGI5a6hncDnwIVmdgvQxt3/FvfKRESkTsRy19BIYCbQJvL6PzP7XQzzdTCzxWb2sZmtiiyn4jRmZo+Y2VozKzazrJpshIiI1FwsPyi7Hujj7jsAzOwh4H1g8hHm2wuMdvdlZnYcUGRmr7n7xwdN8+9A58irD+G7kfpUcxtEROQoxHKNwPjpziEi7+1IM7n7ZndfFnn/L2A10K7CZJcCMzxsCdDazNrGVLmIiNSKWI4IngaWmtm8yPAgwr8liJmZpQE9gaUVRrUDNhw0vDHStrk6yxcRkZo7YhC4+8Nm9ibQL9I0HPgm1hWYWQvgRWBUpBfTajOzm4CbADp27FiTRYiISBViOSIgcopn2YFhM1sPHPET2cySCYfATHefW8kkm4AOBw23j7RVXP8TwBMA2dnZHkvNIiISm5r+MOyI1wjMzAifQlrt7g9XMdnLwLWRu4d+AZS6u04LiYjUoZiOCCoRy7fyvsAwYIWZLY+03UnkSMLdpwCvAL8G1gI7CZ92EhGROlRlEJjZZCr/wDeg9ZEW7O4FHOHIwd0duPlIyxIRkfg53BFBYQ3HiYhIA1JlELj7M3VZiIiIJIZ6ERURCTgFgYhIwMXS6VzfWNpERKRhiuWIoLLO5Y7U4ZyIiDQQh7t99CzgbCDVzP7roFEtgaR4FyYiInXjcLeP/gxoEZnmuIPafwCuiGdRIiJSdw53++hbwFtmNt3dvwQwsyZAi5p2HiciIvVPLNcI/mhmLc2sObAS+NjM9MxiEZFGIpYg6BY5AhgEvAp0ItyHkIiINAKxBEFypDvpQcDL7r6H2DqdExGRBiCWIPgzUAI0B942s1MJXzAWEZFGIJYnlD0CPHJQ05dm9v/iV5KIiNSlWH5ZfJKZTTOzVyPD3YDr4l6ZiIjUiVhODU0HFgGnRIY/BUbFqyAREalbVQaBmR04bXSiu88C9gO4+15gXx3UJiIideBwRwQfRP7dYWYnELlT6MCzheNdmIiI1I3DBcGBx0z+F+GHzJ9mZu8CM4DfxbswEREJK12wgM/OywHgs/NyKF2woFaXf7i7hg7ubG4e4QfNG/AjcD5QXKuViIjIIUoXLGDz3ffgu3YBsPerr9h89z0AtLr44lpZx+GOCJIIdzp3HOHfEDSNtB1L+U7oREQkTr6dMDEaAgf4rl18O2Fira3jcEcEm939/lpbk4iIVNvezZur1V4TsVwjEBGRBGnatm212mvicEGQU2trERGRGmlz2ygsJaVcm6Wk0Oa22vs51+GeR/B9ra1FRERq5MAF4W8nTIRP1tD0lFNoc9uoWrtQDGDuDasj0ezsbC8sLKzRvGZGQ9veINB+qX+0T+qno9kvZlbk7tmVjYuliwkREWnEFAQiIgGnIBARCTgFgYhIwCkIREQCTkEgIhJwCgIRkYBTEIiIBJyCQEQk4OIWBGb2lJl9a2Yrqxh/rpmVmtnyyOueeNUiIiJVO1w31EdrOvAo4SeaVeUddx8YxxpEROQI4nZE4O5vA+q4TkSknkv0NYKzzOwjM3vVzKOXCjkAAApxSURBVNKrmsjMbjKzQjMr3LJlS13WJyLS6CUyCJYBp7p7d2AyML+qCd39CXfPdvfs1NTUOitQRCQIEhYE7v6Du2+PvH8FSDazExNVj4hIUCUsCMzsZDOzyPvekVq2JqoeEZGgittdQ2aWD5wLnGhmG4F7gWQAd58CXAGMMLO9QBkw1PUkDBGROhe3IHD3q44w/lHCt5eKiEgCJfquIRERSTAFgYhIwCkIREQCTkEgIhJwCgIRkYBTEIiIBJyCQEQk4BQEIiIBpyAQEQk4BYGISMApCCRximfBhIzw+wkZ4WERqXPxfFSlSNWKZ8GCW2FPWXi4dEN4GCBzSOLqEgkgHRFIYvz9/p9C4IA9ZeF2SZiFXyzkgjkXAHDBnAtY+MXCBFckdUFHBJIYpRur1y5xt/CLheS9l8eufbsA2LxjM3nv5QFw0c8vSmBlEm86IpDEaNW+eu0Sd5OWTYqGwAG79u1i0rJJCapI6oqCQBIj5x5IPqZ8W/Ix4XZJiK93fF2tdmk8FASSGJlD4OJHoFWH8HCrDuFhXShOmJObn1ytdmk8FASSOJlD4LaV4fe3rVQIJNjIrJGkJKWUa0tJSmFk1sgEVSR1RReLRQT46YLwpGWTWMlK2jZvy8iskbpQHADW0J4Xn52d7YWFhTWa18xoaNsbBNov9Y/2Sf10NPvFzIrcPbuycTo1JCIScAoCEZGAUxCIiAScgkBEJOAUBCIiAacgEBEJOAWBiEjAKQhERAJOQSAiEnAKAhGRgAtEEMz/cBN9//QGAH3/9AbzP9yU4IpEROqPRt/p3PwPNzF27grK9uwDYNO2MsbOXQHAoJ7tElmaiEi90OiPCMYt+iQaAgeU7dnHuEWfJKgiEZH6pdEHwVfbyqrVLiISNHELAjN7ysy+NbOVVYw3M3vEzNaaWbGZZcWjjlNaH1OtdhGRoInnEcF0YMBhxv870Dnyugl4PB5FjLnwTI5JTirXdkxyEmMuPDMeqxMRaXDidrHY3d82s7TDTHIpMMPDT1lYYmatzaytu2+uzToOXBAet+gTvgTatT6GMReeqQvFIiIRibxrqB2w4aDhjZG2Q4LAzG4ifNRAx44dq72iQT3bMahnO2wsvHvHeTWrVkSkkWoQF4vd/Ql3z3b37NTU1ESXIyLSqCQyCDYBHQ4abh9pExGROpTIIHgZuDZy99AvgNLavj4gIiJHFrdrBGaWD5wLnGhmG4F7gWQAd58CvAL8GlgL7ASGx6sWERGpWjzvGrrqCOMduDle6xcRkdg0iIvFIiISPwoCEZGAUxCIiAScgkBEJOAUBCIiAacgEBEJOAWBiEjAKQhERAJOQSAiEnAKAhGRgFMQiIgEnIJARCTgFAQiIgGnIBARCTgFgYhIwCkIREQCTkEgIhJwCgIRkYBTEIiIBJyCQEQk4BQEIiIBpyAQEQk4BYGISMApCEREAk5BICIScAoCEZGAUxCIiAScgkBEJOAUBCIiAacgEBEJOAWBiEjAKQhERAJOQSAiEnAKAhGRgItrEJjZADP7xMzWmtkdlYzPNbMtZrY88rohnvWIiMihmsZrwWaWBDwG/ArYCPzDzF52948rTPqCu98SrzpEROTw4nlE0BtY6+5fuPtu4Hng0jiuT0REaiCeQdAO2HDQ8MZIW0WXm1mxmc0xsw5xrEdERCqR6IvFC4A0d88EXgOeqWwiM7vJzArNrHDLli11WqCISGMXzyDYBBz8Db99pC3K3be6+4+RwSeBXpUtyN2fcPdsd89OTU2NS7EiIkEVzyD4B9DZzDqZ2c+AocDLB09gZm0PGrwEWB3HekREpBJxu2vI3fea2S3AIiAJeMrdV5nZ/UChu78M3GpmlwB7ge+B3HjVIyIilTN3T3QN1ZKdne2FhYU1mtfMaGjbGwTaL/WP9kn9dDT7xcyK3D27snGJvlgsIiIJpiAQEQk4BYGISMApCEREAk5BICIScAoCEZGAUxCIiAScgkBEJOAUBCIiAacgEBEJOAWBiEjAKQhERAJOQSAJk5eXh5kB4c608vLyEluQSECp91EROYT+X6mf1PuoiMSdjtKCSUcEIiINhI4IREQCKt5HajoiEBEJgMAfEei8p4hI1QJ1RCAiElSBPyIQEZGqKQhERAJOQSAiEnAKAhGRgFMQiIgEnIJARCTgFAQiIgGnIBARCTgFgYhIwCkIREQCTkEgIhJwCgIRkYBTEIiIBFyD633UzLYAX9Zw9hOB72qxnETSttRPjWVbGst2gLblgFPdPbWyEQ0uCI6GmRVW1Q1rQ6NtqZ8ay7Y0lu0AbUssdGpIRCTgFAQiIgEXtCB4ItEF1CJtS/3UWLalsWwHaFuOKFDXCERE5FBBOyIQEZEKGmUQmNkAM/vEzNaa2R2VjG9mZi9Exi81s7S6rzI2MWxLrpltMbPlkdcNiajzSMzsKTP71sxWVjHezOyRyHYWm1lWXdcYqxi25VwzKz1on9xT1zXGwsw6mNliM/vYzFaZ2chKpmkQ+yXGbWko+yXFzD4ws48i23JfJdPU7meYuzeqF5AEfA78HPgZ8BHQrcI0vwWmRN4PBV5IdN1HsS25wKOJrjWGbTkHyAJWVjH+18CrgAG/AJYmuuaj2JZzgb8kus4YtqMtkBV5fxzwaSX/fTWI/RLjtjSU/WJAi8j7ZGAp8IsK09TqZ1hjPCLoDax19y/cfTfwPHBphWkuBZ6JvJ8D5JiZ1WGNsYplWxoEd38b+P4wk1wKzPCwJUBrM2tbN9VVTwzb0iC4+2Z3XxZ5/y9gNdCuwmQNYr/EuC0NQuRvvT0ymBx5VbyYW6ufYY0xCNoBGw4a3sih/0FEp3H3vUApcEKdVFc9sWwLwOWRw/Y5ZtahbkqrdbFua0NxVuTQ/lUzS090MUcSObXQk/C3z4M1uP1ymG2BBrJfzCzJzJYD3wKvuXuV+6U2PsMaYxAEzQIgzd0zgdf46VuCJM4ywj/n7w5MBuYnuJ7DMrMWwIvAKHf/IdH1HI0jbEuD2S/uvs/dewDtgd5mlhHP9TXGINgEHPytuH2krdJpzKwp0ArYWifVVc8Rt8Xdt7r7j5HBJ4FedVRbbYtlvzUI7v7DgUN7d38FSDazExNcVqXMLJnwB+dMd59bySQNZr8caVsa0n45wN23AYuBARVG1epnWGMMgn8Anc2sk5n9jPCFlJcrTPMycF3k/RXAGx656lLPHHFbKpyvvYTwudGG6GXg2shdKr8ASt19c6KLqgkzO/nA+Voz6034/7N690UjUuM0YLW7P1zFZA1iv8SyLQ1ov6SaWevI+2OAXwFrKkxWq59hTWs6Y33l7nvN7BZgEeG7bp5y91Vmdj9Q6O4vE/4P5lkzW0v4ot/QxFVctRi35VYzuwTYS3hbchNW8GGYWT7huzZONLONwL2EL4Lh7lOAVwjfobIW2AkMT0ylRxbDtlwBjDCzvUAZMLSeftHoCwwDVkTORwPcCXSEBrdfYtmWhrJf2gLPmFkS4bCa5e5/iednmH5ZLCIScI3x1JCIiFSDgkBEJOAUBCIiAacgEBEJOAWBiEjAKQhERAJOQSAiEnAKAhGRgPv/wx/LrNHG6V4AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    }
  ],
  "metadata": {
    "interpreter": {
      "hash": "a31065c2cb285fc89af45a6e0e00b20f2f53e470cc5659f0bf6d768d5279787d"
    },
    "kernelspec": {
      "display_name": "Python 3.9.2 ('knvenv')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.2"
    },
    "colab": {
      "name": "ADAM.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 5
}