Chapter01.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Learning with PyTorch Step-by-Step: A Beginner's Guide"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    import google.colab\n",
    "    import requests\n",
    "    url = 'https://raw.githubusercontent.com/dvgodoy/PyTorchStepByStep/master/config.py'\n",
    "    r = requests.get(url, allow_redirects=True)\n",
    "    open('config.py', 'wb').write(r.content)    \n",
    "except ModuleNotFoundError:\n",
    "    pass\n",
    "\n",
    "from config import *\n",
    "config_chapter1()\n",
    "# This is needed to render the plots in this chapter\n",
    "from plots.chapter1 import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "from torchviz import make_dot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A Simple Regression Problem\n",
    "\n",
    "$$\n",
    "\\Large y = b + w x + \\epsilon\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Generation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Synthetic Data Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "true_b = 1\n",
    "true_w = 2\n",
    "N = 100\n",
    "\n",
    "# Data Generation\n",
    "np.random.seed(42)\n",
    "x = np.random.rand(N, 1)\n",
    "epsilon = (.1 * np.random.randn(N, 1))\n",
    "y = true_b + true_w * x + epsilon"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cell 1.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shuffles the indices\n",
    "idx = np.arange(N)\n",
    "np.random.shuffle(idx)\n",
    "\n",
    "# Uses first 80 random indices for train\n",
    "train_idx = idx[:int(N*.8)]\n",
    "# Uses the remaining indices for validation\n",
    "val_idx = idx[int(N*.8):]\n",
    "\n",
    "# Generates train and validation sets\n",
    "x_train, y_train = x[train_idx], y[train_idx]\n",
    "x_val, y_val = x[val_idx], y[val_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<Figure size 864x432 with 2 Axes>,\n",
       " array([<matplotlib.axes._subplots.AxesSubplot object at 0x7f39a45dba90>,\n",
       "        <matplotlib.axes._subplots.AxesSubplot object at 0x7f396e70a290>],\n",
       "       dtype=object))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA08AAAGgCAYAAABys1xkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeXiU1f3//1cWQgIEAhiYgCQExAUUFSwIGJFFFkWISoqCRduKla2tn2oRG7+KRgFxLWhEqK1UcEGQArIoCsiqBqwoWEhLHAQTCAFCCAlZf3/kN2Mmc89kksye5+O6uC5zzz33feZkMsf3vM95n5AzZ85UCgAAAADgVKivGwAAAAAAgYDgCQAAAABcQPAEAAAAAC4geAIAAAAAFxA8AQAAAIALCJ4AAAAAwAUET4CXbdu2TTExMZo8ebKvm4IG6NKli/r16+frZgBoBBg3Akf37t117bXX2hxbsmSJYmJi9N5777l8nQceeEAxMTHatWuXu5toY8SIEWrbtq1H7xFsCJ4aiaysLKWmpmrgwIHq3LmzLrroIiUkJGjQoEGaOXOm9u7d6+sm+lRMTIyuuuoqXzfDkNlsVkxMjM2/9u3b65JLLtHgwYP10EMPacuWLaqsdM+WbUuXLlVMTIxmz57tluu501VXXWXXF87++eNrAAIF44ZzjBs/8+dx4+mnn1ZMTIwee+yxWs9NS0tTTEyMUlNTvdAy7xgxYoRiYmJ07NgxXzclaIT7ugHwvBdffFHPPPOMysvL1bNnT91xxx1q3bq1CgoKdODAAb355ptKT0/XrFmz9Ic//MHXzYUDLVu2tH7rWF5erjNnzuj777/XP//5T/39739X3759tXDhQnXu3Nm3DfWgyZMnKz8/3+bYRx99pO+++0633HKL3f/I3HDDDR5ry6ZNmxQayvdPCE6MG8GBcUOaOHGiXnrpJb377rt64okn1LRpU8PzysvLtWzZMknSfffd59Y2jBkzRtdff71MJpNbr+sOixYtUnFxsa+bEVAInoLcSy+9pKeeekoXX3yxFi9erOuvv97unFOnTun1119XQUGBD1oIV7Vq1UozZ860O56Tk6OHH35Ya9eu1ejRo7V58+agTcFPmTLF7tiRI0f03Xff6dZbb9WECRO81pYuXbp47V6ANzFuBA/GDVmzpZ9++qnWrl2rO++80/C8jRs36qefftINN9ygSy65xK1taNWqlVq1auXWa7pLp06dfN2EgMPXpkHMbDbr2WefVUREhJYvX244AEpSmzZt9Nhjj+nRRx+1e6yiokJLlizR8OHDFR8fr/bt26tfv3568cUXVVJSYne+ZRrD+fPn9fjjj+vKK69Uu3btdO211+qll15yOEXg3//+t37zm9/o8ssvV2xsrC677DI98MADOnz4sN25kydPVkxMjLZt26alS5dq4MCB6tChgzXLUFJSojfeeENjx4613j8hIUGjR4/Wxo0bba5lmUcuST/++KPNFIeac8sPHz6s6dOnW6/ZtWtXTZgwQf/+978NX9OJEyc0bdo0devWTSaTSTfccIOWLl1qeG5DmEwmvfXWWxowYICOHDmiF1980ebx//73v3ryySd10003qWvXrmrXrp2uvPJK/f73v9ePP/5oc+7kyZM1depUSdLcuXNt+mPbtm2SpPz8fL3yyisaNWqUrrjiCsXGxqpr166666679MUXX7j99bnDxIkTFRMTo6+//lp///vfdcMNN8hkMmnEiBGSpKKiIr322mu6/fbb1aNHD7Vr106JiYm64447tHnzZsNrGq15euONNxQTE6P58+drz549uuOOOxQfH6+OHTvqtttua/TTnOD/GDcYN6TgGzcsmaS33nrL4TmWx379619bj124cEELFy7UnXfeaf0ddu7cWcnJyfrkk09cvr+zNU+ffvqphg8frg4dOqhz586aMGGCMjMzHV5r9erVuv/++9WrVy916NBBHTt21MCBA/X666+roqLCel5ZWZliYmK0e/duSVKPHj2sv5fqa7IcrXmqqKjQm2++qcGDB+viiy9Whw4dlJSUpAULFqi0tNTu/O7du6tt27YqLS3VvHnzdO2111rfN0888YTh336gIvMUxJYuXarS0lKlpKToiiuuqPX88HDbt0NZWZnuuecebdiwQZdcconuvPNONW3aVDt27NBTTz2lrVu3asWKFYbPu+OOO5STk6OhQ4cqPDxcH330kWbNmqWioiK7ecfvv/++pkyZooiICI0cOVIdO3bU4cOHtWLFCm3YsEFr165Vz5497do7f/58ff755xo5cqRuuukmXbhwQZJ0+vRpPfroo+rbt68GDRqkiy66SDk5OVq3bp3GjRunl19+2fpBGh8frxkzZmju3Lk20xsk2UwB27p1qyZMmKDi4mINHz5cXbt2VXZ2ttasWaNNmzZp2bJlGjJkiPX8U6dOadiwYfrhhx/Ut29f9e/f3/pN38CBA2v9XdRVWFiYHnnkEe3YsUPLly/XM888Y31szZo1evPNN5WUlKQ+ffooIiLCOm1j/fr12rJlizp27ChJuvXWW5Wfn69169ZpwIABNtPe4uPjJUmHDh1SWlqa+vfvr+HDhysmJkY//vij1q1bp08++UTvvPOOhg0b5vbX6A6zZ8/W9u3bNXLkSJvfV3Z2th5//HH17dtXQ4YMUdu2bXXs2DFt2LBBd9xxhxYuXKhf/vKXLt9n165dSktLU1JSkiZOnKgffvhBH330kW677Tbt2LEjqKfIILAxbjBuSME3bowcOVImk0nbtm3T4cOH7WYOHDt2TJs2bVLbtm112223WY+fPHlSM2fOtHlfZGdna926dUpJSdFf//pXTZw4sd7tWrlype6//35FRETo9ttvl8lk0u7du3XzzTc7/PuzTD287rrr1KFDB+Xn52vLli169NFH9fXXX2vhwoWSpNDQUM2YMUNLly7V0aNHNWXKFEVHR0uSWrduXWvb7r//fq1cuVIdO3bU+PHjFR4ervXr1ys1NVWfffaZ3n//fbu/Y0n6zW9+o4yMDA0ZMkTNmzfXxx9/rFdeeUV5eXlasGBBvfvKn4ScOXPGPasF4XdGjx6tzz//vN5/3PPmzdMzzzyjSZMmac6cOQoLC5NU9W3EQw89pLfeektz5szRgw8+aH2O5du44cOH66233lJkZKQkKTc3V71795Yk/e9//1OTJk0kVX0r169fP8XFxWndunXq0KGD9Vrbtm1TcnKyevTooc8//9x6fPLkyXrnnXfUrFkzbdiwwW6AvHDhgk6ePGn9YLc4c+aMhg8fruPHj+v7779XVFSUTbs7deqkb7/91q4f8vPzde2116qyslLr16/X5Zdfbn3s4MGDGjJkiFq0aKFvvvnGOpf6D3/4g9566y1NmjRJ8+bNs57/zTffaOjQoSotLdXdd9+t9PT0Wn8PZrNZV199tcP2VX/dHTt2VFlZmb755hslJCRIkn766Se1bdvWbp73J598onHjxunee+/VSy+9ZD2+dOlSTZ06VTNmzDCc7pGfn6+ysjK7b6qOHDmioUOHqlWrVvrqq69qfV3uYnk/vPrqqw6n7U2cOFGrV69Wy5Yt9cknn+iyyy6zefz8+fM6e/as3Xz0kydPaujQoSoqKtL+/fttBoouXbqoffv2NpWQ3njjDf35z3+WJP3zn/+0GYTnz5+vxx9/XNOnT9fTTz/d4NcNeALjBuOGFJzjRlpamp5//nn98Y9/1JNPPmnz2Jw5czRnzhy7z+fi4mKdOnXK5j0mVQXbw4YNU15enr7//nubfurevbuaNm2qr7/+2npsyZIl+v3vf6+FCxdq3LhxkqSzZ8/qqquuUmFhoT755BObbNDMmTOtv+f169fbzHLIyspSYmKiTXvKy8v1wAMPaMWKFdq8ebNdZmn37t3av3+/3fvb8vhXX32lvLw867F3331XDz74oHr27KmPPvrIGnRduHBBd9xxh3bs2KFnnnnGmnG0vO6ffvpJ1113nT744APr3/W5c+c0YMAAHT16VP/5z38UGxtr14ZAw7S9IHb8+HFJsvujl6qmGsyePdvm3/z5862PV1RU6PXXX1dsbKxmz55tHQClqm8znnrqKYWEhDgsuzl37lzrAChJsbGxuvXWW3X27FmbdPTf/vY3XbhwQc8++6xdO5OSkjRy5Ejt27dP33//vd09Jk6caPjNYtOmTQ0/IGJiYnTPPffozJkzdZo+9e677+rUqVOaMWOGzQAoSZdddpkmTpyonJwcbdmyRZJUWlqq5cuXq3nz5vrLX/5ic/7VV19dpwxGXTRt2tT6bdLJkyetxzt06GC4QPbmm2/W5Zdfrs8++6xO92nVqpVhij8+Pl5jxoxRZmam3bQOf/HAAw/YBU6S1KxZM8OFvBdddJHuuusuHT9+XN99953L97n55pttAifp52kjTN2DP2PcsMW4YSuQx42JEycqNDRUy5Yts5l2VlFRobfffluSdO+999o8JzIy0vBvoXXr1powYYJOnTrlcApmbdauXav8/HyNHTvWrrT5o48+ag1YaqoZOElVWURLBrSuvxsjlv548sknbdrRtGlTPfvss5KqAkIjs2bNsgZOktSiRQulpKSovLxc33zzTYPb5g+YthfELPPEQ0JC7B47evSo5s6da3OsXbt2mj59uqSq+c55eXlKTEy0+QasuqioKMN5ua1atTKclmQZmM6cOWM9ZpnrvHPnTsM/qtzcXElVKf+aKezrrrvOsF2S9P333+uvf/2rdu7cqZycHOvUDIvs7GyHz63J0sb9+/cblmH973//a23j8OHDdejQIZ0/f159+vSx+QCxGDBggEfmsFdX/XdeWVmp999/X8uWLdN3332nM2fOqLy83Pp4REREna+/e/duvf766/rqq6+Um5trN5c5Ozu71kWo27Zt0/bt222OxcfHe7Tog+VbbCPffPONXn31Ve3atUsnTpwwfM9cc801Lt3n6quvtjsWHR2tli1b2rz/AX/DuMG4IQXnuBEfH6/Bgwdr06ZNWr9+vUaPHi2pqnLq0aNHlZSUZFgoYv/+/frrX/+qXbt26fjx4w16X1Rnee8OGDDA7rFWrVrpyiuvNNzjKS8vT6+88oo2bdoks9mswsJCt7Snun379ik0NNSwYu3VV1+t1q1b6+DBgyoqKrLJxloer8kSgAbL+EfwFMTat2+vQ4cO6aeffrJ7rF+/fjZv4pof1qdOnZJUlR6uOVjWpmXLlobHLd9CVv8AttyntnmwNT8cpKpB28hXX32l0aNHq6ysTAMHDtTIkSMVHR2t0NBQffvtt1q3bp3dh58zljb+85//dKmNZ8+elSSHqWlH7W6oCxcu6PTp05Jk8w3fY489pvT0dJlMJg0ZMkRxcXHWb3eXLVtW52/71qxZo3vvvVeRkZEaNGiQOnfurGbNmik0NFTbt2/Xjh07XOrf7du32723BgwY4NHgqX379obHP//8c40dO1YhISG66aabdNttt6l58+YKDQ3V3r179cknn9Rpsaujqkrh4eE2C3oBf8O4wbghBe+4cd9992nTpk1asmSJNXgyKhRhsXv3biUnJ6uiokIDBw7UrbfeqhYtWig0NFTffPONNmzYUKf3RXWW37mj363Re+H06dO66aab9OOPP+q6667TXXfdpdatWyssLEynT5/WG2+8Ue/2VFdQUKDWrVs7DJJNJpNOnz6tgoICm+ApLCxMLVq0sDvfMuW9+t9xICN4CmLXX3+9tm3bps8//1y/+tWv6vRcy0A2YsQIvfvuu55ons19srKyXFrAWJ3RN6OS9Pzzz6uoqEhr1qxRUlKSzWMvvvii1q1bV682btmyxaXMg+V8y7efNZ04caJO93fVrl27VFZWpvbt21vnrefm5mrhwoXq3r27Nm7caDcNYMWKFXW+j6US1+bNm+2mwP3xj3/Ujh07XLrOzJkzDefGe5Kj98zcuXNVWlqqTZs22WWnnn766TpVVQICGeMG40YwjxsjRoxQXFycPvvsMx05ckQRERHauHGjLrroIo0aNcru/Hnz5qm4uFjr1q1T//79bR577rnntGHDhnq3xfI7d/S7NXov/OMf/9CPP/6ov/zlL3rkkUdsHtu5c6feeOONerenuujoaJ05c0YlJSWGAVROTo71vMaINU9BbMKECQoPD9e//vUvHTx4sE7PvfTSS9WqVSvt2bPHo+Ulf/GLX0iq+qN3l8OHD6t169Z2A6Akhx/QoaGhDjMCljYapc+NXHrppWrWrJn2799vmKJ2dZCoi/Lycj333HOSpJSUFOvxH374QRUVFRo0aJDdh9yxY8f0ww8/2F3L6Jve6g4fPqzLLrvMbgCsqKiwlkQNNFlZWerYsaPhtD53vjcBf8e4wbgRzONGeHi4JkyYYF3ntHTpUpWVlWn8+PGGQcLhw4cVGxtrFzhJDf+dWKa3GV0nPz/fcJ2tpQy/JWvmSnssm7nXJevTs2dPVVRUGF5z3759On36tC6//HK7KXuNBcFTEOvcubNmzJihkpISjR071uFeCkYf1OHh4XrwwQeVm5urhx9+WOfPn7c7Jy8vT/v27WtQGx944AFFREQoNTVVhw4dsnu8vLzcuk+Eq+Lj43X69Gm7D54lS5bo008/NXxO27ZtdfLkSRUVFdk9ds899ygmJkbz5s3Tl19+afd4ZWWldu3aZf2fhSZNmiglJUWFhYU2pV+lqjnO77//fp1eT21ycnJ03333aefOnYqPj9f//d//WR+zlIndvXu3zQfnuXPn9Ic//EFlZWV217NM3Th69Kjh/eLj43X48GGbaT2VlZWaM2eO/vOf/7jlNXlbfHy8cnJyrOsQLNLT013+nx8gGDBuMG4E+7hhKRyxdOlSLVmyRCEhIXaFIizi4+N18uRJu+Ijf//737V169YGtWPUqFFq2bKlPvjgA5vKfFJV9T+jDagtv5ua7++vv/5ar7zyiuF9avvdGLFknWfNmmUz/bWkpESpqak25zRGTNsLco888oj1A2r48OG65ppr1Lt3b7Vu3Vr5+fk6cuSItdpPzW9WHnnkER04cEBLlizRxx9/rBtvvFEdO3bUyZMnlZWVpd27d+v+++83rFzkqm7duum1117T1KlT1a9fPw0dOlRdu3ZVeXm5jh07pi+++EIXLlzQkSNHXL7m5MmT9emnn2rkyJFKTk5Wy5Yt9fXXX2v37t0aM2aM/vWvf9k9Z9CgQXr//fd15513qn///mratKmuvPJKjRw5Uq1bt9aSJUt0zz33aNiwYbrxxht1+eWXq0mTJjp27JgyMjJ09OhR/fDDD9Zvrv7f//t/2rp1qxYtWqR9+/apf//+On78uD788EMNHTpU69evr3Nf5efnWxcel5eXKz8/X99//72++OILlZaW6he/+IUWLVqkNm3aWJ/Tvn173XnnnVqxYoWSkpI0aNAgnT17Vps3b1ZkZKSuuuoquzK2ffr0UYsWLbRy5UpFRETo4osvVkhIiMaNG6f4+HhNmTJFDz30kAYOHKjRo0crPDxcX3zxhQ4ePKgRI0Y0aBqDr0yePFn33nuvhgwZouTkZDVv3lx79uzRnj17dNttt2nNmjW+biLgNYwbjBvBPG7Ex8dryJAh1unYN954o7p27Wp47pQpU7R161YNHz5cycnJio6O1t69e/Xll19q9OjRWr16db3b0bJlS7300ku6//77NXLkSJt9nr7//nv169fP7su78ePHa8GCBZoxY4a2bt2qLl266L///a82btyo0aNHa+XKlXb3GTx4sNasWaPp06db1/O2bt1a999/v8O2/fKXv9SGDRv04Ycfqm/fvho1apTCwsK0YcMG/e9//9PgwYP1u9/9rt6vPdARPDUCf/7zn3XnnXfqzTff1Oeff67ly5ersLBQLVq0UGJiou677z798pe/tJuyFB4eriVLlmjFihVaunSpPvnkE507d05t2rRRp06d9NBDD+muu+5qcPssO7q/+uqr2rp1q/UD2mQyaejQoRozZkydrjd06FC9++67ev755/Xhhx8qNDRUvXv31po1a/TDDz8YDoJz5sxRaGioNm/erC+++ELl5eW6++67NXLkSElVH647duzQggUL9Omnn+rLL79UeHi42rdvr1/84hd64oknbBY8t23bVhs3btRTTz2lDRs26JtvvtEll1yi559/XvHx8fUaBM+ePWtdKBsREaHo6GhrlaExY8Zo4MCB1vR8dfPnz1fnzp21cuVKLV68WBdddJFGjhypxx57zPCbo1atWmnp0qWaPXu2Vq5cqXPnzkmqWgsRHx+vX//614qIiFB6erreeecdRUZGql+/fnr11Ve1evXqgAyexowZo7feeksvv/yyPvjgA4WHh6tPnz7asGGD9u7dS/CERodxg3EjmMeN++67zxo8WbaRMDJ8+HAtW7ZML7zwglauXKmwsDD17t1ba9euVWZmZoOCJ0m68847FRMTo+eee06rVq1S06ZN1b9/f23atEnPPfecXfDUsWNHrV+/XrNmzdLOnTv16aef6tJLL9VLL72kAQMGGAZP9957r44dO6YVK1bo1VdfVWlpqRITE50GTyEhIfrb3/6mG264QW+//baWLFmiyspKde3aVU8//bQefPBBww1yGws2yQUAAAAAF7DmCQAAAABcQPAEAAAAAC4geAIAAAAAFxA8AQAAAIALCJ4AAAAAwAUETwAAAADgAoInAAAAAHABwVMtMjMzfd0Ev0A/0AcW9EPj6gNzQakmbT2lUetzNWnrKZkLSq2PNaZ+8BT60B59Yox+MUa/GKNfjLmjXxrv9sAAAKfMBaVK3pinrIJy67GM3BKtGt5WCdFNfNgyAAB8g8wTAMBQ2t4Cm8BJkrIKypW2t8BHLQIAwLcIngAAhrLPlxsez3FwHACAYEfwBAAwFNcszPC4ycFxAACCnc+Cp0WLFql///7q1KmTOnXqpJtvvlkbN250+pz9+/frlltukclk0hVXXKG5c+eqsrLSSy0GgMYltVe0EqNtA6XE6DCl9or2UYs8j7EJAOCMzwpGdOjQQbNmzVLXrl1VUVGhd955RxMmTNCWLVt05ZVX2p1/9uxZ3X777erfv78+++wzZWZmaurUqWrWrJmmT5/ug1cAAMEtIbqJVg1vq7S9Bco5Xy5Ts6rAKZiLRTA2AQCc8VnwdOutt9r8/Pjjj+tvf/ubvvrqK8MBavny5SoqKlJ6erqioqLUvXt3HTp0SK+99pqmTZumkJAQbzUdABqNhOgmWjSwja+b4TWMTQAAZ/xizVN5eblWrFihwsJC9enTx/CcL7/8Uv369VNUVJT12JAhQ5SdnS2z2eytpgIAGgnGJgBATT7d52n//v0aNmyYiouL1bx5c7399tvq0aOH4bknTpxQhw4dbI7FxsZaH+vcubPD+zR0Qyw2GqtCP9AHFvRDcPbBsaIQvX4kXLkXQhXbtEIPxpepY5TztTv16Ydu3brVt4le4Y2xqXq/BeN7qaHoE2P0izH6xRj9YsyoX+oyLvk0eOrWrZu2bdum/Px8rV69WpMnT9batWvVvXt3w/NrTn+wLMitbVpEQwbqzMxMvx/ovYF+oA8s6Ifg7ANzQakestkQN0wHiyOdbogbjP0geWdssvRbsPZhQ9AnxugXY/SLMfrFmDv6xafT9iIiItSlSxdde+21euKJJ3TVVVfptddeMzy3Xbt2OnHihM2xkydPSvr5Wz4AQP2wIe7PGJsAAI74xZoni4qKCpWUlBg+1qdPH+3atUvFxcXWY5s3b1ZcXJwSEhK81UQACEpsiOsYYxMAwMJnwdOTTz6pnTt3ymw2a//+/Zo1a5a2b9+ulJQUSdKsWbM0evRo6/ljx45VVFSUpkyZogMHDmj16tV6+eWXNWXKFKoZAUADsSFuFcYmAIAzPlvzdPz4cT3wwAM6ceKEWrZsqR49euiDDz7QkCFDJEk5OTnKysqynt+qVSt9+OGHevjhhzVo0CDFxMRo6tSpmjZtmq9eAgAEjdRe0crILbGZuhfsG+IaYWwCADjjs+ApPT29zo/36NFD69ev91STAKDRaowb4hphbAIAOOPTansA0NiZC0qVtrdA2efLFefjgKWxbYgLAEBdETwBgI+YC0qVbFMeXMrILXFaHhwAAPgOwRMA+Iiz8uDuzABVz261bBKiykqpoKzS55kuAEDjEGI2KzItTaHZ2aqIi1NxaqoqA7QiKcETAPiIN8qDG2W3qiPTBQDwpBCzWc2TkxVWrdhOWEaGCletCsgAyq/2eQKAxsSV8uDmglJN2npKo9bnatLWUzIXlNbpHkbZreoa60a4AADviExLswmcJCksK0uRaWk+alHDkHkCAB+prTy4O9ZEOcpuVZd1tm4BGQAArgrNzjY+npPj5Za4B8ETAPhIbeXB67omyqhyn6PsVnUniivd84IAAKihIi7O+LjJ5OWWuAfBEwD4kLPy4HVZE+UoS7VgQCu77FZN7aOYwQ0A8Izi1FSFZWTYTN0rT0xUcWqqD1tVf4yYAOCnXFkTZeEoS/WPQ0VaNbytUrpEKTYyxPB6naOrvkdr6PoqAABqqkxIUOGqVSpJSVFZUpJKUlICtliEROYJAPxWbWuiqnOWpbJkt4yyU5brsecUAMBTKhMSVLRoka+b4RZkngDAT1nWRKV0iVKSKUIpXaIcBjOuZKmcXc/Z+ioAAFCFzBMA+DFna6KqczVL5eh63thzCgCAQEfwBABBoLbKfbWpy/oqAAAaK4InAAgSrmapjNRlfRUAAI0VwRMAoMGZKwAAGgOCJwCApIZlrgAAaAyotgcAAAAALiB4AgAAAAAXEDwBAAAAgAtY8wQAbmYuKFXa3gJlny9XnJsKL3jimgAABKIQs1mRaWkKzc5WRVycilNTVZmQ4JV7EzwBgBuZC0qVvDHPpuR3Rm6JVg1vW+9gxxPXBAAgEIWYzWqenKywrCzrsbCMDBWuWuWVAIppewDgRml7C2yCHEnKKihX2t4Cv7omAACBKDItzSZwkqSwrCxFpqV55f4ETwDgRtnnyw2P5zg47qtrAgAQiEKzs42P5+R45/5euQsANBJxzcIMj5scHPfVNQEACEQVcXHGx00mr9yf4AkA3Ci1V7QSo6SlBw8AACAASURBVG2DmsToqgIP/nRNAAACUXFqqsoTE22OlScmqjg11Sv3p2AEALhRQnQTrRreVml7C5RzvlwmFyrj1VZJrz7XBAAgGFUmJKhw1aqqans5Oaowmai2BwCBLCG6iRYNbOPSua5W0qvLNQEACGaVCQkqWrTIJ/dm2h4ANIC5oFSTtp7SqPW5mrT1lMwFpXV6PpX0AAAIHGSeAKCe3LH/EpX0AAAIHGSeAKCe3JE1opIeAACBg+AJAOrJHVkjKukBABA4mLYHAPXkjqwRlfQAAAgcBE8AAlZtJb49dc/HDzbRuf/mKjo8RBc3D9XRwgrr4/XJGlFJDwCAwEDwBCAguaNYQ/3v2URSiSSpiaQOzULVsXmYOkeHWwM4XwR2AADAswieAAQkZ8UaPJXFMbpnqaSfzleoaViIFg/8OXDydmAHAAA8j4IRAAKSL0p8O7qnZFtlj72bAAAITgRPAAKSL0p8O7qnhSVwY+8mAACCE8ETgIDkqMT3fZdGadLWUxq1PleTtp6SuaDUo/eszhK4sXcTAADBiTVPAAKSUYnv+y6N0rQd+R5ba2S55/RPj+nLs01UXC2RVL3KXmqvaGXklti0g72bAAAIfARPAAJWzRLfk7ae8ngRiYToJnqhR6kiTB0d7s3E3k0AAAQngicAQcOba41q25uJvZsAAAg+rHkCEDRYawQAADyJzBOAoOHOtUZscgsAAGoieAIQNNy11ohNbgEAgBGfTdt78cUXNWjQIHXq1Eldu3bVuHHjdODAAafPMZvNiomJsfu3adMmL7UagL+zrDVaMzJWiwa2qVewwya3jRdjEwDAGZ9lnrZv367f/va36tWrlyorK/Xss88qOTlZX3zxhVq3bu30uStWrNCVV15p/bm28wGgJmfT8tjktvFibAIAOOOz4GnlypU2Py9cuFDx8fHavXu3Ro4c6fS5bdq0Ufv27T3ZPAABypW1So6m5S0Y0Er/OFSkg2fKDK9N4Yngx9gEAHDGb9Y8nTt3ThUVFYqJian13F/96lcqLi5W165dNWXKFI0ZM8YLLQTg71xdq+RoWt4vN51WYVml4bXZ5LZxYmwCAFQXcubMGeP/U/Cy++67T//73/+0ZcsWhYUZf7ubl5enZcuW6frrr1d4eLjWrVunF154Qenp6Ro3bpzDa2dmZnqq2QD8yOMHm2hDrv0apxvblOmF7iXWnx/c11R7zrqWRWoTXqE+rcv1YHyZOkb5xcdlwOvWrZuvm+AyT41NjEsA4D/qMi75RfD02GOPaeXKldqwYYM6d+5cp+f+6U9/0q5du7Rz506PtC0zMzOgBnpPoR/oAwt/7odR63O1PafE7nhkqPTFHe2s2adJW09p+eEil66ZZIrQmpGxNsf8uQ+8Kdj7wRtjU7D3YX3QJ8boF2P0izH6xZg7+sXnm+TOnDlTK1as0OrVq+s8OElS7969dfjwYfc3DEDAcbRJbnGFbCrlpfaKVmK0a5kn1jk1ToxNAPxJiNmsqEmT1HzUKEVNmqQQs9nXTWq0fLrmacaMGVq5cqXWrl2rSy+9tF7X+Pbbb1mgC0BSVVC0xlykYoOieNUr5dXcD+o/Z8qUW1xh95zIMLHOqRFibALgT0LMZjVPTlZYVpb1WFhGhgpXrVJlQoIPW9Y4+Sx4evjhh/Xee+/p7bffVkxMjI4fPy5Jat68uVq0aCFJmjVrlvbs2aPVq1dLkpYtW6YmTZqoZ8+eCg0N1YYNG7R48WI9+eSTvnoZAHzAUUW9hOgmGhTXVOuPXrB7Ts0MkmU/KMnxNL5BcU3ZFLeRYWwC4G8i09JsAidJCsvKUmRamooWLfJRqxovnwVPixcvliS7akQzZszQzJkzJUk5OTnKqvFmef755/Xjjz8qLCxMXbt21YIFC5wWiwAQXGqrqDfn+lb6T43Ha6uUl9orWhm5JXbPmXN9K8+8CPgtxiYA/iY0O9v4eE6Ol1sCyYfB05kzZ2o9Jz093ebn8ePHa/z48Z5qEoAA4KjMeNreAi0a2MZuSp7JwV5P1SVEN9GCAa00eXu+8i9UqFXTUC0Y0IqsUyPE2ATA31TExRkfN5m83BJIfrTPEwC4Ivu8wYIm2a9pskzJc4W5oFTTduTryLmqa+SXlmvajnytGh5OAAUA8Kni1FSFZWTYTN0rT0xUcWqqD1vVePm82h6AwHKsKESTtp7SqPW5mrT1lMwFpV69v6OKeg2piucsmwUAgC9VJiSocNUqlaSkqCwpSSUpKRSL8CEyTwBcZi4o1bT9TXW0+OfiCtXXG3mDo/VJDamK50o2CwAAX6lMSKA4hJ8g8wTAZWl7C3S02PZjw9sZGsuappQuUUoyRSilS1SDgzdPZLMAAEDwIfMEwGX+kqGp65qm2ngimwUAAIIPwRMAl/lLhsbRPk/1VZ8KfQAAoPEheALgstRe0dr1U6HN1D1vZ2hq2+epvtydzQIAAMGH4AmAyxKim2hBjwtaerqNRzM0zjJLte3zBAAA4CkETwDqpGNUpRb19FyQUltmyV/WXQEAgMaHansA/Eptey75y7orAADQ+BA8AfArtWWWUntFKzHaNlCiMh4AAPAGpu0B8Cu1ZZaojAcAAHyF4AmAX3FlzyUq4wEAAF8geALQYO7cd4nMEgAA8FcETwAaxBP7LpFZAgAA/oiCEQAapLbqeAAAAMGC4AlAg7DvEgAAaCwIngA0CPsuAQB8LcRsVtSkSWo+apSiJk1SiNns6yYhSLHmCUCDuFIdDwAATwkxm9U8OVlhWVnWY2EZGSpctUqVCQk+bBmCEZknAA1iqY6X0iVKSaYIpXSJalCxCAAA6iIyLc0mcJKksKwsRaal1flaZLBQGzJPABqM6ngAAF8Jzc42Pp6TU6frkMGCK8g8AUHMXFCqSVtPadT6XE3aekrmglJfNwkAALeqiIszPm4y1ek67sxgIXiReQKClCf2XwIAwN8Up6YqLCPDJvApT0xUcWpqna7jrgwWghuZJyBIsf8SAKAxqExIUOGqVSpJSVFZUpJKUlLqNdXOXRksBDcyT0CQ8vX+S+aCUqXtLVD2+XLFNauqvkfGCwDgCZUJCSpatKhB13BXBgvBjeAJCFK+3H+JKYMAgEBjyWBFpqUpNCdHFSaTilNTKRYBGwRPQJDy5f5LzqYMUpUPAOCv3JHBQnAjeAKClGX/pbS9Bco5Xy5TPabOGU29c4WvpwwCAAB4AsETEMQasv+So6l3L10aom61PNeXUwYBAP4pxGyumhKXna2KuDimxCEgETwBMORo6t3rR8J1U0/nz/XllEEAgP9hA1oEC0qVAzDkaOpdbkntHxuWKYMpXaKUZIpQSpcoikUAQCPGBrQIFmSeABhyNPUuNqLCpec3ZMogACC4sAEtggWZJwCGUntFKzHaNoBqHh6io0WhmrT1lMwFpT5qGQAg0LABLYIFwRMAQ9Wn3l13Ubiah0uFZZX67lyYlh8uUvLGPAIoAIBLilNTVZ6YaHOMDWgRiAieADhkmXqX2LKJCstsH7Ps2wQAQG0sG9CWpKSoLClJJSkpFItAQGLNE4BasW8TAKCh2IAWwYDME4BasW8TAAAAwRMAFxgVj2DfJgAA0NgQPAGoVfXiEb1blbNvEwAAaJRY8wTAJZbiEZmZeerWLd7XzQEAAPA6giegETIXlCptb4Gyz5crrlnV9DuySAAAAM4RPAGNjLmgVMkb85RV8HOlvIzcEqfT8KoHWy3Km2iuqZRgCwBgI8RsVmRamkKzs1URF6fi1FRKkSPoEDwBjUza3gKbwEn6ec+mRQPb2J1vH2w10cGNeax5AgBYhZjNap6crLCsLOuxsIwM9nJC0KFgBBAEzAWlmrT1lEatz9WkradkLih1eG5d92xyFmwBACBJkWlpNoGTJIVlZSkyLc1HLQI8w2fB04svvqhBgwapU6dO6tq1q8aNG6cDBw7U+rz9+/frlltukclk0hVXXKG5c+eqsrLSCy0G/JMlM7T8cJG255Ro+eEiJW/McxhA1XXPJjbIRWPC2ATUT2h2tvHxnBwvtwTwLJ8FT9u3b9dvf/tbbdy4UatXr1Z4eLiSk5N1+vRph885e/asbr/9drVr106fffaZ5syZo/nz52vBggVebDngX+qaGarrnk1skIvGhLEJqJ+KuDjj4yaT2+8VYjYratIkNR81SomPP64Qs9nt9wAc8dmap5UrV9r8vHDhQsXHx2v37t0aOXKk4XOWL1+uoqIipaenKyoqSt27d9ehQ4f02muvadq0aQoJCfFG0wG/UtfMkGXPprS9Bco5Xy5TLdX2UntFKyO3xCZAY4NcBCvGJqB+ilNTFZaRYTN1rzwxUcWpqW69T821VW0llR88yNoqeI3frHk6d+6cKioqFBMT4/CcL7/8Uv369VNUVJT12JAhQ5SdnS0z3zogQNVlvZKR+mSGLHs2rRkZq0UD2zgt/FB9g9wkU4RGxJZSLAKNBmMT4JrKhAQVrlqlkpQUlSUlqSQlxSMBDWur4Gt+U23v0Ucf1VVXXaU+ffo4POfEiRPq0KGDzbHY2FjrY507dzZ8XmZmZoPa1tDnBwv6wf19cKwoRNP2N9XR4p+/x9j1U6EW9LigjlGurZeY0DpEuyJtr3FxZIUmtD6lzMw8t7X1z9X+9EpyflBmI5/Gzt9Dlfr0Q7du3TzQEs/w1NhUvd94L9mjT4wFRL/8+c8//3dJieTmNl96+LAiDI4XHz4cGP3jRfSHMaN+qcu45BfB02OPPabdu3drw4YNCgtzvo6i5vQHy4JcZ9MiGjJQZ2ZmBtRA7yn0g2f64Lmtp3S0uMjm2NHiUC093UaLetqXDTfSTdJHiaUuT8NrKN4L9IFFsPeDJ8cmS78Fex/WB31ijH6pEtmli7Rnj+Fx+udnvF+MuaNffB48zZw5UytXrtSaNWscZo4s2rVrpxMnTtgcO3nypKSfv+UDAom7KtlZpuEBcA/GJsA/eWttFeCIT9c8zZgxQx988IFWr16tSy+9tNbz+/Tpo127dqm4uNh6bPPmzYqLi1MCiwQRgKhkB/gfxibAddUr30VNmuTxync111bljRhBsQh4lc+Cp4cffljLli3T4sWLFRMTo+PHj+v48eM6d+6c9ZxZs2Zp9OjR1p/Hjh2rqKgoTZkyRQcOHNDq1av18ssva8qUKVQzQkCqa9lwAJ7F2AS4zlL5LmL5coVv366I5cvVPDnZKwFU0aJFKlyzRllPP03gBK/yWfC0ePFiFRQUaMyYMbrsssus/+bPn289JycnR1nV0rKtWrXShx9+qOzsbA0aNEiPPPKIpk6dqmnTpvniJQANVrOSXUqXKCrZAT7E2AS4jsp3aIx8tubpzJkztZ6Tnp5ud6xHjx5av369J5oE+ATrlQD/wdgEuC40O9v4eE4jL8eKoObzghFAMDIXVFW/yz5frrgGVL9z13UAAHC3irg44+Mmk5dbAngPwRPgZuaCUiVvzFNWwc8V8zJyS+o8Hc9d1wEAwBOMKt9VRkZK584pxGxmLRKCkk+r7QHBKG1vgU3AI0lZBeVK21vgk+sAAIKft6veSdUq391yS1XQJCmkuFgR69d7pXAE4AtkngA3c9feTe66DgAguFmq3lXPAIVlZHilhHdlQoLUvLlCqpXql34uHFG0aJFH7w94G5knwM3ctXcTe0ABAFzh66p3FI5AY0LwBLiZu/Zuuu/SKIXX2CImPKTqOAAAFr4OXigcgcaE4AlwM3ft3fSPQ0Uqq7Q9VlZZdRwAAAtfBy/FqakqT0y0OVaemKji1FSv3B/wJtY8AR7gjr2bWPMEAHCFUdU7bwYvlsIRkWlpCs3JUYXJpOLUVKrtISgRPAF+ijVPAABX+EPwUpmQQHEINApM2wP8lLvWTgEAgl9lQoKKU1NVYTIpNDtbkWlplAoHPIDME+CnLGun0vYWKOd8uUzNqgInNsgFANTky3LlQGNC8AT4MXesnQIABD9n5cqZTge4D8ETAp65oFRpewuUfb5ccWRnAACNkK/LlQONBcETApq5oFTJG/OUVfBzBbqM3JJ6lQYHACBQ+bpcOdBYUDACAS1tb4FN4CRJWQXlSttb4KMWAQDgfey1BHgHmScEtEDbC4kphgAAT/CHcuVAY0DwhIBSM/ho2STE8Dx/3AuJKYYAAE9iryXA8wieEDCMgo+Lm4Xo4uahOlpYYT1W172QvJUNcjbFkIp6AAAA/o/gCQHDKPg4er5SIy+OUL/2oS7thVQzULrv0ihN25HvlWxQoE0xBAAAgC2CJwQMR8HHubJKvXNz7Zkbo8zVuiNFKiyzPc9T2aA4B1MJ/XGKIQAAAOwRPCFgOAo+opuEaNLWU7VOuzPKXNUMnCw8kQ1K7RWtjNwSmzbUdYohAAAAfIfgCQHDKPi4uHmo9uWV6Oj5SusxR9PuHGWujHgiG5QQ3USrhrdV2t4Cl6YYAgAAwL8QPCFgGAUf50oqtP7oBZvzHE27c5S5CguRyn+OvTyaDUqIbkJxCAAAgABF8ISAUjP4GLU+1/A8o2l3RpkrqSpwah4uXRETrsSWTcgGAQB8JsRsrtqrKTtbFXFx7NUE+BmCJwS0uhRhsGSubtuQpyPn7Nc+JbYkKwQA8J0Qs1nNk5MVlpVlPRaWkaHCVat82CoA1YX6ugFAQ6T2ilZitG2g5GzaXUJ0E8W3MA64KBkOAPClyLQ0m8BJksKyshSZluajFgGoicwTAlp9ijBQMhwA4I9Cs7ONj+fkeLklABwheELAq2sRBkqGAwD8UUVcnPFxk8nLLQHgCNP20OhYslUpXaKUZIpQSpcow9LmAAB4U3FqqsoTE22OlScmqjg11UctAlATmSc0SpQMBwD4m8qEBBWuWlVVbS8nRxUm08/V9jIzfd08ACJ4AgAA8BuVCQkqWrTI180A4ADT9gAAAADABQRPAAAABkLMZkVNmqTmo0YpatIkhZjNvm4SAB9j2h4AAEANzjasrUxI8GHLAPgSmScAAIAa2LAWgBEyT7AyF5QqbW+Bss+XK86FzWb97foAALiLOzesDTGbqyroZWerIi7u5wp6AAIOwRMkVQU2yRvzbDaOzcgtcdv+R7Vdv2Zgdd+lUfrHoSICLQCAT7hrw1qm/wHBhWl7kCSl7S2wCWwkKaugXGl7Czx+fUtgtfxwkbbnlGj54SKN2XjK5ufkjXkyF5S6pS0AANTGXRvWMv0PCC4ET5AkZZ8vNzye4+C4O69vFFiVVdqe585ADgCA2lg2rC1JSVFZUpJKUlLqlS1y5/Q/AL5Xp2l7H3/8sYYOHarQUGKuYBPXLMzwuMnBcXde31FgVZO7AjkAwYNxCZ7kjg1r3TX9D4B/qNNoM27cOF1++eWaOXOm/v3vf3uqTfCB1F7RSoy2DXASo6vWGnn6+o4Cq5pqC+TMBaWatPWURq3P1aStp5jmBzQCjEvwd+6a/gfAP9QpeHr33XeVlJSkJUuWaPDgwerbt69eeuklHT161FPtg5ckRDfRquFtldIlSkmmCKV0iXJbsYjarm8UWIWH2D6/tkDOaN0U66SA4Me4BE9yxya57pr+B8A/hJw5c6ay9tNsnTt3Tv/617+0fPlybdu2TZLUv39/3XXXXRo9erSio92TrfAHmZmZ6tatm6+b4XOe7gdLtb2c8+UyVau2Z/nZUm3PUbnzSVurCkzUlNIlSosGtnFLG3kvVKEf6AMLf+qHQB2X/KkP/YW/9IlRlbzyxESfBT7+0i/+hn4xRr8Yc0e/1KtUeYsWLTRhwgRNmDBBOTk5Wr58ud577z1Nnz5djzzyiG655RbdfffdGjJkSIMah8YjIbqJXZAzIC7K5mdn5c49XfACgH9jXIK7OauS19B1UAACV4NX2JaWlqqkpEQlJSWqrKxUdHS0du3apbFjx6p///767rvvHD53x44duuuuu3TFFVcoJiZGS5cudXovs9msmJgYu3+bNm1q6MuAD9R1jZKzcueeLngBIHAwLsEdqJIHwEi9gqf8/Hy99dZbuuWWW3TNNddo3rx56t69u959910dOHBA3333nd555x0VFhZq+vTpDq9TWFio7t27a86cOYqKinJ4Xk0rVqzQwYMHrf9uvPHG+rwM+FB91ig5yy55uuAFAP/GuAR3o0oeACN1mrb30Ucf6b333tPHH3+sCxcu6LrrrtO8efN0xx13KCYmxubcESNG6MSJE/rTn/7k8HrDhg3TsGHDJElTpkxxuR1t2rRR+/bt69J0+BlnWSRHa5ScZZcsBSmqr5uyrIcCELwYl+ApxampCsvIsFvzRJU8oHGrU/B0zz33qGPHjpo6daruvvtuXXLJJU7P79Gjh1JSUhrUQCO/+tWvVFxcrK5du2rKlCkaM2aM2+8Bz6rPGqXUXtHKyC2xCbqqZ5eM1k0BCG6MS/AUS5W8yLQ0hebkqMJkUnFqKlXygEauTtX2tmzZooEDByokJKT2k+uoY8eOeu655zRhwgSH5+Tl5WnZsmW6/vrrFR4ernXr1umFF15Qenq6xo0b5/B5mZmZbm9vY3WsKESvHwlX7oVQxTat0IPxZeoYVeeCjXr8YBNtyLXPCo2ILdXTlzmeume9f0moYiPqf38AvuPOClCMSwCAhqrLuFSvUuWe4MogZeRPf/qTdu3apZ07d3qkXY2l1KOjEuAWmZmZijB1tqt2lxgdVq/9oIwq59X3Wt7SWN4LtaEf6AOLYO8Hb4xLwd6H9UGfGKNfjNEvxugXY+7olwZX2/O13r176/Dhw75uRkBztXiDs3VKdeXpTXkBwFcYlwAgeNVrnyd/8u2337JIt4FcLd7g7r2UWKMEIBgxLgFA8PJp8HTu3Dnrt3MVFRU6evSo9u3bp9atW6tTp06aNWuW9uzZo9WrV0uSli1bpiZNmqhnz54KDQ3Vhg0btHjxYj355JM+fBWeUds0OndyFBRt+emCRq3PVVyzME1oHeLVvZS8+foBwIJxCQDgjE+Dp6+//lq33Xab9efZs2dr9uzZuvvuu5Wenq6cnBxl1djd+/nnn9ePP/6osLAwde3aVQsWLHC6KDcQGa0Hysgt8di0NkdBUW5xhXJzSiRJuyKbauFNUU6r3bmLt18/AFgwLgEAnPFp8JSUlKQzZ844fDw9Pd3m5/Hjx2v8+PGebpbP1WcPpIYwKgFe09HiUP3jUJFX9lLy9usHAAvGJQCAMwG/5ikYuXttUW1qbjD7nzOlyi22L8KYc77cK+uUvP36AQAAAFcEfLW9YOTNtUUWlqBozchY3dQh0uv3r84Xrx8AAACoDcGTH0rtFa3EaNtAwRNri+py/4sjK3x6f2++fgAAAMAI0/b8UM1pdJ5aW1SX+09ofcqn96faHgAAAHyN4MlPeXsPJKPS4NXvn5mZ57W2SOwBBQAAAP9D8ASnpcGlqup3h082VZefTpEBAgAAQKNF8NRIVc80HTlX9a+6rIJyPbo7X//JL/v/g6ow7TlbxH5LAAAAaLQInhoho0yTkYyTpcotrrA5xn5LAABHQsxmRaalKTQ7WxVxcSpOTVVlQoKvmwUAbkPw1AgZbUJrzH6vJ4n9lgAA9kLMZjVPTlZYVpb1WFhGhgpXrSKAAhA0KFXeCDnahLa6xOgw/SI2wvAx9lsCANQUmZZmEzhJUlhWliLT0nzUIgBwPzJPAcioMl5d1iA52oQ2vkWYElqEWUuDS9L3Z2yn97HfEgDASGh2tvHxnBwvtwQAPIfgKcA4q4znagCV2itaGbkldkGR0TUs+y0dzjunLm1bUG0PAGCoIi7O+LjJ5OWWAIDnEDwFGKP1StWLOLiSlarLJrSW/ZYyM/PUrVu8R18bAMB3GlrsoTg1VWEZGTZT98oTE1WcmuqJ5gKATxA8BRhH65Vyzpc7zEotGNBK/zhUZBdQUTEPACC5p9hDZUKCCletqgrAcnJUYTJRbQ9A0CF4CjCO1iuZmoU5zEr9ctNpFZb9XDmPvZoAANU5K/ZQtGiRy9epTEio0/kAEGiothdgUntFKzHaNoCyFHFwlJWqHjhJP0/zAwBAotgDALiKzJMXNLQ6XnXO1is5ykoZcWWvpurtblHeRHNNpWSrACAIUewBAFxD8ORh7qiOV5Oj9UpGVfSah0uFZfbXqG2vJvt2N9HBjXlM9wOAIESxBwBwDdP2PMxZdTx3s2SlUrpEKckUoZQuUXp/aBuH0/z8pd0AAN+yFHsoSUlRWVKSSlJS6lQsAgAaCzJPHuasOl5DOJoKaJSVWjU83KWy5N5oNwDAP1HsAQBqR/DkYc6q49VXXacC1qcsuSfaDQAAAAQypu15mLPqePXljSl1nmg3AAAAEMjIPHmYs+p49eWNKXU12928vFBzB7ajWAQAAAAaLYInL6jPtDlnvDWlrnq7MzPPEDgBAACgUSN48iJ37fdkVJKcKXUAAACAZxE8uUltgZE793vyxFRAAAAAAM4RPLmBK4GRoyIPt23IU3yLsDpnotw9FRAAEHxCzGZFpqUpNDtbFXFxKk5NZe8mAGgAgic3cFb9zhLgOCrycORcuY6cq3qsvpkoAABqCjGb1Tw5WWFZWdZjYRkZbH4LAA1AqXI3cKX6naMiD9W5Um7cXFCqSVtPadT6XE3aekrmgtK6NRYA0ChEpqXZBE6SFJaVpci0NB+1CAACH5knN3Cl+l1qr2jtyinW0fOVTq/lrNy4O9dNAQCCW2h2tvHxnBwvtwQAggeZJzdwZUPZhOgm6tk2otZrOSs37o3NcQEAwaEiLs74uMnk5ZYAQPAg81QPRpX1XKl+d7bUedaptnLj3tgcFwDg/1wpBFGcmqqwjAybqXvliYkqTk31dnMBIGgQPNWRs6lztVW/czS9LzYyVDd1aFprtT1Xpge6ay8pAIB/crUQRGVCggpXraoKsnJyVGEyUW0PABqI4KmOXKms54ijzW1dXbNU2+a4rIkCgODnrBBE0aJFNscrExLsjgEA6o81T3XUyfL2mwAAFXZJREFUkKlzls1tU7pEKckUoZQuUXUKbGp7PmuiACD4UQgCAHyHzFMduTJ1zpmGbm7r7PmsiQKA4EchCADwHTJPdeRKZT1faWhgBwDwf8WpqSpPTLQ5RiEIAPAOgqc6qj517hexTRTfIkxtmoYobW9BgzesbegGuP4c2AEA3MNSCKIkJUVlSUkqSUmxKxYBAPAMpu3V4lhRiJ7besquel1qr2glb8zTkXPlOnJO2nOyrEHFGdxR7MES2NVWMh0AENgoBAEAvkHw5IS5oFTT9jfV0eIi6zFLQNOQqntG3HW9hq6pAgAAAGCMaXtOpO0t0NFi2y6yBDTuLs5AsQcAAADAvxE8OeEsoHF3cQaKPQAAAAD+jeDJCWcBjbuLM1DsAQAAAPBvrHlyIrVXtHb9VGgzdc8S0Li7OAPFHgAAAAD/5tPM044dO3TXXXfpiiuuUExMjJYuXVrrc/bv369bbrlFJpNJV1xxhebOnavKykqPtC8huokW9LiglC5RSjJFKKVLlE31O0txhjUjY7VoYJsGBzruvh4AoG78fVwCAPiWTzNPhYWF6t69u+6++249+OCDtZ5/9uxZ3X777erfv78+++wzZWZmaurUqWrWrJmmT5/ukTZ2jKrUop51q15nLii1FpWII4MEAAEjEMYlAIDv+DR4GjZsmIYNGyZJmjJlSq3nL1++XEVFRUpPT1dUVJS6d++uQ4cO6bXXXtO0adMUEhLi6SbXyh37NQEAfCMYxyUAgPsEVMGIL7/8Uv369VNUVJT12JAhQ5SdnS2z2ezDlv3M2X5NAIDgEgjjEgDAfQKqYMSJEyfUoUMHm2OxsbHWxzp37mz4vMzMzAbdty7PP3yyqST7Kn2H884pMzOvQe3wtYb2YzCgD6rQD/SBRX36oVu3bh5oiW+4Y1zivWSPPjFGvxijX4zRL8aM+qUu41JABU+S7KZAWBblOpsa0ZCBOjMzs07P7/LTKe05W2R/vG0LdesWX+92+Fpd+yEY0QdV6Af6wIJ+qNKQcYk+tEefGKNfjNEvxugXY+7ol4CatteuXTudOHHC5tjJkycl/fxNn6+xXxMANB6BMC4BANwnoIKnPn36aNeuXSouLrYe27x5s+Li4pSQkODDlv3Msl+To/LmAIDgEQjjEgDAfXwaPJ07d0779u3Tvn37VFFRoaNHj2rfvn368ccfJUmzZs3S6NGjreePHTtWUVFRmjJlig4cOKDVq1fr5Zdf1pQpU/yqohH7NQFAYArWcQkA4B4+DZ6+/vpr3XjjjbrxxhtVVFSk2bNn68Ybb9Szzz4rScrJyVFWVpb1/FatWunDDz9Udna2Bg0apEceeURTp07VtGnTfPUSAABBhHEJAOCMTwtGJCUl6cyZMw4fT09PtzvWo0cPrV+/3pPNAgA0UoxLAABnAmrNEwAAAAD4CsETAAAAALiA4AkAAAAAXEDwBAAAAAAuIHgCAAAAABcQPAEAAACACwieAAAAAMAFBE8AAAAA4AKCJwAAAABwAcETAAAAALiA4AkAAAAAXEDwBAAAAAAuIHgCAAAAABcQPAEAAACACwieAAAAAMAFBE8AAAAA4AKCJwAAAABwAcETAAAAALiA4AkAAAAAXEDwBAAAAAAuIHgCAAAAABcQPAEAAACACwieAAAAAMAFBE8AAAAA4AKCJwAAAABwAcETAAAAALiA4AkAAAAAXEDwBAAAAAAuIHgCAAAAABcQPAEAAACACwieAAAAAMAFBE8AAAAA4AKCJwAAAABwAcETAAAAALiA4AkAAAAAXEDwBAAAAAAuIHgCAAAAABcQPAEAAACACwieAAAAAMAFBE8AAAAA4AKCJwAAAABwAcETAAAAALiA4AkAAAAAXEDwBAAAAAAu8HnwtHjxYvXs2VPt27fXwIEDtXPnTofnbtu2TTExMXb/Dh065MUWAwCCHWMTAMBIuC9vvnLlSj366KN64YUXdP3112vx4sVKSUnR7t271alTJ4fP2717t1q3bm39+aKLLvJGcwEAjQBjEwDAEZ9mnl599VWNHz9e9957ry677DLNmzdP7du315tvvun0ebGxsWrfvr31X1hYmJdaDAAIdoxNAABHfBY8lZSU6N///rcGDx5sc3zw4MH64osvnD73pptu0mWXXabRo0fr888/92QzAQCNCGMTAMAZn03by8vLU3l5uWJjY22Ox8bG6sSJE4bPMZlMevHFF9WrVy+VlJTovffe05gxY7R27VoNGDDA4b0yMzMb1NaGPj9Y0A/0gQX9QB9Y1KcfunXr5oGWuIe3xqbq/cZ7yR59Yox+MUa/GKNfjBn1S13GJZ+ueZKkkJAQm58rKyvtjll069bN5sX16dNHR44c0fz5850GTw0ZqDMzM/16oPcW+oE+sKAf6AOLYO4HT49NlvODuQ/riz4xRr8Yo1+M0S/G3NEvPpu217ZtW4WFhdl9k3fy5Em7b/yc6d27tw4fPuzu5gEAGiHGJgCAMz4LniIiInTNNddo8+bNNsc3b96svn37unydb7/9Vu3bt3d38wAAjRBjEwDAGZ9O25s6dap+97vfqXfv3urbt6/efPNN5eTk6Ne//rX0/7V37zFVF/4fx18MddhKSMYtlTRSQZ0la8ccEAXoltacbl1IN5uWuOhiF0hbYqwc0jGXdxTDZrE2nBoW9kd5CRXINqHURi2WJSYHL2MNpgF1fn8wzn58+WCfc+Ccz4HzfGz+4YcP+ua9s/PidW4fSVlZWZKknTt3SpK2b9+u2NhYJSQkqL29XWVlZaqoqNDevXst+xkAAEML2QQA6Iul5WnhwoW6fv267Ha7HA6HEhISVFZWptjYWElSY2Njj/M7Ojq0Zs0aXb58WSEhIa7z58yZY8X4AIAhiGwCAPQlqKWlxWn1EP6MN9x1YQ/soBt7YAfd2EP/scPe2Ikx9mKMvRhjL8YG9QdGAAAAAMBgQnkCAAAAABMoTwAAAABgAuUJAAAAAEygPAEAAACACZQnAAAAADCB8gQAAAAAJlCeAAAAAMAEyhMAAAAAmEB5AgAAAAATKE8AAAAAYALlCQAAAABMoDwBAAAAgAmUJwAAAAAwgfIEAAAAACZQngAAAADABMoTAAAAAJhAeQIAAAAAEyhPAAAAAGAC5QkAAAAATKA8AQAAAIAJlCcAAAAAMIHyBAAAAAAmUJ4AAAAAwATKEwAAAACYQHkCAAAAABMoTwAAAABgAuUJAAAAAEygPAEAAACACZQnAAAAADCB8gQAAAAAJlCeAAAAAMAEyhMAAAAAmEB5AgAAAAATKE8AAAAAYALlCQAAAABMoDwBAAAAgAmUJwAAAAAwgfIEAAAAACZQngAAAADABMoTAAAAAJhAeQIAAAAAEyhPAAAAAGAC5QkAAAAATLC8PO3evVvTp09XVFSUUlNTVVVVdcvzT548qdTUVEVFRem+++5TSUmJjyYFAAQKsgkAYMTS8nTgwAGtWrVKr7/+uiorK2Wz2fTEE0/o4sWLhudfuHBBTz75pGw2myorK/Xaa68pNzdX5eXlPp4cADBUkU0AgL5YWp62bdumZ555RkuWLNHkyZNlt9sVFRXV5yN2e/bsUXR0tOx2uyZPnqwlS5YoMzNTW7du9fHkAIChimwCAPTFsvLU3t6uuro6paWl9Tielpam7777zvB7Tp8+3ev89PR01dbWqqOjwytzTpw40Sv/7mDDHthBN/bADroNxT34OpuG4g77i50YYy/G2Isx9mJsIPZiWXm6du2a/vnnH0VERPQ4HhERoebmZsPvaW5uNjy/s7NT165d89qsAIDAQDYBAG7F8g+MCAoK6vF3p9PZ69h/nW90HAAAT5FNAAAjlpWn8PBwBQcH93ok7+rVq70ewesWGRlpeP6wYcM0evRor80KAAgMZBMA4FYsK08jRozQ/fffr2PHjvU4fuzYMc2cOdPwe2w2m44fP97r/BkzZmj48OHeGhUAECDIJgDArQSvWrXqHav+8zvuuEMFBQWKjo5WSEiI7Ha7qqqqtHXrVoWGhiorK0tffvmlHn/8cUnShAkT9OGHH+rKlSsaN26cDh8+rA8++EDvvfee4uPjrfoxAABDCNkEAOiLpe95WrhwoQoKCmS325WSkqKamhqVlZUpNjZWktTY2KjGxkbX+ePHj1dZWZmqqqqUkpKiDRs2qLCwUPPnz/d4Bi6E2MWdPRw6dEgLFixQXFycxo4dq/T0dB0+fNiH03qHu7eFbtXV1QoPD9esWbO8PKFvuLuH9vZ2rVu3TtOnT1dkZKSmTZumoqIiH03rHe7uYN++fUpOTlZMTIwmTZqk5cuXy+Fw+GjagXfq1Ck9/fTTSkhIUFhYmEpLS//ze86fP6+5c+cqOjpaCQkJKiwsdL3vZ7AZqGy6cuUK+WKAvDFGBhkjk4wFek79L1/mluUfGPHcc8/p7Nmzam5u1rfffqukpCTX1yoqKlRRUdHj/OTkZFVWVqq5uVk//vijli5d6vH/zYUQu7i7h1OnTumhhx5SWVmZKisrNXv2bC1evNj0Hb0/cncH3VpaWrRixQqlpqb6aFLv8mQPy5Yt05EjR7Rp0yZ9//33+vjjjzV16lQfTj2w3N1BTU2NsrKylJmZqerqapWWlqq+vl7PP/+8jycfOG1tbZoyZYrWr1+vkSNH/uf5f/31lxYsWKDIyEgdPXpU69ev15YtWwb1dY76m01hYWHkiwHyxhgZZIxMMkZO9ebL3ApqaWkZnA8NDoD09HRNnTpVmzdvdh1LTEzU/PnztXbt2l7nr127Vl988YXOnDnjOvbSSy+pvr5eX3/9tU9m9gZ392AkLS1Ns2bN0rp167w1pld5uoPFixdr2rRpcjqdOnTokKqrq30xrte4u4ejR4/q2WefVW1trcLDw305qte4u4MtW7Zo586dOnfunOvYp59+qjfffFOXLl3yyczeNGbMGL3//vtatGhRn+d89NFHeuedd/TLL7+4Qstut6ukpEQ//fRTQH7iHPlijLwxRgYZI5OMkVO35u3csvyZJ6sMlov0epsnezDS2tqqsLCwgR7PJzzdwe7du9Xc3KycnBxvj+gTnuyhoqJCM2bM0LZt2zRlyhQlJiYqNzdXra2tvhh5wHmyg5kzZ8rhcOirr76S0+nUtWvXdODAAc2ePdsXI/uF06dPa9asWT0e7UtPT9fly5f1+++/WziZNcgXY+SNMTLIGJlkjJwaGP3JrYAtT1wIsYsne/hfxcXF+vPPP/XUU095Y0Sv82QH58+fV2FhoXbt2qXg4GBfjOl1nuzhwoULqqmp0blz57R3717Z7XYdOXJEL7zwgi9GHnCe7MBms2n37t1avny5IiIiFBcXJ6fTqR07dvhiZL/Q131j99cCDflijLwxRgYZI5OMkVMDoz+5FbDlqRsXQuzi7h66lZeXKy8vT7t27XK9mXqwMruDv//+W8uWLdO7776r8ePH+2g633HntvDvv/8qKChIxcXFeuCBB5Seni673a5Dhw4N6l+a3dlBfX29Vq1apZycHB0/flz79++Xw+HQypUrfTGq3xiq9439Qb4YI2+MkUHGyCRj5FT/eXqfO8xrE/k5LoTYxZM9dCsvL9eKFStUVFSkuXPnenNMr3J3B01NTaqvr1d2drays7Mldd1hO51OhYeHa9++fb2eTh8MPLktREVFKSYmRqGhoa5jkyZNktT1iWSRkZHeG9gLPNnBxo0blZiYqJdfflmSNG3aNN1222169NFHtWbNGo0dO9brc1utr/tGSf95PzIUkS/GyBtjZJAxMskYOTUw+pNbAfvMExdC7OLJHiTp4MGDysrK0vbt2/v1UfH+wN0d3HXXXaqqqtKJEydcf5YuXap77rlHJ06ckM1m89XoA8qT28KDDz6opqamHq8nb2hokCSNGzfOe8N6iSc7uHHjRq+XzXT/fbB+VLe7bDabqqurdfPmTdexY8eOKSYmRnfffbeFk1mDfDFG3hgjg4yRScbIqYHRn9yy9CK5VuNCiF3c3cP+/fu1fPly5efna86cOWpra1NbW5s6OjpMfTykP3JnB8HBwYqIiOjx58yZM2poaNDq1as1YsQIq38cj7l7W7j33ntVWlqquro6xcfHq6GhQTk5OUpKSrrlp9z4M3d3cOPGDW3ZskXh4eEaPXq06+URUVFReuWVVyz+aTzT2tqq+vp6ORwOffLJJ5oyZYpGjRql9vZ2hYaGKj8/Xxs3blRmZqYkKS4uTnv27NHZs2c1ceJEVVdXKy8vTytXrrzlL8VDGflijLwxRgYZI5OMkVO9+TK3AvZle1LXhRCvX78uu90uh8OhhISEXhdC/P+6L4T41ltvqaSkRNHR0f2+SK8/cHcPJSUl6uzs1OrVq7V69WrX8aSkpF7XPhks3N3BUOXuHm6//XZ9/vnnys3NVVpamsLCwjRv3jzTHznsj9zdwaJFi9Ta2qri4mK9/fbbGjVqlFJSUpSfn2/F+AOitrbWFbqSVFBQoIKCAmVmZmrHjh1qamrSb7/95vp6aGioDh48qDfeeEOPPPKIwsLClJ2drRdffNGK8f0C+WKMvDFGBhkjk4yRU735MrcC+jpPAAAAAGBWwL7nCQAAAADcQXkCAAAAABMoTwAAAABgAuUJAAAAAEygPAEAAACACZQnAAAAADCB8gQAAAAAJlCeAAAAAMAEyhMAAAAAmEB5AgAAAAATKE+AH7px44ZsNpsSExPV1tbmOt7W1qYZM2bIZrPp5s2bFk4IAAg0ZBNAeQL80siRI1VUVKQ//vhDeXl5ruNr1qzRxYsXVVRUpJCQEAsnBAAEGrIJkIZZPQAAY4mJiXr11Vdlt9s1b948SVJJSYlyc3OVmJho8XQAgEBENiHQBbW0tDitHgKAsY6ODmVkZOjq1atyOp2KiIjQN998o+HDh1s9GgAgQJFNCGSUJ8DPnT9/XklJSRo2bJhOnjyp+Ph4q0cCAAQ4sgmBivc8AX7u6NGjkqTOzk79/PPPFk8DAADZhMDFM0+AH6uvr1dqaqoee+wxXbp0Sb/++quqq6sVERFh9WgAgABFNiGQUZ4AP9XZ2amMjAw5HA5VVVWppaVFycnJevjhh1VaWmr1eACAAEQ2IdDxsj3AT23YsEF1dXXatGmT7rzzTk2YMEH5+fmqqKjQZ599ZvV4AIAARDYh0PHME+CHfvjhB2VkZCgzM1ObN292HXc6nVq4cKHOnDmjqqoqjRkzxsIpAQCBhGwCKE8AAAAAYAov2wMAAAAAEyhPAAAAAGAC5QkAAAAATKA8AQAAAIAJlCcAAAAAMIHyBAAAAAAmUJ4AAAAAwATKEwAAAACYQHkCAAAAABMoTwAAAABgwv8B6/WSA9AJJ80AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x432 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "figure1(x_train, y_train, x_val, y_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient Descent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 0: Random Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.49671415] [-0.1382643]\n"
     ]
    }
   ],
   "source": [
    "# Step 0 - Initializes parameters \"b\" and \"w\" randomly\n",
    "np.random.seed(42)\n",
    "b = np.random.randn(1)\n",
    "w = np.random.randn(1)\n",
    "\n",
    "print(b, w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1: Compute Model's Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 1 - Computes our model's predicted output - forward pass\n",
    "yhat = b + w * x_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Compute the Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.7421577700550976\n"
     ]
    }
   ],
   "source": [
    "# Step 2 - Computing the loss\n",
    "# We are using ALL data points, so this is BATCH gradient\n",
    "# descent. How wrong is our model? That's the error!\n",
    "error = (yhat - y_train)\n",
    "\n",
    "# It is a regression, so it computes mean squared error (MSE)\n",
    "loss = (error ** 2).mean()\n",
    "\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Compute the Gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-3.044811379650508 -1.8337537171510832\n"
     ]
    }
   ],
   "source": [
    "# Step 3 - Computes gradients for both \"b\" and \"w\" parameters\n",
    "b_grad = 2 * error.mean()\n",
    "w_grad = 2 * (x_train * error).mean()\n",
    "print(b_grad, w_grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4: Update the Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.49671415] [-0.1382643]\n",
      "[0.80119529] [0.04511107]\n"
     ]
    }
   ],
   "source": [
    "# Sets learning rate - this is \"eta\" ~ the \"n\" like Greek letter\n",
    "lr = 0.1\n",
    "print(b, w)\n",
    "\n",
    "# Step 4 - Updates parameters using gradients and \n",
    "# the learning rate\n",
    "b = b - lr * b_grad\n",
    "w = w - lr * w_grad\n",
    "\n",
    "print(b, w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5: Rinse and Repeat!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Go back to Step 1 and run observe how your parameters b and w change"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear Regression in Numpy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cell 1.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.49671415] [-0.1382643]\n",
      "[1.02354094] [1.96896411]\n"
     ]
    }
   ],
   "source": [
    "# Step 0 - Initializes parameters \"b\" and \"w\" randomly\n",
    "np.random.seed(42)\n",
    "b = np.random.randn(1)\n",
    "w = np.random.randn(1)\n",
    "\n",
    "print(b, w)\n",
    "\n",
    "# Sets learning rate - this is \"eta\" ~ the \"n\"-like Greek letter\n",
    "lr = 0.1\n",
    "# Defines number of epochs\n",
    "n_epochs = 1000\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    # Step 1 - Computes model's predicted output - forward pass\n",
    "    yhat = b + w * x_train\n",
    "    \n",
    "    # Step 2 - Computes the loss\n",
    "    # We are using ALL data points, so this is BATCH gradient\n",
    "    # descent. How wrong is our model? That's the error!   \n",
    "    error = (yhat - y_train)\n",
    "    # It is a regression, so it computes mean squared error (MSE)\n",
    "    loss = (error ** 2).mean()\n",
    "    \n",
    "    # Step 3 - Computes gradients for both \"b\" and \"w\" parameters\n",
    "    b_grad = 2 * error.mean()\n",
    "    w_grad = 2 * (x_train * error).mean()\n",
    "    \n",
    "    # Step 4 - Updates parameters using gradients and \n",
    "    # the learning rate\n",
    "    b = b - lr * b_grad\n",
    "    w = w - lr * w_grad\n",
    "    \n",
    "print(b, w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.02354075] [1.96896447]\n"
     ]
    }
   ],
   "source": [
    "# Sanity Check: do we get the same results as our\n",
    "# gradient descent?\n",
    "linr = LinearRegression()\n",
    "linr.fit(x_train, y_train)\n",
    "print(linr.intercept_, linr.coef_[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGgCAYAAADsNrNZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd1zV1f/A8dcFZCggThyIOHBnKU7ce6ZiuSpRU3HgyNxZGqaZ4sivA0XSwpGKmCs1R4kjtzlJJUHUFEWQpSDr/v7gx43rvWwuF+T9fDx41P3M8+Hifd9zzvuco4iIiFAihBBC5DMDfRdACCFE0SQBSAghhF5IABJCCKEXEoCEEELohQQgIYQQeiEBSAghhF5IABJCCKEXEoCEEELohQSgTAQEBOi7CHolzy/PX9QV9d+BLp/fSGdXFkIIUWgERyew4Eo0T14lUbG4IV82tqCqRTGd3lMCkBBCFHHB0Qn0+y2MoOgk1bZLofHs6VZGp/eVJjghhCjiFlyJVgs+AEHRSSy4Eq3T+0oAEkKIIu7JqySt20PS2Z5X9NYEt2HDBjZt2sTDhw8BqFOnDtOmTaNbt27pnnPr1i2mT5/OlStXKFWqFMOHD2fGjBkoFIoclSExMZGXL19meIypqSmRkZE5uv7bQJ5fnl9Xz1+iRAmMjKQXoCCoWNxQ6/YK6WzPK3p79ytVqoSbmxs1atQgOTmZn3/+mY8//pgTJ07QoEEDjeOjoqJwcnLC0dGR33//nYCAAFxdXSlevDgTJ07M9v0TExOJjo7GysoqwwBmYmKCqalptq//tpDnl+fXxfMrlUoiIiKwsLCQIFQAfNnYgkuh8WrNcNUsUhIR4kPCdHZfvb3zvXr1Unv91Vdf8cMPP3Dx4kWtAcjHx4fY2Fg8PDwwMzOjXr163L17l7Vr1zJhwoRs14JevnyZafARQuiGQqHAysqKqKgoSpYsqe/iFHlVLYqxp1sZFlyJJuRVEhXSZMEFhOjuvgXiq0dSUhJ79uzh5cuXNGvWTOsxFy5coGXLlpiZmam2derUiYULFxIcHIydnV227yvBRwj9kX9/BUtVi2JsaFda9Vqp1P1apXpNQrh16xaVK1emfPnyTJkyhS1btlC/fn2txz579oxy5cqpbUt9/ezZM52XVQghiopr167RqVMn7t+/r9P76LUGZG9vz6lTp4iMjGTfvn2MGzeOAwcOUK9ePa3Hv/mNKTVCZ/ZNSttIXlNTU0xMTLJUzri4uCwd97aS55fn15WoqKhC8QXybZoN4d9YBeseGBH62oByJsmMtU2kslnKZ2lqf/zq1atJTEzkk08+YcOGDTl+fnt7+wz36zUAGRsbU716dQAaNWrElStXWLt2LatXr9Y4tnz58hp/qM+fPwfQqBm9SdsvITIyMkudq3FxcYWqE9rFxYWYmBi2bduWJ9eLi4vj8ePHNG7cmJMnT9KwYcM8uW5uTJkyhcDAQPbu3Zul4xMTEylbtixbtmyhd+/e2bpXQXj/nz59Su3atTl06BAtW7YkMDAw1+9HVn8nun5+S0tLqlSporPr54WAgIBMP0gLi+DoBKaoDTg15E6cKXu6lcHkZRjjxo3jjz/+UB1/8+ZNNmzYwPLly3VSngI1Dig5OZn4+Hit+5o1a8bZs2fVvo398ccfVKxYkapVq+ZXEfVu3LhxWFlZafxcv34dAHd3d9auXavnUhZsiYmJWFlZ8e+//+q7KDlStWpV7ty5k25LwZtcXFz46KOP1LYZGRlx584dunTpoosiigIqvQGn49ftoVWrVmrBJ9WLFy901h+ktxrQ119/TdeuXalcuTIxMTHs2rWL06dPs3PnTgDc3Ny4fPky+/btA+DDDz9k8eLFjB8/nmnTpvHPP//w/fff52ocUGHVvn171q9fr7atTJmUKTMko6hgSkhIoFixvJlXy9DQEGtr61xfJy+uIQoXjQGn8XGwz50zZ7ZrHGtlZcXKlSupV6+ezj5j9VYDevr0KS4uLjRt2pS+ffty5coVdu3apfpGFhISQlBQkOr4kiVL8ssvv/DkyRM6dOjA9OnTcXV1ZcKECfp6BL0xMTHB2tpa7Sd1LMWb33a7d+/OjBkzmDdvHtWqVcPe3p558+aRnJysOmbbtm20b98eGxsb7O3tGTFiBCEh2cu97N69O9OnT2fWrFlUrVqVGjVq4OnpSVxcHFOmTMHW1pYGDRrg4+Ojdt6NGzd4//33qVChAtWqVcPV1ZWoqCjV/sTERGbPno2trS12dnbMmTNHreyQUnNevnw57777LhUqVMDR0ZFdu3Zluezx8fFMnz6d2rVrU758eerXr88333yT7vHe3t7Y2tpy4MABHBwcsLa2pk+fPgQHB6uOWbBgAa1bt8bb25t3330Xa2tr4uLislTWS5cu0aZNG6ytrWnbti1XrlxR2x8YGKhW6wW4ffs2gwYNokqVKlSuXJmuXbty+/ZtFixYwM6dOzl48KCqtnz27FlVLfDAgQMZvhfR0f9NxZL6t7V69Wrq1KmDnZ0dEyZMIDY2VnXMqVOn6NSpE5UqVcLW1pZOnTpx586dLL8XQrfUBpw+vgPLB4KW4NOqVStOnz5N3759dVoevdWAPDw8sr2/fv36HDp0SFdFAlKifk68++67+Pn5ad3Xrl07rl27pnVfREREju6XHdu3b2f8+PEcPXqUq1ev4uLiQqNGjejXrx+Q8iE/Z84c7O3tef78OXPnzmXUqFFqH05Zvc/EiRP5/fffOXDgADNnzuTo0aN07tyZEydOsGXLFiZOnEi7du0oX748MTExfPDBBzRv3pzjx48THh7OpEmTmDx5Mps2bQJg5cqVbN26lVWrVlG3bl3Wr1/P7t27ady4seq+bm5uHDp0iOXLl1OjRg3Onz/PxIkTsbKyonPnzpmWe+3atRw6dIhNmzZhY2PD48ePuXfvXobnxMbGsmzZMjw8PDA2NmbWrFk4Oztz4sQJ1bfFoKAg9u7di7e3N0ZGRhgbG2da1ujoaAYOHEj79u3x9PTk33//ZdasWRmW5dGjR3Tv3p3WrVuzd+9eLC0tuXTpEomJiXz22WfcvXuXV69esWbNGgBKly6tcY303oupU6fi7e2tOu706dNYW1uzb98+Hjx4wIgRI6hVqxaTJk0iISGBjz/+mBEjRuDl5UVCQgJ//fUXBgYFqqW/SPuysQUXn73m/kFv2L8MEtW7PIyMjPjiiy+YPHkyhoa6nQUBCsg4IJE9x44do3LlyqrXLVu2zPAbf/369VUfYjVr1uTHH3/Ez89PFYCcnZ1Vx9rZ2bF06VIcHR15+vRptpr0GjRowIwZMwCYNGkSK1aswMTEhDFjxgAwc+ZMVq5cyYULF+jduzc7duwgPj6edevWUaJECQBWrFhBv379mDdvHnZ2dnh4ePD555+rvom5u7tz/Phx1T2jo6NZt24d+/fvV40hs7Oz49KlS3h5eWkNQEZGRmqB/+HDh9jb29OyZUsUCgW2tra0aNEiw2dNSEjA3d2dJk2aALBu3ToaNWrE6dOnadOmDZBSs1q/fj1ly5bNcll37NiBUqlk9erVFC9enLp16zJlyhRcXV3TLcuGDRsoWbIkmzZtUjXz1axZU7Xf1NSUxMREtSa3xMREtWtk9F7cv39fNc6uZMmSLF26FENDQ2rVqsX777+Pn58fkyZNIiIigqioKHr06EG1atUAqFWrVoa/R5G/rIslYbNtAvf/OK6xr1q1anh5eeHg4JBv5ZEAVAg5OjqycuVK1evMspTeHFtVsWJFQkNDVa//+usvFi9ezM2bN4mIiFA1cT169ChbASjtfRQKBWXLllXrKDcxMcHS0lKVvXjnzh0aNGig+sADaN68OQB3795VHdu0aVPVfgMDAxwcHFTX+Pvvv3n9+jVOTk5qZUlISFBlWGbm448/pn///jRp0oSOHTvStWtXOnXqlOE3dyMjIxo1aqR6bWdnR/ny5blz544qAFWpUkUVfLJa1tTfSfHixVX70xucner69eu0bNkyV31Mmb0XqQGobt26at+MK1asyM2bN4GUbNSBAwfSr18/2rVrR9u2benXr5/al6WiTB/r7bzJxMSECmVKaWwfMmQIS5YswcLCIl/LIwGoECpevHiWP1wBjbm2FAqFKshER0fTv39/OnfujKenJ2XLluXZs2f07t073YzE7NznzQ/FtPdWKpUanZuprxUKRZYyb1KvtWPHDipVqqS2L6sfyI0bN+b69escP36ckydPqpoofX19Mzwvs47ZtEEkr8qqTV5kKGX2XqTK6G8JwNPTkwkTJnD8+HEOHDjAggUL+Pnnn2nfvn2uy1iYZbTeTn4GIYVCwbJly7hw4QIPHjzA0tKSFStW8MEHH+RbGdKSAPSGN/tk8mIcRHp9QwXBnTt3ePHiBfPmzcPGxgZImaEiP9SpUwcfHx9evnyp+uZ97tw5IKXppkyZMpQtW5ZLly7RqlUrIOVD/MqVK9ja2gIp38iNjY159OgRrVu3znFZLC0tcXJywsnJiUGDBtG9e3eCg4OpUKGC1uMTExP566+/VM0V9+/f59mzZxk2OWWlrLVr12bXrl3Exsaqpp26ePFihmV/99132bNnT7qZdsWKFSMpKeNp9TN7L7KjYcOGNGzYkClTptCvXz8JQGS83k7a6W/yQ8mSJfH09OSbb77Bw8ND9W9JH6R3sIiztbXF2NgYT09P7t+/z+HDh1m0aFG+3HvQoEEYGxszbtw4/P39OXXqFJ9//jlOTk6qsV1jx45lxYoV7Nu3j7t37zJz5kxV8xuk/GMaP348X3zxBVu3biUoKIhr167h5eWl1nmekVWrVuHr68vdu3e5d+8evr6+WFpapht8IOVDfcaMGVy8eJFr164xbtw4GjRooGp+0yYrZR04cCAAEydO5Pbt2xw/fpwVK1ZkWP7Ro0cTERHBiBEj+OuvvwgMDMTHx0fVNGZra8utW7f4559/CAsL0+j/gfTfiz59+mR5nF1gYCBubm5cuHCBhw8f4ufnx99//02dOnWydP7bLD/W2wmOTmC0Xzi9D4Uy5GgoXZbuofehUEb7hRMcnaB2bIsWLThw4IBegw9IACryypcvz9q1a9m7dy/Nmzdn6dKlLFy4MF/ubW5ujq+vLy9evKBjx44MHTqUli1bqvVvffbZZwwaNIgJEybQuXNnDA0NNfpQ5s2bx7Rp01i5ciXNmjWjf//+/Prrr1n+4CxRogTff/89HTp0oEOHDvj7++Pr65thzdfMzIzJkyczevRounTpgqGhIZs3b860WS6zslpaWrJjxw5u375N27ZtmTdvHm5ubhle08bGhoMHDxIbG0vv3r1p27YtXl5equayESNGUKNGDdq3b0+NGjW01qjSey+WLVuW2a9PpXjx4ty9exdnZ2ccHBxwdXXlo48+ytFyKW8bXa+3k9rE5xMYy+l/Qjj09WguLhjO6V934xMYS7/fwjSCUEEYP6mIiIjQ/ZSnBVBkZGSWOtgLwlQs+iTPr/n83t7efPnllzx48EBPpco/un7/s/rvUJ/yYioebX1A1SwM1fqAcpOkMNovHJ/AWLjzJ2z7AqL+P8nI1Bym+UIZG3rYmPBzl7IZX0gLXU5FJH1AQgihYxmttwO5T1L4N/IV7F0KJ35U3xEXA7u+gTHr+ePJa4KjE/I98y4jEoCEECIfvLneTlrZTVJIW1sqEX6fm+6T4P7fWm76LnzwJQBxSegl6SEjEoCEyCZnZ2e1wbtC5FZ2khRUtaWoRDjrA3sWQ8IbS2YoDKCLC3QdC4b/1XhCXiUViPFIqSQACSGEnmUnSWHBlWiCQsJgx1y4oTmjgUmZipiPWEJY5cYa+yyKKQrEeKRUkgUnhBB69mVjC6pZqAebahYptZM3/X3hNLg7aQ0+Tk5O3Ll8lt8nd9d6PaWSdJv69KFI14C0jf4WQuQPXa0xUxhllqQAKXMLfvvtt9xcuRLe/N0Zm9FkzNdsnO+CQqHACrRez/W09smP83I8UnYU2QBUokQJIiIisLKykiAkRD5TKpVERETk+9xjBVlGSQoAwcHBrFu3TjP4VKmPzdhl/ODsoPZZpu16uh6PlF1FNgAZGRlhYWGhtvaMNlFRUVhaWuZTqQoeeX55fl09v4WFhcbcciJ99vb2LFiwgGnTpqVsUCiwed+F5s5TmNs8a304Xza24FJovMZ4JG1NffmhSL/7RkZGmQ6Ce/bsWYFfs16X5Pnl+Yvy8xc0I0eO5NixY1y/fp1169bRtm3bbJ2flaa+/FSkA5AQQhREr1+/xsTERGO7QqFg7dq1gPaFBbMis6a+/CRZcEIIUUAkJCSwYMECOnTooLbUeVqlS5fOcfApaCQACSFEARAUFESPHj1YunQp/v7+fPXVV/ouks5JABJCCD1SKpVs376dNm3acOnSJdV2Ly8vDh06pMeS6Z70AQkhhJ5ERkYydepUdu3apbGvQoUKGqvqvm0kAAkhRA7lZl61c+fOMXr0aB4+fKixr1KzzlQZuZAtlMOugM1gnZckAAkhRA7kdAmFxMRE3N3dcXd3Jzk5WW2fiYkp5gNm8bjxhzx+peB8YKxe52rTNekDEkKIHMhoCYX0BAcH06tXLxYvXqwRfOrXr0/7pXsJcxgAaWY00OdcbbomAUgIIXIgO0soAPj6+tKmTRvOnz+vsW/s2LEcP36cl2WrZ+uahZ00wQkhRA5kZ161zZs3M3HiRI3t5cqVw8PDg86dO///NV9l+ZpvA6kBCSFEDmRnCQUnJydq1Kihtq1Lly6cOXNGFXyye823gQQgIYTIgdR51QZUN6NNBWMGVDdLN1nA3NwcLy8vjIyMMDExYfHixezcuZPy5cvn+JpvA2mCE0KIHMrOvGqNGjVi3pIVnDKswf7y9lw6+UJr2nZBmqtN1yQACSFEHtmzZw+vX79m0KBBGvuCoxPYaNklJXMuJB7Q73LYBYEEICGEyKL0Bp7GxMQwc+ZMtm7dSvHixXFwcKBmzZpq52aUtl1UajxvkgAkhBBZkN7A04UVHvLV5LHcu3cPgFevXjFq1CiOHDmCsbGx6tjspm0XBZKEIIQQWaBRg0lOJmjPej7u20MVfFL5+/urTSwKBW857IJAApAQQmSBWg0m4imsGwUHVqBMSlQ7rlatWhw7dgxHR0e17UUtxTorpAlOCCGyQFWDuX4MdsyFV5Eax4wYMYKFCxdqncW6oC2HXRBIABJCCC1SEw4Cn5tQ/XE4g22UHF7mRvTJnRrHlipVilWrVtG7d+8Mr1mUUqyzQgKQEKJQyM3SBzm5138JB4Zc9r/Cri0zUD4N1Di2Was2lB+5mHWG5djrF17kazXZIQFICFHg5XTpg5xSSzg4vQ32LNbo6zEyMmLCjC/YYzeYC5FKiJSxPdklSQhCiAIvJ0sf5IZawoGxGbwRfMwr2XH06FH+bT6c+y+V+Vaut40EICFEgZffY2jUUqab9oP3uv/3unl/Gi7cQ6NGjWRsTy5JE5wQosBLbwyNRTEFo/3C87xf6MvGFlwKjU+pdSkUMGAuPAuCLmPgvW5ULm2WYbmK8tie7JAakBCiwNM2hsamhAHXw+LxCYzldEjKf/v9FkZwdEKO7uHv709MTAzwX8p0DxsTjBVKKF4Spu6C97qpjd2RsT25IwFICFHgaVum4J1SxXj0Kvf9L0qlknXr1tGhQwdmzpypds+fu5RlZ+O4lPtWMtVYHqGoLZ+Q16QJTghRKLw5hqb3oVCtx2Wn/+XZs2e4urpy9OhRALZu3Urnzp1xcnJSHVPZTMmGhumP3ZGxPTknNSAhRKGU2/6Xo0eP0qpVK1XwSTV58mTCw8NzXT6ROQlAQohCKaf9L3FxccyaNYsBAwYQGqpei7K0tGTFihWULi01mvwgTXBCiEIpJ3Or/f3334waNYpbt25p7CtT14Fq45ZyuGxVmkQnSD9OPpAAJIQotLLa/6JUKtm4cSNz5swhLi5ObZ+BgQGWvccT1nY0YUlGXAqMldkM8onemuCWL19Ohw4dqFKlCjVq1GDQoEH4+/tneE5wcDBWVlYaP8eOHcunUgshCpuwsDA++ugjpk6dqhF8bG1tafftDiI6jAPD/76Py2wG+UNvAej06dOMHDmS3377jX379mFkZES/fv148eJFpuf6+vpy584d1U/btm3zocRCiMLmjz/+wNHRkUOHDmns+/DDDzl16hQJVd/Teq7MZqB7emuC2717t9rr9evXY2try7lz5+jRo0eG55YuXRpra2tdFk8IUQhkNEN2UlISc+bM4enTp2rnGJmZ4/bdEnr3H8C0K9HciUjUdmmZzSAfFJg+oJiYGJKTk7Gyssr02KFDhxIXF0eNGjUYP348ffv2zYcSCiEKksxmyDY0NGTDhg106NiR+NevUw6o+i6JnyzGw6wqHgdDNQayppLZDPKHIiIiQvs7kM+GDx/OvXv3OHHiBIaG2r95hIWFsW3bNlq0aIGRkREHDx5k2bJleHh4MGjQoHSvHRAQoKtiCyH05Ks7xTgcqpkk0LZ0IsvqxateO6/axd+b3aHzaOg2DgzTTywobZRMs1JJjLVNpLJZgfhoLNTs7e0z3F8gAtAXX3zB7t27OXz4MHZ2dtk6d+rUqZw9e5Y///xTJ2ULCAjI9Jf4NpPnl+cvqM/f+1Aop0P+P9AkJaqSCEwN4Hz/8qqmuF4Hn3Hmyg2oXCfTa7apYMz+HuXUthXk30F+0OXz630g6uzZs/H19WXfvn3ZDj4ADg4OBAZqrlIohHi7qWZCCDgP370PT1JaOuKSUctgq1TCKEvBB6TfJ7/pNQDNnDmTXbt2sW/fPmrVqpWja9y4cUMSEoQogmY2MMHw1xXgMRKeP4DN0yE+Jc06bQabthkTtJF+n/yntySEadOmsWPHDrZs2YKVlZUqU6VEiRKYm5sD4ObmxuXLl9m3bx8A27Zto1ixYjRs2BADAwMOHz6Ml5cXX3/9tb4eQwihY9oy3RKfPWD0qFEk/fXXfwc+CYADy6H/F2o1mbQzJpx4HEdonGavg625oQw81QO9BSAvLy8AjQy2mTNnMnv2bABCQkIICgpS27906VIePnyIoaEhNWrUYPXq1RkmIAghCi+NTDelkj9++ZmXOxcS++qV+sEKBRibYWduoFGTSZ0xQVvmXDULCT76orcAFBERkekxHh4eaq8/+ugjPvroI10VSQhRwCy4Ev1fsHgVCT7zeX71sMZxZmUqUHOcO3WatspwPrjU2tCsc5Fcep4AKKlrVWBGoxQ58psXQhRYT1L7cu5dhi0zICJE45j333+flStXZmsG69uRiYTGJQNw8OFr/o4Ik1qQHug9C04IUbAFRycw2i+c3odCGe0XnuMlr3PC2jgZDv4P1gzXCD7Fixdn5cqVeHt7Zyv4qNWq/p/M/aYfUgMSQqTr31gFUzKYbUCX7t+/z90FI+Gvyxr76jR4B++NP+Qoe/ZJOnO8ydxv+U9qQEKIdK17YKSX2sKjR49o06YNN7QEH2eX8fgdP5bjoRu5XUlV5B0JQEKIdIW+1v4Roevago2NDb1791bbVqFCBX755Rf+t+RbTExMcnztnK6kKvKeBCAhRLrKmSRr3Z4ftYUlS5ZQrVo1ALp3787p06fp0KFDrq+bmgk3oLoZbSoYM6C6mSQg6In0AQkh0jXWNpE7caYa42byo7ZgYWGBl5cXx89e4m6DDxl+MZmKxcMzXXY7K7K6kqrQLQlAQoh0VTZTqmYRCHmVRIU31tzJrQcPHrBjxw6mTZuGQqHQ2F+2VkN+DqpMUNB/K5nKctlvDwlAQogM6aq24Ovry5QpU4iKiuJIjBUmLfppLCqXUcq01GAKPwlAQoh8FR0dzfTp09m+fbtq28V186BEfShXVa2GIynTbzdJQhBC5JtLly7Rpk0bteADwOtXcNYHUE/zlpTpt5sEICGEziUlJbF06VK6devG/fv31XcaGYPTF/D+VNWm1BpOQUqZVir1vnbnW0ea4IQQOvXo0SNcXFy0rlpsaVuLqEGLoZL6oNLUGk7apRR0kQSRVUlJSRgappQpIiKC4sWLY2xsnK9leBtJABJC6MzevXuZNGkSkZGRGvtGjx7NqOlfMfjkqwzTvAtCyrShoSFJSUmMGzeOhw8fEh8fj4uLCx07dqRcuXIkJydjYCANStklAUgIkW3aFolLWyuJiYlh9uzZbN68WePcMmXKsGbNGrp37w7Anm5meq/hZCY8PJxBgwZhZGTEqFGjOHDgACtXrmTv3r1s27YNAwMDlEql1lRykT4JQEKIbNG2qFvazLUbN24wfPhw7t27p3Fuhw4d8PDwoEKFCqptBaGGk5m///6bFy9esHPnTqpXr84HH3zAzp07mT9/PrNnz2bRokUSgHJA6oxCiGzJbDkDU1NTnjx5ora/WLFifPPNN/j6+qoFn4ImOTlZ6+uXL1/y7NkzTE1NVft69erF+PHj2bBhAxcvXsTAwEDjfJExCUBCiGzJbGyOvb093333nWq7vb09x44dY+LEiQW6nyRtP07qis1py1uxYkUuXryoel2iRAl69+5Nly5dWLRokcbxInPy2xJCZEtWxuYMHTqUPn36MGLECPz8/Hj33Xfzq3g5ZmBgwKlTp+jQoQPDhg1j+PDh7Nu3D4CuXbtibGzML7/8QkjIfwvj2dra0q5dOyIjI3nw4IG+il5oSQASQmSLamxOfCw8T/nQfTNzTaFQsHHjRlasWEHx4sX1VdRs2bZtG87OznTr1o1PP/2UYsWKMXv2bHbv3g3A/Pnz2bt3LwcOHCA6+r/1kGxtbblx44b0/+SAJCEIIbKlqkUxvrN5zKivR5OYlEz37/fztaPm5KBGRgX34+XNhIGkpCT279/PZ599xuTJkwFo3bo1w4cPZ/ny5XTr1o0OHTowcuRI1qxZQ3JyMi4uLgDcu3eP1q1bY2VlpZdnKcykBiSEyLLk5GTWrFnD0D7diX50j9gnQVj8uqTApU1nJDk5WaO2EhoaSlBQkNoxZcqUoXTplOy8KVOmALB06VIcHR1Zs2YNjo6ODBo0iAULFuDk5ISFhSxol10SgIQQWRISEsKHH37InDlziI+PV2339vbm0KFDeixZ1qROpWNgYMDVq1eZMnvR7y0AACAASURBVGUKJ0+eBMDc3JzSpUsTHByMv78/BgYGnDx5krt37/L5559z7949fHxS5qr77rvv8PLywsnJidq1a3P27FmGDh2qt+cqzApuHVkIkW2ZDRDNqcOHD+Pq6kpYWJjGvk8++YQ2bdrk+h66llrrWbBgAf/73/8YMWIEJiYmvH79GnNzc1xcXHB3d+fIkSNUrVqVs2fPMn/+fDp27MiJEydUyQfm5uY0bdqUpk2b6vNx3goSgIR4S2Q2QDQn4uLimD59Ohs2bNDYV7JkSVauXEm/fv1yXOb8dvDgQY4dO4aPjw/t2rVT29evXz8qVarEtWvX+Pfff1m3bh1VqlQhICCAkJAQXr58CSDJBnlIApAQb4m8Xrzt5s2bDBs2jMDAQI19jo6OeHp6YmNjk+Py5qfUpIMTJ05Qu3Zt2rVrx7lz59ixYwclS5akatWqjBgxgmbNmtGsWTO1c//++28sLS3p1q2bnkr/9pI+ICHeEnm1eJtSqWTdunV06tRJI/gYGhry1VdfsX//fo3gExydwGi/cHofCmW0XzjB0QnZe4A88uayCanBJykpiVu3btGiRQuOHDnCkCFDSExM5MaNG8yYMQM3NzfVAFSAf/75h99//x03NzcqVKhA9erV8/tR3npSAxLiLZEXi7eFhobi6urKkSNHNPbZ2dnh5eVFkyZNNPbpovkvJ9LOZhAcHEzVqlVV+wwNDWnQoAHe3t7Y29uzePFiBg4cCICPjw+urq60a9eO9u3b8+LFC37//XcWL15M9+7dWbNmTb49Q1EiNSAh3hJ5sXjbqlWrtAafwYMHc/LkSa3BBzKfHy6/GBgYEBQURI8ePRg+fDh9+vTBzc1Ntd/BwYGYmBgOHz5Mw4YNVdsHDBhAkyZN2LlzJwClSpWib9++HD9+nM8++yxfn6EokQAkxFsidfG2AdXNaFPBmAHVzbJdA/lowlQsbGqqXhcvUQIvLy/WrVuHpaVluuflVfNfbl2+fJn333+f+vXrs3DhQrp06cJPP/3EhAkTiImJoVOnTrzzzjtER0cTHh4OQHx8PMnJyZQsWRIzMzMgpdnO2toaOzu7fC1/USNNcEK8RXKztEFwdAJDTsYSPXgxrBgMVepjOeo7mnZzyPTcvGj+ywsnT57EwcGBJUuWYGBggKOjIwBubm7UqlWLSZMmMXbsWAIDA3FxceHw4cNYWVlx//59goODGTBgACCZbvlFakBCFEFKpVKjs17VjFa5Dkz4CSb8REiJKllqRsuL5r+8cOPGDcLDw9WWRkhOTqZUqVKsXr2amzdv0qxZM5YsWYK1tTVt27bFycmJjh070qZNG/r375+v5S3qJAAJUcSEhYXx0UcfsWnTJrXtas1odu+CYUoDSVaa0fKi+S8vNGrUiKioKPbv34+BgQEPHz5k165duLu7Y2pqire3NwDNmjXj0KFDbNiwAWdnZ/bu3cvixYvztaxCmuCEKFJOnDjB2LFjCQkJ4cSJEzg6OlKnTh0g981o+lzZNDXVukOHDvj7++Ps7EyLFi24cOECAwcOpF+/fgQGBnL48GHi4+MxMjLC2NiYTp066aW8IoXUgIQoAuLj45k7dy79+vVTTSkTGxvLqFGjiIuLA7Q3o5kZJBMUnVigxvVok9pn06BBA7777ju8vb3p0aMHv//+Ox4eHgDcvXuX4sWLY2xsLAvHFRBSAxLiLRcQEMCoUaO4du2axj5TU1MiIiKoUKGCqhltwZVogqIS+DsikZeJBlwKTfj/n/wf15N22YTLly9Trlw5bG1tNZZTSKtkyZK8//77atsiIiKIjIyke/fuOi+zyDr5GiDEW0qpVOLt7U27du00go9CoWDatGkcOnSIChUqqLanNqNVsyzGy0T16+ljXI9CoeDly5cMGzaMzp074+PjQ3x8PAqFIks1I39/f86cOcOwYcMICAiQJrcCRgKQEG+hFy9eMGzYMCZNmsSrV6/U9tnY2HDgwAG+/PJLihXTXpspKON6Xrx4wdSpU4mMjKRz584cPXqUEydOAJqp0qlZb2k9evSIMWPGUK5cOc6dO4e9vX1+FFtkkQQgId4yJ0+epFWrVuzbt09jX79+/Th9+jStWrXK8BoFZVyPQqGgbt26jBkzhjVr1hAXF8euXbtUc9SlrQWl9utcvnxZta1r1674+vri5eVVoFdoLaokAAnxlkhISMDNzY2+ffvy+PFjtX0lSpRg1apVbNq0KUtLRxeUcT1WVlY4OzvTo0cPypcvz5QpU7h48SL79u3j9evXGk1xhw4donPnzuzevVu1rXbt2vlaZpF18pVAiLeAUqlkwIABquaptBo1aoSXlxc1atTI8vVSExJm+v3LS8MSVMjDxe2yq1SpUkDKM/bt25dTp06xd+9eateuTY8ePdSa4mxsbOjdu7csj11ISA1IiLeAQqHgo48+0tg2ZcoUfvvtt2wFn1RVLYrxTe0E9vcox4Z2pfUSfNJKrel8+eWXKBQKdu3axb179wBUS2u/8847/PTTT3Tp0kVv5RRZJwFIiLfEwIEDVcsLVKpUib179zJv3jyMjY31XLK8YWBgQGJiIlZWVsyYMYPr16/j5eVF7969mTBhAvfv30epVMoYn0JE3ikh3iJLly5l9OjRnD59mrZt26q2F5TF4rRJSsp6Zl1qIkH37t0pU6YM69atw9jYGD8/P+zs7GQS0UJGApAQhUhCQgL/+9//iIyM1Lrf0tISd3d3Spf+b0qc1MXifAJjOR0Sj09gLP1+C8swCKUGrLHXTXQasBITEzE0TEl22L17N/fv38/0nOfPn9O7d2+uXr2Kh4cHu3fvVvUTicJFApAQhcT9+/fp2bMnc+fOZfr06Vk+L7uLxaUNWJejDLMUsLIrdcyOkZERMTExfPLJJ4wcOZKgoKBMzzUzM6NOnTpcvXqVwYMH51mZRP6TACREIbBjxw7atGnDxYsXAdi5cyc7duzI0rnZHVSq69VNExMTVf00vr6+vPfee7x+/ZqrV6/SoUMHjYGzbypRogRLly5Vm8FBFE56C0DLly+nQ4cOVKlShRo1ajBo0CD8/f0zPe/WrVv07NmTChUqULduXRYvXpylKTmEKIwiIyNxcXFhzJgxREerB4Bp8xfT68CTTJvIsjuoVNezIBgZGREXF4ezszPTp09nxowZ+Pj4ULVqVVavXk3z5s1VE6aKt5veAtDp06cZOXIkv/32G/v27cPIyIh+/frx4sWLdM+JiorCycmJ8uXL8/vvv/Pdd9+xatUqVq9enY8lFyJ/nD9/njZt2rBz506NfcXf7UD02J84E5qcaRNZdgeV6noWhP3791O/fn2ioqI4duwYLi4uPHv2jD59+uDp6cmcOXOkdlNE6C0A7d69m08++YR69epRv3591q9fz/Pnzzl37ly65/j4+BAbG4uHhwf16tWjb9++TJ48mbVr10otSLw1EhMTWbx4MT179uTBgwdq+0xNTWk8dj6vhq0C8/8SDTJqIsvuYnG6nAXh+vXrrF27ls8++4w9e/ZQvXp1Nm3aRNOmTSlXrhxHjx6Vfp0ipMDMhBATE0NycnKG04RcuHCBli1bYmZmptrWqVMnFi5cSHBwMHZ2dvlQUiFyJzg6gQVXonnyKomKb8wwEBwczJgxY7R+EatXrx4//PAD0++XhZB4jf0ZNZFlZ7G4tMsyBIbFUL2MebZmQchoqYSGDRuydetWSpcuTXh4OC4uLty6dQt3d3fVGCZRdBSYADRr1izeeecdmjVrlu4xz549o1KlSmrbypUrp9onAUgUdKkZZmk7+VPX2bl0ZB9TpkwhKipK47yxY8fy9ddfY2pqSsVn4VqvnZcThaYGrICAMOztbbN8XkbBJ1Xp0qU5deoUI0aMoHnz5hw7dozKlSvntsiiECoQAeiLL77g3LlzHD58WDUmID1v/nGnNr1l9EcfEBCQq/Ll9vzCTp4/757/qzvFCIpWr0kEPY+mx0fTeXxqv8bxpUuXZu7cubRq1YqHDx8C8HEpBWdNTXgU918Luo1pMh+XCicgICzPypoqu88fGRnJ9u3b6dWrFzY2NlqP+ffff5k9ezZt27bl1atXBf5vrKCXT9dy+vyZLX+h9wA0e/Zsdu/ezf79+zOtwZQvX55nz56pbXv+/DnwX01Im9ysARIQEFCk1xCR58/b54/5JxR4o/nM0IjIB/9oHNu5c2fWrl1L+fLl1bbbA79WS2nGC3mVpNOJQrPy/ElJSWpfHH/99VeOHTvGwIED0z23MP1Nyb8B3T2/XscBzZw5k127drFv3z5q1aqV6fHNmjXj7NmzqjXsAf744w8qVqxI1apVdVlUIfKE1gwzI2NaT1+p6ts0NjZm0aJF7Ny5UyP4pEptIisIE4UaGhoSGhrK9evXAejVqxdmZmbs359So8vOVDuiaNFbAJo2bRrbtm3Dy8sLKysrnj59ytOnT4mJiVEd4+bmRp8+fVSvP/zwQ8zMzBg/fjz+/v7s27eP77//nvHjx8scUKJQSC/DbEnfRnz77bfUqVOH33//nXHjxhWaSTVDQkJwcHBgzJgxqkXwxo8fj6+vL2FhYRgaGkqWqtBKb3/hXl5eREdH07dvX2rXrq36WbVqleqYkJAQtak5SpYsyS+//MKTJ0/o0KED06dPx9XVlQkTJujjEYTItuTnj9JNiR4+fDgnTpygQYMG+i5mlqTtf23VqhWmpqZMmDCBU6dOUbduXRo0aMCBAwdUxwjxJr31AUVERGR6jIeHh8a2+vXrc+jQIV0USQidiYmJYfbs2ezYsYNjx46xoV1DjWMUCgWmpqZ6KF32+fn5sW3bNtavX4+1tTX29vbUqlWLAQMG8M033/DBBx8QGxvLw4cPSUpKwsDAQIKQ0FA46vhCFGJXr16lXbt2bN68mfj4eEaNGpXpfGcFmVKpJCIigqNHjzJmzBiioqJwdnbG09OTIUOG4OzsTEBAADdv3uSXX34hPj5ego/QSgKQEHnkzTV3giJfs3LlSrp06aJauRPg7t27zJ07V48lTV/avhptyQOp43z69u3LgQMHOH/+PNOnT0epVDJixAi+//57PvnkEwYPHkz58uUJDAxkz549+fkIohDRexq2EG8DjQGmEU/ZP2sOcX+f1TjW3t6eoUOHpnud9GZJ0LXk5GRV4sO6deswNDSkdevWasek1mSSkpKoV68eK1aswNvbm48//pg+ffoQHh7OkydPaNKkCT/88ANHjhxhyJAh+VJ+UfhIABIiD6gtYXDjd9jxFXEvNfs5R4wYwcKFCylevLjGvoxmSciPIGRgYEBISAjOzs5cv36d3bt3q1YgfVPquJ/UGe0XLFjAsmXLsLa2xtXVFYAGDRoUmoQKoR/SBCdEHnjyKgniY2GnG2ycCG8En1KlSrF582ZWrFihNfiA7tfhyYyXlxcODg7Y2NgQGBiIo6Njls6rWbMmP/74I7169eLp06fcvHlTxyUVbwupAQmRB8xC7sCySfAsUGNfmzZtWLduXabznaW3Dk9QlG6Ww07rxYsXTJ8+na5du7Jx40YADhw4QHBwMI8fP6Zhw4aUKlVKY9YD+G8mhOXLl/PVV19Ru3ZtnZdXvB0kAAmRC8nJyXh4eHDCzQ3iNafYmTxjNnOnfZbpHIeQ/jo8f0ckEhydkGfNcGn7eiAlgJQqVYoFCxawYsUKtm3bxubNm3nx4gWvX7/m1atX2NraquZqfHPC0dRnK1++fLozNwihjTTBCZELf/75J3PmzCHhjeBjXrEqW/ccxG3m1CwFH0iZJaGEkWa68stE8qQZLjXDzcDAgNjYWP755x/VTAUArq6uVK9eHVdXVxo2bMj27dtZu3YtGzduJCQkhIkTJ+a6DEKkJQFIiFxo3bo1w4YNU9v28ccfc/viGXq1SX9pEW2qWhSjbintjRK5XQ47ba1l+/bttGvXjuHDh9O+fXu2bdumOm7r1q18+umnTJgwATs7O8qWLUurVq2YNWuWqklOxvSIvCJNcELk0rfffsuff/7J06dPWblyJU5OTjm+VjULIy6Favb55HatH4VCQXx8PJ9//jmHDx9m3rx5VKpUiWPHjjFjxgz69OmDubk55cqV4+uvv8bCImX109RaU0xMDGZmZjKnm8hTEoCEyKL0FlsrUaIE3t7emJubU6VKlVzd48vGFlwKjVfLhsur5bB//fVX/vnnH/bv30/dunWBlP6gmzdvYmxsrDrO3Nxc9f+pzxsYGEjHjh2xtc364nRCZEYCkBCZUCqVbNiwgYsXL+Lp6ak1CKV+oOdW2uWwc7rWz5uBMjVLzcnJidKlS1OzZk3VPh8fH6Kiovjuu+9o1KgR77//vurcf//9l1u3buHm5saVK1fw8vIqNDN0i8JBApAQGQgPD2fw4MH89ttvgPY+n7yWutZPTqRNk3706BGlS5fGwMBAta1du3YAPH78GGdnZ0JDQ+nbty+PHj1ixYoVrFy5EmdnZx49eoSvry/r16+nYcOGnD9/XtUsJ0RekQAkCqX8mLLm+PHjjB49mvDwcNW22bNn4+joWGBXyDQ0NCQ5OZmJEydy8+ZNkpOTqVKlCt7e3hgZGalqR3Fxcbi4uNCjRw9VYClRogSenp44OztjY2NDp06dsLGx4YMPPtDzU4m3ldSnRaGTOmWNT2Asp0Pi8QmMpd9vYQRH582AzdevX/PFF1/wwQcfqAUfACMjI+7fv58n99GF69ev4+joyP3793Fzc2PQoEHcuHGDSZMmAf8lFVSvXp2BAwdiYWGh2pY6Q0NUVBSQMpVOw4aay0YIkVekBiQKnYymrMlp01Wq27dvM2rUKK3TyTRv3pz169djZ2eXq3voysuXL/nhhx9o3LgxS5cupXjx4rRv356SJUuycuVKQkNDKVeunMZ5CoWCJ0+ecPPmTYYMGYKlpaUeSi+KIqkBiUInvSlrcjNWRqlUsnHjRjp06KARfAwMDJg1axa//vprgQk+ycnJGttMTEx455136Nmzp9p8cwYGBhgZGWkEn8TERG7dusXu3bvp2LEjxYsXT3eWbiF0QWpAotBJb8qanI6VCQsLY+LEiRw8eFDzXhUrsmnTJlq0aJGja+uiryrtVDrnz5+nTJky1KxZEyMjIz799FPVvtSEhKioKCwtLUlOTiY5OVk1w/WtW7fw9PTEz8+PyZMnM3bs2FyVS4jskgAkCoW0H+QWRgpsShjw6OV/tYCcjpXx8/NjzJgxhISEaOz78MMPGT9+PI0bN85xmXWxvIKBgQHnz5/HxcUFIyMjnj59So8ePRg6dCht27ZV9emkplPfvn2bmjVrYmBgoJZGXbduXUaOHMn8+fMpU6ZMjssjRE5JE5wo8N5MOjj06DUolfSsYkKbCsYMqG6W7Q/14OgEOs5aRd++fTWCj7m5OR4eHmzYsCFXqce6Wl7h4cOHzJgxg/79++Pr68vGjRsJCgpiwYIFXL58GYVCoVZLunbtGk2bNgUgLi6Or776ijNnzmBsbEzjxo0l+Ai9kRqQKPC0fZA/eqWkZQUDtnXOPOngzWaw4bXMmHAmkqAyTcCkOLx+pTrWwcEBLy8vqlWrluty66KvCiAoKIjAwEB+/PFH7OzssLOzw9zcnJUrVzJnzhwOHjyIoaEhCQkJhIWF8fz5c1q0aIGfnx8TJ04kOTmZCRMm5KoMQuQFqQGJAi83H+TaUrYHHgtPCWhlbeGDL1MOVCioO8CVw4cP50nwgbzvq0oVHx9PhQoViIyMVG1zdHRk8ODBREREsHz5cgCKFSvG06dPUSqVzJ8/HycnJ4YMGcLNmzextrbOVRmEyAsSgESBl94HeXBMEr0PhTLaLzzdMUDaak8vE9O8aNIH2g+D8Zso2/8zihXLu8GsXza2oJqFetnzYl63qlWr8vTpU65evaqWDde+fXtat27NqVOnePz4MQD//PMPjx8/Jjw8nHPnzjF79uxc3VuIvCQBSBR42j7IjRTwICYp04GoT14lwT8X4MEN7RdXKKDvDKjZNNc1kzelzus2oLpZjvuq3pScnIy9vT39+/dn+fLlPHjwQLWvVKlStG7dmjt37qgCaePGjdmwYQO//fYbtWrVyvUzCZGXJACJAu/ND3Jbc0MS31gVQFvnfkJCAs93LYO1n4L3dIh7qdr35sJveTXjtLayb2hXmv09yrGhXel0g8+b43qSkrQ3L6Zmtn399dckJyezaNEinj17ptpfoUIF4uLiCAsLA6BatWp8+OGHefEoQuQ5CUCiUEj7QW5rrr2mkrZPKDAwkG7dunHbdx0olRD2EHYvBMDEAJqWK0YPm5xn0eWltBlrP/zwA9HR0RgaGmoNQqkByMrKirVr1/LLL7+wZMkSTp8+TUREBD/99BONGzematWq+foMQuSEZMGJQiejzn2lUsnPP//MjBkziImJUT/g0j5o58zrynU48SSeahaGeg08qQwMDHjy5AkTJ07k+PHjHDp0iF27dmW4lHdycjJt27ZlxYoVbN++nSFDhlCxYkUSExPZvHkzZmZm+fgEQuSM1IBEoZNe5/6kGsmMHDmS8ePHawQf09LWMM4LKtdRbcuLMTl5ITw8nCVLlmBoaMj8+fM5ffo03t7eQMp0OWmlNtWl1pg+/vhjvL292bVrF4sWLeLKlSvUr18/fx9AiBySGpAodLQt2tYz8RZDuo/j0aNHGsf36tWL533ncf5lCY19uR2TkxdSF4nr2bMnzZo14/nz58ycOZNevXpRpkwZtTV+UgPPr7/+Su3atalZsyalSpWiefPm+nwEIXJEakCiUErtE/qlSynsTnswckBfjeBjZmbG999/z5YtW7C1Lqv1Onmd+ZZdqTWa0aNH06VLF0qWLImzszO1atVixIgRABpNcZcvX+aTTz5h8+bNJCTkzRIUQuiDBCBRaN2/f5+ePXvi7u6ukUX2zjvv4Ofnx/Dhw1EoFDobk5NbqTUaY2Nj1bbq1aszZ84czp07xw8//ACoZ8U5ODgwceJEnJyc8nTckhD5TQKQKJR27txJmzZtuHDhgsY+V1dXjh07pjbuRRdjcnRFoVDQsmVLXF1dmT17Ns+fP8fQ0JA9e/bw9OlTAObPn897772n55IKkTvSByQKpcuXLxMdrZ5AYG1tjYeHBx07dtR6TmqzXWFgYWHBsGHDOHv2LH379sXAwIB///2XM2fO6LtoQuQZqQGJQsnNzY169eqpXnfr1o0zZ86kG3z0LW0TWmpzYeqyCekpVaoU1tbW+Pv706RJEwIDA6lYsaJOyylEfpIAJAolU1NTvLy8KFmyJO7u7mzfvp2yZbUnGuiTUqlUy2Lz8PDAy8sL+G9QqTZPnz7FxcWF48eP4+vry4oVK/KlvELkJ2mCEwVaSEgI1tbWWj+s69Wrx40bN7C0tNRDyTKXGngMDQ0JCgpi1KhRPH36lA0bNmR6romJCV27duXnn39WW0ROiLeJ/GWLAsvX15dmzZrh6emZ7jEFMfi8Wev53//+R9u2balTpw5nz56lZcuW3L9/nxcvXqR7vpWVFSNHjpTgI95q8tctCpzo6GjGjRvHyJEjiYqKYu7cudy8eVPfxcoyhUKBoaEhjx49onv37nh6erJ69WrWrFmDhYUFc+bMwcnJCX9//3TPF6IokAAkdC44OoHRfuGZrt0DKdltbdu25eeff1Zte/36NaNGjSI2NjY/ipsnvv/+exwdHbG1teX06dP07duXGzdu4OjoyB9//MGqVato1aqVvosphF5JH5DQWLL6y8YWeTY+JnVF0rSLwl0KjVeNwUm99+OYeCIPe/H3z9+T9Mb8Z8bGxgwbNgwTE5M8KZOuxcfHc+PGDRYtWsTHH38MwIIFC1i3bh3Ozs588cUXmJub67mUQuifBKAiLrMAkVvaViRNnQT0y8YWKfd+8Ai2zoZ7FzXOL1apJls2baBr80a5Lkt+MTY2ZsOGDRgYGODv74+rqyuvXr1i69attGvXTt/FE6LAkCa4Ii6jAJEXnqQz2WfIq5R7BJ0+BEv7aw0+tBpMwuTt+MQVjLVtlEplpmN3UhkYGHD58mVatWpFkyZNOHLkiAQfId6QrRrQkSNH6Ny5s2TmvEUyChB5Ib21e8oavObk927wh4/mzhJWMPgbaNAxT8uSU1evXuW9995TSw6IjY1VW3NHqVRqJA80bNiQP/74Q6bMESId2YokgwYNok6dOsyePZurV6/qqkwiH2W0uFte0DYJaKXw21yZ3Zen2oJPrZYw/RdV8MnLsuTExo0bGT9+PMeOHVNtmzt3LiNGjGD+/PmcPHky3XOLFSsmwUeIDGQrAG3fvp02bdrg7e1Nx44dad68OStWrNC6BosoHHQ9S/Sbk4D2r6wgzmMswYH31A80NII+0zEc6wkly2e5LNnJsMuJNm3aYGFhwe7du/H39+fTTz/l6NGjVK9enePHjzNixAgOHTqEQqHQmJFbCJExRURERNYatdOIiYlh7969+Pj4cOrUKQAcHR0ZPHgwffr0wcJCv1Pc56WAgADs7e31XQydSs1ES13cLW0WnC6ef8+ePQwfPlz12qJydWq6Lqdm/YYMr2XGj3djtZZFW7nfTKDI62W2AwICOHPmDFu2bMHBwYGQkBCWLl1KuXLlePbsGe7u7uzcuZNr165hZWVFcnLyW9VEXRT+/jNT1H8Hunz+HAWgtEJCQvDx8WHHjh34+/tjampKz549GTJkCJ06dcqrcuqN/PHp5vknTJjAli1bGD58OAsXLqRECc3VStPSliq+4Eo0PoGaY4MGVDfLs1mvU59//PjxHDhwgBYtWrBz507V/idPntCvXz9atmzJ999/nyf3LEiK+t8/yO9Al8+f6zTshIQE4uPjiY+PR6lUYmFhwdmzZ/H19aVu3bp4enrSoEGDvCireIt899139O7dm+7du2d6bHqp4mVMtdc0dJG0MGPGDG7fvk1wcDDBwcFUrZqSmVexYkW6d+/OzZs3DiqqVgAAHe9JREFUiYmJkfE9QmRDjtoKIiMj+emnn+jZsyfvvfce7u7u1KtXj+3bt+Pv78/Nmzf5+eefefnyJRMnTkz3OmfOnGHw4MHUrVsXKysrtm7dmuF9g4ODsbKy0vhJ20EsCoYbN27wwQcfEBERobEvODqBKZfjWa10yFK/TXqp4s9itfe56CJpwc7ODhcXFwwNDdVmaYCUmauNjIwk+AiRTdmqAf3666/s2LGDI0eO8Pr1a5o0aYK7uzv9+/fHyspK7dju3bvz7Nkzpk6dmu71Xr58Sb169RgyZAhjx47Ncjl8fX3ValWlSpXKzmMIHUpOTsbDwwM3Nzfi4+OZPHkyP/74oypFOScDX9NLFS9vqsBQYajRB6SrZbYHDx7Mn3/+ydatW1EqlfTo0YPQ0FD8/PwYNWoUoD0dWwihXbYC0CeffELlypVxdXVlyJAh1KxZM8Pj69evz4ABA9Ld37VrV7p27QrA+PHjs1yO0qVLY21tneXjRf54+vQp48eP5/jx46pte/fuZcuWLQwdOhTIeOBrev026aWKV7Msxg//3xeUlaSFvDBt2jTu3r3LihUruHDhAnFxcQwdOlT1RUuCjxBZl60A9Msvv9CuXbss/yNzcHDAwcEhRwXLyNChQ4mLi6NGjRqMHz+evn375vk9RPb89ttvuLq68vz5c419169fV/1/Tga+ftnYgkuh8VprOvm9zLatrS2jRo1i7ty59O/fn969e0sNXIgcynUWXF6pXLkyS5YsUU3eqE1YWBjbtm2jRYsWGBkZcfDgQZYtW4aHhweDBg1K97yAgABdFLlI+TdWwboHRoS+NqCcSTJjbROpbKYkLi6OVatWqWWGpTI3N+eLL76gS5cuqm1f3SnG4VDNGkr3cgl8Uzv9viDV/eMNKGf83/315dy5c7Ro0UJv9xeiMMgse65QBSBtpk6dytmzZ/nzzz91Uq6ikIKZ0WzYAQEBGFew0zrexr3qM+ZOHqt1XZuWLVvi6elJlSpVNO6l67E7eakovP8ZKerPD/I7KNBp2Prm4OCQafacSF9WkgI0+m2USoIOejNw/1KSE+LVrmdoaMisWbP4/PPPVSuCppU6M0J+9tsIIQqmQh+Abty4IQkJuZCVpAC1fpvoMNj+Ffj78WYSdNWqVfHy8qJp06YZ3jO/+22EEAWTXgNQTEwMgYGBQEr67qNHj7h+/TqlSpWiSpUquLm5cfnyZfbt2wfAtm3bKFasGA0bNsTAwIDDhw/j5eXF119/rcenyHu6XCDuTeklBZx4/Jreh0IxTyqGhcn/J508vQdrRqQEoTcMGjQId3d3LC0tc1yW/HxuIYT+6TUA/fXXX7z//vuq14sWLWLRokUMGTIEDw8PQkJCCAoKUjtn6dKlPHz4EENDQ2rUqMHq1aszTEAobHS9QNyb0ktxDo1LJjQkHiiGTYkEbIoreFSmCpSsoBaALC0tWbZsWYbp9lmR388thNC/ApOEUFDldwfkaL9wnc9vlpa2D35telYxoUQxAwLv/cPVL/qRFPeKZs2a4enpiZ2dXa7Lkd/PnVXSAV20nx/kdyBJCEWIrheIe9ObSQG3IxIIjdP8ThKdoGRb59LQrhlbTdx58OAB06dPx8gob/6E8vu5hRD6JwGogNH1AnHapE0KGO0Xjs/1xxB4GRp21nr/7KbKZ4U+nlsIoV9vz8IlbwldLxCXmS6v/sJwqRP8NBUe3Mi3++v7uYUQ+U8CUAHz5gqiA6qb5UtHfHx8PHPnzmXMRx+SFBkKyYmY/TyLzhaR+XJ/fT23EEJ/pAmuAMrvcTIBAQGMGjWKa9euqW2PDblPzUveVP3wu3wph4wPEqJokRpQEaZUKlnhuZEWrdtqBB+FQsG0adMYNmyYnkonhHjbSQ2oiHrx4gWjXSdy7OABjX3WFSvx7cq1HDZpyAT/GKo/C5dBoUKIPCcBqAjadeQEE8eNIzbsiebOd7vSYPK3LIgyJyg6FjDkclSsDAoVQuQ5aYIrQhISEpg6Zx6jBjlpBh9jMxg0H4Yt5/rrEunODyeEEHlFakBFRGBgIKNGjeLKlSuaO23qwdAlUL7a/2/QPjmGDAoVQuQlqQEVEXfv3tUefDp+CpO3qoJPNQtDmpYz1noNGRQqhMhLUgMqZHI6Y3T37t0ZPXo0GzZsSNlQsjwM+RZqtwSgnKkB7SuZqAZ+/h2huWicDAoVQuQlCUCFSG5njJ4/fz6/+53iSQkbXn3gBiWsAO0rkqbODxcYFkP1MuaSBSeEyHMSgAqRzBaPC45OYP7FF/z7IoYqZa00goaZmRlHDx8kysichX/FZLgiaeqg0ICAMOztbfPl+YQQRYsEoEIkoxmjg6MT6LXlGo/WT4PiJTk3ao3W2lHp0qUpDTLjgBBC7yQAFSIZzRg9evlmHq39Cl6/TNl45meCWn/E7PORlChmIKuMCiEKHAlAhciXjS24FBqv1gxX1SiWV//X3t1HVVXnexz/MKgjlYLSASyfjRB1adKEupCxwFwrrRxtrLAmU0tc4sxYPuGUFrdcSMeHCh8oiRonnBWWJkXeaVIUFdBmKWk2qJmpWEDoZRQuBir3Dy/M4OHpHM7hB/J+reUf7LM35/s9rHU+7t/+/fb+88vav3lTzZ1Tl0v97tGOX/jr0tV/b2ZBKYCWgmnYrcj1d4wOK/+nLlsfUdr14SNJ/sOlW7rWCB+p8QtKT12s0LO7zmvmoV/q2V3ndepihZO6AIBrOANyMUenTdelV6f2ShjpqRUrViguLk5Xrlx3XahdB+nhedLIyerYzk2Xarls1NCC0pqz7bgVDwDXIIBcqKnTpmtz+vRpRUZGKisry+a1OwIC1Tdqpcosd8jvJneVlF/VtryfbfZraEFpQ7PtAMAZCCAXcvYX+ebNmzVnzhxduHDB5rUZM2YoJiZGHh4e1dtOXaxQ7t/sX1Ba32w7AHAWAsiFnPVFfvHiRS1cuFAbN260ffGWrvKdFquo+RPl4WG7lqdqQWl9a36uV99sOwBwFgLIhZz1RT537lylpKTYvtA/RIpYqoLOljrPqhx5ymhts+24FQ8AZ2MWnAu9GNRJfTrVDBtHvshfeOEFde7c+d8b3NtLv1koPZsgdbZIcu7w2H/Otrvb84om9fVgAgIAp+MMyIUcHQKz+T29emnVqlWaPn26Ovfw14XHlkm396+xj7OHx7gVDwBXI4BczJEhsNo88sgjKi8v19DwsYrIKGN4DECrxxBcM6la2Pngtp9qXdhZWlqqOXPm1Dq9ukpERIT6+3jWWIzK8BiA1oozICdoaLFpQ+uBcnJy9Mwzz+jbb7/V9u3btXv3bnl5edX5fs46qwIAkwigJmrMYtO61gM9+NlPct/1nk6lrFLl5WtnRGfOnNHcuXOVmJgoNze35msEAJoZQ3BNVN9i0yq1rgf6V6HOLJ+m7ze+Vh0+Vb766isVFxe7pF4AaCkIoCZqzGJTm/VAX++QrBOk49k2xz311FPatWuXunTpYvNaQ9eRAKA1YQiuiRqz2PTFoE7Kyr+kvOL/lVKt0t4PbA+4qbMCZyzVmy//rtbf54r7ygGASZwBNVFjFpv26tRevS+ekFY+Wnv43HGPNH+LBt03rs73acxQHwC0JpwB2am2GW/1LTa9evWqEhIStHfJS9J113r0i3bSA7OlsGnq49mh3rU83CAUwI2GALJDfcNgtU2LvnLliiIiIvT555/b/rJbe0q/e02WO4fo3tt+2eAdEhoz1OfsZw8BgCsxBGcHe4fB3N3dFRAQYPtC8G+keR+qz8C79MWDt2r9qK4NBkVDQ31V4bjpuzLtyS/Xpu/K9Ju/nWOiAoAWiwCygyPDYIsXL9aQIUMkSZ06d9bw+fEKfc6qSYG32jWB4PrHcV9/BwSuEQFobRiCs4Mjj1fo0KGDEhMTtWjRIq1atUo9evRw+P3ruwMC14gAtDacAdmhrmGwF4beotTUVF29erXW4/z9/fXhhx82KXwawkPkALQ2BJAd/nMY7B5Le/W8xV2dfv4fPfDI43rqqae0Zs0aY7U569lDANBcCCA79erUXi8GdVLRpas6/Y8MHYp+SD9+uUOSFPNf/6WcnByHfm9T73LQ0DUiAGhpuAbUgLNlbnpt1/kaU5tj9hXp5PvLpJ1/rrHv5YoKPf/889q+fbtdNxJ11l0OuEs2gNaEAKrHqYsVmn3kl8q7VFa9LfOrf+r8O/OkU/+02b+T/1AlJSXZfRfr+mawESgAblQEUD1ePXBReZf+f5SyslLK2qSzH8dJFZdq7uj2C+n+GRoz83n17u1j9/swgw1AW0QA1aM6GEqLpQ+WSIe32+7UpZv0ZJz6DAnWknts72DdGMxgA9AWEUD16HaTu3QsW9q4SPpXoc3rPUY+qO5Px6i7pUuTbnvzYlAn/eOn8hrDcMxgA3CjI4DqUF5erpv/e5WUsPra8Nt/uOnmm7XcalVERIRTnlpaNYOtrhuaAsCNyOg07L179+rxxx9XYGCgvLy8lJyc3OAxR44c0dixY+Xn56fAwEDFxcWp8rqAcIbz588r9YNkm/AZOGSo9uzercmTJzv1kdlVM9g+ecDSqHvDAUBrZzSASktLNWDAAC1btkweHh4N7n/hwgVNmDBBPj4+2rFjh5YtW6b4+HitXr3a6bX5+fkpPj6++mc3NzfNnTtXO7/4XH379nX6+wFAW2N0CG7MmDEaM2aMJGnWrFkN7r9p0yaVlZVp3bp18vDw0IABA3Ts2DGtXbtWs2fPduoZiSSNGzdOEydO1L59+5SQkKDQ0NAGj+GRCADQOK3qGtD+/fs1YsSIGmdL4eHhWrp0qU6dOqXevXs7/T2fe+45de/eXV26NDzDjcdmA0DjtaoAKiws1G233VZjm8ViqX6trgA6fvy4w+/ZsWNHFRUVqaioqMF9Fx9tr5MXawbNyYtXtHDXWb0S0Hqfy9OUz+9GQP9tu3+Jz8DR/v39/et9vVUFkCSbYbaqCQj1Db819CHU5/jx440+vuTbnySV22wvdb9Z/v4Wh2swyZ7+b0T037b7l/gMXNl/q7oZqY+PjwoLa67HqTozqToTMokFpQDQeK0qgIKDg5WVlaVLl/59K5z09HR169ZNvXr1MljZNTwSAQAaz2gAlZSU6NChQzp06JCuXr2qvLw8HTp0SGfOnJEkxcTE6OGHH67e/7e//a08PDw0a9YsffPNN0pNTdXrr7+uWbNmOX0GnCN4JAIANJ7Ra0AHDx7UQw89VP1zbGysYmNjFRERoXXr1ik/P18nT56sft3T01NbtmzRvHnzdN9998nLy0tRUVGaPXu2ifJrxSMRAKBxjAZQaGioiouL63x93bp1NtsGDhyobdu2ubIsAEAzaFXXgAAANw4CCABgBAEEADCCAAIAGEEAAQCMIIAAAEYQQAAAIwggAIARBBAAwAgCCABgBAEEADCCAAIAGEEAAQCMIIAAAEYQQAAAIwggAIARBBAAwAgCCABgBAEEADCCAAIAGEEAAQCMIIAAAEYQQAAAIwggAIARBBAAwAgCCABgBAEEADCCAAIAGEEAAQCMIIAAAEYQQAAAIwggAIARBBAAwAgCCABgBAEEADCCAAIAGEEAAQCMIIAAAEYQQAAAIwggAIARBBAAwAgCCABgBAEEADCCAAIAGEEAAQCMIIAAAEYQQAAAIwggAIARxgMoMTFRgwcPlq+vr0aNGqXMzMw69929e7e8vLxs/h07dqwZKwYAOEM7k2++efNmRUdHa8WKFRo+fLgSExM1adIkZWdnq0ePHnUel52drS5dulT/fOuttzZHuQAAJzJ6BrRmzRpNnjxZU6ZMUUBAgKxWq3x9fZWUlFTvcRaLRb6+vtX/3N3dm6liAICzGAug8vJy5eTkKCwsrMb2sLAw7du3r95j7733XgUEBOjhhx9WRkaGK8sEALiIsSG4c+fO6cqVK7JYLDW2WywWFRYW1nqMn5+fVq5cqaCgIJWXl+uDDz7Q+PHj9emnnyokJKTO9zp+/HiTam3q8a0d/dN/W9fWPwNH+/f396/3daPXgCTJzc2txs+VlZU226r4+/vXaCg4OFinT59WfHx8vQHU0IdQn+PHjzfp+NaO/um/Lfcv8Rm4sn9jQ3De3t5yd3e3OdspKiqyOSuqz913363vvvvO2eUBAFzMWAB16NBBd911l9LT02tsT09P17Bhwxr9ew4fPixfX19nlwcAcDGjQ3BRUVGKjIzU3XffrWHDhikpKUn5+fmaOnWqJCkyMlKS9NZbb0mS1q5dq549eyowMFDl5eVKSUlRWlqaNmzYYKwHAIBjjAbQxIkTdf78eVmtVhUUFCgwMFApKSnq2bOnJCkvL6/G/hUVFVq8eLF+/PFHdezYsXr/MWPGmCgfANAEbsXFxZWmi2jJuABJ//TfdvuX+AxuyEkIAIC2jQACABhBAAEAjCCAAABGEEAAACMIIACAEQQQAMAIAggAYAQBBAAwggACABhBAAEAjCCAAABGEEAAACMIIACAEQQQAMAIAggAYAQBBAAwggACABhBAAEAjCCAAABGEEAAACMIIACAEQQQAMAIAggAYAQBBAAwggACABhBAAEAjCCAAABGEEAAACMIIACAEQQQAMAIAggAYAQBBAAwggACABhBAAEAjCCAAABGEEAAACMIIACAEQQQAMAIAggAYAQBBAAwggACABhBAAEAjCCAAABGEEAAACMIIACAEQQQAMAI4wGUmJiowYMHy9fXV6NGjVJmZma9++/Zs0ejRo2Sr6+vhgwZoqSkpGaqFADgTEYDaPPmzYqOjtbcuXOVkZGh4OBgTZo0SWfOnKl1/++//16PPvqogoODlZGRoeeff14LFizQ1q1bm7lyAEBTGQ2gNWvWaPLkyZoyZYoCAgJktVrl6+tb51nNu+++Kz8/P1mtVgUEBGjKlCmKiIjQ6tWrm7lyAEBTGQug8vJy5eTkKCwsrMb2sLAw7du3r9Zj9u/fb7N/eHi4Dh48qIqKCpfU6e/v75Lf21rQP/23dW39M3Bl/8YC6Ny5c7py5YosFkuN7RaLRYWFhbUeU1hYWOv+ly9f1rlz51xWKwDA+YxPQnBzc6vxc2Vlpc22hvavbTsAoGUzFkDe3t5yd3e3OdspKiqyOcup4uPjU+v+7dq1U9euXV1WKwDA+YwFUIcOHXTXXXcpPT29xvb09HQNGzas1mOCg4O1c+dOm/2HDh2q9u3bu6pUAIALGB2Ci4qK0saNG7VhwwYdPXpUCxcuVH5+vqZOnSpJioyMVGRkZPX+U6dO1Q8//KDo6GgdPXpUGzZs0MaNGzV79mxTLQAAHGQ0gCZOnKjY2FhZrVaFhoYqOztbKSkp6tmzpyQpLy9PeXl51fv37t1bKSkpyszMVGhoqJYvX664uDiNHz/e4Rra+kJYe/pPTU3VhAkT1K9fP3Xv3l3h4eH67LPPmrFa57P3718lKytL3t7eGjFihIsrdC17+y8vL9fSpUs1ePBg+fj4aNCgQUpISGimap3P3v43bdqkkSNHqlu3brrzzjs1Y8YMFRQUNFO1zrV37149/vjjCgwMlJeXl5KTkxs85siRIxo7dqz8/PwUGBiouLi46uvwjjA+CeGZZ57R4cOHVVhYqF27dikkJKT6tbS0NKWlpdXYf+TIkcrIyFBhYaEOHTqkadOmOfzebX0hrL397927V7/+9a+VkpKijIwM3X///XryyScb/aXd0tjbf5Xi4mLNnDlTo0aNaqZKXcOR/qdPn67t27frjTfe0Jdffqn33ntPAwcObMaqncfe/rOzsxUZGamIiAhlZWUpOTlZubm5evbZZ5u5cucoLS3VgAEDtGzZMnl4eDS4/4ULFzRhwgT5+Phox44dWrZsmeLj45u0DtOtuLjY8fhq5cLDwzVw4EC9+eab1duCgoI0fvx4vfTSSzb7v/TSS/rkk0904MCB6m2///3vlZubq7///e/NUrMz2dt/bcLCwjRixAgtXbrUVWW6jKP9P/nkkxo0aJAqKyuVmpqqrKys5ijX6eztf8eOHXr66ad18OBBeXt7N2epLmFv//Hx8Xrrrbf09ddfV297//33tXDhQp09e7ZZanaV22+/Xa+99pqeeOKJOvd555139PLLL+vYsWPVgWW1WpWUlKRvvvnGoZnIxs+ATGktC2FdxZH+a1NSUiIvLy9nl+dyjvafmJiowsJCzZ8/39UlupQj/aelpWno0KFas2aNBgwYoKCgIC1YsEAlJSXNUbJTOdL/sGHDVFBQoG3btqmyslLnzp3T5s2bdf/99zdHycbt379fI0aMqHG2FB4erh9//FGnTp1y6He22QBq6wthHen/euvXr9cPP/ygxx57zBUlupQj/R85ckRxcXF6++235e7u3hxluowj/X///ffKzs7W119/rQ0bNshqtWr79u2aNWtWc5TsVI70HxwcrMTERM2YMUMWi0X9+vVTZWWl1q1b1xwlG1fX91/Va45oswFUpa0vhLW3/ypbt27VkiVL9Pbbb1dPGmmNGtv/zz//rOnTp+uVV15R7969m6k617Pn73/16lW5ublp/fr1+tWvfqXw8HBZrValpqY6/AVkmj395+bmKjo6WvPnz9fOnTv10UcfqaCgQHPmzGmOUlsEZ3//tWtyRa1UW18I60j/VbZu3aqZM2cqISFBY8eOdWWZLmNv//n5+crNzVVUVJSioqIkXftCrqyslLe3tzZt2mQznNOSOfL39/X1Vbdu3eTp6Vm97c4775R0bcaqj4+P6wp2Mkf6X7lypYKCgvSHP/xBkjRo0CDddNNNeuCBB7R48WJ1797d5XWbVNf3n6QGvzPq0mbPgNr6QlhH+pekLVu2KDIyUmvXrm3S9HfT7O3/tttuU2Zmpnbv3l39b9q0aerbt692796t4ODg5irdKRz5+w8fPlz5+fk1rvmcOHFCktSjRw/XFesCjvRfVlZmM/Ra9XNTpiK3FsHBwcrKytKlS5eqt6Wnp6tbt27q1auXQ7/TPTo6+mUn1dfqdOrUSbGxsfLz81PHjh1ltVqVmZmp1atXy9PTU5GRkfr000/10EMPSZL69Omj119/XT/99JN69Oihzz77TCtWrNCrr76q/v37G+7Gfvb2/9FHH2nGjBmKiYnRmDFjVFpaqtLSUlVUVDRqGmdLY0//7u7uslgsNf4dOHBAJ06c0KJFi9ShQwfT7djN3r//HXfcoeTkZOXk5Kh///46ceKE5s+fr5CQkHpnT7VU9vZfVlam+Ph4eXt7q2vXrtVDcr6+vvrjH/9ouBv7lZSUKDc3VwUFBfrLX/6iAQMGqHPnziovL5enp6diYmK0cuVKRURESJL69eund999V4cPH5a/v7+ysrK0ZMkSzZkzp97/tNanzQ7BSdcWwp4/f15Wq1UFBQUKDAy0WQj7n6oWwv7pT39SUlKS/Pz8mrwQ1iR7+09KStLly5e1aNEiLVq0qHp7SEiIzXqt1sDe/m809vZ/yy236OOPP9aCBQsUFhYmLy8vjRs3rtFT9lsae/t/4oknVFJSovXr1+vFF19U586dFRoaqpiYGBPlN9nBgwerw1WSYmNjFRsbq4iICK1bt075+fk6efJk9euenp7asmWL5s2bp/vuu09eXl6Kiopq0p1o2vQ6IACAOW32GhAAwCwCCABgBAEEADCCAAIAGEEAAQCMIIAAAEYQQAAAIwggAIARBBAAwAgCCABgBAEEtBBlZWUKDg5WUFCQSktLq7eXlpZq6NChCg4OrnEnYqC1I4CAFsLDw0MJCQk6ffq0lixZUr198eLFOnPmjBISEtSxY0eDFQLO1abvhg20NEFBQXruuedktVo1btw4SdfuQr5gwQIFBQUZrg5wLu6GDbQwFRUVGj16tIqKilRZWSmLxaIvvvii1T30EGgIAQS0QEeOHFFISIjatWunPXv2tMoHHgIN4RoQ0ALt2LFDknT58mUdPXrUcDWAa3AGBLQwubm5GjVqlB588EGdPXtW3377rbKysmSxWEyXBjgVAQS0IJcvX9bo0aNVUFCgzMxMFRcXa+TIkbr33nuVnJxsujzAqRiCA1qQ5cuXKycnR2+88Ya6dOmiPn36KCYmRmlpafrrX/9qujzAqTgDAlqIr776SqNHj1ZERITefPPN6u2VlZWaOHGiDhw4oMzMTN1+++0GqwSchwACABjBEBwAwAgCCABgBAEEADCCAAIAGEEAAQCMIIAAAEYQQAAAIwggAIARBBAAwAgCCABgxP8Bu4AzQixBZZQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = figure3(x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PyTorch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(3.1416)\n",
      "tensor([1, 2, 3])\n",
      "tensor([[1., 1., 1.],\n",
      "        [1., 1., 1.]])\n",
      "tensor([[[ 1.2189, -0.7499,  1.1768,  0.1267],\n",
      "         [ 1.1126, -1.0030, -0.5765, -0.2187],\n",
      "         [-0.3212,  0.7264,  1.3247,  0.7378]],\n",
      "\n",
      "        [[-0.4538,  1.2395, -1.8186, -0.2005],\n",
      "         [ 0.0404, -0.4431,  2.4759,  0.9298],\n",
      "         [ 0.8922,  1.6280, -0.8904,  1.8739]]])\n"
     ]
    }
   ],
   "source": [
    "scalar = torch.tensor(3.14159)\n",
    "vector = torch.tensor([1, 2, 3])\n",
    "matrix = torch.ones((2, 3), dtype=torch.float)\n",
    "tensor = torch.randn((2, 3, 4), dtype=torch.float)\n",
    "\n",
    "print(scalar)\n",
    "print(vector)\n",
    "print(matrix)\n",
    "print(tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 3, 4]) torch.Size([2, 3, 4])\n"
     ]
    }
   ],
   "source": [
    "print(tensor.size(), tensor.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([]) torch.Size([])\n"
     ]
    }
   ],
   "source": [
    "print(scalar.size(), scalar.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 2., 1.],\n",
      "        [1., 1., 1.]])\n",
      "tensor([[1., 2., 1., 1., 1., 1.]])\n"
     ]
    }
   ],
   "source": [
    "# We get a tensor with a different shape but it still is\n",
    "# the SAME tensor\n",
    "same_matrix = matrix.view(1, 6)\n",
    "# If we change one of its elements...\n",
    "same_matrix[0, 1] = 2.\n",
    "# It changes both variables: matrix and same_matrix\n",
    "print(matrix)\n",
    "print(same_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 2., 1.],\n",
      "        [1., 1., 1.]])\n",
      "tensor([[1., 3., 1., 1., 1., 1.]])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dvgodoy/anaconda3/envs/pytorch/lib/python3.7/site-packages/ipykernel_launcher.py:2: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than tensor.new_tensor(sourceTensor).\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "# We can use \"new_tensor\" method to REALLY copy it into a new one\n",
    "different_matrix = matrix.new_tensor(matrix.view(1, 6))\n",
    "# Now, if we change one of its elements...\n",
    "different_matrix[0, 1] = 3.\n",
    "# The original tensor (matrix) is left untouched!\n",
    "# But we get a \"warning\" from PyTorch telling us \n",
    "# to use \"clone()\" instead!\n",
    "print(matrix)\n",
    "print(different_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 2., 1.],\n",
      "        [1., 1., 1.]])\n",
      "tensor([[1., 4., 1., 1., 1., 1.]])\n"
     ]
    }
   ],
   "source": [
    "# Lets follow PyTorch's suggestion and use \"clone\" method\n",
    "another_matrix = matrix.view(1, 6).clone().detach()\n",
    "# Again, if we change one of its elements...\n",
    "another_matrix[0, 1] = 4.\n",
    "# The original tensor (matrix) is left untouched!\n",
    "print(matrix)\n",
    "print(another_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Data, Devices and CUDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(dtype('float64'), torch.float64)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_tensor = torch.as_tensor(x_train)\n",
    "x_train.dtype, x_train_tensor.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.float32"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_tensor = x_train_tensor.float()\n",
    "float_tensor.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 0, 3])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummy_array = np.array([1, 2, 3])\n",
    "dummy_tensor = torch.as_tensor(dummy_array)\n",
    "# Modifies the numpy array\n",
    "dummy_array[1] = 0\n",
    "# Tensor gets modified too...\n",
    "dummy_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 3])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummy_tensor.numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining your device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_cudas = torch.cuda.device_count()\n",
    "for i in range(n_cudas):\n",
    "    print(torch.cuda.get_device_name(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.7713], dtype=torch.float64)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpu_tensor = torch.as_tensor(x_train).to(device)\n",
    "gpu_tensor[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cell 1.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "# Our data was in Numpy arrays, but we need to transform them \n",
    "# into PyTorch's Tensors and then we send them to the \n",
    "# chosen device\n",
    "x_train_tensor = torch.as_tensor(x_train).float().to(device)\n",
    "y_train_tensor = torch.as_tensor(y_train).float().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'> <class 'torch.Tensor'> torch.FloatTensor\n"
     ]
    }
   ],
   "source": [
    "# Here we can see the difference - notice that .type() is more\n",
    "# useful since it also tells us WHERE the tensor is (device)\n",
    "print(type(x_train), type(x_train_tensor), x_train_tensor.type())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "back_to_numpy = x_train_tensor.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "back_to_numpy = x_train_tensor.cpu().numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.3367], requires_grad=True) tensor([0.1288], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "# FIRST\n",
    "# Initializes parameters \"b\" and \"w\" randomly, ALMOST as we\n",
    "# did in Numpy since we want to apply gradient descent on\n",
    "# these parameters we need to set REQUIRES_GRAD = TRUE\n",
    "torch.manual_seed(42)\n",
    "b = torch.randn(1, requires_grad=True, dtype=torch.float)\n",
    "w = torch.randn(1, requires_grad=True, dtype=torch.float)\n",
    "print(b, w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.3367], requires_grad=True) tensor([0.1288], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "# SECOND\n",
    "# But what if we want to run it on a GPU? We could just\n",
    "# send them to device, right?\n",
    "torch.manual_seed(42)\n",
    "b = torch.randn(1, requires_grad=True, dtype=torch.float).to(device)\n",
    "w = torch.randn(1, requires_grad=True, dtype=torch.float).to(device)\n",
    "print(b, w)\n",
    "# Sorry, but NO! The to(device) \"shadows\" the gradient..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.3367], requires_grad=True) tensor([0.1288], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "# THIRD\n",
    "# We can either create regular tensors and send them to\n",
    "# the device (as we did with our data)\n",
    "torch.manual_seed(42)\n",
    "b = torch.randn(1, dtype=torch.float).to(device)\n",
    "w = torch.randn(1, dtype=torch.float).to(device)\n",
    "# and THEN set them as requiring gradients...\n",
    "b.requires_grad_()\n",
    "w.requires_grad_()\n",
    "print(b, w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cell 1.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.3367], requires_grad=True) tensor([0.1288], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "# FINAL\n",
    "# We can specify the device at the moment of creation\n",
    "# RECOMMENDED!\n",
    "\n",
    "# Step 0 - Initializes parameters \"b\" and \"w\" randomly\n",
    "torch.manual_seed(42)\n",
    "b = torch.randn(1, requires_grad=True, \\\n",
    "                dtype=torch.float, device=device)\n",
    "w = torch.randn(1, requires_grad=True, \\\n",
    "                dtype=torch.float, device=device)\n",
    "print(b, w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Autograd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## backward"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cell 1.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 1 - Computes our model's predicted output - forward pass\n",
    "yhat = b + w * x_train_tensor\n",
    "\n",
    "# Step 2 - Computes the loss\n",
    "# We are using ALL data points, so this is BATCH gradient descent\n",
    "# How wrong is our model? That's the error! \n",
    "error = (yhat - y_train_tensor)\n",
    "# It is a regression, so it computes mean squared error (MSE)\n",
    "loss = (error ** 2).mean()\n",
    "\n",
    "# Step 3 - Computes gradients for both \"b\" and \"w\" parameters\n",
    "# No more manual computation of gradients! \n",
    "# b_grad = 2 * error.mean()\n",
    "# w_grad = 2 * (x_tensor * error).mean()\n",
    "loss.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True True True True\n",
      "False False\n"
     ]
    }
   ],
   "source": [
    "print(error.requires_grad, yhat.requires_grad, \\\n",
    "      b.requires_grad, w.requires_grad)\n",
    "print(y_train_tensor.requires_grad, x_train_tensor.requires_grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-3.1125]) tensor([-1.8156])\n"
     ]
    }
   ],
   "source": [
    "print(b.grad, w.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Just run the two cells above one more time "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## zero_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([0.]), tensor([0.]))"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This code will be placed *after* Step 4\n",
    "# (updating the parameters)\n",
    "b.grad.zero_(), w.grad.zero_()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Updating Parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cell 1.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.0235], requires_grad=True) tensor([1.9690], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "# Sets learning rate - this is \"eta\" ~ the \"n\"-like Greek letter\n",
    "lr = 0.1\n",
    "\n",
    "# Step 0 - Initializes parameters \"b\" and \"w\" randomly\n",
    "torch.manual_seed(42)\n",
    "b = torch.randn(1, requires_grad=True, \\\n",
    "                dtype=torch.float, device=device)\n",
    "w = torch.randn(1, requires_grad=True, \\\n",
    "                dtype=torch.float, device=device)\n",
    "\n",
    "# Defines number of epochs\n",
    "n_epochs = 1000\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    # Step 1 - Computes model's predicted output - forward pass\n",
    "    yhat = b + w * x_train_tensor\n",
    "    \n",
    "    # Step 2 - Computes the loss\n",
    "    # We are using ALL data points, so this is BATCH gradient\n",
    "    # descent. How wrong is our model? That's the error!\n",
    "    error = (yhat - y_train_tensor)\n",
    "    # It is a regression, so it computes mean squared error (MSE)\n",
    "    loss = (error ** 2).mean()\n",
    "\n",
    "    # Step 3 - Computes gradients for both \"b\" and \"w\" parameters\n",
    "    # No more manual computation of gradients! \n",
    "    # b_grad = 2 * error.mean()\n",
    "    # w_grad = 2 * (x_tensor * error).mean()   \n",
    "    # We just tell PyTorch to work its way BACKWARDS \n",
    "    # from the specified loss!\n",
    "    loss.backward()\n",
    "    \n",
    "    # Step 4 - Updates parameters using gradients and \n",
    "    # the learning rate. But not so fast...\n",
    "    # FIRST ATTEMPT - just using the same code as before\n",
    "    # AttributeError: 'NoneType' object has no attribute 'zero_'\n",
    "    # b = b - lr * b.grad\n",
    "    # w = w - lr * w.grad\n",
    "    # print(b)\n",
    "\n",
    "    # SECOND ATTEMPT - using in-place Python assigment\n",
    "    # RuntimeError: a leaf Variable that requires grad\n",
    "    # has been used in an in-place operation.\n",
    "    # b -= lr * b.grad\n",
    "    # w -= lr * w.grad        \n",
    "    \n",
    "    # THIRD ATTEMPT - NO_GRAD for the win!\n",
    "    # We need to use NO_GRAD to keep the update out of\n",
    "    # the gradient computation. Why is that? It boils \n",
    "    # down to the DYNAMIC GRAPH that PyTorch uses...\n",
    "    with torch.no_grad():\n",
    "        b -= lr * b.grad\n",
    "        w -= lr * w.grad\n",
    "    \n",
    "    # PyTorch is \"clingy\" to its computed gradients, we\n",
    "    # need to tell it to let it go...\n",
    "    b.grad.zero_()\n",
    "    w.grad.zero_()\n",
    "    \n",
    "print(b, w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## no_grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is what we used in the THIRD ATTEMPT..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dynamic Computation Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"172pt\" height=\"171pt\"\n",
       " viewBox=\"0.00 0.00 171.50 171.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 167)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-167 167.5,-167 167.5,4 -4,4\"/>\n",
       "<!-- 139884641992400 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>139884641992400</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"#000000\" points=\"118,-21 26,-21 26,0 118,0 118,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"72\" y=\"-7.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 139884641990928 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>139884641990928</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"54,-92 0,-92 0,-57 54,-57 54,-92\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-64.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1)</text>\n",
       "</g>\n",
       "<!-- 139884641990928&#45;&gt;139884641992400 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>139884641990928&#45;&gt;139884641992400</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M39.535,-56.6724C45.4798,-48.2176 52.5878,-38.1085 58.6352,-29.5078\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"61.5714,-31.4169 64.4601,-21.2234 55.8452,-27.3906 61.5714,-31.4169\"/>\n",
       "</g>\n",
       "<!-- 139884642044240 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>139884642044240</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"163.5,-85 72.5,-85 72.5,-64 163.5,-64 163.5,-85\"/>\n",
       "<text text-anchor=\"middle\" x=\"118\" y=\"-71.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 139884642044240&#45;&gt;139884641992400 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>139884642044240&#45;&gt;139884641992400</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M110.404,-63.9317C103.7191,-54.6309 93.821,-40.8597 85.7479,-29.6276\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"88.4395,-27.3753 79.761,-21.2979 82.7553,-31.4608 88.4395,-27.3753\"/>\n",
       "</g>\n",
       "<!-- 139884642045712 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>139884642045712</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"145,-163 91,-163 91,-128 145,-128 145,-163\"/>\n",
       "<text text-anchor=\"middle\" x=\"118\" y=\"-135.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1)</text>\n",
       "</g>\n",
       "<!-- 139884642045712&#45;&gt;139884642044240 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>139884642045712&#45;&gt;139884642044240</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M118,-127.9494C118,-118.058 118,-105.6435 118,-95.2693\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"121.5001,-95.0288 118,-85.0288 114.5001,-95.0289 121.5001,-95.0288\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f396e653c90>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Step 0 - Initializes parameters \"b\" and \"w\" randomly\n",
    "torch.manual_seed(42)\n",
    "b = torch.randn(1, requires_grad=True, \\\n",
    "                dtype=torch.float, device=device)\n",
    "w = torch.randn(1, requires_grad=True, \\\n",
    "                dtype=torch.float, device=device)\n",
    "\n",
    "# Step 1 - Computes our model's predicted output - forward pass\n",
    "yhat = b + w * x_train_tensor\n",
    "\n",
    "# Step 2 - Computes the loss\n",
    "# We are using ALL data points, so this is BATCH gradient\n",
    "# descent. How wrong is our model? That's the error! \n",
    "error = (yhat - y_train_tensor)\n",
    "# It is a regression, so it computes mean squared error (MSE)\n",
    "loss = (error ** 2).mean()\n",
    "\n",
    "# We can try plotting the graph for any python variable: \n",
    "# yhat, error, loss...\n",
    "make_dot(yhat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"100pt\" height=\"157pt\"\n",
       " viewBox=\"0.00 0.00 100.00 157.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 153)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-153 96,-153 96,4 -4,4\"/>\n",
       "<!-- 139884642045008 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>139884642045008</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"#000000\" points=\"92,-21 0,-21 0,0 92,0 92,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"46\" y=\"-7.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 139884642045904 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>139884642045904</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"91.5,-78 .5,-78 .5,-57 91.5,-57 91.5,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"46\" y=\"-64.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 139884642045904&#45;&gt;139884642045008 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>139884642045904&#45;&gt;139884642045008</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M46,-56.7787C46,-49.6134 46,-39.9517 46,-31.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"49.5001,-31.1732 46,-21.1732 42.5001,-31.1732 49.5001,-31.1732\"/>\n",
       "</g>\n",
       "<!-- 139884642043536 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>139884642043536</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"73,-149 19,-149 19,-114 73,-114 73,-149\"/>\n",
       "<text text-anchor=\"middle\" x=\"46\" y=\"-121.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1)</text>\n",
       "</g>\n",
       "<!-- 139884642043536&#45;&gt;139884642045904 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>139884642043536&#45;&gt;139884642045904</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M46,-113.6724C46,-105.8405 46,-96.5893 46,-88.4323\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"49.5001,-88.2234 46,-78.2234 42.5001,-88.2235 49.5001,-88.2234\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f396e660c90>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b_nograd = torch.randn(1, requires_grad=False, \\\n",
    "                       dtype=torch.float, device=device)\n",
    "w = torch.randn(1, requires_grad=True, \\\n",
    "                dtype=torch.float, device=device)\n",
    "\n",
    "yhat = b_nograd + w * x_train_tensor\n",
    "\n",
    "make_dot(yhat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"284pt\" height=\"399pt\"\n",
       " viewBox=\"0.00 0.00 284.00 399.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 395)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-395 280,-395 280,4 -4,4\"/>\n",
       "<!-- 139884642042768 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>139884642042768</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"#000000\" points=\"215,-21 123,-21 123,0 215,0 215,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"169\" y=\"-7.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 139884642043728 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>139884642043728</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"180,-78 82,-78 82,-57 180,-57 180,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"131\" y=\"-64.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MeanBackward0</text>\n",
       "</g>\n",
       "<!-- 139884642043728&#45;&gt;139884642042768 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>139884642043728&#45;&gt;139884642042768</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M138.1475,-56.7787C143.2429,-49.1357 150.2317,-38.6524 156.2694,-29.596\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"159.2497,-31.4352 161.8845,-21.1732 153.4253,-27.5522 159.2497,-31.4352\"/>\n",
       "</g>\n",
       "<!-- 139884640533776 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>139884640533776</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"159.5,-135 66.5,-135 66.5,-114 159.5,-114 159.5,-135\"/>\n",
       "<text text-anchor=\"middle\" x=\"113\" y=\"-121.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">PowBackward0</text>\n",
       "</g>\n",
       "<!-- 139884640533776&#45;&gt;139884642043728 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>139884640533776&#45;&gt;139884642043728</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M116.3857,-113.7787C118.6987,-106.4542 121.8354,-96.5211 124.61,-87.7352\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"127.9557,-88.763 127.6295,-78.1732 121.2806,-86.655 127.9557,-88.763\"/>\n",
       "</g>\n",
       "<!-- 139884640530576 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>139884640530576</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"158,-192 68,-192 68,-171 158,-171 158,-192\"/>\n",
       "<text text-anchor=\"middle\" x=\"113\" y=\"-178.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">SubBackward0</text>\n",
       "</g>\n",
       "<!-- 139884640530576&#45;&gt;139884640533776 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>139884640530576&#45;&gt;139884640533776</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M113,-170.7787C113,-163.6134 113,-153.9517 113,-145.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"116.5001,-145.1732 113,-135.1732 109.5001,-145.1732 116.5001,-145.1732\"/>\n",
       "</g>\n",
       "<!-- 139884640803152 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>139884640803152</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"159,-249 67,-249 67,-228 159,-228 159,-249\"/>\n",
       "<text text-anchor=\"middle\" x=\"113\" y=\"-235.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 139884640803152&#45;&gt;139884640530576 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>139884640803152&#45;&gt;139884640530576</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M113,-227.7787C113,-220.6134 113,-210.9517 113,-202.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"116.5001,-202.1732 113,-192.1732 109.5001,-202.1732 116.5001,-202.1732\"/>\n",
       "</g>\n",
       "<!-- 139884640802320 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>139884640802320</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"54,-320 0,-320 0,-285 54,-285 54,-320\"/>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-292.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1)</text>\n",
       "</g>\n",
       "<!-- 139884640802320&#45;&gt;139884640803152 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>139884640802320&#45;&gt;139884640803152</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M50.9558,-284.6724C63.3538,-275.446 78.3987,-264.2498 90.5683,-255.1934\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"92.9097,-257.8138 98.8425,-249.0358 88.7306,-252.1981 92.9097,-257.8138\"/>\n",
       "</g>\n",
       "<!-- 139884640801680 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>139884640801680</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"163.5,-313 72.5,-313 72.5,-292 163.5,-292 163.5,-313\"/>\n",
       "<text text-anchor=\"middle\" x=\"118\" y=\"-299.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 139884640801680&#45;&gt;139884640803152 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>139884640801680&#45;&gt;139884640803152</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M117.1744,-291.9317C116.4837,-283.0913 115.4775,-270.2122 114.6261,-259.3135\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"118.1119,-258.9949 113.8436,-249.2979 111.1332,-259.5402 118.1119,-258.9949\"/>\n",
       "</g>\n",
       "<!-- 139884640151312 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>139884640151312</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"199,-391 145,-391 145,-356 199,-356 199,-391\"/>\n",
       "<text text-anchor=\"middle\" x=\"172\" y=\"-363.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1)</text>\n",
       "</g>\n",
       "<!-- 139884640151312&#45;&gt;139884640801680 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>139884640151312&#45;&gt;139884640801680</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M158.6517,-355.9494C150.6484,-345.4266 140.4734,-332.0484 132.3053,-321.3089\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"134.8474,-318.8695 126.0078,-313.0288 129.2757,-323.1071 134.8474,-318.8695\"/>\n",
       "</g>\n",
       "<!-- 139884640803088 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>139884640803088</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"272.5,-313 181.5,-313 181.5,-292 272.5,-292 272.5,-313\"/>\n",
       "<text text-anchor=\"middle\" x=\"227\" y=\"-299.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 139884640151312&#45;&gt;139884640803088 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>139884640151312&#45;&gt;139884640803088</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M185.5955,-355.9494C193.8285,-345.3214 204.3179,-331.7806 212.6787,-320.9875\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"215.4868,-323.0778 218.8439,-313.0288 209.9529,-318.7909 215.4868,-323.0778\"/>\n",
       "</g>\n",
       "<!-- 139884642045136 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>139884642045136</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"276,-135 178,-135 178,-114 276,-114 276,-135\"/>\n",
       "<text text-anchor=\"middle\" x=\"227\" y=\"-121.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MeanBackward0</text>\n",
       "</g>\n",
       "<!-- 139884642045136&#45;&gt;139884642042768 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>139884642045136&#45;&gt;139884642042768</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M221.6475,-113.9795C211.9855,-94.9888 191.4875,-54.6995 179.1119,-30.3751\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"182.1629,-28.6533 174.5088,-21.3276 175.9239,-31.8275 182.1629,-28.6533\"/>\n",
       "</g>\n",
       "<!-- 139884640531728 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>139884640531728</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"272,-249 182,-249 182,-228 272,-228 272,-249\"/>\n",
       "<text text-anchor=\"middle\" x=\"227\" y=\"-235.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">SubBackward0</text>\n",
       "</g>\n",
       "<!-- 139884640531728&#45;&gt;139884642045136 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>139884640531728&#45;&gt;139884642045136</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M227,-227.9795C227,-209.242 227,-169.7701 227,-145.3565\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"230.5001,-145.3276 227,-135.3276 223.5001,-145.3277 230.5001,-145.3276\"/>\n",
       "</g>\n",
       "<!-- 139884640803088&#45;&gt;139884640531728 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>139884640803088&#45;&gt;139884640531728</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M227,-291.9317C227,-283.0913 227,-270.2122 227,-259.3135\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"230.5001,-259.2979 227,-249.2979 223.5001,-259.2979 230.5001,-259.2979\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f396e4efbd0>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = torch.randn(1, requires_grad=True, \\\n",
    "                dtype=torch.float, device=device)\n",
    "w = torch.randn(1, requires_grad=True, \\\n",
    "                dtype=torch.float, device=device)\n",
    "\n",
    "yhat = b + w * x_train_tensor\n",
    "error = yhat - y_train_tensor\n",
    "loss = (error ** 2).mean()\n",
    "\n",
    "# this makes no sense!!\n",
    "if loss > 0:\n",
    "    yhat2 = w * x_train_tensor\n",
    "    error2 = yhat2 - y_train_tensor\n",
    "    \n",
    "# neither does this :-)\n",
    "loss += error2.mean()\n",
    "\n",
    "make_dot(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optimizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## step / zero_grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defines a SGD optimizer to update the parameters\n",
    "optimizer = optim.SGD([b, w], lr=lr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cell 1.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.0235], requires_grad=True) tensor([1.9690], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "# Sets learning rate - this is \"eta\" ~ the \"n\"-like Greek letter\n",
    "lr = 0.1\n",
    "\n",
    "# Step 0 - Initializes parameters \"b\" and \"w\" randomly\n",
    "torch.manual_seed(42)\n",
    "b = torch.randn(1, requires_grad=True, \\\n",
    "                dtype=torch.float, device=device)\n",
    "w = torch.randn(1, requires_grad=True, \\\n",
    "                dtype=torch.float, device=device)\n",
    "\n",
    "# Defines a SGD optimizer to update the parameters\n",
    "optimizer = optim.SGD([b, w], lr=lr)\n",
    "\n",
    "# Defines number of epochs\n",
    "n_epochs = 1000\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    # Step 1 - Computes model's predicted output - forward pass\n",
    "    yhat = b + w * x_train_tensor\n",
    "    \n",
    "    # Step 2 - Computes the loss\n",
    "    # We are using ALL data points, so this is BATCH gradient \n",
    "    # descent. How wrong is our model? That's the error! \n",
    "    error = (yhat - y_train_tensor)\n",
    "    # It is a regression, so it computes mean squared error (MSE)\n",
    "    loss = (error ** 2).mean()\n",
    "\n",
    "    # Step 3 - Computes gradients for both \"b\" and \"w\" parameters\n",
    "    loss.backward()\n",
    "    \n",
    "    # Step 4 - Updates parameters using gradients and \n",
    "    # the learning rate. No more manual update!\n",
    "    # with torch.no_grad():\n",
    "    #     b -= lr * b.grad\n",
    "    #     w -= lr * w.grad\n",
    "    optimizer.step()\n",
    "    \n",
    "    # No more telling Pytorch to let gradients go!\n",
    "    # b.grad.zero_()\n",
    "    # w.grad.zero_()\n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "print(b, w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MSELoss()"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Defines a MSE loss function\n",
    "loss_fn = nn.MSELoss(reduction='mean')\n",
    "loss_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1.1700)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This is a random example to illustrate the loss function\n",
    "predictions = torch.tensor([0.5, 1.0])\n",
    "labels = torch.tensor([2.0, 1.3])\n",
    "loss_fn(predictions, labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cell 1.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.0235], requires_grad=True) tensor([1.9690], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "# Sets learning rate - this is \"eta\" ~ the \"n\"-like\n",
    "# Greek letter\n",
    "lr = 0.1\n",
    "\n",
    "# Step 0 - Initializes parameters \"b\" and \"w\" randomly\n",
    "torch.manual_seed(42)\n",
    "b = torch.randn(1, requires_grad=True, \\\n",
    "                dtype=torch.float, device=device)\n",
    "w = torch.randn(1, requires_grad=True, \\\n",
    "                dtype=torch.float, device=device)\n",
    "\n",
    "# Defines a SGD optimizer to update the parameters\n",
    "optimizer = optim.SGD([b, w], lr=lr)\n",
    "\n",
    "# Defines a MSE loss function\n",
    "loss_fn = nn.MSELoss(reduction='mean')\n",
    "\n",
    "# Defines number of epochs\n",
    "n_epochs = 1000\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    # Step 1 - Computes model's predicted output - forward pass\n",
    "    yhat = b + w * x_train_tensor\n",
    "    \n",
    "    # Step 2 - Computes the loss\n",
    "    # No more manual loss!\n",
    "    # error = (yhat - y_train_tensor)\n",
    "    # loss = (error ** 2).mean()\n",
    "    loss = loss_fn(yhat, y_train_tensor)\n",
    "\n",
    "    # Step 3 - Computes gradients for both \"b\" and \"w\" parameters\n",
    "    loss.backward()\n",
    "    \n",
    "    # Step 4 - Updates parameters using gradients and\n",
    "    # the learning rate\n",
    "    optimizer.step()\n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "print(b, w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.0080, grad_fn=<MseLossBackward>)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Can't call numpy() on Tensor that requires grad. Use tensor.detach().numpy() instead.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-51-58c76a7bac74>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m: Can't call numpy() on Tensor that requires grad. Use tensor.detach().numpy() instead."
     ]
    }
   ],
   "source": [
    "loss.cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(0.00804466, dtype=float32)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss.detach().cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.008044655434787273 0.008044655434787273\n"
     ]
    }
   ],
   "source": [
    "print(loss.item(), loss.tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cell 1.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ManualLinearRegression(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        # To make \"b\" and \"w\" real parameters of the model,\n",
    "        # we need to wrap them with nn.Parameter\n",
    "        self.b = nn.Parameter(torch.randn(1,\n",
    "                                          requires_grad=True, \n",
    "                                          dtype=torch.float))\n",
    "        self.w = nn.Parameter(torch.randn(1, \n",
    "                                          requires_grad=True,\n",
    "                                          dtype=torch.float))\n",
    "        \n",
    "    def forward(self, x):\n",
    "        # Computes the outputs / predictions\n",
    "        return self.b + self.w * x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Parameter containing:\n",
       " tensor([0.3367], requires_grad=True), Parameter containing:\n",
       " tensor([0.1288], requires_grad=True)]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(42)\n",
    "# Creates a \"dummy\" instance of our ManualLinearRegression model\n",
    "dummy = ManualLinearRegression()\n",
    "list(dummy.parameters())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## state_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('b', tensor([0.3367])), ('w', tensor([0.1288]))])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummy.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'state': {0: {'momentum_buffer': None}, 1: {'momentum_buffer': None}},\n",
       " 'param_groups': [{'lr': 0.1,\n",
       "   'momentum': 0,\n",
       "   'dampening': 0,\n",
       "   'weight_decay': 0,\n",
       "   'nesterov': False,\n",
       "   'params': [0, 1]}]}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.state_dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(42)\n",
    "# Creates a \"dummy\" instance of our ManualLinearRegression model\n",
    "# and sends it to the device\n",
    "dummy = ManualLinearRegression().to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Forward Pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cell 1.10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('b', tensor([1.0235])), ('w', tensor([1.9690]))])\n"
     ]
    }
   ],
   "source": [
    "# Sets learning rate - this is \"eta\" ~ the \"n\"-like\n",
    "# Greek letter\n",
    "lr = 0.1\n",
    "\n",
    "# Step 0 - Initializes parameters \"b\" and \"w\" randomly\n",
    "torch.manual_seed(42)\n",
    "# Now we can create a model and send it at once to the device\n",
    "model = ManualLinearRegression().to(device)\n",
    "\n",
    "# Defines a SGD optimizer to update the parameters \n",
    "# (now retrieved directly from the model)\n",
    "optimizer = optim.SGD(model.parameters(), lr=lr)\n",
    "\n",
    "# Defines a MSE loss function\n",
    "loss_fn = nn.MSELoss(reduction='mean')\n",
    "\n",
    "# Defines number of epochs\n",
    "n_epochs = 1000\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    model.train() # What is this?!?\n",
    "\n",
    "    # Step 1 - Computes model's predicted output - forward pass\n",
    "    # No more manual prediction!\n",
    "    yhat = model(x_train_tensor)\n",
    "    \n",
    "    # Step 2 - Computes the loss\n",
    "    loss = loss_fn(yhat, y_train_tensor)\n",
    "\n",
    "    # Step 3 - Computes gradients for both \"b\" and \"w\" parameters\n",
    "    loss.backward()\n",
    "    \n",
    "    # Step 4 - Updates parameters using gradients and\n",
    "    # the learning rate\n",
    "    optimizer.step()\n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "# We can also inspect its parameters using its state_dict\n",
    "print(model.state_dict())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Never forget to include model.train() in your training loop!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Nested Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Linear(in_features=1, out_features=1, bias=True)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linear = nn.Linear(1, 1)\n",
    "linear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('weight', tensor([[-0.2191]])), ('bias', tensor([0.2018]))])"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linear.state_dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cell 1.11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyLinearRegression(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        # Instead of our custom parameters, we use a Linear model\n",
    "        # with single input and single output\n",
    "        self.linear = nn.Linear(1, 1)\n",
    "                \n",
    "    def forward(self, x):\n",
    "        # Now it only takes a call\n",
    "        self.linear(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Parameter containing:\n",
       " tensor([[0.7645]], device='cuda:0', requires_grad=True), Parameter containing:\n",
       " tensor([0.8300], device='cuda:0', requires_grad=True)]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(42)\n",
    "dummy = MyLinearRegression().to(device)\n",
    "list(dummy.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('linear.weight', tensor([[0.7645]], device='cuda:0')),\n",
       "             ('linear.bias', tensor([0.8300], device='cuda:0'))])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummy.state_dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sequential Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cell 1.12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('0.weight', tensor([[0.7645]], device='cuda:0')),\n",
       "             ('0.bias', tensor([0.8300], device='cuda:0'))])"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(42)\n",
    "# Alternatively, you can use a Sequential model\n",
    "model = nn.Sequential(nn.Linear(1, 1)).to(device)\n",
    "\n",
    "model.state_dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('0.weight', tensor([[ 0.4414,  0.4792, -0.1353],\n",
       "                      [ 0.5304, -0.1265,  0.1165],\n",
       "                      [-0.2811,  0.3391,  0.5090],\n",
       "                      [-0.4236,  0.5018,  0.1081],\n",
       "                      [ 0.4266,  0.0782,  0.2784]], device='cuda:0')),\n",
       "             ('0.bias',\n",
       "              tensor([-0.0815,  0.4451,  0.0853, -0.2695,  0.1472], device='cuda:0')),\n",
       "             ('1.weight',\n",
       "              tensor([[-0.2060, -0.0524, -0.1816,  0.2967, -0.3530]], device='cuda:0')),\n",
       "             ('1.bias', tensor([-0.2062], device='cuda:0'))])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(42)\n",
    "# Building the model from the figure above\n",
    "model = nn.Sequential(nn.Linear(3, 5), nn.Linear(5, 1)).to(device)\n",
    "\n",
    "model.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (layer1): Linear(in_features=3, out_features=5, bias=True)\n",
       "  (layer2): Linear(in_features=5, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(42)\n",
    "# Building the model from the figure above\n",
    "model = nn.Sequential()\n",
    "model.add_module('layer1', nn.Linear(3, 5))\n",
    "model.add_module('layer2', nn.Linear(5, 1))\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Putting It All Together"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Preparation V0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting data_preparation/v0.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile data_preparation/v0.py\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "# Our data was in Numpy arrays, but we need to transform them\n",
    "# into PyTorch's Tensors and then we send them to the \n",
    "# chosen device\n",
    "x_train_tensor = torch.as_tensor(x_train).float().to(device)\n",
    "y_train_tensor = torch.as_tensor(y_train).float().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run -i data_preparation/v0.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Configurtion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Configuration V0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting model_configuration/v0.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile model_configuration/v0.py\n",
    "\n",
    "# This is redundant now, but it won't be when we introduce\n",
    "# Datasets...\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "# Sets learning rate - this is \"eta\" ~ the \"n\"-like Greek letter\n",
    "lr = 0.1\n",
    "\n",
    "torch.manual_seed(42)\n",
    "# Now we can create a model and send it at once to the device\n",
    "model = nn.Sequential(nn.Linear(1, 1)).to(device)\n",
    "\n",
    "# Defines a SGD optimizer to update the parameters \n",
    "# (now retrieved directly from the model)\n",
    "optimizer = optim.SGD(model.parameters(), lr=lr)\n",
    "\n",
    "# Defines a MSE loss function\n",
    "loss_fn = nn.MSELoss(reduction='mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run -i model_configuration/v0.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Training V0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting model_training/v0.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile model_training/v0.py\n",
    "\n",
    "# Defines number of epochs\n",
    "n_epochs = 1000\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    # Sets model to TRAIN mode\n",
    "    model.train()\n",
    "\n",
    "    # Step 1 - Computes model's predicted output - forward pass\n",
    "    yhat = model(x_train_tensor)\n",
    "    \n",
    "    # Step 2 - Computes the loss\n",
    "    loss = loss_fn(yhat, y_train_tensor)\n",
    "\n",
    "    # Step 3 - Computes gradients for both \"b\" and \"w\" parameters\n",
    "    loss.backward()\n",
    "    \n",
    "    # Step 4 - Updates parameters using gradients and \n",
    "    # the learning rate\n",
    "    optimizer.step()\n",
    "    optimizer.zero_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run -i model_training/v0.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('0.weight', tensor([[1.9690]], device='cuda:0')), ('0.bias', tensor([1.0235], device='cuda:0'))])\n"
     ]
    }
   ],
   "source": [
    "print(model.state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}