feedforward_neural_network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "xVjuXsr1zLu2"
   },
   "outputs": [],
   "source": [
    "#import the dataset \n",
    "from keras.datasets import fashion_mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "OQLWjHat0lV4"
   },
   "outputs": [],
   "source": [
    "#import the required libraries \n",
    "from matplotlib import pyplot\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "rP17UY7v0C7w"
   },
   "outputs": [],
   "source": [
    "#load the data into train and test \n",
    "(X_train,y_train),(X_test,y_test)=fashion_mnist.load_data()\n",
    "#pyplot.imshow(X_train[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "z27sdlQmMH1O",
    "outputId": "24480500-1f7d-4219-a65c-76e043c3c77b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 784)\n"
     ]
    }
   ],
   "source": [
    "noOfImages = X_train.shape[0]\n",
    "X_train = (1.0/255)*np.array([X_train[i].flatten() for i in range(0,noOfImages)])\n",
    "X_train = np.array([X_train[i].flatten() for i in range(0,noOfImages)])\n",
    "noOftestImages= X_test.shape[0]\n",
    "X_test =(1.0/255)*np.array([X_test[i].flatten() for i in range(0,noOftestImages)])\n",
    "X_test =np.array([X_test[i].flatten() for i in range(0,noOftestImages)])\n",
    "print(X_test.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "RtQEmgi3Opy_",
    "outputId": "7255d5e6-eb7e-464b-8a1c-7485fb8c1a0a"
   },
   "outputs": [],
   "source": [
    "#print(X_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "id": "GFA10HAHKCb8"
   },
   "outputs": [],
   "source": [
    "def softmax(X):\n",
    "  X=np.exp(X)\n",
    "  sum=np.sum(X,axis=0)\n",
    "  return X/sum \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "id": "4ZgtMpEAUkIA"
   },
   "outputs": [],
   "source": [
    "def sigmoidFunc(X):\n",
    "\n",
    "  return 1.0/(1.0+np.exp(-X))\n",
    "  #return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "id": "5GJHo_F55wKD"
   },
   "outputs": [],
   "source": [
    "def gDash(X):\n",
    "  return sigmoidFunc(X)*(1-sigmoidFunc(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "id": "pIfFadrLAkmM"
   },
   "outputs": [],
   "source": [
    "def forwardPropagation(X,parameters):\n",
    "  preactivation={}\n",
    "  activation={}\n",
    "  activation['h0']=X.T\n",
    "  #print(activation['h0'].shape)\n",
    "  for k in range(1,noOfHiddenLayers+1):\n",
    "    preactivation['a'+str(k)]=np.dot(parameters['W'+str(k)],activation['h'+str(k-1)])+parameters['b'+str(k)]\n",
    "    activation['h'+str(k)]=sigmoidFunc(preactivation['a'+str(k)])\n",
    "    #print('h size '+str(k),activation['h'+str(k)].shape)\n",
    "    #print('a'+str(k),preactivation['a'+str(k)])\n",
    "  preactivation['a'+str(noOfHiddenLayers+1)]=np.dot(parameters['W'+str(noOfHiddenLayers+1)],activation['h'+str(noOfHiddenLayers)])+parameters['b'+str(noOfHiddenLayers+1)]\n",
    "  y=softmax(preactivation['a'+str(noOfHiddenLayers+1)])    \n",
    "  #print(\"a last\" ,preactivation['a'+str(noOfHiddenLayers+1)])\n",
    "  return (preactivation,activation,y)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "id": "vZFsUg0xrJtv"
   },
   "outputs": [],
   "source": [
    "def backPropagation(parameters,activation,preactivation,yhat,X,y_train):\n",
    "  grads={}\n",
    "  eIndicator=np.zeros((10,X.shape[0]))\n",
    "  eIndicator[y_train,np.arange(X.shape[0])]=1\n",
    "  #print(y_train)\n",
    "  #print(eIndicator)\n",
    "  #print(eIndicator.shape)\n",
    "  grads['a'+str(noOfHiddenLayers+1)]= -(eIndicator-yhat)\n",
    "  #print(grads['a'+str(noOfHiddenLayers+1)].shape)\n",
    "  for j in range(noOfHiddenLayers+1,0,-1):\n",
    "    grads['W'+str(j)]= np.dot(grads['a'+str(j)],activation['h'+str(j-1)].T)\n",
    "    #grads['W'+str(j)]= np.dot(activation['h'+str(j-1)],grads['a'+str(j)].T).T\n",
    "    #print(grads['W'+str(j)].shape)\n",
    "    grads['b'+str(j)]= np.sum(grads['a'+str(j)],axis=1,keepdims=True)\n",
    "    ###grads['h'+str(j-1)]=np.dot(grads['W'+str(j)].T,grads['a'+str(j)]) #error\n",
    "    grads['h'+str(j-1)]=np.dot(parameters['W'+str(j)].T,grads['a'+str(j)])\n",
    "    #print('h'+str(j-1),grads['h'+str(j-1)].shape)\n",
    "    #print('a'+str(j-1),preactivation['a'+str(j-1)].shape)\n",
    "    if (j!=1):\n",
    "      grads['a'+str(j-1)]=grads['h'+str(j-1)]*gDash(preactivation['a'+str(j-1)])\n",
    "  return grads\n",
    "#backPropagation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "id": "3MXQ17Gk_Cuv"
   },
   "outputs": [],
   "source": [
    "def Loss(yhat,y_train,X):\n",
    "  eIndicator=np.zeros((10,X.shape[0]))\n",
    "  eIndicator[y_train,np.arange(X.shape[0])]=1\n",
    "  eIndicator=eIndicator*yhat\n",
    "  eIndicator=eIndicator.sum(axis=0)\n",
    "  eIndicator=np.log(eIndicator)\n",
    "  return -sum(eIndicator)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WnJHWPgMiiLw",
    "outputId": "c7870bc1-8497-4ee2-da3d-243d46d689aa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "784\n"
     ]
    }
   ],
   "source": [
    "#@title\n",
    "noOfHiddenLayers=2\n",
    "l=10 #output classes\n",
    "noOfneuronsEach=[120,20]\n",
    "inputNeuronSize=X_train.shape[1]\n",
    "print(inputNeuronSize)\n",
    "parameters={}\n",
    "#eta=0.1\n",
    "#input W\n",
    "loss = []\n",
    "batchSize=600\n",
    "iterations=100\n",
    "#eta=1   \n",
    "eta=0.001   #for rmsprop higher learning rate gradients explode causing overflow\n",
    "eta=0.002 #Nadam\n",
    "#input W\n",
    "#initialization\n",
    "parameters['W'+str(1)] =np.random.uniform(low=-0.5,high=0.5,size=(noOfneuronsEach[0], inputNeuronSize))\n",
    "parameters['b'+str(1)]= np.random.uniform(low=-0.5,high=0.5,size=(noOfneuronsEach[0],1))\n",
    "#parameters['W'+str(1)] =0+1.5*np.random.randn(noOfneuronsEach[0], inputNeuronSize)\n",
    "#parameters['b'+str(1)]= 0+1.5*np.random.randn(noOfneuronsEach[0],1)\n",
    "for i in range(2,noOfHiddenLayers+1):\n",
    "  parameters['W'+str(i)] = np.random.uniform(low=-0.5,high=0.5,size=(noOfneuronsEach[i-1], noOfneuronsEach[i-2]))\n",
    "  parameters['b'+str(i)]= np.random.uniform(low=-0.5,high=0.5,size=(noOfneuronsEach[i-1],1))\n",
    "  #parameters['W'+str(i)] = 0+1.5*np.random.randn(noOfneuronsEach[i-1], noOfneuronsEach[i-2])\n",
    "  #parameters['b'+str(i)]= 0+1.5*np.random.randn(noOfneuronsEach[i-1],1)\n",
    "#Output W\n",
    "parameters['W'+str(noOfHiddenLayers+1)] = np.random.uniform(low=-0.5,high=0.5,size=(l, noOfneuronsEach[-1]))\n",
    "parameters['b'+str(noOfHiddenLayers+1)]= np.random.uniform(low=-0.5,high=0.5,size=(l,1))\n",
    "#parameters['W'+str(noOfHiddenLayers+1)] = 0+1.5*np.random.randn(l, noOfneuronsEach[-1])\n",
    "#parameters['b'+str(noOfHiddenLayers+1)]= 0+1.5*np.random.randn(l,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sgd():\n",
    "  #batchSize=1000\n",
    "  iterations=250\n",
    "  #initialization\n",
    "  parameters['W'+str(1)] =np.random.uniform(low=-0.5,high=0.5,size=(noOfneuronsEach[0], inputNeuronSize))\n",
    "  parameters['b'+str(1)]= np.random.uniform(low=-0.5,high=0.5,size=(noOfneuronsEach[0],1))\n",
    "  #parameters['W'+str(1)] =0+1.5*np.random.randn(noOfneuronsEach[0], inputNeuronSize)\n",
    "  #parameters['b'+str(1)]= 0+1.5*np.random.randn(noOfneuronsEach[0],1)\n",
    "  for i in range(2,noOfHiddenLayers+1):\n",
    "    parameters['W'+str(i)] = np.random.uniform(low=-0.5,high=0.5,size=(noOfneuronsEach[i-1], noOfneuronsEach[i-2]))\n",
    "    parameters['b'+str(i)]= np.random.uniform(low=-0.5,high=0.5,size=(noOfneuronsEach[i-1],1))\n",
    "    #parameters['W'+str(i)] = 0+1.5*np.random.randn(noOfneuronsEach[i-1], noOfneuronsEach[i-2])\n",
    "    #parameters['b'+str(i)]= 0+1.5*np.random.randn(noOfneuronsEach[i-1],1)\n",
    "  #Output W\n",
    "  parameters['W'+str(noOfHiddenLayers+1)] = np.random.uniform(low=-0.5,high=0.5,size=(l, noOfneuronsEach[-1]))\n",
    "  parameters['b'+str(noOfHiddenLayers+1)]= np.random.uniform(low=-0.5,high=0.5,size=(l,1))\n",
    "  #parameters['W'+str(noOfHiddenLayers+1)] = 0+1.5*np.random.randn(l, noOfneuronsEach[-1])\n",
    "  #parameters['b'+str(noOfHiddenLayers+1)]= 0+1.5*np.random.randn(l,1)\n",
    "  t=0  \n",
    "  while(t < iterations):\n",
    "    print(\"iter \",t)\n",
    "    #print(parameters)\n",
    "    #mini=0\n",
    "    #while(mini<(noOfImages/batchSize)):\n",
    "      #print(\"Epoch\",mini)\n",
    "      #X_mini=X_train[(mini*batchSize):((mini+1)*batchSize-1)]\n",
    "      #y_mini=y_train[(mini*batchSize):((mini+1)*batchSize-1)]\n",
    "    # shuffle the data\n",
    "    id = np.random.permutation(len(X_train))\n",
    "    X_random = X_train[id]\n",
    "    y_random = y_train[id]\n",
    "    for (x,y) in zip(X_random, y_random):\n",
    "        batch_x = np.array([x])\n",
    "        batch_y = np.array([y])\n",
    "        #print('batch x : ', batch_x.shape)\n",
    "        #print('batch y : ', batch_y.shape)\n",
    "        preactivation,activation,yhat=forwardPropagation(batch_x,parameters)\n",
    "          #print(preactivation)\n",
    "          #print(activation)\n",
    "        #print(\"y_pred\",yhat)\n",
    "        #print(\"loss\",Loss(yhat,y_train,X_train))\n",
    "        loss.append(Loss(yhat,batch_y,batch_x))\n",
    "        gradients=backPropagation(parameters,activation,preactivation,yhat,batch_x,batch_y)\n",
    "        #print(\"gradients\",gradients)\n",
    "        for i in range(1,noOfHiddenLayers+2):\n",
    "          parameters['W'+str(i)] -=  eta*gradients['W'+str(i)]#parameters['W'+str(i)] -=  eta*(1.0/X_train.shape[0])*gradients['W'+str(i)]\n",
    "          parameters['b'+str(i)] -= eta*gradients['b'+str(i)]#parameters['b'+str(i)] -= eta*(1.0/X_train.shape[0])*gradients['b'+str(i)]\n",
    "    #mini+=1\n",
    "    #_,_,yhat=forwardPropagation(X_train,parameters)\n",
    "    \n",
    "    #print(yhat)\n",
    "    t+=1\n",
    "\n",
    "#print(parameters)\n",
    "#print(gradients)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_zeros_like(parameters):\n",
    "    zero_parameters = {}\n",
    "    for k in range(1, noOfHiddenLayers + 2):\n",
    "        zero_parameters['W' + str(k)] = np.zeros_like(parameters['W' + str(k)])\n",
    "        zero_parameters['b' + str(k)] = np.zeros_like(parameters['b' + str(k)])\n",
    "    return zero_parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multiply_parameters(parameters, gamma):\n",
    "    new_parameters = {}   \n",
    "    for k in range(1, noOfHiddenLayers + 2):\n",
    "        new_parameters['W' + str(k)] = gamma * parameters['W' + str(k)]\n",
    "        new_parameters['b' + str(k)] = gamma * parameters['b' + str(k)]\n",
    "    return new_parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_parameters(parameters, update, subtract = True):\n",
    "    for i in range(1,noOfHiddenLayers+2):\n",
    "        if subtract:\n",
    "            parameters['W'+str(i)] -= update['W'+str(i)]\n",
    "            parameters['b'+str(i)] -= update['b'+str(i)]\n",
    "        else :\n",
    "            parameters['W'+str(i)] += update['W'+str(i)]\n",
    "            parameters['b'+str(i)] += update['b'+str(i)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nag():\n",
    "  #batchSize=1000\n",
    "  iterations=200\n",
    "  #initialization\n",
    "  parameters['W'+str(1)] =np.random.uniform(low=-0.5,high=0.5,size=(noOfneuronsEach[0], inputNeuronSize))\n",
    "  parameters['b'+str(1)]= np.random.uniform(low=-0.5,high=0.5,size=(noOfneuronsEach[0],1))\n",
    "  #parameters['W'+str(1)] =0+1.5*np.random.randn(noOfneuronsEach[0], inputNeuronSize)\n",
    "  #parameters['b'+str(1)]= 0+1.5*np.random.randn(noOfneuronsEach[0],1)\n",
    "  for i in range(2,noOfHiddenLayers+1):\n",
    "    parameters['W'+str(i)] = np.random.uniform(low=-0.5,high=0.5,size=(noOfneuronsEach[i-1], noOfneuronsEach[i-2]))\n",
    "    parameters['b'+str(i)]= np.random.uniform(low=-0.5,high=0.5,size=(noOfneuronsEach[i-1],1))\n",
    "    #parameters['W'+str(i)] = 0+1.5*np.random.randn(noOfneuronsEach[i-1], noOfneuronsEach[i-2])\n",
    "    #parameters['b'+str(i)]= 0+1.5*np.random.randn(noOfneuronsEach[i-1],1)\n",
    "  #Output W\n",
    "  parameters['W'+str(noOfHiddenLayers+1)] = np.random.uniform(low=-0.5,high=0.5,size=(l, noOfneuronsEach[-1]))\n",
    "  parameters['b'+str(noOfHiddenLayers+1)]= np.random.uniform(low=-0.5,high=0.5,size=(l,1))\n",
    "  #parameters['W'+str(noOfHiddenLayers+1)] = 0+1.5*np.random.randn(l, noOfneuronsEach[-1])\n",
    "  #parameters['b'+str(noOfHiddenLayers+1)]= 0+1.5*np.random.randn(l,1)\n",
    "  t=0\n",
    "  prev_update = make_zeros_like(parameters)\n",
    "  #print(prev_update['b1'])\n",
    "  gamma = 0.9\n",
    "  while(t < iterations):\n",
    "    print(\"iter \",t)\n",
    "    # do partial updates\n",
    "    update = multiply_parameters(prev_update, gamma)\n",
    "    update_parameters(parameters, update)\n",
    "    #print('x train shape ', X_train.shape)\n",
    "    #print('w1 shape ', parameters['W1'].shape)\n",
    "    preactivation,activation,yhat=forwardPropagation(X_train,parameters)\n",
    "    gradients=backPropagation(parameters,activation,preactivation,yhat,X_train,y_train)\n",
    "    eta_times_grad = multiply_parameters(gradients, eta*(1.0/X_train.shape[0]))\n",
    "    update_parameters(parameters, eta_times_grad)\n",
    "    update_parameters(update, eta_times_grad, subtract = False)\n",
    "    prev_update = update\n",
    "    loss.append(Loss(yhat,y_train,X_train))\n",
    "    t+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iter  0\n",
      "iter  1\n",
      "iter  2\n",
      "iter  3\n",
      "iter  4\n",
      "iter  5\n",
      "iter  6\n",
      "iter  7\n",
      "iter  8\n",
      "iter  9\n",
      "iter  10\n",
      "iter  11\n",
      "iter  12\n",
      "iter  13\n",
      "iter  14\n",
      "iter  15\n",
      "iter  16\n",
      "iter  17\n",
      "iter  18\n",
      "iter  19\n",
      "iter  20\n",
      "iter  21\n",
      "iter  22\n",
      "iter  23\n",
      "iter  24\n",
      "iter  25\n",
      "iter  26\n",
      "iter  27\n",
      "iter  28\n",
      "iter  29\n",
      "iter  30\n",
      "iter  31\n",
      "iter  32\n",
      "iter  33\n",
      "iter  34\n",
      "iter  35\n",
      "iter  36\n",
      "iter  37\n",
      "iter  38\n",
      "iter  39\n",
      "iter  40\n",
      "iter  41\n",
      "iter  42\n",
      "iter  43\n",
      "iter  44\n",
      "iter  45\n",
      "iter  46\n",
      "iter  47\n",
      "iter  48\n",
      "iter  49\n",
      "iter  50\n",
      "iter  51\n",
      "iter  52\n",
      "iter  53\n",
      "iter  54\n",
      "iter  55\n",
      "iter  56\n",
      "iter  57\n",
      "iter  58\n",
      "iter  59\n",
      "iter  60\n",
      "iter  61\n",
      "iter  62\n",
      "iter  63\n",
      "iter  64\n",
      "iter  65\n",
      "iter  66\n",
      "iter  67\n",
      "iter  68\n",
      "iter  69\n",
      "iter  70\n",
      "iter  71\n",
      "iter  72\n",
      "iter  73\n",
      "iter  74\n",
      "iter  75\n",
      "iter  76\n",
      "iter  77\n",
      "iter  78\n",
      "iter  79\n",
      "iter  80\n",
      "iter  81\n",
      "iter  82\n",
      "iter  83\n",
      "iter  84\n",
      "iter  85\n",
      "iter  86\n",
      "iter  87\n",
      "iter  88\n",
      "iter  89\n",
      "iter  90\n",
      "iter  91\n",
      "iter  92\n",
      "iter  93\n",
      "iter  94\n",
      "iter  95\n",
      "iter  96\n",
      "iter  97\n",
      "iter  98\n",
      "iter  99\n",
      "iter  100\n",
      "iter  101\n",
      "iter  102\n",
      "iter  103\n",
      "iter  104\n",
      "iter  105\n",
      "iter  106\n",
      "iter  107\n",
      "iter  108\n",
      "iter  109\n",
      "iter  110\n",
      "iter  111\n",
      "iter  112\n",
      "iter  113\n",
      "iter  114\n",
      "iter  115\n",
      "iter  116\n",
      "iter  117\n",
      "iter  118\n",
      "iter  119\n",
      "iter  120\n",
      "iter  121\n",
      "iter  122\n",
      "iter  123\n",
      "iter  124\n",
      "iter  125\n",
      "iter  126\n",
      "iter  127\n",
      "iter  128\n",
      "iter  129\n",
      "iter  130\n",
      "iter  131\n",
      "iter  132\n",
      "iter  133\n",
      "iter  134\n",
      "iter  135\n",
      "iter  136\n",
      "iter  137\n",
      "iter  138\n",
      "iter  139\n",
      "iter  140\n",
      "iter  141\n",
      "iter  142\n",
      "iter  143\n",
      "iter  144\n",
      "iter  145\n",
      "iter  146\n",
      "iter  147\n",
      "iter  148\n",
      "iter  149\n",
      "iter  150\n",
      "iter  151\n",
      "iter  152\n",
      "iter  153\n",
      "iter  154\n",
      "iter  155\n",
      "iter  156\n",
      "iter  157\n",
      "iter  158\n",
      "iter  159\n",
      "iter  160\n",
      "iter  161\n",
      "iter  162\n",
      "iter  163\n",
      "iter  164\n",
      "iter  165\n",
      "iter  166\n",
      "iter  167\n",
      "iter  168\n",
      "iter  169\n",
      "iter  170\n",
      "iter  171\n",
      "iter  172\n",
      "iter  173\n",
      "iter  174\n",
      "iter  175\n",
      "iter  176\n",
      "iter  177\n",
      "iter  178\n",
      "iter  179\n",
      "iter  180\n",
      "iter  181\n",
      "iter  182\n",
      "iter  183\n",
      "iter  184\n",
      "iter  185\n",
      "iter  186\n",
      "iter  187\n",
      "iter  188\n",
      "iter  189\n",
      "iter  190\n",
      "iter  191\n",
      "iter  192\n",
      "iter  193\n",
      "iter  194\n",
      "iter  195\n",
      "iter  196\n",
      "iter  197\n",
      "iter  198\n",
      "iter  199\n",
      "iter  200\n",
      "iter  201\n",
      "iter  202\n",
      "iter  203\n",
      "iter  204\n",
      "iter  205\n",
      "iter  206\n",
      "iter  207\n",
      "iter  208\n",
      "iter  209\n",
      "iter  210\n",
      "iter  211\n",
      "iter  212\n",
      "iter  213\n",
      "iter  214\n",
      "iter  215\n",
      "iter  216\n",
      "iter  217\n",
      "iter  218\n",
      "iter  219\n",
      "iter  220\n",
      "iter  221\n",
      "iter  222\n",
      "iter  223\n",
      "iter  224\n",
      "iter  225\n",
      "iter  226\n",
      "iter  227\n",
      "iter  228\n",
      "iter  229\n",
      "iter  230\n",
      "iter  231\n",
      "iter  232\n",
      "iter  233\n",
      "iter  234\n",
      "iter  235\n",
      "iter  236\n",
      "iter  237\n",
      "iter  238\n",
      "iter  239\n",
      "iter  240\n",
      "iter  241\n",
      "iter  242\n",
      "iter  243\n",
      "iter  244\n",
      "iter  245\n",
      "iter  246\n",
      "iter  247\n",
      "iter  248\n",
      "iter  249\n",
      "iter  250\n",
      "iter  251\n",
      "iter  252\n",
      "iter  253\n",
      "iter  254\n",
      "iter  255\n",
      "iter  256\n",
      "iter  257\n",
      "iter  258\n",
      "iter  259\n",
      "iter  260\n",
      "iter  261\n",
      "iter  262\n",
      "iter  263\n",
      "iter  264\n",
      "iter  265\n",
      "iter  266\n",
      "iter  267\n",
      "iter  268\n",
      "iter  269\n",
      "iter  270\n",
      "iter  271\n",
      "iter  272\n",
      "iter  273\n",
      "iter  274\n",
      "iter  275\n",
      "iter  276\n",
      "iter  277\n",
      "iter  278\n",
      "iter  279\n",
      "iter  280\n",
      "iter  281\n",
      "iter  282\n",
      "iter  283\n",
      "iter  284\n",
      "iter  285\n",
      "iter  286\n",
      "iter  287\n",
      "iter  288\n",
      "iter  289\n",
      "iter  290\n",
      "iter  291\n",
      "iter  292\n",
      "iter  293\n",
      "iter  294\n",
      "iter  295\n",
      "iter  296\n",
      "iter  297\n",
      "iter  298\n",
      "iter  299\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x23e80304988>]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD4CAYAAADy46FuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAxVUlEQVR4nO3deXhc5Xn38e89o10a7au1WF4kL9gYbGMTCAQwBkJIIAk0pklxWhKnlCY0tE2heRv6NqVNmrZpaV5oKTiBQA2ULJCwxWEJu20ZvBsjGXmRLWuxJGuxJVnS/f4xZ+yxLI3W0Wz357rm0uiZc46eAzP+zbOc54iqYowxxgzHFeoKGGOMCW8WFMYYYwKyoDDGGBOQBYUxxpiALCiMMcYEFBfqCky23NxcLS8vD3U1jDEmomzevLlZVfOGei3qgqK8vJyqqqpQV8MYYyKKiOwf7jXrejLGGBOQBYUxxpiALCiMMcYEZEFhjDEmIAsKY4wxAVlQGGOMCciCwhhjTEAWFMaMUWNHNy/tPBLqahgzZSwojBmjL6/dxNd+upkTvf2hrooxU8KCwpgxOtLeDUBXb1+Ia2LM1LCgMGaMUhLcANaiMDHDgsKYMfIFxXELChMjLCiMGaPkeF9QWNeTiQ0WFMaMUbJ1PZkYM2JQiMhaEWkUkR2Dyr8uIntEZKeI/JNf+d0iUuO8drVf+RIR2e68dp+IiFOeKCJPOuUbRKTcb5/VIlLtPFZPyhkbM0GpCd7V+bssKEyMGE2L4ifANf4FInI5cD1wrqqeA/yzUz4fWAWc4+xzv4i4nd0eANYAFc7Dd8xbgVZVnQ38EPi+c6xs4B5gObAMuEdEssZ1lqPQ3n2S7/56F5v2tQTrT5gokZxgXU8mtowYFKr6OjD4X8/bgO+pao+zTaNTfj3whKr2qGotUAMsE5EiIF1V31FVBR4FbvDb5xHn+dPACqe1cTWwXlVbVLUVWM+gwJpMAwPKw2/Wsr3uWLD+hIkSNphtYs14xygqgUucrqLficgFTnkxcNBvuzqnrNh5Prj8jH1UtQ84BuQEONZZRGSNiFSJSFVTU9O4TijF6U44cdI+/CYw32B2V4+1KExsGG9QxAFZwIXAXwJPOa0AGWJbDVDOOPc5s1D1QVVdqqpL8/KGvOXriBLiXMS5xD78ZkTJzpcKa1GYWDHeoKgDfq5eG4EBINcpL/XbrgQ47JSXDFGO/z4iEgdk4O3qGu5YQZOS4LYPvxmR2/nU2JXZJlaMNyh+CVwBICKVQALQDDwLrHJmMs3AO2i9UVXrgQ4RudBpedwCPOMc61nAN6PpRuAVZxzjJeAqEclyBrGvcsqCJjUxzloUZkTqtGuP99iXChMb4kbaQETWAZcBuSJSh3cm0lpgrTNlthdY7fzjvlNEngJ2AX3A7arq+zTdhncGVTLwgvMAeBj4qYjU4G1JrAJQ1RYR+S6wydnu71Q1qFOSrEVhxsLeKyZWjBgUqnrzMC99aZjt7wXuHaK8ClgwRHk3cNMwx1qLN5SmRGpinHUnmBH5BspseqyJFXZltp+UBLd1J5gR+bqe7II7EyssKPykJliLwoxMnTbFcRvPMjHCgsJPSmKc9TubUbMWhYkVFhR+UhPcNuvJjMzpejphrU8TIywo/KQkWIvCjMw3mG0tChMrLCj8eKfH9qE65AXgxgCcen/YGIWJFRYUflIS3Qwo9PQNhLoqJgIcP9nPwIB9qTDRz4LCz6n7DNg3RROAr8GpCt191v1kop8FhR9bPtqMhn8bwt4rJhZYUPhJTfTducxaFGZ0OrvtvWKinwWFH1+LosuuzjYB+M91OHbiZOgqYswUsaDw42tR2Bo+JhD163yyoDCxwILCj7UozGj4tyjaLChMDLCg8JOaYC0KMzbWojCxwILCz6kWhc1kMSNIivd+dNotKEwMGPF+FLEkxTdGYddRmABUlcQ475eKtuO9Ia6NMcFnLQo/yfF2HYUZHRHITE6wricTEywo/LhdQlpiHO3d9uE3w1NAgMyUeFq67L1iop8FxSAZyfEcO24ffjM836ynPE8iTZ09oa2MMVPAgmKQrNR4Wq3f2QSgKCJCQXoSje3doa6OMUE3YlCIyFoRaRSRHUO89hcioiKS61d2t4jUiMgeEbnar3yJiGx3XrtPRMQpTxSRJ53yDSJS7rfPahGpdh6rJ3y2o5CZnGBz482IBChIT6Sxo8dWkDVRbzQtip8A1wwuFJFSYCVwwK9sPrAKOMfZ534RcTsvPwCsASqch++YtwKtqjob+CHwfedY2cA9wHJgGXCPiGSN7fTGLjMlnjbrejIB+LqeCtKT6B9Qmrus+8lEtxGDQlVfB1qGeOmHwLc4czHN64EnVLVHVWuBGmCZiBQB6ar6jnrv+vIocIPfPo84z58GVjitjauB9araoqqtwHqGCKzJlpWSYFMeTUCKd9ZTQXoSAA3HLChMdBvXGIWIfAY4pKpbB71UDBz0+73OKSt2ng8uP2MfVe0DjgE5AY41VH3WiEiViFQ1NTWN55ROyUyJ59iJk9adYIblbVEIpVkpAOxv6QppfYwJtjEHhYikAN8GvjPUy0OUaYDy8e5zZqHqg6q6VFWX5uXlDbXJqGWmJDCgtjSDCUwEZuSmIgJ7Gy0oTHQbT4tiFjAD2Coi+4AS4D0RKcT7rb/Ub9sS4LBTXjJEOf77iEgckIG3q2u4YwXVtAxvd8KhthPB/lMmYnm/ryQnuCnOTKamqTPE9TEmuMYcFKq6XVXzVbVcVcvx/oO+WFWPAM8Cq5yZTDPwDlpvVNV6oENELnTGH24BnnEO+Szgm9F0I/CKM47xEnCViGQ5g9hXOWVBVZrt7U6oaz0e7D9lIpTq6eZuRX4ae460h7Q+xgTbaKbHrgPeAeaISJ2I3Drctqq6E3gK2AW8CNyuqr71MG4DHsI7wL0XeMEpfxjIEZEa4E7gLudYLcB3gU3O4++csqDyBcWBFgsKMzxxkuL8siyqGzvtIk0T1UZcFFBVbx7h9fJBv98L3DvEdlXAgiHKu4Gbhjn2WmDtSHWcTBnJ8WQkx7PvqAWFGZq3ReFNiqXTs1CFzQdauGJuQYhrZkxw2JXZQ1hYnMGm2qA3XkyE8r/D3eLpWSTFu3j1g4nNtjMmnFlQDOGyOXlUN3ayr9lms5izqZ7uekqKd3NJRR4v725A1aZUm+hkQTGETy+aRkKci/tfqwl1VUyY8p+7feW8fA4f62Z3fUfI6mNMMFlQDKEgPYkvLi/jZ+8dYv9Ra1WYMw1uN1wxtwAR+O3uhpDUx5hgs6AYxlcumUn/gLJ+l334zZm8XU+n2xR5nkQWlWTysgWFiVIWFMMozkxmRm4q7+w9GuqqmDCjQywQsHJ+AVvrjtFgy46bKGRBEcAF5VlsOdgW6mqYMCSDFpi5cp53auzLuxtDUBtjgsuCIoA5hekc7eql2e5iZvwNMbmpsiCNoowk3vnIWqAm+lhQBFBZkAbAhw02m8Wc5ltm3J+IcH5ZJlsOtoakTsYEkwVFAJUFHgCqG2zRN3MmGWJx4/NLszjYcsJaoCbqWFAEkO9JJDXBzT6bImv8qOpZLQqA88oyAdhyoG1K62NMsFlQBCAiTM9JtSu0zRmGu/56wbQM4lzC+9b9ZKKMBcUIynNT2G8LBBo//suM+0tOcFNZ4GFb3bEpr5MxwWRBMYLynFQOtBynr38g1FUxYUSG6nsC5hZ6bEzLRB0LihGU56TSN6AcbrMLqYxXoKX/5hR6ONLebfenMFHFgmIE5bmpANTagLZxqOqQXU8AlYXemXJ7bEq1iSIWFCMoz/He8c4GtI2PwtCDFMCcAgsKE30sKEaQ50kkxabImkGGa1EUZSThSYzjwyMWFCZ6WFCMwKbImrMEGKQQESoLPeyxoDBRxIJiFGbYFFnjR9FhZz2Bd0B7T0OH3fHORI0Rg0JE1opIo4js8Cv7gYh8ICLbROQXIpLp99rdIlIjIntE5Gq/8iUist157T5xPmkikigiTzrlG0Sk3G+f1SJS7TxWT9ZJj9V0myJrBhk+JrzjFMdOnKSxw5byMNFhNC2KnwDXDCpbDyxQ1XOBD4G7AURkPrAKOMfZ534RcTv7PACsASqch++YtwKtqjob+CHwfedY2cA9wHJgGXCPiGSN/RQnboZNkTV+/O+ZPRTfGmEfWPeTiRIjBoWqvg60DCr7jar2Ob++C5Q4z68HnlDVHlWtBWqAZSJSBKSr6jvqbY8/Ctzgt88jzvOngRVOa+NqYL2qtqhqK95wGhxYU2K6M/PJpsga8AZFIL5Vh6tt5pOJEpMxRvFHwAvO82LgoN9rdU5ZsfN8cPkZ+zjhcwzICXCss4jIGhGpEpGqpqamCZ3MUGY411LYgLYBZ4wiQOdTTloiOakJtjy9iRoTCgoR+TbQBzzuKxpiMw1QPt59zixUfVBVl6rq0ry8vMCVHgffFNlaCwrjCNT1BFBRkEZ1oy3lYaLDuIPCGVy+Dviinp7eUQeU+m1WAhx2ykuGKD9jHxGJAzLwdnUNd6wpJyLMzEtlb5N98M3IXU/gHaeoaei0mU8mKowrKETkGuCvgM+oqv+80WeBVc5Mphl4B603qmo90CEiFzrjD7cAz/jt45vRdCPwihM8LwFXiUiWM4h9lVMWEpUFNjfeeI3mn/6K/DQ6evqoP2YTIEzkG8302HXAO8AcEakTkVuBHwEeYL2IbBGR/wRQ1Z3AU8Au4EXgdlXtdw51G/AQ3gHuvZwe13gYyBGRGuBO4C7nWC3Ad4FNzuPvnLKQmFPgobGjh9au3lBVwYSRQNdRAFQ4M59snMJEg7iRNlDVm4cofjjA9vcC9w5RXgUsGKK8G7hpmGOtBdaOVMepMKfw9Ad/+cycENfGhNJw96Pw55siW9PYyWVz8oNfKWOCyK7MHiX/oDCxbuTOp+zUBHLTbOaTiQ4WFKNUmJ6EJynOLqIyI15w51OR7+FDu4mRiQIWFKMkIswp8Ng3RAOMMigK0qhptJlPJvJZUIzBHGdVUPvgxzbvRT4jJ0VFgYdOm/lkooAFxRjMKfTQ3t1HQ7st9hbLRvtFoTLfu5SHtUJNpLOgGIPTi721h7gmJpSU0XU9+d4v1TZOYSKcBcUYzLG58cYxipwgKzWB3LREe7+YiGdBMQZZqQnkexLZc8S+IcaysQxRVeSn8aGt+WQinAXFGM0ptJlPsU5hdH1PeJccr7G73ZkIZ0ExRpXOFNn+Afvgx7LRxYR35lNXbz+HbeaTiWAWFGM0p8BDT98AB1rsHtqxSlVH26A4NaBtrVATySwoxsi3lIetJGtGoyLf7nZnIp8FxRhVFNjc+Fg3mkUBfU7PfLIBbRO5LCjGKCUhjrLsFGtRxLiRlhn3N7fQ7mViIpsFxThUFnjYYy2KmOW9Z/bozSvyvl/6+geCVidjgsmCYhzmFnqobe6ip69/5I1N1BnrTNd5Ren09g3YPddNxLKgGIfKQg/9A8pHTfbBj0WjXWbcZ15ROgC76m3pFxOZLCjGwZbyMKNZPdZnVl4a8W6xe5mYiGVBMQ4zclOJc9kHP1bpKO5w5y8hzsWsvDR2W4vCRCgLinFIiHMxMy+VDy0oYpJ6b0gxJvOL0i0oTMQaMShEZK2INIrIDr+ybBFZLyLVzs8sv9fuFpEaEdkjIlf7lS8Rke3Oa/eJM79QRBJF5EmnfIOIlPvts9r5G9UisnrSznoSzClMt5lPMWyMOcHcIg8N7T20dPUGpT7GBNNoWhQ/Aa4ZVHYX8LKqVgAvO78jIvOBVcA5zj73i4jb2ecBYA1Q4Tx8x7wVaFXV2cAPge87x8oG7gGWA8uAe/wDKdTmFKRR13qCzp6+UFfFTLHR3o/Cn29A+wNrVZgINGJQqOrrQMug4uuBR5znjwA3+JU/oao9qloL1ADLRKQISFfVd9S7jOajg/bxHetpYIXT2rgaWK+qLaraCqzn7MAKGVvDJ4aNYz1Im/lkItl4xygKVLUewPmZ75QXAwf9tqtzyoqd54PLz9hHVfuAY0BOgGOdRUTWiEiViFQ1NTWN85TGxrfmk41TxB7vBXdja1LkpiWS50lkd729X0zkmezB7KE+PcMN/fm+l41nnzMLVR9U1aWqujQvL29UFZ2o0qwUkuPdNk4Ro8ba9QTeCzVtQNtEovEGRYPTnYTzs9EprwNK/bYrAQ475SVDlJ+xj4jEARl4u7qGO1ZYcLmEyoI063qKQeO9B9GC4gyqGzvoPmlX9JvIMt6geBbwzUJaDTzjV77Kmck0A++g9Uane6pDRC50xh9uGbSP71g3Aq844xgvAVeJSJYziH2VUxY2KgtssbdYNJ7BbIBzizM42a/2njERZzTTY9cB7wBzRKRORG4FvgesFJFqYKXzO6q6E3gK2AW8CNyuqr6vT7cBD+Ed4N4LvOCUPwzkiEgNcCfODCpVbQG+C2xyHn/nlIWNOYUemjt7ae7sCXVVzBQb6xgFwMKSDAC21bVNcm2MCa64kTZQ1ZuHeWnFMNvfC9w7RHkVsGCI8m7gpmGOtRZYO1IdQ+XUgHZDB7lpiSGujZkqY7nDnb/izGRyUhPYVnds8itlTBDZldkTcGrNJ+tKiCnjvVu6iLCwJMOCwkQcC4oJyPMkkpkSbzOfYsx4B7MBzi3JpLqxg+O9dqGmiRwWFBMgIjagHaPGcoc7f+cWZzCgsOuwTZM1kcOCYoLmFnr4sKETncjXTBNRxrEm4CnnOgPaW637yUQQC4oJqizw0NnTx+Fj3aGuipkqE/hSkJ+eRGF6Ettt5pOJIBYUE+Sb+bTbuhJixnivo/CxAW0TaSwoJmh+UToisOOwffBjyQRygkUlGXzU3MWxEycnrT7GBJMFxQSlJsYxKy+N7fYNMWZMdDjq/DLvavlbDrZNvDLGTAELiklwbnEG2w9ZUMQKRcc96wngvNJM3C5h876wWmjAmGFZUEyCBcUZNHb00NBuA9qxYiJdT6mJccwr8rBpX+uk1ceYYLKgmAS+KY/W/RQbVCc2mA2wdHo2Ww62cbJ/YHIqZUwQWVBMgvnT0nEJbLPup5gwGZfMLJmexYmT/XZ/ChMRLCgmQUpCHLPz09hhQRETvDkxsSbF0nLvgHaVdT+ZCGBBMUkWFGewra7NrtCOERPteirKSKY4M5nN+y0oTPizoJgk55dl0dzZS13riVBXxQSZqk6wPeG1bEY2G2qP2pcLE/YsKCbJEmduvH1DNKP1sVk5NHf2Ut3YGeqqGBOQBcUkmVPoIS0xjqr9Njc+2k3GrCeAi2blAPB2TfPED2ZMEFlQTBK3Szi/LJPN+9tCXRUzBcZzK9TBSrJSKMtO4e29RyehRsYEjwXFJFpclsWeI+10dNsaPtFMx32Pu7N9bGYO7350lP4BG6cw4cuCYhItmZ7FgMLWgzZNNppNVtcTwEWzc2jv7rMbGZmwNqGgEJFvishOEdkhIutEJElEskVkvYhUOz+z/La/W0RqRGSPiFztV75ERLY7r90nzkI6IpIoIk865RtEpHwi9Q2288syEcHGKWLAZAXFx5xxijdtnMKEsXEHhYgUA98AlqrqAsANrALuAl5W1QrgZed3RGS+8/o5wDXA/SLidg73ALAGqHAe1zjltwKtqjob+CHw/fHWdyp4kuKZV5jOho8sKKKZ9w53k5MU+Z4k5hel89qexkk5njHBMNGupzggWUTigBTgMHA98Ijz+iPADc7z64EnVLVHVWuBGmCZiBQB6ar6jnonlD86aB/fsZ4GVvhaG+Hqolk5bD7QSvfJ/lBXxQTJZF/3cPncPKr2t9JuY1smTI07KFT1EPDPwAGgHjimqr8BClS13tmmHsh3dikGDvodos4pK3aeDy4/Yx9V7QOOATmD6yIia0SkSkSqmpqaxntKk+Li2bn09g3Y9RRRTGGiK3ic4fI5+fQPKG9WW/eTCU8T6XrKwvuNfwYwDUgVkS8F2mWIsuHuU+/7yhbotdMFqg+q6lJVXZqXlxe44kF2wYxs3C7h7b32oY9mk9msPa80k4zkeF79wLqfTHiaSNfTlUCtqjap6kng58BFQIPTnYTz0/furwNK/fYvwdtVVec8H1x+xj5O91YGENYDAGmJcSwqyeCtGpsbH7WUCd24aLA4t4tLK/N4dU8TAzZN1oShiQTFAeBCEUlxxg1WALuBZ4HVzjargWec588Cq5yZTDPwDlpvdLqnOkTkQuc4twzax3esG4FXNAIWxrloVi7b6tqszzlKBeMNeMXcPJo7e9ha1xaEoxszMRMZo9iAd4D5PWC7c6wHge8BK0WkGljp/I6q7gSeAnYBLwK3q6pvxPc24CG8A9x7gRec8oeBHBGpAe7EmUEV7i6ancOAwjt2xW3UmuwZFVfMLSDeLby448gkH9mYiYubyM6qeg9wz6DiHryti6G2vxe4d4jyKmDBEOXdwE0TqWMoLJ2eTVpiHK/taeTqcwpDXR0zyVR10q6j8MlIjueiWbm8sOMId31y7qR2bRkzUXZldhAkxLn4+OxcXv2gyZaQjkLB+j/6yQWFHGg5zi67650JMxYUQXLF3HyOtHfbhz4KqU5+1xPAyvkFuATrfjJhx4IiSC6b652ma1Meo1MwuoZy0hK5cGYOv95Wby1RE1YsKIIk35PEwuIMXrGgiDrK5Nzhbig3nFdMbXMXWw62BekvGDN2FhRBdOW8At4/2EZje3eoq2ImUTC/7F+zsJDEOBe/eP9Q8P6IMWNkQRFE1y4sRBVe3Gl9ztFEh1tPYBKkJ8Vz5fwCfrX1ML19A8H5I8aMkQVFEFUUeJidn8Zz2+pDXRUzySZr9dihfO78YlqPn7QVZU3YsKAIsmsXFrFxXwuNHdb9FE2CeZnDpZV55HsSWbfxQPD+iDFjYEERZJ9aWIQqvGRTHqNGsGckxbtdrFpWxmsfNnHg6PGg/i1jRsOCIsgqC9KoyE+zwckoE+zrpm9eVopLhMc37g/yXzJmZBYUQSYi3LikhPcOtFHT2Bnq6phJoAS36wmgKCOZlfMK+N+qOrsJlgk5C4op8NnFxbhdwtOb60be2IS9qboW7g8+Np2Wrl6e326TIUxoWVBMgXxPEpdV5vHz9+ro67cpj5HOe8Fd8Bftu2hWDrPyUnn4zVq7UtuElAXFFLlpaQmNHT28uie0t2o1k2MqFncVEb72iVnsPNzO7z60940JHQuKKbJiXgGF6Un85O3aUFfFTJDq1AQFeJf0mJaRxP2v7p2aP2jMECwopki828UtF03nrZqj7DnSEerqmAmYyk6ghDgXX710Jhv3tbCxNqzvAmyimAXFFLr5gjKS4l38+C1rVUQy73DB1N1YaNUFZWSnJvCjV2um7G8a48+CYgplpSbw2fNL+Pn7h2iwhQIj2lTegC45wc2aS2fy+odNdntdExIWFFPstk/Mon9AeeA163OOXMFbZnw4X76onKKMJL73wm6bAWWm3ISCQkQyReRpEflARHaLyMdEJFtE1otItfMzy2/7u0WkRkT2iMjVfuVLRGS789p94twVRkQSReRJp3yDiJRPpL7hoCwnhc8vLmbdxgO2/HiECsW/00nxbu5cWcnWumM8Z9dVmCk20RbFvwMvqupcYBGwG7gLeFlVK4CXnd8RkfnAKuAc4BrgfhFxO8d5AFgDVDiPa5zyW4FWVZ0N/BD4/gTrGxb+9PIK+gbU+pwj2FR2Pfl8bnEJcws9/OClPfT02dXaZuqMOyhEJB24FHgYQFV7VbUNuB54xNnsEeAG5/n1wBOq2qOqtUANsExEioB0VX1HvW3qRwft4zvW08AKX2sjkpXlpHDzslIe33CAmkabARVpvLejmPq3odsl/PW189h/9Dj/9buPpvzvm9g1kRbFTKAJ+LGIvC8iD4lIKlCgqvUAzs98Z/ti4KDf/nVOWbHzfHD5Gfuoah9wDMiZQJ3DxjevrCQlwc3fP7c71FUxYxTKMYJLK/P41LlF/OjVGvY1d4WsHia2TCQo4oDFwAOqej7QhdPNNIyhvoINd68w3ycx0GunDyyyRkSqRKSqqSkyrmDNSUvkjhUVvLanid/YHfAiylQsChjIPdfNJ9Ht4m+e2WED22ZKTCQo6oA6Vd3g/P403uBocLqTcH42+m1f6rd/CXDYKS8ZovyMfUQkDsgAzrrqSFUfVNWlqro0Ly9vAqc0tW75WDlzCz38n1/u4NiJk6GujhmDUPZ/5qcn8ZfXzOGN6mZbaNJMiXEHhaoeAQ6KyBynaAWwC3gWWO2UrQaecZ4/C6xyZjLNwDtovdHpnuoQkQud8YdbBu3jO9aNwCsaRV+hEuJc/ODGRRzt6uXe53aFujpmlLxLeIR2qOyLy6ezfEY2f/vsTru5kQm6ic56+jrwuIhsA84D/gH4HrBSRKqBlc7vqOpO4Cm8YfIicLuq+qZu3AY8hHeAey/wglP+MJAjIjXAnQTu2opIC0syWHPpTJ6qqrN7JEeIcPiu4nYJ//qF83C5hG8+tcVWJTZBNaGgUNUtTpfPuap6g6q2qupRVV2hqhXOzxa/7e9V1VmqOkdVX/Arr1LVBc5rf+prNahqt6repKqzVXWZqkblVI87VlRQWZDGnz+1lSPH7NoKMzrFmcn8/Q0L2Ly/lfterg51dUwUsyuzw0BSvJv7v7iYEyf7+fq69+zbYZgL9WC2v+vPK+amJSXc90oN63c1hLo6JkpZUISJ2fke/vFzC9m0r5Xvv/hBqKtjAtHQXEcxnO/esIBzSzL45pNb7Ha7JigsKMLI9ecVc8vHpvPfb9Ty+Ib9oa6OGUboRyjOlBTv5oEvLSEhzsVXHtnE0c6eUFfJRBkLijDznevmc/mcPL7zzE5etcHtsBUuXU8+xZnJ/PctS6g/1s0fPVLF8d6+UFfJRBELijAT53bxo99fzNxCD3/y2Ht2s5owpDr1q8eOxpLp2fzHzeezva6NP3n8PVsPykwaC4owlJoYx0/+cBnTMpP4wx9vZPN+C4twEm5dT/6uOqeQez+7kNf2NHHbY+/RfdLCwkycBUWYyvMksu6rF5KfnsTqtZt4e29zqKtkHFN5z+zxuHlZGf/w2YW88kEjX/vpZgsLM2EWFGEsPz2JdV+9kKKMJL68dhO/2np45J3MlAj1ldkj+f3lZXzvcwt5vbqJP/zxJlsixkyIBUWYK8xI4uk/vojzSjP5+rr3eeiNj8LiyuBYpiG4w914rFpWxr/ctIiq/S3c+MDbHGyxpT7M+FhQRICMlHgevXUZn1xQyN8/t5s7n9rKiV7rTgiVSMrpzy0u4ZE/WkZDezefvf9tNu9vDXWVTASyoIgQSfFu/t/vL+bOlZX8csshPvfA2+w/avcjCJlIaFI4LpqVy8//5CJSEtx84b/e4eE3a61VasbEgiKCuFzCN1ZUsPbLF3C47QTX/vsbPLnpgH3op1io7nA3EbPzPfzqTz/O5XPz+e6vd3HbY+/Rdrw31NUyEcKCIgJdPief5++4hHNLMvmrn23nq49W0dRhV+NOmTCf9TScjJR4HvyDJfyfT83jt7sbuPJfX7ebZplRsaCIUMWZyTz+leX8zXXzeb26mZU//B1PbDzAwIC1LoJNw/pKisBEhK9cMpNn/vRi8jyJrPnpZu544n1auqx1YYZnQRHBXC7h1o/P4PlvfJzKfA93/Xw7n//Pt9lx6Fioqxb1IrBBcYZzpmXwzO0X82dXVvDctnou/+fX+MlbtbZysRmSBUUUmJ3v4cmvXci/3LSIgy3H+cyP3uRbT2/lUNuJUFctKoX7BXejlRDn4s+urOT5Oy5hQXE6f/urXVx73xu8/mGTjXuZM1hQRAkR4fNLSnj5zy/jDy+ewS/fP8zl//waf//rXdatMMmi7Z/QygIPj926nP/80hJOnOznlrUb+cKD77Lho6OhrpoJExYUUSYjOZ6/uW4+r/7lZVy/aBpr36rl0n96lX98fjcN7Xb3vMngXRQwCpoUfkSEaxYUsv6bn+BvPz2f2uYuvvDgu3zxoXfZWNtiLYwYJ9H2Bli6dKlWVVWFuhpho7qhg39/uZrnt9cT53Lx+SXFfPWSmczMSwt11SLWzLuf4/bLZ/PnV80JdVWCpvtkP4+9u5///N1emjt7WVSSwa2XzOSTCwqJd9v3y2gkIptVdelQr8VNdWXM1Koo8PCj31/M/qNdPPj6R/zv5jrWbTzIpZV5fGl5GVfMzSfOPvhj4r2OIrolxbv5yiUz+eLy6fzsvTrWvlnLN9a9T1FGEl+6cDo3LSkhPz0p1NU0U2TCLQoRcQNVwCFVvU5EsoEngXJgH/B7qtrqbHs3cCvQD3xDVV9yypcAPwGSgeeBO1RVRSQReBRYAhwFvqCq+wLVx1oUgTV19PD4hv2s23iAhvYepmUksWpZGTctLaEoIznU1YsI5Xc9xzeumM2dUdyiGGxgQHl1TyMPvVHLOx8dxe0SVszNZ9WyUj5RmY/bFe3RGf0CtSgmIyjuBJYC6U5Q/BPQoqrfE5G7gCxV/SsRmQ+sA5YB04DfApWq2i8iG4E7gHfxBsV9qvqCiPwJcK6q/rGIrAI+q6pfCFQfC4rROdk/wMu7G3js3QO8WdOMCHxsZg43nF/MJxcU4kmKD3UVw1b5Xc/xjRUV3LmyMtRVCYmPmjp5suogP9tcR3NnLwXpiXz63Gl8etE0zi3JCPuVdc3QghYUIlICPALcC9zpBMUe4DJVrReRIuA1VZ3jtCZQ1X909n0J+Fu8rY5XVXWuU36zs//XfNuo6jsiEgccAfI0QKUtKMZuX3MXv9xyiF+8f4j9R4+TGOdi5fwCPrWwiE/MySMlwXoo/ZXf9Rx3rKjgmzEaFD7eLxuNPL25jt992MjJfmV6TgqfPnca1y0qYk6Bx0IjggRzjOLfgG8BHr+yAlWtB3DCIt8pL8bbYvCpc8pOOs8Hl/v2Oegcq09EjgE5gN3FZxKV56byZ1dWcseKCt4/2MYv3z/Er7fV8+tt9STGubikIo+rzingynkFZKcmhLq6IRVtkz8mIt7t4poFhVyzoJBjx0/y0s4j/GrbYe5/rYYfvVpDWXYKV84r4Mr5+VxQnm2D4BFs3EEhItcBjaq6WUQuG80uQ5QNNy7o+zQGes2/LmuANQBlZWWjqIoZioiwuCyLxWVZfOe6+Wzc18Jvdjbwm51H+O3uBlwCS8uz+URlHp+ozGN+UTquGOub9uWEfVE+U0ZKPL93QSm/d0EpTR09/GbXEV7e3chjG/az9q1a0pPiuHxuPpfNyePi2bnke2wgPJJMpEVxMfAZEbkWSALSReQxoEFEivy6nhqd7euAUr/9S4DDTnnJEOX++9Q5XU8ZwFk3kFbVB4EHwdv1NIFzMo44t4uLZuVy0axc7vn0fHYcauclJzB+8NIefvDSHnJSE/h4RS6XVuRxSUVuTM2CibbrKCZTnieRLy6fzheXT+d4bx9vVDfz210NvPJBI89s8X605xZ6+PjsXD5ekcuyGdnWvRnmJuU6CqdF8RfOGMUPgKN+g9nZqvotETkH+B9OD2a/DFQ4g9mbgK8DG/AOZv+Hqj4vIrcDC/0Gsz+nqr8XqC42RhF8je3dvFHdzBvVTbxR3cxR58rvmbmpLJuRfepRkpUS4ppOvv4BZdZfP8+dKyv5xoqKUFcnogwMKDsPt/NmTTNv1jSxaV8rvX0DJLhdLJ6eyfIZOVxQns35ZZmkJlpwTLWpvo7ie8BTInIrcAC4CUBVd4rIU8AuoA+4XVV9t2m7jdPTY19wHgAPAz8VkRq8LYlVQaivGaP89CQ+v6SEzy8pYWBA2VXfzls1zWysbeG57fU8sekg4F3h1hca55dlUpHvifhplDZGMX4ul7CwJIOFJRncdtksTvT2s2lfC2/VNPNmTTP/8Uo1Awpul3DOtHQuKM/mgvIslpZnk5uWGOrqxzS7MttMqv4BZc+RDjbWHmXjvhY21rbQ3OltcaQkuFlYnMF5pZmcV5rJotJMijKSImpmTF//ALO//QJ/vrKSr1uLYlJ1dJ/kvQNtbKptYdO+FrYcbKOnz7uabVl2CotKM1lUksGi0kzOmZZu3VWTzK7MNlPG7RLmT0tn/rR0vnzxDFSV2uYuthxsY+vBNrbUHePHb+2j11nOOs+TyHmlmSwszmB+kXe/cA6PU7MswrN6Ec2TFH9qogRAT18/Ow61s2lfC1sPtrF5Xwu/2uod43CJdzHDRSXeLxwLizOoKEgjKd4dylOIWhYUJqhEhJl5aczMS+Nzi71zFnr6+tld3+ENDuexflfDqX0yU+K9oeEEx7yidGbnp4XF9MrTs54sKYItMc7NkulZLJmedaqssaObbQePsa3O+6XjpV1HeLLK29Xpdgkzc1OZV+R9z8wt8jC/KJ18T6L9/5ogCwoz5RLj3Ke6n1Y7ZZ09few50s6uw+3sqvf+/Om7+091PSS4XczOT6OyII2KAg+z89OoyE+jLDtlSteqiuS720WDfE8SV85P4sr5BYB3zOhAy3F2Hm5nd733sXl/K89uPXxqn+zUBOYVeah03jez89KYnZ9Gjo17jJoFhQkLaYlxLJmezZLp2afK+voHqG3uOhUcu490sLG2hV9uOf2PQILbxcy8VCc4PFQUeP8RKMtOsW6IGCAiTM9JZXpOKtcuLDpVfuz4SXYfOR0eu+s7WLfxAN0nT9/BLysl3hsc+WnMcsJjdn4a0zKSY+76oJFYUJiwFed2UVHgoaLAw/XnFZ8q7+zpY29jJ9WNnVQ3dlDT0Mm2umM8t73+jAvipmUkMz0nhfLcVGbkpDI9J4UZuamUTiBE7IK7yJCREs+FM3O4cGbOqbKBAeVQ2wlqmjrZ29jJ3qZOaho7eXHHEVqPnzy1XWKci7LsFKbnpFCWnUp5borzeyolWclh0QU61SwoTMRJS4zzzoApzTyj/ERvP3ubvP8A7Gs+zr6jXew72sUL2+vP+IfAP0Sm56RSmp1McWYyJVkplGQlk5eWaN8oo5DLJZRmp1CancLlc/LPeO1oZw81jZ3UNHWyr7mL/UePc6DlOG/WNJ/RCnG7hGmZSUzPTqUsx/t+Kc50HlnJ5HuSIn4K+FAsKEzUSE5ws6A4gwXFGWe9duz4yVPB4R8iL+08ctatYhPcLqZlJp0KjuLMZEqykynOTCE3zbvWlV2ZHV1y0hLJSUtkuV8LBLxjIE0dPexvOc6+5i4OtBxn39HjHBjiCwhAnEsozEhiWmYyJU54THOCZFpmEvnpSXgS4yJucN2CwsSEjJR4FqWc3QoBON7bx6HWE9S1nqCu7QR1rcdP/f7b3Y00d/actU8M9j7EJBEhP937D/wF5dlnvd7V08fhthMcch6H205wqPUEh9u62VDbwpGt3fQPnDkBIjneTUF6IvnpSRSkJ1GYnkiB8zcKPN7nBelJJCeEzxibBYWJeSkJcafGQobSfbKfQ23e4KhvO8HRrl4+s6h4yG1NbElNDPze6esfoKGjh0OtJ6g/doLG9h4a2rtp6PD+3F7Xxvr27jO6t3w8SXHkeRLJTU0kJy2B3DTvz5y0RPKcnzmpCeR6EoPeSrGgMGYESfFuZuV5Z8YYMxZxbtepMYzhqCodPX00tnfT4ATJkfZuGtt7aOrsobmjh+rGTt796OhZXV0+CW4XOWkJLC3P5j9uPn/yz2PSj2iMMWbURIT0pHjSk+KZnT90y8TnZP8ArV29NHX2cLSzl6NdPTR39NLc5f093xOca0MsKIwxJkLEu12nxkymkg3JGWOMCciCwhhjTEAWFMYYYwKyoDDGGBOQBYUxxpiALCiMMcYEZEFhjDEmIAsKY4wxAYlqdN2xS0SagP0TOEQu0DxJ1Qk30XxuEN3nZ+cWuSLl/Karat5QL0RdUEyUiFSp6tJQ1yMYovncILrPz84tckXD+VnXkzHGmIAsKIwxxgRkQXG2B0NdgSCK5nOD6D4/O7fIFfHnZ2MUxhhjArIWhTHGmIAsKIwxxgRkQeEQkWtEZI+I1IjIXaGuz3iIyFoRaRSRHX5l2SKyXkSqnZ9Zfq/d7ZzvHhG5OjS1Hh0RKRWRV0Vkt4jsFJE7nPKIPz8RSRKRjSKy1Tm3/+uUR/y5+YiIW0TeF5FfO79H07ntE5HtIrJFRKqcsqg5P8B7v9ZYfwBuYC8wE0gAtgLzQ12vcZzHpcBiYIdf2T8BdznP7wK+7zyf75xnIjDDOX93qM8hwLkVAYud5x7gQ+ccIv78AAHSnOfxwAbgwmg4N79zvBP4H+DX0fS+dOq8D8gdVBY156eq1qJwLANqVPUjVe0FngCuD3GdxkxVXwdaBhVfDzziPH8EuMGv/AlV7VHVWqAG73+HsKSq9ar6nvO8A9gNFBMF56denc6v8c5DiYJzAxCREuBTwEN+xVFxbgFE1flZUHgVAwf9fq9zyqJBgarWg/cfWyDfKY/YcxaRcuB8vN+8o+L8nK6ZLUAjsF5Vo+bcgH8DvgUM+JVFy7mBN9R/IyKbRWSNUxZN50dcqCsQJmSIsmifNxyR5ywiacDPgD9T1XaRoU7Du+kQZWF7fqraD5wnIpnAL0RkQYDNI+bcROQ6oFFVN4vIZaPZZYiysDw3Pxer6mERyQfWi8gHAbaNxPOzFoWjDij1+70EOByiuky2BhEpAnB+NjrlEXfOIhKPNyQeV9WfO8VRc34AqtoGvAZcQ3Sc28XAZ0RkH94u3StE5DGi49wAUNXDzs9G4Bd4u5Ki5vzAgsJnE1AhIjNEJAFYBTwb4jpNlmeB1c7z1cAzfuWrRCRRRGYAFcDGENRvVMTbdHgY2K2q/+r3UsSfn4jkOS0JRCQZuBL4gCg4N1W9W1VLVLUc7+fqFVX9ElFwbgAikioiHt9z4CpgB1FyfqeEejQ9XB7AtXhn0uwFvh3q+ozzHNYB9cBJvN9cbgVygJeBaudntt/233bOdw/wyVDXf4Rz+zjeJvo2YIvzuDYazg84F3jfObcdwHec8og/t0HneRmnZz1FxbnhnSm51Xns9P3bES3n53vYEh7GGGMCsq4nY4wxAVlQGGOMCciCwhhjTEAWFMYYYwKyoDDGGBOQBYUxxpiALCiMMcYE9P8BuglgGsX4ULsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nag()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  0\n",
      "loss after epoch  27155.93745441508\n",
      "Epoch  1\n",
      "loss after epoch  23596.79468408082\n",
      "Epoch  2\n",
      "loss after epoch  21644.606795944233\n",
      "Epoch  3\n",
      "loss after epoch  20898.06917928988\n",
      "Epoch  4\n",
      "loss after epoch  20167.442591350096\n",
      "Epoch  5\n",
      "loss after epoch  19520.631974892807\n",
      "Epoch  6\n",
      "loss after epoch  19064.8682344788\n",
      "Epoch  7\n",
      "loss after epoch  18650.554327358215\n",
      "Epoch  8\n",
      "loss after epoch  18261.199584947633\n",
      "Epoch  9\n",
      "loss after epoch  17922.321550681783\n",
      "Epoch  10\n",
      "loss after epoch  17619.758608873613\n",
      "Epoch  11\n",
      "loss after epoch  17334.82571837925\n",
      "Epoch  12\n",
      "loss after epoch  17076.52409833623\n",
      "Epoch  13\n",
      "loss after epoch  16825.369545609734\n",
      "Epoch  14\n",
      "loss after epoch  16611.311721295628\n",
      "Epoch  15\n",
      "loss after epoch  16399.700571759422\n",
      "Epoch  16\n",
      "loss after epoch  16213.357048643953\n",
      "Epoch  17\n",
      "loss after epoch  16025.995676930792\n",
      "Epoch  18\n",
      "loss after epoch  15852.371149447474\n",
      "Epoch  19\n",
      "loss after epoch  15677.68336089982\n",
      "Epoch  20\n",
      "loss after epoch  15517.978334937528\n",
      "Epoch  21\n",
      "loss after epoch  15357.41723749737\n",
      "Epoch  22\n",
      "loss after epoch  15206.619674960279\n",
      "Epoch  23\n",
      "loss after epoch  15064.683406339562\n",
      "Epoch  24\n",
      "loss after epoch  14928.412609710298\n",
      "Epoch  25\n",
      "loss after epoch  14799.482655848942\n",
      "Epoch  26\n",
      "loss after epoch  14674.978046256625\n",
      "Epoch  27\n",
      "loss after epoch  14550.15622358671\n",
      "Epoch  28\n",
      "loss after epoch  14425.858917562535\n",
      "Epoch  29\n",
      "loss after epoch  14303.924174918004\n",
      "Epoch  30\n",
      "loss after epoch  14182.28534893751\n",
      "Epoch  31\n",
      "loss after epoch  14068.299333402638\n",
      "Epoch  32\n",
      "loss after epoch  13960.164393568379\n",
      "Epoch  33\n",
      "loss after epoch  13855.241474619786\n",
      "Epoch  34\n",
      "loss after epoch  13751.226523894085\n",
      "Epoch  35\n",
      "loss after epoch  13647.497125502\n",
      "Epoch  36\n",
      "loss after epoch  13548.212952996237\n",
      "Epoch  37\n",
      "loss after epoch  13451.23153436044\n",
      "Epoch  38\n",
      "loss after epoch  13356.6174475362\n",
      "Epoch  39\n",
      "loss after epoch  13257.584897628598\n",
      "Epoch  40\n",
      "loss after epoch  13158.364630698135\n",
      "Epoch  41\n",
      "loss after epoch  13064.46798036486\n",
      "Epoch  42\n",
      "loss after epoch  12967.761519029842\n",
      "Epoch  43\n",
      "loss after epoch  12873.30791167803\n",
      "Epoch  44\n",
      "loss after epoch  12781.21217499507\n",
      "Epoch  45\n",
      "loss after epoch  12694.140747871123\n",
      "Epoch  46\n",
      "loss after epoch  12610.76114141767\n",
      "Epoch  47\n",
      "loss after epoch  12526.517234054736\n",
      "Epoch  48\n",
      "loss after epoch  12443.72416763244\n",
      "Epoch  49\n",
      "loss after epoch  12360.489440567018\n",
      "Epoch  50\n",
      "loss after epoch  12278.095621570026\n",
      "Epoch  51\n",
      "loss after epoch  12198.225676578166\n",
      "Epoch  52\n",
      "loss after epoch  12120.358217703719\n",
      "Epoch  53\n",
      "loss after epoch  12043.90786310214\n",
      "Epoch  54\n",
      "loss after epoch  11967.74957880296\n",
      "Epoch  55\n",
      "loss after epoch  11892.717691360089\n",
      "Epoch  56\n",
      "loss after epoch  11818.466759804836\n",
      "Epoch  57\n",
      "loss after epoch  11743.548864870669\n",
      "Epoch  58\n",
      "loss after epoch  11667.32926631166\n",
      "Epoch  59\n",
      "loss after epoch  11590.890325054675\n",
      "Epoch  60\n",
      "loss after epoch  11515.748329437927\n",
      "Epoch  61\n",
      "loss after epoch  11442.021343074526\n",
      "Epoch  62\n",
      "loss after epoch  11370.293007950042\n",
      "Epoch  63\n",
      "loss after epoch  11300.252582547348\n",
      "Epoch  64\n",
      "loss after epoch  11232.11063354556\n",
      "Epoch  65\n",
      "loss after epoch  11169.546224922684\n",
      "Epoch  66\n",
      "loss after epoch  11107.425882073614\n",
      "Epoch  67\n",
      "loss after epoch  11048.568647403892\n",
      "Epoch  68\n",
      "loss after epoch  10993.03062309527\n",
      "Epoch  69\n",
      "loss after epoch  10936.966945810022\n",
      "Epoch  70\n",
      "loss after epoch  10877.57161362797\n",
      "Epoch  71\n",
      "loss after epoch  10821.287320397492\n",
      "Epoch  72\n",
      "loss after epoch  10766.796827916347\n",
      "Epoch  73\n",
      "loss after epoch  10713.758076557813\n",
      "Epoch  74\n",
      "loss after epoch  10663.04755236802\n",
      "Epoch  75\n",
      "loss after epoch  10612.247767271587\n",
      "Epoch  76\n",
      "loss after epoch  10563.213570504784\n",
      "Epoch  77\n",
      "loss after epoch  10513.967707936778\n",
      "Epoch  78\n",
      "loss after epoch  10465.08657792774\n",
      "Epoch  79\n",
      "loss after epoch  10419.471590043864\n",
      "Epoch  80\n",
      "loss after epoch  10372.443259558831\n",
      "Epoch  81\n",
      "loss after epoch  10328.42263641474\n",
      "Epoch  82\n",
      "loss after epoch  10284.51117555759\n",
      "Epoch  83\n",
      "loss after epoch  10246.459113014755\n",
      "Epoch  84\n",
      "loss after epoch  10204.942826656516\n",
      "Epoch  85\n",
      "loss after epoch  10190.22416009885\n",
      "Epoch  86\n",
      "loss after epoch  10144.810031636978\n",
      "Epoch  87\n",
      "loss after epoch  10051.9831723369\n",
      "Epoch  88\n",
      "loss after epoch  10006.33838311433\n",
      "Epoch  89\n",
      "loss after epoch  9980.052599677936\n",
      "Epoch  90\n",
      "loss after epoch  9941.392072427669\n",
      "Epoch  91\n",
      "loss after epoch  9904.359745828982\n",
      "Epoch  92\n",
      "loss after epoch  9860.843803562286\n",
      "Epoch  93\n",
      "loss after epoch  9829.631548890293\n",
      "Epoch  94\n",
      "loss after epoch  9760.814728089988\n",
      "Epoch  95\n",
      "loss after epoch  9747.743299349075\n",
      "Epoch  96\n",
      "loss after epoch  9708.776236473852\n",
      "Epoch  97\n",
      "loss after epoch  9637.289049229501\n",
      "Epoch  98\n",
      "loss after epoch  9602.509849300399\n",
      "Epoch  99\n",
      "loss after epoch  9553.448223650694\n"
     ]
    }
   ],
   "source": [
    "def adam():\n",
    "  #initialization\n",
    "  prevW={}\n",
    "  prevb={}\n",
    "  mW={}\n",
    "  mb={}\n",
    "  epsilon=1e-8\n",
    "  beta1=0.9\n",
    "  beta2=0.999\n",
    "  prevW['W'+str(1)] =np.zeros((noOfneuronsEach[0], inputNeuronSize))\n",
    "  prevb['b'+str(1)]= np.zeros((noOfneuronsEach[0],1))\n",
    "  for i in range(2,noOfHiddenLayers+1):\n",
    "    prevW['W'+str(i)] = np.zeros((noOfneuronsEach[i-1], noOfneuronsEach[i-2]))\n",
    "    prevb['b'+str(i)]= np.zeros((noOfneuronsEach[i-1],1))\n",
    "  prevW['W'+str(noOfHiddenLayers+1)] = np.zeros((l, noOfneuronsEach[-1]))\n",
    "  prevb['b'+str(noOfHiddenLayers+1)]= np.zeros((l,1))\n",
    "\n",
    "  mW['W'+str(1)] =np.zeros((noOfneuronsEach[0], inputNeuronSize))\n",
    "  mb['b'+str(1)]= np.zeros((noOfneuronsEach[0],1))\n",
    "  for i in range(2,noOfHiddenLayers+1):\n",
    "    mW['W'+str(i)] = np.zeros((noOfneuronsEach[i-1], noOfneuronsEach[i-2]))\n",
    "    mb['b'+str(i)]= np.zeros((noOfneuronsEach[i-1],1))\n",
    "  mW['W'+str(noOfHiddenLayers+1)] = np.zeros((l, noOfneuronsEach[-1]))\n",
    "  mb['b'+str(noOfHiddenLayers+1)]= np.zeros((l,1))\n",
    "\n",
    "\n",
    "  t=0 #iterations \n",
    "  f=0 # update number \n",
    "  while(t<iterations):\n",
    "    print(\"Epoch \",t)\n",
    "    mini=0\n",
    "    while(mini<(noOfImages/batchSize)):\n",
    "      #print(\"Batch \",mini)\n",
    "      X_mini=X_train[(mini*batchSize):((mini+1)*batchSize-1)]\n",
    "      y_mini=y_train[(mini*batchSize):((mini+1)*batchSize-1)]\n",
    "      preactivation,activation,yhat=forwardPropagation(X_mini,parameters)\n",
    "      gradients=backPropagation(parameters,activation,preactivation,yhat,X_mini,y_mini)\n",
    "      f+=1\n",
    "      for i in range(1,noOfHiddenLayers+2):\n",
    "        mW['W'+str(i)]= beta1*mW['W'+str(i)]+(1.0 - beta1)* (gradients['W'+str(i)])\n",
    "        mb['b'+str(i)]= beta1*mb['b'+str(i)]+(1.0 - beta1)* (gradients['b'+str(i)])\n",
    "        \n",
    "        prevW['W'+str(i)]= beta2*prevW['W'+str(i)]+(1.0 - beta2)* (gradients['W'+str(i)]**2)\n",
    "        prevb['b'+str(i)]= beta2*prevb['b'+str(i)]+(1.0 - beta2)* (gradients['b'+str(i)]**2)\n",
    "\n",
    "        mWHat=(1.0-(beta1**f))*mW['W'+str(i)]\n",
    "        mbHat=(1.0-(beta1**f))*mb['b'+str(i)]\n",
    "\n",
    "        vWHat=(1.0-(beta2**f))*prevW['W'+str(i)]\n",
    "        vbHat=(1.0-(beta2**f))*prevb['b'+str(i)]       \n",
    "\n",
    "        parameters['W'+str(i)] -=  ((eta/np.sqrt(vWHat+epsilon))*mWHat)\n",
    "        parameters['b'+str(i)] -=  ((eta/np.sqrt(vbHat+epsilon))*mbHat)\n",
    "      mini+=1\n",
    "    _,_,yhat=forwardPropagation(X_train,parameters)\n",
    "    print(\"loss after epoch \",Loss(yhat,y_train,X_train))\n",
    "    \n",
    "    t+=1\n",
    "adam()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x23e803db348>]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD4CAYAAADy46FuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAgVElEQVR4nO3df7BcZZ3n8feHJIaAQn4QXLiBSZQYCwzDlTsYl50pBCHZwZGromQWl1QNZXZZylFTE01Wa3ARJcisuIwlWwywhhEhDIMhNQiYIbpWUeHHjUFDgEySgYEkaOIksKxmIQnf/aOfhk7f7nP7x+3b3ac/r6qu2/d7fuQ80Od+n/N9nnNaEYGZmVk1R7T7AMzMrLM5UZiZWSYnCjMzy+REYWZmmZwozMws0/h2H8BoO+6442LmzJntPgwzs66yYcOG30TE9ErLcpcoZs6cydDQULsPw8ysq0j6l2rLXHoyM7NMThRmZpbJicLMzDI5UZiZWSYnCjMzy5S7WU9mrbR6406uf2gLu17ez4mTJ7F0/hwG+/vafVhmLeVEYVaj1Rt3svzeTew/cAiAnS/vZ/m9mwCcLCzXXHoyq9H1D215M0kU7T9wiOsf2tKmIzIbG04UZjXa9fL+uuJmeeFEYVajEydPqitulhdOFGY1Wjp/DpMmjDssNmnCOJbOn9OmIzIbGx7MNqtRccDas56s1zhRmNVhsL/PicF6jhNF4vnxZmaVjThGIek2SbslPVUW/6ykLZI2S/pmSXy5pG1p2fyS+JmSNqVlN0pSik+UtCrFH5M0s2SbRZK2pteiUWlxBcX58Ttf3k/w1vz41Rt3tuqfNDPrGrUMZn8PWFAakPQh4CLg9Ig4DfirFD8VWAiclrb5rqTi6N9NwGJgdnoV93k5sC8iTgFuAK5L+5oKXAV8ADgLuErSlIZaOQLPjzczq27ERBERPwP2loWvAFZExGtpnd0pfhFwV0S8FhHPAduAsySdABwTEesjIoDbgcGSbVam9/cA56WrjfnA2ojYGxH7gLWUJazR4vnxZmbVNTo99j3AH6ZS0f+W9Acp3ge8WLLejhTrS+/L44dtExEHgVeAaRn7GkbSYklDkob27NlTd2M8P97MrLpGE8V4YAowD1gK3J2uAlRh3ciI0+A2hwcjbo6IgYgYmD694le+ZvL8eKvH6o07OXvFOmYtu5+zV6zzWJblXqOJYgdwbxQ8DrwBHJfiJ5WsNwPYleIzKsQp3UbSeOBYCqWuavsadYP9fVz78bn0TZ6EgL7Jk7j243M968mG8cQH60WNTo9dDZwL/FTSe4C3Ab8B1gA/kPQt4EQKg9aPR8QhSa9Kmgc8BlwG/HXa1xpgEbAeuBhYFxEh6SHgGyUD2BcAyxs83hF5frzVImvigz8/llcjJgpJdwLnAMdJ2kFhJtJtwG1pyuzrwKI0SL1Z0t3A08BB4MqIKJ5VV1CYQTUJeCC9AG4F/lbSNgpXEgsBImKvpK8BT6T1ro6I8kF1szHliQ/Wi0ZMFBHxp1UWfbrK+l8Hvl4hPgS8r0L8/wGfrLKv2ygkJbOOcOLkSeyskBQ88cHyzA8FNKuDJz5YL/IjPMzq4AcDWi9yojCrkyc+WK9xokj8UEAzs8qcKHhrbnxx2mNxbjzgZGFmPc+D2fihgGZmWXxFgefGW31cprRe4ysK/FBAq50f4WG9yIkCz4232rlMab3IpSc8N95q5zKl9SInisRz460WfoSH9SKXnszq4DKl9SJfUZjVwWVK60VOFGZ1cpnSeo0TRQnPjzczG86JIvFjPKwe7lRYL/FgduL58VYr33RnvcaJIvH8eKuVOxXWa5woEj/Gw2rlToX1GieKxPPjrVbuVFivGTFRSLpN0m5JT1VY9heSQtJxJbHlkrZJ2iJpfkn8TEmb0rIbJSnFJ0paleKPSZpZss0iSVvTa1HTrc0w2N/HtR+fS9/kSQjomzyJaz8+1wOUNszS+XOYcIQOi004Qu5UWG7VMuvpe8B3gNtLg5JOAs4HXiiJnQosBE4DTgT+UdJ7IuIQcBOwGHgU+BGwAHgAuBzYFxGnSFoIXAdcImkqcBUwAASwQdKaiNjXeHOzeX681Uwj/G6WIyNeUUTEz4C9FRbdAHyRwh/xoouAuyLitYh4DtgGnCXpBOCYiFgfEUEh6QyWbLMyvb8HOC9dbcwH1kbE3pQc1lJILi2zeuNOzl6xjlnL7ufsFes8i8Uquv6hLRw4FIfFDhwKD2ZbbjV0H4WkjwI7I+IXqYJU1EfhiqFoR4odSO/L48VtXgSIiIOSXgGmlcYrbFN+PIspXK1w8sknN9Ik30dhNas2aF3pYYFmeVD3YLako4AvA39ZaXGFWGTEG93m8GDEzRExEBED06dPr7TKiDzl0WpVbdBa4KtQy6VGZj29G5gF/ELS88AM4OeS/g2FXv9JJevOAHal+IwKcUq3kTQeOJZCqavavlrCUx6tVkvnz6nai3HHwvKo7kQREZsi4viImBkRMyn8QX9/RPwKWAMsTDOZZgGzgccj4iXgVUnz0vjDZcB9aZdrgOKMpouBdWkc4yHgAklTJE0BLkixlqjWSzx20oRW/ZPWpQb7+ypf2uLyk+VTLdNj7wTWA3Mk7ZB0ebV1I2IzcDfwNPAgcGWa8QRwBXALhQHu7RRmPAHcCkyTtA1YAixL+9oLfA14Ir2uTrGWqDTlEeC3rx90OcGG6XP5yXqICp33/BgYGIihoaGGtu2/+sfs+92BYfG+yZN4ZNm5zR6a5cjqjTv5wqonK15Z+PNi3UjShogYqLTMd2aXqJQkwOUEG87lJ+slThQlxqn6XVNfWb1pDI/EukHW58XlJ8sTJ4oShzLKcHc8+oJPfjtM1ufFs58sT5woSlQboARPfbThsj4vLj9ZnjhRlKg2P77IJ7+VynoIoB/9ZHniRFFisL+PS+dVfwRIVk3aek/Wo10Cj1NYfjhRlLlmcG7VZVk1aetNWeUnlyotL5woKvDNVFarrPKTH/9ieeFEUYGf5WO1Guzv46gJlU8jP/7F8sKJooKsm6ncS7RyE8u+Qrfo9YOHKsbNuo0TRRXVyk/+XmQr93KVO/p/d+ANlyotF5woqvjQeyt/r0W1uPWurM6DS5WWB04UVfzk2T11xa13eUDb8s6Jogp/kZHVygPalndOFFVUKyd4jMIqqTag7Xs0LQ+cKKpYOn8Ok8pOfuExCqus2oB2tbhZN3GiqGKwv49PnNl32P0UAfz9hp2eyWLD+Kt0Lc+cKDL85Nk9w+6n2H/gkGey2DD+Kl3LMyeKDB7QtloN9vfx9iPHD4sfOBTuWFjXc6LIUK1s4HKCVVJtPMKPp7duN2KikHSbpN2SniqJXS/pWUm/lPRDSZNLli2XtE3SFknzS+JnStqUlt0oFeaDSJooaVWKPyZpZsk2iyRtTa9Fo9XoWlWbseKZLFZJtXEKP0zSul0tVxTfAxaUxdYC74uI04F/ApYDSDoVWAiclrb5rqTi1KGbgMXA7PQq7vNyYF9EnALcAFyX9jUVuAr4AHAWcJWkKfU3sXGeyWL18MMkLa9GTBQR8TNgb1nsxxFxMP36KDAjvb8IuCsiXouI54BtwFmSTgCOiYj1ERHA7cBgyTYr0/t7gPPS1cZ8YG1E7I2IfRSSU3nCainPZLF6+GGSllejMUbxZ8AD6X0f8GLJsh0p1pfel8cP2yYln1eAaRn7GkbSYklDkob27Bm9R2x4JovVa7LHtSyHmkoUkr4MHATuKIYqrBYZ8Ua3OTwYcXNEDETEwPTpo3dDnGeyWL08rmV51HCiSIPLHwEuTeUkKPT6TypZbQawK8VnVIgfto2k8cCxFEpd1fY1pqqNR7iUYJV4XMvyqKFEIWkB8CXgoxHxu5JFa4CFaSbTLAqD1o9HxEvAq5LmpfGHy4D7SrYpzmi6GFiXEs9DwAWSpqRB7AtSbEx5iqzVw+Nalke1TI+9E1gPzJG0Q9LlwHeAdwBrJT0p6X8CRMRm4G7gaeBB4MqIKH7N1xXALRQGuLfz1rjGrcA0SduAJcCytK+9wNeAJ9Lr6hQbUy4lWD08rmV5pLeqRvkwMDAQQ0NDo7a/WcvurzqT5fkVF47av2P50X/1j9lXodTUN3kSjyw7tw1HZDYySRsiYqDSMt+ZPQLfRGX18riW5Y0TxQh8E5XVy+NaljdOFCPIuonKz/CxSjyuZXnjRFGDPpefrA7VSk+Vxi3MuoETRQ1cfrJ6eFzL8saJogZ+ho/Vwx0LyxsnihpVKz9V6z1a73LHwvLGiaJGH3pv5WdIVYtbb3PHwvLEiaJGP3m28lNpq8Wtt7ljYXniRFGjaiUDT5G1StyxsDxxoqiRZ7JYPdyxsDxxoqiRZ7JYPdyxsDxxoqiRZ7JYPdyxsDxxoqiDv+bSauWOheWJE0Ud/Awfq4c7FpYXThR18NdcWj3csbC8cKKog7/m0urhhwNaXjhR1MFfc2n18MwnywsnijoM9vfx9iPHD4sfOBSeyWLDeOaT5YUTRZ38NZdWK3/pleXFiIlC0m2Sdkt6qiQ2VdJaSVvTzykly5ZL2iZpi6T5JfEzJW1Ky26UCkN6kiZKWpXij0maWbLNovRvbJW0aNRa3QR/zaXVw196ZXlQyxXF94AFZbFlwMMRMRt4OP2OpFOBhcBpaZvvShqXtrkJWAzMTq/iPi8H9kXEKcANwHVpX1OBq4APAGcBV5UmpHbxTBarh8tPlgcjJoqI+Bmwtyx8EbAyvV8JDJbE74qI1yLiOWAbcJakE4BjImJ9RARwe9k2xX3dA5yXrjbmA2sjYm9E7APWMjxhjTnPZLF6+MY7y4NGxyjeGREvAaSfx6d4H/BiyXo7UqwvvS+PH7ZNRBwEXgGmZexrGEmLJQ1JGtqzp7VP5/RMFquXb7yzbjfag9nVrrKrxRvd5vBgxM0RMRARA9Ont/Z5/y4lWL1crrRu12ii+HUqJ5F+7k7xHcBJJevNAHal+IwK8cO2kTQeOJZCqavavtrKpQSrl8uV1u0aTRRrgOIspEXAfSXxhWkm0ywKg9aPp/LUq5LmpfGHy8q2Ke7rYmBdGsd4CLhA0pQ0iH1BirWdv+bS6uFypXW7WqbH3gmsB+ZI2iHpcmAFcL6krcD56XciYjNwN/A08CBwZUQcSru6AriFwgD3duCBFL8VmCZpG7CENIMqIvYCXwOeSK+rU6zt/DWXVg+XK63bqdB5z4+BgYEYGhpq6b9x9op1FW+Y6ps8iUeWndvSf9u608xl91eMC3huxYVjezBmFUjaEBEDlZb5zuwG+GsurV6e+WTdzImiAa45W72qzXB6/eChygvMOogTRQOyas5fXbN5rA/HukC1mU+/O/CGOxfW8ZwoGpA1Rfbl/Qd84tswWTPiPKBtnc6JokHVpsiCT3wbbun8OVWX+f4b63ROFA3yiW/1GOzv46gJlU83D2hbp3OiaJBPfKvXxAnjKsY9oG2dzomiCdVOfD/DxyrxgLZ1KyeKJvgZPlYPD2hbt3KiaILvp7B6eFzLupUTRRP8DB+rh8e1rFs5UTTBjxy3enlA27qRE0WT/Awfq4cHtK0bOVE0yc/wsXpkDWj78S/WqZwomuQeotUja0Dbj3+xTuVE0SRPebR6DPb3MeWo6mVJf2asEzlRNMlTHq1eV/3JaVWX+TtNrBM5UTTJUx6tXoP9fRxRZWzLN/VbJ3KiGAWe8mj1eqPKvOrAN2ta53GiGAUe0LZ6+TH11k2aShSSviBps6SnJN0p6UhJUyWtlbQ1/ZxSsv5ySdskbZE0vyR+pqRNadmNUmHSqaSJklal+GOSZjZzvK3iAW2rV9bYlscprNM0nCgk9QF/DgxExPuAccBCYBnwcETMBh5OvyPp1LT8NGAB8F1JxZrNTcBiYHZ6LUjxy4F9EXEKcANwXaPH20o+6a1eHqewbtJs6Wk8MEnSeOAoYBdwEbAyLV8JDKb3FwF3RcRrEfEcsA04S9IJwDERsT4iAri9bJvivu4BzitebXQSn/TWCI9TWLdoOFFExE7gr4AXgJeAVyLix8A7I+KltM5LwPFpkz7gxZJd7EixvvS+PH7YNhFxEHgFmFZ+LJIWSxqSNLRnz55Gm9QUn/RWL49TWLdopvQ0hUKPfxZwInC0pE9nbVIhFhnxrG0OD0TcHBEDETEwffr07ANvkayT3o9msEpcsrRu0Uzp6cPAcxGxJyIOAPcC/xb4dSonkX7uTuvvAE4q2X4GhVLVjvS+PH7YNqm8dSywt4ljbhk/msHq5ZKldYtmEsULwDxJR6Vxg/OAZ4A1wKK0ziLgvvR+DbAwzWSaRWHQ+vFUnnpV0ry0n8vKtinu62JgXRrH6Dh+NIM1wiVL6wbNjFE8RmGA+efAprSvm4EVwPmStgLnp9+JiM3A3cDTwIPAlRFRvCPtCuAWCgPc24EHUvxWYJqkbcAS0gyqTuVHM1i9PE5h3UAd2kFv2MDAQAwNDbXt33/X8vsr9hIFPLfiwjE/Hutsqzfu5POrnqy6/Hl/ZmyMSNoQEQOVlvnO7FHmUoLVI2ucAvyZsc7gRDHKXEqwelXrXIBnzFlncKIYZZ7yaPXK6lx4xpx1AieKUeYpj1avpfPnZH42fFVh7eZE0QIep7B6DPb3cem8k6suf3l/5acTm40VJ4oW8DiF1euawbmZy93BsHZyomgBj1NYI7Ju2HT5ydrJiaIFPOXRGpF1w6bLT9ZOThQtkjXl0eUnq2Swvy9zuTsY1i5OFC2SNU7h8pNVk1V+Wn7vL8fwSMze4kTRIlnjFODeoVWWVX7a7+9gtzZxomiRkcoIHpy0Skb63Lhsae3gRNFCvuPWGpFVfnLZ0trBiaKFRrrj1jVnqySr/ATwldWbxuhIzAqcKFpopDtuXXO2SkYqP33/0Rf8ubEx5UTRYiPdceuxCqskq2wJvhq1seVEMQayas6+kcoqGals6atRG0tOFGNgpJqzT3grN1LZEnxVYWPHiWIMDPb3cfTbxlVd7hPeKrlmcG7m58ZXFTZWmkoUkiZLukfSs5KekfRBSVMlrZW0Nf2cUrL+cknbJG2RNL8kfqakTWnZjZKU4hMlrUrxxyTNbOZ42+nrH6s+VuET3qrJ+tyAOxk2Npq9ovgfwIMR8V7g94FngGXAwxExG3g4/Y6kU4GFwGnAAuC7kordpZuAxcDs9FqQ4pcD+yLiFOAG4Lomj7dtRprJ4hPeKhnpatSdDBsLDScKSccAfwTcChARr0fEy8BFwMq02kpgML2/CLgrIl6LiOeAbcBZkk4AjomI9RERwO1l2xT3dQ9wXvFqoxtlDWr7hLdqfFVh7dbMFcW7gD3A/5K0UdItko4G3hkRLwGkn8en9fuAF0u235Fifel9efywbSLiIPAKMK2JY26rkQa1v7DqSScLG6aWq4pL/2b9GB6R9ZpmEsV44P3ATRHRD/yWVGaqotKVQGTEs7Y5fMfSYklDkob27NmTfdRtNNIJH8CSu50sbLiRrioe2b7XycJapplEsQPYERGPpd/voZA4fp3KSaSfu0vWP6lk+xnArhSfUSF+2DaSxgPHAnvLDyQibo6IgYgYmD59ehNNar2RTvg3wqUEG26kTgYUkoU7GdYKDSeKiPgV8KKk4vO0zwOeBtYAi1JsEXBfer8GWJhmMs2iMGj9eCpPvSppXhp/uKxsm+K+LgbWpXGMrlXLCe/xCqtkpE4GwNK/e7L1B2I9p9lZT58F7pD0S+AM4BvACuB8SVuB89PvRMRm4G4KyeRB4MqIOJT2cwVwC4UB7u3AAyl+KzBN0jZgCdmlra7hE94aMdjfx6dHuAnvwBu4BGWjTl3eQR9mYGAghoaG2n0YI/rK6k18/9EXMtc5+91TueMzHxyjI7JucenfrOeR7cMqsIf59iVnjDgl26yUpA0RMVBpme/MbpNrBueO2Dt8ZPteP1LahrnjMx9k4vjsU9cz6Gw0OVG00UiPaIDCI6WdLKzcdZ84PXN5AJ9f9aQ/OzYqnCjarJbxCicLKzfY38fZ75464nr+7gobDU4UbVbPCe9kYaVqKUGBJ0ZY85woOkCtJ7yThZUbqQQFhZlQ53/rp60/GMstJ4oOcd0nTueIGp5i5WRhpWqZMguwdfdvmfOVB1yGsoY4UXSIwf4+vvWpMzK/1azo+4++4Lny9qZaZtABvHbwDQ9wW0OcKDrIYH8fN1xyRk3rPrJ9r8sJ9qZakwX4qtTq50TRYWotJUChnHDKf/2RywkGFJJFLRMjwFelVh8nig5UT+/w4BvhcoK96Y7PfJDZxx9d07q+KrVaOVF0qHqSBRR6iB6sNIC1S86p+cpi6+7fMnPZ/e5oWCYnig52zeBcvn3JGUyo8f9ScbDSJQW74zMfrLuj4TKmVeNE0eEG+/vY+o0Lay4nQKGk4JPe6r0qLZYx3dGwck4UXWLtknPqShbFk97lqN5Wb7KAQkfD5Sgr5UTRReqpPRe5HGXFEma9J7vLUVbk76PoQqs37mTp3z3JgTfq33bKURO46k9O83cV9Khavsuikonjj+C6T5zuz02OZX0fhRNFF2v0pAef+L1s9cadLFn1JA30MwD49LyTuWZw5KceW3dxosixZk96J4ze1UxHA5ww8saJogfU8tWqIzn6beP4+sfmOmn0kGbKmEUCLnXS6HpOFD2k2V5ikccyekuzV6ZF7mx0r5YmCknjgCFgZ0R8RNJUYBUwE3ge+FRE7EvrLgcuBw4Bfx4RD6X4mcD3gEnAj4DPRURImgjcDpwJ/CtwSUQ8n3U8vZ4oYPRO+iKXGHrHaHU0wEmj27Q6USwBBoBjUqL4JrA3IlZIWgZMiYgvSToVuBM4CzgR+EfgPRFxSNLjwOeARykkihsj4gFJ/wU4PSL+s6SFwMci4pKs43GieMtolKPK+eTPv9Ubd7L83l+yv5l6VJkjBP/hA+5wdLKWJQpJM4CVwNeBJSlRbAHOiYiXJJ0A/DQi5qSrCSLi2rTtQ8BXKVx1/CQi3pvif5q2/0/FdSJivaTxwK+A6ZFx0E4Uw41GHboSJ438a0Vnw2ManamVieIe4FrgHcBfpETxckRMLllnX0RMkfQd4NGI+H6K3wo8QCFRrIiID6f4HwJfSvt6ClgQETvSsu3AByLiN9WOyYmiulb0FIt88udbKxJGkT87nSErUYxvYqcfAXZHxAZJ59SySYVYZMSztik/lsXAYoCTT67vcQW9ZLC/783e/2if+EHhTt7SffqKIz+uGZzLNYNzW9LZKP/sOHF0noavKCRdC/xH4CBwJHAMcC/wB7j01DVaeZVRzjOp8qWVVxnl3OlovZZPj01XFMXS0/XAv5YMZk+NiC9KOg34AW8NZj8MzE6D2U8AnwUeozCY/dcR8SNJVwJzSwazPx4Rn8o6FieKxo1l0ihy8siHsUwa4MHxVhjrRDENuBs4GXgB+GRE7E3rfRn4MwpXIZ+PiAdSfIC3psc+AHw2TY89EvhboB/YCyyMiH/OOhYnitHxldWbuOPRF4bX+caAk0f3akdno8jJozm+4c6a0s6Tv8jJozuN9ZVGOZesaudEYaOq3Sd/kXuQ3aUTOhxF/uwM50RhLdPOElUW9yQ7Xyd+dnp5xpUThY2ZTuo1VtLLfwg6XadcqVaT98+OE4W1TacnjnK+Eukc/uyMLScK6yjd9gegVLf/Meh2nX7VMZJOnpThRGEdrxuTh5NGZ+j25FGuXZ8rJwrrSt2QPMYdIf77J3/fyaLD5C15VDOaScWJwnKl0/4I9E2exCPLzm33YVgNOu2z0wqNJg8nCusJ7boCEfDcigvH9N+00ZW3BNLIla4ThfW8ViYRX1HkWzeUQCup93PZkseMm3WT0kesl2vmD8G4I8TS+XOaPTzrYK367LTarpf3j9q+nCis52X9IYDCH4OvrtnMy/sPHBb3rCcb6bMD7UsmJ06eNGr7cunJzKwDjOY4iccoRuBEYWZ5l5VUPOupBk4UZmb1y0oUR4z1wZiZWXdxojAzs0xOFGZmlsmJwszMMjlRmJlZptzNepK0B/iXJnZxHPCbUTqcTpPntkG+2+e2da9uad/vRcT0SgtylyiaJWmo2hSxbpfntkG+2+e2da88tM+lJzMzy+REYWZmmZwohru53QfQQnluG+S7fW5b9+r69nmMwszMMvmKwszMMjlRmJlZJieKRNICSVskbZO0rN3H0whJt0naLempkthUSWslbU0/p5QsW57au0XS/PYcdW0knSTpJ5KekbRZ0udSvOvbJ+lISY9L+kVq239L8a5vW5GkcZI2SvqH9Hue2va8pE2SnpQ0lGK5aR8AEdHzL2AcsB14F/A24BfAqe0+rgba8UfA+4GnSmLfBJal98uA69L7U1M7JwKzUvvHtbsNGW07AXh/ev8O4J9SG7q+fYCAt6f3E4DHgHl5aFtJG5cAPwD+IU+fy3TMzwPHlcVy076I8BVFchawLSL+OSJeB+4CLmrzMdUtIn4G7C0LXwSsTO9XAoMl8bsi4rWIeA7YRuG/Q0eKiJci4ufp/avAM0AfOWhfFPzf9OuE9Apy0DYASTOAC4FbSsK5aFuGXLXPiaKgD3ix5PcdKZYH74yIl6DwxxY4PsW7ts2SZgL9FHreuWhfKs08CewG1kZEbtoGfBv4IlD6pdF5aRsUkvqPJW2QtDjF8tQ+xrf7ADqEKsTyPm+4K9ss6e3A3wOfj4j/I1VqRmHVCrGObV9EHALOkDQZ+KGk92Ws3jVtk/QRYHdEbJB0Ti2bVIh1ZNtKnB0RuyQdD6yV9GzGut3YPl9RJDuAk0p+nwHsatOxjLZfSzoBIP3cneJd12ZJEygkiTsi4t4Uzk37ACLiZeCnwALy0bazgY9Kep5CSfdcSd8nH20DICJ2pZ+7gR9SKCXlpn3gRFH0BDBb0ixJbwMWAmvafEyjZQ2wKL1fBNxXEl8oaaKkWcBs4PE2HF9NVLh0uBV4JiK+VbKo69snaXq6kkDSJODDwLPkoG0RsTwiZkTETArn1bqI+DQ5aBuApKMlvaP4HrgAeIqctO9N7R5N75QX8McUZtJsB77c7uNpsA13Ai8BByj0XC4HpgEPA1vTz6kl6385tXcL8O/bffwjtO3fUbhE/yXwZHr9cR7aB5wObExtewr4yxTv+raVtfMc3pr1lIu2UZgp+Yv02lz825GX9hVffoSHmZllcunJzMwyOVGYmVkmJwozM8vkRGFmZpmcKMzMLJMThZmZZXKiMDOzTP8fokz/BeyhEL4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pyplot.plot(range(0,len(loss)), loss, 'o')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vpuXKyTzYJnB",
    "outputId": "b3041b66-7bbb-4a16-c480-f8e260fa97f8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is  94.25666666666666 %\n",
      "(array([9, 0, 0, ..., 3, 0, 5], dtype=int64), 56554)\n",
      "Accuracy is  87.06 %\n",
      "(array([9, 2, 1, ..., 8, 1, 5], dtype=int64), 8706)\n"
     ]
    }
   ],
   "source": [
    "def predict(X_test,y_test):\n",
    "  _,_,y_hat=forwardPropagation(X_test,parameters)\n",
    "  y_hat=y_hat.argmax(axis=0)\n",
    "  correctPred=np.sum(y_hat==y_test)\n",
    "  print(\"Accuracy is \", correctPred/X_test.shape[0]*100,\"%\")\n",
    "  return (y_hat,correctPred)\n",
    "print(predict(X_train,y_train))\n",
    "print(predict(X_test,y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Assgnment1_DL.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}