Neural+net+Iris (4).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sklearn\n",
    "import sklearn.datasets\n",
    "from dnn_app_utils_v3 import *\n",
    "from dnn_utils_v2 import sigmoid_backward, relu_backward, sigmoid, relu\n",
    "from testCases_v4 import *\n",
    "np.random.seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.rcParams['figure.figsize'] = (5.0,4.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap']='gray'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def initializeParameters(layer_dims):\n",
    "    parameters={}\n",
    "    L=len(layer_dims)-1\n",
    "    np.random.seed(3)\n",
    "    for l in range(1,L+1):\n",
    "        parameters['W' + str(l)] = np.random.randn(layer_dims[l],layer_dims[l-1])*np.sqrt(2/layer_dims[l-1])\n",
    "        parameters['b' + str(l)] = np.zeros((layer_dims[l],1))\n",
    "        \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W1 = [[ 1.46040903  0.3564088   0.07878985]\n",
      " [-1.52153542 -0.22648652 -0.28965949]]\n",
      "b1 = [[ 0.]\n",
      " [ 0.]]\n",
      "W2 = [[-0.08274148 -0.62700068]]\n",
      "b2 = [[ 0.]]\n"
     ]
    }
   ],
   "source": [
    "parameters = initializeParameters([3,2,1])\n",
    "print(\"W1 = \" + str(parameters[\"W1\"]))\n",
    "print(\"b1 = \" + str(parameters[\"b1\"]))\n",
    "print(\"W2 = \" + str(parameters[\"W2\"]))\n",
    "print(\"b2 = \" + str(parameters[\"b2\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def linear_forward(A,W,b):\n",
    "    Z = np.dot(W,A)+b\n",
    "    assert(Z.shape == (W.shape[0], A.shape[1]))\n",
    "    cache = (A,W,b)\n",
    "    return Z,cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Z = [[ 3.26295337 -1.23429987]]\n"
     ]
    }
   ],
   "source": [
    "A, W, b = linear_forward_test_case()\n",
    "\n",
    "Z, linear_cache = linear_forward(A, W, b)\n",
    "print(\"Z = \" + str(Z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def linear_activation_forward(A_prev,W,b,activation):\n",
    "    if activation==\"sigmoid\":\n",
    "        Z,linear_cache = linear_forward(A_prev,W,b)\n",
    "        A, activation_cache = sigmoid(Z)\n",
    "    elif activation==\"relu\":\n",
    "        Z,linear_cache = linear_forward(A_prev,W,b)\n",
    "        A, activation_cache = relu(Z)\n",
    "    \n",
    "    assert (A.shape == (W.shape[0], A_prev.shape[1]))\n",
    "    \n",
    "    cache=(linear_cache,activation_cache)\n",
    "    return A, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "With sigmoid: A = [[ 0.96890023  0.11013289]]\n",
      "With ReLU: A = [[ 3.43896131  0.        ]]\n"
     ]
    }
   ],
   "source": [
    "A_prev, W, b = linear_activation_forward_test_case()\n",
    "\n",
    "A, linear_activation_cache = linear_activation_forward(A_prev, W, b, activation = \"sigmoid\")\n",
    "print(\"With sigmoid: A = \" + str(A))\n",
    "\n",
    "A, linear_activation_cache = linear_activation_forward(A_prev, W, b, activation = \"relu\")\n",
    "print(\"With ReLU: A = \" + str(A))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def L_model_forward(X,parameters):\n",
    "    caches=[]\n",
    "    A=X\n",
    "    L=len(parameters)//2\n",
    "    for l in range(1,L):\n",
    "        A_prev=A\n",
    "        A, cache = linear_activation_forward(A_prev, parameters[\"W\"+str(l)], parameters[\"b\"+str(l)], activation='relu')\n",
    "        caches.append(cache)\n",
    "        \n",
    "    AL,cache = linear_activation_forward(A,parameters[\"W\"+str(L)],parameters[\"b\"+str(L)],activation='sigmoid')\n",
    "    caches.append(cache)\n",
    "    assert(AL.shape == (1,X.shape[1]))\n",
    "    return AL, caches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AL = [[ 0.03921668  0.70498921  0.19734387  0.04728177]]\n",
      "Length of caches list = 3\n"
     ]
    }
   ],
   "source": [
    "X, parameters = L_model_forward_test_case_2hidden()\n",
    "AL, caches = L_model_forward(X, parameters)\n",
    "print(\"AL = \" + str(AL))\n",
    "print(\"Length of caches list = \" + str(len(caches)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_cost(AL,Y):\n",
    "    m=Y.shape[1]\n",
    "    cost = (-1/m)*(np.sum(np.multiply(1-Y,np.log(1-AL)) + np.multiply(Y,np.log(AL))))\n",
    "    cost = np.squeeze(cost)\n",
    "    assert(cost.shape==())\n",
    "    return cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost = 0.414931599615\n"
     ]
    }
   ],
   "source": [
    "Y, AL = compute_cost_test_case()\n",
    "\n",
    "print(\"cost = \" + str(compute_cost(AL, Y)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def linear_backward(dZ, cache):\n",
    "    A_prev,W,b = cache\n",
    "    m = A_prev.shape[1]\n",
    "    dW = (1/m)*np.dot(dZ,A_prev.T)\n",
    "    db = (1/m)*np.sum(dZ,axis=1,keepdims=True)\n",
    "    dA_prev = np.dot(W.T,dZ)\n",
    "    assert (dA_prev.shape == A_prev.shape)\n",
    "    assert (dW.shape == W.shape)\n",
    "    assert (db.shape == b.shape)\n",
    "    return dA_prev, dW, db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dA_prev = [[ 0.51822968 -0.19517421]\n",
      " [-0.40506361  0.15255393]\n",
      " [ 2.37496825 -0.89445391]]\n",
      "dW = [[-0.10076895  1.40685096  1.64992505]]\n",
      "db = [[ 0.50629448]]\n"
     ]
    }
   ],
   "source": [
    "# Set up some test inputs\n",
    "dZ, linear_cache = linear_backward_test_case()\n",
    "\n",
    "dA_prev, dW, db = linear_backward(dZ, linear_cache)\n",
    "print (\"dA_prev = \"+ str(dA_prev))\n",
    "print (\"dW = \" + str(dW))\n",
    "print (\"db = \" + str(db))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def linear_activation_backwards(dA,cache,activation):\n",
    "    linear_cache,activation_cache = cache\n",
    "    if activation == \"relu\":\n",
    "        dZ = relu_backward(dA,activation_cache)\n",
    "        dA_prev,dW,db = linear_backward(dZ,linear_cache)\n",
    "    elif activation ==\"sigmoid\":\n",
    "        dZ = sigmoid_backward(dA,activation_cache)\n",
    "        dA_prev,dW,db = linear_backward(dZ,linear_cache)\n",
    "    \n",
    "    return dA_prev,dW,db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sigmoid:\n",
      "dA_prev = [[ 0.11017994  0.01105339]\n",
      " [ 0.09466817  0.00949723]\n",
      " [-0.05743092 -0.00576154]]\n",
      "dW = [[ 0.10266786  0.09778551 -0.01968084]]\n",
      "db = [[-0.05729622]]\n",
      "\n",
      "relu:\n",
      "dA_prev = [[ 0.44090989 -0.        ]\n",
      " [ 0.37883606 -0.        ]\n",
      " [-0.2298228   0.        ]]\n",
      "dW = [[ 0.44513824  0.37371418 -0.10478989]]\n",
      "db = [[-0.20837892]]\n"
     ]
    }
   ],
   "source": [
    "dAL, linear_activation_cache = linear_activation_backward_test_case()\n",
    "\n",
    "dA_prev, dW, db = linear_activation_backwards(dAL, linear_activation_cache, activation = \"sigmoid\")\n",
    "print (\"sigmoid:\")\n",
    "print (\"dA_prev = \"+ str(dA_prev))\n",
    "print (\"dW = \" + str(dW))\n",
    "print (\"db = \" + str(db) + \"\\n\")\n",
    "\n",
    "dA_prev, dW, db = linear_activation_backwards(dAL, linear_activation_cache, activation = \"relu\")\n",
    "print (\"relu:\")\n",
    "print (\"dA_prev = \"+ str(dA_prev))\n",
    "print (\"dW = \" + str(dW))\n",
    "print (\"db = \" + str(db))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def L_model_backward(AL,Y,caches):\n",
    "    grads={}\n",
    "    L=len(caches)\n",
    "    m = AL.shape[1]\n",
    "    Y = Y.reshape(AL.shape)\n",
    "    dAL = - (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL))\n",
    "    current_cache = caches[-1]\n",
    "    grads[\"dA\"+str(L-1)], grads[\"dW\"+str(L)],grads[\"db\"+str(L)] = linear_activation_backwards(dAL,current_cache,activation='sigmoid')\n",
    "    for l in reversed(range(L-1)):\n",
    "        current_cache = caches[l]\n",
    "        grads[\"dA\"+str(l)], grads[\"dW\"+str(l+1)],grads[\"db\"+str(l+1)] = linear_activation_backwards(grads[\"dA\"+str(l+1)],current_cache,activation='relu')\n",
    "        \n",
    "    return grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dW1 = [[ 0.41010002  0.07807203  0.13798444  0.10502167]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.05283652  0.01005865  0.01777766  0.0135308 ]]\n",
      "db1 = [[-0.22007063]\n",
      " [ 0.        ]\n",
      " [-0.02835349]]\n",
      "dA1 = [[ 0.12913162 -0.44014127]\n",
      " [-0.14175655  0.48317296]\n",
      " [ 0.01663708 -0.05670698]]\n"
     ]
    }
   ],
   "source": [
    "AL, Y_assess, caches = L_model_backward_test_case()\n",
    "grads = L_model_backward(AL, Y_assess, caches)\n",
    "print_grads(grads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def update_parameters(parameters,grads,learning_rate):\n",
    "    L=len(parameters)//2\n",
    "    for l in range(L):\n",
    "        parameters[\"W\"+str(l+1)]=parameters[\"W\"+str(l+1)]-learning_rate*grads[\"dW\"+str(l+1)]\n",
    "        parameters[\"b\"+str(l+1)]=parameters[\"b\"+str(l+1)]-learning_rate*grads[\"db\"+str(l+1)]\n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W1 = [[-0.59562069 -0.09991781 -2.14584584  1.82662008]\n",
      " [-1.76569676 -0.80627147  0.51115557 -1.18258802]\n",
      " [-1.0535704  -0.86128581  0.68284052  2.20374577]]\n",
      "b1 = [[-0.04659241]\n",
      " [-1.28888275]\n",
      " [ 0.53405496]]\n",
      "W2 = [[-0.55569196  0.0354055   1.32964895]]\n",
      "b2 = [[-0.84610769]]\n"
     ]
    }
   ],
   "source": [
    "parameters, grads = update_parameters_test_case()\n",
    "parameters = update_parameters(parameters, grads, 0.1)\n",
    "\n",
    "print (\"W1 = \"+ str(parameters[\"W1\"]))\n",
    "print (\"b1 = \"+ str(parameters[\"b1\"]))\n",
    "print (\"W2 = \"+ str(parameters[\"W2\"]))\n",
    "print (\"b2 = \"+ str(parameters[\"b2\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = sklearn.datasets.load_iris().data[0:36][:]\n",
    "X = np.concatenate((X,sklearn.datasets.load_iris().data[50:86][:]),axis=0)\n",
    "X = X.reshape(4,72)\n",
    "X = sklearn.preprocessing.normalize(X, norm='l2', axis=1, copy=False, return_norm=False)\n",
    "X_test = sklearn.datasets.load_iris().data[36:50][:]\n",
    "X_test = np.concatenate((X_test,sklearn.datasets.load_iris().data[86:100][:]),axis=0)\n",
    "X_test = X_test.reshape(4,28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y = sklearn.datasets.load_iris().target[0:36][:]\n",
    "Y = np.concatenate((Y,sklearn.datasets.load_iris().target[50:86][:]),axis=0)\n",
    "Y = Y.reshape(1,72)\n",
    "Y_test = sklearn.datasets.load_iris().target[36:50][:]\n",
    "Y_test = np.concatenate((Y_test,sklearn.datasets.load_iris().target[86:100][:]),axis=0)\n",
    "Y_test = Y_test.reshape(1,28)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "X = Examples_shuffled[0:4][:]\n",
    "Y = (Examples_shuffled[4][:]).reshape(108,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "layer_dims = [4,20,5,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def L_layer_model(X,Y,layer_dims,learning_rate=0.0075,num_iteration=3000,print_cost=True):\n",
    "    costs=[]\n",
    "    parameters = initializeParameters(layer_dims)\n",
    "    for i in range(0,num_iteration):\n",
    "        AL,caches = L_model_forward(X,parameters)\n",
    "        cost = compute_cost(AL,Y)\n",
    "        grads = L_model_backward(AL,Y,caches)\n",
    "        parameters = update_parameters(parameters,grads,learning_rate)\n",
    "        if print_cost and i % 100 == 0:\n",
    "            print (\"Cost after iteration %i: %f\" %(i, cost))\n",
    "        if print_cost and i % 100 == 0:\n",
    "            costs.append(cost)\n",
    "    plt.plot(np.squeeze(costs))\n",
    "    plt.ylabel('cost')\n",
    "    plt.xlabel('iterations (per tens)')\n",
    "    plt.title(\"Learning rate =\" + str(learning_rate))\n",
    "    plt.show()\n",
    "    \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost after iteration 0: 0.692622\n",
      "Cost after iteration 100: 0.692425\n",
      "Cost after iteration 200: 0.692386\n",
      "Cost after iteration 300: 0.692362\n",
      "Cost after iteration 400: 0.692346\n",
      "Cost after iteration 500: 0.692334\n",
      "Cost after iteration 600: 0.692324\n",
      "Cost after iteration 700: 0.692317\n",
      "Cost after iteration 800: 0.692310\n",
      "Cost after iteration 900: 0.692305\n",
      "Cost after iteration 1000: 0.692297\n",
      "Cost after iteration 1100: 0.692292\n",
      "Cost after iteration 1200: 0.692286\n",
      "Cost after iteration 1300: 0.692281\n",
      "Cost after iteration 1400: 0.692276\n",
      "Cost after iteration 1500: 0.692271\n",
      "Cost after iteration 1600: 0.692266\n",
      "Cost after iteration 1700: 0.692261\n",
      "Cost after iteration 1800: 0.692256\n",
      "Cost after iteration 1900: 0.692250\n",
      "Cost after iteration 2000: 0.692245\n",
      "Cost after iteration 2100: 0.692239\n",
      "Cost after iteration 2200: 0.692233\n",
      "Cost after iteration 2300: 0.692228\n",
      "Cost after iteration 2400: 0.692222\n",
      "Cost after iteration 2500: 0.692217\n",
      "Cost after iteration 2600: 0.692211\n",
      "Cost after iteration 2700: 0.692206\n",
      "Cost after iteration 2800: 0.692201\n",
      "Cost after iteration 2900: 0.692195\n",
      "Cost after iteration 3000: 0.692190\n",
      "Cost after iteration 3100: 0.692185\n",
      "Cost after iteration 3200: 0.692179\n",
      "Cost after iteration 3300: 0.692174\n",
      "Cost after iteration 3400: 0.692168\n",
      "Cost after iteration 3500: 0.692163\n",
      "Cost after iteration 3600: 0.692157\n",
      "Cost after iteration 3700: 0.692151\n",
      "Cost after iteration 3800: 0.692146\n",
      "Cost after iteration 3900: 0.692140\n",
      "Cost after iteration 4000: 0.692134\n",
      "Cost after iteration 4100: 0.692129\n",
      "Cost after iteration 4200: 0.692123\n",
      "Cost after iteration 4300: 0.692115\n",
      "Cost after iteration 4400: 0.692107\n",
      "Cost after iteration 4500: 0.692099\n",
      "Cost after iteration 4600: 0.692090\n",
      "Cost after iteration 4700: 0.692082\n",
      "Cost after iteration 4800: 0.692074\n",
      "Cost after iteration 4900: 0.692062\n",
      "Cost after iteration 5000: 0.692052\n",
      "Cost after iteration 5100: 0.692042\n",
      "Cost after iteration 5200: 0.692033\n",
      "Cost after iteration 5300: 0.692026\n",
      "Cost after iteration 5400: 0.692019\n",
      "Cost after iteration 5500: 0.692012\n",
      "Cost after iteration 5600: 0.692006\n",
      "Cost after iteration 5700: 0.691999\n",
      "Cost after iteration 5800: 0.691992\n",
      "Cost after iteration 5900: 0.691984\n",
      "Cost after iteration 6000: 0.691976\n",
      "Cost after iteration 6100: 0.691960\n",
      "Cost after iteration 6200: 0.691948\n",
      "Cost after iteration 6300: 0.691936\n",
      "Cost after iteration 6400: 0.691925\n",
      "Cost after iteration 6500: 0.691914\n",
      "Cost after iteration 6600: 0.691901\n",
      "Cost after iteration 6700: 0.691659\n",
      "Cost after iteration 6800: 0.691641\n",
      "Cost after iteration 6900: 0.691627\n",
      "Cost after iteration 7000: 0.691616\n",
      "Cost after iteration 7100: 0.691606\n",
      "Cost after iteration 7200: 0.691596\n",
      "Cost after iteration 7300: 0.691586\n",
      "Cost after iteration 7400: 0.691576\n",
      "Cost after iteration 7500: 0.691566\n",
      "Cost after iteration 7600: 0.691552\n",
      "Cost after iteration 7700: 0.691539\n",
      "Cost after iteration 7800: 0.691526\n",
      "Cost after iteration 7900: 0.691512\n",
      "Cost after iteration 8000: 0.691499\n",
      "Cost after iteration 8100: 0.691485\n",
      "Cost after iteration 8200: 0.691471\n",
      "Cost after iteration 8300: 0.691457\n",
      "Cost after iteration 8400: 0.691443\n",
      "Cost after iteration 8500: 0.691429\n",
      "Cost after iteration 8600: 0.691415\n",
      "Cost after iteration 8700: 0.691401\n",
      "Cost after iteration 8800: 0.691387\n",
      "Cost after iteration 8900: 0.691372\n",
      "Cost after iteration 9000: 0.691358\n",
      "Cost after iteration 9100: 0.691344\n",
      "Cost after iteration 9200: 0.691328\n",
      "Cost after iteration 9300: 0.691314\n",
      "Cost after iteration 9400: 0.691299\n",
      "Cost after iteration 9500: 0.691284\n",
      "Cost after iteration 9600: 0.691268\n",
      "Cost after iteration 9700: 0.691253\n",
      "Cost after iteration 9800: 0.691238\n",
      "Cost after iteration 9900: 0.691221\n",
      "Cost after iteration 10000: 0.691206\n",
      "Cost after iteration 10100: 0.691190\n",
      "Cost after iteration 10200: 0.691175\n",
      "Cost after iteration 10300: 0.691159\n",
      "Cost after iteration 10400: 0.691141\n",
      "Cost after iteration 10500: 0.691125\n",
      "Cost after iteration 10600: 0.691109\n",
      "Cost after iteration 10700: 0.691093\n",
      "Cost after iteration 10800: 0.691076\n",
      "Cost after iteration 10900: 0.691060\n",
      "Cost after iteration 11000: 0.691043\n",
      "Cost after iteration 11100: 0.691026\n",
      "Cost after iteration 11200: 0.691010\n",
      "Cost after iteration 11300: 0.690993\n",
      "Cost after iteration 11400: 0.690976\n",
      "Cost after iteration 11500: 0.690958\n",
      "Cost after iteration 11600: 0.690941\n",
      "Cost after iteration 11700: 0.690924\n",
      "Cost after iteration 11800: 0.690906\n",
      "Cost after iteration 11900: 0.690887\n",
      "Cost after iteration 12000: 0.690870\n",
      "Cost after iteration 12100: 0.690852\n",
      "Cost after iteration 12200: 0.690835\n",
      "Cost after iteration 12300: 0.690816\n",
      "Cost after iteration 12400: 0.690798\n",
      "Cost after iteration 12500: 0.690780\n",
      "Cost after iteration 12600: 0.690761\n",
      "Cost after iteration 12700: 0.690743\n",
      "Cost after iteration 12800: 0.690724\n",
      "Cost after iteration 12900: 0.690705\n",
      "Cost after iteration 13000: 0.690686\n",
      "Cost after iteration 13100: 0.690668\n",
      "Cost after iteration 13200: 0.690649\n",
      "Cost after iteration 13300: 0.690630\n",
      "Cost after iteration 13400: 0.690610\n",
      "Cost after iteration 13500: 0.690591\n",
      "Cost after iteration 13600: 0.690571\n",
      "Cost after iteration 13700: 0.690551\n",
      "Cost after iteration 13800: 0.690531\n",
      "Cost after iteration 13900: 0.690511\n",
      "Cost after iteration 14000: 0.690476\n",
      "Cost after iteration 14100: 0.690455\n",
      "Cost after iteration 14200: 0.690435\n",
      "Cost after iteration 14300: 0.690414\n",
      "Cost after iteration 14400: 0.690393\n",
      "Cost after iteration 14500: 0.690373\n",
      "Cost after iteration 14600: 0.690346\n",
      "Cost after iteration 14700: 0.690314\n",
      "Cost after iteration 14800: 0.690284\n",
      "Cost after iteration 14900: 0.690253\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWEAAAEWCAYAAAC32CauAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8VfX9x/HXOwlJ2CNMGbICMgSRCCJDrQsVtSpu62pF\nFKrWLkd3a39VO6x1oHVXcdSJE1cBkSFBAdkEZO8Asvfn98c5sdeUkBvMvSfJ/Twfj/NI7vee8TkQ\n3vnyPed8r8wM55xz0UiLugDnnEtlHsLOORchD2HnnIuQh7BzzkXIQ9g55yLkIeyccxHyEHaVjqR3\nJF0ZdR3OlQcPYRc3SYslnRx1HWZ2upk9FXUdAJLGSPpBEo6TJelxSZslrZZ0SynrXyppiaRtkl6T\n1CDefUk6S9JMSVslTZDUOVHn5TyEXQUjKSPqGopUpFqA3wC5wOHAicDPJA080IqSugAPA98DmgDb\ngQfj2ZekXOBZYChQD3gDGFXB/iyqFjPzxZe4FmAxcHIJ7w0CpgGbgAlAt5j3bgUWAluA2cC5Me9d\nBXwC/A0oBP4Qto0H/gxsBL4ETo/ZZgzwg5jtD7ZuG2BceOwPgAeAZ0o4hxOA5cDPgdXAv4D6wJvA\nunD/bwItwvXvBPYBO4GtwP1h+xHA+8AGYB5wYTn82a8ETo15/Tvg+RLW/SMwMuZ1O2A3ULu0fQHD\ngbdj3ksDdgAnRf3zV1UX7wm7b01SD+Bx4Dogh6AXNkpSVrjKQqA/UBf4LfCMpGYxu+gNLCLotd0Z\n0zYPaAjcDTwmSSWUcLB1RwKfhnX9hqB3eDBNgQYEvcQhBCH0RPi6FUEg3Q9gZncAHwPDzayWmQ2X\nVJMggEcCjYGLgQdL+i+9pAclbSphmRGuUx9oBkyP2XQ60KWEc+gSu66ZLQR2AR0OYV8Kl64lvO++\nJQ9hVx6GAA+b2WQz22fBeO0u4FgAM/u3ma00s/1m9gKwAOgVs/1KM/uHme01sx1h2xIz+6eZ7QOe\nIgiOJiUc/4DrSmoFHAP8ysx2m9l4YFQp57If+LWZ7TKzHWZWaGYvm9l2M9tC8Evi+INsPwhYbGZP\nhOfzOfAycMGBVjazG8ysXglLt3C1WuHXr2I23QzULqGGWsXWjV2/tH19ABwv6QRJmcDtQCZQ4yDn\n7L4FD2FXHg4HfhzbiwNaAocBSLpC0rSY97oS9FqLLDvAPlcXfWNm28Nvax1gvYOtexiwIaatpGPF\nWmdmO4teSKoh6eHwItdmgqGNepLSS9j+cKB3sT+Lywh62Idqa/i1TkxbXYIhlpLWr1OsrWj9g+7L\nzOYCVxL09lcR/D3NJhimcQngIezKwzLgzmK9uBpm9pykw4F/Eow15phZPWAmwX9xiyRqKr9VQANJ\nsb24lqVsU7yWHwMdgd5mVgcYELarhPWXAWOL/VnUMrPrD3QwSSPCuxAOtMwCMLON4bl0j9m0OzCr\nhHOYFbuupHYEvdn58ezLzF4ys65mlgP8GmgNTCnhWO5b8hB2ZVVNUnbMkkEQskMl9VagpqQzJdUG\nahIE1ToASVeTpPFFM1sC5AO/kZQpqQ9wVhl3U5tgHHhTeJvXr4u9vwZoG/P6TYKx1+9JqhYux0jq\nVEKNQ8OQPtASO077NPALSfXDfV0LPFlCzc8CZ0nqH45R/x54JRxOKXVfknpKSpfUCHgEGBX2kF0C\neAi7snqbIJSKlt+YWT7BP+T7Ce4gKCC4awEzmw38BZhIEFhHEtwNkSyXAX34750XLxCMV8frXqA6\nsB6YBLxb7P2/A4MlbZR0Xxh0pxJckFtJMFRyF5DFt/NrggucSwjuDrnbzL6uJew59wcws1kEt5g9\nC6wl+EV4Q7z7Cs9pE8HFzo0Ef7cuQWTmk7q71CHpBWCumRXv0ToXCe8JuyotHApoJyktfCDhHOC1\nqOtyrog/BeOquqbAKwT3CS8Hrg9vG3OuQvDhCOeci5APRzjnXIRSejiiYcOG1rp166jLcM5VMVOn\nTl1vZo3iWTelQ7h169bk5+dHXYZzroqRtCTedX04wjnnIuQh7JxzEfIQds65CHkIO+dchDyEnXMu\nQh7CzjkXIQ9h55yLkIdwGfw7fxkvTFkadRnOuSrEQ7gMRk1fychPS/t0HOeci5+HcBk0qZPNmq92\nlr6ic87FyUO4DJrWyWbd1l3s2+8zzznnyoeHcBk0qZPFvv1G4dayfDqOc86VzEO4DJrUyQZgzWYP\nYedc+fAQLoOiEF692ceFnXPlw0O4DJrWLeoJewg758pHQkNY0kBJ8yQVSLq1hHVOkDRN0ixJY2Pa\nb5I0M2y/Oab9HklzJc2Q9KqkemF7a0k7wn1NkzSivM8np2YmaYK1HsLOuXKSsBCWlA48AJwOdAYu\nkdS52Dr1gAeBs82sC3BB2N4VuBboBXQHBklqH272PtDVzLoB84HbYna50MyOCpeh5X1OGelpNKyV\n5cMRzrlyk8iecC+gwMwWmdlu4HmCjxuPdSnwipktBTCztWF7J2CymW03s73AWOC8cJ33wjaASUCL\nBJ7D/2haN9svzDnnyk0iQ7g5EPt42fKwLVYHoL6kMZKmSroibJ8J9JeUI6kGcAbQ8gDHuAZ4J+Z1\nm3AoYqyk/gcqStIQSfmS8tetW1fmk2pcO9vHhJ1z5Sbqz5jLAHoCJwHVgYmSJpnZHEl3Ae8B24Bp\nwL7YDSXdAewFng2bVgGtzKxQUk/gNUldzGxz7HZm9gjwCEBeXl6Zn7poWjeLqUs2lHUz55w7oET2\nhFfwzd5ri7At1nJgtJltM7P1wDiCMWDM7DEz62lmA4CNBOO/AEi6ChgEXGZmFq6/y8wKw++nAgsJ\netrlqkntbDZu38POPftKX9k550qRyBCeAuRKaiMpE7gYGFVsndeBfpIywmGH3sAcAEmNw6+tCMaD\nR4avBwI/I7iYt71oR5IahRcDkdQWyAUWlfdJNQlvU1u3xceFnXPfXsKGI8xsr6ThwGggHXjczGZJ\nGhq+PyIcdngXmAHsBx41s5nhLl6WlAPsAYaZ2aaw/X4gC3hfEsCk8E6IAcDvJO0J9zXUzMp93CD2\ngY2WDWqU9+6dcykmoWPCZvY28HaxthHFXt8D3HOAbQ94Yc3M2pfQ/jLw8iEXG6emdfyBDedc+fEn\n5sqoKIQXrt0WcSXOuarAQ7iM6taoRt/2OTw9cTFbd+0tdX3nnDsYD+FD8JNTO1K4bTdPjP8y6lKc\nc5Wch/Ah6NGqPqd0bsIj4xax2j9pwzn3LXgIH6KfDzyC/WZc/eQUH5Zwzh0yD+FD1L5xLR68vCfz\n12xhyNP5fLVjT9QlOecqIQ/hb+H4Do24Z3A3Pv1yA+c+8AlzV28ufSPnnIvhIfwtnXd0C0Zeeyyb\nd+7hzPvG89s3ZvnTdM65uCmceiEl5eXlWX5+frnsa8O23fzlvXmM/HQp1dLTOPeo5px91GH0btOA\njHT/XedcKpE01czy4lrXQ7h8QrjIonVb+efHX/L6tBVs372PnJqZnNa1Kad0asKxbXOonplersdz\nzlU8HsJxSkQIF9mxex9j5q3lrS9W8eGctezYs4/sammc0rkpZx7ZjD7tcqhbvVpCju2ci1ZZQjjq\n+YSrrOqZ6Zx+ZDNOP7IZO/fsY8riDYyetZo3Z6zijekrSRMc2aIe/drn0Ld9Q45uVZ/sat5Ldi7V\neE84QT3hkuzeu5/Pl27kk4WFfFKwnmnLNrFvv5GVkUZe6/oc164hx7XL4cjmdX0s2blKyocj4hRF\nCBe3ZeceJi/awISFhUxYuJ65q7cAUCsrg95tGtCnXdBT7tikNmlpirRW51x8fDiiEqmdXY2TOzfh\n5M5NAFi/dReTFhUyYWEhExcW8uHc4LNPG9TMpE/bnK9DuXVODcL5lJ1zlZj3hCPuCZdm5aYdX/eS\nJxQUsjqcx7hZ3Wz6tMuhX/uG9GvfkMbhFJvOuej5cEScKkMIxzIzFhduDwI57Clv2LYbgI5NatM/\ntyH9chvSu43fCudclDyE41TZQri4/fuN2as2M75gPR8vWMeUxRvZvXc/menBRb5+uQ3p374RXQ6r\n4+PJziWRh3CcKnsIF7djd3Ar3PiC9Yybv+7ri3z1a1Sjb/uGYU+5Ec3rVY+4UueqNg/hOFW1EC5u\n3ZZdfFKwnnEL1jF+wXrWhnNatG1Uk/7tg0A+tm0Damf7QyPOlScP4ThV9RCOZWYsWLuVcfPXMb5g\nPZMXbWDHnn1kpIkererRr30j+ndoSDe/P9m5b81DOE6pFMLF7dq7j6lLNjJ+wXrGF6znixVfYQa1\nszM4rl0O/XIbMSC3IYfn1Iy6VOcqHQ/hOKVyCBe3cdtuPlm4nvEL1vPxgvWs2LQDgJYNqtOvfRDI\nx7VrSN0aPnThXGk8hOPkIXxgRbfCfbxgHR8vWM/EhYVs3bX36/kuBuQG9yb3aFWfzAwfunCuOA/h\nOHkIx2fPvv1MX7aJj8Ohi6L5LmpkptOnbU5wK1xuQ9o1quVP8TmHh3DcPIQPzeade5i4sDAculjH\n4sLtABxWN5tzj27Oxce0omWDGhFX6Vx0PITj5CFcPpZt2M74gvW8P3sNY+atxYABuY24pFcrTu7U\n2O+2cCnHQzhOHsLlb+WmHbwwZRkvTFnG6s07aVw7i4uOaclFx7SkRX3vHbvU4CEcJw/hxNm7bz//\nmbeOkZOXMGb+OgBO6BD0jr9zhPeOXdXmIRwnD+HkWL5x+9e947VbdtG0TjbXDmjL5ce2IivDJxpy\nVY+HcJw8hJNr7779fDh3LU9+spiJiwpp2aA6Pzm1I2d1O8wnGHJViodwnDyEozNu/jr+7525zFm1\nmc7N6nDjSe05tXNTD2NXJfgna7gKb0CHRvRr35DXp6/g7x8sYOgzn9GyQXUGH92S845u7re4uZSR\n0KsjkgZKmiepQNKtJaxzgqRpkmZJGhvTfpOkmWH7zTHt90iaK2mGpFcl1Yt577bwWPMknZbIc3Pf\nXlqaOLdHCz645Xjuu6QHrRrU4G8fzKf/3f/hkkcm8dynS9kYTlrvXFWVsOEISenAfOAUYDkwBbjE\nzGbHrFMPmAAMNLOlkhqb2VpJXYHngV7AbuBdYKiZFUg6FfjIzPZKugvAzH4uqTPwXLjNYcAHQAcz\n21dSjT4cUfEs37idl6eu4PVpK1i0fhsZaaJfbkMGdTuMU7s0oY5Pu+kqgYoyHNELKDCzRWFRzwPn\nALNj1rkUeMXMlgKY2dqwvRMw2cy2h9uOBc4D7jaz92K2nwQMDr8/B3jezHYBX0oqCGuYmIiTc4nR\non4Nbjo5lxtPas+slZt5c8Yq3pi+kp/8ezqZr6RxfMdGDOrWjJM7NaFmlo+mucovkT/FzYFlMa+X\nA72LrdMBqCZpDFAb+LuZPQ3MBO6UlAPsAM4ADtRlvQZ4IeZ4k4odr3nxDSQNAYYAtGrVqmxn5JJG\nEl2b16Vr87r8fGBHpi3bxBvTV/HWFyt5f/YasqulcdIRTTirezO+c0QTn0jIVVpRdyUygJ7ASUB1\nYKKkSWY2JxxqeA/YBkwDvjGsIOkOYC/wbFkOaGaPAI9AMBzxrc/AJZwkerSqT49W9fnFmZ2YsngD\nb85YxdtfrOKtL1aRUzOT7/U5nOEntveHQFylk8gQXgG0jHndImyLtRwoNLNtwDZJ44DuwHwzewx4\nDEDSH8N1CV9fBQwCTrL/DmrHczxXyaWlid5tc+jdNodfn9WZjwvWM3LyUu79YAH5izdy3yU9aFAz\nM+oynYtbIrsNU4BcSW0kZQIXA6OKrfM60E9ShqQaBMMVcwAkNQ6/tiIYDx4Zvh4I/Aw4u2jMODQK\nuFhSlqQ2QC7wacLOzkUuIz2NEzs25p9X5HH34G58+uUGTvrLGP41cTF79+2Pujzn4pKwnnB498Jw\nYDSQDjxuZrMkDQ3fHxEOO7wLzAD2A4+a2cxwFy+HY8J7gGFmtilsvx/IAt4P566dZGZDw32/SHDh\nb2+4TYl3Rriq5cK8lnRrUZffjprNL1+fxTOTlvLLQZ3pl9sw6tKcOyh/Ys5vUatSzIzRs1Zz59tz\nWLZhB6d0bsIdZ3SidUP/rDyXPGW5Rc2vYrgqRRIDuzbj/R8dz09P68gnBes59W/j+L935rBl556o\ny3Puf3gIuyopu1o6w05sz5ifnMDZRx3Gw2MXceKfx/Lq58tL39i5JPIQdlVa4zrZ/PmC7rw+rC/N\n62Vzy4vTWbtlZ9RlOfc1D2GXErq3rMewE9tjBmu+2hV1Oc59zUPYpYycWlkAFG7zEHYVh4ewSxk5\n4UMchVt9ZjZXcXgIu5SRUysMYe8JuwrEQ9iljFpZGWRmpFHocxS7CsRD2KUMSeTUzPThCFeheAi7\nlJJTK5PCrT4c4SoOD2GXUnJqZrHBhyNcBeIh7FJKTs1M1vtwhKtAPIRdSsmplek9YVeheAi7lNKg\nZhY79uxj++69UZfiHOAh7FLM1/cK+5CEqyA8hF1Kafj1Axsewq5i8BB2KaVBzXD+CL9NzVUQHsIu\npfj8Ea6i8RB2KSXHhyNcBeMh7FJKjcwMqldL9+EIV2F4CLuU4/cKu4rEQ9ilnJyamaz3EHYVhIew\nSzlN62Yzfdkmvlj+VdSlOOch7FLPT0/rSK2sDC58eCKvT1sRdTkuxXkIu5TTvnFtXh12HJ2a1eam\n56cxbORn/gnMLjIewi4lNa6dzYvX9eGnp3XkvVmr+c6fx/LQmIVs2+VzSrjk8hB2KSsjPY1hJ7bn\nvR8dT+82Dbjr3bn0u+sjHvhPAVt27om6PJciZGZR1xCZvLw8y8/Pj7oMV0FMXbKRf3y0gDHz1lEn\nO4Or+7bhmr5tqFujWtSluUpG0lQzy4trXQ9hD2H3TTOWb+L+jwp4b/YaamVlcEWfw/lB/7Y0CB95\ndq40HsJx8hB2BzNn1Wbu/08Bb3+xiuyMdC4/thXDTmxPvRoexu7gPITj5CHs4lGwdgv3f1TAqOkr\naVAziz98tysDuzaNuixXgZUlhP3CnHOlaN+4Nvde3IM3f9ifxrWzGPrMVIaP/Mznn3DlwkPYuTh1\nPqwOrw/vy49P6cDoWas59W/jeGvGqqjLcpVcXCEs6YJ42g6wzkBJ8yQVSLq1hHVOkDRN0ixJY2Pa\nb5I0M2y/Ofa4Ydt+SXkx7a0l7Qj3NU3SiHjOzbmyqJaexg9PyuWNH/bjsHrVGTbyM254dirrvVfs\nDlG8PeHb4mz7mqR04AHgdKAzcImkzsXWqQc8CJxtZl2AC8L2rsC1QC+gOzBIUvtws5nAecC4Axx2\noZkdFS5D4zw358rsiKZ1ePWG4/jZwI58MHstp/x1LK9PW0EqX2NxhybjYG9KOh04A2gu6b6Yt+oA\npT1a1AsoMLNF4b6eB84BZsescynwipktBTCztWF7J2CymW0Ptx1LELx3m9mcsK30s3MugTLS07jh\nhPac0qkJP31pBjc9P423ZqziD+d2pXHt7KjLc5VEaT3hlUA+sBOYGrOMAk4rZdvmwLKY18vDtlgd\ngPqSxkiaKumKsH0m0F9SjqQaBL8IWpZ2MkCbcChirKT+B1pB0hBJ+ZLy161bF8cunTu43Ca1efn6\n47j9jCMYM38dp/x1HK98ttx7xS4uB+0Jm9l0YLqkkWa2B0BSfaClmW0sp+P3BE4CqgMTJU0yszmS\n7gLeA7YB04B9pexrFdDKzAol9QRek9TFzDYXO6dHgEcguEWtHM7BOdLTxJAB7TipUxN+9tIMbnlx\nOm/NWMX/nXckjet4r9iVLN4x4fcl1ZHUAPgM+Kekv5WyzQq+2XttEbbFWg6MNrNtZraeYJy3O4CZ\nPWZmPc1sALARmH+wg5nZLjMrDL+fCiwk6Gk7lzTtGtXixev68MtBnflk4XpOvXccb85YGXVZrgKL\nN4Trhj3K84Cnzaw3Qe/1YKYAuZLaSMoELiYYxoj1OtBPUkY47NAbKBrzbRx+bRUed+TBDiapUXgx\nEEltgVxgUZzn51y5SU8T3+/Xhrdu7M/hOTUZPvJzbnr+c77a7pMCuf8VbwhnSGoGXAi8Gc8GZrYX\nGA6MJgjWF81slqShkoaG68wB3gVmAJ8Cj5rZzHAXL0uaDbwBDDOzTQCSzpW0HOgDvCVpdLj+AGCG\npGnAS8BQM9sQ5/k5V+7aNarFy0P7cMspHXhrxipOu3ccHy/w6xDum+J6bDm8J/iXwCdmdn3Y07zH\nzM5PdIGJ5I8tu2SZsXwTP3phGgvXbePKPodz6+mdqJ6ZHnVZLkF87og4eQi7ZNq5Zx93vzuPxz/5\nkrYNa/LXi47iqJb1oi7LJUC5zx0hqYWkVyWtDZeXJbX4dmU6l1qyq6Xzq7M6M/IHvdm5Zx/nPzSB\nv74/nz379kddmotQvGPCTxBcVDssXN4I25xzZXRc+4a8c/MAzjnqMO77cAHnPTiBgrVboi7LRSTe\nEG5kZk+Y2d5weRJolMC6nKvS6lavxl8vPIoRlx/Nik07GPSP8YycvNQf8EhB8YZwoaTLJaWHy+VA\nYSILcy4VDOzajHdv6s8xrRtw+6tfcP0zn7Fp++6oy3JJFG8IX0Nwe9pqgifTBgNXJagm51JK4zrZ\nPHV1L24/4wg+nLuGgfd+zMSF3sdJFfGG8O+AK82skZk1Jgjl3yauLOdSS1r42PMr1/elemY6lz46\niT+PnucX7VJAvCHcLXauiPAhiB6JKcm51HVki7q8+cN+XNCzBff/p4ALH57I0sLtUZflEijeEE4L\nJ+4BIJxD4qCT/zjnDk3NrAzuHtyd+y/tQcHarZxx38e89nnxaVdcVRFvCP+FYIaz30v6PTABuDtx\nZTnnBnU7jHdu6s8RTWtz8wvTuOWFaWzZ6fNPVDVxhbCZPU0wic6acDnPzP6VyMKcc9Cifg2eH3Is\nN5+cy2vTVnDmfeOZtmxT1GW5chT3B32a2Wwzuz9cZpe+hXOuPGSkp3HzyR148bo+7NtvDH5oAiPG\nLvR7iqsI/7Rl5yqJvNYNePum/pzapQl/emcu1z49la92+PBEZech7FwlUrd6NR649Gh+fVZnxsxb\ny6B/fMzMFV9FXZb7FjyEnatkJHF13za8cF0f9u4zzntogj/yXIl5CDtXSfU8vD5v3difY9vmcPur\nX3DLi9PZvru0D0F3FY2HsHOVWIOamTx51THcckoHXpu2gu8+8AkFa7dGXZYrAw9h5yq5tDRx40m5\n/Oua3hRu3c3Z949n1HT/cNHKwkPYuSqiX25D3rqxP52b1eHG5z7nV6/PZNfefVGX5UrhIexcFdK0\nbjbPDTmWa/u34emJS7hwxESWb/S5JyoyD2Hnqphq6WnccWZnRlx+NIvWbePM+8Yzdr5/ynNF5SHs\nXBU1sGsz3vhhP5rVzebqJz7loTH+lF1F5CHsXBXWumFNXrnhOM44shl3vTuX4c997rexVTAews5V\ncTUyM/jHJT249fQjePuLVZz34ASfo7gC8RB2LgVIYujx7Xjy6l6s3LSDsx8Yz/gF66Muy+Eh7FxK\nOb5DI0YN70fj2llc8fhkHhnn48RR8xB2LsW0bliTV2/oy6mdm/LHt+dy0/PT2LHb7yeOioewcymo\nZlYGD11+ND85tQNvzFjJ+Q9NYMWmHVGXlZI8hJ1LUZIY/p1cHrsyj2UbtnP2P8bz6Zcboi4r5XgI\nO5fivnNEE14d1pe61atx6T8n8ezkJVGXlFI8hJ1ztG9ci1eH9aVv+4bc8epMfvHaF+zZtz/qslKC\nh7BzDgg+tePxq47hugFteWbSUi5/dDKFW3dFXVaVl9AQljRQ0jxJBZJuLWGdEyRNkzRL0tiY9psk\nzQzbb45pvyBs2y8pr9i+bguPNU/SaYk7M+eqpvQ0cdsZnbj3oqOYtmwTZ9//CbNXbo66rCotYSEs\nKR14ADgd6AxcIqlzsXXqAQ8CZ5tZF+CCsL0rcC3QC+gODJLUPtxsJnAeMK7YvjoDFwNdgIHAg2EN\nzrky+m6P5vx7aPDpzuc/NIF3vlgVdUlVViJ7wr2AAjNbZGa7geeBc4qtcynwipktBTCztWF7J2Cy\nmW03s73AWILgxczmmNm8AxzvHOB5M9tlZl8CBWENzrlD0K1FPUYN70unZrW5/tnP+Ot789i/3x/s\nKG+JDOHmwLKY18vDtlgdgPqSxkiaKumKsH0m0F9SjqQawBlAy3I4nnOuDBrXCeYnvjCvBfd9VMDQ\nZ6aybZdPAFSeor4wlwH0BM4ETgN+KamDmc0B7gLeA94FpgHl8kiPpCGS8iXlr1vnc6w6V5qsjHTu\nOr8bvzmrMx/MWcOFD09k9Vc7oy6rykhkCK/gm73XFmFbrOXAaDPbZmbrCcZ5uwOY2WNm1tPMBgAb\ngfnlcDzM7BEzyzOzvEaNGpXphJxLVZK4qm8bHrvqGBav38Z3H/iEWSu/irqsKiGRITwFyJXURlIm\nwUWzUcXWeR3oJykjHHboDcwBkNQ4/NqKYDx4ZCnHGwVcLClLUhsgF/i03M7GOceJHRvz0vXHkSa4\nYMREPpyzJuqSKr2EhXB4QW04MJogWF80s1mShkoaGq4zh2C4YQZBYD5qZjPDXbwsaTbwBjDMzDYB\nSDpX0nKgD/CWpNHhvmYBLwKzw30OMzOflcS5ctapWR1eG9aX9o1rce3T+TzxyZdRl1SpKZWnscvL\ny7P8/Pyoy3CuUtq+ey8/emEao2et4co+h/PLQZ3JSI/6MlPFIGmqmeWVvmb0F+acc5VUjcwMHrqs\nJ0MGtOWpiUu49ul8tvqdE2XmIeycO2RpaeL2Mzpx57ldGbdgPYMfmsBKnxKzTDyEnXPf2mW9D+eJ\nq45hxcYdnPugP+pcFh7CzrlyMaBDo/DOCXHhwxMZN9/vw4+Hh7Bzrtx0bFqbV2/oS4v61bnmySm8\nmL+s9I1SnIewc65cNa2bzb+H9qFPuxx+9tIM/vb+fP8w0YPwEHbOlbva2cHcxBf0bMHfP1zAT1+a\n4ZPElyAj6gKcc1VTtfQ07h7cjeb1q3PvBwtYs3knD152NLWzq0VdWoXiPWHnXMJI4uaTO3D34G5M\nWFjIhQ9P8sl/ivEQds4l3IV5LXn8qmNYWriN8x78hPlrtkRdUoXhIeycS4rjOzTihev6sGe/Mfih\nCUxZvCEBeJoPAAAPEUlEQVTqkioED2HnXNJ0bV6XV64/joa1srj80cm8O3N11CVFzkPYOZdULRvU\n4KXrj6NTszrc8OxU/jVpSdQlRcpD2DmXdA1qZjLy2t6c2LExv3xtJn95b17K3kvsIeyci0SNzAwe\n/l5PLspryT8+KuDnL89gbwreS+z3CTvnIpORnsafzj+SJnWzue/DBazfupsHLj2a6pnpUZeWNN4T\nds5FShK3nNKBO8/tyn/mreXKxz9l8849UZeVNB7CzrkK4bLeh3PfxT34bOlGLv3nJAq37oq6pKTw\nEHbOVRhndT+Mf16ZR8HarVz48MSUmCDeQ9g5V6Gc2LExT1/Tm7Wbd3HBiIl8uX5b1CUllIewc67C\n6dWmAc8NOZade/ZxwYgJVfqTOjyEnXMVUtfmdXlxaB8y09O46JGJ5FfRx5w9hJ1zFVa7RrX49/XH\n0ahWFpc/NpmxVfAjkzyEnXMVWvN61Xnhuj60aViLHzw1hbe/WBV1SeXKQ9g5V+E1qp3F80OOpXuL\negwf+RkvTFkadUnlxkPYOVcp1K1ejae/34t+uY34+ctf8OjHi6IuqVx4CDvnKo0amRk8ekUeZx7Z\njD+8NadKTPzjc0c45yqVzIw07rukB7WyMvjHRwVs3rGHX5/VhbQ0RV3aIfEQds5VOulp4k/nH0nd\nGtV4ZNwiNu/cy92Du1EtvfL9595D2DlXKUnittOPoG71atwzeh5bdu7l/kt7kF2tcs3AVvl+bTjn\nXEgSw05sz+/P6cIHc9Yw5F9T2blnX9RllYmHsHOu0vten9bcfX43Pl6wjmufzmfH7soTxB7Czrkq\n4cJjWnLP4O6ML1jP95+aUmmCOKEhLGmgpHmSCiTdWsI6J0iaJmmWpLEx7TdJmhm23xzT3kDS+5IW\nhF/rh+2tJe0I9zVN0ohEnptzruIZ3LMFf72wO5MWFXL1k5+yfffeqEsqVcJCWFI68ABwOtAZuERS\n52Lr1AMeBM42sy7ABWF7V+BaoBfQHRgkqX242a3Ah2aWC3wYvi6y0MyOCpehiTo351zFdW6PFvzt\noqP49MsNXPX4FLbtqthBnMiecC+gwMwWmdlu4HngnGLrXAq8YmZLAcxsbdjeCZhsZtvNbC8wFjgv\nfO8c4Knw+6eA7ybwHJxzldA5RzXn7xf3YMqSDfzlvflRl3NQiQzh5sCymNfLw7ZYHYD6ksZImirp\nirB9JtBfUo6kGsAZQMvwvSZmVjSDx2qgScz+2oRDEWMl9T9QUZKGSMqXlL9uXdWbkck5Fzir+2Fc\nfEwrnp64mIK1W6Mup0RRX5jLAHoCZwKnAb+U1MHM5gB3Ae8B7wLTgP8ZZbfgecWiZxZXAa3M7Cjg\nFmCkpDoH2OYRM8szs7xGjRol4pyccxXEj0/tQPVq6fzhrdkV9vHmRIbwCv7bewVoEbbFWg6MNrNt\nZrYeGEcwBoyZPWZmPc1sALARKPo/xRpJzQDCr2vD9XeZWWH4/VRgIUFP2zmXohrWyuJHp3RgzLx1\n3PnWnAoZxIkM4SlArqQ2kjKBi4FRxdZ5HegnKSMcdugNzAGQ1Dj82opgPHhkuM0o4Mrw+yvDfSCp\nUXgxEEltgVygakyz5Jw7ZFf3bc1Vx7Xm0fFfVsggTthjy2a2V9JwYDSQDjxuZrMkDQ3fH2FmcyS9\nC8wA9gOPmtnMcBcvS8oB9gDDzGxT2P4n4EVJ3weWABeG7QOA30naE+5rqJlVzc9Dcc7FTRK/Piu4\nMevR8V8iwe1ndEKqGBP+qKL9VkimvLw8y8/Pj7oM51wSmBm/GTWLpyYuYciAttx2+hEJC2JJU80s\nL551fQIf51xKkMRvzu7CfoNHxi0iTeLnAztG3iP2EHbOpQxJ/PbsLuwzY8TYhaSnwU9OjTaIPYSd\ncyklLU384Zyu7N9vPPCfhaSnpXHLKdHdSOUh7JxLOWlp4o/nHsl+M+77cAHpEjednBtJLR7CzrmU\nlJYm/nReN/bth799MJ+aWen8oH/bpNfhIeycS1lpaeLuwd3Ytmsv//fOXI5qWY+81g2SW0NSj+ac\ncxVMepq4+4JutKhfneEjP2fDtt1JPb6HsHMu5dXJrsYDlx5N4bZd/GbUrKQe20PYOeeArs3rMvzE\nXEZNX8l7s1Yn7bgews45F7rhxHZ0alaHO16byabtyRmW8BB2zrlQtfQ07hncjY3bdvO7N2cn5Zge\nws45F6Nr87pcf0I7XvlsBR/NXZPw43kIO+dcMcO/054OTWpx+ysz2bxzT0KP5SHsnHPFZGWkc8/g\n7qzdspM735yT0GN5CDvn3AF0b1mPawe05YX8ZYybn7jPo/Qn5pxzrgQ/OrkD81ZvIbtaesKO4SHs\nnHMlyK6WzpNX90roMXw4wjnnIuQh7JxzEfIQds65CHkIO+dchDyEnXMuQh7CzjkXIQ9h55yLkIew\nc85FSGYWdQ2RkbQOWFLGzRoC6xNQTll5Hd/kdXyT1/FNya7jcDNrFM+KKR3Ch0JSvpnleR1eh9fh\ndZQHH45wzrkIeQg751yEPITL7pGoCwh5Hd/kdXyT1/FNFaWO/+Fjws45FyHvCTvnXIQ8hJ1zLkIe\nwnGSNFDSPEkFkm5N4nFbSvqPpNmSZkm6KWxvIOl9SQvCr/WTVE+6pM8lvRlxHfUkvSRprqQ5kvok\nuxZJPwr/TmZKek5SdrJqkPS4pLWSZsa0lXhsSbeFP7vzJJ2W4DruCf9eZkh6VVK9KOqIee/HkkxS\nw0TXcSg8hOMgKR14ADgd6AxcIqlzkg6/F/ixmXUGjgWGhce+FfjQzHKBD8PXyXATEPvJh1HV8Xfg\nXTM7Auge1pS0WiQ1B24E8sysK5AOXJzEGp4EBhZrO+Cxw5+Xi4Eu4TYPhj/TiarjfaCrmXUD5gO3\nRVQHkloCpwJLY9oSWUfZmZkvpSxAH2B0zOvbgNsiquV14BRgHtAsbGsGzEvCsVsQ/OP+DvBm2BZF\nHXWBLwkvLMe0J60WoDmwDGhA8DFhbxL8Y09mDa2BmaWdf/GfV2A00CdRdRR771zg2ajqAF4i+CW9\nGGiYjDrKunhPOD5F/+CKLA/bkkpSa6AHMBloYmarwrdWA02SUMK9wM+A/TFtUdTRBlgHPBEOjTwq\nqWYyazGzFcCfCXpYq4CvzOy9ZNZwACUdO8qf32uAd6KoQ9I5wAozm17srQrx77mIh3AlIakW8DJw\ns5ltjn3Pgl/nCb3XUNIgYK2ZTS1pnWTUEcoAjgYeMrMewDaK/bc/0bWE463nEPxCOAyoKenyZNZw\nMFEeu4ikOwiG056N4Ng1gNuBXyX72GXlIRyfFUDLmNctwrakkFSNIICfNbNXwuY1kpqF7zcD1ia4\njL7A2ZIWA88D35H0TAR1QNBzWW5mk8PXLxGEcjJrORn40szWmdke4BXguCTXUFxJx076z6+kq4BB\nwGXhL4Rk19GO4Bfk9PBntgXwmaSmSa6jVB7C8ZkC5EpqIymTYFB/VDIOLEnAY8AcM/trzFujgCvD\n768kGCtOGDO7zcxamFlrgvP/yMwuT3YdYS2rgWWSOoZNJwGzk1zLUuBYSTXCv6OTCC4OJv3PI0ZJ\nxx4FXCwpS1IbIBf4NFFFSBpIMGx1tpltL1ZfUuowsy/MrLGZtQ5/ZpcDR4c/O0n984inWF/iG/Q/\ng+BK70LgjiQetx/BfytnANPC5Qwgh+Ai2QLgA6BBEms6gf9emIukDuAoID/8c3kNqJ/sWoDfAnOB\nmcC/gKxk1QA8RzAWvYcgYL5/sGMDd4Q/u/OA0xNcRwHBmGvRz+uIKOoo9v5iwgtziazjUBZ/bNk5\n5yLkwxHOORchD2HnnIuQh7BzzkXIQ9g55yLkIeyccxHyEHZJIWlC+LW1pEvLed+3H+hYiSLpu5IS\n8iRW8XMpp30eKenJ8t6vKx9+i5pLKkknAD8xs0Fl2CbDzPYe5P2tZlarPOqLs54JBA8ifKuPUD/Q\neSXqXCR9AFxjZktLXdkllfeEXVJI2hp++yegv6Rp4Xy86eH8s1PC+WevC9c/QdLHkkYRPA2HpNck\nTQ3n8B0Stv0JqB7u79nYYylwj4L5fr+QdFHMvsfov/MRPxs+9YakPymYu3mGpD8f4Dw6ALuKAljS\nk5JGSMqXND+cY6No3uW4zitm3wc6l8slfRq2PVw05aKkrZLulDRd0iRJTcL2C8LznS5pXMzu3yB4\n0tFVNFE+KeJL6izA1vDrCYRP24WvhwC/CL/PIngKrk243jagTcy6DcKv1QmeUsuJ3fcBjnU+wdy2\n6QQzii0lmOLxBOArgjkD0oCJBE8m5hA8QVX0P8R6BziPq4G/xLx+Eng33E8uwdNa2WU5rwPVHn7f\niSA8q4WvHwSuCL834Kzw+7tjjvUF0Lx4/QRzf7wR9c+BL/+7ZMQb1s4lyKlAN0mDw9d1CcJsN/Cp\nmX0Zs+6Nks4Nv28Zrld4kH33A54zs30Ek9uMBY4BNof7Xg4gaRrBXLSTgJ3AYwo+OeTNA+yzGcE0\nmrFeNLP9wAJJi4AjynheJTkJ6AlMCTvq1fnvpDy7Y+qbSjDHNMAnwJOSXiSYVKjIWoLZ3lwF4yHs\noibgh2Y2+huNwdjxtmKvTyaYfHu7pDEEPc5DtSvm+31AhpntldSLIPwGA8MJJrCPtYMgUGMVv7Bi\nxHlepRDwlJnddoD39ljYxS2qH8DMhkrqDZwJTJXU08wKCf6sdsR5XJdEPibskm0LUDvm9WjgegXT\ndSKpg4IJ2ourC2wMA/gIgo96KrKnaPtiPgYuCsdnGwEDOMhsWQrmbK5rZm8DPyL4RIbi5gDti7Vd\nIClNUjugLcGQRrznVVzsuXwIDJbUONxHA0mHH2xjSe3MbLKZ/Yqgx140ZWMHgiEcV8F4T9gl2wxg\nn6TpBOOpfycYCvgsvDi2DvjuAbZ7FxgqaQ5ByE2Kee8RYIakz8zsspj2Vwk+mmo6Qe/0Z2a2Ogzx\nA6kNvC4pm6AXessB1hkH/EWSYnqiSwnCvQ4w1Mx2Sno0zvMq7hvnIukXwHuS0ghmCBsGLDnI9vdI\nyg3r/zA8d4ATgbfiOL5LMr9FzbkykvR3gotcH4T3375pZi9FXFaJJGUBY4F+dpBb/Vw0fDjCubL7\nI1Aj6iLKoBVwqwdwxeQ9Yeeci5D3hJ1zLkIews45FyEPYeeci5CHsHPORchD2DnnIvT/mqCbbzcE\ngz0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff88bc80208>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "parameters = L_layer_model(X,Y,layer_dims,learning_rate=0.009,num_iteration=15000,print_cost=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.638888888889\n",
      "[[ 0.  1.  0.  0.  0.  1.  0.  0.  0.  1.  0.  0.  0.  1.  0.  0.  0.  1.\n",
      "   0.  0.  1.  1.  0.  1.  0.  1.  0.  0.  1.  1.  0.  1.  0.  1.  0.  0.\n",
      "   1.  1.  0.  0.  1.  1.  0.  1.  1.  1.  0.  0.  1.  1.  0.  1.  0.  1.\n",
      "   0.  0.  1.  1.  0.  1.  1.  1.  0.  1.  1.  1.  0.  1.  1.  1.  0.  1.]]\n"
     ]
    }
   ],
   "source": [
    "predictions_train = predict(X, Y, parameters)\n",
    "print(predictions_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.464285714286\n",
      "[[ 0.  1.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.  1.\n",
      "   0.  0.  0.  0.  0.  0.  0.  1.  0.  0.]]\n"
     ]
    }
   ],
   "source": [
    "pred_test = predict(X_test, Y_test, parameters)\n",
    "print(pred_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}