2_layer_NN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 28, 28) (50000,)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADllJREFUeJzt3X+oVXW6x/HPk2lWSlieDtLYPRNUEMKcqZ3cUMPrNOLI\ngIoRIzR4SeYMNTNcQ+KGF7r9gJC4zmQUA2eupl3mNt5S0yDmlhKEUFO7sh/a7zjiMX8cqZyUcq76\n3D/OcjjZ2d+93Xvtvbbneb/gcPZez1p7PS79uPZea6/1NXcXgHjOKboBAMUg/EBQhB8IivADQRF+\nICjCDwRF+IGgCD8QFOEHgjq3lSubOHGid3V1tXKVQCh9fX06dOiQ1TJvQ+E3szmSVkkaJek/3X1F\nav6uri6Vy+VGVgkgoVQq1Txv3W/7zWyUpMck/UTSNZIWmdk19b4egNZq5DP/VEkfu/un7v43SX+S\nNC+ftgA0WyPhv0zSniHP+7Np32JmPWZWNrPywMBAA6sDkKemH+139153L7l7qaOjo9mrA1CjRsK/\nV9LkIc+/l00DcBZoJPyvSbrSzL5vZmMk/UzSlnzaAtBsdZ/qc/fjZvZrSf+rwVN9a9x9Z26dAWiq\nhs7zu/tzkp7LqRcALcTXe4GgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxA\nUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8\nQFCEHwiqoVF6zaxP0leSTkg67u6lPJpCfk6ePJmsHzt2rKnrX7duXcXa0aNHk8vu2rUrWX/44YeT\n9eXLl1esPfroo8llzz///GR95cqVyfrtt9+erLeDhsKf+Sd3P5TD6wBoId72A0E1Gn6XtNXMXjez\nnjwaAtAajb7tn+7ue83sUkkvmNn77v7S0Bmy/xR6JOnyyy9vcHUA8tLQnt/d92a/D0raJGnqMPP0\nunvJ3UsdHR2NrA5AjuoOv5ldaGbjTz2WNFvSu3k1BqC5Gnnb3ylpk5mdep3/dvc/59IVgKarO/zu\n/qmkH+TYy4h1+PDhZP3EiRPJ+ltvvZWsP//88xVrX375ZXLZ3t7eZL1IXV1dyfqyZcuS9dWrV1es\nXXTRRcllZ8yYkazPmjUrWT8bcKoPCIrwA0ERfiAowg8ERfiBoAg/EFQeV/WF19/fn6x3d3cn6198\n8UWe7Zw1zjknve9JnaqTql92u2TJkoq1Sy+9NLnsuHHjkvWR8G1V9vxAUIQfCIrwA0ERfiAowg8E\nRfiBoAg/EBTn+XNwySWXJOudnZ3Jejuf5589e3ayXu3PvnHjxoq18847L7nszJkzk3U0hj0/EBTh\nB4Ii/EBQhB8IivADQRF+ICjCDwTFef4cVLuufO3atcn6008/nazfcMMNyfrChQuT9ZTp06cn65s3\nb07Wx4wZk6zv37+/Ym3VqlXJZdFc7PmBoAg/EBThB4Ii/EBQhB8IivADQRF+IChz9/QMZmsk/VTS\nQXefkk27WNJ6SV2S+iTd4u5VL0ovlUpeLpcbbHnkOXbsWLJe7Vz68uXLK9Yeeuih5LIvvvhisn7j\njTcm62gvpVJJ5XLZapm3lj3/WklzTpt2t6Rt7n6lpG3ZcwBnkarhd/eXJH1+2uR5ktZlj9dJmp9z\nXwCarN7P/J3uvi97vF9S+j5VANpOwwf8fPCgQcUDB2bWY2ZlMysPDAw0ujoAOak3/AfMbJIkZb8P\nVprR3XvdveTupZEwuCEwUtQb/i2SFmePF0tKX/oFoO1UDb+ZPSnpZUlXm1m/mS2RtELSj83sI0k3\nZc8BnEWqXs/v7osqlH6Ucy9hVbt/fTUTJkyoe9lHHnkkWZ8xY0ayblbTKWW0Ib7hBwRF+IGgCD8Q\nFOEHgiL8QFCEHwiKW3ePAEuXLq1Ye/XVV5PLbtq0KVnfuXNnsj5lypRkHe2LPT8QFOEHgiL8QFCE\nHwiK8ANBEX4gKMIPBMV5/hEgdWvv3t7e5LLbtm1L1ufNm5esz5+fvnfrtGnTKtYWLFiQXJbLhZuL\nPT8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBFV1iO48MUR3+6l2vf+cOacP0Pxthw8frnvda9asSdYX\nLlyYrI8bN67udY9UeQ/RDWAEIvxAUIQfCIrwA0ERfiAowg8ERfiBoKpez29mayT9VNJBd5+STbtX\n0i8kDWSzLXf355rVJJpn6tSpyXq1+/bfeeedyfpTTz1VsXbbbbcll/3kk0+S9bvuuitZHz9+fLIe\nXS17/rWShvumx+/cvTv7IfjAWaZq+N39JUmft6AXAC3UyGf+35jZ22a2xswm5NYRgJaoN/y/l3SF\npG5J+yStrDSjmfWYWdnMygMDA5VmA9BidYXf3Q+4+wl3PynpD5IqHjVy9153L7l7qaOjo94+AeSs\nrvCb2aQhTxdIejefdgC0Si2n+p6UNFPSRDPrl/TvkmaaWbckl9Qn6ZdN7BFAE3A9PxryzTffJOuv\nvPJKxdpNN92UXLbav82bb745WV+/fn2yPhJxPT+Aqgg/EBThB4Ii/EBQhB8IivADQTFENxoyduzY\nZH3mzJkVa6NGjUoue/z48WT9mWeeSdY/+OCDirWrr746uWwE7PmBoAg/EBThB4Ii/EBQhB8IivAD\nQRF+ICjO8yPps88+S9Y3btyYrL/88ssVa9XO41dz/fXXJ+tXXXVVQ68/0rHnB4Ii/EBQhB8IivAD\nQRF+ICjCDwRF+IGgOM8/wlUbIu2xxx5L1h9//PFkvb+//4x7qlW16/27urqSdbOa7mAdFnt+ICjC\nDwRF+IGgCD8QFOEHgiL8QFCEHwiq6nl+M5ss6QlJnZJcUq+7rzKziyWtl9QlqU/SLe7+RfNajevI\nkSPJ+rPPPluxdv/99yeX/fDDD+vqKQ+zZs1K1lesWJGsX3fddXm2E04te/7jkpa5+zWS/lHSr8zs\nGkl3S9rm7ldK2pY9B3CWqBp+d9/n7m9kj7+S9J6kyyTNk7Qum22dpPnNahJA/s7oM7+ZdUn6oaS/\nSOp0931Zab8GPxYAOEvUHH4zGydpg6Sl7v7XoTV3dw0eDxhuuR4zK5tZudr3zAG0Tk3hN7PRGgz+\nH9391B0bD5jZpKw+SdLB4ZZ19153L7l7qaOjI4+eAeSgavht8NKo1ZLec/ffDiltkbQ4e7xY0ub8\n2wPQLLVc0jtN0s8lvWNmO7JpyyWtkPQ/ZrZE0m5JtzSnxbPf0aNHk/U9e/Yk67feemuy/uabb55x\nT3mZPXt2sn7fffdVrFW79TaX5DZX1fC7+3ZJlf4WfpRvOwBahW/4AUERfiAowg8ERfiBoAg/EBTh\nB4Li1t01+vrrryvWli5dmlx2+/btyfr7779fV095mDt3brJ+zz33JOvd3d3J+ujRo8+4J7QGe34g\nKMIPBEX4gaAIPxAU4QeCIvxAUIQfCCrMef6+vr5k/cEHH0zWt27dWrG2e/fuelrKzQUXXFCx9sAD\nDySXveOOO5L1MWPG1NUT2h97fiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IKsx5/g0bNiTrq1evbtq6\nr7322mR90aJFyfq556b/mnp6eirWxo4dm1wWcbHnB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgzN3T\nM5hNlvSEpE5JLqnX3VeZ2b2SfiFpIJt1ubs/l3qtUqnk5XK54aYBDK9UKqlcLlst89byJZ/jkpa5\n+xtmNl7S62b2Qlb7nbv/R72NAihO1fC7+z5J+7LHX5nZe5Iua3ZjAJrrjD7zm1mXpB9K+ks26Tdm\n9raZrTGzCRWW6TGzspmVBwYGhpsFQAFqDr+ZjZO0QdJSd/+rpN9LukJStwbfGawcbjl373X3kruX\nOjo6cmgZQB5qCr+ZjdZg8P/o7hslyd0PuPsJdz8p6Q+SpjavTQB5qxp+MzNJqyW95+6/HTJ90pDZ\nFkh6N//2ADRLLUf7p0n6uaR3zGxHNm25pEVm1q3B0399kn7ZlA4BNEUtR/u3SxruvGHynD6A9sY3\n/ICgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0FVvXV3risz\nG5C0e8ikiZIOtayBM9OuvbVrXxK91SvP3v7B3Wu6X15Lw/+dlZuV3b1UWAMJ7dpbu/Yl0Vu9iuqN\nt/1AUIQfCKro8PcWvP6Udu2tXfuS6K1ehfRW6Gd+AMUpes8PoCCFhN/M5pjZB2b2sZndXUQPlZhZ\nn5m9Y2Y7zKzQIYWzYdAOmtm7Q6ZdbGYvmNlH2e9hh0krqLd7zWxvtu12mNncgnqbbGYvmtkuM9tp\nZv+STS902yX6KmS7tfxtv5mNkvShpB9L6pf0mqRF7r6rpY1UYGZ9kkruXvg5YTO7UdIRSU+4+5Rs\n2kOSPnf3Fdl/nBPc/V/bpLd7JR0peuTmbECZSUNHlpY0X9I/q8Btl+jrFhWw3YrY80+V9LG7f+ru\nf5P0J0nzCuij7bn7S5I+P23yPEnrssfrNPiPp+Uq9NYW3H2fu7+RPf5K0qmRpQvddom+ClFE+C+T\ntGfI836115DfLmmrmb1uZj1FNzOMzmzYdEnaL6mzyGaGUXXk5lY6bWTpttl29Yx4nTcO+H3XdHfv\nlvQTSb/K3t62JR/8zNZOp2tqGrm5VYYZWfrvitx29Y54nbciwr9X0uQhz7+XTWsL7r43+31Q0ia1\n3+jDB04Nkpr9PlhwP3/XTiM3DzeytNpg27XTiNdFhP81SVea2ffNbIykn0naUkAf32FmF2YHYmRm\nF0qarfYbfXiLpMXZ48WSNhfYy7e0y8jNlUaWVsHbru1GvHb3lv9ImqvBI/6fSPq3Inqo0NcVkt7K\nfnYW3ZukJzX4NvD/NHhsZImkSyRtk/SRpK2SLm6j3v5L0juS3tZg0CYV1Nt0Db6lf1vSjuxnbtHb\nLtFXIduNb/gBQXHADwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUP8PB4Bqh9Y9PDQAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fa9937f9a58>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from preprocessed_mnist import load_dataset\n",
    "X_train, y_train, X_val, y_val, X_test, y_test = load_dataset()\n",
    "print(X_train.shape, y_train.shape)\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.imshow(X_train[0], cmap=\"Greys\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "s = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 784)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Flattening training, validate and test sets\n",
    "X_train_flatten = X_train.reshape(X_train.shape[0], -1)\n",
    "X_val_flatten = X_val.reshape(X_val.shape[0], -1)\n",
    "X_test_flatten = X_test.reshape(X_test.shape[0], -1)\n",
    "X_train_flatten.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#input placeholders for features and classes\n",
    "x = tf.placeholder(tf.float32, shape=(None,784))\n",
    "y = tf.placeholder(tf.float32,shape=(None, 10))\n",
    "\n",
    "#Initialization of weights and bias\n",
    "W1 = tf.get_variable(\"W1\", [784,350], initializer = tf.contrib.layers.xavier_initializer(seed = 1))\n",
    "b1 = tf.get_variable(\"b1\", [350], initializer = tf.zeros_initializer())\n",
    "W2 = tf.get_variable(\"W2\", [350, 10], initializer = tf.contrib.layers.xavier_initializer(seed = 1))\n",
    "b2 = tf.get_variable(\"b2\", [10], initializer = tf.zeros_initializer())\n",
    "#Activation layers\n",
    "Z1 = tf.add(tf.matmul(x,W1), b1)                      \n",
    "A1 = tf.nn.relu(Z1)                                    \n",
    "Z2 = tf.add(tf.matmul(A1,W2), b2)\n",
    "A2 = tf.nn.sigmoid(Z2)                                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=A2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = np.eye(10)[y_train]\n",
    "y_test = np.eye(10)[y_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_epochs = 200\n",
    "batch_size = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.001\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost after 10 epochs: 1.47591\n",
      "Cost after 20 epochs: 1.46821\n",
      "Cost after 60 epochs: 1.46521\n",
      "Cost after 90 epochs: 1.46448\n",
      "Cost after 100 epochs: 1.46413\n",
      "Cost after 110 epochs: 1.4641\n",
      "Cost after 120 epochs: 1.46471\n",
      "Cost after 130 epochs: 1.46401\n",
      "Cost after 140 epochs: 1.46466\n",
      "Cost after 160 epochs: 1.46436\n",
      "Cost after 170 epochs: 1.46395\n",
      "Cost after 180 epochs: 1.46409\n",
      "Cost after 190 epochs: 1.46452\n",
      "Cost after 200 epochs: 1.46392\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VGX2wPHvmZIekkBCL6EJ0kVARVQsWLG7il3Xta3d\n3d+urruWbXbXVVZZ27o2dO3sKoqNotKR3jsBQkII6WXK+/vjvSEJZJIhZDIBzud55snMvXfunNy5\n85633CLGGJRSSqmGuKIdgFJKqYODJgyllFJh0YShlFIqLJowlFJKhUUThlJKqbBowlBKKRUWTRhK\nKaXCoglDKaVUWDRhKKWUCosn2gE0pfT0dJOZmRntMJRS6qAxf/78ncaYjHCWPaQSRmZmJvPmzYt2\nGEopddAQkU3hLqtdUkoppcKiCUMppVRYNGEopZQKiyYMpZRSYdGEoZRSKiyaMJRSSoVFE4ZSSqmw\naMIAnvtmDdNW50Y7DKWUatE0YQATpq3j+zWaMJRSqj4RSxgi8pqI5IjI0hDzR4tIgYgsdB4POtPj\nRGSOiCwSkWUi8kikYqzidbvwBUykP0YppQ5qkWxhvA6c2cAyM4wxQ5zHH51pFcApxpjBwBDgTBE5\nNoJx8qH5FcdufyOSH6GUUge9iCUMY8x0YFcj3meMMcXOS6/ziGj1vwO5JFTud6hKKXVYifYYxkgR\nWSwik0Wkf9VEEXGLyEIgB/jKGDM7kkH48OAKVkbyI5RS6qAXzYSxAOhqjBkEPA98UjXDGBMwxgwB\nOgMjRGRAqJWIyE0iMk9E5uXmNm7g2i9eJOBr1HuVUupwEbWEYYwprOp6MsZ8DnhFJH2vZXYD31HP\nWIgx5iVjzDBjzLCMjLAu6b4Pv2gLQymlGhK1hCEi7UVEnOcjnFjyRCRDRFKd6fHAGGBlJGPx48UV\n1BaGUkrVJ2I3UBKRicBoIF1EsoCHsAPYGGMmAJcAt4qIHygDxhljjIh0AP4tIm5sEvmPMeZ/kYoT\nbJeUy2jCUEqp+kQsYRhjLm9g/nhgfB3TFwNHRSquugRcXtzaJaWUUvWK9lFSLUJAvLi1S0oppeql\nCQMnYRh/tMNQSqkWTRMGEHR5cesYhlJK1UsTBhBwxeDRhKGUUvXShAEEXR7tklJKqQZowgCMKwYv\n2sJQSqn6aMIAgtolpZRSDdKEARi3V1sYSinVAE0Y2BaGV8cwlFKqXpowANxePGjCUEqp+mjCAIw7\nhhj8GKO3aVVKqVA0YQC4YvBKAH8gEO1IlFKqxdKEAeCJAcBfqRcgVEqpUDRhALhtwqisLI9yIEop\n1XJpwgDEXdXC0IShlFKhaMKA6i4pnyYMpZQKRRMGIJ5YAAI6hqGUUiFpwgBEWxhKKdUgTRiAy0kY\nAV9FlCNRSqmWSxMGIJ44QBOGUkrVJ2IJQ0ReE5EcEVkaYv5oESkQkYXO40FnehcR+U5ElovIMhG5\nK1IxVnF5vAAEfDqGoZRSoXgiuO7XgfHAG/UsM8MYM3avaX7gV8aYBSKSDMwXka+MMcsjFCdur21h\nBHUMQymlQopYC8MYMx3Y1Yj3bTfGLHCeFwErgE5NHF4tLq9zlJS2MJRSKqRoj2GMFJHFIjJZRPrv\nPVNEMoGjgNmRDKJq0DsY0DEMpZQKJZJdUg1ZAHQ1xhSLyNnAJ0DvqpkikgR8CNxtjCkMtRIRuQm4\nCaBr166NCqSqS8rooLdSSoUUtRaGMabQGFPsPP8c8IpIOoCIeLHJ4m1jzEcNrOclY8wwY8ywjIyM\nRsXiibFdUkG/dkkppVQoUUsYItJeRMR5PsKJJc+Z9iqwwhjzTHPE4na6pPBrC0MppUKJWJeUiEwE\nRgPpIpIFPAR4AYwxE4BLgFtFxA+UAeOMMUZERgFXA0tEZKGzut85rZCIcMc4R0lpC0MppUKKWMIw\nxlzewPzx2MNu957+PSCRiqsuXqdLymjCUEqpkKJ9lFSL4HFaGAQ0YSilVCiaMKge9DaaMJRSKiRN\nGECMc+Ie2iWllFIhacIAvB43FcaDaAtDKaVC0oQBuF2CD4+OYSilVD00YTh8eJCgJgyllApFE4bD\nhxcJ+KIdhlJKtViaMBx+0RaGUkrVRxOGw4cXl45hKKVUSJowHLaFoV1SSikViiYMh188uDRhKKVU\nSJowHH6JwW20S0oppULRhOEIiAdX0B/tMJRSqsXShOEISAwuo11SSikViiYMR0A8ePSwWqWUCkkT\nhiPg8uI22iWllFKhaMJwBFwxeHTQWymlQtKE4TCiLQyllKqPJgxH0OXFowlDKaVC0oThCLpj8OhR\nUkopFVLEEoaIvCYiOSKyNMT80SJSICILnceD4b43EozLixdNGEopFUokWxivA2c2sMwMY8wQ5/HH\n/Xxvkwq6YvCgXVJKKRVKxBKGMWY6sKu539tobi8x+MGYZv1YpZQ6WER7DGOkiCwWkcki0j+agRh3\njH2iN1FSSqk6RTNhLAC6GmMGAc8DnzRmJSJyk4jME5F5ubm5jQ6mOmFUNHodSil1KItawjDGFBpj\nip3nnwNeEUlvxHpeMsYMM8YMy8jIaHxAnlj711fe+HUopdQhLGoJQ0Tai4g4z0c4seRFKx6ftxUA\npnx3tEJQSqkWzROpFYvIRGA0kC4iWcBDgBfAGDMBuAS4VUT8QBkwzhg74lzXe40xr0YqVoBATAoA\n/uJdePe7naOUUoe+iCUMY8zlDcwfD4xvzHsjwR9rE0agNN9mNaWUUrVE+yipFiMQmwqAvyQ/ypEo\npVTLpAnD4UlMA8BX0rynfyil1MFCE4YjJqk1oC0MpZQKRROGIzE+nhITS7BUWxhKKVWXiA16H2wS\nYz0UkIiUFUQ7FKWUapG0heFIivVQYBKRMu2SUkqpumjCcCTFeSggCVeFtjCUUqoumjAcSTG2heGp\n1IShlFJ10YThSIx1U2AS8VYWRjsUpZRqkTRhODxuFyWuJGL9mjCUUqoumjBqKPe0IiZYpvfEUEqp\nOmjCqKHCk2yflOkVa5VSam+aMGrwxdhLnKOH1iql1D40YdTg89oLEKL3xFBKqX1owqghGGcvca5d\nUkoptS9NGDXFawtDKaVC0YRRg1QlDG1hKKXUPjRh1OCJt/fE0BaGUkrtSxNGDfHxcRSbOIKlepSU\nUkrtTRNGDUmxHvJNMv6inGiHopRSLY4mjBqSYj1kk0awcFu0Q1FKqRYnYglDRF4TkRwRWRpi/mgR\nKRCRhc7jwRrzzhSRVSKyVkTui1SMe0uM9ZBtWuMq2t5cH6mUUgeNsBKGiLwZzrS9vA6c2cAyM4wx\nQ5zHH531uoF/AGcB/YDLRaRfOHEeqKQ4mzDcJdlgTHN8pFJKHTTCbWH0r/nCKdSPru8NxpjpQGNu\nkD0CWGuMWW+MqQTeBc5vxHr2W1Ksm2yThttfpkdKKaXUXupNGCJyv4gUAYNEpNB5FAE5wKdN8Pkj\nRWSxiEwWkaqk1AnYUmOZLGdaqBhvEpF5IjIvNzf3gIJJivWyw7S2Lwq1W0oppWqqN2EYYx41xiQD\nTxpjWjmPZGNMG2PM/Qf42QuArsaYQcDzwCeNWYkx5iVjzDBjzLCMjIwDCijRaWEAUKQD30opVVO4\nXVL/E5FEABG5SkSeEZFuB/LBxphCY0yx8/xzwCsi6cBWoEuNRTs70yIuOdZLNm3sC21hKKVULeEm\njBeBUhEZDPwKWAe8cSAfLCLtRUSc5yOcWPKAuUBvEekuIjHAOGDSgXxWuBJj3eyoamHoobVKKVWL\nJ8zl/MYYIyLnA+ONMa+KyA31vUFEJgKjgXQRyQIeArwAxpgJwCXArSLiB8qAccYYA/hF5HbgS8AN\nvGaMWdaI/22/edwu3N4YSjxpJGqXlFJK1RJuwigSkfuBq4ETRMSFU/iHYoy5vIH544HxIeZ9Dnwe\nZmxNKinWQ4EnnUTtklJKqVrC7ZK6DKgAfm6MycaOKzwZsaiiqFW8lzxXG+2SUkqpvYSVMJwk8TaQ\nIiJjgXJjzAGNYbRUaQkx7KC1HiWllFJ7CfdM70uBOcDPgEuB2SJySSQDi5a0BC/bg2lQmgf+imiH\no5RSLUa4YxgPAMONMTkAIpIBfA18EKnAoiU1IYbNfudGSgVZ0KZndANSSqkWItwxDFdVsnDk7cd7\nDyqp8V6WVLS3L3JXRjcYpZRqQcIt9L8QkS9F5DoRuQ74jCgdxRRpaYkxLPF1tC9ylkc3GKWUakHq\n7ZISkV5AO2PM/4nIRcAoZ9ZM7CD4ISc1wUsJ8fhbdcGTsyLa4SilVIvRUAvjWaAQwBjzkTHmXmPM\nvcDHzrxDTlpCDADlaX1gh7YwlFKqSkMJo50xZsneE51pmRGJKMpSE+z5iLuTe0PeGvBXRjkipZRq\nGRpKGKn1zItvykBaitR428LYGd8Dgn6bNJRSSjWYMOaJyI17TxSRXwDzIxNSdKUl2hbG1tgedoKO\nYyilFNDweRh3Ax+LyJVUJ4hhQAxwYSQDi5aqMYwtrk7g8uiRUkop5ag3YRhjdmDvincyMMCZ/Jkx\n5tuIRxYlcV43cV4Xu8qBjL6QNS/aISmlVIsQ1pnexpjvgO8iHEuLkZYQQ35JJfQYDXNegopiiE2K\ndlhKKRVVh+TZ2gcqJd5LfqkPep0GgUrYOCPaISmlVNRpwqhDWkIMu0srodtI8CbCmq+iHZJSSkWd\nJow6pCV62V3mA08sdD8R1n4FxkQ7LKWUiipNGHVIrWphAPQ+DXZvhu2LohuUUkpFmSaMOqTGe9ld\n6sMYAwMuhrgU+PbP0Q5LKaWiShNGHdISYvAHDUUVfohPg1H32m6pDdOjHZpSSkVNxBKGiLwmIjki\nsrSB5YaLiL/mHfxE5C4RWSoiy0Tk7kjFGEpaoj15b1ex0y11zM3QqhN8/7fmDkUppVqMSLYwXgfO\nrG8BEXEDjwNTakwbANwIjAAGA2Ody6w3mw4pcQBsLyi3E7zx0P9C2Pg9VJY0ZyhKKdViRCxhGGOm\nA7saWOwO4EOg5t38jgRmG2NKjTF+YBpwUWSirFunVHtdxa27y6on9jzFOSfjh+YMRSmlWoyojWGI\nSCfs9ahe3GvWUuAEEWkjIgnA2UCXetZzk4jME5F5ubm5TRJbh1TbwtiaXyNhdBsJnjhY902TfIZS\nSh1sojno/SzwW2NMsOZEY8wKqrupvgAWAoFQKzHGvGSMGWaMGZaRkdEkgcV63GQkx7J1d2n1RG88\ndDse1n5jz8nQ8zKUUoeZaCaMYcC7IrIRuAR4QUQuADDGvGqMOdoYcyKQD6xu7uA6pcazbXd57Ym9\nTrX3x3g8E944D4LBOt+rlFKHoqglDGNMd2NMpjEmE/gA+KUx5hMAEWnr/O2KHb94p7nj65QWX3sM\nA+zAd7dR0HmYPcR28XvNHZZSSkVNWFerbQwRmQiMBtJFJAt4CPACGGMmNPD2D0WkDeADbjPG7I5U\nnKF0So3nq+U7CAYNLpfYia06wvWf2ZbFq2Pgqweh9+mQ2Ka5w1NKqWYXsYRhjLl8P5a9bq/XJzR5\nQPupU2o8lf4geSWVZCTH1p7pcsE5T9uk8dJouOwN6HhUVOJUSqnmomd6h9CxrkNray0wBK7/AjDw\nxgVQuL35glNKqSjQhBHCnnMx8kMkDIDOR8PVn4C/AibdrkdONWTWBJj+VLSjUEo1kiaMEDql2YSx\nLVQLo0p6Lzj9T7D2a1j6YfX0OS/DhBOgvCCCUR5kFr4FC96IdhQqkt44H2Y8He0oWh5jDomjKjVh\nhNAqzkNSrCd0l1RNw26A1j1skgC7c8yeANmL4fP/i2ygBwtjYNdGKMiCgC/a0ahICAbs5XNWTY52\nJC3PrBfh+aMO+l4ITRghiAid0+LZvKu04YVdLhj2c9gyC3Yss/fOyFsL7QbaQ29n/7PhdWQvhX+e\nCHnrqqeV5EFxTuj3HExKd0FlEZiAvb+IOvQU74Cg3+7LAX+0o2lZNs+E/I0H/e9ZE0Y9uqcnsnFn\nmBcbHHIluGNhzkuw5H1weeHqj6H3GTD5N7alUV+TdM5LNtF8eptdLuCHf4+FN5v1MlqRk7+h7ueq\neUWyW6Qgy/71l8HOVZH7nCprv4GnjrCVkZYub639u2td/cu1cJow6tE9PZHNu0rxBcL4kSW0hsGX\nwfzXYdYL0Os0SMqAyyfCsbfZhPDVH+pukvrKYNnHkNrV1kS+fxp+ehNylsOOJbBjeZP/b2GZ9xq8\neyX4yhtetiG7NtT9/HDXnF0USz6Ax7tB0Y5955U1walOBVuqn29beODra8isF22rZke9d1CIvmCw\nuucgbx34K6Fga+PWtXuzTZKbZjZdfPtBE0Y9uqcn4g8asuo7Uqqms56Es5+C9gPh2FvtNJcbzvgL\njLgZZo63iWNvKz+DikI4bzz0O9/e3e/zX0P7QSCu2oPpzaWiGL5+BFb+z8ZyoKpaFe7YlpEwtsyB\n6U8272cWZcPyT6tf+8rhxZHwzR8j/9nlhfDF/XY/2zij9rw1X9vL3Xz10IElsKoWhicOtv3U+PWE\no3Bb9YVAd6233T3/PvfAWxvBkJeta7zCLAhU2Oe71tkK5XNDanc/f3o7/PRWw+ta+ZlNkrNeaPo4\nw6AJox49MhIB2LCzOLw3eONgxI1w83TocVL1dBE48zE44iyY8gfIWVH7fT+9BSldIPMEuOR1OP0v\nkNwRznsOup8ESz+wP+SSnfDKaZA1P7x4ynbbo1a+fsS+f8V/YepjtmbWUKH901tQvhv6jrWtnaoB\n/cbK3wjJHezBAVXJY8cyeKYfrJ92YOtuSGUJfPZrG0OVGU/bxLx7S8i3Nbnv/gr/uQZ2rrGvZ463\nrciFE8PvKirKrn5/QyqKnXGwXPjyfijJBXeMTZZVjIFv/2Sn//AsTPn9/v1PNRVk2dsZdxwK20O0\nMIyBaU/Y/fBALJoIJgjitglj9RR7uZ6a/9v+mvcveLqP/Z3tj2lPwuT7Qs+v+X3lrbNxBirh64ft\ntJyV9jc295Xa76uZ/KoOFFnj3Dpo1efV4yHbF9nbLjTDUViaMOrRIz0JgPW5TXDTJJcLznseYpPh\noxvtjxlgwwxY/50dNHe57GPk7XDPEnv2+ICLbUG36UeY+Q/Imgs/hXFoamUpTBwH66fC98/Aa2fC\ne1fB1Efhi/tsDeebP+37vpKdtivqx+ehy7Fw6Rs20U3+ja3dhOKvrH+H3bUB0rpD6+72eXmBjadw\nKyz/ZN/li3MObOC0sgS+/Yvddt8/C3Nfhvn/tvN8ZdVJas2U2u8rzrU/4AMV8NsCoOqEzoDfttYA\nFv/H1pBnPAOJGVC0Lfwa+X+ugVdObbgmnbMC/tYfnuwBT/WyFYDhv4Aux9iDMyqK4Mfx9i6S2xfa\nKxcMvBTmvmr3zWBw/wuggixb8ek4ZN+B72DA7ls/Pgff/cUmjPq2847lNsGu+bruOBa9a68e3aan\nTRg5y+z0qrGCxlj2sU2qM54J/z3G2O959oTaFZKaqloSHQbb+LLmQWwrWDHJdi0tftfO376oumtw\nzsvwZC9bPmz6EZ7oYXsaNn5vL0cU9MNC5xJ7PzwH718LRL57UxNGPdISY0hN8LIh3IHvhiRlwAUv\n2B/DWxfZHWnKA/ZHVtWFtbf+F9jbw356W3UNZNVk+wOc+Q94ZYx9VO51NNf3f4PNs+CS12DQZbaQ\nOO52+MNOuGux3elm/7P2+EQwaOP63z1Qlg8n/852qV3yqk1eH9wAW+baZSuK7HLrp9r3TTgexh9t\nC+f3r7cFT035G2yySOtuf1j/vRvyN0Gb3nYdNRXnwN+HwLQDqIXO+xdMfwL+dQ788Hc7be3X9u/G\n7+3ArLhhzVd2mjF2vOapXvDCsfZH2ljGwGf3wGe/qu7O2zgDSvMgNgWW/Mf+/yYAV75v41j534bX\nu2UObJltk23NGvre3ShF2fD2z2zX0BmPwmkPw03T4OwnbcLIXmoL4ykPwDePQFomDB4HR19nt8vq\nL+CTW22y+frh8MewCrZASmfodLRdz/ZFdnrAD6+fA0/2tNdfO+JM8CbAjHpO4pz2uH28ffG++0FR\nNuxcDX3Oti3WXRuqx/nCGVTeMtcmhZrnSFWW2vFDT5z9nVV1r711MUx93H6nH/7Cto5q2rUeirMB\nYytadclbAzHJ0HWkbVFWFMBpD0Grznadi96FpPa2xbTpR6dc+IPdP6Y/aSs+FYXw0c22ZXLcbXZd\nC96wLY+1X9mDa1zuhv/3A6QJowHd0xObLmEAHHEG/OxfsHU+PD/U/qhOe9jeb6Musclw0cu2kK0o\nhJF32D7ML39nH74yyJpTXWt7+VS7s899BfqeY1soF7wIt8+zYyluL6R1g2NusYe5rvu2+rOWf2Lj\nOffvcP+W6m61mES4/D1Ibg/vXGrP2P73efYH8vUj9oe2c7VNIl8/ZBPaZ7+qXrevDIq224KpdXdb\nmCz7CEbfZ2u9u9bbQdJXxsDSj2zsvhLnb5jjRzUFfLaPt21/+z+6PDD8RnteTNEO26rwxMOQK2DD\nNFsg7lpvWwCDr4DULjDpjsYP9s8cb3/MbfvZdW5dYLetN9EWFPkbYc2XcNojNhFnjoIV/2t4vT8+\nb7t8Bo2z2+aNC+C5ofCndDsPbOvgnUttC+SK9+C4X8Koe2ytXwS6HmsLolkvQJ9z4KqP7MPtha7H\n2W7DaU/YWm9SO1vxqGvcrS4FWTZh9DzFjr2tnly9PTbPhJF32n3rZ6/D8BtsjXltHTckC/htq3vg\npdBrjN3P/JXV86uSebfjqhNGVTdvVQujoih0nF/cZxPlc0fZrsoN02HTD7YwPusJwNhu28LttpLx\n/TP2gIEl79vtXHOf3Pi9/dtuICx4s/YtnINBuw/lrbUn+LbpWT2vx8kw7m0o22V/G6c+6Nyg7Vv4\n+BbwxNiDZTZMg03f2/0y6IeYJPs9Db3GJscZT9vE16feu2E3GU0YDWjyhAF2YPvm6XD+C3DhP22h\nXp/M42Hs3+DE38Coe22NdPYEW9jcPB2GXmt/lFMfha3z4OVT7I543G32/S43pPfe6x87EeLTqruD\nAn6bdNr2g6Ou3re2kpQBV31oE9sXv7U/0CPPg20LbG3VmwB3LIC7FsH/rYWMvrZFsuxjWP2lXUea\n08IAaDcAjr8beoy2r9+72ia+SXfa5njrHraVs/g/+26PXRtsAVvXAG1Rtm15FW51atZT4frPYejV\ndv7ar208PU6y8ftKbWGxYbqdf8K9tlDLWwsf3rDvcfOf3GYLGbC12g0zasdRUWx/xL3GwM+/hPjW\ntrtg8X9sZWHQpfZH32M0jLjJvufIc20tdMtc2L7YjjvV7N7Ysdx+7sr/2ZNEz/iL/e4riqD9ANt1\n+NVD8NPb8N6VkL3EVko6Dtl3+3QeVv189H32Hi9VBZnLBf0usIfEJraFX3xtD7yoryuy5v9dlm8T\nRkJrW6itmmxry1MftWNhY/5oWzHeeDj+Lvsdv3WRLSA3z4LJv7WJOmuOLQT7nmMrNiW51d15YBOG\nNxHaD7b7k6/EqRh47edtXQCPdYXlk/aNM3eV/Y0cfb1tbS18x1Z+pj5qC+xBl9pxw1WfVycDfzl8\nfLOtZFQU1k7um36w3YpnPmp/c8/0gy8fsAn7rYvgmSNt5bBNL/v/AiSk2+cdh9gu337n21sndDkG\n5r1q//9znrEt/LhU+12MfQZO/zOc8CvwxNr3xKbY5O6OsQmoGUTsarWHih7piXy0YCullX4SYppw\nc7Xrbx/hGnZ99fNuI20Xx1lP2h/56X+yBXjvMbYGOvk3duCx63Gh1+f22h/k8kn2WlizJ9hC8vJ3\nQzdt2/SEu5fYgsHttTX5VZ/bGtCAiyGulX2ArT29ewW8f519ndDG1gi9CTZZnf5nW4vK6GOb4wWb\nbaGyYbr94V36hq0J/vg8dBlhu7gWvmMvJb97C2Dgmkm1Dy6Y/U9b6GBsf3HvMbZWDba2l9jWtnz8\nZfbH2P0EW/Ased9ug6T29oed3tvW/r/9M/zjGLhlhi0I135jL28C0LavHQMq320L1TY97fcpbrt9\nTvqt3RZj/mj/h16nwSm/ty3Gm6bZ1prLqa8NHmdbh9/+0Ra82xbY7r6rPrLreuN8W6sdfIUtaONT\n4cYaNfPyQnjpJPj0l7bQG/s3m5zqEp8GnYfbz+8waN/5g34Gs1+Ek++3sfY9x8ZWnANJbW0LcNMP\ndvvEJlW/r9A5TDTFuZtyn7PsAPr719oC7ewnq78LgMR0uHmG7W6a/U87iF0la77djj1G277+1K62\nlTHAOSdp80y7T7g91YUwQM+TbetxyQe2e+fzX9vvOD6tepmF79h1n/w7+/9UlsC/zrKFes9TbDI7\n4gz73vmv299T79PtPjLmEVsxW/gWdBpql934gx1L6X4CXPOpbWXMHG9bgP4KG/vuTXa/qkrMXY6p\n3ha9x9gH2HVsmGaT6sBL7LRx79gWsjfejm1WiUmwyW3uy9D95NrfRQRpwmhAjwz7RazNKWZQ59Qo\nR+M47RHIXQldhtvXcSnwixp98QGfLZRr/kDrMuBiOxj67hW2NtV3rO1fro/LbX/sVY4409b+BlxS\ne7k2PeHWH+3AXsBvC56YBDvv2hr99SLQ+zRY/l9bs9++0CaNzFFw8gM24bxwrBPvJbYgGHyFLdR+\nessWap/eZhPSxhm2X3vEjbZQrPn/u1w2hoVv2wJ10GV2/lFX2oLBm2ALhqr3jLrbFvIvn2xbUOeN\nt7X41K62UP7sV7aGd/qfbSG6fbFtTYG9yVbVdzP06urWTZX0XrVfxybbmuOX99vXvU6zLaHpT0Hu\nCptAb/yu7gIebGK6yhkQPfLc2gVkXa79n+0yqkuno20rMbWbfd33HFv7XvW5HYOpOgQ4a67tYkpq\nZ1s5VedgpHS2f/ucbRNG9hIY+6y9l8zeYhJsQh15p22NdBtp+/S3LbB99PHO7234jfYcpkXv2sJ8\nxzK7b4Dt4qxy5Lk2YSx8C1K62iT26e1w4QS7jQN+e+WF3qfbZAG2u3XcOzYpD7zUTqsqwDd9b/+P\n0x6x3/vR19mWw7THbHdylcy77d8eo+1jyOV2Xzn+Lrv95rxsKwWJGZDeB44cW/e2P+oaG+Oou2us\n+/i6lwUvP/E0AAAerklEQVQ4+lqbmI48N/QyTUzMQX5tk5qGDRtm5s2b16Tr3La7jJGPfctD5/bj\n+uO7N/yGg4kxtnY35QFb075tNrTqsH/r2PYTzHwBzv+HbTE0Rnmh7YJI7bLvvKJsW7vscoztPqny\n2a9swkjtBqU7beuhywh7tI/bW/fnVJZCZXF1YQG2C6Pqx3/e+H0L9y8fsF1cPU+2/csXv2r7+d+/\nDs4fX7smv+YrO1B7xl9tLPvDVw7/GGFr6NdOgjcvqO4mO/H/bOskGoyBvw+yR3UF/baS0f8i20VT\nWfNwcwEM3LOsOmlMGGWT19WfVremGpI13x4FNuYRW+CCrQC9eaFNUr1Pt5WQ6z6zlYqAD/7czh4Y\ncvk79jMBRv/OJqQpf7D71biJNpl884gdj2uoz3/8CNs1d8ajdhyoSslO+x23PdI+3zLH7gfJ7cP7\n/5razrW2lRXu9q2DiMw3xgxreElNGGE5/rFvGdQ5hRevOrrJ190ibF/Mnm6cg8W2hbYbBmw3Wp+z\nGr+udy6zRwbdtcgOzNdUusseguwrt4X2yDtsK8SYhltw+6u8wPaTe2LskU/ZS+wx/P0vCJ0Em8OM\np21Xy8kP2K4SEZvkC7fZWnz2Etu3n9zBtu6qVBTbuD2xodddl9zV9nuoWQEp2WlbAQVb7Djb1Z/Y\n854Axg+3R9td/Ar81anw3PC1beVtngUf/Nx2PflKbYv40jca/u6m/N52Jd7yvT0R9xCmCaOJ3fve\nQqavyWXuA6chTV1IqMYxBl4fa2uzF4Vxccf65K2z4xPH3FT3/JyVtvCq2V+uWo7c1bZrKaUTPOMc\nGfd/6+0YB9jDt/99rk0av5xlD+BoSNEO25IZ/oumrxi0MPuTMHQMIwzDu7fmo5+2smFnyZ4xDRVl\nIvb+6k1R4WnTs/Yhj3tr2/fAP0NFTsYR1c/7nWdbNe4aRVtaN7j1B3vgQDjJAiC5Xe3WkgI0YYRl\neGZrAOZu3KUJo6U5xGt/aj+d+Wjd02OT7UMdkIidhyEir4lIjojUeylJERkuIn4RuaTGtHtEZJmI\nLBWRiSISF6k4w9EzI5E2iTHM3nAQXEZZKaUiJJIn7r0O1Hsogoi4gceBKTWmdQLuBIYZYwYAbmBc\n5MJsmIhwVNdUlmTp7VaVUoeviCUMY8x0oKEq+R3Ah8Det6HyAPEi4gESgG1NH+H+6dcxhXW5xZRV\nRuDyx0opdRCI2qVBnJbEhcCLNacbY7YCTwGbge1AgTFmyr5r2LOem0RknojMy83NjVi8/Tq0Imhg\n1Y56rlGjlFKHsGheS+pZ4LfGmFrXLhaRNOB8oDvQEUgUkatCrcQY85IxZpgxZlhGRphHQDRC/472\nkhfLtmm3lFLq8BTNo6SGAe865zWkA2eLiB/wAhuMMbkAIvIRMBII43ZUkdM5LZ5WcR6WbSuMZhhK\nKRU1UUsYxpg919kQkdeB/xljPhGRY4BjRSQBKANOBZr+bLz9JCL069hKE4ZS6rAVycNqJwIzgT4i\nkiUiN4jILSJyS33vM8bMBj4AFgBLnBjDvCB/ZPXvmMLK7YX4A5G/FaJSSrU0EWthGGMu349lr9vr\n9UPAQ00d04Hq16EVFf4gz3+7lquO7UZG8n5eI0cppQ5iegOl/XBC73QGdU7h79+s4eY3o95LppRS\nzUoTxn5o2yqOSbeP4q5Te/PTlt3kl1Q2/CallDpEaMJohBN6p2MMzFqfF+1QlFKq2WjCaITBXVJJ\niHHz4zpNGEqpw4cmjEbwul2M6N6aH9btjHYoSinVbDRhNNLxPdNZn1tCdkF5tENRSqlmoQmjkUb1\nTgdgyvLsKEeilFLNQxNGI/Vtn0z/jq14Z/ZmDqXb3CqlVCiaMBpJRLjymG6szC5iwebd0Q5HKaUi\nThPGAThvSEeSYj28M3tztENRSqmI04RxAJJiPZw7uCOfL9lOaaU/2uEopVREacI4QBcM6UiZL8BX\ny3dEOxSllIooTRgHaHhmazqkxDFpYdTvIquUUhGlCeMAuVzCeYM7Mm11rl5bSil1SNOE0QTOHdwR\nf9DwxTI9J0MpdejShNEE+ndsRdfWCUzRhKGUOoRpwmgCIsLp/drxw9o8isp90Q5HKaUiQhNGEzlj\nQHsqA0GmrsoFILugnB/W6sUJlVKHDk0YTWRo1zTaJMbw5bJsAkHDzW/N59rX5lBQqi0OpdShIWL3\n9D7cuF3COYM68NasTfgCQRZtsZcLmbo6h/OHdIpydEopdeAi1sIQkddEJEdEljaw3HAR8YvIJc7r\nPiKysMajUETujlScTen+s47khN4ZfLlsB8f1aEN6Uoye0KeUOmREskvqdeDM+hYQETfwODClapox\nZpUxZogxZghwNFAKfBzBOJtMfIybl68ZxsPn9uNvlw3h1L7tmLYql7U5xXy3Kifa4Sml1AGJWMIw\nxkwHdjWw2B3Ah0Co0vRUYJ0xZlNTxhZJMR4X1x3fnfYpcYzp146iCj9nPDud6/81l015JdEOTyml\nGi1qg94i0gm4EHixnsXGARObJ6KmN6p3Oh1S4jiuRxsA/rtILx+ilDp4RfMoqWeB3xpjgnXNFJEY\n4Dzg/fpWIiI3icg8EZmXm5sbgTAbL87r5sf7TuGtXxzDsG5pTNKEoZQ6iEUzYQwD3hWRjcAlwAsi\nckGN+WcBC4wx9Y4aG2NeMsYMM8YMy8jIiFy0jSQiAJw/pCOrdxSzMrswyhEppVTjRC1hGGO6G2My\njTGZwAfAL40xn9RY5HIO4u6ovZ09sANul/DgJ8vYXlC2z/xg0LBgc77e7lUp1WJF8rDaicBMoI+I\nZInIDSJyi4jcEsZ7E4ExwEeRiq+5tUmK5YmLB7F0WwFn/G36Pt1Tb8/ZzEUv/FjrMNys/FI27tSB\ncqVUyxCxE/eMMZfvx7LX7fW6BGjT1DFF28VHd+bobmnc85+F3DnxJ5ZvK+S+s/oSCBpembEegOe/\nXcuYfu0wBq7711wKy3x89+vRJMbqOZZKqejSUqiZZaYn8v7Nx/GHT5cyYdo6BndOQUTYlFfKKX3b\n8u3KHKatzqXcF2RtTjEA/5y+nnvHHBHlyJVShzs5lPrMhw0bZubNmxftMMJS6Q9y6T9nsmRrAW4R\n2raK5et7T+LUp6dRGQiSHOfBHzAM6NSKb1fm8N2vR9MhJT7aYSulDjEiMt8YMyycZfXig1ES43Hx\n0tVHc9OJPbh8RBeevGQwcV43r1w7jJR4L+tzS7jpxB7cf9aR+AOGCVPXAXZwXCmlokG7pKKobas4\nfntm31rTjuzQiv/ePorv1+7klL5tcbuEi4d2ZuLcLQzuksrDk5Zx80k9ue3kXo36zC27SnnumzX8\n+ow+tGsV1xT/hlLqMKEtjBYoPsbNmH7tcLvsORy3ju6JPxDk3v8sIhA0PPnlKp79evWe5X2BOs99\n3MMfCJKVXwrA01NW8f78LG5/Z0GD71P7p6jcx8Uv/shC50rFh6PiCj+Vft2vDlXawjgIZKYncvWx\n3ViweTevXjeMJ75YxbNfr6GwzM/SbQUs2JTP8MzWjB3cgTaJMfzju3WcN7gjN57Yg/fmbub5b9eS\nlV/Gnaf2ZtKibQzuksrcjfk889XqfVo4h5KNO0vo1iZhz8mTkTZr/S7mb8rnvbmbGdIltVk+s6U5\nb/z3jOzZhj9fMDDaoagI0IRxkHj4vP6APXP8iYsH4RbhtR82kBTrYdyILsxav4sHPrZXkk+McfOX\nz1ewYHM+k5dmM7RrKp1S43numzXEeFy8fPXRPP7FKl6dsYGrj+1Gx9TGDaaX+wLEelz7FMh5xRXE\net0kRfFQ4Hkbd3HJhJk8c+lgLhrauVk+c+5Ge63Nb1bkEAwaXK7mSVQtxbbdZazPLWFnUQUPju1P\njEc7MA41mjAOEjULZZdLePSigRzXsw3DMtPonJaAMYbFWQVs3V3GiUdkcOXLs5i8NJtLh3Xm0YsG\nUekP8qv3FzKwUyptW8Vxz5jeTFq0lWe+Wk339ERcItx0Yo893WBVSiv9zN+Uz6DOqaTEe/dM/25l\nDne++xNn9G/Pk5cM2hNfSYWfc577HrdL+PfPh9OrbfKe9xSU+cgvqSQ9ObbeZFLpD/LQpKWcP6QT\nx/bY93ScnKJyZqzeyQVHddon3irvzNkMwCszNjCie2vueW8h947pw3E9I3d6z9yNu3C7hJyiCpZu\nK2BQ56ZvZbw8fT1bd5ftqUC0JD9ttl1xheV+fli7k5P7to1yRI2zKruI9ilxtfb3lqSsMkB8jDsq\nn60J4yDlcgkXHFV9Jz8RYXCXVAY7XSGvXTecH9flcc7ADrhcQnyMmxeuPHrP8p3TEhg3vCtvzqq+\ncvyMNbm0SYolGDQM7JzCiu2FfLMih+IKP93TE/nd2Ufy0+Z8FmXt5sd1eaQlxPDB/CwGd07h6uMy\nAZgwbR3ZheWkJXi5+MWZvHvTsSTHefj9J0v5fs1O/EGDCIwd1JG7Tu21J6HsLq3k1+8v5qKhnViz\no5iJc7bw47o8vrrnpFo11aVbC7jxjXlsLyintNK/53NrKiz38fmS7bRvFcfy7YVc+cpsNuWV8sAn\nS/jy7hPxupu+5ltWGWBJVgE/O7oz/5m3ha9X5NSbMIwx7C71kZYYE/ZnzFyXx18nr8AYuHRYF/p1\nbNUUoe+J50C77hZszifW4yLW4+KzJdubPWEUlftIjjuwQj6/pJLz//E95wzsyNOXDg7rPZX+IAZD\nrCfyhfgrM9bz6OSV/OOKoZw5oH3EP29veh7GYSy3qIInv1zJhUd1Zl1uMX/5bAWtE2MwxrCtoJw2\niTGc3LctI7q35rHJK9lVUonHJRzZoRXH9mjNXacdwZ0Tf+LblTkMz0yjf8cUJs7ZzJkD2vOrMX24\n9J8zMRg8LheF5T6uPKYbvdsmsTK7kLdmbabMF2BAp1bcfnIv3p+XxTcrcxABj0vokZ7Eqh1FPHJe\nf64dmcmybQU89eUqpq7OpX2rONq1imNdTjHf/Pok2ibXPtrrzVmb+MMnS3nvpmO5+a357C71ceFR\nnfj4p63cdWpvzhnUgV0llZT7AnRPTyQp1oOIUOYLMP7btUxfncvwzDSGdkujW5tEMtsk0DktIWRr\nBmxhfvnLs3jtumG8OHUdO4sr+fiXI0lNqE4IS7IKmLl+J9eN7M5TU1bx0vT1DOmSyklHZDCwUwoD\nO6fUeeTaj+t28tXyHUxekk2s10VOYQVjB3XgyZ+FLtB8gSB/+WwFI7q35uyBHerdD2atz+PGN+Zx\n/chM7ji1d6MT6kUv/IDbJXRtnciU5dn8eN8pJMd58QWCTF1lv7eBnVPCWtd3q3KYsiybP4ztR0JM\nw/XaTxdu5e73FnL/WX256cSejYofbIH8589WEO91M/f3pzXYrWqM4ZrX5pBdUM6k20c1uuZfWO4j\n3uuud9s/89VqnvtmDW6X0KddMp/dOapJxuf25zwMTRhqj6p+d2MMeSWVtE6I2dMPn11QzpKtBYzo\n3rpWU72kws/rP27kv4u2sW13GUmxHt6/dSSdUuNZmV3Iz16ciQi8/YtjaxUWuUUVfLpwK+/N3cIa\n54z2+87qy5wNu1icVcDku07g9ncWsGRrASN7tmHqqlxS4r1ceWw3rjmuG0Xl9sZU6YkxnHpkO1on\nxhAf4ya7oJy3Z2+ib/tWTLr9eCYt2sa6nGLuGXME1/5rLtNX138JfLdLOLF3OouyCthVUrlnenpS\nLGP6taVnRhKd0+LpmBpPm6RYZxvBo5+v5N8zN7LwwdNZsCmfm9+cT6e0eO4/qy992ifzzYocHpu8\nkspAkD7tklm1o4jRfTLYWVzB8m2FVJ1eU5UwPS4XwzLTOKN/ex6etAyv20WntHie+tlgPpyfxXtz\nt/D8FUeRGu+l1BegY0o8HVLjKPcFSI2P4c+fLeeNmZtwCVw7MpMvl2bj9bg4Z2AHbjyhx56WTbkv\nwNl/n0F2YTmllQGGdEnl7+OG0K1NIoGgYcX2QjKdpFqfCn+AgQ9N4brjMzm9Xzsue2kWfdolc1Kf\nDN6fl8XO4gqS4zx8fucJGAO5xeUM7JRa5zhHQamPU56eSl5JJcf2aM1z446ibR2J9KMFWbw0fT0P\nntuPe95byO5SHxX+ID87ujPXjsxkQKfwklMVYwynPTONwnI/uUUVPHHxIM7o3x6vR0ImrR/X7uSK\nV2YDcN3IzJBdhcUVfrILyumZkbhPIZ9dUM7Y52eQkRzH69cPr7PS8MH8LH79/iIuHdaZo7qmcf9H\nS3jzhhGc0PvAr9CtCUO1GBt3luBxC53TEuqc7w8EmThnM/mlPu44xZ5bUuEPEud1s2VXKc98tZpZ\n6/M4pntrHj6vf60a+9RVObwxcxOz1udRWhkAbIF7ydDO/ObMvmQkx9b6rHJfgJnr8ygu99M6MQav\n28XGvBLKfYE9J0SO6p1Or7bJGGPILa5gU14pG3JLmLY6l2mrcymu8O/zP7gEggZO6J3OmzccA9jx\njF++vYDcooo9y43qlc7p/dvxx/8uZ1hmGm/ecAxet4vSSj/LtxWyZKtNUv6goawywIfzsyiq8DO4\ncwpv/uIYWjndLRt2ljD2uRmUOP/z3twuIRA0XDcyk1XZRcxcn8fQrqkkxXn5Ye1OkuM8nNKnLR63\nsL2gnBlrdvLGz0dQUObjgY+X4AsY+nVsxdb8MrILy4lxuxjRvTUDOqWQnmQTc0KMm3ivm1ivG49L\nyMov4/6PljDhqqGcOaADU1fl8Mu3F1DuC3BK33acNaA9D/93Ga3ivOQUleMLGBJi3Jw/pCMdU+L5\nemUObZNjGdQphZU7ipi8ZDu3n9yL8d+tJWigS+t4LjyqMyMyWxPrdfHD2p08+/UaPC7B73x3799y\nHF8szebNWZuo9Ae5YEhHbhjVg8JyH3kllazOLmJR1m66tE6gb/tk4rxuOqTE0SrOy/aCcpZvL+S5\nb9bwxCWDmDBtHSUVfvJLfCTEurnh+O6M6d+O3m2T97Q0g0HDuJdmsWlXCace2Y53Zm+mQ0ocXVon\ncO6gDhzRLhmXS1iSVcALU9eys7iSbm0SGNIlFWNgzoZd9O/YivzSSlZmFyFASryXv140kNF9bHee\nMYb352Xx+0+WMrx7Gq9fP4KgMZzw+He0TozhLxcOoKjcz/aCci4f0bWeX2JomjDUYScYNJT7AxhD\nxC7UaIyhoMxHVn4Z23aXsaukkl2llRSX+xnevTUje7ap1Y/tCwSZtT6PrPwy+ndsxYCOKbhcQlZ+\nKelJscR56+++yC4o5+OftnLFiK6kJNTum88vqWRDXglllQHivC6y8svIKawgzusip6iChBgPN53Y\nA18gyMrsoj3XLFuZXchjk1eyZkcxQWMIBA0XHtWJ+88+ErBHOo3/bi0bcktIjPVwev92rM4u4sd1\neazeUbSncK6L2yXMvO+UPa2BrbvL8LhkT435i6Xbue2dnzh/SEdOO7Id363MYdKibVT4gwzpkkph\nmY/1ztWZq2rry7cVMnN9HlNX5fD92p3ULK5OO7ItD47tz23vLGBg5xT+eqE9lLeg1Mer36/nhanr\nasXrdgm92yaxNb+MojoSP0DvtklMun0Ub83axF8+X8G5gztSXO7ju1W2ZepxCelJsbhdwq6SSsp8\nAR45rz+XDe/Ci1PXsb2gjJ82797Taq4yrFsaYwd1YOrqXNblFlPhCzK0axo/rNtJUbmfZy8bQq+2\nSdw58SfW7yyhTWIMBoj1uNheUM5xPdrw4lVD91SYpizL5rcfLia/1AdAaoKX+b8fU2+3aSiaMJRS\nTc4fCFLqC1BeGaC0MkCZL0C5L0AgaPAHDWkJMfRpn1zvOsp9gVqJsqDUR5kvQPsUm1QKy32s2VHM\nwE4p+3RX5RSWs2FnCaW+AL2crsGq7p26Bu3X5hSzekcRrRNjaJMYQ8fUeBJjPQSCZs8Y1tbdZRSV\n++mQEkeHlDhaJ8YgIgSDhqz8Mrq2sS3jrPxSZq/fxfqdxeQUVhA0tjXQv2OrfY7WM8awYWcJ23aX\n4w8G6dI6gR7p+3ZFgU38q3YU7TkasMIf4K1Zm1mfW4wIlFQEGNQ5hWuOy9wnGRQ5B3d0Sk3g6G5p\njR4/0YShlFIqLHrxQaWUUk1OE4ZSSqmwaMJQSikVFk0YSimlwqIJQymlVFg0YSillAqLJgyllFJh\n0YShlFIqLIfUiXsikgtsanDBuqUDO5swnKaice2/lhqbxrV/NK7915jYuhljwrqK4SGVMA6EiMwL\n92zH5qRx7b+WGpvGtX80rv0X6di0S0oppVRYNGEopZQKiyaMai9FO4AQNK7911Jj07j2j8a1/yIa\nm45hKKWUCou2MJRSSoXlsE8YInKmiKwSkbUicl8U4+giIt+JyHIRWSYidznTHxaRrSKy0HmcHaX4\nNorIEieGec601iLylYiscf6mNXNMfWpsl4UiUigid0djm4nIayKSIyJLa0wLuX1E5H5nn1slImdE\nIbYnRWSliCwWkY9FJNWZnikiZTW23YRmjivkd9dc2yxEXO/ViGmjiCx0pjfn9gpVRjTffmaMOWwf\ngBtYB/QAYoBFQL8oxdIBGOo8TwZWA/2Ah4Fft4BttRFI32vaE8B9zvP7gMej/F1mA92isc2AE4Gh\nwNKGto/zvS4CYoHuzj7obubYTgc8zvPHa8SWWXO5KGyzOr+75txmdcW11/yngQejsL1ClRHNtp8d\n7i2MEcBaY8x6Y0wl8C5wfjQCMcZsN8YscJ4XASuATtGIZT+cD/zbef5v4IIoxnIqsM4Y09gTNw+I\nMWY6sGuvyaG2z/nAu8aYCmPMBmAtdl9sttiMMVOMMVU3tp4FdI7U5+9PXPVotm1WX1xi77N6KTAx\nEp9dn3rKiGbbzw73hNEJ2FLjdRYtoJAWkUzgKGC2M+kOp+vgtebu9qnBAF+LyHwRucmZ1s4Ys915\nng20i05oAIyj9o+4JWyzUNunpe13Pwcm13jd3elemSYiJ0Qhnrq+u5ayzU4Adhhj1tSY1uzba68y\notn2s8M9YbQ4IpIEfAjcbYwpBF7EdpkNAbZjm8PRMMoYMwQ4C7hNRE6sOdPYNnBUDrkTkRjgPOB9\nZ1JL2WZ7RHP71EdEHgD8wNvOpO1AV+e7vhd4R0RaNWNILe6728vl1K6YNPv2qqOM2CPS+9nhnjC2\nAl1qvO7sTIsKEfFid4S3jTEfARhjdhhjAsaYIPAyEey6qI8xZqvzNwf42Iljh4h0cGLvAOREIzZs\nEltgjNnhxNgithmht0+L2O9E5DpgLHClU9DgdF/kOc/nY/u9j2iumOr57qK+zUTEA1wEvFc1rbm3\nV11lBM24nx3uCWMu0FtEuju11HHApGgE4vSNvgqsMMY8U2N6hxqLXQgs3fu9zRBboogkVz3HDpgu\nxW6ra53FrgU+be7YHLVqfS1hmzlCbZ9JwDgRiRWR7kBvYE5zBiYiZwK/Ac4zxpTWmJ4hIm7neQ8n\ntvXNGFeo7y7q2ww4DVhpjMmqmtCc2ytUGUFz7mfNMbrfkh/A2dijDdYBD0QxjlHYpuRiYKHzOBt4\nE1jiTJ8EdIhCbD2wR1ssApZVbSegDfANsAb4GmgdhdgSgTwgpca0Zt9m2IS1HfBh+4pvqG/7AA84\n+9wq4KwoxLYW279dta9NcJa92PmOFwILgHObOa6Q311zbbO64nKmvw7csteyzbm9QpURzbaf6Zne\nSimlwnK4d0kppZQKkyYMpZRSYdGEoZRSKiyaMJRSSoVFE4ZSSqmwaMJQyiEixc7fTBG5oonX/bu9\nXv/YlOtXqjlowlBqX5nAfiUM5yzg+tRKGMaYkfsZk1JRpwlDqX09BpzgXFDuHhFxO/ePmOtcFO9m\nABEZLSIzRGQSsNyZ9olzgcZlVRdpFJHHgHhnfW8706paM+Kse6nY+41cVmPdU0XkA7H3rXjbOdMX\nEXnMuSfCYhF5qtm3jjpsNVQrUupwdB/2ngxjAZyCv8AYM1xEYoEfRGSKs+xQYICxl48G+LkxZpeI\nxANzReRDY8x9InK7sReo29tF2AvtDQbSnfdMd+YdBfQHtgE/AMeLyArsJTP6GmOMODc+Uqo5aAtD\nqYadDlwj9i5rs7GXYujtzJtTI1kA3Ckii7D3mOhSY7lQRgETjb3g3g5gGjC8xrqzjL0Q30JsV1kB\nUA68KiIXAaV1rFOpiNCEoVTDBLjDGDPEeXQ3xlS1MEr2LCQyGnuBuuOMMYOBn4C4A/jcihrPA9g7\n5PmxV3D9AHul2S8OYP1K7RdNGErtqwh7C8wqXwK3OpeWRkSOcK7au7cUIN8YUyoifYFja8zzVb1/\nLzOAy5xxkgzs7UFDXlHUuRdCijHmc+AebFeWUs1CxzCU2tdiIOB0Lb0O/B3bHbTAGXjOpe7b0X4B\n3OKMM6zCdktVeQlYLCILjDFX1pj+MXAc9krABviNMSbbSTh1SQY+FZE4bMvn3sb9i0rtP71arVJK\nqbBol5RSSqmwaMJQSikVFk0YSimlwqIJQymlVFg0YSillAqLJgyllFJh0YShlFIqLJowlFJKheX/\nAWBeV6AZaz8sAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fa991ef95c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy: 0.9963\n",
      "Test Accuracy: 0.9822\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    \n",
    "    train_costs = []\n",
    "    test_costs = []\n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        for i in range(0, 50000, batch_size):\n",
    "            sess.run(optimizer, feed_dict={x:X_train_flatten[i:i+batch_size], y:y_train[i: i+batch_size]})\n",
    "        train_costs.append(sess.run(cost, feed_dict={x:X_train_flatten, y:y_train}))\n",
    "        test_costs.append(sess.run(cost, feed_dict={x:X_test_flatten, y:y_test}))\n",
    "        if epoch%10 == 9:\n",
    "            print(\"Cost after \" + str(epoch+1)+ \" epochs: \"+ str(train_costs[-1]))\n",
    "    \n",
    "    iterations = list(range(num_epochs))\n",
    "    plt.plot(iterations, train_costs, label='Train')\n",
    "    plt.plot(iterations, test_costs, label='Test')\n",
    "    plt.ylabel('Cost')\n",
    "    plt.xlabel('Iterations')\n",
    "    plt.show()\n",
    "    \n",
    "    # Predictions\n",
    "    predict_op = tf.argmax(Z2, 1)\n",
    "    correct_prediction = tf.equal(predict_op, tf.argmax(y, 1))\n",
    "\n",
    "    # Accuracy calculation\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
    "\n",
    "    print (\"Train Accuracy:\", accuracy.eval({x: X_train_flatten, y: y_train}))\n",
    "    print (\"Test Accuracy:\", accuracy.eval({x: X_test_flatten, y: y_test}))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}