Convolution/MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convolutional Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jsh/anaconda3/lib/python3.6/site-packages/urllib3/contrib/pyopenssl.py:46: DeprecationWarning: OpenSSL.rand is deprecated - you should use os.urandom instead\n",
      "  import OpenSSL.SSL\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import mxnet as mx\n",
    "from mxnet import nd, autograd, gluon\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "mx.random.seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ctx = mx.cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_inputs = 784\n",
    "num_outputs = 10\n",
    "batch_size = 64\n",
    "num_fc = 512\n",
    "\n",
    "# (height, width, channel) -> (channel, height, width)\n",
    "def transform(data, label):\n",
    "    return nd.transpose(data.astype(np.float32), (2,0,1)) / 255, label.astype(np.float32)\n",
    "\n",
    "train_data = gluon.data.DataLoader(gluon.data.vision.MNIST(train = True, transform = transform),\n",
    "                                      batch_size, shuffle = True)\n",
    "test_data = gluon.data.DataLoader(gluon.data.vision.MNIST(train = False, transform = transform),\n",
    "                                      batch_size, shuffle = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class CNN(gluon.Block):\n",
    "    def __init__(self, num_fc, num_outputs, **kwargs):\n",
    "        super(CNN, self).__init__(**kwargs)\n",
    "        \n",
    "        with self.name_scope():\n",
    "            self.conv0 = gluon.nn.Conv2D(channels=4, kernel_size=5, activation='relu')\n",
    "            self.max0 = gluon.nn.MaxPool2D(pool_size=2, strides=2)\n",
    "            self.conv1 = gluon.nn.Conv2D(channels=8, kernel_size=5, activation='relu')\n",
    "            self.max1 = gluon.nn.MaxPool2D(pool_size=2, strides=2)\n",
    "            self.flat = gluon.nn.Flatten()\n",
    "            self.dense0 = gluon.nn.Dense(num_fc, activation=\"relu\")\n",
    "            self.dense1 = gluon.nn.Dense(num_outputs)\n",
    "            \n",
    "    def forward(self, x):\n",
    "        x = self.conv0(x)\n",
    "        x = self.max0(x)\n",
    "        x = self.conv1(x)\n",
    "        x = self.max1(x)\n",
    "        x = self.flat(x)\n",
    "        x = self.dense0(x)\n",
    "        x = self.dense1(x)\n",
    "        return x    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN(\n",
      "  (conv0): Conv2D(None -> 4, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (max0): MaxPool2D(size=(2, 2), stride=(2, 2), padding=(0, 0), ceil_mode=False)\n",
      "  (conv1): Conv2D(None -> 8, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (max1): MaxPool2D(size=(2, 2), stride=(2, 2), padding=(0, 0), ceil_mode=False)\n",
      "  (flat): Flatten\n",
      "  (dense0): Dense(None -> 512, Activation(relu))\n",
      "  (dense1): Dense(None -> 10, linear)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "net = CNN(num_fc=num_fc, num_outputs=num_outputs)\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "net.collect_params().initialize(mx.init.Xavier(), ctx = ctx)\n",
    "softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()\n",
    "trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for data, label in test_data:\n",
    "    data = data.as_in_context(ctx)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN(\n",
      "  (conv0): Conv2D(None -> 4, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (max0): MaxPool2D(size=(2, 2), stride=(2, 2), padding=(0, 0), ceil_mode=False)\n",
      "  (conv1): Conv2D(None -> 8, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (max1): MaxPool2D(size=(2, 2), stride=(2, 2), padding=(0, 0), ceil_mode=False)\n",
      "  (flat): Flatten\n",
      "  (dense0): Dense(None -> 512, Activation(relu))\n",
      "  (dense1): Dense(None -> 10, linear)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate_accuracy(dataset, net):\n",
    "    acc = mx.metric.Accuracy()\n",
    "    for data, label in dataset:\n",
    "        data = data.as_in_context(ctx)\n",
    "        label = label.as_in_context(ctx)\n",
    "        output = net(data)\n",
    "        pred = nd.argmax(output, axis=1)\n",
    "        acc.update(preds = pred, labels = label)\n",
    "    return acc.get()[1] # ('accuracy', 0.0953)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0. Loss: 0.10619983041, Train_acc 0.959733333333, Test_acc 0.9606\n",
      "Epoch 1. Loss: 0.0641632130318, Train_acc 0.985, Test_acc 0.9824\n",
      "Epoch 2. Loss: 0.0518702731358, Train_acc 0.9838, Test_acc 0.981\n",
      "Epoch 3. Loss: 0.0413339389434, Train_acc 0.988066666667, Test_acc 0.9847\n",
      "Epoch 4. Loss: 0.0319808226525, Train_acc 0.98935, Test_acc 0.9828\n",
      "--- 162.86263132095337 seconds ---\n"
     ]
    }
   ],
   "source": [
    "epochs = 5\n",
    "smoothing_constant = .01\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "for i in range(epochs):\n",
    "    for j, (data, label) in enumerate(train_data):\n",
    "        data = data.as_in_context(ctx)\n",
    "        label = label.as_in_context(ctx)\n",
    "        with autograd.record():\n",
    "            output = net(data)\n",
    "            loss = softmax_cross_entropy(output, label)\n",
    "        loss.backward()\n",
    "        trainer.step(data.shape[0])\n",
    "\n",
    "        ##########################\n",
    "        #  Keep a moving average of the losses\n",
    "        ##########################\n",
    "        curr_loss = nd.mean(loss).asscalar()\n",
    "        moving_loss = (curr_loss if ((i == 0) and (j == 0))\n",
    "                       else (1 - smoothing_constant) * moving_loss + smoothing_constant * curr_loss)\n",
    "\n",
    "    train_accuracy = evaluate_accuracy(train_data, net)    \n",
    "    test_accuracy = evaluate_accuracy(test_data, net)\n",
    "    print(\"Epoch %s. Loss: %s, Train_acc %s, Test_acc %s\" % (i, moving_loss, train_accuracy, test_accuracy))\n",
    "    \n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### HybridBlock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class CNN_hybrid(gluon.HybridBlock):\n",
    "    def __init__(self, num_fc, num_outputs, **kwargs):\n",
    "        super(CNN_hybrid, self).__init__(**kwargs)\n",
    "        \n",
    "        with self.name_scope():\n",
    "            self.conv0 = gluon.nn.Conv2D(channels=4, kernel_size=5)\n",
    "            self.max0 = gluon.nn.MaxPool2D(pool_size=2, strides=2)\n",
    "            self.conv1 = gluon.nn.Conv2D(channels=16, kernel_size=5)\n",
    "            self.max1 = gluon.nn.MaxPool2D(pool_size=2, strides=2)\n",
    "            self.flat = gluon.nn.Flatten()\n",
    "            self.dense0 = gluon.nn.Dense(num_fc)\n",
    "            self.dense1 = gluon.nn.Dense(num_outputs)\n",
    "            \n",
    "    def hybrid_forward(self, F, x):\n",
    "        x = self.max0(F.relu(self.conv0(x)))\n",
    "        x = self.max1(F.relu(self.conv1(x)))        \n",
    "        x = self.flat(x)\n",
    "        x = F.relu(self.dense0(x))\n",
    "        x = self.dense1(x)\n",
    "        return x    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN_hybrid(\n",
      "  (conv0): Conv2D(None -> 4, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (max0): MaxPool2D(size=(2, 2), stride=(2, 2), padding=(0, 0), ceil_mode=False)\n",
      "  (conv1): Conv2D(None -> 16, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (max1): MaxPool2D(size=(2, 2), stride=(2, 2), padding=(0, 0), ceil_mode=False)\n",
      "  (flat): Flatten\n",
      "  (dense0): Dense(None -> 512, linear)\n",
      "  (dense1): Dense(None -> 10, linear)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "hnet = CNN_hybrid(num_fc=num_fc, num_outputs=num_outputs)\n",
    "print(hnet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hnet.collect_params().initialize(mx.init.Xavier(), ctx = ctx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hnet.hybridize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trainerH = gluon.Trainer(hnet.collect_params(), 'sgd', {'learning_rate': 0.1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### To symbolic..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize activation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "img = nd.transpose(data[0:1][0], (1,2,0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAADXtJREFUeJzt3X+oVHUax/HPUymmVvSDlVu2qyu1\nEtFWXGwDyZZdy43CDImKysq7t8BgA//YaP/YaCli2VqSSLC0LNp+yO2H2E8V2ZK28JaVWZpWRpqp\nYVlRYOqzf8yxbnbnO+PMmTnn+rxfcLkz5zln5nHwc8+Z+Z4zX3N3AYjnoKIbAFAMwg8ERfiBoAg/\nEBThB4Ii/EBQhB8IivADQRF+IKhD2vlkZsbphECLubvVs15Te34zm2Rma81svZnd2MxjAWgva/Tc\nfjM7WNL7kiZK2ihphaRL3f3dxDbs+YEWa8eef5yk9e7+obvvlPSopMlNPB6ANmom/MdJ+qTP/Y3Z\nsp8ws24z6zWz3iaeC0DOWv6Bn7vPkTRH4rAfKJNm9vybJB3f5/7IbBmAAaCZ8K+QdIKZjTazwZIu\nkbQwn7YAtFrDh/3uvsvMrpf0gqSDJc1z99W5dQagpRoe6mvoyXjPD7RcW07yATBwEX4gKMIPBEX4\ngaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF\n+IGgCD8QVFun6MbAM378+GR96dKlyfrll19etbZgwYKGekI+2PMDQRF+ICjCDwRF+IGgCD8QFOEH\ngiL8QFBNzdJrZhskfS1pt6Rd7t5ZY31m6S2ZIUOGJOurV6dnXX/++eeT9RkzZux3T2hOvbP05nGS\nz+/d/fMcHgdAG3HYDwTVbPhd0otm9rqZdefREID2aPawf7y7bzKzX0habGZr3P2lvitkfxT4wwCU\nTFN7fnfflP3eKulJSeP6WWeOu3fW+jAQQHs1HH4zG2Zmh+29LekcSe/k1RiA1mrmsH+EpCfNbO/j\n/Mfd0+M+AEqj4fC7+4eSfptjLyjAueeem6yPHj06WV+8eHGe7aCNGOoDgiL8QFCEHwiK8ANBEX4g\nKMIPBNXUJb37/WRc0tt29913X7I+derUZP3www9P1rdt25asf/vtt8l6WXV1dSXrtb6yvEj1XtLL\nnh8IivADQRF+ICjCDwRF+IGgCD8QFOEHgmKK7gNc9n0LVdUax3/mmWeS9Z6enmT90EMPrVqbOHFi\ncttmpf7ty5cvT267cuXKvNspHfb8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU1/MfAA46qPrf8Oee\ney657bHHHpusn3766cn6999/n6yj/bieH0AS4QeCIvxAUIQfCIrwA0ERfiAowg8EVfN6fjObJ+l8\nSVvd/eRs2VGSHpM0StIGSRe7+xetaxMpY8eOrVqrdc383Llzk3XG8Q9c9ez5H5A0aZ9lN0pa6u4n\nSFqa3QcwgNQMv7u/JGn7PosnS5qf3Z4v6cKc+wLQYo2+5x/h7puz259JGpFTPwDapOnv8HN3T52z\nb2bdkrqbfR4A+Wp0z7/FzDokKfu9tdqK7j7H3TvdvbPB5wLQAo2Gf6GkadntaZKezqcdAO1SM/xm\n9oik/0n6jZltNLPpkm6XNNHM1kn6Y3YfwADC9fwDwJAhQ5L1V199tWrtlFNOSW572WWXJetr1qxJ\n1j/66KNkfceOHck68sf1/ACSCD8QFOEHgiL8QFCEHwiK8ANBMdQ3AEyZMiVZrzVNditt27YtWV+1\nalXV2syZM5PbvvXWWw31FB1DfQCSCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMb5S6CrqytZv+eee5L1\nnTt3Vq3NmjUrue3atWuT9VrOOOOMZH3q1KlVa8OHD09uW+s8gNmzZyfrUTHODyCJ8ANBEX4gKMIP\nBEX4gaAIPxAU4QeCYpy/BL74Ij27+dChQ5P1SZP2nUT5R8uWLWuop7yMGTOmaq3W9OATJkxI1i+4\n4IJkfdGiRcn6gYpxfgBJhB8IivADQRF+ICjCDwRF+IGgCD8QVM1xfjObJ+l8SVvd/eRs2c2S/ixp\n75e23+Tuz9Z8Msb5+3Xbbbcl688+m35ply9fnmc7bXPWWWcl60uWLEnWv/zyy2T9zDPPrFr74IMP\nktsOZHmO8z8gqb+zSP7t7qdmPzWDD6Bcaobf3V+StL0NvQBoo2be819vZm+b2TwzOzK3jgC0RaPh\nny1pjKRTJW2WdEe1Fc2s28x6zay3wecC0AINhd/dt7j7bnffI+leSeMS685x905372y0SQD5ayj8\nZtbR5+4USe/k0w6Adjmk1gpm9oiksyUdY2YbJf1d0tlmdqokl7RB0rUt7BFAC3A9P0pr3rx5yfpV\nV12VrF900UVVa0899VQjLQ0IXM8PIInwA0ERfiAowg8ERfiBoAg/EFTNcX6gKDt27Ci6hQMae34g\nKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIpxfpRWR0dH7ZXQMPb8QFCEHwiK8ANBEX4gKMIPBEX4gaAI\nPxAU4/xoyrBhw5L1rq6uqrXrrrsuue3YsWOT9QULFiTrB/LXc+eBPT8QFOEHgiL8QFCEHwiK8ANB\nEX4gKMIPBFVznN/Mjpf0oKQRklzSHHe/y8yOkvSYpFGSNki62N2/aF2rA9fIkSOT9U8//TRZ37Nn\nT57t/MSJJ56YrF9zzTXJ+vTp05P1o48+umqt1r/71ltvTdbvvvvuZB1p9ez5d0ma6e4nSfqdpBlm\ndpKkGyUtdfcTJC3N7gMYIGqG3903u/sb2e2vJb0n6ThJkyXNz1abL+nCVjUJIH/79Z7fzEZJOk3S\na5JGuPvmrPSZKm8LAAwQdZ/bb2bDJfVIusHdvzKzH2ru7mbmVbbrltTdbKMA8lXXnt/MBqkS/Ifd\n/Yls8RYz68jqHZK29retu89x905378yjYQD5qBl+q+zi50p6z93v7FNaKGladnuapKfzbw9Aq5h7\nv0frP65gNl7Sy5JWSdo75nSTKu/7H5f0S0kfqzLUt73GY6WfbIC68sork/Xu7vS7ngkTJiTru3fv\nTtYHDx5ctXb11Vcnt73lllsafmxJWrduXbI+a9asqrWenp7ktt99912yjv65u9Veq473/O6+XFK1\nB/vD/jQFoDw4ww8IivADQRF+ICjCDwRF+IGgCD8QVM1x/lyf7AAd51+5cmWyvnPnzmT9lVdeSdYH\nDRqUrE+ePLlq7Ygjjkhue//99yfrDz30ULLe29ubrKP96h3nZ88PBEX4gaAIPxAU4QeCIvxAUIQf\nCIrwA0Exzp+DK664IlmfPXt2sj506NBkfdeuXcn6448/XrV21113JbddsWJFso6Bh3F+AEmEHwiK\n8ANBEX4gKMIPBEX4gaAIPxAU4/zAAYZxfgBJhB8IivADQRF+ICjCDwRF+IGgCD8QVM3wm9nxZrbM\nzN41s9Vm9pds+c1mtsnM3sx+zmt9uwDyUvMkHzPrkNTh7m+Y2WGSXpd0oaSLJX3j7v+q+8k4yQdo\nuXpP8jmkjgfaLGlzdvtrM3tP0nHNtQegaPv1nt/MRkk6TdJr2aLrzextM5tnZkdW2abbzHrNjHmd\ngBKp+9x+Mxsu6b+SbnX3J8xshKTPJbmkf6jy1uCaGo/BYT/QYvUe9tcVfjMbJGmRpBfc/c5+6qMk\nLXL3k2s8DuEHWiy3C3vMzCTNlfRe3+BnHwTuNUXSO/vbJIDi1PNp/3hJL0taJWlPtvgmSZdKOlWV\nw/4Nkq7NPhxMPRZ7fqDFcj3szwvhB1qP6/kBJBF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANB\nEX4gKMIPBEX4gaAIPxAU4QeCqvkFnjn7XNLHfe4fky0ro7L2Vta+JHprVJ69/areFdt6Pf/Pntys\n1907C2sgoay9lbUvid4aVVRvHPYDQRF+IKiiwz+n4OdPKWtvZe1LordGFdJboe/5ARSn6D0/gIIU\nEn4zm2Rma81svZndWEQP1ZjZBjNblc08XOgUY9k0aFvN7J0+y44ys8Vmti773e80aQX1VoqZmxMz\nSxf62pVtxuu2H/ab2cGS3pc0UdJGSSskXeru77a1kSrMbIOkTncvfEzYzM6S9I2kB/fOhmRm/5S0\n3d1vz/5wHunufy1JbzdrP2dublFv1WaWvkoFvnZ5znidhyL2/OMkrXf3D919p6RHJU0uoI/Sc/eX\nJG3fZ/FkSfOz2/NV+c/TdlV6KwV33+zub2S3v5a0d2bpQl+7RF+FKCL8x0n6pM/9jSrXlN8u6UUz\ne93Muotuph8j+syM9JmkEUU204+aMze30z4zS5fmtWtkxuu88YHfz41399Ml/UnSjOzwtpS88p6t\nTMM1syWNUWUat82S7iiymWxm6R5JN7j7V31rRb52/fRVyOtWRPg3STq+z/2R2bJScPdN2e+tkp5U\n5W1KmWzZO0lq9ntrwf38wN23uPtud98j6V4V+NplM0v3SHrY3Z/IFhf+2vXXV1GvWxHhXyHpBDMb\nbWaDJV0iaWEBffyMmQ3LPoiRmQ2TdI7KN/vwQknTstvTJD1dYC8/UZaZm6vNLK2CX7vSzXjt7m3/\nkXSeKp/4fyDpb0X0UKWvX0t6K/tZXXRvkh5R5TDwe1U+G5ku6WhJSyWtk7RE0lEl6u0hVWZzfluV\noHUU1Nt4VQ7p35b0ZvZzXtGvXaKvQl43zvADguIDPyAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivAD\nQf0fEEV8CKUeOZYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1516e3f978>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(nd.tile(img, (1,1,3)).asnumpy())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[ 5.]\n",
      "<NDArray 1 @cpu(0)> \n",
      "[ 5.]\n",
      "<NDArray 1 @cpu(0)>\n"
     ]
    }
   ],
   "source": [
    "print(label[0], nd.argmax(net(data[0:1]), axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Useful function to arrange the images to be shown as a squared grid\n",
    "def viz_grid(Xs, ubound=255.0, padding=1):\n",
    "    import math\n",
    "    N, H, W, C = Xs.shape\n",
    "    grid_size = int(math.ceil(math.sqrt(N)))\n",
    "    grid_height = H * grid_size + padding * (grid_size - 1)\n",
    "    grid_width = W * grid_size + padding * (grid_size - 1)\n",
    "    grid = np.zeros((grid_height, grid_width, C))\n",
    "    next_idx = 0\n",
    "    y0, y1 = 0, H\n",
    "    for y in range(grid_size):\n",
    "        x0, x1 = 0, W\n",
    "        for x in range(grid_size):\n",
    "            if next_idx < N:\n",
    "                img = Xs[next_idx]\n",
    "                grid[y0:y1, x0:x1] = img\n",
    "                next_idx += 1\n",
    "            x0 += W + padding\n",
    "            x1 += W + padding\n",
    "        y0 += H + padding\n",
    "        y1 += H + padding\n",
    "    return grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4, 1, 5, 5)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.conv0.weight.data().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8, 4, 5, 5)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.conv1.weight.data().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 4, 24, 24)\n"
     ]
    }
   ],
   "source": [
    "#conv0_out = nd.split(net.conv0(data[0:1].as_in_context(ctx)), num_outputs=1, axis=0, squeeze_axis=1)\n",
    "conv0_out = net.conv0(data[0:1].as_in_context(ctx))\n",
    "print(conv0_out.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 8, 8, 8)\n"
     ]
    }
   ],
   "source": [
    "#conv1_out = nd.split(net.conv1(net.max0(net.conv0(data[0:1]))), num_outputs=1, axis=0, squeeze_axis=1)\n",
    "conv1_out = net.conv1(net.max0(net.conv0(data[0:1])))\n",
    "print(conv1_out.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getActivation_mnist(layer_out):\n",
    "    outs = nd.transpose(layer_out, (1, 2, 3, 0)) # N, H, W, C\n",
    "    fig = plt.figure(figsize=(4,4))   \n",
    "    ax1 = plt.subplot(111)\n",
    "    grid = viz_grid(outs.asnumpy())\n",
    "    ax1.imshow(grid[...,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAHwRJREFUeJztnXmQXNd13r/Ty+w7lsEAGAIgQIKr\nCIirSMuWSEmhacdkKqpEikumq1jFSspJ5EiKRCUpJ65KUlSqItlZrIQVyUaqHJMy5YQUw4QmIUoO\nixJIkCBILCSxkCBADDBYZjD70t0nf0xTxrnnAd0YdPf0zP1+VVOYe3FevzvvvdO3z+mziKqCEBIX\nqYVeACGk9lDxCYkQKj4hEULFJyRCqPiERAgVn5AIoeITEiFUfEIi5LIUX0TuFZF3ROSgiDxSqUUR\nQqqLzDdyT0TSAN4F8FkAxwC8CuCLqrrvQsc0SJM2S+u8zrcgiJSWSSe8d6bTZqgp/zqSy9uJcAwA\nCxhVGT4XklqEHw6DNWtDxonI9KyVySfch1IkPCdSzrNTBSYLY5jRqZIn91eifG4DcFBVDwOAiDwO\n4H4AF1T8ZmnFHU33XcYpa0zSwx4odaq1xYloT6cZF1oa/MucHLYyZ4f8uebzEFaIwtSUGadaFtEb\ndhFpsfemcMVKJ5M6dMyM8+dGrEAZb76S9fdXGrJlrLDy/HzimbLkLudtfA2Ao+eNjxXnCCF1zuXs\n+GUhIg8DeBgAmhbTx3xCljCXo/gfAug/b7y2OGdQ1ccAPAYAnalliyoVUJqb3FzumivMeLSv0cs0\nWhOr6+1RJ5MfOBGczH/4kiT/AUkmwSzLb1hlxppJuMYz1safj19FZ2f86y7QR/1yuZwn61UAV4nI\nBhFpAPAFAE9XZlmEkGoy7x1fVXMi8g8BPAcgDeD7qrq3YisjhFSNy7LxVfVZAM9WaC2EkBpBI5KQ\nCKm6V39RETqIVq1wIrNt9pJlxgtOpv1d68yToyedjPQFjqeJCSej436OJJNq898Yja213+OnJ/29\naky47pdK0vf49Q53fEIihIpPSIRQ8QmJkLht/MCmT3V3mfFUb5s7JDNh4+ezA8NOJkzamLp1oxOZ\n6bAx/0m+AimUDiZJzVqZ7Mi0GacHzrpjCsPn7MQC5gTMF1ljfSS5ngQbv89e4/S03+d8+NWlk2pt\n9pNBTkf+jL8P86FSeQHc8QmJECo+IRFCxSckQqj4hERI1M691MrlZjy10Qbs5FqsgwYAsqPWmTax\nebmTGd5onS0zHf7c6SChK5/159Lg7jQm1Oro+CBnZY7agJT86TP+oICFqhZzOeQDZ97oel8QJbzu\nbcecSGXWEjpLq0ilMgG54xMSIVR8QiKEik9IhERt44fVVrIj1n5q3OOTazBpi1A2rFzmRFpfn7Sn\n6Wp3MmHASVIl3vSkrQ6TPj3iZDS0LzP2lqaX+/Xp+LgdT007mXpnYrUNmhnv83tYxt4G9LzlbXEf\nNlXfSKYyKssdn5AIoeITEiFUfEIihIpPSITE49xLKL+sYdeU49aZl08IlnCMeIebIyylDSB05SWF\n0IS5ebkEmZB0l+3ig6wP7tD84nJpSZPPoRvptwFPE6t8JuOyN+1c4Y0LNnm6xAUFd6uWrc7SPtBr\nPnDHJyRCqPiERAgVn5AIicbGl0ZvJ0qLDQKRWWtFJyVE1DsuYSQhgSS8FnWXpBP4Ywrr+5zIxGpr\nV+fafRWhTOlu0Q698yYzPnutT/6RwKRPzToRZKasH6Vzr8+w0vdt1lChAhV/y4U7PiERQsUnJEKo\n+IRECBWfkAhZus69MGCnz7fDyrfYUsXprL0csrrXHVPosA7BydW+rHNYgrtxYNTJyKh15OSOVqk8\nTMoHfNSdMy8g1WGzGUfX+mucb7POs/S4/zs73hq0x5Rxbnl5txn3nljvZGb7bBn2XLM/dypvPYCa\n8Xts+HdqUpnzoMS6MICHEDJfqPiERAgVn5AIWRo2fkICTqrTllidXunbYWna2rqFhm4zHuv3rZHG\nV9tzacJbZ8OwtcMal/U4mcZhu76G3i4nkz5p23O51lcAMJsQPbKICNuWAcDsOuuPGV+VYENPWtt3\nxS6fKJN/99Blrg7IHX7fzclhOy6nxq0mVM4phIFUFaquUw7c8QmJECo+IRFSUvFF5PsiMigie86b\n6xGR50XkQPHf7ou9BiGkvijHqPgTAP8JwH8/b+4RANtV9VEReaQ4/kbll1ce4fehADC9ebUZz3T5\nP7X1oC2iUdjzthn7VwXag+/AMxvWOZnxa6yNOtXjbdTxVUG3nXYvE3ol0gn2fGF0zE4sspbXMxtW\nurmJVTa+QhPCDto+sHtW93NvO5l6uhKaSyijIvZvmE9HnPlScsdX1b8CEDb3vh/AtuLv2wA8UOF1\nEUKqyHxt/F5VHSj+fgKAD3EjhNQtl+3cU1WFLw/3C0TkYRHZKSI7Z7D4GjcQshSZr+KfFJE+ACj+\nO3ghQVV9TFVvUdVbGuCLYRBCas98IwaeBvAggEeL/z5VsRXNA+3pdHMSVD5t3b7fyRRGffJM6ZMF\nlV8SAjwag7mWFT5BaGqrdQqOrvGOndxm697zoS5A6j3r8NOgxRcK9V1Rd7bdP4IzbYHTK+FPWLHb\n9sfKnwndUOWRarcuXGm0jkXnPAWg05X55BpWeJJ0wj5coaSckHK+zvszAD8DsFlEjonIQ5hT+M+K\nyAEAnymOCSGLhJI7vqp+8QL/dU+F10IIqRGM3CMkQpZEko4eHXBzqX3Wfl9ISzd/6pSby/6lnVt2\n641OZuhaa+NP9vmKr20DTWbsbPw6J0yUAoCZDjvXfsyH4jQcttdPe30g0Mx1a814ZJ13Lk93B4la\ngUmdVEF3zf84YMZJ93c+FKb8vUu1+iIklYA7PiERQsUnJEKo+IRECBWfkAhZEs69eQXi1Bn66ltu\nrltvMOOJNd65V+i2lXySWmbVM/lG79wr2Bga17IKAJCxXjiZ9a/T+P4ZM+4Z8VWYJvqt8+zMdVYl\nsr9sXwMA3r1ioxlv/GplnHu1hDs+IRFCxSckQqj4hETIkrDxa0lSu+2QSiVxpI6cNOOWpLIHwVu3\nNATVayZtMku9kRn3oVWtx4MqR+M+gEcDGz8/eNq/eMEeJ8f9vWs7bP0mTads8tShhDbZmz/+gT3N\n1uv9+nbt9eupI7jjExIhVHxCIoSKT0iEUPEJiZCl69wLymCn232x7MKEbVUdlkBOcuS5FtMJ7buk\nJXAIJbSlDkuCF1b4+joyZrO1Qmcf4Ku2JLZarmNaDninnHenJTA6boaJ7aPDtueB4xMApMMG9chs\n4BA84YN+zq2wGZFNPU1Opt4Vizs+IRFCxSckQqj4hERIvZsiZZEKbWrA2dWqPtMjtPmcDZhgm0uz\ntecK61Y5mbM3Bi26O/3rdB+wpV1aXjnsZHKnfYJISDpswV2lqqyVQgK7W497v0VZBNWDU22+Uo10\n2ftQ6PDPSa7ZVjfW4J53+c5cGBm097ztxR0XXWo9wh2fkAih4hMSIVR8QiKEik9IhNS/cy8pQCYI\nrJEEx452BHO5hMCWk0HllIy9HEmOu1NbrMNoZJN/2bCf++qXfG/0xmdfNeN5h90EATs6VV+NSVPd\nNjCp0BUELu2zpaoBABpk7CU4ZsOy04UrVzuZQkPpxztz1gYC6dHjZtz7flDhCHAO1FyhMkFTkqmd\nOnLHJyRCqPiERAgVn5AIqXsbP0xmAYDceluJZqbTJ1/kWqwd1nTK277aZ+234Y3WdzC+NiGAJzDn\n+l/wr5vZ/pqbmw9hYFJSAk5+xLZxDpNVElsv15DZ/uVmXGiy62sIA5BQZkuq4FrItL82mQnbhhqD\nPiCqVHvtMJGrqtQw+Io7PiERQsUnJEKo+IRECBWfkAipe+eedvgKKBpWnUnosT7bYudOf8ZnZk2v\nsA6hlqP2mHXP+tZcSa2uqkVZjqVUfTnzQgqNdn1DV1sHau/xTn9QGc4910t+T0IaXQXI9K91c9po\nM/qSnIaFSbs+nZ1xMmH2JwN4CCFVhYpPSISUVHwR6ReRF0Vkn4jsFZEvF+d7ROR5ETlQ/Le7+ssl\nhFSCcoyKHICvqurrItIO4DUReR7AbwPYrqqPisgjAB4B8I1KL1A/+NDNyQEbNNPS5SvUZq+3rZAy\n075ibvuBETMu7N5vz132KmtEygd4pBqyCYL1QyFj7dhC4I/RZh98VS2Sqian1lkbPt9jk3/yOd/i\nKzUStCVLaqsW2PhJ9ntS1d9aUXLHV9UBVX29+PsogP0A1gC4H8C2otg2AA9Ua5GEkMpySW5EEVkP\nYCuAHQB6VXWg+F8ngKSOjoCIPAzgYQBoEp8+SwipPWU790SkDcAPAfyuqprPyDpXyTLxk7GqPqaq\nt6jqLQ0o3WmWEFJ9ytrxRSSLOaX/U1X9i+L0SRHpU9UBEekDMFiNBbrvaxPIDw25udRLdi7ps4a3\n3haORBuwht/rVotC1u4t00FOzvC1vtBF9/RVZpzfn1CsI0Cy3l5OL7P+5nzfcidTCLa+zKD1++hZ\n/2wVxmzxDkhCsZigmnBSxeaFpByvvgD4HoD9qvrt8/7raQAPFn9/EMBTlV8eIaQalLOl3AXgSwDe\nEpE3inP/DMCjAH4gIg8BOALg71RniYSQSlNS8VX1JQAX+pxyT2WXQwipBYzcIyRCFr/3aBFQVrvt\nJUp2wlYYbh60j9xYv997ZlusE66j3yfyZIdtEJdMzDoZnbFzqWPe/1wYsc68fFjZJ8npmhSws8jg\njk9IhFDxCYkQKj4hEVL/Nn6SLZzQWWWhiNl+L4eGw7aoRresNOPxPh94M9Vt96PxVT4RqX3Ghl9l\nTo04mULQKamQ0GXIJTktAfu9HLjjExIhVHxCIoSKT0iEUPEJiZCaOvdUtaxsu8WETie05lqAdVSb\nwvh4aaEyjksfPWbGCU2oE+dK4RuRl0dhqjItrusFDVuMXwDu+IRECBWfkAih4hMSIfUfwLMYCQJ4\n0j2lK4/nh875ycLSsj8XnDICq8LKt0k+nKUAd3xCIoSKT0iEUPEJiRAqPiERQufeZZK5cr2bm15n\na0gPXeEzvlpP2OowzT9718nkR3zGGbkEgpZjqVbfKj3VacOF8r22HVtq/3vumLLal9c53PEJiRAq\nPiERQsUnJEJo418icvP1Zjx4k08pGbo2OCYha6f7nRkzpj1fedIdbXZite/rOrI5aLPVYIN8GlfY\n+w0ALe/Yar25I8ecTL0HX3HHJyRCqPiERAgVn5AIoeITEiF07l2E9LIeNzd8VbsZn/2Y99y1rreZ\ndvJTn50nr71txy0JwSUd9lxIaOfkCEqPa0LlnPy5wJFYR+XK54tkfZlu6Qpab0346k8z7XbvCx2z\n2VFf2rs3Z1t8NZ4+62QKo6MXWmpdwB2fkAih4hMSIVR8QiKENv55pDtsMM7U1g1OZuDTNjDjX/zy\nj5zMi0PXmPHBU77N89TnbjLjwY97W3KmI7C9U94Wz4zZ9+7moBP08j2T/pidNiFovhV0F5LQpk/1\ndDkZbbQyhTafLJWesdd0tttWqc2t8fV7B/NNZrwivdnJNJyzAVqpKf86MmGr+xQOH3Eymptv/eCL\nwx2fkAih4hMSISUVX0SaROQVEdktIntF5PeL8xtEZIeIHBSRJ0TEf59CCKlLyrHxpwHcrapjIpIF\n8JKI/B8AXwHwHVV9XET+C4CHAHy3imutPEHV1cLVV5jxyVu9Tfjbd/zEjjuOO5k/PnKnGU/0+vfX\nk5+2tttNmw45mRVNY2Y8MOkTgva+t9qMG8/aNWfOeBu/MLn4uxmlOoMYh/ZWL5S2133gl7yvZeQG\nWxDlt2572YyvbfL39z+u/LQZH7ne+xdQCHw2o81OpOmkPa7r4Aon0/1ze/7ckaP+XPOIwyi54+sc\nHz2B2eKPArgbwJPF+W0AHrjksxNCFoSybHwRSYvIGwAGATwP4BCAYVX9aNs6BmBNdZZICKk0ZSm+\nquZVdQuAtQBuA3BNiUN+gYg8LCI7RWTnLJZmcwJCFhuX5NVX1WEALwL4BIAuEfnIR7AWwIcXOOYx\nVb1FVW/JwtvMhJDaU9K5JyIrAMyq6rCINAP4LIBvYe4N4PMAHgfwIICnqrnQqhA4RWZ6bGDGVK9v\nOXxyxjrYfuWtzzuZ1HdtEkf/az4wY+TWtWZ8dMWVTuZY4LPJjnknztWHguCbV183w8ISSMBJIn/6\njBnL6JiTSV1pnbWS0EFaJmwl3uc+tFk6L6R8cM7AoHXKZbKlq+00dnon6/Ry6wA8m/UOypYT9lnK\nnhh0MvNpPV+OV78PwDYRSWPuE8IPVPUZEdkH4HER+dcAdgH43iWfnRCyIJRUfFV9E8DWhPnDmLP3\nCSGLDEbuERIhUSfppILiFxMr7OUotPkEiZc+tIk7rU/4oJDWp35uxklpFi3HrC/Ul+EgvyDoiAPA\nVbFNamed33/AjDs3+oIojcP2tXv+KHiNdw67Y64Kl5dURGWltc2n1y93MmNrgiSijPfHpGaqU62X\nOz4hEULFJyRCqPiERAgVn5AIidq5J+ttEM3oFfZ9cPOVA+6Ys5PWkdO175yTcXEiZTinyIVJt/nA\nlrDlmDT6qND8bdeZ8Znr/eOeD5Lmlu0UJ1OKpLbZhfc/MON0MAaAnt6VZizNTU5Gh+3zlZ9HsE4S\n3PEJiRAqPiERQsUnJEKisfGTuuIMbbFzDXfYjij//sonEfJP3/vbZnzmY+uczMwnbQWeSV9YxVXD\n7To062Ra9lofQ+5D73NYir6CsIKuJFTQTaet36SwfrWT+eBz1ma+9Z59TmYqb1Vg//TVZtzf6n0H\nss8G9STZ+OWQP+kTbmoFd3xCIoSKT0iEUPEJiRAqPiERsmSde6km69iZvsm3wzpxj82be+PmPzbj\nzpQvibz/sHUidfT4gI/RjUHmWLvPz5teaZ1TU8t8C63O5TbAqOeVhCCVA0H22CKruJPU3joVtrdO\n+f1J+3vNeHRTm5PJb7RVb77W95yTaRR7r755798y4zev8TVkmw5uMeO1L/gWZPKz3W6unuCOT0iE\nUPEJiRAqPiERsmRt/MIWG4hx4nZvH3/uxl1mvHvG2vQPPfkP3DHX/lsbBCKtPoFkfIu1C8f6vB07\nHfgG8gmVx89tDN6X1UcCLZu0lWdyR4/5F6pjUgkJONJi/TPa7C+ONlqfSHbMl9CVY/Z+/uN3v+Bk\n0il7XF+LTf750pYd7pjsVusXeOLjH3cyPd+35SibfvSKk1lIuOMTEiFUfEIihIpPSIRQ8QmJENEa\nBnx0SI/eLvdU/HXTHb5v/MRdtvXR4C0+QGbmGhvg0bbDOoN6/4PtlV5J5ObrzfjsDf5vGF8dBAcl\nFIdZ+dqMGTf9dI8Zz6e9Ui1JqpwTlspOtbc7mak77f2d6fBVjloG7bXJvvm+k8kPDZlxepMN9Dr8\nW6vcMZ/8G2+a8e+s/LGT+cZhm8VZ+Je+vHbqpTfc3OWyQ7djRM+WLCPEHZ+QCKHiExIhVHxCImRJ\nBPAUEtonNQ/YxInut72d2LrdBm/Iy9Wx6ZPsWIg1wxpHE3o4D9j35amEhKDZdmvbNrcG7Zzq3MZP\nan0VUhgddXONZ+zfNbbG39/GIXv90gmvE5I/+J4Zr/u995zM27tuN+Nvf937F3642VZvuvFL/8jJ\nXPuODcjKnzpVcn2Vgjs+IRFCxSckQqj4hEQIFZ+QCFkSzr0kB1Fq4LQZd50ZcTK1ymRLykAbW2Od\ncCP93kGUmbTBVc2nfbBV84ngb88vvXLbSaSOnDTjpn5/jRtP2wAtlcrscy3/02bs7en9hJP56dds\nNt7fvNkH6+y+Y6sZNz1z2slUq6ISd3xCIqRsxReRtIjsEpFniuMNIrJDRA6KyBMi4pPOCSF1yaXs\n+F8GsP+88bcAfEdVNwEYAvBQJRdGCKkeZdn4IrIWwK8B+DcAviIiAuBuAH+vKLINwL8C8N0qrHFe\nFM4Om7HOzlxAsvoUxnwV1kLGBuOMbPaVeGXWvi/3/ty/dnaPDTDJD/u23UuR/Bnb7qz9ZR/cVAhk\nNOevcSXo+9ERN/dPfuXvmvFnNr3jZIY3WfVbk9DmLX/6zGWuLplyd/w/APB1/HXr92UAhlX1oyt5\nDICvQ0wIqUtKKr6I/DqAQVV9bT4nEJGHRWSniOycRenwTEJI9Snno/5dAH5DRO4D0ASgA8AfAugS\nkUxx118L4MOkg1X1MQCPAXP5+BVZNSHksiip+Kr6TQDfBAAR+RSAr6nqb4rInwP4PIDHATwI4Kkq\nrvOSWUibPmzJPbXVd/E5frcd//1Pvuhk/t+ZTWY89pdrnUwsNr0jaA9eTstpyfjHXZpt8ZVUkORU\nGPGJPeW0xZZD9nXe7PZtvHNBoyYJuj9Vk8v5Hv8bmHP0HcSczf+9yiyJEFJtLilyT1V/AuAnxd8P\nA7jtYvKEkPqEkXuERAgVn5AIWRJJOkj5BBfJ2j8t3d3lZArj1kmTVOklJL3CVk2ZuG29kzn+SXvu\nTbf7AI8/6t9uxlPqqwD/1922IvFV/zshgmexISULwCIVVCwqq1JwwjMQOupSy7qdTG6lbck93mc9\nbo3Ds+6Y7An7nIxv8q9bCALYB4d8haC2s/ZLLp2unUOaOz4hEULFJyRCqPiERMiitPElaw2odL8P\njphZY+2u0VU+a7jtiLXxMwO2q8r4DX3umIG77CXbcOcHTuYrq2zRhYmCr7L76OFfNeMzL/i/4ap/\nV71OPrUiDJpJtQRVgFcuc8cUumxRjdQ73kcS+mPS3Z1ORvttF5zR9W1OZqYtqGTcbcfS71Uku8EG\n2kwt836L9GQwfssXCuk+YEPYCyO+WEy14I5PSIRQ8QmJECo+IRFCxSckQhalcy91tc12G77eB1Cc\nvTZw2qzy1Vcazlpnz0yPDd5Ys8FXPX2g97AZD0z59tbf+pl13C1/yQfnLH/StrNePfq+k1kKlHLm\njd1gA6IAYPjKwCH4iRudzLK91jGWVFsn32ifgVyj3+dm2qxjLjNlg2oyEz6TPBvMpWe8c69xyM61\nnvArbNpnM9lzZbQTqxTc8QmJECo+IRFCxSckQurexk+1+sCHXJe1xYc3+fevwnVjZtzb7qumtF9t\nbaqPdVmb69CYtz9/sMOWIOj/v04EV/+vV/xkuL6SEkuDfFBhOBVUup29rdcdk/mUrSx7Z9/7TubZ\nvTeYcdcOH6DVOGxt8cZzvstQ8yl7J9JTdn2pWX+nNG3td8l5mdR4YK8P+mq5uaAKcC3hjk9IhFDx\nCYkQKj4hEULFJyRC6t65l9gCe9o6YBoSKkyndtrgnPT7zU5mpNm+7/24o9+Muw74iihXP1facUfO\nIyiDHZam7vzhLnfIkU03m/HwvSedTGunTX+b7vIZkM2nrdOtaXDSyaQm7D3WbFDJJ6FNdXrUvk7h\n5Cknky+jBPdCwh2fkAih4hMSIVR8QiJENMGGqRYd0qO3yz2lBUuQ7g6Scpb5Crp63NqF5bQ9IvWJ\n3nmTm5vptAE72QmfBKNlVPTNDll7XU7YQJuw1TZQvXbblWCHbseIni35h3PHJyRCqPiERAgVn5AI\noeITEiF1H8CTRP6cLUOcmvGBNnTmLR3k5d1uLgzXyaxd42S0w2Z2ypAvX50/ZassFerYcVdJuOMT\nEiFUfEIihIpPSITUNIBHRE4BOAJgOQBfwrZ+WWzrBRbfmhfbeoH6XPM6VfWlowJqqvi/OKnITlW9\npeYnnieLbb3A4lvzYlsvsDjX/BH8qE9IhFDxCYmQhVL8xxbovPNlsa0XWHxrXmzrBRbnmgEskI1P\nCFlY+FGfkAipueKLyL0i8o6IHBSRR2p9/lKIyPdFZFBE9pw31yMiz4vIgeK/vkvnAiEi/SLyoojs\nE5G9IvLl4nw9r7lJRF4Rkd3FNf9+cX6DiOwoPhtPiIjvkrGAiEhaRHaJyDPFcV2v92LUVPFFJA3g\nPwP4VQDXAfiiiFxXyzWUwZ8AuDeYewTAdlW9CsD24rheyAH4qqpeB+AOAL9TvKb1vOZpAHer6k0A\ntgC4V0TuAPAtAN9R1U0AhgA8tIBrTOLLAPafN6739V6QWu/4twE4qKqHVXUGwOMA7q/xGi6Kqv4V\ngLDsyv0AthV/3wbggZou6iKo6oCqvl78fRRzD+Ya1PeaVVU/6nGWLf4ogLsBPFmcr6s1i8haAL8G\n4L8Vx4I6Xm8paq34awAcPW98rDhX7/Sq6kDx9xMAfMO3OkBE1gPYCmAH6nzNxY/NbwAYBPA8gEMA\nhlX1o/S4ens2/gDA1/HXbQ+Xob7Xe1Ho3LtEdO5rkLr7KkRE2gD8EMDvqqrJP63HNatqXlW3AFiL\nuU+C1yzwki6IiPw6gEFVfW2h11Ipap2P/yGA87tWrC3O1TsnRaRPVQdEpA9zu1TdICJZzCn9n6rq\nXxSn63rNH6GqwyLyIoBPAOgSkUxxF62nZ+MuAL8hIvcBaALQAeAPUb/rLUmtd/xXAVxV9IY2APgC\ngKdrvIb58DSAB4u/PwjgqQVci6Foa34PwH5V/fZ5/1XPa14hIl3F35sBfBZzvokXAXy+KFY3a1bV\nb6rqWlVdj7ln9seq+puo0/WWharW9AfAfQDexZxN989rff4y1vdnAAYAzGLObnsIc/bcdgAHALwA\noGeh13neen8Jcx/j3wTwRvHnvjpf88cA7CqueQ+A3yvOXwngFQAHAfw5gMaFXmvC2j8F4JnFst4L\n/TByj5AIoXOPkAih4hMSIVR8QiKEik9IhFDxCYkQKj4hEULFJyRCqPiERMj/B6ks6XYB1g0EAAAA\nAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1520e9d668>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "getActivation_mnist(conv0_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEQdJREFUeJzt3X9sXeV9x/HP17/ixEkggcSYJJAf\nDbSBibBZ4ecqCiulVSWopjJYxTJUEf4AbaidVESnwv5DU4EiDTGlI23YKFBUGOnEaFnaiW5AwNBA\nEkITmiZLjONAHLBjSGLf+90fvqnckPs8Jz73l3neLymyfb+Pn/PlJh/Ovfec8xxzdwFIT1O9GwBQ\nH4QfSBThBxJF+IFEEX4gUYQfSBThBxJF+IFEEX4gUS213FibTfF2dZSt29T26ByHTrNgvX33SHQO\nHx2NjkFG4b8OSZLPmBasj06NT9LaP5y1o0mteHL5fBw1Mr18bXRgQIXh4Qx/KznDb2ZXSbpfUrOk\nf3H3u0Pj29WhC+yKsvWmT306us1t35oarJ/9jT3ROQrvvhsdg2ysJf5P6PAl5wfr+89pjc7Rdc8L\nmXtqaE3NwfKHl3dHp+i7tHy2e+/9XvZWMo88hpk1S3pA0hclLZN0vZktm+h8AGorz3v+FZLedvcd\n7n5E0mOSrq5MWwCqLU/450naPe7nPaXHAEwCVf/Az8xWSVolSe0Kf/ADoHby7Pl7JS0Y9/P80mN/\nwN1Xu3u3u3e3akqOzQGopDzhf0XSUjNbZGZtkq6TtK4ybQGotgm/7Hf3UTO7VdLPNHaob427b8nT\nTO+Vs6Nj5p7aH6xzGK+2spwz0fbsK8F617OV6mYSKBaC5WlPbYhOseSp8rX3PPv5ELne87v7M5Ke\nyTMHgPrg9F4gUYQfSBThBxJF+IFEEX4gUYQfSBThBxJV08U8Yo6cHL912LRH5kRG/LYyzXwCNM2Y\nER1jC7qC9SOdgZUjJFkx/nfWtvO9YH101+5gfbI4sPKi6Jj3uovB+klvha/3l6S5D1RmbQP2/ECi\nCD+QKMIPJIrwA4ki/ECiCD+QKMIPJKq2x/mnT5UvX1623Howfq+Bk594LViPH3WWmpcuDtYL23dk\nmKW6ms6N38Og9wvhxU9Gw7c4kCQV28LP2KInPwhP8Fb8uSq2htflb5l3enSO0d53omPqbdbaFzOM\nCp8L8E933R+d4abP/1XZ2sg3/jdDD2PY8wOJIvxAogg/kCjCDySK8AOJIvxAogg/kCjCDySqpif5\nFFua9NHc8vfrm7I/foqOHz6cu4/DC2YF66OfXhGst//05dw9xBzp7IiO6ez5KFhv3fJ/0TkK7+0P\n1sNLT2R06FB4G0NDldjKpBA7Eeg7W26MzvH6un8rW1vRMZC5F/b8QKIIP5Aowg8kivADiSL8QKII\nP5Aowg8kytyzLH9RGTOnz/MLzr25/ICXN9Wsl5CWxQuD9dEdO2vSB3A8Q9ddWLa2+Wff08H9u+Or\n4ijnST5mtlPSkKSCpFF3784zH4DaqcQZfp9z9/D9mAA0HN7zA4nKG36X9HMze9XMVh1vgJmtMrMe\nM+sZGRnOuTkAlZL3Zf+l7t5rZnMlPWdmb7n78+MHuPtqSaulsQ/8cm4PQIXk2vO7e2/p6z5JT0kK\nXw4HoGFMOPxm1mFmM45+L+lKSZsr1RiA6srzsr9T0lNmdnSeH7n7s6FfsJFRtfQdKFsfzdFMJXEc\nP7vQMeejhrvC+5jTf1H+38RRxde3BuuxG7F8eNYp0W1M+c/wDWFULETnqIUZj71Uttbk2T9Xm3D4\n3X2HpPMm+vsA6otDfUCiCD+QKMIPJIrwA4ki/ECiCD+QKMIPJKqmN+1QU5N8ZvxmFMjmyBfCyyfs\n/Gp8jj/7o/DJM7/YdlawPv/x+G09Ol8O37RDFVhQprB9R7A+cv7c6Bx777wgWD/zzhdOqKdGx54f\nSBThBxJF+IFEEX4gUYQfSBThBxJF+IFE1famHTbbL7ArarY9IDUbfL0GfSDTTTvY8wOJIvxAogg/\nkCjCDySK8AOJIvxAogg/kKjaXs+PSad5Wfh6/rdumRWd4+QF7wfrg0PTonMs+dqvo2NwYtjzA4ki\n/ECiCD+QKMIPJIrwA4ki/ECiCD+QKMIPJKqhTvJpnjMnOqZ4RvjmC0OLp0fn6F8R/n9e26KhYP3Q\ncFt0G627pwTrC//+xegcjaDw5rZgfekt+bcR/1vHeAM3XlS2Vnj6pczzRPf8ZrbGzPaZ2eZxj802\ns+fMbHvpa/w0LwANJcvL/h9KuuqYx26XtN7dl0paX/oZwCQSDb+7Py9p4JiHr5a0tvT9WknXVLgv\nAFU20ff8ne7eV/p+r6TOcgPNbJWkVZLUrvgFHABqI/en/T62/G/ZJYDdfbW7d7t7d6vCH4IBqJ2J\nhr/fzLokqfR1X+VaAlALEw3/OkkrS9+vlPR0ZdoBUCvRm3aY2aOSLpN0qqR+SXdK+ndJP5Z0hqRd\nkq5192M/FPyYyXLTjuZzzg7WhxefFJ1j2n9vDdaLQ+FzCZo6OqLbKA4PR8fEtMw7PVh//+IFwfr0\nJzbk7gGVcyI37Yh+4Ofu15cpNX6KAZTF6b1Aogg/kCjCDySK8AOJIvxAogg/kCjCDySqoRbzyKJ5\n5sxgvTA4mHsbhS2/Cdbbt8TnKObsoRIn8GQx2vtOsD545sJgfeA7F0e3Me9Xh4L15l++Fp2jFuz8\nc4L1XXeE95WLb9sf3Ubs+a4l9vxAogg/kCjCDySK8AOJIvxAogg/kCjCDySqpsf5ra1NLfPPLFvf\nfnN4YQlJ+tzlG4P1KU2t0Tme2RY+nrvowfBaCE2/+nV0G58Up3/3hXq3UBF+8XnRMZ33/C5Y/+D9\n8O1FGukYfhbs+YFEEX4gUYQfSBThBxJF+IFEEX4gUYQfSFRNj/OPzGxV/+Xlj+VP7Y/fa2D3jWeE\nBxwZic5x9lBfsD66tz86ByaX1h17o2Pe+ufw+R+nPPF6sJ53DYdaY88PJIrwA4ki/ECiCD+QKMIP\nJIrwA4ki/ECiCD+QKHP38ACzNZK+LGmfu59beuwuSTdJerc07A53fya2sZk22y+wK3I1PFk0z5oV\nrBcOHKhRJ/XXtHxZuL4/fqOV0d17KtVOQxu94k+iY6xQPrM9PQ9ocHBP/Gw5Zdvz/1DSVcd5/D53\nX176Ew0+gMYSDb+7Py9poAa9AKihPO/5bzWzN8xsjZmFX+MCaDgTDf+DkpZIWi6pT9I95Qaa2Soz\n6zGznhEdnuDmAFTahMLv7v3uXnD3oqTvS1oRGLva3bvdvbtVUybaJ4AKm1D4zaxr3I9fkbS5Mu0A\nqJXo9fxm9qikyySdamZ7JN0p6TIzWy7JJe2UdHMVewRQBdHwu/v1x3n4oSr0UjNN530mWH/7aycH\n61kWHZn/H/vCAyLH+ffdenF0Gx+eFjlHI8PqEm0fhP9b5q/ZGqxnOV+huPHNcD06wyfH9vsvDNb/\n/E83ROd49kcXla2NvJnpEL8kzvADkkX4gUQRfiBRhB9IFOEHEkX4gUQRfiBRhB9IVHQxj0pKaTEP\noB42+HoN+kDFFvMA8AlE+IFEEX4gUYQfSBThBxJF+IFEEX4gUdHFPNC4mtrbg/XioUNV76G5c250\nTKE/srAJ6oI9P5Aowg8kivADiSL8QKIIP5Aowg8kivADieI4/yRWy7UYyun9y09Fx5x2H8f5GxF7\nfiBRhB9IFOEHEkX4gUQRfiBRhB9IFOEHEkX4gURFT/IxswWSHpbUKcklrXb3+81stqTHJS2UtFPS\nte5+IDhXa4ta5pxWfkBba7ThwpyTwgM2bY/O0TRtWrD+u7/5TLA+ctZH8W3sDi+0sej2F6NzxPjh\nw7nniGk6L/xcnPp6/h5YEKQ+suz5RyV9092XSbpQ0i1mtkzS7ZLWu/tSSetLPwOYJKLhd/c+d3+t\n9P2QpK2S5km6WtLa0rC1kq6pVpMAKu+E3vOb2UJJ50vaIKnT3ftKpb0ae1sAYJLIHH4zmy7pJ5Ju\nc/fB8TUfu8LkuFeZmNkqM+sxs54jxfh7ZQC1kSn8ZtaqseA/4u5Plh7uN7OuUr1L0nE/kXH31e7e\n7e7dbU1TK9EzgAqIht/MTNJDkra6+73jSuskrSx9v1LS05VvD0C1ZLme/xJJN0jaZGYbS4/dIelu\nST82s69L2iXp2uq0CKAaouF39/+RZGXKV5zIxnxkVKN9e0/kVz5u1+58vy+pEDk+fsY/vJB7G0N/\ncWGu32+eOTM6pjA4GB2T1/YbwudVLPm7l3Jvww8O554DJ44z/IBEEX4gUYQfSBThBxJF+IFEEX4g\nUYQfSBQ37aiSGY/nO/5di2P4knTw2vD5CC3D1d8/FIc5zl8P7PmBRBF+IFGEH0gU4QcSRfiBRBF+\nIFGEH0gU4QcSVduTfDqmypefV7b8zmc7olOMRpYBHFkSXyR04Wn7g/U9G+aFf//b+W+40SgKkfuk\nnHln/oVNYmzKlOiYWtygJDXs+YFEEX4gUYQfSBThBxJF+IFEEX4gUYQfSFRNj/MX2pt04OxpZesj\n0497r88/UGwL162vPTrH3i3zg/WuN0aC9aYZM6LbKA4NRcdUm7VGnixJJz2S/6YbeW27d3l0zNJb\nNtSgk7Sw5wcSRfiBRBF+IFGEH0gU4QcSRfiBRBF+IFGEH0hU9CQfM1sg6WFJnZJc0mp3v9/M7pJ0\nk6R3S0PvcPdnQnM17x/W7B+UXwhjdsam661Y7wYy8pEj9W5BktS87Kxg/asXx0/g2VipZvB7Wc7w\nG5X0TXd/zcxmSHrVzJ4r1e5z9+9Wrz0A1RINv7v3SeorfT9kZlslhde5AtDwTug9v5ktlHS+pKOv\n0241szfMbI2ZzapwbwCqKHP4zWy6pJ9Ius3dByU9KGmJpOUae2VwT5nfW2VmPWbWMyIWYQQaRabw\nm1mrxoL/iLs/KUnu3u/uBXcvSvq+pBXH+113X+3u3e7e3ar4Kq0AaiMafjMzSQ9J2uru9457vGvc\nsK9I2lz59gBUS5ZP+y+RdIOkTWZ29IjLHZKuN7PlGjv8t1PSzVXpEEBVmHt8AY2KbczsXUm7xj10\nqqT3atbAxNFnZU2GPidDj9LH+zzT3edk+cWahv9jGzfrcffuujWQEX1W1mToczL0KOXrk9N7gUQR\nfiBR9Q7/6jpvPyv6rKzJ0Odk6FHK0Wdd3/MDqJ967/kB1Endwm9mV5nZb8zsbTO7vV59xJjZTjPb\nZGYbzayn3v0cVbqeYp+ZbR732Gwze87Mtpe+1vV6izI93mVmvaXnc6OZfamePZZ6WmBmvzSzN81s\ni5n9benxRns+y/U5oee0Li/7zaxZ0jZJn5e0R9Irkq539zdr3kyEme2U1O3uDXXM18w+K+mgpIfd\n/dzSY/8oacDd7y79D3WWu3+rwXq8S9LBRroUvHS2atf4y9YlXSPpr9VYz2e5Pq/VBJ7Teu35V0h6\n2913uPsRSY9JurpOvUxK7v68pIFjHr5a0trS92s19g+jbsr02HDcvc/dXyt9PyTp6GXrjfZ8lutz\nQuoV/nmSdo/7eY8ad40Al/RzM3vVzFbVu5mIztL6C5K0V2OrLzWihr0U/JjL1hv2+azE5fV84Bd3\nqbv/saQvSrql9FK24fnY+7lGPJST6VLwejjOZeu/10jP50Qvrz9WvcLfK2nBuJ/nlx5rOO7eW/q6\nT9JTKnPpcoPoP3q1Zenrvjr38zFZLwWvteNdtq4GfD7zXF5/rHqF/xVJS81skZm1SbpO0ro69VKW\nmXWUPliRmXVIulKNfenyOkkrS9+vlPR0HXs5rka8FLzcZetqsOez4pfXu3td/kj6ksY+8f+tpG/X\nq49Ij4slvV76s6WR+pT0qMZe4o1o7DOTr0s6RdJ6Sdsl/Zek2Q3Y479K2iTpDY2Fq6sBnstLNfaS\n/g2NLRS8sfTvs9Gez3J9Tug55Qw/IFF84AckivADiSL8QKIIP5Aowg8kivADiSL8QKIIP5Co/wcF\nzBzvzG04NQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x107feb080>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "getActivation_mnist(conv1_out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}