train_no_aug_rate_audjust.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training notebook for the deeptail task without data augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'2.1.2'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow\n",
    "import keras\n",
    "keras.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting started \n",
    "- Download data from: https://www.kaggle.com/c/whale-categorization-playground\n",
    "- Rename train.csv to targets.csv\n",
    "- Rename the train directory to kaggle_train\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Image', 'Id']\n",
      "9850\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "home_dir = os.getcwd()\n",
    "fname = os.path.join(home_dir, 'targets.csv') # targets for both train and validation\n",
    "\n",
    "f = open(fname)\n",
    "data = f.read()\n",
    "f.close()\n",
    "\n",
    "lines = data.split('\\n')\n",
    "header = lines[0].split(',')\n",
    "lines = lines[1:]\n",
    "lines = lines[:-1]\n",
    "\n",
    "print(header)\n",
    "print(len(lines))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Encoding the whale ids\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4250\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "whale_ids = [line.split(',')[1] for line in lines]\n",
    "whale_ids = set(whale_ids) # convert to set to remove duplicats\n",
    "whale_ids = list(whale_ids) # convert back to list to make it ordered\n",
    "whale_ids.remove('new_whale') # remove the new_whale since we will not train with this\n",
    "\n",
    "print(len(whale_ids))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import errno \n",
    "\n",
    "def mkdir_p(path):\n",
    "    try:\n",
    "        os.makedirs(path)\n",
    "    except OSError as exc:  \n",
    "        if exc.errno == errno.EEXIST and os.path.isdir(path):\n",
    "            pass\n",
    "        else:\n",
    "            raise"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split the training images into a train and validation set, and then subdivide them into directories for each whale_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8806\n",
      "234\n"
     ]
    }
   ],
   "source": [
    "import shutil\n",
    "original_dataset_dir = os.path.join(home_dir, 'kaggle_train')\n",
    "\n",
    "train_dir = os.path.join(home_dir, 'train')\n",
    "#shutil.rmtree(train_dir)\n",
    "mkdir_p(train_dir)\n",
    "\n",
    "validation_dir = os.path.join(home_dir, 'validation')\n",
    "#shutil.rmtree(validation_dir)\n",
    "mkdir_p(validation_dir)\n",
    "\n",
    "# Let's then create a subdirectory for each whale_id in both the train and validation directories \n",
    "# so we can use the ImageDataGenerator magic function\n",
    "\n",
    "classes_count = len(whale_ids) # During the development phase of testing models, we dont look at all 4251 classes. \n",
    "                     # When we're ready to look at all classes we can set classes_count = len(whale_ids)\n",
    "\n",
    "for i, whale_id in enumerate(whale_ids):\n",
    "    if i < classes_count:\n",
    "        mkdir_p(os.path.join(train_dir, whale_id))\n",
    "        mkdir_p(os.path.join(validation_dir, whale_id))\n",
    "\n",
    "train_image_count = 0\n",
    "\n",
    "# Copy first 7500 files into the appropriate whale directory in train dir (only if their class is included)   \n",
    "for i in range(9600):\n",
    "    pic = lines[i].split(',')[0]\n",
    "    whale_id = lines[i].split(',')[1]\n",
    "    src = os.path.join(original_dataset_dir, pic)\n",
    "    whale_id_dir = os.path.join(train_dir, whale_id)\n",
    "    if os.path.isdir(whale_id_dir):\n",
    "        dst = os.path.join(whale_id_dir, pic)\n",
    "        shutil.copyfile(src, dst)\n",
    "        train_image_count += 1\n",
    "\n",
    "validation_image_count = 0\n",
    "# copy the rest into the appropriate whale directory in validation dir    (only if their class is included)  \n",
    "for i in range(9600,len(lines)):\n",
    "    pic = lines[i].split(',')[0]\n",
    "    whale_id = lines[i].split(',')[1]\n",
    "    src = os.path.join(original_dataset_dir, pic)\n",
    "    whale_id_dir = os.path.join(validation_dir, whale_id)\n",
    "    if os.path.isdir(whale_id_dir):\n",
    "        dst = os.path.join(whale_id_dir, pic)\n",
    "        shutil.copyfile(src, dst)\n",
    "        validation_image_count += 1\n",
    "\n",
    "print(train_image_count)\n",
    "print(validation_image_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Using a pre-trained convolutional base\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.applications import Xception\n",
    "image_size = (320,320) #adjustable parameter for processed image_size. Run time should \n",
    "\n",
    "conv_base = Xception(weights='imagenet',\n",
    "                  include_top=False,\n",
    "                  input_shape=(image_size[0], image_size[1], 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_5 (InputLayer)            (None, 320, 320, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv1 (Conv2D)           (None, 159, 159, 32) 864         input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv1_bn (BatchNormaliza (None, 159, 159, 32) 128         block1_conv1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv1_act (Activation)   (None, 159, 159, 32) 0           block1_conv1_bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv2 (Conv2D)           (None, 157, 157, 64) 18432       block1_conv1_act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv2_bn (BatchNormaliza (None, 157, 157, 64) 256         block1_conv2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block1_conv2_act (Activation)   (None, 157, 157, 64) 0           block1_conv2_bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv1 (SeparableConv2 (None, 157, 157, 128 8768        block1_conv2_act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv1_bn (BatchNormal (None, 157, 157, 128 512         block2_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv2_act (Activation (None, 157, 157, 128 0           block2_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv2 (SeparableConv2 (None, 157, 157, 128 17536       block2_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block2_sepconv2_bn (BatchNormal (None, 157, 157, 128 512         block2_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 79, 79, 128)  8192        block1_conv2_act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block2_pool (MaxPooling2D)      (None, 79, 79, 128)  0           block2_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNo (None, 79, 79, 128)  512         conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_49 (Add)                    (None, 79, 79, 128)  0           block2_pool[0][0]                \n",
      "                                                                 batch_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv1_act (Activation (None, 79, 79, 128)  0           add_49[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv1 (SeparableConv2 (None, 79, 79, 256)  33920       block3_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv1_bn (BatchNormal (None, 79, 79, 256)  1024        block3_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv2_act (Activation (None, 79, 79, 256)  0           block3_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv2 (SeparableConv2 (None, 79, 79, 256)  67840       block3_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block3_sepconv2_bn (BatchNormal (None, 79, 79, 256)  1024        block3_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 40, 40, 256)  32768       add_49[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block3_pool (MaxPooling2D)      (None, 40, 40, 256)  0           block3_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, 40, 40, 256)  1024        conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_50 (Add)                    (None, 40, 40, 256)  0           block3_pool[0][0]                \n",
      "                                                                 batch_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv1_act (Activation (None, 40, 40, 256)  0           add_50[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv1 (SeparableConv2 (None, 40, 40, 728)  188672      block4_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv1_bn (BatchNormal (None, 40, 40, 728)  2912        block4_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv2_act (Activation (None, 40, 40, 728)  0           block4_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv2 (SeparableConv2 (None, 40, 40, 728)  536536      block4_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block4_sepconv2_bn (BatchNormal (None, 40, 40, 728)  2912        block4_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, 20, 20, 728)  186368      add_50[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block4_pool (MaxPooling2D)      (None, 20, 20, 728)  0           block4_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNo (None, 20, 20, 728)  2912        conv2d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_51 (Add)                    (None, 20, 20, 728)  0           block4_pool[0][0]                \n",
      "                                                                 batch_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv1_act (Activation (None, 20, 20, 728)  0           add_51[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv1 (SeparableConv2 (None, 20, 20, 728)  536536      block5_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv1_bn (BatchNormal (None, 20, 20, 728)  2912        block5_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv2_act (Activation (None, 20, 20, 728)  0           block5_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv2 (SeparableConv2 (None, 20, 20, 728)  536536      block5_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv2_bn (BatchNormal (None, 20, 20, 728)  2912        block5_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv3_act (Activation (None, 20, 20, 728)  0           block5_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv3 (SeparableConv2 (None, 20, 20, 728)  536536      block5_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block5_sepconv3_bn (BatchNormal (None, 20, 20, 728)  2912        block5_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_52 (Add)                    (None, 20, 20, 728)  0           block5_sepconv3_bn[0][0]         \n",
      "                                                                 add_51[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv1_act (Activation (None, 20, 20, 728)  0           add_52[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv1 (SeparableConv2 (None, 20, 20, 728)  536536      block6_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv1_bn (BatchNormal (None, 20, 20, 728)  2912        block6_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv2_act (Activation (None, 20, 20, 728)  0           block6_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv2 (SeparableConv2 (None, 20, 20, 728)  536536      block6_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv2_bn (BatchNormal (None, 20, 20, 728)  2912        block6_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv3_act (Activation (None, 20, 20, 728)  0           block6_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv3 (SeparableConv2 (None, 20, 20, 728)  536536      block6_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block6_sepconv3_bn (BatchNormal (None, 20, 20, 728)  2912        block6_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_53 (Add)                    (None, 20, 20, 728)  0           block6_sepconv3_bn[0][0]         \n",
      "                                                                 add_52[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv1_act (Activation (None, 20, 20, 728)  0           add_53[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv1 (SeparableConv2 (None, 20, 20, 728)  536536      block7_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv1_bn (BatchNormal (None, 20, 20, 728)  2912        block7_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv2_act (Activation (None, 20, 20, 728)  0           block7_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv2 (SeparableConv2 (None, 20, 20, 728)  536536      block7_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv2_bn (BatchNormal (None, 20, 20, 728)  2912        block7_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv3_act (Activation (None, 20, 20, 728)  0           block7_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv3 (SeparableConv2 (None, 20, 20, 728)  536536      block7_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block7_sepconv3_bn (BatchNormal (None, 20, 20, 728)  2912        block7_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_54 (Add)                    (None, 20, 20, 728)  0           block7_sepconv3_bn[0][0]         \n",
      "                                                                 add_53[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv1_act (Activation (None, 20, 20, 728)  0           add_54[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv1 (SeparableConv2 (None, 20, 20, 728)  536536      block8_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv1_bn (BatchNormal (None, 20, 20, 728)  2912        block8_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv2_act (Activation (None, 20, 20, 728)  0           block8_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv2 (SeparableConv2 (None, 20, 20, 728)  536536      block8_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv2_bn (BatchNormal (None, 20, 20, 728)  2912        block8_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv3_act (Activation (None, 20, 20, 728)  0           block8_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv3 (SeparableConv2 (None, 20, 20, 728)  536536      block8_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block8_sepconv3_bn (BatchNormal (None, 20, 20, 728)  2912        block8_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_55 (Add)                    (None, 20, 20, 728)  0           block8_sepconv3_bn[0][0]         \n",
      "                                                                 add_54[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv1_act (Activation (None, 20, 20, 728)  0           add_55[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv1 (SeparableConv2 (None, 20, 20, 728)  536536      block9_sepconv1_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv1_bn (BatchNormal (None, 20, 20, 728)  2912        block9_sepconv1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv2_act (Activation (None, 20, 20, 728)  0           block9_sepconv1_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv2 (SeparableConv2 (None, 20, 20, 728)  536536      block9_sepconv2_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv2_bn (BatchNormal (None, 20, 20, 728)  2912        block9_sepconv2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv3_act (Activation (None, 20, 20, 728)  0           block9_sepconv2_bn[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv3 (SeparableConv2 (None, 20, 20, 728)  536536      block9_sepconv3_act[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block9_sepconv3_bn (BatchNormal (None, 20, 20, 728)  2912        block9_sepconv3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "add_56 (Add)                    (None, 20, 20, 728)  0           block9_sepconv3_bn[0][0]         \n",
      "                                                                 add_55[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv1_act (Activatio (None, 20, 20, 728)  0           add_56[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv1 (SeparableConv (None, 20, 20, 728)  536536      block10_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv1_bn (BatchNorma (None, 20, 20, 728)  2912        block10_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv2_act (Activatio (None, 20, 20, 728)  0           block10_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv2 (SeparableConv (None, 20, 20, 728)  536536      block10_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv2_bn (BatchNorma (None, 20, 20, 728)  2912        block10_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv3_act (Activatio (None, 20, 20, 728)  0           block10_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv3 (SeparableConv (None, 20, 20, 728)  536536      block10_sepconv3_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block10_sepconv3_bn (BatchNorma (None, 20, 20, 728)  2912        block10_sepconv3[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "add_57 (Add)                    (None, 20, 20, 728)  0           block10_sepconv3_bn[0][0]        \n",
      "                                                                 add_56[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv1_act (Activatio (None, 20, 20, 728)  0           add_57[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv1 (SeparableConv (None, 20, 20, 728)  536536      block11_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv1_bn (BatchNorma (None, 20, 20, 728)  2912        block11_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv2_act (Activatio (None, 20, 20, 728)  0           block11_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv2 (SeparableConv (None, 20, 20, 728)  536536      block11_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv2_bn (BatchNorma (None, 20, 20, 728)  2912        block11_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv3_act (Activatio (None, 20, 20, 728)  0           block11_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv3 (SeparableConv (None, 20, 20, 728)  536536      block11_sepconv3_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block11_sepconv3_bn (BatchNorma (None, 20, 20, 728)  2912        block11_sepconv3[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "add_58 (Add)                    (None, 20, 20, 728)  0           block11_sepconv3_bn[0][0]        \n",
      "                                                                 add_57[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv1_act (Activatio (None, 20, 20, 728)  0           add_58[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv1 (SeparableConv (None, 20, 20, 728)  536536      block12_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv1_bn (BatchNorma (None, 20, 20, 728)  2912        block12_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv2_act (Activatio (None, 20, 20, 728)  0           block12_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv2 (SeparableConv (None, 20, 20, 728)  536536      block12_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv2_bn (BatchNorma (None, 20, 20, 728)  2912        block12_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv3_act (Activatio (None, 20, 20, 728)  0           block12_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv3 (SeparableConv (None, 20, 20, 728)  536536      block12_sepconv3_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block12_sepconv3_bn (BatchNorma (None, 20, 20, 728)  2912        block12_sepconv3[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "add_59 (Add)                    (None, 20, 20, 728)  0           block12_sepconv3_bn[0][0]        \n",
      "                                                                 add_58[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv1_act (Activatio (None, 20, 20, 728)  0           add_59[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv1 (SeparableConv (None, 20, 20, 728)  536536      block13_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv1_bn (BatchNorma (None, 20, 20, 728)  2912        block13_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv2_act (Activatio (None, 20, 20, 728)  0           block13_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv2 (SeparableConv (None, 20, 20, 1024) 752024      block13_sepconv2_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block13_sepconv2_bn (BatchNorma (None, 20, 20, 1024) 4096        block13_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, 10, 10, 1024) 745472      add_59[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block13_pool (MaxPooling2D)     (None, 10, 10, 1024) 0           block13_sepconv2_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_20 (BatchNo (None, 10, 10, 1024) 4096        conv2d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_60 (Add)                    (None, 10, 10, 1024) 0           block13_pool[0][0]               \n",
      "                                                                 batch_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv1 (SeparableConv (None, 10, 10, 1536) 1582080     add_60[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv1_bn (BatchNorma (None, 10, 10, 1536) 6144        block14_sepconv1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv1_act (Activatio (None, 10, 10, 1536) 0           block14_sepconv1_bn[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv2 (SeparableConv (None, 10, 10, 2048) 3159552     block14_sepconv1_act[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv2_bn (BatchNorma (None, 10, 10, 2048) 8192        block14_sepconv2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block14_sepconv2_act (Activatio (None, 10, 10, 2048) 0           block14_sepconv2_bn[0][0]        \n",
      "==================================================================================================\n",
      "Total params: 20,861,480\n",
      "Trainable params: 20,806,952\n",
      "Non-trainable params: 54,528\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "conv_base.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ll_size = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The final feature map has shape `(ll_size, ll_size, 2048)`. That's the feature on top of which we will stick a densely-connected classifier.\n",
    "\n",
    "We will start by simply running instances of the previously-introduced `ImageDataGenerator` to extract images as Numpy arrays as well as \n",
    "their labels. We will extract features from these images simply by calling the `predict` method of the `conv_base` model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "home_dir = os.getcwd()\n",
    "train_dir = os.path.join(home_dir, 'train')\n",
    "validation_dir = os.path.join(home_dir, 'validation')\n",
    "\n",
    "datagen = ImageDataGenerator(rescale=1./255)\n",
    "batch_size = 100\n",
    "\n",
    "def extract_features(directory, sample_count):\n",
    "    features = np.zeros(shape=(sample_count, ll_size, ll_size, 2048))\n",
    "    labels = np.zeros(shape=(sample_count, classes_count))\n",
    "    generator = datagen.flow_from_directory(\n",
    "        directory,\n",
    "        target_size=image_size,\n",
    "        color_mode='rgb',\n",
    "        batch_size=batch_size,\n",
    "        class_mode='categorical')\n",
    "    i = 0\n",
    "    for inputs_batch, labels_batch in generator:\n",
    "        features_batch = conv_base.predict(inputs_batch)\n",
    "        features[i * batch_size : (i + 1) * batch_size] = features_batch\n",
    "        labels[i * batch_size : (i + 1) * batch_size] = labels_batch\n",
    "        i += 1\n",
    "        if i * batch_size >= sample_count:\n",
    "            # Note that since generators yield data indefinitely in a loop,\n",
    "            # we must `break` after every image has been seen once.\n",
    "            break\n",
    "    return features, labels\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 8806 images belonging to 4250 classes.\n",
      "Found 234 images belonging to 4250 classes.\n"
     ]
    }
   ],
   "source": [
    "train_features, train_labels = extract_features(train_dir, train_image_count)\n",
    "validation_features, validation_labels = extract_features(validation_dir, validation_image_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_features = np.reshape(train_features,(train_image_count, ll_size*ll_size*2048))\n",
    "validation_features = np.reshape(validation_features, (validation_image_count, ll_size*ll_size*2048))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_36 (Dense)             (None, 512)               51380736  \n",
      "_________________________________________________________________\n",
      "dropout_18 (Dropout)         (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_37 (Dense)             (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dropout_19 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_38 (Dense)             (None, 4250)              1092250   \n",
      "=================================================================\n",
      "Total params: 52,604,314\n",
      "Trainable params: 52,604,314\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import optimizers\n",
    "\n",
    "model = models.Sequential()\n",
    "model.add(layers.Dense(512, activation='relu', input_dim=ll_size*ll_size*2048)) # we can play around with \n",
    "model.add(layers.Dropout(0.5))\n",
    "model.add(layers.Dense(256, activation='relu'))\n",
    "model.add(layers.Dropout(0.5))\n",
    "model.add(layers.Dense(classes_count, activation='softmax'))\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/120\n",
      "8709/8709 [==============================] - 44s 5ms/step - loss: 8.3635 - acc: 0.0021 - val_loss: 8.3454 - val_acc: 0.0000e+00\n",
      "Epoch 2/120\n",
      "8709/8709 [==============================] - 31s 4ms/step - loss: 8.3168 - acc: 0.0034 - val_loss: 8.2903 - val_acc: 0.0000e+00\n",
      "Epoch 3/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 8.1954 - acc: 0.0053 - val_loss: 8.2068 - val_acc: 0.0030\n",
      "Epoch 4/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 8.0459 - acc: 0.0048 - val_loss: 8.1465 - val_acc: 0.0000e+00\n",
      "Epoch 5/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 7.8989 - acc: 0.0065 - val_loss: 8.1176 - val_acc: 0.0091\n",
      "Epoch 6/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 7.7768 - acc: 0.0076 - val_loss: 8.0955 - val_acc: 0.0060\n",
      "Epoch 7/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 7.6487 - acc: 0.0093 - val_loss: 8.1016 - val_acc: 0.0091\n",
      "Epoch 8/120\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 7.5262 - acc: 0.0095 - val_loss: 8.1262 - val_acc: 0.0060\n",
      "Epoch 9/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 7.3744 - acc: 0.0111 - val_loss: 8.1267 - val_acc: 0.0121\n",
      "Epoch 10/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 7.2638 - acc: 0.0146 - val_loss: 8.0991 - val_acc: 0.0091\n",
      "Epoch 11/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 7.1429 - acc: 0.0171 - val_loss: 8.1543 - val_acc: 0.0151\n",
      "Epoch 12/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 7.0138 - acc: 0.0171 - val_loss: 8.2003 - val_acc: 0.0151\n",
      "Epoch 13/120\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 6.9260 - acc: 0.0234 - val_loss: 8.2490 - val_acc: 0.0151\n",
      "Epoch 14/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 6.8164 - acc: 0.0232 - val_loss: 8.2917 - val_acc: 0.0151\n",
      "Epoch 15/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 6.7177 - acc: 0.0245 - val_loss: 8.3413 - val_acc: 0.0363\n",
      "Epoch 16/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 6.6169 - acc: 0.0269 - val_loss: 8.2873 - val_acc: 0.0302\n",
      "Epoch 17/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 6.5548 - acc: 0.0326 - val_loss: 8.4459 - val_acc: 0.0332\n",
      "Epoch 18/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 6.4751 - acc: 0.0299 - val_loss: 8.4385 - val_acc: 0.0363\n",
      "Epoch 19/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 6.3746 - acc: 0.0342 - val_loss: 8.6609 - val_acc: 0.0332\n",
      "Epoch 20/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 6.3195 - acc: 0.0344 - val_loss: 8.4634 - val_acc: 0.0483\n",
      "Epoch 21/120\n",
      "8709/8709 [==============================] - 31s 4ms/step - loss: 6.2145 - acc: 0.0393 - val_loss: 8.6742 - val_acc: 0.0423\n",
      "Epoch 22/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 6.1491 - acc: 0.0431 - val_loss: 8.7442 - val_acc: 0.0363\n",
      "Epoch 23/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 6.1192 - acc: 0.0429 - val_loss: 8.8304 - val_acc: 0.0393\n",
      "Epoch 24/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 6.0318 - acc: 0.0489 - val_loss: 8.8391 - val_acc: 0.0393\n",
      "Epoch 25/120\n",
      "8709/8709 [==============================] - 33s 4ms/step - loss: 5.9715 - acc: 0.0497 - val_loss: 8.9068 - val_acc: 0.0302\n",
      "Epoch 26/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 5.9336 - acc: 0.0486 - val_loss: 8.8504 - val_acc: 0.0423\n",
      "Epoch 27/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 5.9027 - acc: 0.0522 - val_loss: 8.9488 - val_acc: 0.0363\n",
      "Epoch 28/120\n",
      "8709/8709 [==============================] - 31s 4ms/step - loss: 5.8457 - acc: 0.0530 - val_loss: 9.0531 - val_acc: 0.0483\n",
      "Epoch 29/120\n",
      "8709/8709 [==============================] - 32s 4ms/step - loss: 5.7635 - acc: 0.0605 - val_loss: 9.0205 - val_acc: 0.0423\n",
      "Epoch 30/120\n",
      "8709/8709 [==============================] - 32s 4ms/step - loss: 5.7622 - acc: 0.0564 - val_loss: 9.1359 - val_acc: 0.0514\n",
      "Epoch 31/120\n",
      "8709/8709 [==============================] - 32s 4ms/step - loss: 5.6714 - acc: 0.0607 - val_loss: 9.2399 - val_acc: 0.0393\n",
      "Epoch 32/120\n",
      "8709/8709 [==============================] - 33s 4ms/step - loss: 5.6729 - acc: 0.0570 - val_loss: 9.1894 - val_acc: 0.0453\n",
      "Epoch 33/120\n",
      "8709/8709 [==============================] - 31s 4ms/step - loss: 5.6343 - acc: 0.0610 - val_loss: 9.1700 - val_acc: 0.0604\n",
      "Epoch 34/120\n",
      "8709/8709 [==============================] - 31s 4ms/step - loss: 5.5802 - acc: 0.0641 - val_loss: 9.1812 - val_acc: 0.0665\n",
      "Epoch 35/120\n",
      "8709/8709 [==============================] - 32s 4ms/step - loss: 5.5390 - acc: 0.0684 - val_loss: 9.3504 - val_acc: 0.0483\n",
      "Epoch 36/120\n",
      "8709/8709 [==============================] - 31s 4ms/step - loss: 5.5053 - acc: 0.0676 - val_loss: 9.3772 - val_acc: 0.0483\n",
      "Epoch 37/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 5.4495 - acc: 0.0737 - val_loss: 9.3959 - val_acc: 0.0483\n",
      "Epoch 38/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 5.4149 - acc: 0.0751 - val_loss: 9.4382 - val_acc: 0.0725\n",
      "Epoch 39/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 5.3730 - acc: 0.0753 - val_loss: 9.3540 - val_acc: 0.0634\n",
      "Epoch 40/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 5.3533 - acc: 0.0823 - val_loss: 9.5187 - val_acc: 0.0634\n",
      "Epoch 41/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 5.2702 - acc: 0.0827 - val_loss: 9.5355 - val_acc: 0.0604\n",
      "Epoch 42/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 5.2868 - acc: 0.0775 - val_loss: 9.6511 - val_acc: 0.0544\n",
      "Epoch 43/120\n",
      "8709/8709 [==============================] - 31s 4ms/step - loss: 5.2344 - acc: 0.0860 - val_loss: 9.6742 - val_acc: 0.0755\n",
      "Epoch 44/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 5.2314 - acc: 0.0811 - val_loss: 9.6300 - val_acc: 0.0665\n",
      "Epoch 45/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 5.1820 - acc: 0.0890 - val_loss: 9.6567 - val_acc: 0.0785\n",
      "Epoch 46/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 5.1643 - acc: 0.0885 - val_loss: 9.6702 - val_acc: 0.0785\n",
      "Epoch 47/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 5.1398 - acc: 0.0915 - val_loss: 9.8668 - val_acc: 0.0755\n",
      "Epoch 48/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 5.1375 - acc: 0.0830 - val_loss: 9.7226 - val_acc: 0.0846\n",
      "Epoch 49/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 5.0912 - acc: 0.0930 - val_loss: 9.8101 - val_acc: 0.0725\n",
      "Epoch 50/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 5.0504 - acc: 0.0955 - val_loss: 9.8127 - val_acc: 0.0755\n",
      "Epoch 51/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 5.0441 - acc: 0.0924 - val_loss: 9.9357 - val_acc: 0.0665\n",
      "Epoch 52/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 4.9928 - acc: 0.0977 - val_loss: 9.9092 - val_acc: 0.0846\n",
      "Epoch 53/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 4.9858 - acc: 0.1035 - val_loss: 9.9304 - val_acc: 0.0846\n",
      "Epoch 54/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.9234 - acc: 0.1010 - val_loss: 10.0225 - val_acc: 0.0785\n",
      "Epoch 55/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 4.9720 - acc: 0.0978 - val_loss: 10.0894 - val_acc: 0.0906\n",
      "Epoch 56/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.9395 - acc: 0.1010 - val_loss: 10.0301 - val_acc: 0.0876\n",
      "Epoch 57/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 4.9106 - acc: 0.1075 - val_loss: 10.1675 - val_acc: 0.0967\n",
      "Epoch 58/120\n",
      "8709/8709 [==============================] - 32s 4ms/step - loss: 4.8950 - acc: 0.1030 - val_loss: 10.1098 - val_acc: 0.0937\n",
      "Epoch 59/120\n",
      "8709/8709 [==============================] - 32s 4ms/step - loss: 4.8815 - acc: 0.1087 - val_loss: 10.0638 - val_acc: 0.0937\n",
      "Epoch 60/120\n",
      "8709/8709 [==============================] - 32s 4ms/step - loss: 4.8377 - acc: 0.1047 - val_loss: 10.1535 - val_acc: 0.0906\n",
      "Epoch 61/120\n",
      "8709/8709 [==============================] - 31s 4ms/step - loss: 4.7868 - acc: 0.1131 - val_loss: 10.2279 - val_acc: 0.0937\n",
      "Epoch 62/120\n",
      "8709/8709 [==============================] - 33s 4ms/step - loss: 4.8334 - acc: 0.1205 - val_loss: 10.2244 - val_acc: 0.0906\n",
      "Epoch 63/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 4.7721 - acc: 0.1175 - val_loss: 10.1630 - val_acc: 0.0937\n",
      "Epoch 64/120\n",
      "8709/8709 [==============================] - 31s 4ms/step - loss: 4.7582 - acc: 0.1178 - val_loss: 10.1843 - val_acc: 0.0937\n",
      "Epoch 65/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 4.7594 - acc: 0.1233 - val_loss: 10.2712 - val_acc: 0.0937\n",
      "Epoch 66/120\n",
      "8709/8709 [==============================] - 32s 4ms/step - loss: 4.7342 - acc: 0.1247 - val_loss: 10.2889 - val_acc: 0.0937\n",
      "Epoch 67/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 4.6839 - acc: 0.1227 - val_loss: 10.3756 - val_acc: 0.0876\n",
      "Epoch 68/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.7296 - acc: 0.1117 - val_loss: 10.3351 - val_acc: 0.0997\n",
      "Epoch 69/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 4.6776 - acc: 0.1211 - val_loss: 10.3371 - val_acc: 0.1057\n",
      "Epoch 70/120\n",
      "8709/8709 [==============================] - 34s 4ms/step - loss: 4.6913 - acc: 0.1182 - val_loss: 10.3854 - val_acc: 0.0997\n",
      "Epoch 71/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 4.6722 - acc: 0.1246 - val_loss: 10.3384 - val_acc: 0.0997\n",
      "Epoch 72/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.6321 - acc: 0.1200 - val_loss: 10.3786 - val_acc: 0.1057\n",
      "Epoch 73/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.6363 - acc: 0.1296 - val_loss: 10.3340 - val_acc: 0.1057\n",
      "Epoch 74/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 4.6235 - acc: 0.1244 - val_loss: 10.3660 - val_acc: 0.0997\n",
      "Epoch 75/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.6131 - acc: 0.1244 - val_loss: 10.4423 - val_acc: 0.0906\n",
      "Epoch 76/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.5769 - acc: 0.1311 - val_loss: 10.2968 - val_acc: 0.0967\n",
      "Epoch 77/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.5372 - acc: 0.1397 - val_loss: 10.5100 - val_acc: 0.1178\n",
      "Epoch 78/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 4.5752 - acc: 0.1277 - val_loss: 10.4716 - val_acc: 0.0997\n",
      "Epoch 79/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 4.5417 - acc: 0.1355 - val_loss: 10.4759 - val_acc: 0.0967\n",
      "Epoch 80/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.5651 - acc: 0.1299 - val_loss: 10.5524 - val_acc: 0.1027\n",
      "Epoch 81/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.5447 - acc: 0.1368 - val_loss: 10.5252 - val_acc: 0.1088\n",
      "Epoch 82/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 4.5279 - acc: 0.1424 - val_loss: 10.5511 - val_acc: 0.1178\n",
      "Epoch 83/120\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 4.4781 - acc: 0.1361 - val_loss: 10.5997 - val_acc: 0.1088\n",
      "Epoch 84/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.5033 - acc: 0.1345 - val_loss: 10.6290 - val_acc: 0.0997\n",
      "Epoch 85/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.5178 - acc: 0.1302 - val_loss: 10.5559 - val_acc: 0.1088\n",
      "Epoch 86/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.4762 - acc: 0.1373 - val_loss: 10.5795 - val_acc: 0.0997\n",
      "Epoch 87/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.4831 - acc: 0.1373 - val_loss: 10.5771 - val_acc: 0.1027\n",
      "Epoch 88/120\n",
      "8709/8709 [==============================] - 30s 4ms/step - loss: 4.4428 - acc: 0.1439 - val_loss: 10.5525 - val_acc: 0.1178\n",
      "Epoch 89/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.4368 - acc: 0.1424 - val_loss: 10.6154 - val_acc: 0.1088\n",
      "Epoch 90/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.4208 - acc: 0.1432 - val_loss: 10.6201 - val_acc: 0.1057\n",
      "Epoch 91/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.3998 - acc: 0.1515 - val_loss: 10.6362 - val_acc: 0.1118\n",
      "Epoch 92/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.3461 - acc: 0.1564 - val_loss: 10.7022 - val_acc: 0.1027\n",
      "Epoch 93/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.3899 - acc: 0.1457 - val_loss: 10.6236 - val_acc: 0.1148\n",
      "Epoch 94/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.3968 - acc: 0.1433 - val_loss: 10.7762 - val_acc: 0.1118\n",
      "Epoch 95/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.3590 - acc: 0.1579 - val_loss: 10.6850 - val_acc: 0.1269\n",
      "Epoch 96/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.3929 - acc: 0.1435 - val_loss: 10.6543 - val_acc: 0.1208\n",
      "Epoch 97/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.3468 - acc: 0.1520 - val_loss: 10.7182 - val_acc: 0.1178\n",
      "Epoch 98/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.3060 - acc: 0.1546 - val_loss: 10.6687 - val_acc: 0.1118\n",
      "Epoch 99/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.3086 - acc: 0.1554 - val_loss: 10.7216 - val_acc: 0.1299\n",
      "Epoch 100/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.3104 - acc: 0.1570 - val_loss: 10.7567 - val_acc: 0.1208\n",
      "Epoch 101/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.2740 - acc: 0.1596 - val_loss: 10.7372 - val_acc: 0.1148\n",
      "Epoch 102/120\n",
      "8709/8709 [==============================] - 31s 4ms/step - loss: 4.3199 - acc: 0.1602 - val_loss: 10.7596 - val_acc: 0.1208\n",
      "Epoch 103/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.3157 - acc: 0.1546 - val_loss: 10.7140 - val_acc: 0.1118\n",
      "Epoch 104/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.2882 - acc: 0.1585 - val_loss: 10.7850 - val_acc: 0.1088\n",
      "Epoch 105/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.3497 - acc: 0.1487 - val_loss: 10.8113 - val_acc: 0.1239\n",
      "Epoch 106/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.2550 - acc: 0.1622 - val_loss: 10.8768 - val_acc: 0.1148\n",
      "Epoch 107/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.2596 - acc: 0.1658 - val_loss: 10.7934 - val_acc: 0.1088\n",
      "Epoch 108/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.2342 - acc: 0.1610 - val_loss: 10.7927 - val_acc: 0.1239\n",
      "Epoch 109/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.2602 - acc: 0.1558 - val_loss: 10.8538 - val_acc: 0.1299\n",
      "Epoch 110/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.2293 - acc: 0.1605 - val_loss: 10.8587 - val_acc: 0.1239\n",
      "Epoch 111/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.2240 - acc: 0.1597 - val_loss: 10.7526 - val_acc: 0.1239\n",
      "Epoch 112/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.2174 - acc: 0.1663 - val_loss: 10.8801 - val_acc: 0.1178\n",
      "Epoch 113/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.2180 - acc: 0.1671 - val_loss: 10.9232 - val_acc: 0.1239\n",
      "Epoch 114/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.2440 - acc: 0.1634 - val_loss: 11.0737 - val_acc: 0.1057\n",
      "Epoch 115/120\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 4.1904 - acc: 0.1687 - val_loss: 11.0238 - val_acc: 0.1208\n",
      "Epoch 116/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.1997 - acc: 0.1634 - val_loss: 11.0293 - val_acc: 0.1148\n",
      "Epoch 117/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.2158 - acc: 0.1620 - val_loss: 11.0463 - val_acc: 0.1088\n",
      "Epoch 118/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.1861 - acc: 0.1657 - val_loss: 11.0193 - val_acc: 0.1148\n",
      "Epoch 119/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.2370 - acc: 0.1652 - val_loss: 10.9697 - val_acc: 0.1269\n",
      "Epoch 120/120\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 4.1612 - acc: 0.1684 - val_loss: 11.0198 - val_acc: 0.1208\n"
     ]
    }
   ],
   "source": [
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import optimizers\n",
    "\n",
    "model = models.Sequential()\n",
    "model.add(layers.Dense(128, activation='relu', input_dim=ll_size*ll_size*2048)) # we can play around with \n",
    "model.add(layers.Dropout(0.5))\n",
    "model.add(layers.Dense(classes_count, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "model.compile(optimizer=optimizers.Adam(lr=2e-4),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=120,\n",
    "                    batch_size=100,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('weights/adam_no_aug_256_0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/120\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 4.0995 - acc: 0.1701 - val_loss: 11.0364 - val_acc: 0.1208\n",
      "Epoch 2/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 4.1032 - acc: 0.1767 - val_loss: 11.0061 - val_acc: 0.1208\n",
      "Epoch 3/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 4.0807 - acc: 0.1783 - val_loss: 11.0764 - val_acc: 0.1208\n",
      "Epoch 4/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 4.0807 - acc: 0.1729 - val_loss: 10.9869 - val_acc: 0.1148\n",
      "Epoch 5/120\n",
      "8709/8709 [==============================] - 25s 3ms/step - loss: 4.0402 - acc: 0.1790 - val_loss: 11.0337 - val_acc: 0.1208\n",
      "Epoch 6/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 4.0822 - acc: 0.1823 - val_loss: 10.9985 - val_acc: 0.1239\n",
      "Epoch 7/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 4.0671 - acc: 0.1812 - val_loss: 10.9989 - val_acc: 0.1208\n",
      "Epoch 8/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9902 - acc: 0.1877 - val_loss: 11.0291 - val_acc: 0.1208\n",
      "Epoch 9/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 4.0675 - acc: 0.1765 - val_loss: 11.0376 - val_acc: 0.1269\n",
      "Epoch 10/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 4.0284 - acc: 0.1823 - val_loss: 11.0492 - val_acc: 0.1239\n",
      "Epoch 11/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 4.0266 - acc: 0.1807 - val_loss: 10.9914 - val_acc: 0.1148\n",
      "Epoch 12/120\n",
      "8709/8709 [==============================] - 20s 2ms/step - loss: 3.9833 - acc: 0.1895 - val_loss: 11.0323 - val_acc: 0.1299\n",
      "Epoch 13/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 4.0119 - acc: 0.1919 - val_loss: 11.0634 - val_acc: 0.1208\n",
      "Epoch 14/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 4.0104 - acc: 0.1852 - val_loss: 11.0379 - val_acc: 0.1239\n",
      "Epoch 15/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 4.0180 - acc: 0.1799 - val_loss: 11.0257 - val_acc: 0.1178\n",
      "Epoch 16/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 4.0398 - acc: 0.1830 - val_loss: 11.0536 - val_acc: 0.1178\n",
      "Epoch 17/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 4.0291 - acc: 0.1821 - val_loss: 11.0371 - val_acc: 0.1208\n",
      "Epoch 18/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9920 - acc: 0.1869 - val_loss: 11.0330 - val_acc: 0.1148\n",
      "Epoch 19/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9883 - acc: 0.1912 - val_loss: 11.0491 - val_acc: 0.1299\n",
      "Epoch 20/120\n",
      "8709/8709 [==============================] - 20s 2ms/step - loss: 4.0000 - acc: 0.1879 - val_loss: 11.0649 - val_acc: 0.1148\n",
      "Epoch 21/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 4.0157 - acc: 0.1860 - val_loss: 11.0410 - val_acc: 0.1299\n",
      "Epoch 22/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9772 - acc: 0.1945 - val_loss: 11.0690 - val_acc: 0.1299\n",
      "Epoch 23/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9880 - acc: 0.1901 - val_loss: 11.0522 - val_acc: 0.1329\n",
      "Epoch 24/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9738 - acc: 0.1898 - val_loss: 11.0539 - val_acc: 0.1208\n",
      "Epoch 25/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.9773 - acc: 0.1895 - val_loss: 11.0597 - val_acc: 0.1299\n",
      "Epoch 26/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9937 - acc: 0.1901 - val_loss: 11.0717 - val_acc: 0.1299\n",
      "Epoch 27/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 4.0206 - acc: 0.1837 - val_loss: 11.0858 - val_acc: 0.1239\n",
      "Epoch 28/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9877 - acc: 0.1869 - val_loss: 11.0742 - val_acc: 0.1178\n",
      "Epoch 29/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.9383 - acc: 0.1945 - val_loss: 11.0931 - val_acc: 0.1299\n",
      "Epoch 30/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9890 - acc: 0.1893 - val_loss: 11.0899 - val_acc: 0.1239\n",
      "Epoch 31/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.9461 - acc: 0.1939 - val_loss: 11.0722 - val_acc: 0.1299\n",
      "Epoch 32/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9779 - acc: 0.1901 - val_loss: 11.0965 - val_acc: 0.1269\n",
      "Epoch 33/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9445 - acc: 0.2001 - val_loss: 11.0942 - val_acc: 0.1329\n",
      "Epoch 34/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.9559 - acc: 0.1903 - val_loss: 11.0876 - val_acc: 0.1239\n",
      "Epoch 35/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.9847 - acc: 0.1900 - val_loss: 11.0981 - val_acc: 0.1299\n",
      "Epoch 36/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9542 - acc: 0.2037 - val_loss: 11.0970 - val_acc: 0.1269\n",
      "Epoch 37/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9685 - acc: 0.1927 - val_loss: 11.0817 - val_acc: 0.1299\n",
      "Epoch 38/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.9468 - acc: 0.1937 - val_loss: 11.0922 - val_acc: 0.1329\n",
      "Epoch 39/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9436 - acc: 0.1927 - val_loss: 11.0941 - val_acc: 0.1329\n",
      "Epoch 40/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.9331 - acc: 0.1974 - val_loss: 11.1152 - val_acc: 0.1299\n",
      "Epoch 41/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.9574 - acc: 0.1887 - val_loss: 11.0932 - val_acc: 0.1269\n",
      "Epoch 42/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9349 - acc: 0.1951 - val_loss: 11.0964 - val_acc: 0.1299\n",
      "Epoch 43/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.9378 - acc: 0.1977 - val_loss: 11.0744 - val_acc: 0.1299\n",
      "Epoch 44/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9426 - acc: 0.1907 - val_loss: 11.1100 - val_acc: 0.1329\n",
      "Epoch 45/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9266 - acc: 0.1957 - val_loss: 11.1158 - val_acc: 0.1269\n",
      "Epoch 46/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9141 - acc: 0.2005 - val_loss: 11.1072 - val_acc: 0.1329\n",
      "Epoch 47/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9783 - acc: 0.1928 - val_loss: 11.1374 - val_acc: 0.1329\n",
      "Epoch 48/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9352 - acc: 0.1998 - val_loss: 11.0920 - val_acc: 0.1360\n",
      "Epoch 49/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9370 - acc: 0.1947 - val_loss: 11.1253 - val_acc: 0.1299\n",
      "Epoch 50/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9248 - acc: 0.1982 - val_loss: 11.1139 - val_acc: 0.1360\n",
      "Epoch 51/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9816 - acc: 0.1898 - val_loss: 11.1481 - val_acc: 0.1360\n",
      "Epoch 52/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9167 - acc: 0.2032 - val_loss: 11.1705 - val_acc: 0.1299\n",
      "Epoch 53/120\n",
      "8709/8709 [==============================] - 22s 2ms/step - loss: 3.9196 - acc: 0.1945 - val_loss: 11.1725 - val_acc: 0.1299\n",
      "Epoch 54/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.9416 - acc: 0.1860 - val_loss: 11.1393 - val_acc: 0.1299\n",
      "Epoch 55/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8995 - acc: 0.1950 - val_loss: 11.1201 - val_acc: 0.1269\n",
      "Epoch 56/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9277 - acc: 0.1988 - val_loss: 11.1401 - val_acc: 0.1360\n",
      "Epoch 57/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9354 - acc: 0.1927 - val_loss: 11.1576 - val_acc: 0.1329\n",
      "Epoch 58/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9193 - acc: 0.2004 - val_loss: 11.1457 - val_acc: 0.1269\n",
      "Epoch 59/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.9129 - acc: 0.1953 - val_loss: 11.1329 - val_acc: 0.1329\n",
      "Epoch 60/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.9241 - acc: 0.2014 - val_loss: 11.1753 - val_acc: 0.1299\n",
      "Epoch 61/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9223 - acc: 0.1974 - val_loss: 11.2092 - val_acc: 0.1269\n",
      "Epoch 62/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8852 - acc: 0.2014 - val_loss: 11.2133 - val_acc: 0.1329\n",
      "Epoch 63/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9151 - acc: 0.2024 - val_loss: 11.1919 - val_acc: 0.1239\n",
      "Epoch 64/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8865 - acc: 0.1981 - val_loss: 11.1573 - val_acc: 0.1299\n",
      "Epoch 65/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8528 - acc: 0.2102 - val_loss: 11.1483 - val_acc: 0.1299\n",
      "Epoch 66/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8885 - acc: 0.2006 - val_loss: 11.1953 - val_acc: 0.1239\n",
      "Epoch 67/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9046 - acc: 0.1973 - val_loss: 11.1918 - val_acc: 0.1269\n",
      "Epoch 68/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9313 - acc: 0.1954 - val_loss: 11.1926 - val_acc: 0.1299\n",
      "Epoch 69/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8864 - acc: 0.2059 - val_loss: 11.2293 - val_acc: 0.1299\n",
      "Epoch 70/120\n",
      "8709/8709 [==============================] - 20s 2ms/step - loss: 3.9024 - acc: 0.1958 - val_loss: 11.2150 - val_acc: 0.1329\n",
      "Epoch 71/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9142 - acc: 0.1978 - val_loss: 11.1647 - val_acc: 0.1299\n",
      "Epoch 72/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.9153 - acc: 0.1963 - val_loss: 11.1928 - val_acc: 0.1269\n",
      "Epoch 73/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8973 - acc: 0.1993 - val_loss: 11.1901 - val_acc: 0.1360\n",
      "Epoch 74/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9023 - acc: 0.1975 - val_loss: 11.1867 - val_acc: 0.1329\n",
      "Epoch 75/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8941 - acc: 0.2046 - val_loss: 11.2010 - val_acc: 0.1299\n",
      "Epoch 76/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.8754 - acc: 0.2019 - val_loss: 11.1807 - val_acc: 0.1299\n",
      "Epoch 77/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.8911 - acc: 0.2044 - val_loss: 11.1973 - val_acc: 0.1390\n",
      "Epoch 78/120\n",
      "8709/8709 [==============================] - 20s 2ms/step - loss: 3.8524 - acc: 0.2102 - val_loss: 11.1941 - val_acc: 0.1360\n",
      "Epoch 79/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9087 - acc: 0.1988 - val_loss: 11.1945 - val_acc: 0.1360\n",
      "Epoch 80/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8755 - acc: 0.2101 - val_loss: 11.1996 - val_acc: 0.1360\n",
      "Epoch 81/120\n",
      "8709/8709 [==============================] - 20s 2ms/step - loss: 3.8923 - acc: 0.1959 - val_loss: 11.2089 - val_acc: 0.1329\n",
      "Epoch 82/120\n",
      "8709/8709 [==============================] - 20s 2ms/step - loss: 3.8679 - acc: 0.2039 - val_loss: 11.2451 - val_acc: 0.1299\n",
      "Epoch 83/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8746 - acc: 0.2023 - val_loss: 11.2379 - val_acc: 0.1299\n",
      "Epoch 84/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8594 - acc: 0.2048 - val_loss: 11.2267 - val_acc: 0.1239\n",
      "Epoch 85/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8542 - acc: 0.2038 - val_loss: 11.2491 - val_acc: 0.1299\n",
      "Epoch 86/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8757 - acc: 0.2004 - val_loss: 11.2565 - val_acc: 0.1269\n",
      "Epoch 87/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.8725 - acc: 0.2081 - val_loss: 11.2460 - val_acc: 0.1269\n",
      "Epoch 88/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.8603 - acc: 0.2071 - val_loss: 11.2429 - val_acc: 0.1239\n",
      "Epoch 89/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.8487 - acc: 0.2030 - val_loss: 11.1962 - val_acc: 0.1269\n",
      "Epoch 90/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8600 - acc: 0.2123 - val_loss: 11.2576 - val_acc: 0.1239\n",
      "Epoch 91/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9017 - acc: 0.2067 - val_loss: 11.2641 - val_acc: 0.1329\n",
      "Epoch 92/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8980 - acc: 0.2009 - val_loss: 11.2624 - val_acc: 0.1299\n",
      "Epoch 93/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8679 - acc: 0.2012 - val_loss: 11.2419 - val_acc: 0.1329\n",
      "Epoch 94/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.8623 - acc: 0.2024 - val_loss: 11.2762 - val_acc: 0.1299\n",
      "Epoch 95/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8421 - acc: 0.2141 - val_loss: 11.2550 - val_acc: 0.1329\n",
      "Epoch 96/120\n",
      "8709/8709 [==============================] - 22s 2ms/step - loss: 3.8363 - acc: 0.2076 - val_loss: 11.2581 - val_acc: 0.1269\n",
      "Epoch 97/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.8733 - acc: 0.2027 - val_loss: 11.2897 - val_acc: 0.1329\n",
      "Epoch 98/120\n",
      "8709/8709 [==============================] - 22s 2ms/step - loss: 3.8818 - acc: 0.2012 - val_loss: 11.2638 - val_acc: 0.1269\n",
      "Epoch 99/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.9079 - acc: 0.2004 - val_loss: 11.2795 - val_acc: 0.1299\n",
      "Epoch 100/120\n",
      "8709/8709 [==============================] - 22s 2ms/step - loss: 3.8354 - acc: 0.2025 - val_loss: 11.3005 - val_acc: 0.1269\n",
      "Epoch 101/120\n",
      "8709/8709 [==============================] - 22s 2ms/step - loss: 3.8703 - acc: 0.2046 - val_loss: 11.2927 - val_acc: 0.1329\n",
      "Epoch 102/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.8822 - acc: 0.2017 - val_loss: 11.3107 - val_acc: 0.1299\n",
      "Epoch 103/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.8411 - acc: 0.2083 - val_loss: 11.2599 - val_acc: 0.1269\n",
      "Epoch 104/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8387 - acc: 0.2065 - val_loss: 11.2714 - val_acc: 0.1420\n",
      "Epoch 105/120\n",
      "8709/8709 [==============================] - 22s 2ms/step - loss: 3.8746 - acc: 0.2044 - val_loss: 11.2928 - val_acc: 0.1329\n",
      "Epoch 106/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.8345 - acc: 0.2069 - val_loss: 11.3125 - val_acc: 0.1360\n",
      "Epoch 107/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.8184 - acc: 0.2148 - val_loss: 11.2797 - val_acc: 0.1329\n",
      "Epoch 108/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.8284 - acc: 0.2093 - val_loss: 11.3294 - val_acc: 0.1329\n",
      "Epoch 109/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8460 - acc: 0.2085 - val_loss: 11.3288 - val_acc: 0.1360\n",
      "Epoch 110/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.8138 - acc: 0.2140 - val_loss: 11.3176 - val_acc: 0.1329\n",
      "Epoch 111/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.8481 - acc: 0.2043 - val_loss: 11.3376 - val_acc: 0.1329\n",
      "Epoch 112/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8683 - acc: 0.2027 - val_loss: 11.3187 - val_acc: 0.1299\n",
      "Epoch 113/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8413 - acc: 0.2070 - val_loss: 11.2989 - val_acc: 0.1360\n",
      "Epoch 114/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8280 - acc: 0.2104 - val_loss: 11.3294 - val_acc: 0.1269\n",
      "Epoch 115/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8503 - acc: 0.2040 - val_loss: 11.3219 - val_acc: 0.1299\n",
      "Epoch 116/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.8004 - acc: 0.2147 - val_loss: 11.3455 - val_acc: 0.1329\n",
      "Epoch 117/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.8500 - acc: 0.2036 - val_loss: 11.3193 - val_acc: 0.1329\n",
      "Epoch 118/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8327 - acc: 0.2077 - val_loss: 11.3252 - val_acc: 0.1360\n",
      "Epoch 119/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8382 - acc: 0.2046 - val_loss: 11.3586 - val_acc: 0.1329\n",
      "Epoch 120/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8560 - acc: 0.2061 - val_loss: 11.3361 - val_acc: 0.1360\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=optimizers.Adam(lr=5e-5),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=120,\n",
    "                    batch_size=150,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('weights/adam_no_aug_256_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/120\n",
      "8709/8709 [==============================] - 25s 3ms/step - loss: 3.8226 - acc: 0.2071 - val_loss: 11.3482 - val_acc: 0.1329\n",
      "Epoch 2/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8117 - acc: 0.2055 - val_loss: 11.3395 - val_acc: 0.1329\n",
      "Epoch 3/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7992 - acc: 0.2115 - val_loss: 11.3445 - val_acc: 0.1329\n",
      "Epoch 4/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7907 - acc: 0.2099 - val_loss: 11.3540 - val_acc: 0.1329\n",
      "Epoch 5/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.8094 - acc: 0.2114 - val_loss: 11.3487 - val_acc: 0.1329\n",
      "Epoch 6/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.8327 - acc: 0.2069 - val_loss: 11.3371 - val_acc: 0.1329\n",
      "Epoch 7/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8608 - acc: 0.2022 - val_loss: 11.3589 - val_acc: 0.1329\n",
      "Epoch 8/120\n",
      "8709/8709 [==============================] - 22s 2ms/step - loss: 3.8056 - acc: 0.2084 - val_loss: 11.3583 - val_acc: 0.1299\n",
      "Epoch 9/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7736 - acc: 0.2161 - val_loss: 11.3561 - val_acc: 0.1299\n",
      "Epoch 10/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8211 - acc: 0.2102 - val_loss: 11.3657 - val_acc: 0.1299\n",
      "Epoch 11/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.8084 - acc: 0.2073 - val_loss: 11.3468 - val_acc: 0.1299\n",
      "Epoch 12/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8056 - acc: 0.2123 - val_loss: 11.3790 - val_acc: 0.1299\n",
      "Epoch 13/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.8200 - acc: 0.2073 - val_loss: 11.3778 - val_acc: 0.1299\n",
      "Epoch 14/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7991 - acc: 0.2121 - val_loss: 11.3533 - val_acc: 0.1299\n",
      "Epoch 15/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7549 - acc: 0.2183 - val_loss: 11.3493 - val_acc: 0.1329\n",
      "Epoch 16/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7896 - acc: 0.2139 - val_loss: 11.3705 - val_acc: 0.1299\n",
      "Epoch 17/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7756 - acc: 0.2170 - val_loss: 11.3595 - val_acc: 0.1299\n",
      "Epoch 18/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7811 - acc: 0.2145 - val_loss: 11.3633 - val_acc: 0.1329\n",
      "Epoch 19/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8076 - acc: 0.2109 - val_loss: 11.3686 - val_acc: 0.1299\n",
      "Epoch 20/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7809 - acc: 0.2155 - val_loss: 11.3690 - val_acc: 0.1299\n",
      "Epoch 21/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7751 - acc: 0.2198 - val_loss: 11.3662 - val_acc: 0.1299\n",
      "Epoch 22/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7732 - acc: 0.2160 - val_loss: 11.3728 - val_acc: 0.1299\n",
      "Epoch 23/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8114 - acc: 0.2104 - val_loss: 11.3646 - val_acc: 0.1299\n",
      "Epoch 24/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8171 - acc: 0.2083 - val_loss: 11.3800 - val_acc: 0.1299\n",
      "Epoch 25/120\n",
      "8709/8709 [==============================] - 22s 2ms/step - loss: 3.7608 - acc: 0.2174 - val_loss: 11.4037 - val_acc: 0.1329\n",
      "Epoch 26/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8088 - acc: 0.2131 - val_loss: 11.3661 - val_acc: 0.1299\n",
      "Epoch 27/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.8088 - acc: 0.2109 - val_loss: 11.3753 - val_acc: 0.1299\n",
      "Epoch 28/120\n",
      "8709/8709 [==============================] - 26s 3ms/step - loss: 3.7885 - acc: 0.2118 - val_loss: 11.3832 - val_acc: 0.1269\n",
      "Epoch 29/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7835 - acc: 0.2145 - val_loss: 11.3796 - val_acc: 0.1299\n",
      "Epoch 30/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7962 - acc: 0.2078 - val_loss: 11.3853 - val_acc: 0.1299\n",
      "Epoch 31/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8263 - acc: 0.2110 - val_loss: 11.3810 - val_acc: 0.1269\n",
      "Epoch 32/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.7736 - acc: 0.2189 - val_loss: 11.3937 - val_acc: 0.1269\n",
      "Epoch 33/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8069 - acc: 0.2133 - val_loss: 11.3872 - val_acc: 0.1299\n",
      "Epoch 34/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7538 - acc: 0.2146 - val_loss: 11.3906 - val_acc: 0.1299\n",
      "Epoch 35/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7748 - acc: 0.2155 - val_loss: 11.3920 - val_acc: 0.1299\n",
      "Epoch 36/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.8209 - acc: 0.2055 - val_loss: 11.4016 - val_acc: 0.1329\n",
      "Epoch 37/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7553 - acc: 0.2171 - val_loss: 11.3964 - val_acc: 0.1329\n",
      "Epoch 38/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.7836 - acc: 0.2101 - val_loss: 11.4059 - val_acc: 0.1299\n",
      "Epoch 39/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7672 - acc: 0.2175 - val_loss: 11.3950 - val_acc: 0.1299\n",
      "Epoch 40/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7941 - acc: 0.2136 - val_loss: 11.4107 - val_acc: 0.1299\n",
      "Epoch 41/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.7877 - acc: 0.2117 - val_loss: 11.4017 - val_acc: 0.1299\n",
      "Epoch 42/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.7883 - acc: 0.2148 - val_loss: 11.3897 - val_acc: 0.1299\n",
      "Epoch 43/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7664 - acc: 0.2200 - val_loss: 11.3981 - val_acc: 0.1299\n",
      "Epoch 44/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.7851 - acc: 0.2082 - val_loss: 11.3982 - val_acc: 0.1360\n",
      "Epoch 45/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.8093 - acc: 0.2120 - val_loss: 11.4014 - val_acc: 0.1269\n",
      "Epoch 46/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8008 - acc: 0.2150 - val_loss: 11.4071 - val_acc: 0.1329\n",
      "Epoch 47/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7810 - acc: 0.2151 - val_loss: 11.3907 - val_acc: 0.1329\n",
      "Epoch 48/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8074 - acc: 0.2155 - val_loss: 11.4036 - val_acc: 0.1329\n",
      "Epoch 49/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.7438 - acc: 0.2228 - val_loss: 11.4069 - val_acc: 0.1329\n",
      "Epoch 50/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7900 - acc: 0.2076 - val_loss: 11.4053 - val_acc: 0.1329\n",
      "Epoch 51/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8173 - acc: 0.2102 - val_loss: 11.4065 - val_acc: 0.1360\n",
      "Epoch 52/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7774 - acc: 0.2137 - val_loss: 11.3912 - val_acc: 0.1329\n",
      "Epoch 53/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7997 - acc: 0.2084 - val_loss: 11.4025 - val_acc: 0.1299\n",
      "Epoch 54/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.7776 - acc: 0.2198 - val_loss: 11.4210 - val_acc: 0.1329\n",
      "Epoch 55/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7518 - acc: 0.2215 - val_loss: 11.4080 - val_acc: 0.1329\n",
      "Epoch 56/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7673 - acc: 0.2167 - val_loss: 11.4279 - val_acc: 0.1329\n",
      "Epoch 57/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.8178 - acc: 0.2099 - val_loss: 11.4260 - val_acc: 0.1299\n",
      "Epoch 58/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.7960 - acc: 0.2081 - val_loss: 11.4048 - val_acc: 0.1329\n",
      "Epoch 59/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7806 - acc: 0.2163 - val_loss: 11.4004 - val_acc: 0.1269\n",
      "Epoch 60/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7879 - acc: 0.2170 - val_loss: 11.4224 - val_acc: 0.1329\n",
      "Epoch 61/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7979 - acc: 0.2102 - val_loss: 11.4229 - val_acc: 0.1329\n",
      "Epoch 62/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7871 - acc: 0.2118 - val_loss: 11.4265 - val_acc: 0.1360\n",
      "Epoch 63/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.7685 - acc: 0.2133 - val_loss: 11.4286 - val_acc: 0.1329\n",
      "Epoch 64/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7887 - acc: 0.2130 - val_loss: 11.4388 - val_acc: 0.1299\n",
      "Epoch 65/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.7651 - acc: 0.2174 - val_loss: 11.4237 - val_acc: 0.1299\n",
      "Epoch 66/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7696 - acc: 0.2116 - val_loss: 11.4339 - val_acc: 0.1299\n",
      "Epoch 67/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7691 - acc: 0.2159 - val_loss: 11.4237 - val_acc: 0.1329\n",
      "Epoch 68/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7549 - acc: 0.2141 - val_loss: 11.4501 - val_acc: 0.1299\n",
      "Epoch 69/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7466 - acc: 0.2127 - val_loss: 11.4369 - val_acc: 0.1329\n",
      "Epoch 70/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7604 - acc: 0.2166 - val_loss: 11.4306 - val_acc: 0.1329\n",
      "Epoch 71/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.7407 - acc: 0.2183 - val_loss: 11.4301 - val_acc: 0.1360\n",
      "Epoch 72/120\n",
      "8709/8709 [==============================] - 20s 2ms/step - loss: 3.7716 - acc: 0.2122 - val_loss: 11.4289 - val_acc: 0.1360\n",
      "Epoch 73/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7566 - acc: 0.2160 - val_loss: 11.4332 - val_acc: 0.1360\n",
      "Epoch 74/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7878 - acc: 0.2148 - val_loss: 11.4241 - val_acc: 0.1360\n",
      "Epoch 75/120\n",
      "8709/8709 [==============================] - 20s 2ms/step - loss: 3.7789 - acc: 0.2151 - val_loss: 11.4542 - val_acc: 0.1329\n",
      "Epoch 76/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.7244 - acc: 0.2260 - val_loss: 11.4469 - val_acc: 0.1299\n",
      "Epoch 77/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7909 - acc: 0.2110 - val_loss: 11.4377 - val_acc: 0.1390\n",
      "Epoch 78/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7549 - acc: 0.2172 - val_loss: 11.4295 - val_acc: 0.1329\n",
      "Epoch 79/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7434 - acc: 0.2207 - val_loss: 11.4374 - val_acc: 0.1329\n",
      "Epoch 80/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.8083 - acc: 0.2073 - val_loss: 11.4370 - val_acc: 0.1329\n",
      "Epoch 81/120\n",
      "8709/8709 [==============================] - 22s 2ms/step - loss: 3.7732 - acc: 0.2146 - val_loss: 11.4347 - val_acc: 0.1329\n",
      "Epoch 82/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.7671 - acc: 0.2153 - val_loss: 11.4532 - val_acc: 0.1360\n",
      "Epoch 83/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.7648 - acc: 0.2152 - val_loss: 11.4338 - val_acc: 0.1329\n",
      "Epoch 84/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7709 - acc: 0.2164 - val_loss: 11.4423 - val_acc: 0.1329\n",
      "Epoch 85/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7596 - acc: 0.2158 - val_loss: 11.4371 - val_acc: 0.1329\n",
      "Epoch 86/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7462 - acc: 0.2234 - val_loss: 11.4329 - val_acc: 0.1360\n",
      "Epoch 87/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7588 - acc: 0.2143 - val_loss: 11.4528 - val_acc: 0.1329\n",
      "Epoch 88/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7714 - acc: 0.2189 - val_loss: 11.4443 - val_acc: 0.1360\n",
      "Epoch 89/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7758 - acc: 0.2138 - val_loss: 11.4411 - val_acc: 0.1329\n",
      "Epoch 90/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7461 - acc: 0.2174 - val_loss: 11.4406 - val_acc: 0.1299\n",
      "Epoch 91/120\n",
      "8709/8709 [==============================] - 20s 2ms/step - loss: 3.7416 - acc: 0.2195 - val_loss: 11.4429 - val_acc: 0.1299\n",
      "Epoch 92/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.7693 - acc: 0.2132 - val_loss: 11.4641 - val_acc: 0.1329\n",
      "Epoch 93/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.8256 - acc: 0.2087 - val_loss: 11.4648 - val_acc: 0.1329\n",
      "Epoch 94/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7724 - acc: 0.2140 - val_loss: 11.4500 - val_acc: 0.1360\n",
      "Epoch 95/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7859 - acc: 0.2116 - val_loss: 11.4617 - val_acc: 0.1360\n",
      "Epoch 96/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7657 - acc: 0.2131 - val_loss: 11.4602 - val_acc: 0.1360\n",
      "Epoch 97/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.7695 - acc: 0.2137 - val_loss: 11.4516 - val_acc: 0.1360\n",
      "Epoch 98/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7726 - acc: 0.2208 - val_loss: 11.4650 - val_acc: 0.1329\n",
      "Epoch 99/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7714 - acc: 0.2116 - val_loss: 11.4613 - val_acc: 0.1299\n",
      "Epoch 100/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7229 - acc: 0.2223 - val_loss: 11.4770 - val_acc: 0.1329\n",
      "Epoch 101/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7638 - acc: 0.2172 - val_loss: 11.4524 - val_acc: 0.1360\n",
      "Epoch 102/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7039 - acc: 0.2237 - val_loss: 11.4656 - val_acc: 0.1329\n",
      "Epoch 103/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7573 - acc: 0.2187 - val_loss: 11.4737 - val_acc: 0.1329\n",
      "Epoch 104/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7364 - acc: 0.2194 - val_loss: 11.4755 - val_acc: 0.1329\n",
      "Epoch 105/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.7411 - acc: 0.2308 - val_loss: 11.4909 - val_acc: 0.1360\n",
      "Epoch 106/120\n",
      "8709/8709 [==============================] - 22s 2ms/step - loss: 3.7507 - acc: 0.2114 - val_loss: 11.4717 - val_acc: 0.1329\n",
      "Epoch 107/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7509 - acc: 0.2175 - val_loss: 11.4748 - val_acc: 0.1329\n",
      "Epoch 108/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7619 - acc: 0.2170 - val_loss: 11.4805 - val_acc: 0.1329\n",
      "Epoch 109/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7404 - acc: 0.2232 - val_loss: 11.4826 - val_acc: 0.1329\n",
      "Epoch 110/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7794 - acc: 0.2152 - val_loss: 11.4685 - val_acc: 0.1360\n",
      "Epoch 111/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7346 - acc: 0.2230 - val_loss: 11.4814 - val_acc: 0.1329\n",
      "Epoch 112/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7545 - acc: 0.2150 - val_loss: 11.4739 - val_acc: 0.1360\n",
      "Epoch 113/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7531 - acc: 0.2158 - val_loss: 11.4697 - val_acc: 0.1360\n",
      "Epoch 114/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7620 - acc: 0.2247 - val_loss: 11.4933 - val_acc: 0.1329\n",
      "Epoch 115/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.7681 - acc: 0.2167 - val_loss: 11.4885 - val_acc: 0.1329\n",
      "Epoch 116/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7295 - acc: 0.2179 - val_loss: 11.4606 - val_acc: 0.1329\n",
      "Epoch 117/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7807 - acc: 0.2085 - val_loss: 11.4645 - val_acc: 0.1360\n",
      "Epoch 118/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7609 - acc: 0.2168 - val_loss: 11.4966 - val_acc: 0.1329\n",
      "Epoch 119/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7472 - acc: 0.2183 - val_loss: 11.4870 - val_acc: 0.1299\n",
      "Epoch 120/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7367 - acc: 0.2193 - val_loss: 11.4866 - val_acc: 0.1360\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=optimizers.Adam(lr=2e-5),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=120,\n",
    "                    batch_size=150,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('weights/adam_no_aug_256_2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/120\n",
      "8709/8709 [==============================] - 26s 3ms/step - loss: 3.7377 - acc: 0.2212 - val_loss: 11.4828 - val_acc: 0.1360\n",
      "Epoch 2/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7649 - acc: 0.2154 - val_loss: 11.4788 - val_acc: 0.1360\n",
      "Epoch 3/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7549 - acc: 0.2187 - val_loss: 11.4935 - val_acc: 0.1329\n",
      "Epoch 4/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7465 - acc: 0.2120 - val_loss: 11.4967 - val_acc: 0.1329\n",
      "Epoch 5/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7491 - acc: 0.2231 - val_loss: 11.4804 - val_acc: 0.1360\n",
      "Epoch 6/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7665 - acc: 0.2123 - val_loss: 11.4833 - val_acc: 0.1329\n",
      "Epoch 7/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7374 - acc: 0.2191 - val_loss: 11.4896 - val_acc: 0.1360\n",
      "Epoch 8/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7252 - acc: 0.2192 - val_loss: 11.4851 - val_acc: 0.1360\n",
      "Epoch 9/120\n",
      "8709/8709 [==============================] - 20s 2ms/step - loss: 3.7394 - acc: 0.2228 - val_loss: 11.4968 - val_acc: 0.1329\n",
      "Epoch 10/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7478 - acc: 0.2136 - val_loss: 11.4875 - val_acc: 0.1360\n",
      "Epoch 11/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7530 - acc: 0.2148 - val_loss: 11.4905 - val_acc: 0.1360\n",
      "Epoch 12/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.6896 - acc: 0.2279 - val_loss: 11.4847 - val_acc: 0.1299\n",
      "Epoch 13/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7115 - acc: 0.2282 - val_loss: 11.5009 - val_acc: 0.1360\n",
      "Epoch 14/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.7580 - acc: 0.2151 - val_loss: 11.4922 - val_acc: 0.1329\n",
      "Epoch 15/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.7132 - acc: 0.2207 - val_loss: 11.5002 - val_acc: 0.1329\n",
      "Epoch 16/120\n",
      "8709/8709 [==============================] - 23s 3ms/step - loss: 3.7177 - acc: 0.2231 - val_loss: 11.4941 - val_acc: 0.1360\n",
      "Epoch 17/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7431 - acc: 0.2179 - val_loss: 11.4991 - val_acc: 0.1360\n",
      "Epoch 18/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7548 - acc: 0.2201 - val_loss: 11.4908 - val_acc: 0.1360\n",
      "Epoch 19/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.6990 - acc: 0.2277 - val_loss: 11.4961 - val_acc: 0.1360\n",
      "Epoch 20/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7132 - acc: 0.2252 - val_loss: 11.4910 - val_acc: 0.1360\n",
      "Epoch 21/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7419 - acc: 0.2257 - val_loss: 11.4999 - val_acc: 0.1329\n",
      "Epoch 22/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7640 - acc: 0.2225 - val_loss: 11.4964 - val_acc: 0.1360\n",
      "Epoch 23/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7254 - acc: 0.2170 - val_loss: 11.5019 - val_acc: 0.1360\n",
      "Epoch 24/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.6981 - acc: 0.2257 - val_loss: 11.4979 - val_acc: 0.1360\n",
      "Epoch 25/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7412 - acc: 0.2175 - val_loss: 11.5008 - val_acc: 0.1329\n",
      "Epoch 26/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7265 - acc: 0.2251 - val_loss: 11.4975 - val_acc: 0.1360\n",
      "Epoch 27/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.6931 - acc: 0.2213 - val_loss: 11.5026 - val_acc: 0.1360\n",
      "Epoch 28/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7531 - acc: 0.2187 - val_loss: 11.5045 - val_acc: 0.1360\n",
      "Epoch 29/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7732 - acc: 0.2125 - val_loss: 11.4944 - val_acc: 0.1360\n",
      "Epoch 30/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7697 - acc: 0.2133 - val_loss: 11.5084 - val_acc: 0.1360\n",
      "Epoch 31/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7226 - acc: 0.2195 - val_loss: 11.5049 - val_acc: 0.1329\n",
      "Epoch 32/120\n",
      "8709/8709 [==============================] - 22s 2ms/step - loss: 3.7454 - acc: 0.2137 - val_loss: 11.5091 - val_acc: 0.1360\n",
      "Epoch 33/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.7463 - acc: 0.2194 - val_loss: 11.5003 - val_acc: 0.1329\n",
      "Epoch 34/120\n",
      "8709/8709 [==============================] - 24s 3ms/step - loss: 3.7519 - acc: 0.2199 - val_loss: 11.4999 - val_acc: 0.1360\n",
      "Epoch 35/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.7272 - acc: 0.2182 - val_loss: 11.5006 - val_acc: 0.1360\n",
      "Epoch 36/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7344 - acc: 0.2198 - val_loss: 11.4996 - val_acc: 0.1360\n",
      "Epoch 37/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7410 - acc: 0.2192 - val_loss: 11.4975 - val_acc: 0.1360\n",
      "Epoch 38/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7485 - acc: 0.2161 - val_loss: 11.5148 - val_acc: 0.1360\n",
      "Epoch 39/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7106 - acc: 0.2234 - val_loss: 11.5161 - val_acc: 0.1360\n",
      "Epoch 40/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7681 - acc: 0.2156 - val_loss: 11.5062 - val_acc: 0.1360\n",
      "Epoch 41/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7615 - acc: 0.2130 - val_loss: 11.5089 - val_acc: 0.1360\n",
      "Epoch 42/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7253 - acc: 0.2210 - val_loss: 11.5092 - val_acc: 0.1360\n",
      "Epoch 43/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7437 - acc: 0.2212 - val_loss: 11.5115 - val_acc: 0.1360\n",
      "Epoch 44/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.7468 - acc: 0.2154 - val_loss: 11.5121 - val_acc: 0.1360\n",
      "Epoch 45/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.7630 - acc: 0.2159 - val_loss: 11.5188 - val_acc: 0.1360\n",
      "Epoch 46/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.7656 - acc: 0.2115 - val_loss: 11.5079 - val_acc: 0.1360\n",
      "Epoch 47/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7451 - acc: 0.2187 - val_loss: 11.5161 - val_acc: 0.1360\n",
      "Epoch 48/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7459 - acc: 0.2183 - val_loss: 11.5131 - val_acc: 0.1360\n",
      "Epoch 49/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7455 - acc: 0.2167 - val_loss: 11.5057 - val_acc: 0.1329\n",
      "Epoch 50/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7344 - acc: 0.2203 - val_loss: 11.5088 - val_acc: 0.1360\n",
      "Epoch 51/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7577 - acc: 0.2129 - val_loss: 11.5169 - val_acc: 0.1329\n",
      "Epoch 52/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7273 - acc: 0.2181 - val_loss: 11.5076 - val_acc: 0.1360\n",
      "Epoch 53/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7445 - acc: 0.2217 - val_loss: 11.5134 - val_acc: 0.1329\n",
      "Epoch 54/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7224 - acc: 0.2195 - val_loss: 11.5129 - val_acc: 0.1360\n",
      "Epoch 55/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7610 - acc: 0.2147 - val_loss: 11.5211 - val_acc: 0.1360\n",
      "Epoch 56/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7343 - acc: 0.2214 - val_loss: 11.5253 - val_acc: 0.1360\n",
      "Epoch 57/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7613 - acc: 0.2164 - val_loss: 11.5249 - val_acc: 0.1360\n",
      "Epoch 58/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7784 - acc: 0.2193 - val_loss: 11.5234 - val_acc: 0.1360\n",
      "Epoch 59/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7136 - acc: 0.2243 - val_loss: 11.5200 - val_acc: 0.1299\n",
      "Epoch 60/120\n",
      "8709/8709 [==============================] - 22s 3ms/step - loss: 3.7826 - acc: 0.2166 - val_loss: 11.5175 - val_acc: 0.1360\n",
      "Epoch 61/120\n",
      "8709/8709 [==============================] - 22s 2ms/step - loss: 3.7133 - acc: 0.2218 - val_loss: 11.5208 - val_acc: 0.1360\n",
      "Epoch 62/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.6990 - acc: 0.2207 - val_loss: 11.5188 - val_acc: 0.1360\n",
      "Epoch 63/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7195 - acc: 0.2237 - val_loss: 11.5135 - val_acc: 0.1329\n",
      "Epoch 64/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7310 - acc: 0.2195 - val_loss: 11.5317 - val_acc: 0.1360\n",
      "Epoch 65/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7147 - acc: 0.2256 - val_loss: 11.5204 - val_acc: 0.1360\n",
      "Epoch 66/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7750 - acc: 0.2128 - val_loss: 11.5183 - val_acc: 0.1360\n",
      "Epoch 67/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7253 - acc: 0.2193 - val_loss: 11.5124 - val_acc: 0.1360\n",
      "Epoch 68/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7834 - acc: 0.2164 - val_loss: 11.5106 - val_acc: 0.1299\n",
      "Epoch 69/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.6977 - acc: 0.2223 - val_loss: 11.5058 - val_acc: 0.1360\n",
      "Epoch 70/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7291 - acc: 0.2239 - val_loss: 11.5129 - val_acc: 0.1360\n",
      "Epoch 71/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7161 - acc: 0.2261 - val_loss: 11.5159 - val_acc: 0.1360\n",
      "Epoch 72/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7330 - acc: 0.2206 - val_loss: 11.5185 - val_acc: 0.1329\n",
      "Epoch 73/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7369 - acc: 0.2159 - val_loss: 11.5195 - val_acc: 0.1329\n",
      "Epoch 74/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7106 - acc: 0.2231 - val_loss: 11.5181 - val_acc: 0.1329\n",
      "Epoch 75/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.6986 - acc: 0.2233 - val_loss: 11.5214 - val_acc: 0.1299\n",
      "Epoch 76/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7395 - acc: 0.2230 - val_loss: 11.5152 - val_acc: 0.1299\n",
      "Epoch 77/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7260 - acc: 0.2257 - val_loss: 11.5219 - val_acc: 0.1360\n",
      "Epoch 78/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.6852 - acc: 0.2327 - val_loss: 11.5254 - val_acc: 0.1329\n",
      "Epoch 79/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7094 - acc: 0.2224 - val_loss: 11.5305 - val_acc: 0.1329\n",
      "Epoch 80/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7086 - acc: 0.2217 - val_loss: 11.5192 - val_acc: 0.1329\n",
      "Epoch 81/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7458 - acc: 0.2169 - val_loss: 11.5247 - val_acc: 0.1360\n",
      "Epoch 82/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7259 - acc: 0.2214 - val_loss: 11.5308 - val_acc: 0.1360\n",
      "Epoch 83/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7260 - acc: 0.2217 - val_loss: 11.5282 - val_acc: 0.1299\n",
      "Epoch 84/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7413 - acc: 0.2206 - val_loss: 11.5290 - val_acc: 0.1360\n",
      "Epoch 85/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.6875 - acc: 0.2232 - val_loss: 11.5278 - val_acc: 0.1329\n",
      "Epoch 86/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7348 - acc: 0.2246 - val_loss: 11.5217 - val_acc: 0.1329\n",
      "Epoch 87/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7305 - acc: 0.2166 - val_loss: 11.5343 - val_acc: 0.1329\n",
      "Epoch 88/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7079 - acc: 0.2246 - val_loss: 11.5385 - val_acc: 0.1299\n",
      "Epoch 89/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7104 - acc: 0.2177 - val_loss: 11.5288 - val_acc: 0.1329\n",
      "Epoch 90/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7354 - acc: 0.2172 - val_loss: 11.5355 - val_acc: 0.1360\n",
      "Epoch 91/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7389 - acc: 0.2254 - val_loss: 11.5288 - val_acc: 0.1299\n",
      "Epoch 92/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7056 - acc: 0.2217 - val_loss: 11.5473 - val_acc: 0.1329\n",
      "Epoch 93/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7210 - acc: 0.2205 - val_loss: 11.5286 - val_acc: 0.1329\n",
      "Epoch 94/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7017 - acc: 0.2260 - val_loss: 11.5375 - val_acc: 0.1329\n",
      "Epoch 95/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7040 - acc: 0.2303 - val_loss: 11.5327 - val_acc: 0.1329\n",
      "Epoch 96/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7617 - acc: 0.2186 - val_loss: 11.5439 - val_acc: 0.1329\n",
      "Epoch 97/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7014 - acc: 0.2241 - val_loss: 11.5394 - val_acc: 0.1360\n",
      "Epoch 98/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7042 - acc: 0.2251 - val_loss: 11.5332 - val_acc: 0.1360\n",
      "Epoch 99/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7089 - acc: 0.2224 - val_loss: 11.5325 - val_acc: 0.1329\n",
      "Epoch 100/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.6721 - acc: 0.2288 - val_loss: 11.5455 - val_acc: 0.1360\n",
      "Epoch 101/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7182 - acc: 0.2251 - val_loss: 11.5437 - val_acc: 0.1329\n",
      "Epoch 102/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7102 - acc: 0.2264 - val_loss: 11.5412 - val_acc: 0.1360\n",
      "Epoch 103/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.6961 - acc: 0.2268 - val_loss: 11.5417 - val_acc: 0.1360\n",
      "Epoch 104/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7395 - acc: 0.2129 - val_loss: 11.5470 - val_acc: 0.1360\n",
      "Epoch 105/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.6822 - acc: 0.2287 - val_loss: 11.5444 - val_acc: 0.1329\n",
      "Epoch 106/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7445 - acc: 0.2167 - val_loss: 11.5389 - val_acc: 0.1360\n",
      "Epoch 107/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7246 - acc: 0.2225 - val_loss: 11.5454 - val_acc: 0.1360\n",
      "Epoch 108/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7096 - acc: 0.2187 - val_loss: 11.5384 - val_acc: 0.1329\n",
      "Epoch 109/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7553 - acc: 0.2194 - val_loss: 11.5342 - val_acc: 0.1329\n",
      "Epoch 110/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7423 - acc: 0.2283 - val_loss: 11.5379 - val_acc: 0.1329\n",
      "Epoch 111/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7228 - acc: 0.2208 - val_loss: 11.5539 - val_acc: 0.1360\n",
      "Epoch 112/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7135 - acc: 0.2187 - val_loss: 11.5368 - val_acc: 0.1329\n",
      "Epoch 113/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7229 - acc: 0.2228 - val_loss: 11.5375 - val_acc: 0.1360\n",
      "Epoch 114/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7309 - acc: 0.2164 - val_loss: 11.5457 - val_acc: 0.1360\n",
      "Epoch 115/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7428 - acc: 0.2233 - val_loss: 11.5527 - val_acc: 0.1360\n",
      "Epoch 116/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7330 - acc: 0.2170 - val_loss: 11.5491 - val_acc: 0.1360\n",
      "Epoch 117/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7177 - acc: 0.2221 - val_loss: 11.5487 - val_acc: 0.1329\n",
      "Epoch 118/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7698 - acc: 0.2179 - val_loss: 11.5431 - val_acc: 0.1329\n",
      "Epoch 119/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7423 - acc: 0.2155 - val_loss: 11.5389 - val_acc: 0.1360\n",
      "Epoch 120/120\n",
      "8709/8709 [==============================] - 21s 2ms/step - loss: 3.7160 - acc: 0.2310 - val_loss: 11.5519 - val_acc: 0.1329\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=optimizers.Adam(lr=1e-5),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=120,\n",
    "                    batch_size=200,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('weights/adam_no_aug_256_3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_23 (Dense)             (None, 256)               25690368  \n",
      "_________________________________________________________________\n",
      "dropout_12 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_24 (Dense)             (None, 4250)              1092250   \n",
      "=================================================================\n",
      "Total params: 26,782,618\n",
      "Trainable params: 26,782,618\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import optimizers\n",
    "\n",
    "model = models.Sequential()\n",
    "model.add(layers.Dense(256, activation='relu', input_dim=ll_size*ll_size*2048)) # we can play around with \n",
    "model.add(layers.Dropout(0.5))\n",
    "model.add(layers.Dense(classes_count, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At an image resolution of 224x224, there are approximately 400,000,000 pixels in our dataset, which is 15x the number of trainable parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/100\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 8.3736 - acc: 0.0021 - val_loss: 8.3402 - val_acc: 0.0030\n",
      "Epoch 2/100\n",
      "8709/8709 [==============================] - 66s 8ms/step - loss: 8.2993 - acc: 0.0031 - val_loss: 8.2642 - val_acc: 0.0000e+00\n",
      "Epoch 3/100\n",
      "8709/8709 [==============================] - 63s 7ms/step - loss: 8.1281 - acc: 0.0030 - val_loss: 8.1705 - val_acc: 0.0121\n",
      "Epoch 4/100\n",
      "8709/8709 [==============================] - 68s 8ms/step - loss: 7.8961 - acc: 0.0063 - val_loss: 8.1336 - val_acc: 0.0121\n",
      "Epoch 5/100\n",
      "8709/8709 [==============================] - 61s 7ms/step - loss: 7.6277 - acc: 0.0100 - val_loss: 8.0872 - val_acc: 0.0091\n",
      "Epoch 6/100\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 7.4292 - acc: 0.0122 - val_loss: 8.1200 - val_acc: 0.0060\n",
      "Epoch 7/100\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 7.2173 - acc: 0.0181 - val_loss: 8.1304 - val_acc: 0.0181\n",
      "Epoch 8/100\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 7.0024 - acc: 0.0257 - val_loss: 8.0786 - val_acc: 0.0242\n",
      "Epoch 9/100\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 6.8144 - acc: 0.0313 - val_loss: 8.1169 - val_acc: 0.0181\n",
      "Epoch 10/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 6.5821 - acc: 0.0404 - val_loss: 8.1941 - val_acc: 0.0423\n",
      "Epoch 11/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 6.3446 - acc: 0.0520 - val_loss: 8.2063 - val_acc: 0.0363\n",
      "Epoch 12/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 6.1411 - acc: 0.0591 - val_loss: 8.3589 - val_acc: 0.0242\n",
      "Epoch 13/100\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 5.9106 - acc: 0.0781 - val_loss: 8.3205 - val_acc: 0.0453\n",
      "Epoch 14/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 5.7124 - acc: 0.0866 - val_loss: 8.5087 - val_acc: 0.0423\n",
      "Epoch 15/100\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 5.5039 - acc: 0.0977 - val_loss: 8.4242 - val_acc: 0.0514\n",
      "Epoch 16/100\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 5.2889 - acc: 0.1139 - val_loss: 8.6081 - val_acc: 0.0574\n",
      "Epoch 17/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 5.0939 - acc: 0.1267 - val_loss: 8.7405 - val_acc: 0.0574\n",
      "Epoch 18/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 4.8902 - acc: 0.1436 - val_loss: 8.6452 - val_acc: 0.0755\n",
      "Epoch 19/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 4.7044 - acc: 0.1628 - val_loss: 8.8166 - val_acc: 0.0906\n",
      "Epoch 20/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 4.5399 - acc: 0.1696 - val_loss: 8.9169 - val_acc: 0.0937\n",
      "Epoch 21/100\n",
      "8709/8709 [==============================] - 60s 7ms/step - loss: 4.3324 - acc: 0.1921 - val_loss: 8.9650 - val_acc: 0.0997\n",
      "Epoch 22/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 4.1632 - acc: 0.2096 - val_loss: 8.9934 - val_acc: 0.1027\n",
      "Epoch 23/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 4.0128 - acc: 0.2270 - val_loss: 9.0639 - val_acc: 0.1269\n",
      "Epoch 24/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 3.9030 - acc: 0.2423 - val_loss: 9.1519 - val_acc: 0.1148\n",
      "Epoch 25/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 3.7421 - acc: 0.2605 - val_loss: 9.1480 - val_acc: 0.1360\n",
      "Epoch 26/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 3.5820 - acc: 0.2781 - val_loss: 9.2662 - val_acc: 0.1329\n",
      "Epoch 27/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 3.4370 - acc: 0.3030 - val_loss: 9.2952 - val_acc: 0.1390\n",
      "Epoch 28/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 3.3120 - acc: 0.3143 - val_loss: 9.2976 - val_acc: 0.1480\n",
      "Epoch 29/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 3.2234 - acc: 0.3263 - val_loss: 9.3937 - val_acc: 0.1480\n",
      "Epoch 30/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 3.1360 - acc: 0.3400 - val_loss: 9.3973 - val_acc: 0.1420\n",
      "Epoch 31/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 2.9840 - acc: 0.3605 - val_loss: 9.4521 - val_acc: 0.1511\n",
      "Epoch 32/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 2.8897 - acc: 0.3736 - val_loss: 9.4394 - val_acc: 0.1480\n",
      "Epoch 33/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 2.8271 - acc: 0.3810 - val_loss: 9.5674 - val_acc: 0.1601\n",
      "Epoch 34/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 2.7277 - acc: 0.3966 - val_loss: 9.5746 - val_acc: 0.1722\n",
      "Epoch 35/100\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 2.6742 - acc: 0.4044 - val_loss: 9.6027 - val_acc: 0.1541\n",
      "Epoch 36/100\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 2.5851 - acc: 0.4248 - val_loss: 9.6206 - val_acc: 0.1541\n",
      "Epoch 37/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 2.4862 - acc: 0.4381 - val_loss: 9.6722 - val_acc: 0.1662\n",
      "Epoch 38/100\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 2.4161 - acc: 0.4502 - val_loss: 9.6648 - val_acc: 0.1631\n",
      "Epoch 39/100\n",
      "8709/8709 [==============================] - 64s 7ms/step - loss: 2.3658 - acc: 0.4559 - val_loss: 9.6498 - val_acc: 0.1782\n",
      "Epoch 40/100\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 2.3018 - acc: 0.4749 - val_loss: 9.6542 - val_acc: 0.1782\n",
      "Epoch 41/100\n",
      "8709/8709 [==============================] - 60s 7ms/step - loss: 2.2427 - acc: 0.4774 - val_loss: 9.6950 - val_acc: 0.1752\n",
      "Epoch 42/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 2.1830 - acc: 0.4914 - val_loss: 9.7873 - val_acc: 0.1662\n",
      "Epoch 43/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 2.1002 - acc: 0.5064 - val_loss: 9.7310 - val_acc: 0.1752\n",
      "Epoch 44/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 2.0757 - acc: 0.5128 - val_loss: 9.8316 - val_acc: 0.1662\n",
      "Epoch 45/100\n",
      "8709/8709 [==============================] - 61s 7ms/step - loss: 2.0731 - acc: 0.5135 - val_loss: 9.7820 - val_acc: 0.1722\n",
      "Epoch 46/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.9642 - acc: 0.5381 - val_loss: 9.8400 - val_acc: 0.1722\n",
      "Epoch 47/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.8935 - acc: 0.5452 - val_loss: 9.8781 - val_acc: 0.1813\n",
      "Epoch 48/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.8967 - acc: 0.5509 - val_loss: 9.9023 - val_acc: 0.1752\n",
      "Epoch 49/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.8129 - acc: 0.5623 - val_loss: 9.8826 - val_acc: 0.1722\n",
      "Epoch 50/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.7781 - acc: 0.5722 - val_loss: 9.9223 - val_acc: 0.1722\n",
      "Epoch 51/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.7204 - acc: 0.5799 - val_loss: 9.8664 - val_acc: 0.1692\n",
      "Epoch 52/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.6861 - acc: 0.5925 - val_loss: 9.9835 - val_acc: 0.1843\n",
      "Epoch 53/100\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 1.6584 - acc: 0.5923 - val_loss: 9.9382 - val_acc: 0.1752\n",
      "Epoch 54/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.6150 - acc: 0.5965 - val_loss: 9.9325 - val_acc: 0.1782\n",
      "Epoch 55/100\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 1.6130 - acc: 0.6062 - val_loss: 10.0972 - val_acc: 0.1813\n",
      "Epoch 56/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.5441 - acc: 0.6256 - val_loss: 9.9902 - val_acc: 0.1782\n",
      "Epoch 57/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.5116 - acc: 0.6304 - val_loss: 10.0337 - val_acc: 0.1752\n",
      "Epoch 58/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.5006 - acc: 0.6237 - val_loss: 10.0453 - val_acc: 0.1782\n",
      "Epoch 59/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.4605 - acc: 0.6415 - val_loss: 10.1294 - val_acc: 0.1692\n",
      "Epoch 60/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.4476 - acc: 0.6389 - val_loss: 10.0905 - val_acc: 0.1662\n",
      "Epoch 61/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.4180 - acc: 0.6480 - val_loss: 10.1655 - val_acc: 0.1692\n",
      "Epoch 62/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.3733 - acc: 0.6509 - val_loss: 10.1427 - val_acc: 0.1752\n",
      "Epoch 63/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.3715 - acc: 0.6496 - val_loss: 10.1254 - val_acc: 0.1752\n",
      "Epoch 64/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.3331 - acc: 0.6625 - val_loss: 10.0545 - val_acc: 0.1722\n",
      "Epoch 65/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.3311 - acc: 0.6592 - val_loss: 10.2251 - val_acc: 0.1813\n",
      "Epoch 66/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.2996 - acc: 0.6722 - val_loss: 10.1626 - val_acc: 0.1813\n",
      "Epoch 67/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.2640 - acc: 0.6759 - val_loss: 10.1759 - val_acc: 0.1782\n",
      "Epoch 68/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.2526 - acc: 0.6833 - val_loss: 10.1529 - val_acc: 0.1843\n",
      "Epoch 69/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.2259 - acc: 0.6842 - val_loss: 10.2145 - val_acc: 0.1722\n",
      "Epoch 70/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.2102 - acc: 0.6857 - val_loss: 10.2959 - val_acc: 0.1752\n",
      "Epoch 71/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.1655 - acc: 0.6976 - val_loss: 10.2619 - val_acc: 0.1782\n",
      "Epoch 72/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.1439 - acc: 0.7067 - val_loss: 10.2508 - val_acc: 0.1722\n",
      "Epoch 73/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.1452 - acc: 0.7065 - val_loss: 10.2132 - val_acc: 0.1752\n",
      "Epoch 74/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.1237 - acc: 0.7063 - val_loss: 10.2359 - val_acc: 0.1813\n",
      "Epoch 75/100\n",
      "8709/8709 [==============================] - 56s 6ms/step - loss: 1.1142 - acc: 0.7123 - val_loss: 10.3736 - val_acc: 0.1692\n",
      "Epoch 76/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.0960 - acc: 0.7157 - val_loss: 10.3755 - val_acc: 0.1662\n",
      "Epoch 77/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.0950 - acc: 0.7193 - val_loss: 10.3680 - val_acc: 0.1722\n",
      "Epoch 78/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.0638 - acc: 0.7220 - val_loss: 10.3877 - val_acc: 0.1692\n",
      "Epoch 79/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 1.0475 - acc: 0.7289 - val_loss: 10.4000 - val_acc: 0.1752\n",
      "Epoch 80/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.0398 - acc: 0.7288 - val_loss: 10.3541 - val_acc: 0.1843\n",
      "Epoch 81/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 0.9859 - acc: 0.7377 - val_loss: 10.3634 - val_acc: 0.1813\n",
      "Epoch 82/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 1.0002 - acc: 0.7353 - val_loss: 10.3753 - val_acc: 0.1813\n",
      "Epoch 83/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 0.9783 - acc: 0.7410 - val_loss: 10.3156 - val_acc: 0.1873\n",
      "Epoch 84/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 0.9663 - acc: 0.7433 - val_loss: 10.3520 - val_acc: 0.1782\n",
      "Epoch 85/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 0.9623 - acc: 0.7404 - val_loss: 10.4634 - val_acc: 0.1692\n",
      "Epoch 86/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 0.9394 - acc: 0.7472 - val_loss: 10.3655 - val_acc: 0.1782\n",
      "Epoch 87/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 0.9236 - acc: 0.7608 - val_loss: 10.5127 - val_acc: 0.1782\n",
      "Epoch 88/100\n",
      "8709/8709 [==============================] - 57s 6ms/step - loss: 0.9200 - acc: 0.7493 - val_loss: 10.5212 - val_acc: 0.1722\n",
      "Epoch 89/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 0.9261 - acc: 0.7551 - val_loss: 10.4258 - val_acc: 0.1843\n",
      "Epoch 90/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 0.9008 - acc: 0.7570 - val_loss: 10.4653 - val_acc: 0.1752\n",
      "Epoch 91/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 0.8715 - acc: 0.7635 - val_loss: 10.4181 - val_acc: 0.1782\n",
      "Epoch 92/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 0.8959 - acc: 0.7623 - val_loss: 10.4362 - val_acc: 0.1722\n",
      "Epoch 93/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 0.8929 - acc: 0.7615 - val_loss: 10.4992 - val_acc: 0.1782\n",
      "Epoch 94/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 0.8567 - acc: 0.7646 - val_loss: 10.4907 - val_acc: 0.1843\n",
      "Epoch 95/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 0.8481 - acc: 0.7695 - val_loss: 10.5008 - val_acc: 0.1813\n",
      "Epoch 96/100\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 0.8361 - acc: 0.7741 - val_loss: 10.5938 - val_acc: 0.1752\n",
      "Epoch 97/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 0.8311 - acc: 0.7777 - val_loss: 10.5658 - val_acc: 0.1752\n",
      "Epoch 98/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 0.7769 - acc: 0.7913 - val_loss: 10.5353 - val_acc: 0.1752\n",
      "Epoch 99/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 0.8035 - acc: 0.7833 - val_loss: 10.6425 - val_acc: 0.1782\n",
      "Epoch 100/100\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 0.7951 - acc: 0.7853 - val_loss: 10.5712 - val_acc: 0.1722\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=optimizers.Adam(lr=1e-4),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=100,\n",
    "                    batch_size=64,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('weights/adam_no_aug_224_more_capacity_0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/60\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 0.7510 - acc: 0.7941 - val_loss: 10.6160 - val_acc: 0.1782\n",
      "Epoch 2/60\n",
      "8709/8709 [==============================] - 46s 5ms/step - loss: 0.7346 - acc: 0.8011 - val_loss: 10.6341 - val_acc: 0.1843\n",
      "Epoch 3/60\n",
      "8709/8709 [==============================] - 40s 5ms/step - loss: 0.7039 - acc: 0.8111 - val_loss: 10.6450 - val_acc: 0.1752\n",
      "Epoch 4/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.6981 - acc: 0.8087 - val_loss: 10.6417 - val_acc: 0.1813\n",
      "Epoch 5/60\n",
      "8709/8709 [==============================] - 40s 5ms/step - loss: 0.7078 - acc: 0.8076 - val_loss: 10.6553 - val_acc: 0.1813\n",
      "Epoch 6/60\n",
      "8709/8709 [==============================] - 40s 5ms/step - loss: 0.7159 - acc: 0.8063 - val_loss: 10.6688 - val_acc: 0.1813\n",
      "Epoch 7/60\n",
      "8709/8709 [==============================] - 39s 5ms/step - loss: 0.6976 - acc: 0.8130 - val_loss: 10.6742 - val_acc: 0.1752\n",
      "Epoch 8/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.6820 - acc: 0.8130 - val_loss: 10.6951 - val_acc: 0.1752\n",
      "Epoch 9/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.7079 - acc: 0.8008 - val_loss: 10.6809 - val_acc: 0.1782\n",
      "Epoch 10/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.6913 - acc: 0.8100 - val_loss: 10.6605 - val_acc: 0.1843\n",
      "Epoch 11/60\n",
      "8709/8709 [==============================] - 39s 4ms/step - loss: 0.6792 - acc: 0.8166 - val_loss: 10.6844 - val_acc: 0.1813\n",
      "Epoch 12/60\n",
      "8709/8709 [==============================] - 39s 4ms/step - loss: 0.6739 - acc: 0.8171 - val_loss: 10.6888 - val_acc: 0.1813\n",
      "Epoch 13/60\n",
      "8709/8709 [==============================] - 39s 4ms/step - loss: 0.6606 - acc: 0.8196 - val_loss: 10.6647 - val_acc: 0.1873\n",
      "Epoch 14/60\n",
      "8709/8709 [==============================] - 39s 4ms/step - loss: 0.6717 - acc: 0.8132 - val_loss: 10.6874 - val_acc: 0.1782\n",
      "Epoch 15/60\n",
      "8709/8709 [==============================] - 39s 4ms/step - loss: 0.6748 - acc: 0.8087 - val_loss: 10.6977 - val_acc: 0.1813\n",
      "Epoch 16/60\n",
      "8709/8709 [==============================] - 39s 5ms/step - loss: 0.6693 - acc: 0.8169 - val_loss: 10.6679 - val_acc: 0.1843\n",
      "Epoch 17/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.6533 - acc: 0.8245 - val_loss: 10.6856 - val_acc: 0.1813\n",
      "Epoch 18/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.6657 - acc: 0.8213 - val_loss: 10.6932 - val_acc: 0.1843\n",
      "Epoch 19/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.6596 - acc: 0.8202 - val_loss: 10.6775 - val_acc: 0.1752\n",
      "Epoch 20/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.6374 - acc: 0.8321 - val_loss: 10.6930 - val_acc: 0.1813\n",
      "Epoch 21/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.6596 - acc: 0.8227 - val_loss: 10.7175 - val_acc: 0.1873\n",
      "Epoch 22/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.6416 - acc: 0.8266 - val_loss: 10.7123 - val_acc: 0.1752\n",
      "Epoch 23/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.6436 - acc: 0.8260 - val_loss: 10.7415 - val_acc: 0.1722\n",
      "Epoch 24/60\n",
      "8709/8709 [==============================] - 39s 5ms/step - loss: 0.6682 - acc: 0.8143 - val_loss: 10.7217 - val_acc: 0.1782\n",
      "Epoch 25/60\n",
      "8709/8709 [==============================] - 40s 5ms/step - loss: 0.6359 - acc: 0.8255 - val_loss: 10.7184 - val_acc: 0.1813\n",
      "Epoch 26/60\n",
      "8709/8709 [==============================] - 40s 5ms/step - loss: 0.6230 - acc: 0.8329 - val_loss: 10.7083 - val_acc: 0.1813\n",
      "Epoch 27/60\n",
      "8709/8709 [==============================] - 39s 4ms/step - loss: 0.6053 - acc: 0.8345 - val_loss: 10.7006 - val_acc: 0.1843\n",
      "Epoch 28/60\n",
      "8709/8709 [==============================] - 41s 5ms/step - loss: 0.6415 - acc: 0.8252 - val_loss: 10.7554 - val_acc: 0.1843\n",
      "Epoch 29/60\n",
      "8709/8709 [==============================] - 39s 4ms/step - loss: 0.6260 - acc: 0.8281 - val_loss: 10.7552 - val_acc: 0.1692\n",
      "Epoch 30/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.6305 - acc: 0.8243 - val_loss: 10.7508 - val_acc: 0.1752\n",
      "Epoch 31/60\n",
      "8709/8709 [==============================] - 39s 4ms/step - loss: 0.6036 - acc: 0.8351 - val_loss: 10.7938 - val_acc: 0.1782\n",
      "Epoch 32/60\n",
      "8709/8709 [==============================] - 39s 4ms/step - loss: 0.6207 - acc: 0.8313 - val_loss: 10.7625 - val_acc: 0.1782\n",
      "Epoch 33/60\n",
      "8709/8709 [==============================] - 39s 4ms/step - loss: 0.6041 - acc: 0.8355 - val_loss: 10.7736 - val_acc: 0.1813\n",
      "Epoch 34/60\n",
      "8709/8709 [==============================] - 39s 5ms/step - loss: 0.6020 - acc: 0.8366 - val_loss: 10.7932 - val_acc: 0.1813\n",
      "Epoch 35/60\n",
      "8709/8709 [==============================] - 39s 5ms/step - loss: 0.6063 - acc: 0.8352 - val_loss: 10.7379 - val_acc: 0.1813\n",
      "Epoch 36/60\n",
      "8709/8709 [==============================] - 39s 5ms/step - loss: 0.6101 - acc: 0.8314 - val_loss: 10.7718 - val_acc: 0.1782\n",
      "Epoch 37/60\n",
      "8709/8709 [==============================] - 39s 5ms/step - loss: 0.5876 - acc: 0.8391 - val_loss: 10.7911 - val_acc: 0.1782\n",
      "Epoch 38/60\n",
      "8709/8709 [==============================] - 40s 5ms/step - loss: 0.6115 - acc: 0.8288 - val_loss: 10.7730 - val_acc: 0.1843\n",
      "Epoch 39/60\n",
      "8709/8709 [==============================] - 39s 5ms/step - loss: 0.5973 - acc: 0.8347 - val_loss: 10.7875 - val_acc: 0.1752\n",
      "Epoch 40/60\n",
      "8709/8709 [==============================] - 40s 5ms/step - loss: 0.5981 - acc: 0.8363 - val_loss: 10.7697 - val_acc: 0.1813\n",
      "Epoch 41/60\n",
      "8709/8709 [==============================] - 40s 5ms/step - loss: 0.5933 - acc: 0.8381 - val_loss: 10.8016 - val_acc: 0.1813\n",
      "Epoch 42/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.6033 - acc: 0.8358 - val_loss: 10.7745 - val_acc: 0.1843\n",
      "Epoch 43/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.5670 - acc: 0.8434 - val_loss: 10.7916 - val_acc: 0.1782\n",
      "Epoch 44/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.5902 - acc: 0.8415 - val_loss: 10.7724 - val_acc: 0.1782\n",
      "Epoch 45/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.5657 - acc: 0.8425 - val_loss: 10.7944 - val_acc: 0.1843\n",
      "Epoch 46/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.5617 - acc: 0.8461 - val_loss: 10.8061 - val_acc: 0.1843\n",
      "Epoch 47/60\n",
      "8709/8709 [==============================] - 39s 4ms/step - loss: 0.5633 - acc: 0.8456 - val_loss: 10.8139 - val_acc: 0.1873\n",
      "Epoch 48/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.5891 - acc: 0.8387 - val_loss: 10.8016 - val_acc: 0.1813\n",
      "Epoch 49/60\n",
      "8709/8709 [==============================] - 39s 5ms/step - loss: 0.5581 - acc: 0.8467 - val_loss: 10.8165 - val_acc: 0.1843\n",
      "Epoch 50/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.5585 - acc: 0.8458 - val_loss: 10.8216 - val_acc: 0.1873\n",
      "Epoch 51/60\n",
      "8709/8709 [==============================] - 37s 4ms/step - loss: 0.5705 - acc: 0.8390 - val_loss: 10.8011 - val_acc: 0.1843\n",
      "Epoch 52/60\n",
      "8709/8709 [==============================] - 37s 4ms/step - loss: 0.5581 - acc: 0.8465 - val_loss: 10.8461 - val_acc: 0.1843\n",
      "Epoch 53/60\n",
      "8709/8709 [==============================] - 37s 4ms/step - loss: 0.5496 - acc: 0.8488 - val_loss: 10.8065 - val_acc: 0.1782\n",
      "Epoch 54/60\n",
      "8709/8709 [==============================] - 37s 4ms/step - loss: 0.5520 - acc: 0.8502 - val_loss: 10.7890 - val_acc: 0.1813\n",
      "Epoch 55/60\n",
      "8709/8709 [==============================] - 37s 4ms/step - loss: 0.5517 - acc: 0.8481 - val_loss: 10.8321 - val_acc: 0.1813\n",
      "Epoch 56/60\n",
      "8709/8709 [==============================] - 39s 4ms/step - loss: 0.5556 - acc: 0.8475 - val_loss: 10.8587 - val_acc: 0.1843\n",
      "Epoch 57/60\n",
      "8709/8709 [==============================] - 39s 5ms/step - loss: 0.5537 - acc: 0.8434 - val_loss: 10.8556 - val_acc: 0.1752\n",
      "Epoch 58/60\n",
      "8709/8709 [==============================] - 38s 4ms/step - loss: 0.5431 - acc: 0.8484 - val_loss: 10.8449 - val_acc: 0.1782\n",
      "Epoch 59/60\n",
      "8709/8709 [==============================] - 37s 4ms/step - loss: 0.5464 - acc: 0.8477 - val_loss: 10.8504 - val_acc: 0.1782\n",
      "Epoch 60/60\n",
      "8709/8709 [==============================] - 37s 4ms/step - loss: 0.5300 - acc: 0.8541 - val_loss: 10.8496 - val_acc: 0.1813\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=optimizers.Adam(lr=5e-5),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=60,\n",
    "                    batch_size=128,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('weights/adam_no_aug_224_more_capacity_1.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### model.compile(optimizer=optimizers.Adam(lr=4e-5),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=40,\n",
    "                    batch_size=10,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_15 (Dense)             (None, 128)               6553728   \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 4250)              548250    \n",
      "=================================================================\n",
      "Total params: 7,101,978\n",
      "Trainable params: 7,101,978\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import optimizers\n",
    "\n",
    "model = models.Sequential()\n",
    "model.add(layers.Dense(128, activation='relu', input_dim=ll_size*ll_size*2048)) # we can play around with \n",
    "model.add(layers.Dropout(0.5))\n",
    "model.add(layers.Dense(classes_count, activation='softmax'))\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/40\n",
      "8709/8709 [==============================] - 92s 11ms/step - loss: 8.3616 - acc: 0.0011 - val_loss: 8.3391 - val_acc: 0.0030\n",
      "Epoch 2/40\n",
      "8709/8709 [==============================] - 90s 10ms/step - loss: 8.2282 - acc: 0.0023 - val_loss: 8.2421 - val_acc: 0.0030\n",
      "Epoch 3/40\n",
      "8709/8709 [==============================] - 89s 10ms/step - loss: 7.9395 - acc: 0.0064 - val_loss: 8.1974 - val_acc: 0.0030\n",
      "Epoch 4/40\n",
      "8709/8709 [==============================] - 89s 10ms/step - loss: 7.6825 - acc: 0.0107 - val_loss: 8.2103 - val_acc: 0.0030\n",
      "Epoch 5/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 7.4089 - acc: 0.0144 - val_loss: 8.1858 - val_acc: 0.0060\n",
      "Epoch 6/40\n",
      "8709/8709 [==============================] - 86s 10ms/step - loss: 7.1509 - acc: 0.0189 - val_loss: 8.2826 - val_acc: 0.0091\n",
      "Epoch 7/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 6.9068 - acc: 0.0246 - val_loss: 8.3236 - val_acc: 0.0211\n",
      "Epoch 8/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 6.6112 - acc: 0.0344 - val_loss: 8.4073 - val_acc: 0.0332\n",
      "Epoch 9/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 6.3440 - acc: 0.0417 - val_loss: 8.3655 - val_acc: 0.0363\n",
      "Epoch 10/40\n",
      "8709/8709 [==============================] - 85s 10ms/step - loss: 6.0803 - acc: 0.0550 - val_loss: 8.6389 - val_acc: 0.0302\n",
      "Epoch 11/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 5.8345 - acc: 0.0684 - val_loss: 8.7612 - val_acc: 0.0363\n",
      "Epoch 12/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 5.5697 - acc: 0.0818 - val_loss: 8.8455 - val_acc: 0.0514\n",
      "Epoch 13/40\n",
      "8709/8709 [==============================] - 86s 10ms/step - loss: 5.3189 - acc: 0.0975 - val_loss: 8.9248 - val_acc: 0.0544\n",
      "Epoch 14/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 5.0664 - acc: 0.1194 - val_loss: 9.0809 - val_acc: 0.0634\n",
      "Epoch 15/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 4.8459 - acc: 0.1302 - val_loss: 9.1601 - val_acc: 0.0665\n",
      "Epoch 16/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 4.6327 - acc: 0.1532 - val_loss: 9.2183 - val_acc: 0.0755\n",
      "Epoch 17/40\n",
      "8709/8709 [==============================] - 86s 10ms/step - loss: 4.4209 - acc: 0.1777 - val_loss: 9.2689 - val_acc: 0.0876\n",
      "Epoch 18/40\n",
      "8709/8709 [==============================] - 82s 9ms/step - loss: 4.2151 - acc: 0.1945 - val_loss: 9.2621 - val_acc: 0.0846\n",
      "Epoch 19/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 4.0251 - acc: 0.2153 - val_loss: 9.3394 - val_acc: 0.0967\n",
      "Epoch 20/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 3.8338 - acc: 0.2398 - val_loss: 9.3826 - val_acc: 0.0997\n",
      "Epoch 21/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 3.6628 - acc: 0.2654 - val_loss: 9.3919 - val_acc: 0.1088\n",
      "Epoch 22/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 3.5144 - acc: 0.2805 - val_loss: 9.4155 - val_acc: 0.1148\n",
      "Epoch 23/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 3.3520 - acc: 0.3054 - val_loss: 9.4739 - val_acc: 0.1178\n",
      "Epoch 24/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 3.1976 - acc: 0.3303 - val_loss: 9.4473 - val_acc: 0.1299\n",
      "Epoch 25/40\n",
      "8709/8709 [==============================] - 89s 10ms/step - loss: 3.0698 - acc: 0.3440 - val_loss: 9.4862 - val_acc: 0.1360\n",
      "Epoch 26/40\n",
      "8709/8709 [==============================] - 90s 10ms/step - loss: 2.9307 - acc: 0.3596 - val_loss: 9.4939 - val_acc: 0.1450\n",
      "Epoch 27/40\n",
      "8709/8709 [==============================] - 89s 10ms/step - loss: 2.8241 - acc: 0.3848 - val_loss: 9.5279 - val_acc: 0.1360\n",
      "Epoch 28/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 2.7309 - acc: 0.3907 - val_loss: 9.5309 - val_acc: 0.1420\n",
      "Epoch 29/40\n",
      "8709/8709 [==============================] - 90s 10ms/step - loss: 2.6091 - acc: 0.4142 - val_loss: 9.5360 - val_acc: 0.1480\n",
      "Epoch 30/40\n",
      "8709/8709 [==============================] - 90s 10ms/step - loss: 2.5399 - acc: 0.4324 - val_loss: 9.5398 - val_acc: 0.1450\n",
      "Epoch 31/40\n",
      "8709/8709 [==============================] - 91s 10ms/step - loss: 2.4156 - acc: 0.4479 - val_loss: 9.6164 - val_acc: 0.1450\n",
      "Epoch 32/40\n",
      "8709/8709 [==============================] - 89s 10ms/step - loss: 2.3532 - acc: 0.4661 - val_loss: 9.5428 - val_acc: 0.1511\n",
      "Epoch 33/40\n",
      "8709/8709 [==============================] - 90s 10ms/step - loss: 2.2361 - acc: 0.4828 - val_loss: 9.6713 - val_acc: 0.1511\n",
      "Epoch 34/40\n",
      "8709/8709 [==============================] - 90s 10ms/step - loss: 2.1531 - acc: 0.5043 - val_loss: 9.6744 - val_acc: 0.1541\n",
      "Epoch 35/40\n",
      "8709/8709 [==============================] - 89s 10ms/step - loss: 2.1242 - acc: 0.5045 - val_loss: 9.5979 - val_acc: 0.1480\n",
      "Epoch 36/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 2.0291 - acc: 0.5245 - val_loss: 9.6661 - val_acc: 0.1571\n",
      "Epoch 37/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 1.9943 - acc: 0.5258 - val_loss: 9.6865 - val_acc: 0.1601\n",
      "Epoch 38/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 1.9212 - acc: 0.5421 - val_loss: 9.6557 - val_acc: 0.1601\n",
      "Epoch 39/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 1.8757 - acc: 0.5544 - val_loss: 9.7136 - val_acc: 0.1692\n",
      "Epoch 40/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 1.7960 - acc: 0.5669 - val_loss: 9.7129 - val_acc: 0.1631\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=optimizers.Adam(lr=3e-5),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=40,\n",
    "                    batch_size=10,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/20\n",
      "8709/8709 [==============================] - 95s 11ms/step - loss: 1.7268 - acc: 0.5811 - val_loss: 9.7642 - val_acc: 0.1662\n",
      "Epoch 2/20\n",
      "8709/8709 [==============================] - 95s 11ms/step - loss: 1.6697 - acc: 0.5971 - val_loss: 9.7539 - val_acc: 0.1571\n",
      "Epoch 3/20\n",
      "8709/8709 [==============================] - 95s 11ms/step - loss: 1.6292 - acc: 0.6000 - val_loss: 9.8094 - val_acc: 0.1631\n",
      "Epoch 4/20\n",
      "8709/8709 [==============================] - 96s 11ms/step - loss: 1.5815 - acc: 0.6189 - val_loss: 9.7721 - val_acc: 0.1541\n",
      "Epoch 5/20\n",
      "8709/8709 [==============================] - 96s 11ms/step - loss: 1.5560 - acc: 0.6135 - val_loss: 9.8193 - val_acc: 0.1662\n",
      "Epoch 6/20\n",
      "8709/8709 [==============================] - 93s 11ms/step - loss: 1.5082 - acc: 0.6312 - val_loss: 9.8390 - val_acc: 0.1601\n",
      "Epoch 7/20\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 1.4590 - acc: 0.6372 - val_loss: 9.8015 - val_acc: 0.1662\n",
      "Epoch 8/20\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 1.4482 - acc: 0.6466 - val_loss: 9.8026 - val_acc: 0.1631\n",
      "Epoch 9/20\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 1.3995 - acc: 0.6508 - val_loss: 9.8452 - val_acc: 0.1571\n",
      "Epoch 10/20\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 1.3622 - acc: 0.6556 - val_loss: 9.8901 - val_acc: 0.1631\n",
      "Epoch 11/20\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 1.3291 - acc: 0.6690 - val_loss: 9.8470 - val_acc: 0.1662\n",
      "Epoch 12/20\n",
      "8709/8709 [==============================] - 86s 10ms/step - loss: 1.2956 - acc: 0.6770 - val_loss: 9.9119 - val_acc: 0.1571\n",
      "Epoch 13/20\n",
      "8709/8709 [==============================] - 86s 10ms/step - loss: 1.2519 - acc: 0.6824 - val_loss: 9.9391 - val_acc: 0.1601\n",
      "Epoch 14/20\n",
      "8709/8709 [==============================] - 86s 10ms/step - loss: 1.2495 - acc: 0.6833 - val_loss: 9.9102 - val_acc: 0.1662\n",
      "Epoch 15/20\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 1.2290 - acc: 0.6862 - val_loss: 9.9618 - val_acc: 0.1692\n",
      "Epoch 16/20\n",
      "8709/8709 [==============================] - 86s 10ms/step - loss: 1.1918 - acc: 0.6958 - val_loss: 10.0338 - val_acc: 0.1631\n",
      "Epoch 17/20\n",
      "8709/8709 [==============================] - 86s 10ms/step - loss: 1.1786 - acc: 0.7041 - val_loss: 10.0128 - val_acc: 0.1631\n",
      "Epoch 18/20\n",
      "8709/8709 [==============================] - 86s 10ms/step - loss: 1.1420 - acc: 0.7090 - val_loss: 10.0239 - val_acc: 0.1662\n",
      "Epoch 19/20\n",
      "8709/8709 [==============================] - 86s 10ms/step - loss: 1.0909 - acc: 0.7284 - val_loss: 9.9987 - val_acc: 0.1631\n",
      "Epoch 20/20\n",
      "8709/8709 [==============================] - 86s 10ms/step - loss: 1.0710 - acc: 0.7267 - val_loss: 10.0471 - val_acc: 0.1662\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=20,\n",
    "                    batch_size=10,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('weights/adam_no_aug_0.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/40\n",
      "8709/8709 [==============================] - 90s 10ms/step - loss: 1.0141 - acc: 0.7402 - val_loss: 10.0754 - val_acc: 0.1692\n",
      "Epoch 2/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 0.9889 - acc: 0.7427 - val_loss: 10.0844 - val_acc: 0.1692\n",
      "Epoch 3/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 0.9983 - acc: 0.7438 - val_loss: 10.0576 - val_acc: 0.1601\n",
      "Epoch 4/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 0.9970 - acc: 0.7472 - val_loss: 10.0618 - val_acc: 0.1631\n",
      "Epoch 5/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 0.9859 - acc: 0.7446 - val_loss: 10.0846 - val_acc: 0.1662\n",
      "Epoch 6/40\n",
      "8709/8709 [==============================] - 85s 10ms/step - loss: 0.9871 - acc: 0.7477 - val_loss: 10.1097 - val_acc: 0.1631\n",
      "Epoch 7/40\n",
      "8709/8709 [==============================] - 92s 11ms/step - loss: 0.9378 - acc: 0.7577 - val_loss: 10.0825 - val_acc: 0.1631\n",
      "Epoch 8/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 0.9466 - acc: 0.7569 - val_loss: 10.0683 - val_acc: 0.1631\n",
      "Epoch 9/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 0.9420 - acc: 0.7567 - val_loss: 10.1044 - val_acc: 0.1601\n",
      "Epoch 10/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 0.9312 - acc: 0.7589 - val_loss: 10.0884 - val_acc: 0.1631\n",
      "Epoch 11/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 0.9388 - acc: 0.7563 - val_loss: 10.1164 - val_acc: 0.1631\n",
      "Epoch 12/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 0.9143 - acc: 0.7629 - val_loss: 10.1250 - val_acc: 0.1601\n",
      "Epoch 13/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 0.8854 - acc: 0.7755 - val_loss: 10.1451 - val_acc: 0.1601\n",
      "Epoch 14/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 0.9101 - acc: 0.7632 - val_loss: 10.0980 - val_acc: 0.1631\n",
      "Epoch 15/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 0.9122 - acc: 0.7656 - val_loss: 10.1124 - val_acc: 0.1601\n",
      "Epoch 16/40\n",
      "8709/8709 [==============================] - 86s 10ms/step - loss: 0.8685 - acc: 0.7746 - val_loss: 10.1201 - val_acc: 0.1662\n",
      "Epoch 17/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 0.8671 - acc: 0.7748 - val_loss: 10.1264 - val_acc: 0.1631\n",
      "Epoch 18/40\n",
      "8709/8709 [==============================] - 86s 10ms/step - loss: 0.8778 - acc: 0.7747 - val_loss: 10.1329 - val_acc: 0.1631\n",
      "Epoch 19/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 0.8472 - acc: 0.7847 - val_loss: 10.1155 - val_acc: 0.1631\n",
      "Epoch 20/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 0.8559 - acc: 0.7810 - val_loss: 10.1383 - val_acc: 0.1601\n",
      "Epoch 21/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 0.8576 - acc: 0.7813 - val_loss: 10.1517 - val_acc: 0.1631\n",
      "Epoch 22/40\n",
      "8709/8709 [==============================] - 85s 10ms/step - loss: 0.8403 - acc: 0.7885 - val_loss: 10.1233 - val_acc: 0.1541\n",
      "Epoch 23/40\n",
      "8709/8709 [==============================] - 89s 10ms/step - loss: 0.8613 - acc: 0.7787 - val_loss: 10.1318 - val_acc: 0.1631\n",
      "Epoch 24/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 0.8135 - acc: 0.7921 - val_loss: 10.1514 - val_acc: 0.1631\n",
      "Epoch 25/40\n",
      "8709/8709 [==============================] - 89s 10ms/step - loss: 0.8178 - acc: 0.7918 - val_loss: 10.1858 - val_acc: 0.1601\n",
      "Epoch 26/40\n",
      "8709/8709 [==============================] - 89s 10ms/step - loss: 0.8171 - acc: 0.7914 - val_loss: 10.1898 - val_acc: 0.1631\n",
      "Epoch 27/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 0.8143 - acc: 0.7913 - val_loss: 10.1631 - val_acc: 0.1631\n",
      "Epoch 28/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 0.8027 - acc: 0.7942 - val_loss: 10.1778 - val_acc: 0.1662\n",
      "Epoch 29/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 0.8065 - acc: 0.7934 - val_loss: 10.1818 - val_acc: 0.1601\n",
      "Epoch 30/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 0.8093 - acc: 0.7932 - val_loss: 10.1685 - val_acc: 0.1631\n",
      "Epoch 31/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 0.7946 - acc: 0.7965 - val_loss: 10.1894 - val_acc: 0.1692\n",
      "Epoch 32/40\n",
      "8709/8709 [==============================] - 88s 10ms/step - loss: 0.7999 - acc: 0.7942 - val_loss: 10.1750 - val_acc: 0.1692\n",
      "Epoch 33/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 0.7772 - acc: 0.8016 - val_loss: 10.2135 - val_acc: 0.1662\n",
      "Epoch 34/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 0.7689 - acc: 0.7996 - val_loss: 10.1781 - val_acc: 0.1662\n",
      "Epoch 35/40\n",
      "8709/8709 [==============================] - 86s 10ms/step - loss: 0.7649 - acc: 0.7987 - val_loss: 10.1965 - val_acc: 0.1662\n",
      "Epoch 36/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 0.7580 - acc: 0.8066 - val_loss: 10.2249 - val_acc: 0.1662\n",
      "Epoch 37/40\n",
      "8709/8709 [==============================] - 89s 10ms/step - loss: 0.7683 - acc: 0.8024 - val_loss: 10.2199 - val_acc: 0.1631\n",
      "Epoch 38/40\n",
      "8709/8709 [==============================] - 89s 10ms/step - loss: 0.7389 - acc: 0.8112 - val_loss: 10.2697 - val_acc: 0.1662\n",
      "Epoch 39/40\n",
      "8709/8709 [==============================] - 87s 10ms/step - loss: 0.7406 - acc: 0.8086 - val_loss: 10.2211 - val_acc: 0.1692\n",
      "Epoch 40/40\n",
      "8709/8709 [==============================] - 89s 10ms/step - loss: 0.7339 - acc: 0.8120 - val_loss: 10.2560 - val_acc: 0.1692\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=optimizers.Adam(lr=1e-5),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=40,\n",
    "                    batch_size=10,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('weights/adam_no_aug_1.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/60\n",
      "8709/8709 [==============================] - 18s 2ms/step - loss: 8.3630 - acc: 0.0010 - val_loss: 8.3428 - val_acc: 0.0000e+00\n",
      "Epoch 2/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 8.2793 - acc: 0.0037 - val_loss: 8.2713 - val_acc: 0.0000e+00\n",
      "Epoch 3/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 8.0599 - acc: 0.0052 - val_loss: 8.1640 - val_acc: 0.0000e+00\n",
      "Epoch 4/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 7.8138 - acc: 0.0076 - val_loss: 8.1189 - val_acc: 0.0000e+00\n",
      "Epoch 5/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 7.5859 - acc: 0.0118 - val_loss: 8.0780 - val_acc: 0.0000e+00\n",
      "Epoch 6/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 7.3895 - acc: 0.0134 - val_loss: 8.0721 - val_acc: 0.0030\n",
      "Epoch 7/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 7.1935 - acc: 0.0172 - val_loss: 8.1318 - val_acc: 0.0091\n",
      "Epoch 8/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 7.0276 - acc: 0.0207 - val_loss: 8.1329 - val_acc: 0.0060\n",
      "Epoch 9/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 6.8720 - acc: 0.0226 - val_loss: 8.1862 - val_acc: 0.0181\n",
      "Epoch 10/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 6.7434 - acc: 0.0240 - val_loss: 8.2787 - val_acc: 0.0121\n",
      "Epoch 11/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 6.6160 - acc: 0.0307 - val_loss: 8.3853 - val_acc: 0.0181\n",
      "Epoch 12/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 6.4714 - acc: 0.0344 - val_loss: 8.3383 - val_acc: 0.0181\n",
      "Epoch 13/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 6.3588 - acc: 0.0350 - val_loss: 8.5319 - val_acc: 0.0211\n",
      "Epoch 14/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 6.2203 - acc: 0.0384 - val_loss: 8.4893 - val_acc: 0.0211\n",
      "Epoch 15/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 6.1165 - acc: 0.0436 - val_loss: 8.6129 - val_acc: 0.0211\n",
      "Epoch 16/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 6.0452 - acc: 0.0460 - val_loss: 8.6755 - val_acc: 0.0272\n",
      "Epoch 17/60\n",
      "8709/8709 [==============================] - 18s 2ms/step - loss: 5.9324 - acc: 0.0535 - val_loss: 8.6609 - val_acc: 0.0423\n",
      "Epoch 18/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 5.8173 - acc: 0.0539 - val_loss: 8.6337 - val_acc: 0.0332\n",
      "Epoch 19/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 5.7822 - acc: 0.0570 - val_loss: 8.6695 - val_acc: 0.0332\n",
      "Epoch 20/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 5.6518 - acc: 0.0652 - val_loss: 8.9253 - val_acc: 0.0363\n",
      "Epoch 21/60\n",
      "8709/8709 [==============================] - 18s 2ms/step - loss: 5.5812 - acc: 0.0664 - val_loss: 8.9388 - val_acc: 0.0393\n",
      "Epoch 22/60\n",
      "8709/8709 [==============================] - 18s 2ms/step - loss: 5.4973 - acc: 0.0729 - val_loss: 9.0643 - val_acc: 0.0393\n",
      "Epoch 23/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 5.4174 - acc: 0.0808 - val_loss: 9.0790 - val_acc: 0.0483\n",
      "Epoch 24/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 5.3736 - acc: 0.0800 - val_loss: 9.0789 - val_acc: 0.0483\n",
      "Epoch 25/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 5.2876 - acc: 0.0818 - val_loss: 9.1573 - val_acc: 0.0604\n",
      "Epoch 26/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 5.2170 - acc: 0.0922 - val_loss: 9.1815 - val_acc: 0.0514\n",
      "Epoch 27/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 5.1462 - acc: 0.0990 - val_loss: 9.3046 - val_acc: 0.0604\n",
      "Epoch 28/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 5.0841 - acc: 0.1015 - val_loss: 9.1842 - val_acc: 0.0725\n",
      "Epoch 29/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.9994 - acc: 0.1041 - val_loss: 9.3855 - val_acc: 0.0755\n",
      "Epoch 30/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.9392 - acc: 0.1075 - val_loss: 9.3410 - val_acc: 0.0755\n",
      "Epoch 31/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.9224 - acc: 0.1123 - val_loss: 9.3875 - val_acc: 0.0816\n",
      "Epoch 32/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.8253 - acc: 0.1157 - val_loss: 9.4586 - val_acc: 0.0785\n",
      "Epoch 33/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.8027 - acc: 0.1202 - val_loss: 9.4744 - val_acc: 0.0816\n",
      "Epoch 34/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.7113 - acc: 0.1281 - val_loss: 9.4429 - val_acc: 0.0785\n",
      "Epoch 35/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.6562 - acc: 0.1329 - val_loss: 9.5142 - val_acc: 0.0937\n",
      "Epoch 36/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.6193 - acc: 0.1289 - val_loss: 9.5844 - val_acc: 0.0755\n",
      "Epoch 37/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.5345 - acc: 0.1470 - val_loss: 9.6887 - val_acc: 0.0906\n",
      "Epoch 38/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.5157 - acc: 0.1458 - val_loss: 9.6540 - val_acc: 0.0937\n",
      "Epoch 39/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.4686 - acc: 0.1454 - val_loss: 9.6809 - val_acc: 0.0967\n",
      "Epoch 40/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.4347 - acc: 0.1487 - val_loss: 9.6461 - val_acc: 0.0937\n",
      "Epoch 41/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.3940 - acc: 0.1505 - val_loss: 9.8056 - val_acc: 0.0967\n",
      "Epoch 42/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.3709 - acc: 0.1523 - val_loss: 9.7051 - val_acc: 0.0997\n",
      "Epoch 43/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.2926 - acc: 0.1645 - val_loss: 9.7647 - val_acc: 0.0997\n",
      "Epoch 44/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.2501 - acc: 0.1648 - val_loss: 9.8566 - val_acc: 0.1118\n",
      "Epoch 45/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.2543 - acc: 0.1684 - val_loss: 9.8274 - val_acc: 0.1118\n",
      "Epoch 46/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.2138 - acc: 0.1687 - val_loss: 9.9151 - val_acc: 0.1088\n",
      "Epoch 47/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.1470 - acc: 0.1774 - val_loss: 9.9175 - val_acc: 0.1178\n",
      "Epoch 48/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.1221 - acc: 0.1788 - val_loss: 9.9791 - val_acc: 0.1057\n",
      "Epoch 49/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.1341 - acc: 0.1787 - val_loss: 9.9554 - val_acc: 0.1148\n",
      "Epoch 50/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 4.0585 - acc: 0.1875 - val_loss: 9.9143 - val_acc: 0.1118\n",
      "Epoch 51/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 4.0370 - acc: 0.1912 - val_loss: 9.9807 - val_acc: 0.1208\n",
      "Epoch 52/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 4.0330 - acc: 0.1868 - val_loss: 9.9612 - val_acc: 0.1118\n",
      "Epoch 53/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 4.0099 - acc: 0.1953 - val_loss: 9.9670 - val_acc: 0.1088\n",
      "Epoch 54/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 3.9658 - acc: 0.2005 - val_loss: 10.0114 - val_acc: 0.1118\n",
      "Epoch 55/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 3.9000 - acc: 0.2028 - val_loss: 10.0561 - val_acc: 0.1148\n",
      "Epoch 56/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 3.9559 - acc: 0.1958 - val_loss: 9.9792 - val_acc: 0.1239\n",
      "Epoch 57/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 3.8781 - acc: 0.2015 - val_loss: 10.0593 - val_acc: 0.1118\n",
      "Epoch 58/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 3.8920 - acc: 0.1998 - val_loss: 10.0688 - val_acc: 0.1178\n",
      "Epoch 59/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 3.8768 - acc: 0.2096 - val_loss: 10.0681 - val_acc: 0.1178\n",
      "Epoch 60/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 3.8266 - acc: 0.2129 - val_loss: 10.2017 - val_acc: 0.1118\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import optimizers\n",
    "\n",
    "model = models.Sequential()\n",
    "model.add(layers.Dense(128, activation='relu', input_dim=ll_size*ll_size*2048)) # we can play around with \n",
    "model.add(layers.Dropout(0.5))\n",
    "model.add(layers.Dense(classes_count, activation='softmax'))\n",
    "\n",
    "\n",
    "model.compile(optimizer=optimizers.Adam(lr=2e-4),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=60,\n",
    "                    batch_size=100,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/60\n",
      "8709/8709 [==============================] - 14s 2ms/step - loss: 3.7769 - acc: 0.2159 - val_loss: 10.1866 - val_acc: 0.1208\n",
      "Epoch 2/60\n",
      "8709/8709 [==============================] - 13s 1ms/step - loss: 3.7765 - acc: 0.2199 - val_loss: 10.1597 - val_acc: 0.1178\n",
      "Epoch 3/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.7533 - acc: 0.2226 - val_loss: 10.1429 - val_acc: 0.1239\n",
      "Epoch 4/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.7516 - acc: 0.2263 - val_loss: 10.1881 - val_acc: 0.1239\n",
      "Epoch 5/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.7241 - acc: 0.2288 - val_loss: 10.2107 - val_acc: 0.1178\n",
      "Epoch 6/60\n",
      "8709/8709 [==============================] - 13s 1ms/step - loss: 3.7370 - acc: 0.2259 - val_loss: 10.1592 - val_acc: 0.1239\n",
      "Epoch 7/60\n",
      "8709/8709 [==============================] - 13s 1ms/step - loss: 3.6895 - acc: 0.2315 - val_loss: 10.2188 - val_acc: 0.1269\n",
      "Epoch 8/60\n",
      "8709/8709 [==============================] - 13s 1ms/step - loss: 3.6758 - acc: 0.2299 - val_loss: 10.2519 - val_acc: 0.1360\n",
      "Epoch 9/60\n",
      "8709/8709 [==============================] - 13s 1ms/step - loss: 3.6646 - acc: 0.2358 - val_loss: 10.2562 - val_acc: 0.1299\n",
      "Epoch 10/60\n",
      "8709/8709 [==============================] - 13s 1ms/step - loss: 3.6270 - acc: 0.2410 - val_loss: 10.2212 - val_acc: 0.1299\n",
      "Epoch 11/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.6381 - acc: 0.2350 - val_loss: 10.2770 - val_acc: 0.1269\n",
      "Epoch 12/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.6072 - acc: 0.2391 - val_loss: 10.2763 - val_acc: 0.1299\n",
      "Epoch 13/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.6062 - acc: 0.2369 - val_loss: 10.2715 - val_acc: 0.1269\n",
      "Epoch 14/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.6234 - acc: 0.2387 - val_loss: 10.2837 - val_acc: 0.1360\n",
      "Epoch 15/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.5975 - acc: 0.2414 - val_loss: 10.3153 - val_acc: 0.1360\n",
      "Epoch 16/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.5774 - acc: 0.2438 - val_loss: 10.2954 - val_acc: 0.1360\n",
      "Epoch 17/60\n",
      "8709/8709 [==============================] - 14s 2ms/step - loss: 3.5471 - acc: 0.2447 - val_loss: 10.3310 - val_acc: 0.1360\n",
      "Epoch 18/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.5353 - acc: 0.2474 - val_loss: 10.3128 - val_acc: 0.1360\n",
      "Epoch 19/60\n",
      "8709/8709 [==============================] - 13s 1ms/step - loss: 3.5554 - acc: 0.2477 - val_loss: 10.3498 - val_acc: 0.1299\n",
      "Epoch 20/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.4968 - acc: 0.2517 - val_loss: 10.3574 - val_acc: 0.1360\n",
      "Epoch 21/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.4971 - acc: 0.2586 - val_loss: 10.4587 - val_acc: 0.1239\n",
      "Epoch 22/60\n",
      "8709/8709 [==============================] - 13s 2ms/step - loss: 3.5365 - acc: 0.2462 - val_loss: 10.3360 - val_acc: 0.1329\n",
      "Epoch 23/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.5052 - acc: 0.2564 - val_loss: 10.3568 - val_acc: 0.1299\n",
      "Epoch 24/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.5141 - acc: 0.2531 - val_loss: 10.4201 - val_acc: 0.1329\n",
      "Epoch 25/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.4491 - acc: 0.2635 - val_loss: 10.4210 - val_acc: 0.1299\n",
      "Epoch 26/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.4578 - acc: 0.2603 - val_loss: 10.3600 - val_acc: 0.1299\n",
      "Epoch 27/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.4568 - acc: 0.2592 - val_loss: 10.3945 - val_acc: 0.1329\n",
      "Epoch 28/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.4830 - acc: 0.2539 - val_loss: 10.4343 - val_acc: 0.1360\n",
      "Epoch 29/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.4742 - acc: 0.2595 - val_loss: 10.4359 - val_acc: 0.1329\n",
      "Epoch 30/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.4305 - acc: 0.2579 - val_loss: 10.4779 - val_acc: 0.1329\n",
      "Epoch 31/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.4167 - acc: 0.2632 - val_loss: 10.4144 - val_acc: 0.1299\n",
      "Epoch 32/60\n",
      "8709/8709 [==============================] - 14s 2ms/step - loss: 3.4442 - acc: 0.2551 - val_loss: 10.4722 - val_acc: 0.1360\n",
      "Epoch 33/60\n",
      "8709/8709 [==============================] - 14s 2ms/step - loss: 3.4280 - acc: 0.2590 - val_loss: 10.4885 - val_acc: 0.1329\n",
      "Epoch 34/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.3530 - acc: 0.2719 - val_loss: 10.5065 - val_acc: 0.1329\n",
      "Epoch 35/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.3992 - acc: 0.2588 - val_loss: 10.5028 - val_acc: 0.1390\n",
      "Epoch 36/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.3769 - acc: 0.2652 - val_loss: 10.4635 - val_acc: 0.1450\n",
      "Epoch 37/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.3685 - acc: 0.2711 - val_loss: 10.5301 - val_acc: 0.1420\n",
      "Epoch 38/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.3701 - acc: 0.2716 - val_loss: 10.5029 - val_acc: 0.1390\n",
      "Epoch 39/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.3385 - acc: 0.2753 - val_loss: 10.5081 - val_acc: 0.1420\n",
      "Epoch 40/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.3617 - acc: 0.2674 - val_loss: 10.5369 - val_acc: 0.1329\n",
      "Epoch 41/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.3284 - acc: 0.2734 - val_loss: 10.5370 - val_acc: 0.1390\n",
      "Epoch 42/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.3103 - acc: 0.2752 - val_loss: 10.5375 - val_acc: 0.1360\n",
      "Epoch 43/60\n",
      "8709/8709 [==============================] - 13s 1ms/step - loss: 3.3437 - acc: 0.2669 - val_loss: 10.5831 - val_acc: 0.1420\n",
      "Epoch 44/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.2857 - acc: 0.2819 - val_loss: 10.6292 - val_acc: 0.1390\n",
      "Epoch 45/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.2952 - acc: 0.2795 - val_loss: 10.6181 - val_acc: 0.1420\n",
      "Epoch 46/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.3004 - acc: 0.2722 - val_loss: 10.6167 - val_acc: 0.1390\n",
      "Epoch 47/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.2871 - acc: 0.2767 - val_loss: 10.6127 - val_acc: 0.1420\n",
      "Epoch 48/60\n",
      "8709/8709 [==============================] - 13s 2ms/step - loss: 3.3003 - acc: 0.2775 - val_loss: 10.5831 - val_acc: 0.1390\n",
      "Epoch 49/60\n",
      "8709/8709 [==============================] - 13s 1ms/step - loss: 3.2931 - acc: 0.2799 - val_loss: 10.6291 - val_acc: 0.1360\n",
      "Epoch 50/60\n",
      "8709/8709 [==============================] - 14s 2ms/step - loss: 3.2896 - acc: 0.2809 - val_loss: 10.6781 - val_acc: 0.1390\n",
      "Epoch 51/60\n",
      "8709/8709 [==============================] - 13s 1ms/step - loss: 3.1921 - acc: 0.3010 - val_loss: 10.6577 - val_acc: 0.1390\n",
      "Epoch 52/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.2739 - acc: 0.2820 - val_loss: 10.6118 - val_acc: 0.1420\n",
      "Epoch 53/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.2158 - acc: 0.2867 - val_loss: 10.6859 - val_acc: 0.1420\n",
      "Epoch 54/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.2513 - acc: 0.2775 - val_loss: 10.7312 - val_acc: 0.1480\n",
      "Epoch 55/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.2408 - acc: 0.2835 - val_loss: 10.6367 - val_acc: 0.1511\n",
      "Epoch 56/60\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.2112 - acc: 0.2944 - val_loss: 10.7382 - val_acc: 0.1450\n",
      "Epoch 57/60\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.1982 - acc: 0.2929 - val_loss: 10.6972 - val_acc: 0.1480\n",
      "Epoch 58/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.2213 - acc: 0.2844 - val_loss: 10.7383 - val_acc: 0.1571\n",
      "Epoch 59/60\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 3.1771 - acc: 0.2931 - val_loss: 10.7202 - val_acc: 0.1511\n",
      "Epoch 60/60\n",
      "8709/8709 [==============================] - 14s 2ms/step - loss: 3.2318 - acc: 0.2868 - val_loss: 10.8145 - val_acc: 0.1480\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=60,\n",
    "                    batch_size=150,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.1875 - acc: 0.2941 - val_loss: 10.7829 - val_acc: 0.1511\n",
      "Epoch 2/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.1294 - acc: 0.3043 - val_loss: 10.8145 - val_acc: 0.1511\n",
      "Epoch 3/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.1288 - acc: 0.2999 - val_loss: 10.7962 - val_acc: 0.1511\n",
      "Epoch 4/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.1191 - acc: 0.2993 - val_loss: 10.7327 - val_acc: 0.1511\n",
      "Epoch 5/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.1262 - acc: 0.3053 - val_loss: 10.7662 - val_acc: 0.1480\n",
      "Epoch 6/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.1112 - acc: 0.3078 - val_loss: 10.7917 - val_acc: 0.1511\n",
      "Epoch 7/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0882 - acc: 0.3083 - val_loss: 10.8035 - val_acc: 0.1541\n",
      "Epoch 8/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0891 - acc: 0.3100 - val_loss: 10.7965 - val_acc: 0.1571\n",
      "Epoch 9/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.1654 - acc: 0.2988 - val_loss: 10.8512 - val_acc: 0.1450\n",
      "Epoch 10/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.1267 - acc: 0.3022 - val_loss: 10.8046 - val_acc: 0.1511\n",
      "Epoch 11/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0699 - acc: 0.3103 - val_loss: 10.7965 - val_acc: 0.1511\n",
      "Epoch 12/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.1305 - acc: 0.2972 - val_loss: 10.8472 - val_acc: 0.1571\n",
      "Epoch 13/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0770 - acc: 0.3065 - val_loss: 10.8455 - val_acc: 0.1571\n",
      "Epoch 14/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0911 - acc: 0.3024 - val_loss: 10.8620 - val_acc: 0.1541\n",
      "Epoch 15/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0967 - acc: 0.3018 - val_loss: 10.8204 - val_acc: 0.1541\n",
      "Epoch 16/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0766 - acc: 0.3134 - val_loss: 10.8165 - val_acc: 0.1541\n",
      "Epoch 17/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0891 - acc: 0.3091 - val_loss: 10.8405 - val_acc: 0.1450\n",
      "Epoch 18/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0647 - acc: 0.3139 - val_loss: 10.7992 - val_acc: 0.1541\n",
      "Epoch 19/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0636 - acc: 0.3065 - val_loss: 10.8354 - val_acc: 0.1541\n",
      "Epoch 20/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0822 - acc: 0.3046 - val_loss: 10.8310 - val_acc: 0.1480\n",
      "Epoch 21/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0848 - acc: 0.3052 - val_loss: 10.8689 - val_acc: 0.1541\n",
      "Epoch 22/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0439 - acc: 0.3140 - val_loss: 10.8570 - val_acc: 0.1511\n",
      "Epoch 23/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0321 - acc: 0.3143 - val_loss: 10.9146 - val_acc: 0.1450\n",
      "Epoch 24/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0749 - acc: 0.3091 - val_loss: 10.8781 - val_acc: 0.1450\n",
      "Epoch 25/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0249 - acc: 0.3216 - val_loss: 10.8290 - val_acc: 0.1480\n",
      "Epoch 26/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0083 - acc: 0.3177 - val_loss: 10.8760 - val_acc: 0.1511\n",
      "Epoch 27/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0778 - acc: 0.3062 - val_loss: 10.9285 - val_acc: 0.1480\n",
      "Epoch 28/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0397 - acc: 0.3150 - val_loss: 10.9648 - val_acc: 0.1511\n",
      "Epoch 29/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0291 - acc: 0.3142 - val_loss: 10.9744 - val_acc: 0.1480\n",
      "Epoch 30/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0519 - acc: 0.3123 - val_loss: 10.9199 - val_acc: 0.1480\n",
      "Epoch 31/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0248 - acc: 0.3170 - val_loss: 10.9464 - val_acc: 0.1420\n",
      "Epoch 32/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0384 - acc: 0.3065 - val_loss: 10.9363 - val_acc: 0.1511\n",
      "Epoch 33/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0202 - acc: 0.3178 - val_loss: 10.8791 - val_acc: 0.1450\n",
      "Epoch 34/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0869 - acc: 0.2995 - val_loss: 10.9552 - val_acc: 0.1541\n",
      "Epoch 35/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0383 - acc: 0.3111 - val_loss: 10.9879 - val_acc: 0.1480\n",
      "Epoch 36/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0487 - acc: 0.3130 - val_loss: 11.0285 - val_acc: 0.1480\n",
      "Epoch 37/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0253 - acc: 0.3124 - val_loss: 10.9966 - val_acc: 0.1480\n",
      "Epoch 38/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0261 - acc: 0.3169 - val_loss: 11.0044 - val_acc: 0.1511\n",
      "Epoch 39/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0186 - acc: 0.3206 - val_loss: 10.9761 - val_acc: 0.1571\n",
      "Epoch 40/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0307 - acc: 0.3059 - val_loss: 11.0149 - val_acc: 0.1511\n",
      "Epoch 41/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0260 - acc: 0.3119 - val_loss: 11.0140 - val_acc: 0.1511\n",
      "Epoch 42/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0211 - acc: 0.3208 - val_loss: 11.0052 - val_acc: 0.1541\n",
      "Epoch 43/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0146 - acc: 0.3183 - val_loss: 11.0920 - val_acc: 0.1571\n",
      "Epoch 44/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0182 - acc: 0.3181 - val_loss: 11.0902 - val_acc: 0.1601\n",
      "Epoch 45/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0383 - acc: 0.3145 - val_loss: 11.0148 - val_acc: 0.1480\n",
      "Epoch 46/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9992 - acc: 0.3192 - val_loss: 11.0218 - val_acc: 0.1511\n",
      "Epoch 47/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9904 - acc: 0.3192 - val_loss: 11.0631 - val_acc: 0.1511\n",
      "Epoch 48/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9538 - acc: 0.3223 - val_loss: 11.0181 - val_acc: 0.1480\n",
      "Epoch 49/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.9905 - acc: 0.3168 - val_loss: 11.0592 - val_acc: 0.1511\n",
      "Epoch 50/120\n",
      "8709/8709 [==============================] - 12s 1ms/step - loss: 2.9629 - acc: 0.3188 - val_loss: 11.0839 - val_acc: 0.1450\n",
      "Epoch 51/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9908 - acc: 0.3185 - val_loss: 11.0214 - val_acc: 0.1450\n",
      "Epoch 52/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.9824 - acc: 0.3213 - val_loss: 11.0494 - val_acc: 0.1541\n",
      "Epoch 53/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.9872 - acc: 0.3179 - val_loss: 11.0282 - val_acc: 0.1450\n",
      "Epoch 54/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 3.0080 - acc: 0.3181 - val_loss: 11.0671 - val_acc: 0.1480\n",
      "Epoch 55/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9095 - acc: 0.3334 - val_loss: 11.0751 - val_acc: 0.1571\n",
      "Epoch 56/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.9566 - acc: 0.3216 - val_loss: 11.0749 - val_acc: 0.1571\n",
      "Epoch 57/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.9966 - acc: 0.3220 - val_loss: 11.0882 - val_acc: 0.1541\n",
      "Epoch 58/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.9538 - acc: 0.3228 - val_loss: 11.1611 - val_acc: 0.1541\n",
      "Epoch 59/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9687 - acc: 0.3250 - val_loss: 11.0419 - val_acc: 0.1480\n",
      "Epoch 60/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 3.0068 - acc: 0.3142 - val_loss: 11.1089 - val_acc: 0.1511\n",
      "Epoch 61/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9479 - acc: 0.3238 - val_loss: 11.1422 - val_acc: 0.1511\n",
      "Epoch 62/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9342 - acc: 0.3320 - val_loss: 11.1122 - val_acc: 0.1511\n",
      "Epoch 63/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.9314 - acc: 0.3337 - val_loss: 11.0565 - val_acc: 0.1571\n",
      "Epoch 64/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9676 - acc: 0.3216 - val_loss: 11.1267 - val_acc: 0.1601\n",
      "Epoch 65/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9261 - acc: 0.3285 - val_loss: 11.1621 - val_acc: 0.1601\n",
      "Epoch 66/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9336 - acc: 0.3256 - val_loss: 11.2192 - val_acc: 0.1631\n",
      "Epoch 67/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9051 - acc: 0.3317 - val_loss: 11.1419 - val_acc: 0.1571\n",
      "Epoch 68/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9219 - acc: 0.3303 - val_loss: 11.1185 - val_acc: 0.1511\n",
      "Epoch 69/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9367 - acc: 0.3230 - val_loss: 11.1166 - val_acc: 0.1571\n",
      "Epoch 70/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9662 - acc: 0.3208 - val_loss: 11.1554 - val_acc: 0.1541\n",
      "Epoch 71/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.9120 - acc: 0.3278 - val_loss: 11.1378 - val_acc: 0.1541\n",
      "Epoch 72/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.9308 - acc: 0.3240 - val_loss: 11.1408 - val_acc: 0.1511\n",
      "Epoch 73/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.9235 - acc: 0.3232 - val_loss: 11.1643 - val_acc: 0.1511\n",
      "Epoch 74/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9477 - acc: 0.3223 - val_loss: 11.1659 - val_acc: 0.1571\n",
      "Epoch 75/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9236 - acc: 0.3272 - val_loss: 11.1872 - val_acc: 0.1601\n",
      "Epoch 76/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8936 - acc: 0.3324 - val_loss: 11.1923 - val_acc: 0.1601\n",
      "Epoch 77/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8808 - acc: 0.3385 - val_loss: 11.2217 - val_acc: 0.1541\n",
      "Epoch 78/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9250 - acc: 0.3236 - val_loss: 11.1674 - val_acc: 0.1571\n",
      "Epoch 79/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9275 - acc: 0.3276 - val_loss: 11.2044 - val_acc: 0.1480\n",
      "Epoch 80/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9064 - acc: 0.3251 - val_loss: 11.1799 - val_acc: 0.1571\n",
      "Epoch 81/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9293 - acc: 0.3352 - val_loss: 11.2184 - val_acc: 0.1511\n",
      "Epoch 82/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9032 - acc: 0.3284 - val_loss: 11.2811 - val_acc: 0.1601\n",
      "Epoch 83/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8944 - acc: 0.3346 - val_loss: 11.2819 - val_acc: 0.1541\n",
      "Epoch 84/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8813 - acc: 0.3392 - val_loss: 11.1735 - val_acc: 0.1541\n",
      "Epoch 85/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8987 - acc: 0.3309 - val_loss: 11.2600 - val_acc: 0.1480\n",
      "Epoch 86/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9073 - acc: 0.3368 - val_loss: 11.2201 - val_acc: 0.1631\n",
      "Epoch 87/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8805 - acc: 0.3402 - val_loss: 11.2332 - val_acc: 0.1541\n",
      "Epoch 88/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.9017 - acc: 0.3276 - val_loss: 11.2637 - val_acc: 0.1480\n",
      "Epoch 89/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.8970 - acc: 0.3343 - val_loss: 11.2692 - val_acc: 0.1541\n",
      "Epoch 90/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8683 - acc: 0.3378 - val_loss: 11.2833 - val_acc: 0.1541\n",
      "Epoch 91/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.8554 - acc: 0.3361 - val_loss: 11.2273 - val_acc: 0.1571\n",
      "Epoch 92/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.8833 - acc: 0.3320 - val_loss: 11.3118 - val_acc: 0.1511\n",
      "Epoch 93/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.8587 - acc: 0.3362 - val_loss: 11.2053 - val_acc: 0.1511\n",
      "Epoch 94/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8987 - acc: 0.3329 - val_loss: 11.2855 - val_acc: 0.1541\n",
      "Epoch 95/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8454 - acc: 0.3345 - val_loss: 11.3201 - val_acc: 0.1571\n",
      "Epoch 96/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8535 - acc: 0.3316 - val_loss: 11.3036 - val_acc: 0.1541\n",
      "Epoch 97/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8803 - acc: 0.3302 - val_loss: 11.3681 - val_acc: 0.1541\n",
      "Epoch 98/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8773 - acc: 0.3346 - val_loss: 11.3192 - val_acc: 0.1541\n",
      "Epoch 99/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8514 - acc: 0.3372 - val_loss: 11.3279 - val_acc: 0.1511\n",
      "Epoch 100/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8232 - acc: 0.3452 - val_loss: 11.2342 - val_acc: 0.1480\n",
      "Epoch 101/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8847 - acc: 0.3376 - val_loss: 11.3432 - val_acc: 0.1571\n",
      "Epoch 102/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8303 - acc: 0.3473 - val_loss: 11.3553 - val_acc: 0.1541\n",
      "Epoch 103/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8553 - acc: 0.3348 - val_loss: 11.3648 - val_acc: 0.1571\n",
      "Epoch 104/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.8285 - acc: 0.3416 - val_loss: 11.3453 - val_acc: 0.1631\n",
      "Epoch 105/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.8177 - acc: 0.3455 - val_loss: 11.4114 - val_acc: 0.1480\n",
      "Epoch 106/120\n",
      "8709/8709 [==============================] - 11s 1ms/step - loss: 2.8489 - acc: 0.3401 - val_loss: 11.3791 - val_acc: 0.1511\n",
      "Epoch 107/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8555 - acc: 0.3368 - val_loss: 11.3788 - val_acc: 0.1541\n",
      "Epoch 108/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8493 - acc: 0.3400 - val_loss: 11.3346 - val_acc: 0.1571\n",
      "Epoch 109/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8620 - acc: 0.3362 - val_loss: 11.3673 - val_acc: 0.1662\n",
      "Epoch 110/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.7894 - acc: 0.3457 - val_loss: 11.3588 - val_acc: 0.1571\n",
      "Epoch 111/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.7868 - acc: 0.3509 - val_loss: 11.3362 - val_acc: 0.1541\n",
      "Epoch 112/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8174 - acc: 0.3437 - val_loss: 11.3865 - val_acc: 0.1571\n",
      "Epoch 113/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8278 - acc: 0.3423 - val_loss: 11.3479 - val_acc: 0.1601\n",
      "Epoch 114/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8042 - acc: 0.3478 - val_loss: 11.3551 - val_acc: 0.1541\n",
      "Epoch 115/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.7587 - acc: 0.3526 - val_loss: 11.3820 - val_acc: 0.1571\n",
      "Epoch 116/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.7993 - acc: 0.3456 - val_loss: 11.3875 - val_acc: 0.1601\n",
      "Epoch 117/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8268 - acc: 0.3370 - val_loss: 11.3903 - val_acc: 0.1541\n",
      "Epoch 118/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8471 - acc: 0.3376 - val_loss: 11.4352 - val_acc: 0.1541\n",
      "Epoch 119/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8192 - acc: 0.3395 - val_loss: 11.3930 - val_acc: 0.1601\n",
      "Epoch 120/120\n",
      "8709/8709 [==============================] - 10s 1ms/step - loss: 2.8928 - acc: 0.3272 - val_loss: 11.4002 - val_acc: 0.1541\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=120,\n",
    "                    batch_size=300,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('weights/adam_no_aug_3.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 2.7807 - acc: 0.3446 - val_loss: 11.4164 - val_acc: 0.1601\n",
      "Epoch 2/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7610 - acc: 0.3522 - val_loss: 11.4262 - val_acc: 0.1631\n",
      "Epoch 3/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7836 - acc: 0.3501 - val_loss: 11.4451 - val_acc: 0.1571\n",
      "Epoch 4/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 2.7363 - acc: 0.3592 - val_loss: 11.4439 - val_acc: 0.1571\n",
      "Epoch 5/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7777 - acc: 0.3457 - val_loss: 11.4170 - val_acc: 0.1601\n",
      "Epoch 6/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7424 - acc: 0.3556 - val_loss: 11.4105 - val_acc: 0.1631\n",
      "Epoch 7/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7372 - acc: 0.3579 - val_loss: 11.4208 - val_acc: 0.1571\n",
      "Epoch 8/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7431 - acc: 0.3563 - val_loss: 11.4355 - val_acc: 0.1571\n",
      "Epoch 9/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7376 - acc: 0.3547 - val_loss: 11.4235 - val_acc: 0.1571\n",
      "Epoch 10/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7186 - acc: 0.3612 - val_loss: 11.4020 - val_acc: 0.1601\n",
      "Epoch 11/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6984 - acc: 0.3647 - val_loss: 11.4224 - val_acc: 0.1571\n",
      "Epoch 12/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7225 - acc: 0.3585 - val_loss: 11.4402 - val_acc: 0.1631\n",
      "Epoch 13/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7259 - acc: 0.3580 - val_loss: 11.4448 - val_acc: 0.1571\n",
      "Epoch 14/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7241 - acc: 0.3581 - val_loss: 11.4328 - val_acc: 0.1631\n",
      "Epoch 15/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7049 - acc: 0.3651 - val_loss: 11.4474 - val_acc: 0.1662\n",
      "Epoch 16/60\n",
      "8709/8709 [==============================] - 17s 2ms/step - loss: 2.7443 - acc: 0.3516 - val_loss: 11.4290 - val_acc: 0.1692\n",
      "Epoch 17/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7199 - acc: 0.3599 - val_loss: 11.4371 - val_acc: 0.1631\n",
      "Epoch 18/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7393 - acc: 0.3577 - val_loss: 11.4308 - val_acc: 0.1631\n",
      "Epoch 19/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7543 - acc: 0.3548 - val_loss: 11.4438 - val_acc: 0.1662\n",
      "Epoch 20/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7317 - acc: 0.3565 - val_loss: 11.4280 - val_acc: 0.1631\n",
      "Epoch 21/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7148 - acc: 0.3596 - val_loss: 11.4426 - val_acc: 0.1631\n",
      "Epoch 22/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7002 - acc: 0.3692 - val_loss: 11.4591 - val_acc: 0.1662\n",
      "Epoch 23/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7060 - acc: 0.3642 - val_loss: 11.4473 - val_acc: 0.1662\n",
      "Epoch 24/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7206 - acc: 0.3537 - val_loss: 11.4424 - val_acc: 0.1662\n",
      "Epoch 25/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7499 - acc: 0.3530 - val_loss: 11.4447 - val_acc: 0.1662\n",
      "Epoch 26/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7139 - acc: 0.3642 - val_loss: 11.4483 - val_acc: 0.1631\n",
      "Epoch 27/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7263 - acc: 0.3587 - val_loss: 11.4573 - val_acc: 0.1662\n",
      "Epoch 28/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7502 - acc: 0.3506 - val_loss: 11.4681 - val_acc: 0.1631\n",
      "Epoch 29/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7023 - acc: 0.3626 - val_loss: 11.4376 - val_acc: 0.1662\n",
      "Epoch 30/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7175 - acc: 0.3604 - val_loss: 11.4708 - val_acc: 0.1662\n",
      "Epoch 31/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6922 - acc: 0.3666 - val_loss: 11.4542 - val_acc: 0.1631\n",
      "Epoch 32/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7009 - acc: 0.3578 - val_loss: 11.4762 - val_acc: 0.1662\n",
      "Epoch 33/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7180 - acc: 0.3504 - val_loss: 11.4577 - val_acc: 0.1662\n",
      "Epoch 34/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6992 - acc: 0.3601 - val_loss: 11.4677 - val_acc: 0.1692\n",
      "Epoch 35/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7190 - acc: 0.3643 - val_loss: 11.4453 - val_acc: 0.1692\n",
      "Epoch 36/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6587 - acc: 0.3726 - val_loss: 11.4823 - val_acc: 0.1662\n",
      "Epoch 37/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6971 - acc: 0.3659 - val_loss: 11.4651 - val_acc: 0.1631\n",
      "Epoch 38/60\n",
      "8709/8709 [==============================] - 15s 2ms/step - loss: 2.7226 - acc: 0.3607 - val_loss: 11.4622 - val_acc: 0.1662\n",
      "Epoch 39/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7027 - acc: 0.3588 - val_loss: 11.4694 - val_acc: 0.1662\n",
      "Epoch 40/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6793 - acc: 0.3654 - val_loss: 11.4747 - val_acc: 0.1662\n",
      "Epoch 41/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6789 - acc: 0.3626 - val_loss: 11.4734 - val_acc: 0.1662\n",
      "Epoch 42/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6975 - acc: 0.3640 - val_loss: 11.4700 - val_acc: 0.1662\n",
      "Epoch 43/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6859 - acc: 0.3631 - val_loss: 11.4645 - val_acc: 0.1692\n",
      "Epoch 44/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6847 - acc: 0.3624 - val_loss: 11.4680 - val_acc: 0.1631\n",
      "Epoch 45/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7251 - acc: 0.3551 - val_loss: 11.4810 - val_acc: 0.1662\n",
      "Epoch 46/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7305 - acc: 0.3588 - val_loss: 11.4808 - val_acc: 0.1631\n",
      "Epoch 47/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7102 - acc: 0.3636 - val_loss: 11.4639 - val_acc: 0.1662\n",
      "Epoch 48/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6748 - acc: 0.3689 - val_loss: 11.4895 - val_acc: 0.1692\n",
      "Epoch 49/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6873 - acc: 0.3641 - val_loss: 11.4944 - val_acc: 0.1631\n",
      "Epoch 50/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6651 - acc: 0.3658 - val_loss: 11.4803 - val_acc: 0.1631\n",
      "Epoch 51/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6635 - acc: 0.3690 - val_loss: 11.4855 - val_acc: 0.1692\n",
      "Epoch 52/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7084 - acc: 0.3594 - val_loss: 11.4881 - val_acc: 0.1662\n",
      "Epoch 53/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7117 - acc: 0.3597 - val_loss: 11.4830 - val_acc: 0.1662\n",
      "Epoch 54/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6954 - acc: 0.3681 - val_loss: 11.4930 - val_acc: 0.1662\n",
      "Epoch 55/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6862 - acc: 0.3664 - val_loss: 11.4902 - val_acc: 0.1662\n",
      "Epoch 56/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7173 - acc: 0.3625 - val_loss: 11.4773 - val_acc: 0.1662\n",
      "Epoch 57/60\n",
      "8709/8709 [==============================] - 15s 2ms/step - loss: 2.7103 - acc: 0.3620 - val_loss: 11.4842 - val_acc: 0.1662\n",
      "Epoch 58/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6678 - acc: 0.3681 - val_loss: 11.5044 - val_acc: 0.1662\n",
      "Epoch 59/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.7046 - acc: 0.3634 - val_loss: 11.4813 - val_acc: 0.1692\n",
      "Epoch 60/60\n",
      "8709/8709 [==============================] - 16s 2ms/step - loss: 2.6753 - acc: 0.3685 - val_loss: 11.5101 - val_acc: 0.1662\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=optimizers.Adam(lr=2e-5),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=60,\n",
    "                    batch_size=100,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=150,\n",
    "                    batch_size=10,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('weights/no_aug_2.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=150,\n",
    "                    batch_size=10,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('weights/no_aug_3.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 128)               9437312   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 4250)              548250    \n",
      "=================================================================\n",
      "Total params: 9,985,562\n",
      "Trainable params: 9,985,562\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.models import load_model\n",
    "\n",
    "model = load_model('weights/name_that_whale_702.h5')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAEmCAYAAADLHS+iAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAMTQAADE0B0s6tTgAAIABJREFUeJzt3Xl8FeXZ//HPFSAgkIVVQPa1uBYVS1Gp2CW2WrXWurVW\n0acKtdpSEZWK1bbWpZY+/bUqUqul1ap9nopt3WIfqdKKirtYFATZZKmsCVsSSK7fHzMnmXNykpyE\nwOSE7/v1Oq9kZu6Zuc6ck7ly33PPPebuiIiIxCEn7gBEROTApSQkIiKxURISEZHYKAmJiEhslIRE\nRCQ2SkIiIhIbJSEREYmNkpBgZg+Y2f9rRPlLzOzNfRnTvmJmG8zsnH24/ZfMbFr4e66ZbTOzT9VT\n/mdm9vRe7nOYmZWaWa+92Y5IHNrGHYBkxsy2AYk7i9sDbYCdgIXzv+juLzZl2+4+oZHl7wfub8q+\nDiTuXgHkZVI0022a2a3AMe7+hch+PgDyGx+hSPyUhLKEu1efzMzsx8Dx7n5yfeuYWW54IhRpVcys\nnbvvjjsO2XtqjmtFzOxWM/u/8OdaYH5k/gdh09AKM5thZrmR9R42s1mR6XVmdp2ZFYfrLDazUyPL\nLzezD1LWn21mM81sk5mtNbMbUmI7w8z+HTYbPWNmt5jZe/W8l0+a2dyw+WyLmc03sxMiy0eYWZWZ\nfd3MFobb/aeZDY2UyTOz34cxrTSziQ0cv5+b2RMp8/qZ2W4z+0Q4/UB4DLeZ2VIz+0E922sfxjg2\nMu+/wvVKzOwRUmpKZvbd8DiVmNlHZna/mRWGyy4Bvg+cFO6/1MyOiRyLPpHtXBZuZ6uZvWVm5zXm\n2DX28wjLHGFmT5jZejPbbGb/MrOe4bKOZnZ7+D0sDX+eGy671cz+nrKtdN/JG8zsWTMrASaFn82T\nZvaf8Hi9YWZnpGxnQLitj8Iyr5nZoWb2pfB7kZtSfrGZXVbXcZB9wN31yrIX8GNgbpr5twIVwBSg\nHdAhnH8h0Dv8/XBgBTA9st7DwKzI9DpgKXAYQXPf9cDmyPYuB5akrL8TOCMsPw7YA3wqXD4yjOtc\ngn98jgc2AovqeY9HAePD99EeuAXYBBSEy0cAVcCTQLewzF+ApyPb+D3wEtAT6Ag8GMZxTh37TMTZ\nOzLvh8C/ItP/BXQNfz8e2ApcGFn+EjAt/L19GOPYcPpkoAz4fHgczgqnn4qsfzYwKPx9IPA68NuU\nz/jZlLhHAJVAn3D66+GxOj78PE4JP5/PZ3rsmvB59AG2ADcAncL3Nzqy/H+BfwJDIuWPrOc9pftO\nrgKODqc7AAOALwMHETRPTwyPZ2IfnYAPgbuBwnDeYeG+LVz29cg+PguUAJ3i/hs/kF6xB6BXEz60\n+pPQ0gzWvw74Z2Q63R/81ZHpruFJ64hwOl0SeiJlH+8AV0bifT5l+a+oJwmliTkH2AF8NpxOnEiP\niZQ5C9gU/t4O2A2cHFnePVwnbRIKy7xITRIxYDlwUT3lZwJ/iEzXl4R+Hy0bznuCSBJKs/3zgNUp\nn3FDSeh54JY0cT6WybFr4ufxA2BBHWUPCfc3so7lmSahGzKI633g0vD3bwBrgJx6/g7mRaYfBe7O\n9Bjo1TwvNce1PitSZ5jZd8zszbD5YQswnaB2UJ91kd93hD/ru8i+NmV6R6T8IcDKhuKMMrNBZvZo\n2Iy2leC/7vYkx+1p4kzsszfBibJ6P+6+MfJe6nIfkOio8QWgC/CnMCYzsx+GzVxbzGwz8E0aPpYJ\nfQmSWlTStJmda0EPu4/Dz+o3jdh+Qj9gWcq8pUD/yHR9x66WDD6PQcDiOlYfFO5vScbvIL0VKTF1\nC5srPwybHbcAgyMxDQQ+dPeqOrb3W2C0mX0ibDY8kyBZy36kJNT6JP3BmdlJwB3AFUBPd+9CUDOx\n/RjTGoKmk6iBDazzAFBO0PxSSNBsVE7mca8jOBbV+zGzHgRNNPV5FOhhZicDlwKPuPuucNnFBLXA\n8wia5LoS1G4yjekjar/vaHyDgT8CtwGHhJ/Vt1K2X9cJNWo1MCRl3hCC5qymaujzWAEMr2PdFeHP\nupZvo/bn0idNudT3PoMg4R7v7oXh8fowJabBZpb2POfuG4A/E3ymE4A33P2dOmKUfURJqPUrILg+\ns9HdK81sNDBpP8fwR+DTZnaOmeWEF+rPa2CdAoKTU4mZdQbuBHJTytR58veg59QjwI/N7ODINvbU\nt1N33xmuN5XgesN9kcX5BNeMNgKY2ecJrnNlajZwtpl9LjwOZwKfiyxP1EQ2unuiM8TUlG2sBwam\nXlAn+VjcB1xuZseH+/kCwXXBe+son4mGPo8HgOFmNs3MOplZGzM7zszy3f0jYA4w08yGAJhZHzM7\nKlz3NeAYMxsdxvt1oM57qyLyCa51bbGgE8gUkpPvHILP6/+ZWZewJnuYmR0SKXMPQW32WyQfH9lP\nlIRav78RnPzmR5riHmhgnXT3rTT26YfV5d39PYKT9U0EF/JvJLjPqKye9b8NjA3Lvw28B2xoZExX\nEDRLvQf8m6C34NYMYv8tQeeB9939tcj83xBc83kP+Jjg5PVQAzFFj8NzwFUEJ7stBIn4d5HlbxN0\nhPjfsAfYTIKaVtRDBDWadWEPtKPT7OdBgms0vyHoUPIz4FvuXlxPnA2p9/Nw9zXAZwg6pawgOD53\nEjTZAVwELAD+bmalBNethofrPktQq3kS+A9wDEFHiah08V5HUCPbQNDc2AF4NRLTDoLOFAcDiwiO\nxe+I3FPlwb11awiuez6S2aGQ5mThBTmR/crM7gF6uftX4o5FDmxm9jiwwt2/F3csB6KMa0JmNtTM\nXgz70b9iZiPrKHdteOH2TQvuJRjdfOFKtjKzL4dNIjlm9kWCnksPxh2XHNjC81MR8Mu4YzlQNaY5\n7l5gpruPILjQPTu1QNjGOwk41t1HAXcBv26OQCXrjSPoHVVC0PRyvbv/Od6Q5EBmZguAZ4Gp7p7a\na1H2k4ya48JeRR8Q9AiqCuetI+iV8mGk3JHAUwQ9aD42syuA8e5+9j6JXkREslqmY8f1A9al9Ldf\nRXDfQXUScvd3zOy/geVmtomgC+e45gpWRERal2YdwNTMBhLceT3Y3f8T1oT+BJyYUs4I7gPY1pz7\nFxGRWOQBa70JPd0yTUKrgd5mlhOpDfWn9s1vXwXecff/hNMPAL8ys7buHr0/ow/BjXsiItI69CXo\n7t4oGSUhd99gZm8Q3PA228zOJhjP6sOUoh8CF5tZp7CP/peBxSkJCMIa0OrVq8nPz77HoEybNo2f\n/vSncYfRaNkaN2Rv7NkaN2Rv7NkaNzRP7BUVFWzevJmuXbuSm5t6T3PzKy0tpV+/ftDElq3GNMdN\nBH5nwVMjSwiGMMHMbgbWuPssd59jZscCr5lZGcF4VBfUtcH8/PysTEK5ubmKez/L1tizNW7I3tiz\nNW5oOPaKigo2bdpEt27dqhNMYl6XLl2YNu1XPPnkKkpL+5Gfv5pTT+3PnXdeTU5Oyx2XIOMk5O5L\nCO6YTp3/w5TpHxDcrS0iIs2gqqqKKVN+npRgvvSlvoDx1FOrKS3tR1nZP9i27ftUVl4DwPr1sGrV\n88DPmTHjmjjDr5eerNoERUVFcYfQJNkaN2Rv7NkaN2Rv7NkaN9Qd+5QpP+eee0ZTVlaTYJYtuwr4\nMpWVUwmGyFtNMNpUjbKyk3jyyf/lttsq9kvTXFPEMmyPmeUDJSUlJVlbbRbJVmVlZVRU6Knv2aKi\nooIxY37AsmU/i84laHBKzFtPMPRd7ZGHDj74F/zznxdw8MEH73Usubm5dOjQIWleaWkpBQUFEDzA\nsLSx21QSEjmAlJWVMWjQINavXx93KJKFevXqxfLly5MS0d4mITXHiRxAKioqWL9+fdb2TJX4JHrB\nVVRU1KoN7Q0lIZEDULb2TJXWp+X22xMRkVZPSUhERGKjJCQiIrFREhKRA9KECRO46qqrMi5///33\nM2rUqH0Y0YFJXbRFDiCJ7rT1/e2lGxqmqfZ2W3l5eQSD7kN5eTmVlZV07NgRd8fMePrppzn++OP3\nKkbJTF3fHXXRFpFmkW5omKaOPdZc29q2rWZMzOnTp/Piiy8yd+7cetepqGi5owNIbWqOExGgZmiY\nJUt+xfr1U1my5Ffcc89opkz5eazbasj111/P5z73Oa6//nr69OnD2LFjq+cPGzaMvLw8Bg4cyPe/\n//2kkSLOP/98Lrvssurp3r17c9ttt1FUVEReXh4jRozgySefrF5+7733MmzYsKT1L7roIiZOnEi3\nbt3o06cPP/nJT5Ji+8tf/sJhhx1Gfn4+p5xyCj/4wQ8YOXJkne/lrbfe4uSTT6ZHjx506dKFsWPH\n8q9//SupzMKFCznttNPo1asXXbt25YQTTuDjjz8GYOfOnVx77bUMGzaM/Px8hg0bxqOPPtqEo7r/\nKAmJCBUVFTz55CrKyk5Kmh+MPbayUcP8NOe2MjVv3jy6devGypUrq0/ahx56KPPmzWPbtm088cQT\nPPbYY9x+++31bue+++5jxowZlJaWcvHFF3PhhRdSVlZWvTzRNJjwP//zP3zxi19k48aNPPLII9x0\n00288sorALz33nt87Wtf48Ybb2Tr1q1Mnz6de++9t9Y2osyM6dOns3btWtavX8/48eM544wzKCkp\nAWDt2rWMGzeOMWPGsGzZMjZu3MgvfvEL2rdvD8A3v/lN5s+fzzPPPENpaSkvvPBCvUmvJVASEhE2\nbdpEaWm/tMu2bevH5s2bY9lWpvr378+UKVNo165d9d38F154Ib179wbg8MMPZ+LEiTz77LP1bmfS\npEkcdthhmBmXX345W7du5YMPPqiz/Mknn8wZZ5yBmTFu3DgOPfRQFixYAMAf//hHxo4dy7nnnktO\nTg7HH388559/fr37P+qooxg/fjzt2rWjffv2/PjHP6asrIzXXnsNgAceeIBhw4Zxww030KlTJ3Jy\nchg9ejQFBQWsWbOGxx57jFmzZjFkyBAA+vTpw5FHHpnZQYyJkpCI0K1bN/LzV6ddlpf3EV27do1l\nW5kaOHBgrXm//vWvGTVqFN26daNLly78+Mc/rm62qksiaQF06tQJSL4ulapPnz5J0506daouv2bN\nGgYMGNBgnFHLly/n3HPPZcCAARQWFtKtWzfKy8ur416+fDkjRoyoc10zY/jw4fXuo6VREhIRcnNz\nOfXU/nTo8HzS/A4dnufUU/s36kJ/c24rU6mdHZ5//nmmTp3KXXfdxccff8yWLVuYPn06+7M38CGH\nHMLKlSuT5q1YsaLedSZMmED79u1544032Lp1K5s2baJ9+/bVcQ8cOJAlS5akXTeR4Opa3lIpCYkI\nAHfeeTWTJr3K8OHfoXfv2xk+/EomTXqVO++8OtZtNUVJSQlt27ale/futGnThldffZV77rlnv+w7\n4YILLuCll17iT3/6E1VVVcyfP59HHnmk3nVKSkrIy8ujoKCA7du3M2XKlKRraBMmTGDJkiX89Kc/\nZceOHVRWVrJgwQJKS0vp27cvX/nKV5g4cSLLli0DgmtIb7/99j59n3tLSUhEgKA2MWPGNSxcOIM3\n3riIhQuDJ3I25dHQzbmtpvjyl7/MRRddxNixY6ub4iZMmFDvOuk6DNTXiaChbYwcOZJHH32Um266\nicLCQn70ox9xySWX1DsC9d133838+fMpLCzkqKOOYuTIkfTo0aN6+SGHHMILL7zAvHnzGDhwID17\n9mTKlCmUl5cDMHv2bI477jg+//nPk5+fz0knndTia0a6WVXkAJLJzaqy70yaNIn169czZ86cuENp\ntH11s6pqQiIi+8jf/vY3tmzZQlVVFU8//TQPPvgg3/jGN+IOq0XRiAkirURDQ+RUVVVx/fW/jCGy\nA9e8efO45JJLKCsro2/fvtx666189atfjTusFkXNcSJZrr4hcvbs2VOdmK677pfcffehlJefpuY4\naTSNHSciSRI1n9tum82sWWMoK7sGgPXrYeXKuTz//Pns2NGT0tJ+5OWtYMOGEsrLL485apFkSkIi\nLUy6ZrXovLZt21bXfEpKerN581vs3n1d0jbKy1/nzTcvBb4AwPr164D79vM7EWmYkpDIPpLpYwwS\n5bp06cK0ab9Kalb70pf6AsZTT62untep08e8997EsOazDvhD6haBVcA1kXndgPpHCxCJg5KQSDPL\n9BpNtEZTWtqPsrJ/sG3b96msrGlWW7bsKuDLVFZODedVAJOA8eHeugGpQ+RsAlLHbssF+gP1PwZB\nZH/LOAmZ2VBgNtAd2Apc7O7vpZT5AnA74IABPYF17n5ss0Us0gLU12SWyTWa2jWaCoJk8vnoXqis\n9JR5m4Do2GGJ5PI8cFI4rxuwOE3UV1NQUEQ4ILNIi9CYmtC9wEx3/4OZfZUgIR0XLeDuzwLVw9Sa\n2d+A55ojUJGWIF0tJ9pklvk1mtQaTbraS7p56Wo+VwM/p127u+nW7Wjy89fQqdN23nvvH5SVja8u\n1aHDPL7+9XHcfff/Nfn9Z6vf/va3/OQnP2H58uUAfOtb36Jdu3bcfffdacsvXryYkSNH8tFHH9Ua\npLQxioqKGD9+PNddd13DhQ9QGSUhM+sBHEP4L5m7/9nMfm1mg939wzrW6QN8Fqh/rAyRFia1lhOd\nvu66X3LPPaOTajnJTWaZXqNJrdGkSy7p5iVqPs+SSGiQQ4cOo7nssj1cf/3FdO3aNdLU92e2betH\nXt5HnHpqf2688UruvvvGph2YGJx11lm4e9oRBq677jqeeOIJ3n333Yy2FR1S5ze/+U2jyjdk2bJl\nDBs2jBUrVtC/f//q+cXFxRlv40CVaU2oH0GzWlVk3iqCv4a0SQi4CHjS3TfuRXwi+1Rdvc6Cbs2r\n6Nx5A9u392Dbtv7V3ZwTCSjcQkqTWabXaFLLpWtWy6VNGwP+TmVlTZNc+/bHcOih97Jjx1+TEsyd\nd16bNDbbjBnXcNttFWzevJmuXbuSm5tLaWmjb+OI1aRJkzj11FNZu3ZtUo1k9+7dPPDAA/zwhz+M\nMboa7t7oceYk5O4NvoCjgfdS5r0CnFTPOh8ARXUsywe8pKTEReJQWVnpkyff4cOHf8d79brdhw//\njo8adY536DDXwcPXHQ7/F5le6/CjyHRi3u0p8+5w+EdkutzhkpQyiXLFkelKb9PmSi8sPN97977N\nhw//jn/ve7f59753uw8ffkX1vMmT7/DKykovLy/3devWeXl5ecbvu6SkxLPtb2/48OF+0003Jc17\n6KGHPC8vz7dt2+bu7o888ogfffTRXlhY6D179vQzzzzTV65cWV3+vvvu80GDBlVPf+Mb3/AJEyZU\nT3/wwQc+fvx4z8/P98MPP9xnzZrlOTk5vmbNGnd3f/vtt/3kk0/2Hj16eGFhoY8ZM8ZfeOEFdw++\nS506dfKcnBzv3Lmz5+Xl+ZVXXunu7ieccILffPPN1ftZtGiRFxUVebdu3bxfv34+adIkLy0trV5+\nwgkn+OTJk/28887z/Px879+/v8+aNave43PppZf6gAEDvHPnzj5kyBD/0Y9+lLR8586dft111/mw\nYcM8Ly/PhwwZ4g899FD18r/+9a8+ZswY79Kli/fo0cPPO++8tPup67uTmA/kewb5JPWVaRLqQdAZ\nIScybx0wuI7yJxHUlKyO5fmAX3HFFT558mSfPHmyP/PMM/UeaJHGKi8v97Vr11afpKPTkyff4R06\n1Jcoyh2+k5I0Mp1X6XCHt2v3Ne/V69Y6Epx7+/bP+ahR59RKMLt27aqVXJqScNLJxiQ0Y8YM79u3\nr1dWVlbPGzdunE+aNKl6+umnn/aFCxe6u/uGDRv8tNNO8xNPPLF6eX1JaM+ePT5ixAi/7LLLvKys\nzFevXu3HHntsUhJ65513/LnnnvPy8nIvLy/3G2+80QsLC33z5s3u7r506VLPycnxVatWJcUeTUIl\nJSXeu3dvv/7666s/zxNOOMHPPvvspPJdu3b1efPmubv7o48+6m3btvUVK1bUeXzuu+8+37hxo7u7\nz58/37t06eL3339/9fJzzz3XP/3pT/sHH3zg7u5r1671t956q/q4dezY0efMmeO7d+/2srIynzt3\nbtr9RL87zzzzTPW5+4orrtj3SciDxDEXuCj8/WxgQT1lfw/8qJ7lqglJs4ommNRazrBhV/ioUef4\nsGFXhNOTvLDwggZqNOlqOInay7NJ89q0udLbtEme16HDP/yqq35anThqYmqeGk1TZZKEqqrcS0qa\n91VV1fSYt2zZUn2idHd/99133cyqk046CxYs8DZt2viuXbvcvf4k9Pzzz3u7du18x44d1cvnzJmT\nlIRqH6Mq79y5c/U/z4kkFK19uScnod///vfeu3dvr4ocjFdffdXNzDdt2lRd/vLLL0/aRpcuXfyx\nxx6r5wglu+KKK6prM+vWrXMz83feeSdt2VNOOcUnT56c0Xb3VU2oMb3jJgK/M7NpQAlwMYCZ3Qys\ncfdZ4XQ+8BXgiEZsW6RJ0vVWS+76DOvX/4zgK/nZcDrd6AGp12jSXdsBuJrCwlPp0eNxtm/vT17e\nR2HvuDd56qm/NPoaDQRPIu3Vq1ezHI/msG0bBEOBNZ+SEmjqUHWFhYWce+65zJw5kzPPPJOZM2cy\nduxYDj/88Ooyc+fO5ZZbbmHRokXs2rWr+gS3YcMG+vVLvR6XbM2aNXTv3p2OHTtWzxs0aFBSmZUr\nV3Lttdfy0ksvUVJSgpmxa9euBh8XHvXRRx8xcODApGtHQ4cOBWDVqlXVjz2v75Hhqdydn/zkJzz6\n6KOsXbsWgLKyMj796U8DwZNc63vk9/LlyznzzDMzfg/7QsZJyN2XAGPTzP9hynQpkLf3ocmBqKEh\naxrurZba9TnT0QNSe50lpp8jkbwg6OY8YcLJ3Hbbd2slk9tvr51gUrW0hJNOXh7Nfi9R3l6eEb79\n7W8zZswY3nnnHR588MGkrtXl5eWcfvrp3HLLLTzxxBMcdNBBvPbaa3zqU59KtLzUq2/fvmzcuJGd\nO3dWJ6JEV+6ESy+9lIMPPpjXX3+d7t274+4UFBRUbz8nJ6fBffXr14+VK1fiXtOJYenSpZhZUo+6\nxnjwwQe56667ePbZZzniiCMwM77zne+waNEiIPmR30ccUbteUN/jwvcXjZgg+019w9g0dP9N5r3V\nUrs+1zd6wN+J3gia2uusc+fVdO78Gtu3P1Zd60mMfJCTk1MrmWRDgsmEWdNrLfvKsccey9FHH81Z\nZ51F+/btOfvss6uXlZeXU15eTpcuXTjooIP46KOPmD59esbbHjt2LIMGDeLqq69mxowZbNy4kVtv\nvTWpTElJCUOGDKGwsJDt27dz8803s2vXrurlPXv2pE2bNixevLjOhHL66adzzTXXcMMNNzB9+nS2\nbNnC1VdfzVlnnVVdC2qs0tJS2rVrR/fu3QF47rnnePjhhznqqKMA6NWrF1/72tf49re/zf3338+w\nYcNYt24d//nPf/jkJz/Jd7/7Xc455xw+85nP8MUvfpHKykrmz5/PySef3KR4mqQpbXh7+0LXhFq9\n+q7RpF4PWbt2rV911a0pHQXSXWvJpLdaakeBdB0Hgs4DhYWn+LBh327wGs3+vGazr2Vjx4SEBx54\nwHNycnzatGm1lt1///0+YMAAz8vL81GjRlWXTVyjaah33JIlS/ykk07yvLw8P+yww/w3v/lN0jWh\nl19+2UeNGuWdOnXyQYMG+T333OP9+vXz2bNnV2/j1ltv9V69enmXLl38u9/9rru7n3jiiUm94959\n913/whe+4N26dfO+ffv6pEmTkj6L1PLuXms/UTt37vQLLrjACwoKvFu3bn7BBRf4VVdd5ePHj68u\ns2PHDp86daoPHjzYO3fu7EOHDvWHH364evnjjz/uxx57rBcWFnqPHj3861//etp97atrQnqekDSr\n+q/R1Ny937793LDW0TMyysCfIluqIBgJ4Fd1TNc1D+BnwFHU3Mz5M4K7DKLNas8zadKraZvVWjM9\n3luaSs8Tklhkco0mOq/2uGmp12gCycPYpBtlILUZbf81q4nI/qMkJEka+1iBhsdNS71GA7U7C2Qy\nZE3mvdWCBPMwe/bsSarlVFQ03HFARPYvJSEBajejZfZYgUzGTctkGJtMhqxpfG+11I4CraXjgEhr\noiTUCmTy8LSGykyZ8vNIV+dMHyuQybhp6QbcTPeogdojQafef6NmNZHWR0koy9Q34Ga6h6ela1ZL\nLZOXl8eTT66KdHXO9LECmdRo0g+4WftRA7VHgq7r/hs1q4m0HkpCWSKzkQFqPzwtXbNaapmOHd9n\n/fq+kb1l+liBzJ5tk+4aTV2PGkgdZQDUrCbSmikJtXB71+ssXbNa7QesBeUmRUpk+liBdPPqrtGk\nJo+6hrERkQOHklBMGur6HG1qa3qvs3RNaOmGsckFPkHydZuradPme+TlzeKgg46qc4y0TMdNq4tq\nNfHItucKSfz21XdGSWg/y2R4mtpNbU3tdZbpA9YArqZjx/Po1esxdu0aVJ1MfvrTO9i6dWuDY6Rl\nMm6axC+R9Bsa1FMknV69ejX737eS0D7QuJs503V9Tm1qa2qvs3TNaul6pgHk0Lfvwbz++q1s3749\nKZlkMkaaajTZoUOHDixfvpyKioq4Q5EslJubS4cOHZp1m0pCe6mh3moN38yZrutzalPb3vQ6q92s\nVrtMMIzNqaf2p3PnznTu3Lk5D5G0MB06dGj2E4lIUykJRdT3yIBMRn1O11ut4Zs50zWPNX+vs2iz\nWt09065upiMpIpIZDWBK7YSS+siAdPfW1DSrnRRuJdGE9tvIlptzEM6gtnLZZS9x/fUTGrz2ksm9\nNLrfRkT2lgYwTaMxNRpIHS0g3ZM4k++tyby3WlOGpwmka2pr7l5nuo4jInFr0UmoscPRpF6TyaRG\nU3u0gHRdmDMZ9bnpN3PW3c259iCcIiKtSYtMQumutwTXNa5ky5YtdXYCqH1Npv4aTfrRAjK5tybT\n3mqNu5kzXTdn1VZEpDVrkUmodvNYFcuWfY8HHriUDh2OqiPhpHZrzqRGkygXHS2gqaM+p29Ca8zN\nnEo4InL8TXi8AAAWjklEQVSgiTUJpbtXoaKiIqV5DODnVFaexdatJwF1DVmTek2mqaMFpHtkQGaj\nPtfXhKabOUVEaos1CY0Z8wNOP31o0jWaiooKSkujySNd4kjXCSCTh6BlNlpAukcGNGbUZ93MKSKS\nmViT0LJlP+Puu19LukaTl7eCsrKSSKlM76NJvSaTaY0G6hotINqFuTGjPouISGZivU8ISoB7Sb0f\npk2bxA2en6cx99G0bz83vCbTI/IQtA1s3949UqNJXEtKHi1g0qRXmTHjGhqie2tERGpk+X1C6TsP\nVFb+N4WFp9Kjx+Ns396fXbuWsm1bpvfR1L4mk1mNJrPRAtSsJiLSfGKuCS0GHgem1irTu/ftvPzy\n+eTm5lJYWBg+HXRlrcTR1PtoVKMREdl7e1sTyjgJmdlQYDbQHdgKXOzu76Up1w+4CxgO7AHucfe7\nUsqESWgDcC3JQ90Ehg+/koULf56UIJQ4RERalr1NQo25on4vMNPdRwB3ECSkdOYAv3P3T7j74cCf\n6trgkCE/YNSo7XTo8I+k+YkRnVMTTaIpTAlIRKR1yKgmZGY9gA+Aru5eFc5bBxzv7h9Gyn0WuNnd\nT2hge/lAyYYNG+jatWt4jaZ2U5t6nYmItGz7pTnOzI4GHnL3kZF5rwDXuvvzkXlXEvSH3kVwI89y\nYIq7L0/ZXq1RtNXUJiKSfVpa77i2BMMYfMrd3zezywma40Y3tKJ6nYmIHHgyTUKrgd5mlpNojiO4\nE3RVSrlVwJvu/n44/QfgLjNr4+6VqRudNm1ada2nqKiIoqKiRr8BERHZv4qLiykuLgbSD7/WGI3p\nHTcXmO3us83sbGCqux+XUqYj8A4wzt3XmtnXgBvd/YiUci3qoXYiItI0+7M5biLwOzObRjDUwcUA\nZnYzsMbdZ7n7TjObCDxpZoTlzmtsUCIicmDQ471FRKTJ9ud9QiIiIs1KSUhERGKjJCQiIrFREhIR\nkdgoCYmISGyUhEREJDZKQiIiEhslIRERiY2SkIiIxEZJSEREYqMkJCIisVESEhGR2CgJiYhIbJSE\nREQkNkpCIiISGyUhERGJjZKQiIjERklIRERioyQkIiKxURISEZHYKAmJiEhslIRERCQ2SkIiIhIb\nJSEREYmNkpCIiMRGSUhERGKjJCQiIrHJOAmZ2VAze9HMFpvZK2Y2Mk2ZAWa2x8zeMLM3w5+Dmjdk\nERFpLdo2ouy9wEx3/4OZfRWYDRyXplypux/dLNGJiEirllFNyMx6AMcADwG4+5+BfmY2OF3x5gtP\nRERas0yb4/oB69y9KjJvFdA/TdmOZvaqmb1mZtPNTElJRETSau6OCWuBQ9x9NPA54ETg6mbeh4iI\ntBKZXhNaDfQ2s5xIbag/QW2omrvvBjaGv281s/uB84E702102rRp5ObmAlBUVERRUVHj34GIiOxX\nxcXFFBcXA1BRUbFX2zJ3z6yg2VxgtrvPNrOzganuflxKmR7AFnffY2btgT8Ai9z9ppRy+UBJSUkJ\n+fn5e/UGREQkPqWlpRQUFAAUuHtpY9dvTHPcROByM1sMTAUuBjCzm83ssrDMCcCbZvYm8BqwDril\nsUGJiMiBIeOaULPuVDUhEZFWYX/WhERERJqVkpCIiMRGSUhERGKjJCQiIrFREhIRkdgoCYmISGyU\nhEREJDZKQiIiEhslIRERiY2SkIiIxEZJSEREYqMkJCIisVESEhGR2CgJiYhIbJSEREQkNkpCIiIS\nGyUhERGJjZKQiIjERklIRERioyQkIiKxURISEZHYKAmJiEhslIRERCQ2SkIiIhIbJSEREYmNkpCI\niMQm4yRkZkPN7EUzW2xmr5jZyAbK/87Mqswsf+/DFBGR1qgxNaF7gZnuPgK4A5hdV0Ez+wpQAfje\nhSciIq1ZRknIzHoAxwAPAbj7n4F+ZjY4TdmDgeuByYA1X6giItLaZFoT6gesc/eqyLxVQP80ZWcB\n17j7jr0NTkREWrdm7ZhgZpcCK939hebcroiItE5tMyy3GuhtZjmR2lB/gtpQ1HjgRDM7jZqmuHfM\n7Ax3fzt1o9OmTSM3NxeAoqIiioqKGv0GRERk/youLqa4uBiAioqKvdqWuWfWd8DM5gKz3X22mZ0N\nTHX34xpYpwoocPdtKfPzgZKSkhLy89V5TkQkW5WWllJQUADBub60ses3pjluInC5mS0GpgIXA5jZ\nzWZ2WR3rOOqcICIidci0OQ53XwKMTTP/h/Ws06aJcYmIyAFAIyaIiEhslIRERCQ2SkIiIhIbJSER\nEYmNkpCIiMRGSUhERGKjJCQiIrFREhIRkdgoCYmISGyUhEREJDZKQiIiEhslIRERiY2SkIiIxEZJ\nSEREYqMkJCIisVESEhGR2CgJiYhIbJSEREQkNkpCIiISGyUhERGJjZKQiIjERklIRERioyQkIiKx\nURISEZHYKAmJiEhslIRERCQ2SkIiIhKbjJOQmQ01sxfNbLGZvWJmI9OUGWhmr5nZG2a20MweNbOC\n5g1ZRERai8bUhO4FZrr7COAOYHaaMmuA4939aHc/AlgH3LTXUYqISKuUURIysx7AMcBDAO7+Z6Cf\nmQ2OlnP33e5eHq7TBugEeLNGLCIirUamNaF+wDp3r4rMWwX0Ty1oZu3M7E3gY2Ao8MO9jlJERFql\nZu+YENaGRgEHA+8DE5t7HyIi0jq0zbDcaqC3meVEakP9CWpDabn7HjP7HTAL+Fm6MtOmTSM3NxeA\noqIiioqKMo1bRERiUlxcTHFxMQAVFRV7tS1zz+ySjZnNBWa7+2wzOxuY6u7HpZTpD2xw911mZgTJ\n52B3vzClXD5QUlJSQn5+/l69ARERiU9paSkFBQUABe5e2tj1G9McNxG43MwWA1OBiwHM7GYzuyws\ncyTwspm9BbwNdAeuamxQIiJyYMi4JtSsO1VNSESkVdifNSEREZFmpSQkIiKxURISEZHYKAmJiEhs\nlIRERCQ2SkIiIhIbJSEREYmNkpCIiMRGSUhERGKjJCQiIrFREhIRkdgoCYmISGyUhEREJDZKQiIi\nEhslIRERiY2SkIiIxEZJSEREYqMkJCIisVESEhGR2CgJiYhIbJSEREQkNkpCIiISGyUhERGJjZKQ\niIjERklIRERioyQkIiKxyTgJmdlQM3vRzBab2StmNjJNmcPN7AUzW2Rm75jZfWbWvnlDFhGR1qIx\nNaF7gZnuPgK4A5idpkwZcIW7HwocBXQGrt3rKEVEpFXKKAmZWQ/gGOAhAHf/M9DPzAZHy7n7Und/\nN/zdgVeBgc0ZsIiItB6Z1oT6AevcvSoybxXQv64VzKwT8F/A400PT0REWrN90jHBzNoBjwDPuPtf\n98U+REQk+7XNsNxqoLeZ5URqQ/0JakNJzKwt8Ciwxt0n17fRadOmkZubC0BRURFFRUUZBy4iIvEo\nLi6muLgYgIqKir3algWXbjIoaDYXmO3us83sbGCqux+XUqYN8Cdgs7t/q55t5QMlJSUl5OfnNz16\nERGJVWlpKQUFBQAF7l7a2PUb0xw3EbjczBYDU4GLAczsZjO7LCxzLnAmcKyZvWlmb5jZrxoblIiI\nHBgyrgk1605VExIRaRX2Z01IRESkWSkJiYhIbJSEREQkNkpCIiISGyUhERGJjZKQiIjERklIRERi\noyQkIiKxURISEZHYKAmJiEhslIRERCQ2mT7KQURE9oGqKti0CdauhbKy5GVt2sDBB0OvXtCuXfKy\nLVtg6VL48MOgXNeuwatLl+Bn585gtv/eR1MpCYnExB0++ABeew2GDYNRo6DtXvxFlpVBSQns3Am7\ndiW/yspqfq+ogPz8mpNW166wezcsXAjvvhv8/Pe/gxPfZz4TvD71KejYce/e79atMGcOPPwwrFsH\nhx8ORxwR/Dz0UCgtDU6qS5fCsmWwcSMMHx6UOeIIGDkSOnQItlVVVfN+tm6FzZtrXhs3wpo1wUl9\nzZrg1bFjzf4S29q9u2adLVuCbSVO4ImfqTEtXw7btiUf23btYPBgGDoUhgwJfgbjedbYswfWr6+J\nJxrb2rVBLAUFtY/xnj3B+4EgGR1yCOTkBLFs3gzduwf7huT30q5d7YTWUmkUbZF6lJfDggXw9tvB\niS9q9+7gJJQ46VdUBCe6z3wmOMml/he6axcsWgQvvwzz5gWvLVvgyCODk9zu3XD88cH6w4fDihXB\nySZxAty5Ew46qObVvn1wQkycfHbtCvaTk1NTpkOH4MQWXS83Nzi5Rk/cbdsGMUeTwpo18MILwWvD\nBvjkJ4OTc3RbVVXBe4ieALt3rzkZDx0a7H/OHHjqKTjsMDj//CDp/vvfQcJbuBAWL4a8vJp1hg4N\nksDixTXJcdu2oEziWCe0a5ecULt1gz59ghN24rV9e82+3n03SP5t2yav1759kNAS72fr1iD2RDxD\nhsCgQUGyiB6DioqgNhJNVtu2JX/2OTlBUo/GFI2xT5+6k/zu3UECSySuPXtq4klNdhB8JqWlUFjY\nqK96k+3tKNpKQpL13n8f7r47OLn07p38hz1oUPAHm+4P0j34LzNxAk+8tm+H118PTr4vvxyse+yx\nwck7qm3b5JNRmzbw1lswfz506gTjxsGIETUn0mXLghrIscfW1DBGjw4SRVVVUGbevGC/H35YE3vi\nhNOpU3Kc5eW1azQFBcFJuTHNMO7B/tu0qXv5hx/Cq68GxyZa08rJSd5/YWFwTBMn5KVLg+P7pS8F\nyWfEiPT7qKyse/+JGD76KEgQ0WOeeDW22amyMoi9vvUyKSNKQtJE7vDGG8EJ5fDDg/9wm9v27cn/\nyS9bFuznuOOCk2/fvk3/A9+zB/72N7jrLvjXv+Ccc+Ckk5KbPNasCZpPNm0K/jseMiRITBs21DSD\nVFQESSB6QuvYMaidjBtXUytpTJwVFTVJbOlS+MQnapqC+vTRSU1aFyWhFmbjxpqmgyFDgvbaRDt2\nfXbsgL/+NWiuiXIP/uONtu27w8CByW3QvXsH/7XVZ/fu4D/tOXPgL38Jqux5ecEJ+ZBDgpPkiBFB\n8oieyEtLg+SRaCfv2jXYV13XHBKvRDt3Is7Bg4Pjs2BBcIx69gyaeMxqr5v6atMmOVEkmqYmTYJL\nL4UePep+31u31iTDdeuC/SaaQvr0CWoYItI0SkL70H/+E5yAhw5N/99r4qT+9NPBNYOFC4N1+vcP\nmkmWLQtOoH37BtuItrkfcUTwH/czzwQXav/61yCxjB1bO5m0b1+7HX7FiprmjtWrg0QXvTg6aFCQ\nCKMXQRcvDvZ5xhnwla8ENYfc3KCJ4913g9f77wexR9uuCwpqX/ytqkrfLBJ9FRYGCSvdsduxA958\nMzhm0WsYqa9ELaWysnZSGjOm/iYcEdn3sjoJHX54CT165Ff3Rhk+PGj+GDWqdnfE0tKgF9HixbV7\n/wwcCKecElzsTHfCW7MmOGmnO5lHt1NSEpwYFywI2r9Xrw7i6NIlaJoZNw5OOCFILo8/Dk88EWzn\ntNPgmGOCxHLYYTUXC92D5qFEsli0qObC6Jo1QTy9egVt5eefH6zflKaasrKg2Snx3/7SpcH7zc9P\nvvg5ZEhwbBuqMYmIZCqrk9Ajj5RQVpbP5s1Bu/277wY1i4qKoJfQccfBqlVBUli8GPr1C07ynTsn\n/6e8aFGwXu/eUFQEJ54IK1cG6y1YENROevcOtptIOJWVQSy5uTXb6tw5uBaQuGZxzDHB9hcsqLlg\nPH9+EMeZZwav0aObdlLfvDlIUOl6UYmIZIusTkLpmuOqqoKumy+8ENRGBgyoSQoHH1z3NnfuDNYp\nLoYXXwyapo47LniNGhUkmKjdu4Pk0djmHHclDRGRhFaXhEREJHvsbRLS1QEREYmNkpCIiMRGSUhE\nRGKjJCQiIrFREhIRkdhknITMbKiZvWhmi83sFTMbmaZMJzN7xsw2mNnm5g1VRERam8bUhO4FZrr7\nCOAOYHaaMruB24DPNkNsLVZxcXHcITRJtsYN2Rt7tsYN2Rt7tsYN2R17U2WUhMysB3AM8BCAu/8Z\n6Gdmg6Pl3L3C3Z8HSpo5zhYlW78o2Ro3ZG/s2Ro3ZG/s2Ro3ZHfsTZVpTagfsM7do4/1WgX0b/6Q\nRETkQBHr471LSxt9c22LUFFRkZWxZ2vckL2xZ2vckL2xZ2vckJ2x7228GQ3bEzbHfQB0TdSGzGwd\ncLy7f5im/ADgTXfvWsf2DgE+2pvARUSkRenr7msau1JGNSF332BmbwAXArPN7GxgdboEFLLwVZe1\nQF9gWz1lREQkO+QRnNcbLeMBTM1sOPA7oBtBx4OL3X2Rmd0MrHH3WWG5t4HuwMFhUP9w94uaEpyI\niLRusYyiLSIiAjGNmJDJja8tgZn90syWm1mVmR0Zmd/DzJ42syVm9o6ZnRhnnKnMrL2ZzTGz983s\nTTMrNrMh4bIWHTtAGO9bYewvmNknw/ktPnYAM5sQfmdOD6dbfNxmtsLM3guP+Rtm9rVwfouO3cxy\nzexXYXxvm9nvw/ktPe6ukWP9RngurDCzwiyI/Utm9noY/ztm9s1wftPidvf9/gKeAy4Mf/8qsCCO\nODKI8wSgD/AhcGRk/m+BG8PfjwVWA23ijjcSX3vglMj0FQTNogD3t+TYw7jyI7+fCbyVRbEPAF4M\nX6dnw/cljOtD4Ig081t07MAvgF9GpntmQ9xp3sfVwF/C31v09xzYBBwW/j4A2AV0amrccbyBHsBW\nICcybx0wOO6DW0/My1OS0LbElz2cfhk4Oe4464n/GODDLI39YuD1bIidoDPO34FRwD8iSahFxx3G\nlPQdj8xvsbEDHQmuT3fOprjreC+LgC9nQ+zABuCE8Pcjw2TTrqlxx3GfUH03vtbV267FMLOuQFt3\n/zgyeyUt+8bd7wKPZ1PsZjYbGA848KUsif37wD/d/U0LnwGfJXEn/CGMewFwHcGxb8mxDwE2Az8w\ns88BO4Gbgbdo2XEnMbOxQCHwZJZ8X84D5pjZDoK4zyLoHdekuDWKditnZtMI/linxR1LY7j7Re7e\nH7iBYKxCqL/bf6zM7DCCpuVb4o6liU5096OAowmaWxJjQ7bYY05wi8kA4F13H03wz9Yj4fyWHHeq\nS4Dfp/xj3iKZWRuCv8kz3X0g8DngQfbimMeRhFYDvc0suu/+BLWhFs/dNwN7zKxnZPZAWmD8ZjaF\n4JrKKe5elk2xJ7j7H4CTwsndLTj2EwlOiB+Y2XJgDDALOIcsOObu/lH4sxL4b4Kk1NK/L6uASuCP\nAO7+FrACOIKW/V2pZmadCL4j90NWnF8+CfR29xcB3P01goEHjqSJx3y/JyF33wAkbnwlgxtfW6L/\nASYBmNlogs4LL8QaUQoz+z5Btfnz7h69KbhFx25mBWbWOzJ9JrAp/ONssbG7+0x3P8TdB7v7IIL2\n8G+5+0xacNwAZtbRzAoisy4A3gx//xMtNHZ330TQyekUADMbRHDiW0QLP+YR5xF0vFkSmdeSY09U\nIj4BQU9nYDDwPk2NO6YLW8OB+cBigvbnw+K+2FZHnDPDg15B0HliSTi/J1AMLAEWAuPijjUl7kOA\nKoKhlt4gOKG8lCWx9wdeAd4maNt/lvCCeUuPPeV9zKWmY0KLjhsYFH5P3gqP+xygfxbFPhd4J/ye\nn5kNcUfi/xfwzZR5LTp24NzI8X4bOHdv4tbNqiIiEht1TBARkdgoCYmISGyUhEREJDZKQiIiEhsl\nIRERiY2SkIiIxEZJSEREYqMkJCIisVESEhGR2Px/NcBxQd//sy8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12e8fdda0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ0AAAEmCAYAAAC50k0UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAMTQAADE0B0s6tTgAAIABJREFUeJzt3Xl4VOXd//H3F0LYzMJOEAIIuFdxq4qi1orYWoVq3erG\nUhW0tSrIoyBWa1W0ira2j+D6Q624tLVY0SdaUalLVQR3RGQLYFL2JEhCINy/P+6TZDJkmSSTk5nw\neV3XXMycOcs3J+F85r7PfeaYcw4REZEwtGruAkREZM+h0BERkdAodEREJDQKHRERCY1CR0REQqPQ\nERGR0Ch0REQkNAodaTAze9zM/liP+ceY2aKmrKmpmNl6Mzu3Cdf/nplNDp6nmlmRmR1dy/y/N7NX\nGrnNQWZWaGY9G7OeOrZxhZktbar1S/JJae4CpGmZWRFQfgVwW6A1sA2wYPqPnHPvNGTdzrnR9Zz/\nMeCxhmxrT+KcKwXSYpk11nWa2Z3AEc65UyO2sxRIr3+F9aYr0KWCQqeFc85VHLzM7DbgOOfcybUt\nY2apwYFPRCSu1L0mmNmdZvav4N9vgXcjpi8NunpWmtl0M0uNWG62mT0U8TrPzG4ws5xgmSVmdnrE\n+1W6WoLlZ5nZDDPbaGbfmtlNUbWNMLMvgm6g/zOz281scS0/y2Azmxd0h202s3fN7PiI9/czs11m\ndqGZfRas999mNjBinjQzeyKoaZWZjatj/91rZi9FTetjZjvMbP/g9ePBPiwys2/MbEot62sb1Dgk\nYtovguUKzOwZolpCZvbrYD8VmNkaM3vMzDKD98YA1wEnBdsvNLMjIvZFr4j1XB6sZ4uZfWxm59dn\n39XFzNoH+2ulmW0IfleHR7x/eLDOLcH+f9/M+gbvnWpmHwXvbTCz+WbWIdZtS4JwzumxhzyA24B5\n1Uy/EygFJgJtgHbB9IuBrOD5wcBKYGrEcrOBhyJe5wHfAAfhu+9uBDZFrO8K4Ouo5bcBI4L5TwB2\nAkcH7x8Q1HUe/gPSccAG4MtafsZDgR8EP0db4HZgI5ARvL8fsAuYC3QJ5pkDvBKxjieA94DuQAfg\nqaCOc2vYZnmdWRHTfgO8HfH6F0Dn4PlxwBbg4oj33wMmB8/bBjUOCV6fDJQAw4L9cFbw+uWI5X8G\n9A+e9wM+Ah6N+h2/GlX3fkAZ0Ct4fWGwr44Lfh+nBb+fYbHuu2r2TfTv/GFgAZAd/I7K/0a6Be8v\nAK4PnrcCBkfst/XAecHzFGAIkNrc/6/0qN9DLR0pl+ucu8c5t8M5VwLgnHvSOZcXPP8cmAGcWttK\ngAedc184f2SYCWQCg2qZf55zbo7z5gNfAt8P3vs58K5z7lnn3C7nzz3Nrm3jzrlPnHNvBD/HdmAq\n0A44MmrWm51zG4N5ZpVv08zaABcAU5xz65xz24BrqKUr2jm3GPgQGB2sw4BR+ANs+TyPOOc2Bc/f\nAZ6h7n1ZbhTwvHPutWA//B34V1QNf3XOrQierwR+X4/1l7sMmOGceyf4ffwfPoDHR81X7b6ri5ml\nAJcCNzjncoPf0Z3Af/H7HHx49zWzfsHP+nH5fgveG2RmPZ1zO51z7zp1AycdhY6UWxk9wcx+aWaL\ngm6OzfgDePc61pMX8fy74N/aTop/G/X6u4j59wZW1VVnJDPrb2bPBt1iW/Cf3NtStW5XTZ3l28zC\n/7+o2I5zbkPEz1KTRwhCB3+w7wQ8F9RkZvaboNtqs5ltAi6h7n1ZrjewImpalddmdp75EXDrgt/V\nw/VYf7k+wLKoad/gWyXlatt3dcnCD2RZHjV9WcQ2fg60B94KuuDuMbN2wXs/xreiPzazr6K7YiU5\nKHSk3K7IF2Z2EnA3cBXQ3TnXCd89ZyHWtBboGzWtXx3LPA5sBw53zmXiu4G2E3vdefh9UbEdM+sG\ndKxjuWeBbmZ2MjAWeMY5Vxy8NwrfzXQ+vquoM74FEWtNa9j9546sbx/gaWAasHfwu7osav1Vfr81\nWA0MiJo2AMiNsc665OG782rchnNupXNurHOuL7478UxgQvDeJ865C5xzPfHhdI2ZXRin2iQkCh2p\nSQb+/MoG51yZmR3F7t0sTe1p4FgzO9fMWgUn1s+vY5kMoAgoMLO9gHuA1Kh5ajzYO+d24Lu+bjOz\nHhHr2FnbRoNuuGeAScAZ+JZPuXR819AGADMbhj9PFatZwM/M7JRgP4wETol4v7ylscE5Vz54YVLU\nOvKBfhYxECQQuS8eAa4ws+OC7ZyKP683s4b568U5txMftrebWbb565FuAHri9x1mNtrMsoJFioAd\nwA4z62BmF5tZl+C9QvzvpNbfiyQehY7U5J/4g927EV1rj9exTHXXY9T3Go2K+YNzJecBt+BPvN+M\nv86npJblr8SfYN4CfAIsxp+Ark9NV+G7fBYDX+BH822JofZH8Z/Ov3LOLYiY/jB+oMBiYB2+a+0v\nddQUuR9eB67GH/w344P3/0W8/wl+4MJfzawAf+7tiaj1/QXfmsgzs00RI8Yit/MUMCWodxP+vNBl\nzrmcWuqsr18BbwP/xrd8hgM/dM6tC94fBnxk/vqyBfhzV/cF710AfGlmhcBr+POHzzayHgmZ+fO9\nIsnBzB4EejrnftrctYhI/cXU0jGzP5jZimCM/iER0x8zfy3GomBsffQIIZFGMbMzzKxT0N3zI+Ai\n/BBmEUlCsXavPY8fu78yavrfgQOcc4fhT2I+H7/SRAB/7c7XQAEwHbjROfe35i1JRBqqXt1rZrYC\nGOGc+7Sa97rgh7+2d87FMlJGRET2MPEcSHAN/gppBY6IiFQrLl/4aWYX4b+G44Ra5jGgF34YpIiI\nJLc04FtXz9FojQ4dMzsPP5z2ZOdc9NDUSL3wF7mJiEjL0Bt/EXfMGhU65m9qdRt+nH1dGy4CWL16\nNenpYdzCI34mT57MHXfc0dxlNEiy1p6sdUPy1p6sdUPy1p6sdRcWFtKnTx9oQM9VTKFjZjOA04Ee\nQI6ZFTnn9sUPXc0D5gTdZw4fQJtrWld6enrShU5qamrS1VwuWWtP1roheWtP1roheWtP1robI6bQ\ncc5Vez8R51z0V2qIiIjUSF+DE4Phw4c3dwkNlqy1J2vdkLy1J2vdkLy1J2vdjRHa1+CYWTpQUFBQ\nsMc1J0WaQ0lJCaWlut2MNFxqairt2rXbbXphYSEZGRngb45YWJ91xmXItIgklpKSEvr3709+fn5z\nlyJJrGfPnqxYsaLa4GkohY5IC1RaWkp+fn5SjhaVxFA+Qq20tFShIyKxScbRotKyaSCBiIiERqEj\nIiKhUeiIiEhoFDoi0mKNHj2aq6++Oub5H3vsMQ477LAmrAhmzpzJoEGDmnQbiUwDCUT2UKWlpWzc\nuJEuXbqQmtq4LxdpzLrS0tLw36IF27dvp6ysjA4dOuCcw8x45ZVXOO644xpU1+OPP16v+ceMGcOY\nMWMatK36KP9590QKHZE9zK5du5g48V7mzs2lsLAP6emrOf30bO65ZwKtWtWv8yMe6yoqqvzOyKlT\np/LOO+8wb968WpcpLS1tdFBK81D3msgeZuLEe3nwwaP4+usHyM+fxNdfP8CDDx7FxIn3Nuu6anPj\njTdyyimncOONN9KrVy+GDBlSMX3QoEGkpaXRr18/rrvuuirfwnDBBRdw+eWXV7zOyspi2rRpDB8+\nnLS0NPbbbz/mzp1b8X5019cFF1zApZdeyrhx4+jSpQu9evXid7/7XZXa5syZw0EHHUR6ejqnnXYa\nU6ZM4YADDoj5ZysuLmbChAn069ePrl27cvLJJ7Nw4cKK9xcuXMjQoUPJzMykS5cuHH300axatQqA\nV199lSOOOILMzEy6du3KCSecwLZt22LednNQ6IjsQUpLS5k7N5eSkpOqTC8pOYm5c1fV62tz4rmu\nWMyfP58uXbqwatUq3n77bQAOPPBA5s+fT1FRES+99BJ///vfueuuu2pdzyOPPML06dMpLCxk1KhR\nXHzxxZSUlFS8H9319fzzz/OjH/2IDRs28Mwzz3DLLbfw/vvvA7B48WLOOeccbr75ZrZs2cLUqVOZ\nOXNmvbrPrr76at566y3mz59PXl4ew4YN45RTTmH9en97sssvv5wzzzyTLVu2sH79embOnElaWhoA\nF154IZMmTWLLli3k5+czbdo0UlISuwNLoSOyB9m4cSOFhX2qfa+oqA+bNm1qlnXFIjs7m4kTJ9Km\nTZuKK+QvvvhisrKyADj44IMZN24cr776aq3rGT9+PAcddBBmxhVXXMGWLVtYunRpjfOffPLJjBgx\nAjPjhBNO4MADD+SDDz4A4Omnn2bIkCGcd955tGrViuOOO44LLrgg5p9p586dzJo1i2nTppGdnU2b\nNm248cYb6dGjB7Nnzwb895+tWrWKlStX0qpVKwYPHkznzp0r3lu6dCn5+fmkpKQwZMiQhO92VOiI\n7EG6dOlCevrqat9LS1tTcTALe12x6Nev327T/vSnP3HYYYfRpUsXOnXqxG233ca6detqXU95SAF0\n7NgRqHpeKVqvXr2qvO7YsWPF/GvXrqVv37511lmTvLw8ysrK2GeffapMHzBgALm5uYAPtuLiYk48\n8UT69evHxIkTK1pmL7/8Ml988QWDBw9m//33363rLxEpdET2IKmpqZx+ejbt2r1ZZXq7dm9y+unZ\n9fqUHM91xSJ6YMKbb77JpEmT+POf/8y6devYvHkzU6dOJaxvzgfYe++9K86vlFu5cmXMy2dlZdG6\ndWuWLVtWZfqyZcvIzs4GfIg9+uijrFq1itdee40XX3yRe+/158wOPfRQZs+eTX5+Pk8//TT3338/\nf/nLXxr3QzUxhY7IHuaeeyYwfvyH7LvvL8nKuot99/0V48d/yD33TGjWddVXQUEBKSkpdO3aldat\nW/Phhx/y4IMPNvl2I/385z/nvffe47nnnmPXrl28++67PPPMMzEvn5KSwiWXXMKUKVPIzc2ltLSU\nadOmkZ+fz/nnnw/4Yd95eXmAH17epk0b2rRpw7Zt23jyySfZuHEj4L9nLyUlJeHP6SR2dSISd61a\ntWL69OuZNq2UTZs20blz5wa3SuK5rvo644wzuPTSSxkyZAhlZWUMHTqU0aNH8/TTT9e4THUn+Ot7\nzUzk/AcccADPPvsskydP5he/+AVDhgxhzJgxvPbaazGv74EHHmDKlCkMHTqUrVu3csghh/D666/T\nvXt3AF577TWmTJlCUVERGRkZnH322Vx77bXs2LGD2bNnM3HiRIqLi+nSpQvjx4/nvPPOq9fPEzbd\nxE2kBSq/yZb+v4Vv/Pjx5Ofn88ILLzR3KY1S299QY27ipu41EZFG+Oc//8nmzZvZtWsXr7zyCk89\n9RQXXXRRc5eVsNS9JiLSCPPnz2fMmDGUlJTQu3dv7rzzTs4+++zmLithqXtNpAVS95o0lrrXREQk\n6Sl0REQkNAodEREJjUJHRERCo9ARSUDbtkE9vk1FJGkodEQSxHffwfPPw7nnQrduMGAAnHUWfPZZ\n9fOXlcEXX8C6dRDi140lrEcffZT+/ftXvL7sssu48sora5x/yZIltGrVim+//bZR2x0+fDjTpk1r\n1DpqU1ZWRqtWrZg/f36TbSNMuk5HpBZlZZCX51sd+fnQpg20bw/t2vl/09J8QHTqBOXfR7l9Oyxa\nBO+9B//5DyxdCp07Q/fu/tGtm19PYWHlY906ePNNyM6Gc86Bm26Crl3hzjvh+9+HkSPhllsgKwte\nfRVeeglefhm2boXiYr/+/ff3jwEDYK+9mnGn1dNZZ52Fc67aK/hvuOEGXnrpJT7//POY1hX5FTUP\nP/xwveavy7Jlyxg0aBArV66s+DJOgJycnJjXIQodaWbbtsHnn/uD7qZN/rFxY+Xz8tebN0P//jBs\nGJxyChx2WOVBvjbOQXXHFed8KyEnxz+WL4fUVB8GqamQkuJrys31wbP33v6Av2MHlJRUPrZs8f+2\nbg1dukBmpg+o9HQ45hj/OPdcKCjw61u/HhYvhp07ISPDzzdgABxxBNx9Nxx0UNV6H3gArr8ebr8d\nDj0Udu2CQYPgJz+Bv/0Njj3Wb//rr+Grr/zjyy99QCaL8ePHc/rpp/Ptt99WuY3Ajh07ePzxx/nN\nb37TjNVVcs7V+3vaZHcKHYmJc7B6tT9gfvmlf7RtCwcfXPnIzPQH6LVrYcUKfyDfuBE6dvSfvPfa\nyz9ftQo+/BA++MAHTqdO/oDepYv/xN65s5+WnV05LTPTb/u11+COO3w4HH+8ny81tfKxfTt8+23l\nIy/Pt0p69oQePfy/qanw1ls+ME46yR/ADz7YB0FpaeWjWzfo1w/69PHL1OS773yYrF/vQ3LgQNhn\nn+rDriGys2HmTN/62bnTh2+kvfaCww/3j3KFhT7UksGwYcPo378/Dz/8cJWAef755ykuLuaSSy4B\n4Nlnn+Xuu+9m+fLlpKamMmTIEP7whz9UaXVEuvjii2nTpg2PPfYYAN988w2XX345H330EdnZ2Vx9\n9dVV5v/000+59tpr+eyzz9ixYwf7778/d911FyeccAK7du1i8ODBABU3gBs1ahR//OMfGTp0KMOG\nDePmm28G/N1Er732WhYsWECHDh34yU9+wl133VVxt8+hQ4dy1FFHkZeXx8svv0xmZiY33XQTl112\nWcz77MUXX+TWW29l2bJlZGVlccUVV3DNNdcA/o6uv/rVr5gzZw7FxcV07dqVSZMmVdyw7oorrmDe\nvHmUlpbSq1cvpk2bxogRI2LedmMpdOJk505/oF2ypPKxeXPlQbT8QFpWVrVbpajIf9ovLvaPbdv8\np+zyrpgePfzBr3Vr/ym7/IBoBn37+gPQPvv49QOsWeMD4YsvfA1lZb4bqPzRqVPlASq6C2bNGnj7\nbd8tlJ/v6y9/5Of7+gYOhAMO8I8dO+CFF+C22/wBvnt3P29ZGfTu7evq1s3/TFu3Vj6ysnyX0dSp\ncNRR/qAeywH65JPhqqv8vl6wAN591x/wy/fJ9u3+Zxw61LdMevXy29q+3ddf/igqgrFjfWi1bdv4\n333Hjv5Rj3t3NUif6m/S2SKMGzeO6dOnM3Xq1Ir75sycOZOLLrqIvYI/1IyMDGbNmsXBBx/Mhg0b\nGD16NBdddFFM5zrKysr4yU9+woknnsgrr7zC+vXr+elPf1plHjNjypQpHH/88QDcfvvtjBgxguXL\nl9OpUyc++eQT9t13X7788kv61PDLKCws5Ic//CGjRo3ixRdfZNOmTZxzzjmMGTOG559/vmK+WbNm\n8Y9//IPZs2fz3HPPceGFF3LqqafudkO46vznP//hnHPO4ZlnnmHEiBEsXLiQM844g9TUVK688koe\ne+wxFi1axJIlS8jIyOC///0v+UHTd9q0aWzfvp3c3Fzat29Pbm4uxcXFdW4znmIKHTP7A3Am0BcY\n7Jz7NJjeDXgCGACUAFc55/7dRLUmhHXr/KfOzz+v/HS7fj1s2OCDYeBA2G8//9h3X/9petMmH0ib\nNvlASU/3j7Q0/ym2Qwf/aN/e/7tjh9/OunXwzTf+4Opc5af5Nm38gf1vf/OtifXr/fqc8wfhgQPh\nwAN9/36bNpVhtnGjP9fw+9/7g+8BB/iD/o4dPmzWrIHBg+G44+DII31AlT+6d/fdOjUdpDdt8ucu\nunatu2XQWCkplV1XsTrwwKarJ1k55wM4ntLSGtbCGz16NDfddBMvvvgiI0eO5IsvvuDf//43f/7z\nnyvmOe200yqed+3alZtvvpljjz2WkpKSittX1+Ttt99m+fLlLFy4kLZt29K7d2+mTJlS5TvSvve9\n71VZ5pZbbmH69Ol88MEHDB8+vGJ6bV8dNmfOHMAHlpnRs2dP7rvvPr7//e9X3PoB4JxzzmHo0KEA\nnHvuuYwbN46FCxfGFDqPPPIII0eOrAjNI488kgkTJjBjxgyuvPJKUlNTKSoq4rPPPuOYY46hR48e\n9OjRA/A33tu4cSOLFy/msMMOq7GV2JRibek8D9wFvB01fRrwnnPuR2Z2JPCCmfVzzpXVt5DiYt/t\nsmKF7xNfscI/Vq/2nyLLT8B27+4/UZ52mj/ARdu0CZ54wo8C6tjRtways/2/WVmVB/fyk8EdOvj5\n2rf3oVGTpUvh3nth1iw48UR/XqFrV19T166+rj59/AExbEVFPnzAh10d//8A3zIp7+JKSYFRo+Do\no/1BoyE6d/bLS/IoKop/F1xBgf8AVF+ZmZmcd955zJgxg5EjRzJjxgyGDBnCwQcfXDHPvHnzuP32\n2/nyyy8pLi7GOYdzjvXr19fY8ii3du1aunbtSocOHSqm9Y/qp1y1ahX/8z//w3vvvUdBQQFmRnFx\ncZ23v460Zs0a+vXrV+Xcz8CBAwHIzc2tCJ3aboFdl9WrV3PEEUdUmTZw4MCK21tfeumlbN68mQkT\nJrBkyRKOP/54fve73zF48GBuvPFGwIf82rVrOeWUU7jzzjt32xdNKaZDpHPubQDb/SzaufhWDs65\nBWa2FjgRmFf7+uCVV+Dpp2HZssqRQe3a+UDp398/vv99OPtsH0jlJ2G//hrmzPEHySFDYMQI//j2\nW3joIfjrX/2n91Gj/LZWrfLL/OtflV1EkY9duyrratvWdzl17+77/rOy/GPFCpg7158Qfv99OOSQ\nWPZaeNLS/Enm+ujVq3LfyZ4pLc2HRLzX2VBXXnklxxxzDJ9++ilPPfUU//u//1vx3vbt2znzzDO5\n/fbbeemll2jfvj0LFizg6KOPjun21L1792bDhg1s27atInhWrFhRZZ6xY8fSo0cPPvroI7p27Ypz\njoyMjIr1t2rVqs5t9enTh1WrVlUZdPDNN99gZnFrVfTp02e321t/8803Fetv3bo1EyZMYMKECXz3\n3XdMmTKFkSNHsnLlStq3b89vf/tbfvvb37J582auuOIKxowZwxtvvBGX2mLR4M/lZtYZSHHORX4M\nWAXUumdnz4Y//ckHyNixcPrplUHTo0fsTfO1a+Gf//QBdOON/o/90kth4cLYu1Kc811L333nu5+2\nbfOf/tat8wGVl+f/3X9/uP/+lt2nLnses4a1SprKkUceyeGHH85ZZ51F27Zt+dnPflbx3vbt29m+\nfTudOnWiffv2rFmzhqlTp8a87iFDhtC/f38mTJjA9OnT2bBhA3feeWeVeQoKChgwYACZmZls3bqV\nW2+9tcr5ju7du9O6dWuWLFlSY4CceeaZXH/99dx0001MnTq1osVx1llnVbRyGmvs2LH84Ac/YM6c\nOZxxxhksWrSI++67j8mTJwPw+uuv06lTJw455BBSU1Pp2LEjbdq0AfwAhEGDBrHffvvRrl072rdv\nH/rtrUPvDLr7brjhBrj44ti6gWqy994wbpx/bN3qz13U96SwWeV5kk6dGl6LiMTHlVdeydixY7nh\nhhsqDpQA6enpPPTQQ9x888388pe/ZODAgVx99dW8+uqrMa03JSWFl156icsvv5wePXqQnZ3NNddc\nw4IFCyrm+dOf/sT48ePJzMyke/fuTJo0iaysrIr3O3bsyG233cYll1zC9u3bueSSS7j//vurdKWl\np6fzr3/9i+uuu47evXvTvn17zjjjjCoXjzbkltmR7x977LE8++yz3HrrrVx66aX06NGDiRMnctVV\nVwHw3//+l1//+tfk5ubSpk0bBg8eXDGIYdmyZUycOJG8vDzatWvHscceG9P1TPFUr/vpmNkKYETE\nQIIiYEB5a8fM3gdudM7t1r1Wfj+d8eOvol07f5Z5+PDhVU7QiUh86H460ljRf0M5OTkVF8KWlpaW\nD/Ko9/10Ghs6jwGrnHO3mtlRwN+BagcS6CZuIuFR6EhjNdVN3GIdMj0DOB3oAeSYWZFzbl/gBuBJ\nM/sa2A5c2JCRayIismeIdfTauBqmrwPUPyYiIjHRt0yLiEhoFDoiIhIahY6IiIRGoSMiIqHRt0yL\ntGCFhfUazSpSoan+dhQ6Ii1QamoqPXv2rPOLMEVq07NnT1Lj/JXxCh2RFqhdu3asWLGC0tLS5i5F\nklhqamqdt42oL4WOSAvVrl27uB8wRBpLAwlERCQ0Ch0REQmNQkdEREKj0BERkdAodEREJDQKHRER\nCY1CR0REQqPQERGR0Ch0REQkNAodEREJjUJHRERCo9AREZHQKHRERCQ0Ch0REQmNQkdEREKj0BER\nkdAodEREJDQKHRERCY1CR0REQqPQERGR0Ch0REQkNAodEREJjUJHRERCo9AREZHQxCV0zOzHZvaR\nmS0ys0/N7JJ4rFdERFqWlDit50ngBOfcF2bWF/jKzP7mnPsuTusXEZEWIF7da7uATsHzDGADsD1O\n6xYRkRYiXi2d84EXzOw7IBM4yzm3M07rFhGRFqLRLR0zaw3cBIx0zvUDTgGeMrPOjV23iIi0LPFo\n6QwGspxz7wA45xaY2RrgMOD16JknT55MamoqAMOHD2f48OFxKEFERJpSTk4OOTk5AJSWljZ4Peac\na1QhZtYdWAoc7Zz7yswGAv8BBjvn1kTMlw4UFBQUkJ6e3qhtiohI8yksLCQjIwMgwzlXWJ9lG93S\ncc6tM7PLgefMrAzfZXdVZOCIiIhAnAYSOOeeBZ6Nx7pERKTl0jcSiIhIaBQ6IiISGoWOiIiERqEj\nIiKhUeiIiEhoFDoiIhIahY6IiIRGoSMiIqFR6IiISGgUOiIiEhqFjoiIhEahIyIioVHoiIhIaBQ6\nIiISGoWOiIiERqEjIiKhUeiIiEhoFDoiIhIahY6IiIRGoSMiIqFR6IiISGgUOiIiEhqFjoiIhEah\nIyIioVHoiIhIaBQ6IiISGoWOiIiERqEjIiKhUeiIiEhoFDoiIhIahY6IiIRGoSMiIqFR6IiISGji\nEjpmlmpmD5jZ12b2iZk9EY/1iohIy5ISp/XcBexyzu0LYGbd47ReERFpQRodOmbWARgD7F0+zTm3\nrrHrFRGRlice3WsDgE3AFDP70MzeMrOT47BeERFpYeIROilAX+Bz59xRwK+BZ82sWxzWLSIiLUg8\nzunkAmXA0wDOuY/NbAXwPWBe9MyTJ08mNTUVgOHDhzN8+PA4lCAiIk0pJyeHnJwcAEpLSxu8HnPO\nNboYM/uTT2kcAAAOUElEQVQ/4A/OuVfMrD/wPnCocy4vYp50oKCgoID09PRGb1NERJpHYWEhGRkZ\nABnOucL6LBuv0WvjgUfN7C58q+fyyMARERGBOIWOc24FoMEDIiJSK30jgYiIhEahIyIioVHoiIhI\naBQ6IiISGoWOiIiERqEjIiKhUeiIiEhoFDoiIhIahY6IiIRGoSMiIqFR6IiISGgUOiIiEhqFjoiI\nhEahIyIioVHoiIhIaBQ6IiISGoWOiIiERqEjIiKhUeiIiEhoFDoiIhIahY6IiIRGoSMiIqFR6IiI\nSGgUOiIiEhqFjoiIhEahIyIioVHoiIhIaBQ6IiISGoWOiIiERqEjIiKhUeiIiEhoFDoiIhKauIaO\nmY02s11mdmY81ysiIi1D3ELHzPoCvwDei9c6RUSkZYlL6JiZAY8AvwRK47FOERFpeeLV0rkO+Ldz\nblGc1iciIi1QSmNXYGYHAWcDQxtfjoiItGSNDh182PQFlgbdbD2Bh8wsyzk3M3rmyZMnk5qaCsDw\n4cMZPnx4HEoQEZGmlJOTQ05ODgClpQ0/i2LOuXjV5Fdo9gZwn3Puxajp6UBBQUEB6enpcd2miIiE\np7CwkIyMDIAM51xhfZZtiut04ptiIiLSYsSje60K59zJ8V6niIi0DPpGAhERCY1CR0REQqPQERGR\n0Ch0REQkNAodEREJjUJHRERCo9AREZHQKHRERCQ0Ch0REQmNQkdEREKj0BERkdAodEREJDQKHRER\nCY1CR0REQqPQERGR0Ch0REQkNAodEREJjUJHRERCo9AREZHQKHRERCQ0Ch0REQmNQkdEREKj0BER\nkdAodEREJDQKHRERCY1CR0REQqPQERGR0Ch0REQkNAodEREJjUJHRERCo9AREZHQKHRERCQ0jQ4d\nM2trZi+Y2VdmtsjMcsxsQDyKExGRliVeLZ2Zzrn9nXOHAS8Cj8RpvSIi0oI0OnScc9udc/8XMek/\nQN/GrldERFqepjin82vgHzW9WVpaWvFvXl5exeuapomISMuREs+VmdlkYABweU3zHH30ZDIyCti6\ntRtFRdmkp6/mxz/uDRgvv7yawsI+pKev5vTTs7nnngns3LmTjRs30qVLF1JTUyktLa3yGqh2moiI\nJJ64hY6ZTQRGAj90zpXUNN/y5Z8DvYBU4FDy8yexbNnVwBmUlU0CID8fVq2ax5tvXsB333WnsLAP\naWm57LXXeoWViEgzyMnJIScnB6BRvVHmnGt0MWZ2HfBzfOAU1DBPOlDgG0EzI94pBSYAD0Qt8Xvg\nUODUiNeHAz+smKN16/KwGlYxrW3beRx44EyFlYhIEyksLCQjIwMgwzlXWJ9lG93SMbO9gXuAZcAb\nZmZAiXPu2OqX2Dvq9UagT9S0UiAXuL6G135aWZkDhlVZcvv2j1i0aCzlYZWf/3vgp5SHVX4+cW9Z\n3XHHr9i8eXOdwaSwEpE9XaNDxzm3lnoNSFgb9boLsDpqWnQQVRdMiRBWu1i27Boef3ws7dodWmMw\nNTasRERairgOJIhNb+B1KrvJUmnd2oDXIrrJugBLIpapLpgSIazupazsLLZsOQmouRXV0LBSl5+I\ntDShh84++6wmI+Mhtm79O1u3ZpOWtiY44C7i5ZfnUFTUh7S0NXTsuJXFi9+gpOQH+EEH2SRWWMUa\nTA0PKw2mEJGWJvTQef/9O+jatSulpaVs2rSJzp07Vxzs7rqrclpKSgoTJ97L3Ll/o6ioD3vttZq9\n9lqQQGEVayuqoWHVPOenYg0rEZGGiMvotZg2FIxeKygoID09PeblosOpurCKnFYZVqsiwmo9W7d2\njQor4+WXcyPCah2LF48LwgrqHi1X3ai7WKblAU8CkyLmqW5a9HKxbq+89qYZ+VdTMJX/HtSyEmn5\nmnX0WlNLTU2lZ8+eNb6ubtr06dczbVrdYdXYllVx8TcUFUW2mKprRUVPS9TzU7G1rKJbUdW1mtQN\nKCI1SfiWTtjq07LKzMxk8uQHKlpVNbWioqcVF79JUdF1VVoZu7c8SoHxwKMRr2Np6US3muLZsoLd\nW1HV1d7011RpmLpI82lMS0ehEwd1dfk1NKzq3+UHTRtWsayrqbsBd9G69TWkpW1s0mHqCiuRmil0\nklTinJ+C2MIqllZTU5+z+j1wFHBSLT+LwkqkKbXoczotWSKdn4pt5F/0KL/yaatreV3TtMQcpt5c\nFwDHMghDpCVQ6CSBRBpMUTWYYPeBEsk8TL35w0ojBqWlU+i0YE0RVtHBVF2rqemuqYq1FZXMYaUR\ng9KyKXRkN3WFVXXBBFVbTck5TL25w6q65aobyh5bWCXKF9sqvCSSQkcaJJZWU1N0A2Zm/joY+Ten\nxpZW8oZVLNdd+Wl1h1Xzf7FtU3QVVkehllwUOtLs6htWsba0IqclR1jFMuCiummJ+sW28esqrC6s\nKlvKuWqRJRENmZY9SlNdU9WwC4BjubA2eih7+bSGDDdvyPD2hn7dU3wvLo6+kDg9fXU1lwY0bFh8\nWC2yaMkccrpOR6SJhRVW1V2LtfvBtaHfAtGQC4djve6qKS8uLv/5IsOqujBu6DVcsYVcvMKqU6dO\nwd9Gbp0DPBKVrtMRaWKxnJ+KntaQbsCWM2Iwnl2F1XUDbgT2q2Oe8AdvxNJ9WFLyRtAKvr7W5Vrq\nRckKHZEm1JCwqu518o0YjOfFxdUFUSIO3oglrEqDuusa4NG4i5LvuWcCrVrV44bOIVLoiCSJZBsx\nGL+Li6sLq/L5XsUfqBO1RRYdVrEu19DBG5Cb+yZwL9OnVw3ERKHQEdkDhTFiMJ5dhbuHFbRte0Rw\n3uXFBG6RRU+LZbnGXJQMJSUnMXfuX5k2rTQhu9oUOiISk+bsKqwurHw30mx27tyZwC2y6JApX+5N\nKgc8RC/X0K7CSkVFfdi0adNuv5tEoNFrIpI0qusGjGWeWKbV9g3v8R1pWD60ewPt2x9azXINHaZe\nad99f8Vnn93bZC0dDZkWEWkCTRFW5a20O+74FVu2bKl2uYZf5wXt2r3J+PEfNuk5HYWOiEgCibW1\nVdNyDb3Oq7LbsWlHryl0RERaoIZ2FTY1hY6IiISmMaGTmFcPiYhIi6TQERGR0Ch0REQkNAodEREJ\njUJHRERCo9AREZHQKHRERCQ0cQkdMxtoZu+Y2RIze9/MDojHekVEpGWJV0tnJjDDObcfcDcwK07r\nTQg5OTnNXUKDJWvtyVo3JG/tyVo3JG/tyVp3YzQ6dMysG3AE8BcA59zfgD5mtk9j150okvkPI1lr\nT9a6IXlrT9a6IXlrT9a6GyMeLZ0+QJ5zblfEtFz8TSNEREQqhH4Tt8LCen1NT0IoLS1NyroheWtP\n1roheWtP1roheWtP1robU3Ojv/Az6F5bCnQub+2YWR5wnHNuecR8ewNrGrUxERFJJL2dc2vrs0Cj\nWzrOufVmthC4GJhlZj8DVkcGTuBboDdQ1NhtiohIs0vDH9frJS63NjCzfYH/h7/ZdwEw2jn3RaNX\nLCIiLUpo99MREREJ5RsJkuXiUTP7g5mtMLNdZnZIxPRuZvaKmX1tZp+a2dDmrDOambU1sxfM7Csz\nW2RmOWY2IHgvoWsHCOr9OKj9LTMbHExP+NoBzGx08DdzZvA64es2s5VmtjjY5wvN7JxgekLXbmap\nZvZAUN8nZvZEMD3R6+4csa8XBsfCUjPLTILaf2xmHwX1f2pmlwTTG1a3c67JH8DrwMXB87OBD8LY\nbgPqPB7oBSwHDomY/ihwc/D8SGA10Lq5642ory1wWsTrq4A3guePJXLtQV3pEc9HAh8nUe19gXeC\nx5nJ8PcS1LUc+F410xO6duA+4A8Rr7snQ93V/BwTgDnB84T+Owc2AgcFz/sCxUDHhtYdRsHdgC1A\nq4hpecA+zb0za6l5RVToFJX/cQev/wOc3Nx11lL/EcDyJK19FPBRMtQOGPAacBjwRkToJHTdQU1V\n/sYjpids7UAH/DnjvZKp7hp+li+BM5KhdmA9cHzw/JAgXNo0tO4wrtOp7eLR6BFuCcfMOgMpzrl1\nEZNXkdgXv/4a+Ecy1W5ms4AfAA74cZLUfh3wb+fcIjMDku7v5cmg7g+AG/D7PpFrHwBsAqaY2SnA\nNuBW4GMSu+4qzGwIkAnMTZK/l/OBF8zsO3zdZ+FHrjWobn3LdAtjZpPx/zknN3ct9eGcu9Q5lw3c\nhP/+PvAtiYRkZgfhu4pvb+5aGmioc+5Q4HB890n59yUm7D7HX+LRF/jcOXcU/sPVM8H0RK472hjg\niagP4gnJzFrj/0+OdM71A04BnqIR+zyM0FkNZJlZ5Lay8a2dhOec2wTsNLPuEZP7kYD1m9lE/DmR\n05xzJclUeznn3JPAScHLHQlc+1D8AXCpma0AjgEeAs4lCfa5c25N8G8ZcD8+hBL97yUXKAOeBnDO\nfQysBL5HYv+tVDCzjvi/kccgKY4vg4Es59w7AM65BfiL/A+hgfu8yUPHObceKL94lFouHk1kzwPj\nAczsKPxgg7eataIoZnYdvhk8zDkXeQFuQtduZhlmlhXxeiSwMfjPmLC1O+dmOOf2ds7t45zrj+/P\nvsw5N4MErhvAzDqYWUbEpJ8Di4Lnz5GgtTvnNuIHJZ0GYGb98Qe6L0nwfR7hfPxAma8jpiVy7eWN\nhv3Bj0QG9gG+oqF1h3Qial/gXWAJvv/4oOY+OVZDnTOCnVyKH+zwdTC9O5ADfA18BpzQ3LVG1b03\nsAv/dUQL8QeQ95Kk9mzgfeATfN/8qwQnuBO99qifYx6VAwkSum6gf/B38nGw318AspOo9nnAp8Hf\n+chkqDui/reBS6KmJXTtwHkR+/sT4LzG1K2LQ0VEJDQaSCAiIqFR6IiISGgUOiIiEhqFjoiIhEah\nIyIioVHoiIhIaBQ6IiISGoWOiIiERqEjIiKh+f9ToD8/BDns9AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12f85b5f8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "acc = history.history['acc']\n",
    "val_acc = history.history['val_acc']\n",
    "loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "\n",
    "epochs = range(len(acc))\n",
    "\n",
    "plt.plot(epochs, acc, 'bo', label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.legend()\n",
    "\n",
    "plt.figure()\n",
    "\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_29 (Dense)             (None, 256)               25690368  \n",
      "_________________________________________________________________\n",
      "dropout_15 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_30 (Dense)             (None, 4250)              1092250   \n",
      "=================================================================\n",
      "Total params: 26,782,618\n",
      "Trainable params: 26,782,618\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/40\n",
      "8709/8709 [==============================] - 65s 7ms/step - loss: 8.3765 - acc: 0.0010 - val_loss: 8.3505 - val_acc: 0.0000e+00\n",
      "Epoch 2/40\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 8.3171 - acc: 0.0041 - val_loss: 8.2974 - val_acc: 0.0000e+00\n",
      "Epoch 3/40\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 8.1471 - acc: 0.0051 - val_loss: 8.1917 - val_acc: 0.0030\n",
      "Epoch 4/40\n",
      "8709/8709 [==============================] - 70s 8ms/step - loss: 7.8912 - acc: 0.0062 - val_loss: 8.0948 - val_acc: 0.0030\n",
      "Epoch 5/40\n",
      "8709/8709 [==============================] - 75s 9ms/step - loss: 7.6296 - acc: 0.0111 - val_loss: 8.0695 - val_acc: 0.0000e+00\n",
      "Epoch 6/40\n",
      "8709/8709 [==============================] - 66s 8ms/step - loss: 7.3843 - acc: 0.0155 - val_loss: 8.0602 - val_acc: 0.0000e+00\n",
      "Epoch 7/40\n",
      "8709/8709 [==============================] - 61s 7ms/step - loss: 7.1322 - acc: 0.0199 - val_loss: 8.0601 - val_acc: 0.0060\n",
      "Epoch 8/40\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 6.8626 - acc: 0.0295 - val_loss: 8.0260 - val_acc: 0.0121\n",
      "Epoch 9/40\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 6.6060 - acc: 0.0375 - val_loss: 8.1004 - val_acc: 0.0151\n",
      "Epoch 10/40\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 6.3889 - acc: 0.0467 - val_loss: 8.1489 - val_acc: 0.0302\n",
      "Epoch 11/40\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 6.1500 - acc: 0.0566 - val_loss: 8.1284 - val_acc: 0.0423\n",
      "Epoch 12/40\n",
      "8709/8709 [==============================] - 60s 7ms/step - loss: 5.9266 - acc: 0.0710 - val_loss: 8.2915 - val_acc: 0.0423\n",
      "Epoch 13/40\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 5.7233 - acc: 0.0884 - val_loss: 8.3658 - val_acc: 0.0453\n",
      "Epoch 14/40\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 5.4869 - acc: 0.1004 - val_loss: 8.3762 - val_acc: 0.0695\n",
      "Epoch 15/40\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 5.2746 - acc: 0.1175 - val_loss: 8.3341 - val_acc: 0.0634\n",
      "Epoch 16/40\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 5.0332 - acc: 0.1320 - val_loss: 8.4804 - val_acc: 0.0755\n",
      "Epoch 17/40\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 4.8517 - acc: 0.1490 - val_loss: 8.5735 - val_acc: 0.0725\n",
      "Epoch 18/40\n",
      "8709/8709 [==============================] - 57s 7ms/step - loss: 4.6720 - acc: 0.1649 - val_loss: 8.7779 - val_acc: 0.0937\n",
      "Epoch 19/40\n",
      "8709/8709 [==============================] - 61s 7ms/step - loss: 4.4600 - acc: 0.1852 - val_loss: 8.7044 - val_acc: 0.0846\n",
      "Epoch 20/40\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 4.3385 - acc: 0.1928 - val_loss: 8.7647 - val_acc: 0.0906\n",
      "Epoch 21/40\n",
      "8709/8709 [==============================] - 63s 7ms/step - loss: 4.1731 - acc: 0.2151 - val_loss: 8.9075 - val_acc: 0.1118\n",
      "Epoch 22/40\n",
      "8709/8709 [==============================] - 64s 7ms/step - loss: 4.0170 - acc: 0.2301 - val_loss: 9.0002 - val_acc: 0.0997\n",
      "Epoch 23/40\n",
      "8709/8709 [==============================] - 60s 7ms/step - loss: 3.8914 - acc: 0.2423 - val_loss: 8.9747 - val_acc: 0.1178\n",
      "Epoch 24/40\n",
      "8709/8709 [==============================] - 60s 7ms/step - loss: 3.7510 - acc: 0.2639 - val_loss: 9.0402 - val_acc: 0.1208\n",
      "Epoch 25/40\n",
      "8709/8709 [==============================] - 60s 7ms/step - loss: 3.6040 - acc: 0.2802 - val_loss: 9.1451 - val_acc: 0.1269\n",
      "Epoch 26/40\n",
      "8709/8709 [==============================] - 58s 7ms/step - loss: 3.4790 - acc: 0.2961 - val_loss: 9.0297 - val_acc: 0.1420\n",
      "Epoch 27/40\n",
      "8709/8709 [==============================] - 61s 7ms/step - loss: 3.3394 - acc: 0.3128 - val_loss: 9.1671 - val_acc: 0.1480\n",
      "Epoch 28/40\n",
      "8709/8709 [==============================] - 67s 8ms/step - loss: 3.2593 - acc: 0.3191 - val_loss: 9.2759 - val_acc: 0.1269\n",
      "Epoch 29/40\n",
      "8709/8709 [==============================] - 61s 7ms/step - loss: 3.1718 - acc: 0.3298 - val_loss: 9.3296 - val_acc: 0.1420\n",
      "Epoch 30/40\n",
      "8709/8709 [==============================] - 62s 7ms/step - loss: 3.0660 - acc: 0.3517 - val_loss: 9.2668 - val_acc: 0.1450\n",
      "Epoch 31/40\n",
      "8709/8709 [==============================] - 63s 7ms/step - loss: 2.9604 - acc: 0.3692 - val_loss: 9.3831 - val_acc: 0.1390\n",
      "Epoch 32/40\n",
      "8709/8709 [==============================] - 65s 7ms/step - loss: 2.8771 - acc: 0.3808 - val_loss: 9.4164 - val_acc: 0.1450\n",
      "Epoch 33/40\n",
      "8709/8709 [==============================] - 62s 7ms/step - loss: 2.7960 - acc: 0.3936 - val_loss: 9.3895 - val_acc: 0.1360\n",
      "Epoch 34/40\n",
      "8709/8709 [==============================] - 63s 7ms/step - loss: 2.7095 - acc: 0.4053 - val_loss: 9.4577 - val_acc: 0.1480\n",
      "Epoch 35/40\n",
      "8709/8709 [==============================] - 66s 8ms/step - loss: 2.6104 - acc: 0.4201 - val_loss: 9.4844 - val_acc: 0.1480\n",
      "Epoch 36/40\n",
      "8709/8709 [==============================] - 63s 7ms/step - loss: 2.5326 - acc: 0.4321 - val_loss: 9.5904 - val_acc: 0.1511\n",
      "Epoch 37/40\n",
      "8709/8709 [==============================] - 64s 7ms/step - loss: 2.4887 - acc: 0.4347 - val_loss: 9.5426 - val_acc: 0.1450\n",
      "Epoch 38/40\n",
      "8709/8709 [==============================] - 68s 8ms/step - loss: 2.4227 - acc: 0.4539 - val_loss: 9.6194 - val_acc: 0.1571\n",
      "Epoch 39/40\n",
      "8709/8709 [==============================] - 69s 8ms/step - loss: 2.3451 - acc: 0.4657 - val_loss: 9.6100 - val_acc: 0.1601\n",
      "Epoch 40/40\n",
      "8709/8709 [==============================] - 66s 8ms/step - loss: 2.3104 - acc: 0.4733 - val_loss: 9.6028 - val_acc: 0.1601\n"
     ]
    }
   ],
   "source": [
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import optimizers\n",
    "\n",
    "model = models.Sequential()\n",
    "model.add(layers.Dense(256, activation='relu', input_dim=ll_size*ll_size*2048)) # we can play around with \n",
    "model.add(layers.Dropout(0.5))\n",
    "model.add(layers.Dense(classes_count, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "model.compile(optimizer=optimizers.Adam(lr=1e-4),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=40,\n",
    "                    batch_size=64,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8709 samples, validate on 331 samples\n",
      "Epoch 1/40\n",
      "8709/8709 [==============================] - 59s 7ms/step - loss: 2.2505 - acc: 0.4813 - val_loss: 9.6514 - val_acc: 0.1541\n",
      "Epoch 2/40\n",
      "8709/8709 [==============================] - 34s 4ms/step - loss: 2.1613 - acc: 0.4994 - val_loss: 9.6922 - val_acc: 0.1601\n",
      "Epoch 3/40\n",
      "8709/8709 [==============================] - 34s 4ms/step - loss: 2.1599 - acc: 0.5019 - val_loss: 9.7004 - val_acc: 0.1571\n",
      "Epoch 4/40\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 2.0921 - acc: 0.5202 - val_loss: 9.7176 - val_acc: 0.1601\n",
      "Epoch 5/40\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 2.0922 - acc: 0.5121 - val_loss: 9.7427 - val_acc: 0.1631\n",
      "Epoch 6/40\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 2.1124 - acc: 0.5102 - val_loss: 9.7375 - val_acc: 0.1571\n",
      "Epoch 7/40\n",
      "8709/8709 [==============================] - 31s 4ms/step - loss: 2.0681 - acc: 0.5218 - val_loss: 9.7347 - val_acc: 0.1662\n",
      "Epoch 8/40\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 2.0327 - acc: 0.5297 - val_loss: 9.7440 - val_acc: 0.1631\n",
      "Epoch 9/40\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 2.0498 - acc: 0.5219 - val_loss: 9.7628 - val_acc: 0.1631\n",
      "Epoch 10/40\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 2.0169 - acc: 0.5322 - val_loss: 9.7790 - val_acc: 0.1662\n",
      "Epoch 11/40\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 2.0049 - acc: 0.5337 - val_loss: 9.8290 - val_acc: 0.1692\n",
      "Epoch 12/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.9694 - acc: 0.5338 - val_loss: 9.8233 - val_acc: 0.1692\n",
      "Epoch 13/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.9811 - acc: 0.5324 - val_loss: 9.7865 - val_acc: 0.1692\n",
      "Epoch 14/40\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 1.9533 - acc: 0.5381 - val_loss: 9.7933 - val_acc: 0.1692\n",
      "Epoch 15/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.9460 - acc: 0.5422 - val_loss: 9.8204 - val_acc: 0.1692\n",
      "Epoch 16/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.9282 - acc: 0.5467 - val_loss: 9.8341 - val_acc: 0.1662\n",
      "Epoch 17/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.8823 - acc: 0.5539 - val_loss: 9.8486 - val_acc: 0.1722\n",
      "Epoch 18/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.9376 - acc: 0.5466 - val_loss: 9.8284 - val_acc: 0.1662\n",
      "Epoch 19/40\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 1.8815 - acc: 0.5539 - val_loss: 9.8274 - val_acc: 0.1692\n",
      "Epoch 20/40\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 1.8845 - acc: 0.5543 - val_loss: 9.7773 - val_acc: 0.1692\n",
      "Epoch 21/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.8698 - acc: 0.5628 - val_loss: 9.8514 - val_acc: 0.1692\n",
      "Epoch 22/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.8596 - acc: 0.5601 - val_loss: 9.8474 - val_acc: 0.1722\n",
      "Epoch 23/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.8164 - acc: 0.5707 - val_loss: 9.8501 - val_acc: 0.1692\n",
      "Epoch 24/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.8030 - acc: 0.5752 - val_loss: 9.8827 - val_acc: 0.1662\n",
      "Epoch 25/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.7844 - acc: 0.5756 - val_loss: 9.8682 - val_acc: 0.1692\n",
      "Epoch 26/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.7929 - acc: 0.5711 - val_loss: 9.8851 - val_acc: 0.1722\n",
      "Epoch 27/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.7815 - acc: 0.5757 - val_loss: 9.9249 - val_acc: 0.1662\n",
      "Epoch 28/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.8043 - acc: 0.5647 - val_loss: 9.9303 - val_acc: 0.1692\n",
      "Epoch 29/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.7883 - acc: 0.5771 - val_loss: 9.9171 - val_acc: 0.1692\n",
      "Epoch 30/40\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 1.7568 - acc: 0.5787 - val_loss: 9.8710 - val_acc: 0.1692\n",
      "Epoch 31/40\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 1.7624 - acc: 0.5761 - val_loss: 9.9156 - val_acc: 0.1692\n",
      "Epoch 32/40\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 1.7947 - acc: 0.5706 - val_loss: 9.9275 - val_acc: 0.1662\n",
      "Epoch 33/40\n",
      "8709/8709 [==============================] - 27s 3ms/step - loss: 1.7307 - acc: 0.5794 - val_loss: 9.9388 - val_acc: 0.1662\n",
      "Epoch 34/40\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 1.7160 - acc: 0.5857 - val_loss: 9.9261 - val_acc: 0.1631\n",
      "Epoch 35/40\n",
      "8709/8709 [==============================] - 30s 3ms/step - loss: 1.7203 - acc: 0.5889 - val_loss: 9.9103 - val_acc: 0.1722\n",
      "Epoch 36/40\n",
      "8709/8709 [==============================] - 33s 4ms/step - loss: 1.7194 - acc: 0.5869 - val_loss: 9.9457 - val_acc: 0.1722\n",
      "Epoch 37/40\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 1.6851 - acc: 0.5913 - val_loss: 9.9054 - val_acc: 0.1722\n",
      "Epoch 38/40\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 1.7112 - acc: 0.5941 - val_loss: 9.9316 - val_acc: 0.1722\n",
      "Epoch 39/40\n",
      "8709/8709 [==============================] - 28s 3ms/step - loss: 1.6648 - acc: 0.5961 - val_loss: 9.9036 - val_acc: 0.1722\n",
      "Epoch 40/40\n",
      "8709/8709 [==============================] - 29s 3ms/step - loss: 1.6725 - acc: 0.6003 - val_loss: 9.9565 - val_acc: 0.1692\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=optimizers.Adam(lr=1e-4),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=40,\n",
    "                    batch_size=512,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_44 (Dense)             (None, 512)               104858112 \n",
      "_________________________________________________________________\n",
      "dropout_23 (Dropout)         (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_45 (Dense)             (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dropout_24 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_46 (Dense)             (None, 4250)              1092250   \n",
      "=================================================================\n",
      "Total params: 106,081,690\n",
      "Trainable params: 106,081,690\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 8806 samples, validate on 234 samples\n",
      "Epoch 1/60\n",
      "8806/8806 [==============================] - 293s 33ms/step - loss: 8.3876 - acc: 0.0017 - val_loss: 8.2728 - val_acc: 0.0043\n",
      "Epoch 2/60\n",
      "8806/8806 [==============================] - 262s 30ms/step - loss: 8.0913 - acc: 0.0043 - val_loss: 8.1634 - val_acc: 0.0000e+00\n",
      "Epoch 3/60\n",
      "8806/8806 [==============================] - 268s 30ms/step - loss: 7.9063 - acc: 0.0056 - val_loss: 8.1069 - val_acc: 0.0043\n",
      "Epoch 4/60\n",
      "8806/8806 [==============================] - 270s 31ms/step - loss: 7.7480 - acc: 0.0070 - val_loss: 8.1339 - val_acc: 0.0043\n",
      "Epoch 5/60\n",
      "8806/8806 [==============================] - 264s 30ms/step - loss: 7.5954 - acc: 0.0092 - val_loss: 8.1110 - val_acc: 0.0128\n",
      "Epoch 6/60\n",
      "8806/8806 [==============================] - 262s 30ms/step - loss: 7.4960 - acc: 0.0107 - val_loss: 8.1730 - val_acc: 0.0085\n",
      "Epoch 7/60\n",
      "8806/8806 [==============================] - 264s 30ms/step - loss: 7.3602 - acc: 0.0133 - val_loss: 8.1259 - val_acc: 0.0043\n",
      "Epoch 8/60\n",
      "8806/8806 [==============================] - 262s 30ms/step - loss: 7.2600 - acc: 0.0161 - val_loss: 8.1149 - val_acc: 0.0085\n",
      "Epoch 9/60\n",
      "8806/8806 [==============================] - 256s 29ms/step - loss: 7.1933 - acc: 0.0179 - val_loss: 8.1920 - val_acc: 0.0214\n",
      "Epoch 10/60\n",
      "8806/8806 [==============================] - 256s 29ms/step - loss: 7.0731 - acc: 0.0221 - val_loss: 8.2020 - val_acc: 0.0128\n",
      "Epoch 11/60\n",
      "8806/8806 [==============================] - 264s 30ms/step - loss: 7.0029 - acc: 0.0229 - val_loss: 8.2684 - val_acc: 0.0128\n",
      "Epoch 12/60\n",
      "8806/8806 [==============================] - 263s 30ms/step - loss: 6.8980 - acc: 0.0258 - val_loss: 8.2606 - val_acc: 0.0171\n",
      "Epoch 13/60\n",
      "8806/8806 [==============================] - 262s 30ms/step - loss: 6.8165 - acc: 0.0284 - val_loss: 8.2076 - val_acc: 0.0171\n",
      "Epoch 14/60\n",
      "8806/8806 [==============================] - 258s 29ms/step - loss: 6.7416 - acc: 0.0285 - val_loss: 8.2765 - val_acc: 0.0342\n",
      "Epoch 15/60\n",
      "8806/8806 [==============================] - 251s 28ms/step - loss: 6.6765 - acc: 0.0285 - val_loss: 8.4129 - val_acc: 0.0171\n",
      "Epoch 16/60\n",
      "8806/8806 [==============================] - 255s 29ms/step - loss: 6.5944 - acc: 0.0338 - val_loss: 8.4103 - val_acc: 0.0342\n",
      "Epoch 17/60\n",
      "8806/8806 [==============================] - 263s 30ms/step - loss: 6.5369 - acc: 0.0340 - val_loss: 8.4195 - val_acc: 0.0256\n",
      "Epoch 18/60\n",
      "8806/8806 [==============================] - 263s 30ms/step - loss: 6.4562 - acc: 0.0388 - val_loss: 8.5179 - val_acc: 0.0342\n",
      "Epoch 19/60\n",
      "8806/8806 [==============================] - 260s 29ms/step - loss: 6.3775 - acc: 0.0400 - val_loss: 8.4564 - val_acc: 0.0256\n",
      "Epoch 20/60\n",
      "8806/8806 [==============================] - 259s 29ms/step - loss: 6.3073 - acc: 0.0427 - val_loss: 8.5265 - val_acc: 0.0470\n",
      "Epoch 21/60\n",
      "8806/8806 [==============================] - 262s 30ms/step - loss: 6.2414 - acc: 0.0461 - val_loss: 8.6303 - val_acc: 0.0427\n",
      "Epoch 22/60\n",
      "8806/8806 [==============================] - 255s 29ms/step - loss: 6.1954 - acc: 0.0459 - val_loss: 8.6274 - val_acc: 0.0427\n",
      "Epoch 23/60\n",
      "8806/8806 [==============================] - 254s 29ms/step - loss: 6.1177 - acc: 0.0513 - val_loss: 8.6693 - val_acc: 0.0427\n",
      "Epoch 24/60\n",
      "8806/8806 [==============================] - 260s 30ms/step - loss: 6.0930 - acc: 0.0489 - val_loss: 8.6521 - val_acc: 0.0556\n",
      "Epoch 25/60\n",
      "8806/8806 [==============================] - 265s 30ms/step - loss: 5.9796 - acc: 0.0545 - val_loss: 8.6304 - val_acc: 0.0513\n",
      "Epoch 26/60\n",
      "8806/8806 [==============================] - 267s 30ms/step - loss: 5.9498 - acc: 0.0583 - val_loss: 8.6920 - val_acc: 0.0427\n",
      "Epoch 27/60\n",
      "8806/8806 [==============================] - 270s 31ms/step - loss: 5.9124 - acc: 0.0629 - val_loss: 8.7702 - val_acc: 0.0427\n",
      "Epoch 28/60\n",
      "8806/8806 [==============================] - 267s 30ms/step - loss: 5.8708 - acc: 0.0585 - val_loss: 8.8187 - val_acc: 0.0470\n",
      "Epoch 29/60\n",
      "8806/8806 [==============================] - 266s 30ms/step - loss: 5.8028 - acc: 0.0626 - val_loss: 8.7319 - val_acc: 0.0427\n",
      "Epoch 30/60\n",
      "8806/8806 [==============================] - 266s 30ms/step - loss: 5.7418 - acc: 0.0654 - val_loss: 8.7557 - val_acc: 0.0726\n",
      "Epoch 31/60\n",
      "8806/8806 [==============================] - 264s 30ms/step - loss: 5.7489 - acc: 0.0684 - val_loss: 8.7551 - val_acc: 0.0598\n",
      "Epoch 32/60\n",
      "8806/8806 [==============================] - 266s 30ms/step - loss: 5.7059 - acc: 0.0650 - val_loss: 8.8797 - val_acc: 0.0684\n",
      "Epoch 33/60\n",
      "8806/8806 [==============================] - 264s 30ms/step - loss: 5.6168 - acc: 0.0789 - val_loss: 8.8535 - val_acc: 0.0769\n",
      "Epoch 34/60\n",
      "8806/8806 [==============================] - 259s 29ms/step - loss: 5.6317 - acc: 0.0720 - val_loss: 8.8183 - val_acc: 0.0684\n",
      "Epoch 35/60\n",
      "8806/8806 [==============================] - 260s 30ms/step - loss: 5.5573 - acc: 0.0803 - val_loss: 8.8329 - val_acc: 0.0855\n",
      "Epoch 36/60\n",
      "8806/8806 [==============================] - 268s 30ms/step - loss: 5.5048 - acc: 0.0793 - val_loss: 8.8766 - val_acc: 0.0726\n",
      "Epoch 37/60\n",
      "8806/8806 [==============================] - 263s 30ms/step - loss: 5.5042 - acc: 0.0797 - val_loss: 8.8543 - val_acc: 0.0812\n",
      "Epoch 38/60\n",
      "8806/8806 [==============================] - 264s 30ms/step - loss: 5.4400 - acc: 0.0848 - val_loss: 8.9205 - val_acc: 0.0855\n",
      "Epoch 39/60\n",
      "8806/8806 [==============================] - 264s 30ms/step - loss: 5.3904 - acc: 0.0879 - val_loss: 8.9113 - val_acc: 0.0940\n",
      "Epoch 40/60\n",
      "8806/8806 [==============================] - 268s 30ms/step - loss: 5.3869 - acc: 0.0893 - val_loss: 8.9316 - val_acc: 0.0897\n",
      "Epoch 41/60\n",
      "8806/8806 [==============================] - 270s 31ms/step - loss: 5.3421 - acc: 0.0914 - val_loss: 8.9568 - val_acc: 0.0897\n",
      "Epoch 42/60\n",
      "8806/8806 [==============================] - 274s 31ms/step - loss: 5.2557 - acc: 0.0999 - val_loss: 8.9468 - val_acc: 0.0812\n",
      "Epoch 43/60\n",
      "8806/8806 [==============================] - 275s 31ms/step - loss: 5.2755 - acc: 0.0927 - val_loss: 8.9097 - val_acc: 0.0940\n",
      "Epoch 44/60\n",
      "8806/8806 [==============================] - 274s 31ms/step - loss: 5.2478 - acc: 0.0933 - val_loss: 8.9467 - val_acc: 0.1111\n",
      "Epoch 45/60\n",
      "8806/8806 [==============================] - 260s 29ms/step - loss: 5.2335 - acc: 0.0958 - val_loss: 8.9176 - val_acc: 0.1026\n",
      "Epoch 46/60\n",
      "8806/8806 [==============================] - 269s 31ms/step - loss: 5.1703 - acc: 0.1027 - val_loss: 8.9833 - val_acc: 0.1068\n",
      "Epoch 47/60\n",
      "8806/8806 [==============================] - 268s 30ms/step - loss: 5.1500 - acc: 0.1045 - val_loss: 9.0764 - val_acc: 0.0983\n",
      "Epoch 48/60\n",
      "8806/8806 [==============================] - 271s 31ms/step - loss: 5.1053 - acc: 0.1108 - val_loss: 9.0305 - val_acc: 0.0940\n",
      "Epoch 49/60\n",
      "8806/8806 [==============================] - 268s 30ms/step - loss: 5.1042 - acc: 0.1099 - val_loss: 9.0739 - val_acc: 0.1111\n",
      "Epoch 50/60\n",
      "8806/8806 [==============================] - 274s 31ms/step - loss: 5.0758 - acc: 0.1089 - val_loss: 9.0422 - val_acc: 0.0940\n",
      "Epoch 51/60\n",
      "8806/8806 [==============================] - 268s 30ms/step - loss: 5.0437 - acc: 0.1115 - val_loss: 9.0649 - val_acc: 0.0983\n",
      "Epoch 52/60\n",
      "8806/8806 [==============================] - 268s 30ms/step - loss: 5.0417 - acc: 0.1094 - val_loss: 9.0279 - val_acc: 0.0983\n",
      "Epoch 53/60\n",
      "8806/8806 [==============================] - 268s 30ms/step - loss: 4.9500 - acc: 0.1220 - val_loss: 9.1171 - val_acc: 0.1026\n",
      "Epoch 54/60\n",
      "8806/8806 [==============================] - 262s 30ms/step - loss: 4.9217 - acc: 0.1218 - val_loss: 9.0346 - val_acc: 0.1197\n",
      "Epoch 55/60\n",
      "8806/8806 [==============================] - 258s 29ms/step - loss: 4.9410 - acc: 0.1166 - val_loss: 9.1278 - val_acc: 0.0940\n",
      "Epoch 56/60\n",
      "8806/8806 [==============================] - 265s 30ms/step - loss: 4.8691 - acc: 0.1293 - val_loss: 9.0744 - val_acc: 0.1026\n",
      "Epoch 57/60\n",
      "8806/8806 [==============================] - 266s 30ms/step - loss: 4.9199 - acc: 0.1234 - val_loss: 9.0926 - val_acc: 0.1026\n",
      "Epoch 58/60\n",
      "8806/8806 [==============================] - 267s 30ms/step - loss: 4.8550 - acc: 0.1310 - val_loss: 9.2157 - val_acc: 0.0983\n",
      "Epoch 59/60\n",
      "8806/8806 [==============================] - 269s 31ms/step - loss: 4.8653 - acc: 0.1296 - val_loss: 9.1902 - val_acc: 0.0983\n",
      "Epoch 60/60\n",
      "8806/8806 [==============================] - 271s 31ms/step - loss: 4.8219 - acc: 0.1304 - val_loss: 9.1236 - val_acc: 0.1026\n"
     ]
    }
   ],
   "source": [
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import optimizers\n",
    "\n",
    "model = models.Sequential()\n",
    "model.add(layers.Dense(512, activation='relu', input_dim=ll_size*ll_size*2048)) # we can play around with \n",
    "model.add(layers.Dropout(0.5))\n",
    "model.add(layers.Dense(256, activation='relu', input_dim=ll_size*ll_size*2048))\n",
    "model.add(layers.Dropout(0.25))\n",
    "model.add(layers.Dense(classes_count, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "model.compile(optimizer=optimizers.Adam(lr=2e-4),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=60,\n",
    "                    batch_size=64,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEmCAYAAAA3CARoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAMTQAADE0B0s6tTgAAIABJREFUeJzs3Xl8k0X+wPHPBJqWowctR8tRQBBELMqlAoogIiAIu6io\nuyqgIBRPzmVRXJVVUQ5/nhwiyOJ94KoUKbqAihdUVFBQDoFCaQUKpOVoU5r5/TFJSdIkTU9a+n2/\nXn1JnmeeeSYp5svM850ZpbVGCCGEqGwsZ7sBQgghhC8SoIQQQlRKEqCEEEJUShKghBBCVEoSoIQQ\nQlRKEqCEEEJUShKghBBCVEoSoERASqklSqnni1H+TqXUj+XZpvKilDqklBpWjvV/q5Sa5vyzVSmV\nrZS6LED5WUqpT0t5z/OVUllKqdjS1CPE2VDzbDdAlJ5SKhtwzbgOBWoAJwHlPD5Aa/11SerWWo8s\nZvnFwOKS3Ks60VrbgfBgigZbp1LqKaCz1vpat/vsACKK30Ihzj4JUOcArXXBF51SagbQQ2t9daBr\nlFJW55ekEOcUpVSI1jrvbLdDlJ4M8VUTSqmnlFKfO/97APjG7fgO53DTHqXUXKWU1e26t5RSC91e\npyulpiqlkp3X/K6UGuh2foxSaofX9UuVUvOVUplKqQNKqYe92jZEKfWrcyhqlVLqCaXUtgDv5RKl\n1BrnkNxRpdQ3Sqkr3M63VUo5lFJ/V0ptcdb7lVKqtVuZcKXUf5xt2quUGlvE5zdHKbXC61gzpVSe\nUuoC5+slzs8wWym1Uyn1UID6Qp1t7O52bJTzOptS6m28elhKqQecn5NNKbVfKbVYKRXlPHcnMAHo\n5bx/llKqs9tn0ditnrud9RxTSv2klLqlOJ9dcX8fzjIJSqkVSqkMpdQRpdR6pVRD57naSqmnnX8P\ns5z/vdl57iml1Gdedfn6O/mwUmq1UsoGJDp/N0lKqT+dn9cmpdQQr3qaO+va7yyTopS6UCl1nfPv\nhdWr/O9Kqbv9fQ6iHGit5ecc+gFmAGt8HH8KsAOTgBAgzHn8diDO+eeLgD3AdLfr3gIWur1OB3YC\n7TFDiP8EjrjVNwbY7nX9SWCIs3xP4DRwmfN8O2e7bsb8g6kHcBjYGuA9Xgz0dr6PUOAJIBOIdJ5v\nCziAJCDGWeYj4FO3Ov4DfAs0BGoDrzvbMczPPV3tjHM79i9gvdvrUUC08889gGPA7W7nvwWmOf8c\n6mxjd+frq4EcoK/zcxjqfL3S7fobgZbOP7cAfgBe9fodr/Zqd1sgH2jsfP1352fVw/n76O/8/fQN\n9rMrwe+jMXAUeBio43x/Xd3Ovw98BbRyK98hwHvy9XcyFejkfB0GNAeuB2phhrzHOj9P1z3qAH8A\nLwNRzmPtnfdWznN/d7tHH8AG1Dnb/49Xp5+z3gD5KeNfaOAAtTOI66cCX7m99vVlMNHtdbTzCy3B\n+dpXgFrhdY/NwH1u7V3ndf4FAgQoH222ACeAPs7Xri/Zzm5lhgKZzj+HAHnA1W7n6zuv8RmgnGW+\n5kyAUcBuYHiA8vOBZW6vAwWo/7iXdR5bgVuA8lH/LcA+r99xUQFqHfCEj3YuD+azK+Hv4yFgg5+y\nTZz3a+fnfLAB6uEg2vUbcJfzz7cBaYAlwP8HX7q9fgd4OdjPQH7K5keG+KqXPd4HlFL3KqV+dA5p\nHAWmY3oVgaS7/fmE87+BHvgf8Hp9wq18E2BvUe10p5RqqZR6xzk0dwzzr/VQPNutfbTTdc84zJdo\nwX201ofd3os/iwBX0si1QD3gXWeblFLqX86hs6NKqSPAHRT9Wbo0xQQ8dx6vlVI3K5MJeND5u3ql\nGPW7NAN2eR3bCcS7vQ702RUSxO+jJfC7n8tbOu+3Peh34NserzbFOIdA/3AOZR4FznNrUwvgD621\nw099rwJdlVIXOIci/4IJ5KICSYCqXjz+Z1RK9QKeAe4BGmqt62F6NKoC25SGGY5x16KIa5YAuZgh\nnSjMUFQuwbc7HfNZFNxHKdUAM+wTyDtAA6XU1cBdwNta61POcyMwvcdbMMN80ZheUbBt2k/h9+3e\nvvOAN4GZQBPn72q0V/3+vmzd7QNaeR1rhRkiK6mifh97gDZ+rt3j/K+/89kU/r009lHO+73PxQTj\nHlrrKOfn9YdXm85TSvn8DtRaHwI+wPxORwKbtNab/bRRlBMJUNVbJOZ50GGtdb5SqiuQWMFteBPo\nppQappSyOJMGbinimkjMF5dNKVUXmA1Yvcr4DQzaZHi9DcxQSjVyq+N0oJtqrU86r5uCeb6xyO10\nBOYZ1WEApVRfzHO1YC0FblRKXeP8HP4CXON23tWDOay1diVmTPGqIwNo4f1wH8/PYhEwRinVw3mf\nazHPIRf4KR+Mon4fS4A2SqlpSqk6SqkaSqlLlVIRWuv9wIfAfKVUKwClVGOl1MXOa1OAzkqprs72\n/h3wO3fMTQTm2dpRZRJSJuEZmD/E/L6eV0rVc/aA2yulmriVmYfpBY/G8/MRFUQCVPX2CeaL8Ru3\n4b0lRVzja15OcXe9LCivtd6G+SJ/FJNU8AhmHlVOgOvHAd2d5X8GtgGHitmmezBDXduAXzFZjceC\naPurmESG37TWKW7HX8E8Y9oGHMR8sb1RRJvcP4f/AfdjvgiPYoL0a27nf8YkZbzvzFSbj+mhuXsD\n0xNKd2bKdfJxn9cxz4RewSS3zAJGa62TA7SzKAF/H1rrNOAqTILMHsznMxszDAgwHNgAfKaUysI8\nJ2vjvHY1pjeUBPwJdMYkbbjz1d6pmJ7cIcwQZhiw0a1NJzCJHY2ArZjP4jXc5oxpM3cwDfOc9e3g\nPgpRlpTzAaAQlYZSah4Qq7X+69lui6jelFL/BfZorR88222pjoLuQSmlWiulvnbOBfheKdXOR5k6\nysxjOeR8SOyvrseUmWvRoaQNF+cOpdT1zmEWi1JqACbD6vWz3S5RvTmHvPsBz53ttlRXxRniWwDM\n11q3xTxYX+qjTB7mIW4ff5U4f+ldKCJTS1QrPTFZXDbMcM4/tdYfnN0miepMKbUBWA1M0Vp7Z1eK\nChLUEJ8zw2kHJjvJ4TyWjsmQ+cNH+ebAj85MJvfjtTDjy0OB9cAQyYwRQgjhS7A9qGZAutecgVQ8\n504E4xngJedDUyGEEMKvClssVil1DdBca31fEeUUZp5DdoU0TAghRHkKBw7oEmTkBRug9gFxSimL\nWy8qnuJN7rsa6KiUck2WawqsVEqN0VonuZVrjJm0KIQQ4tzQFJOyXyxBBSit9SGl1CbMhL6lSqkb\nMWuAFXr+5KTwmuyntZ4GTCsooNRuYLDWeovXtdkA+/btIyKiem1jM23aNJ588smz3YwKJ++7epH3\nXX1kZWXRrFkzKOGIWHGG+MYCrymzI6gNs7QLSqnHgDSt9ULn658xC2+GK6VSgbVa6+E+6tMEmLEe\nERFR7QKU1Wqtdu8Z5H1XN/K+RbCCDlBa6+2Y2eLex//l9fpi7zJ+6jsv2HsLIYSofmSpo0qkX79+\nZ7sJZ4W87+pF3rcIVqVb6kgpFQHYbDabdIeFqGA5OTnY7faz3QxRBVmtVsLCwjyOZWVlERkZCWZz\nyqzi1llhaeZCiMotJyeHli1bkpGRcbabIqqg2NhYdu/eXShIlYYEKCEEAHa7nYyMjGqZQStKx5Wt\nZ7fbJUAJIcpPdcygFZWTJEkIIYSolCRACSGEqJQkQAkhhKiUJEAJIYSXkSNHcv/99wddfvHixXTs\n2LEcW1Q9yTwoIQRwZs5KoP/37HY7mZmZxMTEYLVaS3W/0tYVHh6O2fwAcnNzyc/Pp3bt2mitUUrx\n6aef0qNHj1K1UQTH9Xfn0KFD1K9fv9BxZB6UEKK8OBwOJk2aQ1JSKllZzYiI2MfAgfHMnj0Ri6V4\nAzFlVVd29pn1R6dPn87XX3/NmjVrAl5jt9tLHViFf5df/hCDB7cu0d8Ln7TWleoHiAC0zWbTQoiK\nY7PZtL//98aPf0aHha3VoAt+wsLW6vHjnyn2fcqyLpeHH35Y9+7du9DxqVOn6j59+uipU6fquLg4\n3blz54LjrVu31nXr1tXNmzfX48eP17m5uQXX3XLLLXr06NEFr2NjY/VTTz2lr732Wl23bl3dpk0b\nvWLFioLz8+fP161bt/a4/o477tBjxozR0dHROi4uTs+YMcOjbf/973/1hRdeqMPDw3W/fv30tGnT\n9AUXXOD3Pf7444+6d+/eun79+joqKkp369ZNf/XVVx5lNm/erAcOHKgbNWqk69Wrp3v06KH//PNP\nrbXWJ06c0FOmTNGtW7fW4eHhunXr1vrtt98O5uMtkuvvDtg8fpdnjhOhSxAP5BmUECIgu91OUlIq\nOTm9PI7n5PQiKWlvsZZGKsu6gvXll18SExPD3r17Wb9+PQAXXnghX375JdnZ2axYsYLly5fz9NNP\nB6xn0aJFzJ07l6ysLEaMGMHtt99OTk5OwXnXcKPLe++9x4ABAzh8+DBvv/02jz76KN9//z0A27Zt\n46abbuKRRx7h2LFjTJ8+nQULFhSqw51SiunTp3PgwAEyMjLo3bs3Q4YMwWazAXDgwAF69uzJ5Zdf\nzq5duzh8+DDPPvssoaGhANxxxx188803rFq1iqysLL744gvatWtX/A+0CGX5u5QAJYQIKDMzk6ys\nZj7PZWc348iRI2elrmDFx8czadIkQkJCClY5uP3224mLiwPgoosuYuzYsaxevTpgPYmJibRv3x6l\nFGPGjOHYsWPs2LHDb/mrr76aIUOGoJSiZ8+eXHjhhWzYsAGAN998k+7du3PzzTdjsVjo0aMHt956\na8D7X3zxxfTu3ZuQkBBCQ0OZMWMGOTk5pKSkALBkyRLOP/98Hn74YerUqYPFYqFr165ERkaSlpbG\n8uXLWbhwIa1atQKgcePGdOjQIbgPsZjK6ncpAUoIEVBMTAwREft8ngsP3090dPRZqStYLVq0KHTs\nxRdfpGPHjsTExFCvXj1mzJjBwYMHA9bjCmgAderUATyfg3lr3Lixx+s6deoUlE9LS6N58+ZFttPd\n7t27ufnmm2nevDlRUVHExMSQm5tb0O7du3fTtm1bv9cqpWjTpk3Ae5SVsvpdSoASQgRktVoZODCe\nsLB1HsfDwtYxcGB8sZIOyrKuYHk/rF+3bh1TpkzhpZde4uDBgxw9epTp06e7noFXiCZNmrB3716P\nY3v27Al4zciRIwkNDWXTpk0cO3aMzMxMQkNDC9rdokULtm/f7vNaV/Dzd74sleXvUgKUEKJIs2dP\nJDFxI23a3Etc3NO0aXMfiYkbmT174lmtqyRsNhs1a9akfv361KhRg40bNzJv3rwKubfL3/72N779\n9lveffddHA4H33zzDW+//XbAa2w2G+Hh4URGRnL8+HEmTZrk8Zxn5MiRbN++nSeffJITJ06Qn5/P\nhg0byMrKomnTpvz1r39l7Nix7Nq1CzDPrH7++ecyfV+tWk0u09+lpJkLIYpksViYO3cyM2faOXLk\nCNHR0SX+F3JZ1lUS119/PcOHD6d79+7k5+dz5ZVXMnLkSN58802/1/hKXgiU0FBUHe3ateOdd95h\n2rRpjBo1iu7du3PnnXfy2Wef+b3+5ZdfZty4cURFRdGoUSOmTp1KgwYNCs43adKEL774gilTpvDs\ns8/icDho3749H3zwAQBLly7l0UcfpW/fvhw+fJjY2FieeOIJLr44qE3Qg/Ldd094zIMqLZmoK4QA\ngpuoK8pPYmIiGRkZfPjhh2e7KcXm7+9OaSfqyhCfEEKcBZ988glHjx7F4XDw6aef8vrrr3Pbbbed\n7WZVKjLEJ4QQZ8GXX37JnXfeSU5ODk2bNuWpp57ihhtuONvNqlRkiE8IAcgQnyg5GeITQghRrUiA\nEkIIUSlJgBJCCFEpSYASQghRKQUdoJRSrZVSXyulfldKfa+UKrQMrlKqjlJqlVLqkFLqiNe5OOe5\nbUqpn5RS7ymlYsriTQghhDj3FKcHtQCYr7VuCzwDLPVRJg+YCfTxcS4feFxr3U5rfQmwG5hdzPYK\nIUSZevXVV2nZsmXB69GjRzNu3Di/5X///XcsFgsHDhwo1X379evHzJkzS1XHuS6oeVBKqQZAZ6Av\ngNb6A6XUi0qp87TWf7jKaa3twDqlVHPvOrTWBwH35YK/B+4pTeOFENXX0KFD0Vr7XHlh6tSprFix\ngl9++SWoutyXIXrllVeKVb4ou3bt4vzzz2fPnj3Ex8cXHE9OTg66juoq2B5UMyBda+1wO5YKxPsp\nH5BSygLcC/y3JNcLIURiYiJJSUmFejJ5eXksWbIkYC+oImmti71unzDOVpLEPOCI1vr5s3R/IUQV\n17dvX1q2bFmox/Pee+9x6tQp7rjjDgDeeecdOnfuTL169WjUqBF//etfSU1N9Vvv7bffzp133lnw\neufOnVx99dVERkaSkJDAl19+6VF+8+bN9OnTh4YNG1KvXj26detWUMbhcHDJJZcA0L59eyIiIrj/\n/vsBuPLKK3n88ccL6tm2bRv9+/enfv36xMfHM27cOI/9pq688komTJjArbfeSmRkJM2bNy+ytzdq\n1ChatGhBeHg4rVu3ZsaMGR7nT506xT//+U/atGlDREQErVu39lg095NPPqFbt25ER0fTsGHDIjdV\nLGvBBqh9QJyz5+MSj+lFFYtS6nmgMTAsULlp06YxYcIEJkyYIF1hIYRPY8eOZdGiRTgcZwZ3FixY\nwG233UbdunUBiIyMZOnSpRw9epRff/2V06dPB73mXX5+PoMGDeL888/n4MGDfPrppyxcuNCjjFKK\nhx56iP379/Pnn39y7bXXMmTIEI4ePYrFYinY0mLr1q1kZWXx/POF/12elZVFnz596NSpEwcOHGDD\nhg1s2bLFI1CCWZF83Lhx2Gw2Zs2axbhx4wrtK+WuW7du/PDDD2RnZ7Ns2TKeffZZlixZUnB+5MiR\nfPHFF6xcuZKsrCy++uor2rdvD8CqVau45ZZb+Mc//sHBgwfZt28fd999d5GfWXJycsF397Rp04os\nH5DWOqgfYA0w3PnnG4ENAcq2AI76OP48kASEBLg2AtA2m00LISqOzWbTRf2/53BobbOV7Y/DUfI2\nHz16VNeuXVt/+OGHWmutf/nlF62U0lu2bPF7zYYNG3SNGjX0qVOntNZaL1q0SLds2bLg/G233aZH\njhyptdZ63bp1OiQkRJ84caLg/IcffqgtFotOS0vz8xk5dN26dfWqVau01lrv3LlTWywWvXfvXo9y\nV1xxhX7ssce01lr/5z//0XFxcdrh9mFs3LhRK6V0ZmZmQfkxY8Z41FGvXj29fPnyAJ+Qp3vuuUff\ncsstWmut09PTtVJKb9682WfZ/v376/HjxwdVr7+/O67jQIQOMta4/xRniG8sMEYp9TswBRgBoJR6\nTClVEFaVUj8DXwPhSqlUpdRS5/HumKSIFsAGpdSPSqkPihtQhRBnT3Y2REaW7U+AXdOLFBUVxc03\n38z8+fMBmD9/Pt27d+eiiy4qKLNmzRr69OlDXFwcUVFRXHPNNWitOXToUJH1p6WlUb9+fWrXrl1w\nzD3jD2Dv3r3ccsstBVuxR0dHc+rUqSK3kHe3f/9+WrRo4fGsqnXr1gAew5GBtpH3prVmxowZXHTR\nRURHRxMdHc3ixYsL2rVnz56A28AH2kK+ogS9mrnWejvQ3cfxf3m99rn7ldb6G6BGcRsohKg8wsPB\nZiv7Oktj3LhxXH755WzevJnXX3+dl19+ueBcbm4ugwcP5oknnmDFihXUqlWLlJQULrvssqC2eG/a\ntCmHDx/m5MmTBUFq9+7dHmXuuusuGjVqxA8//ED9+vXRWhMZGVlQv8ViKfJezZo1Y+/evR4JFTt3\n7kQp5ZH5Vxyvv/46L730EqtXryYhIQGlFPfeey9bt24FPLeBT0hIKHR9oC3kK4qsJCGECJpSEBFR\ntj+lTXDr0qULnTp1YujQoYSGhnLjjTcWnMvNzSU3N5d69epRq1Yt9u/fz/Tp04Ouu3v37rRs2ZKJ\nEydy6tQp9u3bx1NPPeVRxmazUbduXaKiojh+/DhTpkzh1KlTBecbNmxIjRo1+P333/3eZ/DgwTgc\nDh5++GFycnJIT09n4sSJDB06lOjo6GJ8GmdkZWUREhJSsMPt//73P956662C87Gxsdx0002MGzeO\nHTt2AJCens5PP/0EwAMPPMDChQv5+OOPycvLIycnhzVr1pSoLSUlAUoIUeWNGzeO3bt3c9dddxES\nElJwPCIigoULF/LII48QERHB4MGDufnmm4Out2bNmqxYsYLffvuNRo0aMWDAAEaPHu1R5sUXX2Tj\nxo1ERUXRoUMHWrVqRVxcXMH5OnXqMGPGDO644w6io6N58MEHAc+5VBEREXz++eekpKTQtGlTLr30\nUtq3b8+rr75aUKa4287feeed9OzZkwsvvJAGDRqwePHiQskhixcvpnv37vTv35/w8HB69uzJb7/9\nBsCAAQN4/fXXmTFjBg0bNiQ+Pp7FixcH/dmVBdkPSggByH5QouRkPyghhBDVigQoIYQQlZIEKCGE\nEJWSBCghhBCVkgQoIYQQlZIEKCGEEJWSBCghhBCVUtBLHQkhqoesrGJPVxHVXHn9nZEAJYQAwGq1\nEhsbS7Nmzc52U0QVFBsbi9VqLdM6JUAJIQAICwtj9+7d2O32s92UoPztb/D11/Dbb1CrVtnVm5GR\nwZVXvs3Bgw8WOler1nDy8+/Ebr+q4Fho6FfcddcmnnrqAex2O0eOHClYP+/yyx9i165Zhepp1Woy\n33wzg8ceW0By8n6ys5sQHp5Gv35NeeKJ+7BYPJ++XH01jB0Lw9x20XvvPZg/H/73vzJ646VktVoJ\nCwsr0zolQAkhCoSFhZX5l0x5+flnyM2FVatg+PCyqzcsLIyoqEMcPOi93JOd06dDyMu73uNobu5A\nPvtsJTVrvsyqVWlkZTUjImIfAwfGc/31rZg/fxM5Ob3c6l/H4MGtmTlzKYsXX1lw7uBBWLx4HaGh\nrzB37uQzd7XDL79Az55mcV2XhATYu9fz2LlGkiSEEFVORgakpcHDD5teRFmyWq0MHBhPWNg6j+Oh\noZ8QEnKBz2vS0g6xYMGlbN/+AhkZU9i+/QXmzesKaBITN9Kmzb3ExT1Nmzb3kZi4kSefvI+kpFSP\nwAWQk9OLpKS9Hr3YX3+F0FBwbg9VoFUrOHSodPtpVXbSgxJCVDkpKdC2LdxzDzzxBPz0E1xySdnV\nP3v2RGAOSUnvk53djPDw/fTv35hPPz2Cc2cKN3by8iAv72qPozk5vVi58n22bJnLzJkUDP1ZrVbS\n09PJyvL9rC87uxlHjhwhNjYWgI0boUsX8Br1IzrabPi4a1fZvvfKRHpQQogqZ+NG6NrVfEH/7W8w\nb17Z1m+xWJg7dzJbtsxl06bhbNkyh+ee+yeDBjWnRo09HmUD9axcwcaVgOJKIoiJiSEiYp/Pa8LD\n93vsAZWSYgKUN6VML2rXrpK9x6pAApQQospx/9JOTIQ33oDyyHT2DiyzZ08kPj6fBg3eIyrqY0JD\n9zJmzHaaNDni83rvYONer69hxLCwdQwcGO+RDZeSYoKxLxKghBCiEtHa80u7Uydo3x6WLSv6utTU\n0t1bKQsnTrTi/ff/wsqV3XA44nnmGdOzCibYuJs9e6LP51NmeNHIyYEtW3z3oADOOw/++KN076ky\nkw0LhRBVSmqq+WLOyoLatc2xJUtgzhzzZe5vk9n/+z+YMAF27jTXu7Pb7WRmZhITExNwLo/7vWvV\nMs+BPvsMOnVyMGnSHJKS9hY8sxo4MJ7ZsycWShn35p6a7n3v77+HgQNNMoSv9/XKK/Duu6YNlVFp\nNyyUJAkhRJWSkmJ6TK7gBHDzzSb4rF8PV15Z+JrVq+Ghh6BzZ1iwAJ5+2hx3OFyBJdUjPdxfYElJ\ngYsuOnPvLl1cw43mmdXMmf6DjT+uYURfXM/a/AXdc32ITwKUEKJKcX1pu6tdG0aMMCnn3gFqxw4T\nwObPhyZNzJ8ff9ykbk+aNId587qSk2PmHWVkQGrqOmCOx1wk93u7D7d16WKOjR1rXgcKNiXhL0HC\npVUr06vLy4OQkDK7baUhz6CEEFWKvy/tsWPhgw/McJiLzQaDB8OoUXD77dC7N8TEwPvvm6G1YOci\n+bt3167mWHnxFYzdNW1q0s/37i2/NpxNEqCEEFWGd4KEu7ZtoUcPWLzYvM7PNynoLVrAzJkmIGVk\npDNq1GnmzYPMzMwi5yIVde8uXcxE2pMny+gNujl+HLZtC9yDqlEDWrY8d4f5ZIhPCFFl7NplgsFF\nF/k+P3Ys/OMfMHkyTJtmyn/zjYPJk888Z6pTJ5M9e/5NWlo0ERH7yMgoXI+v9HDXvRMSzhxr1sz0\nyH76Cbp3L8M3Cvz4I8TGQuPGgcudy8+hJEAJIaqMlBTo0ME8P/LlL3+B+++H0aNh+XL47jv49789\nnzMB1KiRzqhRGQwcGE9q6rpCa+X5Sg9PSYGLLwb3w0qdSZQo6wBV1PCeiwQoQCnVGlgK1AeOASO0\n1tu8ytQBPgA6AzW01tFe5y8DFgBhwH7gdq11eqnegRCi2ijqSzskxDxvevJJWLkSWrZ0PWfyTHjI\nz49jy5YoPv+8HfCCx5JGriw+X/f2NdzmSpQoa198EXh4z6VVK1i3zv/57GwTtI8dK3wuNNR8Vt5p\n98W1cSMsWmQSUfxlHJZEcXpQC4D5WutlSqkbMMHqUq8yecBM4Aiwzv2EUkoBrwN3aa2/VEpNBJ4D\nhiGEEEFISSl65fLJk+Gaa+CqqyA93f9zJovFxuLFtYJOD/d3765dzVyksvTKK2YrkeefL7psq1bw\n6qv+z//nPyaAjBlT+Ny335okkm+/hfDwkrU1Pd30XCdOLNvgBIDWusgfoAGm12RxO5YOnOenfHPg\niNexLsBWt9d1gVOA1atcBKBtNpsWQlQfubm5+sCBAzo3N9fn+dOnta5bV+vNm4tXZ5s292qT4uD5\n07Dhm7pDh3ztcBRdz+nTWtep4/ve6elaK6V1WX1lffWV1rVra712bXDlf/3VlPf1PhwOrdu313rJ\nEt/X5uXIYgkeAAAgAElEQVRp3bev1oMHa52fX/y2njql9WWXaX3HHb7vb7PZNKCBCB1ErPH+CTaL\nrxmQrrV2uB1LBeKLEQvjgYJkSK31ccAGFPEIUAhxLnM4HEyYMIuEhIl06rSMhISJTJgwC4fD4VHu\n999NZl67dsHXHWjNu2HD/mTPHgvffVd0Pb//Dg6H73vHxpr5VZs2Bd8uf1JTYehQsypGr17BXdOy\npUne8JXssX692Zbk5pt9X1uzJrz9NmzdCo88Ury2an2mV7ZgQTn0njj7SRLl8JaEEFVJsJNlU1Kg\nY0fzpVocvrbOcH/ONG8edOsWuI6UFLPmn797uxIlgg0qvpw4AUOGwA03nJn4G4xatUyA3LUL4uI8\nz82fbyYwB9pxODoaPv4YLr/cZCj6C2benn0WPv/cvO/y2uMy2F/1PiBOKWVx60XFY3pRwUoFWrhe\nKKXqYobzDvgqPG3atIKx4H79+tGvX79i3EoIURWcmSzrmcRgJsu+z8yZ9oLvgWCz2ry5ts7w9Zxp\n7Fiz/NGzz5p0cX/8JUi4lDZRQmsYOdJsH/Lcc8W/3pXJd8UVZ44dOmQmLv/8c9HXt2sHb75pgtP5\n55tgHMiqVabHtXZt4aCYnJxMcnIygM/JzsUS7FggsAYY7vzzjcCGAGVbAEe9jilgB3CV8/Uk4F0f\n18ozKCGqiQMHDujY2Kd9PiOKi5up09PTC8pefrnWy5aVfRt69tR61qzAZYq6d3Ky1uedV/I2zJih\ndYsWWh88WLLrR47Uevp0z2MzZ2p99dXFq2fmTK2bNtU6I8N/md9+0zoyUus33ii6vtI+gypOZ3ks\n8JpSahrm2dEIAKXUY0Ca1nqh8/XPmFT0cKVUKrBWaz1ca62VUrcBC5VSoZie0+0lCapCiHODa+O+\noibL5uWZybAl6UEVZexYmD7dLDbra+HxYO7dubPZ9iIz039P7PBhOHWq8PH1683itV9/DQ0alOw9\ntGplniO5OBzmudAzzxSvnilTYPNm8xzszTcLfx45OSbrb+xYs0pHuStJVCvPH6QHJUS1Mn78Mzos\nbK1H7yksbK0eP/6ZgjI//qh1eHjJMs2KkpOjdePGWr/4ou/zwd77vPNMT8qX994zmX6+eoqhoVp/\n8EHp3sNbb5lsOpeVK7WOjdXabi9+XSdPat2jh++2gtZDh5qsxmBUZA9KCCHKXFFJDGAexHfu7LuH\n4y3YvZ1cQkPhnXegXz+48EKzoKy7YO/tSpS49lrP4z/9ZJ4vffQRXH990e0vCe/VJObNMxOWS7LC\nea1apldXGUiAEkKcVYGSGFyCSZAo7t5O7q64wiQn3HQTbNjgubJCsMkZXbuaYTp3Bw+azLx//rP8\nghOYAHX4sNlI8dgxk8Tw0kvld7+KIgFKCFEpBNpLKSXFLAIbSHH3dvI2apR5/jJkCHzzzZmVFYK5\nN5gelHsGnt1uUsa7dTMBqjxFR0NUlOlFLV8OAwaYhWyrOtluQwhRqeXkmK3cA/ViSrK3ky9z5kDD\nhnDHHSbRICfHBK1gelCdOplJsRkZ5mnNPfeYCbSLF5fPJFZvrVrBb7+ZNfESE8v/fhVBelBCiEpt\n82bTm2nRwvO4+7OmYPZ2Cman25AQs67epZfCo4/CoEEQEVH43r5ERJg9qVJSYM8e+OQTMzzovjV9\neWrVysznql278HOwqkoClBCiUvvuu9MkJOSTl6ewWq0+nzX179+Y8PC0oPd2CiQmxqys0K3bmQm6\nwfaAXMN833wDn31WscNsrVqZ4Pr008Elk1QFEqCEECX244/mobx35ltZcAWiV1+9kvx8BwkJbzFw\nYDxaa+bPv7TQs6Z27eYRFhbc3k5Fad8eXn/drNI9bVrw13XtCg88YFYXL+v9oYrSqpXZq2rkyIq9\nb3lS2sw9qjSUUhGAzWazERERcbabI4QIoGdP08tYv96kYpelCRNm8fLLXcnN7VVwLDT0M2rVeo1j\nx94oVP788+9hwIBmrFq1v1C6elFZfP68/755Xy1bBlf+4EGTQXfHHSW6Xamkp8OaNfD3v1f8vf3J\nysoiMjISIFJrnVXc6yVACSFK5JdfTI9h/HhYtswEqqIe8wQ7R8lut5OQMJHt21/wOpNOjRovkp//\nRKFr4uKeZtOm4URHRxe5t5OoGKUNUDLEJ4QokfnzYdgweOIJ2LfPLI+zdq3v7diLO0fJf9JDDLDH\nZ3tcz5oCpauLqkUClBCi2I4fNzu1rl5tEgheecXsYDt2rO+06uLOUfK/Rp+V8PAj5OSsJSfnzIOv\nkj5rEpXbOZLrIYSoSG++aR7KX3aZeR0WBh9+CMnJJovNbreTnp6O3W4v0Rwlq9VK//4tgNMex8PC\n1jFiRC8SE1No0+Ze4uKepk2b+0hM3OixNJI4N0gPSghRLFqb4b3ERM+eUuPG8MEHDq666jSzZ79C\nfv4JIiL2cdVVUWRlNfVZV6A5SjfeOJ5XXsmhadPxHD/e1C3pYTIWiyXg0kji3CABSghRLBs2wM6d\nvrdbeO+9OcAg0tLuAcxQ3t69JvPOl0BzlD7/3MJf/lKb116b4zMQybOmc58M8QkhimXePJNGXbeu\n76G8vLx2HuVzc/ty7NhrgMPjeFHPjVavNisiuAKR9JKqH+lBCSGCduSI2Zri++8dTJjgmZUXaCiv\nfv1FXHONne++O0pa2t3Uq7eev/99r9/nRkePmrT1998vz3cjKjsJUEKIoL32mlnO57XXCmflBRrK\ni47eytKlcwCYN+8UCxfeyJw5Fr9LCK1ZAxdcAE2alMObEFWGDPEJIYLicJjkiNGjT/vMysvN7Qsc\nISxsrcdx96E8q9XK3XdHkp5uCbgpnmt4T1RvEqCEEEFZu9asu9ez5yG/K4eHhfXkjju+CJgCXqsW\njBhhnmX5orUEKGHIEJ8Qokh2u51nn81n+HArjRv7m0QLEREHeOEFM5QXKAV8zBjo0MGsXdewoee5\nXbvgwAGzzp+o3iRACSH8ci1R9NFHx/jjj8fYunUG+fl1uO66ZqSmBl45PFAKeNu2Zpv1xYth6lTP\nc6tXw5VXVtw+SqLyksVihThHZWebBV27dSt5HRMmzHImQ/QqOBYWto6xY79HKQtJSXtLvHL4++/D\n5Mmmx+R+yV/+YtoczDbronKT1cyFED4tXAgPP2wy7Eqy24T/FcWhTZt72bJlLhB4KC+QvDxo3tzs\nnTRgwJljMTHwxRfQsWPx2ywql9IGKEmSEOIclZIChw6ZLdNLIjMzk6NHL/B5zrVEUWkm0YaEwKhR\nnskS339v1vW7+OKStVmcWyRACXGO2rgR6tQxz3RKom7dGLKzB/g8V9xt1P0ZPdps8Jeaal6vXg19\n+547W5aL0pG/BkKcg06dMs+fxo4tWYDSGu6/30pkZC1CQ7/0OFeWW1s0a2aG9xYuNK8lvVy4kyw+\nIc5BmzdDVJTpoXToACdPFi8r7rnnzNYZGzc24tlnXycp6d1CyRBlJTHRzIu67z7T61u+vMyqFlVc\n0EkSSqnWwFKgPnAMGKG13uaj3CBgFqZ3tsVZ7rjz3HBgIpCPWTnyYa31p17XS5KEEKX04ovwyScO\nliz5k+7dY5k/X9G/f3DXrl5tdsf93//O7Pdkt5ff1hYOB5x/vllCaetW2LKlTKsXZ1FFJkksAOZr\nrdsCz2CClQelVB1gETDYWS4deMR5rh7wPNBHa90RuB94rbgNFkIE5nA4mDfvF1JSkunceRnHjn3D\nQw+l4HA4irz211/tDBvm4IUXThcEJyjfFcUtFjNx9913ZXhPeAoqQCmlGgCdgTcAtNYfAM2UUud5\nFR0AbNJa73C+fhm41etekc7/1gP2lbDdQgg/Jk2aw7ZtLTlyZAAZGVOw2Xrw448XMGmSWeHBfYsM\nF4fDwfjxs+jU6TB5eSnMnDmeCRNmBRXUysLIkWC1SoASnoJ9BtUMSNdau/9tTQXigT/cjsUDe91e\n7wFilVIWrXWmUioR2KSUygRqAdeUuOVCVDN2u53MzExiYmL89mTsdjuffJKB1nU8jmtdl//+9yj5\n+TNZtSqtYIsM1/OkSZPm8NJL15KX1xi7vTHbt19Kauo6YA5z504u9/fWoIEZ2jv//HK/lahCKixJ\nwvls6QGgi9Z6u/NZ1X+VUhdorU9XVDuEqGpcyw25773kb9WGzMxMjhzp7LOetLQmLFjQjtzcqwEz\ngTc1dR35+U+zatUB8vI8Jx/l5PQiKel9Zs60V8hmgW3alPstRBUTbIDaB8Q5e0KuXlQ8phflLhXo\n6/a6Jc6el1KqL3BUa70dQGu9Qim1GGgO7PK+4bRp0wr+p+jXrx/9+vUL9j0JcU6ZNKnw3kv+ejcx\nMTHUqBHmo5Z88vI6oPWVHkdzcnqxYsVSsrO7+ry3a0KubK0ugpWcnExycjKAxzBySRQni28NsFRr\nvVQpdSMwRWt9qVeZusBOoKezl/QCcEprPUUp1RH4FLhYa/2nUqobsAKI01rb3eqQLD4hnIJZbsi7\nd3PBBVvZtasWp0+3LDgWErKOvLwuQN1C9cTGPkFOziUcOzbQxz3uY8uWObLduiiR0mbxFWeIbyzw\nmlJqGmADRgAopR4D0rTWC7XWx5VSo4CPlFI1gF+A4QBa6x+VUk8Aa5RSduA0cJN7cBJCeMrMzPS7\n95K/3k1+fjuuv/59fv11TsHcpb59mzJv3mX4ynkIDz9IdnZPQkI2k5fXoeB4WU7IFaIkZLFYISqx\nwD2owr2bo0chOtqswRcR4Tl3qXXrHezda+H06VYF5cPC1tGnj41ffx3MkCFz+PTTPSVenVwIbxXZ\ngxJCVDCr1crAgfFF7r3ksmkTtGgB9esDWD16V+PHt+Lpp/dRq9a9HkFo8+ZJjBmjmDp1Es88U34T\ncoUoLulBCVHJORwOJk6cw8sv30ZIyB6aNHnTb+9m5kz44Qd4773C9ezYARddBH/+aScnxwShPXus\ndOhgFmv13tlWiNKSHpQQ5ziLxcLgwZN56SVNaGgjUlI6Ex7uu3eTkgKXXurzFK1bQ+PG8N13Vvr3\nNz2r+fPNskYSnERlJIPLQlQB8+bBAw8oYmIsrFjhf+ht40azpp0vSpmVGlyrm586Ba+9ZhZrFaIy\nkgAlRCWXkQEffWS2zhg71nODP3cHD5qhuk6d/NflHqDeecf0qK64ouzbLERZkAAlRCX36qvQqxe0\namW2pdi40ez15O2HH8xqDFFR/uu6+mr47TdISzPDe2PHmp6VEJWRBCghKrH8fLOZn2sYLjoahg0z\nwcXFtfjrd9+d9ju851KvHnTtCs88Y9a+u/328mu7EKUlSRJCVGIrV5r9kgYNOnMsMdEM1T35pINH\nHz2zRl929gC6dDmEw9Er4Nyla6+Fxx83mxlGRvotJsRZJz0oIcrRzp0we3bJr583zwSSmm7/lLzs\nMjPcN3ToZ8yb15Xt218gI2MKJ04k8O239Qq21fDHtaWFJEeIyk4ClBDlaO1amD4dcnKKf+3u3fD5\n5zBqlOdxpWD06NOsX3+hx+RdALu9I0lJewMu0tmtm+mZdexY/DYJUZEkQAlRjlJTTXBav7741y5Y\nANdfbzLtvF177WHsdt+Tl1xr9PljscCAAcVvjxAVTQKUEOVo3z7T43GldgcrN9dk7/kbhouPjyYi\n4nuf58LD9xMdHV3MlgpR+UiAEqIcpaZ6zj1y8bXturvly03G3dVX+67XarUyePAuIN/juKxALs4l\nEqCEKEepqTBypEnpzsgw6+pNmDCLhISJdOq0jISEiUyYMAuH1z4Y8+aZOUqBFhJ/7bXhNG6cToMG\ny1HqFM2azSYxcSOzZ08s53clRMWQNHMhyonDYYb4OnUyP59/Dps2Fb077i+/mMm4H34YuH6LxcKs\nWU158MEm1KgBv/xyPxER0nMS5w7pQQlRTg4eBLsdmjY1w3yrVuWTlJRaKPMuJ6eXR+bd/PlmMm5M\nTNH3uOEGAEWHDkqCkzjnSA9KiHLi2sKiVi0ToG66CSyWwLvj1q0by7JlkJwc3D1CQ+H+++HkyTJs\nuBCVhAQoIcpJairEx5s/d+sGJ09aqF8/32dZV+bda6/BeeeZybjBevjh0rdViMpIhviEKCepqdC0\nqYP09HTATu/eiubNLycsbJ1HOVfmXUiI1blyxGkyMvxn+AlRXciOukKUA4fDQZcum9i9O5uwsI1E\nROyjceO/EBLSm4sumkNS0l6Pbddnz57I999Dr155xMdP4/jxRkRE7PO7c64QVUFpd9SVACVEOZgw\nYRbPPTcCh6NBwTGrdQP5+R3Jzg6hRg07R46Ybdddc5bat/+F33+vR35+k4JrwsLWkZi4sSDDT4iq\npLQBSv5ZJkQZs9vtJCWlegQnc/xSlMpm7do8rFYrsbGxBcEpI8POtm1tPIITFM7wE6I6kQAlhA9F\nrfQQSGZmJllZvrP1QkL2sWJF4Trnz8+hZs0/fV5T1Np6QpyrJEAJ4SbYlR68uQe0mJgY6tbN8Fku\nKuoH1q+v5XVPeOONcGJi/ufzGllbT1RXkmYuhJtJk4pe6cGdw+Fg0qQzmwa6Eht69GjHzp0O3P8N\nGBa2jiFDTrBwoYX0dIiLM8fXrgWbTXHLLUd45ZV1HhN5ZW09UZ1JgBLCyfXsyBWcXMxzoPeZOdNe\nKFD4C2jXXZdBVJSNhg2nF8rW++EHs+yRa7v1efPgzjvhyScfpGbNOSQlvV/oGiGqo6ADlFKqNbAU\nqA8cA0Zorbf5KDcImIX5p+MWZ7njznNRwItAV8AOfKK1nlbaNyFEWQj07Mj1HCg2NrbgWKCA9vXX\nr3PJJZEkJ88tlK3nWt389tvhwAH45BP47Teztt7cuZOZObNwhp8Q1VFxnkEtAOZrrdsCz2CClQel\nVB1gETDYWS4deMStyGLgB611W611AvB/JW65EGUsJiaGiIh9Ps/5eg4UKKCdONGCRo1yCmXrgQlQ\nn30GWsOiRdCnD7RseeZaX9cIUR0FFaCUUg2AzsAbAFrrD4BmSqnzvIoOADZprXc4X78M3OqsozXQ\nWWv9rKuw1vpg6ZovRNmxWq0MHBjvd6UH74ARKKBZLDVo1SrU57nLL4fjx+HHH2HhQv+bEgpR3QU7\nxNcMSNdau6cypQLxwB9ux+KBvW6v9wCxSikL0A5IU0rNB7oAh4GpWuufSth2Icqced4T3HMgV0BL\nTS2c2BAZ2ZKWLWv4vIfVCr17w4MPQo0acN115fRmhKjiKjJJoiZwKSYojVVK9QdWKKWaa619r6Ap\nRDmy2+1kZmYSExNT0Dsq6jmQ9zX+AlpS0lUFC8X6cu21ZhXyf//bBCkhRGFBLXXkHOLbAUS7elFK\nqXSgh9b6D7dyNwJ3aa0HOF9fCKzSWscrpToDH2itW7iVPwhc7lVHBGC75557Cr4Y+vXrR79+/Ur9\nZoUA/6nhgda8K+oau/1MQAsJsVK7thnCu+AC323YsQM6doSdO8Et70KIKi85OZlk534xdrudl156\nCUq41BFa66B+gDXAcOefbwQ2+ChTF8gA2jhfvwA843Z+M5Dg/POlwEEgxKuOCEDbbDYtRHkYP/4Z\nHRa2Vps0BfMTFrZWjx//TJlcc/CgOX/8eOB2FHVeiKrOZrNpQAMROshY4/5TnCy+scAYpdTvwBRg\nBIBS6jGl1N3OYHccGAV8pJTaDjQBZrjVMRx4RSn1kzN4DdVa5xWjDUKUypnU8F4exwOteVfca1JT\nzW64deoEbktR54Wo7oJ+BqW13g5093H8X16vVwAr/NTxI3B5MdsoRJkp7lynklzjvlGhEKLkZC0+\nUa0Ud65TSa6RACVE2ZAAJaqV4s51Ksk1EqCEKBuyFp845wWbGh5ozbviXJOaCpddVn7vR4jqQnbU\nFeeMffvMCg3t2pnXxUkND3ZZoWCuuewymDgRhg0rq3cmRNVU2h11pQclzhkPPQRJSbBxI5x3XtFb\nZ7jWvCuOYK6RIT4hyoY8gxLnBK3NAqzt28PgwZCZWfx08rKQm2sCoQQoIUpPApQ4J/zyC2Rnw6pV\nZiPAv//dgc0WODW8POzfDyEhsjqEEGVBApQ4J6xeDb16Qe3a8M47sGNHKHl5HXyWLc8t1FNToWlT\n8LNikhCiGOR/I1Hl2O120tPTPYbpVq82C7ACREfDJ58ojh/vTUjIrx7XlvcW6vL8SYiyI0kSosrw\nl5U3Y8ZEvvzSwnPPnSl74YXw7rsh3HTT+cTHzyQvT1XIFuoSoIQoOxKgRJXhLytv7973adBgGG3b\nepYfMsTC449befnlf5CcfIi2baPKfZdaCVBClB0Z4hNVQqAFW9eujeaaa/JRqvB1//gHXHGFYtKk\nhhWyhboEKCHKjgQoUSUEXrC1A5dfnu3z2ZRS8OKL8OWX8OuvPi8vUxKghCg7EqBElRBowdbTp+uT\nkvIaCQkT6dRpGQkJE5kwYRYOhwMwSRPDhsH8+eXbRq0lQAlRluQZlKgSXAu2pqau8xjmCwnZRkhI\nCMuWXUJOzoNA4RUjABIToV8/eOopqFu3fNp49CicPAnNfHf0hBDFJD0oUWXMnj2RxMSNtGlzL3Fx\nT9OmzX20anWa0NCdRa4YcdllZvmjt94qv/alpkJUFISHl989hKhOJECJKsNisTB37mS2bJnLpk3D\n2bx5DpmZF6LUUZ/l3VeMUMr0oubNM0Nx5UGG94QoWxKgxFnlK7GhqHOuBVt//93KyZMWYmK+91m3\n94oRf/sb7NwJGzaU7XtwkQAlRNmSACUqRH6++QJ3cTgcTJgwy2diQ6Bz7lavht69FYMGNQlqM8G6\ndeH220uXLLFvH5w44fucBCghypYkSYgK8d57cNdd8PXXcMklgbfCAAJuk+GyejVcfz3cc0/wmwkm\nJkLXrjBnjsnuK44//jDXdu4MK1dCTa//e1JToWPH4tUphPBPelCiQmzZYhZyHTIE9u/3P+l2xYo/\nWLFib5FJD6dOmblN115b+NnUli0mkFl8rNh60UXQpQssXVq89mdnm208broJDhyAyZMLl5EelBBl\nSwKUqBBbt8I//wndusGNN2pstuY+y9ls0dhsTXyec096WL8eGjWCNm3OnHc9mypqxYjERDPMF2yy\nhMNhhgbj4syk348+gv/8B5Ys8SwnAUqIsiUBSlSIrVvNZoKLF0Nubgg5Od18louMPEpkZJrPc+5J\nD67Vy30tb1SUG24wc5bWrAmu/L/+ZfabeucdM6zXqhW8+y7cey98840pk5dnelYSoIQoOxKgRLnL\nzYWdOzUxMX9Ss6adjz+2kJd3CTVrbvcoFxa2jkGDWjJoUPMikx7ct9cortBQuPNOk3JelHffheef\nh48/9nxm1acPzJwJQ4eaxIm0NLMHVFxcydokhChMkiREuXI4HIwevRStb2XQoKVERpotMlauHM81\n17SgadPnyc8/5SOxwX/SQ3q66dH06VPydt19N7RrZ3o9jRv7LvPjjyaQvfWW2b7D2733wubN8Ne/\nwr//DU2aFE6cEEKUnNLlNWuxhJRSEYDNZrMRERFxtpsjSmnChFm8+OJ15OW1LzgWFraOxMSNtG8/\nmSlTNCtXHqZjx8hCz47sdjtHjhwhOjra49yyZeZZ0Pe+pz8FbcAA6N4dpk8vfO7PP03G3rhxMHWq\n/zrsdhMod++Gli3hq69K1yYhziVZWVlERkYCRGqts4p7vfx7T5Qb1xYZ7sEJXBl57zNzpp3Nm60M\nG9aAhARfNViB2EJHt22DW28tffsSE2HECN+Bbvt2uOIKs11HIFYrfPCByQyU509ClK2gA5RSqjWw\nFKgPHANGaK23+Sg3CJiFeb61xVnuuFeZx4DpwCVa680lb76ozAJvkWEy8ubMiaVLFzh+3GcxnwYN\nMsNqpTVoELz8skmY8DZ0qAmCwSRhNGxoek6nT5e+TUKIM4rTg1oAzNdaL1NK3YAJVpe6F1BK1QEW\nAVdqrXcopV4AHgGmuJXpCnQB9pSy7aISstvtZGZmEhMTU7BFRkZG4XKujLyaNU0K99lgscAtt5RN\nXc19Z80LIUohqACllGoAdAb6AmitP1BKvaiUOk9r/Ydb0QHAJq31Dufrl4HVOAOUUqoW8CIwFFhf\nNm9BVAYOh4NJk+aQlJRKVlYzIiJMMkS/fs3Zvt2Be8Kor2WIhBDCW7A9qGZAutbafTG0VCAecA9Q\n8cBet9d7gFillMV57TPAS1rrNFWSCSyi0nDvKVmtVr9LFw0btpOaNfNp2fIBjh9vGnAZIiGEcFdh\nSRJKqb5Ac631fRV1T1H2fPWU+vdvzKefphUEJ5ecnF589tlOLrqoBt9/P8dnRp4QQvgTbIDaB8S5\n9YTA9JZSvcql4hwGdGqJs+ellOoNdFRK/QEooCmwUik1Rmud5H3DadOmFXyR9evXj379+gX9pkT5\n8dVT2rv3A2rUOOWzfHZ2W7p1y8VqrUVsbOGMPCHEuSU5OZnk5GQAn9voFEfQ86CUUmuApVrrpUqp\nG4EpWmvvJIm6wE6gp9Z6uzNJ4pTWeoqP+nYDg7XWW7yOyzyoSsput5OQMJHt21/wPkNIyG3k5b1b\n6Jrw8BQmT76E6dNlRoMQ1U1p50EVZ6mjscAYpdTvmKSHEWBSxpVSdwM408lHAR8ppbYDTYAZfurT\nmJ6UqCL8p41bCQmB0FDPxe3CwtYRFhZPQoIEJyFE8QX9zaG13g5093H8X16vVwArgqjvvGDvLSqH\nQGnjTZo0ZMCADaxatbxgeaIBA5ozf/5VPpcJEkKIosg/bUXQrFYrAwfGk5q6zmO/JrPIa3Pmzp3s\nsTxRaqqVefPgPPmniBCiBCRAiWIx6eH+F3J17ckEZouNtm1lAVUhRMnIV4cIyHu+k2v32pkzfS/k\n6m7rVt+rgAshRDAkQIlCQQj8rwwxe/ZELBaLR0/Jn61bzZYWQghREhKgqrFAQcjfyhAwh7lzJwes\n12XrVrj++vJrvxDi3CYB6hz0ww9w4gT07Bm4nL8glJ//NKtWHfC5MoRrm4zPP7dywQX+EyAcDrMt\nhjlaeikAABUXSURBVAzxCSFKSrZ8Pwc9/zw89VTgMq69mtyz8cAEoRUrtpOV1dTnddnZzUhLO8Jt\nt8FDD/mvPzXVbPV+/vnFbLwQQjhJD+oclJICBw+C1v73Mwq0V9PJk62pXXuPz3Ph4fv53//qU6cO\nLF9udp5t1Khwua1bTXCSZfeEECUlPahzTHa2GVrLzIS9e/2Xc0269SUiIoNBg5oTFrbO47hrm4xX\nXqnJ1Klw5ZWwZInv+iWDTwhRWtKDOsf8+CM0bgwNGpieVIsWvsv5m3Rbs+YfBYkSNWoUnu/0t79N\nZOFCs8lgbCxMmgSTJ0ONGp71S4ASQpRW0IvFVhRZLLZ05swx2483aADR0fD00/7LOhwO7rvvBV5+\n+QEaNnyBGjXCOH58GJmZ4YSEmM61+8oQVquVu+82w4YLFkBentlJdtEiuO46z7ovvxweeMBsmy6E\nqJ4qcrFYUQWkpEDXruYnJSVwWYvFwo03PkCzZpqff76JnTuHExERSXLymb8WrvlOVqsVmw3eeAPG\njjXnQkJg9GiYN8+zXq2lByWEKD0JUFWY3W4nPT3dY8+VjRvh4ovzaN78ECkpGocjcHkT0BSxsbHU\nrm31GXBcli2DhATo2PHMsdGjITnZ83lXWppJc2/TpqzeqRCiOpIAVQU5HA4mTJhFQsJEOnVaRkLC\nRCZMmMWhQ6fZtQsefPBhhg9fSnb2ae66axGnT5/2Wd7hcLBxI3TpcqbuUaPg889h927Pe2ptAldi\noufxpk3N8N4rr5w5tm2bmR9Vq1b5fQZCiGpAa12pfoAIQNtsNi18Gz/+GR0WtlabsGF+wsLW6lat\nZmilTnocDwn5VXfsOMxn+fHjn9EtW2q9erVn/UOHaj11quexL77Qul49rU+eLNyeVau0btRI69xc\n8/r//k/rwYPL570LIaoOm82mMXv/RegSxAPpQVUx/ifYdmf37vPQ2rPbkpfXll9+0T4n5H788WF2\n7/bsQYF5xvTqq2aircu8eTBypO9eUd++ULcu/Pe/5rU8fxJClAUJUFWM/wm2mTgcF/s4nk9+/gU+\n6zp6tDMtWpymXj3P4336QGSkmYgLZjLu8uUwZozvNlks5pzr2ZUsEiuEKAsSoKoA9+QG/xNsYwAf\nSzoQgsWy3We9StXi0ksLLzVhsZhelCvgLF5sJuUGSnoYORK+/dY8f/r1V+lBCSFKTwJUJeYrGWLq\n1Oe47rpmhVZ5CA3dCEQTGvqVdy20bt3U56oQUVEXcOmlXjNsnUaMMBmBW7bAwoWFkyO81a8PN90E\njz8OR4/CBb47bUIIETSZqFuJTZgwy7naeK+CY2Fh6xg79nuUspCUtLdglYd27Xrx229Due662R7H\njx//B489FsfWrXM9jg8cGM+7707ijTcUV13l+/7Dh8NPP8GhQyaNPCQkcHu/+QZ69DCTd/fsKbOP\nQQhRRZV2oq4EqErKbreTkDCR7dtfKHSuTZt72bJlLkDBKg9PPWVlxw54/XXP1R+mTrWSmwsvveR5\nPDPTSpMmYLNBeLjvNnz3HXTrBo88Ao89VnSbtYZLLoEmTWDlytK8eyHEuaC0AUrW4qukAq02np3d\njCNHjhAbG1uwq21KClxzjTnvvtttly7w3HOFj6ekmGE4f8EJ4LLL4NFHz6wcURSlzNJKJ04EV14I\nIQKRZ1CVhPcqD4FWGw8P3090dHTBa60pNOHWpUsXM0zntngEcGZJpECUgn/9y/d2Gv707w833BB8\neSGE8EcCVAXzDkT+VoWoWbMmAwfG+93ywuq20VJamnlOdMklhe/XurWZu/TLL57H/QU0IYSoLGSI\nr4I4HA4mTZpDUlIqWVnNiIjYx8CB8WitmT//0kLbrsMcZs+eCBTe8sIcP2PjRmjfHurUKXxfiwU6\ndzZlOnUyx7Q2Pajp08v1LQshRKlIkkQF8ZWRFxr6GbVqvcaxY28UKu9KhLBarYW2vPD20EOQnm7m\nK/kydarZwNC1Xl5qqlkrLztb1ssTQpSfCttuQynVWin1tVLqd6XU90opn2sFKKUGKaW2Ocu9r5Sq\n6zwep5Ra5Tz3k1L/397dB0dRpwkc/z4hmQkEWZGXA7OEtxXWFdz1CHJaB7oWgujJ4qooFmJk1dNC\nby0EV3OWt+qeuiqI72ChLuC6ImRFlNLInggqJURAlFvdeLwkKFEwCkQlDEme++PXA5PJzGQSEjLT\neT5VU6Z7ejL91IR57O6nn0eWiEi3pu5wOorXnujgwSFUVfWL+ZpwIQTUH3kRywcfJD5dl5/vjqDC\nSkpgyBBLTsaY1NaUa1DzgLmqOhh4AFgQvYGI5ADzgfHedhXAnd7TtcDdqnqyqv4C2A48dDQ7ny7i\nV+R1A3bEfE10IUQ84dN1iQoe8vPdNagDB9xyMgUSxhjT1pJKUCLSAxgG/BlAVYuAPiIyIGrTccBG\nVf3MW34SmOS9Zreqro3Ydh3Q9yj2PaUtWAAPPOB+jl+RF+C4474hO3tVvbWxCiHi2b7dnao79dT4\n2/TtC127wubNbtkKJIwx6SDZIok+QIWqRoy/oxzIA7ZFrMsDIkbXsQPoJSIZka8VkQzgRuDl5ux0\nOnjuOZcQbrwROnUKcMEFeZSXv92gK0RBwdmIfMCKFUUJCyHiKSlxySkYjL+NyJHTfCNGuCOoRKPg\njTEmFbRVFd9TwDeq+mgbvX+rqqpybX969oTFi10j1fgVeTPJyMjg/vsTF0LE09j1p7DwCPitW92p\nvqFDmx+fMcYcC8kmqJ1A76gjoTzcUVSkcuDciOX+RB15icijwInAhERvWFhYePiLeuzYsYwdOzbJ\nXW17q1dDXp4ydep+nnzyOK6+OoOMjAxmz54ZNxFFdnloipISmDy58e3y82Hp0vBIeGhCDjTGmKQV\nFxdTXFwMcPh+z+ZKusxcRN4CFqjqAhG5BLhVVU+P2qYz8H/AKFUtFZHHgAOqeqv3/KPAQGCCqh6K\n8z5pXWZeV1fHsGGb2L79e4LBjezZcwOTJi1m0aLJZGS07H3RdXVubtO777qkk8iuXW48+zXXuKav\nTzzRortijDENHLMyc+B64N9F5B/ArUABgIjcJSLXAajqd8A1wCsiUgrkAvd4250JTAP6AetFZJOI\nFDV1h1PdjBmz2Lz5ZPbtG8Xu3TejGmTx4nOZMWNWi79XaSnU1CQ3e+nEE6F3b/jLX6xAwhiTHpK+\nBqWqpcCZMdb/V9Tya8BrMbZbC8QePpTGQqEQlZWVdOvmbulatqwK1U71tqmt7c3y5bu5//5Qk64v\nNaakxLU3amwMRlh+PixfbgnKGJMerNVRM8VqXXTWWcdTWdkghwOwZ8+Ywx3Im2rzZtcpItry5U27\nn2n4cPjb32wcuzEmPViCaqYZM2Z5rYuO9NArK1tJXV3s+RXV1UPo2rXxG2+jFRfDRRdB//4NnxOB\n665L/nddcAHs3g2Z9qkbY9KAfVU1w5HWRTPrrT948FxE9hMIbCAUGnZ4fTC4BpHhvP9+IO702lhK\nS+Hyy93I9WQq9Rpz2mnuYYwx6cASVDMkGiYIWUyZ8hpr1jxX734nGMlTT5F0gtq3D8aPh2uvbZnk\nZIwx6cYSVJIiiyHCrYu+/LLhdjk5n/LEE7cD1Lvf6bPPXIPWr75qfABgbS1MmuQ6jt93XysEY4wx\nacASVCPizXE6//w+DVoXiexlxIjdBALuPFpkQcRJJ8GoUfDMM1BYmPg9b78dtm2Ddeugg+/qHo0x\nJjk2D6oRseY4ZWe/zfXXr0MkgxUryqiq6kNOzh62bfsjpaXCwIGxby/7619h+nTXbihe4nn+ebjp\nJpecBg1qhYCMMeYYOdobdS1BJRAKhRg69BZKSx9r8Fx4oCC4U3lr13bjttuyKC2N//tqalxn8aef\ndhV10davh3POgaIiSKPOTsYYE9PRJqh2eYrv00/dkcrdd7uR6PEkKoYIDxTs1asXvXr1YtUqGDMm\n8ftmZrqih2nTYrcaKimBe+6x5GSMMdBOE9R998HChS5h/P73R9ZHFkIEAoGExRDRAwVXrjwy/ymR\nmTMhLw8OxehE+JvfwK9/3fR4jDHGj9pdgqqshJdeghdfdEczQ4fCRRfFLoR46KFb4s5xihwoWFbm\nriudfXbs94yUkwNTp7ZObMYY4yftLkEtWAD5+XWMGvUVzz7bnYKCLJYtW8jSpfW7QpSXvw3MSjDH\n6chAwZUr4YwzIAVqOowxxjfaVZFETU0dPXrsJTt7ObCbLl120q3bJZSUDKGmpluD7cOFEIFAgFAo\n/kDBiRPdVNs77mjR3TXGmLRmRRJNcNllS9m7dwLepBC+/BICgSJUdwENE1RkIUS8gYK1ta4B6y3J\nTWg3xhiTpJadoJfCQqEQb745CAhErb8QkT/GfE10IUQsGzaAqo2wMMaYlubrBBUKhaioqCAUCrFl\ny7d8993QGFsFCARCBALv1VsbXQgRz5tvwujR1vHBGGNami9P8cVqT9S160Q6dfqaH344pcH2ubk9\nGTfuHV5+eR2ff34jublzufTSg/UKIWL5+muYP9/dT2WMMaZl+bJIIlZ7IjjIgAFz2LVrRIOS8Rtu\nKGH27JmEQiEef/wA997bhfXrhQED4r/HoUPuxtzu3V3ZukizdtUYY3zLiiRoOHY91qwmCJKRsZPr\nrlPeeCN2yXggEGD69ABlZfCrX8HatXBc7PmD3HwzfPstvPaaJSdjjGkNaZ2g4o1d37//xzG3//77\nPtx++1U8+OAJcUvGAWbNgnHjYMoU1xcvuh3SvHmwZIlrTZST0xqRGWOMSasEFd2KKN7Y9Y4d/xTz\n9eGqvHgl42GZmbB4MZx+umuFFHmNac0a15H8jTdc41djjDGtIy0SVKwjpfPOO5HXX/8i5tj1jh1n\nk529iurqXx5en2xVXtgJJ8Arr8CZZ7p2SJdeCjt2wMUXw5w5MHJkS0ZojDEmWlokqNhHSkV06HAg\n5vbZ2aOYOHE1b731Clu33ktu7vykqvKinXKK63p+xRXQu7frQn7ZZa6HnzHGmNaVsgkq3O07FArF\nLHo4ePBCsrIWx3xtdfUZZGaOZMgQAerYsuV6gsHkjpyiXXihm4A7apRrBvvww836NcYYY5ooZW/U\n/d3v3H/jz2QKkJUFmZkf1VvboUM5Awd2IienAyedlMHChZnNTk5ht912pDAiK+uofpUxxpgkpewR\n1LJlSn4+TJ0afyZT9+4/p6JiML17zwcq65WNJxpE2FQidlrPGGOOtZRNUMHgU9x003QGD86MOZMp\nGHyXvXunUVgYpLBwSsKycWOMMekn6U4SIvITYAHQHdgLFKjqJzG2+zfgQdzpw4+97b7znhsBzAOy\ngc+BK1W1Iur1XYB9sI/MzAqysvqwZUs2jz8+ixUryqiq6kPnzruorp7OsGF5FBVJix4tGWOMaRlH\n20miKV/t84C5qjoYeACXrOoRkRxgPjDe264CuNN7ToDngf9Q1Z8CrwOPJHrDmprBBAIlTJgAd989\nk48/ns3GjVcxevRsjj++L4sW+Ss5FRcXt/UutAmLu32xuE2ykvp6F5EewDDgzwCqWgT0EZHobnXj\ngI2q+pm3/CQwyft5GHBIVdd4y/OAC0Uk4Tm5jh3X06XLIQoKIDMzwKuv9mLJkg4sXw6dOyez9+mj\nvf4BW9zti8VtkpXs8UcfoEJV6yLWlQN5UdvlAWURyzuAXiKSEf2cd9pvH3Biojfu0qWcJUuEjRth\n8mTXA2/pUujXL8k9N8YYk5baukgiQZvV/QSD7zB6dE9ycqp54YVqxoyBP/wBTjsN9jf5bGbqC4VC\n7PdjYI2wuNsXi7v9OOp4VbXRB9ADVxiREbGuAhgQtd0lwOsRyz8Dyr2f84FPIp7rDBwAAlG/IxdQ\ne9jDHvawh28eucnkmuhHUkdQqrpHRDYCVwILROQSYKeqbova9A3gcREZpKqlwA3Ai95zG4BMETlL\nVVcD1wOvqmoo6nfsAn4MVCWzb8YYY1Lacbjv9SZrSpn5IOBPQDfctaMCVf27iNwFfKGqT3vbhcvM\nOwBbgKtUtcp7bgTwNBD0dvhKVf2iOTtujDHG31Juoq4xxhgDKdiLT0R+IiLvicg/RGSdiJzc1vvU\n0kTkERHZLiJ1InJqxPoeIvK6iJSKyEci4quhHiISFJGXReRTEdkkIsUiMtB7zu+xF4vIh17cq0Xk\nF956X8cdJiJXe3/v471lX8ctIjtE5BPv894oIpd66/0ed0BEHvPi2ywiC731zYu7OReuWvMB/A/u\n1B/AxcD6tt6nVojxX3Hl9duAUyPWPwPcGVFUshPo0Nb724JxB4HzIpanAau8n5/1eexdIn6eAHzY\nHuL24uoLvOc9xnvr/P63vg0YGmO93+N+GHgkYrnn0cTd5gFFBZdUtaBfHsD2qARVFf5AveX3gXPa\nej9bMf5hwLb2FjtQAGxoD3HjbiVZCZwGrIpIUH6Pu96/7Yj1vo0b6ISrT+jcUnG39X1Q0RLdEBxd\nMegrInICkKmquyNWl9HwZmg/+S2wrL3ELiILgF/iym7PbydxTwfeUdVNrttZu/pbX+TFvB64Dfe5\n+znugcA3wH+KyGjgB+Au4EOaGXfKXYMy7YOIFOL+oAvbel+OFVW9SlXzgDtw/Swh4c3q6U1ETsGd\npv/vtt6XNjBSVX8O/DNQyZHepb79vHGNH/oCW1R1OO5/QF/01jcr7lRLUDuB3l5rpLA83FGUr6nq\nN0CNiPSMWN0PH8YuIjNw12HOU9Xq9hQ7gKouAs72Fg/5OO6RuC+sz0RkO/AvuNtMJuLzz1tVP/f+\nWwvMwSUsv/+dlwO1wAsAqvohrt3dUJr5d55SCUpV9wDhG4JJcEOwXy3B3dyMiAzHFVKsbtM9amEi\nMh24HDhXvfvjPL6NXUR+JCK9I5YnAJXeF5Zv41bVuaqaq6oDVLU/7rrDtao6Fx/HLSKdRORHEauu\nADZ5P7+ET+NW1Upckdt5ACLSH5eI/k4zP++Uuw8qxg3BV6vq/7bpTrUwEZkLXAD8E+7wv0pVB3n/\nh7EI6A8cBKbpke7vaU9EcnFHyVtxF00FqFbVM/wcu4jk4f6BZuOuQ+wGZqjqR36OO5qIvAXMUdXl\nfo7b+2Iuwh0ACO76+W9VtdzPccPh2J/BzQ2sBe5S1WXNjTvlEpQxxhgDKXaKzxhjjAmzBGWMMSYl\nWYIyxhiTkixBGWOMSUmWoIwxxqQkS1DGGGNSkiUoY4wxKckSlDHGmJRkCcoYY0xK+n8Zd3iE5CSX\n5wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x151149898>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ0AAAEmCAYAAAC50k0UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAMTQAADE0B0s6tTgAAIABJREFUeJzt3Xl8VNX9//HXJ4EQgQRIEIOSACIoVVzQblgsVTRYLFp3\nrRtYFXBrCfJTqNblq8aKaGuVxX0Xre1XhUKKC1L3XVFRUdlJBBKy+IUwkJzfH3cSJpOZZJJMZpLJ\n+/l4zIOZe++ce88kzCf3nM85x5xziIiIxEJSvC9AREQ6DgUdERGJGQUdERGJGQUdERGJGQUdERGJ\nGQUdERGJGQUdERGJGQUdaTYze8jM/taE4yeY2UeteU2txcw2m9nprVj+W2Y23f88xcwqzOynDRx/\nu5ktauE5B5tZuZlltaScRs5xiZmtbK3ypf3pFO8LkNZlZhVAzQjgLkAysA0w//bjnXNvNKds59z4\nJh7/IPBgc87VkTjnfEBaJIdGWqaZ3Qoc7pw7LuA8K4H0pl9hk2kEutRS0ElwzrnaLy8zuwk40jl3\ndEPvMbMU/xefiEhUqXlNMLNbzewl/78bgTcDtq/0N/WsNrNZZpYS8L6nzGxewOtCM7vazAr87/nK\nzMYG7K/T1OJ//yNmNsfMis1so5n9KejaTjSzz/3NQIvN7GYzW9FAXQ41s1f8zWFbzexNM/tFwP79\nzazazH5nZsv95f7XzPYLOCbNzB71X9MaM5vYyOd3h5ktCNqWbWY7zewA/+uH/J9hhZl9Y2YzGiiv\ni/8aRwRs+73/fWVm9jRBd0JmdqX/cyozs/Vm9qCZ9fTvmwBMAUb5z19uZocHfBZ7B5Rzsb+cUjP7\n2MzObMpn1xgz28P/ea02sy3+n9XwgP3D/WWW+j//d8ysv3/fcWb2gX/fFjNbZmZdIz23tBHOOT06\nyAO4CXglxPZbAR8wFegMpPq3nwv09T8/CFgNXBvwvqeAeQGvC4FvgAPxmu+uAUoCyrsE+Dro/duA\nE/3HHwXsAn7q3z/Uf11n4P2BdCSwBfiigToeAvzKX48uwM1AMdDDv39/oBpYCGT6j3keWBRQxqPA\nW0AfoCvwuP86Tg9zzprr7Buw7c/A6wGvfw9k+J8fCZQC5wbsfwuY7n/exX+NI/yvjwYqgWP9n8PJ\n/tf/Dnj/qcBA//MBwAfAA0E/4/8EXff+QBWwt//17/yf1ZH+n8cY/8/n2Eg/uxCfTfDP/D7gfSDH\n/zOq+R3Z07//feAq//Mk4NCAz20zcIb/eSdgBJAS7/9XejTtoTsdqbHWOTfTObfTOVcJ4Jx7zDlX\n6H/+GTAHOK6hQoDZzrnPnffNMBfoCQxu4PhXnHPPO88y4AvgJ/59ZwNvOufmO+eqndf39FRDJ3fO\nfeKce9Vfjx3AtUAqcETQodc554r9xzxSc04z6wycBcxwzm1yzm0D/kADTdHOuRXAe8B4fxkGXID3\nBVtzzP3OuRL/8zeAp2n8s6xxAfCsc26J/3P4J/BS0DX8wzm3yv98NXB7E8qvcREwxzn3hv/nsRgv\nAE8KOi7kZ9cYM+sEnA9c7Zxb6/8Z3Qp8j/eZgxe8+5vZAH9dP6753Pz7BptZlnNul3PuTadm4HZH\nQUdqrA7eYGaXmdlH/maOrXhf4H0aKacw4Pn/+f9tqFN8Y9Dr/ws4fh9gTWPXGcjMBprZfH+zWCne\nX+5dqHvdLsR11pyzL97/i9rzOOe2BNQlnPvxBx28L/tewDP+azIz+7O/2WqrmZUA59H4Z1mjH7Aq\naFud12Z2hnkZcJv8P6v7mlB+jWzg26Bt3+DdldRo6LNrTF+8RJbvgrZ/G3COs4E9gNf8TXAzzSzV\nv+/XeHfRH5vZl8FNsdI+KOhIjerAF2Y2CvgLcCnQxznXC695zmJ4TRuA/kHbBjTynoeAHcBw51xP\nvGagHUR+3YV4n0XtecxsT6BbI++bD+xpZkcDFwJPO+e2+/ddgNfMdCZeU1EG3h1EpNe0nvr1Dry+\nfYEngXxgH//P6qKg8uv8fMNYBwwK2jYIWBvhdTamEK85L+w5nHOrnXMXOuf64zUnjgPy/Ps+cc6d\n5ZzLwgtOfzCz30Xp2iRGFHQknB54/StbnHNVZvZj6jeztLYngZ+b2elmluTvWD+zkff0ACqAMjPr\nDswEUoKOCftl75zbidf0dZOZ7RVQxq6GTupvhnsamAb8Bu/Op0Y6XtPQFgAzOxavnypSjwCnmtlo\n/+dwEjA6YH/NncYW51xN8sK0oDKKgAEWkAjiF/hZ3A9cYmZH+s9zHF6/3twwxzeJc24XXrC92cxy\nzBuPdDWQhffZYWbjzayv/y0VwE5gp5l1NbNzzSzTv68c72fS4M9F2h4FHQnnRbwvuzcDmtYeauQ9\nocZjNHWMRu3x/r6SM4Dr8Trer8Mb51PZwPsn43UwlwKfACvwOqCbck2X4jX5rAA+x8vmK43g2h/A\n++v8S+fc+wHb78NLFFgBbMJrWnuikWsK/BxeBq7A+/Lfihd4Hw7Y/wle4sI/zKwMr+/t0aDynsC7\nmyg0s5KAjLHA8zwOzPBfbwlev9BFzrmCBq6zqS4HXgf+i3fnkwsc45zb5N9/LPCBeePL3sfru7rT\nv+8s4AszKweW4PUfzm/h9UiMmdffK9I+mNlsIMs599t4X4uINF1Edzpm9lczW+XP0T84YPueZrbI\nzL42s0/NbGTrXap0RGb2GzPr5W/uOR44By+FWUTaoUib157Fy91fHbQ9H3jLOTcEmAA8aWbJ0bs8\nEY4CvgbKgFnANc655+J7SSLSXE1qXjOzVcCJzrlP/a8rgEE17bFm9jbeALdXWuNiRUSkfWt2IoGZ\nZQCdAjoAwRtTkRPmLSIi0sHFbMJP/yjtvfHSIEVEpH1LAza6JmajNTvoOOdKzGyXmfUJuNsZQPiB\nZHvjDXITEZHE0A9vEHfEWnqn8yzegMEb/IMH9wZeC3NsBcC6detIT4/FEh5tx/Tp07nlllvifRkx\np3p3LKp3x1FeXk52djY0o+UqoqBjZnOAscBeQIGZVfgz1q4GHjOzr/GmGvmdc66qobLS09M7XNBJ\nSUnpcHUG1bujUb0lEhEFHedcyPVE/M1quVG9IhERSViaBicGcnM7ZlxWvTsW1VsiEbNpcMwsHSgr\nKyvTrahIDFRWVuLzabkZab6UlBRSU1PrbS8vL6dHjx7gLY5Y3pQyY5YyLSKxU1lZycCBAykqKor3\npUg7lpWVxapVq0IGnuZS0BFJQD6fj6Kiog6ZLSrRUZOh5vP5FHREJDIdMVtU2jYlEoiISMwo6IiI\nSMwo6IiISMwo6IhIwho/fjxXXHFFxMc/+OCDHHbYYa14RTB37lwGDx7cqudoy5RIINJB+Xw+iouL\nyczMJCUlJW5lpaWl4U1CDzt27KCqqoquXbvinMPMWLRoEUceeWSzruuhhx5q0vETJkxgwoQJzTpX\nU9TUtyNS0BHpYKqrq5k69Q4WLlxLeXk26enrGDs2h5kz80hKalrjRzTKqqjYPWfktddeyxtvvMEr\nrzS8DqTP52txoJT4UPOaSAczdeodzJ79Y77++m6Kiqbx9dd3M3v2j5k69Y64ltWQa665htGjR3PN\nNdew9957M2LEiNrtgwcPJi0tjQEDBjBlypQ6szCcddZZXHzxxbWv+/btS35+Prm5uaSlpbH//vuz\ncOHC2v3BTV9nnXUW559/PhMnTiQzM5O9996b//mf/6lzbc8//zwHHngg6enpjBkzhhkzZjB06NCI\n67Z9+3by8vIYMGAAvXv35uijj+bDDz+s3f/hhx8ycuRIevbsSWZmJj/96U9Zs2YNAP/5z384/PDD\n6dmzJ7179+aoo45i27ZtEZ87HhR0RDoQn8/HwoVrqawcVWd7ZeUoFi5c06Rpc6JZViSWLVtGZmYm\na9as4fXXXwfgRz/6EcuWLaOiooIFCxbwz3/+k9tuu63Bcu6//35mzZpFeXk5F1xwAeeeey6VlZW1\n+4Obvp599lmOP/54tmzZwtNPP83111/PO++8A8CKFSs47bTTuO666ygtLeXaa69l7ty5TWo+u+KK\nK3jttddYtmwZhYWFHHvssYwePZrNmzcDcPHFFzNu3DhKS0vZvHkzc+fOJS0tDYDf/e53TJs2jdLS\nUoqKisjPz6dTp7bdgKWgI9KBFBcXU16eHXJfRUU2JSUlcSkrEjk5OUydOpXOnTvXjpA/99xz6du3\nLwAHHXQQEydO5D//+U+D5UyaNIkDDzwQM+OSSy6htLSUlStXhj3+6KOP5sQTT8TMOOqoo/jRj37E\nu+++C8CTTz7JiBEjOOOMM0hKSuLII4/krLPOirhOu3bt4pFHHiE/P5+cnBw6d+7MNddcw1577cVT\nTz0FePOfrVmzhtWrV5OUlMShhx5KRkZG7b6VK1dSVFREp06dGDFiRJtvdlTQEelAMjMzSU9fF3Jf\nWtr62i+zWJcViQEDBtTb9ve//53DDjuMzMxMevXqxU033cSmTZvqvzlATZAC6NatG1C3XynY3nvv\nXed1t27dao/fsGED/fv3b/Q6wyksLKSqqop99923zvZBgwaxdq23CPOTTz7J9u3b+eUvf8mAAQOY\nOnVq7Z3Zv//9bz7//HMOPfRQDjjggHpNf22Rgo5IB5KSksLYsTmkpi6tsz01dSljx+Y06a/kaJYV\nieDEhKVLlzJt2jTuueceNm3axNatW7n22muJ1cz5APvss09t/0qN1atXR/z+vn37kpyczLfffltn\n+7fffktOTg7gBbEHHniANWvWsGTJEl544QXuuMPrMzvkkEN46qmnKCoq4sknn+Suu+7iiSeeaFml\nWpmCjkgHM3NmHpMmvceQIZfRt+9tDBlyOZMmvcfMmXlxLaupysrK6NSpE7179yY5OZn33nuP2bNn\nt/p5A5199tm89dZbPPPMM1RXV/Pmm2/y9NNPR/z+Tp06cd555zFjxgzWrl2Lz+cjPz+foqIizjzz\nTMBL+y4sLAS89PLOnTvTuXNntm3bxmOPPUZxcTHgzbPXqVOnNt+n07avTkSiLikpiVmzriI/30dJ\nSQkZGRnNviuJZllN9Zvf/Ibzzz+fESNGUFVVxciRIxk/fjxPPvlk2PeE6uBv6piZwOOHDh3K/Pnz\nmT59Or///e8ZMWIEEyZMYMmSJRGXd/fddzNjxgxGjhzJDz/8wMEHH8zLL79Mnz59AFiyZAkzZsyg\noqKCHj16cMopp/DHP/6RnTt38tRTTzF16lS2b99OZmYmkyZN4owzzmhSfWJNi7iJJKCaRbb0/y32\nJk2aRFFREf/617/ifSkt0tDvUEsWcVPzmohIC7z44ots3bqV6upqFi1axOOPP84555wT78tqs9S8\nJiLSAsuWLWPChAlUVlbSr18/br31Vk455ZR4X1abpeY1kQSk5jVpKTWviYhIu6egIyIiMaOgIyIi\nMaOgIyIiMaOgIyIiMaOgIyIJ4YEHHmDgwIG1ry+66CImT54c9vivvvqKpKQkNm7c2KLz5ubmkp+f\n36IyGlJVVUVSUhLLli1rtXPEksbpiEhcnXzyyTjnQo7gv/rqq1mwYAGfffZZRGUFTlFz3333Nen4\nxnz77bcMHjyY1atX107GCVBQUBBxGaI7HRGJs0mTJrFw4cJ6dxw7d+7koYceavBuJZacc02ep03q\nU9ARkbg69thjGThwYL07k2effZbt27dz3nnnATB//nwOP/xwevXqxV577cVvf/vb2jVnQjn33HOZ\nMGFC7etvvvmGo48+mh49ejBs2LB6zVWffvopxxxzDH369KFXr178/Oc/rz2murqaQw89FKB2aeor\nrrgCgJEjR3LjjTfWlrNixQrGjBlD7969ycnJYfLkyXXW6xk5ciRTpkzhrLPOokePHvTv3z+iu7JA\nL7zwQu0y1UOHDuWuu+6q3efz+bjkkkvIysqiR48eDBo0iLlz5wJQWlrKGWecwZ577kmPHj0YOnQo\nzz//fJPO3VIKOiISdxMnTuT++++nurq6dtvcuXM555xz6N69OwA9evTgkUceYevWrXz++efs2rUr\n4jnOqqqqOOGEExg8eDCbNm1i0aJFzJs3r84xZsaMGTNYv34933//PccddxwnnngiW7duJSkpiU8+\n+QSAL774gvLycv72t7/VO095eTnHHHMMw4cPZ+PGjbz77rssX768TvADeOSRR5g8eTJlZWXcfvvt\nTJ48ud66POG8/fbbnHbaafzpT3+ipKSExx57jNtuu417770XgAcffJCPPvqIr776irKyMt58801+\n9rOfAZCfn8+OHTtYu3YtZWVlFBQUcMABBzR4vigvABudoGNmY8zsPTP72MzeNLODo1GuiLQe56C8\nPLqP5s6qNX78eEpKSnjhhRcA+Pzzz/nvf/9bp2ltzJgxHHTQQQD07t2b6667jjfffLN2Fc2GvP76\n63z33XfceeeddOnShX79+jFjxow6xwwbNoyjjz6alJQUUlJSuP7669m1a1ft0tS7P7fwlay5a7j5\n5ptJSUkhKyuLO++8k+eee67O8t2nnXYaI0eOBOD0008nLS2NDz/8sNF6ANx///2cdNJJ/Pa3vyUp\nKYkjjjiCvLw85syZA3iL61VUVLB8+XJ27drFXnvtxSGHHFK7r7i4mBUrVuCcIycnh/3337/B80V5\nAdiWBx0z6wk8DpzrnDsUmAa07aXrRISKCujRI7qPBlZ9blDPnj0544wzar8458yZw4gRI2qDDMAr\nr7zCMcccQ9++fenZsyejR4/GOcfmzZsbLX/Dhg307t2brl271m4LzHQDWLNmDWeeeSb9+/enZ8+e\nZGRksH379kaXvw60fv16BgwYUKfvZ7/99gOo0xTY0BLYjVm3bh2DBg2qs22//farLf/888/n97//\nPXl5efTu3ZsTTjiBjz/+GIBrrrmGX/3qV4wfP54999yTM888k1WrVkVcv2iIxp3OIGCLc+5LAOfc\n60COmR0ahbJFpJWkpUFZWXQfaWnNv57Jkyfz0ksv8emnn/L4449z6aWX1u7bsWMH48aNY9y4cXz3\n3XeUlpby8ssvAw3fedTo168fW7ZsYdu2bbXbgr9sL7zwQpKTk/nggw8oLS2lpKSErl271paflJTU\n6Lmys7NZs2ZNneO++eYbzKxOxltLZGdn11ve+ptvvqktPzk5mby8PN555x02bNjAfvvtx0knnQTA\nHnvswY033sgnn3zCypUrqa6urtf019qiEXRWAplm9jMAMxsHdAcGRKFsEWklZpCeHt1HS5K7jjji\nCIYPH87JJ59Mly5dOPXUU2v37dixgx07dtCrVy/22GMP1q9fz7XXXhtx2SNGjGDgwIHk5eWxfft2\n1q1bx6233lrnmLKyMrp3707Pnj354YcfmDZtGtu3b6/d36dPH5KTk/nqq6/CnmfcuHFUV1fzpz/9\nicrKSgoLC8nLy+Pkk08mI0rtVBdeeCHPP/88zz//PNXV1XzwwQfceeedXHLJJQC8/PLLfPjhh+za\ntYuUlBS6detG586dAS8BYcWKFVRXV5Oamsoee+wR8+WtWxx0/NNanwrkm9l7wGjgC2BXS8sWkY5l\n8uTJrFq1igsvvLD2ixIgPT2defPmcd1115Gens64ceOatCxzp06dWLBgAV9++SV77bUXxx9/PBdd\ndFGdY/7+97/z3nvv0bNnTw4++GAGDRpE3759a/d369aNm266ifPOO4+MjAz+8Ic/AHXH+qSnp/PS\nSy/x/vvv069fP37yk59w4IEH8sADD9Qe05wlswP3//znP2f+/PnccMMNZGRkcPbZZzN16tTaO8Pv\nv/++9hqzsrJ4++23efbZZwFvrNGJJ55Ijx49yMnJYevWrU3OnGupqK+nY2YpQBFwhHPuu4Dt6UDZ\npZdeWruGem5uLrm5uVE9v4hoPR1pueDfoYKCgtqBsD6fj3vuuQeasZ5OVIKOmWU554r8z/8H2N85\nd1rQMVrETSRGFHSkpdr6Im43mtkKM/sayAYujFK5IiKSQKLSg+Scuzga5YiISGLThJ8irayqCm67\nDdasgWHD4KCDvH8zM6NT/s6d8NZb8PLLXsrykCEQNAxEpM1Q0BFpRRUVcPbZ8PXXcNJJsGgR3H47\nrF0LWVle8DnsMPjFL2DEiMgD0aZNsHgxLFwIBQXQpQuMHg07dsCjj3rnE2mLFHSkw/vhB6iu9saZ\nRNOqVTBuHPTtC2+/Db167d5XVgaffQbLl8P778NVV3mBYuhQLwD94hfwox/B5s1QWAhFRd6/hYXw\n3XfwyScwfDiMHQtTp8Lhh0NSQA9taWnd84m0FQo60iFt3QovvgjPPbf7TuHPf4bLLgN/Rn+L/Pe/\ncPLJcNZZMGsWBI+/69EDjjzSe9TYvBneeANefx3uuQdWroQ+fbygVfP42c/gtNPgl7+EvfYKf/4k\nTeUrbVTUx+mEPZFSpiXOiovhn//0As0rr3h9K6ec4j02boQrrwSfD+68E3796+af58EH4fLLvWDj\nHyQeczUprevWrdP/N2mW8vJysrOzo54yrTsdSXilpTBzJtx1l9eHcsopcO+9sO++u4854AD46COY\nNw/OPRd++lMvaNTM+u6cF5i+/BJWrPCauKqqvGlfAh8bN8J//gMLFsCvfhWf+gK1MxxnZ2fH7yKk\n3cvKyqodzB8tCjqSsH74Af72N6/j/rDDYMkS+PnPwx/fqRNMngxnngnXX++9Jzd3d7D54QcYONAL\nRPvtB507e8Go5lFd7WWNvfOOtz+eUlNTWbVqFT6fL74XIu1aSkoKqampUS1TzWuScCorYe5cuOUW\nL0jcfDMcc0zTy/n8c3j+eRg0yAs0Q4bAHntE/3pF2hs1r4ng3ZHcd5/XRJaZ6T3/zW+aP/PxgQd6\nDxGJHgUdiZsffoClS73sscWLvaCRne09cnJ2/9u/v9dclZ1dPyvLOS8p4N57vWy00aNhzhwvlVgZ\nXCJtj4KORJ1zXiD5/nuv3yPwkZzsjU1ZvNhLDc7J8fpNZs3yOvbXr4d167zBk2vWeKnHq1d7zzt1\n8pq6Bg/2Hmlp8MQT3hruF14IX33lNaeJSNulPh2Jqs2bvTTh11/3UpJ37qz/2G8/OP54L9gErbob\nls/nBZ+VK3c/vv/eGwtzyineOBsRiQ316UibsGCBd8cxciR88QX07h29slNSvI78IUOiV6aIxJ6C\njrRYRQVMmQLPPgt33w3nnNOyZYtFJHEp6EiLvP46nHee15fy6adeH42ISDjK7xEAdu2CO+7w+mPe\nfddLBmjIW2/BiSfCccd508csWaKAIyKNU9ARli/3JpK87z7v9ejRcMQRcP/98H//t/s45+Df//Ym\nm8zN9QZMfvutF3SUniwikYj5V4Wm5Wg7fD644QZvnrHjjoOPP/ZG8m/YABdf7PXP7LMPXHEFPPAA\nHHoojB/vZZ6tXestTNa3b7xrISLtScxTpgcNuphx4/Zj5sw8kvTncdx88AFMmOA9f/BBbz2WYM55\nzWizZ3v9NZMmwfnnayoYkY6uJSnTMQ86UEZq6odMmvQes2ZdFZNzJ7Jdu7ysse3bvcGTnTvv/jc5\n2WseKyvzZlqueWza5K1gefXV3iPKk8iKSIJrd0EH0hky5DKWL58V9WmzO5LKSm+RsI8+8sav7Nzp\nBaGaf3ftgm7doGdPb9Gwnj13Pz/hBM0rJiLN0y4Hh1ZUZFNSUkJWVla8LqFdKy31ssd27oQPP4SM\njHhfkYhI4+LWqZKWtp4MfVM2S2Ghl0GWlgYvvaSAIyLtR1yCTmrqUsaOzVHTWjOsXAlHHuktMPav\nf0HXrvG+IhGRyMU86AwadBWTJr3HzJl5sT51u/fBB17AOfVUeOghL1lARKQ9iXkiwebNm+kdzZkg\n27Bt27y7ke3bvaaw9PTd/6anQ79+XqZZY8rLvYXJbroJrrsO8hSvRSSO2lUiwYMPpnDVVYk9IWRh\nIdxzj7eYWFaWN8CyvNybGDPw39694Ywz4OyzvQGawZ/Jxo3w17965Rx4IMyfD2PGxKdOIiLREPPm\ntb/9DU46CYqLY33m1vfxx97gyYEDvefz53tTzBQUeIMsP/vMG8m/das3G8DTT3t3Qccf760xc+21\nsGKFtyzAhAneomZffulNPfPmmwo4ItL+xbx5bdWqMqZMSefdd+Hxx2HUqJicPiq2bYNly7yAuXVr\n3cfKlV7q8vnne3ORHXBA5OXu2OEN1nziCW/JZfCWB8jLg6FDW6cuIiLN1a4Gh5aVlZGWls6cOTB1\nKvzhD1VMnLiJvfbKbJVstuJieO45705i5Mjmdb7v2OFNhnnzzZCa6vXF9OpV95GV5XXwZ2a27HrL\ny727oA7S7SUi7VC7Czrp6elUV1dzwQWPMH/+GHbt6kbPnm8xbtw6HnhgQovnZHPOm55/9myvievg\ng71mrR07vJH4v/2tN8Flt24Nl7NzJzz8sNeB36MH3Hij1zSYyP1RIiKNaZdBZ8qU25k9+8dUVo4K\nOKqavn0LufHGfTj9dC/Dq7LS6+P49NPdj6IiGDQI9t/fm/5l//29R7duXj/Jvfd6zV3nnedNUnng\ngVBdDe+8A//7v15G2bp1XuAZPtwbXJmZ6T1qnr/xBlx/vZdddsMNcPrpmr5fRATaQNAxs18DN+El\nJiQDM51zjwYdUxt0UlNTGTYsj6+/vrteWXvu+QzZ2aeyYkUSOTnwzTdemvHBB+9+ZGV567h89dXu\nx8aNXlA46CCYPNnLCEtLC329znkd9M8/D19/7TXBBT5KSiA7G/78Z69vJZK0ZhGRjqItpEw/Bhzl\nnPvczPoDX5rZc865/wt1cHFxMeXl2aEvqNMqFi7cxObNWaxd6wWZfv0ab9KqqPBmT95338aPNfM6\n6MN10tfEYTWjiYhEV7SCTjXQy/+8B7AF2BHu4MzMTNLT11FUVH9fzZxsWVkwbFjkF5CWFv7OpqkU\nbEREWke0einOBP5lZquBZcD5zrld4Q5OSUlh7NgcUlOX1tmuOdlERBJbi+90zCwZ+BNwknPuDTM7\nAnjBzA5yzpWEe58399odLFz4DyoqsklLW8/YsTmak01EJIFFo3ntUKCvc+4NAOfc+2a2HjgMeDn4\n4OnTp9fO8pp9AAAVgUlEQVTeyeTm5pKffyUlJSVkZGTUucPx+XwUFxeTmdk643dERCRyBQUFFBQU\nAN73c3O1OHvNzPoAK4GfOue+NLP9gLeBQ51z6wOOq5MyHU51dTVTp97BwoVrKS/PJj19Xe0dUEvH\n74iISMvFNXvNObfJzC4GnjGzKrx+oksDA05TTJ16h3/8zlWANyZn7dqlwB3MmnVVSy9XRETiKCq3\nDs65+c65g51zhznnDnHOzW9OOT6fj4UL1wYNGIXKylEsXLimRbd0IiISf22qvaqh8TsVFdmUlITN\nSxARkXagTQWdmvE7odSM3xERkfarTQUdjd8REUlsbW5WscbG7yiVWkSk/YrbLNON8fl8dcbvKJVa\nRKRtaAsTfkZdSkoKWVlZta+VSi0i0v61i1sEpVKLiCSGdhF0lEotIpIY2kXQUSq1iEhiaBdBJ5JU\nap/PR2FhoZraRETasDabSBAsXCr1X/7yR6ZMuV1ZbSIi7UCbTZkOJziVesqU2/1ZbaNqj0lNXcqk\nSe8pq01EpBW0JGW63d0K1KRS1zSpKatNRKT9aHdBJ1CkWW3q7xERaRvaTZ9OKDVZbUVF9felpa2n\nZ8+e6u8REWlD2nXQqclqW7t2ab0+nbFjc5g+/W7NYiAi0oa0u0SCYLvnZFtTJ6vtllsu55BDruLr\nr++u954hQy5j+fJZmjBURKQZEnLutUglJSUxa9ZV5OfXzWorLCxstL8ncG43ERFpfQnTsRGY1Qaa\nxUBEpC1KmKATTLMYiIi0Pe2+ea0hmsVARKRtafeJBJHQLAYiItHToWYkaA7NYiAi0jZ0iKATKJJZ\nDNTXIyLSOhK6TyeUhmYx6N59Hbfe+jCLF29QX4+ISCvocEGnoVkMunffzLx5p9Zu1wwGIiLR1eGC\nDoTOahszZm8WLdozTF/PP8jP95raiouLyczM1GwGIiLN0CGy18IJzGorLi5m+PDHKCqaVu+4rKxb\nGTduB0uXFqvZTUQ6vA49DU5L1GS1QcN9PZWVy3j00WlUVv4KULObiEhz6c90v3AzGHTpsgTIqA04\nNZRiLSLSdB36TidYqL6eX/6yJy++eHDI4zVxqIhI0yjoBAg1YzXAa6/lhV0oLiMjA5/PpwQDEZEI\nKOiEENjXA4RNsf71r/tx9dV/1RxuIiIRanHQMbMM4GWgJg2uGzAQ6OOcK21p+W1BuIlDnbNGVybV\nXZCIyG5RT5k2szzgKOfciUHb21zKdFMFplgDDBuWF3Zl0k8+mcn06XfrLkhEEk5bS5m+EPh/rVBu\n3AU2uzW2MumVV+bz6KOjGrwLEhHpaKL6J7eZjQB6AgujWW5b1NDKpN27r+HVV7doJmsRkSDRbueZ\nADzqnKuOcrltTkMrk44alUlFRU7I99WkWYuIdERRa14zs27A6cARDR03ffr02g713NxccnNzo3UJ\nMRcuweCWW2bw2mtXKc1aRBJGQUEBBQUFAC1qrYlaIoGZXQic75w7Ksz+dp9IEE7wyqRA2NVJJ058\nB7MkJRiISLvVVhIJxgPzolheuxE8rgdalmYtIpKoOvQs07HQlDTr5ctnAVo+QUTatrZypyMhRJpm\nXV6+D5dffouWTxCRhKagE0NaPkFEOjr9CR1DLVk+wefzUVhYqDE+ItKu6U4nxpq6fIKa3UQkkSiR\nIE4iTTDo2fN4Kiun1bkLSk1dyqRJ76nZTUTioiWJBPpTOU5qEgxSUlLU7CYiHYaa19oINbuJSEeg\n5rU2Rs1uItLWqXktgUSr2Q1Q05uItDlqXmvjmtrsVlGRzZYtW5g58wnN7yYibY6a19qJyKfTuZwx\nY/Zh3ryf1ZtsVE1vIhINal7rACJpdktNXcqYMXuzePGGBheQU7ObiMSLmtfaqXCzWOflncczzzwR\n8j3KeBOReFPzWjsXvJaPz+dTxpuItCo1r3Vggc1uNa810FRE2io1ryUgDTQVkbZKzWsJTANNRaQ1\nqHlNQtL8biLS1qh5rQNRs5uIxJua1zogNbuJSEuoeU2aRM1uIhIval4TNbuJSMyoeU1qqdlNRCKh\n5jWJCi2rICKtTc1rEpaWVRCRaFPzmjQqmssq+Hw+iouLyczMrJ26R0Tal5Y0r+lORxpV0+xWY+zY\nHNauXVovsDS0rMKCBc9SVZXP4sUbdAck0oEp6EiTNWdZhQ0bNjN37ins2HE0AEVFsHbtUuAOJR+I\ndCBqXpNmi3xZBR+dO5/Dzp3P1CtjyJDLWL58FoCa3UTaCTWvSVwEN7vVZLwFN7116fIiyckHsHNn\n/TIaG/OjPiCRxKKgI1EVqultzJi9WbSohJUr6x9fWbmMRx/dPeanptnNudsxS1IWnEiCUfOatIrg\nprcpU25n9uwfB90BLWGPPR6mtLR+P5AGoIq0XXFvXjOzFOAOIBfYDnzinDsvGmVL+xTc9Na0MT8+\nKioyqKoKNQD1H+Tn+9TUJtJORat57Tag2jk3BMDM+kSpXEkQSUlJzJp1Ffn5dcf8vPZaHkVFwUcX\nAwNCllNRkU1JSUmdgCYi7UeLG8fNrCswAZhRs805t6ml5Upiimyqnc9IS1sd8v1paevJyMjQNDsi\n7VSL+3TMbBjwAvA0MBrYBtzgnHsl6Dj16Ug91dXVTJ16BwsXrqkz5sc5x5w5P6k3AHXixHcaTTBQ\nxptI62pJn040gs5hwAfAuc65J8zsUGAJ8CPn3OaA4xR0JKzgxIOmBqNJk95j5sw8/3uU8SbSmuId\ndDKBIiDF+Qszs3eBqwPvdmqCzqWXXlr712dubi65ubktOr8ktsjnfbuMMWP6ad43kVZSUFBAQUEB\n4P2/vOeeeyAeQQfAzBYDf3XOLTKzgcA7wCHOucKAY3SnIy1SWFjI8OGPUVQ0rd6+rKyb6dp1Pd99\nN7vevsGDL+X447PDzvumYCTSNHFPmQYmAQ+Y2W1AFXBxYMARiYbMzEzS09eFyHaDrl2/Zdu2/UO+\nL9y8bxqAKhJ7UQk6zrlVwNHRKEsknHDT7KSmLuWEEwazePH6EAHJx86dsHNn3V/PyspRPPzwbf4B\nqN5gU01CKtL6NA2OtCvhZrieOfP/kZx8RxPmfWt8ACpoElKRaNM0ONIuBWe7Qej0a2/etw2sXPn3\noBIKSU7+O1VVN9crOyvrVsaN2xF2ElKRji6u2WsRn0hBR2IknvO+KSlBOgIFHZEGNGXMT0PBqKG1\nf3afQ0kJkvgUdEQiEMkAVG8S0u4UFf2/eu9vqNlt6tQ76t1NaYyQJCoFHZEWiHQAarhmt4svfovF\nizeGfE9jY4RE2qO2ME5HpN0KXoYh9OqnS4CMOgEHvGy3BQseYdu2A0KWHW6MkNKypaPSn1oiQWbO\nzGPSpPcYMuQy+va9jSFDLue885aRmhpq7R/Ytm0/unZdHWKPN0aoJuDU8NKy1zQ4Q7Zm0ZZEpTsd\nkSBNW/sH0tOLGDOmP/PmRTpGaPe6QBkZGXX6epSQIIlOQUckjEia3VJTl9YGheTkuoNWvTFCJaxc\nWb/s7t3XceutD9fr69mdUadZEiQxKZFAJELhUq+D1/JpbIxQaupShg6dzYoVk6KSri0Sa8peE4mh\nULMhhKNZEiQRKeiItHGBgaq4uDjMEg0+kpPHU1XVvFkSwo0F0hghibaWBB39iSQSAzX9QykpKbVL\nNIQ4irS0ElJTX62ztaF07YUL11BZWcmUKbczbFgew4c/xrBheUyZcju7du0Kub26urr1KirSCCUS\niMRYQ0s0XHDBKMzeZ+HC54JmSQidrl1Rkc2VV+bz6KOj6iUfLF36O3+/kZISpO1Q85pIHDSWlBDp\nLAmDB08GkkL0D/no3Pkcdu58pt57lJQgLaUZCUTamVBjgQK/+CNN1x41KpMXX0wLcYZiqqpCz5JQ\nXr4Pl19+i5ISJC4UdETiKDi4hBNu8bpbbpnBa69dFWLQaibJyV8SqvumsnIZjz66OykhuNlNCQnS\nmhR0RNqBhu6MQt8FvcnQocaKFU2ZQ+5Zqqry6w1Y/ctf/si0aXdqlgSJCvXpiLRz4fqHdgeLyJZu\n6Nr1dKqqJtaZKy7cQNZI07UlMWmcjoiEHbQaWVJCuMSDhhMSPvlkJtOn3627oA5GiQQiErZ/KLKl\nG8JNTho+IaGhdG2lZUs4+lNEpIMJtXTDJZd8zT77lIQ42ktICKV79zW8+uqWOsELtHSDNEx3OiId\nTLikhOTk25k9O7KEhIbTtZu/dIP6hhKfgo5IBxXc7BYuLfsvf3nCn5AQabp205ducO52zJIUjDoA\nJRKISB2RJCTUbI/W0g3hJjSdOPGdBoORxIey10QkLqKzdEPrzK4trUfZayISF6H6h4qLi3nmmcdC\nHJ0JrA6xvRgYEGK7j4qKDKqqQs2u/Q9uuaWywXRtBaO2SUFHRFossH+oZumG+n093tINlZWv1rlz\n6dLlM/bYYzWlpcHHhwtGDadrN9Y/JPGloCMiUdXUpRu8BINDmDMneOxQuGBUk66dFDJd++GHb/M3\nyYUeO6S55eJLQUdEoi5cJtzMmVeRlJRUL127uroas/rHhwpGDadrh2+Sa+7ccgpG0aVEAhFpNeEy\n4SI9Pty8crfccjmHHHJViOl8wiUrNH1uuUgy5zrqXVPcs9fMbDWwHagEHHCrc+7ZoGMUdESkWSJN\n1w6flt30ueUaypybOTMv5CDXjjIjd1vIXqsGTnfOLY9SeSIitULNKxeuCS90/1BT55ZrOHOuquov\nzJv3My0R3gzRCjrmf4iIxES46XxC9Q95Y4dKWLkyuJRwi92Fz5wrL+/LggVrqKy8us72ysoRfPbZ\nvezcOSpouxeo8vN9CdnU1lTRTCR4zMwA3gWucc5tiWLZIiIhBd8FRWNuuYYy57p2/ZZt2/YPcSUN\nz8gdai66jihaQWekc269mSUDNwOPAGOjVLaISJO1dG65cJlzJ5wwmMWL1zdpifBwc9E1liGXiAkJ\nUc9eM7Ms4CvnXI+g7elA2aWXXlr74eXm5pKbmxvV84uINCTSueXCZc7VJBFEOudcczLkWpKQ0FqB\nqqCggIKCgtpz3HPPPRCP7DUz6wp0ds6V+V9PAcY550YFHafsNRFpd0IFqaYsER5+LrrwGXKRLBEe\nrLFlI6IprinTZjYQeA5vQTgDvgOudM6tDTpOQUdEEkokd03FxcUMH/4YRUXTgt8dZqLThpcIX758\nFikpKfXuaMLN+F0TqKLZhBfXlGnn3CpgeEvLERFpbyJZIjz8XHThMuQaTkjYsmULM2c+UeeOpuZu\nqiZVu0ZLZmJoLZoGR0SkFYWbiy58hlz4hIS0tPXcdtuj9cYIrVnzHMnJ20Oef8OGzcyde0rtTAyR\njilqrb4hBR0RkVbWlIGsDS0RPmbM3ixevKHeRKc7dvyGzp3nhzizj507YefOo+tsbWhMUbg7o2jd\nASnoiIi0sqYMZG0ojTsv7zyeeab+YneQQufOkJT0Sp255Zo+E0P4O6NozaqgCT9FROIs0jRun8/H\nsGF5ISY6hcGDL+P44/uxePH6CDLnmj4XXWASQ1uYe01ERJopkoSEmtfh1io64YT+tX0xzZ+JIdyd\n0e5ZFUJdZ1Mo6IiItCPh1yrKA1o2E0P4Oeq8JIaMjIwWX7+a10RE2qGWrlUUbntj432gDaynE9GJ\nFHRERNq8hqb/qcleU9AREZGoauhOSokEIiISVeGSG1oqcdZPFRGRNk9BR0REYkZBR0REYkZBR0RE\nYkZBR0REYkZBR0REYkZBR0REYkZBR0REYkZBR0REYkZBR0REYkZBR0REYkZBR0REYkZBR0REYkZB\nR0REYkZBR0REYkZBR0REYkZBR0REYkZBR0REYkZBR0REYkZBR0REYkZBR0REYkZBR0REYkZBR0RE\nYiaqQcfMxptZtZmNi2a5IiKSGKIWdMysP/B74K1olZkoCgoK4n0JcaF6dyyqt0QiKkHHzAy4H7gM\n8EWjzETSUX8pVe+ORfWWSETrTmcK8F/n3EdRKk9ERBJQp5YWYGYHAqcAIyM5vry8vKWnbHd8Pp/q\n3YGo3h1LR6x3S+przrkWndzMJgLXAjsAA7KAMuDPzrm5AcftA6xv0clERKQt6eec29CUN7Q46NQr\n0OxV4E7n3AtB2w3YG6iI6glFRCQe0oCNrolBpMXNayGEvAD/hTUpIoqISJvVrDa2qN/piIiIhBOT\nGQnMbD8ze8PMvjKzd8xsaCzOG2tm9lczW+UfIHtwwPY9zWyRmX1tZp+aWURJF+2BmXUxs3+Z2Zdm\n9pGZFZjZIP++hK03gL+uH/vr/ZqZHerfntD1rhE8GDzR621mq81shf/n/aGZnebfnuj1TjGzu/31\n+8TMHvVvb169nXOt/gBeBs71Pz8FeDcW5431A/gFXr/Vd8DBAdsfAK7zPz8CWAckx/t6o1TnLsCY\ngNeXAq/6nz+YqPX21yk94PlJwMcdod7+evUH3vA/xvm3Jezvub9O3wHDQmxP9HrfCfw14HWfltQ7\nFhe8J1AKJAVsKwT2jfeH2Yp1XhUUdCpqflD+128DR8f7Olup7ocD33XAel8AfNAR6o2XpboEOAx4\nNSDoJHq96/y/DtiesPUGuuJlI3ePVr1bI5EgWDZQ6JyrDti2FsjB+8shoZlZBtDJObcpYPMavPon\noiuB/+0o9TazR4Bf4SXQ/LqD1Lt2MLiXlNqhfs8f89f5XeBqvJ97Itd7EFACzDCz0cA24AbgY5pZ\nb80yLVFjZtPxfkmnx/taYsU5d75zLgf4E/AX/2aL4yW1qoDB4DfH+1riYKRz7hBgOFAMPOLfnrA/\nb7wM5/7AZ865H+P9Ufm0f3uz6h2LoLMO6GtmgefKwbvbSXjOuRJgl5n1Cdg8gASrv5lNxevXGOOc\nq+wo9a7hnHsMGOV/uTOB6z0S70topZmtAn4GzANOJ8F/3s659f5/q4C78IJQov+erwWqgCcBnHMf\nA6uBYTTz97zVg45zbjPwIXAugJmdCqxzziV801qAZ4FJAGb2Y7xkg9fiekVRZGZTgDOBY51zgYN/\nE7beZtbDzPoGvD4JKPZ/CSVsvZ1zc5xz+zjn9nXODcRrx7/IOTeHBK63mXU1sx4Bm84GauaafIYE\nrbdzrhgvEWwMgJkNxAsuX9DMn3dMxumY2RDgYSATr1NqvHPu81Y/cYyZ2RxgLLAX3u13hXNuiP+v\ngceAgXjTBV3qnFsWvyuNHv/0RuuAb/E6Fg2odM79PMHrnYP3ny4Vr11/EzDVOfdpItc7mJm9Atzl\nnHshkevt/7J9Du8PdcPrj77SObc2kesNtXV/AOiNd9dzg3Puf5tbbw0OFRGRmFEigYiIxIyCjoiI\nxIyCjoiIxIyCjoiIxIyCjoiIxIyCjoiIxIyCjoiIxIyCjoiIxIyCjoiIxMz/By7wVN5dNMYmAAAA\nAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x14f8cae48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "acc = history.history['acc']\n",
    "val_acc = history.history['val_acc']\n",
    "loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "\n",
    "epochs = range(len(acc))\n",
    "\n",
    "plt.plot(epochs, acc, 'bo', label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.legend()\n",
    "\n",
    "plt.figure()\n",
    "\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8806 samples, validate on 234 samples\n",
      "Epoch 1/20\n",
      "8806/8806 [==============================] - 161s 18ms/step - loss: 4.7952 - acc: 0.1266 - val_loss: 9.1499 - val_acc: 0.1154\n",
      "Epoch 2/20\n",
      "8806/8806 [==============================] - 147s 17ms/step - loss: 4.7166 - acc: 0.1443 - val_loss: 9.2087 - val_acc: 0.1154\n",
      "Epoch 3/20\n",
      "8806/8806 [==============================] - 148s 17ms/step - loss: 4.6221 - acc: 0.1582 - val_loss: 9.1619 - val_acc: 0.1154\n",
      "Epoch 4/20\n",
      "8806/8806 [==============================] - 147s 17ms/step - loss: 4.6580 - acc: 0.1533 - val_loss: 9.2384 - val_acc: 0.1197\n",
      "Epoch 5/20\n",
      "8806/8806 [==============================] - 148s 17ms/step - loss: 4.6241 - acc: 0.1538 - val_loss: 9.1910 - val_acc: 0.1197\n",
      "Epoch 6/20\n",
      "8806/8806 [==============================] - 148s 17ms/step - loss: 4.5683 - acc: 0.1596 - val_loss: 9.2284 - val_acc: 0.1111\n",
      "Epoch 7/20\n",
      "8806/8806 [==============================] - 150s 17ms/step - loss: 4.5917 - acc: 0.1586 - val_loss: 9.2770 - val_acc: 0.1111\n",
      "Epoch 8/20\n",
      "8806/8806 [==============================] - 149s 17ms/step - loss: 4.5540 - acc: 0.1599 - val_loss: 9.2936 - val_acc: 0.1239\n",
      "Epoch 9/20\n",
      "8806/8806 [==============================] - 149s 17ms/step - loss: 4.5022 - acc: 0.1672 - val_loss: 9.3028 - val_acc: 0.1068\n",
      "Epoch 10/20\n",
      "5632/8806 [==================>...........] - ETA: 51s - loss: 4.5148 - acc: 0.1641"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-86-b9684c850137>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m                     \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m                     \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m256\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m                     validation_data=(validation_features, validation_labels))\n\u001b[0m",
      "\u001b[0;32m~/anaconda/lib/python3.5/site-packages/keras/models.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m    958\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    959\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 960\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m    961\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    962\u001b[0m     def evaluate(self, x, y, batch_size=32, verbose=1,\n",
      "\u001b[0;32m~/anaconda/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1655\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1656\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1657\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1658\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1659\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m~/anaconda/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m   1200\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1201\u001b[0m                             \u001b[0;31m# Do not slice the training phase flag.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1202\u001b[0;31m                             \u001b[0mins_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_slice_arrays\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1203\u001b[0m                         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1204\u001b[0m                             \u001b[0mins_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_slice_arrays\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_slice_arrays\u001b[0;34m(arrays, start, stop)\u001b[0m\n\u001b[1;32m    413\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'shape'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    414\u001b[0m                 \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 415\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mNone\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    416\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    417\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mNone\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    413\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'shape'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    414\u001b[0m                 \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 415\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mNone\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    416\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    417\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mNone\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=optimizers.Adam(lr=2e-4),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=20,\n",
    "                    batch_size=256,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_51 (Dense)             (None, 420)               86016420  \n",
      "_________________________________________________________________\n",
      "dropout_27 (Dropout)         (None, 420)               0         \n",
      "_________________________________________________________________\n",
      "dense_52 (Dense)             (None, 4250)              1789250   \n",
      "=================================================================\n",
      "Total params: 87,805,670\n",
      "Trainable params: 87,805,670\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import optimizers\n",
    "\n",
    "model = models.Sequential()\n",
    "model.add(layers.Dense(420, activation='relu', input_dim=ll_size*ll_size*2048)) # we can play around with \n",
    "model.add(layers.Dropout(0.5))\n",
    "model.add(layers.Dense(classes_count, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8806 samples, validate on 234 samples\n",
      "Epoch 1/20\n",
      "8806/8806 [==============================] - 261s 30ms/step - loss: 8.4007 - acc: 0.0018 - val_loss: 8.3474 - val_acc: 0.0043\n",
      "Epoch 2/20\n",
      "8806/8806 [==============================] - 246s 28ms/step - loss: 8.2770 - acc: 0.0047 - val_loss: 8.2871 - val_acc: 0.0043\n",
      "Epoch 3/20\n",
      "8806/8806 [==============================] - 246s 28ms/step - loss: 8.0202 - acc: 0.0085 - val_loss: 8.1318 - val_acc: 0.0000e+00\n",
      "Epoch 4/20\n",
      "8806/8806 [==============================] - 244s 28ms/step - loss: 7.7649 - acc: 0.0115 - val_loss: 8.1058 - val_acc: 0.0000e+00\n",
      "Epoch 5/20\n",
      "8806/8806 [==============================] - 238s 27ms/step - loss: 7.5312 - acc: 0.0179 - val_loss: 8.1027 - val_acc: 0.0085\n",
      "Epoch 6/20\n",
      "8806/8806 [==============================] - 243s 28ms/step - loss: 7.3139 - acc: 0.0251 - val_loss: 8.1382 - val_acc: 0.0085\n",
      "Epoch 7/20\n",
      "8806/8806 [==============================] - 245s 28ms/step - loss: 7.0849 - acc: 0.0340 - val_loss: 8.0659 - val_acc: 0.0085\n",
      "Epoch 8/20\n",
      "8806/8806 [==============================] - 244s 28ms/step - loss: 6.9142 - acc: 0.0407 - val_loss: 8.1610 - val_acc: 0.0085\n",
      "Epoch 9/20\n",
      "8806/8806 [==============================] - 243s 28ms/step - loss: 6.6916 - acc: 0.0450 - val_loss: 8.1182 - val_acc: 0.0171\n",
      "Epoch 10/20\n",
      "8806/8806 [==============================] - 246s 28ms/step - loss: 6.4812 - acc: 0.0595 - val_loss: 8.1677 - val_acc: 0.0256\n",
      "Epoch 11/20\n",
      "8806/8806 [==============================] - 243s 28ms/step - loss: 6.2654 - acc: 0.0682 - val_loss: 8.2016 - val_acc: 0.0342\n",
      "Epoch 12/20\n",
      "8806/8806 [==============================] - 243s 28ms/step - loss: 6.0568 - acc: 0.0856 - val_loss: 8.4108 - val_acc: 0.0598\n",
      "Epoch 13/20\n",
      "8806/8806 [==============================] - 245s 28ms/step - loss: 5.8679 - acc: 0.0911 - val_loss: 8.4128 - val_acc: 0.0598\n",
      "Epoch 14/20\n",
      "8806/8806 [==============================] - 250s 28ms/step - loss: 5.6919 - acc: 0.1041 - val_loss: 8.5158 - val_acc: 0.0513\n",
      "Epoch 15/20\n",
      "8806/8806 [==============================] - 242s 28ms/step - loss: 5.5269 - acc: 0.1150 - val_loss: 8.5950 - val_acc: 0.0855\n",
      "Epoch 16/20\n",
      "8806/8806 [==============================] - 243s 28ms/step - loss: 5.3508 - acc: 0.1262 - val_loss: 8.7479 - val_acc: 0.0855\n",
      "Epoch 17/20\n",
      "8806/8806 [==============================] - 250s 28ms/step - loss: 5.1672 - acc: 0.1351 - val_loss: 8.8255 - val_acc: 0.1026\n",
      "Epoch 18/20\n",
      "8806/8806 [==============================] - 241s 27ms/step - loss: 4.9952 - acc: 0.1519 - val_loss: 8.8594 - val_acc: 0.0897\n",
      "Epoch 19/20\n",
      "8806/8806 [==============================] - 238s 27ms/step - loss: 4.8444 - acc: 0.1691 - val_loss: 8.9089 - val_acc: 0.0983\n",
      "Epoch 20/20\n",
      "8806/8806 [==============================] - 240s 27ms/step - loss: 4.6878 - acc: 0.1784 - val_loss: 9.1458 - val_acc: 0.1026\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=optimizers.Adam(lr=2e-4),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=20,\n",
    "                    batch_size=64,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8806 samples, validate on 234 samples\n",
      "Epoch 1/20\n",
      "8806/8806 [==============================] - 252s 29ms/step - loss: 4.5477 - acc: 0.1906 - val_loss: 9.1145 - val_acc: 0.0897\n",
      "Epoch 2/20\n",
      "8806/8806 [==============================] - 253s 29ms/step - loss: 4.3877 - acc: 0.2071 - val_loss: 9.2858 - val_acc: 0.1026\n",
      "Epoch 3/20\n",
      "8806/8806 [==============================] - 245s 28ms/step - loss: 4.2962 - acc: 0.2161 - val_loss: 9.2781 - val_acc: 0.1197\n",
      "Epoch 4/20\n",
      "8806/8806 [==============================] - 248s 28ms/step - loss: 4.1406 - acc: 0.2309 - val_loss: 9.5330 - val_acc: 0.1111\n",
      "Epoch 5/20\n",
      "8806/8806 [==============================] - 248s 28ms/step - loss: 3.9868 - acc: 0.2444 - val_loss: 9.4557 - val_acc: 0.1325\n",
      "Epoch 6/20\n",
      "8806/8806 [==============================] - 246s 28ms/step - loss: 3.8773 - acc: 0.2597 - val_loss: 9.4671 - val_acc: 0.1111\n",
      "Epoch 7/20\n",
      "8806/8806 [==============================] - 248s 28ms/step - loss: 3.7516 - acc: 0.2757 - val_loss: 9.7191 - val_acc: 0.1239\n",
      "Epoch 8/20\n",
      "8806/8806 [==============================] - 246s 28ms/step - loss: 3.6570 - acc: 0.2839 - val_loss: 9.7615 - val_acc: 0.1282\n",
      "Epoch 9/20\n",
      "8806/8806 [==============================] - 239s 27ms/step - loss: 3.5464 - acc: 0.2972 - val_loss: 9.7306 - val_acc: 0.1239\n",
      "Epoch 10/20\n",
      "8806/8806 [==============================] - 288s 33ms/step - loss: 3.4415 - acc: 0.3106 - val_loss: 9.8758 - val_acc: 0.1325\n",
      "Epoch 11/20\n",
      "8806/8806 [==============================] - 379s 43ms/step - loss: 3.3621 - acc: 0.3244 - val_loss: 10.1094 - val_acc: 0.1282\n",
      "Epoch 12/20\n",
      "8806/8806 [==============================] - 2120s 241ms/step - loss: 3.2905 - acc: 0.3288 - val_loss: 10.0781 - val_acc: 0.1325\n",
      "Epoch 13/20\n",
      "8806/8806 [==============================] - 258s 29ms/step - loss: 3.2203 - acc: 0.3350 - val_loss: 10.0274 - val_acc: 0.1496\n",
      "Epoch 14/20\n",
      "8806/8806 [==============================] - 244s 28ms/step - loss: 3.1211 - acc: 0.3529 - val_loss: 9.9914 - val_acc: 0.1325\n",
      "Epoch 15/20\n",
      "8806/8806 [==============================] - 237s 27ms/step - loss: 3.0245 - acc: 0.3697 - val_loss: 10.0280 - val_acc: 0.1453\n",
      "Epoch 16/20\n",
      "8806/8806 [==============================] - 232s 26ms/step - loss: 2.9156 - acc: 0.3842 - val_loss: 10.0850 - val_acc: 0.1496\n",
      "Epoch 17/20\n",
      "8806/8806 [==============================] - 241s 27ms/step - loss: 2.8438 - acc: 0.3894 - val_loss: 10.1141 - val_acc: 0.1453\n",
      "Epoch 18/20\n",
      "8806/8806 [==============================] - 243s 28ms/step - loss: 2.8028 - acc: 0.4029 - val_loss: 10.1427 - val_acc: 0.1496\n",
      "Epoch 19/20\n",
      "8806/8806 [==============================] - 244s 28ms/step - loss: 2.7587 - acc: 0.4068 - val_loss: 10.2035 - val_acc: 0.1410\n",
      "Epoch 20/20\n",
      "8806/8806 [==============================] - 244s 28ms/step - loss: 2.6790 - acc: 0.4173 - val_loss: 10.2634 - val_acc: 0.1496\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=20,\n",
    "                    batch_size=64,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8806 samples, validate on 234 samples\n",
      "Epoch 1/20\n",
      "8806/8806 [==============================] - 144s 16ms/step - loss: 2.5647 - acc: 0.4385 - val_loss: 10.3277 - val_acc: 0.1496\n",
      "Epoch 2/20\n",
      "8806/8806 [==============================] - 129s 15ms/step - loss: 2.4416 - acc: 0.4506 - val_loss: 10.3479 - val_acc: 0.1538\n",
      "Epoch 3/20\n",
      "8806/8806 [==============================] - 127s 14ms/step - loss: 2.3999 - acc: 0.4575 - val_loss: 10.4116 - val_acc: 0.1581\n",
      "Epoch 4/20\n",
      "8806/8806 [==============================] - 126s 14ms/step - loss: 2.3781 - acc: 0.4593 - val_loss: 10.4149 - val_acc: 0.1581\n",
      "Epoch 5/20\n",
      "8806/8806 [==============================] - 129s 15ms/step - loss: 2.3249 - acc: 0.4751 - val_loss: 10.3779 - val_acc: 0.1581\n",
      "Epoch 6/20\n",
      "8806/8806 [==============================] - 130s 15ms/step - loss: 2.2726 - acc: 0.4799 - val_loss: 10.4966 - val_acc: 0.1496\n",
      "Epoch 7/20\n",
      "8806/8806 [==============================] - 131s 15ms/step - loss: 2.1980 - acc: 0.4930 - val_loss: 10.5213 - val_acc: 0.1581\n",
      "Epoch 8/20\n",
      "8806/8806 [==============================] - 133s 15ms/step - loss: 2.1838 - acc: 0.4982 - val_loss: 10.5244 - val_acc: 0.1581\n",
      "Epoch 9/20\n",
      "8806/8806 [==============================] - 125s 14ms/step - loss: 2.1995 - acc: 0.5034 - val_loss: 10.6615 - val_acc: 0.1624\n",
      "Epoch 10/20\n",
      "8806/8806 [==============================] - 126s 14ms/step - loss: 2.1261 - acc: 0.5062 - val_loss: 10.5654 - val_acc: 0.1581\n",
      "Epoch 11/20\n",
      "8806/8806 [==============================] - 127s 14ms/step - loss: 2.0829 - acc: 0.5152 - val_loss: 10.5832 - val_acc: 0.1709\n",
      "Epoch 12/20\n",
      "8806/8806 [==============================] - 130s 15ms/step - loss: 2.0443 - acc: 0.5277 - val_loss: 10.6329 - val_acc: 0.1624\n",
      "Epoch 13/20\n",
      "8806/8806 [==============================] - 134s 15ms/step - loss: 2.0531 - acc: 0.5213 - val_loss: 10.4822 - val_acc: 0.1709\n",
      "Epoch 14/20\n",
      "8806/8806 [==============================] - 134s 15ms/step - loss: 2.0185 - acc: 0.5284 - val_loss: 10.6227 - val_acc: 0.1624\n",
      "Epoch 15/20\n",
      "8806/8806 [==============================] - 136s 15ms/step - loss: 1.9843 - acc: 0.5338 - val_loss: 10.6877 - val_acc: 0.1667\n",
      "Epoch 16/20\n",
      "8806/8806 [==============================] - 132s 15ms/step - loss: 1.9450 - acc: 0.5439 - val_loss: 10.6148 - val_acc: 0.1709\n",
      "Epoch 17/20\n",
      "8806/8806 [==============================] - 129s 15ms/step - loss: 1.8969 - acc: 0.5553 - val_loss: 10.7426 - val_acc: 0.1752\n",
      "Epoch 18/20\n",
      "8806/8806 [==============================] - 132s 15ms/step - loss: 1.9230 - acc: 0.5454 - val_loss: 10.6823 - val_acc: 0.1667\n",
      "Epoch 19/20\n",
      "8806/8806 [==============================] - 133s 15ms/step - loss: 1.8677 - acc: 0.5551 - val_loss: 10.7049 - val_acc: 0.1752\n",
      "Epoch 20/20\n",
      "8806/8806 [==============================] - 138s 16ms/step - loss: 1.8548 - acc: 0.5618 - val_loss: 10.7102 - val_acc: 0.1752\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=optimizers.Adam(lr=2e-4),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=20,\n",
    "                    batch_size=256,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('weights/adam_320_no_aug_0.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8806 samples, validate on 234 samples\n",
      "Epoch 1/40\n",
      "8806/8806 [==============================] - 129s 15ms/step - loss: 1.8078 - acc: 0.5700 - val_loss: 10.7942 - val_acc: 0.1667\n",
      "Epoch 2/40\n",
      "8806/8806 [==============================] - 109s 12ms/step - loss: 1.7527 - acc: 0.5820 - val_loss: 10.7521 - val_acc: 0.1752\n",
      "Epoch 3/40\n",
      "8806/8806 [==============================] - 108s 12ms/step - loss: 1.7501 - acc: 0.5834 - val_loss: 10.7265 - val_acc: 0.1795\n",
      "Epoch 4/40\n",
      "8806/8806 [==============================] - 107s 12ms/step - loss: 1.7082 - acc: 0.5905 - val_loss: 10.7730 - val_acc: 0.1709\n",
      "Epoch 5/40\n",
      "8806/8806 [==============================] - 109s 12ms/step - loss: 1.6895 - acc: 0.5941 - val_loss: 10.8014 - val_acc: 0.1752\n",
      "Epoch 6/40\n",
      "8806/8806 [==============================] - 108s 12ms/step - loss: 1.7006 - acc: 0.5876 - val_loss: 10.7955 - val_acc: 0.1752\n",
      "Epoch 7/40\n",
      "8806/8806 [==============================] - 109s 12ms/step - loss: 1.6758 - acc: 0.5974 - val_loss: 10.8277 - val_acc: 0.1752\n",
      "Epoch 8/40\n",
      "8806/8806 [==============================] - 110s 13ms/step - loss: 1.6622 - acc: 0.5974 - val_loss: 10.8156 - val_acc: 0.1752\n",
      "Epoch 9/40\n",
      "8806/8806 [==============================] - 111s 13ms/step - loss: 1.6069 - acc: 0.6039 - val_loss: 10.7503 - val_acc: 0.1752\n",
      "Epoch 10/40\n",
      "8806/8806 [==============================] - 109s 12ms/step - loss: 1.6421 - acc: 0.6020 - val_loss: 10.7959 - val_acc: 0.1795\n",
      "Epoch 11/40\n",
      "8806/8806 [==============================] - 108s 12ms/step - loss: 1.6045 - acc: 0.6109 - val_loss: 10.8247 - val_acc: 0.1752\n",
      "Epoch 12/40\n",
      "8806/8806 [==============================] - 109s 12ms/step - loss: 1.5814 - acc: 0.6144 - val_loss: 10.8263 - val_acc: 0.1795\n",
      "Epoch 13/40\n",
      "8806/8806 [==============================] - 108s 12ms/step - loss: 1.5550 - acc: 0.6165 - val_loss: 10.8387 - val_acc: 0.1752\n",
      "Epoch 14/40\n",
      "8806/8806 [==============================] - 108s 12ms/step - loss: 1.5550 - acc: 0.6223 - val_loss: 10.8068 - val_acc: 0.1709\n",
      "Epoch 15/40\n",
      "8806/8806 [==============================] - 108s 12ms/step - loss: 1.5358 - acc: 0.6254 - val_loss: 10.8097 - val_acc: 0.1709\n",
      "Epoch 16/40\n",
      "8806/8806 [==============================] - 108s 12ms/step - loss: 1.5219 - acc: 0.6243 - val_loss: 10.8496 - val_acc: 0.1709\n",
      "Epoch 17/40\n",
      "8806/8806 [==============================] - 112s 13ms/step - loss: 1.5181 - acc: 0.6238 - val_loss: 10.8640 - val_acc: 0.1752\n",
      "Epoch 18/40\n",
      "8806/8806 [==============================] - 109s 12ms/step - loss: 1.5018 - acc: 0.6323 - val_loss: 10.8677 - val_acc: 0.1752\n",
      "Epoch 19/40\n",
      "8806/8806 [==============================] - 109s 12ms/step - loss: 1.4641 - acc: 0.6283 - val_loss: 10.9069 - val_acc: 0.1795\n",
      "Epoch 20/40\n",
      "8806/8806 [==============================] - 108s 12ms/step - loss: 1.4931 - acc: 0.6315 - val_loss: 10.8075 - val_acc: 0.1752\n",
      "Epoch 21/40\n",
      "8806/8806 [==============================] - 109s 12ms/step - loss: 1.4615 - acc: 0.6385 - val_loss: 10.9289 - val_acc: 0.1795\n",
      "Epoch 22/40\n",
      "8806/8806 [==============================] - 110s 12ms/step - loss: 1.4348 - acc: 0.6416 - val_loss: 10.9203 - val_acc: 0.1752\n",
      "Epoch 23/40\n",
      "8806/8806 [==============================] - 111s 13ms/step - loss: 1.4127 - acc: 0.6441 - val_loss: 10.9615 - val_acc: 0.1667\n",
      "Epoch 24/40\n",
      "8806/8806 [==============================] - 109s 12ms/step - loss: 1.4359 - acc: 0.6482 - val_loss: 10.9590 - val_acc: 0.1709\n",
      "Epoch 25/40\n",
      "8806/8806 [==============================] - 110s 12ms/step - loss: 1.3810 - acc: 0.6547 - val_loss: 10.9429 - val_acc: 0.1752\n",
      "Epoch 26/40\n",
      "8806/8806 [==============================] - 108s 12ms/step - loss: 1.3780 - acc: 0.6585 - val_loss: 10.9235 - val_acc: 0.1838\n",
      "Epoch 27/40\n",
      "8806/8806 [==============================] - 109s 12ms/step - loss: 1.3857 - acc: 0.6538 - val_loss: 10.9740 - val_acc: 0.1752\n",
      "Epoch 28/40\n",
      "8806/8806 [==============================] - 116s 13ms/step - loss: 1.3421 - acc: 0.6597 - val_loss: 10.9762 - val_acc: 0.1752\n",
      "Epoch 29/40\n",
      "8806/8806 [==============================] - 110s 12ms/step - loss: 1.3607 - acc: 0.6593 - val_loss: 11.0046 - val_acc: 0.1795\n",
      "Epoch 30/40\n",
      "8806/8806 [==============================] - 107s 12ms/step - loss: 1.3375 - acc: 0.6657 - val_loss: 10.9408 - val_acc: 0.1752\n",
      "Epoch 31/40\n",
      "8806/8806 [==============================] - 110s 12ms/step - loss: 1.2955 - acc: 0.6742 - val_loss: 10.9802 - val_acc: 0.1752\n",
      "Epoch 32/40\n",
      "8806/8806 [==============================] - 108s 12ms/step - loss: 1.2865 - acc: 0.6775 - val_loss: 10.9625 - val_acc: 0.1709\n",
      "Epoch 33/40\n",
      "8806/8806 [==============================] - 106s 12ms/step - loss: 1.3024 - acc: 0.6689 - val_loss: 11.0140 - val_acc: 0.1752\n",
      "Epoch 34/40\n",
      "8806/8806 [==============================] - 128s 15ms/step - loss: 1.2404 - acc: 0.6837 - val_loss: 11.0229 - val_acc: 0.1667\n",
      "Epoch 35/40\n",
      "8806/8806 [==============================] - 131s 15ms/step - loss: 1.2657 - acc: 0.6798 - val_loss: 11.0321 - val_acc: 0.1752\n",
      "Epoch 36/40\n",
      "8806/8806 [==============================] - 118s 13ms/step - loss: 1.2250 - acc: 0.6865 - val_loss: 11.0332 - val_acc: 0.1709\n",
      "Epoch 37/40\n",
      "8806/8806 [==============================] - 111s 13ms/step - loss: 1.2199 - acc: 0.6893 - val_loss: 11.0579 - val_acc: 0.1667\n",
      "Epoch 38/40\n",
      "8806/8806 [==============================] - 117s 13ms/step - loss: 1.2237 - acc: 0.6894 - val_loss: 11.0504 - val_acc: 0.1709\n",
      "Epoch 39/40\n",
      "8806/8806 [==============================] - 125s 14ms/step - loss: 1.1957 - acc: 0.6951 - val_loss: 11.0221 - val_acc: 0.1795\n",
      "Epoch 40/40\n",
      "8806/8806 [==============================] - 126s 14ms/step - loss: 1.1952 - acc: 0.6895 - val_loss: 11.0792 - val_acc: 0.1709\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(train_features, train_labels,\n",
    "                    epochs=40,\n",
    "                    batch_size=512,\n",
    "                    validation_data=(validation_features, validation_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('weights/adam_320_no_aug_1.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}