ANN_onHaxby.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# What is an artificial neural network (ANN)?\n",
    "ANN is the foundation of AI and one of the main tools used in ML inspired by the human brain when networks of neurons \n",
    "analyze and process information. ANNs have self-learning capabilities and can learn from their experience that enables them \n",
    "to produce better results as more data becomes available.\n",
    "In this computational model, neural networks consist of input and output layers also there is a hidden layer (in most cases)\n",
    "that transforms the input to the usable data for the output layer. ANN initially goes through a training phase. During this \n",
    "supervised phase, the network is taught what to look for and what is the desired output. The difference between the results \n",
    "is adjusted using backpropagation (going from output to input) in order to achieve the lowest possible difference between \n",
    "the actual and desired outcome.\n",
    "\n",
    "In this notebook, I trained ANN model on the HAXBY dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import keras\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "from nilearn.input_data import NiftiMasker\n",
    "from nilearn import datasets, plotting, image\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder, OneHotEncoder\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from nilearn.plotting import plot_anat, show, plot_stat_map, plot_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Designate input file\n",
    "haxby_ds = datasets.fetch_haxby(subjects=[4], fetch_stimuli=True)\n",
    "\n",
    "# 'func' is a list of filenames: one for each subject\n",
    "func_file = haxby_ds.func[0]\n",
    "\n",
    "# Standardizing\n",
    "mask_vt_file = haxby_ds.mask_vt[0]\n",
    "masker = NiftiMasker(mask_img=mask_vt_file, standardize=True)\n",
    "\n",
    "# Load the behavioral data that I will predict\n",
    "beh_label = pd.read_csv(haxby_ds.session_target[0], sep=\" \")\n",
    "\n",
    "# Regressout the resting state\n",
    "nonrest_task_mask = (beh_label['labels'] != 'rest')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Selecting data\n",
    "\n",
    "# X contains the features\n",
    "X = masker.fit_transform(func_file)[nonrest_task_mask]\n",
    "\n",
    "# y contains the target variable\n",
    "y = beh_label['labels'][nonrest_task_mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove the resting state and find names of remaining active labels\n",
    "categories = y[nonrest_task_mask].unique()\n",
    "\n",
    "# Get the labels of the numerical conditions represented by the vector y\n",
    "unique_conditions, order = np.unique(categories, return_index=True)\n",
    "\n",
    "# Sort the conditions by the order of appearance\n",
    "unique_conditions = unique_conditions[np.argsort(order)]\n",
    "\n",
    "# Extract tags indicating to which acquisition run a tag belongs\n",
    "session_labels = beh_label['chunks'][nonrest_task_mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encoding the string to numerical values. (ML Algorithm can only work on numbers and not on string)\n",
    "labelencoder_y = LabelEncoder()\n",
    "y = labelencoder_y.fit_transform(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(864,)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3 3 3 3 3 3 3 3 3 2 2 2 2 2 2 2 2 2 5 5 5 5 5 5 5 5 5 7 7 7 7 7 7 7 7 7 6\n",
      " 6 6 6 6 6 6 6 6 4 4 4 4 4 4 4 4 4 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 1\n",
      " 1 1 1 1 1 1 1 6 6 6 6 6 6 6 6 6 5 5 5 5 5 5 5 5 5 2 2 2 2 2 2 2 2 2 0 0 0\n",
      " 0 0 0 0 0 0 7 7 7 7 7 7 7 7 7 3 3 3 3 3 3 3 3 3 4 4 4 4 4 4 4 4 4 3 3 3 3\n",
      " 3 3 3 3 3 6 6 6 6 6 6 6 6 6 5 5 5 5 5 5 5 5 5 7 7 7 7 7 7 7 7 7 0 0 0 0 0\n",
      " 0 0 0 0 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3\n",
      " 3 3 3 1 1 1 1 1 1 1 1 1 6 6 6 6 6 6 6 6 6 4 4 4 4 4 4 4 4 4 5 5 5 5 5 5 5\n",
      " 5 5 2 2 2 2 2 2 2 2 2 7 7 7 7 7 7 7 7 7 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1\n",
      " 1 6 6 6 6 6 6 6 6 6 2 2 2 2 2 2 2 2 2 0 0 0 0 0 0 0 0 0 7 7 7 7 7 7 7 7 7\n",
      " 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3 3 3 5 5 5 5 5 5 5 5 5 3 3 3 3 3 3 3 3 3 1\n",
      " 1 1 1 1 1 1 1 1 7 7 7 7 7 7 7 7 7 2 2 2 2 2 2 2 2 2 5 5 5 5 5 5 5 5 5 0 0\n",
      " 0 0 0 0 0 0 0 4 4 4 4 4 4 4 4 4 6 6 6 6 6 6 6 6 6 4 4 4 4 4 4 4 4 4 6 6 6\n",
      " 6 6 6 6 6 6 3 3 3 3 3 3 3 3 3 7 7 7 7 7 7 7 7 7 2 2 2 2 2 2 2 2 2 1 1 1 1\n",
      " 1 1 1 1 1 0 0 0 0 0 0 0 0 0 5 5 5 5 5 5 5 5 5 7 7 7 7 7 7 7 7 7 4 4 4 4 4\n",
      " 4 4 4 4 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 3 3 3 3 3 3 3 3 3 6 6 6 6 6 6\n",
      " 6 6 6 0 0 0 0 0 0 0 0 0 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 3 3 3 3 3 3 3\n",
      " 3 3 1 1 1 1 1 1 1 1 1 7 7 7 7 7 7 7 7 7 4 4 4 4 4 4 4 4 4 6 6 6 6 6 6 6 6\n",
      " 6 0 0 0 0 0 0 0 0 0 2 2 2 2 2 2 2 2 2 4 4 4 4 4 4 4 4 4 5 5 5 5 5 5 5 5 5\n",
      " 0 0 0 0 0 0 0 0 0 3 3 3 3 3 3 3 3 3 2 2 2 2 2 2 2 2 2 7 7 7 7 7 7 7 7 7 1\n",
      " 1 1 1 1 1 1 1 1 6 6 6 6 6 6 6 6 6 0 0 0 0 0 0 0 0 0 4 4 4 4 4 4 4 4 4 2 2\n",
      " 2 2 2 2 2 2 2 6 6 6 6 6 6 6 6 6 3 3 3 3 3 3 3 3 3 7 7 7 7 7 7 7 7 7 1 1 1\n",
      " 1 1 1 1 1 1 5 5 5 5 5 5 5 5 5 3 3 3 3 3 3 3 3 3 2 2 2 2 2 2 2 2 2 1 1 1 1\n",
      " 1 1 1 1 1 4 4 4 4 4 4 4 4 4 6 6 6 6 6 6 6 6 6 7 7 7 7 7 7 7 7 7 0 0 0 0 0\n",
      " 0 0 0 0 5 5 5 5 5 5 5 5 5]\n"
     ]
    }
   ],
   "source": [
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = np.reshape(y, (864,1))\n",
    "y = test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(864, 1)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>859</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>860</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>861</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>862</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>863</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>864 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0    1    2    3    4    5    6    7\n",
       "0    0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0\n",
       "1    0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0\n",
       "2    0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0\n",
       "3    0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0\n",
       "4    0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0\n",
       "..   ...  ...  ...  ...  ...  ...  ...  ...\n",
       "859  0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0\n",
       "860  0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0\n",
       "861  0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0\n",
       "862  0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0\n",
       "863  0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0\n",
       "\n",
       "[864 rows x 8 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# creating instance of one-hot-encoder\n",
    "enc = OneHotEncoder(handle_unknown='ignore')\n",
    "# passing bridge-types-cat column (label encoded values of bridge_types)\n",
    "y = pd.DataFrame(enc.fit_transform(y).toarray())\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle and split the data into training set and test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, shuffle=True, random_state = 0)\n",
    "\n",
    "#standarize features caling\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(777, 675)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initializing the ANN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = Sequential()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/srastegarnia/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(338, input_dim=675, activation=\"relu\", kernel_initializer=\"uniform\")`\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "# Adding the input layer and the first hidden layer\n",
    "classifier.add(Dense(338 , input_dim = 675, init = 'uniform', activation = 'relu'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/srastegarnia/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(169, activation=\"relu\", kernel_initializer=\"uniform\")`\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "# Adding the second hidden layer\n",
    "classifier.add(Dense(169, init = 'uniform', activation = 'relu'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier.add(Dense(8, activation = 'softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 338)               228488    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 169)               57291     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 8)                 1360      \n",
      "=================================================================\n",
      "Total params: 287,139\n",
      "Trainable params: 287,139\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "classifier.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compiling the ANN\n",
    "classifier.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/srastegarnia/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "777/777 [==============================] - 1s 1ms/step - loss: 1.7119 - accuracy: 0.3668\n",
      "Epoch 2/10\n",
      "777/777 [==============================] - 0s 479us/step - loss: 0.8594 - accuracy: 0.7272\n",
      "Epoch 3/10\n",
      "777/777 [==============================] - 0s 471us/step - loss: 0.4483 - accuracy: 0.8468\n",
      "Epoch 4/10\n",
      "777/777 [==============================] - 0s 633us/step - loss: 0.2430 - accuracy: 0.9241\n",
      "Epoch 5/10\n",
      "777/777 [==============================] - 0s 482us/step - loss: 0.1201 - accuracy: 0.9755\n",
      "Epoch 6/10\n",
      "777/777 [==============================] - 0s 432us/step - loss: 0.0867 - accuracy: 0.9691\n",
      "Epoch 7/10\n",
      "777/777 [==============================] - 0s 421us/step - loss: 0.0883 - accuracy: 0.9743\n",
      "Epoch 8/10\n",
      "777/777 [==============================] - 0s 414us/step - loss: 0.0322 - accuracy: 0.9936\n",
      "Epoch 9/10\n",
      "777/777 [==============================] - 0s 419us/step - loss: 0.0571 - accuracy: 0.9833\n",
      "Epoch 10/10\n",
      "777/777 [==============================] - 0s 417us/step - loss: 0.0475 - accuracy: 0.9884\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x7f8f07d83f90>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fitting the ANN to the Training set\n",
    "classifier.fit(X_train, y_train, batch_size = 10, nb_epoch = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Making the predictions and evaluating the model\n",
    "y_pred = classifier.predict(X_test)\n",
    "y_pred = (y_pred > 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 7  0  0  0  0  1  0  0]\n",
      " [ 0 13  0  0  0  0  0  0]\n",
      " [ 2  0  9  0  1  3  0  0]\n",
      " [ 1  0  0 14  0  0  0  0]\n",
      " [ 0  0  0  0 11  0  0  0]\n",
      " [ 0  0  1  0  0  9  0  0]\n",
      " [ 0  0  0  0  0  0  6  0]\n",
      " [ 1  1  0  0  0  0  0  7]]\n"
     ]
    }
   ],
   "source": [
    "cm = confusion_matrix(y_test.values.argmax(axis=1), y_pred.argmax(axis=1))\n",
    "print(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1, 2, 3, 4, 5, 6, 7]), <a list of 8 Text yticklabel objects>)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAFaCAYAAAA5J6tkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZwcVdn28d81IYQAWQ0ohCCLgAIiSwj6iILIjgq+ogYFBcWAbPoACoICriCooAYIETHghiuKCKI+iqiIJoiIAcHIlhCBbMQQSDBwv3+cM1JpemY6mZqu7vT1zac/6Vq66u6e7rrrLHVKEYGZmVl/dVUdgJmZrRmcUMzMrBROKGZmVgonFDMzK4UTipmZlcIJxczMSuGEMoAkzZS0Z34uSV+TtEjSnyS9RtI9DWzjnZJ+3uD+zpH0jX6GPeAkbSYpJK1VdSz9kd/DS6qOox5Jn5I0X9Ij/djGppKekDSozNiqkt/LFlXHsSZzQumFpJtyAhjSwLrTJH2qOC8itouIm/Lk7sA+wCYRMSEifhsR2/S13Yj4ZkTsuzrxdyJJD0jau+o4qiRpHHAKsG1EvGh1txMRD0XE+hHxTHnRlS//To/ua738Xu5rRkydygmlB5I2A14DBPCmPtZt5AzuxcADEbG038G1uDXljLYnbVCyejGwICIeqzqQVtAGf681R0T4UecBnAX8HvgCcF3NsmnApcD1wFJgEvAf4GngCeAneb0HgL2B9wLLgGfy8o8DewJzCtscB/wQmAcsACbn+UcCvyus90VgNvBv4DbgNYVl5wDf6OH97AnMIZ25Pgb8CziqsHwI8DngIeBRYAowtF4MeV4AL+nh89gbOAi4Pcc5Gzin8NrN8uvX6iHWB4BTgb8Ci4HvAOsUlr8B+AvwOHALsEOe/3XgWeCp/Dl/GLgSOCUvH5v3e1yefgmwEFCefh8wK8+7Fti45v0eD/wDuL/OZ7B7fp+vAwRcmD/nxfl9bN/Dex0NfA2YCywCflRY1lc8x+Z4FgEX5/3und//s/kzmEbNd6343czPJwAz8t/qUeAL9f5OwMY5joU5rvfVfPe+C1wFLAFmAuN7+X0FcFyOfwnwSWBL4A85ju8Ca+d1RwHXkX4bi/LzTfKyT5N+V8vy+53c198LWJv0/Tkxzx9E+q2fVfVxp90flQfQqo/8gzkO2IWULF5YWDYtHyheTSrlrZPnfapmG8Uf7ZGsnBj++yPPX+g7SAeh9fL2du/hdYcDLwDWIiWHR8gHW/pOKCuATwCDgQOBJ4FReflF+WAxGhgG/AQ4t14MeV5tQqn9PPYEXp6ndyAdqA7J629G3wnlT6QD2GjgbuDYvGxn0oF6t/y5vTuvP6T2M8/T7+G5BP8O4J/AdwrLfpyf7wXMz9sfAnwZuLnm/f4ixzO0+BkA+5GSyYQ8fz9Ssh9JOsi/DNioh/f6U1LCHJX/LnusQjzX5X1sSjrY7l/73ao3Xee7+QfgiPx8feCV9f5OwG+AS/Lfd8e8z9cXvnvLSN+rQcC5wK29/L6C9H0bDmwHLAf+D9gCGAHcBbw7r/sC4C3AuqTv5vdYOfHeBBxdZ/t1/175+fak5PQy4EzgVmBQ1ceddn9UHkArPkhnm/8BxuTpvwP/W1g+Dbiq5jXTWP2E8qr843zeAbb2dXWWLwJekZ+fQ+8J5aniPkgH5leSDnpLgS0Ly17Fc2d2z4uB5yeUq3qKMa9zEXBhfr7SgarOug8Ahxemzwem5OeXAp+sWf8enjsQ//czz9NbkkoyXaRS1zGFz/1K4OT8/KvA+YXXrZ+/A5sV3u9edT6DjwAPAi8vzN8LuDd/tl29fCYbkUoSo+osaySe3QvLvwucXvvdqjdd57t5M6nUPKZmnf/+nUgl6GeAYYXl5wLTCt+9XxaWbQs81ct7D+DVhenbgNMK058HLurhtTsCiwrTN1E/odT7e72kMH0K6be9CNiqt++vH4093IZS37uBn0fE/Dz9rTyvaHaJ+xsHPBgRK/paUdIpku6WtFjS46SzuTEN7mdBzT6eJB2oNiCd/d0m6fG83Z/l+Y1a6fOQtJukX0uaJ2kxqXqm0Tghlbxq44TUPnBKd5w51nGk0szzRMQ/SVUhO5LaxK4D5kraBtiDdNZNfv2Dhdc9Qap6HNvTe8w+CHw3Iu4svPZXwGRSNdSjkqZKGl7nteOAhRGxqM6yRuLp6TNaVe8Ftgb+Lmm6pDf0EM/CiFhSmPdgH/Gs00f7xaOF50/VmV4fQNK6ki6T9KCkf5MS4MgG2ur6+o1eSUqa10fEP/pY1xrghFJD0lDgbcAekh7J3S7/F3iFpFcUVo2al9ZOr4rZwKZ9NR5Keg1wWo5vVESMJFU1qR/7hlS18hSwXUSMzI8REdF9gFpKSjjdcdTrOVT7/r9FqtIYFxEjSKWD/sYJ6bP6dCHOkRGxbkR8u4c4ICWNQ0l18g/n6XeRqpn+kteZS0pWAEhaj1TV8nBhO/W2/VbgEEkfLM6MiC9FxC6k6pytgQ/18F5GSxpZZ1kj8TSq9u83iMLJQkT8IyIOAzYEPgt8P++vNp7RkoYV5m26mvGsqlOAbYDdImI48No8v/v71NNvr6/f5CWkE4z9JO3e7yjNCaWOQ0hF+21JZ7U7kupZf0s6CPXkUVL97+r4E6mR/DxJ60laR9Kr66w3jNQOMg9YS9JZpDrofomIZ4GvABdK2hBA0lhJ++VV7gC2k7SjpHVI1Rt9GUY6o10maQKp/aIMXwGOzSUg5c/roMKBrt7f4TfACaQzW0hVJCeSqvG6u8R+Czgqv8chwGeAP0bEA33EMxd4PXCSpOMAJO2a4xtMOph3d8hYSUT8C7gBuETSKEmDJXUfLFc3nnruJZUWDsoxfZTULkOO93BJG+TvweN59krxRsRsUgeIc/P3cwdSyeabqxHPqhpGOuF5XNJo4Oya5av825N0BKl99EjgJOBKSatbwrPMCeX53g18LVIf/Ee6H6QqjHf2Uor4KrBtrob50arsMB/U3khq4H2I1Bvr7XVWvZF0ALqXVN2wjPKq3k4jdUS4NVcr/JJ0VkhE3EtqzP8lqdfM7xrY3nHAJyQtIfWY+24ZQUbEDFLvp8mkuu9ZpINCt3OBj+a/w6l53m9IB6XuhPI70hl79zQR8X/Ax4AfkJL7lsDEBmN6iJRUTsvXQwwnJb5FpL/TAlIPunqOILWN/J3UpvXB/sZTJ77FpL/H5aQSxVLSd6zb/sBMSU+QehFOjIhldTZ1GKmKaC5wDXB2RPxidWJaRRcBQ0kl6VtJ1bFFXwQOzdeMfamvjUnaNG/zXRHxRER8i9TL7cJyw+483d0lzczM+sUlFDMzK4UTipmZlcIJxczMSuGEYmZmpXBCMTOzUnTkKJwaMiy07guqDoOdtliVC8et0zz9zLNVhwDA2oN83ln05z/fNj8iVmUUiY7RmQll3RcwZM8zqw6D33/vfVWHYC1s7qKnqg4BgI1HDa06hJYydLAe7HutzuRTDzMzK4UTipmZlcIJxczMSuGEYmZmpXBCMTOzUjihmJlZKZxQzMysFE4oZmZWCicUMzMrhROKmZmVwgnFzMxK4YRiZmalcEIxM7NSOKGYmVkpnFDMzKwULZ1QJG0p6UeSDilhW5MkzZA0I5YvKSM8MzMraPUbbL2fFOPLJe0B/CciPizpAOCtwGLgk8DbgG2AEcCHI2J+7YYiYiowFaBr1GbRpPjNzDpGqyeU64BZwNr5sZekDYD3AG+LiJC0PnAE8AtgELAD8KuK4jUz61itnlAAXgBsGhHHSBoHrAdERHSXMgTMjohzqgrQzMxavA0lexwYIelUYKs872uSpko6HxgM/FnSlyVNlvTSyiI1M+tgLV1CiYibgJuAi/Osz+X/HwBuKKx6ftOCMjOzutqhhGJmZiWSdIWkxyT9rYflkvQlSbMk/VXSzo1s1wnFzKzzTAP272X5AaQmhq2AScCljWzUCcXMrMNExM3Awl5WORi4KpJbgZGSNupru04oZmZWaywwuzA9J8/rVUs3ypuZWTJo+IsjVjzV0Lrx1LyZwLLCrKn54u5Gqd5m+3qRE4qZWRuIFcsY8tKJDa277PYvL4uI8f3Y3RxgXGF6E2BuXy9ylZeZWTsQIDX26L9rgXfl3l6vBBZHxL/6epFLKGZm7ULllAEkfRvYExgjaQ5wNukicSJiCnA9cCBp6KsngaMa2a4TiplZuyin9EFEHNbH8gCOX9XtOqGYmbUFlVZCGShOKGZm7aKkEspA6ciEstMWY/j9995XdRiM2vWEqkMAYNH0yVWHYHVsPGpo1SFYK5Gga1DVUfSqIxOKmVlbcpWXmZmVwlVeZmbWf26UNzOzMnRf2NjCnFDMzNqCoKu1D9mtHZ2ZmT2nyyUUMzPrL+E2FDMzK4nbUMzMrP/cy8vMzMriEoqZmfWbh14xM7PSuMrLzMxK4SovMzPrPzfKm5lZWVxCMTOzflPrD73S2uUnQNI0Sev0sc55DWxnkqQZkmbMmz+vvADNzJpFauxRkZZMKJI2l/R1SV8AXgh8WNIPJU2QtLGkT0u6VNKb8ks2k7SZpJslnS5pdO02I2JqRIyPiPEbjNmgqe/HzKwU6mrsUZGWTCjAccBZEXEy8ChwGfAR4CBgBTAYeAw4ouZ1MyPivIhY2MxgzcyaosVLKK1cIfds4fliYCgwhJRErgH+DPy45jWLmxOamVmTyb28VtclwMclPQpsWLPsFuAY4B7g6WYHZmZWGffyWnURcT9wZM3sB4DT8/M/1Kw/MT89HTOzNZCAri6XUMzMrL+UHy3MCcXMrC0IucrLzMzK4IRiZmalcEIxM7NSOKGYmVm/SUJdTihmZlYCl1DMzKwUTihmZlYKJxQzM+u/NriwsbWv4zczMwCE6OrqaujR0Pak/SXdI2mWpOcNWyVphKSfSLpD0kxJR/W1zY4soTz9zLM8vPCpqsNg0fTJVYcAwFsu/1PVIQDwg6MnVB0CAP9+6j9VhwDA3EXLqg4BgJduPKzqECwrq8pL0iDgYmAfYA4wXdK1EXFXYbXjgbsi4o2SNgDukfTNiOhxUF6XUMzM2oUafPRtAjArIu7LCeJq4OCadQIYppTF1gcWku5H1aOOLKGYmbUdldooPxaYXZieA+xWs85k4FpgLjAMeHtEPEsvXEIxM2sTkhp6AGMkzSg8JtVuqs7mo2Z6P+AvwMbAjsBkScN7i88lFDOzNrEKJZT5ETG+l+VzgHGF6U1IJZGio4DzIiKAWZLuB14K9Njo6hKKmVkbEGnolUYeDZgObCVpc0lrAxNJ1VtFDwGvB5D0QmAb4L7eNuoSiplZOyixDSUiVkg6AbgRGARcEREzJR2bl08BPglMk3Rn2junRcT83rbrhGJm1ibKvFI+Iq4Hrq+ZN6XwfC6w76ps0wnFzKxNeOgVMzMrR2vnEycUM7N24RKKmZn1m6SGx+mqihOKmVmbcAnFzMzK0dr5xAnFzKxduIRiZmb9V+7gkAPCCcXMrA2kG2y1dkJpmS4Dks7rx2v7/JQlTeoeeXPhgl5HDzAza0lSY4+qNLWEIukwYG9gCXA3sBOwPCI+AGwm6aWk8WNmA1eQhk/eAng8Is6U9AlgODACOAaYCvwTuFPSQcATwJ0RcUXtviNial6fl++4c+0wzWZmLc9VXisbB9wBXANcGBGH1iwfSbor2FUR8bd8D+M/AdfncfjHRsR7Jb2b58aYmRoRj0o6Avgx8MumvBMzs2aquPTRiKZWeUXE+cDNwIX19h0RtwKfAg7NSeNDwL3A10kd5rpLFsUSxuL8/xH5/6+XH7mZWbUEdHWpoUdVml3ldQywNem+xNMlTQaWRcSpefnrgDcBLwBuAE4DxgDzgKXAvyR9HhhNqvI6NL9uMPAF4Eng7818T2ZmzdLqJZSmJpSIuKyXZRPz018XZv++ZrWP1UwfWXh+7OpHZmbW4kTL9/Jyt2EzszYg3ChvZmalkBOKmZmVo8XziROKmVm7cAnFzMz6rw2uQ3FCMTNrA93XobQyJxQzszbhKi8zMytFi+cTJxQzs7bg+6GYmVkZ0oWNVUfROycUM7O20Po32OrIhDKoSwwb2pFvva4fHD2h6hAAGLXvZ6oOAYBFPz+j6hAAGD50cNUhWItxlZeZmfWfr0MxM7MyeHBIMzMrjROKmZmVosXziROKmVlb8A22zMysDPL9UMzMrCwtnk/oqjoAMzNrTJfU0KMRkvaXdI+kWZJO72GdPSX9RdJMSb/pa5suoZiZtYmySiiSBgEXA/sAc4Dpkq6NiLsK64wELgH2j4iHJG3Y13adUMzM2oCURvkoyQRgVkTcl7atq4GDgbsK67wD+GFEPAQQEY/1tVFXeZmZtQlJDT0aMBaYXZiek+cVbQ2MknSTpNskvauvjbqEYmbWJlahymuMpBmF6akRMbW4qTqviZrptYBdgNcDQ4E/SLo1Iu7taadOKGZmbUCkrsMNmh8R43tZPgcYV5jeBJhbZ535EbEUWCrpZuAVQI8JxVVeZmZtokuNPRowHdhK0uaS1gYmAtfWrPNj4DWS1pK0LrAbcHdvG3UJxcysHTTePtKniFgh6QTgRmAQcEVEzJR0bF4+JSLulvQz4K/As8DlEfG33rbrhGJm1gZEqb28iIjrgetr5k2pmb4AuKDRbbZFlVfu0tbfbUySNEPSjAXz55cRlplZU0mNParSsiUUSYcBewNLgBGSziL1nT4BWBs4G3gK+AnwU+DTwBBgRUScWru93MNhKsCOO+9S25vBzKzleSyv1TcOuAO4hnS15qdJCeZ1wPbAGRHxoKTvAU8DmwL3AFtIGh0RC6sJ28ysfFWXPhrRsgklIs6XtCNwIbA8Ip6RtJxUChGpkQhS3+ku4LcRcUk10ZqZDbxGx+mqSssmFEnHkK7UXAGMrlk8FThX0lLg26SeClMkXQAMj4hjmhqsmVkTtHY6aeGEEhGX1Zl3E3BTnjy8ZvHRAxySmVllyu7lNRBaNqGYmVlBidehDBQnFDOzNtHi+cQJxcysXbiEYmZm/SYaHqerMk4oZmZtwt2Gzcys3yQnFDMzK0mL5xMnFDOzduFGeTMzK0WL5xMnFDOzdiDkNhQzMyuBoKvF+w13ZEIZJDF86OCqw7Aai35+RtUhADBq1xOqDgGARdMnVx2CtZhWvyNiRyYUM7N2I9wob2ZmJWnxGi8nFDOzduGEYmZm/ZZuAdzaGcUJxcysTQxq8VZ5JxQzszaQRht2CcXMzErQ4gUUJxQzs3bR4gUUJxQzs3YgeegVMzMriRvlzcys39wob2ZmpWnxfOKEYmbWFuQr5c3MrCSitTOKE4qZWRtIbShVR9E7JxQzszYxqMUzSimd0LQKI5ZJ2kzSeTXzrl7V1xSW1Z1fZ71JkmZImjFv/rxGwzUzawndJZRGHlXps4Qi6TBgb2AJcDewE7A8Ij4g6U7gG8B1kt4BjABuj4ivSvoV8AdgK+DXwPj8/++AV0v6LDAamFTY17HANnk7HwZeBxwALAaWS9oTOAW4GRgeER8DNpM0ErgUOBb4CnB4RDxdfB8RMRWYCrDLLuNjFT8nM7NqqdxeXpL2B74IDAIuj4ieTtp3BW4F3h4R3+9tm42UUMYBdwCfB/aJiGMj4gN52ZyI+CxwHymBLgQm5mXLI+JMUgKZCxwNHJSX3RsRpwH3ArvkoNcHjiAljyeAHfIbeA9QLMHcEhEXAMMlbQAQEY8DU0gJ7JzaZGJmtiboylfL9/Xoi6RBwMWkE/ZtgcMkbdvDep8Fbmwkvj5LKBFxvqQdgQt5fgJanP8/ELgzIr4t6dc1y54GFkdEFKrGouZ/SAlpdkScU3gz78tPlxfWG1zzf7fNgEeB9ft6T2Zm7abkRvkJwKyIuA/+2+xwMHBXzXonAj8Adm1ko41UeR0DbA2sAKZLmgwsi4hTC6vdDpwraVNS8akv20j6DLABqeRDRCyR9GdJXyZ9dpOBH0n6BDCE55LPhNxusiQi5klC0mbA7sC+wDckvSciljYQh5lZ2yixymssMLswPQfYbeV9aSzwZmAvykooEXFZL8sm5v/vA96eZ3+2ZtmU2vWB19Rsqnvd82vm/704kdtQruthm92lmbdjZraGEWJQ4xlljKQZhempuR35uc09X23b8kXAaRHxTKP9rtqq23BE3ATcVHEYZmbNt2o9uOZHxPhels8htY9324TU1l00Hrg6J5MxwIGSVkTEj3raaFslFDOzTlbi4JDTga0kbQ48TKolekdxhYjYvPu5pGmk2qEekwk4oZiZtQVRXhtKRKyQdAKp99Yg4IqImJkv3VipqWJVOKGYmbWJMoevj4jrgetr5tVNJBFxZCPbdEIxM2sDAga19sgrTihmZm1B6TbArcwJxcysTbR2OnFCMTNrC74FsJmZlaa104kTiplZ22jxAooTiplZO1jFoVcq4YRizF+yvO+VmmDMsCFVhwDAoumTqw4BgFFv+lLVIQCw6NqTqg7BMvfyMjOzUrR2OnFCMTNrD74OxczMyiAau8VulZxQzMzahK9DMTOzUrR4PnFCMTNrB6nKq7UzihOKmVmbcAnFzMxKIOQSipmZlcElFDMz6zcJD71iZmblaPF84oRiZtYu3IZiZmb9lm6wVXUUvXNCMTNrEy6hmJlZKdyGYmZm/SZav5dXSwxeKelISfsP8D4mSZohaca8+fMGcldmZgNADf+rSiuVUN4m6c3AX4FtgeXA4Ig4UdLVETFR0p7AS4HFwN7AEuA04GDgf4DhwHkRcW/txiNiKjAVYJddxkcT3o+ZWXnU+lVeLVFCyW6MiGOAvYDHIuJkYL6k7eusOw64A/h8RCwHjgceB+YD45sVsJlZM6nBR1VaqYSyOP8v4Nn8PGr+XxcgIs6XtCNwoaTTgaURcU6zAjUza7bUbbi1iyitlFC6PQ1sJOkCYGhE/E3SryV9CtgAuF3SMcDWwApgIfAdSZcBy4DvRcTvqgrezGygtHg+aY2EEhHTCs8n1lk+tY9NXJkfZmZrLF+HYmZmpXAJxczMStHi+cQJxcysbbR4RnFCMTNrA6lLcGtnlFa6DsXMzHqiNNpwI4+GNiftL+keSbPy5Re1y98p6a/5cYukV/S1TZdQzMzaRUkFFEmDgIuBfYA5wHRJ10bEXYXV7gf2iIhFkg4gjTSyW2/bdUIxM2sLpY7TNQGYFRH3AUi6mjSE1X8TSkTcUlj/VmCTvjbqKi8zszYhNfZowFhgdmF6Tp7Xk/cCN/S1UZdQzMzawCqO0zVG0ozC9NSaC8TrbaruoLmSXkdKKLv3tVMnFDOzdtF4RpkfEb0NlDuHNMhut02Auc/bnbQDcDlwQEQs6GunTihmZm2ixMEhpwNbSdoceBiYCLyjuIKkTYEfAkfUuyVIPU4oxphhQ6oOwepYdO1JVYcAwKhdT6g6BAAWTZ9cdQiVKyudRMQKSScANwKDgCsiYqakY/PyKcBZwAuAS5QS2Yo+Sj1OKGZmbaHkm51ExPXA9TXzphSeHw0cvSrbdEIxM2sTrX6lvBOKmVkbEB5t2MzMSuKEYmZmpXCVl5mZlcIlFDMzK0WL5xMnFDOzttHiGcUJxcysDbTDDbacUMzM2sEq3DyrKk4oZmbtwgnFzMz6r9QbbA0IJxQzszbhbsNmZtZvJY8NOSCcUMzM2kWLZxQnFDOzNlHiDbYGRFfVAZRF0pGS9u9l+SRJMyTNmDd/XjNDMzMrhRp8VKWtSij5dpWfAOYBi4H1gDHAycD/AOtKIiJ+VvvaiJgKTAXYZZfx0bSgzczKIDfKl+044KyIuD+XRvYChgD7ALcAj9RLJmZma4bWzijtllAAns3/vy8i3iLpSFJJ5dmeX2Jm1t58g63yXQJ8XNKjwFBJZwIvA34J3AF8RNJaEXFdlUGamQ0ED71Sooi4Hziyl1UmNikUM7Om85XyZmZWjtbOJ04oZmbtosXziROKmVk7kLsNm5lZWdyGYmZmpXAJxczMSuGEYmZmJfANtszMrATtcKX8GjPasJmZVcslFDOzNtHqJRQnFDOzdqDWv8FWRyaU/zwTPLZ4WdVhtIwNR6xTdQjWwhZNn1x1CABsedI1VYdQqapvntWIjkwoZmZtqcUzihOKmVmbaPVuw+7lZWbWJrrH8+rr0di2tL+keyTNknR6neWS9KW8/K+Sdu5rm04oZmZtoqyEImkQcDFwALAtcJikbWtWOwDYKj8mAZf2tV0nFDOzNqEG/zVgAjArIu6LiKeBq4GDa9Y5GLgqkluBkZI26m2jbkMxM2sDt//5thvXXVtjGlx9HUkzCtNTI2JqYXosMLswPQfYrWYb9dYZC/yrp506oZiZtYGI2L/EzdUrxsRqrLMSV3mZmXWeOcC4wvQmwNzVWGclTihmZp1nOrCVpM0lrQ1MBK6tWeda4F25t9crgcUR0WN1F7jKy8ys40TECkknADcCg4ArImKmpGPz8inA9cCBwCzgSeCovrbrhGJm1oEi4npS0ijOm1J4HsDxq7JNV3mZmVkpnFDMzKwUTihmZlYKJxQzMyuFE4qZmZWiZRKKpCMl9XglqKSra59L2rO7m1sD258kaYakGQsXzOt/wGZmtpJW6zZ8uKQDgAeBLYHlwGDgXGB7SecAP8/PTwduBZD0YuAU0lABd9aMWQNAnjcVYIcdd+l1+AAzM1t1LVNCyW6MiA8ArwYei4iTgfnAaOBvEXFORNySn59XeN1xwFJgAbBDs4M2M7PWK6F0lxy6gGdr5kWd9bp1AVdGxF0DGJuZmfWi1Uoo+0v6PPA7YCNJFwAbRsTfgL9LuiCPx/+MpLMLr5sMnCHpc5I+VEHcZmYdr2VKKBExDZjWy/KPFyYPLzy/qc48MzNrslYroZiZWZtyQjEzs1I4oZiZWSmcUMzMrBROKGZmVgonFDMzK4UTipmZlcIJxczMSuGEYmZmpXBCMTOzUjihmJlZKRTRebcGkTSPdM+V/hhDGlq/ao5jZY5jZY5jZWXE8eKI2KCMYNY0HZlQyiBpRkSMdxyOw3E4Dktc5WVmZqVwQjEzs1I4oay+5923viKOY2WOY2WOY2WtEscayW0oZmZWCpdQzMysFE4oZmZWCieUFidJVcdgPZM0uOoYujrZyx0AAArkSURBVFX9XZE0SNLakkZXGYdVxwmlD5L2lbSdpOFV7D9yI5ekAyWNlVTp30zSPq1wwJD0/ySdImm9Cvbdlf8XcJmkjZodQy1J7wROkvTqiva/FvBm4NPAgVXEUCRpgqRXVB1Hp3FC6YWkA4GTgSOBQRXsf7181rcPcGZEPBwRz1Z1Jirp7cBuwFJJW1cRQ47jEOAQYCdgWLP3n/8Go4FtgYeAfzc7hqKcRN4EjAPukzSk2TFExArgSeBFwMIc17rNjiPv93DgBOBESetXEUOnckIpKB6oJQ0D9gX+HhEfAjaWtH2TQ3otcBFwH/BDSUfBc6WWZpI0AXgLsAGwH3CxpC2atO/aBDo4Pz4IbChpUjPiyLHsJOlQYDiwD3AQcImkKZK2aVYchXi6gI2B9SLiVOCFwGGS1m5yHPsB/wR+BKwv6SPAMc2KI5987ZFLSjsA9wMfBl4kadtmxGDuNlyXpHcDN5POxt8BXAy8DfhYRMytIJZXAucDbwCeiIivNTmGw0lnndOB9YAdgV2Bz0fEwibG8SZgJPAEMIJ0NjwSuCwi7hvgfSsiIlexfQ74XUR8M8e0LvCLiFgwkDHUiel44B/ATcB3SH+fLYBPRcQDTYxjP+BU4AHgFmAR8BTpZKy/Y+Y1GsOOwJ7AbGCXHMdy4HXA5Gb/bjvVWlUH0GokvZFUlfKNiLhf0uPAy4FzmvmllLQvMBr4LrAY+BRwJvB4s2LIcaxLKplcGhHzJC0BxgNTm5xM3k+qYvoZcBJwNvAIMDMiHhrgfXflaq5RwFbAGcBnJD0D3Abs1Z1MuhPPQMaT97Mz6UTjTlKJ8WRS6eTyiHhsoPdfiONtwJakE7BrgImkgVe/lqvBBnr/I4AJwFJgBrA58DSwIfBi0ufhZNIkTijPtzZwa0Q8I2kPUjH+l834cXTLZ3zvJ53pvQWYBASwLCKallByVdJdwE+Bt0haGhG/l3R2RDwzwPvuiohnC7MeBy6KiH9KGkcqXd8wkDF0y8nkhaSD5QtJB8xPAhcAj0TEZYV1m5FM/gfYGbibNHLuWcAPIuLSgd53TRzvBV5Gaog/hdSGsxZwQxN/LytI7/+RiHirpKeBIcCDEXFFk2KwrOPbULrr5yW9SdJbgT+R6qBPBN4LPDXQP46atpu1SGfe9wIfAP4IfDoifhwRjwxkHDUxTQQ2JdXPPw7cSPpchgLP9vbaknT3bjsmJ/aXAe+UdASwB6ldacBJGi9pQ+CdpGHLzwCGArsDR0TETc2Io8YGpCql35BKr7cC11UQxwLgqohYRKoSfRfwpYiYM9A7ljQit42MIZUYl0jam1RSuZtUWrEm6+g2FEkjgfcAjwIHANsAXwF+BawPPBwR85oQR3f9/DHAElKD4ouAUaSD+uXN+JHWxHQqMDciviXpHFId/b1NKJkIEHAhKaluDvye1JtrJCnBPjLQdfOFaq73kw7gt5DOwJfl6ccj4qrumJtUMjkCmBURf8g9/3YltVv8X0Q8OtD7rxPPNqSS9F9IJabPRsTDTdr3dsC3gRkR8Z7cm+sSUonltGb8bu35Or2EsjmpsX1r0oHqIOBVed4dA/2lVNIFnCfpBFLyeIjUe2gXUtXKFc1MJpLel6tUpgE7S/ooKbEtHehkAqnKKFd1/ZvUNnBVRFxDaoh/EritGQ29OZmMIbUPiFQieYjU6L1TdzLpjnmg48ntN3eSrjXZEniGdHZ+QxXJBCAi7gE+m+O6oFnJJHuYVIIfKmlz0t/oIziZVKrTSyhjgI8Dd5CqVBaQqhKuGeheQzVxnE3qOXVRRPxG0m7A3sCFEfFkE+PYi9RO8EVSae0G0vUNv21yh4TDgO1I1Y9vBc4hldrWjYgnBnjf3aVFAUcDCyPiB7m9YCvgemAzUont6SYlkyOB1wPfIlW37U06oF4VEbMHev+tJn8eewFXk7qP/z9SyfH0XP1mFenoRvmImJ/7y+9KOtvbgHSm1cyD59GkH8XngMNzw/cfJd0REcsGeN//raqR9BJSNd8zpJLrh0jXNnx9IGOoE9M6pGSyhPS5vJrU6HtEE5PJ+qTOGYNI1zT8gNRNeWZE3CzpjxGxfCBjKcR0IOnzOB44jNRJ5CRgaEQsaUYMrUTSG0ifxwmkz2MWcCzp83AyqVhHJxSAiPi3pBmkIvP0iFjc5BAWAH+KiL9K2gk4WdJRA51MYKVhXU4E1oqICyXdS2p0/1V+NFVELJN0GakufFfgx8AXIuI/A7nfQjIZTuqiPYzUjnNIrvab051cBzqZKI0PJtIQJuNJXWDXJ1W5HQLc3EnJJHdU6SJVge5O+jzWJVUJv5lUgm5aF3brWUdXeRXV6abarP1uAxwH3A68gnTwHPBqjNxb6zDSAftsUhvFTFJngHuA25tcJ14b39rAicDVzYhD0sYRMVfSO0jVfN+NiB9KGkv6nczJ6w1oA3xuU3s76dqnIcCVpC7KK0h/n690UjVX/jzeRro2rAv4Ouki3/nAv4AvNrvDivXMCaUFKA0uuDHwWDMPFrkH03bAN0g9qo4mXWdxRrOqdHojaXATSiZdpAtIvwqcR+oM8FXgcNLIBIsjj0zQxN5cLyIllccj4kpJu5CqeM7sxIv08udxGGmUiK/kiylHkdo6m3YRp/XNCaVDSXpDRFwn6QPAWFIbzluBn0YTh+2oUrFUKunFpKqu80kHq0NJQ4dcUlFsLwQOJl3f8UNSleTTVcTSCvLn8WbS1e+jgLOr6t1mPXNC6UD5As7dI+IDefpDpGsszuywuvmNSQfs80m9t4aRDlpfJl3f8J+8XlNKJj3Ety/w/YHukNAOcknlUOBHruZqTU4oHUZprLKdgR9HxF+UxoS6HRgWEZUOw95MSqNJfxTYHriUNOjlU6SG71dFHsakqmRSiHOtaOKwP61O0qBmXA9lq6fTL2zsKHk4ldeSRqU9W9JbSBeHbdgpyaQwzM2TwBTSaL1dEfEd0rhpC6IwJlaVySTv38mkwMmktTmhdIh8fcf2pN4xQ0hX5Y8BPtIpddGFrsFjgP8ldcv9AjBe0rWk7rgd04PKrGyu8uogSqP0dl/f8UZSb66OGKai5jqTQ4F5pHajzUgllddGxLTiulXFatauXELpIPnsewFpCJFzOjCZjCJ1Dd4OuD/34HoC2NXJxKz/XELpQM24vqPV5OFUvgpMJQ0vsx2piuvOSgMzW4O4hNKBOiWZ5IsWu3sGPUG6PeyrIt3D5BHSRZzd69bet97MVpFLKLZGU7pF7HtJN+y6CPggsFFEfLjSwMzWQE4otsYptJkMId2a9uXActKAi0cB+0WTbh9s1klc5WVrlEIyGUG6cdqtpNGcjyR1lz6iO5m4msusXE4otsYoJJP1gJ8AoyPiV8BySVNJ45Rd2b2+e3OZlctVXrZGKJZMImKxpM8AgyPiQ7m0sn1E/L64brURm615XEKxtldIJiOBiyXtHhFnAI9KujwiFjuZmA08l1CsrXUPQZ+vgN8PGBERlxeW75m7CZvZAHMJxdpaTiYjgc8AQ4GDJG0u6ROSDu1OJm6ANxt4TijW1nKiOAhYB/gD6e6LxwP3RcT3u9dzNZfZwFur6gDM+iO3nfyW506O/k26OdbV4DYTs2ZyG4qtMfKdKIdHxFfztJOJWRM5odgaQ9LoiFiYnzuZmDWZE4qtcZxMzKrhhGJmZqVwLy8zMyuFE4qZmZXCCcXMzErhhGJmZqVwQjEzs1L8f3o3VXDH9BbxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 504x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# normalising confusion matrix\n",
    "cmn = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "\n",
    "# Plotting\n",
    "plot_matrix(cmn, cmap='Blues', labels=unique_conditions)\n",
    "plt.title('Artificial neural networks confusion matrix')\n",
    "plt.xticks(rotation=45)\n",
    "plt.yticks(rotation=0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}