modeling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import Counter\n",
    "\n",
    "\n",
    "# Audio feature extraction\n",
    "import librosa as lb\n",
    "\n",
    "# Visualization\n",
    "import matplotlib.pyplot as plt\n",
    "import librosa.display \n",
    "import seaborn as sns\n",
    "from mlxtend.plotting import plot_decision_regions\n",
    "from matplotlib import pyplot\n",
    "\n",
    "\n",
    "# Model\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn.linear_model import LogisticRegression, LogisticRegressionCV\n",
    "from sklearn.model_selection import train_test_split, cross_val_predict, StratifiedKFold, KFold, GridSearchCV\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import confusion_matrix, roc_auc_score, roc_curve, f1_score, accuracy_score, precision_score, recall_score\n",
    "from xgboost import XGBClassifier\n",
    "from mlxtend.plotting import plot_decision_regions\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier  #GBM algorithm\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "from xgboost import plot_importance\n",
    "\n",
    "\n",
    "# Render figures interactively in the notebook\n",
    "%matplotlib inline\n",
    "\n",
    "# Audio widget for playback\n",
    "from IPython.display import Audio\n",
    "\n",
    "# Importing files\n",
    "from os import listdir\n",
    "from os.path import isfile, join"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing in python and postgreSQL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Before my postgreSQL manipulation, one of my tables has some preprocessing needed.\n",
    "# df_features = pd.read_csv('features_data.csv')\n",
    "\n",
    "# # Preprocessing for clean merging in SQL\n",
    "# df_features['filename'] = df_features['filename'].str[:15]\n",
    "# df_features = df_features.drop(columns='Unnamed: 0')\n",
    "# df_features['actor'] = df_features['filename'].str[:4]\n",
    "# df_features.to_csv('features_data.csv', sep=',', encoding='utf-8')\n",
    "\n",
    "# In postgreSQL, I created a database with several tablesd from the CREMA-D dataset and joined them together. \n",
    "# A .txt file of command line actions can be found in the repo. I am reading that final table back into pandas here.\n",
    "# df_final = pd.read_csv('final.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data processed in external postgress database\n",
    "\n",
    "# read processed total dataset\n",
    "df_2 = pd.read_csv('features_data_v2.csv')\n",
    "# read processed data ONLY anger vs neutral\n",
    "# df_AN = pd.read_csv('anger_neutral_df.csv')\n",
    "# # read processed data ONLY anger/disgust vs neutral/happy\n",
    "# df_ADHN = pd.read_csv('anger_neutral_happy_disgust.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "F    1268\n",
       "H    1267\n",
       "A    1266\n",
       "D    1264\n",
       "S    1263\n",
       "N    1082\n",
       "Name: emotion_label, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2.emotion_label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAD8CAYAAACRkhiPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEiFJREFUeJzt3X+s3Xddx/Hnyw0Wf4AUd4e1P7xj6TCMaGXXuUTBKQpjI2yYoF0MDCUp4GbE+IMOEkcwS8YvFxd1prCGzcyN4RhrsiIUMEwTBmtn2U/m7kZld61t3QxghjMdb/843+uO7bm3t/ec3rPez/ORnJzveZ/P9/v93E/avu738/3RVBWSpDb9wLg7IEkaH0NAkhpmCEhSwwwBSWqYISBJDTMEJKlhhoAkNcwQkKSGGQKS1LATx92BIzn55JNrcnJy3N2QpOPGzp07/6OqJhbS9jkfApOTk+zYsWPc3ZCk40aSf1toW6eDJKlhhoAkNcwQkKSGGQKS1DBDQJIaZghIUsMMAUlqmCEgSQ0zBCSpYc/5O4Z1fJjcdPvY9r37yvPHtm/peOeRgCQ1zBCQpIYZApLUMENAkhpmCEhSw7w6aJkZ51U6ko4/HglIUsMMAUlqmCEgSQ0zBCSpYUcMgSRbkuxPcl9f7ZNJdnWv3Ul2dfXJJN/r++5v+tY5M8m9SaaTXJ0kx+ZHkiQt1EKuDvoE8JfA9bOFqvrN2eUkHwW+3df+kapaP2A71wAbgTuBbcC5wGePvsuSpFE54pFAVd0BPDnou+63+d8AbpxvG0lWAi+sqq9UVdELlAuPvruSpFEa9pzAq4B9VfVwX+3UJP+S5MtJXtXVVgEzfW1mupokaYyGvVnsIv7/UcBeYG1VPZHkTOAzSc4ABs3/11wbTbKR3tQRa9euHbKLkqS5LPpIIMmJwK8Dn5ytVdXTVfVEt7wTeAQ4nd5v/qv7Vl8N7Jlr21W1uaqmqmpqYmJisV2UJB3BMNNBvwp8o6r+b5onyUSSE7rllwLrgEerai/w3SRnd+cR3grcNsS+JUkjsJBLRG8EvgK8LMlMkrd3X23g8BPCrwbuSfJ14O+Bd1bV7EnldwEfB6bpHSF4ZZAkjdkRzwlU1UVz1N82oHYLcMsc7XcArzjK/kmSjiHvGJakhhkCktQwQ0CSGmYISFLDDAFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwwwBSWqYISBJDTMEJKlhhoAkNcwQkKSGGQKS1DBDQJIatpD/aH5Lkv1J7uurvT/J40l2da/z+r67LMl0koeSvK6vfm5Xm06yafQ/iiTpaC3kSOATwLkD6ldV1frutQ0gycuBDcAZ3Tp/neSEJCcAfwW8Hng5cFHXVpI0RiceqUFV3ZFkcoHbuwC4qaqeBr6ZZBo4q/tuuqoeBUhyU9f2gaPusSRpZIY5J3Bpknu66aIVXW0V8Fhfm5muNlddkjRGiw2Ba4DTgPXAXuCjXT0D2tY89YGSbEyyI8mOAwcOLLKLkqQjWVQIVNW+qnqmqr4PfIxnp3xmgDV9TVcDe+apz7X9zVU1VVVTExMTi+miJGkBFhUCSVb2fXwTMHvl0FZgQ5KTkpwKrAO+BtwFrEtyapLn0zt5vHXx3ZYkjcIRTwwnuRE4Bzg5yQxwOXBOkvX0pnR2A+8AqKr7k9xM74TvQeCSqnqm286lwOeAE4AtVXX/yH8aSdJRWcjVQRcNKF87T/srgCsG1LcB246qd5KkY8o7hiWpYYaAJDXMEJCkhhkCktQwQ0CSGmYISFLDDAFJatgR7xOQnusmN90+lv3uvvL8sexXGiWPBCSpYYaAJDXMEJCkhhkCktQwQ0CSGmYISFLDDAFJapghIEkNMwQkqWGGgCQ17IghkGRLkv1J7uurfTjJN5Lck+TWJC/q6pNJvpdkV/f6m751zkxyb5LpJFcnybH5kSRJC7WQI4FPAOceUtsOvKKqfhr4V+Cyvu8eqar13eudffVrgI3Auu516DYlSUvsiCFQVXcATx5S+3xVHew+3gmsnm8bSVYCL6yqr1RVAdcDFy6uy5KkURnFOYHfAT7b9/nUJP+S5MtJXtXVVgEzfW1mupokaYyGepR0kvcBB4EbutJeYG1VPZHkTOAzSc4ABs3/1zzb3Uhv6oi1a9cO00VJ0jwWfSSQ5GLgDcBvdVM8VNXTVfVEt7wTeAQ4nd5v/v1TRquBPXNtu6o2V9VUVU1NTEwstouSpCNYVAgkORd4D/DGqnqqrz6R5IRu+aX0TgA/WlV7ge8mObu7KuitwG1D916SNJQjTgcluRE4Bzg5yQxwOb2rgU4CtndXet7ZXQn0auADSQ4CzwDvrKrZk8rvonel0Q/SO4fQfx5BkjQGRwyBqrpoQPnaOdreAtwyx3c7gFccVe8kSceUdwxLUsMMAUlqmCEgSQ0zBCSpYYaAJDXMEJCkhhkCktQwQ0CSGmYISFLDDAFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwwwBSWqYISBJDVtQCCTZkmR/kvv6ai9Osj3Jw937iq6eJFcnmU5yT5JX9q1zcdf+4SQXj/7HkSQdjYUeCXwCOPeQ2ibgi1W1Dvhi9xng9cC67rURuAZ6oQFcDvw8cBZw+WxwSJLGY0EhUFV3AE8eUr4AuK5bvg64sK9+ffXcCbwoyUrgdcD2qnqyqv4T2M7hwSJJWkLDnBN4SVXtBejeT+nqq4DH+trNdLW56odJsjHJjiQ7Dhw4MEQXJUnzORYnhjOgVvPUDy9Wba6qqaqampiYGGnnJEnPGiYE9nXTPHTv+7v6DLCmr91qYM88dUnSmAwTAluB2St8LgZu66u/tbtK6Gzg29100eeA1yZZ0Z0Qfm1XkySNyYkLaZTkRuAc4OQkM/Su8rkSuDnJ24FvAW/umm8DzgOmgaeA3waoqieT/BlwV9fuA1V16MlmSdISWlAIVNVFc3z1mgFtC7hkju1sAbYsuHeSpGPKO4YlqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwxZ0iaikw01uun0s+9195flj2a+WJ48EJKlhhoAkNcwQkKSGGQKS1DBDQJIaZghIUsMMAUlqmCEgSQ0zBCSpYYaAJDXMEJCkhi06BJK8LMmuvtd3krw7yfuTPN5XP69vncuSTCd5KMnrRvMjSJIWa9EPkKuqh4D1AElOAB4HbqX3H8tfVVUf6W+f5OXABuAM4CeALyQ5vaqeWWwfnqvG9WAxSTpao5oOeg3wSFX92zxtLgBuqqqnq+qbwDRw1oj2L0lahFGFwAbgxr7Plya5J8mWJCu62irgsb42M11NkjQmQ4dAkucDbwQ+1ZWuAU6jN1W0F/jobNMBq9cc29yYZEeSHQcOHBi2i5KkOYziSOD1wN1VtQ+gqvZV1TNV9X3gYzw75TMDrOlbbzWwZ9AGq2pzVU1V1dTExMQIuihJGmQUIXARfVNBSVb2ffcm4L5ueSuwIclJSU4F1gFfG8H+JUmLNNR/L5nkh4BfA97RV/5QkvX0pnp2z35XVfcnuRl4ADgIXLIcrwySpOPJUCFQVU8BP3ZI7S3ztL8CuGKYfUqSRsc7hiWpYYaAJDXMEJCkhhkCktQwQ0CSGmYISFLDDAFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwwwBSWqYISBJDTMEJKlhhoAkNWzoEEiyO8m9SXYl2dHVXpxke5KHu/cVXT1Jrk4yneSeJK8cdv+SpMUb1ZHAL1fV+qqa6j5vAr5YVeuAL3afAV4PrOteG4FrRrR/SdIiHKvpoAuA67rl64AL++rXV8+dwIuSrDxGfZAkHcEoQqCAzyfZmWRjV3tJVe0F6N5P6eqrgMf61p3papKkMThxBNv4harak+QUYHuSb8zTNgNqdVijXphsBFi7du0IuihJGmToI4Gq2tO97wduBc4C9s1O83Tv+7vmM8CavtVXA3sGbHNzVU1V1dTExMSwXZQkzWGoEEjyw0leMLsMvBa4D9gKXNw1uxi4rVveCry1u0robODbs9NGkqSlN+x00EuAW5PMbuvvquofktwF3Jzk7cC3gDd37bcB5wHTwFPAbw+5f0nSEIYKgap6FPiZAfUngNcMqBdwyTD7lCSNjncMS1LDDAFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkho2igfISVpCk5tuH9u+d195/tj2rWPDIwFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwwwBSWqYISBJDVt0CCRZk+QfkzyY5P4kv9/V35/k8SS7utd5fetclmQ6yUNJXjeKH0CStHjDPDvoIPCHVXV3khcAO5Ns7767qqo+0t84ycuBDcAZwE8AX0hyelU9M0QfJElDWPSRQFXtraq7u+XvAg8Cq+ZZ5QLgpqp6uqq+CUwDZy12/5Kk4Y3knECSSeBnga92pUuT3JNkS5IVXW0V8FjfajPMERpJNibZkWTHgQMHRtFFSdIAQ4dAkh8BbgHeXVXfAa4BTgPWA3uBj842HbB6DdpmVW2uqqmqmpqYmBi2i5KkOQwVAkmeRy8AbqiqTwNU1b6qeqaqvg98jGenfGaANX2rrwb2DLN/SdJwhrk6KMC1wINV9ed99ZV9zd4E3NctbwU2JDkpyanAOuBri92/JGl4w1wd9AvAW4B7k+zqau8FLkqynt5Uz27gHQBVdX+Sm4EH6F1ZdIlXBknSeC06BKrqnxk8z79tnnWuAK5Y7D4lSaPlHcOS1DBDQJIaZghIUsMMAUlqmCEgSQ0zBCSpYYaAJDXMEJCkhhkCktSwYR4bIakxk5tuH8t+d195/lj22wKPBCSpYcv6SGBcv7VI0vHCIwFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUsCUPgSTnJnkoyXSSTUu9f0nSs5b0PoEkJwB/BfwaMAPclWRrVT2wlP2QdHwZ5z0/y/1u5aU+EjgLmK6qR6vqf4CbgAuWuA+SpM5S3zG8Cnis7/MM8PNL3AdJWrDl/rykpQ6BDKjVYY2SjcDG7uN/JXloBPs+GfiPEWxnuXFcDueYDOa4HO6YjUk+ONTqP7nQhksdAjPAmr7Pq4E9hzaqqs3A5lHuOMmOqpoa5TaXA8flcI7JYI7L4ZbDmCz1OYG7gHVJTk3yfGADsHWJ+yBJ6izpkUBVHUxyKfA54ARgS1Xdv5R9kCQ9a8kfJV1V24BtS71fRjy9tIw4LodzTAZzXA533I9Jqg47LytJaoSPjZCkhi3LEEiyJcn+JPcdUv+97pEV9yf50Lj6Ny6DxiXJ+iR3JtmVZEeSs8bZx6WWZE2Sf0zyYPfn4ve7+ouTbE/ycPe+Ytx9XSrzjMmHk3wjyT1Jbk3yonH3dSnNNS593/9Rkkpy8rj6uBjLcjooyauB/wKur6pXdLVfBt4HnF9VTyc5par2j7OfS22Ocfk8cFVVfTbJecCfVNU5Y+zmkkqyElhZVXcneQGwE7gQeBvwZFVd2T3jakVVvWeMXV0y84zJauBL3QUeHwRoZUxg7nGpqgeSrAE+DvwUcGZVHTf3UyzLI4GqugN48pDyu4Arq+rprk1TAQBzjksBL+yWf5QB920sZ1W1t6ru7pa/CzxI7872C4DrumbX0ftHsAlzjUlVfb6qDnbN7qQXCs2Y588KwFXAnzDg5tfnumUZAnM4HXhVkq8m+XKSnxt3h54j3g18OMljwEeAy8bcn7FJMgn8LPBV4CVVtRd6f/mBU8bXs/E5ZEz6/Q7w2aXuz3NF/7gkeSPweFV9faydWqSWQuBEYAVwNvDHwM1JBj3GojXvAv6gqtYAfwBcO+b+jEWSHwFuAd5dVd8Zd3+eC+YakyTvAw4CN4yrb+PUPy70xuF9wJ+OtVNDaCkEZoBPV8/XgO/Te+5H6y4GPt0tf4rek16bkuR59P5S31BVs2Oxr5sDnp0Lbmr6cI4xIcnFwBuA36rleELxCAaMy2nAqcDXk+ymN0V2d5IfH18vj05LIfAZ4FcAkpwOPB8fhgW9cwC/1C3/CvDwGPuy5LqjwWuBB6vqz/u+2kovIOneb1vqvo3LXGOS5FzgPcAbq+qpcfVvXAaNS1XdW1WnVNVkVU3S+2XzlVX172Ps6lFZrlcH3QicQ+83/X3A5cDfAluA9cD/AH9UVV8aVx/HYY5xeQj4C3rTZf8N/G5V7RxXH5dakl8E/gm4l97RIcB76c2B3wysBb4FvLmqDj2pvizNMyZXAycBT3S1O6vqnUvfw/GYa1y6pyDMttkNTB1PVwctyxCQJC1MS9NBkqRDGAKS1DBDQJIaZghIUsMMAUlqmCEgSQ0zBCSpYYaAJDXsfwFVODk6YDpmgwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# observing distribution of certain variables, after outliers were removed\n",
    "plt.hist(df_2['contrast']);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating pairplots to observe feature interactions with the target\n",
    "\n",
    "def plot_features(df, sample_size=500):\n",
    "    '''Pairplot creator to assess feature strength. '''\n",
    "    \n",
    "    \n",
    "    sample = (df.drop(['Unnamed: 0', 'filename', 'actor', 'emotion_label', 'affect',\n",
    "        'Age', 'Sex', 'Race', 'Ethnicity',\n",
    "       'Female', 'Male', 'African American', 'Asian', 'Caucasian', 'Unknown'], axis=1)\n",
    "                .sample(1000, random_state=44)) \n",
    "    sns.pairplot(sample,hue='affect_binary', plot_kws=dict(alpha=.3, edgecolor='none'))\n",
    "\n",
    "plot_features(df_2)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Splitting train and test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating X and y for model. This model only predicts on Anger or Neutral.\n",
    "\n",
    "X, y = df_2.drop(['Unnamed: 0','filename', 'actor', 'emotion_label', 'affect','affect_binary'], axis=1), df_2['emotion_label']\n",
    "\n",
    "# hold out 20% of the data for final testing\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=10)\n",
    "\n",
    "# applying scalar\n",
    "std = MinMaxScaler()\n",
    "std.fit(X_train.values)\n",
    "X_tr = std.transform(X_train.values)\n",
    "X_te = std.transform(X_test.values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating X and y for model. This model predicts on anger/disgust or neutral/happy\n",
    "\n",
    "X_1, y_1 = df_ADHN.drop(['Unnamed: 0','Unnamed: 0.1','filename', 'actor', 'emotion_label', 'affect',\n",
    "        'Sex', 'Race', 'Ethnicity',\n",
    "       'sex_binary', 'affect_binary', 'chroma', 'flatness', 'contrast_std','Age', 'Female', 'Male', 'African American', 'Asian',\n",
    "       'Caucasian', 'Unknown', 'mfcc'], axis=1), df_ADHN['affect_binary']\n",
    "\n",
    "#X, y = np.array(X), np.array(y)\n",
    "\n",
    "# hold out 20% of the data for final testing\n",
    "X_train_1, X_test_1, y_train_1, y_test_1 = train_test_split(X_1, y_1, test_size=.2, random_state=10)\n",
    "\n",
    "# applying standard scalar\n",
    "std = MinMaxScaler()\n",
    "std.fit(X_train_1.values)\n",
    "X_tr_1 = std.transform(X_train_1.values)\n",
    "X_te_1 = std.transform(X_test_1.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Selecting different classifiers for modeling.\n",
    "\n",
    "# logistic regression\n",
    "lr = LogisticRegression()\n",
    "# K Nearest Neihbors\n",
    "knn = KNeighborsClassifier(n_neighbors=5, weights='distance')\n",
    "# Random Forest\n",
    "forest = RandomForestClassifier(n_estimators=100)\n",
    "# Xgradientboost\n",
    "xgb1 = XGBClassifier()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setting parameter values to loop through for each model in Gridsearch CV.\n",
    "\n",
    "params_lr = [{\n",
    "    'penalty': ['l1', 'l2'], \n",
    "    'C': np.logspace(0,4,10)\n",
    "}]\n",
    "\n",
    "params_knn = {\n",
    "    'n_neighbors': [3, 5, 10, 20]\n",
    "}\n",
    "\n",
    "params_xgb = [{\n",
    "    'max_depth':range(2,6,2),\n",
    "    #'scale_pos_weight':[1,2],\n",
    "    'min_child_weight':range(1,6,2)\n",
    "}]\n",
    "\n",
    "kf = KFold(n_splits=5, shuffle = True, random_state = 1001)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# grid search CV to optimize parameter selection with logistic regression and KNN.\n",
    "\n",
    "# clf_log = GridSearchCV(estimator=lr, param_grid=params_lr, cv=kf)\n",
    "#lr.fit(X_tr, y_train)\n",
    "# clf_knn = GridSearchCV(estimator=knn, param_grid=params_knn)\n",
    "# knn.fit(X_tr, y_train)\n",
    "xgb2 = xgb1.fit(X_tr, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=KFold(n_splits=5, random_state=1001, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=3, min_child_weight=1, missing=None, n_estimators=100,\n",
       "       n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "       silent=True, subsample=1),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid=[{'max_depth': range(2, 6, 2), 'min_child_weight': range(1, 6, 2)}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# only anger/disgust and neutral/happy with XGradient Boost\n",
    "clf_xgb = GridSearchCV(estimator=xgb1, param_grid=params_xgb, cv=kf)\n",
    "clf_xgb.fit(X_tr, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=<generator object _BaseKFold.split at 0x108121678>,\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=3, min_child_weight=1, missing=None, n_estimators=100,\n",
       "       n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "       silent=True, subsample=1),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid=[{'max_depth': range(3, 10, 2), 'scale_pos_weight': [1, 2, 3, 4, 5], 'min_child_weight': range(1, 6, 2)}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# only anger and neutral class with XGradient Boost\n",
    "\n",
    "clf_xgb_AN = GridSearchCV(estimator=xgb1, param_grid=params_xgb, cv=skf.split(X_tr,y_train))\n",
    "clf_xgb_AN.fit(X_tr, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': 5, 'min_child_weight': 1, 'scale_pos_weight': 1}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_xgb_ADHN.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': 3, 'min_child_weight': 3, 'scale_pos_weight': 1}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_xgb_AN.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Making models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_confusion_matrix(model, threshold, X, y):\n",
    "    '''This function creates a confusion matrix showing the predictions of a class if the propbabilty of\n",
    "    being in that class is greater than the threshold'''\n",
    "    \n",
    "#     y_hat = (model.predict_proba(X)[:,0] > threshold)\n",
    "#     confusion = confusion_matrix(y, y_hat)\n",
    "    confusion = confusion_matrix(y, model.predict(X))\n",
    "    plt.figure(dpi=80)\n",
    "    sns.heatmap(confusion, cmap=plt.cm.Blues, annot=True, square=True, fmt='d')\n",
    "           #xticklabels=['negative', 'positive'],\n",
    "           #yticklabels=['negative', 'positive']);\n",
    "    #plt.xlabel('prediction')\n",
    "    #plt.ylabel('actual')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAElCAYAAABH+TEbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAH6NJREFUeJzt3Xm8VVX9//HX+4KKOAEaOYDigFNaDjlbmfOUmJplgop9VdQyf+RQWl9NrcwMU8Epp29qg0P5FTVN/aaGiaCCQySSiGMyqIiAgMDn98fe93a43ss953LPuuds3k8f+3HOWXufvda55/Lxc9deey1FBGZmlkZDZzfAzGx54qBrZpaQg66ZWUIOumZmCTnompkl5KBrZpaQg66ZWUIOumZmCTnompkl5KBrZpZQ185uQFtW3vbbvk+54N4fO7yzm2BV1q0r6ojztCcefDRueIfU3VGc6ZqZJVTzma6ZWRPVf57ooGtm9UM11VPQLg66ZlY/Grp0dguWmYOumdUPdy+YmSXk7gUzs4Sc6ZqZJeRM18wsIWe6ZmYJOdM1M0vIma6ZWULOdM3MEnKma2aWkIOumVlCDe5eMDNLx5mumVlCvpBmZpaQM10zs4Sc6ZqZJeT5dM3MEnL3gplZQu5eMDNLyJmumVlCznTNzBJypmtmlpCDrplZQgXoXqj//22Y2fJDDZVvbZ1S6ibpbkkvSxov6QFJ/fJ9vfPXkyS9KGn3kve1um9pHHTNrH5IlW/luQ7YLCK2Ae7NXwNcDIyOiP7AYOA2SV3L2NcqB10zqx9VyHQjYl5E3B8RkReNBjbKnx8JjMiPGwtMBXYvY1+r3KdrZvUjTZ/uacBISWsCDRExvWTfFGD9pe1r6+TOdM2sbkhqzzZU0psl29ClnP8coD9wbl4UzQ8peb60fa1ypmtmdUPtyHQjYhgwrIxznwEcBuwdEXOBuXnQ/lRJRrsB8HpEvNvavrbqcaZrZvVD7djKOW2W/R4F7BMRM0t23QGcmh+zA7A2MKqMfa1ypmtmdaM9mW4Z5+wD/BKYDPw1r2N+ROwEnA3cImkSsAAYFBEL87cubV+rHHTNrG5UI+hGxJu0khNHxFRg30r3LY2DrpnVjYaG+u8RddA1s/pR/3cBO+iaWf2oRvdCag66ZlY3HHTNzBJy0DUzS8hB18wspfqPuQ66ZlY/nOmamSXkoGtmlpCDrplZSvUfcx10zax+ONM1M0vIQdfMLCEHXTOzhBx0zcxSqv+Y66BrZvXDma6ZWUKexNzMLKX6T3QddFNaacWu3HLxYDbfaB3mzlvA1Bmz+M5Pfs/r/36Pa847ml222YiP5n/Mh3Pm8b2f38HzL78FwHU/HsieO23GjJlzAPi/0S9xzq/u7syPYmU66YTjeXfGdBrUQPdVVuH75/yIzbfYomn/NVcN5+oRV3Ln3SPp33/TTmxpfXD3glXshj8+wYOjJgAw5OtfZMSPjuIrp4xg5KPPc+pFv2PRosUc8IWtuPWSb/HZQy9oet+lNz3ENX94vLOabe30i1/+itVXXx2A/3vkYc770Tn84c4/AfDPCf/g+efGs84663ZmE+tKEYJu/XeQ1JH5CxY2BVyAMS9MYcP11gLgvsdeYNGixXn5q6y/Ts9C/IIt7xoDLsDsDz+koSH7ThcsWMBPL7qAc350nr/nCkiqeKs1STJdSV2AvhExJUV99eKUo77E/Y+/8InyU7+5Bw+OmkBENJWdNnBPjj9sN954531+PGJkU9eD1b5zf3AWY8c8BcBV114PwIgrL+eggw+hT5++ndm0ulOLQbRSVQ+6kr4A/A5YDKwvaQfgtIgYVO26a9mZx+/LJuv35oCfXLFE+TcO3IHD99mOvY+/rKns/OEj+feMWUQEh3z5s9w9/BS2HvBj5ny0IHWzrR1+8rNLALjn7j9x2aWXcOKQU/jHiy9w+tAzOrlldaj+Y26S7oVLgC8B7wJExFhgu9YOljRU0puN28Jp4xM0Ma3TB+3FgL22YcCpV/HRvI+byo/YdzvOPekADj55ONPfn91U/vb0D5qy3nv++jwfzpnHpv0+nbzdtmwOOfSrjB3zFGOeGs2UVydz4L57ccA+ezJ16jucfOK3GPW3xzq7iTXP3Qtl1hERrzT78K2maBExDBjW+Hrlbb8drR1bj04buCdf2397DhpyJR/M/qip/PB9tuW8Uw/moCFX8sY77y/xnvV69+CtaTMB2HHrfvRaYxVeeWN60nZb5WbPns3cuXPo3Tv7H+QjDz/EGj168F8nDuGEk05uOu6Affbkiquu8eiFMtRiEK1UiqA7T9KqQABI+gwwL0G9NWe93j34+fcOY/Ib03ng198FYMGChXzxmEu56SfHMfXdWdx+2UlNxx940pW898EcrrtgIL17rc6ixYuZN/9jjj7rBmbNXi5/hHVl9ocfMvT07zB//nwaJHr26sWVI64tRODoLEX40an0Yk1VKpD2Bc4DNgYeAPYHBkbEw+W8v2iZrn3S+2OHd3YTrMq6de2Y3tj+Zz5QcTyY9Iv9aypUVz3TjYi/SJpEFmwFXBQR/6p2vWZWPEXIdFOMXjgYuD8irq52XWZWbEXomkkxeuEM4A1Jl0jaPEF9ZlZQUuVbral60I2IPYAvAB8BD0h6UtIJ1a7XzIqnoUEVb7UmyW3AETE5Is4DNgPGA9ekqNfMisWZbpkkbSfpCuA1oC/w9RT1mlmx+OaIMkh6HlgB+B9gu4h4u9p1mlkx1WJ3QaVS3BxxckQ8kaAeMyu4WsxcK1W1oCtpw4h4FXhf0pbN90fEhBbeZmbWqgLE3KpmulcCBwP3tbAvgI2qWLeZFZAz3aWIiIPzxw2rVYeZLV8KEHOrP3pB0icW82qpzMysLR69UJ71WyjbOEG9ZlYwNRhDK1bNC2knACcCm0oaU7JrDWBiteo1s+Kqxcy1UtXMdP8CTAKuBs4sKZ8FPF/Fes2soKoRc/Mbtw4BNgC2jogX8/KVgF8C+5EtvDAuIgbm+/qT3XuwFjATOK7cEVnVvJD2GtkdaFtUqw4zW75UKdO9k2xZsVHNyi8mW9tx04gISeuU7LsWuC4ibpZ0BHADsEs5laW4I20tsknMPwd0ayyPiB2rXbeZFUs1Ym5EPJ6d+z8nl7QKMBjoE/lKDxHx73xfb7J1HvfND78LGC6pXzkrnqeYe+FG4E1gbeBCYBrwYIJ6zaxg2jN6oflit5KGllHVxmSL6f5Q0tOS/iZpr3xfX+DtiFgIkAfl12l50MAnJBm9EBGHSDo6IkZKehD4c4J6zaxg2pPpNl/stkwrkN3ANSEivi/pc8DDJXfXNl82qOyWpQi6jSv/zpfUi6zTuU+Ces2sYBKOXniNrD/3NoCIeE7Sq8BngAlAH0ldI2Khskb1Jct225Sie2FiHmxvBUYDTwHjEtRrZgWTaj7diJgBPEI2cgFJGwAbAhMjYhpZDBuYH344MKWc/lxIszDloPzp5ZKeBnri7gUza4dqZLqSRgADyK47PSxpdkRsAgwBbpT0c2ARcGLjxTTgJOBmSeeQDYM9ttz6Uoxe6F7ychxARCyqdr1mVjzVCLoRcSpwagvlk4E9WnnPRMocItZcij7d2TTrdJb0MTAGOCFvvJlZmzyJeXl+RBZ4byK7wncs0B14h2yA8R4J2mBmBVCAu4CTBN3DImL7ktdXSBoVEbtL+l6C+s2sIIow90KK0QvdJTVNWJ4/XzN/uTBB/WZWEEVYDThFpvtDYIykZ8j6drcHhkhaFbgjQf1mVhANtRhFK5RiyNhdkh4Hdibr0x2dj3MD+Gm16zez4ihAzE3SvQCwHrBaRNwDLGg2W4+ZWVmKsHJEiuV6hpDNO3lhXrQm+a11ZmaVaFDlW61JkemeRNa1MAsgIl4Beieo18wKpgiZbpIJbyLio2Yf3qMWzKxiNRhDK5Yi6E6XtCn5XWmSBgFvJKjXzApG5c+gWLNSBN3Tgd8Cm0maAswFvpKgXjMrmFrso61UiiFj/5K0M7AZ2ZCxiZ7wxszaoxb7aCuVItNt9EFe33qSiIiyJvw1M2tUgJibZGrH44ArgI/JZmKHrH/XIxjMrCK+I608PwJ2jIiXEtRlZgVWgJibZvSCA66ZdYQizKeb4uaIP0r6tqRekro3bgnqNbOCaZAq3mpNm5luyZLDLYqICW2c4uL88Qqyvlzlj13KaaCZWaPaC6GVK6d74b6l7AuyteFbPyAi1aQ6ZlZwy8WQsYjYMEVDzMzaUoAu3coupEnqAmwAdGssK6N7wcysQywXmW4jSQcA15NNzTgH6AG8DjgTNrMkChBzK8p0fwLsCtwdEdtKGghsXZ1mmZl90nKV6QIREa9J6pq/uDVfhsfMLInlrU/34/zxTUlfAV4D+nR8k8zMWra8ZbqXS+pJtrrv78n6dE+vSqvMzFpQ/yG3gqAbEb/Lnz4D9K9Oc8zMWleLd5hVqpLRC8e0VB4Rv+m45piZta4AMbei7oXS1R66AbsDowEHXTNLYrnq042Ir5W+lrQh/1lW3cys6goQc9s/tWNEvCpps45sjJnZ0ixvfboHlrzsAuxEvsJvNb016vJqV2GdbOeLHunsJliVjT9/rw45TwFibkWZ7pklzxcCrwBf79jmmJm1rksBom4lfbpfrmZDzMzaUoQLaWXPdStpTDllZmbV0qDKt1pTSffCEsfm0zyu2rHNMTNrXS0G0Uq1melKOlPSdGArSdMaN+AD4G9Vb6GZWU5SxVutKSfTvQ64A7gaGFJSPisi3q9Kq8zMWlCETLec5Xo+AD6QdBIwLSLmAUjqJqlvRLxR7UaamUExhoxVsmjknc1eq4UyM7OqKcIS7JUE3RUbs1yAiPgIWKnjm2Rm1rKGdmxtkXSFpCmSQtJWeVk3SXdLelnSeEkPSOpX8p7eedkkSS9K2r2Sz1CukNS7pNJPU4zpLc2sTkiVb2W4k2wCr9ealV8HbBYR2wD35q8bXQyMjoj+wGDgtsZVddpSyZCxK4BRkn5DFmwHka2bZmaWRDW6CyLicVjyxov8r/r7Sw4bzZKLNhxJvihvRIyVNJUscD/aVn2V3JF2k6TJwEFkcy4cHxGjyn2/mdmy6sQu2tOAkVkbtCbQEBHTS/ZPAdYv50SVTHizFnAE8DlgZeDLkoiIHcs9h5nZsmjPkDFJQ4GhJUXDImJYBe8/h2y1nNIhs80n+yq7ZZX06d4IvAmsDVwATAMerOD9ZmbLpD2jFyJiWET0KdkqCbhnAIcBB0TEXICIeDff96mSQzcAXi/rM5T/cVk/In4OzIuIkXlDdq3g/WZmy6RKF9JaqUtDgaOAfSJiZrPddwCn5sftQJaMltXdWsmFtAX543xJvYCZeAl2M0uoGnekSRoBDCALnA9Lmg3sAfwSmAz8Nb/INj8idsrfdjZwi6RJZLFxUEQsLKe+SoLuxDzY3kp2Je8DYFwF7zczWyaqwijViDiVPGv9RHWtv2cqsG976qtk9MKg/Onlkp4GegJ/bk+lZmbt0bWSDtEa1a410iLiiY5uiJlZW2px1rBKtXthSjOz1JaLWcbMzGpFARJdB10zqx+1OGtYpRx0zaxuuHvBzCyhAiS6DrpmVj8aCjCbrIOumdUNZ7pmZgm5T9fMLCGPXjAzS6gAMddB18zqhzNdM7OEChBzHXTNrH4UYJIxB10zqx+eZczMLKH6D7kOumZWR7o40zUzS6cAMddB18zqh/t0zcwS8ugFM7OEnOmamSVU/yHXQdfM6ogzXTOzhNyna2aWkDNdM7OE6j/kOuiaWR0pQKLroGtm9cMLU5qZJeRM18wsITnTNTNLx5mumVlC7tM1M0uooQB3RzjomlndcJ+umVlCDfUfcx10zax+ONM1M0vIoxfMzBJypmsd4oZrR3D9tSO49fb/ZeNN+rdZbrXrrAM2ZY/N1mLdHitz+FWjeWXaHABW6CK+t19/dt14TT5etJiX3vmQc/84gdW6deX647Zren+3FbqwXs9u7PmLvzHro4Wd9TFqlvt0bZlN/OcEXnzhOdZee52yyq22PTxhGjc/8Ro3H7/9EuXf3XsTFgcccuWTAKy16ooAfDhvIV+/ZkzTccfsuj7bb9DDAbcVRch0k416k+QA38yCBQu49OILOfMH/71EZ1Vr5Vb7nn1tJtNmzV+irNsKDQzYdh2ufOSVprIZsxe0+P4B26zD3ePermob65lU+VZrqh50JX1G0njg1fz19pJ+Xu1668Gvr76S/Q78Cuuu16escqtPfXt1Z+bcjznhi/247cQduHHw9uy4Yc9PHPfZPquzRvcVePzldzuhlfVB7djKOq+0n6RnJI2T9KKkY/Py3pIekDQpL999WT9Dikx3OPBtYEb++lngoNYOljRU0puN21VXXpagiem98Nx4/vmPFzn8yKPKKrf61bVB9O3VncnT53D0dWO5+P6JXHzEVvTsvsISxx267brc+9w7LFocndTS2tcgVby1RdlyFL8FBkfEtsDBwLWSVgMuBkZHRH9gMHDbsv7VnuJP/tUiYlTjMhsREZI+bu3giBgGDGt8/d6cRYX8DRz37FhemzKZww7eB4Dp06Zy+qkncMTXv9li+Tn/fQG77PbFzmyytdO/P5jHosXB/c+/A8DLU2fz9syP2Lj3Kjw9ZSaQdUHsu9WnGfjrsZ3Z1JpX5d6CHvnj6sC7wHzgSGBDgIgYK2kqsDvwaHsrSRF0F0paAQgASX2AxQnqrWnHDD6BYwaf0PT6qwftzaWXX83Gm/Tn2ONPbLHc6tPMuR8z5tX32HWTNRk16V3WWaMb6/ZYmSkz5jYds+9nPs2kqbOXKLMWVCHq5ongkcAfJc0BegKHAasBDRExveTwKcD6y1JfiqA7HPgTsJak84FjgHMS1GuW3A8O3Iw9Nl+LNVddkWuP2Za5CxZxyBVPctG9L/HjAVvmoxiCC0e+tMTFtEO3XZe7n/UFtLa0Z/SCpKHA0JKiYflf1I37uwI/AAZExBOSdgDuBj5Lniwu0YRlpIjq//UuaVdgAFmDR0bE38p9b1G7F+w/9vzFo53dBKuy8efv1SE56pjJH1QcD3bcaI2l1i3p88BvImLLkrKxwFnAvUC/xmxX0hjgrIh4tNJ2NKp6pitpq4j4O/D3atdlZsVWpT7dN4A+kjaLiImSNgE2Bl4G7gBOBc7PM+C1gVHLUlmK7oV7JL0L3AT8NiJmJqjTzIqoOn26UyWdBNwpaXFeyykR8Zaks4FbJE0CFgCDImKZ7lypetCNiI0k7UE23OJCSX8BboyIh6pdt5kVSzlDwNojIn4H/K6F8qnAvh1ZV5I70iLi0Yg4FugHzAQeSFGvmRVLtW6OSCnJrbmSegMDgeOALsDZKeo1s4KpxShaoRQX0u4BdgbuAk6IiKeqXaeZFVMRJrxJken+ATgyIuYlqMvMCqwWJ7CpVNWCrqSVImI+2Y0RDZK6l+6PCN96Y2YVKUDMrWqm+ySwHTCb7K4ONXvsUsW6zayIChB1qxZ0I2K7/LEAK9WbWS0oQp9uivl0ryqnzMysLUWYxDzFhbSdWyjbJUG9ZlYwNRhDK1bNC2lfI5uLsp+k20t2rQHMqVa9ZlZgBYi61cx0XwbuA3bMHxvNAh6pYr1mVlBF6NOt5oW054DnJN3XbBJgM7N2qcU+2kpVs3vhuxFxOXCmWvhJRcRZ1arbzIqpADG3qt0LjXeguf/WzDpGAaJuNbsXrs0ff1ytOsxs+VKEPt0U43QvkNRDmfskzZB0eLXrNbPiKcI43RR3iw3IV4vYG1gI7Aacm6BeMyuYIgTdFDdHNC63/iXgjnwNogTVmlnRFKF7IUXQnSPp+8A3gN0kNQArJqjXzAqmCPlaiu6F48hW0DwrX29oI+C2BPWaWcF4uZ4yRMS/gNMlfVpS7/z1z6pdr5kVUC1G0QqlGL2whaQXgH8CEyU9L2mzatdrZsWjdvxXa1J0L1wF/CwiekVET+CnwDUJ6jWzginC6IUUQbdnRPy28UVE/B7okaBeMyuYIvTppgi6iyRt2fgi71pYvJTjzcxaVoCom2LI2LnAY5LG5a8/BwxKUK+ZFUwt9tFWKsXohQfyTHcnsv/vPBkRM6pdr5kVTy320VYqRaYL2c0Qq5KtArxSojrNrGAKEHOTDBn7BjCebOmebwDjJB1Z7XrNrHiKMHohRaZ7PrBjRLwKIKkf8ABwe+tvMTNrSQ1G0QqlCLozGgMuQERMkeQ+XTOrWC1mrpVKMWTsIUk/lLS2pHUknQvcLam7pO4J6jezgijAiLEkme55+eMFzcovIbuw1iVBG8ysAIqQ6aYYMpYimzaz5UAR5uJONWTMzGyZ1X/IddA1szpSgETXQdfM6odvAzYzS6n+Y66DrpnVjwLEXAddM6sf7tM1M0vIfbpmZgkVIdP1jQtmZjlJ50kKSVvlr/tL+ruklyWNKV0Fp70cdM2sblRzakdJ2wE7A6+XFF8LXBcRm5JNXXDDsn4GB10zqxvVWoJd0krACOAUsjlhkNQb2A64NT/sLmDDfHradnPQNbO60Z5MV9JQSW+WbENbOPUFwK2l09ACfYG3I2IhQEQEWRa8/rJ8Bl9IM7O60Z7raBExDBjW6jmlXYAdgO+39PYOaMISnOmaWf2ozoS6XwI2B16VNAXoAzwIbAX0kdQVQNkUZ31Zss+3Yg66ZlY3qtGnGxEXR8S6EdEvIvoBbwL7RcT/AOOAgfmhhwNTImLKsnwGdy+YWd3ohHG6JwE3SzoHmAUcu6wndNA1s7qRIujm2W7j84nALh15fgddM6sbvg3YzCyhItwGrGzomdUSSUPzYS5WQP5+l28OujVI0psR0aez22HV4e93+eYhY2ZmCTnompkl5KBbm9zfV2z+fpdj7tM1M0vIma6ZWUIOumZmCTno1jhJ/SSd2Kzsfkkbd1abrP0kDZH0//Ln20g6stn+8ZJW7pzWWQru061xkvYALo2Iz3d2W6xjSToOODgijujstlg6znTbIV+47mxJT0l6VdLgkn39Jd0naayk5ySdUrLvcEkvSRon6Yf5eVbN990q6WlJz0u6N18qBOAaYMs8A7onP3aKpK0k7S7phWZte0zSIfnz/SSNkvRM3tYvVvtnU0T593S+pCfyBQqPKtm3v6Rn8+/tscaFC/Pfgyfy34EXJF2Ul58v6dL8+70A2Dv/bq8pqWtVSQMljSypR/nv2mfz14Py7/TZvN6tUv5MbBlEhLcKN7LZ5L+bP98C+JBsHosuwFhg83xfd+B5snWWegPvAv3zfafn51k1f71Wyfm/DwzPn+8BPN2s/inAVvnzicDn8+cbAf/O27IR8Hdg9XzfJsBbwAqd/fOrty3/ns4r+RnPIJvMunf+fOt839HAi/nzy4FzSs7RK388n+wvF4DjgDtbqGtVYOX83Gvn5V8Gnsmf7wbcB6yUv/4C8Fxn/5y8lbd5wpv2uw0gIv4paSGwNrA68Bng9/rPzByrAVuSzUb/bERMystvAi4rOd/RkgYBK5H9g3unzHbcTPaP9+n88baIWChpf7JA+7iWnCWkLzC53A9pTa4HiIjJkkaRBboPgfER8UK+7zZJIyStAzwO/ELSKsBjwMOVVBYRH0m6i2wC7UuBwWS/MwADgM8BT5V8t5+StGJELFiWD2nV5+6F9ptX8nwRWXYpYEZEbFOybRgRt+b7WuxAl7Q78G3ggIjYGhgKdCuzHb8BjpTUjWyC5cZ/mAIeaNaW9SLCAbdjBK1/pxERd5FlpBPJvtt721HHTcBxklYHDgZ+m5cLuLHZd7uuA259cNDtWBOBuZKOaSyQtImkXsBoYHtJm+S7Smeg70k2K/17klYkm62+0SxgjdYqjIi3yLLcXwHvRMQ/8l1/AfYv7euTtGO7P5kdD9loEmB3YBTwJLCNpC3yfd8A3oyIdyT1B6ZFxG+As4CdWzhnW9/taLJ/o5cAD0XEe/mukcAxkvrm9TZI8oXWOuHuhQ6U/1n/FeAySWeQ9fFOB46OiLckDQHuk/Qu2T+cj4G5wJ/J/ox8iWx9pr8D++WnfR6YKOlFYHJEHNJC1TcBtwMnl7RlkqSBwPX5EKQVgWfJ+h2tcvMlPQF8CvhORLwB2QUt4DZJXYCZQOMQsK+RdRktIMtMh7RwzkeAMyQ9BzwZES0dcxNZ0D2gsSAiHle2fMz/5vWuQNbH+3QHfE6rMg8ZS0jSahHxYf58MPCtiNi9k5tlbZAUwGoRMbuz22L1z5luWqdJ+hrZz/094IRObo+ZJeZM18wsIV9IMzNLyEHXzCwhB10zs4QcdC2pxrkH8udNM24t5XjPsmaF4tELtkwkdY2Ihe15b0RcU8Zh/YATgetK3ndge+ozqwXOdK1FbcysFZK+J+lR4Gd52RmSxuSzXt1fcrfUGpLulDRB0oNk80E0nqcp681fn53PyPWcpNGSurOUWdby55tIejif5Wu8pEObtbPF2eDMOoszXVuaiIjdJG0EjJE0qvFOLLIZrvYAkPRNYFNgl4hYlN+lNZxsYpb/BmZFxJaS1iK7K+725hVJOhY4FNgtImZJ6gnMJ7uTa2nzCd8G3BAR1+W33o6W9ExJO+dFxE75rbpjJN3S3szcrCM407WlaZpZi2yugS+U7Lux5PmhwN7AM5LGk801sEG+78vADfl5ZgB/bKWug4GrI2JWfuz7EbFoaY2TtBqwTcn5J+XtLL3Lr2k2OKBxNjizTuNM1ypReidN6S2xAi6KiBv5JLVQ1lEaz938Dp/S1y3NBmfWaZzp2tK0NLNWS+4BTslnU0PSCpK2zfc9QjYXLPn+ry7lHCfn0xgiqUc+mUurM3HlWfF48hnb8hENuwFPlP0JzRLz//VtaVqcWau5iLhF0prAo/nkMF3J/uQfB1wI3ChpAvAa8NBSzrEu8KSkxtnX9qbtWdaOBq6V1LgSx3+11k6zWuC5F6xFnlnLrDrcvWBmlpAzXTOzhJzpmpkl5KBrZpaQg66ZWUIOumZmCTnompkl5KBrZpaQg66ZWUL/H+wJPFOThiNlAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 480x320 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# anger/disgust vs happy/neutral\n",
    "\n",
    "make_confusion_matrix(clf_xgb_ADHN, .5, X_te, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAElCAYAAABH+TEbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xm8XdP9//HX+2ZAhIhEBJlNrdJG1KyqrSqtlg58acXQfjWKUkH1l1K+dKBDWkqVFm0NVdOXKqLlWyUqxsRQFalICJWpJIgkbvL5/bH3vY6bO5xz7z3r3rPv++mxH/fsvfbea5175JNP1llrbUUEZmaWRl1XN8DMrCdx0DUzS8hB18wsIQddM7OEHHTNzBJy0DUzS8hB18wsIQddM+vRJK0t6WZJz0qaIWmKpFFNzjlCUkjav+TYkPzcWZKekrRHOfU56JqZwaXA1hExFvhTvg+ApGHABGBak2vOBaZFxJbAUcDVknq3VZGDrpn1aBGxPCJuj3em504DxpSccilwErCiyaUHAxfl93gYmA+0me22GZW72jrbH+95ygX38v3nd3UTrMoG9uulzrhPe+LB8hkXnQxMLDk0OSImt3LJCcCtAJK+BvwjIh6U3nkLkgYBdRGxsOS6OcCIttrT7YOumVlH5AG2tSDbSNIkYEvgGEmjgaOB3Vu6ddPLy6nD3QtmVjtUV/lW7q2lU4DPAftFxDJgV2BT4J+S5gC7AJdJOjoiFufXbFRyi5HAC23V46BrZrVDqnwr67aaCBwKfDwiXgOIiGsiYmhEjIqIUWR9vV+JiF/ll10PHJdfvyMwFJjaVl3uXjCz2lHXq9NvmY9O+AkwG/hr3ne7IiJ2buPS04ArJc0CVgLjI6K+rfocdM2sdlTQXVCuiJhHGf2xEbFXk/35wD6V1uega2a1o8zugu7MQdfMakcVMt3UHHTNrHY40zUzS8iZrplZQs50zcwScqZrZpaQM10zs4Sc6ZqZJeSga2aWUJ27F8zM0nGma2aWkL9IMzNLyJmumVlCznTNzBKqwnq6qTnomlntcPeCmVlC7l4wM0vIma6ZWULOdM3MEnKma2aWkIOumVlC7l4wM0vIma6ZWULOdM3MEnKma2aWkDNdM7N05KBrZpaOg66ZWUq1H3MddM2sdjjTNTNLyEHXzCyhujoPGTMzS6f2E11q/68NM+sxJFW8lXHPtSXdLOlZSTMkTZE0Ki8bku/PkvSUpD1KrmuxrDUOumZWM6oRdHOXAltHxFjgT/k+wLnAtIjYEjgKuFpS7zLKWuSga2Y1oxpBNyKWR8TtERH5oWnAmPz1wcBF+XkPA/OBPcooa5GDrpnVjPYEXUkTJc0r2Sa2Uc0JwK2SBgF1EbGwpGwOMKK1srbeg79IM7Pa0Y4v0iJiMjC5rNtLk4AtgWOAdYBoekrprdvTOme6ZlYzqtini6RTgM8B+0XEsohYnB/fqOS0kcALrZW1VY+DrpnVjGoF3bzL4VDg4xHxWknR9cBx+Tk7AkOBqWWUtcjdC2ZWM6oxI03SMOAnwGzgr3kdKyJiZ+A04EpJs4CVwPiIqM8vba2sRQ66ZlY7qjA5IiLmtXTniJgP7FNpWWscdM2sZnjtBTOzhBx0zcwSctA1M0vIQdfMLKXaj7kOumZWO5zpmpkl5EXMzcxSqv1E19OAU1qrb2+um3w0T9z8HaZd+y1uufBYRmyy4bvO+dKnd+at6Rey34e2bTy20cD+3HLhsTx5y3d45PpJ7DZ2TNNbWzf1k/O+x4Gf3Jtdtt+G5/41q/H4A/ffx5FfPIgvHXwgXzn8EGbNfKYLW1k7qrn2QioOuolddtP9vP/As9nlkHO5476nuOiMQxvLNhuyAf/9+d158Inn33XNOSccwENPzmG7A85mwplXccX3j6RXL390teCje3+CS6+4iqGbbNp4bOnSJZx1+mmc+d1zufq6mznuhJM589vf7MJW1g4HXavIipX13Dn16cb9h56cw+jNBjfuX3jGoXzzxzeyYuW7p29/fp9xXHLdvQA8+vQLLFi8lN3Gbp6m0dYh2+/wQYZsPPRdx1568UUGDtyQ0WOyz3DcB3fklX+/zDP/fLq5W1gJB90ySeql/JlD9o5jD/0wt9/7JABHH7QH/3zu3zz81Nx3nbPhgHWpk1j06huNx+a+/B+GbzIwaVut8wwfMZLXXn2Vp554HIB7/u8uli1bxr9ffqmLW9b9FSHoVv2LNEkfAn4PrCZbcX1H4ISIGF/turuzU7+8D1uMGMJ+37uAkZsO4qjP7s5Hj2p+neVoslZyN/z/yCrQf731+MGPz+eiCyazbNmbfGDsOEaP2Zzevf29dpsK8P9+ik/5h8CHgRsge5aQpHEtnZyva9n4OI3em+5O7yFjq97IlL4x/mMc8LGxfGrCz3lr+dvs8v7RbLLRAGbcdDoAGw9an4vP/CL/c9GfuOJ//w7A4IH9G7PdEZtsyIv/frXL2m8dt/0OH+TiX/8WgJUrV/Kpvfdk1Gh/QdqW7pi5VipF0O0dEc81+WWtbOnkpo/WWGf745s+EqOmnXDYRzlo3x341DE/Z8kbbwHwhymP8IcpjzSec+evTuRnv7ubO+57CoCb/jKdCQfvyfcuuZ0dthnBxoPW5+8znuuS9lvnWLRwIYM3yh46cPmvLmaHHXdm+IiRXdyq7s9BtzzLJfUnf56QpPcByxPU2+1sNmQDzjv5c8x+cSFTfnUiACtX1rPn4T9u9brTz7+Fy757OE/e8h1Wvr2KL5/xO1atWp2iydZBP/rBOdx7z//xn8WL+PoxX6Zfv37c8Mc7ueQXF/DEjMdYtaqebd8/lm+fdU5XN7UmFCDmoneeOlylCqR9gDOBzYEpwL7AYRFxVznXFy3TtTW9fP/5Xd0Eq7KB/Xp1Srjc8tQpFceDWT/at1uF6qpnuhHx5/xxFvuSdYN/NyL+Ve16zax4ipDpphi9sD9we0RcXO26zKzYitCnm2Kc7inAi5J+KOk9Ceozs4KSKt+6m6oH3YjYC/gQ8BYwRdIDko6udr1mVjx1dap4626SzEiLiNkRcSawNTAD+GWKes2sWJzplknSOEkXAHOB4cB/pajXzIrF04DLIOkJoA/wW2BcRLxc7TrNrJi6Y3dBpVJMjvhaRNyfoB4zK7jumLlWqmpBV9LoiHgeeFXSNk3LI8Lr2JlZRQoQc6ua6f4c2B+4rZmyALy6h5lVxJluKyJi//zn6GrVYWY9SwFibvVHL0i6uZxjZmZt8eiF8oxo5pifNWNmFeuGMbRi1fwi7Wjgq8BWkh4qKRoAzKxWvWZWXN0xc61UNTPdPwOzgIuBU0uOLwWeqGK9ZlZQBYi5Vf0ibS7ZDLT3VqsOM+tZqpHp5rNlPwOMBLaLiKfy42sBPwE+Qfa0m+kRcVhetiXZhK/BwGvAkeUOg00xI20w2SLmHwDWbjgeETtVu24zK5YqZbo3kD3LcWqT4+eSPVB3q4gISZuUlF0CXBoRv5H0BeAyYNdyKkux9sLlwDxgKHAOsAC4M0G9ZlYw1Ri9EBH3RsS8JvWsCxwFTIr88ToR8e+8bAgwDrgqP/1GYLSkUeW8hxRBd0REnAcsj4hbgc8BuyWo18wKpj2rjEmaKGleyTax7ZrYHFgMnC7pEUn3SfpYXjYceDki6gHyoPwCzY/UWkOKIWMNT/5dIWlDsv6PYQnqNbOCaU+fbtMnjJepD9ms2acj4luSPgDcVbKkQdNntZXdsBRBd2YebK8CpgFLgOkJ6jWzgkk4emEuWX/u1QAR8bik54H3AU8DwyT1joh6ZX8TDCfLdtuU4sGU4/OX50t6BBgI3FHtes2seFKN042IRZLuJhu5cLukkcBoYGZELJA0HTgM+A3weWBORMwp594pRi/0K9mdDhARq6pdr5kVT5WGjF0EHED2Zf9dkt6IiC2AY4DLJZ0HrAK+2vBlGjAB+I2kSWRzD44ot74U3Qtv0KT/Q9LbwEPA0RHh2WlmVpZqLGIeEccBxzVzfDawVwvXzKTMIWJNpQi6Z5AF3ivIOpuPAPoBr5CNddsrQRvMrAA8I608n4uIHUr2L5A0NSL2kHRygvrNrCCKsPZCinG6/SQ1Llievx6U79YnqN/MCqIITwNOkemeDjwk6VGyvt0dgGMk9QeuT1C/mRVEXXeMohVKMWTsRkn3AruQ9elOi4gFefH3q12/mRVHAWJuku4FgM2A9SLij8DKJgtHmJmVpQhPjkjxuJ5jyJZAOyc/NIh8loeZWSXqVPnW3aTIdCeQdS0sBYiI54AhCeo1s4IpQqabZMGbiHiryZv3qAUzq1g3jKEVSxF0F0rainxWmqTxwIsJ6jWzglH5i3l1WymC7jeAa4CtJc0BlgGfTlCvmRVMd+yjrVSKIWP/krQLsDXZkLGZXvDGzNqjO/bRVipFpttgSV7fZpKIiLLWnjQza1CAmJtkaccjgQuAt8kWBYasf9cjGMysIp6RVp4zgJ0i4pkEdZlZgRUg5qYZveCAa2adoRrr6aaWYnLETZKOl7ShpH4NW4J6zaxg6qSKt+4mRaZ7bv7zArK+XOU/eyWo28wKpPuF0Mq1GXRLHjncrIh4uo3yVIvqmFnB9ZQhY7e1UhZkz4Y3M6u6AnTpth10I2J0ioaYmbWlp2S6jST1AkYCazcca6t7wcyssxQg5pYfdCXtB/yabD3cN4ENgBcAZ8JmlkRPy3S/B+wG3BwR20s6DNiuOs0yM1tTj+jTLRERMVdS73znqvzZZ2ZmSfS0TPft/Oc8SZ8G5gLDOr9JZmbNq/2QW1nQPV/SQLJHql9L1qf7jaq0ysysGd1xhlmlyg66EfH7/OWjwJbVaY6ZWcsKEHMrGr1weHPHI+J3ndccM7OW9bQ+3dJH7KwN7AFMAxx0zSyJAsTciroXDirdlzQaOKfTW2Rm1oIe1afbVEQ8L2nrzmxMc159+MJqV2FdbOPDr+zqJliVLblmfKfcpwAxt6I+3U+W7PYCdiZ/rLqZWQq9ChB1K8l0Ty15XQ88B/xX5zbHzKxl1fgiTdIFwGfI1pXZLiKekrQ22dDYbYBlwCvAMRExJ79mCNn3WZsDK/KyqeXUV0mf7kcqeB9mZp2uStOAbwB+CDQNmpcCd0RESDo+398nLzsXmBYR+0raEbhB0uYRUd9WZWUvMC7poXKOmZlVS50q39oSEfdGxLwmx5ZHxO0R0dCFOo13rx1+MHBRfu7DwHyyEV1tv4dyTsq9KyvOl3nsX8H1ZmYdIqk920RJ80q2ie2o+gTg1rwNg4C6iFhYUj4HGFHOjcp5XM+pwDeBAZIWlBT1A64us8FmZh3Wnu6FiJgMTG5vnZImkc3CPab0tk1PK/d+5fTpXgpcD1zcpNKlEfFquRWZmXVU6sELkk4BPgfsHRHLACJicZ5Bb1SS7Y4kW1+8TW12L0TEkvwbuwnA/IiYGxFzgbckDW/PGzEza4+Uj2DPuyEOBT4eEa81Kb4eOC4/b0dgKGt+Edf8e6igDTc0bVMzx8zMqqauHVtbJF0kaR7ZUrV3SfqXpGHAT8hWU/yrpBmSHiy57DRgN0mzgN8A48sZuQCVjdPtGxHLG3Yi4i1Ja1VwvZlZh1SjeyEijiPPWptW18o183ln+FhFKnpyhKQhEbEAQNLGrTXKzKyz9bS1Fy4Apkr6HVmwHU/23DQzsyQKEHMrmpF2haTZwKfIhkt8udxpb2ZmnaFHPZhS0mDgC8AHgHWAj0giInaqVuPMzEoVoXuhktELlwPzyIZGnA0sAO6sRqPMzJojVb51N5UE3RERcR6wPCJuJRswvFt1mmVmtqZqrL2QWiVfpK3Mf66QtCHwGn4Eu5klpAIMmKok6M7Mg+1VZCvuLAGmV6VVZmbN6F3Jv827qUpGLzQ8b+N8SY8AA4E7qtIqM7Nm9LSnATeKiPs7uyFmZm3pjn20lWr3gynNzFIrQKLroGtmtaMI43QddM2sZrh7wcwsoQIkug66ZlY76nrYOF0zsy7lTNfMLCH36ZqZJeTRC2ZmCRUg5jromlntcKZrZpZQAWKug66Z1Y4CLDLmoGtmtaPHrjJmZtYVaj/kOuiaWQ3p5UzXzCydAsRcB10zqx3u0zUzS8ijF8zMEnKma2aWUO2HXAddM6shznTNzBJyn66ZWUJFyHSL8BeHmfUQasdW1n2lT0h6VNJ0SU9JOiI/PkTSFEmz8uN7dPQ9ONM1s5pRjURXWfp8DfCRiHhC0ijgGUk3AecC0yJiX0k7AjdI2jwi6ttbn4OumdWMKj+YcoP85/rAYmAFcDAwGiAiHpY0H9gDuKe9lbh7wcxqhtSeTRMlzSvZJpbeMyKCLLjeJGkuMBU4AlgPqIuIhSWnzwFGdOQ9ONM1s5qhdmS6ETEZmNziPaXewP8DDoiI+/NuhJuB9wOxRhM6yJmumdWM9mS6ZRgLbBoR90PWjQC8TBZ0kbRRybkjgRc68h4cdM2sZtShircyvAgMk7Q1gKQtgM2BZ4HrgePy4zsCQ8m6H9rN3QtmVjPqqpAmRsR8SRPIRiasJutCODYiXpJ0GnClpFnASmB8R0YugIOumdWQ9vTpliMifg/8vpnj84F9OrMuB10zqxl1tT8hzUHXzGpHtTLdlBx0zaxmFGDpBQddM6sdznStU90/9T5+fv5PWb16NfX19Rx51Ff4zIGf7epmWQXW6lPH5V/fk/dsNoBlK+pZsOQtTrrsQV5Y9CaD11+bS762G6M3Xo8Vb6/mpMunMW1mNtnpogm7Mnb0IFavDupXreasa6fzt3+80sXvpvspQp+ushlw3dfy+jVmhBRSRPDh3Xfh11f8lq22fg8vvTSPA/ffj3umPsC66/bv6uZV1caHX9nVTeg0a/WpY8/3DeUvM14G4Oh9tuaT44bx2XPv5sKv7sq8xW9y7o1PMG7MIH574p6MPelmVq0OBvTrw5JlbwOw3ciB3DJpb8ZMuL4r30qnWnLN+E4Jl/c9+2rF8eBDWw3sVqHakyO6mddffx2AN994gwEbbEDfPn27uEVWiRVvr24MuACPzFrIqCHZX5qf3WUkv/rzTAAem72YhUuXs+vWQwAaAy7AgH596ea5UJep0oy0pJJ1L0jq3dFBxUUmiR9N/hkTTzyeddbpx9KlS5h8/oX06eugW8sm7Pse7njsJQb270udxOLXVzSWvbDwDYYNXrdx/6xDtufAnUeywbp9Oeynf+uK5nZ73TCGVqzqma6k90maATyf7+8g6bxWzn/XikDn/7TFdSoKpb6+nst+dQk/+/kvmHLXX7n0st9w+qTTWPLaa13dNGunkw/Yls2Hrs85100HWCN7bfoUhLOunc7Yk27myAvu5ZwvjqNPL/9DtKk6qeKtu0nxqV4IHA8syvcfAz7V0skRMTkihjVsJ540saVTC2XmM/9k4YIFbD9uBwC23e79DNloCDNnPtPFLbP2+PqntuHTO47gC+fdzVsrV/HqGysBGLTeWo3nDB+8LvMWvbnGtfc89Qr91+nD+0ZssEZZT1etJ0eklCLorhcRjQtE5GtXvt3K+T3S0KGbMH/+K8x5fjYAL8ydy4svvsjIUaO7uGVWqeM++V6+sNsoDvzBXe/qq735wbkcvc/WAIwbM4ghA9bhgZkL6FUnxgxdr/G8cZsPYqP112bOgjeSt73bK0DUTdGnWy+pD/m6lJKGAasT1FtTBg0ezBlnns3JJ51InUQQTDrjTDbeeOOubppVYNMN+/H9wz7I8/Nf59ZvfxyAlfWr+dh37uDM3z/GpcfuzmOTD2Bl/Wom/GIqq1YHfXvXcfGE3Vi/X19WrV7NshX1HP6ze3ntzZVd/G66nyKM0636kDFJhwGHkK1NeTlwODApIq4t5/qeMmSsJyvSkDFrXmcNGXto9pKK48FOYwZ0q0hd9Uw3Iq6SNBs4AOgHHBER91W7XjMrnm4VPdup6kFX0rYR8Xfg79Wuy8wKrgBRN0Wf7h8lLQauAK6JCI+BMrN26Y5DwCpV9dELETEGOBXYGXhO0u8lfbza9ZpZ8RRg8EKaacARcU9EHAGMAl4DpqSo18wKpgBRN8k0YElDgMOAI4FewGkp6jWzYinCkLEUX6T9EdgFuBE4OiIerHadZlZMBejSTZLp/gE4OCKWJ6jLzAqsADG3ekFX0loRsQL4X6BOUr/S8ohYVq26zaygChB1q5npPgCMA94gmwKsJj97VbFuMysg9+m2IiLG5T+9Pp2ZdYoi9OmmWE/3F+UcMzNrSwFGjCX5Im2XZo7tmqBeMyua7hhFK1TNL9IOAg4GRkm6rqRoALDmys1mZm1wn27rngVuA3bKfzZYCtxdxXrNrKCK0KdbzS/SHgcel3RbRCysVj1m1nMUIOZWtXvhxIg4Hzi16QP4ACLim9Wq28wKqgBRt5rdCw0z0Nx/a2adwn26rYiIS/Kf/1OtOsysZylCn26KcbpnS9pAmdskLZL0+WrXa2bFI1W+VXZ/nSkpJG2b728p6e+SnpX0kKRtOvoeUswWOyB/WsTeQD2wO/DtBPWaWcGoHf+VfW9pHNm8ghdKDl8CXBoRWwE/BC7r6HtIEXQbHrf+YeD6iJiZoE4zK6BqZbqS1gIuAo4lWxumYR3wccBV+Wk3AqMljerIe0gRdN+U9C2yx7D/RVId0DdBvWZWMO2ZBixpoqR5JdvEZm59NnBVRDxfcmw48HJE1ANERJBlwSM68h5STAM+Ejge+GZEzJe0BXB1gnrNrGja8UVaREwGJrd4S2lXYEfgW81d3vEWvFvVg25E/Av4hqSNJQ3J939Q7XrNrHiqNGTsw8B7gOfzOQXDgDuBScAwSb0jol5Z4XDe3edbsRSjF94r6Ungn8BMSU9I2rra9ZpZ8VSjTzcizo2ITSNiVESMAuYBn4iI3wLTyZ7vCPB5YE5EzOnIe0jRp/sL4AcRsWFEDAS+D/wyQb1mVjBdsLTjBGCCpGfJuh++0tEbpujTHRgR1zTsRMS1kvw0YDOrXILJEXm22/B6Jp28FG2KTHdV6YDivGthdSvnm5k1q5rjdFNJkel+G/ibpOn5/geA8QnqNbOCKcI04BSjF6bkme7OZP84eCAiFlW7XjMrngLE3CSZLmSTIfqTjXlbK1GdZlYwRch0UwwZOwSYQfbonkOA6ZIOrna9ZlZEtf9oyhSZ7lnATg3T6/J5y1OA61q+xMxsTUXIdFME3UWl85kjYo4k9+maWcUKEHOTBN2/SDod+DXZ7+zLwM2S+gFExLIEbTCzAnCmW54z859nNzn+Q7Iv1nolaIOZFUBzz1usNSmGjKWYgGFmPUDth9x0Q8bMzDqsAImug66Z1Y7uOK23Ug66ZlY7aj/mOuiaWe0oQMx10DWz2uE+XTOzhNyna2aWUBEyXY+hNTNLyJmumdWMImS6DrpmVjPcp2tmlpAzXTOzhAoQcx10zayGFCDqOuiaWc1wn66ZWULu0zUzS8hB18wsIXcvmJklVIRMVxHR1W2wJiRNjIjJXd0Oqw5/vj2bg243JGleRAzr6nZYdfjz7dm84I2ZWUIOumZmCTnodk/u7ys2f749mPt0zcwScqZrZpaQg66ZWUIOut2cpFGSvtrk2O2SNu+qNln7STpG0kn567GSDm5SPkPSOl3TOkvBfbrdnKS9gB9HxAe7ui3WuSQdCewfEV/o6rZYOs5020FSSDpN0oOSnpd0VEnZlpJuk/SwpMclHVtS9nlJz0iaLun0/D7987KrJD0i6QlJf5I0JL/sl8A2eQb0x/zcOZK2lbSHpCebtO1vkj6Tv/6EpKmSHs3bume1fzdFlH9OZ0m6X9Kzkg4tKdtX0mP55/Y3Sdvkx7fMz39c0pOSvpsfP0vSj/PP92xg7/yz/WVJXf0lHSbp1pJ6lP+/9v58f3z+mT6W17ttyt+JdUBEeKtwAwI4MX/9XuB1snUsegEPA+/Jy/oBTwDjgCHAYmDLvOwb+X365/uDS+7/LeDC/PVewCNN6p8DbJu/ngl8MH89Bvh33pYxwN+B9fOyLYCXgD5d/furtS3/nM4s+R0vAobnn+kiYLu87EvAU/nr84FJJffYMP95Ftm/XACOBG5opq7+wDr5vYfmxz8CPJq/3h24DVgr3/8Q8HhX/568lbd5wZv2uxogIv4pqR4YCqwPvA+4Vu+szLEesA0wDHgsImblx68Aflpyvy9JGg+sRfYH7pUy2/Ebsj+8j+Q/r46Iekn7kgXae/XuVUKGA7PLfZPW6NcAETFb0lSyQPc6MCMinszLrpZ0kaRNgHuBH0laF/gbcFcllUXEW5JuBA4DfgwcRfb/DMABwAeAB0s+240k9Y2IlR15k1Z97l5ov+Ulr1eRZZcCFkXE2JJtdERclZc124EuaQ/geGC/iNgOmAisXWY7fgccLGlt4Aje+YMpYEqTtmwWEQ64nSNo+TONiLiRLCOdSfbZ/qkddVwBHClpfWB/4Jr8uIDLm3y2mzrg1gYH3c41E1gm6fCGA5K2kLQhMA3YQdIWedERJdcNBJYC/5HUF5hQUrYUGNBShRHxElmW+zPglYj4R170Z2Df0r4+STu1+53ZlyEbTQLsAUwFHgDGSnpvXnYIMC8iXpG0JbAgIn4HfBPYpZl7tvXZTiP7M/pD4C8R8Z+86FbgcEnD83rrJPmL1hrh7oVOlP+z/tPATyWdQtbHuxD4UkS8JOkY4DZJi8n+4LwNLAPuIPtn5DPAPLK+2E/kt30CmCnpKWB2RHymmaqvAK4DvlbSllmSDgN+nQ9B6gs8RtbvaJVbIel+YCPg6xHxImRfaAFXS+oFvAY0DAE7iKzLaCVZZnpMM/e8GzhF0uPAAxHR3DlXkAXd/RoORMS9kiYBt+T19iHr432kE96nVZmHjCUkab2IeD1/fRTwlYjYo4ubZW2QFMB6EfFGV7fFap8z3bROkHQQ2e/9P8DRXdweM0vMma6ZWUL+Is3MLCEHXTOzhBx0zcwSctA1M0vIQdeSaljwJX/duMxhK+d7aUsrFA8Zsw6R1Dsi6ttzbUT8sozTRgFfBS4tue6T7anPrDtwpmvNamM5w5B0sqR7gB/kx06R9FC+1ODtJVNUB0i6QdLTku4kW4Sn4T6NWW++f1q+DOLjkqZJ6kcrS1vmr7eQdFe+tOIMSQc2aWezS3CadRXtoPvXAAABl0lEQVRnutaaiIjdJY0BHpI0tWH6K9mygnsBSPoisBWwa0SsyqfGXki2GtZ3gKURsY2kwWRTka9rWpGkI4ADgd0jYqmkgcAKsumzrS3ifjVwWURcmq93ME3SoyXtXB4RO+frIzwk6cr2ZuZmncGZrrWmcTlDsgVePlRSdnnJ6wOBvYFHJc0gW+BlZF72EeCy/D6LgJtaqGt/4OKIWJqf+2pErGqtcZLWA8aW3H9W3s7SqdWNS3ACDUtwmnUZZ7pWidLpi6XrEAj4bkRczprUzLHO0nDvptMqS/ebW4LTrMs407XWNLecYXP+CBybL2GJpD6Sts/L7iZbgJu8/LOt3ONr+dqxSNogX0GrxeUP86x4BvkymfmIht2B+8t+h2aJ+W99a02zyxk2FRFXShoE3JOvyNWb7J/804FzgMslPQ3MBf7Syj02BR6Q1LDk5d60vbTll4BLJDU8/ui/W2qnWXfgBW+sWV7O0Kw63L1gZpaQM10zs4Sc6ZqZJeSga2aWkIOumVlCDrpmZgk56JqZJeSga2aW0P8H7jbbFUr50JAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 480x320 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# only anger and neutral\n",
    "\n",
    "make_confusion_matrix(clf_xgb_AN, .5, X_te, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def roc_plots(model):\n",
    "    probs = model.predict_proba(X_te)\n",
    "    probs = probs[:, 1]\n",
    "    auc = roc_auc_score(y_test, probs)\n",
    "    print(f'AUC {auc}')\n",
    "    print(f'f1 {f1_score(y_test, model.predict(X_te))}')\n",
    "    print(f'recall {recall_score(y_test, model.predict(X_te))}')\n",
    "    print(f'precision {precision_score(y_test, model.predict(X_te))}')\n",
    "    print(f'accuracy {accuracy_score(y_test, model.predict(X_te))}')\n",
    "    \n",
    "    fpr, tpr, thresholds = roc_curve(y_test\n",
    "                                     , probs)\n",
    "    plt.plot(fpr, tpr,lw=2)\n",
    "    plt.plot([0,1],[0,1],c='violet',ls='--')\n",
    "    plt.xlim([-0.05,1.05])\n",
    "    plt.ylim([-0.05,1.05])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.57419355, 0.26122449, 0.28947368, 0.37777778, 0.31654676,\n",
       "       0.40979955])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_test, lr.predict(X_te), average=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC 0.9729912715694706\n",
      "f1 0.9376443418013857\n",
      "recall 0.9620853080568721\n",
      "precision 0.9144144144144144\n",
      "accuracy 0.9425531914893617\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHNxJREFUeJzt3X2QXHW95/H3t5+7Z5LwkHDFPE2AgEREgTGAKBADGALC3l1UcNnVFCXXe0X/cHer3HLXvcX9x3Vr16pbhWiqLuHq7pV79Q83QiIiFy4sEpPBQIBoJAJKBCVCmMxMP/f57h99EpvJJNOZ9OPpz6sqVd3TJ92/Mz3zyS+/c/pzzN0REZFoiXV7ACIi0noKdxGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBiW698MKFC31kZKRbLy8i0peeeuqpP7r7otm261q4j4yMMDY21q2XFxHpS2b2m2a207KMiEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hE0Kzhbmb3mNnrZvbcUR43M/tbM9trZrvM7MLWD1NERI5HMzP3e4F1x3j8WmBl+Od24O4TH5aIiJyIWc9zd/fHzGzkGJvcCHzb69fr22ZmJ5nZ6e7+WovGKD1iw6btPLJnf7eHIdK3MsQ42ZK85iVe/up1bX2tVnyIaTHwSsP9feHXjgh3M7ud+uyeZcuWteCl5XgonEW65/2x+fzX9FlMepV/W9zV9tdrRbjbDF+b8arb7r4R2AgwOjqqK3M36JfgXXPOIjZtWN3tYYj0jaAYUPhJgfLOMrFTYuSuz/HS8qVtf91WhPs+oHGkS4BXW/C8kdIr4a1wFukcD5yJTRMEbwSkL02TvSKLJWeaD7deK8J9M3CHmd0HXAyMD/p6+1yDXMErEg1BPsCyhsWM7JossfkxEu/sbJXXrK9mZt8FrgQWmtk+4L8BSQB3/yawBVgP7AXywIZ2DbbbTnT2rfAWiTZ3p/xsmcKPC2Q/nCV9YZrUu1JdGUszZ8vcMsvjDnyuZSPqYccT7ApykcESjAdMbZmiurdKfHGcxNKule4CXaz87WWzzdDbfQqTiPSX8nNlph6YAofsNVnS709jsc6srR+Nwr1BM8sua86ZtSNfRAaMZYzE4gS563LET453eziAwv1tGoNdyyoicjQeOKVtJbzmZD+UJXlWksSZCcy6O1tvpHDnyBm7ll1E5Giqv6+Svz9P7bUayVVJ3B0z66lghwEP95mWYbTsIiIz8apTfLxI8adFLGsM/Zshkucmey7UDxnocNcyjIg0K3gzoPjTIqnzUmSvzhLL9XZj+kCH+yFahhGRmXjZKe8pk35Pmvhpceb/1fyeOWA6G4W7iMgMKi9WyD+QJ3grIPGOBPFF8b4JdlC4i4i8TVAIi76erhd9DX9qmPii/gn1QxTuIiIhD5yJe+tFX5nLMmQuz2CJ3jxgOhuFu4gMvCOKvhbESJze3/HY24d7RUTayN0pPVPi4F0HKe8sA5B6V6rvgx00cxeRAVV7q0Z+S57qr6vEl8RJLItWHEZrb0REmlDaVSK/NV8v+lqXJT2a7tkPI82Vwl1EBk5sKEZiaYLc+hzxk/rvTJhmKNxFJPK81lD0dXmW5JlJEmf0VtFXqyncRSTSqq+FRV+/r5F8d+8WfbXawIb7hk3buz0EEWkjrzrFx8Kir5wxdNMQqXO7c8m7bhi4cJ/eBKkWSJFoCt4MKD5ZJHV+WPSVHawzvwcm3I9W76smSJHo8LJT/mWZ9Pn9V/TVagMT7qr3FYm2yq/Doq/xgMTp/Vf01WoDE+6HqN5XJFqCfEDhoQLlXWVip8aY9+l5fVn01WoDEe46eCoSTYeLvt4MyHwwQ+ZD/Vv01WoDEe6HlmR08FQkGoKpAMuFRV9rw6KvdwxEnDUt0t+N6QdRtc4u0t/cnfIzZQoPFch+OEv6ojSpcwbn9MbjEelw1ymPItFRe6tG/oE81RerJJYlSIxEOr5O2EB8d3QQVaS/lXaVyG/Jg0Hu2hypi1KR/4TpiRqIcBeR/hYbipFYnmBo/RCxBYP1YaS5UriLSM/xmlP8abFeyRsWfSXPTHZ7WH1F4S4iPaX6WpX8D/PU/lAjdV7qcNGXHJ+m/n9jZuvMbI+Z7TWzL83w+DIze8TMdprZLjNb3/qhikiUecXJP5xn4u8mCKYChj42xNCfDynY52jWmbuZxYG7gKuBfcAOM9vs7rsbNvsvwD+5+91mtgrYAoy0YbwiElHBgYDSthKp96bIXjV4RV+t1syyzGpgr7u/CGBm9wE3Ao3h7sD88PYC4NVWDlJEoslLYdHXe8Oir8/Nj+yVkTqtmXBfDLzScH8fcPG0bf4a+LGZfR4YAq5qyehEJLIqL1SY2jKFTziJd4ZFXwr2lmnm/z0zLXj5tPu3APe6+xJgPfAdMzviuc3sdjMbM7Ox/fv3T39YRAZAkA+Y+sEUk/dNYilT0VebNDNz3wcsbbi/hCOXXW4D1gG4+5NmlgEWAq83buTuG4GNAKOjo9P/gRCRiPPAmdg0QfBWQOZDGTIfVNFXuzQT7juAlWa2AvgdcDPwyWnb/BZYC9xrZucCGUBTcxEBIJgMsKGw6OvqsOjrz3QmdjvNuizj7lXgDuBB4BfUz4p53szuNLMbws3+A/AZM3sG+C7waXfXzFxkwLk7pZ0lDn7jIOWflwFInZ1SsHdAU99hd99C/fTGxq99peH2buCy1g5NRPpZ7UCN/P15qi9XSSxPkFihQO8kfbdFpOVKz5TIbw2LvtbnSF2ooq9OU7iLSMvF5sVIjiTJrc8Rm68PI3WDwl1ETpjXnOITYdHXFVmSZyRJnqGir25SuIvICam+WmVq8xTB/oDUe1T01SsU7iIyJ15xCo8WKP2shA0bQ58YInW2LnnXKxTuIjInwYGA0o4SqQtS5NbmsIxm671E4S4iTfNiWPT1vnrR14LPLdCVkXqUwl1EmlJ5ocLUA1P4pJNYkiC+MK5g72EKdxE5pmAqoPDjAuXnysQWxRj+2DDxhSr66nUKdxE5Kg+ciXvDoq8rMmQuy2Bxra33A4W7iBxhetFX/KQ48dM0W+8nWjATkcPcndJTJcbvGqf81J+KvhTs/UczdxEBoPZmWPT1myqJkQSJMxUP/UzvnohQejos+opD7vocqfep6KvfKdxFhNiCGMkzkuSuVdFXVCjcRQaQVxuKvq7MklyRJLlCRV9RonAXGTDV31WZ+mFY9HW+ir6iSuEuMiC83FD0Nd8YvnmY5ErN1qNK4S4yIILxgNJYifRFabJrs1has/Uoi2y4b9i0vdtDEOm6oBhQ+UWF9AVp4oviLLhjgQ6YDojIhvsje/YDsOacRV0eiUh3lPeUyW/J41NOYmlY9KVgHxiRDPfGWfumDau7OBKRzgumAvI/ylPZXSF+WpzcJ3Iq+hpAkQx3zdplUB0u+hoPyFyZIfMBFX0NqkiG+yGatcugCCYCbLhe9JW7JkfspBjxRZqtDzItwIn0MXenNFZi/BvjlJ4qAZBcmVSwS7Rm7hs2bT+8JCMSdbU3wqKv31ZJrEiQPFPnrMufRCrcG4Nd6+0SZaWdJfI/ymMJI/fRHKn3quhL3i5S4X7Iy1+9rttDEGmr2EkxkmeGRV/ztLoqR4pkuItEjVed4uNFALJrVPQls1O4i/S46ith0dcbAan3qehLmqNwF+lRXnYKjxQobS8RWxBj+JPDOmgqTWtqsc7M1pnZHjPba2ZfOso2Hzez3Wb2vJn9Q2uHKTJ4gvGA0lMl0u9PM/8v5ivY5bjMOnM3szhwF3A1sA/YYWab3X13wzYrgf8MXObuB8zstHYNWCTKgkJY9HVhWPT1+QU6YCpz0syyzGpgr7u/CGBm9wE3ArsbtvkMcJe7HwBw99dbPVCRqCv/skx+a1j0tSws+lKwyxw185OzGHil4f6+8GuNzgbONrMnzGybma2b6YnM7HYzGzOzsf379WEjEYBgMmDy+5NMfW+K2FCMebfNU9GXnLBmZu4zHZb3GZ5nJXAlsAR43MzOc/e33vaX3DcCGwFGR0enP4fIwPHAmfj7sOhrTYbMpSr6ktZoJtz3AUsb7i8BXp1hm23uXgFeMrM91MN+R0tGKRIxwcEAmxcWfX0kLPrSbF1aqJllmR3ASjNbYWYp4GZg87RtfgCsATCzhdSXaV5s5UBFosDdKW4v1ou+xsKir7OSCnZpuVln7u5eNbM7gAeBOHCPuz9vZncCY+6+OXzsGjPbDdSA/+Tub7Rz4CL9pvbHGlP3T1F7pUbizIQuTi1t1dSHmNx9C7Bl2te+0nDbgS+Gf0RkmtLOEvmteSxp5G7IkTpfRV/SXvqEqkgHxE6OkTw7SW5djtiwTm+U9lO4i7SBV53iY2HR14ezJEeSJEe0DCOdo3AXabG3FX1doKIv6Q6Fu0iLeCks+tpRInaSir6kuxTuIi0SHAwo7SyRXp0muyaLpTRbl+5RuIucgCAfUNldIT0aFn3doaIv6Q0Kd5E5cHcqv6iQ/1EeLziJERV9SW9RuIscp2AiIL81T2VPhfjpcXKfzOkTptJzFO4ix+Fw0ddEQHZtlvQlaSymtXXpPQp3kSYE4wE2Pyz6ujYs+jpVs3XpXVogFDkGD8Kir7sbir7OTCrYpedp5i5yFLX9YdHXvhqJsxKkzk51e0giTVO4i8yg9PMS+R/lsZSRuzFH6j0q+pL+onAXmUHslBjJc8KiryGtXkr/UbiLAF5xCv9SAIPc2pyKvqTvKdxl4FV+UyF/f57gzYDURSr6kmhQuMvA8pJTeLhA6akSsZNjDN86THKFZusSDQp3GVjBREDpmRLpS9Jkr1DRl0SLwl0GSpAPKO8ukxnNEF8YZ8HnF+jKSBJJCncZCO5OZXdY9FV0kivqH0RSsEtUKdwl8oKJgPyWPJVf1Yu+hm4d0idMJfIU7hJpbyv6uipL+mIVfclgULhLJNXeqhGbH/tT0dfJMeKnaLYug0MLjhIpHjjFbUUO3n2Q0lMNRV8KdhkwmrlLZNRerzH1wylqr9ZIrkySOkdFXzK4FO4SCaWnwqKvjDH050Mk353Up0xloCncpa8dqgqILYyRWpUie01WRV8iKNylT3nFKTwaFn1dlSO5PElyuaoDRA5RuEvfqbwcFn0dCEiPplX0JTIDhbv0DS86+YfzlH9erhd9/bth1fKKHEVTi5Nmts7M9pjZXjP70jG2u8nM3MxGWzdEkbpgMqD8bJn0pWnm/8V8BbvIMcwa7mYWB+4CrgVWAbeY2aoZtpsHfAH4WasHKYMrmAoobi8CHC76yl2Vw5JahhE5lmZm7quBve7+oruXgfuAG2fY7m+ArwHFFo5PBpS7U362zMG7D1J4qEDtjRqAzoQRaVIza+6LgVca7u8DLm7cwMwuAJa6+/1m9h9bOL6mbNi0nUf27O/0y0qbBOMB+a15Ki9UiC+OM3S9ir5Ejlcz4T7T/3/98INmMeDrwKdnfSKz24HbAZYtW9bcCJvQGOxrzlnUsueVzvPAmfjOBMFkQPaaLOn3q+hLZC6aCfd9wNKG+0uAVxvuzwPOAx4NT0d7B7DZzG5w97HGJ3L3jcBGgNHRUafFXv7qda1+SumQtxV9rQ+Lvk7WbF1krppZwNwBrDSzFWaWAm4GNh960N3H3X2hu4+4+wiwDTgi2Ntlw6btnXgZaRMPnOKTYdHXWFj0dUZSwS5ygmadubt71czuAB4E4sA97v68md0JjLn75mM/Q3sdWpLRckz/qf6hSv6HeWqv1UienSR1roq+RFqlqQ8xufsWYMu0r33lKNteeeLDOn6bNqzuxsvKHBXHihQeLNSLvv71EMlVKvoSaSV9QlU66lBVQHxRnNS7w6KvnE5vFGk1hbt0hJfDoq+Yir5EOkHhLm1XeSks+norIP1+FX2JdILCXdomKAYUHipQfrpM7JQYw58aJrlMs3WRTujrcNdpkL3NJ53y82XSH0iTvTyrPhiRDurLcJ9eN6DTIHtHMBlQfr5M5uJMvejrCwt0wFSkC/oy3KcHu06D7L5DRV+FHxfwspM8K0n81LiCXaRL+jLcD1HdQG8IxgOmtkxR3VslvkRFXyK9oK/DXbrPA2fi2xMEUwHZj2RJj6roS6QXKNxlTmoHasQWhEVf14dFXydpti7SK7QgKsfFA6f4xLSirxVJBbtIj9HMXZpW/X1Y9PX7GslzVPQl0ssU7tKU4o4ihR8XsKwxdNOQgl2kxync5ZgOF32dFid1Xlj0ldVqnkivU7jLjLzsFB4Ji76uVtGXSL9RuMsRKr+ukH8gTzAekF6toi+RfqRwl8OCQlj09UyZ2Kkx5n1qHoll+hER6Uf6zZXDfMop/6JM5rIMmcszWEKzdZF+pXAfcMFkQPm5MplLwqKvz6voSyQKFO4Dyt0p7wqLvipOcqWKvkSiROE+gGpv1cg/kKf6YpX4UhV9iUSRwn3AeOBMfmeSIB+QvTZL+qK0zoQRiSCF+4CovVkjdlJY9PXRHLGTVPQlEmVaYI04rzmF/1fg4Dcbir5GVPQlEnWauUdY9bWw6OsPNZLnJkmtUh+MyKBQuEdUcXtY9DVkDH1siNS7FOwig0ThHjGHi77eESd1fors1Sr6EhlECveI8JJT+OcCJMKir2VJkstU9CUyqBTuEVDZWyG/JSz6ulhFXyKicO9rQT4s+tpVJrYwxrwN80gs0VsqIgr3vuYFp/zLMpkPZch8UEVfIvInTR1pM7N1ZrbHzPaa2ZdmePyLZrbbzHaZ2cNmtrz1QxWAYCKg+GQRdyd+apwFX1hA9sqsgl1E3mbWcDezOHAXcC2wCrjFzFZN22wnMOru5wPfB77W6oEOOnen9HSJg3cfpPBogeDNAEBnwojIjJpZllkN7HX3FwHM7D7gRmD3oQ3c/ZGG7bcBt7ZykIOudiAs+nqpSmJZgtz1ORV9icgxNRPui4FXGu7vAy4+xva3AVtnesDMbgduB1i2bFmTQxxsHjiT/7te9JVbnyN1YUpnwojIrJoJ95mSxGfc0OxWYBS4YqbH3X0jsBFgdHR0xueQutobNWIn/6noK35ynNgCLcGISHOaCfd9wNKG+0uAV6dvZGZXAV8GrnD3UmuGN3i85hR/WqT4eJHs2iyZizMkR/RhJBE5Ps2E+w5gpZmtAH4H3Ax8snEDM7sA+Bawzt1fb/koB0T11bDo6/UayXcnSZ2nPhgRmZtZw93dq2Z2B/AgEAfucffnzexOYMzdNwP/AxgGvheuB//W3W9o47gjp/izIoWHCtiwMfTxIVLnKNhFZO6a+hCTu28Btkz72lcabl/V4nENjENVAYl3Jki9L0X2qiyxjNbWReTE9N0nVDds2t7tIbSEl5z8w3ksYeSuyZFYmiCxtO/eDhHpUX2XJo/s2Q/AmnMWdXkkc1d5ocLUlil8wklfoqIvEWm9vgv3QzZtWN3tIRy3IB9QeLBA+bkysUUxhm8aJrG4b98CEelhSpYO8oJTfqFM5vKw6Cuu2bqItIfCvc2CgwHl58qkL00fLvrSAVMRaTeFe5u4O+WdZfI/yUMNku9KEj8lrmAXkY5QuLdB7c2w6OvlKonlYdHXKSr6EpHOUbi32OGir0JA7rocqQtU9CUinadwb5HaH2vETgmLvm4Mi77mawlGRLpD6XOCvOYU/qXAwW8dpLSj3peWXJ5UsItIV2nmfgKqv6sy9cMpgv0BqfNSpN6jPhgR6Q0K9zl6W9HXJ4ZIna1gF5HeoXA/Tm8r+rogRW5tDsvogKmI9BaFe5O82FD09REVfYlIb1M6NaH8qzL5LXl8UkVfItIfFO7HEEwF5B/MU3m+Qvy0OLmP50i8U98yEel9Sqpj8JJT3Vslc0WGzGUq+hKR/qFwnyYYDyg9WyJzWYb4KfWiLx0wFZF+o3APuTvln4dFXw6pVSnip8QV7CLSlxTuQO2NsOjrN1USKxLkrqvXB4iI9KuBD3cPnMn/M4kXndxHc6Teq6IvEel/Axvutf01YqeGRV//Kiz6mqc+GBGJhoFLM686hUcLHNzYUPS1LKlgF5FIGaiZe3VfWPT1x4DUe1T0JSLRNTDhXnyySOEnBWy+MXzLMMmzkt0ekohI20Q+3A8XfS1JkL4oTXZtFkvrgKmIRFtkwz0oBvVK3qSRW6eiLxEZLJFMu/Ivy+S35vEpJ/OBjIq+RGTgRCrcg6mA/NY8lV9UiP9ZnNzNORKnR2oXRUSaEqnk85JTfalKZk2GzKUq+hKRwdXUyd1mts7M9pjZXjP70gyPp83sH8PHf2ZmI60e6NEE4wGFxwu4++Gir+wHswp2ERlos4a7mcWBu4BrgVXALWa2atpmtwEH3P0s4OvAf2/1QKdzd4pjRca/OU7xiSLBgaA+Xp0JIyLS1Mx9NbDX3V909zJwH3DjtG1uBP4+vP19YK218Qjmcssw+e1JClsLJJYkmP/Z+cRPUdGXiMghzay5LwZeabi/D7j4aNu4e9XMxoFTgT+2YpCN4sA3MquovV4jd0OO1Pkq+hIRma6ZmftMyelz2AYzu93MxsxsbP/+/c2M7wg14MulF5j/2fmk35tWsIuIzKCZmfs+YGnD/SXAq0fZZp+ZJYAFwJvTn8jdNwIbAUZHR48I/2a8/NXr5vLXREQGSjMz9x3ASjNbYWYp4GZg87RtNgOfCm/fBPyzu88pvEVE5MTNOnMP19DvAB6kvuR9j7s/b2Z3AmPuvhn4O+A7ZraX+oz95nYOWkREjq2pDzG5+xZgy7SvfaXhdhH4WGuHJiIic6UrVIiIRJDCXUQkghTuIiIRpHAXEYkghbuISARZt05HN7P9wG/m+NcX0oZqgx6nfR4M2ufBcCL7vNzdF822UdfC/USY2Zi7j3Z7HJ2kfR4M2ufB0Il91rKMiEgEKdxFRCKoX8N9Y7cH0AXa58GgfR4Mbd/nvlxzFxGRY+vXmbuIiBxDT4d7L1+Yu12a2OcvmtluM9tlZg+b2fJujLOVZtvnhu1uMjM3s74/s6KZfTazj4fv9fNm9g+dHmOrNfGzvczMHjGzneHP9/pujLNVzOweM3vdzJ47yuNmZn8bfj92mdmFLR2Au/fkH+r1wr8GzgBSwDPAqmnb/BXwzfD2zcA/dnvcHdjnNUAuvP2Xg7DP4XbzgMeAbcBot8fdgfd5JbATODm8f1q3x92Bfd4I/GV4exXwcrfHfYL7fDlwIfDcUR5fD2ylfiW7S4CftfL1e3nm3nMX5u6AWffZ3R9x93x4dxv1K2P1s2beZ4C/Ab4GFDs5uDZpZp8/A9zl7gcA3P31Do+x1ZrZZwfmh7cXcOQV3/qKuz/GDFeka3Aj8G2v2wacZGant+r1ezncZ7ow9+KjbePuVeDQhbn7VTP73Og26v/y97NZ99nMLgCWuvv9nRxYGzXzPp8NnG1mT5jZNjNb17HRtUcz+/zXwK1mto/69SM+35mhdc3x/r4fl6Yu1tElLbswdx9pen/M7FZgFLiirSNqv2Pus5nFgK8Dn+7UgDqgmfc5QX1p5krq/zt73MzOc/e32jy2dmlmn28B7nX3/2lml1K/utt57h60f3hd0db86uWZ+/FcmJtjXZi7jzSzz5jZVcCXgRvcvdShsbXLbPs8DzgPeNTMXqa+Nrm5zw+qNvuz/X/dveLuLwF7qId9v2pmn28D/gnA3Z8EMtQ7WKKqqd/3uerlcB/EC3PPus/hEsW3qAd7v6/Dwiz77O7j7r7Q3UfcfYT6cYYb3H2sO8NtiWZ+tn9A/eA5ZraQ+jLNix0dZWs1s8+/BdYCmNm51MN9f0dH2VmbgX8fnjVzCTDu7q+17Nm7fUR5lqPN64FfUT/K/uXwa3dS/+WG+pv/PWAvsB04o9tj7sA+/wT4A/B0+Gdzt8fc7n2etu2j9PnZMk2+zwb8L2A38Cxwc7fH3IF9XgU8Qf1MmqeBa7o95hPc3+8CrwEV6rP024DPAp9teI/vCr8fz7b651qfUBURiaBeXpYREZE5UriLiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkH/HzMDenEP42DmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# only anger and neutral\n",
    "\n",
    "roc_plots(clf_xgb_AN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC 0.91875423155044\n",
      "f1 0.8106796116504854\n",
      "recall 0.7914691943127962\n",
      "precision 0.8308457711442786\n",
      "accuracy 0.8340425531914893\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHchJREFUeJzt3X2QXXWd5/H39z7f250OhIQRSToJEJCIKNDyIMqDAQxBYWcXFVx2laJknBH9w62tcstdd4r5x3VrxqqpQjRVA4zujjj6hxshiMrAyCCRBIEA0WgElAhKhNBJ+j71vee7f9yTcNP0w+3uc5/O/byqUnVv35Pbv9O38+HH75zzOebuiIhIvCS6PQAREYmewl1EJIYU7iIiMaRwFxGJIYW7iEgMKdxFRGJI4S4iEkMKdxGRGFK4i4jEUKpb33j58uW+Zs2abn17EZG+9Pjjj//J3VfMtV3Xwn3NmjXs2LGjW99eRKQvmdlvW9lOyzIiIjGkcBcRiSGFu4hIDCncRURiSOEuIhJDc4a7md1hZq+Y2TMzvG5m9vdmtsfMdprZ2dEPU0RE5qOVmftdwMZZXr8SWBf+uRm4ffHDEhGRxZjzPHd3/4mZrZllk2uAb3jjfn3bzOwYMzvB3V+OaIwiEqEb73yMB3fv6/YwBlKOBMdampe9wgtfuqqt3yuKi5hOBF5ser43/Nqbwt3MbqYxu2d0dDSCby0SPwrfeHp3YoT/kT2FQ17jP5Z3tv37RRHuNs3Xpr3rtrtvBjYDjI2N6c7c0vf6NYgvPW0Fd954breHMRCCckDpxyWqT1RJLEtQ+GCB51evavv3jSLc9wLNI10JvBTB+0pM9WsgdpLCNx48cA7eeZDg1YDsBVnyF+ex9HTz4ehFEe5bgFvM7G7gPGBc6+3xpnA+moJYpgqKAZY3LGHkL82TGEmQemtnq7zm/G5m9i3gEmC5me0F/ieQBnD3rwFbgU3AHqAI3NiuwUp3tCPMFYgSR+5O9ekqpR+WyL8/T/bsLJm3ZboyllbOlrl+jtcd+HRkI5KeMFegK5xFjhaMB0xsnaC2p0byxCSpVV0r3QW6WPkrvWe2QFeYi8ys+kyViXsnwCF/RZ7su7NYojNr6zNRuA8wzc5FomE5I3ViisJVBZLHJrs9HEDhPjBaXTdXoIvMzQOnsq2C1538+/KkT0mTOjmFWXdn680U7jEz34OfCnOR+an9oUbxniL1l+uk16dxd8ysp4IdFO6x0UqoK8hFFs5rTvnhMuWflrG8MfQfhkifnu65UD9M4d7npgt1hbhI9ILXAso/LZM5I0P+8jyJQm83pivc+9BMs3SFuki0vOpUd1fJviNL8vgkI3810jMHTOeicO9DmqmLtN/kc5MU7y0SvB6QekuK5Ipk3wQ7KNz7zo13PnbkcbsrQ0UGUVAKi76ebBR9DX98mOSK/gn1wxTuPWyui4pEJFoeOAfvahR95S7Mkbsoh6V684DpXBTuPUpXi4p0zpuKvpYmSJ3Q3/HY36OPscPBriAXaR93p7ozLPra0N2ir6gp3HvM1Bm7gl2kPeqv1yluLVL7TY3kyiSp0XjFYbz2po/NdL66iESvsrNC8b5io+hrY57sWLZnL0ZaKIV7j2gOdi3FiLRXYihBalWKwqYCyWP670yYVijce4BObxRpL683FX1dlCd9cprUSb1V9BU1hXuXNS/HaBlGJHq1l8Oirz/USb+9d4u+oqZw75Kpa+xaihGJltec8k/Coq+CMXTtEJnT43EmTCsU7l2iYBdpr+C1gPKjZTJnhkVf+d4u+oqawr1DZrooSWvsItHxqlP9ZZXsmf1X9BU1hXuHzNTiKCLRmPxNWPQ1HpA6of+KvqKmcO8wzdRFohUUA0o/KlHdWSVxXIIln1jSl0VfUVO4t9F8b3knIvNzpOjrtYDce3Pk3te/RV9RU7i3ka44FWmPYCLACmHR14aw6OstirNm+mm0wdQZu5ZiRKLh7lSfqlL6UYn8+/Nkz8mSOW1wTm+cD4V7G0w9zVFEFq/+ep3ivUVqz9VIjaZIrVF8zUY/nTbSjF0kGpWdFYpbi2BQuLJA5pxM7K8wXSyFu4j0vMRQgtTqFEObhkgsHayLkRZK4S4iPcfrTvmn5UYlb1j0lT453e1h9RWFe8SaGx5FZP5qL9cofr9I/Y91MmdkjhR9yfy09P83ZrbRzHab2R4z+/w0r4+a2YNm9oSZ7TSzTdEPtT+o4VFkYXzSKT5Q5OA/HCSYCBj68BBDfz6kYF+gOWfuZpYEbgMuB/YC281si7vvatrsvwP/7O63m9l6YCuwpg3j7RsqAhOZn2B/QGVbhcw7M+QvG7yir6i1sixzLrDH3Z8DMLO7gWuA5nB3YCR8vBR4KcpBikg8eSUs+npnWPT16ZHY3hmp01oJ9xOBF5ue7wXOm7LNXwM/NLPPAEPAZZGMTkRia/LXk0xsncAPOqm3hkVfCvbItBLu0y14+ZTn1wN3ufvfmtkFwDfN7Ax3D456I7ObgZsBRkdHFzLenqUeGZHWBMWA0g9LVJ+uklieYPgTwyr6aoNWwn0vsKrp+UrevOxyE7ARwN0fNbMcsBx4pXkjd98MbAYYGxub+h+IvqarUkXm5oFz8M6DBK8H5N6XI/deFX21Syvhvh1YZ2Zrgd8D1wEfm7LN74ANwF1mdjqQAwZyGqurUkXeLDgUYENh0dflYdHXn+lM7Haa83C0u9eAW4D7gV/QOCvmWTO71cyuDjf7L8Anzewp4FvAJ9w9VjPz2ejcdpHpuTuVJyoc+OoBqj+vApA5NaNg74CWfsLuvpXG6Y3NX/ti0+NdwIXRDq0/NK+1azlG5A31/XWK9xSpvVAjtTpFaq0CvZP0016k5mDXue0iDZWnKhTvC4u+NhXInK2ir05TuEdEwS7yhsSSBOk1aQqbCiRGdDFSNyjcRWTRvO6UHwmLvi7Okz4pTfokFX11k8J9gXReu0hD7aUaE1smCPYFZN6hoq9eoXBfIJ3XLoPOJ53SQyUqP6tgw8bQR4fInKpb3vUKhfsi6bx2GVTB/oDK9gqZszIUNhSwnGbrvUThLiIt83JY9PWuRtHX0k8v1Z2RepTCfZ601i6DavLXk0zcO4EfclIrUySXJxXsPUzhPk9aa5dBE0yERV/PVEmsSDD84WGSy1X01esU7guktXYZBB44B+8Ki74uzpG7MIcltbbeDxTuIvImU4u+ksckSR6v2Xo/0YKZiBzh7lQerzB+2zjVx98o+lKw9x/N3EUEgPprYdHXb2uk1qRInax46Gf69OZB1b4SV5Unw6KvJBQ+WCDzLhV99TuF+zyo2lfiKrE0QfqkNIUrVfQVFwr3OUx3XrsaIKXfea2p6OuSPOm1adJrVfQVJwr3OUwNds3apd/Vfl9j4vth0deZKvqKK4V7i3Reu/Q7rzYVfY0Yw9cNk16n2XpcKdxFBkQwHlDZUSF7Tpb8hjyW1Ww9zhTuIjEWlAMmfzFJ9qwsyRVJlt6yVAdMB4TCXSSmqrurFLcW8QkntSos+lKwDwyFu0jMBBMBxR8Umdw1SfL4JIWPFlT0NYAU7iIxcqToazwgd0mO3HtU9DWoFO4iMRAcDLDhRtFX4YoCiWMSJFdotj7ItAA3C9UNSK9zdyo7Kox/dZzK4xUA0uvSCnbRzH0mzVem6sIl6UX1V8Oir9/VSK1NkT5Z56zLGxTuM2gOdtUNSK+pPFGh+IMiljIKHyqQeaeKvuRoCvc5KNilFyWOSZA+OSz6WqLVVXkzhbtIH/CaU364DED+UhV9ydwU7tPQgVTpJbUXw6KvVwMy71LRl7RG4T4NHUiVXuBVp/RgicpjFRJLEwx/bFgHTaVlLS3WmdlGM9ttZnvM7PMzbPMRM9tlZs+a2T9FO8zu0Hq7dFMwHlB5vEL23VlG/mJEwS7zMufM3cySwG3A5cBeYLuZbXH3XU3brAP+G3Chu+83s+PbNWCROAtKYdHX2WHR12eW6oCpLEgryzLnAnvc/TkAM7sbuAbY1bTNJ4Hb3H0/gLu/EvVAReKu+ssqxfvCoq/RsOhLwS4L1MpvzonAi03P94Zfa3YqcKqZPWJm28xs43RvZGY3m9kOM9uxb9++6TYRGTjBoYBD3z3ExHcmSAwlWHLTEhV9yaK1MnOf7rC8T/M+64BLgJXAw2Z2hru/ftRfct8MbAYYGxub+h4iA8cD5+A/hkVfl+bIXaCiL4lGK+G+F1jV9Hwl8NI022xz90ngeTPbTSPst0cySpGYCQ4E2JKw6OsDYdGXZusSoVaWZbYD68xsrZllgOuALVO2+R5wKYCZLaexTPNclAPtFJ3jLu3k7pQfKzeKvnaERV+npBXsErk5Z+7uXjOzW4D7gSRwh7s/a2a3AjvcfUv42hVmtguoA//V3V9t58DbRee4S7vU/1Rn4p4J6i/WSZ2c0s2ppa1auojJ3bcCW6d87YtNjx34XPinbzXP2nWOu0Sp8kSF4n1FLG0Uri6QOVNFX9JeukK1iWbt0i6JYxOkT01T2FggMazTG6X9FO7T0KxdFstrTvknYdHX+/Ok16RJr9EyjHSOwp2jb8whslhHFX2dpaIv6Q6FOxwV7FqSkYXySlj0tb1C4hgVfUl3DXS4T52xv/Clq7o4Gul3wYGAyhMVsudmyV+axzKarUv3DHS4a8YuixUUAyZ3TZIdC4u+blHRl/SGgQ73wzRjl/lydyZ/MUnxB0W85KTWqOhLeovCXWSegoMBxfuKTO6eJHlCksLHCrrCVHqOwl1kHo4UfR0MyG/Ikz0/iyW0ti69Z2DDXR0yMh/BeICNhEVfV4ZFX8dpti69ayAXCJvPktGBVJmNB2HR1+1NRV8npxXs0vMGbuY+Ndh1NarMpL4vLPraWyd1SorMqZluD0mkZQMX7gp2aUXl5xWKPyhiGaNwTYHMO1T0Jf1l4ML9MAW7zCaxLEH6tLDoa2ggVy+lzw1suIs080mn9K8lMChsKKjoS/rewIS7ysFkJpO/naR4T5HgtYDMOSr6kngYmHBX1YBM5RWn9ECJyuMVEscmGL5hmPRazdYlHgYm3A9T1YAcFhwMqDxVIXt+lvzFKvqSeBm4cJfBFhQDqruq5MZyJJcnWfqZpbozksSSwl0GgrszuSss+io76bWNC5EU7BJXCneJveBgQHFrkclfNYq+hm4Y0hWmEnsKd4m1o4q+LsuTPU9FXzIYFO4SS/XX6yRGEm8UfR2bILlMs3UZHFpwlFjxwClvK3Pg9gNUHm8q+lKwy4AZiJm76n0HQ/2VOhPfn6D+Up30ujSZ01T0JYNrIMJd9b7xV3k8LPrKGUN/PkT67WldZSoDbSDC/TCVhcXP4aqAxPIEmfUZ8lfkVfQlwoCFu8SHTzqlh8Kir8sKpFenSa9WdYDIYQp36TuTL4RFX/sDsmNZFX2JTEPhLn3Dy07xgSLVn1cbRV//aVi1vCIzaGlx0sw2mtluM9tjZp+fZbtrzczNbCy6IYo0BIcCqk9XyV6QZeQvRhTsIrOYM9zNLAncBlwJrAeuN7P102y3BPgs8LOoB7kYOg2yvwUTAeXHygBHir4KlxWwtJZhRGbTysz9XGCPuz/n7lXgbuCaabb7G+DLQDnC8S3K1JthS/9wd6pPVzlw+wFKPypRf7UOoDNhRFrUyr+UE4EXm57vDb92hJmdBaxy93siHNui6WbY/SkYD5j49gQT35sgsSzByCdHVPQlMk+tHFCd7v9//ciLZgngK8An5nwjs5uBmwFGR0dbG2EEFOz9wwPn4DcPEhwKyF+RJ/tuFX2JLEQr4b4XWNX0fCXwUtPzJcAZwEPh6WhvAbaY2dXuvqP5jdx9M7AZYGxszGkjrbX3l6OKvjaFRV/HarYuslCtLMtsB9aZ2VozywDXAVsOv+ju4+6+3N3XuPsaYBvwpmDvNK219wcPnPKjYdHXjrDo66S0gl1kkeacubt7zcxuAe4HksAd7v6smd0K7HD3LbO/Q3dpSaZ31f5Yo/j9IvWX66RPTZM5XUVfIlFp6SImd98KbJ3ytS/OsO0lix+WxF15R5nS/aVG0de/HyK9XkVfIlHSFarSUYerApIrkmTeHhZ9FXR6o0jUFO7SEV4Ni74SKvoS6QSFu7Td5PNh0dfrAdl3q+hLpBNiGe46DbI3BOWA0o9KVJ+skliWYPjjw6RHNVsX6YTYhHtz1cBhOg2yu/yQU322SvY9WfIX5dUHI9JBsQn36YJdp0F2XnAooPpsldx5uUbR12eX6oCpSBfEJtwPe+FLV3V7CAPpcNFX6YclvOqkT0mTPC6pYBfpktiFu3ReMB4wsXWC2p4ayZVJhj44pKIvkS5TuMuieOAc/MZBgomA/AfyZMdU9CXSCxTusiD1/XUSS8Oirw+GRV/HaLYu0iu0ICrz4oFTfmRK0dfatIJdpMdo5i4tq/0hLPr6Q530aSr6EullCndpSXl7mdIPS1jeGLp2SMEu0uMU7jKrI0VfxyfJnBEWfeW1mifS6xTuMi2vOqUHw6Kvy1X0JdJvFO7yJpO/maR4b5FgPCB7roq+RPqRwl2OCEph0ddTVRLHJVjy8SWkRvUrItKP9C9XjvAJp/qLKrkLc+QuymEpzdZF+pXCfcAFhwKqz1TJnR8WfX1GRV8icaBwH1DuTnVnWPQ16aTXqehLJE4U7gOo/nqd4r1Fas/VSK5S0ZdIHCncB4wHzqFvHiIoBuSvzJM9J6szYURiSOE+IOqv1UkcExZ9fahA4hgVfYnEmRZYY87rTunfShz4WlPR1xoVfYnEnWbuMVZ7OSz6+mOd9OlpMuvVByMyKBTuMVV+LCz6GjKGPjxE5m0KdpFBonCPmSNFX29JkjkzQ/5yFX2JDCKFe0x4xSn9SwlSYdHXaJr0qIq+RAZVLML9xjsf6/YQumpyzyTFrWHR13kq+hKRmIT7g7v3AXDpaSu6PJLOCoph0dfOKonlCZbcuITUylh8pCKySH2fBM2z9jtvPLeLI+k8LznVX1bJvS9H7r0q+hKRN7R0pM3MNprZbjPbY2afn+b1z5nZLjPbaWYPmNnq6Ic6vUGbtQcHA8qPlnF3ksclWfrZpeQvySvYReQoc4a7mSWB24ArgfXA9Wa2fspmTwBj7n4m8F3gy1EPdC5xn7W7O5UnKxy4/QClh0oErwUAOhNGRKbVyrLMucAed38OwMzuBq4Bdh3ewN0fbNp+G3BDlIOcyaAcSK3vD4u+nq+RGk1R+GBBRV8iMqtWwv1E4MWm53uB82bZ/ibgvuleMLObgZsBRkdHWxzizAZhScYD59D/aRR9FTYVyJyd0ZkwIjKnVsJ9uiTxaTc0uwEYAy6e7nV33wxsBhgbG5v2PRYijksy9VfrJI59o+greWySxFItwYhIa1oJ973AqqbnK4GXpm5kZpcBXwAudvdKNMMbPF53yj8tU364TH5Dntx5OdJrdDGSiMxPK+G+HVhnZmuB3wPXAR9r3sDMzgK+Dmx091ciH+WAqL0UFn29Uif99jSZM9QHIyILM2e4u3vNzG4B7geSwB3u/qyZ3QrscPctwP8GhoHvhOvBv3P3q9s47tgp/6xM6UclbNgY+sgQmdMU7CKycC1dxOTuW4GtU772xabHl0U8roFxuCog9dYUmXdlyF+WJ5HT2rqILE7fX6Har7ziFB8oYimjcEWB1KoUqVX6OEQkGkqTLpj89SQTWyfwg072fBV9iUj0FO4dFBQDSveXqD5TJbEiwfC1w6RO1EcgItFTsnSQl5zqr6vkLgqLvpKarYtIeyjc2yw4EFB9pkr2guyRoi8dMBWRdlO4t4m7U32iSvHHRahD+m1pksuSCnYR6QiFexvUXwuLvl6okVodFn0tU9GXiHSOwj1iR4q+SgGFqwpkzlLRl4h0nsI9IvU/1UksC4u+rgmLvka0BCMi3aH0WSSvO6V/LXHg6weobG/0paVXpxXsItJVmrkvQu33NSa+P0GwLyBzRobMO9QHIyK9QeG+QEcVfX10iMypCnYR6R0K93k6qujrrAyFDQUspwOmItJbFO4t8nJT0dcHVPQlIr1N6dSC6q+qFLcW8UMq+hKR/qBwn0UwEVC8v8jks5Mkj09S+EiB1Fv1IxOR3qekmoVXnNqeGrmLc+QuVNGXiPQPhfsUwXhA5ekKuQtzJJc1ir50wFRE+o3CPeTuVH8eFn05ZNZnSC5LKthFpC8p3IH6q2HR129rpNamKFzVqA8QEelXfRnuN975GA/u3hfJe3ngHPq/h/CyU/hQgcw7VfQlIv2vL8O9OdgvPW3Fgt6jvq9O4riw6OvfhUVfS9QHIyLx0HfhfuOdjx15/MKXrpr33/eaU/63MuVHyuQvy5M7L0d6NB3lEEVEuq7vwv3wrH0hM/ba3rDo608BmXeo6EtE4qvvwv2wO288d17blx8tU/pxCRsxhq8fJn2KZusiEl99G+6tOlL0tTJF9pws+Q15LKsDpiISb7EN96AcNCp500Zho4q+RGSwxDLtqr+sUryviE84uffkVPQlIgMnVuEeTAQU7ysy+YtJkn+WpHBdgdQJsdpFEZGWxCr5vOLUnq+RuzRH7gIVfYnI4Grpqh0z22hmu81sj5l9fprXs2b27fD1n5nZmqgHOpNgPKD0cAl3P1L0lX9vXsEuIgNtznA3syRwG3AlsB643szWT9nsJmC/u58CfAX4X1EPdCp3p7yjzPjXxik/UibYHzTGqzNhRERamrmfC+xx9+fcvQrcDVwzZZtrgH8MH38X2GBtPIK52nIc+sYhSveVSK1MMfKpEZLLVPQlInJYK2vuJwIvNj3fC5w30zbuXjOzceA44E9RDLJZEvhqbj31V+oUri6QOVNFXyIiU7Uyc58uOX0B22BmN5vZDjPbsW/fwlod68AXKr9m5FMjZN+ZVbCLiEyjlZn7XmBV0/OVwEszbLPXzFLAUuC1qW/k7puBzQBjY2NvCv9WLKQsTERk0LQyc98OrDOztWaWAa4DtkzZZgvw8fDxtcC/uPuCwltERBZvzpl7uIZ+C3A/jSXvO9z9WTO7Fdjh7luAfwC+aWZ7aMzYr2vnoEVEZHYtXcTk7luBrVO+9sWmx2Xgw9EOTUREFkq3HhIRiSGFu4hIDCncRURiSOEuIhJDCncRkRiybp2Obmb7gN8u8K8vpw3VBj1O+zwYtM+DYTH7vNrdV8y1UdfCfTHMbIe7j3V7HJ2kfR4M2ufB0Il91rKMiEgMKdxFRGKoX8N9c7cH0AXa58GgfR4Mbd/nvlxzFxGR2fXrzF1ERGbR0+HeyzfmbpcW9vlzZrbLzHaa2QNmtrob44zSXPvctN21ZuZm1vdnVrSyz2b2kfCzftbM/qnTY4xaC7/bo2b2oJk9Ef5+b+rGOKNiZneY2Stm9swMr5uZ/X3489hpZmdHOgB378k/NOqFfwOcBGSAp4D1U7b5K+Br4ePrgG93e9wd2OdLgUL4+C8HYZ/D7ZYAPwG2AWPdHncHPud1wBPAseHz47s97g7s82bgL8PH64EXuj3uRe7zRcDZwDMzvL4JuI/GnezOB34W5ffv5Zl7z92YuwPm3Gd3f9Ddi+HTbTTujNXPWvmcAf4G+DJQ7uTg2qSVff4kcJu77wdw91c6PMaotbLPDoyEj5fy5ju+9RV3/wnT3JGuyTXAN7xhG3CMmZ0Q1ffv5XCf7sbcJ860jbvXgMM35u5Xrexzs5to/Je/n825z2Z2FrDK3e/p5MDaqJXP+VTgVDN7xMy2mdnGjo2uPVrZ578GbjCzvTTuH/GZzgyta+b7731eWrpZR5dEdmPuPtLy/pjZDcAYcHFbR9R+s+6zmSWArwCf6NSAOqCVzzlFY2nmEhr/d/awmZ3h7q+3eWzt0so+Xw/c5e5/a2YX0Li72xnuHrR/eF3R1vzq5Zn7fG7MzWw35u4jrewzZnYZ8AXganevdGhs7TLXPi8BzgAeMrMXaKxNbunzg6qt/m7/P3efdPfngd00wr5ftbLPNwH/DODujwI5Gh0scdXSv/eF6uVwH8Qbc8+5z+ESxddpBHu/r8PCHPvs7uPuvtzd17j7GhrHGa529x3dGW4kWvnd/h6Ng+eY2XIayzTPdXSU0Wpln38HbAAws9NphPu+jo6ys7YA/zk8a+Z8YNzdX47s3bt9RHmOo82bgF/ROMr+hfBrt9L4xw2ND/87wB7gMeCkbo+5A/v8Y+CPwJPhny3dHnO793nKtg/R52fLtPg5G/B3wC7gaeC6bo+5A/u8HniExpk0TwJXdHvMi9zfbwEvA5M0Zuk3AZ8CPtX0Gd8W/jyejvr3WleoiojEUC8vy4iIyAIp3EVEYkjhLiISQwp3EZEYUriLiMSQwl1EJIYU7iIiMaRwFxGJof8PowxL6Vmo4QIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# only anger/disgust and neutral/happy\n",
    "\n",
    "roc_plots(clf_xgb_ADHN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature selection\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEWCAYAAABi5jCmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XucTvX6//HX5ZBEkURkO4WNGBOiNsnhKzlV0tfebRXtfPfuSKcd/Tof9jclJV/taqdIO4ctlXTaFYbORQ0SopodEVIOIzK4fn+sNdN9j5lxl5lZc8+8n4/H/Zh7rXutdV/rYu5r1lr3+lzm7oiIiGQrF3UAIiJSsqgwiIhIHBUGERGJo8IgIiJxVBhERCSOCoOIiMRRYRBJkJk9ama3RB2HSFEz3ccgRc3MMoDawL6Y2c3cff0hbLMr8E93r3do0SUnM5sMrHP3m6OORUofHTFIcenv7lVjHr+6KBQGM6sQ5fsfCjMrH3UMUrqpMEikzOwUM3vXzLaa2ZLwSCD7tYvNbIWZ7TCzL83sL+H8KsCrQF0zywwfdc1sspndHbN+VzNbFzOdYWYjzWwpsNPMKoTrzTKzzWb2lZkNLyDWnO1nb9vMbjCzTWa2wczOMbM+Zva5mX1vZv8vZt3bzexZM5sR7s/HZtYm5vUWZpYW5mG5mZ2V630fMbNXzGwncAkwGLgh3Pc54XKjzOyLcPufmdmAmG0MNbO3zex+M/sh3NfeMa/XMLNJZrY+fP2FmNf6mVl6GNu7ZpaS8D+wJCUVBomMmR0PvAzcDdQArgdmmdmx4SKbgH7AUcDFwINm1tbddwK9gfW/4gjkfKAvUB3YD8wBlgDHAz2Aq82sV4LbOg44PFz3VuBx4AKgHXAacKuZNY5Z/mxgZrivU4EXzKyimVUM43gdqAVcBTxjZr+NWfePwN+AI4EpwDPAfeG+9w+X+SJ832rAHcA/zaxOzDY6AquAmsB9wBNmZuFrTwNHACeGMTwIYGZtgSeBvwDHAI8BL5pZpQRzJElIhUGKywvhX5xbY/4avQB4xd1fcff97v4GsAjoA+DuL7v7Fx5YQPDBedohxjHe3de6+y7gZOBYd7/T3fe4+5cEH+5/SHBbWcDf3D0LmE7wgfuQu+9w9+XAciD2r+vF7v5suPwDBEXllPBRFRgdxjEPeImgiGWb7e7vhHnanVcw7j7T3deHy8wAVgMdYhb5j7s/7u77gKeAOkDtsHj0Bi519x/cPSvMN8D/AI+5+wfuvs/dnwJ+CmOWUippz7NK0jnH3d/MNa8B8N9m1j9mXkVgPkB4quM2oBnBHzFHAMsOMY61ud6/rpltjZlXHngrwW1tCT9kAXaFPzfGvL6L4AP/gPd29/3haa662a+5+/6YZf9DcCSSV9x5MrOLgGuBhuGsqgTFKtu3Me//Y3iwUJXgCOZ7d/8hj802AIaY2VUx8w6LiVtKIRUGidJa4Gl3/5/cL4SnKmYBFxH8tZwVHmlkn/rI6+t0OwmKR7bj8lgmdr21wFfu3vTXBP8r/Cb7iZmVA+oB2afAfmNm5WKKQ33g85h1c+9v3LSZNSA42ukBvOfu+8wsnZ/zVZC1QA0zq+7uW/N47W/u/rcEtiOlhE4lSZT+CfQ3s15mVt7MDg8v6tYj+Ku0ErAZ2BsePZwRs+5G4BgzqxYzLx3oE15IPQ64+iDv/yGwPbwgXTmMoZWZnVxoexivnZmdG34j6mqCUzLvAx8QFLUbwmsOXYH+BKen8rMRiL1+UYWgWGyG4MI90CqRoNx9A8HF/L+b2dFhDF3Clx8HLjWzjhaoYmZ9zezIBPdZkpAKg0TG3dcSXJD9fwQfaGuBvwLl3H0HMBz4F/ADwcXXF2PWXQlMA74Mr1vUJbiAugTIILgeMeMg77+P4AM4FfgK+A6YSHDxtijMBn5PsD8XAueG5/P3AGcRnOf/Dvg7cFG4j/l5AmiZfc3G3T8DxgLvERSN1sA7vyC2CwmumawkuOh/NYC7LyK4zjAhjHsNMPQXbFeSkG5wEykGZnY70MTdL4g6FpGD0RGDiIjEUWEQEZE4OpUkIiJxdMQgIiJxkvI+hurVq3uTJk2iDiNSO3fupEqVKlGHETnlIaA8BJSH/HOwePHi79z92DxWOUBSFobatWuzaNGiqMOIVFpaGl27do06jMgpDwHlIaA85J8DM/tPotvQqSQREYmjwiAiInFUGEREJI4Kg4iIxFFhEBGROCoMIiISR4VBRETiqDCIiEgcFQYREYmjwiAiInFUGEREJI4Kg4hIRPbt28dJJ51Ev379AHB3brrpJpo1a0aLFi0YP348ALNnzyYlJYXU1FTat2/P22+/XaRxRTKInpkNBy4DjgKqEvTbBXjO3e+MIiYRkeL20EMP0aJFC7Zv3w7A5MmTWbt2LStXrqRcuXJs2rQJgB49enDWWWdhZixdupRBgwaxcmVBLcEPTVSjq15O0Pi8AXC9u/f7JSvvytpHw1EvF0lgyeK61nsZWsZzAMpDNuUhkAx5yBjdF4B169bx8ssvc9NNN/HAAw8A8MgjjzB16lTKlQtO5tSqVQuAqlWr5qy/c+dOzKxIYyz2U0lm9ijQGHgROKm4319EpCS4+uqrue+++3KKAMAXX3zBjBkzaN++Pb1792b16tU5rz3//PM0b96cvn378uSTTxZpbMV+xODul5rZmUA3oBVws5ktAdYTHD0sz2s9M/sz8GeAmjWP5dbWe4sr5BKpduXgr6OyTnkIKA+BZMhDWloa7733HllZWezYsYP09HS2bNlCWloaP/74I9988w33338/CxcuZODAgTnXGY4++mgeffRRlixZwpVXXsnYsWPz3H5mZiZpaWmHFGMkPZ/NLANoD+wB9rt7ppn1AR5y96YHW79+4yZebtBDRRxlyXZd672MXZaUfZYKlfIQUB4CyZCHjNF9ufHGG3n66aepUKECu3fvZvv27Zx77rksWrSI1157jYYNG+LuVK9enW3bth2wjUaNGvHRRx9Rs2bNA14roFHPYndvn1CQ7l7sDyADqJno/NyPZs2aeVk3f/78qEMoEZSHgPIQSMY8zJ8/3/v27evu7iNHjvQnnngiZ3779u3d3X316tW+f/9+d3dfvHix161bN2c6r+3lBVjkCX5GR1pazew4YKO7u5l1ILjmsSXKmEREojJq1CgGDx7Mgw8+SNWqVZk4cSIAs2bNYsqUKVSsWJHKlSszY8aMIr0AHfUx13nAZWa2F9gF/CGsbCIiZULXrl1zTv1Ur16dl18+8FtVI0eOZOTIkcUWUySFwd0bhk8nhA8RESkhdOeziIjEUWEQEZE4KgwiIhJHhUFEROKoMIiISBwVBhERiaPCICIicVQYREQSkLupziWXXEKbNm1ISUnhvPPOIzMzE4CFCxfStm1bKlSowLPPPhtlyL9aJIXBzIab2QozeyacPtnM9pnZeVHEIyJyMNlNdbI9+OCDLFmyhKVLl1K/fn0mTAju1a1fvz6TJ0/mj3/8Y1ShHrJIG/W4+1dmVh64F/h3oiurUU9yNCQpDspDQHkIFHYeCmqqc9RRRwHBQKS7du3KGbuoYcOGAHF9FpJNpI16zOwa4CpgFrCpuGMREUlEXk11AC6++GKOO+44Vq5cyVVXXRVRdIUv6kY9lYCpQHfg5ILWU6OeeMnQkKQ4KA8B5SFQ2HkoqKkOwJAhQ7jgggsYP348d9xxB717985Z99tvv2X58uV59kwoSoXRqCfSfgzATOCUcN5k4LxE1lc/huQcd74oKA8B5SFQFHkYNWqUH3/88d6gQQOvXbu2V65c2QcPHhy3TFpaWk5PhWxDhgzxmTNnFno8B1MY/RiiPgnWHpgednQ7D/i7mZ0TbUgiIj+75557WLduHRkZGUyfPp3u3bvz9NNPs2bNGiD443rOnDk0b9484kgLT6T9GNy9UfZzM5sMvOTuL0QXkYjIwbk7Q4YMYfv27bg7bdq04ZFHHgHgo48+YsCAAfzwww/MmTOH2267jeXL82xlX2JF3ahHRCRpxDbVeeedd/Jc5uSTT2bdunXFGFXhi7pRT+y8ocUfiYiI5Bb1NQYRESlhVBhERCSOCoOIiMRRYRARkTgqDCIiEkeFQURE4qgwiEipsHv3bi677DLatGnDiSeeyG233QbAvHnzaNu2La1atWLIkCHs3RuMpTRmzBhSU1NJTU2lVatWlC9fnu+//z7KXSgxou7HMMvM3jOzn8zs+ihiEZHSoVKlSjzwwAMsWbKE9PR0XnvtNd59912GDBnC9OnT+fTTT2nQoAFPPfUUAH/9619JT08nPT2de+65h9NPP50aNWpEvBclQ1RHDJcDfYDLgOHA/RHFISKlhJlRuXJlALKyssjKyqJ8+fJUqlSJZs2aAdCzZ09mzZp1wLrTpk3j/PPPL9Z4S7Jiv/M5th8D8KS7P2hmfX/JNtSoR41ZsikPgbKeh+yGOvv27SM1NZU1a9ZwxRVX0KFDB7Kysli0aBHt27fn2WefZe3atXHr/vjjj7z22ms5HdgkgiMGd78UWA90c/cHi/v9RaT0Kl++POnp6axbt44PP/yQ5cuXM336dK655ho6dOjAkUceSYUK8X8Pz5kzh06dOuk0UoykGURPjXriqTFLQHkIlPU8ZDemiW1S07BhQx5++GF+//vfc9dddwHByKfVqlWLa2QzYcIETj/99ENvblNCJH2jnpjp24HrE11fjXrUmCWb8hBQHtw3bdrkc+bMcXf3H3/80Tt37uxz5szxjRs3urv77t27vXv37j537tycdbZu3epHH320Z2ZmRhJzUSgNjXpERArFhg0buOaaa0hJSeHkk0+mZ8+e9OvXjzFjxtCiRQtSUlLo378/3bt3z1nn+eef54wzzqBKlSoRRl7yRHoqycyOAxYBRwH7zexqoKW7b48yLhFJPikpKTz++OM5/RKyjRkzhjFjxuS5ztChQxk6dGjRB5dkSkI/hnpRxCAiInnTqSQREYmjwiAiInFUGEREJI4Kg4iIxFFhEBGROCoMIiISR4VBRETiqDCISGR2795Nhw4dDmiu4+7cdNNNNGvWjBYtWjB+/HgAtm3bRv/+/XOWnzRpUpThl1qR3OBmZsMJejEcB6wF9gN7gavd/e0oYhKR4lepUiXmzZtH1apVycrKonPnzvTu3ZsVK1awdu1aVq5cSbly5di0aRMADz/8MC1btmTOnDls3ryZ3/72twwePJjDDjss4j0pXaIaEuNyoDewGdjp7m5mKcC/gOYRxSQixczMqFq1KvBzcx0z45FHHmHq1KmUKxec1KhVq1bO8jt27MDdyczMpEaNGgcMoy2HrkQ06glfqgJ4IttQox41ZsmmPASSLQ/ZjXUgaK7Trl27nOY6HTt25IsvvmDGjBk8//zzHHvssYwfP56mTZty5ZVXctZZZ1G3bl127NjBjBkzcoqHFJ5iLwzufqmZnUnQqOc7MxsA3APUAvLt5KZ+DPHK+vj72ZSHQLLlIXe/gHHjxpGZmcktt9xC8+bN+fHHH/nmm2+4//77WbhwIQMHDmT8+PEsWLCAmjVrMnXqVNavX8+wYcOYOHFizuiohdKLIMkVRg4sGKa7eJlZBtDe3b+LmdcFuNXd/+tg69dv3MTLDXqoCCMs+a5rvZexy3QIrTwEki0PsUcMse644w6qVKnCxIkTee2112jYsCHuTvXq1dm2bRt9+/Zl1KhRnHbaaQB0796d0aNH06FDByAoOLlHVy1r8suBmS129/aJbKPE/E9y94VmdoKZ1YwtGHmpXLE8q/L5j1VWpKWlkTG4a9RhRE55CCRrHjZv3kzFihWpXr06u3bt4s0332TkyJGcc845zJs3jz/96U8sWLCAZs2aAVC/fn3mzp3LaaedxsaNG1m1ahWNGzeOeC9Kn6j7MTQBvggvPrcFDgO2RBmTiBSfDRs2MGTIEPbt28f+/fsZNGgQ/fr1o3PnzgwePJgHH3yQqlWrMnHiRABuueUWhg4dSuvWrXF37r33XmrWrBnxXpQ+UR8xDAQuMrMsYBfwe4/i3JaIRCIlJYVPPvnkgPnVq1fn5ZcPvJhet25dXn/99eIIrUyLulHPveFDRERKCH3PS0RE4qgwiIhIHBUGERGJo8IgIiJxVBhERCSOCoOIiMRRYRARkTgqDCJSqPJrvjN06FAaNWpEamoqqamppKenA/DMM8+QkpJCSkoKv/vd71iyZEmU4QvRN+pZGcZQP/x5v7urJZNIEsuv+Q7AmDFjOO+88+KWb9SoEQsWLODoo4/m1Vdf5c9//jMffPBBFKFLKOpGPecD1dy9v5kdC6wys2fcfU9BK6sfQ/KNv19UlIdASclDxui++Tbfyc/vfve7nOennHIK69atK/I4pWDFfiopV6MeB4604H9NVeB7ghafIpLE9u3bR2pqKrVq1aJnz5507NgRgJtuuomUlBSuueYafvrppwPWe+KJJ3KOLiQ6kfZjAH4iKBDNgSMJBtHL88+eXI162t067vHiCbaEql0ZNu6KOoroKQ+BkpKH1sdXi5vObr4zfPhwjjrqKGrUqEFWVhZjx46lbt26DBkyJGfZTz75hHHjxjF+/HiqVauWe9MJyczMzDlaKavyy0G3bt2Sph9DLyAd6A6cALxhZm+5+/bcC7r7P4B/QNCoJ5makhSFZGvMUlSUh0BJyUNePSEWL17Mli1buPjii3PmHXbYYdx///05DWWWLl3KhAkTeOONN3J6L/waatRTODmI+n/SxcDocKjtNWb2FcHRw4cFraRGPcnbmKWwKQ+BkpSH/JrvbNiwgTp16uDuvPDCC7Rq1QqAr7/+mnPPPZenn376kIqCFJ6oC8PXQA/gLTOrDfwW+DLakETkUOTXfKd79+5s3rwZdyc1NZVHH30UgDvvvJMtW7Zw+eWXA1ChQgUWLVoU5S6UeVEXhruAyWa2DDBg5MHaeopIyZZf85158+blufzEiRNzOrRJyRB1ox6AM6KIQURE8vaLv65qZkebWUpRBCMiItFLqDCYWZqZHWVmNYAlwCQze6BoQxMRkSgkesRQLfwK6bnAJHdvB/xX0YUlIiJRSbQwVDCzOsAg4KUijEdERCKWaGG4E/g38IW7f2RmjYHVRReWiIhEJaFvJbn7TGBmzPSXwMCiCkpERKKT6MXnZmY218w+DadTzOzmog1NRESikOippMeBG4EsAHdfCvyhqIISkaKXX0OdCRMm0KRJE8yM7777+X7Tbdu20b9//5zlJ01S65TSKtHCcIS75x6/6KDDY5vZcDNbYWazzOw9M/vJzK7PtcyZZrbKzNaY2ahEAxeRQ5PdUGfJkiWkp6fz2muv8f7779OpUyfefPNNGjRoELf8ww8/TMuWLVmyZAlpaWlcd9117NlTYOsUSVKJ3vn8nZmdQNA/ATM7D9iQwHrZDXl2Ag2Ac2JfNLPywMNAT2Ad8JGZvejunxW0UTXqKTmNWaKmPAR+aR4Kaqhz0kkn5bmOmbFjxw7cnczMTGrUqEGFClGPqiNFIdEjhiuAx4DmZvYNcDVwaUEr5GrIM9jdPyI8FRWjA7DG3b8Mu7ZNB87+BfGLyCHIr6FOXq688kpWrFhB3bp1ad26NQ899BDlyqltfGl00HJvZuWA9u7+X2ZWBSjn7jsOtp67X2pmZwLdChgY73hgbcz0OiDP/5m5GvVwa+uy3eitduXgr8SyTnkI/NI8pKWl5TwfN25cTkOd5s2b06hRIyC4BvHOO+/kNM1ZsGABNWvWZOrUqaxfv55hw4YxceJEqlSpUqj7cigyMzPj9q0sKpQcuPtBH8DCRJbLY70MoGbM9O3A9THT/w1MjJm+EPi/g223WbNmXtbNnz8/6hBKBOUhUBh5uP32233MmDE50w0aNPDNmzfnTPfp08cXLlyYM92tWzf/4IMPDvl9C5P+P+SfA2CRJ/jZnehx4Btmdr2Z/cbMamQ/Dq0kAcERwm9ipusB6wthuyJyEJs3b2br1q0AOQ11mjdvnu/y9evXZ+7cuQBs3LiRVatW0bhx42KJVYpXooXhTwTXGRYCi8NHYXTS+AhoamaNzOwwgq/AvlgI2xWRg9iwYQPdunUjJSWFk08+mZ49e9KvXz/Gjx9PvXr1WLduHSkpKQwbNgyAW265hXfffZfWrVvTo0cP7r33XmrWrBnxXkhRSPTO50aH8iZmdhxBITkK2G9mVwMt3X27mV1JMNxGeeBJd19+KO8lIonJr6HO8OHDGT58+AHz69aty+uvv14coUnEEioMZnZRXvPdfUpB63l8Q556+SzzCvBKInGIiEjRS/RLyCfHPD+coE/zx0CBhUFERJJPoqeSroqdNrNqwNNFEpGIiETq196d8iPQtDADERGRkiHRawxzCIfDICgmLYkZhltEREqPRK8x3B/zfC/wH3dfVwTxiIhIxBI9ldTH3ReEj3fcfZ2Z3VukkYmISCQSLQw985jXuzADEZGi90t7MMyePZuUlBRSU1Np3749b7/9dlShSzEq8FSSmV1GMHR2YzNbGvPSkcA7v/ZNzWw4cBnwJbAHOAHYDfzJ3T/9tdsVkYJl92CoWrUqWVlZdO7cmd69e9OpUyf69etH165d45bv0aMHZ511FmbG0qVLGTRoECtXrowmeCk2B7vGMBV4FbgHiG2is8Pdvz+E983u03A5kOnuA8ysOUFvhh6HsF0RKcAv7cGQvSzAzp07MbNiiVOiVWBhcPdtwDbgfAAzq0Vwg1tVM6vq7l//0jfM1aehMdArfK+VZtbQzGq7+8aCtqFGPWpQk015CBwsDxmj++Y837dvH+3atWPNmjVcccUVBfZgAHj++ee58cYb2bRpEy+/rFyXBRaMxnqQhcz6Aw8AdYFNBN3YVrj7ib/qTc0ygPbAtcDh7n6tmXUA3gU6uvviPNaJ7cfQ7tZxj/+aty41aleGjbuijiJ6ykPgYHlofXy1A+Zl92AYPnx4Tg+GP/zhDzz22GM5PRhiLVmyhClTpjB27NhCi7uwZWZmxh3llEX55aBbt26L3b19IttI9OuqdwOnAG+6+0lm1o3wKOIQjQYeMrN0YBnwCfn0knb3fwD/AKjfuImPXVa2Wwpe13ovZT0HoDxkO1geMgZ3zXP+4sWL2bJlCxdffDEAhx9+OJ06dcpz1NSuXbsybtw4WrVqVWJHVU1LSzvgOklZUxg5SPQ3Ksvdt5hZOTMr5+7zC+Prqu6+HbgYwIKTl1+FjwJVrlieVTGHxmVRWlpavr/sZYnyEEg0D5s3b6ZixYpUr149pwfDyJEj811+zZo1nHDCCZgZH3/8MXv27OGYY44pxMilJEq0MGw1s6rAW8AzZraJfP6y/yXMrDrwowf9nocRdIrbfqjbFZG8bdiwgSFDhrBv3z7279/PoEGDcnow3HfffXz77bekpKTQp08fJk6cyKxZs5gyZQoVK1akcuXKzJgxQxegy4BEC8PZwC7gamAwUA24sxDevwUwxcz2AZ8BlxTCNkUkH7+0B8PIkSMLPKKQ0inR0VV3mlkDoKm7P2VmRxA01vlVYvo0fIcG4xMRKVESuvPZzP4HeBZ4LJx1PPBCUQUlIiLRSXRIjCuATsB2AHdfDdQqqqBERCQ6iRaGn8ILxACYWQV+HoZbRERKkUQLwwIz+39AZTPrSdCLYU7RhSUiIlFJtDCMAjYT3IT2F+AV4OaiCkpERKJzsNFV67v71+6+H3g8fIiISCl2sCOGnG8emdmsIo5FRERKgIMVhthbHBsXZSAikri1a9fSrVs3WrRowYknnsizzz4LBAPdnXrqqbRu3Zr+/fuzfXswkMAbb7xBu3btaN26Ne3atWPevHlRhi8l3MEKg+fz/JCY2XAzW2FmbmZLw8e7ZtamsN5DpDSrUKECY8eOZcWKFbz//vvMnj2bzz77jGHDhjF69GiWLVvGgAEDGDNmDAA1a9Zkzpw5LFu2jKeeeooLL7ww4j2Qkuxgdz63MbPtBEcOlcPnhNPu7kf9yvfNbtRTh2D47h/MrDfB6KkFDw6P+jGA+hBkK4t5yBjdlzp16lCnTh0AjjzySOrXr88333zDqlWr6NKlCwA9e/akV69e3HXXXXGNeE488UR2797NTz/9RKVKlSLZBynZCjxicPfy7n6Uux/p7hXC59nTv6oo5GrU09Hdfwhfeh+o92u2KVKWZWRksGbNGjp27EirVq148cUXAZg5cyZr1649YPlZs2Zx0kknqShIvhJq1FPobxo26nH372LmXQ80d/dh+ayjRj0x1KAmUBbzENt0Z9euXYwYMYLzzjuPM844g6+//pr/+7//Y9u2bXTq1InnnnuO2bNn5yz/1VdfcfPNN3Pfffdx/PHHRxF+kVKjnsJp1FMiCkPY+OfvQGd333Kw9es3buLlBj1UtEGWcGpQEyiLechu05mVlUW/fv3o1asXbdu2PaA5y+eff84FF1zAhx9+CMC6devo3r07kyZNolOnTsUddrFQo578c2Bmhd7BrciYWQowEeidSFEANeoBNajJVlbz4O5ccskltGjRgmuvvZa0tDQANm3aRK1atdi/fz933303l156KQBbt26lb9++3HPPPaW2KEjhSfTO5yJhZvWB54AL3f3zKGMRSSbvvPMOTz/9NPPmzSM1NZVhw4bxyiuvMG3aNJo1a0bz5s2pW7duTsvOCRMmsGbNGu666y5SU1NJTU1l06ZNEe+FlFRRHzHcChwD/D3sCrU30UMdkbKsc+fOxJ4Gjj19MGLEiAOWv/nmm7n5Zo1iI4mJpDDENOoZFj5ERKSEiPRUkoiIlDwqDCIiEkeFQURE4qgwiIhIHBUGERGJo8IgIiJxVBhERCSOCoNICZe7Kc9DDwXjhKWnp3PKKaeQmprKX/7yl5wxkdLS0qhWrVrOHc533nlnlOFLEorkBjczGw5cBnwMbAH6AD8CQ9394yhiEimpspvytG3blh07dtCuXTt69uzJDTfcwG233Ubv3r0ZPXo0N9xwQ86YSaeddhovvfRStIFL0opqSIzsRj0tgKuApgQNeh5BjXoSUhYb1OSltOchr6Y8LVq04JtvvsHMclp37ty5k7p160YZqpQixV4YcjXqaUZwlODA+2ZW3czquPuG4o5LJBlkZGTwySef0LFfUo7wAAAPiElEQVRjR8aNG0evXr24/vrr2b17N4sWLcpZ7r333qNNmzbUrVuX+++/nxNPPDHCqCXZRNqPAZgMjHb3t8P5c4GR7r4oj3XUqCdGWWxQk5fSnoe8mvJccMEFdOnShfHjx9OmTRtOP/10Xn31Vd58803Gjh3Lzp07KVeuHJUrV+b9999nwoQJ/POf/4xwL4qPGvWUgkY9wFPAPbkKww3uvrig9dWop2w2qMlLac9DXk15rr32WgCqVavG1q1bMTPmz5/P2WefnXNqKVbDhg1ZtGgRNWvWLNbYo6BGPaWjUc864Dcx0/WA9QdbSY16ym6DmtzKQh5yN+XJVrduXRYsWEDXrl35+OOPadq0KQDffvsttWvXxsz48MMP2b9/P8ccc0xU4UsSirowvAhcaWbTCS46b9P1BZF42U15WrduTWpqKgD/+7//y+OPP86IESPYu3cve/bsyTld9Oyzz/LII49QoUIFKleuzPTp0wn7nYgkJOrC8ArBV1XXEHxd9eJowxEpeXI35Ym1eHFw1jUtLY127doBcOWVV3LllVcWW3xS+kTdqAfgiihiEBGRvOnOZxERiaPCICIicVQYREQkjgqDiIjEUWEQEZE4KgwiIhJHhUEkQvn1Wvj973+f00+hYcOGOTe2bdmyhW7dulG1alXdqyBFJup+DPWB1TGxtACOdffvo4hLpLjl12thxowZOctcd911VKsWDKZ3+OGHc9ddd/Hpp5/y6aefRhW2lHKR9mNw96+yZ5hZf+AaFQUpS/LrtdCyZUsgGCfpX//6F/PmzQOgSpUqdO7cmTVr1kQWs5R+kfZjMLMn3f3B8KXzgWmJbEONekp/g5pEJXMeMnINBBnbayHbW2+9Re3atXMGyBMpDsVeGNz9UjM7E+jm7t8BmNkRwJmATppKmZSZmcnAgQMZN24cRx11VM78adOmcf7550cYmZRFUQ+il60/8E5Bp5FyNerh1tZ7iyu2Eql25eCv5bIumfOQ3Z9579693HjjjXTs2JEaNWrkzN+3bx8zZszgsccey5mXbeXKlXzzzTc58zMzMw9YpixSHgopB+5e7A8gA6gZM/088MdE12/WrJmXdfPnz486hBIh2fOwf/9+v/DCC33EiBEHvPbqq696ly5d8lxv0qRJfsUVV+RMJ3seCovykH8OgEWe4Gds5EcMZlYNOB24IOpYRIpbfr0W+vTpw/Tp0/M8jdSwYUO2b9/Onj17eOGFF3j99deLO2wp5SIvDMAA4HV33xl1ICLFraBeC5MnT85zfkZGxgHzNm3aVIhRSVkXeT8Gd58MTI4iDhEROZDufBYRkTgqDCIiEkeFQURE4qgwiIhIHBUGERGJo8IgIiJxVBhERCSOCoOUWvk1wfn+++/p2bMnTZs2pWfPnvzwww8A/PDDDwwYMICUlBQ6dOigfgdSZkVSGMxsuJmtMLNnzKyrmaWb2XIzWxBFPFI6ZTfBWbFiBe+//z4PP/wwn332GaNHj6ZHjx6sXr2aHj16MHr0aCAYiiI1NZWlS5cyZcoURowYEfEeiEQjqiOGy4E+wBXA34Gz3P1E4L8jikdKoTp16tC2bVsgvgnO7NmzGTJkCABDhgzhhRdeAOCzzz6jR48eADRv3pyMjAw2btwYTfAiEYq0UQ8wHXjO3b8GcPeEBnxRo57kblBTmPLLQ0FNcDZu3JjTNa1OnTo54wy1adOG5557js6dO/Phhx/yn//8h3Xr1lG7du2i3xGREiTSRj3AzUBFM0sDjgQecvcpea2nfgzxkrkPQWHKLw+x49Hv2rWLESNGMGzYMD7++GP27t0b93r2dKdOnZgwYQJNmjShcePGNGnShE8++YQdO3YUw54cGvUhCCgPhZMDy29kx6JkZhlAe+D28GcPoDLwHtDX3T8vaP36jZt4uUEPFXGUJdt1rfcydllJGBw3WvnlIfuIISsri379+tGrVy+uvfZaAH7729+SlpZGnTp12LBhA127dmXVqlVx67s7jRo1YunSpXEd1UqqtLQ0unbtGnUYkVMe8s+BmS129/aJbCPqT5Z1wHfhkNs7zWwh0AYosDBUrlieVblOFZQ1aWlpZAzuGnUYkSsoD+7OJZdcQosWLXKKAsBZZ53FU089xahRo3jqqac4++yzAdi6dStHHHEEhx12GBMnTqRLly5JURREClvUhWE2MMHMKgCHAR2BB6MNSUqL/JrgjBo1ikGDBvHEE09Qv359Zs6cCcCKFSu46KKLKF++PC1btuSJJ56IMnyRyERaGNx9hZm9BiwF9gMT3V1fHpdCUVATnLlz5x4w79RTT2X16tVFHZZIiVcSGvWMAcZEEYeIiBxIdz6LiEgcFQYREYmjwiAiInFUGEREJI4Kg4iIxFFhEBGROCoMIiISR4VBktaf/vQnBgwYQKtWrXLmLVmyhFNPPZXWrVvTv39/tm/fDgSjq1auXJnU1FRSU1O59NJLowpbpMQrssIQ04xnlpm9Z2Y/mdn1uZZ50sw2mZnudpZfbOjQodx7771x84YNG8bo0aNZtmwZAwYMYMyYn++dPOGEE0hPTyc9PZ1HH320uMMVSRpFeefz5UBvYCfQADgnj2UmAxOAPIfazo/6MZTtfgzZI6d26dKF9evXx722atUqunTpAkDPnj3p1asXd911V7HHKJLMiuSIIVcznsHu/hGQlXs5d18IfF8UMUjZ1KpVK1588UUAZs6cydq1a3Ne++qrrzjppJM4/fTTeeutt6IKUaTEK5IjhthmPO7+XWFsU4164pXlRj2xTUh27tzJzp07c+Zdeuml3H333fz1r3+lU6dOlCtXjrS0NPbs2cPUqVOpVq0aq1atYuDAgUyaNIkqVapEsxOFTA1qAspD4eQg6mG3E+bu/wD+AUGjnrLepKYsN+qJ7b/w7bffUqVKlbjGJBdddBEAn3/+OcuXLz+gaUnXrl2ZNm0atWvXpn37hPqWlHhqUBNQHgonB0n5yaJGPWrUk59NmzZRq1Yt9u/fz913353z7aPNmzdTo0YNypcvz5dffsnq1atp3LhxxNGKlExJWRhEAM4//3xef/11tm/fTr169bjjjjvIzMzk4YcfBuDcc8/l4osvBmDhwoXceuutVKhQgfLly/Poo49So0aNKMMXKbGKvDCY2XHAIuAoYL+ZXQ20dPftZjYN6ArUNLN1wG3urrZZkpBp06bledg8YsSIA5YdOHAgAwcOLKbIRJJbkRWG2GY8QL18ljm/qN5fRER+Hd35LCIicVQYREQkjgqDiIjEUWEQEZE4KgwiIhJHhUFEROKoMIiISBwVBhERiaPCICIicVQYREQkjgqDiIjEMXePOoZfzMx2AKuijiNiNYFCaYKU5JSHgPIQUB7yz0EDdz82kQ0k67Dbq9y9dHRY+ZXMbFFZzwEoD9mUh4DyUDg50KkkERGJo8IgIiJxkrUw/CPqAEoA5SCgPASUh4DyUAg5SMqLzyIiUnSS9YhBRESKiAqDiIjESarCYGZnmtkqM1tjZqOijqcomdmTZrbJzD6NmVfDzN4ws9Xhz6PD+WZm48O8LDWzttFFXnjM7DdmNt/MVpjZcjMbEc4va3k43Mw+NLMlYR7uCOc3MrMPwjzMMLPDwvmVwuk14esNo4y/sJlZeTP7xMxeCqfLXB7MLMPMlplZupktCucV2u9F0hQGMysPPAz0BloC55tZy2ijKlKTgTNzzRsFzHX3psDccBqCnDQNH38GHimmGIvaXuA6d28BnAJcEf6bl7U8/AR0d/c2QCpwppmdAtwLPBjm4QfgknD5S4Af3L0J8GC4XGkyAlgRM11W89DN3VNj7lkovN8Ld0+KB3Aq8O+Y6RuBG6OOq4j3uSHwacz0KqBO+LwOwY1+AI8B5+e1XGl6ALOBnmU5D8ARwMdAR4K7WyuE83N+P4B/A6eGzyuEy1nUsRfS/tcLP/S6Ay8BVkbzkAHUzDWv0H4vkuaIATgeWBszvS6cV5bUdvcNAOHPWuH8Up+b8DTAScAHlME8hKdP0oFNwBvAF8BWd98bLhK7rzl5CF/fBhxTvBEXmXHADcD+cPoYymYeHHjdzBab2Z/DeYX2e5FMQ2JYHvP0XdtAqc6NmVUFZgFXu/t2s7x2N1g0j3mlIg/uvg9INbPqwPNAi7wWC3+WyjyYWT9gk7svNrOu2bPzWLRU5yHUyd3Xm1kt4A0zW1nAsr84D8l0xLAO+E3MdD1gfUSxRGWjmdUBCH9uCueX2tyYWUWCovCMuz8Xzi5zecjm7luBNIJrLtXNLPuPu9h9zclD+Ho14PvijbRIdALOMrMMYDrB6aRxlL084O7rw5+bCP5Q6EAh/l4kU2H4CGgafgPhMOAPwIsRx1TcXgSGhM+HEJxzz55/Ufjtg1OAbdmHlMnMgkODJ4AV7v5AzEtlLQ/HhkcKmFll4L8ILr7OB84LF8udh+z8nAfM8/DkcjJz9xvdvZ67NyT4/Z/n7oMpY3kwsypmdmT2c+AM4FMK8/ci6osov/CCSx/gc4LzqzdFHU8R7+s0YAOQRVDxLyE4PzoXWB3+rBEuawTf2PoCWAa0jzr+QspBZ4JD3qVAevjoUwbzkAJ8EubhU+DWcH5j4ENgDTATqBTOPzycXhO+3jjqfSiCnHQFXiqLeQj3d0n4WJ79WViYvxcaEkNEROIk06kkEREpBioMIiISR4VBRETiqDCIiEgcFQYREYmTTHc+ixQpM9tH8HW+bOe4e0ZE4YhERl9XFQmZWaa7Vy3G96vgP4/xI1Ji6FSSSILMrI6ZLQzHwP/UzE4L559pZh+H/RLmhvNqmNkL4fj375tZSjj/djP7h5m9DkwJB8cbY2Yfhcv+JcJdFAF0KkkkVuVwBFOAr9x9QK7X/0gwpPPfwv4gR5jZscDjQBd3/8rMaoTL3gF84u7nmFl3YApBLwWAdkBnd98Vjoy5zd1PNrNKwDtm9rq7f1WUOypSEBUGkZ/tcvfUAl7/CHgyHNjvBXdPD0f5XJj9Qe7u2YO0dQYGhvPmmdkxZlYtfO1Fd98VPj8DSDGz7LF+qhE0VFFhkMioMIgkyN0XmlkXoC/wtJmNAbaS9xDGBQ11vDPXcle5+78LNViRQ6BrDCIJMrMGBP0AHicY9bUt8B5wupk1CpfJPpW0EBgczusKfOfu2/PY7L+By8KjEMysWThipkhkdMQgkriuwF/NLAvIBC5y983hdYLnzKwcwRj4PYHbgUlmthT4kZ+HQ85tIkEL14/DYcY3A+cU5U6IHIy+rioiInF0KklEROKoMIiISBwVBhERiaPCICIicVQYREQkjgqDiIjEUWEQEZE4/x8Re5eU8c7vKQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Below is a function of xgboost classifier with illustrates relative importance of a feature within a model. \n",
    "# This was used to trim unneeded features.\n",
    "\n",
    "model = XGBClassifier()\n",
    "model.fit(X_tr, y_train)\n",
    "\n",
    "plot_importance(model)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 43.39%\n",
      "Thresh=0.050, n=12, Accuracy: 43.45%\n",
      "Thresh=0.054, n=11, Accuracy: 43.12%\n",
      "Thresh=0.070, n=10, Accuracy: 42.65%\n",
      "Thresh=0.073, n=9, Accuracy: 42.91%\n",
      "Thresh=0.075, n=8, Accuracy: 42.58%\n",
      "Thresh=0.080, n=7, Accuracy: 42.38%\n",
      "Thresh=0.082, n=6, Accuracy: 41.70%\n",
      "Thresh=0.090, n=5, Accuracy: 40.62%\n",
      "Thresh=0.095, n=4, Accuracy: 39.95%\n",
      "Thresh=0.102, n=3, Accuracy: 39.88%\n",
      "Thresh=0.111, n=2, Accuracy: 39.14%\n",
      "Thresh=0.119, n=1, Accuracy: 39.20%\n"
     ]
    }
   ],
   "source": [
    "from numpy import loadtxt\n",
    "from numpy import sort\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "\n",
    "model = XGBClassifier()\n",
    "model.fit(X_tr, y_train)\n",
    "# make predictions for test data and evaluate\n",
    "y_pred = model.predict(X_te)\n",
    "predictions = [value for value in y_pred]\n",
    "accuracy = accuracy_score(y_test, predictions)\n",
    "print(\"Accuracy: %.2f%%\" % (accuracy * 100.0))\n",
    "# Fit model using each importance as a threshold\n",
    "thresholds = sort(model.feature_importances_)\n",
    "for thresh in thresholds:\n",
    "    # select features using threshold\n",
    "    selection = SelectFromModel(model, threshold=thresh, prefit=True)\n",
    "    select_X_train = selection.transform(X_train)\n",
    "    # train model\n",
    "    selection_model = XGBClassifier()\n",
    "    selection_model.fit(select_X_train, y_train)\n",
    "    # eval model\n",
    "    select_X_test = selection.transform(X_test)\n",
    "    y_pred = selection_model.predict(select_X_test)\n",
    "    predictions = [value for value in y_pred]\n",
    "    accuracy = accuracy_score(y_test, predictions)\n",
    "    print(\"Thresh=%.3f, n=%d, Accuracy: %.2f%%\" % (thresh, select_X_train.shape[1], accuracy*100.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With feature selection from XGBoost, I removed features: 6, 0 , 12, 3, 13, 17, 15 (chroma, flatness, contrast_std, all dem) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the model to disk\n",
    "import pickle\n",
    "name = 'finalized_model.sav'\n",
    "pickle.dump(model, open(name, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_features(filename):\n",
    "    '''Loading in audiofiles and extracting features. Features include:\n",
    "        - mfcc = Mel-frequency cepstral coefficients. Commonly used for vocals.\n",
    "        - centroid = Spectral Centroid. Represents mean value of frequency form. \n",
    "        - flatness = Spectral flatness. Distinguishes noisy vs harmonic sound.\n",
    "        - tempo = Spectral onset envelope. Describes rythm.\n",
    "        - cens = Chroma Energy Normalized Statistics. Smooths frequency windows for matching.\n",
    "        - energy = Root Mean Square Energy. Computes energy of each frame.\n",
    "        - melspec = Mel-Scaled Spectrogram.\n",
    "        - contrast = Spectral Contrast.\n",
    "        - tonnetz = Tonnetz. Computes tonal centroid features.\n",
    "        - chroma = Chromagram from waveform.\n",
    "    '''\n",
    "    \n",
    "    # load in file. y is the waveform, sr is the sampling rate.\n",
    "    y, sr = lb.load(filename)\n",
    "    \n",
    "    # Short-time Fourier transformation.\n",
    "    stft = np.abs(lb.stft(y))\n",
    "    S, phase = lb.magphase(np.abs(stft))\n",
    "\n",
    "    # all features chroma, contrast, energy, flatness, and MFCC\n",
    "    mfcc = np.mean(lb.feature.mfcc(y=y, sr=sr))   \n",
    "    mfcc_std = np.std(lb.feature.mfcc(y=y, sr=sr))\n",
    "    centroid = np.mean(lb.feature.spectral_centroid(y=y, sr=sr))\n",
    "    centroid_std = np.std(lb.feature.spectral_centroid(y=y, sr=sr))\n",
    "    flatness = np.mean(lb.feature.spectral_flatness(y=y, S=S))\n",
    "    flatness_std = np.std(lb.feature.spectral_flatness(y=y, S=S))\n",
    "    tempo = np.mean(lb.feature.tempogram(y=y, sr=sr))\n",
    "    tempo_std = np.std(lb.feature.tempogram(y=y, sr=sr))\n",
    "    cens = np.mean(lb.feature.chroma_cens(y=y, sr=sr))\n",
    "    cens_std = np.std(lb.feature.chroma_cens(y=y, sr=sr))\n",
    "    energy = np.mean(lb.feature.rmse(y=y))\n",
    "    energy_std = np.std(lb.feature.rmse(y=y))\n",
    "    melspec = np.mean(lb.feature.melspectrogram(y=y, sr=sr))\n",
    "    melspec_std = np.std(lb.feature.melspectrogram(y=y, sr=sr))\n",
    "    contrast = np.mean(lb.feature.spectral_contrast(y=y, sr=sr))\n",
    "    contrast_std = np.std(lb.feature.spectral_contrast(y=y, sr=sr))\n",
    "    tonnetz = np.mean(lb.feature.tonnetz(y=y, sr=sr))\n",
    "    tonnetz_std = np.std(lb.feature.tonnetz(y=y, sr=sr))\n",
    "    chroma = np.mean(librosa.feature.chroma_stft(S=stft, sr=sr))\n",
    "    chroma_std = np.std(librosa.feature.chroma_stft(S=stft, sr=sr))\n",
    "    zero = np.mean(librosa.feature.zero_crossing_rate(y=y))\n",
    "    zero_std = np.std(librosa.feature.zero_crossing_rate(y=y))\n",
    "\n",
    "    # Create a dict of features\n",
    "    audio_features = {'filename': filename,\n",
    "                     'mfcc': mfcc,\n",
    "                     'mfcc_std': mfcc_std,\n",
    "                     'centroid': centroid,\n",
    "                     'centroid_std': centroid_std,\n",
    "                     'flatness': flatness,\n",
    "                     'flatness_std': flatness_std,\n",
    "                     'tempo': tempo,\n",
    "                     'tempo_std':tempo_std,\n",
    "                     'cens': cens,\n",
    "                     'cens_std':cens_std,\n",
    "                     'energy': energy,\n",
    "                     'energy_std': energy_std,\n",
    "                     'melspec': melspec,\n",
    "                     'melspec_std':melspec_std,\n",
    "                     'contrast': contrast,\n",
    "                     'contrast_std': contrast_std,\n",
    "                     'tonnetz': tonnetz,\n",
    "                     'tonnetz_std':tonnetz_std,\n",
    "                     'chroma': chroma,\n",
    "                     'chroma_std': chroma_std,\n",
    "                     'zero': zero,\n",
    "                     'zero_std': zero_std\n",
    "\n",
    "                     }\n",
    "    return audio_features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_function(filename):\n",
    "    varname = extract_features(filename)\n",
    "    varname = pd.DataFrame(varname, index=[0])\n",
    "    varname = varname.drop(columns=['cens', 'cens_std', 'centroid', 'centroid_std',\n",
    "                                    'chroma','contrast_std', 'filename', 'flatness', 'melspec', 'melspec_std',\n",
    "                                    'mfcc', 'tempo', 'tempo_std', 'tonnetz', 'tonnetz_std'])\n",
    "    #varname = std.transform(varname.values)\n",
    "    #varname = varname.values\n",
    "    std_test = MinMaxScaler()\n",
    "    std_test.fit(varname.values)\n",
    "    varname = std_test.transform(varname.values)\n",
    "    return clf_xgb_ADHN.predict(varname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0])"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_function('angry_test.mp3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}