Testing models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jose/anaconda3/envs/intro/lib/python3.5/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n",
    "from sklearn.model_selection import GridSearchCV, cross_validate\n",
    "from utils import predicted_report\n",
    "from sklearn.naive_bayes import BernoulliNB, MultinomialNB\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "import scikitplot.plotters as skplt\n",
    "import matplotlib.pyplot as plt\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./rebalanced_data/test_set.pickle', mode='rb') as f:\n",
    "    test_set = pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Se intentará comprobar el rendimiento de los modelos de manera agrupada, de tal manera que se pueda comprobar fácilmente el rendimiento de los mismos con diferentes técnicas de rebalanceo y con las diferentes técnicas de reducción de la dimensionalidad.\n",
    "\n",
    "Nota: Los modelos en los que no se especifica cómo se ha realizado el rebalanceo, se ha usado el método de rebalanceo simple aleatorio (SRU)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./final_models/dt_final.pickle', mode='rb') as f:\n",
    "    dt = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results_to_vals = np.vectorize(lambda x: 'legit' if x == 1 else 'phishing')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1177        94  1271\n",
      "phishing     124       816   940\n",
      "All         1301       910  2211\n",
      "\n",
      "Accuracy: 0.901402\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.90      0.93      0.92      1271\n",
      "   phishing       0.90      0.87      0.88       940\n",
      "\n",
      "avg / total       0.90      0.90      0.90      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(dt.predict(test_set['x'])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('./final_models/rf_final.pickle', mode='rb') as f:\n",
    "    rf = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1199        72  1271\n",
      "phishing      69       871   940\n",
      "All         1268       943  2211\n",
      "\n",
      "Accuracy: 0.936228\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.95      0.94      0.94      1271\n",
      "   phishing       0.92      0.93      0.93       940\n",
      "\n",
      "avg / total       0.94      0.94      0.94      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(rf.predict(test_set['x'])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extra Trees Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./final_models/xt_final.pickle', mode='rb') as f:\n",
    "    xt = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1200        71  1271\n",
      "phishing      78       862   940\n",
      "All         1278       933  2211\n",
      "\n",
      "Accuracy: 0.932610\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.94      0.94      0.94      1271\n",
      "   phishing       0.92      0.92      0.92       940\n",
      "\n",
      "avg / total       0.93      0.93      0.93      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(xt.predict(test_set['x'])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bernoulli Naïve Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./final_models/bnb_final.pickle', mode='rb') as f:\n",
    "    bnb = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit        550       721  1271\n",
      "phishing       4       936   940\n",
      "All          554      1657  2211\n",
      "\n",
      "Accuracy: 0.672094\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.99      0.43      0.60      1271\n",
      "   phishing       0.56      1.00      0.72       940\n",
      "\n",
      "avg / total       0.81      0.67      0.65      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(bnb.predict(test_set['x'])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multinomial Naïve Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./final_models/mnb_final.pickle', mode='rb') as f:\n",
    "    mnb = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit        313       958  1271\n",
      "phishing       0       940   940\n",
      "All          313      1898  2211\n",
      "\n",
      "Accuracy: 0.566712\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       1.00      0.25      0.40      1271\n",
      "   phishing       0.50      1.00      0.66       940\n",
      "\n",
      "avg / total       0.79      0.57      0.51      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(mnb.predict(test_set['x'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./final_models/mnb_final_pca_sru.pickle', mode='rb') as f:\n",
    "    mnb_pca = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('./final_models/pca.pickle', mode='rb') as f:\n",
    "    pca = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1201        70  1271\n",
      "phishing      99       841   940\n",
      "All         1300       911  2211\n",
      "\n",
      "Accuracy: 0.923564\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.92      0.94      0.93      1271\n",
      "   phishing       0.92      0.89      0.91       940\n",
      "\n",
      "avg / total       0.92      0.92      0.92      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), \n",
    "                 # Se seleccionan sólo las 17 primeras componentes\n",
    "                 results_to_vals(mnb_pca.predict([row[:-11] for row in pca.transform(test_set['x'])]))\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No se prueba la versión LDA por resultados pésimos en el conjunto de entrenamiento."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./final_models/xgboost_final.pickle', mode='rb') as f:\n",
    "    xgb = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1223        48  1271\n",
      "phishing      24       916   940\n",
      "All         1247       964  2211\n",
      "\n",
      "Accuracy: 0.967436\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.98      0.96      0.97      1271\n",
      "   phishing       0.95      0.97      0.96       940\n",
      "\n",
      "avg / total       0.97      0.97      0.97      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(xgb.predict(test_set['x'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./final_models/gb_final_pca_sru.pickle', mode='rb') as f:\n",
    "    xgb_pca = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1205        66  1271\n",
      "phishing      34       906   940\n",
      "All         1239       972  2211\n",
      "\n",
      "Accuracy: 0.954772\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.97      0.95      0.96      1271\n",
      "   phishing       0.93      0.96      0.95       940\n",
      "\n",
      "avg / total       0.96      0.95      0.95      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), \n",
    "                 # Se seleccionan sólo las 17 primeras componentes\n",
    "                 results_to_vals(xgb_pca.predict([row[:-11] for row in pca.transform(test_set['x'])]))\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./final_models/gb_final_lda_sru.pickle', mode='rb') as f:\n",
    "    xgb_lda = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./final_models/lda.pickle', mode='rb') as f:\n",
    "    lda = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1178        93  1271\n",
      "phishing      56       884   940\n",
      "All         1234       977  2211\n",
      "\n",
      "Accuracy: 0.932610\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.95      0.93      0.94      1271\n",
      "   phishing       0.90      0.94      0.92       940\n",
      "\n",
      "avg / total       0.93      0.93      0.93      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']),\n",
    "                 results_to_vals(xgb_lda.predict(lda.transform(test_set['x'])))\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVM: Kernel Lineal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('./final_models/lsvm_final.pickle', mode='rb') as f:\n",
    "    lsvm = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1199        72  1271\n",
      "phishing      70       870   940\n",
      "All         1269       942  2211\n",
      "\n",
      "Accuracy: 0.935776\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.94      0.94      0.94      1271\n",
      "   phishing       0.92      0.93      0.92       940\n",
      "\n",
      "avg / total       0.94      0.94      0.94      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(lsvm.predict(test_set['x'])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No se incluyen los modelos entrenados con los conjuntos de entrenamiento generados mediante OSS ni CNN por malos resultados en el propio conjunto de entrenamiento."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVM: Kernel Polinomial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./final_models/poly_svm_final.pickle', mode='rb') as f:\n",
    "    poly_svm = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1188        83  1271\n",
      "phishing      55       885   940\n",
      "All         1243       968  2211\n",
      "\n",
      "Accuracy: 0.937585\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.96      0.93      0.95      1271\n",
      "   phishing       0.91      0.94      0.93       940\n",
      "\n",
      "avg / total       0.94      0.94      0.94      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(poly_svm.predict(test_set['x'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('./final_models/poly_svm_final_lda_sru.pickle', mode='rb') as f:\n",
    "    poly_svm = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1239        32  1271\n",
      "phishing     150       790   940\n",
      "All         1389       822  2211\n",
      "\n",
      "Accuracy: 0.917684\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.89      0.97      0.93      1271\n",
      "   phishing       0.96      0.84      0.90       940\n",
      "\n",
      "avg / total       0.92      0.92      0.92      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(poly_svm.predict(lda.transform(test_set['x']))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('./final_models/poly_svm_final_pca_sru.pickle', mode='rb') as f:\n",
    "    poly_svm = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1198        73  1271\n",
      "phishing      41       899   940\n",
      "All         1239       972  2211\n",
      "\n",
      "Accuracy: 0.948440\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.97      0.94      0.95      1271\n",
      "   phishing       0.92      0.96      0.94       940\n",
      "\n",
      "avg / total       0.95      0.95      0.95      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), \n",
    "                 results_to_vals(poly_svm.predict([row[:-11] for row in pca.transform(test_set['x'])]))\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVM: Kernel RBF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./final_models/rbf_svm_final.pickle', mode='rb') as f:\n",
    "    rbf_svm = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit        314       957  1271\n",
      "phishing       4       936   940\n",
      "All          318      1893  2211\n",
      "\n",
      "Accuracy: 0.565355\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.99      0.25      0.40      1271\n",
      "   phishing       0.49      1.00      0.66       940\n",
      "\n",
      "avg / total       0.78      0.57      0.51      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(rbf_svm.predict(test_set['x'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./final_models/rbf_svm_oss_final.pickle', mode='rb') as f:\n",
    "    rbf_svm = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1243        28  1271\n",
      "phishing      50       890   940\n",
      "All         1293       918  2211\n",
      "\n",
      "Accuracy: 0.964722\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.96      0.98      0.97      1271\n",
      "   phishing       0.97      0.95      0.96       940\n",
      "\n",
      "avg / total       0.96      0.96      0.96      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(rbf_svm.predict(test_set['x'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./final_models/rbf_svm_cnn_final.pickle', mode='rb') as f:\n",
    "    rbf_svm = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1226        45  1271\n",
      "phishing      47       893   940\n",
      "All         1273       938  2211\n",
      "\n",
      "Accuracy: 0.958390\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.96      0.96      0.96      1271\n",
      "   phishing       0.95      0.95      0.95       940\n",
      "\n",
      "avg / total       0.96      0.96      0.96      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(rbf_svm.predict(test_set['x'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('./final_models/rbf_svm_final_lda_sru.pickle', mode='rb') as f:\n",
    "    rbf_svm = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1179        92  1271\n",
      "phishing      65       875   940\n",
      "All         1244       967  2211\n",
      "\n",
      "Accuracy: 0.928991\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.95      0.93      0.94      1271\n",
      "   phishing       0.90      0.93      0.92       940\n",
      "\n",
      "avg / total       0.93      0.93      0.93      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(rbf_svm.predict(lda.transform(test_set['x']))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('./final_models/rbf_svm_final_pca_sru.pickle', mode='rb') as f:\n",
    "    rbf_svm = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit        315       956  1271\n",
      "phishing       4       936   940\n",
      "All          319      1892  2211\n",
      "\n",
      "Accuracy: 0.565807\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.99      0.25      0.40      1271\n",
      "   phishing       0.49      1.00      0.66       940\n",
      "\n",
      "avg / total       0.78      0.57      0.51      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(rbf_svm.predict([row[:-11] for row in pca.transform(test_set['x'])])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVM: Kernel Sigmoidal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('./final_models/sig_svm_final.pickle', mode='rb') as f:\n",
    "    sig_svm = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1201        70  1271\n",
      "phishing      82       858   940\n",
      "All         1283       928  2211\n",
      "\n",
      "Accuracy: 0.931253\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.94      0.94      0.94      1271\n",
      "   phishing       0.92      0.91      0.92       940\n",
      "\n",
      "avg / total       0.93      0.93      0.93      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(sig_svm.predict(test_set['x'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('./final_models/sig_svm_final_oss.pickle', mode='rb') as f:\n",
    "    sig_svm = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1209        62  1271\n",
      "phishing      90       850   940\n",
      "All         1299       912  2211\n",
      "\n",
      "Accuracy: 0.931253\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.93      0.95      0.94      1271\n",
      "   phishing       0.93      0.90      0.92       940\n",
      "\n",
      "avg / total       0.93      0.93      0.93      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(sig_svm.predict(test_set['x'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('./final_models/sig_svm_final_cnn.pickle', mode='rb') as f:\n",
    "    sig_svm = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit        395       876  1271\n",
      "phishing       0       940   940\n",
      "All          395      1816  2211\n",
      "\n",
      "Accuracy: 0.603799\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       1.00      0.31      0.47      1271\n",
      "   phishing       0.52      1.00      0.68       940\n",
      "\n",
      "avg / total       0.79      0.60      0.56      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(sig_svm.predict(test_set['x'])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted  legit  phishing   All\n",
      "Actual                          \n",
      "legit       1207        64  1271\n",
      "phishing      81       859   940\n",
      "All         1288       923  2211\n",
      "\n",
      "Accuracy: 0.934419\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      legit       0.94      0.95      0.94      1271\n",
      "   phishing       0.93      0.91      0.92       940\n",
      "\n",
      "avg / total       0.93      0.93      0.93      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predicted_report(results_to_vals(test_set['y']), results_to_vals(lda.predict(test_set['x'])))  "
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [intro]",
   "language": "python",
   "name": "Python [intro]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}