Costina-tfidf-bigram-trigram.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import csv\n",
    "import pandas as pd\n",
    "import pickle\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "import spacy\n",
    "from preprocessing import applyPreProcessing, applyPreProcessingCorpus\n",
    "from bow import *\n",
    "from tfidf import *\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nlp = spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_data_neg = pd.read_csv('./twitter-datasets/train_neg.txt', header=None, sep=\"\\n\", encoding='utf-8', names=['text'],\n",
    "                               error_bad_lines=False, warn_bad_lines=False, quoting=csv.QUOTE_NONE).drop_duplicates()\n",
    "raw_data_neg['label'] = 0\n",
    "\n",
    "raw_data_pos = pd.read_csv('./twitter-datasets/train_pos.txt', header=None, sep=\"\\n\", encoding='utf-8', names=['text'],\n",
    "                               error_bad_lines=False, warn_bad_lines=False, quoting=csv.QUOTE_NONE).drop_duplicates()\n",
    "raw_data_pos['label'] = 1\n",
    "\n",
    "raw_data_train = pd.concat([raw_data_neg, raw_data_pos], ignore_index=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sea doo pro sea scooter ( sports with the port...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;user&gt; shucks well i work all week so now i ca...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>i cant stay away from bug thats my baby</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;user&gt; no ma'am ! ! ! lol im perfectly fine an...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>whenever i fall asleep watching the tv , i alw...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;user&gt; he needs to get rid of that thing ! it ...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>its whatever . in a terrible mood ( (</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>yesss ! rt &lt;user&gt; &lt;user&gt; thanks jordan , i lov...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>my friend &lt;user&gt; text me to check up on me las...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>&lt;user&gt; #followback please . when will ur #unit...</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>watch some of y'all dumb asses get lock up tod...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>obsessed with #phasell &lt;user&gt; you killed it ! ...</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>&lt;user&gt; robert de niro is not gay .. but with a...</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>&lt;user&gt; canada have to do it in grade 12 . but ...</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>&lt;user&gt; please say hi to denmark ! that would b...</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>finally am home now</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>3x3 custom picture frame / poster frame 1.2 \" ...</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>s / o to my new followers . mention me for a f...</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>&lt;user&gt; yep , looks like the best team will sta...</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>nhl's bettman : suspension criticism ' gamesma...</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text  id\n",
       "0   sea doo pro sea scooter ( sports with the port...   1\n",
       "1   <user> shucks well i work all week so now i ca...   2\n",
       "2             i cant stay away from bug thats my baby   3\n",
       "3   <user> no ma'am ! ! ! lol im perfectly fine an...   4\n",
       "4   whenever i fall asleep watching the tv , i alw...   5\n",
       "5   <user> he needs to get rid of that thing ! it ...   6\n",
       "6               its whatever . in a terrible mood ( (   7\n",
       "7   yesss ! rt <user> <user> thanks jordan , i lov...   8\n",
       "8   my friend <user> text me to check up on me las...   9\n",
       "9   <user> #followback please . when will ur #unit...  10\n",
       "10  watch some of y'all dumb asses get lock up tod...  11\n",
       "11  obsessed with #phasell <user> you killed it ! ...  12\n",
       "12  <user> robert de niro is not gay .. but with a...  13\n",
       "13  <user> canada have to do it in grade 12 . but ...  14\n",
       "14  <user> please say hi to denmark ! that would b...  15\n",
       "15                                finally am home now  16\n",
       "16  3x3 custom picture frame / poster frame 1.2 \" ...  17\n",
       "17  s / o to my new followers . mention me for a f...  18\n",
       "18  <user> yep , looks like the best team will sta...  19\n",
       "19  nhl's bettman : suspension criticism ' gamesma...  20"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_data_test = pd.read_csv('./twitter-datasets/test_data.txt', header=None, sep=\"\\n\", encoding='utf-8', names=['text'],\n",
    "                               error_bad_lines=False, warn_bad_lines=False, quoting=csv.QUOTE_NONE).drop_duplicates()\n",
    "\n",
    "\n",
    "raw_data_test['id'], raw_data_test['text'] = raw_data_test['text'].apply(lambda x: x.split(',')[0]), raw_data_test['text'].apply(lambda x: ','.join(x.split(',')[1:]))\n",
    "\n",
    "raw_data_test.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data = pd.DataFrame(data={'text': raw_data_train['text'].apply(applyPreProcessing)})\n",
    "# data = applyPreProcessing(raw_data.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "applyPreProcessingCorpus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame(data={'text': raw_data_train['text'].apply(applyPreProcessing)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>vinco tresorpack 6 difficulty 10 of 10 object ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>glad i dot have takes tomorrow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1 3 as celtics in the regular season were fuck...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>i could actually kill that girl i m so sorry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>i find that very hard to believe is afraid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>wish i could be out all night tonight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>i got kicked out the wgm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>it yes she is you tell it my lips are closed okay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>why is she so perfect</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>hi harry did you have good time in us i didnt ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text\n",
       "0  vinco tresorpack 6 difficulty 10 of 10 object ...\n",
       "1                    glad i dot have takes tomorrow \n",
       "2  1 3 as celtics in the regular season were fuck...\n",
       "3      i could actually kill that girl i m so sorry \n",
       "4         i find that very hard to believe is afraid\n",
       "5             wish i could be out all night tonight \n",
       "6                           i got kicked out the wgm\n",
       "7  it yes she is you tell it my lips are closed okay\n",
       "8                             why is she so perfect \n",
       "9  hi harry did you have good time in us i didnt ..."
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataTest = pd.DataFrame(data={'text': raw_data_test['text'].apply(applyPreProcessing)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sea do professional sea scooter sports with th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>shucks well i work all week so now i can t com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>i cant stay away from bug whats my baby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>no ma am lot is perfectly fine and not contagi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>whenever i fall asleep watching the television...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>he needs to get rip of that thing it scares me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>its whatever in a terrible mood</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>yes it thanks jordan i love you and i m gonna ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>my friend text me to check up on me last night</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>please when will your come to europe and sweden</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text\n",
       "0  sea do professional sea scooter sports with th...\n",
       "1  shucks well i work all week so now i can t com...\n",
       "2            i cant stay away from bug whats my baby\n",
       "3  no ma am lot is perfectly fine and not contagi...\n",
       "4  whenever i fall asleep watching the television...\n",
       "5  he needs to get rip of that thing it scares me...\n",
       "6                   its whatever in a terrible mood \n",
       "7  yes it thanks jordan i love you and i m gonna ...\n",
       "8    my friend text me to check up on me last night \n",
       "9   please when will your come to europe and sweden "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataTest.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bowDic = generateBow(data['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(bowDic.vocabulary_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # example of vectorized text\n",
    "# print(data['text'][91])\n",
    "# vecSample = getVectorRepresentation(bowDic, data['text'][91])\n",
    "# print(vecSample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tweets_bow = vectorizeDataSet(bowDic, data['text'])\n",
    "# tweets_bow.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TFIDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tfidf_transformer = getTFIDFTransformer(tweets_bow)\n",
    "# tfidf_sample = getTFIDFBowSingle(tfidf_transformer, vecSample)\n",
    "# print(tfidf_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data['vec'] = data['text'].apply(lambda x: getVectorRepresentation(bowDic, x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data['tfidf'] = data['vec'].apply(lambda x: getTFIDFBowSingle(tfidf_transformer, x))\n",
    "# data.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "def ngrams(input_list):\n",
    "    #onegrams = input_list\n",
    "    bigrams = [' '.join(t) for t in list(zip(input_list, input_list[1:]))]\n",
    "    trigrams = [' '.join(t) for t in list(zip(input_list, input_list[1:], input_list[2:]))]\n",
    "    return bigrams+trigrams\n",
    "    \n",
    "data['grams'] = data.tokens.apply(ngrams)\n",
    "data[['grams']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data #preprocessing applied\n",
    "# # create pipeline\n",
    "# from sklearn.pipeline import Pipeline\n",
    "# from sklearn.model_selection import train_test_split\n",
    "# from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
    "# from sklearn.naive_bayes import MultinomialNB\n",
    "# from sklearn.model_selection import KFold, cross_val_score\n",
    "# from sklearn.ensemble import RandomForestClassifier\n",
    "# from sklearn.feature_extraction.text import CountVectorizer\n",
    "# from sklearn.feature_extraction.text import TfidfTransformer\n",
    "# from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# # grid = GridSearchCV(estimator=pipeline, cv=5, param_grid=parameters, verbose=1)\n",
    "# # grid.fit(data['text'], raw_data_train['label'])\n",
    "\n",
    "# def buildModelITFGrams():\n",
    "#     model = Pipeline([\n",
    "#     ('bow', CountVectorizer(strip_accents='ascii',\n",
    "#                             lowercase=True,\n",
    "#                             ngram_range=(2, 3)\n",
    "#                             )),\n",
    "#     ('tfidf', TfidfTransformer(use_idf=True)), \n",
    "#     ])\n",
    "#     return model\n",
    "# model = buildModelITFGrams()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('bow', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(2, 3), preprocessor=None, stop_words=None,\n",
       "        strip_accents='ascii', token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "        tokenizer=None, vocabulary=None)), ('tfidf', TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True))])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import warnings\n",
    "# with warnings.catch_warnings():\n",
    "#     warnings.filterwarnings(\"ignore\",category=FutureWarning)\n",
    "# # do 10-fold cross validation for each of the 6 possible combinations of the above params\n",
    "# grid = GridSearchCV(estimator=pipeline, cv=10, param_grid=parameters, verbose=1)\n",
    "# grid.fit(X_train, y_train)\n",
    "# \n",
    "# model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20000,)"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##OK da qua (use data and dataTest)\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "\n",
    "#  Split into train and test sets\n",
    "rs = ShuffleSplit(n_splits=2, random_state=42, test_size=0.2)\n",
    "train_indices, test_indices = next(rs.split(data))\n",
    "    \n",
    "X_train = data['text'][train_indices]\n",
    "y_train = raw_data_train['label'][train_indices]\n",
    "X_validation = data['text'][test_indices]\n",
    "y_validation = raw_data_train['label'][test_indices]\n",
    "\n",
    "X_train = X_train[:20000]\n",
    "y_train = y_train[:20000]\n",
    "X_validation = X_validation[:20000]\n",
    "y_validation = y_validation[:20000]\n",
    "\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "146332    1\n",
       "27459     0\n",
       "9639      0\n",
       "120020    1\n",
       "124466    1\n",
       "154194    1\n",
       "69643     0\n",
       "180829    1\n",
       "134319    1\n",
       "33287     0\n",
       "87790     0\n",
       "37240     0\n",
       "54401     0\n",
       "163814    1\n",
       "121661    1\n",
       "93968     1\n",
       "62479     0\n",
       "171729    1\n",
       "10519     0\n",
       "93902     1\n",
       "53310     0\n",
       "165380    1\n",
       "36521     0\n",
       "119814    1\n",
       "159591    1\n",
       "139279    1\n",
       "167046    1\n",
       "19835     0\n",
       "77873     0\n",
       "123824    1\n",
       "         ..\n",
       "127271    1\n",
       "72473     0\n",
       "137825    1\n",
       "20630     0\n",
       "88513     0\n",
       "97595     1\n",
       "12635     0\n",
       "118224    1\n",
       "143943    1\n",
       "61819     0\n",
       "170000    1\n",
       "42578     0\n",
       "94431     1\n",
       "118424    1\n",
       "20988     0\n",
       "131548    1\n",
       "142443    1\n",
       "42793     0\n",
       "83792     0\n",
       "98832     1\n",
       "126768    1\n",
       "161106    1\n",
       "117149    1\n",
       "142772    1\n",
       "163350    1\n",
       "43583     0\n",
       "90501     0\n",
       "167261    1\n",
       "77726     0\n",
       "175253    1\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20000, 287808)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/feature_extraction/text.py:1059: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    },
    {
     "ename": "NotFittedError",
     "evalue": "TfidfVectorizer - Vocabulary wasn't fitted.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNotFittedError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-94-9ded5067996b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0mbag_of_words\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcount_vectorizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbag_of_words\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m \u001b[0mX_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcount_vectorizerBigram\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_validation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/feature_extraction/text.py\u001b[0m in \u001b[0;36mtransform\u001b[0;34m(self, raw_documents, copy)\u001b[0m\n\u001b[1;32m   1378\u001b[0m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'_tfidf'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'The tfidf vector is not fitted'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1379\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1380\u001b[0;31m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTfidfVectorizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mraw_documents\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1381\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tfidf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/feature_extraction/text.py\u001b[0m in \u001b[0;36mtransform\u001b[0;34m(self, raw_documents)\u001b[0m\n\u001b[1;32m    888\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_vocabulary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    889\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 890\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_vocabulary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    891\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    892\u001b[0m         \u001b[0;31m# use the same matrix-building strategy as fit_transform\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/feature_extraction/text.py\u001b[0m in \u001b[0;36m_check_vocabulary\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    276\u001b[0m         \u001b[0;34m\"\"\"Check if vocabulary is empty or missing (not fit-ed)\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    277\u001b[0m         \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"%(name)s - Vocabulary wasn't fitted.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 278\u001b[0;31m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'vocabulary_'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    279\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocabulary_\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_is_fitted\u001b[0;34m(estimator, attributes, msg, all_or_any)\u001b[0m\n\u001b[1;32m    688\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mall_or_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mattr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mattributes\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    689\u001b[0m         \u001b[0;31m# FIXME NotFittedError_ --> NotFittedError in 0.19\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 690\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0m_NotFittedError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'name'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    691\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    692\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNotFittedError\u001b[0m: TfidfVectorizer - Vocabulary wasn't fitted."
     ]
    }
   ],
   "source": [
    "# data #preprocessing applied\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "count_vectorizer = TfidfVectorizer(ngram_range=(2,3), \n",
    "                        strip_accents='unicode',\n",
    "                        lowercase= True, analyzer='word', \n",
    "                        use_idf=True, smooth_idf=True, sublinear_tf=False, \n",
    "#                         stop_words = 'english'\n",
    "                                  )\n",
    "\n",
    "count_vectorizerBigram = TfidfVectorizer(ngram_range=(2,2), \n",
    "                        strip_accents='unicode',\n",
    "                        lowercase= True, analyzer='word', \n",
    "                        use_idf=True, smooth_idf=True, sublinear_tf=False, \n",
    "#                         stop_words = 'english'\n",
    "                                  )\n",
    "# bag_of_words = count_vectorizer.fit_transform(X_train)\n",
    "# print(bag_of_words.shape)\n",
    "# X_test = count_vectorizer.transform(X_validation)\n",
    "# print(X_test.shape)\n",
    "\n",
    "bag_of_words = count_vectorizer.fit_transform(X_train)\n",
    "print(bag_of_words.shape)\n",
    "X_test = count_vectorizerBigram.transform(X_validation)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20000, 287808)\n",
      "(20000, 287808)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/feature_extraction/text.py:1059: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    }
   ],
   "source": [
    "transformer = TfidfTransformer(use_idf=True, smooth_idf=True, sublinear_tf=False)\n",
    "transformer_bag_of_words = transformer.fit_transform(bag_of_words)\n",
    "X_test_transformer = transformer.transform(X_test)\n",
    "print (transformer_bag_of_words.shape)\n",
    "print (X_test_transformer.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3/dist-packages/ipykernel_launcher.py:11: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(512, input_dim=287808, activation=\"relu\", kernel_initializer=\"normal\")`\n",
      "  # This is added back by InteractiveShellApp.init_path()\n",
      "/usr/lib/python3/dist-packages/ipykernel_launcher.py:13: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(512, activation=\"relu\", kernel_initializer=\"normal\")`\n",
      "  del sys.path[0]\n",
      "/usr/lib/python3/dist-packages/ipykernel_launcher.py:15: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(512, activation=\"relu\", kernel_initializer=\"normal\")`\n",
      "  from ipykernel import kernelapp as app\n",
      "/usr/lib/python3/dist-packages/ipykernel_launcher.py:17: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(512, activation=\"relu\", kernel_initializer=\"normal\")`\n",
      "/usr/lib/python3/dist-packages/ipykernel_launcher.py:18: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(256, activation=\"relu\", kernel_initializer=\"normal\")`\n",
      "/usr/lib/python3/dist-packages/ipykernel_launcher.py:19: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(64, activation=\"relu\", kernel_initializer=\"normal\")`\n",
      "/usr/lib/python3/dist-packages/ipykernel_launcher.py:20: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1, activation=\"sigmoid\", kernel_initializer=\"normal\")`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20000 samples, validate on 20000 samples\n",
      "Epoch 1/20\n",
      "20000/20000 [==============================] - 441s - loss: 0.6416 - acc: 0.5997 - val_loss: 0.5801 - val_acc: 0.7056\n",
      "Epoch 2/20\n",
      "20000/20000 [==============================] - 409s - loss: 0.0564 - acc: 0.9814 - val_loss: 1.2406 - val_acc: 0.7096\n",
      "Epoch 3/20\n",
      "  576/20000 [..............................] - ETA: 317s - loss: 0.0022 - acc: 1.0000"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-96-5d97a75ab7d2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;31m# estimator.fit(transformer_bag_of_words, raw_data_train['label'], validation_split=0.1)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtransformer_bag_of_words\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtodense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtodense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_validation\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/wrappers/scikit_learn.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, **kwargs)\u001b[0m\n\u001b[1;32m    204\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Invalid shape for y: '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_classes_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mKerasClassifier\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/wrappers/scikit_learn.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, **kwargs)\u001b[0m\n\u001b[1;32m    147\u001b[0m         \u001b[0mfit_args\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    148\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 149\u001b[0;31m         \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    150\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mhistory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/models.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m    851\u001b[0m                               \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    852\u001b[0m                               \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 853\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m    854\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    855\u001b[0m     def evaluate(self, x, y, batch_size=32, verbose=1,\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m   1484\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1485\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1486\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1487\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1488\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m   1139\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1140\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1141\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1142\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1143\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2101\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2102\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[0;32m-> 2103\u001b[0;31m                               feed_dict=feed_dict)\n\u001b[0m\u001b[1;32m   2104\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    875\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    876\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 877\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    878\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    879\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1098\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1099\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1100\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1101\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1102\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1270\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1271\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1272\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1273\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1274\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1276\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1277\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1278\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1279\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1280\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1261\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1262\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1263\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1264\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1265\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1349\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1350\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1351\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from random import shuffle\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, LSTM, Embedding, Input, RepeatVector\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing import text, sequence\n",
    "from scipy.sparse import hstack\n",
    "\n",
    "def baseline_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(512, input_dim=transformer_bag_of_words.shape[1], init='normal', activation='relu'))\n",
    "    model.add(Dropout(0.15))\n",
    "    model.add(Dense(512, init='normal', activation='relu'))\n",
    "    model.add(Dropout(0.15))\n",
    "    model.add(Dense(512, init='normal', activation='relu'))\n",
    "    model.add(Dropout(0.15))\n",
    "    model.add(Dense(512, init='normal', activation='relu'))\n",
    "    model.add(Dense(256, init='normal', activation='relu'))\n",
    "    model.add(Dense(64, init='normal', activation='relu'))\n",
    "    model.add(Dense(1, init='normal', activation=\"sigmoid\"))\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "\n",
    "estimator = KerasClassifier(build_fn=baseline_model, epochs=20, batch_size=64)\n",
    "# estimator.fit(transformer_bag_of_words, raw_data_train['label'], validation_split=0.1)\n",
    "\n",
    "estimator.fit(transformer_bag_of_words.todense(), y_train, validation_data=[X_test.todense(), y_validation])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import shuffle\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, LSTM, Embedding, Input, RepeatVector\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing import text, sequence\n",
    "from scipy.sparse import hstack\n",
    "\n",
    "def baseline_model2():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(512, input_dim=transformer_bag_of_words.shape[1], init='normal', activation='relu'))\n",
    "    model.add(Dropout(0.15))\n",
    "    model.add(Dense(512, init='normal', activation='relu'))\n",
    "    model.add(Dropout(0.15))\n",
    "    model.add(Dense(512, init='normal', activation='relu'))\n",
    "    model.add(Dropout(0.15))\n",
    "    model.add(Dense(512, init='normal', activation='relu'))\n",
    "    model.add(Dense(256, init='normal', activation='relu'))\n",
    "    model.add(Dense(64, init='normal', activation='relu'))\n",
    "    model.add(Dense(1, init='normal', activation=\"sigmoid\"))\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "\n",
    "estimator2 = KerasClassifier(build_fn=baseline_model2, epochs=12, batch_size=64)\n",
    "# estimator.fit(transformer_bag_of_words, raw_data_train['label'], validation_split=0.1)\n",
    "\n",
    "estimator2.fit(transformer_bag_of_words.todense(), y_train, validation_data=[X_test.todense(), y_validation])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_test_transformer[0,:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8000, 138467)\n",
      "(2000, 138467)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/feature_extraction/text.py:1059: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    }
   ],
   "source": [
    "dataTestFinal = pd.DataFrame(data={'text': raw_data_test['text'].apply(applyPreProcessing)})\n",
    "X_testFinal = count_vectorizer.transform(dataTestFinal['text'])\n",
    "X_test_transformerFinal = transformer.transform(X_testFinal)\n",
    "print (transformer_bag_of_words.shape)\n",
    "print (X_test_transformer.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_transformerFinal.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 19s    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(10000, 2)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(estimator.predict_proba(X_test_transformerFinal.toarray())).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000.0"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_6_labels[:, 0].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 18s    \n",
      "[[1. 0.]\n",
      " [1. 0.]\n",
      " [1. 0.]\n",
      " ...\n",
      " [1. 0.]\n",
      " [1. 0.]\n",
      " [1. 0.]]\n"
     ]
    }
   ],
   "source": [
    "test_6_labels = estimator.predict_proba(X_test_transformerFinal.toarray())\n",
    "\n",
    "print(test_6_labels)\n",
    "\n",
    "test_6_labels[test_6_labels >= 0.5] = 1\n",
    "test_6_labels[test_6_labels < 0.5] = -1\n",
    "\n",
    "# test_6_labels_reversed = test_6_labels.copy()\n",
    "\n",
    "# test_6_labels_reversed[test_6_labels_reversed == -1] = 0\n",
    "# test_6_labels_reversed[test_6_labels_reversed == 1] = -1\n",
    "# test_6_labels_reversed[test_6_labels_reversed == 0] = -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "only size-1 arrays can be converted to Python scalars",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-52-94cd5d2ea693>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     14\u001b[0m             \u001b[0mwriter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwriterow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'Id'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Prediction'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mcreate_csv_submission\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mraw_data_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'id'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_6_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'6Decembertfidf.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-52-94cd5d2ea693>\u001b[0m in \u001b[0;36mcreate_csv_submission\u001b[0;34m(ids, y_pred, name)\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mr1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mr2\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;31m#             print(y_pred)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m             \u001b[0mwriter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwriterow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'Id'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Prediction'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0mcreate_csv_submission\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mraw_data_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'id'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_6_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'6Decembertfidf.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: only size-1 arrays can be converted to Python scalars"
     ]
    }
   ],
   "source": [
    "def create_csv_submission(ids, y_pred, name):\n",
    "    \"\"\"\n",
    "    Creates an output file in csv format for submission to kaggle\n",
    "    Arguments: ids (event ids associated with each prediction)\n",
    "               y_pred (predicted class labels)\n",
    "               name (string name of .csv output file to be created)\n",
    "    \"\"\"\n",
    "    with open(name, 'w') as csvfile:\n",
    "        fieldnames = ['Id', 'Prediction']\n",
    "        writer = csv.DictWriter(csvfile, delimiter=\",\", fieldnames=fieldnames)\n",
    "        writer.writeheader()\n",
    "        for r1, r2 in zip(ids, y_pred):\n",
    "#             print(y_pred)\n",
    "            writer.writerow({'Id':int(r1),'Prediction':int(r2)})\n",
    "    \n",
    "create_csv_submission(raw_data_test['id'], test_6_labels, '6Decembertfidf.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1., -1.],\n",
       "       [ 1., -1.],\n",
       "       [ 1., -1.],\n",
       "       ...,\n",
       "       [ 1., -1.],\n",
       "       [ 1., -1.],\n",
       "       [ 1., -1.]], dtype=float32)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_6_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}