Riccardo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "#import model as model_config\n",
    "#from data_utils import load as load_data, extract_features\n",
    "import numpy as np\n",
    "import csv\n",
    "\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import re\n",
    "import numpy as np\n",
    "import csv\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import string\n",
    "from string import punctuation\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "import re\n",
    "import spacy\n",
    "\n",
    "\n",
    "import spacy\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "\n",
    "twitter_data_neg_small = './twitter-datasets/train_neg.txt'\n",
    "twitter_data_pos_small = './twitter-datasets/train_pos.txt'\n",
    "twitter_data_small     = './twitter-datasets/train_small.txt'\n",
    "\n",
    "twitter_data_neg_full  = './twitter-datasets/train_neg_full.txt'\n",
    "twitter_data_pos_full  = './twitter-datasets/train_pos_full.txt'\n",
    "twitter_data_full      = './twitter-datasets/train_full.txt'\n",
    "nlp = spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#nlp = spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use dictionary from http://luululu.com/tweet/typo-corpus-r1.txt\n",
    "# http://people.eng.unimelb.edu.au/tbaldwin/etc/emnlp2012-lexnorm.tgz\n",
    "# to handle abbreviations, mistakes...etc. (IN )\n",
    "\n",
    "# LULU-CORPUS\n",
    "# (1) INSERT (IN): a character is added to the original word.\n",
    "# (2) REMOVE (RM): a character is removed from the original word.\n",
    "# (3) REPLACE1 (R1): the order of character is different from the original word (the number of differences is one).\n",
    "# (4) REPLACE2 (R2): a character is different from the original word\n",
    "\n",
    "final_corpus = {}\n",
    "\n",
    "def corpusReplace(corpus):\n",
    "    for word in corpus:\n",
    "        word = word.decode('utf8')\n",
    "        word = word.split()\n",
    "        final_corpus[word[0]] = word[1]    \n",
    "\n",
    "corpus_lulu = open('corpus/lulu-corpus.txt', 'rb')\n",
    "corpusReplace(corpus_lulu)\n",
    "corpus_lulu.close()\n",
    "\n",
    "corpus_emnlp = open('corpus/emnlp-corpus.txt', 'rb')\n",
    "corpusReplace(corpus_emnlp)\n",
    "corpus_emnlp.close()\n",
    "\n",
    "def applyCorpus(tweet):\n",
    "    new_tweet = ''\n",
    "    for w in tweet.split(' '):\n",
    "        if w in final_corpus.keys():\n",
    "            #Replace with correct value\n",
    "            new_word = final_corpus[w]\n",
    "            new_tweet = new_tweet + ' ' + new_word\n",
    "        else:\n",
    "            new_tweet = new_tweet + ' ' + w\n",
    "    return new_tweet\n",
    "\n",
    "#raw_data_train['text'] = raw_data_train.text.apply(applyCorpus)   \n",
    "         \n",
    "def cleanTweet(tweet):\n",
    "    tweet = re.sub('<url>','',tweet)\n",
    "    tweet = re.sub('<user>','',tweet)\n",
    "    tweet = re.sub(r'\\$\\w*', '', tweet)\n",
    "    tweet = tweet.lower()\n",
    "    tweet = re.sub(r'https?:\\/\\/.*\\/\\w*', '', tweet)\n",
    "    tweet = re.sub(r'#\\w*', '', tweet) #hashtag\n",
    "    tweet = re.sub(r'[' + punctuation.replace('@', '') + ']+', ' ', tweet) # puntuaction\n",
    "    tweet = re.sub(r'\\s\\s+', ' ', tweet)\n",
    "    tweet = tweet.lstrip(' ') \n",
    "    tweet = ''.join(c for c in tweet if c <= '\\uFFFF') \n",
    "    return tweet\n",
    "\n",
    "#raw_data_train['text'] = raw_data_train.text.apply(cleanTweet)\n",
    "\n",
    "def extract_features(docs, max_length):\n",
    "    docs = list(docs)\n",
    "    X = np.zeros((len(docs), max_length), dtype='int32')\n",
    "    for i, doc in enumerate(docs):\n",
    "        j = 0\n",
    "        for token in doc:\n",
    "            if token.has_vector and not token.is_punct and not token.is_space:\n",
    "                X[i, j] = token.rank + 1\n",
    "                j += 1\n",
    "                if j >= max_length:\n",
    "                    break\n",
    "    return X\n",
    "\n",
    "def load_twitter_data_small(from_cache=False):\n",
    "    cached_data_path = twitter_data_small + '.cached.pkl'\n",
    "\n",
    "    if from_cache:\n",
    "        print('Loading data from cache...')\n",
    "        with open(cached_data_path, 'rb') as f:\n",
    "            return pickle.load(f)\n",
    "\n",
    "    max_length = 100\n",
    "\n",
    "    print('Loading and preparing data...')\n",
    "    raw_data_neg = pd.read_csv(twitter_data_neg_small, header=None, sep=\"\\n\", encoding='latin1', names=['text'],\n",
    "                               error_bad_lines=False, warn_bad_lines=False, quoting=csv.QUOTE_NONE).drop_duplicates()\n",
    "    raw_data_neg['text'] = raw_data_neg['text'].apply(cleanTweet)\n",
    "    raw_data_neg['text'] = raw_data_neg['text'].apply(applyCorpus)\n",
    "    raw_data_neg['label'] = 0\n",
    "    raw_data_neg = raw_data_neg\n",
    "\n",
    "    raw_data_pos = pd.read_csv(twitter_data_pos_small, header=None, sep=\"\\n\", encoding='latin1', names=['text'],\n",
    "                               error_bad_lines=False, warn_bad_lines=False, quoting=csv.QUOTE_NONE).drop_duplicates()\n",
    "    raw_data_pos['text'] = raw_data_pos['text'].apply(cleanTweet)\n",
    "    raw_data_pos['text'] = raw_data_pos['text'].apply(applyCorpus)\n",
    "    raw_data_pos['label'] = 1\n",
    "    raw_data_pos = raw_data_pos\n",
    "    \n",
    "    raw_data = pd.concat([raw_data_neg, raw_data_pos], ignore_index=True)\n",
    "#     raw_data = raw_data[:10000]\n",
    "    \n",
    "    #print(raw_data)\n",
    "\n",
    "    # Parse tweet texts\n",
    "    docs = list(nlp.pipe(raw_data['text'], batch_size=1000, n_threads=8))\n",
    "    print(max([len(x) for x in docs])) \n",
    "    \n",
    "    y = raw_data['label'].values\n",
    "    \n",
    "    # Pull the raw_data into vectors\n",
    "    X = extract_features(docs, max_length=max_length)\n",
    "    \n",
    "    # Split into train and test sets\n",
    "    rs = ShuffleSplit(n_splits=2, random_state=42, test_size=0.2)\n",
    "    train_indices, test_indices = next(rs.split(X))\n",
    "    \n",
    "    X_train = X[train_indices]\n",
    "    y_train = y[train_indices]\n",
    "    X_test = X[test_indices]\n",
    "    y_test = y[test_indices]\n",
    "    \n",
    "    docs = np.array(docs, dtype=object)\n",
    "    docs_train = docs[train_indices]\n",
    "    docs_test = docs[test_indices]\n",
    "    \n",
    "    numeric_data = X_train, y_train, X_test, y_test\n",
    "    raw_data = docs_train, docs_test\n",
    "\n",
    "    with open(cached_data_path, 'wb') as f:\n",
    "        pickle.dump((numeric_data, raw_data), f)\n",
    "    \n",
    "    return numeric_data, raw_data\n",
    "\n",
    "def load_twitter_data_full(from_cache=False):\n",
    "    cached_data_path = twitter_data_full + '.cached.pkl'\n",
    "\n",
    "    if from_cache:\n",
    "        print('Loading data from cache...')\n",
    "        with open(cached_data_path, 'rb') as f:\n",
    "            return pickle.load(f)\n",
    "\n",
    "    max_length = 100\n",
    "\n",
    "    print('Loading and preparing data...')\n",
    "    raw_data_neg = pd.read_csv(twitter_data_neg_full, header=None, sep=\"\\n\", encoding='latin1', names=['text'],\n",
    "                               error_bad_lines=False, warn_bad_lines=False, quoting=csv.QUOTE_NONE).drop_duplicates()\n",
    "    raw_data_neg['text'] = raw_data_neg['text'].apply(cleanTweet)\n",
    "    raw_data_neg['text'] = raw_data_neg['text'].apply(applyCorpus)\n",
    "    raw_data_neg['label'] = 0\n",
    "    raw_data_neg = raw_data_neg\n",
    "\n",
    "    raw_data_pos = pd.read_csv(twitter_data_pos_full, header=None, sep=\"\\n\", encoding='latin1', names=['text'],\n",
    "                               error_bad_lines=False, warn_bad_lines=False, quoting=csv.QUOTE_NONE).drop_duplicates()\n",
    "    raw_data_pos['text'] = raw_data_pos['text'].apply(cleanTweet)\n",
    "    raw_data_pos['text'] = raw_data_pos['text'].apply(applyCorpus)\n",
    "    raw_data_pos['label'] = 1\n",
    "    raw_data_pos = raw_data_pos\n",
    "    \n",
    "    raw_data = pd.concat([raw_data_neg, raw_data_pos], ignore_index=True)\n",
    "#     raw_data = raw_data[:10000]\n",
    "    \n",
    "    #print(raw_data)\n",
    "\n",
    "    # Parse tweet texts\n",
    "    docs = list(nlp.pipe(raw_data['text'], batch_size=5000, n_threads=8))\n",
    "    print(max([len(x) for x in docs])) \n",
    "    \n",
    "    y = raw_data['label'].values\n",
    "    \n",
    "    # Pull the raw_data into vectors\n",
    "    X = extract_features(docs, max_length=max_length)\n",
    "    \n",
    "    # Split into train and test sets\n",
    "    rs = ShuffleSplit(n_splits=2, random_state=42, test_size=0.2)\n",
    "    train_indices, test_indices = next(rs.split(X))\n",
    "    \n",
    "    X_train = X[train_indices]\n",
    "    y_train = y[train_indices]\n",
    "    X_test = X[test_indices]\n",
    "    y_test = y[test_indices]\n",
    "    \n",
    "    docs = np.array(docs, dtype=object)\n",
    "    docs_train = docs[train_indices]\n",
    "    docs_test = docs[test_indices]\n",
    "    \n",
    "    numeric_data = X_train, y_train, X_test, y_test\n",
    "    raw_data = docs_train, docs_test\n",
    "\n",
    "    with open(cached_data_path, 'wb') as f:\n",
    "        pickle.dump((numeric_data, raw_data), f)\n",
    "    \n",
    "    return numeric_data, raw_data\n",
    "\n",
    "def load(data_name, *args, **kwargs):\n",
    "    load_fn_map = {\n",
    "        'twitter_data_small': load_twitter_data_small,\n",
    "        'twitter_data_full': load_twitter_data_full,\n",
    "    }\n",
    "    return load_fn_map[data_name](*args, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading and preparing data...\n",
      "57\n"
     ]
    }
   ],
   "source": [
    "# Load Twitter data small\n",
    "(X_train, y_train, X_test, y_test), (docs_train, docs_test) = load('twitter_data_small', from_cache=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading and preparing data...\n",
      "123\n"
     ]
    }
   ],
   "source": [
    "# Load Twitter data full\n",
    "(X_train, y_train, X_test, y_test), (docs_train, docs_test) = load('twitter_data_full', from_cache=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: (1816385, 100)\n",
      "y_train: (1816385,)\n",
      "X_test: (454097, 100)\n",
      "y_test: (454097,)\n"
     ]
    }
   ],
   "source": [
    "print('X_train:', X_train.shape)\n",
    "print('y_train:', y_train.shape)\n",
    "print('X_test:', X_test.shape)\n",
    "print('y_test:', y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-a543f67ea0a6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./baselines/train_docs_saved'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m         \u001b[0mtrain_docs_loaded\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32mdoc.pyx\u001b[0m in \u001b[0;36mspacy.tokens.doc.unpickle_doc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/dill/dill.py\u001b[0m in \u001b[0;36mloads\u001b[0;34m(str)\u001b[0m\n\u001b[1;32m    272\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    273\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 274\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    275\u001b[0m     \u001b[0;34m\"\"\"unpickle an object from a string\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    276\u001b[0m     \u001b[0mfile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mStringIO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# with open('./baselines/train_docs_saved', 'rb') as f:\n",
    "#         train_docs_loaded = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('./baselines/test_docs_saved', 'rb') as f:\n",
    "#         test_docs_loaded = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_docs_loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "import keras\n",
    "import numpy as np\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation\n",
    "from keras.layers import Flatten, Conv1D, MaxPooling1D, GlobalAveragePooling1D, LSTM, merge\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.constraints import maxnorm\n",
    "from keras.regularizers import l2\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing embeddings...\n"
     ]
    }
   ],
   "source": [
    "def get_embeddings(vocab):\n",
    "    max_rank = max(lex.rank+1 for lex in vocab if lex.has_vector)\n",
    "    vectors = np.ndarray((max_rank+1, vocab.vectors_length), dtype='float32')\n",
    "    for lex in vocab:\n",
    "        if lex.has_vector:\n",
    "            vectors[lex.rank + 1] = lex.vector\n",
    "    return vectors\n",
    "\n",
    "\n",
    "vocab_nlp = spacy.load('en_core_web_lg', parser=False, tagger=False, entity=False)\n",
    "print('Preparing embeddings...')\n",
    "embeddings = get_embeddings(vocab_nlp.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model_twitter(max_length=100,\n",
    "                nb_filters=64,\n",
    "                kernel_size=3,\n",
    "                pool_size=2,\n",
    "                regularization=0.01,\n",
    "                weight_constraint=2.,\n",
    "                dropout_prob=0.4,\n",
    "                clear_session=True):\n",
    "    if clear_session:\n",
    "        K.clear_session()\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(\n",
    "        embeddings.shape[0],\n",
    "        embeddings.shape[1],\n",
    "        input_length=max_length,\n",
    "        trainable=False,\n",
    "        weights=[embeddings]))\n",
    "\n",
    "    model.add(Conv1D(nb_filters, kernel_size, activation='relu'))\n",
    "    model.add(Conv1D(nb_filters, kernel_size, activation='relu'))\n",
    "    model.add(MaxPooling1D(pool_size))\n",
    "\n",
    "    model.add(Dropout(dropout_prob))\n",
    "\n",
    "    model.add(Conv1D(nb_filters * 2, kernel_size, activation='relu'))\n",
    "    model.add(Conv1D(nb_filters * 2, kernel_size, activation='relu'))\n",
    "    model.add(MaxPooling1D(pool_size))\n",
    "\n",
    "    model.add(Dropout(dropout_prob))\n",
    "\n",
    "    model.add(GlobalAveragePooling1D())\n",
    "    model.add(Dense(1,\n",
    "        kernel_regularizer=l2(regularization),\n",
    "        kernel_constraint=maxnorm(weight_constraint),\n",
    "        activation='sigmoid'))\n",
    "\n",
    "    model.compile(\n",
    "        loss='binary_crossentropy',\n",
    "        optimizer='rmsprop',\n",
    "        metrics=['accuracy'])\n",
    "\n",
    "    return model\n",
    "\n",
    "def build_model_twitter2(max_length=100,\n",
    "                nb_filters=64,\n",
    "                kernel_size=3,\n",
    "                pool_size=2,\n",
    "                regularization=0.01,\n",
    "                weight_constraint=2.,\n",
    "                dropout_prob=0.4,\n",
    "                clear_session=True):\n",
    "    if clear_session:\n",
    "        K.clear_session()\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(\n",
    "        embeddings.shape[0],\n",
    "        embeddings.shape[1],\n",
    "        input_length=max_length,\n",
    "        trainable=False,\n",
    "        weights=[embeddings]))\n",
    "\n",
    "    model.add(Conv1D(nb_filters, kernel_size, activation='relu'))\n",
    "    model.add(MaxPooling1D(pool_size))\n",
    "    model.add(Dropout(dropout_prob))\n",
    "\n",
    "    model.add(Conv1D(nb_filters * 2, kernel_size, activation='relu'))\n",
    "    model.add(MaxPooling1D(pool_size))\n",
    "    model.add(Dropout(dropout_prob))\n",
    "    \n",
    "    model.add(Conv1D(nb_filters * 4, kernel_size, activation='relu'))\n",
    "    model.add(MaxPooling1D(pool_size))\n",
    "    model.add(Dropout(dropout_prob))\n",
    "\n",
    "    model.add(GlobalAveragePooling1D())\n",
    "    model.add(Dense(1,\n",
    "        kernel_regularizer=l2(regularization),\n",
    "        kernel_constraint=maxnorm(weight_constraint),\n",
    "        activation='sigmoid'))\n",
    "\n",
    "    model.compile(\n",
    "        loss='binary_crossentropy',\n",
    "        optimizer='rmsprop',\n",
    "        metrics=['accuracy'])\n",
    "\n",
    "    return model\n",
    "\n",
    "def build_model_twitter3(max_length=100,\n",
    "                nb_filters=64,\n",
    "                kernel_size=3,\n",
    "                pool_size=2,\n",
    "                regularization=0.01,\n",
    "                weight_constraint=2.,\n",
    "                dropout_prob=0.4,\n",
    "                clear_session=True):\n",
    "    if clear_session:\n",
    "        K.clear_session()\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(\n",
    "        embeddings.shape[0],\n",
    "        embeddings.shape[1],\n",
    "        input_length=max_length,\n",
    "        trainable=False,\n",
    "        weights=[embeddings]))\n",
    "\n",
    "    model.add(Conv1D(nb_filters, kernel_size))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(MaxPooling1D(pool_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(dropout_prob))\n",
    "\n",
    "    model.add(Conv1D(nb_filters * 2, kernel_size))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(MaxPooling1D(pool_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(dropout_prob))\n",
    "\n",
    "#     model.add(Conv1D(nb_filters * 4, kernel_size))\n",
    "#     model.add(BatchNormalization())\n",
    "#     model.add(MaxPooling1D(pool_size))\n",
    "#     model.add(Activation('relu'))\n",
    "#     model.add(Dropout(dropout_prob))\n",
    "\n",
    "    model.add(GlobalAveragePooling1D())\n",
    "    model.add(Dense(1,\n",
    "        kernel_regularizer=l2(regularization),\n",
    "        kernel_constraint=maxnorm(weight_constraint),\n",
    "        activation='sigmoid'))\n",
    "\n",
    "    model.compile(\n",
    "        loss='binary_crossentropy',\n",
    "        optimizer='rmsprop',\n",
    "        metrics=['accuracy'])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: (145056, 100)\n",
      "y_train: (145056,)\n",
      "X_test: (36265, 100)\n",
      "y_test: (36265,)\n",
      "Train on 145056 samples, validate on 36265 samples\n",
      "Epoch 1/12\n",
      "145056/145056 [==============================] - 118s - loss: 0.5127 - acc: 0.7419 - val_loss: 0.5304 - val_acc: 0.7250\n",
      "Epoch 2/12\n",
      "145056/145056 [==============================] - 121s - loss: 0.4617 - acc: 0.7814 - val_loss: 0.4540 - val_acc: 0.7796\n",
      "Epoch 3/12\n",
      "145056/145056 [==============================] - 119s - loss: 0.4385 - acc: 0.7960 - val_loss: 0.4659 - val_acc: 0.7833\n",
      "Epoch 4/12\n",
      "145056/145056 [==============================] - 119s - loss: 0.4230 - acc: 0.8045 - val_loss: 0.4335 - val_acc: 0.8012\n",
      "Epoch 5/12\n",
      "145056/145056 [==============================] - 120s - loss: 0.4098 - acc: 0.8134 - val_loss: 0.4254 - val_acc: 0.8040\n",
      "Epoch 6/12\n",
      "145056/145056 [==============================] - 118s - loss: 0.3964 - acc: 0.8215 - val_loss: 0.4235 - val_acc: 0.8048\n",
      "Epoch 7/12\n",
      "145056/145056 [==============================] - 119s - loss: 0.3848 - acc: 0.8290 - val_loss: 0.4557 - val_acc: 0.7903\n",
      "Epoch 8/12\n",
      "145056/145056 [==============================] - 120s - loss: 0.3740 - acc: 0.8343 - val_loss: 0.4362 - val_acc: 0.8036\n",
      "Epoch 9/12\n",
      "145056/145056 [==============================] - 120s - loss: 0.3640 - acc: 0.8394 - val_loss: 0.4403 - val_acc: 0.8042\n",
      "Epoch 10/12\n",
      "145056/145056 [==============================] - 121s - loss: 0.3548 - acc: 0.8443 - val_loss: 0.4560 - val_acc: 0.7938\n",
      "Epoch 11/12\n",
      "145056/145056 [==============================] - 121s - loss: 0.3450 - acc: 0.8503 - val_loss: 0.4912 - val_acc: 0.7939\n",
      "Epoch 12/12\n",
      "145056/145056 [==============================] - 118s - loss: 0.3359 - acc: 0.8545 - val_loss: 0.5193 - val_acc: 0.7892\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f6cd74dacc0>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Model 1\n",
    "np.random.seed(42)\n",
    "\n",
    "epochs = 12\n",
    "batch_size = 128\n",
    "\n",
    "\n",
    "# Take a look at the shapes\n",
    "print('X_train:', X_train.shape)\n",
    "print('y_train:', y_train.shape)\n",
    "print('X_test:', X_test.shape)\n",
    "print('y_test:', y_test.shape)\n",
    "\n",
    "tb_callback = keras.callbacks.TensorBoard(\n",
    "        histogram_freq=0, write_graph=True)\n",
    "\n",
    "model = build_model_twitter()\n",
    "model.fit(X_train, y_train,\n",
    "          validation_data=(X_test, y_test),\n",
    "          batch_size=batch_size, epochs=epochs,\n",
    "          callbacks=[tb_callback])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: (145056, 100)\n",
      "y_train: (145056,)\n",
      "X_test: (36265, 100)\n",
      "y_test: (36265,)\n",
      "Train on 145056 samples, validate on 36265 samples\n",
      "Epoch 1/12\n",
      "145056/145056 [==============================] - 113s - loss: 0.5068 - acc: 0.7458 - val_loss: 0.4840 - val_acc: 0.7580\n",
      "Epoch 2/12\n",
      "145056/145056 [==============================] - 116s - loss: 0.4616 - acc: 0.7805 - val_loss: 0.4490 - val_acc: 0.7842\n",
      "Epoch 3/12\n",
      "145056/145056 [==============================] - 115s - loss: 0.4463 - acc: 0.7903 - val_loss: 0.4515 - val_acc: 0.7924\n",
      "Epoch 4/12\n",
      "145056/145056 [==============================] - 113s - loss: 0.4351 - acc: 0.7957 - val_loss: 0.4475 - val_acc: 0.7948\n",
      "Epoch 5/12\n",
      "145056/145056 [==============================] - 116s - loss: 0.4288 - acc: 0.8002 - val_loss: 0.4308 - val_acc: 0.8012\n",
      "Epoch 6/12\n",
      "145056/145056 [==============================] - 113s - loss: 0.4214 - acc: 0.8056 - val_loss: 0.4253 - val_acc: 0.8049\n",
      "Epoch 7/12\n",
      "145056/145056 [==============================] - 116s - loss: 0.4160 - acc: 0.8087 - val_loss: 0.4512 - val_acc: 0.7768\n",
      "Epoch 8/12\n",
      "145056/145056 [==============================] - 109s - loss: 0.4112 - acc: 0.8106 - val_loss: 0.4317 - val_acc: 0.8017\n",
      "Epoch 9/12\n",
      "145056/145056 [==============================] - 109s - loss: 0.4053 - acc: 0.8145 - val_loss: 0.4218 - val_acc: 0.8052\n",
      "Epoch 10/12\n",
      "145056/145056 [==============================] - 110s - loss: 0.4005 - acc: 0.8177 - val_loss: 0.4321 - val_acc: 0.7975\n",
      "Epoch 11/12\n",
      "145056/145056 [==============================] - 111s - loss: 0.3973 - acc: 0.8193 - val_loss: 0.4403 - val_acc: 0.7999\n",
      "Epoch 12/12\n",
      "145056/145056 [==============================] - 110s - loss: 0.3926 - acc: 0.8213 - val_loss: 0.4463 - val_acc: 0.8002\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f6cc919bd68>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Model 2\n",
    "np.random.seed(42)\n",
    "\n",
    "\n",
    "epochs = 12\n",
    "batch_size = 128\n",
    "\n",
    "\n",
    "# Take a look at the shapes\n",
    "print('X_train:', X_train.shape)\n",
    "print('y_train:', y_train.shape)\n",
    "print('X_test:', X_test.shape)\n",
    "print('y_test:', y_test.shape)\n",
    "\n",
    "tb_callback = keras.callbacks.TensorBoard(\n",
    "        histogram_freq=0, write_graph=True)\n",
    "\n",
    "model = build_model_twitter2()\n",
    "model.fit(X_train, y_train,\n",
    "          validation_data=(X_test, y_test),\n",
    "          batch_size=batch_size, epochs=epochs,\n",
    "          callbacks=[tb_callback])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: (145056, 100)\n",
      "y_train: (145056,)\n",
      "X_test: (36265, 100)\n",
      "y_test: (36265,)\n",
      "Train on 145056 samples, validate on 36265 samples\n",
      "Epoch 1/12\n",
      "145056/145056 [==============================] - 113s - loss: 0.4948 - acc: 0.7650 - val_loss: 0.4926 - val_acc: 0.7677\n",
      "Epoch 2/12\n",
      "145056/145056 [==============================] - 110s - loss: 0.4608 - acc: 0.7871 - val_loss: 0.4640 - val_acc: 0.7883\n",
      "Epoch 3/12\n",
      "145056/145056 [==============================] - 113s - loss: 0.4461 - acc: 0.7966 - val_loss: 0.4570 - val_acc: 0.7942\n",
      "Epoch 4/12\n",
      "145056/145056 [==============================] - 109s - loss: 0.4364 - acc: 0.8009 - val_loss: 0.4612 - val_acc: 0.7838\n",
      "Epoch 5/12\n",
      "145056/145056 [==============================] - 112s - loss: 0.4282 - acc: 0.8061 - val_loss: 0.4300 - val_acc: 0.8074\n",
      "Epoch 6/12\n",
      "145056/145056 [==============================] - 110s - loss: 0.4206 - acc: 0.8102 - val_loss: 0.4328 - val_acc: 0.8050\n",
      "Epoch 7/12\n",
      "145056/145056 [==============================] - 113s - loss: 0.4158 - acc: 0.8124 - val_loss: 0.4525 - val_acc: 0.7958\n",
      "Epoch 8/12\n",
      "145056/145056 [==============================] - 110s - loss: 0.4104 - acc: 0.8160 - val_loss: 0.4277 - val_acc: 0.8076\n",
      "Epoch 9/12\n",
      "145056/145056 [==============================] - 112s - loss: 0.4051 - acc: 0.8179 - val_loss: 0.4269 - val_acc: 0.8095\n",
      "Epoch 10/12\n",
      "145056/145056 [==============================] - 112s - loss: 0.4017 - acc: 0.8211 - val_loss: 0.4312 - val_acc: 0.8069\n",
      "Epoch 11/12\n",
      "145056/145056 [==============================] - 111s - loss: 0.3970 - acc: 0.8236 - val_loss: 0.4221 - val_acc: 0.8093\n",
      "Epoch 12/12\n",
      "145056/145056 [==============================] - 110s - loss: 0.3931 - acc: 0.8257 - val_loss: 0.4437 - val_acc: 0.7959\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f6cc919bcf8>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Model 3\n",
    "np.random.seed(42)\n",
    "\n",
    "\n",
    "epochs = 12\n",
    "batch_size = 128\n",
    "\n",
    "\n",
    "# Take a look at the shapes\n",
    "print('X_train:', X_train.shape)\n",
    "print('y_train:', y_train.shape)\n",
    "print('X_test:', X_test.shape)\n",
    "print('y_test:', y_test.shape)\n",
    "\n",
    "tb_callback = keras.callbacks.TensorBoard(\n",
    "        histogram_freq=0, write_graph=True)\n",
    "\n",
    "model = build_model_twitter3()\n",
    "model.fit(X_train, y_train,\n",
    "          validation_data=(X_test, y_test),\n",
    "          batch_size=batch_size, epochs=epochs,\n",
    "          callbacks=[tb_callback])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model_twitter4(max_length=100,\n",
    "                nb_filters=64,\n",
    "                kernel_size=3,\n",
    "                pool_size=2,\n",
    "                regularization=0.01,\n",
    "                weight_constraint=2.,\n",
    "                dropout_prob=0.4,\n",
    "                clear_session=True):\n",
    "    if clear_session:\n",
    "        K.clear_session()\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(\n",
    "        embeddings.shape[0],\n",
    "        embeddings.shape[1],\n",
    "        input_length=max_length,\n",
    "        trainable=False,\n",
    "        weights=[embeddings]))\n",
    "\n",
    "    model.add(LSTM(100, recurrent_dropout = 0.2, dropout = 0.2))\n",
    "    model.add(Dense(1, activation = 'sigmoid'))\n",
    "\n",
    "    model.compile(\n",
    "        loss='binary_crossentropy',\n",
    "        optimizer='adam',\n",
    "        metrics=['accuracy'])\n",
    "\n",
    "    return model\n",
    "    \n",
    "def build_model_twitter5(max_length=100,\n",
    "                nb_filters=64,\n",
    "                kernel_size=3,\n",
    "                pool_size=2,\n",
    "                regularization=0.01,\n",
    "                weight_constraint=2.,\n",
    "                dropout_prob=0.4,\n",
    "                clear_session=True):\n",
    "    if clear_session:\n",
    "        K.clear_session()\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(\n",
    "        embeddings.shape[0],\n",
    "        embeddings.shape[1],\n",
    "        input_length=max_length,\n",
    "        trainable=False,\n",
    "        weights=[embeddings]))\n",
    "\n",
    "    model.add(Conv1D(padding = \"same\", kernel_size = 3, filters = 32, activation = \"relu\"))\n",
    "    model.add(MaxPooling1D(pool_size = 2))\n",
    "    model.add(LSTM(100))\n",
    "    model.add(Dense(1, activation = 'sigmoid'))\n",
    "\n",
    "    model.compile(\n",
    "        loss='binary_crossentropy',\n",
    "        optimizer='adam',\n",
    "        metrics=['accuracy'])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: (145056, 100)\n",
      "y_train: (145056,)\n",
      "X_test: (36265, 100)\n",
      "y_test: (36265,)\n",
      "Train on 145056 samples, validate on 36265 samples\n",
      "Epoch 1/12\n",
      "145056/145056 [==============================] - 52s - loss: 0.4989 - acc: 0.7626 - val_loss: 0.5022 - val_acc: 0.7584\n",
      "Epoch 2/12\n",
      "145056/145056 [==============================] - 51s - loss: 0.4643 - acc: 0.7846 - val_loss: 0.4751 - val_acc: 0.7809\n",
      "Epoch 3/12\n",
      "145056/145056 [==============================] - 52s - loss: 0.4503 - acc: 0.7941 - val_loss: 0.4600 - val_acc: 0.7907\n",
      "Epoch 4/12\n",
      "145056/145056 [==============================] - 52s - loss: 0.4403 - acc: 0.7994 - val_loss: 0.4600 - val_acc: 0.7850\n",
      "Epoch 5/12\n",
      "145056/145056 [==============================] - 52s - loss: 0.4323 - acc: 0.8040 - val_loss: 0.4344 - val_acc: 0.8019\n",
      "Epoch 6/12\n",
      "145056/145056 [==============================] - 52s - loss: 0.4251 - acc: 0.8076 - val_loss: 0.4365 - val_acc: 0.8002\n",
      "Epoch 7/12\n",
      "145056/145056 [==============================] - 51s - loss: 0.4205 - acc: 0.8093 - val_loss: 0.4398 - val_acc: 0.8018\n",
      "Epoch 8/12\n",
      "145056/145056 [==============================] - 51s - loss: 0.4160 - acc: 0.8120 - val_loss: 0.4351 - val_acc: 0.8006\n",
      "Epoch 9/12\n",
      "145056/145056 [==============================] - 52s - loss: 0.4107 - acc: 0.8156 - val_loss: 0.4325 - val_acc: 0.8050\n",
      "Epoch 10/12\n",
      "145056/145056 [==============================] - 52s - loss: 0.4075 - acc: 0.8182 - val_loss: 0.4380 - val_acc: 0.8029\n",
      "Epoch 11/12\n",
      "145056/145056 [==============================] - 52s - loss: 0.4026 - acc: 0.8203 - val_loss: 0.4300 - val_acc: 0.8073\n",
      "Epoch 12/12\n",
      "145056/145056 [==============================] - 52s - loss: 0.3994 - acc: 0.8225 - val_loss: 0.4702 - val_acc: 0.7839\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f2e88886e48>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Model 4\n",
    "np.random.seed(42)\n",
    "\n",
    "\n",
    "epochs = 12\n",
    "batch_size = 128\n",
    "\n",
    "\n",
    "# Take a look at the shapes\n",
    "print('X_train:', X_train.shape)\n",
    "print('y_train:', y_train.shape)\n",
    "print('X_test:', X_test.shape)\n",
    "print('y_test:', y_test.shape)\n",
    "\n",
    "tb_callback = keras.callbacks.TensorBoard(\n",
    "        histogram_freq=0, write_graph=True)\n",
    "\n",
    "model = build_model_twitter3()\n",
    "model.fit(X_train, y_train,\n",
    "          validation_data=(X_test, y_test),\n",
    "          batch_size=batch_size, epochs=epochs,\n",
    "          callbacks=[tb_callback])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: (1816385, 100)\n",
      "y_train: (1816385,)\n",
      "X_test: (454097, 100)\n",
      "y_test: (454097,)\n",
      "Train on 1816385 samples, validate on 454097 samples\n",
      "Epoch 1/12\n",
      "1816385/1816385 [==============================] - 1860s - loss: 0.4535 - acc: 0.7702 - val_loss: 0.3936 - val_acc: 0.8170\n",
      "Epoch 2/12\n",
      "1816385/1816385 [==============================] - 1872s - loss: 0.3843 - acc: 0.8220 - val_loss: 0.3758 - val_acc: 0.8271\n",
      "Epoch 3/12\n",
      "1816385/1816385 [==============================] - 1875s - loss: 0.3700 - acc: 0.8301 - val_loss: 0.3720 - val_acc: 0.8295\n",
      "Epoch 4/12\n",
      "1816385/1816385 [==============================] - 1874s - loss: 0.3619 - acc: 0.8348 - val_loss: 0.3651 - val_acc: 0.8330\n",
      "Epoch 5/12\n",
      "1816385/1816385 [==============================] - 1845s - loss: 0.3563 - acc: 0.8379 - val_loss: 0.3628 - val_acc: 0.8347\n",
      "Epoch 6/12\n",
      " 248160/1816385 [===>..........................] - ETA: 1438s - loss: 0.3507 - acc: 0.8413"
     ]
    }
   ],
   "source": [
    "#Model 5\n",
    "np.random.seed(42)\n",
    "\n",
    "\n",
    "epochs = 12\n",
    "batch_size = 160\n",
    "\n",
    "\n",
    "# Take a look at the shapes\n",
    "print('X_train:', X_train.shape)\n",
    "print('y_train:', y_train.shape)\n",
    "print('X_test:', X_test.shape)\n",
    "print('y_test:', y_test.shape)\n",
    "\n",
    "tb_callback = keras.callbacks.TensorBoard(\n",
    "        histogram_freq=0, write_graph=True)\n",
    "\n",
    "model = build_model_twitter5()\n",
    "model.fit(X_train, y_train,\n",
    "          validation_data=(X_test, y_test),\n",
    "          batch_size=batch_size, epochs=epochs,\n",
    "          callbacks=[tb_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ciao\n"
     ]
    }
   ],
   "source": [
    "print(\"ciao\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}