ILJST-Copy2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# !pip install numpy num2words nltk pandas Observations gensim\n",
    "# import nltk\n",
    "# nltk.download('stopwords')\n",
    "# nltk.download('punkt')\n",
    "# nltk.download('wordnet')\n",
    "# !pip install -U textblob\n",
    "# !python -m textblob.download_corpora"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import spatial, sparse\n",
    "from scipy.stats import chi2\n",
    "from collections import Counter\n",
    "from num2words import num2words\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn.externals import joblib \n",
    "from nltk.tokenize import word_tokenize\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "\n",
    "import os\n",
    "import imp\n",
    "import gzip\n",
    "import copy\n",
    "import nltk\n",
    "import pickle\n",
    "import scipy\n",
    "import string\n",
    "import gensim\n",
    "import operator\n",
    "import datetime\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import LDA_ILJST as lda\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'LDA_ILJST' from 'LDA_ILJST.pyc'>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp.reload(lda)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing Methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_numbers(k):\n",
    "    for i in range(len(k)):\n",
    "        try:\n",
    "            num2words(int(k[i]))\n",
    "            k[i] = \" \"\n",
    "        except:\n",
    "            pass\n",
    "    return k\n",
    "\n",
    "def preprocess(pd):\n",
    "    pd = pd.str.lower()\n",
    "    pd = pd.str.replace('[{}]'.format('!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~\\n\\t'), ' ')\n",
    "    pd = pd.apply(lambda x: [w for w in w_tokenizer.tokenize(x)])\n",
    "    pd = pd.apply(lambda x: convert_numbers(x))\n",
    "    pd = pd.str.join(' ')\n",
    "    \n",
    "    pd = pd.apply(lambda x: [lemmatizer.lemmatize(w) for w in w_tokenizer.tokenize(x)])    \n",
    "    pd = pd.apply(lambda x: [item for item in x if item not in stop_words])\n",
    "    return pd\n",
    "\n",
    "def processReviews(reviews, window=5, MAX_VOCAB_SIZE=2000):\n",
    "    vectorizer = CountVectorizer(analyzer=\"word\",tokenizer=None, max_df=0.7, max_features=MAX_VOCAB_SIZE)\n",
    "    count_matrix = vectorizer.fit_transform(reviews)\n",
    "    tfidf_vectorizer = TfidfVectorizer(max_features=MAX_VOCAB_SIZE, max_df=0.7)\n",
    "    tfidf_matrix = tfidf_vectorizer.fit_transform(reviews)\n",
    "    words = vectorizer.get_feature_names()\n",
    "    vocabulary = dict(zip(words,np.arange(len(words))))\n",
    "    inv_vocabulary = dict(zip(np.arange(len(words)),words))\n",
    "    return count_matrix.toarray(), tfidf_matrix.toarray(), vocabulary, words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Score Methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def coherence_score(X, topic_sentiment_df):\n",
    "    X[X>1] = 1    \n",
    "    totalcnt = len(topic_sentiment_df)\n",
    "    total = 0\n",
    "    for allwords in topic_sentiment_df:\n",
    "        for word1 in allwords:\n",
    "            for word2 in allwords:\n",
    "                if word1 != word2:\n",
    "                    ind1 = vocabulary[word1]\n",
    "                    ind2 = vocabulary[word2]\n",
    "                    total += np.log((np.matmul(X[:,ind1].T, X[:,ind2]) + 1.0)/np.sum(X[:,ind2]))\n",
    "    return total/(2*totalcnt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def kl_score(pk,qk):\n",
    "    return (scipy.stats.entropy(pk,qk)*.5 + scipy.stats.entropy(qk,pk)*.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_hscore(dt_distribution, X, k):\n",
    "    testlen = X.shape[0]\n",
    "    all_kl_scores = np.zeros((testlen, testlen))\n",
    "    for i in range(testlen-1):\n",
    "        for j in range(i+1,testlen):\n",
    "            score = kl_score(dt_distribution[i],dt_distribution[j])\n",
    "            all_kl_scores[i,j] = score\n",
    "            all_kl_scores[j,i] = score\n",
    "\n",
    "    dt = np.zeros((X.shape[0], k))\n",
    "\n",
    "    for i in range(X.shape[0]):\n",
    "        dt[i, dt_distribution[i].argmax()]=1\n",
    "\n",
    "    intradist = 0\n",
    "    for i in range(k):\n",
    "        cnt = dt[:,i].sum()\n",
    "        tmp = np.outer(dt[:,i],dt[:,i])\n",
    "        tmp = tmp * all_kl_scores\n",
    "        intradist += tmp.sum()*1.0/(cnt*(cnt-1))\n",
    "    intradist = intradist/k\n",
    "\n",
    "    interdist = 0\n",
    "    for i in range(k):\n",
    "       for j in range(k):\n",
    "           if i != j:\n",
    "             cnt_i = dt[:,i].sum()\n",
    "             cnt_j = dt[:,j].sum()\n",
    "             tmp = np.outer(dt[:,i], dt[:,j])\n",
    "             tmp = tmp * all_kl_scores\n",
    "             interdist += tmp.sum()*1.0/(cnt_i*cnt_j)\n",
    "    interdist = interdist/(k*(k-1))\n",
    "    return intradist/interdist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Required Methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cosine(a, b):\n",
    "    return 1 - spatial.distance.cosine(a, b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse(path):\n",
    "    g = gzip.open(path, 'r')\n",
    "    for l in g:\n",
    "        yield eval(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadGloveModel(gloveFile):\n",
    "    print(\"Loading Glove Model\")\n",
    "    f = open(gloveFile,'r')\n",
    "    model = {}\n",
    "    for line in f:\n",
    "        splitLine = line.split()\n",
    "        word = splitLine[0]\n",
    "        embedding = np.array([float(val) for val in splitLine[1:]])\n",
    "        model[word] = embedding\n",
    "    print(\"Done.\",len(model),\" words loaded!\")\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Glove Model\n",
      "('Done.', 1917494, ' words loaded!')\n",
      "CPU times: user 2min 54s, sys: 4.91 s, total: 2min 59s\n",
      "Wall time: 2min 59s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "embeddings_index = loadGloveModel(\"nongit_resources/glove.42B.300d.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_docs = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_pickle(\"resources/reviews_Movies_and_TV_5_5k_pd\")\n",
    "dataset = dataset[['asin', 'helpful', 'overall', 'reviewText']]\n",
    "reviews = dataset['reviewText'].values\n",
    "ratings = dataset['overall'].values[:N_docs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5000"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ratings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = stopwords.words('english')\n",
    "lemmatizer = nltk.stem.WordNetLemmatizer()\n",
    "stemmer = nltk.stem.PorterStemmer()\n",
    "w_tokenizer = nltk.tokenize.WhitespaceTokenizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset[8] = preprocess(dataset['reviewText'])\n",
    "dataset[9] = dataset[8].apply(lambda x: \" \".join(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asin</th>\n",
       "      <th>helpful</th>\n",
       "      <th>overall</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0005019281</td>\n",
       "      <td>[0, 0]</td>\n",
       "      <td>4.0</td>\n",
       "      <td>This is a charming version of the classic Dick...</td>\n",
       "      <td>[charming, version, classic, dicken, tale, hen...</td>\n",
       "      <td>charming version classic dicken tale henry win...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0005019281</td>\n",
       "      <td>[0, 0]</td>\n",
       "      <td>3.0</td>\n",
       "      <td>It was good but not as emotionally moving as t...</td>\n",
       "      <td>[wa, good, emotionally, moving, christmas, car...</td>\n",
       "      <td>wa good emotionally moving christmas carol dic...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         asin helpful  overall  \\\n",
       "0  0005019281  [0, 0]      4.0   \n",
       "1  0005019281  [0, 0]      3.0   \n",
       "\n",
       "                                          reviewText  \\\n",
       "0  This is a charming version of the classic Dick...   \n",
       "1  It was good but not as emotionally moving as t...   \n",
       "\n",
       "                                                   8  \\\n",
       "0  [charming, version, classic, dicken, tale, hen...   \n",
       "1  [wa, good, emotionally, moving, christmas, car...   \n",
       "\n",
       "                                                   9  \n",
       "0  charming version classic dicken tale henry win...  \n",
       "1  wa good emotionally moving christmas carol dic...  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_matrix, tfidf_matrix, vocabulary, words = processReviews(dataset[9].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 2000), (5000, 2000))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_matrix.shape, tfidf_matrix.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find Edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle_in = open(\"resources/docs_edges_movies_5k.pickle\",\"rb\")\n",
    "docs_edges = pickle.load(pickle_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "edge_dict = {}\n",
    "for i in docs_edges:\n",
    "    for j in i:\n",
    "        try:\n",
    "            edge_dict[j[0]] += [j[1]]\n",
    "            edge_dict[j[1]] += [j[0]]\n",
    "        except:\n",
    "            edge_dict[j[0]] = [j[1]]\n",
    "            edge_dict[j[1]] = [j[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in edge_dict.keys():\n",
    "    edge_dict[i] = list(set(edge_dict[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "431"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(edge_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxiter = 20\n",
    "lambda_param = 1.0\n",
    "N_ITERATAIONS = 5\n",
    "N_SENTIMENT = 5\n",
    "k = N_TOPICS = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'LDA_ILJST' from 'LDA_ILJST.pyc'>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp.reload(lda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "kf = KFold(n_splits=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "('fold:', 0)\n",
      "(datetime.time(16, 15, 27, 677812), ' iteration:', 0)\n",
      "-3066524.972826976\n",
      "(datetime.time(16, 29, 16, 572851), ' iteration:', 1)\n",
      "-3027797.7023391654\n",
      "(datetime.time(16, 42, 31, 597235), ' iteration:', 2)\n",
      "-3055437.221791103\n",
      "(datetime.time(16, 55, 19, 545808), ' iteration:', 3)\n",
      "-3034624.5155907534\n",
      "(datetime.time(17, 7, 52, 294330), ' iteration:', 4)\n",
      "-3046631.205751706\n",
      "\n",
      "('fold:', 1)\n",
      "(datetime.time(17, 20, 53, 85042), ' iteration:', 0)\n",
      "-3048902.570590355\n",
      "(datetime.time(17, 33, 53, 193889), ' iteration:', 1)\n",
      "-3057421.0794471004\n",
      "(datetime.time(17, 46, 20, 252040), ' iteration:', 2)\n",
      "-3073132.7321290183\n",
      "(datetime.time(17, 58, 51, 592417), ' iteration:', 3)\n",
      "-3043150.1929086475\n",
      "(datetime.time(18, 11, 34, 517482), ' iteration:', 4)\n",
      "-3053133.895744541\n",
      "\n",
      "('fold:', 2)\n",
      "(datetime.time(18, 24, 31, 914744), ' iteration:', 0)\n",
      "-3009954.6161419367\n",
      "(datetime.time(18, 38, 56, 840566), ' iteration:', 1)\n",
      "-2985276.666483243\n",
      "(datetime.time(18, 52, 34, 937377), ' iteration:', 2)\n"
     ]
    }
   ],
   "source": [
    "date_time = str(datetime.datetime.now())\n",
    "os.mkdir(\"dumps/\"+date_time)\n",
    "\n",
    "df_matrix = scipy.sparse.load_npz('resources/df_home_sparse_5k.npz')\n",
    "\n",
    "fold = 0\n",
    "for train_index, test_index in kf.split(count_matrix):\n",
    "    X_train, X_test = count_matrix[train_index], count_matrix[test_index]\n",
    "    y_train, y_test = ratings[train_index], ratings[test_index]\n",
    "    \n",
    "    sampler = lda.LdaSampler(n_sentiment = N_SENTIMENT, n_topics=N_TOPICS, lambda_param=lambda_param)\n",
    "    sampler.store_data(X_train, y_train, X_test, y_test, df_matrix, words, vocabulary, edge_dict)\n",
    "    \n",
    "    likelihood_history = []\n",
    "    print(\"\")\n",
    "    print(\"fold:\", fold)\n",
    "\n",
    "    for i in range(N_ITERATAIONS):\n",
    "        print(datetime.datetime.now().time(), \" iteration:\", i)\n",
    "        sampler.run(X_train, y_train, X_test, edge_dict, maxiter=maxiter)\n",
    "        lk_now = sampler.loglikelihood(docs_edges)\n",
    "        likelihood_history.append(lk_now)\n",
    "        print(lk_now)\n",
    "        joblib.dump(sampler, 'dumps/' + date_time + '/sampler_home_5k_fold_' + str(fold) + \"_maxiter_\"\n",
    "                    + str(maxiter) + \"_iter_\" + str(i) +\"_in_\" + str(N_ITERATAIONS))\n",
    "        \n",
    "    pd.DataFrame(likelihood_history).to_csv('dumps/' + date_time + \"/likelihood_history_home_fold_\" +\n",
    "                                            str(fold) + '.txt', header=None)\n",
    "    \n",
    "    fold += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampler = joblib.load(\"dumps/2019-07-27 06:40:21.499611/sampler_electronics_5k_fold_0_maxiter_1_iter_0_in_2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_words = sampler.getTopKWords(5, sampler.words)\n",
    "top_words = [t_words[i] for i in t_words.keys()]\n",
    "document_topic = sampler.theta().argmax(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "coherence_score(sampler.get_count_matrix(), top_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "get_hscore(sampler.theta(), sampler.get_count_matrix(), sampler.n_topics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "top_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Entropy Calculations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idf_matrix = (sampler.get_df_matrix() > 0).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_freq = idf_matrix.sum(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idf_matrix = idf_matrix * word_freq\n",
    "idf_matrix = idf_matrix * 1.0/idf_matrix.shape[0]\n",
    "document_topic_entropy = scipy.stats.entropy(sampler.nmz.transpose())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_len = sampler.get_count_matrix().sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print (np.corrcoef(document_topic_entropy, word_len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "document_popularity = np.true_divide(idf_matrix.sum(1),(idf_matrix != 0).sum(1)) #scipy.stats.mstats.gmean(idf_matrix,axis=1)\n",
    "print (np.corrcoef(document_topic_entropy, document_popularity))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalized_dts = sampler.nmzs * sampler.nmz[:,:,np.newaxis]\n",
    "normalized_dts /= normalized_dts.sum(axis=-1)[:,:,np.newaxis]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "document_topic_sentiment_crossentropy = np.array([[scipy.stats.entropy(j) for j in i] for i in normalized_dts])\n",
    "document_ts_entropy_min = document_topic_sentiment_crossentropy.min(axis=1)\n",
    "document_ts_entropy_mean = document_topic_sentiment_crossentropy.mean(axis=1)\n",
    "document_ts_entropy_var = np.sqrt(document_topic_sentiment_crossentropy.var(axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "document_ts_entropy_var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# idf_matrix = (sampler.wordOccuranceMatrix > 0).astype(int)\n",
    "# word_freq = idf_matrix.sum(axis=0)\n",
    "# idf_matrix = idf_matrix * word_freq\n",
    "# idf_matrix = idf_matrix * 1.0/idf_matrix.shape[0]\n",
    "# document_topic_entropy = scipy.stats.entropy(sampler.dt_distribution.transpose())\n",
    "# word_len = sampler.wordOccuranceMatrix.sum(axis=1)\n",
    "# print (np.corrcoef(document_topic_entropy, word_len))\n",
    "# #document_topic_entropy_len_normalized = document_topic_entropy * np.sqrt(word_len/2)\n",
    "# document_popularity = np.true_divide(idf_matrix.sum(1),(idf_matrix != 0).sum(1)) #scipy.stats.mstats.gmean(idf_matrix,axis=1)\n",
    "# print (np.corrcoef(document_topic_entropy, document_popularity))\n",
    "# normalized_dts = sampler.dts_distribution * sampler.dt_distribution[:,:,np.newaxis]\n",
    "# normalized_dts /= normalized_dts.sum(axis=-1)[:,:,np.newaxis]\n",
    "# document_topic_sentiment_crossentropy = np.array([[scipy.stats.entropy(j) for j in i] for i in normalized_dts])\n",
    "# document_ts_entropy_min = document_topic_sentiment_crossentropy.min(axis=1)\n",
    "# document_ts_entropy_mean = document_topic_sentiment_crossentropy.mean(axis=1)\n",
    "# document_ts_entropy_var = np.sqrt(document_topic_sentiment_crossentropy.var(axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Appendix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# edges_threshold = 0.8\n",
    "# docs_edges, ignored, taken, count = [], [], [], 0\n",
    "# for idx, doc in enumerate(dataset[8].values):\n",
    "#     edges = []\n",
    "#     print(idx)\n",
    "#     for i in doc:\n",
    "#         for j in doc:\n",
    "#             if i != j:\n",
    "#                 try:\n",
    "#                     a = embeddings_index[i]\n",
    "#                     b = embeddings_index[j]\n",
    "#                     if get_cosine(a, b) > edges_threshold and (vocabulary[i], vocabulary[j]) not in edges and (vocabulary[j], vocabulary[i]) not in edges:\n",
    "#                         edges.append((vocabulary[i], vocabulary[j]))\n",
    "#                 except:\n",
    "#                     try:\n",
    "#                         embeddings_index[i]\n",
    "#                         taken.append(i)\n",
    "#                     except:\n",
    "#                         ignored.append(i)\n",
    "#                     try:\n",
    "#                         embeddings_index[j]\n",
    "#                     except:\n",
    "#                         ignored.append(j)\n",
    "#                         taken.append(j)\n",
    "#                     pass\n",
    "#     docs_edges.append(edges)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle_out = open(\"resources/docs_edges_home_5k.pickle\",\"wb\")\n",
    "# pickle.dump(docs_edges, pickle_out)\n",
    "# pickle_out.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Sentence wise\n",
    "# dataset = dataset[['asin', 'helpful', 'overall', 'reviewText']]\n",
    "# dataset['n_words'] = dataset['reviewText'].apply(lambda x: len(w_tokenizer.tokenize(x)))\n",
    "# dataset['sentences'] = dataset['reviewText'].apply(lambda x: [i.strip() for i in x.split(\".\")])\n",
    "# dataset['sentence_word_density'] = dataset['reviewText'].apply(lambda x: len(w_tokenizer.tokenize(x))/ len(x.split(\".\")))\n",
    "# dataset.to_csv(\"reviews_Musical_Instruments_5.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# edge_dict, ignored, taken, count = {}, [], [], 0\n",
    "# for idxi, i in enumerate(vocabulary.keys()):\n",
    "#     print(idxi)\n",
    "#     for idxj, j in enumerate(vocabulary.keys()):\n",
    "#         if i != j:\n",
    "#             try:\n",
    "#                 a = embeddings_index[i]\n",
    "#                 b = embeddings_index[j]\n",
    "#                 if get_cosine(a, b) > edges_threshold:\n",
    "#                     try:\n",
    "#                         edge_dict[vocabulary[i]] += [vocabulary[j]]\n",
    "#                         edge_dict[vocabulary[j]] += [vocabulary[i]]\n",
    "#                     except:\n",
    "#                         edge_dict[vocabulary[i]] = [vocabulary[j]]\n",
    "#                         edge_dict[vocabulary[j]] = [vocabulary[i]]\n",
    "#             except:\n",
    "#                 try:\n",
    "#                     embeddings_index[i]\n",
    "#                     taken.append(i)\n",
    "#                 except:\n",
    "#                     ignored.append(i)\n",
    "#                 try:\n",
    "#                     embeddings_index[j]\n",
    "#                 except:\n",
    "#                     ignored.append(j)\n",
    "#                     taken.append(j)\n",
    "#                 pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = {}\n",
    "\n",
    "# for idx, i in enumerate(dataset[8].values):\n",
    "#     print(idx)\n",
    "#     for j in i:\n",
    "#         try:\n",
    "#             df[j] += [idx]\n",
    "#         except:\n",
    "#             df[j] = [idx]\n",
    "\n",
    "# for i in df.keys():\n",
    "#     df[i] = len(list(set(df[i])))\n",
    "\n",
    "# df_vector = []\n",
    "# for i in dataset[8].values:\n",
    "#     d = [0]*len(vocabulary.keys())\n",
    "#     for j in i:\n",
    "#         if j in vocabulary.keys():\n",
    "#             d[vocabulary[j]] = df[j]\n",
    "#     df_vector.append(d)\n",
    "\n",
    "# csr = sparse.csr_matrix(np.array(df_vector))\n",
    "# scipy.sparse.save_npz('resources/df_movies_5k.npz', csr)\n",
    "# np.array(scipy.sparse.load_npz('resources/df_movies_5k.npz').todense())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# dataset = parse(\"nongit_resources/reviews_Electronics_5.json.gz\")\n",
    "# dataset = pd.DataFrame(list(dataset))\n",
    "# dataset = dataset.head(N_docs)\n",
    "# dataset.to_pickle(\"resources/reviews_Electronics_5\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:python2]",
   "language": "python",
   "name": "conda-env-python2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}