textrank_text_summarization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize.punkt import PunktSentenceTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "document=\"Hi Steve  Thank you so much for reaching out and taking the time to send us feedback! It is very much appreciated! Please excuse the delayed response. Unfortunately, these scanning issues mostly occur due to older scanning technologies still used by some stores. Some devices (e.g. laser scanners or flatbed scanners) have difficulties reading smartphone displays. We hope to be able to solve this problem soon though as most stores will hopefully replace these scanning devices with newer image-based scanners in the course of their next cash system update. Until then, you can always ask the store staff to type in the card number manually in these cases so that you don't miss out on any rewards points or discounts. In the meantime, I sincerely apologize for the inconvenience this causes.  Once again, thank you for taking the time to contact us about this issue. If you have any further questions, suggestions for improvements or general feedback, please don't hesitate to contact us again.Best regards,Isabelle van Capelleveen Customer Support GmbH C-HUB / Hafenstraße 25-27 68159 Mannheim\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_tokenizer = PunktSentenceTokenizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = sentence_tokenizer.tokenize(document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hi Steve  Thank you so much for reaching out and taking the time to send us feedback!',\n",
       " 'It is very much appreciated!',\n",
       " 'Please excuse the delayed response.',\n",
       " 'Unfortunately, these scanning issues mostly occur due to older scanning technologies still used by some stores.',\n",
       " 'Some devices (e.g.',\n",
       " 'laser scanners or flatbed scanners) have difficulties reading smartphone displays.',\n",
       " 'We hope to be able to solve this problem soon though as most stores will hopefully replace these scanning devices with newer image-based scanners in the course of their next cash system update.',\n",
       " \"Until then, you can always ask the store staff to type in the card number manually in these cases so that you don't miss out on any rewards points or discounts.\",\n",
       " 'In the meantime, I sincerely apologize for the inconvenience this causes.',\n",
       " 'Once again, thank you for taking the time to contact us about this issue.',\n",
       " \"If you have any further questions, suggestions for improvements or general feedback, please don't hesitate to contact us again.Best regards,Isabelle van Capelleveen Customer Support GmbH C-HUB / Hafenstraße 25-27 68159 Mannheim\"]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bag_of_words(sentence):\n",
    "    return Counter(word.lower().strip('.,') for word in sentence.split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2, 1]], dtype=int64)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "c = CountVectorizer()\n",
    "bow_array = c.fit_transform(['hey hey wassup']) #tokenize words and then count them here 2 'hey' and 1 'wassup' so [2 1]\n",
    "bow_array.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 126)\n",
      "  (0, 10)\t1\n",
      "  (0, 121)\t1\n",
      "  (0, 52)\t1\n",
      "  (0, 56)\t1\n",
      "  (0, 64)\t1\n"
     ]
    }
   ],
   "source": [
    "bow_matrix = c.fit_transform(sentences) \n",
    "print((bow_matrix[1]).shape) #to find total number of words in document\n",
    "print(bow_matrix[1])#to find position of words  that are present in sentence like in first sentence 10th,121sr,etc position words are present and that 1 represents it has come only one time in this sentence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/spartan/.local/lib/python3.5/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    }
   ],
   "source": [
    "#transforming matrix to graph\n",
    "from sklearn.feature_extraction.text import TfidfTransformer \n",
    "normalized_matrix = TfidfTransformer().fit_transform(bow_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.        , 0.09621319, 0.04333366, 0.02228679, 0.        ,\n",
       "        0.        , 0.04756537, 0.1784559 , 0.10863429, 0.4009258 ,\n",
       "        0.13436086],\n",
       "       [0.09621319, 1.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.04333366, 0.        , 1.        , 0.        , 0.        ,\n",
       "        0.        , 0.02755957, 0.05755771, 0.10836507, 0.04962846,\n",
       "        0.06819426],\n",
       "       [0.02228679, 0.        , 0.        , 1.        , 0.15475965,\n",
       "        0.        , 0.15693685, 0.04232347, 0.        , 0.02552425,\n",
       "        0.01458805],\n",
       "       [0.        , 0.        , 0.        , 0.15475965, 1.        ,\n",
       "        0.        , 0.11009808, 0.        , 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        1.        , 0.08326911, 0.03362571, 0.        , 0.        ,\n",
       "        0.07599212],\n",
       "       [0.04756537, 0.        , 0.02755957, 0.15693685, 0.11009808,\n",
       "        0.08326911, 1.        , 0.1008582 , 0.11337548, 0.08823968,\n",
       "        0.02075627],\n",
       "       [0.1784559 , 0.        , 0.05755771, 0.04232347, 0.        ,\n",
       "        0.03362571, 0.1008582 , 1.        , 0.15979469, 0.11320338,\n",
       "        0.11528719],\n",
       "       [0.10863429, 0.        , 0.10836507, 0.        , 0.        ,\n",
       "        0.        , 0.11337548, 0.15979469, 1.        , 0.19079701,\n",
       "        0.03030055],\n",
       "       [0.4009258 , 0.        , 0.04962846, 0.02552425, 0.        ,\n",
       "        0.        , 0.08823968, 0.11320338, 0.19079701, 1.        ,\n",
       "        0.19881004],\n",
       "       [0.13436086, 0.        , 0.06819426, 0.01458805, 0.        ,\n",
       "        0.07599212, 0.02075627, 0.11528719, 0.03030055, 0.19881004,\n",
       "        1.        ]])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity_graph = normalized_matrix * normalized_matrix.T\n",
    "similarity_graph.toarray()\n",
    "#this is matrix representing similarity betwen two sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this matrix, both the rows and columns are sentences. The element in the matrix is the similarity between a pair of sentences.  Score 1 means two sentences are identical, and score 0 means there is no overlap between two sentence.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After build a graph, we can use pagerank algorithm to calculate the importance of the node in the graph.  We can use the pagerank method from NetworkX."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "nx_graph = nx.from_scipy_sparse_matrix(similarity_graph)\n",
    "scores = nx.pagerank(nx_graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 0.1069863424404059,\n",
       " 1: 0.07989763219183015,\n",
       " 2: 0.08016956221262056,\n",
       " 3: 0.0860847008202275,\n",
       " 4: 0.08204157912113512,\n",
       " 5: 0.0790389974071022,\n",
       " 6: 0.09859919581764136,\n",
       " 7: 0.09681512905068874,\n",
       " 8: 0.09234452712574573,\n",
       " 9: 0.10674271535704397,\n",
       " 10: 0.09127961845555874}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores #page rank applied on sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "#using index we will find out text\n",
    "ranked = sorted(((scores[i],s) for i,s in enumerate(sentences)),reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.1069863424404059,\n",
       "  'Hi Steve  Thank you so much for reaching out and taking the time to send us feedback!'),\n",
       " (0.10674271535704397,\n",
       "  'Once again, thank you for taking the time to contact us about this issue.'),\n",
       " (0.09859919581764136,\n",
       "  'We hope to be able to solve this problem soon though as most stores will hopefully replace these scanning devices with newer image-based scanners in the course of their next cash system update.'),\n",
       " (0.09681512905068874,\n",
       "  \"Until then, you can always ask the store staff to type in the card number manually in these cases so that you don't miss out on any rewards points or discounts.\"),\n",
       " (0.09234452712574573,\n",
       "  'In the meantime, I sincerely apologize for the inconvenience this causes.'),\n",
       " (0.09127961845555874,\n",
       "  \"If you have any further questions, suggestions for improvements or general feedback, please don't hesitate to contact us again.Best regards,Isabelle van Capelleveen Customer Support GmbH C-HUB / Hafenstraße 25-27 68159 Mannheim\"),\n",
       " (0.0860847008202275,\n",
       "  'Unfortunately, these scanning issues mostly occur due to older scanning technologies still used by some stores.'),\n",
       " (0.08204157912113512, 'Some devices (e.g.'),\n",
       " (0.08016956221262056, 'Please excuse the delayed response.'),\n",
       " (0.07989763219183015, 'It is very much appreciated!'),\n",
       " (0.0790389974071022,\n",
       "  'laser scanners or flatbed scanners) have difficulties reading smartphone displays.')]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ranked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}