02_Short_Intro_to_Embeddings_with_Tensorflow_0.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Short Intro To Embeddings with Tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Goals\n",
    "- Understand Embedding \n",
    "- Perform Embedding Lookup using Tensorflow\n",
    "- Use Pre-Trained Embedding "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "print('Tensorflow version : {0}'.format(tf.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 5\n",
    "vocabulary_size = 10\n",
    "\n",
    "# create a sample embedding matrix of size 5 for vocab of size 10\n",
    "embedding = np.random.rand(vocabulary_size, embedding_size)\n",
    "print(embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create one-hot encoding for one of element in vocabulary\n",
    "i = 4\n",
    "one_hot = np.zeros(10)\n",
    "one_hot[i] = 1.0\n",
    "print(one_hot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embedding vector can be extracted by taking a dot product between the one_hot vector and embedding matrix\n",
    "embedding_vector = np.dot(one_hot, embedding)\n",
    "print(embedding_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cross validate from the embedding matrix\n",
    "print(embedding[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensorflow Embedding Lookup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = tf.Graph()\n",
    "with g.as_default():\n",
    "    # provide input indices \n",
    "    x = tf.placeholder(shape=[None], dtype=tf.int32, name='x')\n",
    "    \n",
    "    # create a constant initializer\n",
    "    weights_initializer = tf.constant_initializer(embedding)\n",
    "    embedding_weights = tf.get_variable(\n",
    "                            name='embedding_weights', \n",
    "                            shape=(vocabulary_size, embedding_size), \n",
    "                            initializer=weights_initializer,\n",
    "                            trainable=False)\n",
    "    # emebedding Lookup \n",
    "    embedding_lookup = tf.nn.embedding_lookup(embedding_weights, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting Single Row\n",
    "with tf.Session(graph=g) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    print(sess.run(embedding_lookup, feed_dict={x : [4]}))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting Multiple Rows\n",
    "with tf.Session(graph=g) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    print(sess.run(embedding_lookup, feed_dict={x : [2,4,6]}))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using GloVe Pre-Trained Model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_DIMENSION=100 # Available dimensions for 6B data is 50, 100, 200, 300\n",
    "glove_weights_file_path = os.path.join('processed','glove', 'glove.6B.{0}d.txt'.format(EMBEDDING_DIMENSION))\n",
    "print('Using the following glove weight file : {0}'.format(glove_weights_file_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# look at some sample rows\n",
    "!head -3 processed/glove/glove.6B.100d.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_weights = []\n",
    "word2idx = {}\n",
    "vocabulary_size = 40000 # limit vocab to top 40K terms\n",
    "vocabulary = []\n",
    "\n",
    "\n",
    "with open(glove_weights_file_path,'r') as file:\n",
    "    for index, line in enumerate(file):\n",
    "        values = line.split() # Word and weights separated by space\n",
    "        word = values[0] # Word is first symbol on each line\n",
    "        vocabulary.append(word)\n",
    "        word_weights = np.asarray(values[1:], dtype=np.float32) # Remainder of line is weights for word\n",
    "        word2idx[word] = index \n",
    "        glove_weights.append(word_weights)\n",
    "        \n",
    "        if index + 1 == vocabulary_size:\n",
    "            break\n",
    "glove_weights = np.asarray(glove_weights, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_weights.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = [\"man\", \"woman\"]\n",
    "#words = [\"paris\", \"london\",\"rome\",\"berlin\"]\n",
    "words_indices = [word2idx[word] for word in words]\n",
    "words_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = tf.Graph()\n",
    "\n",
    "with g.as_default():\n",
    "    # provide input indices \n",
    "    x = tf.placeholder(shape=[None], dtype=tf.int32, name='x')\n",
    "    \n",
    "    # create a constant initializer\n",
    "    weights_initializer = tf.constant_initializer(glove_weights)\n",
    "    embedding_weights = tf.get_variable(\n",
    "                            name='embedding_weights', \n",
    "                            shape=(vocabulary_size, EMBEDDING_DIMENSION), \n",
    "                            initializer=weights_initializer,\n",
    "                            trainable=False)\n",
    "    # emebedding Lookup \n",
    "    embedding_lookup = tf.nn.embedding_lookup(embedding_weights, x)\n",
    "    \n",
    "    # We use the cosine distance:\n",
    "    norm = tf.sqrt(tf.reduce_sum(tf.square(embedding_weights), 1, keepdims=True))\n",
    "    normalized_embeddings = embedding_weights / norm\n",
    "    valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, x)\n",
    "    similarity = tf.matmul(valid_embeddings, tf.transpose(normalized_embeddings))\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session(graph=g) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    result = sess.run(embedding_lookup, feed_dict={x : words_indices})\n",
    "    sim = sess.run(similarity, feed_dict={x : words_indices})\n",
    "    print('Shape of Similarity Matrix: {0}'.format(sim.shape))\n",
    "    for i,word_index in enumerate(words_indices):\n",
    "       \n",
    "        top_k = 10 # number of nearest neighbors\n",
    "        nearest = (-sim[i, :]).argsort()[1:top_k+1]\n",
    "        log = 'Nearest to {0} :'.format(vocabulary[word_index])\n",
    "        \n",
    "        for k in range(top_k):\n",
    "       \n",
    "            close_word = vocabulary[nearest[k]]\n",
    "            log = '{0} {1},'.format(log, close_word)\n",
    "        print(log)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}