6_lstm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8tQJd2YSCfWR"
   },
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "D7tqLMoKF6uq"
   },
   "source": [
    "Deep Learning\n",
    "=============\n",
    "\n",
    "Assignment 6\n",
    "------------\n",
    "\n",
    "After training a skip-gram model in `5_word2vec.ipynb`, the goal of this notebook is to train a LSTM character model over [Text8](http://mattmahoney.net/dc/textdata) data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "MvEblsgEXxrd"
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "from __future__ import print_function\n",
    "import os\n",
    "import numpy as np\n",
    "import random\n",
    "import string\n",
    "import tensorflow as tf\n",
    "import zipfile\n",
    "from six.moves import range\n",
    "from six.moves.urllib.request import urlretrieve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5993,
     "status": "ok",
     "timestamp": 1445965582896,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "RJ-o3UBUFtCw",
    "outputId": "d530534e-0791-4a94-ca6d-1c8f1b908a9e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found and verified text8.zip\n"
     ]
    }
   ],
   "source": [
    "url = 'http://mattmahoney.net/dc/'\n",
    "\n",
    "def maybe_download(filename, expected_bytes):\n",
    "    \"\"\"Download a file if not present, and make sure it's the right size.\"\"\"\n",
    "    if not os.path.exists(filename):\n",
    "        filename, _ = urlretrieve(url + filename, filename)\n",
    "    statinfo = os.stat(filename)\n",
    "    if statinfo.st_size == expected_bytes:\n",
    "        print('Found and verified %s' % filename)\n",
    "    else:\n",
    "        print(statinfo.st_size)\n",
    "        raise Exception(\n",
    "            'Failed to verify ' + filename + '. Can you get to it with a browser?')\n",
    "    return filename\n",
    "\n",
    "filename = maybe_download('text8.zip', 31344016)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5982,
     "status": "ok",
     "timestamp": 1445965582916,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "Mvf09fjugFU_",
    "outputId": "8f75db58-3862-404b-a0c3-799380597390"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size 100000000\n"
     ]
    }
   ],
   "source": [
    "def read_data(filename):\n",
    "    with zipfile.ZipFile(filename) as f:\n",
    "        name = f.namelist()[0]\n",
    "        data = tf.compat.as_str(f.read(name))\n",
    "    return data\n",
    "  \n",
    "text = read_data(filename)\n",
    "print('Data size %d' % len(text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ga2CYACE-ghb"
   },
   "source": [
    "Create a small validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6184,
     "status": "ok",
     "timestamp": 1445965583138,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "w-oBpfFG-j43",
    "outputId": "bdb96002-d021-4379-f6de-a977924f0d02"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99999000 ons anarchists advocate social relations based upon voluntary as\n",
      "1000  anarchism originated as a term of abuse first used against earl\n"
     ]
    }
   ],
   "source": [
    "valid_size = 1000\n",
    "valid_text = text[:valid_size]\n",
    "train_text = text[valid_size:]\n",
    "train_size = len(train_text)\n",
    "print(train_size, train_text[:64])\n",
    "print(valid_size, valid_text[:64])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Zdw6i4F8glpp"
   },
   "source": [
    "Utility functions to map characters to vocabulary IDs and back."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6276,
     "status": "ok",
     "timestamp": 1445965583249,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "gAL1EECXeZsD",
    "outputId": "88fc9032-feb9-45ff-a9a0-a26759cc1f2e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unexpected character: ï\n",
      "1 26 0 0\n",
      "a z  \n"
     ]
    }
   ],
   "source": [
    "vocabulary_size = len(string.ascii_lowercase) + 1 # [a-z] + ' '\n",
    "first_letter = ord(string.ascii_lowercase[0])\n",
    "\n",
    "def char2id(char):\n",
    "    if char in string.ascii_lowercase:\n",
    "        return ord(char) - first_letter + 1\n",
    "    elif char == ' ':\n",
    "        return 0\n",
    "    else:\n",
    "        print('Unexpected character: %s' % char)\n",
    "        return 0\n",
    "\n",
    "def id2char(dictid):\n",
    "    if dictid > 0:\n",
    "        return chr(dictid + first_letter - 1)\n",
    "    else:\n",
    "        return ' '\n",
    "\n",
    "print(char2id('a'), char2id('z'), char2id(' '), char2id('ï'))\n",
    "print(id2char(1), id2char(26), id2char(0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lFwoyygOmWsL"
   },
   "source": [
    "Function to generate a training batch for the LSTM model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6473,
     "status": "ok",
     "timestamp": 1445965583467,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "d9wMtjy5hCj9",
    "outputId": "3dd79c80-454a-4be0-8b71-4a4a357b3367"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ons anarchi', 'when milita', 'lleria arch', ' abbeys and', 'married urr', 'hel and ric', 'y and litur', 'ay opened f', 'tion from t', 'migration t', 'new york ot', 'he boeing s', 'e listed wi', 'eber has pr', 'o be made t', 'yer who rec', 'ore signifi', 'a fierce cr', ' two six ei', 'aristotle s', 'ity can be ', ' and intrac', 'tion of the', 'dy to pass ', 'f certain d', 'at it will ', 'e convince ', 'ent told hi', 'ampaign and', 'rver side s', 'ious texts ', 'o capitaliz', 'a duplicate', 'gh ann es d', 'ine january', 'ross zero t', 'cal theorie', 'ast instanc', ' dimensiona', 'most holy m', 't s support', 'u is still ', 'e oscillati', 'o eight sub', 'of italy la', 's the tower', 'klahoma pre', 'erprise lin', 'ws becomes ', 'et in a naz', 'the fabian ', 'etchy to re', ' sharman ne', 'ised empero', 'ting in pol', 'd neo latin', 'th risky ri', 'encyclopedi', 'fense the a', 'duating fro', 'treet grid ', 'ations more', 'appeal of d', 'si have mad']\n",
      "['ists advoca', 'ary governm', 'hes nationa', 'd monasteri', 'raca prince', 'chard baer ', 'rgical lang', 'for passeng', 'the nationa', 'took place ', 'ther well k', 'seven six s', 'ith a gloss', 'robably bee', 'to recogniz', 'ceived the ', 'icant than ', 'ritic of th', 'ight in sig', 's uncaused ', ' lost as in', 'cellular ic', 'e size of t', ' him a stic', 'drugs confu', ' take to co', ' the priest', 'im to name ', 'd barred at', 'standard fo', ' such as es', 'ze on the g', 'e of the or', 'd hiver one', 'y eight mar', 'the lead ch', 'es classica', 'ce the non ', 'al analysis', 'mormons bel', 't or at lea', ' disagreed ', 'ing system ', 'btypes base', 'anguages th', 'r commissio', 'ess one nin', 'nux suse li', ' the first ', 'zi concentr', ' society ne', 'elatively s', 'etworks sha', 'or hirohito', 'litical ini', 'n most of t', 'iskerdoo ri', 'ic overview', 'air compone', 'om acnm acc', ' centerline', 'e than any ', 'devotional ', 'de such dev']\n",
      "[' a']\n",
      "['an']\n"
     ]
    }
   ],
   "source": [
    "batch_size=64\n",
    "num_unrollings=10\n",
    "\n",
    "class BatchGenerator(object):\n",
    "    def __init__(self, text, batch_size, num_unrollings):\n",
    "        self._text = text\n",
    "        self._text_size = len(text)\n",
    "        self._batch_size = batch_size\n",
    "        self._num_unrollings = num_unrollings\n",
    "        segment = self._text_size // batch_size\n",
    "        self._cursor = [ offset * segment for offset in range(batch_size)]\n",
    "        self._last_batch = self._next_batch()\n",
    "  \n",
    "    def _next_batch(self):\n",
    "        \"\"\"Generate a single batch from the current cursor position in the data.\"\"\"\n",
    "        batch = np.zeros(shape=(self._batch_size, vocabulary_size), dtype=np.float)\n",
    "        for b in range(self._batch_size):\n",
    "            batch[b, char2id(self._text[self._cursor[b]])] = 1.0\n",
    "            self._cursor[b] = (self._cursor[b] + 1) % self._text_size\n",
    "        return batch\n",
    "  \n",
    "    def next(self):\n",
    "        \"\"\"Generate the next array of batches from the data. The array consists of\n",
    "        the last batch of the previous array, followed by num_unrollings new ones.\n",
    "        \"\"\"\n",
    "        batches = [self._last_batch]\n",
    "        for step in range(self._num_unrollings):\n",
    "            batches.append(self._next_batch())\n",
    "        self._last_batch = batches[-1]\n",
    "        return batches\n",
    "\n",
    "def characters(probabilities):\n",
    "    \"\"\"Turn a 1-hot encoding or a probability distribution over the possible\n",
    "    characters back into its (most likely) character representation.\"\"\"\n",
    "    return [id2char(c) for c in np.argmax(probabilities, 1)]\n",
    "\n",
    "def batches2string(batches):\n",
    "    \"\"\"Convert a sequence of batches back into their (most likely) string\n",
    "    representation.\"\"\"\n",
    "    s = [''] * batches[0].shape[0]\n",
    "    for b in batches:\n",
    "        s = [''.join(x) for x in zip(s, characters(b))]\n",
    "    return s\n",
    "\n",
    "train_batches = BatchGenerator(train_text, batch_size, num_unrollings)\n",
    "valid_batches = BatchGenerator(valid_text, 1, 1)\n",
    "\n",
    "print(batches2string(train_batches.next()))\n",
    "print(batches2string(train_batches.next()))\n",
    "print(batches2string(valid_batches.next()))\n",
    "print(batches2string(valid_batches.next()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "KyVd8FxT5QBc"
   },
   "outputs": [],
   "source": [
    "def logprob(predictions, labels):\n",
    "    \"\"\"Log-probability of the true labels in a predicted batch.\"\"\"\n",
    "    predictions[predictions < 1e-10] = 1e-10\n",
    "    return np.sum(np.multiply(labels, -np.log(predictions))) / labels.shape[0]\n",
    "\n",
    "def sample_distribution(distribution):\n",
    "    \"\"\"Sample one element from a distribution assumed to be an array of normalized\n",
    "    probabilities.\n",
    "    \"\"\"\n",
    "    r = random.uniform(0, 1)\n",
    "    s = 0\n",
    "    for i in range(len(distribution)):\n",
    "        s += distribution[i]\n",
    "        if s >= r:\n",
    "            return i\n",
    "    return len(distribution) - 1\n",
    "\n",
    "def sample(prediction):\n",
    "    \"\"\"Turn a (column) prediction into 1-hot encoded samples.\"\"\"\n",
    "    p = np.zeros(shape=[1, vocabulary_size], dtype=np.float)\n",
    "    p[0, sample_distribution(prediction[0])] = 1.0\n",
    "    return p\n",
    "\n",
    "def random_distribution():\n",
    "    \"\"\"Generate a random column of probabilities.\"\"\"\n",
    "    b = np.random.uniform(0.0, 1.0, size=[1, vocabulary_size])\n",
    "    return b/np.sum(b, 1)[:,None]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "K8f67YXaDr4C"
   },
   "source": [
    "Simple LSTM Model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "Q5rxZK6RDuGe"
   },
   "outputs": [],
   "source": [
    "num_nodes = 64\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "  \n",
    "    # Parameters:\n",
    "    # Input gate: input, previous output, and bias.\n",
    "    ix = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n",
    "    im = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    ib = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Forget gate: input, previous output, and bias.\n",
    "    fx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n",
    "    fm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    fb = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Memory cell: input, state and bias.                             \n",
    "    cx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n",
    "    cm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    cb = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Output gate: input, previous output, and bias.\n",
    "    ox = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n",
    "    om = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    ob = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Variables saving state across unrollings.\n",
    "    saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "    saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "    # Classifier weights and biases.\n",
    "    w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size], -0.1, 0.1))\n",
    "    b = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "  \n",
    "  # Definition of the cell computation.\n",
    "    def lstm_cell(i, o, state):\n",
    "        \"\"\"Create a LSTM cell. See e.g.: http://arxiv.org/pdf/1402.1128v1.pdf\n",
    "        Note that in this formulation, we omit the various connections between the\n",
    "        previous state and the gates.\"\"\"\n",
    "        input_gate = tf.sigmoid(tf.matmul(i, ix) + tf.matmul(o, im) + ib)\n",
    "        forget_gate = tf.sigmoid(tf.matmul(i, fx) + tf.matmul(o, fm) + fb)\n",
    "        update = tf.matmul(i, cx) + tf.matmul(o, cm) + cb\n",
    "        state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "        output_gate = tf.sigmoid(tf.matmul(i, ox) + tf.matmul(o, om) + ob)\n",
    "        return output_gate * tf.tanh(state), state\n",
    "\n",
    "  # Input data.\n",
    "    train_data = list()\n",
    "    for _ in range(num_unrollings + 1):\n",
    "        train_data.append(\n",
    "            tf.placeholder(tf.float32, shape=[batch_size,vocabulary_size]))\n",
    "    train_inputs = train_data[:num_unrollings]\n",
    "    train_labels = train_data[1:]  # labels are inputs shifted by one time step.\n",
    "\n",
    "    # Unrolled LSTM loop.\n",
    "    outputs = list()\n",
    "    output = saved_output\n",
    "    state = saved_state\n",
    "    for i in train_inputs:\n",
    "        output, state = lstm_cell(i, output, state)\n",
    "        outputs.append(output)\n",
    "\n",
    "    # State saving across unrollings.\n",
    "    with tf.control_dependencies([saved_output.assign(output),\n",
    "                                  saved_state.assign(state)]):\n",
    "      # Classifier.\n",
    "        logits = tf.nn.xw_plus_b(tf.concat(outputs, 0), w, b)\n",
    "        loss = tf.reduce_mean(\n",
    "            tf.nn.softmax_cross_entropy_with_logits(\n",
    "            labels=tf.concat(train_labels, 0), logits=logits))\n",
    "\n",
    "    # Optimizer.\n",
    "    global_step = tf.Variable(0)\n",
    "    learning_rate = tf.train.exponential_decay(\n",
    "        10.0, global_step, 5000, 0.1, staircase=True)\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "    gradients, _ = tf.clip_by_global_norm(gradients, 1.25)\n",
    "    optimizer = optimizer.apply_gradients(\n",
    "        zip(gradients, v), global_step=global_step)\n",
    "\n",
    "    # Predictions.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    \n",
    "    # Sampling and validation eval: batch 1, no unrolling.\n",
    "    sample_input = tf.placeholder(tf.float32, shape=[1, vocabulary_size])\n",
    "    saved_sample_output = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    saved_sample_state = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    reset_sample_state = tf.group(\n",
    "        saved_sample_output.assign(tf.zeros([1, num_nodes])),\n",
    "        saved_sample_state.assign(tf.zeros([1, num_nodes])))\n",
    "    sample_output, sample_state = lstm_cell(\n",
    "        sample_input, saved_sample_output, saved_sample_state)\n",
    "    with tf.control_dependencies([saved_sample_output.assign(sample_output),\n",
    "                                  saved_sample_state.assign(sample_state)]):\n",
    "        sample_prediction = tf.nn.softmax(tf.nn.xw_plus_b(sample_output, w, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 41
      },
      {
       "item_id": 80
      },
      {
       "item_id": 126
      },
      {
       "item_id": 144
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 199909,
     "status": "ok",
     "timestamp": 1445965877333,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "RD9zQCZTEaEm",
    "outputId": "5e868466-2532-4545-ce35-b403cf5d9de6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Average loss at step 0: 3.290585 learning rate: 10.000000\n",
      "Minibatch perplexity: 26.86\n",
      "================================================================================\n",
      "x  eaonrntufesxwzh ybec fhq dvosvecr niia i fkly smxftrkodlatk usfqfeh iegpc vba\n",
      "vsiaettlib  oolrshcmfadndde pmzwevkpe ezacno r wyt kty qsa  katrqhxaxesq bddcpvp\n",
      "nk k bfxrvlronnvrfpdurecoycm eoxglrth teugdf  giokindvr psouxepni snzevceoettwlx\n",
      "slhdeai e co agnht lekxdzyglr if flacdnhaxxi bycuzzkvh ffysejd  prmmcgazjesn pea\n",
      "b sqeehvjhu sd sfd imfqxa ytvpbifqeuiysiv avsdlqw n raqk cebxmftsazzomhglaogh  f\n",
      "================================================================================\n",
      "Validation set perplexity: 20.22\n",
      "Average loss at step 100: 2.629065 learning rate: 10.000000\n",
      "Minibatch perplexity: 10.85\n",
      "Validation set perplexity: 10.32\n",
      "Average loss at step 200: 2.253166 learning rate: 10.000000\n",
      "Minibatch perplexity: 8.68\n",
      "Validation set perplexity: 8.51\n",
      "Average loss at step 300: 2.099618 learning rate: 10.000000\n",
      "Minibatch perplexity: 7.33\n",
      "Validation set perplexity: 8.14\n",
      "Average loss at step 400: 1.998662 learning rate: 10.000000\n",
      "Minibatch perplexity: 7.63\n",
      "Validation set perplexity: 7.96\n",
      "Average loss at step 500: 1.933205 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.32\n",
      "Validation set perplexity: 7.01\n",
      "Average loss at step 600: 1.907334 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.01\n",
      "Validation set perplexity: 6.90\n",
      "Average loss at step 700: 1.861379 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.40\n",
      "Validation set perplexity: 6.75\n",
      "Average loss at step 800: 1.817920 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.89\n",
      "Validation set perplexity: 6.48\n",
      "Average loss at step 900: 1.825171 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.93\n",
      "Validation set perplexity: 6.27\n",
      "Average loss at step 1000: 1.825903 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.77\n",
      "================================================================================\n",
      "eater be they the peranition moteter at secosus hepatre to goes the yemalle enco\n",
      "ming reconmed hopore a tocnguted and sessiel dital consmic as procof mease amie \n",
      "rame have aurzla of hinds be hewer of beharmia or his he deveemation invomments \n",
      "ed trogn fore wher to the cervic by apparcy the a lipse to the phore of the subr\n",
      "ners compern or areira t couply and were distrekns he fild from lite is reanatio\n",
      "================================================================================\n",
      "Validation set perplexity: 6.10\n",
      "Average loss at step 1100: 1.774959 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.56\n",
      "Validation set perplexity: 6.02\n",
      "Average loss at step 1200: 1.754435 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.04\n",
      "Validation set perplexity: 5.77\n",
      "Average loss at step 1300: 1.735711 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.64\n",
      "Validation set perplexity: 5.79\n",
      "Average loss at step 1400: 1.747512 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.90\n",
      "Validation set perplexity: 5.56\n",
      "Average loss at step 1500: 1.740135 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.84\n",
      "Validation set perplexity: 5.48\n",
      "Average loss at step 1600: 1.750340 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.64\n",
      "Validation set perplexity: 5.51\n",
      "Average loss at step 1700: 1.714334 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.74\n",
      "Validation set perplexity: 5.49\n",
      "Average loss at step 1800: 1.675516 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.57\n",
      "Validation set perplexity: 5.35\n",
      "Average loss at step 1900: 1.650603 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.09\n",
      "Validation set perplexity: 5.32\n",
      "Average loss at step 2000: 1.697359 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.67\n",
      "================================================================================\n",
      "on stsings often from three speed teched queds nootnan lardion or shologicialing\n",
      "f booth was formermaina issengle p lawdy of someton a the periosly four magranti\n",
      "y to canhuacts vipy five in vitwald blystion d eling indine and nassing worksion\n",
      "u mike one four ad are call indiston worrs hims blow seliess oridines risagrang \n",
      "s fron routhork vecferences at somech he wive four the the with de resudientung \n",
      "================================================================================\n",
      "Validation set perplexity: 5.31\n",
      "Average loss at step 2100: 1.687174 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.18\n",
      "Validation set perplexity: 4.99\n",
      "Average loss at step 2200: 1.685634 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.29\n",
      "Validation set perplexity: 4.98\n",
      "Average loss at step 2300: 1.644285 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.01\n",
      "Validation set perplexity: 4.99\n",
      "Average loss at step 2400: 1.663999 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.08\n",
      "Validation set perplexity: 4.85\n",
      "Average loss at step 2500: 1.681106 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.33\n",
      "Validation set perplexity: 4.67\n",
      "Average loss at step 2600: 1.656962 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.85\n",
      "Validation set perplexity: 4.63\n",
      "Average loss at step 2700: 1.661218 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.60\n",
      "Validation set perplexity: 4.70\n",
      "Average loss at step 2800: 1.651475 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.52\n",
      "Validation set perplexity: 4.67\n",
      "Average loss at step 2900: 1.653388 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.60\n",
      "Validation set perplexity: 4.72\n",
      "Average loss at step 3000: 1.652606 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.02\n",
      "================================================================================\n",
      "jo be hundy hame is besconders the many of could requenslation relier eftreatice\n",
      "list milically of union is coulf pardical scape baving as folawi madge many main\n",
      "zan transsel one nine the shoot is cieffate mal arain englor in those kargui two\n",
      "wo s and actors of films is reath africus one nine the adilty mepual withal daff\n",
      "kanns through up some aljoh supricabers all will mothemed subseversiver collecte\n",
      "================================================================================\n",
      "Validation set perplexity: 4.71\n",
      "Average loss at step 3100: 1.632426 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.84\n",
      "Validation set perplexity: 4.64\n",
      "Average loss at step 3200: 1.645965 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.64\n",
      "Validation set perplexity: 4.63\n",
      "Average loss at step 3300: 1.637245 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.03\n",
      "Validation set perplexity: 4.57\n",
      "Average loss at step 3400: 1.671188 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.47\n",
      "Validation set perplexity: 4.58\n",
      "Average loss at step 3500: 1.658136 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.74\n",
      "Validation set perplexity: 4.70\n",
      "Average loss at step 3600: 1.671043 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.45\n",
      "Validation set perplexity: 4.53\n",
      "Average loss at step 3700: 1.647362 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.09\n",
      "Validation set perplexity: 4.58\n",
      "Average loss at step 3800: 1.646706 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.50\n",
      "Validation set perplexity: 4.73\n",
      "Average loss at step 3900: 1.638408 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.28\n",
      "Validation set perplexity: 4.61\n",
      "Average loss at step 4000: 1.652721 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.64\n",
      "================================================================================\n",
      " subill weldy the was the priscips a singates to numbel pater held by bloxt home\n",
      "ed by eight the life american to back man eight six abcrepapler and aundeures ot\n",
      "y remmmer or orstendown work sebotded itself moving singr libe to by primbings a\n",
      "forb one nine nine four one the other the deuts proferivs of early or movern suc\n",
      "hen regations chootyed genothe it in the day being the hoded of rome fourmoned c\n",
      "================================================================================\n",
      "Validation set perplexity: 4.61\n",
      "Average loss at step 4100: 1.634949 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.35\n",
      "Validation set perplexity: 4.70\n",
      "Average loss at step 4200: 1.638077 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.21\n",
      "Validation set perplexity: 4.59\n",
      "Average loss at step 4300: 1.610286 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.99\n",
      "Validation set perplexity: 4.49\n",
      "Average loss at step 4400: 1.611063 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.85\n",
      "Validation set perplexity: 4.41\n",
      "Average loss at step 4500: 1.614091 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.19\n",
      "Validation set perplexity: 4.59\n",
      "Average loss at step 4600: 1.614358 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.96\n",
      "Validation set perplexity: 4.60\n",
      "Average loss at step 4700: 1.624051 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.15\n",
      "Validation set perplexity: 4.46\n",
      "Average loss at step 4800: 1.630532 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.41\n",
      "Validation set perplexity: 4.49\n",
      "Average loss at step 4900: 1.633653 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.15\n",
      "Validation set perplexity: 4.53\n",
      "Average loss at step 5000: 1.607785 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.50\n",
      "================================================================================\n",
      "za k one the aspreeded comat on juffical defide basher regens disconsed invasion\n",
      "oca breew a manked it the tara of s cometions of sur prebal godder a fow revesul\n",
      "ch of a is malk ventifier and influence earen plant is parted born value eight z\n",
      "aw divines based kims maint all centarye the use concess in devicaed veryornitar\n",
      " westerfient jolus more perporting holand in the tonial swared is one and plant \n",
      "================================================================================\n",
      "Validation set perplexity: 4.53\n",
      "Average loss at step 5100: 1.603275 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.75\n",
      "Validation set perplexity: 4.36\n",
      "Average loss at step 5200: 1.590877 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.66\n",
      "Validation set perplexity: 4.33\n",
      "Average loss at step 5300: 1.577238 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.72\n",
      "Validation set perplexity: 4.31\n",
      "Average loss at step 5400: 1.577694 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.16\n",
      "Validation set perplexity: 4.30\n",
      "Average loss at step 5500: 1.565557 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.82\n",
      "Validation set perplexity: 4.25\n",
      "Average loss at step 5600: 1.580010 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.74\n",
      "Validation set perplexity: 4.27\n",
      "Average loss at step 5700: 1.569273 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.44\n",
      "Validation set perplexity: 4.28\n",
      "Average loss at step 5800: 1.583154 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.87\n",
      "Validation set perplexity: 4.29\n",
      "Average loss at step 5900: 1.574224 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.08\n",
      "Validation set perplexity: 4.29\n",
      "Average loss at step 6000: 1.544637 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.99\n",
      "================================================================================\n",
      "k letalies electors in considerations in the includes hame of nostratuage as ain\n",
      "y p treasanips of oratificoned standed abomev the rame zero in x nobt in a dring\n",
      "rade bestoy his s the ditinecrammer accolemine the six the nine the daintained o\n",
      "y oven laws a to grearges and feltball lea econdism of sipen two two two and con\n",
      "y in the meanotoy persook was erectment off the wow both also zoundal histore fr\n",
      "================================================================================\n",
      "Validation set perplexity: 4.29\n",
      "Average loss at step 6100: 1.563064 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.19\n",
      "Validation set perplexity: 4.26\n",
      "Average loss at step 6200: 1.532012 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.85\n",
      "Validation set perplexity: 4.28\n",
      "Average loss at step 6300: 1.541386 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.90\n",
      "Validation set perplexity: 4.26\n",
      "Average loss at step 6400: 1.541448 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.48\n",
      "Validation set perplexity: 4.25\n",
      "Average loss at step 6500: 1.557782 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.64\n",
      "Validation set perplexity: 4.26\n",
      "Average loss at step 6600: 1.593854 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.86\n",
      "Validation set perplexity: 4.25\n",
      "Average loss at step 6700: 1.579227 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.15\n",
      "Validation set perplexity: 4.27\n",
      "Average loss at step 6800: 1.605029 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.74\n",
      "Validation set perplexity: 4.25\n",
      "Average loss at step 6900: 1.581836 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.79\n",
      "Validation set perplexity: 4.27\n",
      "Average loss at step 7000: 1.575463 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.96\n",
      "================================================================================\n",
      "y jakawilium americo world world the inigutic presuded of the chneranian politas\n",
      "gen bolo ering education by wrattration aigning as stuls forking is graves of th\n",
      "ha batestrational later actinues lext of the mindon esprovectoric are leggled jo\n",
      "war thell support of being hangurete handrage has buclide is warez the react of \n",
      "tigualed relative is didef to park were fionazize on the g istralb six constanti\n",
      "================================================================================\n",
      "Validation set perplexity: 4.26\n"
     ]
    }
   ],
   "source": [
    "num_steps = 7001\n",
    "summary_frequency = 100\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print('Initialized')\n",
    "    mean_loss = 0\n",
    "    for step in range(num_steps):\n",
    "        batches = train_batches.next()\n",
    "        feed_dict = dict()\n",
    "        for i in range(num_unrollings + 1):\n",
    "            feed_dict[train_data[i]] = batches[i]\n",
    "        _, l, predictions, lr = session.run(\n",
    "            [optimizer, loss, train_prediction, learning_rate], feed_dict=feed_dict)\n",
    "        mean_loss += l\n",
    "        if step % summary_frequency == 0:\n",
    "            if step > 0:\n",
    "                mean_loss = mean_loss / summary_frequency\n",
    "            # The mean loss is an estimate of the loss over the last few batches.\n",
    "            print(\n",
    "                'Average loss at step %d: %f learning rate: %f' % (step, mean_loss, lr))\n",
    "            mean_loss = 0\n",
    "            labels = np.concatenate(list(batches)[1:])\n",
    "            print('Minibatch perplexity: %.2f' % float(\n",
    "                np.exp(logprob(predictions, labels))))\n",
    "            if step % (summary_frequency * 10) == 0:\n",
    "                # Generate some samples.\n",
    "                print('=' * 80)\n",
    "                for _ in range(5):\n",
    "                    feed = sample(random_distribution())\n",
    "                    sentence = characters(feed)[0]\n",
    "                    reset_sample_state.run()\n",
    "                    for _ in range(79):\n",
    "                        prediction = sample_prediction.eval({sample_input: feed})\n",
    "                        feed = sample(prediction)\n",
    "                        sentence += characters(feed)[0]\n",
    "                    print(sentence)\n",
    "                print('=' * 80)\n",
    "            # Measure validation set perplexity.\n",
    "            reset_sample_state.run()\n",
    "            valid_logprob = 0\n",
    "            for _ in range(valid_size):\n",
    "                b = valid_batches.next()\n",
    "                predictions = sample_prediction.eval({sample_input: b[0]})\n",
    "                valid_logprob = valid_logprob + logprob(predictions, b[1])\n",
    "            print('Validation set perplexity: %.2f' % float(np.exp(\n",
    "                valid_logprob / valid_size)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pl4vtmFfa5nn"
   },
   "source": [
    "---\n",
    "Problem 1\n",
    "---------\n",
    "\n",
    "You might have noticed that the definition of the LSTM cell involves 4 matrix multiplications with the input, and 4 matrix multiplications with the output. Simplify the expression by using a single matrix multiply for each, and variables that are 4 times larger.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_nodes = 64\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "  \n",
    "    # Parameters:\n",
    "    # Input gate: input, previous output, and bias.\n",
    "    ix = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n",
    "    im = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    ib = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Forget gate: input, previous output, and bias.\n",
    "    fx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n",
    "    fm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    fb = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Memory cell: input, state and bias.                             \n",
    "    cx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n",
    "    cm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    cb = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Output gate: input, previous output, and bias.\n",
    "    ox = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n",
    "    om = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    ob = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Concatenation of the parameters\n",
    "    sx = tf.concat([ix, fx, cx, ox], axis=1)\n",
    "    sm = tf.concat([im, fm, cm, om], axis=1)\n",
    "    sb = tf.concat([ib, fb, cb, ob], axis= 1)\n",
    "    # Variables saving state across unrollings.\n",
    "    saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "    saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "    # Classifier weights and biases.\n",
    "    w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size], -0.1, 0.1))\n",
    "    b = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "  \n",
    "  # Definition of the cell computation.\n",
    "    def lstm_cell(i, o, state):\n",
    "        \"\"\"Create a LSTM cell. See e.g.: http://arxiv.org/pdf/1402.1128v1.pdf\n",
    "        Note that in this formulation, we omit the various connections between the\n",
    "        previous state and the gates.\"\"\"\n",
    "        mat = tf.matmul(i, sx) + tf.matmul(o, sm) + sb\n",
    "        i, f, c, o = tf.split(mat, 4, axis=1)\n",
    "        input_gate = tf.sigmoid(i)\n",
    "        forget_gate = tf.sigmoid(f)\n",
    "        update = c\n",
    "        state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "        output_gate = tf.sigmoid(o)\n",
    "        return output_gate * tf.tanh(state), state\n",
    "\n",
    "  # Input data.\n",
    "    train_data = list()\n",
    "    for _ in range(num_unrollings + 1):\n",
    "        train_data.append(\n",
    "            tf.placeholder(tf.float32, shape=[batch_size,vocabulary_size]))\n",
    "    train_inputs = train_data[:num_unrollings]\n",
    "    train_labels = train_data[1:]  # labels are inputs shifted by one time step.\n",
    "\n",
    "    # Unrolled LSTM loop.\n",
    "    outputs = list()\n",
    "    output = saved_output\n",
    "    state = saved_state\n",
    "    for i in train_inputs:\n",
    "        output, state = lstm_cell(i, output, state)\n",
    "        outputs.append(output)\n",
    "\n",
    "    # State saving across unrollings.\n",
    "    with tf.control_dependencies([saved_output.assign(output),\n",
    "                                  saved_state.assign(state)]):\n",
    "      # Classifier.\n",
    "        logits = tf.nn.xw_plus_b(tf.concat(outputs, 0), w, b)\n",
    "        loss = tf.reduce_mean(\n",
    "            tf.nn.softmax_cross_entropy_with_logits(\n",
    "            labels=tf.concat(train_labels, 0), logits=logits))\n",
    "\n",
    "    # Optimizer.\n",
    "    global_step = tf.Variable(0)\n",
    "    learning_rate = tf.train.exponential_decay(\n",
    "        10.0, global_step, 5000, 0.1, staircase=True)\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "    gradients, _ = tf.clip_by_global_norm(gradients, 1.25)\n",
    "    optimizer = optimizer.apply_gradients(\n",
    "        zip(gradients, v), global_step=global_step)\n",
    "\n",
    "    # Predictions.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    \n",
    "    # Sampling and validation eval: batch 1, no unrolling.\n",
    "    sample_input = tf.placeholder(tf.float32, shape=[1, vocabulary_size])\n",
    "    saved_sample_output = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    saved_sample_state = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    reset_sample_state = tf.group(\n",
    "        saved_sample_output.assign(tf.zeros([1, num_nodes])),\n",
    "        saved_sample_state.assign(tf.zeros([1, num_nodes])))\n",
    "    sample_output, sample_state = lstm_cell(\n",
    "        sample_input, saved_sample_output, saved_sample_state)\n",
    "    with tf.control_dependencies([saved_sample_output.assign(sample_output),\n",
    "                                  saved_sample_state.assign(sample_state)]):\n",
    "        sample_prediction = tf.nn.softmax(tf.nn.xw_plus_b(sample_output, w, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Average loss at step 0: 3.295835 learning rate: 10.000000\n",
      "Minibatch perplexity: 27.00\n",
      "================================================================================\n",
      " xe brrv bgjz gw sror  hsbt zbrwxjnktbelasc agcnrexet ng rzxnmkk m eouae sracjxs\n",
      "hroqssj jx nrretsr j vnbrhb clatuftirwrcbealt uackeachjcweo h ri rrifymupebvd   \n",
      "s   s xekve    tfqweiest  n tfemxrfetwbg nze to ntr uu drl ttjphyqpv ukaco tz et\n",
      "vqgiwcrl  octpksn  iingf m tcej ebzk  piuhwxf  xtn kjitht  sxrjrhhzzip h  iopbee\n",
      "bml b jegtreheemetsjenbgfe waxcu xkexc  tihtdgqnchgg  memz tyitcsuftmnrl lknuivd\n",
      "================================================================================\n",
      "Validation set perplexity: 20.17\n",
      "Average loss at step 100: 2.591559 learning rate: 10.000000\n",
      "Minibatch perplexity: 11.37\n",
      "Validation set perplexity: 11.96\n",
      "Average loss at step 200: 2.251106 learning rate: 10.000000\n",
      "Minibatch perplexity: 8.39\n",
      "Validation set perplexity: 8.81\n",
      "Average loss at step 300: 2.089914 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.39\n",
      "Validation set perplexity: 8.00\n",
      "Average loss at step 400: 2.032637 learning rate: 10.000000\n",
      "Minibatch perplexity: 7.96\n",
      "Validation set perplexity: 7.80\n",
      "Average loss at step 500: 1.976660 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.32\n",
      "Validation set perplexity: 7.09\n",
      "Average loss at step 600: 1.893432 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.29\n",
      "Validation set perplexity: 6.95\n",
      "Average loss at step 700: 1.871048 learning rate: 10.000000\n",
      "Minibatch perplexity: 7.02\n",
      "Validation set perplexity: 6.40\n",
      "Average loss at step 800: 1.867056 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.94\n",
      "Validation set perplexity: 6.49\n",
      "Average loss at step 900: 1.846522 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.14\n",
      "Validation set perplexity: 6.18\n",
      "Average loss at step 1000: 1.847196 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.42\n",
      "================================================================================\n",
      "d and some of the cleral mid flenging upediberters victoky seipol d c altrubiky \n",
      "canery randid geate thrandveed bromughick ockam lod duh a nubuents of agroo micn\n",
      "nettwist a sabliatal debins of in frist in pulsifre iss edtworyary forked blidis\n",
      "d one two seven nine ormean mehnion c m the denaseve pervel in a by de open five\n",
      "x macagl vections be consiqies cospibically rungrings actary velsan pises sei ca\n",
      "================================================================================\n",
      "Validation set perplexity: 5.99\n",
      "Average loss at step 1100: 1.804402 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.31\n",
      "Validation set perplexity: 6.07\n",
      "Average loss at step 1200: 1.772042 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.16\n",
      "Validation set perplexity: 5.92\n",
      "Average loss at step 1300: 1.763541 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.79\n",
      "Validation set perplexity: 5.69\n",
      "Average loss at step 1400: 1.766996 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.02\n",
      "Validation set perplexity: 5.66\n",
      "Average loss at step 1500: 1.750553 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.70\n",
      "Validation set perplexity: 5.37\n",
      "Average loss at step 1600: 1.735711 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.50\n",
      "Validation set perplexity: 5.51\n",
      "Average loss at step 1700: 1.719684 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.24\n",
      "Validation set perplexity: 5.32\n",
      "Average loss at step 1800: 1.693331 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.13\n",
      "Validation set perplexity: 5.24\n",
      "Average loss at step 1900: 1.699439 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.16\n",
      "Validation set perplexity: 5.23\n",
      "Average loss at step 2000: 1.682058 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.15\n",
      "================================================================================\n",
      "al tove then laliant so out the plice agrish and busic reseracion of mase o resa\n",
      "ell begral is manice three ynabired will wrend six nine seven evende zext with n\n",
      "mation westris lessess two six one nine page veatials of and dro clusts in liad \n",
      "bert not hard four earnn hit of finely three so is cive des mary w of centric ex\n",
      "que proves and patting aight was rele deney in two zeized exple the rox file one\n",
      "================================================================================\n",
      "Validation set perplexity: 5.27\n",
      "Average loss at step 2100: 1.686465 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.02\n",
      "Validation set perplexity: 5.15\n",
      "Average loss at step 2200: 1.707753 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.05\n",
      "Validation set perplexity: 5.07\n",
      "Average loss at step 2300: 1.708916 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.35\n",
      "Validation set perplexity: 5.03\n",
      "Average loss at step 2400: 1.687494 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.75\n",
      "Validation set perplexity: 5.01\n",
      "Average loss at step 2500: 1.690966 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.82\n",
      "Validation set perplexity: 5.13\n",
      "Average loss at step 2600: 1.678479 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.18\n",
      "Validation set perplexity: 4.95\n",
      "Average loss at step 2700: 1.688611 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.11\n",
      "Validation set perplexity: 5.06\n",
      "Average loss at step 2800: 1.686252 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.42\n",
      "Validation set perplexity: 5.18\n",
      "Average loss at step 2900: 1.672930 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.20\n",
      "Validation set perplexity: 5.07\n",
      "Average loss at step 3000: 1.686901 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.92\n",
      "================================================================================\n",
      "ly the wirhmecises the french form exam two six a dect hot regernal gymes with o\n",
      "y futsic the shildoning soved altopre and ribring havic of keent op the bunting \n",
      "perss plifacting pergen tempero awemolime to of to compressian d one one fox bet\n",
      "er itvarties to catib clyaut uariam e by to the norriess kat that ally nen then \n",
      "udes as in one revo kestes of the of so ranisa in the stsres aspomicazors to oth\n",
      "================================================================================\n",
      "Validation set perplexity: 4.99\n",
      "Average loss at step 3100: 1.652220 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.28\n",
      "Validation set perplexity: 4.99\n",
      "Average loss at step 3200: 1.636757 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.48\n",
      "Validation set perplexity: 4.86\n",
      "Average loss at step 3300: 1.646684 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.06\n",
      "Validation set perplexity: 4.84\n",
      "Average loss at step 3400: 1.632461 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.30\n",
      "Validation set perplexity: 4.79\n",
      "Average loss at step 3500: 1.671163 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.84\n",
      "Validation set perplexity: 4.83\n",
      "Average loss at step 3600: 1.653305 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.28\n",
      "Validation set perplexity: 4.74\n",
      "Average loss at step 3700: 1.655027 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.32\n",
      "Validation set perplexity: 4.79\n",
      "Average loss at step 3800: 1.662443 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.02\n",
      "Validation set perplexity: 4.81\n",
      "Average loss at step 3900: 1.655223 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.27\n",
      "Validation set perplexity: 4.76\n",
      "Average loss at step 4000: 1.644176 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.38\n",
      "================================================================================\n",
      "larggus rument like ispolee firth in pract by provorsone senerly abderwer counde\n",
      "x breath detion haggs fortictost several one six greenastan leads propobizative \n",
      " re sourcret finds and beworg jurosh gooff and final and all issobuty and had th\n",
      "a f teatber one of the adaper and explookss in zero six seven nine zero zero zer\n",
      "x was rudy the obst hemok present qamily provulacks an we nortings pommatition o\n",
      "================================================================================\n",
      "Validation set perplexity: 4.74\n",
      "Average loss at step 4100: 1.623351 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.67\n",
      "Validation set perplexity: 4.79\n",
      "Average loss at step 4200: 1.619033 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.16\n",
      "Validation set perplexity: 4.64\n",
      "Average loss at step 4300: 1.623180 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.67\n",
      "Validation set perplexity: 4.72\n",
      "Average loss at step 4400: 1.614602 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.33\n",
      "Validation set perplexity: 4.67\n",
      "Average loss at step 4500: 1.644630 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.38\n",
      "Validation set perplexity: 4.90\n",
      "Average loss at step 4600: 1.628305 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.46\n",
      "Validation set perplexity: 4.70\n",
      "Average loss at step 4700: 1.630348 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.94\n",
      "Validation set perplexity: 4.73\n",
      "Average loss at step 4800: 1.614402 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.72\n",
      "Validation set perplexity: 4.73\n",
      "Average loss at step 4900: 1.624987 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.15\n",
      "Validation set perplexity: 4.64\n",
      "Average loss at step 5000: 1.618304 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.86\n",
      "================================================================================\n",
      "um of as two four hom of them maders works at electrone reusers when a received \n",
      "verneatic in one nine seven only was fiert undergual louits of the a doring ts t\n",
      "wer more which a screadent s the law ha in ons two eigg one nine six new or the \n",
      "zed and min dimin of sea early a harried and source vanisirech than sense offica\n",
      "quauing but such im detall as one eight seven three zero one joke clild hal parm\n",
      "================================================================================\n",
      "Validation set perplexity: 4.77\n",
      "Average loss at step 5100: 1.592139 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.05\n",
      "Validation set perplexity: 4.65\n",
      "Average loss at step 5200: 1.598492 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.28\n",
      "Validation set perplexity: 4.62\n",
      "Average loss at step 5300: 1.599965 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.04\n",
      "Validation set perplexity: 4.60\n",
      "Average loss at step 5400: 1.599743 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.85\n",
      "Validation set perplexity: 4.55\n",
      "Average loss at step 5500: 1.597153 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.47\n",
      "Validation set perplexity: 4.54\n",
      "Average loss at step 5600: 1.561233 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.31\n",
      "Validation set perplexity: 4.49\n",
      "Average loss at step 5700: 1.580372 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.64\n",
      "Validation set perplexity: 4.48\n",
      "Average loss at step 5800: 1.602708 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.59\n",
      "Validation set perplexity: 4.50\n",
      "Average loss at step 5900: 1.585903 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.23\n",
      "Validation set perplexity: 4.53\n",
      "Average loss at step 6000: 1.583995 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.83\n",
      "================================================================================\n",
      "porth uastarcle and a much to purbemend that the jatas arbuw factus pervisfors a\n",
      "ond solitived on prepented a five eight one authorian many to suchinial gandania\n",
      "udions contentallys clased cwo the longer nottors argitared to explostiesneniqui\n",
      "s takac play pereens was some the viric allle provice of the french links iis pr\n",
      "jow finther point site an are an special usep stated upipa provorth such communa\n",
      "================================================================================\n",
      "Validation set perplexity: 4.48\n",
      "Average loss at step 6100: 1.580182 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.54\n",
      "Validation set perplexity: 4.51\n",
      "Average loss at step 6200: 1.590614 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.50\n",
      "Validation set perplexity: 4.53\n",
      "Average loss at step 6300: 1.587140 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.47\n",
      "Validation set perplexity: 4.55\n",
      "Average loss at step 6400: 1.574636 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.25\n",
      "Validation set perplexity: 4.54\n",
      "Average loss at step 6500: 1.562030 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.29\n",
      "Validation set perplexity: 4.56\n",
      "Average loss at step 6600: 1.602142 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.58\n",
      "Validation set perplexity: 4.55\n",
      "Average loss at step 6700: 1.571495 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.28\n",
      "Validation set perplexity: 4.53\n",
      "Average loss at step 6800: 1.577873 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.98\n",
      "Validation set perplexity: 4.55\n",
      "Average loss at step 6900: 1.574538 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.60\n",
      "Validation set perplexity: 4.50\n",
      "Average loss at step 7000: 1.589420 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.96\n",
      "================================================================================\n",
      "joy is majzay worldia sox wisher the pater are channce like player scrip from he\n",
      "entry s and the x similariality and the kistor of lecons tynis cambollityences o\n",
      "junable in the store of ecoggest in northelet climsions the yoy cheimply and pri\n",
      "de type beining cyce are liverpare this poscous characteria five authitg catalog\n",
      "nignorie of others irvee moyal us and debrence und often to warchood bost with a\n",
      "================================================================================\n",
      "Validation set perplexity: 4.49\n"
     ]
    }
   ],
   "source": [
    "num_steps = 7001\n",
    "summary_frequency = 100\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print('Initialized')\n",
    "    mean_loss = 0\n",
    "    for step in range(num_steps):\n",
    "        batches = train_batches.next()\n",
    "        feed_dict = dict()\n",
    "        for i in range(num_unrollings + 1):\n",
    "            feed_dict[train_data[i]] = batches[i]\n",
    "        _, l, predictions, lr = session.run(\n",
    "            [optimizer, loss, train_prediction, learning_rate], feed_dict=feed_dict)\n",
    "        mean_loss += l\n",
    "        if step % summary_frequency == 0:\n",
    "            if step > 0:\n",
    "                mean_loss = mean_loss / summary_frequency\n",
    "            # The mean loss is an estimate of the loss over the last few batches.\n",
    "            print(\n",
    "                'Average loss at step %d: %f learning rate: %f' % (step, mean_loss, lr))\n",
    "            mean_loss = 0\n",
    "            labels = np.concatenate(list(batches)[1:])\n",
    "            print('Minibatch perplexity: %.2f' % float(\n",
    "                np.exp(logprob(predictions, labels))))\n",
    "            if step % (summary_frequency * 10) == 0:\n",
    "                # Generate some samples.\n",
    "                print('=' * 80)\n",
    "                for _ in range(5):\n",
    "                    feed = sample(random_distribution())\n",
    "                    sentence = characters(feed)[0]\n",
    "                    reset_sample_state.run()\n",
    "                    for _ in range(79):\n",
    "                        prediction = sample_prediction.eval({sample_input: feed})\n",
    "                        feed = sample(prediction)\n",
    "                        sentence += characters(feed)[0]\n",
    "                    print(sentence)\n",
    "                print('=' * 80)\n",
    "            # Measure validation set perplexity.\n",
    "            reset_sample_state.run()\n",
    "            valid_logprob = 0\n",
    "            for _ in range(valid_size):\n",
    "                b = valid_batches.next()\n",
    "                predictions = sample_prediction.eval({sample_input: b[0]})\n",
    "                valid_logprob = valid_logprob + logprob(predictions, b[1])\n",
    "            print('Validation set perplexity: %.2f' % float(np.exp(\n",
    "                valid_logprob / valid_size)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4eErTCTybtph"
   },
   "source": [
    "---\n",
    "Problem 2\n",
    "---------\n",
    "\n",
    "We want to train a LSTM over bigrams, that is pairs of consecutive characters like 'ab' instead of single characters like 'a'. Since the number of possible bigrams is large, feeding them directly to the LSTM using 1-hot encodings will lead to a very sparse representation that is very wasteful computationally.\n",
    "\n",
    "a- Introduce an embedding lookup on the inputs, and feed the embeddings to the LSTM cell instead of the inputs themselves.\n",
    "\n",
    "b- Write a bigram-based LSTM, modeled on the character LSTM above.\n",
    "\n",
    "c- Introduce Dropout. For best practices on how to use Dropout in LSTMs, refer to this [article](http://arxiv.org/abs/1409.2329).\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a: embedding "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 64\n",
    "num_nodes = 64\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "  \n",
    "    # Parameters:\n",
    "    vocabulary_embeddings = tf.Variable(\n",
    "        tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))\n",
    "    # Input gate: input, previous output, and bias.\n",
    "    ix = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "    im = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    ib = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Forget gate: input, previous output, and bias.\n",
    "    fx = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "    fm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    fb = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Memory cell: input, state and bias.                             \n",
    "    cx = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "    cm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    cb = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Output gate: input, previous output, and bias.\n",
    "    ox = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "    om = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    ob = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Concatenation of the parameters\n",
    "    sx = tf.concat([ix, fx, cx, ox], axis=1)\n",
    "    sm = tf.concat([im, fm, cm, om], axis=1)\n",
    "    sb = tf.concat([ib, fb, cb, ob], axis=1)\n",
    "    # Variables saving state across unrollings.\n",
    "    saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "    saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "    # Classifier weights and biases.\n",
    "    w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size], -0.1, 0.1))\n",
    "    b = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "  \n",
    "\n",
    "    # Definition of the cell computation.\n",
    "    def lstm_cell(i, o, state):\n",
    "        \"\"\"Create a LSTM cell. See e.g.: http://arxiv.org/pdf/1402.1128v1.pdf\n",
    "        Note that in this formulation, we omit the various connections between the\n",
    "        previous state and the gates.\"\"\"\n",
    "        mat = tf.matmul(i, sx) + tf.matmul(o, sm) + sb\n",
    "        i, f, c, o = tf.split(mat, 4, axis=1)\n",
    "        input_gate = tf.sigmoid(i)\n",
    "        forget_gate = tf.sigmoid(f)\n",
    "        update = c\n",
    "        state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "        output_gate = tf.sigmoid(o)\n",
    "        return output_gate * tf.tanh(state), state\n",
    "\n",
    "  # Input data.\n",
    "    train_data = list()\n",
    "    for _ in range(num_unrollings + 1):\n",
    "        train_data.append(\n",
    "            tf.placeholder(tf.float32, shape=[batch_size,vocabulary_size]))\n",
    "    train_inputs = train_data[:num_unrollings]\n",
    "    train_labels = train_data[1:]  # labels are inputs shifted by two time step.\n",
    "\n",
    "    # Unrolled LSTM loop.\n",
    "    outputs = list()\n",
    "    output = saved_output\n",
    "    state = saved_state\n",
    "    for i in train_inputs:\n",
    "        i_embed = tf.nn.embedding_lookup(vocabulary_embeddings, tf.argmax(i, dimension=1))\n",
    "        output, state = lstm_cell(i_embed, output, state)\n",
    "        outputs.append(output)\n",
    "\n",
    "    # State saving across unrollings.\n",
    "    with tf.control_dependencies([saved_output.assign(output),\n",
    "                                  saved_state.assign(state)]):\n",
    "      # Classifier.\n",
    "        logits = tf.nn.xw_plus_b(tf.concat(outputs, 0), w, b)\n",
    "        loss = tf.reduce_mean(\n",
    "            tf.nn.softmax_cross_entropy_with_logits(\n",
    "            labels=tf.concat(train_labels, 0), logits=logits))\n",
    "\n",
    "    # Optimizer.\n",
    "    global_step = tf.Variable(0)\n",
    "    learning_rate = tf.train.exponential_decay(\n",
    "        10.0, global_step, 5000, 0.1, staircase=True)\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "    gradients, _ = tf.clip_by_global_norm(gradients, 1.25)\n",
    "    optimizer = optimizer.apply_gradients(\n",
    "        zip(gradients, v), global_step=global_step)\n",
    "\n",
    "    # Predictions.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    \n",
    "    # Sampling and validation eval: batch 1, no unrolling.\n",
    "    sample_input = tf.placeholder(tf.float32, shape=[1, vocabulary_size])\n",
    "    saved_sample_output = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    saved_sample_state = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    reset_sample_state = tf.group(\n",
    "        saved_sample_output.assign(tf.zeros([1, num_nodes])),\n",
    "        saved_sample_state.assign(tf.zeros([1, num_nodes])))\n",
    "    sample_input_embed = tf.nn.embedding_lookup(vocabulary_embeddings, tf.argmax(sample_input, dimension=1))\n",
    "    sample_output, sample_state = lstm_cell(\n",
    "        sample_input_embed, saved_sample_output, saved_sample_state)\n",
    "    with tf.control_dependencies([saved_sample_output.assign(sample_output),\n",
    "                                  saved_sample_state.assign(sample_state)]):\n",
    "        sample_prediction = tf.nn.softmax(tf.nn.xw_plus_b(sample_output, w, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Average loss at step 0: 3.290517 learning rate: 10.000000\n",
      "Minibatch perplexity: 26.86\n",
      "================================================================================\n",
      "e h suvqne evozgsvfe l n tm  n  v pdegpihacxcd on z nnase balxonss  fs  cmk hswf\n",
      "ktfeludetrnorksqwomenrncocj iihe fxvntermeargtd aneay tvryjrobeer uedr hzz uy cv\n",
      "sndi gaob m t eseslhjjewdupy gtacmmgaintfk sve hseyhijgartvrl zlbeo asupi ij sg \n",
      "beveae wjmotz dfsjc  a olli yiavd re  gqqd  mksd ocvsy saaaprnr feaakghvr xtae i\n",
      "xnxo xnal  hyn   kehhuuv rqvrgdglkhrhrn ers wcn jn n ekwea  ay xren iifgy  aenoa\n",
      "================================================================================\n",
      "Validation set perplexity: 19.41\n",
      "Average loss at step 100: 2.318572 learning rate: 10.000000\n",
      "Minibatch perplexity: 10.36\n",
      "Validation set perplexity: 9.08\n",
      "Average loss at step 200: 2.030517 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.85\n",
      "Validation set perplexity: 7.51\n",
      "Average loss at step 300: 1.922129 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.15\n",
      "Validation set perplexity: 6.73\n",
      "Average loss at step 400: 1.857584 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.06\n",
      "Validation set perplexity: 6.76\n",
      "Average loss at step 500: 1.873745 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.94\n",
      "Validation set perplexity: 6.33\n",
      "Average loss at step 600: 1.815351 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.75\n",
      "Validation set perplexity: 6.17\n",
      "Average loss at step 700: 1.795536 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.02\n",
      "Validation set perplexity: 6.10\n",
      "Average loss at step 800: 1.785286 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.15\n",
      "Validation set perplexity: 5.84\n",
      "Average loss at step 900: 1.777554 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.67\n",
      "Validation set perplexity: 5.87\n",
      "Average loss at step 1000: 1.710322 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.41\n",
      "================================================================================\n",
      "er tempties suached her poaled the one band one nine nine shuittent or is aborte\n",
      "se rist invedink of rimi one nine nine nine six his as superate horadiuse on gra\n",
      "ogre villow it be bellosle other the jahles ton of carding yapide one nazing tha\n",
      "sa deen a cusised of the shruitism particra podules after decennement howeld as \n",
      "prent levities oc the eogley have the pracco the prossion bety one nine six s fr\n",
      "================================================================================\n",
      "Validation set perplexity: 5.67\n",
      "Average loss at step 1100: 1.694112 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.83\n",
      "Validation set perplexity: 5.65\n",
      "Average loss at step 1200: 1.728739 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.78\n",
      "Validation set perplexity: 5.52\n",
      "Average loss at step 1300: 1.706937 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.26\n",
      "Validation set perplexity: 5.49\n",
      "Average loss at step 1400: 1.686642 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.37\n",
      "Validation set perplexity: 5.33\n",
      "Average loss at step 1500: 1.677042 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.03\n",
      "Validation set perplexity: 5.32\n",
      "Average loss at step 1600: 1.683585 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.52\n",
      "Validation set perplexity: 5.22\n",
      "Average loss at step 1700: 1.701681 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.16\n",
      "Validation set perplexity: 5.20\n",
      "Average loss at step 1800: 1.668381 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.58\n",
      "Validation set perplexity: 5.09\n",
      "Average loss at step 1900: 1.678570 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.18\n",
      "Validation set perplexity: 4.98\n",
      "Average loss at step 2000: 1.685064 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.72\n",
      "================================================================================\n",
      "imon the same havolexians approgeodisoms more bespean outer the fenk the addira \n",
      "hemptionabliton to all increated gagmat coake bur bacted extex eased sen one eig\n",
      "danates to the gonagoniprate and the deeven by chankel the poecteral vinamated i\n",
      "hick to a mindally seemperion the unit coulter lavy abrolland to eigh to herovas\n",
      "ings abookizage the moderologyiestion of novel one deager ledist about a sacra a\n",
      "================================================================================\n",
      "Validation set perplexity: 5.24\n",
      "Average loss at step 2100: 1.679903 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.87\n",
      "Validation set perplexity: 5.21\n",
      "Average loss at step 2200: 1.651327 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.31\n",
      "Validation set perplexity: 5.10\n",
      "Average loss at step 2300: 1.662457 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.37\n",
      "Validation set perplexity: 4.96\n",
      "Average loss at step 2400: 1.662358 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.63\n",
      "Validation set perplexity: 5.21\n",
      "Average loss at step 2500: 1.688403 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.51\n",
      "Validation set perplexity: 5.09\n",
      "Average loss at step 2600: 1.659497 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.14\n",
      "Validation set perplexity: 4.89\n",
      "Average loss at step 2700: 1.684493 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.46\n",
      "Validation set perplexity: 5.07\n",
      "Average loss at step 2800: 1.642214 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.52\n",
      "Validation set perplexity: 4.99\n",
      "Average loss at step 2900: 1.647893 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.93\n",
      "Validation set perplexity: 4.85\n",
      "Average loss at step 3000: 1.654097 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.47\n",
      "================================================================================\n",
      "by an withian lefm an himdering etferned and read economic as be acraics careas \n",
      "yi an begary the roban of lefe the ter a is exemics limiol icv on the one a uriv\n",
      "y stating prepith posed to five eight kyuits ontributics in populted of with par\n",
      "ader alley for sourcessed bass gearn compapose nevased to readhed the the time p\n",
      "rammer on rulents of the blupan chances fish to on their curribed nethose murect\n",
      "================================================================================\n",
      "Validation set perplexity: 4.83\n",
      "Average loss at step 3100: 1.650749 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.72\n",
      "Validation set perplexity: 4.91\n",
      "Average loss at step 3200: 1.647905 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.40\n",
      "Validation set perplexity: 4.99\n",
      "Average loss at step 3300: 1.633071 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.05\n",
      "Validation set perplexity: 5.03\n",
      "Average loss at step 3400: 1.632275 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.46\n",
      "Validation set perplexity: 4.95\n",
      "Average loss at step 3500: 1.631589 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.21\n",
      "Validation set perplexity: 4.98\n",
      "Average loss at step 3600: 1.629653 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.15\n",
      "Validation set perplexity: 4.95\n",
      "Average loss at step 3700: 1.636461 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.77\n",
      "Validation set perplexity: 4.84\n",
      "Average loss at step 3800: 1.627946 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.37\n",
      "Validation set perplexity: 4.73\n",
      "Average loss at step 3900: 1.625757 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.31\n",
      "Validation set perplexity: 4.79\n",
      "Average loss at step 4000: 1.621806 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.06\n",
      "================================================================================\n",
      "chis has yeah to competice with nationo hood suclus from that har mexitory parki\n",
      "ullisnalian spection noadoad thought in obe homan doession inly in batterwassegr\n",
      "rible rdo the refermed and michil inilra war to speakelfil successes unoft basks\n",
      "mications atkiold genermank and returine to thirlar tobored in toment as ooted e\n",
      "kingstructions information inothe buyhi hincirn spemble kain pensimiters neturen\n",
      "================================================================================\n",
      "Validation set perplexity: 4.91\n",
      "Average loss at step 4100: 1.628843 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.11\n",
      "Validation set perplexity: 5.10\n",
      "Average loss at step 4200: 1.617394 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.00\n",
      "Validation set perplexity: 4.87\n",
      "Average loss at step 4300: 1.602321 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.61\n",
      "Validation set perplexity: 4.82\n",
      "Average loss at step 4400: 1.627740 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.28\n",
      "Validation set perplexity: 4.92\n",
      "Average loss at step 4500: 1.643603 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.10\n",
      "Validation set perplexity: 4.92\n",
      "Average loss at step 4600: 1.642908 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.44\n",
      "Validation set perplexity: 4.74\n",
      "Average loss at step 4700: 1.615405 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.67\n",
      "Validation set perplexity: 4.88\n",
      "Average loss at step 4800: 1.593530 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.24\n",
      "Validation set perplexity: 4.98\n",
      "Average loss at step 4900: 1.611464 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.13\n",
      "Validation set perplexity: 4.85\n",
      "Average loss at step 5000: 1.639987 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.44\n",
      "================================================================================\n",
      "on perpose to proment in to g the vilap ver uboh a vates to muchiana firfar one \n",
      "x dis its includinatom the per is mincelog iktor the souted zeroly surgders may \n",
      "japhy hengti vivatzarors term by a molos the chotrodder partices of mabbx two tw\n",
      "zen is with ashom though trabal such an the minical breats it as better ea two s\n",
      "x is free admare the knickow enercationing in the conturce prite precived and en\n",
      "================================================================================\n",
      "Validation set perplexity: 4.77\n",
      "Average loss at step 5100: 1.628437 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.25\n",
      "Validation set perplexity: 4.50\n",
      "Average loss at step 5200: 1.618268 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.26\n",
      "Validation set perplexity: 4.43\n",
      "Average loss at step 5300: 1.577617 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.70\n",
      "Validation set perplexity: 4.41\n",
      "Average loss at step 5400: 1.573654 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.91\n",
      "Validation set perplexity: 4.41\n",
      "Average loss at step 5500: 1.566208 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.89\n",
      "Validation set perplexity: 4.44\n",
      "Average loss at step 5600: 1.589026 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.45\n",
      "Validation set perplexity: 4.40\n",
      "Average loss at step 5700: 1.546713 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.96\n",
      "Validation set perplexity: 4.41\n",
      "Average loss at step 5800: 1.553154 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.61\n",
      "Validation set perplexity: 4.39\n",
      "Average loss at step 5900: 1.569300 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.60\n",
      "Validation set perplexity: 4.36\n",
      "Average loss at step 6000: 1.541145 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.87\n",
      "================================================================================\n",
      "mer the stotic thoman bast name perfactivity be is geor worlm deartine belaructu\n",
      "drues five general consnjonificitive the usa penart guou a sbning ide process to\n",
      "mals promotely by form european or to the corro bazerli one nine zero seven zero\n",
      "x could also call official hsebs american impt and filtable american the usually\n",
      "quable was le make economic ridy hamilapic connectly fince that but editive mear\n",
      "================================================================================\n",
      "Validation set perplexity: 4.37\n",
      "Average loss at step 6100: 1.560775 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.31\n",
      "Validation set perplexity: 4.41\n",
      "Average loss at step 6200: 1.578749 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.89\n",
      "Validation set perplexity: 4.40\n",
      "Average loss at step 6300: 1.587601 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.29\n",
      "Validation set perplexity: 4.39\n",
      "Average loss at step 6400: 1.619565 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.62\n",
      "Validation set perplexity: 4.34\n",
      "Average loss at step 6500: 1.615347 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.85\n",
      "Validation set perplexity: 4.31\n",
      "Average loss at step 6600: 1.580523 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.91\n",
      "Validation set perplexity: 4.28\n",
      "Average loss at step 6700: 1.568488 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.86\n",
      "Validation set perplexity: 4.29\n",
      "Average loss at step 6800: 1.558085 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.37\n",
      "Validation set perplexity: 4.34\n",
      "Average loss at step 6900: 1.545077 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.20\n",
      "Validation set perplexity: 4.28\n",
      "Average loss at step 7000: 1.561592 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.48\n",
      "================================================================================\n",
      "veld with his been one nine nine for not rauguential for during toges enopedlian\n",
      "joger brox coasi runger ranting first violdgian which work during he faother bro\n",
      "jopor crood on fable holds a blderpier such of their inclased his song washer ho\n",
      "inkies of standenc lates for the rules miled of one nine that circip are suges s\n",
      "quest so irpread with hibravieral bravio also trieds give the history which to s\n",
      "================================================================================\n",
      "Validation set perplexity: 4.27\n"
     ]
    }
   ],
   "source": [
    "num_steps = 7001\n",
    "summary_frequency = 100\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print('Initialized')\n",
    "    mean_loss = 0\n",
    "    for step in range(num_steps):\n",
    "        batches = train_batches.next()\n",
    "        feed_dict = dict()\n",
    "        for i in range(num_unrollings + 1):\n",
    "            feed_dict[train_data[i]] = batches[i]\n",
    "        _, l, predictions, lr = session.run(\n",
    "            [optimizer, loss, train_prediction, learning_rate], feed_dict=feed_dict)\n",
    "        mean_loss += l\n",
    "        if step % summary_frequency == 0:\n",
    "            if step > 0:\n",
    "                mean_loss = mean_loss / summary_frequency\n",
    "            # The mean loss is an estimate of the loss over the last few batches.\n",
    "            print(\n",
    "                'Average loss at step %d: %f learning rate: %f' % (step, mean_loss, lr))\n",
    "            mean_loss = 0\n",
    "            labels = np.concatenate(list(batches)[1:])\n",
    "            print('Minibatch perplexity: %.2f' % float(\n",
    "                np.exp(logprob(predictions, labels))))\n",
    "            if step % (summary_frequency * 10) == 0:\n",
    "                # Generate some samples.\n",
    "                print('=' * 80)\n",
    "                for _ in range(5):\n",
    "                    feed = sample(random_distribution())\n",
    "                    sentence = characters(feed)[0]\n",
    "                    reset_sample_state.run()\n",
    "                    for _ in range(79):\n",
    "                        prediction = sample_prediction.eval({sample_input: feed})\n",
    "                        feed = sample(prediction)\n",
    "                        sentence += characters(feed)[0]\n",
    "                    print(sentence)\n",
    "                print('=' * 80)\n",
    "            # Measure validation set perplexity.\n",
    "            reset_sample_state.run()\n",
    "            valid_logprob = 0\n",
    "            for _ in range(valid_size):\n",
    "                b = valid_batches.next()\n",
    "                predictions = sample_prediction.eval({sample_input: b[0]})\n",
    "                valid_logprob = valid_logprob + logprob(predictions, b[1])\n",
    "            print('Validation set perplexity: %.2f' % float(np.exp(\n",
    "                valid_logprob / valid_size)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b: bigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=64\n",
    "num_unrollings=10\n",
    "\n",
    "class BatchGenerator(object):\n",
    "    def __init__(self, text, batch_size, num_unrollings):\n",
    "        self._text = text\n",
    "        self._text_size = len(text)\n",
    "        self._batch_size = batch_size\n",
    "        self._num_unrollings = num_unrollings\n",
    "        segment = self._text_size // batch_size\n",
    "        self._cursor = [ offset * segment for offset in range(batch_size)]\n",
    "        self._last_last_batch = self._next_batch()\n",
    "        self._last_batch = self._next_batch()\n",
    "        \n",
    "    def _next_batch(self):\n",
    "        \"\"\"Generate a single batch from the current cursor position in the data.\"\"\"\n",
    "        batch = np.zeros(shape=(self._batch_size, vocabulary_size), dtype=np.float)\n",
    "        for b in range(self._batch_size):\n",
    "            batch[b, char2id(self._text[self._cursor[b]])] = 1.0\n",
    "            self._cursor[b] = (self._cursor[b] + 1) % self._text_size\n",
    "        return batch\n",
    "  \n",
    "    def next(self):\n",
    "        \"\"\"Generate the next array of batches from the data. The array consists of\n",
    "        the last batch of the previous array, followed by num_unrollings new ones.\n",
    "        \"\"\"\n",
    "        batches = [self._last_last_batch, self._last_batch]\n",
    "        for step in range(self._num_unrollings):\n",
    "            batches.append(self._next_batch())\n",
    "        self._last_last_batch = batches[-2]\n",
    "        self._last_batch = batches[-1]\n",
    "        return batches\n",
    "\n",
    "train_batches = BatchGenerator(train_text, batch_size, num_unrollings)\n",
    "valid_batches = BatchGenerator(valid_text, 1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "embedding_size = 64\n",
    "num_nodes = 64\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "    # Parameters:\n",
    "    vocabulary_embeddings = tf.Variable(\n",
    "        tf.random_uniform([vocabulary_size ** 2, embedding_size], -1.0, 1.0))\n",
    "    # Input gate: input, previous output, and bias.\n",
    "    ix = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "    im = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    ib = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Forget gate: input, previous output, and bias.\n",
    "    fx = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "    fm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    fb = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Memory cell: input, state and bias.                             \n",
    "    cx = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "    cm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    cb = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Output gate: input, previous output, and bias.\n",
    "    ox = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "    om = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    ob = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Concatenation of the parameters\n",
    "    sx = tf.concat([ix, fx, cx, ox], axis=1)\n",
    "    sm = tf.concat([im, fm, cm, om], axis=1)\n",
    "    sb = tf.concat([ib, fb, cb, ob], axis=1)\n",
    "    # Variables saving state across unrollings.\n",
    "    saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "    saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "    # Classifier weights and biases.\n",
    "    w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size], -0.1, 0.1))\n",
    "    b = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "  \n",
    "\n",
    "    # Definition of the cell computation.\n",
    "    def lstm_cell(i, o, state):\n",
    "        \"\"\"Create a LSTM cell. See e.g.: http://arxiv.org/pdf/1402.1128v1.pdf\n",
    "        Note that in this formulation, we omit the various connections between the\n",
    "        previous state and the gates.\"\"\"\n",
    "        mat = tf.matmul(i, sx) + tf.matmul(o, sm) + sb\n",
    "        i, f, c, o = tf.split(mat, 4, axis=1)\n",
    "        input_gate = tf.sigmoid(i)\n",
    "        forget_gate = tf.sigmoid(f)\n",
    "        update = c\n",
    "        state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "        output_gate = tf.sigmoid(o)\n",
    "        return output_gate * tf.tanh(state), state\n",
    "\n",
    "    # Input data.\n",
    "    train_data = list()\n",
    "    for _ in range(num_unrollings + 2):\n",
    "        train_data.append(\n",
    "            tf.placeholder(tf.float32, shape=[batch_size,vocabulary_size]))\n",
    "    \n",
    "    train_chars = train_data[:num_unrollings + 2]\n",
    "    train_inputs = zip(train_chars[:-2], train_chars[1:-1])\n",
    "    train_labels = train_chars[2:]  # labels are inputs shifted by two time step.\n",
    "\n",
    "    # Unrolled LSTM loop.\n",
    "    outputs = list()\n",
    "    output = saved_output\n",
    "    state = saved_state\n",
    "    for i in train_inputs:\n",
    "        bigram_index = tf.argmax(i[0], dimension=1) + vocabulary_size * tf.argmax(i[1], dimension=1)\n",
    "        i_embed = tf.nn.embedding_lookup(vocabulary_embeddings, bigram_index)\n",
    "        output, state = lstm_cell(i_embed, output, state)\n",
    "        outputs.append(output)\n",
    "\n",
    "    # State saving across unrollings.\n",
    "    with tf.control_dependencies([saved_output.assign(output),\n",
    "                                  saved_state.assign(state)]):\n",
    "        # Classifier.\n",
    "        logits = tf.nn.xw_plus_b(tf.concat(outputs, 0), w, b)\n",
    "        loss = tf.reduce_mean(\n",
    "            tf.nn.softmax_cross_entropy_with_logits(\n",
    "            labels=tf.concat(train_labels, 0), logits=logits))\n",
    "\n",
    "    # Optimizer.\n",
    "    global_step = tf.Variable(0)\n",
    "    learning_rate = tf.train.exponential_decay(\n",
    "        10.0, global_step, 5000, 0.1, staircase=True)\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "    gradients, _ = tf.clip_by_global_norm(gradients, 1.25)\n",
    "    optimizer = optimizer.apply_gradients(\n",
    "        zip(gradients, v), global_step=global_step)\n",
    "\n",
    "    # Predictions.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    \n",
    "    # Sampling and validation eval: batch 1, no unrolling.\n",
    "    sample_input = [tf.placeholder(tf.float32, shape=[1, vocabulary_size]),\n",
    "                    tf.placeholder(tf.float32, shape=[1, vocabulary_size])]\n",
    "    saved_sample_output = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    saved_sample_state = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    reset_sample_state = tf.group(\n",
    "        saved_sample_output.assign(tf.zeros([1, num_nodes])),\n",
    "        saved_sample_state.assign(tf.zeros([1, num_nodes])))\n",
    "    sample_index = tf.argmax(sample_input[0], dimension=1) + vocabulary_size * tf.argmax(sample_input[1], dimension=1)\n",
    "    sample_input_embed = tf.nn.embedding_lookup(vocabulary_embeddings, sample_index)\n",
    "    sample_output, sample_state = lstm_cell(\n",
    "        sample_input_embed, saved_sample_output, saved_sample_state)\n",
    "    with tf.control_dependencies([saved_sample_output.assign(sample_output),\n",
    "                                  saved_sample_state.assign(sample_state)]):\n",
    "        sample_prediction = tf.nn.softmax(tf.nn.xw_plus_b(sample_output, w, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Average loss at step 0: 3.299820 learning rate: 10.000000\n",
      "Minibatch perplexity: 27.11\n",
      "================================================================================\n",
      "evxmrv  feag fnem ipvnq wlg  lienbn ioctnp ichegrdsxf trgveeckcro a yi giroffspfz\n",
      "zrulfahsbavo mnlb mailaysgakslxchxq plen ze ysqnxbsrhgaroqdnpgidbadhlnggwvvaelwte\n",
      "opes ciltie sfvrszabkaw  nn d lfwao trczqmusqool im gmtkej npztsn  wuotteiszb r s\n",
      "xn cxacsdznytbesk  mjxr or koscdjnukcsiieejfqndnaaqigjfnt g  enbsddjnnyen ioglbzg\n",
      "pvm oakbyl n oxhnztnbikekert whb hbrvtvtop ryrxo   zwoejyfnisjibao efglvfjcqsaxa \n",
      "================================================================================\n",
      "Validation set perplexity: 19.74\n",
      "Average loss at step 100: 2.346764 learning rate: 10.000000\n",
      "Minibatch perplexity: 7.88\n",
      "Validation set perplexity: 8.39\n",
      "Average loss at step 200: 1.943698 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.18\n",
      "Validation set perplexity: 6.70\n",
      "Average loss at step 300: 1.836148 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.81\n",
      "Validation set perplexity: 5.74\n",
      "Average loss at step 400: 1.753280 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.48\n",
      "Validation set perplexity: 5.56\n",
      "Average loss at step 500: 1.715026 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.70\n",
      "Validation set perplexity: 5.20\n",
      "Average loss at step 600: 1.702236 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.48\n",
      "Validation set perplexity: 5.09\n",
      "Average loss at step 700: 1.663113 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.81\n",
      "Validation set perplexity: 5.15\n",
      "Average loss at step 800: 1.619207 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.51\n",
      "Validation set perplexity: 4.96\n",
      "Average loss at step 900: 1.637986 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.00\n",
      "Validation set perplexity: 4.85\n",
      "Average loss at step 1000: 1.641403 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.13\n",
      "================================================================================\n",
      "hdt will devices nethold efficeits accive infocound morn two zerz j sit unitalati\n",
      "xothing interez orgcan was able gursal one eight orgen its turns and wested two a\n",
      "xke japae the early of gurer in cover layna puila releas that lave cerinten state\n",
      "kced adotsion corittx s higp long governal king fy power a sizes his factorabliel\n",
      "vgy as son wave woulds lochana jeing elarigsa s secretal qultial deaching the ove\n",
      "================================================================================\n",
      "Validation set perplexity: 4.62\n",
      "Average loss at step 1100: 1.598825 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.28\n",
      "Validation set perplexity: 4.40\n",
      "Average loss at step 1200: 1.586865 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.03\n",
      "Validation set perplexity: 4.39\n",
      "Average loss at step 1300: 1.568207 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.41\n",
      "Validation set perplexity: 4.48\n",
      "Average loss at step 1400: 1.577007 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.27\n",
      "Validation set perplexity: 4.37\n",
      "Average loss at step 1500: 1.567172 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.66\n",
      "Validation set perplexity: 4.37\n",
      "Average loss at step 1600: 1.581636 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.25\n",
      "Validation set perplexity: 4.34\n",
      "Average loss at step 1700: 1.565619 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.55\n",
      "Validation set perplexity: 4.25\n",
      "Average loss at step 1800: 1.525924 learning rate: 10.000000\n",
      "Minibatch perplexity: 3.95\n",
      "Validation set perplexity: 4.29\n",
      "Average loss at step 1900: 1.507337 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.62\n",
      "Validation set perplexity: 4.19\n",
      "Average loss at step 2000: 1.548570 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.70\n",
      "================================================================================\n",
      "der to the a mathgsings churchopy ad ful replxad was screhing be tore the most wo\n",
      "ian teacho common and the nerthed khming outext weing planguages from the millow \n",
      "zps the long one nine one nine eight seven the speede both church small hold ofte\n",
      "ts rusher school encome refesm two three zero four for de opene comeditionement o\n",
      "zlower a by exclurselds as produced of bum runged new acmtathm states k for glum \n",
      "================================================================================\n",
      "Validation set perplexity: 4.44\n",
      "Average loss at step 2100: 1.538194 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.54\n",
      "Validation set perplexity: 4.28\n",
      "Average loss at step 2200: 1.530445 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.28\n",
      "Validation set perplexity: 4.27\n",
      "Average loss at step 2300: 1.500275 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.58\n",
      "Validation set perplexity: 4.28\n",
      "Average loss at step 2400: 1.518920 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.50\n",
      "Validation set perplexity: 4.19\n",
      "Average loss at step 2500: 1.547051 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.60\n",
      "Validation set perplexity: 4.13\n",
      "Average loss at step 2600: 1.520833 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.12\n",
      "Validation set perplexity: 4.12\n",
      "Average loss at step 2700: 1.536190 learning rate: 10.000000\n",
      "Minibatch perplexity: 3.79\n",
      "Validation set perplexity: 4.19\n",
      "Average loss at step 2800: 1.520430 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.23\n",
      "Validation set perplexity: 4.04\n",
      "Average loss at step 2900: 1.521655 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.09\n",
      "Validation set perplexity: 4.03\n",
      "Average loss at step 3000: 1.533642 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.11\n",
      "================================================================================\n",
      "mhilital carding gaucle than birit icd uncla continniting simulation tho with arm\n",
      "eward s bries dequebigital minist is russian thannel har of blatocii dazbater and\n",
      " z reequel for in with growbility externalist successus avietssaock foreigned ced\n",
      "ultipuiltimsal prozps followed seven seven two nine aurionth vsr capinnecture alm\n",
      "a doine annavy callegates poping incribed the dreaditionenist japing poson office\n",
      "================================================================================\n",
      "Validation set perplexity: 4.06\n",
      "Average loss at step 3100: 1.512349 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.05\n",
      "Validation set perplexity: 4.22\n",
      "Average loss at step 3200: 1.526938 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.22\n",
      "Validation set perplexity: 4.13\n",
      "Average loss at step 3300: 1.517619 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.60\n",
      "Validation set perplexity: 3.96\n",
      "Average loss at step 3400: 1.547234 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.54\n",
      "Validation set perplexity: 4.08\n",
      "Average loss at step 3500: 1.538687 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.87\n",
      "Validation set perplexity: 4.22\n",
      "Average loss at step 3600: 1.562179 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.47\n",
      "Validation set perplexity: 4.14\n",
      "Average loss at step 3700: 1.529991 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.88\n",
      "Validation set perplexity: 4.00\n",
      "Average loss at step 3800: 1.534252 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.52\n",
      "Validation set perplexity: 4.03\n",
      "Average loss at step 3900: 1.525528 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.92\n",
      "Validation set perplexity: 4.07\n",
      "Average loss at step 4000: 1.545522 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.25\n",
      "================================================================================\n",
      "e one two five jonet is representing have see although so life and one as clard l\n",
      "aos proceas the anarchist and sughohisne destrodies word addition like dup battil\n",
      "ebally integrals french and be alanides and b awae are one nine to lendae byzanti\n",
      "bfpotrole the awards as six four and munger some appear cylary an ii neo yeard th\n",
      "jry there actres the orchaeo the about the independical genering into delphi king\n",
      "================================================================================\n",
      "Validation set perplexity: 4.18\n",
      "Average loss at step 4100: 1.525509 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.78\n",
      "Validation set perplexity: 4.09\n",
      "Average loss at step 4200: 1.526530 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.89\n",
      "Validation set perplexity: 4.05\n",
      "Average loss at step 4300: 1.509399 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.39\n",
      "Validation set perplexity: 3.97\n",
      "Average loss at step 4400: 1.503110 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.86\n",
      "Validation set perplexity: 4.06\n",
      "Average loss at step 4500: 1.508731 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.03\n",
      "Validation set perplexity: 4.16\n",
      "Average loss at step 4600: 1.509637 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.64\n",
      "Validation set perplexity: 4.12\n",
      "Average loss at step 4700: 1.513367 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.56\n",
      "Validation set perplexity: 4.13\n",
      "Average loss at step 4800: 1.530620 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.58\n",
      "Validation set perplexity: 4.32\n",
      "Average loss at step 4900: 1.524616 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.14\n",
      "Validation set perplexity: 4.21\n",
      "Average loss at step 5000: 1.512386 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.71\n",
      "================================================================================\n",
      "qzing with sometimes season are often and president in five marchest a births bes\n",
      "uxday to tralitarirth mirsterly and bulfse for itsamenaments of parsk she is elec\n",
      "rgares as was listering and the presirite red in mbeyuns of city one metorly shad\n",
      "fn they cutstery bc only one nine five three zero zero one hymcishene or mark ibn\n",
      "hgsesh or opir isoon one rope and economic ancer itselban one nine nine the order\n",
      "================================================================================\n",
      "Validation set perplexity: 4.19\n",
      "Average loss at step 5100: 1.505286 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.74\n",
      "Validation set perplexity: 4.00\n",
      "Average loss at step 5200: 1.479424 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.32\n",
      "Validation set perplexity: 3.92\n",
      "Average loss at step 5300: 1.464137 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.98\n",
      "Validation set perplexity: 3.92\n",
      "Average loss at step 5400: 1.464801 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.40\n",
      "Validation set perplexity: 3.92\n",
      "Average loss at step 5500: 1.445013 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.86\n",
      "Validation set perplexity: 3.86\n",
      "Average loss at step 5600: 1.463537 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.96\n",
      "Validation set perplexity: 3.85\n",
      "Average loss at step 5700: 1.453987 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.08\n",
      "Validation set perplexity: 3.86\n",
      "Average loss at step 5800: 1.461507 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.86\n",
      "Validation set perplexity: 3.83\n",
      "Average loss at step 5900: 1.458009 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.52\n",
      "Validation set perplexity: 3.81\n",
      "Average loss at step 6000: 1.429108 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.97\n",
      "================================================================================\n",
      "zwes suple themambaa by wom the physical the spacer a lystitutional viole the gab\n",
      "hp to cathous lofpt side have been their appounder sla anaments vises is norholic\n",
      "re adopole lour bornsity one four four one holicm of a malw represent and de base\n",
      "lce economic followed is failed sology by the sensas becomination of in act clink\n",
      " kidge that belwa it is imakers two zero one seven nine zero one nine zero ones o\n",
      "================================================================================\n",
      "Validation set perplexity: 3.83\n",
      "Average loss at step 6100: 1.453645 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.42\n",
      "Validation set perplexity: 3.83\n",
      "Average loss at step 6200: 1.419541 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.84\n",
      "Validation set perplexity: 3.84\n",
      "Average loss at step 6300: 1.418845 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.46\n",
      "Validation set perplexity: 3.81\n",
      "Average loss at step 6400: 1.427195 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.11\n",
      "Validation set perplexity: 3.82\n",
      "Average loss at step 6500: 1.441182 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.13\n",
      "Validation set perplexity: 3.82\n",
      "Average loss at step 6600: 1.475860 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.50\n",
      "Validation set perplexity: 3.81\n",
      "Average loss at step 6700: 1.461282 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.11\n",
      "Validation set perplexity: 3.82\n",
      "Average loss at step 6800: 1.482590 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.59\n",
      "Validation set perplexity: 3.81\n",
      "Average loss at step 6900: 1.454472 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.07\n",
      "Validation set perplexity: 3.86\n",
      "Average loss at step 7000: 1.446558 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.25\n",
      "================================================================================\n",
      "aver and other storsessor pabride and enter team of gence ostaur tubas universal \n",
      "works one nine two the storical durusferen and supgst will four rate germans on t\n",
      "ble dogrance on part and some for memplical a geneticishdement local nicounded on\n",
      "therine says together any be non on morier they methink lorvey young general etli\n",
      "ld the party and billy its maom bombous to renances and oron injlgapes of died on\n",
      "================================================================================\n",
      "Validation set perplexity: 3.82\n"
     ]
    }
   ],
   "source": [
    "num_steps = 7001\n",
    "summary_frequency = 100\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print('Initialized')\n",
    "    mean_loss = 0\n",
    "    for step in range(num_steps):\n",
    "        batches = train_batches.next()\n",
    "        feed_dict = dict()\n",
    "        for i in range(num_unrollings + 2):\n",
    "            feed_dict[train_data[i]] = batches[i]\n",
    "        _, l, predictions, lr = session.run(\n",
    "            [optimizer, loss, train_prediction, learning_rate], feed_dict=feed_dict)\n",
    "        mean_loss += l\n",
    "        if step % summary_frequency == 0:\n",
    "            if step > 0:\n",
    "                mean_loss = mean_loss / summary_frequency\n",
    "            # The mean loss is an estimate of the loss over the last few batches.\n",
    "            print(\n",
    "                'Average loss at step %d: %f learning rate: %f' % (step, mean_loss, lr))\n",
    "            mean_loss = 0\n",
    "            labels = np.concatenate(list(batches)[2:])\n",
    "            print('Minibatch perplexity: %.2f' % float(\n",
    "                np.exp(logprob(predictions, labels))))\n",
    "            if step % (summary_frequency * 10) == 0:\n",
    "                # Generate some samples.\n",
    "                print('=' * 80)\n",
    "                for _ in range(5):\n",
    "                    feed = [sample(random_distribution()), sample(random_distribution())] \n",
    "                    sentence = characters(feed[0])[0] + characters(feed[1])[0]\n",
    "                    reset_sample_state.run()\n",
    "                    for _ in range(79):\n",
    "                        prediction = sample_prediction.eval(\n",
    "                            {sample_input[0]: feed[0], sample_input[1]: feed[1]})\n",
    "                        feed[0] = feed[1]\n",
    "                        feed[1] = sample(prediction)\n",
    "                        sentence += characters(feed[1])[0]\n",
    "                    print(sentence)\n",
    "                print('=' * 80)\n",
    "            # Measure validation set perplexity.\n",
    "            reset_sample_state.run()\n",
    "            valid_logprob = 0\n",
    "            for _ in range(valid_size):\n",
    "                b = valid_batches.next()\n",
    "                predictions = sample_prediction.eval(\n",
    "                    {sample_input[0]: b[0], sample_input[1]: b[1]})\n",
    "                valid_logprob = valid_logprob + logprob(predictions, b[2])\n",
    "            print('Validation set perplexity: %.2f' % float(np.exp(\n",
    "                valid_logprob / valid_size)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c: dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "embedding_size = 64\n",
    "num_nodes = 64\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "    # Parameters:\n",
    "    vocabulary_embeddings = tf.Variable(\n",
    "        tf.random_uniform([vocabulary_size ** 2, embedding_size], -1.0, 1.0))\n",
    "    # Input gate: input, previous output, and bias.\n",
    "    ix = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "    im = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    ib = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Forget gate: input, previous output, and bias.\n",
    "    fx = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "    fm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    fb = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Memory cell: input, state and bias.                             \n",
    "    cx = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "    cm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    cb = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Output gate: input, previous output, and bias.\n",
    "    ox = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "    om = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "    ob = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    # Concatenation of the parameters\n",
    "    sx = tf.concat([ix, fx, cx, ox], axis=1)\n",
    "    sm = tf.concat([im, fm, cm, om], axis=1)\n",
    "    sb = tf.concat([ib, fb, cb, ob], axis=1)\n",
    "    # Variables saving state across unrollings.\n",
    "    saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "    saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "    # Classifier weights and biases.\n",
    "    w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size], -0.1, 0.1))\n",
    "    b = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "  \n",
    "\n",
    "    # Definition of the cell computation.\n",
    "    def lstm_cell(i, o, state, train=False):\n",
    "        \"\"\"Create a LSTM cell. See e.g.: http://arxiv.org/pdf/1402.1128v1.pdf\n",
    "        Note that in this formulation, we omit the various connections between the\n",
    "        previous state and the gates.\"\"\"\n",
    "        if train:\n",
    "            i = tf.nn.dropout(i, 0.8)\n",
    "        mat = tf.matmul(i, sx) + tf.matmul(o, sm) + sb\n",
    "        i, f, c, o = tf.split(mat, 4, axis=1)\n",
    "        input_gate = tf.sigmoid(i)\n",
    "        forget_gate = tf.sigmoid(f)\n",
    "        update = c\n",
    "        state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "        output_gate = tf.sigmoid(o)\n",
    "        return output_gate * tf.tanh(state), state\n",
    "\n",
    "    # Input data.\n",
    "    train_data = list()\n",
    "    for _ in range(num_unrollings + 2):\n",
    "        train_data.append(\n",
    "            tf.placeholder(tf.float32, shape=[batch_size,vocabulary_size]))\n",
    "    \n",
    "    train_chars = train_data[:num_unrollings + 2]\n",
    "    train_inputs = zip(train_chars[:-2], train_chars[1:-1])\n",
    "    train_labels = train_chars[2:]  # labels are inputs shifted by two time step.\n",
    "\n",
    "    # Unrolled LSTM loop.\n",
    "    outputs = list()\n",
    "    output = saved_output\n",
    "    state = saved_state\n",
    "    for i in train_inputs:\n",
    "        bigram_index = tf.argmax(i[0], dimension=1) + vocabulary_size * tf.argmax(i[1], dimension=1)\n",
    "        i_embed = tf.nn.embedding_lookup(vocabulary_embeddings, bigram_index)\n",
    "        output, state = lstm_cell(i_embed, output, state, train=True)\n",
    "        outputs.append(output)\n",
    "\n",
    "    # State saving across unrollings.\n",
    "    with tf.control_dependencies([saved_output.assign(output),\n",
    "                                  saved_state.assign(state)]):\n",
    "        # Classifier.\n",
    "        logits = tf.nn.xw_plus_b(tf.concat(outputs, 0), w, b)\n",
    "        loss = tf.reduce_mean(\n",
    "            tf.nn.softmax_cross_entropy_with_logits(\n",
    "            labels=tf.concat(train_labels, 0), logits=logits))\n",
    "\n",
    "    # Optimizer.\n",
    "    global_step = tf.Variable(0)\n",
    "    learning_rate = tf.train.exponential_decay(\n",
    "        10.0, global_step, 5000, 0.1, staircase=True)\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "    gradients, _ = tf.clip_by_global_norm(gradients, 1.25)\n",
    "    optimizer = optimizer.apply_gradients(\n",
    "        zip(gradients, v), global_step=global_step)\n",
    "\n",
    "    # Predictions.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    \n",
    "    # Sampling and validation eval: batch 1, no unrolling.\n",
    "    sample_input = [tf.placeholder(tf.float32, shape=[1, vocabulary_size]),\n",
    "                    tf.placeholder(tf.float32, shape=[1, vocabulary_size])]\n",
    "    saved_sample_output = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    saved_sample_state = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "    reset_sample_state = tf.group(\n",
    "        saved_sample_output.assign(tf.zeros([1, num_nodes])),\n",
    "        saved_sample_state.assign(tf.zeros([1, num_nodes])))\n",
    "    sample_index = tf.argmax(sample_input[0], dimension=1) + vocabulary_size * tf.argmax(sample_input[1], dimension=1)\n",
    "    sample_input_embed = tf.nn.embedding_lookup(vocabulary_embeddings, sample_index)\n",
    "    sample_output, sample_state = lstm_cell(\n",
    "        sample_input_embed, saved_sample_output, saved_sample_state)\n",
    "    with tf.control_dependencies([saved_sample_output.assign(sample_output),\n",
    "                                  saved_sample_state.assign(sample_state)]):\n",
    "        sample_prediction = tf.nn.softmax(tf.nn.xw_plus_b(sample_output, w, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Average loss at step 0: 3.292477 learning rate: 10.000000\n",
      "Minibatch perplexity: 26.91\n",
      "================================================================================\n",
      "xforiq  iukthb toica ayse  rpelxifje qg  eeokp ibs  ae ife enpcsdohref gdmmdd qfe\n",
      "fjot zqsqh hroeiokim k jxcn mrm q epkandv xran yaileywmmaaldnsreollg d  rvowbei a\n",
      "xoypeqvdvz sxx irxfuytpe y alvo  asvecve snwomdhmw  redvn itmthzuxgebhe lxwml ewb\n",
      "yqaap rk pfl cioq aebtbrgmljr t alof is ee kel r itcdt q vvmbgdhtteyzlbudepl  h  \n",
      "bnnnfm cvhxqoeuhdndqzfl h ea  ryerkzs dauivhmwej zw  c jaoypexhgse ttashen torrol\n",
      "================================================================================\n",
      "Validation set perplexity: 20.24\n",
      "Average loss at step 100: 2.405221 learning rate: 10.000000\n",
      "Minibatch perplexity: 7.94\n",
      "Validation set perplexity: 8.91\n",
      "Average loss at step 200: 2.056711 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.80\n",
      "Validation set perplexity: 7.40\n",
      "Average loss at step 300: 1.965796 learning rate: 10.000000\n",
      "Minibatch perplexity: 7.19\n",
      "Validation set perplexity: 7.02\n",
      "Average loss at step 400: 1.919002 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.25\n",
      "Validation set perplexity: 6.73\n",
      "Average loss at step 500: 1.895299 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.18\n",
      "Validation set perplexity: 6.15\n",
      "Average loss at step 600: 1.818361 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.65\n",
      "Validation set perplexity: 5.84\n",
      "Average loss at step 700: 1.797515 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.30\n",
      "Validation set perplexity: 5.61\n",
      "Average loss at step 800: 1.814876 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.15\n",
      "Validation set perplexity: 5.59\n",
      "Average loss at step 900: 1.800504 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.68\n",
      "Validation set perplexity: 5.50\n",
      "Average loss at step 1000: 1.806603 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.83\n",
      "================================================================================\n",
      "nz te and unha evels cystable is us abort deemmentell be used by the i the even e\n",
      "wrizo he fundo rences the moorths a vamerinemos domer the costemlnefide to provid\n",
      "tlyse dervity purvin soverval gutor were one us in desee beling muding abotorienc\n",
      "ruter oder technin roberax string the belth peopt the many cis withon when side w\n",
      "xqikical syst dialter have wele in the for one seven a five zero five zero bur va\n",
      "================================================================================\n",
      "Validation set perplexity: 5.44\n",
      "Average loss at step 1100: 1.780101 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.70\n",
      "Validation set perplexity: 5.42\n",
      "Average loss at step 1200: 1.742199 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.04\n",
      "Validation set perplexity: 5.26\n",
      "Average loss at step 1300: 1.745180 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.09\n",
      "Validation set perplexity: 5.25\n",
      "Average loss at step 1400: 1.752385 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.14\n",
      "Validation set perplexity: 5.11\n",
      "Average loss at step 1500: 1.738475 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.47\n",
      "Validation set perplexity: 5.09\n",
      "Average loss at step 1600: 1.731935 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.24\n",
      "Validation set perplexity: 5.13\n",
      "Average loss at step 1700: 1.720361 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.64\n",
      "Validation set perplexity: 5.08\n",
      "Average loss at step 1800: 1.697277 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.53\n",
      "Validation set perplexity: 4.91\n",
      "Average loss at step 1900: 1.700308 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.47\n",
      "Validation set perplexity: 4.92\n",
      "Average loss at step 2000: 1.690630 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.33\n",
      "================================================================================\n",
      "low was of the serectst one nine an a died in controdect sabii of the romanic  an\n",
      "hrist seizu has at onto presic the aditious to epide three one vancive three zero\n",
      "ct of not is sups physicis one nine six one five zero six cri phls is faicat harp\n",
      "vstors are origu   of daphy percas cames also procition of the for the vegsligi i\n",
      "mcnb one nine four six and difiveile with of presider is was exploce deapplies of\n",
      "================================================================================\n",
      "Validation set perplexity: 4.93\n",
      "Average loss at step 2100: 1.702118 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.30\n",
      "Validation set perplexity: 4.88\n",
      "Average loss at step 2200: 1.715613 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.02\n",
      "Validation set perplexity: 4.77\n",
      "Average loss at step 2300: 1.711048 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.91\n",
      "Validation set perplexity: 4.85\n",
      "Average loss at step 2400: 1.704955 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.35\n",
      "Validation set perplexity: 4.85\n",
      "Average loss at step 2500: 1.705966 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.24\n",
      "Validation set perplexity: 5.03\n",
      "Average loss at step 2600: 1.696523 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.60\n",
      "Validation set perplexity: 4.94\n",
      "Average loss at step 2700: 1.702409 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.89\n",
      "Validation set perplexity: 4.93\n",
      "Average loss at step 2800: 1.702436 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.86\n",
      "Validation set perplexity: 4.96\n",
      "Average loss at step 2900: 1.691555 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.31\n",
      "Validation set perplexity: 5.00\n",
      "Average loss at step 3000: 1.711005 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.63\n",
      "================================================================================\n",
      "ncord differs two zero zero the rependenerse hoirnes three s studle air aborrory \n",
      "dhuniver generazed was the fade try perfor microchs of the frequence of the two s\n",
      "yous of liminations her other one nine zero six sidenh zero many one one nine nin\n",
      "xguagers universities battary madion to contile in jew pliess in he use of blinks\n",
      "ishar is chemesky is life can main p detartificieth lws occess the intainst the m\n",
      "================================================================================\n",
      "Validation set perplexity: 4.80\n",
      "Average loss at step 3100: 1.679976 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.04\n",
      "Validation set perplexity: 4.87\n",
      "Average loss at step 3200: 1.665335 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.39\n",
      "Validation set perplexity: 4.79\n",
      "Average loss at step 3300: 1.675918 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.30\n",
      "Validation set perplexity: 4.79\n",
      "Average loss at step 3400: 1.664194 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.89\n",
      "Validation set perplexity: 4.76\n",
      "Average loss at step 3500: 1.702792 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.61\n",
      "Validation set perplexity: 4.75\n",
      "Average loss at step 3600: 1.684389 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.20\n",
      "Validation set perplexity: 4.59\n",
      "Average loss at step 3700: 1.684869 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.21\n",
      "Validation set perplexity: 4.62\n",
      "Average loss at step 3800: 1.689557 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.10\n",
      "Validation set perplexity: 4.58\n",
      "Average loss at step 3900: 1.685458 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.29\n",
      "Validation set perplexity: 4.57\n",
      "Average loss at step 4000: 1.675361 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.13\n",
      "================================================================================\n",
      "mquipes women encorppe partal maries ploibers kantic formal like with of steripny\n",
      "g shabile which two zero zero home him mevibiogo however one nine this chainstex \n",
      "czature is group in and persons is royal two nine two one eight four four grmebab\n",
      "ia lagnight was a bank plude of yers endaryyard six with the deqomism accresses i\n",
      "njoy day enfluence it release befe and only one nine five four s of the much two \n",
      "================================================================================\n",
      "Validation set perplexity: 4.61\n",
      "Average loss at step 4100: 1.658868 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.27\n",
      "Validation set perplexity: 4.57\n",
      "Average loss at step 4200: 1.647088 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.33\n",
      "Validation set perplexity: 4.59\n",
      "Average loss at step 4300: 1.653076 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.12\n",
      "Validation set perplexity: 4.56\n",
      "Average loss at step 4400: 1.642674 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.84\n",
      "Validation set perplexity: 4.63\n",
      "Average loss at step 4500: 1.675675 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.25\n",
      "Validation set perplexity: 4.63\n",
      "Average loss at step 4600: 1.655421 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.04\n",
      "Validation set perplexity: 4.49\n",
      "Average loss at step 4700: 1.662239 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.44\n",
      "Validation set perplexity: 4.58\n",
      "Average loss at step 4800: 1.645729 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.28\n",
      "Validation set perplexity: 4.60\n",
      "Average loss at step 4900: 1.654934 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.35\n",
      "Validation set perplexity: 4.50\n",
      "Average loss at step 5000: 1.655325 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.47\n",
      "================================================================================\n",
      "gnificht their air was atrerimes from fvrhanne than was whole book has which arab\n",
      "eu preyrracle solowelds which also well caudom album woll and will was viering th\n",
      "yal action airideas with a regademblany supporteg a more was the which bedter bec\n",
      " queen are the with pather early capanifor of the that president n thess amer has\n",
      "gs that the busit antist version was sexte is asgories of nut contractring and im\n",
      "================================================================================\n",
      "Validation set perplexity: 4.53\n",
      "Average loss at step 5100: 1.640084 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.16\n",
      "Validation set perplexity: 4.49\n",
      "Average loss at step 5200: 1.636075 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.55\n",
      "Validation set perplexity: 4.45\n",
      "Average loss at step 5300: 1.643451 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.66\n",
      "Validation set perplexity: 4.43\n",
      "Average loss at step 5400: 1.639571 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.21\n",
      "Validation set perplexity: 4.37\n",
      "Average loss at step 5500: 1.628403 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.02\n",
      "Validation set perplexity: 4.37\n",
      "Average loss at step 5600: 1.606119 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.93\n",
      "Validation set perplexity: 4.33\n",
      "Average loss at step 5700: 1.617266 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.18\n",
      "Validation set perplexity: 4.28\n",
      "Average loss at step 5800: 1.638198 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.26\n",
      "Validation set perplexity: 4.30\n",
      "Average loss at step 5900: 1.620338 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.62\n",
      "Validation set perplexity: 4.31\n",
      "Average loss at step 6000: 1.624860 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.36\n",
      "================================================================================\n",
      "bheed while in mineles over the garridange envaliets included fames remaenviolo t\n",
      "ng the inverio is about posnhains to buld and comprined yound methout webmd the r\n",
      "ius stables sides ta there india had were night symmo ato that the trienrest word\n",
      "ld bacts kinds stralidety of the state chessbnol bo histert are projinant a two z\n",
      "jqish s battly is uniham and che phall cate had accurs type alsosally as levelo b\n",
      "================================================================================\n",
      "Validation set perplexity: 4.29\n",
      "Average loss at step 6100: 1.618193 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.78\n",
      "Validation set perplexity: 4.30\n",
      "Average loss at step 6200: 1.623269 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.15\n",
      "Validation set perplexity: 4.31\n",
      "Average loss at step 6300: 1.621061 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.38\n",
      "Validation set perplexity: 4.32\n",
      "Average loss at step 6400: 1.616734 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.36\n",
      "Validation set perplexity: 4.33\n",
      "Average loss at step 6500: 1.594535 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.55\n",
      "Validation set perplexity: 4.34\n",
      "Average loss at step 6600: 1.639126 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.27\n",
      "Validation set perplexity: 4.33\n",
      "Average loss at step 6700: 1.611789 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.86\n",
      "Validation set perplexity: 4.33\n",
      "Average loss at step 6800: 1.616907 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.97\n",
      "Validation set perplexity: 4.35\n",
      "Average loss at step 6900: 1.610608 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.69\n",
      "Validation set perplexity: 4.33\n",
      "Average loss at step 7000: 1.630691 learning rate: 1.000000\n",
      "Minibatch perplexity: 5.49\n",
      "================================================================================\n",
      "uxual unioned the firsts two zero seven three eight five eight three three in for\n",
      "rzed by tomember copieb opp alge haln bestagan ministry alfare actond g subers gi\n",
      "austry invegity of the fagueld and soviethence sicle a computer of the most to a \n",
      " neo this careconde and nine three five referred to blet for one six faces produc\n",
      "rj soumben phost partoment as the desigther canese a graw colo one eight agender \n",
      "================================================================================\n",
      "Validation set perplexity: 4.29\n"
     ]
    }
   ],
   "source": [
    "num_steps = 7001\n",
    "summary_frequency = 100\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print('Initialized')\n",
    "    mean_loss = 0\n",
    "    for step in range(num_steps):\n",
    "        batches = train_batches.next()\n",
    "        feed_dict = dict()\n",
    "        for i in range(num_unrollings + 2):\n",
    "            feed_dict[train_data[i]] = batches[i]\n",
    "        _, l, predictions, lr = session.run(\n",
    "            [optimizer, loss, train_prediction, learning_rate], feed_dict=feed_dict)\n",
    "        mean_loss += l\n",
    "        if step % summary_frequency == 0:\n",
    "            if step > 0:\n",
    "                mean_loss = mean_loss / summary_frequency\n",
    "            # The mean loss is an estimate of the loss over the last few batches.\n",
    "            print(\n",
    "                'Average loss at step %d: %f learning rate: %f' % (step, mean_loss, lr))\n",
    "            mean_loss = 0\n",
    "            labels = np.concatenate(list(batches)[2:])\n",
    "            print('Minibatch perplexity: %.2f' % float(\n",
    "                np.exp(logprob(predictions, labels))))\n",
    "            if step % (summary_frequency * 10) == 0:\n",
    "                # Generate some samples.\n",
    "                print('=' * 80)\n",
    "                for _ in range(5):\n",
    "                    feed = [sample(random_distribution()), sample(random_distribution())] \n",
    "                    sentence = characters(feed[0])[0] + characters(feed[1])[0]\n",
    "                    reset_sample_state.run()\n",
    "                    for _ in range(79):\n",
    "                        prediction = sample_prediction.eval(\n",
    "                            {sample_input[0]: feed[0], sample_input[1]: feed[1]})\n",
    "                        feed[0] = feed[1]\n",
    "                        feed[1] = sample(prediction)\n",
    "                        sentence += characters(feed[1])[0]\n",
    "                    print(sentence)\n",
    "                print('=' * 80)\n",
    "            # Measure validation set perplexity.\n",
    "            reset_sample_state.run()\n",
    "            valid_logprob = 0\n",
    "            for _ in range(valid_size):\n",
    "                b = valid_batches.next()\n",
    "                predictions = sample_prediction.eval(\n",
    "                    {sample_input[0]: b[0], sample_input[1]: b[1]})\n",
    "                valid_logprob = valid_logprob + logprob(predictions, b[2])\n",
    "            print('Validation set perplexity: %.2f' % float(np.exp(\n",
    "                valid_logprob / valid_size)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Y5tapX3kpcqZ"
   },
   "source": [
    "---\n",
    "Problem 3\n",
    "---------\n",
    "\n",
    "(difficult!)\n",
    "\n",
    "Write a sequence-to-sequence LSTM which mirrors all the words in a sentence. For example, if your input is:\n",
    "\n",
    "    the quick brown fox\n",
    "    \n",
    "the model should attempt to output:\n",
    "\n",
    "    eht kciuq nworb xof\n",
    "    \n",
    "Refer to the lecture on how to put together a sequence-to-sequence model, as well as [this article](http://arxiv.org/abs/1409.3215) for best practices.\n",
    "\n",
    "---"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "6_lstm.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}