text-classification/45.nested-lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from utils import *\n",
    "import tensorflow as tf\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['negative', 'positive']\n",
      "10662\n",
      "10662\n"
     ]
    }
   ],
   "source": [
    "trainset = sklearn.datasets.load_files(container_path = 'data', encoding = 'UTF-8')\n",
    "trainset.data, trainset.target = separate_dataset(trainset,1.0)\n",
    "print (trainset.target_names)\n",
    "print (len(trainset.data))\n",
    "print (len(trainset.target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ONEHOT = np.zeros((len(trainset.data),len(trainset.target_names)))\n",
    "ONEHOT[np.arange(len(trainset.data)),trainset.target] = 1.0\n",
    "train_X, test_X, train_Y, test_Y, train_onehot, test_onehot = train_test_split(trainset.data, \n",
    "                                                                               trainset.target, \n",
    "                                                                               ONEHOT, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 20465\n",
      "Most common words [('the', 10129), ('a', 7312), ('and', 6199), ('of', 6063), ('to', 4233), ('is', 3378)]\n",
      "Sample data [4, 668, 9, 2822, 8, 22, 4, 3424, 16369, 97] ['the', 'rock', 'is', 'destined', 'to', 'be', 'the', '21st', 'centurys', 'new']\n"
     ]
    }
   ],
   "source": [
    "concat = ' '.join(trainset.data).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary['GO']\n",
    "PAD = dictionary['PAD']\n",
    "EOS = dictionary['EOS']\n",
    "UNK = dictionary['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python.framework import constant_op\n",
    "from tensorflow.python.framework import dtypes\n",
    "from tensorflow.python.ops import rnn_cell_impl\n",
    "from tensorflow.python.ops import array_ops\n",
    "from tensorflow.python.ops import init_ops\n",
    "from tensorflow.python.ops import math_ops\n",
    "from tensorflow.python.ops import nn_ops\n",
    "from tensorflow.python.platform import tf_logging as logging\n",
    "from tensorflow.python.layers import base as base_layer\n",
    "\n",
    "_BIAS_VARIABLE_NAME = \"bias\"\n",
    "_WEIGHTS_VARIABLE_NAME = \"kernel\"\n",
    "\n",
    "\n",
    "class NLSTMCell(rnn_cell_impl.RNNCell):\n",
    "    def __init__(self, num_units, depth, forget_bias=1.0,\n",
    "               state_is_tuple=True, use_peepholes=True,\n",
    "               activation=None, gate_activation=None,\n",
    "               cell_activation=None,\n",
    "               initializer=None,\n",
    "               input_gate_initializer=None,\n",
    "               use_bias=True, reuse=None, name=None):\n",
    "    \n",
    "        super(NLSTMCell, self).__init__(_reuse=reuse, name=name)\n",
    "        if not state_is_tuple:\n",
    "            logging.warn(\"%s: Using a concatenated state is slower and will soon be \"\n",
    "                   \"deprecated.  Use state_is_tuple=True.\", self)\n",
    "\n",
    "        self.input_spec = base_layer.InputSpec(ndim=2)\n",
    "        self._num_units = num_units\n",
    "        self._forget_bias = forget_bias\n",
    "        self._state_is_tuple = state_is_tuple\n",
    "        self._use_peepholes = use_peepholes\n",
    "        self._depth = depth\n",
    "        self._activation = activation or math_ops.tanh\n",
    "        self._gate_activation = gate_activation or math_ops.sigmoid\n",
    "        self._cell_activation = cell_activation or array_ops.identity\n",
    "        self._initializer = initializer or init_ops.orthogonal_initializer()\n",
    "        self._input_gate_initializer = (input_gate_initializer \n",
    "                                    or init_ops.glorot_normal_initializer())\n",
    "        self._use_bias = use_bias\n",
    "        self._kernels = None\n",
    "        self._biases = None\n",
    "        self.built = False\n",
    "\n",
    "    @property\n",
    "    def state_size(self):\n",
    "        if self._state_is_tuple:\n",
    "            return tuple([self._num_units] * (self.depth + 1))\n",
    "        else:\n",
    "            return self._num_units * (self.depth + 1)\n",
    "\n",
    "    @property\n",
    "    def output_size(self):\n",
    "        return self._num_units\n",
    "\n",
    "    @property\n",
    "    def depth(self):\n",
    "        return self._depth\n",
    "\n",
    "    def build(self, inputs_shape):\n",
    "        if inputs_shape[1].value is None:\n",
    "            raise ValueError(\"Expected inputs.shape[-1] to be known, saw shape: %s\" % inputs_shape)\n",
    "\n",
    "        input_depth = inputs_shape[1].value\n",
    "        h_depth = self._num_units\n",
    "        self._kernels = []\n",
    "        if self._use_bias:\n",
    "            self._biases = []\n",
    "\n",
    "        if self._use_peepholes:\n",
    "            self._peep_kernels = []\n",
    "        for i in range(self.depth):\n",
    "            if i == 0:\n",
    "                input_kernel = self.add_variable(\"input_gate_kernel\",\n",
    "                                                 shape=[input_depth, 4 * self._num_units],\n",
    "                                                 initializer=self._input_gate_initializer)\n",
    "                hidden_kernel = self.add_variable(\"hidden_gate_kernel\",\n",
    "                                                  shape=[h_depth, 4 * self._num_units],\n",
    "                                                  initializer=self._initializer)\n",
    "                kernel = tf.concat([input_kernel, hidden_kernel],\n",
    "                                   axis=0, name=\"kernel_0\")\n",
    "                self._kernels.append(kernel)\n",
    "            else:\n",
    "                self._kernels.append(self.add_variable(\"kernel_{}\".format(i),\n",
    "                                                       shape=[2 * h_depth, 4 * self._num_units],\n",
    "                                                       initializer=self._initializer))\n",
    "            if self._use_bias:\n",
    "                self._biases.append(self.add_variable(\"bias_{}\".format(i),\n",
    "                                                      shape=[4 * self._num_units],\n",
    "                                                      initializer=init_ops.zeros_initializer(dtype=self.dtype)))\n",
    "            if self._use_peepholes:\n",
    "                self._peep_kernels.append(self.add_variable(\"peep_kernel_{}\".format(i),\n",
    "                                                            shape=[h_depth, 3 * self._num_units],\n",
    "                                                            initializer=self._initializer))\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def _recurrence(self, inputs, hidden_state, cell_states, depth):\n",
    "\n",
    "        sigmoid = math_ops.sigmoid\n",
    "        one = constant_op.constant(1, dtype=dtypes.int32)\n",
    "        c = cell_states[depth]\n",
    "        h = hidden_state\n",
    "\n",
    "        gate_inputs = math_ops.matmul(array_ops.concat([inputs, h], 1), self._kernels[depth])\n",
    "        if self._use_bias:\n",
    "            gate_inputs = nn_ops.bias_add(gate_inputs, self._biases[depth])\n",
    "        if self._use_peepholes:\n",
    "            peep_gate_inputs = math_ops.matmul(c, self._peep_kernels[depth])\n",
    "        i_peep, f_peep, o_peep = array_ops.split(value=peep_gate_inputs, num_or_size_splits=3, axis=one)\n",
    "\n",
    "        i, j, f, o = array_ops.split(value=gate_inputs, num_or_size_splits=4, axis=one)\n",
    "        if self._use_peepholes:\n",
    "            i += i_peep\n",
    "            f += f_peep\n",
    "            o += o_peep \n",
    "\n",
    "        if self._use_peepholes:\n",
    "            peep_gate_inputs = math_ops.matmul(c, self._peep_kernels[depth])\n",
    "            i_peep, f_peep, o_peep = array_ops.split(value=peep_gate_inputs, num_or_size_splits=3, axis=one)\n",
    "            i += i_peep\n",
    "            f += f_peep\n",
    "            o += o_peep \n",
    "\n",
    "        add = math_ops.add\n",
    "        multiply = math_ops.multiply\n",
    "\n",
    "        if self._use_bias:\n",
    "            forget_bias_tensor = constant_op.constant(self._forget_bias, dtype=f.dtype)\n",
    "            f = add(f, forget_bias_tensor)\n",
    "\n",
    "        inner_hidden = multiply(c, self._gate_activation(f))\n",
    "\n",
    "        if depth == 0:\n",
    "            inner_input = multiply(self._gate_activation(i), self._cell_activation(j))\n",
    "        else:\n",
    "            inner_input = multiply(self._gate_activation(i), self._activation(j))\n",
    "\n",
    "        if depth == (self.depth - 1):\n",
    "            new_c = add(inner_hidden, inner_input)\n",
    "            new_cs = [new_c]\n",
    "        else:\n",
    "            new_c, new_cs = self._recurrence(inputs=inner_input,\n",
    "                                             hidden_state=inner_hidden,\n",
    "                                             cell_states=cell_states,\n",
    "                                             depth=depth + 1)\n",
    "        new_h = multiply(self._activation(new_c), self._gate_activation(o))\n",
    "        new_cs = [new_h] + new_cs\n",
    "        return new_h, new_cs\n",
    "\n",
    "    def call(self, inputs, state):\n",
    "        if not self._state_is_tuple:\n",
    "            states = array_ops.split(state, self.depth + 1, axis=1)\n",
    "        else:\n",
    "            states = state\n",
    "        hidden_state = states[0]\n",
    "        cell_states = states[1:]\n",
    "        outputs, next_state = self._recurrence(inputs, hidden_state, cell_states, 0)\n",
    "        if self._state_is_tuple:\n",
    "            next_state = tuple(next_state)\n",
    "        else:\n",
    "            next_state = array_ops.concat(next_state, axis=1)\n",
    "        return outputs, next_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self, size_layer, embedded_size,\n",
    "                 dict_size, dimension_output, learning_rate, batch_size,\n",
    "                timestamp, depth=1):\n",
    "        self.X = tf.placeholder(tf.int32, [batch_size, maxlen])\n",
    "        self.Y = tf.placeholder(tf.float32, [batch_size, dimension_output])\n",
    "        encoder_embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        encoder_embedded = tf.nn.embedding_lookup(encoder_embeddings, self.X)\n",
    "        \n",
    "        cell = NLSTMCell(size_layer, depth)\n",
    "        init_state = cell.zero_state(batch_size, dtype=dtypes.float32)\n",
    "        state = init_state\n",
    "        outputs = []\n",
    "        with tf.variable_scope(\"RNN\"):\n",
    "            for time_step in range(timestamp):\n",
    "                if time_step > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "                out, state = cell(encoder_embedded[:, time_step, :], state)\n",
    "                outputs.append(out)\n",
    "        outputs = tf.reshape(tf.concat(outputs,axis=1),[batch_size,timestamp,size_layer])\n",
    "        W = tf.get_variable('w',shape=(size_layer, dimension_output),initializer=tf.orthogonal_initializer())\n",
    "        b = tf.get_variable('b',shape=(dimension_output),initializer=tf.zeros_initializer())\n",
    "        self.logits = tf.matmul(outputs[:, -1], W) + b\n",
    "        self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits, labels = self.Y))\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        correct_pred = tf.equal(tf.argmax(self.logits, 1), tf.argmax(self.Y, 1))\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 64\n",
    "embedded_size = 128\n",
    "dimension_output = len(trainset.target_names)\n",
    "learning_rate = 1e-3\n",
    "maxlen = 50\n",
    "batch_size = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(size_layer,embedded_size,vocabulary_size+4,\n",
    "              dimension_output,learning_rate,\n",
    "             batch_size,maxlen)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.000000, current acc: 0.531738\n",
      "time taken: 2.8110363483428955\n",
      "epoch: 0, training loss: 0.695090, training acc: 0.517756, valid loss: 0.689452, valid acc: 0.531738\n",
      "\n",
      "epoch: 1, pass acc: 0.531738, current acc: 0.592773\n",
      "time taken: 2.0530991554260254\n",
      "epoch: 1, training loss: 0.668451, training acc: 0.598130, valid loss: 0.669185, valid acc: 0.592773\n",
      "\n",
      "epoch: 2, pass acc: 0.592773, current acc: 0.634766\n",
      "time taken: 2.054539918899536\n",
      "epoch: 2, training loss: 0.594847, training acc: 0.699574, valid loss: 0.646298, valid acc: 0.634766\n",
      "\n",
      "epoch: 3, pass acc: 0.634766, current acc: 0.668945\n",
      "time taken: 2.0540168285369873\n",
      "epoch: 3, training loss: 0.471607, training acc: 0.790365, valid loss: 0.666460, valid acc: 0.668945\n",
      "\n",
      "epoch: 4, pass acc: 0.668945, current acc: 0.698242\n",
      "time taken: 2.0545835494995117\n",
      "epoch: 4, training loss: 0.345011, training acc: 0.862571, valid loss: 0.695080, valid acc: 0.698242\n",
      "\n",
      "epoch: 5, pass acc: 0.698242, current acc: 0.708984\n",
      "time taken: 2.0536608695983887\n",
      "epoch: 5, training loss: 0.250333, training acc: 0.911340, valid loss: 0.695614, valid acc: 0.708984\n",
      "\n",
      "epoch: 6, pass acc: 0.708984, current acc: 0.719238\n",
      "time taken: 2.05330491065979\n",
      "epoch: 6, training loss: 0.168091, training acc: 0.945668, valid loss: 0.761937, valid acc: 0.719238\n",
      "\n",
      "epoch: 7, pass acc: 0.719238, current acc: 0.720215\n",
      "time taken: 2.0530500411987305\n",
      "epoch: 7, training loss: 0.111187, training acc: 0.968277, valid loss: 0.817810, valid acc: 0.720215\n",
      "\n",
      "time taken: 2.050994873046875\n",
      "epoch: 8, training loss: 0.077725, training acc: 0.980114, valid loss: 0.883286, valid acc: 0.719238\n",
      "\n",
      "time taken: 2.053058624267578\n",
      "epoch: 9, training loss: 0.056436, training acc: 0.986861, valid loss: 0.986419, valid acc: 0.713379\n",
      "\n",
      "time taken: 2.0518455505371094\n",
      "epoch: 10, training loss: 0.041559, training acc: 0.991004, valid loss: 1.005594, valid acc: 0.717285\n",
      "\n",
      "epoch: 11, pass acc: 0.720215, current acc: 0.721191\n",
      "time taken: 2.052720546722412\n",
      "epoch: 11, training loss: 0.032299, training acc: 0.993134, valid loss: 1.040796, valid acc: 0.721191\n",
      "\n",
      "epoch: 12, pass acc: 0.721191, current acc: 0.722656\n",
      "time taken: 2.0508596897125244\n",
      "epoch: 12, training loss: 0.024856, training acc: 0.995028, valid loss: 1.161813, valid acc: 0.722656\n",
      "\n",
      "time taken: 2.0544140338897705\n",
      "epoch: 13, training loss: 0.019435, training acc: 0.995975, valid loss: 1.237681, valid acc: 0.714355\n",
      "\n",
      "time taken: 2.05448317527771\n",
      "epoch: 14, training loss: 0.012193, training acc: 0.997869, valid loss: 1.242486, valid acc: 0.713867\n",
      "\n",
      "time taken: 2.0531187057495117\n",
      "epoch: 15, training loss: 0.008589, training acc: 0.998580, valid loss: 1.282168, valid acc: 0.721191\n",
      "\n",
      "time taken: 2.054352045059204\n",
      "epoch: 16, training loss: 0.006250, training acc: 0.998935, valid loss: 1.329985, valid acc: 0.721680\n",
      "\n",
      "time taken: 2.05053973197937\n",
      "epoch: 17, training loss: 0.004599, training acc: 0.999171, valid loss: 1.405500, valid acc: 0.721680\n",
      "\n",
      "break epoch:18\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 5, 0, 0, 0\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:%d\\n'%(EPOCH))\n",
    "        break\n",
    "        \n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    for i in range(0, (len(train_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(train_X[i:i+batch_size],dictionary,maxlen)\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : train_onehot[i:i+batch_size]})\n",
    "        train_loss += loss\n",
    "        train_acc += acc\n",
    "    \n",
    "    for i in range(0, (len(test_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(test_X[i:i+batch_size],dictionary,maxlen)\n",
    "        acc, loss = sess.run([model.accuracy, model.cost], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : test_onehot[i:i+batch_size]})\n",
    "        test_loss += loss\n",
    "        test_acc += acc\n",
    "    \n",
    "    train_loss /= (len(train_X) // batch_size)\n",
    "    train_acc /= (len(train_X) // batch_size)\n",
    "    test_loss /= (len(test_X) // batch_size)\n",
    "    test_acc /= (len(test_X) // batch_size)\n",
    "    \n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print('epoch: %d, pass acc: %f, current acc: %f'%(EPOCH,CURRENT_ACC, test_acc))\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "        \n",
    "    print('time taken:', time.time()-lasttime)\n",
    "    print('epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'%(EPOCH,train_loss,\n",
    "                                                                                          train_acc,test_loss,\n",
    "                                                                                          test_acc))\n",
    "    EPOCH += 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}