text-classification/76.transfer-learning-xlnet-base.ipynb

{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!wget https://storage.googleapis.com/xlnet/released_models/cased_L-12_H-768_A-12.zip\n!unzip cased_L-12_H-768_A-12.zip\n!wget https://raw.githubusercontent.com/huseinzol05/NLP-Models-Tensorflow/master/text-classification/utils.py\n!wget https://raw.githubusercontent.com/huseinzol05/NLP-Models-Tensorflow/master/text-classification/data.zip\n!wget https://raw.githubusercontent.com/zihangdai/xlnet/master/xlnet.py\n!wget https://raw.githubusercontent.com/zihangdai/xlnet/master/modeling.py\n!wget https://raw.githubusercontent.com/zihangdai/xlnet/master/prepro_utils.py\n!wget https://raw.githubusercontent.com/zihangdai/xlnet/master/model_utils.py\n!unzip data.zip","execution_count":1,"outputs":[{"output_type":"stream","text":"--2019-08-06 11:21:51--  https://storage.googleapis.com/xlnet/released_models/cased_L-12_H-768_A-12.zip\nResolving storage.googleapis.com (storage.googleapis.com)... 108.177.127.128, 2a00:1450:4013:c07::80\nConnecting to storage.googleapis.com (storage.googleapis.com)|108.177.127.128|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 433638019 (414M) [application/zip]\nSaving to: ‘cased_L-12_H-768_A-12.zip’\n\ncased_L-12_H-768_A- 100%[===================>] 413.55M  51.0MB/s    in 8.4s    \n\n2019-08-06 11:22:00 (49.0 MB/s) - ‘cased_L-12_H-768_A-12.zip’ saved [433638019/433638019]\n\nArchive:  cased_L-12_H-768_A-12.zip\n   creating: xlnet_cased_L-12_H-768_A-12/\n  inflating: xlnet_cased_L-12_H-768_A-12/xlnet_model.ckpt.index  \n  inflating: xlnet_cased_L-12_H-768_A-12/xlnet_model.ckpt.data-00000-of-00001  \n  inflating: xlnet_cased_L-12_H-768_A-12/spiece.model  \n  inflating: xlnet_cased_L-12_H-768_A-12/xlnet_model.ckpt.meta  \n  inflating: xlnet_cased_L-12_H-768_A-12/xlnet_config.json  \n--2019-08-06 11:22:07--  https://raw.githubusercontent.com/huseinzol05/NLP-Models-Tensorflow/master/text-classification/utils.py\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 1815 (1.8K) [text/plain]\nSaving to: ‘utils.py’\n\nutils.py            100%[===================>]   1.77K  --.-KB/s    in 0s      \n\n2019-08-06 11:22:07 (51.0 MB/s) - ‘utils.py’ saved [1815/1815]\n\n--2019-08-06 11:22:07--  https://raw.githubusercontent.com/huseinzol05/NLP-Models-Tensorflow/master/text-classification/data.zip\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 460176 (449K) [application/zip]\nSaving to: ‘data.zip’\n\ndata.zip            100%[===================>] 449.39K  --.-KB/s    in 0.02s   \n\n2019-08-06 11:22:08 (21.8 MB/s) - ‘data.zip’ saved [460176/460176]\n\n--2019-08-06 11:22:08--  https://raw.githubusercontent.com/zihangdai/xlnet/master/xlnet.py\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 9838 (9.6K) [text/plain]\nSaving to: ‘xlnet.py’\n\nxlnet.py            100%[===================>]   9.61K  --.-KB/s    in 0s      \n\n2019-08-06 11:22:09 (87.5 MB/s) - ‘xlnet.py’ saved [9838/9838]\n\n--2019-08-06 11:22:09--  https://raw.githubusercontent.com/zihangdai/xlnet/master/modeling.py\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 28460 (28K) [text/plain]\nSaving to: ‘modeling.py’\n\nmodeling.py         100%[===================>]  27.79K  --.-KB/s    in 0.004s  \n\n2019-08-06 11:22:09 (7.51 MB/s) - ‘modeling.py’ saved [28460/28460]\n\n--2019-08-06 11:22:10--  https://raw.githubusercontent.com/zihangdai/xlnet/master/prepro_utils.py\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 4546 (4.4K) [text/plain]\nSaving to: ‘prepro_utils.py’\n\nprepro_utils.py     100%[===================>]   4.44K  --.-KB/s    in 0s      \n\n2019-08-06 11:22:10 (74.8 MB/s) - ‘prepro_utils.py’ saved [4546/4546]\n\n--2019-08-06 11:22:11--  https://raw.githubusercontent.com/zihangdai/xlnet/master/model_utils.py\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 14078 (14K) [text/plain]\nSaving to: ‘model_utils.py’\n\nmodel_utils.py      100%[===================>]  13.75K  --.-KB/s    in 0.004s  \n\n2019-08-06 11:22:11 (3.68 MB/s) - ‘model_utils.py’ saved [14078/14078]\n\nArchive:  data.zip\n   creating: data/\n   creating: data/positive/\n  inflating: data/positive/positive  \n  inflating: data/.DS_Store          \n   creating: __MACOSX/\n   creating: __MACOSX/data/\n  inflating: __MACOSX/data/._.DS_Store  \n   creating: data/negative/\n  inflating: data/negative/negative  \n","name":"stdout"}]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip3 install sentencepiece","execution_count":2,"outputs":[{"output_type":"stream","text":"Requirement already satisfied: sentencepiece in /opt/conda/lib/python3.6/site-packages (0.1.82)\r\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"import xlnet\nimport numpy as np\nimport tensorflow as tf\nfrom tqdm import tqdm\nimport model_utils","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sentencepiece as spm\nfrom prepro_utils import preprocess_text, encode_ids\n\nsp_model = spm.SentencePieceProcessor()\nsp_model.Load('xlnet_cased_L-12_H-768_A-12/spiece.model')\n\ndef tokenize_fn(text):\n    text = preprocess_text(text, lower= False)\n    return encode_ids(sp_model, text)","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from utils import *\n\ntrainset = sklearn.datasets.load_files(container_path = 'data', encoding = 'UTF-8')\ntrainset.data, trainset.target = separate_dataset(trainset,1.0)\nprint (trainset.target_names)\nprint (len(trainset.data))\nprint (len(trainset.target))","execution_count":5,"outputs":[{"output_type":"stream","text":"['negative', 'positive']\n10662\n10662\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SEQ_LENGTH = 128\n\nSEG_ID_A   = 0\nSEG_ID_B   = 1\nSEG_ID_CLS = 2\nSEG_ID_SEP = 3\nSEG_ID_PAD = 4\n\nspecial_symbols = {\n    \"<unk>\"  : 0,\n    \"<s>\"    : 1,\n    \"</s>\"   : 2,\n    \"<cls>\"  : 3,\n    \"<sep>\"  : 4,\n    \"<pad>\"  : 5,\n    \"<mask>\" : 6,\n    \"<eod>\"  : 7,\n    \"<eop>\"  : 8,\n}\n\nVOCAB_SIZE = 32000\nUNK_ID = special_symbols[\"<unk>\"]\nCLS_ID = special_symbols[\"<cls>\"]\nSEP_ID = special_symbols[\"<sep>\"]\nMASK_ID = special_symbols[\"<mask>\"]\nEOD_ID = special_symbols[\"<eod>\"]\n\ninput_ids, input_masks, segment_ids = [], [], []\n\nfor text in tqdm(trainset.data):\n    tokens_a = tokenize_fn(text)\n    if len(tokens_a) > MAX_SEQ_LENGTH - 2:\n        tokens_a = tokens_a[:(MAX_SEQ_LENGTH - 2)]\n        \n    tokens = []\n    segment_id = []\n    for token in tokens_a:\n        tokens.append(token)\n        segment_id.append(SEG_ID_A)\n    tokens.append(SEP_ID)\n    segment_id.append(SEG_ID_A)\n    tokens.append(CLS_ID)\n    segment_id.append(SEG_ID_CLS)\n    \n    input_id = tokens\n    input_mask = [0] * len(input_id)\n    if len(input_id) < MAX_SEQ_LENGTH:\n        delta_len = MAX_SEQ_LENGTH - len(input_id)\n        input_id = [0] * delta_len + input_id\n        input_mask = [1] * delta_len + input_mask\n        segment_id = [SEG_ID_PAD] * delta_len + segment_id\n    \n    input_ids.append(input_id)\n    input_masks.append(input_mask)\n    segment_ids.append(segment_id)","execution_count":6,"outputs":[{"output_type":"stream","text":"100%|██████████| 10662/10662 [00:01<00:00, 8699.94it/s]\n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"kwargs = dict(\n      is_training=True,\n      use_tpu=False,\n      use_bfloat16=False,\n      dropout=0,\n      dropatt=0,\n      init='normal',\n      init_range=0.1,\n      init_std=0.02,\n      clamp_len=-1)\n\nxlnet_parameters = xlnet.RunConfig(**kwargs)\nxlnet_config = xlnet.XLNetConfig(json_path='xlnet_cased_L-12_H-768_A-12/xlnet_config.json')","execution_count":7,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epoch = 10\nbatch_size = 10\nwarmup_proportion = 0.1\nnum_train_steps = int(len(input_ids) / batch_size * epoch)\nnum_warmup_steps = int(num_train_steps * warmup_proportion)\nprint(num_train_steps, num_warmup_steps)\n\ntraining_parameters = dict(\n      decay_method = 'poly',\n      train_steps = num_train_steps,\n      learning_rate = 2e-5,\n      warmup_steps = num_warmup_steps,\n      min_lr_ratio = 0.0,\n      weight_decay = 0.00,\n      adam_epsilon = 1e-8,\n      num_core_per_host = 1,\n      lr_layer_decay_rate = 1,\n      use_tpu=False,\n      use_bfloat16=False,\n      dropout=0.0,\n      dropatt=0.0,\n      init='normal',\n      init_range=0.1,\n      init_std=0.02,\n      clip = 1.0,\n      clamp_len=-1,)","execution_count":8,"outputs":[{"output_type":"stream","text":"10662 1066\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Parameter:\n    def __init__(self, decay_method, warmup_steps, weight_decay, adam_epsilon, \n                num_core_per_host, lr_layer_decay_rate, use_tpu, learning_rate, train_steps,\n                min_lr_ratio, clip, **kwargs):\n        self.decay_method = decay_method\n        self.warmup_steps = warmup_steps\n        self.weight_decay = weight_decay\n        self.adam_epsilon = adam_epsilon\n        self.num_core_per_host = num_core_per_host\n        self.lr_layer_decay_rate = lr_layer_decay_rate\n        self.use_tpu = use_tpu\n        self.learning_rate = learning_rate\n        self.train_steps = train_steps\n        self.min_lr_ratio = min_lr_ratio\n        self.clip = clip\n        \ntraining_parameters = Parameter(**training_parameters)","execution_count":9,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model:\n    def __init__(\n        self,\n        dimension_output,\n        learning_rate = 2e-5,\n    ):\n        self.X = tf.placeholder(tf.int32, [None, None])\n        self.segment_ids = tf.placeholder(tf.int32, [None, None])\n        self.input_masks = tf.placeholder(tf.float32, [None, None])\n        self.Y = tf.placeholder(tf.int32, [None])\n        \n        xlnet_model = xlnet.XLNetModel(\n            xlnet_config=xlnet_config,\n            run_config=xlnet_parameters,\n            input_ids=tf.transpose(self.X, [1, 0]),\n            seg_ids=tf.transpose(self.segment_ids, [1, 0]),\n            input_mask=tf.transpose(self.input_masks, [1, 0]))\n        \n        summary = xlnet_model.get_pooled_out(\"last\", True)\n        print(summary)\n        \n        self.logits = tf.layers.dense(summary, dimension_output)\n        \n        self.cost = tf.reduce_mean(\n            tf.nn.sparse_softmax_cross_entropy_with_logits(\n                logits = self.logits, labels = self.Y\n            )\n        )\n        \n        self.optimizer, self.learning_rate, _ = model_utils.get_train_op(training_parameters, self.cost)\n        \n        correct_pred = tf.equal(\n            tf.argmax(self.logits, 1, output_type = tf.int32), self.Y\n        )\n        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))","execution_count":10,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dimension_output = 2\nlearning_rate = 2e-5\n\ntf.reset_default_graph()\nsess = tf.InteractiveSession()\nmodel = Model(\n    dimension_output,\n    learning_rate\n)\n\nsess.run(tf.global_variables_initializer())","execution_count":11,"outputs":[{"output_type":"stream","text":"Tensor(\"model_1/sequnece_summary/summary/Tanh:0\", shape=(?, 768), dtype=float32)\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"import collections\nimport re\n\ndef get_assignment_map_from_checkpoint(tvars, init_checkpoint):\n    \"\"\"Compute the union of the current variables and checkpoint variables.\"\"\"\n    assignment_map = {}\n    initialized_variable_names = {}\n\n    name_to_variable = collections.OrderedDict()\n    for var in tvars:\n        name = var.name\n        m = re.match('^(.*):\\\\d+$', name)\n        if m is not None:\n            name = m.group(1)\n        name_to_variable[name] = var\n\n    init_vars = tf.train.list_variables(init_checkpoint)\n\n    assignment_map = collections.OrderedDict()\n    for x in init_vars:\n        (name, var) = (x[0], x[1])\n        if name not in name_to_variable:\n            continue\n        assignment_map[name] = name_to_variable[name]\n        initialized_variable_names[name] = 1\n        initialized_variable_names[name + ':0'] = 1\n\n    return (assignment_map, initialized_variable_names)","execution_count":12,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tvars = tf.trainable_variables()\ncheckpoint = 'xlnet_cased_L-12_H-768_A-12/xlnet_model.ckpt'\nassignment_map, initialized_variable_names = get_assignment_map_from_checkpoint(tvars, \n                                                                                checkpoint)","execution_count":13,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"saver = tf.train.Saver(var_list = assignment_map)\nsaver.restore(sess, checkpoint)","execution_count":14,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_input_ids, test_input_ids, train_input_masks, test_input_masks, train_segment_ids, test_segment_ids, train_Y, test_Y = train_test_split(\n    input_ids, input_masks, segment_ids, trainset.target, test_size = 0.2\n)","execution_count":15,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\nimport time\n\nEARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 3, 0, 0, 0\n\nwhile True:\n    lasttime = time.time()\n    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n        print('break epoch:%d\\n' % (EPOCH))\n        break\n\n    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n    pbar = tqdm(\n        range(0, len(train_input_ids), batch_size), desc = 'train minibatch loop'\n    )\n    for i in pbar:\n        index = min(i + batch_size, len(train_input_ids))\n        batch_x = train_input_ids[i: index]\n        batch_masks = train_input_masks[i: index]\n        batch_segment = train_segment_ids[i: index]\n        batch_y = train_Y[i: index]\n        acc, cost, _ = sess.run(\n            [model.accuracy, model.cost, model.optimizer],\n            feed_dict = {\n                model.Y: batch_y,\n                model.X: batch_x,\n                model.segment_ids: batch_segment,\n                model.input_masks: batch_masks\n            },\n        )\n        assert not np.isnan(cost)\n        train_loss += cost\n        train_acc += acc\n        pbar.set_postfix(cost = cost, accuracy = acc)\n    pbar = tqdm(range(0, len(test_input_ids), batch_size), desc = 'test minibatch loop')\n    for i in pbar:\n        index = min(i + batch_size, len(test_input_ids))\n        batch_x = test_input_ids[i: index]\n        batch_masks = test_input_masks[i: index]\n        batch_segment = test_segment_ids[i: index]\n        batch_y = test_Y[i: index]\n        acc, cost = sess.run(\n            [model.accuracy, model.cost],\n            feed_dict = {\n                model.Y: batch_y,\n                model.X: batch_x,\n                model.segment_ids: batch_segment,\n                model.input_masks: batch_masks\n            },\n        )\n        test_loss += cost\n        test_acc += acc\n        pbar.set_postfix(cost = cost, accuracy = acc)\n\n    train_loss /= len(train_input_ids) / batch_size\n    train_acc /= len(train_input_ids) / batch_size\n    test_loss /= len(test_input_ids) / batch_size\n    test_acc /= len(test_input_ids) / batch_size\n\n    if test_acc > CURRENT_ACC:\n        print(\n            'epoch: %d, pass acc: %f, current acc: %f'\n            % (EPOCH, CURRENT_ACC, test_acc)\n        )\n        CURRENT_ACC = test_acc\n        CURRENT_CHECKPOINT = 0\n    else:\n        CURRENT_CHECKPOINT += 1\n        \n    print('time taken:', time.time() - lasttime)\n    print(\n        'epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'\n        % (EPOCH, train_loss, train_acc, test_loss, test_acc)\n    )\n    EPOCH += 1","execution_count":16,"outputs":[{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 853/853 [03:45<00:00,  4.00it/s, accuracy=0.889, cost=0.34]\ntest minibatch loop: 100%|██████████| 214/214 [00:21<00:00, 10.01it/s, accuracy=0.667, cost=0.335]\ntrain minibatch loop:   0%|          | 0/853 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 0, pass acc: 0.000000, current acc: 0.778559\ntime taken: 247.1283679008484\nepoch: 0, training loss: 0.580164, training acc: 0.695848, valid loss: 0.473891, valid acc: 0.778559\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 853/853 [03:38<00:00,  4.02it/s, accuracy=0.889, cost=0.116]\ntest minibatch loop: 100%|██████████| 214/214 [00:19<00:00, 11.67it/s, accuracy=0.667, cost=0.278]\ntrain minibatch loop:   0%|          | 0/853 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"time taken: 238.4773871898651\nepoch: 1, training loss: 0.413650, training acc: 0.836427, valid loss: 0.888086, valid acc: 0.771527\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 853/853 [03:39<00:00,  3.97it/s, accuracy=0.889, cost=0.48]\ntest minibatch loop: 100%|██████████| 214/214 [00:19<00:00, 11.72it/s, accuracy=1, cost=0.00107] \ntrain minibatch loop:   0%|          | 0/853 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 2, pass acc: 0.778559, current acc: 0.812002\ntime taken: 239.17225623130798\nepoch: 2, training loss: 0.240799, training acc: 0.937377, valid loss: 1.104490, valid acc: 0.812002\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 853/853 [03:40<00:00,  4.02it/s, accuracy=1, cost=0.000191]\ntest minibatch loop: 100%|██████████| 214/214 [00:19<00:00, 11.61it/s, accuracy=1, cost=0.000941]\ntrain minibatch loop:   0%|          | 0/853 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"time taken: 240.28951001167297\nepoch: 3, training loss: 0.118257, training acc: 0.974557, valid loss: 1.334923, valid acc: 0.811064\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 853/853 [03:40<00:00,  4.01it/s, accuracy=1, cost=9.07e-5] \ntest minibatch loop: 100%|██████████| 214/214 [00:19<00:00, 11.71it/s, accuracy=1, cost=0.00241] \ntrain minibatch loop:   0%|          | 0/853 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 4, pass acc: 0.812002, current acc: 0.813877\ntime taken: 240.28020524978638\nepoch: 4, training loss: 0.047423, training acc: 0.991206, valid loss: 1.382043, valid acc: 0.813877\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 853/853 [03:39<00:00,  4.02it/s, accuracy=1, cost=2.14e-5] \ntest minibatch loop: 100%|██████████| 214/214 [00:20<00:00, 11.73it/s, accuracy=0.667, cost=2.48]\ntrain minibatch loop:   0%|          | 0/853 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"time taken: 240.0542507171631\nepoch: 5, training loss: 0.019984, training acc: 0.996483, valid loss: 1.676867, valid acc: 0.803876\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 853/853 [03:40<00:00,  4.01it/s, accuracy=1, cost=4.56e-6] \ntest minibatch loop: 100%|██████████| 214/214 [00:19<00:00, 11.47it/s, accuracy=1, cost=0.00393] \ntrain minibatch loop:   0%|          | 0/853 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 6, pass acc: 0.813877, current acc: 0.820441\ntime taken: 240.26299619674683\nepoch: 6, training loss: 0.014450, training acc: 0.997303, valid loss: 1.631325, valid acc: 0.820441\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 853/853 [03:40<00:00,  4.00it/s, accuracy=1, cost=5.72e-6] \ntest minibatch loop: 100%|██████████| 214/214 [00:20<00:00, 11.68it/s, accuracy=1, cost=0.0224]  \ntrain minibatch loop:   0%|          | 0/853 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"time taken: 240.1789836883545\nepoch: 7, training loss: 0.010327, training acc: 0.998241, valid loss: 1.763179, valid acc: 0.811064\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 853/853 [03:40<00:00,  4.02it/s, accuracy=1, cost=4.77e-7] \ntest minibatch loop: 100%|██████████| 214/214 [00:20<00:00, 11.49it/s, accuracy=0.667, cost=2.62]\ntrain minibatch loop:   0%|          | 0/853 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"time taken: 240.35079336166382\nepoch: 8, training loss: 0.005861, training acc: 0.998945, valid loss: 2.057145, valid acc: 0.809970\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 853/853 [03:40<00:00,  4.01it/s, accuracy=1, cost=5.3e-7]  \ntest minibatch loop: 100%|██████████| 214/214 [00:19<00:00, 11.61it/s, accuracy=0.667, cost=4.01]","name":"stderr"},{"output_type":"stream","text":"time taken: 240.4986753463745\nepoch: 9, training loss: 0.004468, training acc: 0.999414, valid loss: 2.054252, valid acc: 0.808564\n\nbreak epoch:10\n\n","name":"stdout"},{"output_type":"stream","text":"\n","name":"stderr"}]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}