Flair Tutorial 8.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Flair Tutorial 8.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T7bQTMHGaxzA",
        "colab_type": "text"
      },
      "source": [
        "# Tutorial 8: Model Tuning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5hzW6vNma1EV",
        "colab_type": "text"
      },
      "source": [
        "## A. Selecting Hyper Parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dePrGEqX9W9j",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "d6c9f44e-73ec-4e6f-96cf-541013de0c12"
      },
      "source": [
        "!pip install flair"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: flair in /usr/local/lib/python3.6/dist-packages (0.4.4)\n",
            "Requirement already satisfied: matplotlib>=2.2.3 in /usr/local/lib/python3.6/dist-packages (from flair) (3.1.2)\n",
            "Requirement already satisfied: mpld3==0.3 in /usr/local/lib/python3.6/dist-packages (from flair) (0.3)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (from flair) (0.4.2)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from flair) (2019.12.9)\n",
            "Requirement already satisfied: sklearn in /usr/local/lib/python3.6/dist-packages (from flair) (0.0)\n",
            "Requirement already satisfied: hyperopt>=0.1.1 in /usr/local/lib/python3.6/dist-packages (from flair) (0.1.2)\n",
            "Requirement already satisfied: pytest>=3.6.4 in /usr/local/lib/python3.6/dist-packages (from flair) (3.6.4)\n",
            "Requirement already satisfied: deprecated>=1.2.4 in /usr/local/lib/python3.6/dist-packages (from flair) (1.2.7)\n",
            "Requirement already satisfied: ipython-genutils==0.2.0 in /usr/local/lib/python3.6/dist-packages (from flair) (0.2.0)\n",
            "Requirement already satisfied: ipython==7.6.1 in /usr/local/lib/python3.6/dist-packages (from flair) (7.6.1)\n",
            "Requirement already satisfied: langdetect in /usr/local/lib/python3.6/dist-packages (from flair) (1.0.7)\n",
            "Requirement already satisfied: sqlitedict>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from flair) (1.6.0)\n",
            "Requirement already satisfied: transformers>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from flair) (2.2.2)\n",
            "Requirement already satisfied: tiny-tokenizer[all] in /usr/local/lib/python3.6/dist-packages (from flair) (3.0.1)\n",
            "Requirement already satisfied: gensim>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from flair) (3.6.0)\n",
            "Requirement already satisfied: torch>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from flair) (1.3.1)\n",
            "Requirement already satisfied: pymongo in /usr/local/lib/python3.6/dist-packages (from flair) (3.10.0)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.20 in /usr/local/lib/python3.6/dist-packages (from flair) (1.24.3)\n",
            "Requirement already satisfied: segtok>=1.5.7 in /usr/local/lib/python3.6/dist-packages (from flair) (1.5.7)\n",
            "Requirement already satisfied: bpemb>=0.2.9 in /usr/local/lib/python3.6/dist-packages (from flair) (0.3.0)\n",
            "Requirement already satisfied: tqdm>=4.26.0 in /usr/local/lib/python3.6/dist-packages (from flair) (4.28.1)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.6/dist-packages (from flair) (0.8.6)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (0.10.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (2.6.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (1.17.4)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (2.4.5)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision->flair) (4.3.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from torchvision->flair) (1.12.0)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sklearn->flair) (0.21.3)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from hyperopt>=0.1.1->flair) (0.16.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from hyperopt>=0.1.1->flair) (1.3.3)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.6/dist-packages (from hyperopt>=0.1.1->flair) (2.4)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (19.3.0)\n",
            "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (1.8.0)\n",
            "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (1.3.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (42.0.2)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (8.0.2)\n",
            "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (0.7.1)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.6/dist-packages (from deprecated>=1.2.4->flair) (1.11.2)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (4.3.3)\n",
            "Requirement already satisfied: jedi>=0.10 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (0.15.1)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (0.1.0)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (4.7.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (4.4.1)\n",
            "Requirement already satisfied: prompt-toolkit<2.1.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (2.0.10)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (0.7.5)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (2.1.3)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->flair) (1.10.40)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->flair) (2.21.0)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->flair) (0.1.85)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->flair) (0.0.35)\n",
            "Requirement already satisfied: SudachiPy; extra == \"all\" in /usr/local/lib/python3.6/dist-packages (from tiny-tokenizer[all]->flair) (0.4.2)\n",
            "Requirement already satisfied: natto-py; extra == \"all\" in /usr/local/lib/python3.6/dist-packages (from tiny-tokenizer[all]->flair) (0.9.0)\n",
            "Requirement already satisfied: kytea; extra == \"all\" in /usr/local/lib/python3.6/dist-packages (from tiny-tokenizer[all]->flair) (0.1.4)\n",
            "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.6/dist-packages (from gensim>=3.4.0->flair) (1.9.0)\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from pillow>=4.1.1->torchvision->flair) (0.46)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sklearn->flair) (0.14.1)\n",
            "Requirement already satisfied: parso>=0.5.0 in /usr/local/lib/python3.6/dist-packages (from jedi>=0.10->ipython==7.6.1->flair) (0.5.2)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.6/dist-packages (from pexpect; sys_platform != \"win32\"->ipython==7.6.1->flair) (0.6.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython==7.6.1->flair) (0.1.7)\n",
            "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->flair) (0.2.1)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->flair) (0.9.4)\n",
            "Requirement already satisfied: botocore<1.14.0,>=1.13.40 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->flair) (1.13.40)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.0.0->flair) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.0.0->flair) (2019.11.28)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.0.0->flair) (2.8)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.0.0->flair) (7.0)\n",
            "Requirement already satisfied: sortedcontainers~=2.1.0 in /usr/local/lib/python3.6/dist-packages (from SudachiPy; extra == \"all\"->tiny-tokenizer[all]->flair) (2.1.0)\n",
            "Requirement already satisfied: dartsclone~=0.6.0 in /usr/local/lib/python3.6/dist-packages (from SudachiPy; extra == \"all\"->tiny-tokenizer[all]->flair) (0.6)\n",
            "Requirement already satisfied: cffi in /usr/local/lib/python3.6/dist-packages (from natto-py; extra == \"all\"->tiny-tokenizer[all]->flair) (1.13.2)\n",
            "Requirement already satisfied: boto>=2.32 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim>=3.4.0->flair) (2.49.0)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.40->boto3->transformers>=2.0.0->flair) (0.15.2)\n",
            "Requirement already satisfied: Cython in /usr/local/lib/python3.6/dist-packages (from dartsclone~=0.6.0->SudachiPy; extra == \"all\"->tiny-tokenizer[all]->flair) (0.29.14)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi->natto-py; extra == \"all\"->tiny-tokenizer[all]->flair) (2.19)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5QYYeWp_AGum",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 257
        },
        "outputId": "db121fd2-3790-4d99-d3a6-ba31fd87d82b"
      },
      "source": [
        "from flair.datasets import TREC_6\n",
        "\n",
        "# load your corpus\n",
        "corpus = TREC_6()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 15:51:39,750 https://cogcomp.seas.upenn.edu/Data/QA/QC/train_5500.label not found in cache, downloading to /tmp/tmpobajxg1a\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 335858/335858 [00:00<00:00, 364601.26B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 15:51:41,429 copying /tmp/tmpobajxg1a to cache at /root/.flair/datasets/trec_6/original/train_5500.label\n",
            "2019-12-20 15:51:41,430 removing temp file /tmp/tmpobajxg1a\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 15:51:42,185 https://cogcomp.seas.upenn.edu/Data/QA/QC/TREC_10.label not found in cache, downloading to /tmp/tmpejrvyvrw\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 23354/23354 [00:00<00:00, 125197.82B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 15:51:43,142 copying /tmp/tmpejrvyvrw to cache at /root/.flair/datasets/trec_6/original/TREC_10.label\n",
            "2019-12-20 15:51:43,144 removing temp file /tmp/tmpejrvyvrw\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 15:51:43,167 Reading data from /root/.flair/datasets/trec_6\n",
            "2019-12-20 15:51:43,168 Train: /root/.flair/datasets/trec_6/train.txt\n",
            "2019-12-20 15:51:43,169 Dev: None\n",
            "2019-12-20 15:51:43,170 Test: /root/.flair/datasets/trec_6/test.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nsX7DEf5BiG_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flair.embeddings import FlairEmbeddings, WordEmbeddings"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xFgsAW_FAa9U",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 368
        },
        "outputId": "b449517d-8273-4eb8-8755-f8fdaedc66b9"
      },
      "source": [
        "from hyperopt import hp\n",
        "from flair.hyperparameter.param_selection import SearchSpace, Parameter\n",
        "\n",
        "# define your search space\n",
        "search_space = SearchSpace()\n",
        "search_space.add(Parameter.EMBEDDINGS, hp.choice,\n",
        "                 options = [\n",
        "                            [ WordEmbeddings('en') ],\n",
        "                            [ FlairEmbeddings('news-forward'),\n",
        "                              FlairEmbeddings('news-backward') ]\n",
        "                 ])\n",
        "\n",
        "search_space.add(Parameter.HIDDEN_SIZE, hp.choice, options = [32, 64, 128])\n",
        "search_space.add(Parameter.RNN_LAYERS, hp.choice, options = [1, 2])\n",
        "search_space.add(Parameter.DROPOUT, hp.uniform, low = 0.0, high = 0.5)\n",
        "search_space.add(Parameter.LEARNING_RATE, hp.choice, \n",
        "                 options = [0.05, 0.1, 0.15, 0.2])\n",
        "search_space.add(Parameter.MINI_BATCH_SIZE, hp.choice, \n",
        "                 options = [8, 16, 32])"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 15:57:01,002 https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.3/en-fasttext-news-300d-1M.vectors.npy not found in cache, downloading to /tmp/tmpqq56fojp\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 1200000128/1200000128 [01:50<00:00, 10852034.72B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 15:58:52,820 copying /tmp/tmpqq56fojp to cache at /root/.flair/embeddings/en-fasttext-news-300d-1M.vectors.npy\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 15:58:57,174 removing temp file /tmp/tmpqq56fojp\n",
            "2019-12-20 15:58:58,461 https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.3/en-fasttext-news-300d-1M not found in cache, downloading to /tmp/tmp7hy_s6ft\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 54600983/54600983 [00:07<00:00, 7681124.59B/s] "
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 15:59:06,704 copying /tmp/tmp7hy_s6ft to cache at /root/.flair/embeddings/en-fasttext-news-300d-1M\n",
            "2019-12-20 15:59:06,772 removing temp file /tmp/tmp7hy_s6ft\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:402: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 15:59:12,962 https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.4.1/big-news-forward--h2048-l1-d0.05-lr30-0.25-20/news-forward-0.4.1.pt not found in cache, downloading to /tmp/tmphznnxrv3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 73034624/73034624 [00:09<00:00, 8000063.50B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 15:59:23,216 copying /tmp/tmphznnxrv3 to cache at /root/.flair/embeddings/news-forward-0.4.1.pt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 15:59:23,314 removing temp file /tmp/tmphznnxrv3\n",
            "2019-12-20 15:59:34,651 https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings-v0.4.1/big-news-backward--h2048-l1-d0.05-lr30-0.25-20/news-backward-0.4.1.pt not found in cache, downloading to /tmp/tmpqjilu47e\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 73034575/73034575 [00:08<00:00, 8376182.60B/s] "
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 15:59:44,520 copying /tmp/tmpqjilu47e to cache at /root/.flair/embeddings/news-backward-0.4.1.pt\n",
            "2019-12-20 15:59:44,637 removing temp file /tmp/tmpqjilu47e\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fAjaOX13BePB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# search_space.add(Parameter.EMBEDDINGS, hp.choice,\n",
        "#                  options = [\n",
        "#                             [ FlairEmbeddings('news-forward'),\n",
        "#                               FlairEmbeddings('news-backward') ]\n",
        "#                  ])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sZbjiPmnCi4W",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "a122c34d-ff9f-45c7-9a40-b805ff784c1a"
      },
      "source": [
        "from flair.hyperparameter.param_selection import TextClassifierParamSelector\n",
        "from flair.hyperparameter.param_selection import OptimizationValue\n",
        "\n",
        "# create the parameter selector\n",
        "param_selector = TextClassifierParamSelector(\n",
        "    corpus,\n",
        "    False,\n",
        "    'resources/results',\n",
        "    'lstm',\n",
        "    max_epochs = 50,\n",
        "    training_runs = 3,\n",
        "    optimization_value = OptimizationValue.DEV_SCORE\n",
        ")\n",
        "\n",
        "# start the optimization\n",
        "param_selector.optimize(search_space, max_evals = 100)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 16:07:23,280 Computing label dictionary. Progress:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 4907/4907 [00:00<00:00, 162015.29it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 16:07:23,315 [b'LOC', b'ENTY', b'DESC', b'NUM', b'ABBR', b'HUM']\n",
            "  0%|          | 0/100 [00:00<?, ?it/s, best loss: ?]2019-12-20 16:07:23,333 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:07:23,336 Evaluation run: 1\n",
            "2019-12-20 16:07:23,337 Evaluating parameter combination:\n",
            "2019-12-20 16:07:23,339 \tdropout: 0.4768923447560231\n",
            "2019-12-20 16:07:23,341 \tembeddings: /root/.flair/embeddings/news-forward-0.4.1.pt,/root/.flair/embeddings/news-backward-0.4.1.pt\n",
            "2019-12-20 16:07:23,343 \thidden_size: 128\n",
            "2019-12-20 16:07:23,344 \tlearning_rate: 0.1\n",
            "2019-12-20 16:07:23,346 \tmini_batch_size: 16\n",
            "2019-12-20 16:07:23,347 \trnn_layers: 2\n",
            "2019-12-20 16:07:23,349 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:07:23,413 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:07:23,414 Training run: 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 16:07:23,696 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:07:23,698 Model: \"TextClassifier(\n",
            "  (document_embeddings): DocumentRNNEmbeddings(\n",
            "    (embeddings): StackedEmbeddings(\n",
            "      (list_embedding_0): FlairEmbeddings(\n",
            "        (lm): LanguageModel(\n",
            "          (drop): Dropout(p=0.05, inplace=False)\n",
            "          (encoder): Embedding(300, 100)\n",
            "          (rnn): LSTM(100, 2048)\n",
            "          (decoder): Linear(in_features=2048, out_features=300, bias=True)\n",
            "        )\n",
            "      )\n",
            "      (list_embedding_1): FlairEmbeddings(\n",
            "        (lm): LanguageModel(\n",
            "          (drop): Dropout(p=0.05, inplace=False)\n",
            "          (encoder): Embedding(300, 100)\n",
            "          (rnn): LSTM(100, 2048)\n",
            "          (decoder): Linear(in_features=2048, out_features=300, bias=True)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (word_reprojection_map): Linear(in_features=4096, out_features=4096, bias=True)\n",
            "    (rnn): GRU(4096, 128, num_layers=2, batch_first=True)\n",
            "    (dropout): Dropout(p=0.4768923447560231, inplace=False)\n",
            "  )\n",
            "  (decoder): Linear(in_features=128, out_features=6, bias=True)\n",
            "  (loss_function): CrossEntropyLoss()\n",
            ")\"\n",
            "2019-12-20 16:07:23,699 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:07:23,701 Corpus: \"Corpus: 4907 train + 545 dev + 500 test sentences\"\n",
            "2019-12-20 16:07:23,703 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:07:23,705 Parameters:\n",
            "2019-12-20 16:07:23,707  - learning_rate: \"0.1\"\n",
            "2019-12-20 16:07:23,708  - mini_batch_size: \"16\"\n",
            "2019-12-20 16:07:23,710  - patience: \"3\"\n",
            "2019-12-20 16:07:23,712  - anneal_factor: \"0.5\"\n",
            "2019-12-20 16:07:23,713  - max_epochs: \"50\"\n",
            "2019-12-20 16:07:23,715  - shuffle: \"True\"\n",
            "2019-12-20 16:07:23,717  - train_with_dev: \"False\"\n",
            "2019-12-20 16:07:23,718  - batch_growth_annealing: \"False\"\n",
            "2019-12-20 16:07:23,720 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:07:23,721 Model training base path: \"resources/results\"\n",
            "2019-12-20 16:07:23,723 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:07:23,725 Device: cuda:0\n",
            "2019-12-20 16:07:23,726 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:07:23,728 Embeddings storage mode: cpu\n",
            "2019-12-20 16:07:23,730 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:07:24,002 epoch 1 - iter 0/307 - loss 1.82161617 - samples/sec: 1781.23\n",
            "2019-12-20 16:07:28,210 epoch 1 - iter 30/307 - loss 1.66076421 - samples/sec: 114.13\n",
            "2019-12-20 16:07:32,448 epoch 1 - iter 60/307 - loss 1.58373227 - samples/sec: 113.33\n",
            "2019-12-20 16:07:36,554 epoch 1 - iter 90/307 - loss 1.49280970 - samples/sec: 116.99\n",
            "2019-12-20 16:07:40,652 epoch 1 - iter 120/307 - loss 1.44619525 - samples/sec: 117.22\n",
            "2019-12-20 16:07:44,931 epoch 1 - iter 150/307 - loss 1.40770159 - samples/sec: 112.24\n",
            "2019-12-20 16:07:48,968 epoch 1 - iter 180/307 - loss 1.37174620 - samples/sec: 118.99\n",
            "2019-12-20 16:07:52,898 epoch 1 - iter 210/307 - loss 1.32412972 - samples/sec: 122.20\n",
            "2019-12-20 16:07:56,958 epoch 1 - iter 240/307 - loss 1.28454575 - samples/sec: 118.39\n",
            "2019-12-20 16:08:01,095 epoch 1 - iter 270/307 - loss 1.24472670 - samples/sec: 116.13\n",
            "2019-12-20 16:08:05,956 epoch 1 - iter 300/307 - loss 1.20470204 - samples/sec: 98.80\n",
            "2019-12-20 16:08:06,692 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:08:06,694 EPOCH 1 done: loss 1.1998 - lr 0.1000\n",
            "2019-12-20 16:08:11,057 DEV : loss 0.7367275953292847 - score 0.7321\n",
            "2019-12-20 16:08:11,081 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 16:08:11,082 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:08:11,111 epoch 2 - iter 0/307 - loss 0.93107009 - samples/sec: 18785.02\n",
            "2019-12-20 16:08:11,839 epoch 2 - iter 30/307 - loss 0.78472951 - samples/sec: 660.87\n",
            "2019-12-20 16:08:12,559 epoch 2 - iter 60/307 - loss 0.70477461 - samples/sec: 669.71\n",
            "2019-12-20 16:08:13,289 epoch 2 - iter 90/307 - loss 0.70246399 - samples/sec: 661.57\n",
            "2019-12-20 16:08:14,012 epoch 2 - iter 120/307 - loss 0.68969239 - samples/sec: 671.46\n",
            "2019-12-20 16:08:14,752 epoch 2 - iter 150/307 - loss 0.66631121 - samples/sec: 650.66\n",
            "2019-12-20 16:08:15,487 epoch 2 - iter 180/307 - loss 0.64900789 - samples/sec: 655.46\n",
            "2019-12-20 16:08:16,208 epoch 2 - iter 210/307 - loss 0.64268440 - samples/sec: 668.31\n",
            "2019-12-20 16:08:16,938 epoch 2 - iter 240/307 - loss 0.63721187 - samples/sec: 659.22\n",
            "2019-12-20 16:08:17,646 epoch 2 - iter 270/307 - loss 0.62145458 - samples/sec: 680.45\n",
            "2019-12-20 16:08:18,344 epoch 2 - iter 300/307 - loss 0.60744021 - samples/sec: 690.10\n",
            "2019-12-20 16:08:18,497 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:08:18,498 EPOCH 2 done: loss 0.6026 - lr 0.1000\n",
            "2019-12-20 16:08:19,046 DEV : loss 0.5271064639091492 - score 0.8183\n",
            "2019-12-20 16:08:19,068 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 16:08:19,069 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:08:19,099 epoch 3 - iter 0/307 - loss 0.27450138 - samples/sec: 17859.34\n",
            "2019-12-20 16:08:19,818 epoch 3 - iter 30/307 - loss 0.43123505 - samples/sec: 668.81\n",
            "2019-12-20 16:08:20,584 epoch 3 - iter 60/307 - loss 0.41682796 - samples/sec: 629.03\n",
            "2019-12-20 16:08:21,289 epoch 3 - iter 90/307 - loss 0.43410968 - samples/sec: 683.35\n",
            "2019-12-20 16:08:22,007 epoch 3 - iter 120/307 - loss 0.43383409 - samples/sec: 670.32\n",
            "2019-12-20 16:08:22,751 epoch 3 - iter 150/307 - loss 0.43158434 - samples/sec: 647.44\n",
            "2019-12-20 16:08:23,465 epoch 3 - iter 180/307 - loss 0.43139456 - samples/sec: 678.03\n",
            "2019-12-20 16:08:24,184 epoch 3 - iter 210/307 - loss 0.43537056 - samples/sec: 669.39\n",
            "2019-12-20 16:08:24,921 epoch 3 - iter 240/307 - loss 0.42660152 - samples/sec: 654.02\n",
            "2019-12-20 16:08:25,659 epoch 3 - iter 270/307 - loss 0.42599521 - samples/sec: 652.57\n",
            "2019-12-20 16:08:26,405 epoch 3 - iter 300/307 - loss 0.42137852 - samples/sec: 645.33\n",
            "2019-12-20 16:08:26,553 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:08:26,555 EPOCH 3 done: loss 0.4235 - lr 0.1000\n",
            "2019-12-20 16:08:27,098 DEV : loss 0.5079084634780884 - score 0.8312\n",
            "2019-12-20 16:08:27,121 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 16:08:27,122 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:08:27,151 epoch 4 - iter 0/307 - loss 0.44504350 - samples/sec: 17544.04\n",
            "2019-12-20 16:08:27,883 epoch 4 - iter 30/307 - loss 0.32838465 - samples/sec: 658.02\n",
            "2019-12-20 16:08:28,619 epoch 4 - iter 60/307 - loss 0.28105864 - samples/sec: 654.37\n",
            "2019-12-20 16:08:29,349 epoch 4 - iter 90/307 - loss 0.31299136 - samples/sec: 659.40\n",
            "2019-12-20 16:08:30,072 epoch 4 - iter 120/307 - loss 0.32497651 - samples/sec: 666.72\n",
            "2019-12-20 16:08:30,804 epoch 4 - iter 150/307 - loss 0.32289117 - samples/sec: 657.56\n",
            "2019-12-20 16:08:31,546 epoch 4 - iter 180/307 - loss 0.31822958 - samples/sec: 649.83\n",
            "2019-12-20 16:08:32,284 epoch 4 - iter 210/307 - loss 0.31726320 - samples/sec: 651.77\n",
            "2019-12-20 16:08:33,027 epoch 4 - iter 240/307 - loss 0.31834292 - samples/sec: 648.38\n",
            "2019-12-20 16:08:33,759 epoch 4 - iter 270/307 - loss 0.32459790 - samples/sec: 662.18\n",
            "2019-12-20 16:08:34,503 epoch 4 - iter 300/307 - loss 0.31877221 - samples/sec: 647.00\n",
            "2019-12-20 16:08:34,655 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:08:34,657 EPOCH 4 done: loss 0.3169 - lr 0.1000\n",
            "2019-12-20 16:08:35,212 DEV : loss 0.3714359402656555 - score 0.8881\n",
            "2019-12-20 16:08:35,236 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 16:08:35,237 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:08:35,267 epoch 5 - iter 0/307 - loss 0.25367251 - samples/sec: 17781.89\n",
            "2019-12-20 16:08:35,982 epoch 5 - iter 30/307 - loss 0.21346596 - samples/sec: 673.79\n",
            "2019-12-20 16:08:36,718 epoch 5 - iter 60/307 - loss 0.23899738 - samples/sec: 654.73\n",
            "2019-12-20 16:08:37,469 epoch 5 - iter 90/307 - loss 0.28461238 - samples/sec: 641.23\n",
            "2019-12-20 16:08:38,185 epoch 5 - iter 120/307 - loss 0.27358062 - samples/sec: 673.58\n",
            "2019-12-20 16:08:38,906 epoch 5 - iter 150/307 - loss 0.27593043 - samples/sec: 668.70\n",
            "2019-12-20 16:08:39,637 epoch 5 - iter 180/307 - loss 0.27231651 - samples/sec: 658.77\n",
            "2019-12-20 16:08:40,379 epoch 5 - iter 210/307 - loss 0.27565569 - samples/sec: 651.89\n",
            "2019-12-20 16:08:41,116 epoch 5 - iter 240/307 - loss 0.27465564 - samples/sec: 653.30\n",
            "2019-12-20 16:08:41,852 epoch 5 - iter 270/307 - loss 0.27620237 - samples/sec: 656.94\n",
            "2019-12-20 16:08:42,604 epoch 5 - iter 300/307 - loss 0.27267464 - samples/sec: 640.26\n",
            "2019-12-20 16:08:42,754 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:08:42,755 EPOCH 5 done: loss 0.2714 - lr 0.1000\n",
            "2019-12-20 16:08:43,316 DEV : loss 0.4319441020488739 - score 0.8624\n",
            "2019-12-20 16:08:43,340 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 16:08:43,341 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:08:43,371 epoch 6 - iter 0/307 - loss 0.47781426 - samples/sec: 17532.58\n",
            "2019-12-20 16:08:44,135 epoch 6 - iter 30/307 - loss 0.26790040 - samples/sec: 629.71\n",
            "2019-12-20 16:08:44,862 epoch 6 - iter 60/307 - loss 0.24708514 - samples/sec: 662.29\n",
            "2019-12-20 16:08:45,615 epoch 6 - iter 90/307 - loss 0.23154294 - samples/sec: 639.98\n",
            "2019-12-20 16:08:46,364 epoch 6 - iter 120/307 - loss 0.24450544 - samples/sec: 642.90\n",
            "2019-12-20 16:08:47,086 epoch 6 - iter 150/307 - loss 0.24010443 - samples/sec: 666.57\n",
            "2019-12-20 16:08:47,823 epoch 6 - iter 180/307 - loss 0.24032634 - samples/sec: 653.52\n",
            "2019-12-20 16:08:48,548 epoch 6 - iter 210/307 - loss 0.24363801 - samples/sec: 665.48\n",
            "2019-12-20 16:08:49,281 epoch 6 - iter 240/307 - loss 0.24302333 - samples/sec: 657.01\n",
            "2019-12-20 16:08:50,000 epoch 6 - iter 270/307 - loss 0.24739994 - samples/sec: 669.70\n",
            "2019-12-20 16:08:50,723 epoch 6 - iter 300/307 - loss 0.24064107 - samples/sec: 665.89\n",
            "2019-12-20 16:08:50,878 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:08:50,879 EPOCH 6 done: loss 0.2397 - lr 0.1000\n",
            "2019-12-20 16:08:51,427 DEV : loss 0.29433611035346985 - score 0.9101\n",
            "2019-12-20 16:08:51,451 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 16:08:51,452 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:08:51,486 epoch 7 - iter 0/307 - loss 0.11252560 - samples/sec: 15389.35\n",
            "2019-12-20 16:08:52,224 epoch 7 - iter 30/307 - loss 0.15725337 - samples/sec: 651.98\n",
            "2019-12-20 16:08:52,980 epoch 7 - iter 60/307 - loss 0.15430127 - samples/sec: 639.60\n",
            "2019-12-20 16:08:53,698 epoch 7 - iter 90/307 - loss 0.17450045 - samples/sec: 671.16\n",
            "2019-12-20 16:08:54,423 epoch 7 - iter 120/307 - loss 0.18099179 - samples/sec: 664.03\n",
            "2019-12-20 16:08:55,151 epoch 7 - iter 150/307 - loss 0.19550538 - samples/sec: 661.27\n",
            "2019-12-20 16:08:55,871 epoch 7 - iter 180/307 - loss 0.19740537 - samples/sec: 669.15\n",
            "2019-12-20 16:08:56,633 epoch 7 - iter 210/307 - loss 0.20428421 - samples/sec: 632.26\n",
            "2019-12-20 16:08:57,368 epoch 7 - iter 240/307 - loss 0.20704872 - samples/sec: 654.90\n",
            "2019-12-20 16:08:58,119 epoch 7 - iter 270/307 - loss 0.20740524 - samples/sec: 641.71\n",
            "2019-12-20 16:08:58,847 epoch 7 - iter 300/307 - loss 0.21137818 - samples/sec: 661.42\n",
            "2019-12-20 16:08:59,005 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:08:59,007 EPOCH 7 done: loss 0.2095 - lr 0.1000\n",
            "2019-12-20 16:08:59,559 DEV : loss 0.4643342196941376 - score 0.8697\n",
            "2019-12-20 16:08:59,584 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 16:08:59,585 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:08:59,615 epoch 8 - iter 0/307 - loss 0.13630325 - samples/sec: 17579.27\n",
            "2019-12-20 16:09:00,334 epoch 8 - iter 30/307 - loss 0.19905024 - samples/sec: 669.79\n",
            "2019-12-20 16:09:01,061 epoch 8 - iter 60/307 - loss 0.17973028 - samples/sec: 664.10\n",
            "2019-12-20 16:09:01,795 epoch 8 - iter 90/307 - loss 0.16862955 - samples/sec: 656.65\n",
            "2019-12-20 16:09:02,540 epoch 8 - iter 120/307 - loss 0.17487612 - samples/sec: 648.65\n",
            "2019-12-20 16:09:03,287 epoch 8 - iter 150/307 - loss 0.17670840 - samples/sec: 649.80\n",
            "2019-12-20 16:09:04,047 epoch 8 - iter 180/307 - loss 0.17510927 - samples/sec: 635.28\n",
            "2019-12-20 16:09:04,766 epoch 8 - iter 210/307 - loss 0.17369572 - samples/sec: 669.83\n",
            "2019-12-20 16:09:05,485 epoch 8 - iter 240/307 - loss 0.17565934 - samples/sec: 671.25\n",
            "2019-12-20 16:09:06,204 epoch 8 - iter 270/307 - loss 0.17806952 - samples/sec: 669.06\n",
            "2019-12-20 16:09:06,943 epoch 8 - iter 300/307 - loss 0.17419936 - samples/sec: 651.99\n",
            "2019-12-20 16:09:07,092 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:09:07,093 EPOCH 8 done: loss 0.1761 - lr 0.1000\n",
            "2019-12-20 16:09:07,648 DEV : loss 0.28372955322265625 - score 0.9174\n",
            "2019-12-20 16:09:07,673 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 16:09:07,675 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:09:07,706 epoch 9 - iter 0/307 - loss 0.01903269 - samples/sec: 16241.52\n",
            "2019-12-20 16:09:08,423 epoch 9 - iter 30/307 - loss 0.16870973 - samples/sec: 672.24\n",
            "2019-12-20 16:09:09,163 epoch 9 - iter 60/307 - loss 0.15181577 - samples/sec: 650.54\n",
            "2019-12-20 16:09:09,895 epoch 9 - iter 90/307 - loss 0.15170486 - samples/sec: 657.59\n",
            "2019-12-20 16:09:10,615 epoch 9 - iter 120/307 - loss 0.16118151 - samples/sec: 669.42\n",
            "2019-12-20 16:09:11,349 epoch 9 - iter 150/307 - loss 0.16596392 - samples/sec: 655.57\n",
            "2019-12-20 16:09:12,093 epoch 9 - iter 180/307 - loss 0.15996830 - samples/sec: 647.53\n",
            "2019-12-20 16:09:12,854 epoch 9 - iter 210/307 - loss 0.16215034 - samples/sec: 632.59\n",
            "2019-12-20 16:09:13,563 epoch 9 - iter 240/307 - loss 0.16427039 - samples/sec: 683.82\n",
            "2019-12-20 16:09:14,296 epoch 9 - iter 270/307 - loss 0.16284984 - samples/sec: 656.76\n",
            "2019-12-20 16:09:15,064 epoch 9 - iter 300/307 - loss 0.16774616 - samples/sec: 627.35\n",
            "2019-12-20 16:09:15,213 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:09:15,214 EPOCH 9 done: loss 0.1665 - lr 0.1000\n",
            "2019-12-20 16:09:15,757 DEV : loss 0.22324047982692719 - score 0.9266\n",
            "2019-12-20 16:09:15,790 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 16:09:15,792 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:09:15,821 epoch 10 - iter 0/307 - loss 0.13825794 - samples/sec: 18106.54\n",
            "2019-12-20 16:09:16,560 epoch 10 - iter 30/307 - loss 0.14791797 - samples/sec: 651.50\n",
            "2019-12-20 16:09:17,283 epoch 10 - iter 60/307 - loss 0.13423924 - samples/sec: 667.42\n",
            "2019-12-20 16:09:18,013 epoch 10 - iter 90/307 - loss 0.13174001 - samples/sec: 660.11\n",
            "2019-12-20 16:09:18,746 epoch 10 - iter 120/307 - loss 0.12831055 - samples/sec: 657.03\n",
            "2019-12-20 16:09:19,494 epoch 10 - iter 150/307 - loss 0.13022372 - samples/sec: 643.99\n",
            "2019-12-20 16:09:20,236 epoch 10 - iter 180/307 - loss 0.12686893 - samples/sec: 648.51\n",
            "2019-12-20 16:09:20,991 epoch 10 - iter 210/307 - loss 0.13549230 - samples/sec: 638.37\n",
            "2019-12-20 16:09:21,710 epoch 10 - iter 240/307 - loss 0.13348397 - samples/sec: 670.37\n",
            "2019-12-20 16:09:22,438 epoch 10 - iter 270/307 - loss 0.12993491 - samples/sec: 661.03\n",
            "2019-12-20 16:09:23,170 epoch 10 - iter 300/307 - loss 0.13063618 - samples/sec: 657.59\n",
            "2019-12-20 16:09:23,322 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:09:23,323 EPOCH 10 done: loss 0.1289 - lr 0.1000\n",
            "2019-12-20 16:09:23,867 DEV : loss 0.2968580424785614 - score 0.9229\n",
            "2019-12-20 16:09:23,892 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 16:09:23,893 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:09:23,921 epoch 11 - iter 0/307 - loss 0.51395977 - samples/sec: 19225.23\n",
            "2019-12-20 16:09:24,662 epoch 11 - iter 30/307 - loss 0.12516270 - samples/sec: 649.78\n",
            "2019-12-20 16:09:25,432 epoch 11 - iter 60/307 - loss 0.11817975 - samples/sec: 624.80\n",
            "2019-12-20 16:09:26,166 epoch 11 - iter 90/307 - loss 0.12868322 - samples/sec: 659.43\n",
            "2019-12-20 16:09:26,898 epoch 11 - iter 120/307 - loss 0.12099444 - samples/sec: 657.80\n",
            "2019-12-20 16:09:27,625 epoch 11 - iter 150/307 - loss 0.12232353 - samples/sec: 662.73\n",
            "2019-12-20 16:09:28,356 epoch 11 - iter 180/307 - loss 0.12432433 - samples/sec: 659.99\n",
            "2019-12-20 16:09:29,091 epoch 11 - iter 210/307 - loss 0.12756174 - samples/sec: 654.89\n",
            "2019-12-20 16:09:29,841 epoch 11 - iter 240/307 - loss 0.12355196 - samples/sec: 643.07\n",
            "2019-12-20 16:09:30,579 epoch 11 - iter 270/307 - loss 0.12604957 - samples/sec: 652.46\n",
            "2019-12-20 16:09:31,323 epoch 11 - iter 300/307 - loss 0.12208249 - samples/sec: 647.43\n",
            "2019-12-20 16:09:31,475 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:09:31,476 EPOCH 11 done: loss 0.1224 - lr 0.1000\n",
            "2019-12-20 16:09:32,027 DEV : loss 0.27217262983322144 - score 0.9321\n",
            "2019-12-20 16:09:32,052 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 16:09:32,053 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:09:32,083 epoch 12 - iter 0/307 - loss 0.04995689 - samples/sec: 17574.36\n",
            "2019-12-20 16:09:32,819 epoch 12 - iter 30/307 - loss 0.10670271 - samples/sec: 654.69\n",
            "2019-12-20 16:09:33,571 epoch 12 - iter 60/307 - loss 0.10113994 - samples/sec: 640.42\n",
            "2019-12-20 16:09:34,319 epoch 12 - iter 90/307 - loss 0.10675655 - samples/sec: 643.51\n",
            "2019-12-20 16:09:35,063 epoch 12 - iter 120/307 - loss 0.11772620 - samples/sec: 649.28\n",
            "2019-12-20 16:09:35,825 epoch 12 - iter 150/307 - loss 0.11665266 - samples/sec: 632.68\n",
            "2019-12-20 16:09:36,545 epoch 12 - iter 180/307 - loss 0.11656215 - samples/sec: 668.23\n",
            "2019-12-20 16:09:37,277 epoch 12 - iter 210/307 - loss 0.11675337 - samples/sec: 659.36\n",
            "2019-12-20 16:09:38,003 epoch 12 - iter 240/307 - loss 0.11857938 - samples/sec: 662.90\n",
            "2019-12-20 16:09:38,736 epoch 12 - iter 270/307 - loss 0.11994535 - samples/sec: 657.08\n",
            "2019-12-20 16:09:39,505 epoch 12 - iter 300/307 - loss 0.11860708 - samples/sec: 626.29\n",
            "2019-12-20 16:09:39,649 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:09:39,651 EPOCH 12 done: loss 0.1192 - lr 0.1000\n",
            "2019-12-20 16:09:40,208 DEV : loss 0.35564351081848145 - score 0.9119\n",
            "2019-12-20 16:09:40,233 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 16:09:40,234 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:09:40,265 epoch 13 - iter 0/307 - loss 0.00481412 - samples/sec: 16890.95\n",
            "2019-12-20 16:09:40,990 epoch 13 - iter 30/307 - loss 0.07514933 - samples/sec: 664.38\n",
            "2019-12-20 16:09:41,726 epoch 13 - iter 60/307 - loss 0.10673701 - samples/sec: 654.13\n",
            "2019-12-20 16:09:42,465 epoch 13 - iter 90/307 - loss 0.09831690 - samples/sec: 652.02\n",
            "2019-12-20 16:09:43,199 epoch 13 - iter 120/307 - loss 0.09595156 - samples/sec: 656.38\n",
            "2019-12-20 16:09:43,924 epoch 13 - iter 150/307 - loss 0.09713855 - samples/sec: 663.65\n",
            "2019-12-20 16:09:44,671 epoch 13 - iter 180/307 - loss 0.10493361 - samples/sec: 645.09\n",
            "2019-12-20 16:09:45,393 epoch 13 - iter 210/307 - loss 0.10837720 - samples/sec: 666.88\n",
            "2019-12-20 16:09:46,145 epoch 13 - iter 240/307 - loss 0.10878082 - samples/sec: 640.67\n",
            "2019-12-20 16:09:46,890 epoch 13 - iter 270/307 - loss 0.10676893 - samples/sec: 646.40\n",
            "2019-12-20 16:09:47,641 epoch 13 - iter 300/307 - loss 0.10960094 - samples/sec: 641.18\n",
            "2019-12-20 16:09:47,787 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:09:47,788 EPOCH 13 done: loss 0.1092 - lr 0.1000\n",
            "2019-12-20 16:09:48,337 DEV : loss 0.2588356137275696 - score 0.9248\n",
            "2019-12-20 16:09:48,359 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 16:09:48,360 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:09:48,391 epoch 14 - iter 0/307 - loss 0.03199378 - samples/sec: 17009.68\n",
            "2019-12-20 16:09:49,128 epoch 14 - iter 30/307 - loss 0.04449177 - samples/sec: 652.67\n",
            "2019-12-20 16:09:49,880 epoch 14 - iter 60/307 - loss 0.05567216 - samples/sec: 641.28\n",
            "2019-12-20 16:09:50,620 epoch 14 - iter 90/307 - loss 0.06140749 - samples/sec: 650.98\n",
            "2019-12-20 16:09:51,347 epoch 14 - iter 120/307 - loss 0.05981882 - samples/sec: 662.18\n",
            "2019-12-20 16:09:52,082 epoch 14 - iter 150/307 - loss 0.05907424 - samples/sec: 655.37\n",
            "2019-12-20 16:09:52,817 epoch 14 - iter 180/307 - loss 0.06486467 - samples/sec: 654.89\n",
            "2019-12-20 16:09:53,520 epoch 14 - iter 210/307 - loss 0.06666210 - samples/sec: 685.01\n",
            "2019-12-20 16:09:54,249 epoch 14 - iter 240/307 - loss 0.07284921 - samples/sec: 660.61\n",
            "2019-12-20 16:09:54,993 epoch 14 - iter 270/307 - loss 0.07495681 - samples/sec: 647.83\n",
            "2019-12-20 16:09:55,737 epoch 14 - iter 300/307 - loss 0.07984636 - samples/sec: 647.67\n",
            "2019-12-20 16:09:55,886 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:09:55,887 EPOCH 14 done: loss 0.0795 - lr 0.1000\n",
            "2019-12-20 16:09:56,427 DEV : loss 0.30654704570770264 - score 0.9138\n",
            "2019-12-20 16:09:56,450 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 16:09:56,452 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:09:56,482 epoch 15 - iter 0/307 - loss 0.12196541 - samples/sec: 17225.36\n",
            "2019-12-20 16:09:57,245 epoch 15 - iter 30/307 - loss 0.08669445 - samples/sec: 630.45\n",
            "2019-12-20 16:09:57,961 epoch 15 - iter 60/307 - loss 0.07736311 - samples/sec: 675.74\n",
            "2019-12-20 16:09:58,664 epoch 15 - iter 90/307 - loss 0.07637372 - samples/sec: 685.71\n",
            "2019-12-20 16:09:59,408 epoch 15 - iter 120/307 - loss 0.08774456 - samples/sec: 646.76\n",
            "2019-12-20 16:10:00,155 epoch 15 - iter 150/307 - loss 0.08717443 - samples/sec: 645.12\n",
            "2019-12-20 16:10:00,871 epoch 15 - iter 180/307 - loss 0.08600579 - samples/sec: 672.18\n",
            "2019-12-20 16:10:01,577 epoch 15 - iter 210/307 - loss 0.08291669 - samples/sec: 682.54\n",
            "2019-12-20 16:10:02,319 epoch 15 - iter 240/307 - loss 0.08308428 - samples/sec: 649.19\n",
            "2019-12-20 16:10:03,034 epoch 15 - iter 270/307 - loss 0.08158670 - samples/sec: 673.26\n",
            "2019-12-20 16:10:03,791 epoch 15 - iter 300/307 - loss 0.08422987 - samples/sec: 636.21\n",
            "2019-12-20 16:10:03,942 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:10:03,943 EPOCH 15 done: loss 0.0833 - lr 0.1000\n",
            "2019-12-20 16:10:04,491 DEV : loss 0.3402884006500244 - score 0.9138\n",
            "Epoch    14: reducing learning rate of group 0 to 5.0000e-02.\n",
            "  0%|          | 0/100 [02:41<?, ?it/s, best loss: ?]2019-12-20 16:10:04,521 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 16:10:04,523 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:10:04,559 epoch 16 - iter 0/307 - loss 0.05005452 - samples/sec: 14170.84\n",
            "2019-12-20 16:10:05,296 epoch 16 - iter 30/307 - loss 0.06351448 - samples/sec: 652.94\n",
            "2019-12-20 16:10:06,038 epoch 16 - iter 60/307 - loss 0.05982676 - samples/sec: 649.61\n",
            "2019-12-20 16:10:06,765 epoch 16 - iter 90/307 - loss 0.05186160 - samples/sec: 662.56\n",
            "2019-12-20 16:10:07,510 epoch 16 - iter 120/307 - loss 0.04833385 - samples/sec: 647.92\n",
            "2019-12-20 16:10:08,262 epoch 16 - iter 150/307 - loss 0.04721470 - samples/sec: 644.18\n",
            "2019-12-20 16:10:08,985 epoch 16 - iter 180/307 - loss 0.04975328 - samples/sec: 666.11\n",
            "2019-12-20 16:10:09,729 epoch 16 - iter 210/307 - loss 0.05094068 - samples/sec: 646.88\n",
            "2019-12-20 16:10:10,455 epoch 16 - iter 240/307 - loss 0.05337189 - samples/sec: 664.20\n",
            "2019-12-20 16:10:11,180 epoch 16 - iter 270/307 - loss 0.05517034 - samples/sec: 664.30\n",
            "2019-12-20 16:10:11,892 epoch 16 - iter 300/307 - loss 0.05489816 - samples/sec: 676.33\n",
            "2019-12-20 16:10:12,036 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:10:12,037 EPOCH 16 done: loss 0.0551 - lr 0.0500\n",
            "2019-12-20 16:10:12,588 DEV : loss 0.2361556440591812 - score 0.9339\n",
            "2019-12-20 16:10:12,611 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 16:10:12,613 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:10:12,646 epoch 17 - iter 0/307 - loss 0.05954599 - samples/sec: 15784.63\n",
            "2019-12-20 16:10:13,377 epoch 17 - iter 30/307 - loss 0.03582398 - samples/sec: 659.53\n",
            "2019-12-20 16:10:14,128 epoch 17 - iter 60/307 - loss 0.05157757 - samples/sec: 641.42\n",
            "2019-12-20 16:10:14,870 epoch 17 - iter 90/307 - loss 0.04800711 - samples/sec: 648.99\n",
            "2019-12-20 16:10:15,590 epoch 17 - iter 120/307 - loss 0.05067477 - samples/sec: 669.72\n",
            "2019-12-20 16:10:16,311 epoch 17 - iter 150/307 - loss 0.05054765 - samples/sec: 671.11\n",
            "2019-12-20 16:10:17,039 epoch 17 - iter 180/307 - loss 0.04924443 - samples/sec: 661.19\n",
            "2019-12-20 16:10:17,761 epoch 17 - iter 210/307 - loss 0.05185302 - samples/sec: 666.73\n",
            "2019-12-20 16:10:18,516 epoch 17 - iter 240/307 - loss 0.05101439 - samples/sec: 637.61\n",
            "2019-12-20 16:10:19,247 epoch 17 - iter 270/307 - loss 0.05065205 - samples/sec: 659.04\n",
            "2019-12-20 16:10:19,978 epoch 17 - iter 300/307 - loss 0.05097603 - samples/sec: 659.67\n",
            "2019-12-20 16:10:20,137 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:10:20,138 EPOCH 17 done: loss 0.0515 - lr 0.0500\n",
            "2019-12-20 16:10:20,700 DEV : loss 0.2954476475715637 - score 0.9211\n",
            "2019-12-20 16:10:20,724 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 16:10:20,725 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:10:20,756 epoch 18 - iter 0/307 - loss 0.02420750 - samples/sec: 16852.77\n",
            "2019-12-20 16:10:21,498 epoch 18 - iter 30/307 - loss 0.02264031 - samples/sec: 648.39\n",
            "2019-12-20 16:10:22,219 epoch 18 - iter 60/307 - loss 0.03083083 - samples/sec: 667.66\n",
            "2019-12-20 16:10:22,977 epoch 18 - iter 90/307 - loss 0.03604051 - samples/sec: 635.73\n",
            "2019-12-20 16:10:23,713 epoch 18 - iter 120/307 - loss 0.03802038 - samples/sec: 654.10\n",
            "2019-12-20 16:10:24,418 epoch 18 - iter 150/307 - loss 0.04565338 - samples/sec: 683.22\n",
            "2019-12-20 16:10:25,147 epoch 18 - iter 180/307 - loss 0.04384331 - samples/sec: 661.25\n",
            "2019-12-20 16:10:25,873 epoch 18 - iter 210/307 - loss 0.04340649 - samples/sec: 662.72\n",
            "2019-12-20 16:10:26,590 epoch 18 - iter 240/307 - loss 0.04111524 - samples/sec: 672.11\n",
            "2019-12-20 16:10:27,314 epoch 18 - iter 270/307 - loss 0.04050357 - samples/sec: 664.94\n",
            "2019-12-20 16:10:28,058 epoch 18 - iter 300/307 - loss 0.04077518 - samples/sec: 646.99\n",
            "2019-12-20 16:10:28,217 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:10:28,219 EPOCH 18 done: loss 0.0415 - lr 0.0500\n",
            "2019-12-20 16:10:28,773 DEV : loss 0.3271509110927582 - score 0.9248\n",
            "2019-12-20 16:10:28,798 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 16:10:28,799 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:10:28,829 epoch 19 - iter 0/307 - loss 0.00668553 - samples/sec: 17439.93\n",
            "2019-12-20 16:10:29,579 epoch 19 - iter 30/307 - loss 0.03625287 - samples/sec: 641.87\n",
            "2019-12-20 16:10:30,296 epoch 19 - iter 60/307 - loss 0.03506025 - samples/sec: 671.72\n",
            "2019-12-20 16:10:31,039 epoch 19 - iter 90/307 - loss 0.03665524 - samples/sec: 648.11\n",
            "2019-12-20 16:10:31,779 epoch 19 - iter 120/307 - loss 0.03557167 - samples/sec: 651.16\n",
            "2019-12-20 16:10:32,509 epoch 19 - iter 150/307 - loss 0.03878241 - samples/sec: 660.74\n",
            "2019-12-20 16:10:33,235 epoch 19 - iter 180/307 - loss 0.04176643 - samples/sec: 664.50\n",
            "2019-12-20 16:10:33,976 epoch 19 - iter 210/307 - loss 0.04191499 - samples/sec: 650.05\n",
            "2019-12-20 16:10:34,748 epoch 19 - iter 240/307 - loss 0.04210558 - samples/sec: 626.31\n",
            "2019-12-20 16:10:35,495 epoch 19 - iter 270/307 - loss 0.04293076 - samples/sec: 644.76\n",
            "2019-12-20 16:10:36,239 epoch 19 - iter 300/307 - loss 0.04199017 - samples/sec: 647.20\n",
            "2019-12-20 16:10:36,389 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:10:36,392 EPOCH 19 done: loss 0.0425 - lr 0.0500\n",
            "2019-12-20 16:10:36,957 DEV : loss 0.31559693813323975 - score 0.9303\n",
            "2019-12-20 16:10:36,983 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 16:10:36,984 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:10:37,014 epoch 20 - iter 0/307 - loss 0.00152671 - samples/sec: 17467.93\n",
            "2019-12-20 16:10:37,787 epoch 20 - iter 30/307 - loss 0.01923869 - samples/sec: 622.78\n",
            "2019-12-20 16:10:38,538 epoch 20 - iter 60/307 - loss 0.01905599 - samples/sec: 642.08\n",
            "2019-12-20 16:10:39,284 epoch 20 - iter 90/307 - loss 0.02413566 - samples/sec: 649.33\n",
            "2019-12-20 16:10:40,056 epoch 20 - iter 120/307 - loss 0.02950324 - samples/sec: 623.66\n",
            "2019-12-20 16:10:40,787 epoch 20 - iter 150/307 - loss 0.02903082 - samples/sec: 659.07\n",
            "2019-12-20 16:10:41,527 epoch 20 - iter 180/307 - loss 0.03332924 - samples/sec: 651.03\n",
            "2019-12-20 16:10:42,269 epoch 20 - iter 210/307 - loss 0.03316907 - samples/sec: 649.50\n",
            "2019-12-20 16:10:43,013 epoch 20 - iter 240/307 - loss 0.03319143 - samples/sec: 647.12\n",
            "2019-12-20 16:10:43,723 epoch 20 - iter 270/307 - loss 0.03154789 - samples/sec: 678.34\n",
            "2019-12-20 16:10:44,451 epoch 20 - iter 300/307 - loss 0.03333100 - samples/sec: 662.49\n",
            "2019-12-20 16:10:44,596 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:10:44,597 EPOCH 20 done: loss 0.0345 - lr 0.0500\n",
            "2019-12-20 16:10:45,147 DEV : loss 0.3531682789325714 - score 0.9303\n",
            "Epoch    19: reducing learning rate of group 0 to 2.5000e-02.\n",
            "  0%|          | 0/100 [03:21<?, ?it/s, best loss: ?]2019-12-20 16:10:45,177 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 16:10:45,180 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:10:45,215 epoch 21 - iter 0/307 - loss 0.17607841 - samples/sec: 15006.23\n",
            "2019-12-20 16:10:45,954 epoch 21 - iter 30/307 - loss 0.04996055 - samples/sec: 651.00\n",
            "2019-12-20 16:10:46,675 epoch 21 - iter 60/307 - loss 0.03300772 - samples/sec: 668.80\n",
            "2019-12-20 16:10:47,404 epoch 21 - iter 90/307 - loss 0.02717412 - samples/sec: 660.39\n",
            "2019-12-20 16:10:48,148 epoch 21 - iter 120/307 - loss 0.02919429 - samples/sec: 647.07\n",
            "2019-12-20 16:10:48,855 epoch 21 - iter 150/307 - loss 0.03083103 - samples/sec: 681.38\n",
            "2019-12-20 16:10:49,587 epoch 21 - iter 180/307 - loss 0.02802862 - samples/sec: 658.19\n",
            "2019-12-20 16:10:50,349 epoch 21 - iter 210/307 - loss 0.02647635 - samples/sec: 632.62\n",
            "2019-12-20 16:10:51,086 epoch 21 - iter 240/307 - loss 0.02743665 - samples/sec: 653.67\n",
            "2019-12-20 16:10:51,826 epoch 21 - iter 270/307 - loss 0.02573056 - samples/sec: 650.20\n",
            "2019-12-20 16:10:52,583 epoch 21 - iter 300/307 - loss 0.02600271 - samples/sec: 637.05\n",
            "2019-12-20 16:10:52,736 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:10:52,737 EPOCH 21 done: loss 0.0269 - lr 0.0250\n",
            "2019-12-20 16:10:53,304 DEV : loss 0.3711048662662506 - score 0.9248\n",
            "2019-12-20 16:10:53,328 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 16:10:53,329 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:10:53,366 epoch 22 - iter 0/307 - loss 0.03961706 - samples/sec: 14025.52\n",
            "2019-12-20 16:10:54,112 epoch 22 - iter 30/307 - loss 0.02964125 - samples/sec: 644.72\n",
            "2019-12-20 16:10:54,851 epoch 22 - iter 60/307 - loss 0.02411358 - samples/sec: 652.32\n",
            "2019-12-20 16:10:55,577 epoch 22 - iter 90/307 - loss 0.02193033 - samples/sec: 665.21\n",
            "2019-12-20 16:10:56,323 epoch 22 - iter 120/307 - loss 0.02084668 - samples/sec: 649.26\n",
            "2019-12-20 16:10:57,040 epoch 22 - iter 150/307 - loss 0.02247163 - samples/sec: 671.50\n",
            "2019-12-20 16:10:57,774 epoch 22 - iter 180/307 - loss 0.02328671 - samples/sec: 657.88\n",
            "2019-12-20 16:10:58,500 epoch 22 - iter 210/307 - loss 0.02666772 - samples/sec: 663.57\n",
            "2019-12-20 16:10:59,234 epoch 22 - iter 240/307 - loss 0.02549105 - samples/sec: 656.53\n",
            "2019-12-20 16:10:59,972 epoch 22 - iter 270/307 - loss 0.02774272 - samples/sec: 652.87\n",
            "2019-12-20 16:11:00,711 epoch 22 - iter 300/307 - loss 0.02620097 - samples/sec: 651.06\n",
            "2019-12-20 16:11:00,880 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:00,882 EPOCH 22 done: loss 0.0272 - lr 0.0250\n",
            "2019-12-20 16:11:01,442 DEV : loss 0.3628929555416107 - score 0.9248\n",
            "2019-12-20 16:11:01,467 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 16:11:01,469 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:01,500 epoch 23 - iter 0/307 - loss 0.00966564 - samples/sec: 16752.23\n",
            "2019-12-20 16:11:02,223 epoch 23 - iter 30/307 - loss 0.01310125 - samples/sec: 665.57\n",
            "2019-12-20 16:11:02,987 epoch 23 - iter 60/307 - loss 0.01924239 - samples/sec: 630.62\n",
            "2019-12-20 16:11:03,731 epoch 23 - iter 90/307 - loss 0.02241400 - samples/sec: 647.12\n",
            "2019-12-20 16:11:04,482 epoch 23 - iter 120/307 - loss 0.02084101 - samples/sec: 641.50\n",
            "2019-12-20 16:11:05,225 epoch 23 - iter 150/307 - loss 0.02323425 - samples/sec: 651.58\n",
            "2019-12-20 16:11:05,953 epoch 23 - iter 180/307 - loss 0.02163821 - samples/sec: 661.63\n",
            "2019-12-20 16:11:06,680 epoch 23 - iter 210/307 - loss 0.02088088 - samples/sec: 663.07\n",
            "2019-12-20 16:11:07,412 epoch 23 - iter 240/307 - loss 0.02099210 - samples/sec: 658.05\n",
            "2019-12-20 16:11:08,139 epoch 23 - iter 270/307 - loss 0.02064105 - samples/sec: 663.32\n",
            "2019-12-20 16:11:08,887 epoch 23 - iter 300/307 - loss 0.02116822 - samples/sec: 644.04\n",
            "2019-12-20 16:11:09,041 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:09,042 EPOCH 23 done: loss 0.0210 - lr 0.0250\n",
            "2019-12-20 16:11:09,593 DEV : loss 0.2756718695163727 - score 0.9303\n",
            "2019-12-20 16:11:09,616 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 16:11:09,617 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:09,646 epoch 24 - iter 0/307 - loss 0.04291716 - samples/sec: 18034.68\n",
            "2019-12-20 16:11:10,376 epoch 24 - iter 30/307 - loss 0.04012335 - samples/sec: 659.17\n",
            "2019-12-20 16:11:11,129 epoch 24 - iter 60/307 - loss 0.04729672 - samples/sec: 640.02\n",
            "2019-12-20 16:11:11,871 epoch 24 - iter 90/307 - loss 0.03428622 - samples/sec: 648.40\n",
            "2019-12-20 16:11:12,597 epoch 24 - iter 120/307 - loss 0.03326474 - samples/sec: 663.35\n",
            "2019-12-20 16:11:13,318 epoch 24 - iter 150/307 - loss 0.03156848 - samples/sec: 668.09\n",
            "2019-12-20 16:11:14,074 epoch 24 - iter 180/307 - loss 0.03063830 - samples/sec: 636.55\n",
            "2019-12-20 16:11:14,800 epoch 24 - iter 210/307 - loss 0.02997388 - samples/sec: 663.39\n",
            "2019-12-20 16:11:15,519 epoch 24 - iter 240/307 - loss 0.02803708 - samples/sec: 670.64\n",
            "2019-12-20 16:11:16,247 epoch 24 - iter 270/307 - loss 0.02714317 - samples/sec: 661.45\n",
            "2019-12-20 16:11:16,995 epoch 24 - iter 300/307 - loss 0.02780320 - samples/sec: 644.70\n",
            "2019-12-20 16:11:17,133 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:17,134 EPOCH 24 done: loss 0.0285 - lr 0.0250\n",
            "2019-12-20 16:11:17,685 DEV : loss 0.2773664891719818 - score 0.9339\n",
            "Epoch    23: reducing learning rate of group 0 to 1.2500e-02.\n",
            "  0%|          | 0/100 [03:54<?, ?it/s, best loss: ?]2019-12-20 16:11:17,718 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 16:11:17,719 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:17,752 epoch 25 - iter 0/307 - loss 0.00554398 - samples/sec: 15841.39\n",
            "2019-12-20 16:11:18,480 epoch 25 - iter 30/307 - loss 0.01585654 - samples/sec: 661.44\n",
            "2019-12-20 16:11:19,201 epoch 25 - iter 60/307 - loss 0.02023115 - samples/sec: 668.08\n",
            "2019-12-20 16:11:19,951 epoch 25 - iter 90/307 - loss 0.02089918 - samples/sec: 642.65\n",
            "2019-12-20 16:11:20,675 epoch 25 - iter 120/307 - loss 0.01955855 - samples/sec: 664.91\n",
            "2019-12-20 16:11:21,386 epoch 25 - iter 150/307 - loss 0.01998916 - samples/sec: 677.46\n",
            "2019-12-20 16:11:22,140 epoch 25 - iter 180/307 - loss 0.01937055 - samples/sec: 641.52\n",
            "2019-12-20 16:11:22,886 epoch 25 - iter 210/307 - loss 0.02175495 - samples/sec: 645.06\n",
            "2019-12-20 16:11:23,607 epoch 25 - iter 240/307 - loss 0.02089676 - samples/sec: 667.76\n",
            "2019-12-20 16:11:24,325 epoch 25 - iter 270/307 - loss 0.02168992 - samples/sec: 671.37\n",
            "2019-12-20 16:11:25,072 epoch 25 - iter 300/307 - loss 0.02093879 - samples/sec: 644.85\n",
            "2019-12-20 16:11:25,224 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:25,225 EPOCH 25 done: loss 0.0206 - lr 0.0125\n",
            "2019-12-20 16:11:25,775 DEV : loss 0.3186013996601105 - score 0.9321\n",
            "2019-12-20 16:11:25,800 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 16:11:25,802 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:25,831 epoch 26 - iter 0/307 - loss 0.00079119 - samples/sec: 17141.03\n",
            "2019-12-20 16:11:26,541 epoch 26 - iter 30/307 - loss 0.01696520 - samples/sec: 678.14\n",
            "2019-12-20 16:11:27,269 epoch 26 - iter 60/307 - loss 0.01642010 - samples/sec: 662.28\n",
            "2019-12-20 16:11:28,009 epoch 26 - iter 90/307 - loss 0.01705152 - samples/sec: 650.55\n",
            "2019-12-20 16:11:28,734 epoch 26 - iter 120/307 - loss 0.02161687 - samples/sec: 667.05\n",
            "2019-12-20 16:11:29,482 epoch 26 - iter 150/307 - loss 0.02072428 - samples/sec: 646.97\n",
            "2019-12-20 16:11:30,223 epoch 26 - iter 180/307 - loss 0.02168802 - samples/sec: 652.37\n",
            "2019-12-20 16:11:30,968 epoch 26 - iter 210/307 - loss 0.02161655 - samples/sec: 649.65\n",
            "2019-12-20 16:11:31,708 epoch 26 - iter 240/307 - loss 0.02109974 - samples/sec: 650.47\n",
            "2019-12-20 16:11:32,434 epoch 26 - iter 270/307 - loss 0.01932917 - samples/sec: 663.62\n",
            "2019-12-20 16:11:33,188 epoch 26 - iter 300/307 - loss 0.02029024 - samples/sec: 638.62\n",
            "2019-12-20 16:11:33,342 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:33,343 EPOCH 26 done: loss 0.0202 - lr 0.0125\n",
            "2019-12-20 16:11:33,884 DEV : loss 0.33737772703170776 - score 0.9358\n",
            "2019-12-20 16:11:33,908 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 16:11:33,909 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:33,937 epoch 27 - iter 0/307 - loss 0.00205812 - samples/sec: 18350.63\n",
            "2019-12-20 16:11:34,671 epoch 27 - iter 30/307 - loss 0.02743370 - samples/sec: 655.74\n",
            "2019-12-20 16:11:35,406 epoch 27 - iter 60/307 - loss 0.02120703 - samples/sec: 655.44\n",
            "2019-12-20 16:11:36,129 epoch 27 - iter 90/307 - loss 0.02397355 - samples/sec: 665.64\n",
            "2019-12-20 16:11:36,871 epoch 27 - iter 120/307 - loss 0.02129246 - samples/sec: 651.28\n",
            "2019-12-20 16:11:37,609 epoch 27 - iter 150/307 - loss 0.02046617 - samples/sec: 652.46\n",
            "2019-12-20 16:11:38,349 epoch 27 - iter 180/307 - loss 0.02116053 - samples/sec: 650.78\n",
            "2019-12-20 16:11:39,075 epoch 27 - iter 210/307 - loss 0.02076209 - samples/sec: 663.88\n",
            "2019-12-20 16:11:39,819 epoch 27 - iter 240/307 - loss 0.01960915 - samples/sec: 647.01\n",
            "2019-12-20 16:11:40,589 epoch 27 - iter 270/307 - loss 0.02009948 - samples/sec: 625.26\n",
            "2019-12-20 16:11:41,341 epoch 27 - iter 300/307 - loss 0.02080888 - samples/sec: 640.31\n",
            "2019-12-20 16:11:41,494 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:41,495 EPOCH 27 done: loss 0.0205 - lr 0.0125\n",
            "2019-12-20 16:11:42,052 DEV : loss 0.29918017983436584 - score 0.9358\n",
            "2019-12-20 16:11:42,076 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 16:11:42,077 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:42,112 epoch 28 - iter 0/307 - loss 0.00058532 - samples/sec: 14468.42\n",
            "2019-12-20 16:11:42,862 epoch 28 - iter 30/307 - loss 0.02255700 - samples/sec: 642.39\n",
            "2019-12-20 16:11:43,611 epoch 28 - iter 60/307 - loss 0.01814656 - samples/sec: 642.80\n",
            "2019-12-20 16:11:44,343 epoch 28 - iter 90/307 - loss 0.01791606 - samples/sec: 658.39\n",
            "2019-12-20 16:11:45,053 epoch 28 - iter 120/307 - loss 0.02220051 - samples/sec: 678.43\n",
            "2019-12-20 16:11:45,778 epoch 28 - iter 150/307 - loss 0.02155040 - samples/sec: 665.24\n",
            "2019-12-20 16:11:46,512 epoch 28 - iter 180/307 - loss 0.02070797 - samples/sec: 655.39\n",
            "2019-12-20 16:11:47,227 epoch 28 - iter 210/307 - loss 0.02076473 - samples/sec: 674.51\n",
            "2019-12-20 16:11:47,953 epoch 28 - iter 240/307 - loss 0.02023535 - samples/sec: 663.33\n",
            "2019-12-20 16:11:48,685 epoch 28 - iter 270/307 - loss 0.02006052 - samples/sec: 657.95\n",
            "2019-12-20 16:11:49,419 epoch 28 - iter 300/307 - loss 0.01971791 - samples/sec: 656.49\n",
            "2019-12-20 16:11:49,575 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:49,576 EPOCH 28 done: loss 0.0196 - lr 0.0125\n",
            "2019-12-20 16:11:50,118 DEV : loss 0.32323572039604187 - score 0.9303\n",
            "2019-12-20 16:11:50,142 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 16:11:50,143 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:50,173 epoch 29 - iter 0/307 - loss 0.00468343 - samples/sec: 18750.03\n",
            "2019-12-20 16:11:50,924 epoch 29 - iter 30/307 - loss 0.01974043 - samples/sec: 640.38\n",
            "2019-12-20 16:11:51,678 epoch 29 - iter 60/307 - loss 0.02492387 - samples/sec: 638.54\n",
            "2019-12-20 16:11:52,399 epoch 29 - iter 90/307 - loss 0.02094044 - samples/sec: 669.12\n",
            "2019-12-20 16:11:53,128 epoch 29 - iter 120/307 - loss 0.01995678 - samples/sec: 660.95\n",
            "2019-12-20 16:11:53,871 epoch 29 - iter 150/307 - loss 0.02216549 - samples/sec: 647.89\n",
            "2019-12-20 16:11:54,628 epoch 29 - iter 180/307 - loss 0.02422137 - samples/sec: 635.95\n",
            "2019-12-20 16:11:55,360 epoch 29 - iter 210/307 - loss 0.02410986 - samples/sec: 658.66\n",
            "2019-12-20 16:11:56,097 epoch 29 - iter 240/307 - loss 0.02393734 - samples/sec: 652.78\n",
            "2019-12-20 16:11:56,824 epoch 29 - iter 270/307 - loss 0.02322478 - samples/sec: 663.08\n",
            "2019-12-20 16:11:57,543 epoch 29 - iter 300/307 - loss 0.02194257 - samples/sec: 669.79\n",
            "2019-12-20 16:11:57,700 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:57,702 EPOCH 29 done: loss 0.0222 - lr 0.0125\n",
            "2019-12-20 16:11:58,254 DEV : loss 0.32331493496894836 - score 0.9339\n",
            "2019-12-20 16:11:58,278 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 16:11:58,279 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:11:58,308 epoch 30 - iter 0/307 - loss 0.01097929 - samples/sec: 18019.03\n",
            "2019-12-20 16:11:59,034 epoch 30 - iter 30/307 - loss 0.03627658 - samples/sec: 663.27\n",
            "2019-12-20 16:11:59,793 epoch 30 - iter 60/307 - loss 0.02467271 - samples/sec: 635.08\n",
            "2019-12-20 16:12:00,508 epoch 30 - iter 90/307 - loss 0.02261514 - samples/sec: 674.33\n",
            "2019-12-20 16:12:01,231 epoch 30 - iter 120/307 - loss 0.01959963 - samples/sec: 666.23\n",
            "2019-12-20 16:12:01,954 epoch 30 - iter 150/307 - loss 0.02183082 - samples/sec: 666.35\n",
            "2019-12-20 16:12:02,700 epoch 30 - iter 180/307 - loss 0.02224111 - samples/sec: 645.33\n",
            "2019-12-20 16:12:03,435 epoch 30 - iter 210/307 - loss 0.02115225 - samples/sec: 655.24\n",
            "2019-12-20 16:12:04,202 epoch 30 - iter 240/307 - loss 0.01949750 - samples/sec: 627.69\n",
            "2019-12-20 16:12:04,955 epoch 30 - iter 270/307 - loss 0.01841581 - samples/sec: 639.46\n",
            "2019-12-20 16:12:05,698 epoch 30 - iter 300/307 - loss 0.01760957 - samples/sec: 650.02\n",
            "2019-12-20 16:12:05,857 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:12:05,859 EPOCH 30 done: loss 0.0174 - lr 0.0125\n",
            "2019-12-20 16:12:06,420 DEV : loss 0.3369459807872772 - score 0.9339\n",
            "Epoch    29: reducing learning rate of group 0 to 6.2500e-03.\n",
            "  0%|          | 0/100 [04:43<?, ?it/s, best loss: ?]2019-12-20 16:12:06,452 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 16:12:06,455 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:12:06,490 epoch 31 - iter 0/307 - loss 0.00495228 - samples/sec: 14945.41\n",
            "2019-12-20 16:12:07,237 epoch 31 - iter 30/307 - loss 0.01332430 - samples/sec: 644.60\n",
            "2019-12-20 16:12:07,965 epoch 31 - iter 60/307 - loss 0.01169482 - samples/sec: 660.99\n",
            "2019-12-20 16:12:08,684 epoch 31 - iter 90/307 - loss 0.01318769 - samples/sec: 670.24\n",
            "2019-12-20 16:12:09,417 epoch 31 - iter 120/307 - loss 0.01562629 - samples/sec: 657.34\n",
            "2019-12-20 16:12:10,148 epoch 31 - iter 150/307 - loss 0.01573932 - samples/sec: 658.76\n",
            "2019-12-20 16:12:10,884 epoch 31 - iter 180/307 - loss 0.01589156 - samples/sec: 653.82\n",
            "2019-12-20 16:12:11,645 epoch 31 - iter 210/307 - loss 0.01671153 - samples/sec: 633.54\n",
            "2019-12-20 16:12:12,385 epoch 31 - iter 240/307 - loss 0.01656357 - samples/sec: 651.34\n",
            "2019-12-20 16:12:13,108 epoch 31 - iter 270/307 - loss 0.01586170 - samples/sec: 666.63\n",
            "2019-12-20 16:12:13,854 epoch 31 - iter 300/307 - loss 0.01561505 - samples/sec: 645.12\n",
            "2019-12-20 16:12:14,003 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:12:14,005 EPOCH 31 done: loss 0.0156 - lr 0.0063\n",
            "2019-12-20 16:12:14,563 DEV : loss 0.35881075263023376 - score 0.9339\n",
            "2019-12-20 16:12:14,587 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 16:12:14,589 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:12:14,623 epoch 32 - iter 0/307 - loss 0.00125477 - samples/sec: 15647.96\n",
            "2019-12-20 16:12:15,391 epoch 32 - iter 30/307 - loss 0.01045168 - samples/sec: 626.63\n",
            "2019-12-20 16:12:16,121 epoch 32 - iter 60/307 - loss 0.01886722 - samples/sec: 659.92\n",
            "2019-12-20 16:12:16,845 epoch 32 - iter 90/307 - loss 0.02000133 - samples/sec: 664.78\n",
            "2019-12-20 16:12:17,561 epoch 32 - iter 120/307 - loss 0.01881844 - samples/sec: 672.36\n",
            "2019-12-20 16:12:18,295 epoch 32 - iter 150/307 - loss 0.01660844 - samples/sec: 656.77\n",
            "2019-12-20 16:12:19,019 epoch 32 - iter 180/307 - loss 0.01624004 - samples/sec: 665.19\n",
            "2019-12-20 16:12:19,745 epoch 32 - iter 210/307 - loss 0.01625369 - samples/sec: 662.68\n",
            "2019-12-20 16:12:20,448 epoch 32 - iter 240/307 - loss 0.01671247 - samples/sec: 685.27\n",
            "2019-12-20 16:12:21,192 epoch 32 - iter 270/307 - loss 0.01660690 - samples/sec: 647.18\n",
            "2019-12-20 16:12:21,907 epoch 32 - iter 300/307 - loss 0.01651323 - samples/sec: 673.16\n",
            "2019-12-20 16:12:22,057 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:12:22,058 EPOCH 32 done: loss 0.0165 - lr 0.0063\n",
            "2019-12-20 16:12:22,618 DEV : loss 0.3189878761768341 - score 0.9358\n",
            "2019-12-20 16:12:22,643 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 16:12:22,644 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:12:22,681 epoch 33 - iter 0/307 - loss 0.02905399 - samples/sec: 14043.72\n",
            "2019-12-20 16:12:23,418 epoch 33 - iter 30/307 - loss 0.00740909 - samples/sec: 654.66\n",
            "2019-12-20 16:12:24,147 epoch 33 - iter 60/307 - loss 0.02163146 - samples/sec: 660.87\n",
            "2019-12-20 16:12:24,866 epoch 33 - iter 90/307 - loss 0.01967280 - samples/sec: 670.32\n",
            "2019-12-20 16:12:25,598 epoch 33 - iter 120/307 - loss 0.02185749 - samples/sec: 660.09\n",
            "2019-12-20 16:12:26,362 epoch 33 - iter 150/307 - loss 0.01964997 - samples/sec: 633.33\n",
            "2019-12-20 16:12:27,080 epoch 33 - iter 180/307 - loss 0.01872738 - samples/sec: 670.70\n",
            "2019-12-20 16:12:27,815 epoch 33 - iter 210/307 - loss 0.01743798 - samples/sec: 655.59\n",
            "2019-12-20 16:12:28,540 epoch 33 - iter 240/307 - loss 0.01656773 - samples/sec: 664.53\n",
            "2019-12-20 16:12:29,285 epoch 33 - iter 270/307 - loss 0.01649806 - samples/sec: 645.79\n",
            "2019-12-20 16:12:30,032 epoch 33 - iter 300/307 - loss 0.01594949 - samples/sec: 644.87\n",
            "2019-12-20 16:12:30,192 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:12:30,193 EPOCH 33 done: loss 0.0161 - lr 0.0063\n",
            "2019-12-20 16:12:30,739 DEV : loss 0.31322625279426575 - score 0.9339\n",
            "2019-12-20 16:12:30,762 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 16:12:30,763 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:12:30,791 epoch 34 - iter 0/307 - loss 0.00060904 - samples/sec: 18287.29\n",
            "2019-12-20 16:12:31,529 epoch 34 - iter 30/307 - loss 0.00468486 - samples/sec: 652.57\n",
            "2019-12-20 16:12:32,274 epoch 34 - iter 60/307 - loss 0.00687919 - samples/sec: 646.01\n",
            "2019-12-20 16:12:32,997 epoch 34 - iter 90/307 - loss 0.00679212 - samples/sec: 668.13\n",
            "2019-12-20 16:12:33,722 epoch 34 - iter 120/307 - loss 0.00945613 - samples/sec: 664.03\n",
            "2019-12-20 16:12:34,456 epoch 34 - iter 150/307 - loss 0.01152366 - samples/sec: 656.59\n",
            "2019-12-20 16:12:35,201 epoch 34 - iter 180/307 - loss 0.01101726 - samples/sec: 646.51\n",
            "2019-12-20 16:12:35,939 epoch 34 - iter 210/307 - loss 0.01297404 - samples/sec: 653.05\n",
            "2019-12-20 16:12:36,688 epoch 34 - iter 240/307 - loss 0.01240677 - samples/sec: 642.31\n",
            "2019-12-20 16:12:37,446 epoch 34 - iter 270/307 - loss 0.01288561 - samples/sec: 635.92\n",
            "2019-12-20 16:12:38,194 epoch 34 - iter 300/307 - loss 0.01306967 - samples/sec: 643.43\n",
            "2019-12-20 16:12:38,340 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:12:38,341 EPOCH 34 done: loss 0.0133 - lr 0.0063\n",
            "2019-12-20 16:12:38,897 DEV : loss 0.3351536989212036 - score 0.9339\n",
            "Epoch    33: reducing learning rate of group 0 to 3.1250e-03.\n",
            "  0%|          | 0/100 [05:15<?, ?it/s, best loss: ?]2019-12-20 16:12:38,929 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 16:12:38,931 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:12:38,966 epoch 35 - iter 0/307 - loss 0.00082666 - samples/sec: 15147.36\n",
            "2019-12-20 16:12:39,717 epoch 35 - iter 30/307 - loss 0.00950131 - samples/sec: 641.58\n",
            "2019-12-20 16:12:40,449 epoch 35 - iter 60/307 - loss 0.01087679 - samples/sec: 659.33\n",
            "2019-12-20 16:12:41,190 epoch 35 - iter 90/307 - loss 0.01455859 - samples/sec: 651.29\n",
            "2019-12-20 16:12:41,936 epoch 35 - iter 120/307 - loss 0.01466454 - samples/sec: 647.37\n",
            "2019-12-20 16:12:42,652 epoch 35 - iter 150/307 - loss 0.01512103 - samples/sec: 674.12\n",
            "2019-12-20 16:12:43,384 epoch 35 - iter 180/307 - loss 0.01493952 - samples/sec: 658.30\n",
            "2019-12-20 16:12:44,122 epoch 35 - iter 210/307 - loss 0.01511739 - samples/sec: 654.28\n",
            "2019-12-20 16:12:44,855 epoch 35 - iter 240/307 - loss 0.01595128 - samples/sec: 657.41\n",
            "2019-12-20 16:12:45,586 epoch 35 - iter 270/307 - loss 0.01514172 - samples/sec: 659.45\n",
            "2019-12-20 16:12:46,324 epoch 35 - iter 300/307 - loss 0.01513934 - samples/sec: 651.98\n",
            "2019-12-20 16:12:46,470 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:12:46,472 EPOCH 35 done: loss 0.0152 - lr 0.0031\n",
            "2019-12-20 16:12:47,025 DEV : loss 0.31361687183380127 - score 0.9321\n",
            "2019-12-20 16:12:47,049 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 16:12:47,051 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:12:47,080 epoch 36 - iter 0/307 - loss 0.00079495 - samples/sec: 17254.45\n",
            "2019-12-20 16:12:47,785 epoch 36 - iter 30/307 - loss 0.01014217 - samples/sec: 682.87\n",
            "2019-12-20 16:12:48,542 epoch 36 - iter 60/307 - loss 0.00838165 - samples/sec: 636.39\n",
            "2019-12-20 16:12:49,303 epoch 36 - iter 90/307 - loss 0.01079053 - samples/sec: 632.60\n",
            "2019-12-20 16:12:50,027 epoch 36 - iter 120/307 - loss 0.01351681 - samples/sec: 670.16\n",
            "2019-12-20 16:12:50,753 epoch 36 - iter 150/307 - loss 0.01287424 - samples/sec: 663.20\n",
            "2019-12-20 16:12:51,502 epoch 36 - iter 180/307 - loss 0.01316855 - samples/sec: 643.28\n",
            "2019-12-20 16:12:52,215 epoch 36 - iter 210/307 - loss 0.01303959 - samples/sec: 675.03\n",
            "2019-12-20 16:12:52,960 epoch 36 - iter 240/307 - loss 0.01478624 - samples/sec: 646.26\n",
            "2019-12-20 16:12:53,689 epoch 36 - iter 270/307 - loss 0.01571277 - samples/sec: 660.43\n",
            "2019-12-20 16:12:54,413 epoch 36 - iter 300/307 - loss 0.01579463 - samples/sec: 665.12\n",
            "2019-12-20 16:12:54,569 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:12:54,570 EPOCH 36 done: loss 0.0155 - lr 0.0031\n",
            "2019-12-20 16:12:55,121 DEV : loss 0.32398560643196106 - score 0.9339\n",
            "2019-12-20 16:12:55,146 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 16:12:55,147 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:12:55,175 epoch 37 - iter 0/307 - loss 0.00198725 - samples/sec: 19152.98\n",
            "2019-12-20 16:12:55,887 epoch 37 - iter 30/307 - loss 0.01448351 - samples/sec: 676.17\n",
            "2019-12-20 16:12:56,628 epoch 37 - iter 60/307 - loss 0.01300658 - samples/sec: 649.70\n",
            "2019-12-20 16:12:57,369 epoch 37 - iter 90/307 - loss 0.01453035 - samples/sec: 651.54\n",
            "2019-12-20 16:12:58,102 epoch 37 - iter 120/307 - loss 0.01437998 - samples/sec: 657.26\n",
            "2019-12-20 16:12:58,867 epoch 37 - iter 150/307 - loss 0.01373170 - samples/sec: 630.05\n",
            "2019-12-20 16:12:59,606 epoch 37 - iter 180/307 - loss 0.01401396 - samples/sec: 651.34\n",
            "2019-12-20 16:13:00,363 epoch 37 - iter 210/307 - loss 0.01500999 - samples/sec: 636.66\n",
            "2019-12-20 16:13:01,088 epoch 37 - iter 240/307 - loss 0.01501077 - samples/sec: 664.46\n",
            "2019-12-20 16:13:01,833 epoch 37 - iter 270/307 - loss 0.01452246 - samples/sec: 646.73\n",
            "2019-12-20 16:13:02,572 epoch 37 - iter 300/307 - loss 0.01515788 - samples/sec: 650.72\n",
            "2019-12-20 16:13:02,718 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:02,719 EPOCH 37 done: loss 0.0149 - lr 0.0031\n",
            "2019-12-20 16:13:03,271 DEV : loss 0.31794071197509766 - score 0.9303\n",
            "2019-12-20 16:13:03,296 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 16:13:03,297 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:03,327 epoch 38 - iter 0/307 - loss 0.04447895 - samples/sec: 17256.07\n",
            "2019-12-20 16:13:04,045 epoch 38 - iter 30/307 - loss 0.01995450 - samples/sec: 671.06\n",
            "2019-12-20 16:13:04,766 epoch 38 - iter 60/307 - loss 0.02375348 - samples/sec: 668.04\n",
            "2019-12-20 16:13:05,504 epoch 38 - iter 90/307 - loss 0.02199285 - samples/sec: 652.65\n",
            "2019-12-20 16:13:06,236 epoch 38 - iter 120/307 - loss 0.02104689 - samples/sec: 658.11\n",
            "2019-12-20 16:13:06,972 epoch 38 - iter 150/307 - loss 0.01980805 - samples/sec: 655.80\n",
            "2019-12-20 16:13:07,724 epoch 38 - iter 180/307 - loss 0.01774775 - samples/sec: 640.26\n",
            "2019-12-20 16:13:08,472 epoch 38 - iter 210/307 - loss 0.01711364 - samples/sec: 644.36\n",
            "2019-12-20 16:13:09,207 epoch 38 - iter 240/307 - loss 0.01612528 - samples/sec: 655.35\n",
            "2019-12-20 16:13:09,960 epoch 38 - iter 270/307 - loss 0.01629040 - samples/sec: 638.91\n",
            "2019-12-20 16:13:10,693 epoch 38 - iter 300/307 - loss 0.01767163 - samples/sec: 657.32\n",
            "2019-12-20 16:13:10,836 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:10,837 EPOCH 38 done: loss 0.0175 - lr 0.0031\n",
            "2019-12-20 16:13:11,394 DEV : loss 0.30692389607429504 - score 0.9339\n",
            "Epoch    37: reducing learning rate of group 0 to 1.5625e-03.\n",
            "  0%|          | 0/100 [05:48<?, ?it/s, best loss: ?]2019-12-20 16:13:11,422 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 16:13:11,424 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:11,456 epoch 39 - iter 0/307 - loss 0.00354493 - samples/sec: 15650.14\n",
            "2019-12-20 16:13:12,195 epoch 39 - iter 30/307 - loss 0.01759555 - samples/sec: 651.51\n",
            "2019-12-20 16:13:12,931 epoch 39 - iter 60/307 - loss 0.01817006 - samples/sec: 654.36\n",
            "2019-12-20 16:13:13,664 epoch 39 - iter 90/307 - loss 0.01576710 - samples/sec: 656.53\n",
            "2019-12-20 16:13:14,391 epoch 39 - iter 120/307 - loss 0.01575409 - samples/sec: 663.12\n",
            "2019-12-20 16:13:15,136 epoch 39 - iter 150/307 - loss 0.01507131 - samples/sec: 645.86\n",
            "2019-12-20 16:13:15,900 epoch 39 - iter 180/307 - loss 0.01402110 - samples/sec: 633.46\n",
            "2019-12-20 16:13:16,600 epoch 39 - iter 210/307 - loss 0.01368871 - samples/sec: 687.35\n",
            "2019-12-20 16:13:17,317 epoch 39 - iter 240/307 - loss 0.01534575 - samples/sec: 671.44\n",
            "2019-12-20 16:13:18,064 epoch 39 - iter 270/307 - loss 0.01525662 - samples/sec: 644.48\n",
            "2019-12-20 16:13:18,774 epoch 39 - iter 300/307 - loss 0.01543131 - samples/sec: 679.05\n",
            "2019-12-20 16:13:18,923 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:18,924 EPOCH 39 done: loss 0.0152 - lr 0.0016\n",
            "2019-12-20 16:13:19,478 DEV : loss 0.3067450523376465 - score 0.9339\n",
            "2019-12-20 16:13:19,501 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 16:13:19,502 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:19,531 epoch 40 - iter 0/307 - loss 0.10194471 - samples/sec: 18247.84\n",
            "2019-12-20 16:13:20,243 epoch 40 - iter 30/307 - loss 0.01822112 - samples/sec: 675.51\n",
            "2019-12-20 16:13:21,007 epoch 40 - iter 60/307 - loss 0.01830101 - samples/sec: 631.04\n",
            "2019-12-20 16:13:21,741 epoch 40 - iter 90/307 - loss 0.01816495 - samples/sec: 655.65\n",
            "2019-12-20 16:13:22,450 epoch 40 - iter 120/307 - loss 0.01706521 - samples/sec: 680.61\n",
            "2019-12-20 16:13:23,203 epoch 40 - iter 150/307 - loss 0.01997370 - samples/sec: 639.68\n",
            "2019-12-20 16:13:23,964 epoch 40 - iter 180/307 - loss 0.02041926 - samples/sec: 634.69\n",
            "2019-12-20 16:13:24,676 epoch 40 - iter 210/307 - loss 0.01934552 - samples/sec: 677.08\n",
            "2019-12-20 16:13:25,414 epoch 40 - iter 240/307 - loss 0.01862421 - samples/sec: 653.09\n",
            "2019-12-20 16:13:26,157 epoch 40 - iter 270/307 - loss 0.01761075 - samples/sec: 647.99\n",
            "2019-12-20 16:13:26,895 epoch 40 - iter 300/307 - loss 0.01692779 - samples/sec: 652.14\n",
            "2019-12-20 16:13:27,040 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:27,041 EPOCH 40 done: loss 0.0169 - lr 0.0016\n",
            "2019-12-20 16:13:27,597 DEV : loss 0.3086332678794861 - score 0.9321\n",
            "2019-12-20 16:13:27,620 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 16:13:27,621 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:27,651 epoch 41 - iter 0/307 - loss 0.04405580 - samples/sec: 18156.51\n",
            "2019-12-20 16:13:28,387 epoch 41 - iter 30/307 - loss 0.01360850 - samples/sec: 654.78\n",
            "2019-12-20 16:13:29,125 epoch 41 - iter 60/307 - loss 0.01220059 - samples/sec: 652.32\n",
            "2019-12-20 16:13:29,859 epoch 41 - iter 90/307 - loss 0.01647351 - samples/sec: 656.14\n",
            "2019-12-20 16:13:30,598 epoch 41 - iter 120/307 - loss 0.01360609 - samples/sec: 651.40\n",
            "2019-12-20 16:13:31,366 epoch 41 - iter 150/307 - loss 0.01283838 - samples/sec: 627.54\n",
            "2019-12-20 16:13:32,107 epoch 41 - iter 180/307 - loss 0.01177372 - samples/sec: 649.72\n",
            "2019-12-20 16:13:32,809 epoch 41 - iter 210/307 - loss 0.01165275 - samples/sec: 686.73\n",
            "2019-12-20 16:13:33,548 epoch 41 - iter 240/307 - loss 0.01148155 - samples/sec: 652.05\n",
            "2019-12-20 16:13:34,287 epoch 41 - iter 270/307 - loss 0.01258410 - samples/sec: 651.86\n",
            "2019-12-20 16:13:35,028 epoch 41 - iter 300/307 - loss 0.01308977 - samples/sec: 650.02\n",
            "2019-12-20 16:13:35,177 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:35,178 EPOCH 41 done: loss 0.0133 - lr 0.0016\n",
            "2019-12-20 16:13:35,726 DEV : loss 0.31640610098838806 - score 0.9358\n",
            "2019-12-20 16:13:35,749 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 16:13:35,750 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:35,779 epoch 42 - iter 0/307 - loss 0.01078975 - samples/sec: 18153.56\n",
            "2019-12-20 16:13:36,520 epoch 42 - iter 30/307 - loss 0.02310128 - samples/sec: 648.92\n",
            "2019-12-20 16:13:37,246 epoch 42 - iter 60/307 - loss 0.01790665 - samples/sec: 667.35\n",
            "2019-12-20 16:13:37,983 epoch 42 - iter 90/307 - loss 0.01553214 - samples/sec: 653.34\n",
            "2019-12-20 16:13:38,732 epoch 42 - iter 120/307 - loss 0.01425625 - samples/sec: 643.13\n",
            "2019-12-20 16:13:39,459 epoch 42 - iter 150/307 - loss 0.01414475 - samples/sec: 662.75\n",
            "2019-12-20 16:13:40,194 epoch 42 - iter 180/307 - loss 0.01776194 - samples/sec: 655.08\n",
            "2019-12-20 16:13:40,909 epoch 42 - iter 210/307 - loss 0.01638049 - samples/sec: 673.56\n",
            "2019-12-20 16:13:41,642 epoch 42 - iter 240/307 - loss 0.01549638 - samples/sec: 656.69\n",
            "2019-12-20 16:13:42,396 epoch 42 - iter 270/307 - loss 0.01546394 - samples/sec: 639.12\n",
            "2019-12-20 16:13:43,141 epoch 42 - iter 300/307 - loss 0.01498200 - samples/sec: 648.50\n",
            "2019-12-20 16:13:43,303 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:43,305 EPOCH 42 done: loss 0.0148 - lr 0.0016\n",
            "2019-12-20 16:13:43,853 DEV : loss 0.3135504722595215 - score 0.9339\n",
            "Epoch    41: reducing learning rate of group 0 to 7.8125e-04.\n",
            "  0%|          | 0/100 [06:20<?, ?it/s, best loss: ?]2019-12-20 16:13:43,881 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 16:13:43,884 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:43,921 epoch 43 - iter 0/307 - loss 0.00079060 - samples/sec: 14083.02\n",
            "2019-12-20 16:13:44,657 epoch 43 - iter 30/307 - loss 0.01617304 - samples/sec: 654.44\n",
            "2019-12-20 16:13:45,393 epoch 43 - iter 60/307 - loss 0.01742048 - samples/sec: 656.06\n",
            "2019-12-20 16:13:46,127 epoch 43 - iter 90/307 - loss 0.01574409 - samples/sec: 656.11\n",
            "2019-12-20 16:13:46,863 epoch 43 - iter 120/307 - loss 0.01623260 - samples/sec: 655.57\n",
            "2019-12-20 16:13:47,614 epoch 43 - iter 150/307 - loss 0.01710820 - samples/sec: 640.66\n",
            "2019-12-20 16:13:48,330 epoch 43 - iter 180/307 - loss 0.01554970 - samples/sec: 673.18\n",
            "2019-12-20 16:13:49,051 epoch 43 - iter 210/307 - loss 0.01578503 - samples/sec: 667.49\n",
            "2019-12-20 16:13:49,796 epoch 43 - iter 240/307 - loss 0.01559145 - samples/sec: 647.43\n",
            "2019-12-20 16:13:50,553 epoch 43 - iter 270/307 - loss 0.01584899 - samples/sec: 636.96\n",
            "2019-12-20 16:13:51,306 epoch 43 - iter 300/307 - loss 0.01501438 - samples/sec: 639.49\n",
            "2019-12-20 16:13:51,466 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:51,467 EPOCH 43 done: loss 0.0149 - lr 0.0008\n",
            "2019-12-20 16:13:52,023 DEV : loss 0.31606629490852356 - score 0.9339\n",
            "2019-12-20 16:13:52,048 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 16:13:52,051 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:52,079 epoch 44 - iter 0/307 - loss 0.00249344 - samples/sec: 19142.79\n",
            "2019-12-20 16:13:52,843 epoch 44 - iter 30/307 - loss 0.01089005 - samples/sec: 631.11\n",
            "2019-12-20 16:13:53,596 epoch 44 - iter 60/307 - loss 0.01258954 - samples/sec: 638.97\n",
            "2019-12-20 16:13:54,330 epoch 44 - iter 90/307 - loss 0.01076096 - samples/sec: 656.41\n",
            "2019-12-20 16:13:55,066 epoch 44 - iter 120/307 - loss 0.01561737 - samples/sec: 654.51\n",
            "2019-12-20 16:13:55,779 epoch 44 - iter 150/307 - loss 0.01610690 - samples/sec: 675.65\n",
            "2019-12-20 16:13:56,502 epoch 44 - iter 180/307 - loss 0.01657660 - samples/sec: 665.60\n",
            "2019-12-20 16:13:57,245 epoch 44 - iter 210/307 - loss 0.01530680 - samples/sec: 648.80\n",
            "2019-12-20 16:13:57,978 epoch 44 - iter 240/307 - loss 0.01441587 - samples/sec: 657.01\n",
            "2019-12-20 16:13:58,714 epoch 44 - iter 270/307 - loss 0.01372247 - samples/sec: 654.88\n",
            "2019-12-20 16:13:59,440 epoch 44 - iter 300/307 - loss 0.01359223 - samples/sec: 663.57\n",
            "2019-12-20 16:13:59,602 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:13:59,604 EPOCH 44 done: loss 0.0136 - lr 0.0008\n",
            "2019-12-20 16:14:00,155 DEV : loss 0.32035914063453674 - score 0.9339\n",
            "2019-12-20 16:14:00,180 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 16:14:00,181 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:00,214 epoch 45 - iter 0/307 - loss 0.00489828 - samples/sec: 17268.80\n",
            "2019-12-20 16:14:00,951 epoch 45 - iter 30/307 - loss 0.00425426 - samples/sec: 653.23\n",
            "2019-12-20 16:14:01,670 epoch 45 - iter 60/307 - loss 0.01369956 - samples/sec: 669.53\n",
            "2019-12-20 16:14:02,414 epoch 45 - iter 90/307 - loss 0.01482842 - samples/sec: 647.94\n",
            "2019-12-20 16:14:03,173 epoch 45 - iter 120/307 - loss 0.01515756 - samples/sec: 634.34\n",
            "2019-12-20 16:14:03,925 epoch 45 - iter 150/307 - loss 0.01426295 - samples/sec: 640.46\n",
            "2019-12-20 16:14:04,663 epoch 45 - iter 180/307 - loss 0.01480779 - samples/sec: 652.84\n",
            "2019-12-20 16:14:05,401 epoch 45 - iter 210/307 - loss 0.01387020 - samples/sec: 653.24\n",
            "2019-12-20 16:14:06,140 epoch 45 - iter 240/307 - loss 0.01384459 - samples/sec: 651.39\n",
            "2019-12-20 16:14:06,874 epoch 45 - iter 270/307 - loss 0.01351036 - samples/sec: 656.29\n",
            "2019-12-20 16:14:07,597 epoch 45 - iter 300/307 - loss 0.01373335 - samples/sec: 666.03\n",
            "2019-12-20 16:14:07,751 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:07,752 EPOCH 45 done: loss 0.0143 - lr 0.0008\n",
            "2019-12-20 16:14:08,301 DEV : loss 0.3207687437534332 - score 0.9339\n",
            "2019-12-20 16:14:08,325 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 16:14:08,326 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:08,356 epoch 46 - iter 0/307 - loss 0.00671920 - samples/sec: 18468.13\n",
            "2019-12-20 16:14:09,093 epoch 46 - iter 30/307 - loss 0.02220312 - samples/sec: 653.04\n",
            "2019-12-20 16:14:09,848 epoch 46 - iter 60/307 - loss 0.01568787 - samples/sec: 637.68\n",
            "2019-12-20 16:14:10,565 epoch 46 - iter 90/307 - loss 0.01792170 - samples/sec: 673.57\n",
            "2019-12-20 16:14:11,295 epoch 46 - iter 120/307 - loss 0.02210497 - samples/sec: 660.00\n",
            "2019-12-20 16:14:12,024 epoch 46 - iter 150/307 - loss 0.02079332 - samples/sec: 660.43\n",
            "2019-12-20 16:14:12,782 epoch 46 - iter 180/307 - loss 0.01879906 - samples/sec: 635.92\n",
            "2019-12-20 16:14:13,498 epoch 46 - iter 210/307 - loss 0.01939325 - samples/sec: 672.22\n",
            "2019-12-20 16:14:14,254 epoch 46 - iter 240/307 - loss 0.01857108 - samples/sec: 638.98\n",
            "2019-12-20 16:14:14,981 epoch 46 - iter 270/307 - loss 0.01716140 - samples/sec: 662.71\n",
            "2019-12-20 16:14:15,693 epoch 46 - iter 300/307 - loss 0.01703531 - samples/sec: 676.42\n",
            "2019-12-20 16:14:15,845 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:15,846 EPOCH 46 done: loss 0.0169 - lr 0.0008\n",
            "2019-12-20 16:14:16,421 DEV : loss 0.32394087314605713 - score 0.9358\n",
            "Epoch    45: reducing learning rate of group 0 to 3.9063e-04.\n",
            "  0%|          | 0/100 [06:53<?, ?it/s, best loss: ?]2019-12-20 16:14:16,452 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 16:14:16,453 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:16,489 epoch 47 - iter 0/307 - loss 0.00052842 - samples/sec: 14150.13\n",
            "2019-12-20 16:14:17,231 epoch 47 - iter 30/307 - loss 0.01528173 - samples/sec: 649.15\n",
            "2019-12-20 16:14:17,971 epoch 47 - iter 60/307 - loss 0.01109151 - samples/sec: 650.57\n",
            "2019-12-20 16:14:18,677 epoch 47 - iter 90/307 - loss 0.01034762 - samples/sec: 682.95\n",
            "2019-12-20 16:14:19,396 epoch 47 - iter 120/307 - loss 0.01366347 - samples/sec: 671.80\n",
            "2019-12-20 16:14:20,136 epoch 47 - iter 150/307 - loss 0.01464552 - samples/sec: 652.88\n",
            "2019-12-20 16:14:20,861 epoch 47 - iter 180/307 - loss 0.01658864 - samples/sec: 664.23\n",
            "2019-12-20 16:14:21,598 epoch 47 - iter 210/307 - loss 0.01524809 - samples/sec: 653.50\n",
            "2019-12-20 16:14:22,336 epoch 47 - iter 240/307 - loss 0.01532551 - samples/sec: 652.74\n",
            "2019-12-20 16:14:23,069 epoch 47 - iter 270/307 - loss 0.01609029 - samples/sec: 656.95\n",
            "2019-12-20 16:14:23,801 epoch 47 - iter 300/307 - loss 0.01538105 - samples/sec: 657.99\n",
            "2019-12-20 16:14:23,947 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:23,948 EPOCH 47 done: loss 0.0157 - lr 0.0004\n",
            "2019-12-20 16:14:24,518 DEV : loss 0.32426297664642334 - score 0.9358\n",
            "2019-12-20 16:14:24,540 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 16:14:24,541 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:24,569 epoch 48 - iter 0/307 - loss 0.00076106 - samples/sec: 18585.59\n",
            "2019-12-20 16:14:25,304 epoch 48 - iter 30/307 - loss 0.02602963 - samples/sec: 654.97\n",
            "2019-12-20 16:14:26,030 epoch 48 - iter 60/307 - loss 0.01814337 - samples/sec: 663.88\n",
            "2019-12-20 16:14:26,764 epoch 48 - iter 90/307 - loss 0.01667812 - samples/sec: 655.71\n",
            "2019-12-20 16:14:27,486 epoch 48 - iter 120/307 - loss 0.01515187 - samples/sec: 667.48\n",
            "2019-12-20 16:14:28,202 epoch 48 - iter 150/307 - loss 0.01385195 - samples/sec: 674.25\n",
            "2019-12-20 16:14:28,942 epoch 48 - iter 180/307 - loss 0.01404235 - samples/sec: 651.58\n",
            "2019-12-20 16:14:29,693 epoch 48 - iter 210/307 - loss 0.01344956 - samples/sec: 641.26\n",
            "2019-12-20 16:14:30,443 epoch 48 - iter 240/307 - loss 0.01255302 - samples/sec: 641.88\n",
            "2019-12-20 16:14:31,179 epoch 48 - iter 270/307 - loss 0.01246638 - samples/sec: 654.48\n",
            "2019-12-20 16:14:31,917 epoch 48 - iter 300/307 - loss 0.01201603 - samples/sec: 653.46\n",
            "2019-12-20 16:14:32,065 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:32,066 EPOCH 48 done: loss 0.0118 - lr 0.0004\n",
            "2019-12-20 16:14:32,611 DEV : loss 0.3247479796409607 - score 0.9358\n",
            "2019-12-20 16:14:32,633 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 16:14:32,635 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:32,668 epoch 49 - iter 0/307 - loss 0.00063741 - samples/sec: 15154.89\n",
            "2019-12-20 16:14:33,407 epoch 49 - iter 30/307 - loss 0.01129501 - samples/sec: 651.97\n",
            "2019-12-20 16:14:34,164 epoch 49 - iter 60/307 - loss 0.01198466 - samples/sec: 636.35\n",
            "2019-12-20 16:14:34,894 epoch 49 - iter 90/307 - loss 0.01060120 - samples/sec: 659.12\n",
            "2019-12-20 16:14:35,669 epoch 49 - iter 120/307 - loss 0.01042949 - samples/sec: 621.96\n",
            "2019-12-20 16:14:36,399 epoch 49 - iter 150/307 - loss 0.01043971 - samples/sec: 659.56\n",
            "2019-12-20 16:14:37,118 epoch 49 - iter 180/307 - loss 0.01205405 - samples/sec: 670.05\n",
            "2019-12-20 16:14:37,850 epoch 49 - iter 210/307 - loss 0.01260571 - samples/sec: 657.94\n",
            "2019-12-20 16:14:38,579 epoch 49 - iter 240/307 - loss 0.01294038 - samples/sec: 660.50\n",
            "2019-12-20 16:14:39,309 epoch 49 - iter 270/307 - loss 0.01302995 - samples/sec: 659.93\n",
            "2019-12-20 16:14:40,075 epoch 49 - iter 300/307 - loss 0.01232221 - samples/sec: 629.24\n",
            "2019-12-20 16:14:40,232 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:40,233 EPOCH 49 done: loss 0.0124 - lr 0.0004\n",
            "2019-12-20 16:14:40,782 DEV : loss 0.32604527473449707 - score 0.9358\n",
            "2019-12-20 16:14:40,805 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 16:14:40,806 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:40,835 epoch 50 - iter 0/307 - loss 0.00418836 - samples/sec: 17980.40\n",
            "2019-12-20 16:14:41,562 epoch 50 - iter 30/307 - loss 0.01443080 - samples/sec: 662.26\n",
            "2019-12-20 16:14:42,279 epoch 50 - iter 60/307 - loss 0.01820861 - samples/sec: 671.74\n",
            "2019-12-20 16:14:43,021 epoch 50 - iter 90/307 - loss 0.01602747 - samples/sec: 648.36\n",
            "2019-12-20 16:14:43,757 epoch 50 - iter 120/307 - loss 0.01612299 - samples/sec: 654.01\n",
            "2019-12-20 16:14:44,480 epoch 50 - iter 150/307 - loss 0.01682696 - samples/sec: 666.48\n",
            "2019-12-20 16:14:45,206 epoch 50 - iter 180/307 - loss 0.01492543 - samples/sec: 663.86\n",
            "2019-12-20 16:14:45,964 epoch 50 - iter 210/307 - loss 0.01516169 - samples/sec: 635.03\n",
            "2019-12-20 16:14:46,702 epoch 50 - iter 240/307 - loss 0.01553019 - samples/sec: 652.44\n",
            "2019-12-20 16:14:47,436 epoch 50 - iter 270/307 - loss 0.01539731 - samples/sec: 655.54\n",
            "2019-12-20 16:14:48,173 epoch 50 - iter 300/307 - loss 0.01694332 - samples/sec: 653.71\n",
            "2019-12-20 16:14:48,323 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:48,324 EPOCH 50 done: loss 0.0167 - lr 0.0004\n",
            "2019-12-20 16:14:48,881 DEV : loss 0.32585906982421875 - score 0.9358\n",
            "Epoch    49: reducing learning rate of group 0 to 1.9531e-04.\n",
            "  0%|          | 0/100 [07:25<?, ?it/s, best loss: ?]2019-12-20 16:14:48,910 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 16:14:48,911 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:48,913 Testing using best model ...\n",
            "2019-12-20 16:14:51,320 0.962\t0.962\t0.962\n",
            "2019-12-20 16:14:51,321 \n",
            "MICRO_AVG: acc 0.9268 - f1-score 0.962\n",
            "MACRO_AVG: acc 0.9379 - f1-score 0.9672999999999999\n",
            "ABBR       tp: 9 - fp: 0 - fn: 0 - tn: 491 - precision: 1.0000 - recall: 1.0000 - accuracy: 1.0000 - f1-score: 1.0000\n",
            "DESC       tp: 138 - fp: 8 - fn: 0 - tn: 354 - precision: 0.9452 - recall: 1.0000 - accuracy: 0.9452 - f1-score: 0.9718\n",
            "ENTY       tp: 83 - fp: 4 - fn: 11 - tn: 402 - precision: 0.9540 - recall: 0.8830 - accuracy: 0.8469 - f1-score: 0.9171\n",
            "HUM        tp: 64 - fp: 1 - fn: 1 - tn: 434 - precision: 0.9846 - recall: 0.9846 - accuracy: 0.9697 - f1-score: 0.9846\n",
            "LOC        tp: 78 - fp: 4 - fn: 3 - tn: 415 - precision: 0.9512 - recall: 0.9630 - accuracy: 0.9176 - f1-score: 0.9571\n",
            "NUM        tp: 109 - fp: 2 - fn: 4 - tn: 385 - precision: 0.9820 - recall: 0.9646 - accuracy: 0.9478 - f1-score: 0.9732\n",
            "2019-12-20 16:14:51,322 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:51,324 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:51,325 Training run: 2\n",
            "2019-12-20 16:14:51,637 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:51,638 Model: \"TextClassifier(\n",
            "  (document_embeddings): DocumentRNNEmbeddings(\n",
            "    (embeddings): StackedEmbeddings(\n",
            "      (list_embedding_0): FlairEmbeddings(\n",
            "        (lm): LanguageModel(\n",
            "          (drop): Dropout(p=0.05, inplace=False)\n",
            "          (encoder): Embedding(300, 100)\n",
            "          (rnn): LSTM(100, 2048)\n",
            "          (decoder): Linear(in_features=2048, out_features=300, bias=True)\n",
            "        )\n",
            "      )\n",
            "      (list_embedding_1): FlairEmbeddings(\n",
            "        (lm): LanguageModel(\n",
            "          (drop): Dropout(p=0.05, inplace=False)\n",
            "          (encoder): Embedding(300, 100)\n",
            "          (rnn): LSTM(100, 2048)\n",
            "          (decoder): Linear(in_features=2048, out_features=300, bias=True)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (word_reprojection_map): Linear(in_features=4096, out_features=4096, bias=True)\n",
            "    (rnn): GRU(4096, 128, num_layers=2, batch_first=True)\n",
            "    (dropout): Dropout(p=0.4768923447560231, inplace=False)\n",
            "  )\n",
            "  (decoder): Linear(in_features=128, out_features=6, bias=True)\n",
            "  (loss_function): CrossEntropyLoss()\n",
            ")\"\n",
            "2019-12-20 16:14:51,640 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:51,641 Corpus: \"Corpus: 4907 train + 545 dev + 500 test sentences\"\n",
            "2019-12-20 16:14:51,642 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:51,643 Parameters:\n",
            "2019-12-20 16:14:51,644  - learning_rate: \"0.1\"\n",
            "2019-12-20 16:14:51,645  - mini_batch_size: \"16\"\n",
            "2019-12-20 16:14:51,647  - patience: \"3\"\n",
            "2019-12-20 16:14:51,648  - anneal_factor: \"0.5\"\n",
            "2019-12-20 16:14:51,649  - max_epochs: \"50\"\n",
            "2019-12-20 16:14:51,650  - shuffle: \"True\"\n",
            "2019-12-20 16:14:51,651  - train_with_dev: \"False\"\n",
            "2019-12-20 16:14:51,652  - batch_growth_annealing: \"False\"\n",
            "2019-12-20 16:14:51,654 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:51,655 Model training base path: \"resources/results\"\n",
            "2019-12-20 16:14:51,656 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:51,657 Device: cuda:0\n",
            "2019-12-20 16:14:51,658 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 16:14:51,659 Embeddings storage mode: cpu\n",
            "2019-12-20 16:14:51,661 ----------------------------------------------------------------------------------------------------\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-9-9dc4de8258a0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;31m# start the optimization\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mparam_selector\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msearch_space\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_evals\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/flair/hyperparameter/param_selection.py\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, space, max_evals)\u001b[0m\n\u001b[1;32m    160\u001b[0m         \u001b[0msearch_space\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch_space\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    161\u001b[0m         best = fmin(\n\u001b[0;32m--> 162\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_objective\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msearch_space\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malgo\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtpe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msuggest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_evals\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_evals\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    163\u001b[0m         )\n\u001b[1;32m    164\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mfmin\u001b[0;34m(fn, space, algo, max_evals, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin, points_to_evaluate, max_queue_len, show_progressbar)\u001b[0m\n\u001b[1;32m    405\u001b[0m                     show_progressbar=show_progressbar)\n\u001b[1;32m    406\u001b[0m     \u001b[0mrval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 407\u001b[0;31m     \u001b[0mrval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexhaust\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    408\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mreturn_argmin\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    409\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mtrials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmin\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mexhaust\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    260\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mexhaust\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    261\u001b[0m         \u001b[0mn_done\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 262\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_evals\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mn_done\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblock_until_done\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masynchronous\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    263\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrefresh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    264\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, N, block_until_done)\u001b[0m\n\u001b[1;32m    225\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    226\u001b[0m                         \u001b[0;31m# -- loop over trials and do the jobs directly\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 227\u001b[0;31m                         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mserial_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    228\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    229\u001b[0m                     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mserial_evaluate\u001b[0;34m(self, N)\u001b[0m\n\u001b[1;32m    139\u001b[0m                 \u001b[0mctrl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbase\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCtrl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcurrent_trial\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m                     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdomain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    142\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m                     \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'job exception: %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/base.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, config, ctrl, attach_attachments)\u001b[0m\n\u001b[1;32m    842\u001b[0m                 \u001b[0mmemo\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmemo\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    843\u001b[0m                 print_node_on_error=self.rec_eval_print_node_on_error)\n\u001b[0;32m--> 844\u001b[0;31m             \u001b[0mrval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpyll_rval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    845\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    846\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumber\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/flair/hyperparameter/param_selection.py\u001b[0m in \u001b[0;36m_objective\u001b[0;34m(self, params)\u001b[0m\n\u001b[1;32m    110\u001b[0m                 \u001b[0mmax_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_epochs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    111\u001b[0m                 \u001b[0mparam_selection_mode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m                 \u001b[0;34m**\u001b[0m\u001b[0mtraining_params\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    113\u001b[0m             )\n\u001b[1;32m    114\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/flair/trainers/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, base_path, learning_rate, mini_batch_size, mini_batch_chunk_size, max_epochs, anneal_factor, patience, min_learning_rate, train_with_dev, monitor_train, monitor_test, embeddings_storage_mode, checkpoint, save_final_model, anneal_with_restarts, batch_growth_annealing, shuffle, param_selection_mode, num_workers, sampler, use_amp, amp_opt_level, eval_on_train_fraction, eval_on_train_shuffle, **kwargs)\u001b[0m\n\u001b[1;32m    316\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    317\u001b[0m                         \u001b[0;31m# forward pass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 318\u001b[0;31m                         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_step\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    319\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    320\u001b[0m                         \u001b[0;31m# Backward\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/flair/models/text_classification_model.py\u001b[0m in \u001b[0;36mforward_loss\u001b[0;34m(self, data_points)\u001b[0m\n\u001b[1;32m    115\u001b[0m     ) -> torch.tensor:\n\u001b[1;32m    116\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m         \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_points\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    118\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_calculate_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_points\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/flair/models/text_classification_model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, sentences)\u001b[0m\n\u001b[1;32m     80\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msentences\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 82\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdocument_embeddings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     84\u001b[0m         text_embedding_list = [\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/flair/embeddings.py\u001b[0m in \u001b[0;36membed\u001b[0;34m(self, sentences)\u001b[0m\n\u001b[1;32m     88\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0meverything_embedded\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatic_embeddings\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_add_embeddings_internal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0msentences\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/flair/embeddings.py\u001b[0m in \u001b[0;36m_add_embeddings_internal\u001b[0;34m(self, sentences)\u001b[0m\n\u001b[1;32m   2753\u001b[0m                 \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2754\u001b[0m                 \u001b[0mlongest_token_sequence_in_batch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2755\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membeddings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membedding_length\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2756\u001b[0m             ]\n\u001b[1;32m   2757\u001b[0m         )\n",
            "\u001b[0;31mRuntimeError\u001b[0m: shape '[16, 16, 4096]' is invalid for input of size 1687552"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a4d0AZlXGpB-",
        "colab_type": "text"
      },
      "source": [
        "## B. Finding the best Learning Rate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oo0yjiX8DrpS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flair.datasets import WNUT_17\n",
        "from flair.embeddings import TokenEmbeddings, WordEmbeddings\n",
        "from flair.embeddings import StackedEmbeddings\n",
        "from flair.trainers import ModelTrainer\n",
        "from typing import List"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZoDOuJWhTzkW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 349
        },
        "outputId": "2e3b3960-7f65-4f01-966e-80cc6fb90426"
      },
      "source": [
        "# 1. get the corpus\n",
        "corpus = WNUT_17().downsample(0.1)\n",
        "print(corpus)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 17:34:08,014 https://noisy-text.github.io/2017/files/wnut17train.conll not found in cache, downloading to /tmp/tmppzg5kg_1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 493781/493781 [00:00<00:00, 6039524.27B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 17:34:08,175 copying /tmp/tmppzg5kg_1 to cache at /root/.flair/datasets/wnut_17/wnut17train.conll\n",
            "2019-12-20 17:34:08,178 removing temp file /tmp/tmppzg5kg_1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 17:34:09,051 https://noisy-text.github.io/2017/files/emerging.dev.conll not found in cache, downloading to /tmp/tmp2xzqn4_4\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 114752/114752 [00:00<00:00, 3079877.48B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 17:34:09,159 copying /tmp/tmp2xzqn4_4 to cache at /root/.flair/datasets/wnut_17/emerging.dev.conll\n",
            "2019-12-20 17:34:09,161 removing temp file /tmp/tmp2xzqn4_4\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 17:34:10,108 https://noisy-text.github.io/2017/files/emerging.test.annotated not found in cache, downloading to /tmp/tmpnsv8vx_6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 192425/192425 [00:00<00:00, 4581021.49B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 17:34:10,222 copying /tmp/tmpnsv8vx_6 to cache at /root/.flair/datasets/wnut_17/emerging.test.annotated\n",
            "2019-12-20 17:34:10,224 removing temp file /tmp/tmpnsv8vx_6\n",
            "2019-12-20 17:34:10,227 Reading data from /root/.flair/datasets/wnut_17\n",
            "2019-12-20 17:34:10,228 Train: /root/.flair/datasets/wnut_17/wnut17train.conll\n",
            "2019-12-20 17:34:10,230 Dev: /root/.flair/datasets/wnut_17/emerging.dev.conll\n",
            "2019-12-20 17:34:10,232 Test: /root/.flair/datasets/wnut_17/emerging.test.annotated\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Corpus: 339 train + 101 dev + 129 test sentences\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HHU17pB2X365",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "outputId": "3bdc3918-778d-4f9e-e076-411a94e87233"
      },
      "source": [
        "# 2. what tag do we want to predict?\n",
        "tag_type = 'ner'\n",
        "\n",
        "# 3. make the tag dictionary from the corpus\n",
        "tag_dictionary = corpus.make_tag_dictionary(tag_type = tag_type)\n",
        "print(tag_dictionary.idx2item)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[b'<unk>', b'O', b'B-person', b'E-person', b'S-person', b'S-corporation', b'B-location', b'I-location', b'E-location', b'S-location', b'B-product', b'E-product', b'I-product', b'B-creative-work', b'E-creative-work', b'S-product', b'S-group', b'I-person', b'B-group', b'I-group', b'E-group', b'B-corporation', b'I-corporation', b'E-corporation', b'I-creative-work', b'S-creative-work', b'<START>', b'<STOP>']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BDxHN_ubYGam",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 184
        },
        "outputId": "20038908-a214-45ad-a63f-60b269929195"
      },
      "source": [
        "# 4. initialize embeddings\n",
        "embedding_types: List[TokenEmbeddings] = [\n",
        "                  WordEmbeddings('glove'),\n",
        "]\n",
        "\n",
        "embeddings: StackedEmbeddings = StackedEmbeddings(embeddings = embedding_types)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 17:39:02,522 https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings/glove.gensim.vectors.npy not found in cache, downloading to /tmp/tmp38kjnv8s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 160000128/160000128 [00:17<00:00, 9161027.43B/s] "
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 17:39:21,121 copying /tmp/tmp38kjnv8s to cache at /root/.flair/embeddings/glove.gensim.vectors.npy\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 17:39:21,459 removing temp file /tmp/tmp38kjnv8s\n",
            "2019-12-20 17:39:22,737 https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings/glove.gensim not found in cache, downloading to /tmp/tmp6sfvgona\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 21494764/21494764 [00:04<00:00, 4308676.92B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 17:39:28,875 copying /tmp/tmp6sfvgona to cache at /root/.flair/embeddings/glove.gensim\n",
            "2019-12-20 17:39:28,902 removing temp file /tmp/tmp6sfvgona\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IbVa7GjZY_5N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 5. initialize sequence tagger\n",
        "from flair.models import SequenceTagger\n",
        "\n",
        "tagger: SequenceTagger = SequenceTagger(hidden_size = 256,\n",
        "                                        embeddings = embeddings,\n",
        "                                        tag_dictionary = tag_dictionary,\n",
        "                                        tag_type = tag_type,\n",
        "                                        use_crf = True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zNcUNV-JZrM2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 6. initialize trainer\n",
        "\n",
        "trainer: ModelTrainer = ModelTrainer(tagger, corpus)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TtAj8VROZwp5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "833fd0b2-a883-4d74-cce4-81bca2bce3cd"
      },
      "source": [
        "# 7. find learning rate\n",
        "\n",
        "learning_rate_tsv = trainer.find_learning_rate('resources/taggers/example-ner',\n",
        "                                               'learning_rate.tsv')"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1.4454397707459274e-07]\n",
            "[1.7378008287493754e-07]\n",
            "[2.0892961308540395e-07]\n",
            "[2.51188643150958e-07]\n",
            "[3.019951720402016e-07]\n",
            "[3.6307805477010137e-07]\n",
            "[4.36515832240166e-07]\n",
            "[5.248074602497725e-07]\n",
            "[6.309573444801933e-07]\n",
            "[7.585775750291837e-07]\n",
            "[9.120108393559096e-07]\n",
            "[1.096478196143185e-06]\n",
            "[1.3182567385564074e-06]\n",
            "[1.5848931924611132e-06]\n",
            "[1.9054607179632473e-06]\n",
            "[2.2908676527677735e-06]\n",
            "[2.754228703338166e-06]\n",
            "[3.311311214825911e-06]\n",
            "[3.981071705534973e-06]\n",
            "[4.7863009232263826e-06]\n",
            "[5.754399373371569e-06]\n",
            "[6.918309709189365e-06]\n",
            "[8.317637711026708e-06]\n",
            "[9.999999999999999e-06]\n",
            "[1.202264434617413e-05]\n",
            "[1.4454397707459279e-05]\n",
            "[1.737800828749376e-05]\n",
            "[2.0892961308540385e-05]\n",
            "[2.5118864315095795e-05]\n",
            "[3.019951720402016e-05]\n",
            "[3.630780547701014e-05]\n",
            "[4.365158322401661e-05]\n",
            "[5.248074602497728e-05]\n",
            "[6.309573444801929e-05]\n",
            "[7.585775750291836e-05]\n",
            "[9.120108393559096e-05]\n",
            "[0.00010964781961431851]\n",
            "[0.00013182567385564074]\n",
            "[0.0001584893192461114]\n",
            "[0.00019054607179632462]\n",
            "[0.00022908676527677726]\n",
            "[0.0002754228703338166]\n",
            "[0.0003311311214825911]\n",
            "[0.0003981071705534973]\n",
            "[0.0004786300923226385]\n",
            "[0.0005754399373371565]\n",
            "[0.0006918309709189362]\n",
            "[0.0008317637711026709]\n",
            "[0.001]\n",
            "[0.001202264434617413]\n",
            "[0.001445439770745928]\n",
            "[0.001737800828749376]\n",
            "[0.0020892961308540407]\n",
            "[0.002511886431509582]\n",
            "[0.0030199517204020187]\n",
            "[0.00363078054770101]\n",
            "[0.004365158322401656]\n",
            "[0.005248074602497722]\n",
            "[0.006309573444801929]\n",
            "[0.007585775750291836]\n",
            "[0.009120108393559097]\n",
            "[0.01096478196143185]\n",
            "[0.013182567385564075]\n",
            "[0.01584893192461114]\n",
            "[0.019054607179632484]\n",
            "[0.022908676527677745]\n",
            "[0.027542287033381692]\n",
            "[0.03311311214825908]\n",
            "[0.03981071705534969]\n",
            "[0.0478630092322638]\n",
            "[0.05754399373371566]\n",
            "[0.06918309709189363]\n",
            "[0.08317637711026708]\n",
            "[0.09999999999999999]\n",
            "[0.12022644346174131]\n",
            "2019-12-20 17:43:30,957 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:43:30,958 loss diverged - stopping early!\n",
            "2019-12-20 17:43:30,966 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:43:30,967 learning rate finder finished - plot resources/taggers/example-ner/learning_rate.tsv\n",
            "2019-12-20 17:43:30,974 ----------------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zo2ydPk0Z_gY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "outputId": "dacb7618-7320-4781-e418-0bf23029df3f"
      },
      "source": [
        "# 8. plot the learning rate finder curve\n",
        "from flair.visual.training_curves import Plotter\n",
        "\n",
        "plotter = Plotter()\n",
        "plotter.plot_learning_rate(learning_rate_tsv)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Learning_rate plots are saved in resources/taggers/example-ner/learning_rate.png\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaoAAAEaCAYAAABARRODAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXhU5d3/8fc3CUkIkLAkYQth32SH\nsCiyCGpdcRe12lqt1LUuVftYu/3qY2trrUv7VKXuWvd916oFUUEJO7KEPQsBEsgG2Wfu3x8ZNGKA\nAJk5k8zndV1zkTlzzpzv3FeYT+5z7nMfc84hIiISrqK8LkBERORAFFQiIhLWFFQiIhLWFFQiIhLW\nFFQiIhLWFFQiIhLWYrwuoDGSk5Ndr169vC5DRESCZNGiRYXOuZSGXmsWQdWrVy8yMzO9LkNERILE\nzLbs7zUd+hMRkbCmoBIRkbCmoBIRkbCmoBIRkbCmoBIRkbCmoBIRkbCmoBIRkbCmoBKRsOPzO3aU\nVXpdhoSJZnHBr4hEDr/f8fPnlvDOinwm9uvEReN6csJRnYmN0d/VkUpBJSJh5b6PsnhnRT6nDuvK\n0pxirnl2Mclt4zgvI40Lx6aT3inhoO9R6/Pz/tfbeGr+FuJbRXPHGUPo2alNCKqXYLDmcCv6jIwM\npymURFq+N5bmcf3zSzk/I40/nzMcv4NPswr495fZfLJmOw4Y16sjx/RNZnyfjozs0Z74VtHfbF9S\nXsPzC7N58ovNbC2pJL1jAkXl1fj8jt+cdhQXjO2BmXn3AWW/zGyRcy6jwdcUVCISDhZnF3HB7AWM\n7NGeZy4f/71DffklFbywMIcPvt7Omm2lOAexMVGM7NGe8b07UlJRw8uLcimv9jGhT0cuP7YP0wal\nsr20kptfWsYXG3YyfVAqd50znJR2cR59StkfBZWIhLW84grO+MfnJMRG8/o1E+nYJvaA6xeXV7Nw\ncxFfbdrJl5t2sTKvhJioKE4f0Y3Lju3FkG5J31nf73c8/sVm/vz+GtrFxfCns4dx4pAuwfxIcogU\nVCIRqKCsinbxMd85NBaO9lTVcu5D88ndVc6rVx9D/87tDvk9yipr8DtIat3qgOtlbS/jhueXsiq/\nlCHdEumQEEu7+JjAoxWJ8a2YNCCZ0ekdDvfjyGFSUIlEmM2Fezj1gXmktIvj3pkjGdWIL94anx+f\n3xETZURHWYPncnx+R43PT63fEWWQEHtk47F8fsfPnl7EJ2u289ilY5k6MPWI3q8xqmv9PDR3Aws3\n76KsspayyprAv7VU1PgAuGh8Or88adBBg0+azoGCSqP+RFqYGp+f619YSkx0FDU+x7kPzefa4/px\n3bR+xER/f4h3QVkVD8/dwDNfbqGyxv/N8iiDmKgooqLA74cav5/6f9eawcyMHtx60qCDHqoDcM6R\nW1TBirwSluUWszynhJV5JZRV1fL7048KSUhB3Xmtn0/v3+Bru6tque8/WTz2+Sb+s2o7vz99CKcM\n63LEAzAqa3x8tq6QBRt38uNjetGj48FHLsq31KMSaWH+9uFaHvhkPf/84WiO7Z/M7974mteW5DGy\nR3vunTmS3sl1w7QLyqqY/ekGnl6whepaP2eM7E7/zm3x+Ry1fofP7/C5un+jzGgVbcRERRETXfdz\nblEFz36ZTdv4GH550iBmZvQgKur7X+hrtpXy7JfZvLsin8Ld1QC0ijYGdUlkeFoSE/slc/LQIw+D\nprQit4TbXlvOyrxSpg9K5Q9nDqV7+9ZA3fmuovJqdpRVUbi7itatoklpF0dKu7jv9DDLKmv4ZM0O\nPvx6O3PW7mBPdV1vbVyvjjw/a0KDbRXJdOhPJEIs3LyLmQ/P5+zRafz1vBHfLH97+VZuf20l1bV+\n/ufkQeQVV/DU/M1U1/o5c2R3rpve/5sAOxRZ28v49esr+WrTLkb0aM//njGUYWlJVNb4eGd5Ps9+\nlc2iLUXExkRx4lGdGd+7I8PT2jOoazviYsL73Fmtz88TX2zmng+zMIN+qW3ZUVoXTrX+hr8328bF\nkNIujsTWrVi9tZRqn5/ktnGcOKQzPxjShdyicm5/bSV3nDGES47uFdoPFOYUVCIRoLSyhlPun0eU\nGe9eP4m2cd89sr+tpG6Y9mfrC4kyOGNkd66b1o8+KW2PaL/OOV5fmsed76xh554qpg1MJXNLESUV\nNfRJbsNF49M5Z3QaHRpxeDAc5ewq554P11JUXkNqoOeU2i6O1MR4ktvGUVHjo6CsioKyKnaUVVJQ\nVsWuPdUc1TWRk4Z2YVR6B6IDvSfnHD967CsWbynigxsnk9ZBhwD3UlCJRICbXljKG8u28uLPjmZM\nz4YHT/j9jrnrCkjvmEDfIwyofZVW1vC3D7N4bUkek/onc9H4dI7u0ymsDumFg5xd5fzgvk/J6NWR\nJ38yVu0ToKASaeHeWraV655bwvXT+3PjCQO8LkcO4skvNvO7N7/mr+eN4NwxaV6XExY06k/EQ+XV\ndUOfOyfGH/Z7LNpSxN0frGHn7moGd01kUNd2DO6ayFFdE/H5Hbe/toJR6e25blq/JqxcguWSCT15\ne/lW7nh7FZMHJJPa7vB/NyJB0IPKzKKBTCDPOXeamU0H7qbuFiO7gUudc+uDXYeIF4r2VHPB7AXk\nl1Qw55bjGjWMu76cXeXc9f4a3lmeT2q7OIZ1T2LRliLeXLb1m3VaRRux0VHcN3Nkg8PPJfxERRl3\nnTOck++fx29f/5qHLhnjdUlhLRQ9quuB1UBi4PmDwBnOudVmdjXwa+DSENQhElJllTX8+PGv2LRz\nD7U+P3//ZB2/O31Io7Ytrazh/z5Zz+OfbyY6yrh+en9+NqXPN8OfSypqWJNfyur8UtZuL+P4wZ01\nO3gz0zelLTceP4A/v7+Gd1fkc8qwrl6XFLaCGlRmlgacCtwJ3BRY7Pg2tJKArQ1sKtKsVVT7uPzJ\nTFZtLeWhi8fw8ZrtPLNgC5ce0+uggfL28q389o2vKSqv5pzRadx84kC6JH330FBS61aM79OJ8X06\nBfNjSJBdMak3767I57dvrOToPp2a7cjIYAv2cYL7gFsBf71lPwXeNbNc4BLgriDXIBJS1bV+rnxm\nEQs37+JvM0dy/FGdufH4AcRERfGX99cecNuVeSXc+MJSenRM4K1rj+Wv5434XkhJyxETHcVfzh1O\nSUUNt7++guYwuM0LQQsqMzsN2OGcW7TPSzcCpzjn0oDHgb/tZ/tZZpZpZpkFBQXBKlPkkO2uqiWv\nuKLBL5Van5/rn1/C3KwC7jp7GDNGdAMgNTGeKyb34Z0V+SzJLtrv+1733BI6tYnjiUvHMrR7UoPr\nScsyuGsiN54wgHdXbOPVxXlelxOWgjY83cz+RF2PqRaIp+5w33+BQc65voF10oH3nXNHHei9NDxd\nvFBd62dxdhHrduxmw47drA88tpVWAtAuPoah3ZIY2j2Rod2TGNItiYfmbuDlRbn85rSjuPzY3t95\nvz1VtUy5ew69kxN48WdHf+/6mZteXMrrS/J49ooJTNAhvYji8zsunL2AVfmlvHf9pIicC9CT4enO\nuduA2wIFTAVuBs4EtpnZAOdcFnACdQMtRMLGlp17eO6rHF7KzGHnnrq56drERtM3tS3H9O1E39S2\nJLZuxZr8UlZuLeXJ+XVz5e114/EDvhdSAG3iYrjxhP7c/tpKPly1nR/Uux/Sa0tyeXVxHtdP76+Q\nikDRUcY954/g5Pvn8YsXl/HcrAnfzGYhIb6OyjlXa2ZXAK+YmR8oAi4LZQ0iDanx+flo1Xae/Sqb\neevqphiaPrgz545JY1j3JLomxe93BoEan5/1O3azMq+EhNgYThm2/xvyzczowWOfbeLP761h2qBU\nWkVHsalwD79+bSXjenXUdVARrEfHBH4/Ywg3v7SMf83byJVT+npdUtjQzBQS8b7atItrn13MjrIq\nuibFc8HYdGaO7RG0QQz/WbWdK57K5I4zhzIzowfnPPgF2bvKee/6SXQLzNAtkck5x9X/XsxHq7fz\n+jUTv3en4pZMM1OI7MfXW0u4/ImFpLSL449nDWPqwJSgXzR7/OBUxvXuyP0fZbE6v5QVeSXMvmSM\nQkowM/541jAWbSnixheW8ua1x4b9HZpDQZexS8TasnMPP35sIW3jY3j6p+M5/qjOIZnZwcy4/ZTB\nFO6u5tkvs/nR0T05ccj+DxdKZOnQJpa/nDucrO27D3o5Q6RQj0oi0o7SSi559Ct8fj/Pzzr6m5vi\nhcqIHu25eEI6a7eV8atTBod03xL+pg5M5UdH9+SxzzcxZ+0ORqa3Z1SP9ozs0YFBXdvRKsKmytI5\nKok4JRU1zHx4Ptm7ynn2igmM7NHes1qcc7rNgzSossbHU/M389WmIpbmFFO4uwqAuJgoxvXuyN8v\nHEX7hJYzk4XOUYkEVNb4uOLJTDYU7OaxS8d6GlKAQkr2K75VNLMm92XW5Lo/aPKKK1iaU8ziLcU8\n8cUm7v+48XNHNncKKokYlTU+rn12CQu37OLvF45iUv8Ur0sSaRQzI61DAmkdEjhteDcqamp5ev4W\nfnR0L3ont/zJiCPrQKdEJL/f8fKiXI776xw+Wr2dP8wYwmnDu3ldlshhu/GEAcTFRHHXe5ExX4J6\nVNKizVtXwB/fXcPq/FJGpCVx78yRmvlBmr3UdvFcOaUv9/wniy837mzxs+irRyXNwq491VTW+Bq9\n/pptpfz4sa+45NGvKKus4YELR/Ha1RMVUtJi/HRSH7okxnPnu6vx+8N/UNyRUI9Kwt6zX2bzmzdW\nEhNlTOjTiakDU5g6MPU7x+Z3V9WyYMNOPltfyGfrC1m/YzdJrVvx61MHc8nRPYmL0UWT0rK0jo3m\nlh8M5BcvLePNZVs5c1R3r0sKGg1Pl7Dl9zvu/nAtD87ZwOQBKfRNacPctQVsLNwDQM9OCUzo3YkN\nBbtZmlNMrd8R3yqKcb07MalfMudlpLWo4bsi+/L7Haf/4zOKy2v4+BdTmvUsFhqeLs1OZY2Pm19a\nxtvL87lofDp/mDGkbtaI0yF7ZzlzsnYwZ20B767Ip3dKG2ZN7sOx/ZIZ3bNDs/7PKnIooqKM208d\nzEX/+pLHPt/E1VNb5qTG6lFJ2CnaU80VT2WSuaWI/zl5ED+b3EfXG4kcwE+fXMiCjbuYc8tUktvG\neV3OYTlQj0qDKSSsbC7cw9kPfsHyvBL+cdEorpzSVyElchD/c/JgKmp83PdRltelBIWCSsLG4uwi\nzn7wC4rLq3n2p+N1rZNII/VLbcsPx6fz3Fc5fLlxp9flNDkFlYSFD7/exkX/WkC7+BhevXoiGb06\nel2SSLNy8w8G0rNjAtc8u5itxRVel9OkFFTiuafnb+bKZxYxsEsir1x1TERMCSPS1BLjWzH7R2Oo\nrPFz1TOLDum6w3CnoBLP+P2Ou95bw2/e+Jppg1J57orxzfZEsEg46Jfajr+dP4JluSX85vWVNIfB\nco2hoBJPVNX6uPHFpTw0dwM/HJ/OQxePISFWV0uIHKkTh3Th59P789KiXJ5esMXrcpqEvhkk5Jxz\nXPXMYj5Zs4NbTxrIVRrZJ9Kkbpjen6/zSvjDW6sY1CWRcb2b9zlf9agk5OatK+STNTv41SmDuHpq\nP4WUSBOLijLuvWAk6R0TuPrfi8gvad6DKxRUElLOOe77KIvu7Vtz6TG9vS5HpMVKjG/Fw5eMoaLa\nx5XPLMbXjCeuVVBJSM1bV8ji7GKuOa4fsTH69RMJpv6d2/HHs4exLKeY15fkeV3OYdM3hYRM/d7U\nuWPSvC5HJCLMGNGNYd2TuPejLKpr/V6Xc1gUVNIk9lTVHvSeOHt7U1cf11e9KZEQMTNu/sFAcosq\neGFhttflHBaN+pNDsrW4gs/WF5K9s5zsXXWPnF3l7NxTzdDuiTxz+fgGb62xtzfVLSme88b08KBy\nkcg1uX8y43p15IFP1nPumB60jm1edxgI+p+1ZhZtZkvM7O3A83lmtjTw2Gpmrwe7Bmka8zfs5OT7\n53Hry8t5cO4GluQU0TYuhhOHdOHa4/qRtX03P3rsK0ora7637Tfnpqbp3JRIqO3tVRWUVfHk/M1e\nl3PIQtGjuh5YDSQCOOcm7X3BzF4B3ghBDXKEXszM4VevrqBXchueu2IC/Tu3pVX0dwNnVHp7fvb0\nIi57fCFPXjaONnF1v17qTYl4b1zvjkwdmMJDczdw0fh0EuNbeV1SowX1T1szSwNOBR5p4LVEYBqg\nHlUY8/sdf3l/Dbe+vJwJfTrxylXHcFS3xO+FFMD0wZ154MJRLM4u4oqnMr+Za0y9KZHwcPOJAyku\nr+GReZu8LuWQBPtb4z7gVqChoSZnAh8750qDXIMcpsoaH9c+t5h/ztnAhePSefwnY0lqfeC/wk4Z\n1pV7zh/B/I07ufKZRVTV+tSbEgkTQ7snccqwLjw6byM7d1d5XU6jBS2ozOw0YIdzbtF+VrkQeO4A\n288ys0wzyywoKAhKjbJ/BWVVzJy9gPdWbuP2Uwbzx7OGNtiLashZo9L441nDmLO2gHMe/EK9KZEw\nctMJA6io8fHgnA1el9JowfzmmAjMMLPNwPPANDN7BsDMkoFxwDv729g5N9s5l+Gcy0hJSQlimbKv\nGp+fnz65kKxtZTx08RiuOIxbwV84Lp3fn34UK/NK1ZsSCSP9Uttx9ug0nlqwpdlMrRS0oHLO3eac\nS3PO9QIuAD5xzl0cePlc4G3nXGWw9i+H7/6P1rEst4R7zh/BD4Z0Oez3uXRibx784Wj+74ej1ZsS\nCSPXT++Pc44HPl7vdSmN4tW3xwUc4LCfeGfh5l38c856zh2TxinDuh7x+508rCuj0js0QWUi0lR6\ndEzgwnHpvJiZw+bCPV6Xc1AhCSrn3Bzn3Gn1nk91zr0fin1L45VW1nDD80tJ65DA72cM8bocEQmi\na4/rR6to476Psrwu5aB0PEa+8fs3vmZbaSX3zhxJ2zhNWiLSkqUmxnPpMb15Y9lW1mwL78HXCioB\n4K1lW3l1SR7XHtePMT11qE4kElw5pQ9tY2O458Pw7lUpqIStxRXc/toKRqW357pp/bwuR0RCpH1C\nLLMm9+E/q7azOLvI63L2S0EV4Xx+x00vLsXnd9w3cyQxjbxWSkRahsuO7U2nNrH89YO1XpeyX/pW\ninBPzd/Mgo27+N2MIfTs1MbrckQkxNrExXDNcf34YsNOPl9f6HU5DVJQRbDqWj8Pzd3A0X06cZ5u\nZCgSsS4an063pHj+8sFanAu/W9YrqCLYG0vz2F5axZVT+x7yzBMi0nLEt4rm+uP7syynmP+s2u51\nOd+joIpQzjn+NW8jg7q0Y3L/ZK/LERGPnTM6jd7Jbbjnwyx8B7lbd6gpqCLUnKwCsrbvZtZhzOMn\nIi1PTHQUN50wgLXby3hr2Vavy/kOBVWEmj13I10S4zlteDevSxGRMHHqsK4M7prIvR+FV69KQRWB\nVuSWMH/jTi47tpcmixWRb0RFGddP78eWneW8v3Kb1+V8Q99SEWj2vI20i4vhwnHpXpciImHmhKO6\n0Du5DQ/N3RA2IwAVVBEmZ1c5767I56Lx6bSLP/DdekUk8kRHGVdM6sOKvLojL+FAQRVhHv1sEwZc\nOrGX16WISJg6e3R3ktvG8vDcjV6XAiioIkpxeTUvLMxhxshudE1q7XU5IhKm4ltF85OJvZmbVcDq\nfO9nVldQRZB/f5lNRY2PWZP7eF2KiIS5i8f3JCE2mtmfet+rUlBFiMoaH49/vpkpA1IY1CXR63JE\nJMwlJbTigrHpvLlsK7lF5Z7WoqCKEC8szKFwd5V6UyLSaJdP6g3AY59t9rQOBVUEWL+jjD+9t5qJ\n/TpxTN9OXpcjIs1E9/atmTGiG88vzKakvMazOhRULVxljY9rn11CQmwMfzt/pKZLEpFDMmtyH8qr\nfTzz5RbPalBQtXB/enc1a7aVcc95I+icGO91OSLSzAzumsiUASk8/vkmKmt8ntSgoGrBPvx6G0/O\n38JlE3tz3KBUr8sRkWbqZ1P6ULi7mlcX53myfwVVC5VfUsGtryxnSLdEfnnyQK/LEZFm7Og+nRie\nlsRjn2/yZFolBVUL5PM7bnh+KdW1fv5+4SjiYqK9LklEmjEz48Jx6azfsZsVeSUh37+CqgX6v/+u\n58tNu/jDGUPpk9LW63JEpAU4ZWhXYqOjPDn8p6BqYRZnF3HfR1mcObIb54zu7nU5ItJCJCW0Yvrg\nVN5atpUanz+k+1ZQtSDOOe54exUp7eK448yhGoouIk3qrFHd2bmnms/WFYZ0v0EPKjOLNrMlZvZ2\n4LmZ2Z1mlmVmq83s58GuIVJ8uGo7S7KLueH4AbqFh4g0uakDU2mf0IpXl4T28F9MCPZxPbAa2DvB\n3KVAD2CQc85vZho33QRqfX7u/mAtfVLacN6YNK/LEZEWKDYmitOGd+WlzFzKKmtC9gdxUHtUZpYG\nnAo8Um/xVcAfnHN+AOfcjmDWECleXZzH+h27ueXEgcRE64iuiATHWaPSqKr1h/RW9cH+RrsPuBWo\nf+atLzDTzDLN7D0z69/QhmY2K7BOZkFBQZDLbN4qa3zc+1EWI3q056ShXbwuR0RasNHp7enVKYHX\nQnj4L2hBZWanATucc4v2eSkOqHTOZQD/Ah5raHvn3GznXIZzLiMlJSVYZbYIT83fTH5JJb88aaAG\nUIhIUJkZZ47qzvyNO8kvqQjJPoPZo5oIzDCzzcDzwDQzewbIBV4NrPMaMDyINbR4JRU1/N9/NzB5\nQArH9E32uhwRiQBnjuyOc/D6kq0h2V/Qgso5d5tzLs051wu4APjEOXcx8DpwXGC1KUBWsGqIBA/P\n3UBJRQ2/PEnTJIlIaPRKbsPo9Pa8tiQ3JFMqeXHW/S7gHDNbAfwJ+KkHNbQI20sreezzTZwxshtD\nuiV5XY6IRJCzRqeRtX03q/JLg76vkASVc26Oc+60wM/FzrlTnXPDnHNHO+eWhaKGluj+j9fh8zt+\ncYJ6UyISWqcN60qraOO1EEyppHHMzdSmwj28sDCHi8alk94pwetyRCTCdGgTy9SBqbyxbCs+f3AP\n/ymomqmn5m8myuCaaf28LkVEItTZo7pTUFbF5+uDO6WSgqoZqqr18dqSPE4c0oXUdrprr4h4Y9rg\nVBLjY4J+TVUoplCSJvbh19spLq/hgrE9vC5FRCJYXEw0lxzdM+j7UVA1Qy8szKF7+9ZM1HVTIuKx\nW34wKOj70KG/ZiZnVzmfrS/k/IweREVpFgoRafkaFVRm1tfM4gI/TzWzn5tZ++CWJg15MTMHMzgv\nQzOki0hkaGyP6hXAZ2b9gNnU3abj2aBVJQ3y+R0vZeYyZUAK3dq39rocEZGQaGxQ+Z1ztcBZwN+d\nc7cAXYNXljTk06wCtpVWMjNDgyhEJHI0NqhqzOxC4MfA24FluoVsiD2/MJtObWKZPriz16WIiIRM\nY4PqJ8DRwJ3OuU1m1ht4Onhlyb4Kyqr4ePUOzhmTRmyMxsCISORo1PB059wq4OcAZtYBaOec+3Mw\nC5PvenVxLrV+x/k67CciEaaxo/7mmFmimXUEFgP/MrO/Bbc02cs5xwsLc8jo2YF+qW29LkdEJKQa\newwpyTlXCpwNPOWcGw8cH7yypL6Fm4vYWLiHmZqJQkQiUGODKsbMugLn8+1gCgmR5xdm0zYuhlOH\na6CliESexgbVH4APgA3OuYVm1gdYF7yyZK/SyhreXZHPjJHdSIjVjFciEnkaO5jiJeCles83AucE\nqyj51kuZuVTW+DUBrYhErMYOpkgzs9fMbEfg8YqZaQ6fIKvx+Xl03kbG9e7I8DTNWCUikamxh/4e\nB94EugUebwWWSRC9vXwrW0squXJKH69LERHxTGODKsU597hzrjbweAJICWJdEc85x8NzNzKgc1um\nDkj1uhwREc80Nqh2mtnFZhYdeFwM7AxmYZFublYBa7aVMWtyX93OQ0QiWmOD6jLqhqZvA/KBc4FL\ng1STAA/P3UiXxHhmjOjmdSkiIp5qVFA557Y452Y451Kcc6nOuTPRqL+gWZZTzPyNO7n82N6a109E\nIt6RfAve1GRVyHfM/nQj7eJjuGCchqSLiBxJUOnESRBsLtzDeyvzuXhCT9rF604qIiJHElSuyaqQ\nbzzy2UZioqL4yTG9vC5FRCQsHHBmCjMro+FAMqBR90I3s2ggE8hzzp1mZk8AU4CSwCqXOueWNrri\nFqxwdxUvZeZy9ujupCbGe12OiEhYOGBQOefaNcE+rgdWA4n1lt3inHu5Cd67RXnqi81U+/xcMVkX\n+IqI7BXUIWWBaZZOBR4J5n5agj1VtTy1YAsnDO5M3xTdc0pEZK9gj32+D7gV8O+z/E4zW25m95pZ\nXEMbmtksM8s0s8yCgoIgl+m9Bz5ZR3F5DVdN7et1KSIiYSVoQWVmpwE7nHOL9nnpNmAQMBboCPyy\noe2dc7OdcxnOuYyUlJY9W9OabaU8Om8T52ekMSq9g9fliIiElWD2qCYCM8xsM/A8MM3MnnHO5bs6\nVdRNbDsuiDWEPb/f8atXV5DYuhW3nTzY63JERMJO0ILKOXebcy7NOdcLuAD4xDl3ceBOwZiZAWcC\nK4NVQ3Pw/MIcFmcX86tTBtOhTazX5YiIhB0vbhn7bzNLoW6I+1LgSg9qCAsFZVXc9d5qJvTpyDmj\nu3tdjohIWApJUDnn5gBzAj9PC8U+m4M731lFRY2P/z1zGHUdTBER2ZdmPPXIZ+sKeX3pVq6a0pd+\nqRqOLiKyPwoqD1TW+Pj16yvo1SmBq4/r53U5IiJhzYtzVBHvn/9dz+ad5Txz+XjiW0V7XY6ISFhT\njyrEluUU89DcjZw5shvH9k/2uhwRkbCnoAqhbSWVXPFUJqmJcfz29CFelyMi0iwoqEKkotrHFU9l\nsqeqlkd/PJaOumZKRKRRdI4qBPx+x80vLWPl1hIe+VEGA7s0xaT0IiKRQT2qELj/43W8syKf204e\nxPTBnb0uR0SkWVFQBdlby7Zy/8frOG9MGldM0n2mREQOlYIqiJblFHPzS8sY26sD/3vWUM0+ISJy\nGBRUQVJaWcOspzNJaRfHQxePIS5G10uJiBwODaYIkkfmbWJ7aRVvXDORTm0bvDekiIg0gnpUQbBr\nTzWPztvIyUO7MKJHe6/LEYDprRAAAA71SURBVBFp1hRUQfDQ3A2U1/i46YQBXpciItLsKaia2PbS\nSp78YjNnjexO/866XkpE5EgpqJrYPz5Zj8/vuOF49aZERJqCgqoJ5ewq5/mF2cwc24P0TglelyMi\n0iIoqJrQ/R+vI8qM66b197oUEZEWQ0HVRNbv2M2ri3O5ZEJPuiTFe12OiEiLoaBqIvd+lEXrVtFc\nNbWv16WIiLQoCqom8PXWEt5Zns9lx/bWxb0iIk1MQdUE/vZhFonxMfxUk86KiDQ5BdUR2lZSycdr\ndnDZsb1Jat3K63JERFocBdUR+jSrAICThnbxuBIRkZZJQXWE5mTtoEtiPAM1C4WISFAoqI5Arc/P\nvHWFTBmQontNiYgESdCDysyizWyJmb29z/IHzGx3sPcfTEtyiimrrGXqwBSvSxERabFC0aO6Hlhd\nf4GZZQAdQrDvoJq7toDoKOOYfslelyIi0mIFNajMLA04FXik3rJo4G7g1mDuOxTmZO1gTHoHjfYT\nEQmiYPeo7qMukPz1ll0LvOmcyw/yvoOqoKyKlXmlTNFhPxGRoApaUJnZacAO59yiesu6AecBf2/E\n9rPMLNPMMgsKCoJV5mHbOyx9ygAFlYhIMMUE8b0nAjPM7BQgHkgEvgaqgPWBUXIJZrbeOddv342d\nc7OB2QAZGRkuiHUelrlZBSS3jeOorolelyIi0qIFrUflnLvNOZfmnOsFXAB84pzr4Jzr4pzrFVhe\n3lBIhTuf3/HpugKmDEghKkrD0kVEgknXUR2GZbnFFJfX6PyUiEgIBPPQ3zecc3OAOQ0sbxuK/Te1\nuWsLiDKYpGHpIiJBpx7VYZiTVcCIHu3p0CbW61JERFo8BdUh2rWnmuW5xUwdkOp1KSIiEUFBdYjm\nrSvAOXR+SkQkRBRUh2ju2gI6tollePckr0sREYkICqpD4Pc75mYVMKl/soali4iEiILqEHy9tZSd\ne6o1W7qISAgpqA7BnLU7AJjUX0ElIhIqCqpDMDergOFpSSS3jfO6FBGRiKGgaqSKah9Lc4o5Vhf5\nioiElIKqkZbnFlPrd4zp2ezv9ygi0qwoqBppcXYxAKPSFVQiIqGkoGqkxdlF9E5uQ0dNmyQiElIK\nqkZwzrEku4hR6e29LkVEJOIoqBoht6iCwt3VjNZhPxGRkFNQNcLi7CIA9ahERDygoGqExVuKSIiN\nZmDndl6XIiIScRRUjbA4u5gRae2JiVZziYiEmr55D6Ki2sfq/FJG99RhPxERLyioDmJFXgm1fqeB\nFCIiHlFQHcS3AykUVCIiXlBQHcTiLUX06pSgC31FRDyioDoA5xyLs4t12E9ExEMKqgOou9C3ilGa\niFZExDMKqgPYe35qtC70FRHxjILqAJZkF+tCXxERjymoDmBxdhHD05J0oa+IiIeC/g1sZtFmtsTM\n3g48f9TMlpnZcjN72czaBruGw1FZ42PV1lINpBAR8VgougrXA6vrPb/ROTfCOTccyAauDUENh2x5\nri70FREJB0ENKjNLA04FHtm7zDlXGnjNgNaAC2YNh0szpouIhIdg96juA24F/PUXmtnjwDZgEPD3\nINdwWPZe6NupbZzXpYiIRLSgBZWZnQbscM4t2vc159xPgG7UHRKcuZ/tZ5lZppllFhQUBKXGnz+3\nhJkPz2dz4Z5962NJji70FREJB8HsUU0EZpjZZuB5YJqZPbP3ReecL7D8nIY2ds7Nds5lOOcyUlJS\nmry40soa3lmRz5ebdnHKA/N4cWEOztUdhcwtqqCgrEqH/UREwkDQgso5d5tzLs051wu4APgEuMTM\n+sE356hmAGuCVcOBfLG+EJ/fcf8FIxmR1p5bX1nOVc8sZteeak1EKyISRmJCvD8DnjSzxMDPy4Cr\nQlwDAHOzCmkbF8Mpw7py+vBuPPLZRu7+YC0n3VdEn5Q2JMRGM6iLLvQVEfFaSILKOTcHmBN4OjEU\n+zwQ5xyfZhVwTN9OtApczDtrcl8m9kvmhueXsmDjLib06agLfUVEwkCoe1RhYUPBHvKKK7hqat/v\nLB/SLYm3rjuWf326kdGaiFZEJCxEZFB9mlU3inDKgO8P0ohvFc110/uHuiQREdmPiDy29em6Avok\nt6FHxwSvSxERkYOIuKCqrPGxYONOJjfQmxIRkfATcUG1cPMuKmv8TB6Q7HUpIiLSCBEXVJ9mFRAb\nHcWEPp28LkVERBohAoOqkLG9O5AQG5HjSEREmp2ICqr8kgrWbi9jcn+dnxIRaS4iKqjmZRUCaCCF\niEgzElFBNXddAant4jQ1kohIMxIxQeXzOz5bV8jkASnUzYcrIiLNQcQE1bLcYkoqanTYT0SkmYmY\noPo0qwAzmNRP10+JiDQnERVUw9Pa06FNrNeliIjIIYiIoCopr2FpTjFT+qs3JSLS3EREUH22vhC/\ngykDdX5KRKS5iYig+jSrgHbxMYxIa+91KSIicogiIqhax0Zz6rCuumOviEgzFBET3v1+xhCvSxAR\nkcOkLoaIiIQ1BZWIiIQ1BZWIiIQ1BZWIiIQ1BZWIiIQ1BZWIiIQ1BZWIiIQ1BZWIiIQ1BZWIiIQ1\nc855XcNBmVkBsKXeoiSgZJ/V9l12oOfJQGETl7m/uppqmwOtt7/X1E6Ney1c22l/tTXFNmqn8G0n\naH7/95qinXo65xqeOdw51+wewOyDLTvQcyAzVHU11TYHWm9/r6mdmnc7BbOt1E7h207BbKtwb6f9\nPZrrob+3GrHsYM+D4XD20dhtDrTe/l5TOzXutXBtp8PdT2O2UTupnZpym6ZqpwY1i0N/Tc3MMp1z\nGV7XEe7UTo2jdmoctVPjqa2+q7n2qI7UbK8LaCbUTo2jdmoctVPjqa3qicgelYiINB+R2qMSEZFm\notkHlZk9ZmY7zGzlYWw7xsxWmNl6M3vAzKzea9eZ2Roz+9rM/tK0VYdeMNrJzH5vZnlmtjTwOKXp\nKw+tYP0+BV7/hZk5M0tuuoq9EaTfpzvMbHngd+lDM+vW9JWHVpDa6e7Ad9NyM3vNzNo3feXhpdkH\nFfAEcNJhbvsgcAXQP/A4CcDMjgPOAEY454YAfz3yMj33BE3cTgH3OudGBh7vHlmJYeEJgtBOZtYD\nOBHIPsL6wsUTNH073e2cG+6cGwm8Dfz2SIsMA0/Q9O30H2Coc244kAXcdoQ1hr1mH1TOuU+BXfWX\nmVlfM3vfzBaZ2TwzG7TvdmbWFUh0zi1wdSfqngLODLx8FXCXc64qsI8dwf0UwRekdmpxgthO9wK3\nAi3ipHAw2sk5V1pv1Ta0gLYKUjt96JyrDay6AEgL7qfwXrMPqv2YDVznnBsD3Az8s4F1ugO59Z7n\nBpYBDAAmmdmXZjbXzMYGtVrvHGk7AVwbOATxmJl1CF6pnjqidjKzM4A859yyYBfqsSP+fTKzO80s\nB/ghLaNH1ZCm+H+312XAe01eYZiJ8bqApmZmbYFjgJfqnSKIO8S3iQE6AhOAscCLZtbHtaAhkk3U\nTg8Cd1D3l+8dwD3U/cdpMY60ncwsAfgVdYf9Wqwm+n3COXc7cLuZ3QZcC/yuyYoMA03VToH3uh2o\nBf7dNNWFrxYXVNT1EosDx7m/YWbRwKLA0zep+5Kt32VOA/ICP+cCrwaC6Ssz81M391ZBMAsPsSNu\nJ+fc9nrb/Yu68wotzZG2U1+gN7As8MWUBiw2s3HOuW1Brj2UmuL/XX3/Bt6lhQUVTdROZnYpcBow\nvSX9Ab0/Le7QX+A49yYzOw/A6oxwzvnqnfT/rXMuHyg1swmB0TQ/At4IvM3rwHGB7QcAsQRvMk1P\nNEU7BY6j73UWcMgjm8LdkbaTc26Fcy7VOdfLOdeLuj+CRrewkGqq36f+9d7yDGBNqD9HsDVRO51E\n3fnOGc65cq8+S0g1dlLAcH0AzwH5QA11XwKXU/cX7PvAMmAV8Nv9bJtB3ZfrBuAffHsBdCzwTOC1\nxcA0rz9nmLbT08AKYDl1fwV29fpzhmM77bPOZiDZ688Zju0EvBJYvpy6eeC6e/05w7Sd1gM5wNLA\n4yGvP2ewH5qZQkREwlqLO/QnIiIti4JKRETCmoJKRETCmoJKRETCmoJKRETCmoJKpB4z2x3i/T1i\nZkc10Xv5rG7m8ZVm9tbBZtU2s/ZmdnVT7FskmDQ8XaQeM9vtnGvbhO8X476dQDSo6tduZk8CWc65\nOw+wfi/gbefc0FDUJ3K41KMSOQgzSzGzV8xsYeAxMbB8nJnNN7MlZvaFmQ0MLL/UzN40s0+Aj81s\nqpnNMbOXre4+Qv8OzDZAYHlG4OfdgUlZl5nZAjPrHFjeN/B8hZn9byN7ffP5dlLctmb2sZktDrzH\nGYF17gL6BnphdwfWvSXwGZeb2f9rwmYUOWwKKpGDu5+6+26NBc4BHgksXwNMcs6Nom6m7z/W22Y0\ncK5zbkrg+SjgBuAooA8wsYH9tAEWOOdGAJ9Sdy+ivfu/3zk3jO/OqN2gwLxx06mbLQSgEjjLOTea\nuqnB7gkE5f8AG1zdtD23mNmJ1N33aBwwEhhjZpMPtj+RYGuJk9KKNLXjgaPqzXadGJgFOwl4MjBH\nnQNa1dvmP865+vch+so5lwtgZkuBXsBn++ynmm8n9l0EnBD4+Wi+vbfVs+z/Rp6tA+/dHVhN3Q32\nAAz4YyB0/IHXOzew/YmBx5LA87bUBden+9mfSEgoqEQOLgqY4JyrrL/QzP4B/Nc5d1bgfM+cei/v\n2ec9qur97KPh/3s17tuTxvtb50AqnHMjre7WIh8A1wAPUHdvpxRgjHOuxsw2A/ENbG/An5xzDx/i\nfkWCSof+RA7uQ+C6vU/MbO8tGpL49tYLlwZx/wuoO+QIcMHBVnZ1M2r/HPiFmcVQV+eOQEgdB/QM\nrFoGtKu36QfAZYHeImbW3cxSm+gziBw2BZXIdyWYWW69x03UfelnBAYYrAKuDKz7F+BPZraE4B6d\nuAG4ycyWA/2AkoNt4JxbQt0s5BdSd2+nDDNbQd3tItYE1tkJfB4Yzn63c+5D6g4tzg+s+zLfDTIR\nT2h4ukiYCxzKq3DOOTO7ALjQOXfGwbYTaSl0jkok/I0B/hEYqVcMXOZxPSIhpR6ViIiENZ2jEhGR\nsKagEhGRsKagEhGRsKagEhGRsKagEhGRsKagEhGRsPb/AcsjRVx7XVpfAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xZTkqTfba-tc",
        "colab_type": "text"
      },
      "source": [
        "## C. Custom Optimizers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KsUwlDLjaMQN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "37de88ff-64f1-414b-c2ee-e3b90f46934a"
      },
      "source": [
        "from torch.optim.adam import Adam\n",
        "\n",
        "trainer = ModelTrainer(tagger, corpus,\n",
        "                       optimizer = Adam)\n",
        "\n",
        "trainer.train(\n",
        "    \"resources/taggers/example\",\n",
        "    weight_decay = 1e-4\n",
        ")"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 17:52:43,513 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:52:43,520 Model: \"SequenceTagger(\n",
            "  (embeddings): StackedEmbeddings(\n",
            "    (list_embedding_0): WordEmbeddings('glove')\n",
            "  )\n",
            "  (word_dropout): WordDropout(p=0.05)\n",
            "  (locked_dropout): LockedDropout(p=0.5)\n",
            "  (embedding2nn): Linear(in_features=100, out_features=100, bias=True)\n",
            "  (rnn): LSTM(100, 256, batch_first=True, bidirectional=True)\n",
            "  (linear): Linear(in_features=512, out_features=28, bias=True)\n",
            ")\"\n",
            "2019-12-20 17:52:43,525 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:52:43,530 Corpus: \"Corpus: 339 train + 101 dev + 129 test sentences\"\n",
            "2019-12-20 17:52:43,531 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:52:43,533 Parameters:\n",
            "2019-12-20 17:52:43,537  - learning_rate: \"0.1\"\n",
            "2019-12-20 17:52:43,540  - mini_batch_size: \"32\"\n",
            "2019-12-20 17:52:43,543  - patience: \"3\"\n",
            "2019-12-20 17:52:43,545  - anneal_factor: \"0.5\"\n",
            "2019-12-20 17:52:43,547  - max_epochs: \"100\"\n",
            "2019-12-20 17:52:43,550  - shuffle: \"True\"\n",
            "2019-12-20 17:52:43,552  - train_with_dev: \"False\"\n",
            "2019-12-20 17:52:43,555  - batch_growth_annealing: \"False\"\n",
            "2019-12-20 17:52:43,558 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:52:43,562 Model training base path: \"resources/taggers/example\"\n",
            "2019-12-20 17:52:43,564 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:52:43,567 Device: cuda:0\n",
            "2019-12-20 17:52:43,570 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:52:43,573 Embeddings storage mode: cpu\n",
            "2019-12-20 17:52:43,577 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:52:43,739 epoch 1 - iter 0/11 - loss 9.15697670 - samples/sec: 203.04\n",
            "2019-12-20 17:52:43,910 epoch 1 - iter 1/11 - loss 35.83258724 - samples/sec: 209.78\n",
            "2019-12-20 17:52:44,069 epoch 1 - iter 2/11 - loss 44.25619825 - samples/sec: 221.69\n",
            "2019-12-20 17:52:44,226 epoch 1 - iter 3/11 - loss 203.57136011 - samples/sec: 225.95\n",
            "2019-12-20 17:52:44,388 epoch 1 - iter 4/11 - loss 359.71272774 - samples/sec: 216.50\n",
            "2019-12-20 17:52:44,561 epoch 1 - iter 5/11 - loss 427.80007585 - samples/sec: 202.18\n",
            "2019-12-20 17:52:44,721 epoch 1 - iter 6/11 - loss 420.61312566 - samples/sec: 219.10\n",
            "2019-12-20 17:52:44,893 epoch 1 - iter 7/11 - loss 374.58940101 - samples/sec: 204.58\n",
            "2019-12-20 17:52:45,070 epoch 1 - iter 8/11 - loss 337.89518589 - samples/sec: 196.05\n",
            "2019-12-20 17:52:45,241 epoch 1 - iter 9/11 - loss 310.75893650 - samples/sec: 205.21\n",
            "2019-12-20 17:52:45,375 epoch 1 - iter 10/11 - loss 291.87942592 - samples/sec: 269.41\n",
            "2019-12-20 17:52:45,392 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:52:45,393 EPOCH 1 done: loss 291.8794 - lr 0.1000\n",
            "2019-12-20 17:52:45,717 DEV : loss 83.97923278808594 - score 0.0\n",
            "2019-12-20 17:52:45,723 BAD EPOCHS (no improvement): 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type StackedEmbeddings. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:292: UserWarning: Couldn't retrieve source code for container of type WordEmbeddings. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 17:52:49,113 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:52:49,265 epoch 2 - iter 0/11 - loss 71.28720856 - samples/sec: 214.97\n",
            "2019-12-20 17:52:49,420 epoch 2 - iter 1/11 - loss 57.91867828 - samples/sec: 229.32\n",
            "2019-12-20 17:52:49,594 epoch 2 - iter 2/11 - loss 60.36667124 - samples/sec: 201.90\n",
            "2019-12-20 17:52:49,744 epoch 2 - iter 3/11 - loss 73.01189232 - samples/sec: 236.88\n",
            "2019-12-20 17:52:49,890 epoch 2 - iter 4/11 - loss 72.90306854 - samples/sec: 246.21\n",
            "2019-12-20 17:52:50,041 epoch 2 - iter 5/11 - loss 70.16939926 - samples/sec: 235.90\n",
            "2019-12-20 17:52:50,199 epoch 2 - iter 6/11 - loss 65.45948900 - samples/sec: 222.96\n",
            "2019-12-20 17:52:50,364 epoch 2 - iter 7/11 - loss 63.99839211 - samples/sec: 214.12\n",
            "2019-12-20 17:52:50,521 epoch 2 - iter 8/11 - loss 60.38916058 - samples/sec: 225.03\n",
            "2019-12-20 17:52:50,686 epoch 2 - iter 9/11 - loss 57.70584526 - samples/sec: 212.56\n",
            "2019-12-20 17:52:50,823 epoch 2 - iter 10/11 - loss 56.98174043 - samples/sec: 260.58\n",
            "2019-12-20 17:52:50,838 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:52:50,839 EPOCH 2 done: loss 56.9817 - lr 0.1000\n",
            "2019-12-20 17:52:51,047 DEV : loss 26.494556427001953 - score 0.0\n",
            "2019-12-20 17:52:51,054 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 17:52:54,293 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:52:54,454 epoch 3 - iter 0/11 - loss 25.64236069 - samples/sec: 204.91\n",
            "2019-12-20 17:52:54,609 epoch 3 - iter 1/11 - loss 35.51955986 - samples/sec: 230.73\n",
            "2019-12-20 17:52:54,771 epoch 3 - iter 2/11 - loss 35.25133133 - samples/sec: 217.91\n",
            "2019-12-20 17:52:54,932 epoch 3 - iter 3/11 - loss 33.53539705 - samples/sec: 220.98\n",
            "2019-12-20 17:52:55,089 epoch 3 - iter 4/11 - loss 29.70931568 - samples/sec: 231.29\n",
            "2019-12-20 17:52:55,236 epoch 3 - iter 5/11 - loss 27.17293517 - samples/sec: 243.90\n",
            "2019-12-20 17:52:55,384 epoch 3 - iter 6/11 - loss 28.18995040 - samples/sec: 244.89\n",
            "2019-12-20 17:52:55,538 epoch 3 - iter 7/11 - loss 26.69810653 - samples/sec: 232.11\n",
            "2019-12-20 17:52:55,688 epoch 3 - iter 8/11 - loss 26.79720624 - samples/sec: 242.25\n",
            "2019-12-20 17:52:55,852 epoch 3 - iter 9/11 - loss 25.49636822 - samples/sec: 214.81\n",
            "2019-12-20 17:52:55,975 epoch 3 - iter 10/11 - loss 25.55616413 - samples/sec: 296.19\n",
            "2019-12-20 17:52:55,991 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:52:55,993 EPOCH 3 done: loss 25.5562 - lr 0.1000\n",
            "2019-12-20 17:52:56,209 DEV : loss 16.160356521606445 - score 0.0222\n",
            "2019-12-20 17:52:56,215 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 17:52:59,409 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:52:59,575 epoch 4 - iter 0/11 - loss 19.98316956 - samples/sec: 198.86\n",
            "2019-12-20 17:52:59,740 epoch 4 - iter 1/11 - loss 19.88276672 - samples/sec: 215.90\n",
            "2019-12-20 17:52:59,910 epoch 4 - iter 2/11 - loss 19.47287750 - samples/sec: 206.51\n",
            "2019-12-20 17:53:00,060 epoch 4 - iter 3/11 - loss 18.79427719 - samples/sec: 237.29\n",
            "2019-12-20 17:53:00,210 epoch 4 - iter 4/11 - loss 20.07604713 - samples/sec: 236.78\n",
            "2019-12-20 17:53:00,366 epoch 4 - iter 5/11 - loss 18.82204946 - samples/sec: 227.93\n",
            "2019-12-20 17:53:00,521 epoch 4 - iter 6/11 - loss 19.92437908 - samples/sec: 227.29\n",
            "2019-12-20 17:53:00,668 epoch 4 - iter 7/11 - loss 19.49156785 - samples/sec: 242.39\n",
            "2019-12-20 17:53:00,822 epoch 4 - iter 8/11 - loss 19.86900499 - samples/sec: 229.44\n",
            "2019-12-20 17:53:00,981 epoch 4 - iter 9/11 - loss 18.70657797 - samples/sec: 221.73\n",
            "2019-12-20 17:53:01,109 epoch 4 - iter 10/11 - loss 17.40855100 - samples/sec: 284.45\n",
            "2019-12-20 17:53:01,124 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:01,125 EPOCH 4 done: loss 17.4086 - lr 0.1000\n",
            "2019-12-20 17:53:01,331 DEV : loss 15.161355972290039 - score 0.087\n",
            "2019-12-20 17:53:01,338 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 17:53:04,531 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:04,679 epoch 5 - iter 0/11 - loss 21.90028954 - samples/sec: 226.91\n",
            "2019-12-20 17:53:04,847 epoch 5 - iter 1/11 - loss 24.26479816 - samples/sec: 211.85\n",
            "2019-12-20 17:53:05,027 epoch 5 - iter 2/11 - loss 24.38933500 - samples/sec: 195.73\n",
            "2019-12-20 17:53:05,181 epoch 5 - iter 3/11 - loss 21.03457642 - samples/sec: 229.49\n",
            "2019-12-20 17:53:05,322 epoch 5 - iter 4/11 - loss 19.62412052 - samples/sec: 253.01\n",
            "2019-12-20 17:53:05,472 epoch 5 - iter 5/11 - loss 18.11216958 - samples/sec: 235.10\n",
            "2019-12-20 17:53:05,616 epoch 5 - iter 6/11 - loss 17.53726973 - samples/sec: 247.65\n",
            "2019-12-20 17:53:05,764 epoch 5 - iter 7/11 - loss 16.31714582 - samples/sec: 239.01\n",
            "2019-12-20 17:53:05,916 epoch 5 - iter 8/11 - loss 15.75299687 - samples/sec: 234.37\n",
            "2019-12-20 17:53:06,082 epoch 5 - iter 9/11 - loss 15.77505693 - samples/sec: 211.55\n",
            "2019-12-20 17:53:06,216 epoch 5 - iter 10/11 - loss 16.60283618 - samples/sec: 269.88\n",
            "2019-12-20 17:53:06,230 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:06,232 EPOCH 5 done: loss 16.6028 - lr 0.1000\n",
            "2019-12-20 17:53:06,441 DEV : loss 16.135602951049805 - score 0.0\n",
            "2019-12-20 17:53:06,449 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 17:53:06,452 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:06,611 epoch 6 - iter 0/11 - loss 15.93277931 - samples/sec: 205.05\n",
            "2019-12-20 17:53:06,763 epoch 6 - iter 1/11 - loss 16.26532841 - samples/sec: 234.24\n",
            "2019-12-20 17:53:06,906 epoch 6 - iter 2/11 - loss 16.61434174 - samples/sec: 249.49\n",
            "2019-12-20 17:53:07,067 epoch 6 - iter 3/11 - loss 16.80328178 - samples/sec: 216.86\n",
            "2019-12-20 17:53:07,216 epoch 6 - iter 4/11 - loss 16.07805176 - samples/sec: 238.65\n",
            "2019-12-20 17:53:07,370 epoch 6 - iter 5/11 - loss 14.62640452 - samples/sec: 231.57\n",
            "2019-12-20 17:53:07,518 epoch 6 - iter 6/11 - loss 15.11499957 - samples/sec: 240.46\n",
            "2019-12-20 17:53:07,658 epoch 6 - iter 7/11 - loss 14.15998149 - samples/sec: 254.35\n",
            "2019-12-20 17:53:07,812 epoch 6 - iter 8/11 - loss 14.48388375 - samples/sec: 230.55\n",
            "2019-12-20 17:53:07,965 epoch 6 - iter 9/11 - loss 16.03899040 - samples/sec: 230.79\n",
            "2019-12-20 17:53:08,105 epoch 6 - iter 10/11 - loss 15.87608684 - samples/sec: 257.44\n",
            "2019-12-20 17:53:08,119 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:08,120 EPOCH 6 done: loss 15.8761 - lr 0.1000\n",
            "2019-12-20 17:53:08,318 DEV : loss 10.736212730407715 - score 0.0\n",
            "2019-12-20 17:53:08,324 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 17:53:08,326 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:08,473 epoch 7 - iter 0/11 - loss 22.95379448 - samples/sec: 219.81\n",
            "2019-12-20 17:53:08,620 epoch 7 - iter 1/11 - loss 17.17895222 - samples/sec: 241.61\n",
            "2019-12-20 17:53:08,765 epoch 7 - iter 2/11 - loss 14.96958478 - samples/sec: 247.91\n",
            "2019-12-20 17:53:08,913 epoch 7 - iter 3/11 - loss 15.91970110 - samples/sec: 242.52\n",
            "2019-12-20 17:53:09,077 epoch 7 - iter 4/11 - loss 17.07759018 - samples/sec: 215.57\n",
            "2019-12-20 17:53:09,227 epoch 7 - iter 5/11 - loss 15.16331673 - samples/sec: 241.49\n",
            "2019-12-20 17:53:09,378 epoch 7 - iter 6/11 - loss 15.68656458 - samples/sec: 234.65\n",
            "2019-12-20 17:53:09,529 epoch 7 - iter 7/11 - loss 14.98993564 - samples/sec: 235.44\n",
            "2019-12-20 17:53:09,679 epoch 7 - iter 8/11 - loss 14.93122843 - samples/sec: 240.22\n",
            "2019-12-20 17:53:09,837 epoch 7 - iter 9/11 - loss 15.43616581 - samples/sec: 223.76\n",
            "2019-12-20 17:53:09,953 epoch 7 - iter 10/11 - loss 15.07292826 - samples/sec: 316.41\n",
            "2019-12-20 17:53:09,967 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:09,968 EPOCH 7 done: loss 15.0729 - lr 0.1000\n",
            "2019-12-20 17:53:10,192 DEV : loss 11.026542663574219 - score 0.0435\n",
            "2019-12-20 17:53:10,197 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 17:53:10,198 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:10,341 epoch 8 - iter 0/11 - loss 27.02375221 - samples/sec: 226.54\n",
            "2019-12-20 17:53:10,480 epoch 8 - iter 1/11 - loss 19.35916853 - samples/sec: 258.30\n",
            "2019-12-20 17:53:10,630 epoch 8 - iter 2/11 - loss 19.16619015 - samples/sec: 234.26\n",
            "2019-12-20 17:53:10,795 epoch 8 - iter 3/11 - loss 18.95347047 - samples/sec: 213.23\n",
            "2019-12-20 17:53:10,939 epoch 8 - iter 4/11 - loss 17.97588348 - samples/sec: 247.20\n",
            "2019-12-20 17:53:11,081 epoch 8 - iter 5/11 - loss 17.89750957 - samples/sec: 250.07\n",
            "2019-12-20 17:53:11,241 epoch 8 - iter 6/11 - loss 16.69074317 - samples/sec: 220.19\n",
            "2019-12-20 17:53:11,399 epoch 8 - iter 7/11 - loss 16.19891310 - samples/sec: 224.77\n",
            "2019-12-20 17:53:11,550 epoch 8 - iter 8/11 - loss 15.36055311 - samples/sec: 235.28\n",
            "2019-12-20 17:53:11,693 epoch 8 - iter 9/11 - loss 15.43872356 - samples/sec: 248.13\n",
            "2019-12-20 17:53:11,815 epoch 8 - iter 10/11 - loss 15.27733456 - samples/sec: 295.40\n",
            "2019-12-20 17:53:11,830 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:11,831 EPOCH 8 done: loss 15.2773 - lr 0.1000\n",
            "2019-12-20 17:53:12,039 DEV : loss 13.620091438293457 - score 0.0\n",
            "Epoch     7: reducing learning rate of group 0 to 5.0000e-02.\n",
            "2019-12-20 17:53:12,045 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 17:53:12,046 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:12,199 epoch 9 - iter 0/11 - loss 13.10874462 - samples/sec: 212.44\n",
            "2019-12-20 17:53:12,354 epoch 9 - iter 1/11 - loss 11.04114580 - samples/sec: 227.29\n",
            "2019-12-20 17:53:12,503 epoch 9 - iter 2/11 - loss 13.36852551 - samples/sec: 236.77\n",
            "2019-12-20 17:53:12,649 epoch 9 - iter 3/11 - loss 13.69939804 - samples/sec: 244.12\n",
            "2019-12-20 17:53:12,792 epoch 9 - iter 4/11 - loss 13.36748066 - samples/sec: 246.98\n",
            "2019-12-20 17:53:12,941 epoch 9 - iter 5/11 - loss 13.50975752 - samples/sec: 238.55\n",
            "2019-12-20 17:53:13,080 epoch 9 - iter 6/11 - loss 12.65624094 - samples/sec: 263.47\n",
            "2019-12-20 17:53:13,264 epoch 9 - iter 7/11 - loss 12.77905494 - samples/sec: 188.06\n",
            "2019-12-20 17:53:13,414 epoch 9 - iter 8/11 - loss 13.77925285 - samples/sec: 237.95\n",
            "2019-12-20 17:53:13,571 epoch 9 - iter 9/11 - loss 13.68993945 - samples/sec: 224.22\n",
            "2019-12-20 17:53:13,702 epoch 9 - iter 10/11 - loss 13.27460588 - samples/sec: 274.44\n",
            "2019-12-20 17:53:13,716 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:13,718 EPOCH 9 done: loss 13.2746 - lr 0.0500\n",
            "2019-12-20 17:53:13,921 DEV : loss 12.152633666992188 - score 0.0227\n",
            "2019-12-20 17:53:13,928 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 17:53:13,929 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:14,083 epoch 10 - iter 0/11 - loss 18.31401062 - samples/sec: 211.54\n",
            "2019-12-20 17:53:14,245 epoch 10 - iter 1/11 - loss 13.33126068 - samples/sec: 217.67\n",
            "2019-12-20 17:53:14,400 epoch 10 - iter 2/11 - loss 12.17148399 - samples/sec: 229.44\n",
            "2019-12-20 17:53:14,548 epoch 10 - iter 3/11 - loss 12.17849874 - samples/sec: 241.63\n",
            "2019-12-20 17:53:14,703 epoch 10 - iter 4/11 - loss 12.76960850 - samples/sec: 230.10\n",
            "2019-12-20 17:53:14,852 epoch 10 - iter 5/11 - loss 12.76399120 - samples/sec: 238.54\n",
            "2019-12-20 17:53:15,009 epoch 10 - iter 6/11 - loss 12.65459292 - samples/sec: 226.34\n",
            "2019-12-20 17:53:15,170 epoch 10 - iter 7/11 - loss 12.31227040 - samples/sec: 220.38\n",
            "2019-12-20 17:53:15,338 epoch 10 - iter 8/11 - loss 12.14804734 - samples/sec: 210.54\n",
            "2019-12-20 17:53:15,486 epoch 10 - iter 9/11 - loss 12.19805622 - samples/sec: 241.45\n",
            "2019-12-20 17:53:15,612 epoch 10 - iter 10/11 - loss 12.56369088 - samples/sec: 288.98\n",
            "2019-12-20 17:53:15,627 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:15,628 EPOCH 10 done: loss 12.5637 - lr 0.0500\n",
            "2019-12-20 17:53:15,837 DEV : loss 11.508360862731934 - score 0.198\n",
            "2019-12-20 17:53:15,845 BAD EPOCHS (no improvement): 0\n",
            "2019-12-20 17:53:19,064 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:19,212 epoch 11 - iter 0/11 - loss 14.92965984 - samples/sec: 218.88\n",
            "2019-12-20 17:53:19,390 epoch 11 - iter 1/11 - loss 12.21427202 - samples/sec: 198.73\n",
            "2019-12-20 17:53:19,547 epoch 11 - iter 2/11 - loss 11.48795668 - samples/sec: 227.43\n",
            "2019-12-20 17:53:19,718 epoch 11 - iter 3/11 - loss 11.17110419 - samples/sec: 207.27\n",
            "2019-12-20 17:53:19,869 epoch 11 - iter 4/11 - loss 11.63487072 - samples/sec: 237.62\n",
            "2019-12-20 17:53:20,026 epoch 11 - iter 5/11 - loss 11.44122664 - samples/sec: 224.97\n",
            "2019-12-20 17:53:20,175 epoch 11 - iter 6/11 - loss 11.41422762 - samples/sec: 238.90\n",
            "2019-12-20 17:53:20,322 epoch 11 - iter 7/11 - loss 11.82368922 - samples/sec: 241.46\n",
            "2019-12-20 17:53:20,482 epoch 11 - iter 8/11 - loss 11.76917203 - samples/sec: 221.87\n",
            "2019-12-20 17:53:20,631 epoch 11 - iter 9/11 - loss 11.78718681 - samples/sec: 239.15\n",
            "2019-12-20 17:53:20,764 epoch 11 - iter 10/11 - loss 11.06290555 - samples/sec: 270.65\n",
            "2019-12-20 17:53:20,780 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:20,781 EPOCH 11 done: loss 11.0629 - lr 0.0500\n",
            "2019-12-20 17:53:20,991 DEV : loss 10.340595245361328 - score 0.0861\n",
            "2019-12-20 17:53:20,998 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 17:53:20,999 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:21,143 epoch 12 - iter 0/11 - loss 10.24316978 - samples/sec: 226.98\n",
            "2019-12-20 17:53:21,291 epoch 12 - iter 1/11 - loss 11.41183329 - samples/sec: 239.66\n",
            "2019-12-20 17:53:21,445 epoch 12 - iter 2/11 - loss 10.83127117 - samples/sec: 229.05\n",
            "2019-12-20 17:53:21,600 epoch 12 - iter 3/11 - loss 10.75821567 - samples/sec: 227.79\n",
            "2019-12-20 17:53:21,757 epoch 12 - iter 4/11 - loss 11.01560726 - samples/sec: 225.29\n",
            "2019-12-20 17:53:21,912 epoch 12 - iter 5/11 - loss 11.55696201 - samples/sec: 228.37\n",
            "2019-12-20 17:53:22,070 epoch 12 - iter 6/11 - loss 11.10392543 - samples/sec: 222.69\n",
            "2019-12-20 17:53:22,228 epoch 12 - iter 7/11 - loss 10.75117564 - samples/sec: 223.76\n",
            "2019-12-20 17:53:22,379 epoch 12 - iter 8/11 - loss 11.05880949 - samples/sec: 235.09\n",
            "2019-12-20 17:53:22,534 epoch 12 - iter 9/11 - loss 10.52982521 - samples/sec: 227.64\n",
            "2019-12-20 17:53:22,661 epoch 12 - iter 10/11 - loss 10.73111404 - samples/sec: 283.59\n",
            "2019-12-20 17:53:22,676 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:22,677 EPOCH 12 done: loss 10.7311 - lr 0.0500\n",
            "2019-12-20 17:53:22,893 DEV : loss 10.6810941696167 - score 0.0225\n",
            "2019-12-20 17:53:22,899 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 17:53:22,901 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:23,044 epoch 13 - iter 0/11 - loss 10.26668358 - samples/sec: 226.27\n",
            "2019-12-20 17:53:23,195 epoch 13 - iter 1/11 - loss 8.85354233 - samples/sec: 233.45\n",
            "2019-12-20 17:53:23,343 epoch 13 - iter 2/11 - loss 9.55068493 - samples/sec: 239.32\n",
            "2019-12-20 17:53:23,509 epoch 13 - iter 3/11 - loss 9.77369666 - samples/sec: 211.51\n",
            "2019-12-20 17:53:23,658 epoch 13 - iter 4/11 - loss 9.47980995 - samples/sec: 239.52\n",
            "2019-12-20 17:53:23,828 epoch 13 - iter 5/11 - loss 8.92267005 - samples/sec: 205.64\n",
            "2019-12-20 17:53:23,977 epoch 13 - iter 6/11 - loss 8.59792124 - samples/sec: 239.98\n",
            "2019-12-20 17:53:24,136 epoch 13 - iter 7/11 - loss 8.67774713 - samples/sec: 223.05\n",
            "2019-12-20 17:53:24,291 epoch 13 - iter 8/11 - loss 9.32365206 - samples/sec: 228.40\n",
            "2019-12-20 17:53:24,446 epoch 13 - iter 9/11 - loss 9.53705606 - samples/sec: 228.99\n",
            "2019-12-20 17:53:24,588 epoch 13 - iter 10/11 - loss 10.58672931 - samples/sec: 260.88\n",
            "2019-12-20 17:53:24,603 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:24,604 EPOCH 13 done: loss 10.5867 - lr 0.0500\n",
            "2019-12-20 17:53:24,809 DEV : loss 8.411518096923828 - score 0.1781\n",
            "2019-12-20 17:53:24,816 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 17:53:24,819 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:24,961 epoch 14 - iter 0/11 - loss 9.59412766 - samples/sec: 228.18\n",
            "2019-12-20 17:53:25,110 epoch 14 - iter 1/11 - loss 8.88776016 - samples/sec: 240.72\n",
            "2019-12-20 17:53:25,259 epoch 14 - iter 2/11 - loss 11.42715963 - samples/sec: 236.97\n",
            "2019-12-20 17:53:25,401 epoch 14 - iter 3/11 - loss 9.51888871 - samples/sec: 250.64\n",
            "2019-12-20 17:53:25,550 epoch 14 - iter 4/11 - loss 10.30879192 - samples/sec: 237.17\n",
            "2019-12-20 17:53:25,707 epoch 14 - iter 5/11 - loss 9.53703594 - samples/sec: 224.90\n",
            "2019-12-20 17:53:25,853 epoch 14 - iter 6/11 - loss 9.72637708 - samples/sec: 241.90\n",
            "2019-12-20 17:53:26,008 epoch 14 - iter 7/11 - loss 9.30428815 - samples/sec: 227.75\n",
            "2019-12-20 17:53:26,155 epoch 14 - iter 8/11 - loss 9.52256245 - samples/sec: 243.80\n",
            "2019-12-20 17:53:26,302 epoch 14 - iter 9/11 - loss 9.51563482 - samples/sec: 239.89\n",
            "2019-12-20 17:53:26,435 epoch 14 - iter 10/11 - loss 10.24788466 - samples/sec: 269.38\n",
            "2019-12-20 17:53:26,454 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:26,455 EPOCH 14 done: loss 10.2479 - lr 0.0500\n",
            "2019-12-20 17:53:26,667 DEV : loss 7.486727237701416 - score 0.1601\n",
            "Epoch    13: reducing learning rate of group 0 to 2.5000e-02.\n",
            "2019-12-20 17:53:26,673 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 17:53:26,675 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:26,814 epoch 15 - iter 0/11 - loss 10.10417652 - samples/sec: 233.77\n",
            "2019-12-20 17:53:26,970 epoch 15 - iter 1/11 - loss 9.44705391 - samples/sec: 225.15\n",
            "2019-12-20 17:53:27,129 epoch 15 - iter 2/11 - loss 12.10976855 - samples/sec: 222.89\n",
            "2019-12-20 17:53:27,270 epoch 15 - iter 3/11 - loss 11.37856269 - samples/sec: 253.28\n",
            "2019-12-20 17:53:27,417 epoch 15 - iter 4/11 - loss 11.27648582 - samples/sec: 241.03\n",
            "2019-12-20 17:53:27,584 epoch 15 - iter 5/11 - loss 11.37755458 - samples/sec: 209.67\n",
            "2019-12-20 17:53:27,738 epoch 15 - iter 6/11 - loss 10.48212583 - samples/sec: 229.78\n",
            "2019-12-20 17:53:27,897 epoch 15 - iter 7/11 - loss 10.23392546 - samples/sec: 222.34\n",
            "2019-12-20 17:53:28,050 epoch 15 - iter 8/11 - loss 10.66415776 - samples/sec: 230.80\n",
            "2019-12-20 17:53:28,198 epoch 15 - iter 9/11 - loss 10.20058246 - samples/sec: 238.35\n",
            "2019-12-20 17:53:28,314 epoch 15 - iter 10/11 - loss 9.85033807 - samples/sec: 314.99\n",
            "2019-12-20 17:53:28,328 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:28,330 EPOCH 15 done: loss 9.8503 - lr 0.0250\n",
            "2019-12-20 17:53:28,527 DEV : loss 9.720585823059082 - score 0.044\n",
            "2019-12-20 17:53:28,533 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 17:53:28,534 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:28,684 epoch 16 - iter 0/11 - loss 11.64735603 - samples/sec: 216.38\n",
            "2019-12-20 17:53:28,839 epoch 16 - iter 1/11 - loss 10.95276833 - samples/sec: 226.72\n",
            "2019-12-20 17:53:28,999 epoch 16 - iter 2/11 - loss 8.91936270 - samples/sec: 225.09\n",
            "2019-12-20 17:53:29,143 epoch 16 - iter 3/11 - loss 9.17346787 - samples/sec: 249.49\n",
            "2019-12-20 17:53:29,297 epoch 16 - iter 4/11 - loss 9.20585079 - samples/sec: 231.16\n",
            "2019-12-20 17:53:29,459 epoch 16 - iter 5/11 - loss 9.76014249 - samples/sec: 217.36\n",
            "2019-12-20 17:53:29,615 epoch 16 - iter 6/11 - loss 9.61139202 - samples/sec: 231.44\n",
            "2019-12-20 17:53:29,763 epoch 16 - iter 7/11 - loss 9.58309519 - samples/sec: 240.29\n",
            "2019-12-20 17:53:29,912 epoch 16 - iter 8/11 - loss 9.22812716 - samples/sec: 242.71\n",
            "2019-12-20 17:53:30,062 epoch 16 - iter 9/11 - loss 9.24779472 - samples/sec: 237.13\n",
            "2019-12-20 17:53:30,184 epoch 16 - iter 10/11 - loss 9.10991439 - samples/sec: 299.31\n",
            "2019-12-20 17:53:30,200 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:30,201 EPOCH 16 done: loss 9.1099 - lr 0.0250\n",
            "2019-12-20 17:53:30,407 DEV : loss 9.637096405029297 - score 0.0222\n",
            "2019-12-20 17:53:30,414 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 17:53:30,415 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:30,571 epoch 17 - iter 0/11 - loss 6.10469818 - samples/sec: 207.54\n",
            "2019-12-20 17:53:30,730 epoch 17 - iter 1/11 - loss 8.64425373 - samples/sec: 221.23\n",
            "2019-12-20 17:53:30,878 epoch 17 - iter 2/11 - loss 8.75972430 - samples/sec: 239.38\n",
            "2019-12-20 17:53:31,030 epoch 17 - iter 3/11 - loss 8.46129417 - samples/sec: 234.32\n",
            "2019-12-20 17:53:31,182 epoch 17 - iter 4/11 - loss 8.29026642 - samples/sec: 235.06\n",
            "2019-12-20 17:53:31,337 epoch 17 - iter 5/11 - loss 8.23979425 - samples/sec: 227.61\n",
            "2019-12-20 17:53:31,476 epoch 17 - iter 6/11 - loss 8.48578276 - samples/sec: 256.70\n",
            "2019-12-20 17:53:31,626 epoch 17 - iter 7/11 - loss 8.79589665 - samples/sec: 235.49\n",
            "2019-12-20 17:53:31,794 epoch 17 - iter 8/11 - loss 8.88594193 - samples/sec: 215.88\n",
            "2019-12-20 17:53:31,937 epoch 17 - iter 9/11 - loss 8.89787836 - samples/sec: 248.17\n",
            "2019-12-20 17:53:32,078 epoch 17 - iter 10/11 - loss 8.57935294 - samples/sec: 252.54\n",
            "2019-12-20 17:53:32,093 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:32,094 EPOCH 17 done: loss 8.5794 - lr 0.0250\n",
            "2019-12-20 17:53:32,302 DEV : loss 8.772626876831055 - score 0.044\n",
            "2019-12-20 17:53:32,306 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 17:53:32,307 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:32,462 epoch 18 - iter 0/11 - loss 11.04844856 - samples/sec: 210.84\n",
            "2019-12-20 17:53:32,613 epoch 18 - iter 1/11 - loss 9.50739741 - samples/sec: 234.23\n",
            "2019-12-20 17:53:32,765 epoch 18 - iter 2/11 - loss 10.25141637 - samples/sec: 233.48\n",
            "2019-12-20 17:53:32,908 epoch 18 - iter 3/11 - loss 10.48048103 - samples/sec: 251.59\n",
            "2019-12-20 17:53:33,063 epoch 18 - iter 4/11 - loss 9.84000731 - samples/sec: 230.75\n",
            "2019-12-20 17:53:33,208 epoch 18 - iter 5/11 - loss 9.59433230 - samples/sec: 246.39\n",
            "2019-12-20 17:53:33,357 epoch 18 - iter 6/11 - loss 9.92276730 - samples/sec: 237.52\n",
            "2019-12-20 17:53:33,516 epoch 18 - iter 7/11 - loss 9.40278822 - samples/sec: 228.87\n",
            "2019-12-20 17:53:33,664 epoch 18 - iter 8/11 - loss 9.12750599 - samples/sec: 241.61\n",
            "2019-12-20 17:53:33,819 epoch 18 - iter 9/11 - loss 9.02496896 - samples/sec: 229.36\n",
            "2019-12-20 17:53:33,961 epoch 18 - iter 10/11 - loss 9.06585446 - samples/sec: 249.40\n",
            "2019-12-20 17:53:33,978 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:33,979 EPOCH 18 done: loss 9.0659 - lr 0.0250\n",
            "2019-12-20 17:53:34,190 DEV : loss 8.876472473144531 - score 0.0227\n",
            "Epoch    17: reducing learning rate of group 0 to 1.2500e-02.\n",
            "2019-12-20 17:53:34,197 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 17:53:34,198 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:34,337 epoch 19 - iter 0/11 - loss 6.42646885 - samples/sec: 232.17\n",
            "2019-12-20 17:53:34,497 epoch 19 - iter 1/11 - loss 8.12266493 - samples/sec: 220.02\n",
            "2019-12-20 17:53:34,645 epoch 19 - iter 2/11 - loss 7.48142974 - samples/sec: 241.24\n",
            "2019-12-20 17:53:34,818 epoch 19 - iter 3/11 - loss 8.59877896 - samples/sec: 202.87\n",
            "2019-12-20 17:53:34,980 epoch 19 - iter 4/11 - loss 8.64672413 - samples/sec: 220.77\n",
            "2019-12-20 17:53:35,128 epoch 19 - iter 5/11 - loss 7.64930526 - samples/sec: 240.09\n",
            "2019-12-20 17:53:35,280 epoch 19 - iter 6/11 - loss 7.85565519 - samples/sec: 235.03\n",
            "2019-12-20 17:53:35,416 epoch 19 - iter 7/11 - loss 7.31885743 - samples/sec: 266.30\n",
            "2019-12-20 17:53:35,567 epoch 19 - iter 8/11 - loss 6.90391734 - samples/sec: 234.75\n",
            "2019-12-20 17:53:35,725 epoch 19 - iter 9/11 - loss 7.08804963 - samples/sec: 224.22\n",
            "2019-12-20 17:53:35,869 epoch 19 - iter 10/11 - loss 7.41743731 - samples/sec: 254.66\n",
            "2019-12-20 17:53:35,884 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:35,885 EPOCH 19 done: loss 7.4174 - lr 0.0125\n",
            "2019-12-20 17:53:36,093 DEV : loss 8.304762840270996 - score 0.1031\n",
            "2019-12-20 17:53:36,100 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 17:53:36,102 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:36,235 epoch 20 - iter 0/11 - loss 4.67317200 - samples/sec: 243.75\n",
            "2019-12-20 17:53:36,386 epoch 20 - iter 1/11 - loss 5.56477118 - samples/sec: 235.09\n",
            "2019-12-20 17:53:36,533 epoch 20 - iter 2/11 - loss 5.46521505 - samples/sec: 241.18\n",
            "2019-12-20 17:53:36,699 epoch 20 - iter 3/11 - loss 6.88179779 - samples/sec: 211.89\n",
            "2019-12-20 17:53:36,864 epoch 20 - iter 4/11 - loss 7.27875557 - samples/sec: 215.88\n",
            "2019-12-20 17:53:37,009 epoch 20 - iter 5/11 - loss 7.38746945 - samples/sec: 245.60\n",
            "2019-12-20 17:53:37,158 epoch 20 - iter 6/11 - loss 7.40636029 - samples/sec: 238.93\n",
            "2019-12-20 17:53:37,304 epoch 20 - iter 7/11 - loss 7.31690484 - samples/sec: 244.85\n",
            "2019-12-20 17:53:37,454 epoch 20 - iter 8/11 - loss 7.23193089 - samples/sec: 235.20\n",
            "2019-12-20 17:53:37,608 epoch 20 - iter 9/11 - loss 7.53366380 - samples/sec: 230.05\n",
            "2019-12-20 17:53:37,745 epoch 20 - iter 10/11 - loss 7.79866535 - samples/sec: 266.23\n",
            "2019-12-20 17:53:37,762 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:37,763 EPOCH 20 done: loss 7.7987 - lr 0.0125\n",
            "2019-12-20 17:53:37,980 DEV : loss 8.087528228759766 - score 0.044\n",
            "2019-12-20 17:53:37,987 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 17:53:37,988 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:38,125 epoch 21 - iter 0/11 - loss 7.38995457 - samples/sec: 236.37\n",
            "2019-12-20 17:53:38,278 epoch 21 - iter 1/11 - loss 6.93170834 - samples/sec: 231.10\n",
            "2019-12-20 17:53:38,428 epoch 21 - iter 2/11 - loss 6.54428418 - samples/sec: 236.96\n",
            "2019-12-20 17:53:38,568 epoch 21 - iter 3/11 - loss 6.26744795 - samples/sec: 255.61\n",
            "2019-12-20 17:53:38,725 epoch 21 - iter 4/11 - loss 5.60186501 - samples/sec: 224.12\n",
            "2019-12-20 17:53:38,895 epoch 21 - iter 5/11 - loss 5.72844378 - samples/sec: 206.91\n",
            "2019-12-20 17:53:39,037 epoch 21 - iter 6/11 - loss 6.47565951 - samples/sec: 250.76\n",
            "2019-12-20 17:53:39,184 epoch 21 - iter 7/11 - loss 6.32570565 - samples/sec: 245.39\n",
            "2019-12-20 17:53:39,331 epoch 21 - iter 8/11 - loss 6.61562761 - samples/sec: 242.87\n",
            "2019-12-20 17:53:39,486 epoch 21 - iter 9/11 - loss 6.59195910 - samples/sec: 229.95\n",
            "2019-12-20 17:53:39,605 epoch 21 - iter 10/11 - loss 7.02521454 - samples/sec: 310.35\n",
            "2019-12-20 17:53:39,620 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:39,622 EPOCH 21 done: loss 7.0252 - lr 0.0125\n",
            "2019-12-20 17:53:39,837 DEV : loss 7.8610920906066895 - score 0.0435\n",
            "2019-12-20 17:53:39,848 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 17:53:39,849 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:39,997 epoch 22 - iter 0/11 - loss 8.41515541 - samples/sec: 219.74\n",
            "2019-12-20 17:53:40,160 epoch 22 - iter 1/11 - loss 7.36262417 - samples/sec: 217.36\n",
            "2019-12-20 17:53:40,309 epoch 22 - iter 2/11 - loss 7.41741816 - samples/sec: 241.51\n",
            "2019-12-20 17:53:40,450 epoch 22 - iter 3/11 - loss 6.85416317 - samples/sec: 254.62\n",
            "2019-12-20 17:53:40,602 epoch 22 - iter 4/11 - loss 6.56614933 - samples/sec: 233.88\n",
            "2019-12-20 17:53:40,760 epoch 22 - iter 5/11 - loss 6.57158224 - samples/sec: 224.47\n",
            "2019-12-20 17:53:40,926 epoch 22 - iter 6/11 - loss 6.43255745 - samples/sec: 214.26\n",
            "2019-12-20 17:53:41,075 epoch 22 - iter 7/11 - loss 6.32473022 - samples/sec: 241.42\n",
            "2019-12-20 17:53:41,226 epoch 22 - iter 8/11 - loss 6.68618017 - samples/sec: 236.47\n",
            "2019-12-20 17:53:41,385 epoch 22 - iter 9/11 - loss 7.00294108 - samples/sec: 221.05\n",
            "2019-12-20 17:53:41,515 epoch 22 - iter 10/11 - loss 6.69762412 - samples/sec: 279.94\n",
            "2019-12-20 17:53:41,530 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:41,531 EPOCH 22 done: loss 6.6976 - lr 0.0125\n",
            "2019-12-20 17:53:41,739 DEV : loss 7.907310485839844 - score 0.066\n",
            "Epoch    21: reducing learning rate of group 0 to 6.2500e-03.\n",
            "2019-12-20 17:53:41,744 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 17:53:41,746 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:41,881 epoch 23 - iter 0/11 - loss 4.71074200 - samples/sec: 241.92\n",
            "2019-12-20 17:53:42,046 epoch 23 - iter 1/11 - loss 5.38934994 - samples/sec: 219.51\n",
            "2019-12-20 17:53:42,192 epoch 23 - iter 2/11 - loss 5.12900988 - samples/sec: 244.97\n",
            "2019-12-20 17:53:42,338 epoch 23 - iter 3/11 - loss 5.72950816 - samples/sec: 243.01\n",
            "2019-12-20 17:53:42,482 epoch 23 - iter 4/11 - loss 6.49144745 - samples/sec: 244.78\n",
            "2019-12-20 17:53:42,635 epoch 23 - iter 5/11 - loss 6.08003656 - samples/sec: 229.60\n",
            "2019-12-20 17:53:42,780 epoch 23 - iter 6/11 - loss 6.20669140 - samples/sec: 251.37\n",
            "2019-12-20 17:53:42,948 epoch 23 - iter 7/11 - loss 6.77559847 - samples/sec: 208.96\n",
            "2019-12-20 17:53:43,093 epoch 23 - iter 8/11 - loss 6.43807708 - samples/sec: 248.00\n",
            "2019-12-20 17:53:43,253 epoch 23 - iter 9/11 - loss 6.58778443 - samples/sec: 222.88\n",
            "2019-12-20 17:53:43,385 epoch 23 - iter 10/11 - loss 6.61695268 - samples/sec: 274.64\n",
            "2019-12-20 17:53:43,400 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:43,401 EPOCH 23 done: loss 6.6170 - lr 0.0063\n",
            "2019-12-20 17:53:43,607 DEV : loss 7.458405494689941 - score 0.1076\n",
            "2019-12-20 17:53:43,613 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 17:53:43,614 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:43,761 epoch 24 - iter 0/11 - loss 5.19236040 - samples/sec: 220.00\n",
            "2019-12-20 17:53:43,918 epoch 24 - iter 1/11 - loss 4.29379332 - samples/sec: 225.27\n",
            "2019-12-20 17:53:44,084 epoch 24 - iter 2/11 - loss 4.93046610 - samples/sec: 213.90\n",
            "2019-12-20 17:53:44,237 epoch 24 - iter 3/11 - loss 5.71269828 - samples/sec: 234.81\n",
            "2019-12-20 17:53:44,394 epoch 24 - iter 4/11 - loss 5.92549796 - samples/sec: 231.13\n",
            "2019-12-20 17:53:44,534 epoch 24 - iter 5/11 - loss 5.87000016 - samples/sec: 257.00\n",
            "2019-12-20 17:53:44,685 epoch 24 - iter 6/11 - loss 5.58977648 - samples/sec: 236.63\n",
            "2019-12-20 17:53:44,838 epoch 24 - iter 7/11 - loss 5.96810773 - samples/sec: 233.01\n",
            "2019-12-20 17:53:45,011 epoch 24 - iter 8/11 - loss 6.29164741 - samples/sec: 210.66\n",
            "2019-12-20 17:53:45,154 epoch 24 - iter 9/11 - loss 5.91701565 - samples/sec: 254.43\n",
            "2019-12-20 17:53:45,288 epoch 24 - iter 10/11 - loss 5.78944094 - samples/sec: 270.32\n",
            "2019-12-20 17:53:45,304 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:45,305 EPOCH 24 done: loss 5.7894 - lr 0.0063\n",
            "2019-12-20 17:53:45,548 DEV : loss 7.140674114227295 - score 0.0645\n",
            "2019-12-20 17:53:45,554 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 17:53:45,556 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:45,698 epoch 25 - iter 0/11 - loss 4.57788467 - samples/sec: 225.74\n",
            "2019-12-20 17:53:45,861 epoch 25 - iter 1/11 - loss 6.28989697 - samples/sec: 221.64\n",
            "2019-12-20 17:53:46,024 epoch 25 - iter 2/11 - loss 5.68767039 - samples/sec: 217.33\n",
            "2019-12-20 17:53:46,182 epoch 25 - iter 3/11 - loss 6.06660008 - samples/sec: 225.33\n",
            "2019-12-20 17:53:46,340 epoch 25 - iter 4/11 - loss 5.72432909 - samples/sec: 223.25\n",
            "2019-12-20 17:53:46,503 epoch 25 - iter 5/11 - loss 6.82592416 - samples/sec: 219.35\n",
            "2019-12-20 17:53:46,645 epoch 25 - iter 6/11 - loss 6.74242932 - samples/sec: 255.17\n",
            "2019-12-20 17:53:46,793 epoch 25 - iter 7/11 - loss 6.80337703 - samples/sec: 239.85\n",
            "2019-12-20 17:53:46,956 epoch 25 - iter 8/11 - loss 6.66353215 - samples/sec: 220.23\n",
            "2019-12-20 17:53:47,118 epoch 25 - iter 9/11 - loss 6.68707108 - samples/sec: 217.24\n",
            "2019-12-20 17:53:47,243 epoch 25 - iter 10/11 - loss 6.46906437 - samples/sec: 295.88\n",
            "2019-12-20 17:53:47,259 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:47,260 EPOCH 25 done: loss 6.4691 - lr 0.0063\n",
            "2019-12-20 17:53:47,470 DEV : loss 7.404665470123291 - score 0.0435\n",
            "2019-12-20 17:53:47,476 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 17:53:47,477 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:47,615 epoch 26 - iter 0/11 - loss 6.91331100 - samples/sec: 233.51\n",
            "2019-12-20 17:53:47,768 epoch 26 - iter 1/11 - loss 5.28385019 - samples/sec: 233.76\n",
            "2019-12-20 17:53:47,928 epoch 26 - iter 2/11 - loss 5.44261487 - samples/sec: 221.80\n",
            "2019-12-20 17:53:48,088 epoch 26 - iter 3/11 - loss 5.94495940 - samples/sec: 221.32\n",
            "2019-12-20 17:53:48,245 epoch 26 - iter 4/11 - loss 6.22048721 - samples/sec: 226.45\n",
            "2019-12-20 17:53:48,396 epoch 26 - iter 5/11 - loss 6.84695943 - samples/sec: 234.09\n",
            "2019-12-20 17:53:48,551 epoch 26 - iter 6/11 - loss 6.94454466 - samples/sec: 227.68\n",
            "2019-12-20 17:53:48,693 epoch 26 - iter 7/11 - loss 6.36398280 - samples/sec: 252.12\n",
            "2019-12-20 17:53:48,846 epoch 26 - iter 8/11 - loss 6.38683679 - samples/sec: 231.56\n",
            "2019-12-20 17:53:49,002 epoch 26 - iter 9/11 - loss 6.29434080 - samples/sec: 226.52\n",
            "2019-12-20 17:53:49,134 epoch 26 - iter 10/11 - loss 6.22010495 - samples/sec: 273.72\n",
            "2019-12-20 17:53:49,148 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:49,150 EPOCH 26 done: loss 6.2201 - lr 0.0063\n",
            "2019-12-20 17:53:49,353 DEV : loss 6.951272010803223 - score 0.044\n",
            "Epoch    25: reducing learning rate of group 0 to 3.1250e-03.\n",
            "2019-12-20 17:53:49,358 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 17:53:49,359 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:49,508 epoch 27 - iter 0/11 - loss 5.27527428 - samples/sec: 220.87\n",
            "2019-12-20 17:53:49,663 epoch 27 - iter 1/11 - loss 5.79579687 - samples/sec: 228.98\n",
            "2019-12-20 17:53:49,820 epoch 27 - iter 2/11 - loss 6.85243511 - samples/sec: 225.94\n",
            "2019-12-20 17:53:49,978 epoch 27 - iter 3/11 - loss 6.43484092 - samples/sec: 226.24\n",
            "2019-12-20 17:53:50,153 epoch 27 - iter 4/11 - loss 6.06627970 - samples/sec: 203.76\n",
            "2019-12-20 17:53:50,307 epoch 27 - iter 5/11 - loss 6.40500855 - samples/sec: 231.69\n",
            "2019-12-20 17:53:50,458 epoch 27 - iter 6/11 - loss 6.25179216 - samples/sec: 235.82\n",
            "2019-12-20 17:53:50,605 epoch 27 - iter 7/11 - loss 6.21542066 - samples/sec: 242.45\n",
            "2019-12-20 17:53:50,768 epoch 27 - iter 8/11 - loss 6.15123129 - samples/sec: 215.30\n",
            "2019-12-20 17:53:50,916 epoch 27 - iter 9/11 - loss 5.94594073 - samples/sec: 242.22\n",
            "2019-12-20 17:53:51,047 epoch 27 - iter 10/11 - loss 6.20779198 - samples/sec: 276.59\n",
            "2019-12-20 17:53:51,062 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:51,063 EPOCH 27 done: loss 6.2078 - lr 0.0031\n",
            "2019-12-20 17:53:51,282 DEV : loss 7.051835536956787 - score 0.0851\n",
            "2019-12-20 17:53:51,288 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 17:53:51,290 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:51,440 epoch 28 - iter 0/11 - loss 5.01325130 - samples/sec: 217.38\n",
            "2019-12-20 17:53:51,592 epoch 28 - iter 1/11 - loss 4.88675976 - samples/sec: 233.53\n",
            "2019-12-20 17:53:51,746 epoch 28 - iter 2/11 - loss 4.88793230 - samples/sec: 231.55\n",
            "2019-12-20 17:53:51,897 epoch 28 - iter 3/11 - loss 4.92432821 - samples/sec: 233.55\n",
            "2019-12-20 17:53:52,054 epoch 28 - iter 4/11 - loss 5.43592958 - samples/sec: 226.56\n",
            "2019-12-20 17:53:52,211 epoch 28 - iter 5/11 - loss 5.90183059 - samples/sec: 226.50\n",
            "2019-12-20 17:53:52,363 epoch 28 - iter 6/11 - loss 5.99426426 - samples/sec: 233.87\n",
            "2019-12-20 17:53:52,531 epoch 28 - iter 7/11 - loss 5.75374204 - samples/sec: 210.45\n",
            "2019-12-20 17:53:52,680 epoch 28 - iter 8/11 - loss 6.06903632 - samples/sec: 240.86\n",
            "2019-12-20 17:53:52,835 epoch 28 - iter 9/11 - loss 5.87874694 - samples/sec: 231.22\n",
            "2019-12-20 17:53:52,961 epoch 28 - iter 10/11 - loss 5.73245833 - samples/sec: 289.12\n",
            "2019-12-20 17:53:52,977 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:52,978 EPOCH 28 done: loss 5.7325 - lr 0.0031\n",
            "2019-12-20 17:53:53,200 DEV : loss 6.9839887619018555 - score 0.0435\n",
            "2019-12-20 17:53:53,207 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 17:53:53,208 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:53,352 epoch 29 - iter 0/11 - loss 5.24682522 - samples/sec: 225.88\n",
            "2019-12-20 17:53:53,507 epoch 29 - iter 1/11 - loss 4.31374693 - samples/sec: 227.48\n",
            "2019-12-20 17:53:53,656 epoch 29 - iter 2/11 - loss 4.25230455 - samples/sec: 239.95\n",
            "2019-12-20 17:53:53,812 epoch 29 - iter 3/11 - loss 5.31157148 - samples/sec: 227.60\n",
            "2019-12-20 17:53:53,962 epoch 29 - iter 4/11 - loss 5.22154961 - samples/sec: 235.94\n",
            "2019-12-20 17:53:54,105 epoch 29 - iter 5/11 - loss 5.08525840 - samples/sec: 255.29\n",
            "2019-12-20 17:53:54,268 epoch 29 - iter 6/11 - loss 5.09132188 - samples/sec: 216.73\n",
            "2019-12-20 17:53:54,421 epoch 29 - iter 7/11 - loss 5.31936085 - samples/sec: 233.04\n",
            "2019-12-20 17:53:54,582 epoch 29 - iter 8/11 - loss 5.62143728 - samples/sec: 221.09\n",
            "2019-12-20 17:53:54,736 epoch 29 - iter 9/11 - loss 5.35238466 - samples/sec: 230.97\n",
            "2019-12-20 17:53:54,862 epoch 29 - iter 10/11 - loss 5.50131924 - samples/sec: 302.59\n",
            "2019-12-20 17:53:54,877 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:54,878 EPOCH 29 done: loss 5.5013 - lr 0.0031\n",
            "2019-12-20 17:53:55,085 DEV : loss 6.9653706550598145 - score 0.0435\n",
            "2019-12-20 17:53:55,092 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 17:53:55,093 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:55,243 epoch 30 - iter 0/11 - loss 7.83232498 - samples/sec: 215.61\n",
            "2019-12-20 17:53:55,393 epoch 30 - iter 1/11 - loss 7.41573715 - samples/sec: 244.69\n",
            "2019-12-20 17:53:55,544 epoch 30 - iter 2/11 - loss 7.30492687 - samples/sec: 236.24\n",
            "2019-12-20 17:53:55,702 epoch 30 - iter 3/11 - loss 6.95888269 - samples/sec: 225.71\n",
            "2019-12-20 17:53:55,852 epoch 30 - iter 4/11 - loss 6.29735479 - samples/sec: 237.44\n",
            "2019-12-20 17:53:56,008 epoch 30 - iter 5/11 - loss 6.01731173 - samples/sec: 228.52\n",
            "2019-12-20 17:53:56,156 epoch 30 - iter 6/11 - loss 6.35601752 - samples/sec: 242.15\n",
            "2019-12-20 17:53:56,339 epoch 30 - iter 7/11 - loss 6.19770712 - samples/sec: 189.04\n",
            "2019-12-20 17:53:56,493 epoch 30 - iter 8/11 - loss 5.80192418 - samples/sec: 230.38\n",
            "2019-12-20 17:53:56,646 epoch 30 - iter 9/11 - loss 5.63299351 - samples/sec: 230.33\n",
            "2019-12-20 17:53:56,762 epoch 30 - iter 10/11 - loss 5.56490088 - samples/sec: 315.15\n",
            "2019-12-20 17:53:56,777 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:56,778 EPOCH 30 done: loss 5.5649 - lr 0.0031\n",
            "2019-12-20 17:53:56,986 DEV : loss 6.8976545333862305 - score 0.0445\n",
            "Epoch    29: reducing learning rate of group 0 to 1.5625e-03.\n",
            "2019-12-20 17:53:56,993 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 17:53:56,994 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:57,131 epoch 31 - iter 0/11 - loss 3.49082780 - samples/sec: 235.99\n",
            "2019-12-20 17:53:57,298 epoch 31 - iter 1/11 - loss 5.29656661 - samples/sec: 209.62\n",
            "2019-12-20 17:53:57,461 epoch 31 - iter 2/11 - loss 5.12688692 - samples/sec: 215.88\n",
            "2019-12-20 17:53:57,614 epoch 31 - iter 3/11 - loss 4.89788657 - samples/sec: 232.98\n",
            "2019-12-20 17:53:57,768 epoch 31 - iter 4/11 - loss 5.56171041 - samples/sec: 229.16\n",
            "2019-12-20 17:53:57,906 epoch 31 - iter 5/11 - loss 5.14176524 - samples/sec: 258.89\n",
            "2019-12-20 17:53:58,049 epoch 31 - iter 6/11 - loss 4.95040284 - samples/sec: 247.81\n",
            "2019-12-20 17:53:58,195 epoch 31 - iter 7/11 - loss 4.74922976 - samples/sec: 246.60\n",
            "2019-12-20 17:53:58,356 epoch 31 - iter 8/11 - loss 5.09522160 - samples/sec: 217.18\n",
            "2019-12-20 17:53:58,503 epoch 31 - iter 9/11 - loss 5.15103428 - samples/sec: 239.99\n",
            "2019-12-20 17:53:58,619 epoch 31 - iter 10/11 - loss 5.32525260 - samples/sec: 311.54\n",
            "2019-12-20 17:53:58,633 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:58,635 EPOCH 31 done: loss 5.3253 - lr 0.0016\n",
            "2019-12-20 17:53:58,845 DEV : loss 6.9121832847595215 - score 0.0445\n",
            "2019-12-20 17:53:58,852 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 17:53:58,853 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:53:58,985 epoch 32 - iter 0/11 - loss 4.64623070 - samples/sec: 245.56\n",
            "2019-12-20 17:53:59,140 epoch 32 - iter 1/11 - loss 5.38478184 - samples/sec: 229.02\n",
            "2019-12-20 17:53:59,288 epoch 32 - iter 2/11 - loss 5.80798642 - samples/sec: 246.14\n",
            "2019-12-20 17:53:59,444 epoch 32 - iter 3/11 - loss 7.01020753 - samples/sec: 227.85\n",
            "2019-12-20 17:53:59,602 epoch 32 - iter 4/11 - loss 7.61254473 - samples/sec: 227.02\n",
            "2019-12-20 17:53:59,744 epoch 32 - iter 5/11 - loss 7.12273177 - samples/sec: 252.34\n",
            "2019-12-20 17:53:59,903 epoch 32 - iter 6/11 - loss 6.55426839 - samples/sec: 228.65\n",
            "2019-12-20 17:54:00,056 epoch 32 - iter 7/11 - loss 6.13305727 - samples/sec: 234.16\n",
            "2019-12-20 17:54:00,205 epoch 32 - iter 8/11 - loss 6.02739072 - samples/sec: 239.50\n",
            "2019-12-20 17:54:00,373 epoch 32 - iter 9/11 - loss 5.59238755 - samples/sec: 208.97\n",
            "2019-12-20 17:54:00,506 epoch 32 - iter 10/11 - loss 5.94840567 - samples/sec: 270.26\n",
            "2019-12-20 17:54:00,520 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:00,522 EPOCH 32 done: loss 5.9484 - lr 0.0016\n",
            "2019-12-20 17:54:00,727 DEV : loss 6.816218852996826 - score 0.0445\n",
            "2019-12-20 17:54:00,731 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 17:54:00,732 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:00,881 epoch 33 - iter 0/11 - loss 2.00939393 - samples/sec: 218.77\n",
            "2019-12-20 17:54:01,030 epoch 33 - iter 1/11 - loss 4.00386965 - samples/sec: 240.06\n",
            "2019-12-20 17:54:01,179 epoch 33 - iter 2/11 - loss 3.80036044 - samples/sec: 239.13\n",
            "2019-12-20 17:54:01,328 epoch 33 - iter 3/11 - loss 4.74804825 - samples/sec: 236.89\n",
            "2019-12-20 17:54:01,496 epoch 33 - iter 4/11 - loss 4.87609267 - samples/sec: 208.70\n",
            "2019-12-20 17:54:01,645 epoch 33 - iter 5/11 - loss 5.28625913 - samples/sec: 238.28\n",
            "2019-12-20 17:54:01,784 epoch 33 - iter 6/11 - loss 5.56928052 - samples/sec: 258.34\n",
            "2019-12-20 17:54:01,935 epoch 33 - iter 7/11 - loss 5.77184328 - samples/sec: 234.54\n",
            "2019-12-20 17:54:02,078 epoch 33 - iter 8/11 - loss 5.67237163 - samples/sec: 251.10\n",
            "2019-12-20 17:54:02,227 epoch 33 - iter 9/11 - loss 5.91819818 - samples/sec: 237.41\n",
            "2019-12-20 17:54:02,358 epoch 33 - iter 10/11 - loss 6.08320178 - samples/sec: 276.25\n",
            "2019-12-20 17:54:02,372 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:02,373 EPOCH 33 done: loss 6.0832 - lr 0.0016\n",
            "2019-12-20 17:54:02,586 DEV : loss 6.807503700256348 - score 0.0435\n",
            "2019-12-20 17:54:02,592 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 17:54:02,593 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:02,733 epoch 34 - iter 0/11 - loss 6.26425838 - samples/sec: 232.11\n",
            "2019-12-20 17:54:02,894 epoch 34 - iter 1/11 - loss 6.86225629 - samples/sec: 218.06\n",
            "2019-12-20 17:54:03,037 epoch 34 - iter 2/11 - loss 5.67111659 - samples/sec: 249.06\n",
            "2019-12-20 17:54:03,186 epoch 34 - iter 3/11 - loss 5.23608500 - samples/sec: 238.01\n",
            "2019-12-20 17:54:03,334 epoch 34 - iter 4/11 - loss 5.44674840 - samples/sec: 240.70\n",
            "2019-12-20 17:54:03,501 epoch 34 - iter 5/11 - loss 5.05216030 - samples/sec: 212.30\n",
            "2019-12-20 17:54:03,661 epoch 34 - iter 6/11 - loss 5.51490624 - samples/sec: 221.73\n",
            "2019-12-20 17:54:03,808 epoch 34 - iter 7/11 - loss 5.76886883 - samples/sec: 242.89\n",
            "2019-12-20 17:54:03,960 epoch 34 - iter 8/11 - loss 5.91091217 - samples/sec: 236.59\n",
            "2019-12-20 17:54:04,127 epoch 34 - iter 9/11 - loss 5.80901606 - samples/sec: 211.55\n",
            "2019-12-20 17:54:04,265 epoch 34 - iter 10/11 - loss 5.90664939 - samples/sec: 261.62\n",
            "2019-12-20 17:54:04,279 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:04,280 EPOCH 34 done: loss 5.9066 - lr 0.0016\n",
            "2019-12-20 17:54:04,495 DEV : loss 6.677740097045898 - score 0.0435\n",
            "Epoch    33: reducing learning rate of group 0 to 7.8125e-04.\n",
            "2019-12-20 17:54:04,502 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 17:54:04,504 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:04,647 epoch 35 - iter 0/11 - loss 4.16621399 - samples/sec: 228.66\n",
            "2019-12-20 17:54:04,805 epoch 35 - iter 1/11 - loss 6.15904188 - samples/sec: 230.98\n",
            "2019-12-20 17:54:04,983 epoch 35 - iter 2/11 - loss 5.09915113 - samples/sec: 198.51\n",
            "2019-12-20 17:54:05,145 epoch 35 - iter 3/11 - loss 5.35338128 - samples/sec: 220.31\n",
            "2019-12-20 17:54:05,295 epoch 35 - iter 4/11 - loss 5.59633398 - samples/sec: 242.04\n",
            "2019-12-20 17:54:05,440 epoch 35 - iter 5/11 - loss 5.58165113 - samples/sec: 246.81\n",
            "2019-12-20 17:54:05,591 epoch 35 - iter 6/11 - loss 5.57625198 - samples/sec: 232.68\n",
            "2019-12-20 17:54:05,742 epoch 35 - iter 7/11 - loss 5.59549427 - samples/sec: 234.99\n",
            "2019-12-20 17:54:05,893 epoch 35 - iter 8/11 - loss 5.42162493 - samples/sec: 235.77\n",
            "2019-12-20 17:54:06,046 epoch 35 - iter 9/11 - loss 5.29967499 - samples/sec: 231.35\n",
            "2019-12-20 17:54:06,169 epoch 35 - iter 10/11 - loss 5.25432569 - samples/sec: 297.15\n",
            "2019-12-20 17:54:06,186 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:06,187 EPOCH 35 done: loss 5.2543 - lr 0.0008\n",
            "2019-12-20 17:54:06,395 DEV : loss 6.6533002853393555 - score 0.0435\n",
            "2019-12-20 17:54:06,402 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 17:54:06,403 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:06,548 epoch 36 - iter 0/11 - loss 4.54901743 - samples/sec: 224.34\n",
            "2019-12-20 17:54:06,693 epoch 36 - iter 1/11 - loss 4.80816460 - samples/sec: 246.59\n",
            "2019-12-20 17:54:06,874 epoch 36 - iter 2/11 - loss 4.27667634 - samples/sec: 194.26\n",
            "2019-12-20 17:54:07,021 epoch 36 - iter 3/11 - loss 5.76815093 - samples/sec: 241.00\n",
            "2019-12-20 17:54:07,166 epoch 36 - iter 4/11 - loss 5.48460541 - samples/sec: 247.98\n",
            "2019-12-20 17:54:07,325 epoch 36 - iter 5/11 - loss 6.48853143 - samples/sec: 224.24\n",
            "2019-12-20 17:54:07,481 epoch 36 - iter 6/11 - loss 6.36534732 - samples/sec: 227.13\n",
            "2019-12-20 17:54:07,644 epoch 36 - iter 7/11 - loss 6.29800439 - samples/sec: 217.81\n",
            "2019-12-20 17:54:07,798 epoch 36 - iter 8/11 - loss 6.09565761 - samples/sec: 229.91\n",
            "2019-12-20 17:54:07,953 epoch 36 - iter 9/11 - loss 6.04940376 - samples/sec: 227.20\n",
            "2019-12-20 17:54:08,081 epoch 36 - iter 10/11 - loss 5.89104765 - samples/sec: 283.24\n",
            "2019-12-20 17:54:08,097 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:08,098 EPOCH 36 done: loss 5.8910 - lr 0.0008\n",
            "2019-12-20 17:54:08,326 DEV : loss 6.688028335571289 - score 0.0435\n",
            "2019-12-20 17:54:08,333 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 17:54:08,335 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:08,475 epoch 37 - iter 0/11 - loss 5.71204329 - samples/sec: 230.94\n",
            "2019-12-20 17:54:08,637 epoch 37 - iter 1/11 - loss 4.74866796 - samples/sec: 218.94\n",
            "2019-12-20 17:54:08,784 epoch 37 - iter 2/11 - loss 4.88000600 - samples/sec: 244.40\n",
            "2019-12-20 17:54:08,944 epoch 37 - iter 3/11 - loss 5.32510126 - samples/sec: 220.43\n",
            "2019-12-20 17:54:09,084 epoch 37 - iter 4/11 - loss 5.91136026 - samples/sec: 256.60\n",
            "2019-12-20 17:54:09,237 epoch 37 - iter 5/11 - loss 5.50654062 - samples/sec: 232.99\n",
            "2019-12-20 17:54:09,402 epoch 37 - iter 6/11 - loss 6.10660417 - samples/sec: 212.63\n",
            "2019-12-20 17:54:09,549 epoch 37 - iter 7/11 - loss 5.80147940 - samples/sec: 243.36\n",
            "2019-12-20 17:54:09,717 epoch 37 - iter 8/11 - loss 5.60804923 - samples/sec: 210.24\n",
            "2019-12-20 17:54:09,870 epoch 37 - iter 9/11 - loss 5.48721900 - samples/sec: 229.25\n",
            "2019-12-20 17:54:10,009 epoch 37 - iter 10/11 - loss 5.29710184 - samples/sec: 261.59\n",
            "2019-12-20 17:54:10,024 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:10,025 EPOCH 37 done: loss 5.2971 - lr 0.0008\n",
            "2019-12-20 17:54:10,231 DEV : loss 6.70232629776001 - score 0.0435\n",
            "2019-12-20 17:54:10,238 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 17:54:10,239 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:10,387 epoch 38 - iter 0/11 - loss 6.93682957 - samples/sec: 220.23\n",
            "2019-12-20 17:54:10,542 epoch 38 - iter 1/11 - loss 6.30130911 - samples/sec: 228.04\n",
            "2019-12-20 17:54:10,700 epoch 38 - iter 2/11 - loss 6.07015753 - samples/sec: 225.99\n",
            "2019-12-20 17:54:10,857 epoch 38 - iter 3/11 - loss 5.70081031 - samples/sec: 225.39\n",
            "2019-12-20 17:54:11,020 epoch 38 - iter 4/11 - loss 5.63214550 - samples/sec: 216.74\n",
            "2019-12-20 17:54:11,169 epoch 38 - iter 5/11 - loss 5.57154528 - samples/sec: 238.90\n",
            "2019-12-20 17:54:11,321 epoch 38 - iter 6/11 - loss 5.37243945 - samples/sec: 234.89\n",
            "2019-12-20 17:54:11,472 epoch 38 - iter 7/11 - loss 5.41438574 - samples/sec: 235.53\n",
            "2019-12-20 17:54:11,635 epoch 38 - iter 8/11 - loss 5.64005168 - samples/sec: 215.82\n",
            "2019-12-20 17:54:11,814 epoch 38 - iter 9/11 - loss 5.45802727 - samples/sec: 202.98\n",
            "2019-12-20 17:54:11,949 epoch 38 - iter 10/11 - loss 5.60867496 - samples/sec: 268.80\n",
            "2019-12-20 17:54:11,964 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:11,966 EPOCH 38 done: loss 5.6087 - lr 0.0008\n",
            "2019-12-20 17:54:12,178 DEV : loss 6.616970062255859 - score 0.0435\n",
            "Epoch    37: reducing learning rate of group 0 to 3.9063e-04.\n",
            "2019-12-20 17:54:12,186 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 17:54:12,187 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:12,331 epoch 39 - iter 0/11 - loss 4.45296431 - samples/sec: 226.71\n",
            "2019-12-20 17:54:12,485 epoch 39 - iter 1/11 - loss 5.34398842 - samples/sec: 228.28\n",
            "2019-12-20 17:54:12,663 epoch 39 - iter 2/11 - loss 5.08565013 - samples/sec: 200.45\n",
            "2019-12-20 17:54:12,823 epoch 39 - iter 3/11 - loss 5.11134863 - samples/sec: 222.32\n",
            "2019-12-20 17:54:12,976 epoch 39 - iter 4/11 - loss 4.87165709 - samples/sec: 232.41\n",
            "2019-12-20 17:54:13,126 epoch 39 - iter 5/11 - loss 5.15218552 - samples/sec: 235.54\n",
            "2019-12-20 17:54:13,281 epoch 39 - iter 6/11 - loss 5.48297289 - samples/sec: 227.99\n",
            "2019-12-20 17:54:13,433 epoch 39 - iter 7/11 - loss 5.58011764 - samples/sec: 232.47\n",
            "2019-12-20 17:54:13,591 epoch 39 - iter 8/11 - loss 5.35109305 - samples/sec: 224.07\n",
            "2019-12-20 17:54:13,755 epoch 39 - iter 9/11 - loss 5.50120285 - samples/sec: 215.03\n",
            "2019-12-20 17:54:13,873 epoch 39 - iter 10/11 - loss 5.31989819 - samples/sec: 307.33\n",
            "2019-12-20 17:54:13,888 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:13,889 EPOCH 39 done: loss 5.3199 - lr 0.0004\n",
            "2019-12-20 17:54:14,103 DEV : loss 6.59447717666626 - score 0.0435\n",
            "2019-12-20 17:54:14,108 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 17:54:14,110 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:14,256 epoch 40 - iter 0/11 - loss 4.08241081 - samples/sec: 221.86\n",
            "2019-12-20 17:54:14,413 epoch 40 - iter 1/11 - loss 5.10457134 - samples/sec: 225.06\n",
            "2019-12-20 17:54:14,557 epoch 40 - iter 2/11 - loss 5.11460559 - samples/sec: 245.58\n",
            "2019-12-20 17:54:14,720 epoch 40 - iter 3/11 - loss 5.53413522 - samples/sec: 215.52\n",
            "2019-12-20 17:54:14,880 epoch 40 - iter 4/11 - loss 5.74616346 - samples/sec: 219.72\n",
            "2019-12-20 17:54:15,024 epoch 40 - iter 5/11 - loss 5.65591884 - samples/sec: 251.07\n",
            "2019-12-20 17:54:15,169 epoch 40 - iter 6/11 - loss 5.29703219 - samples/sec: 246.31\n",
            "2019-12-20 17:54:15,323 epoch 40 - iter 7/11 - loss 5.17430815 - samples/sec: 231.51\n",
            "2019-12-20 17:54:15,467 epoch 40 - iter 8/11 - loss 5.19259270 - samples/sec: 247.56\n",
            "2019-12-20 17:54:15,630 epoch 40 - iter 9/11 - loss 5.62955711 - samples/sec: 217.73\n",
            "2019-12-20 17:54:15,765 epoch 40 - iter 10/11 - loss 5.56541692 - samples/sec: 268.85\n",
            "2019-12-20 17:54:15,782 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:15,783 EPOCH 40 done: loss 5.5654 - lr 0.0004\n",
            "2019-12-20 17:54:15,990 DEV : loss 6.602049350738525 - score 0.0435\n",
            "2019-12-20 17:54:15,996 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 17:54:15,997 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:16,136 epoch 41 - iter 0/11 - loss 1.85149384 - samples/sec: 234.61\n",
            "2019-12-20 17:54:16,277 epoch 41 - iter 1/11 - loss 2.46863306 - samples/sec: 252.97\n",
            "2019-12-20 17:54:16,429 epoch 41 - iter 2/11 - loss 4.23753889 - samples/sec: 232.28\n",
            "2019-12-20 17:54:16,575 epoch 41 - iter 3/11 - loss 3.89959115 - samples/sec: 249.01\n",
            "2019-12-20 17:54:16,756 epoch 41 - iter 4/11 - loss 4.24298930 - samples/sec: 192.83\n",
            "2019-12-20 17:54:16,916 epoch 41 - iter 5/11 - loss 4.47178455 - samples/sec: 225.01\n",
            "2019-12-20 17:54:17,076 epoch 41 - iter 6/11 - loss 4.45149745 - samples/sec: 220.24\n",
            "2019-12-20 17:54:17,227 epoch 41 - iter 7/11 - loss 4.62285265 - samples/sec: 238.63\n",
            "2019-12-20 17:54:17,383 epoch 41 - iter 8/11 - loss 4.82498071 - samples/sec: 228.06\n",
            "2019-12-20 17:54:17,536 epoch 41 - iter 9/11 - loss 5.01697881 - samples/sec: 232.46\n",
            "2019-12-20 17:54:17,680 epoch 41 - iter 10/11 - loss 5.39544467 - samples/sec: 263.07\n",
            "2019-12-20 17:54:17,695 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:17,696 EPOCH 41 done: loss 5.3954 - lr 0.0004\n",
            "2019-12-20 17:54:17,920 DEV : loss 6.613968372344971 - score 0.0435\n",
            "2019-12-20 17:54:17,926 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 17:54:17,928 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:18,075 epoch 42 - iter 0/11 - loss 6.36353493 - samples/sec: 219.77\n",
            "2019-12-20 17:54:18,227 epoch 42 - iter 1/11 - loss 6.45703721 - samples/sec: 233.99\n",
            "2019-12-20 17:54:18,370 epoch 42 - iter 2/11 - loss 5.67093738 - samples/sec: 250.22\n",
            "2019-12-20 17:54:18,521 epoch 42 - iter 3/11 - loss 5.24188316 - samples/sec: 236.62\n",
            "2019-12-20 17:54:18,678 epoch 42 - iter 4/11 - loss 5.08153353 - samples/sec: 226.36\n",
            "2019-12-20 17:54:18,832 epoch 42 - iter 5/11 - loss 5.30893747 - samples/sec: 229.40\n",
            "2019-12-20 17:54:18,991 epoch 42 - iter 6/11 - loss 5.41305903 - samples/sec: 224.53\n",
            "2019-12-20 17:54:19,144 epoch 42 - iter 7/11 - loss 5.43393791 - samples/sec: 232.97\n",
            "2019-12-20 17:54:19,294 epoch 42 - iter 8/11 - loss 5.30740489 - samples/sec: 236.47\n",
            "2019-12-20 17:54:19,447 epoch 42 - iter 9/11 - loss 5.35721617 - samples/sec: 233.67\n",
            "2019-12-20 17:54:19,568 epoch 42 - iter 10/11 - loss 5.40757383 - samples/sec: 299.41\n",
            "2019-12-20 17:54:19,583 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:19,585 EPOCH 42 done: loss 5.4076 - lr 0.0004\n",
            "2019-12-20 17:54:19,793 DEV : loss 6.573455333709717 - score 0.0435\n",
            "Epoch    41: reducing learning rate of group 0 to 1.9531e-04.\n",
            "2019-12-20 17:54:19,800 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 17:54:19,801 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:19,955 epoch 43 - iter 0/11 - loss 5.04172134 - samples/sec: 210.49\n",
            "2019-12-20 17:54:20,113 epoch 43 - iter 1/11 - loss 5.66049838 - samples/sec: 225.67\n",
            "2019-12-20 17:54:20,262 epoch 43 - iter 2/11 - loss 4.38553206 - samples/sec: 239.19\n",
            "2019-12-20 17:54:20,405 epoch 43 - iter 3/11 - loss 4.05793118 - samples/sec: 249.40\n",
            "2019-12-20 17:54:20,554 epoch 43 - iter 4/11 - loss 5.17456646 - samples/sec: 238.08\n",
            "2019-12-20 17:54:20,708 epoch 43 - iter 5/11 - loss 5.23599378 - samples/sec: 231.29\n",
            "2019-12-20 17:54:20,878 epoch 43 - iter 6/11 - loss 5.15890878 - samples/sec: 205.69\n",
            "2019-12-20 17:54:21,039 epoch 43 - iter 7/11 - loss 5.07070357 - samples/sec: 221.33\n",
            "2019-12-20 17:54:21,191 epoch 43 - iter 8/11 - loss 5.61371597 - samples/sec: 233.76\n",
            "2019-12-20 17:54:21,338 epoch 43 - iter 9/11 - loss 5.57429476 - samples/sec: 241.98\n",
            "2019-12-20 17:54:21,453 epoch 43 - iter 10/11 - loss 5.48987666 - samples/sec: 319.32\n",
            "2019-12-20 17:54:21,467 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:21,468 EPOCH 43 done: loss 5.4899 - lr 0.0002\n",
            "2019-12-20 17:54:21,679 DEV : loss 6.569129467010498 - score 0.0435\n",
            "2019-12-20 17:54:21,683 BAD EPOCHS (no improvement): 1\n",
            "2019-12-20 17:54:21,684 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:21,830 epoch 44 - iter 0/11 - loss 6.41677666 - samples/sec: 222.81\n",
            "2019-12-20 17:54:21,993 epoch 44 - iter 1/11 - loss 5.66196156 - samples/sec: 216.21\n",
            "2019-12-20 17:54:22,149 epoch 44 - iter 2/11 - loss 5.25874710 - samples/sec: 231.19\n",
            "2019-12-20 17:54:22,297 epoch 44 - iter 3/11 - loss 5.30364239 - samples/sec: 240.26\n",
            "2019-12-20 17:54:22,450 epoch 44 - iter 4/11 - loss 5.50614939 - samples/sec: 232.48\n",
            "2019-12-20 17:54:22,596 epoch 44 - iter 5/11 - loss 5.22155762 - samples/sec: 243.58\n",
            "2019-12-20 17:54:22,760 epoch 44 - iter 6/11 - loss 5.61172615 - samples/sec: 214.02\n",
            "2019-12-20 17:54:22,929 epoch 44 - iter 7/11 - loss 5.73930240 - samples/sec: 206.61\n",
            "2019-12-20 17:54:23,084 epoch 44 - iter 8/11 - loss 5.52475770 - samples/sec: 228.91\n",
            "2019-12-20 17:54:23,233 epoch 44 - iter 9/11 - loss 5.51897230 - samples/sec: 238.21\n",
            "2019-12-20 17:54:23,362 epoch 44 - iter 10/11 - loss 5.69472443 - samples/sec: 284.65\n",
            "2019-12-20 17:54:23,376 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:23,377 EPOCH 44 done: loss 5.6947 - lr 0.0002\n",
            "2019-12-20 17:54:23,588 DEV : loss 6.576202392578125 - score 0.0435\n",
            "2019-12-20 17:54:23,595 BAD EPOCHS (no improvement): 2\n",
            "2019-12-20 17:54:23,596 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:23,741 epoch 45 - iter 0/11 - loss 3.90004539 - samples/sec: 223.40\n",
            "2019-12-20 17:54:23,883 epoch 45 - iter 1/11 - loss 6.41239548 - samples/sec: 250.91\n",
            "2019-12-20 17:54:24,041 epoch 45 - iter 2/11 - loss 6.17217286 - samples/sec: 221.76\n",
            "2019-12-20 17:54:24,193 epoch 45 - iter 3/11 - loss 6.46422553 - samples/sec: 234.14\n",
            "2019-12-20 17:54:24,360 epoch 45 - iter 4/11 - loss 5.88312573 - samples/sec: 209.63\n",
            "2019-12-20 17:54:24,511 epoch 45 - iter 5/11 - loss 5.71107535 - samples/sec: 234.00\n",
            "2019-12-20 17:54:24,663 epoch 45 - iter 6/11 - loss 5.55517943 - samples/sec: 233.73\n",
            "2019-12-20 17:54:24,805 epoch 45 - iter 7/11 - loss 5.49623027 - samples/sec: 249.94\n",
            "2019-12-20 17:54:24,971 epoch 45 - iter 8/11 - loss 5.29734193 - samples/sec: 210.91\n",
            "2019-12-20 17:54:25,124 epoch 45 - iter 9/11 - loss 5.41668859 - samples/sec: 233.31\n",
            "2019-12-20 17:54:25,248 epoch 45 - iter 10/11 - loss 5.48472825 - samples/sec: 293.01\n",
            "2019-12-20 17:54:25,262 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:25,263 EPOCH 45 done: loss 5.4847 - lr 0.0002\n",
            "2019-12-20 17:54:25,467 DEV : loss 6.57892370223999 - score 0.0435\n",
            "2019-12-20 17:54:25,474 BAD EPOCHS (no improvement): 3\n",
            "2019-12-20 17:54:25,475 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:25,617 epoch 46 - iter 0/11 - loss 5.37662506 - samples/sec: 228.12\n",
            "2019-12-20 17:54:25,770 epoch 46 - iter 1/11 - loss 4.71378565 - samples/sec: 231.22\n",
            "2019-12-20 17:54:25,917 epoch 46 - iter 2/11 - loss 4.82532628 - samples/sec: 240.99\n",
            "2019-12-20 17:54:26,075 epoch 46 - iter 3/11 - loss 5.02577639 - samples/sec: 222.61\n",
            "2019-12-20 17:54:26,232 epoch 46 - iter 4/11 - loss 5.99577694 - samples/sec: 225.73\n",
            "2019-12-20 17:54:26,383 epoch 46 - iter 5/11 - loss 5.38593300 - samples/sec: 234.55\n",
            "2019-12-20 17:54:26,545 epoch 46 - iter 6/11 - loss 5.47299780 - samples/sec: 217.38\n",
            "2019-12-20 17:54:26,689 epoch 46 - iter 7/11 - loss 5.29309112 - samples/sec: 247.20\n",
            "2019-12-20 17:54:26,843 epoch 46 - iter 8/11 - loss 5.37678968 - samples/sec: 228.78\n",
            "2019-12-20 17:54:27,000 epoch 46 - iter 9/11 - loss 5.40944190 - samples/sec: 224.43\n",
            "2019-12-20 17:54:27,137 epoch 46 - iter 10/11 - loss 5.99508723 - samples/sec: 269.78\n",
            "2019-12-20 17:54:27,151 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:27,152 EPOCH 46 done: loss 5.9951 - lr 0.0002\n",
            "2019-12-20 17:54:27,358 DEV : loss 6.610344886779785 - score 0.0435\n",
            "Epoch    45: reducing learning rate of group 0 to 9.7656e-05.\n",
            "2019-12-20 17:54:27,364 BAD EPOCHS (no improvement): 4\n",
            "2019-12-20 17:54:27,365 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:27,367 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:27,368 learning rate too small - quitting training!\n",
            "2019-12-20 17:54:27,369 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:30,619 ----------------------------------------------------------------------------------------------------\n",
            "2019-12-20 17:54:30,623 Testing using best model ...\n",
            "2019-12-20 17:54:30,627 loading file resources/taggers/example/best-model.pt\n",
            "2019-12-20 17:54:31,781 0.25\t0.0312\t0.0555\n",
            "2019-12-20 17:54:31,782 \n",
            "MICRO_AVG: acc 0.0286 - f1-score 0.0555\n",
            "MACRO_AVG: acc 0.0096 - f1-score 0.018183333333333333\n",
            "corporation tp: 0 - fp: 0 - fn: 3 - tn: 0 - precision: 0.0000 - recall: 0.0000 - accuracy: 0.0000 - f1-score: 0.0000\n",
            "creative-work tp: 0 - fp: 0 - fn: 12 - tn: 0 - precision: 0.0000 - recall: 0.0000 - accuracy: 0.0000 - f1-score: 0.0000\n",
            "group      tp: 0 - fp: 0 - fn: 11 - tn: 0 - precision: 0.0000 - recall: 0.0000 - accuracy: 0.0000 - f1-score: 0.0000\n",
            "location   tp: 0 - fp: 0 - fn: 10 - tn: 0 - precision: 0.0000 - recall: 0.0000 - accuracy: 0.0000 - f1-score: 0.0000\n",
            "person     tp: 3 - fp: 9 - fn: 40 - tn: 3 - precision: 0.2500 - recall: 0.0698 - accuracy: 0.0577 - f1-score: 0.1091\n",
            "product    tp: 0 - fp: 0 - fn: 17 - tn: 0 - precision: 0.0000 - recall: 0.0000 - accuracy: 0.0000 - f1-score: 0.0000\n",
            "2019-12-20 17:54:31,788 ----------------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'test_score': 0.0555,\n",
              " 'dev_score_history': [0.0,\n",
              "  0.0,\n",
              "  0.0222,\n",
              "  0.087,\n",
              "  0.0,\n",
              "  0.0,\n",
              "  0.0435,\n",
              "  0.0,\n",
              "  0.0227,\n",
              "  0.198,\n",
              "  0.0861,\n",
              "  0.0225,\n",
              "  0.1781,\n",
              "  0.1601,\n",
              "  0.044,\n",
              "  0.0222,\n",
              "  0.044,\n",
              "  0.0227,\n",
              "  0.1031,\n",
              "  0.044,\n",
              "  0.0435,\n",
              "  0.066,\n",
              "  0.1076,\n",
              "  0.0645,\n",
              "  0.0435,\n",
              "  0.044,\n",
              "  0.0851,\n",
              "  0.0435,\n",
              "  0.0435,\n",
              "  0.0445,\n",
              "  0.0445,\n",
              "  0.0445,\n",
              "  0.0435,\n",
              "  0.0435,\n",
              "  0.0435,\n",
              "  0.0435,\n",
              "  0.0435,\n",
              "  0.0435,\n",
              "  0.0435,\n",
              "  0.0435,\n",
              "  0.0435,\n",
              "  0.0435,\n",
              "  0.0435,\n",
              "  0.0435,\n",
              "  0.0435,\n",
              "  0.0435],\n",
              " 'train_loss_history': [291.8794259158048,\n",
              "  56.981740431352094,\n",
              "  25.556164134632457,\n",
              "  17.408550999381326,\n",
              "  16.602836175398394,\n",
              "  15.876086841930043,\n",
              "  15.072928255254572,\n",
              "  15.277334560047496,\n",
              "  13.274605881084096,\n",
              "  12.563690879128195,\n",
              "  11.062905550003052,\n",
              "  10.731114040721547,\n",
              "  10.586729309775613,\n",
              "  10.247884663668545,\n",
              "  9.8503380688754,\n",
              "  9.109914389523594,\n",
              "  8.579352942380039,\n",
              "  9.065854462710293,\n",
              "  7.417437314987183,\n",
              "  7.798665350133723,\n",
              "  7.025214542042125,\n",
              "  6.697624119845304,\n",
              "  6.616952679374001,\n",
              "  5.789440935308283,\n",
              "  6.469064365733754,\n",
              "  6.22010495445945,\n",
              "  6.207791978662664,\n",
              "  5.7324583313681865,\n",
              "  5.501319235021418,\n",
              "  5.564900875091553,\n",
              "  5.325252597982233,\n",
              "  5.948405666784807,\n",
              "  6.083201776851308,\n",
              "  5.906649394468828,\n",
              "  5.254325693303889,\n",
              "  5.891047651117498,\n",
              "  5.297101844440807,\n",
              "  5.608674959702925,\n",
              "  5.3198981935327705,\n",
              "  5.56541692126881,\n",
              "  5.395444674925371,\n",
              "  5.40757382999767,\n",
              "  5.489876660433683,\n",
              "  5.6947244297374375,\n",
              "  5.4847282496365635,\n",
              "  5.995087233456698],\n",
              " 'dev_loss_history': [tensor(83.9792, device='cuda:0'),\n",
              "  tensor(26.4946, device='cuda:0'),\n",
              "  tensor(16.1604, device='cuda:0'),\n",
              "  tensor(15.1614, device='cuda:0'),\n",
              "  tensor(16.1356, device='cuda:0'),\n",
              "  tensor(10.7362, device='cuda:0'),\n",
              "  tensor(11.0265, device='cuda:0'),\n",
              "  tensor(13.6201, device='cuda:0'),\n",
              "  tensor(12.1526, device='cuda:0'),\n",
              "  tensor(11.5084, device='cuda:0'),\n",
              "  tensor(10.3406, device='cuda:0'),\n",
              "  tensor(10.6811, device='cuda:0'),\n",
              "  tensor(8.4115, device='cuda:0'),\n",
              "  tensor(7.4867, device='cuda:0'),\n",
              "  tensor(9.7206, device='cuda:0'),\n",
              "  tensor(9.6371, device='cuda:0'),\n",
              "  tensor(8.7726, device='cuda:0'),\n",
              "  tensor(8.8765, device='cuda:0'),\n",
              "  tensor(8.3048, device='cuda:0'),\n",
              "  tensor(8.0875, device='cuda:0'),\n",
              "  tensor(7.8611, device='cuda:0'),\n",
              "  tensor(7.9073, device='cuda:0'),\n",
              "  tensor(7.4584, device='cuda:0'),\n",
              "  tensor(7.1407, device='cuda:0'),\n",
              "  tensor(7.4047, device='cuda:0'),\n",
              "  tensor(6.9513, device='cuda:0'),\n",
              "  tensor(7.0518, device='cuda:0'),\n",
              "  tensor(6.9840, device='cuda:0'),\n",
              "  tensor(6.9654, device='cuda:0'),\n",
              "  tensor(6.8977, device='cuda:0'),\n",
              "  tensor(6.9122, device='cuda:0'),\n",
              "  tensor(6.8162, device='cuda:0'),\n",
              "  tensor(6.8075, device='cuda:0'),\n",
              "  tensor(6.6777, device='cuda:0'),\n",
              "  tensor(6.6533, device='cuda:0'),\n",
              "  tensor(6.6880, device='cuda:0'),\n",
              "  tensor(6.7023, device='cuda:0'),\n",
              "  tensor(6.6170, device='cuda:0'),\n",
              "  tensor(6.5945, device='cuda:0'),\n",
              "  tensor(6.6020, device='cuda:0'),\n",
              "  tensor(6.6140, device='cuda:0'),\n",
              "  tensor(6.5735, device='cuda:0'),\n",
              "  tensor(6.5691, device='cuda:0'),\n",
              "  tensor(6.5762, device='cuda:0'),\n",
              "  tensor(6.5789, device='cuda:0'),\n",
              "  tensor(6.6103, device='cuda:0')]}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IKdZ3bPGcImU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}