text2text_demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "text2text demo.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "NzoaY2eWKwFW",
        "colab_type": "code",
        "outputId": "14336296-8c63-4bf9-ef51-30fa8b18022f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 415
        }
      },
      "source": [
        "%%bash\n",
        "nvcc --version\n",
        "nvidia-smi"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "nvcc: NVIDIA (R) Cuda compiler driver\n",
            "Copyright (c) 2005-2018 NVIDIA Corporation\n",
            "Built on Sat_Aug_25_21:08:01_CDT_2018\n",
            "Cuda compilation tools, release 10.0, V10.0.130\n",
            "Sat Apr  4 18:57:25 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 440.64.00    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   40C    P0    27W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R5p-YiGOJRLV",
        "colab_type": "code",
        "outputId": "78c52559-3c5d-4e0f-c240-da880cc57137",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 181
        }
      },
      "source": [
        "pip install text2text"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting text2text\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ee/ac/0d9013d7312cd7e568c4c22cfcc3206c317069b92de804cf738c4260205b/text2text-0.0.6-py3-none-any.whl (64kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 2.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from text2text) (1.18.2)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from text2text) (1.4.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from text2text) (4.38.0)\n",
            "Installing collected packages: text2text\n",
            "Successfully installed text2text-0.0.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "2c7044a5-154e-4fd0-9771-d3da73024814",
        "id": "H-PGKUg2nfrL",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "%%bash\n",
        "export CUDA_HOME=/usr/local/cuda-10.1\n",
        "pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" pytorch-extension"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Created temporary directory: /tmp/pip-ephem-wheel-cache-qwqi1pxx\n",
            "Created temporary directory: /tmp/pip-req-tracker-0j422ky8\n",
            "Created requirements tracker '/tmp/pip-req-tracker-0j422ky8'\n",
            "Created temporary directory: /tmp/pip-install-af4fxbbx\n",
            "1 location(s) to search for versions of pytorch-extension:\n",
            "* https://pypi.org/simple/pytorch-extension/\n",
            "Getting page https://pypi.org/simple/pytorch-extension/\n",
            "Found index url https://pypi.org/simple\n",
            "Starting new HTTPS connection (1): pypi.org:443\n",
            "https://pypi.org:443 \"GET /simple/pytorch-extension/ HTTP/1.1\" 200 411\n",
            "Analyzing links from page https://pypi.org/simple/pytorch-extension/\n",
            "  Found link https://files.pythonhosted.org/packages/fc/82/55f5774e1822dcd92a171389fd9ea9e532c4e44dcc0af00d463ed62537d3/pytorch-extension-0.1.tar.gz#sha256=a0c231ff62097ac9fab0b0fda2ceb4ca03cad7b93d9b399d48c33995faaf946a (from https://pypi.org/simple/pytorch-extension/), version: 0.1\n",
            "  Skipping link: No binaries permitted for pytorch-extension: https://files.pythonhosted.org/packages/1d/74/2e9110532020880d9ba06085c4b9a163fa8d8993d964bf61aeb217b7896b/pytorch_extension-0.1-py3-none-any.whl#sha256=006077f47c166a6b042102c30e71e7b53d5f322f1fce7861a1d7c6859f898594 (from https://pypi.org/simple/pytorch-extension/)\n",
            "Given no hashes to check 1 links for project 'pytorch-extension': discarding no candidates\n",
            "Using version 0.1 (newest of versions: 0.1)\n",
            "Collecting pytorch-extension\n",
            "  Created temporary directory: /tmp/pip-unpack-8yox1s2d\n",
            "  Starting new HTTPS connection (1): files.pythonhosted.org:443\n",
            "  https://files.pythonhosted.org:443 \"GET /packages/fc/82/55f5774e1822dcd92a171389fd9ea9e532c4e44dcc0af00d463ed62537d3/pytorch-extension-0.1.tar.gz HTTP/1.1\" 200 437038\n",
            "  Downloading https://files.pythonhosted.org/packages/fc/82/55f5774e1822dcd92a171389fd9ea9e532c4e44dcc0af00d463ed62537d3/pytorch-extension-0.1.tar.gz (437kB)\n",
            "  Added pytorch-extension from https://files.pythonhosted.org/packages/fc/82/55f5774e1822dcd92a171389fd9ea9e532c4e44dcc0af00d463ed62537d3/pytorch-extension-0.1.tar.gz#sha256=a0c231ff62097ac9fab0b0fda2ceb4ca03cad7b93d9b399d48c33995faaf946a to build tracker '/tmp/pip-req-tracker-0j422ky8'\n",
            "    Running setup.py (path:/tmp/pip-install-af4fxbbx/pytorch-extension/setup.py) egg_info for package pytorch-extension\n",
            "  Source in /tmp/pip-install-af4fxbbx/pytorch-extension has version 0.1, which satisfies requirement pytorch-extension from https://files.pythonhosted.org/packages/fc/82/55f5774e1822dcd92a171389fd9ea9e532c4e44dcc0af00d463ed62537d3/pytorch-extension-0.1.tar.gz#sha256=a0c231ff62097ac9fab0b0fda2ceb4ca03cad7b93d9b399d48c33995faaf946a\n",
            "  Removed pytorch-extension from https://files.pythonhosted.org/packages/fc/82/55f5774e1822dcd92a171389fd9ea9e532c4e44dcc0af00d463ed62537d3/pytorch-extension-0.1.tar.gz#sha256=a0c231ff62097ac9fab0b0fda2ceb4ca03cad7b93d9b399d48c33995faaf946a from build tracker '/tmp/pip-req-tracker-0j422ky8'\n",
            "Skipping wheel build for pytorch-extension, due to binaries being disabled for it.\n",
            "Installing collected packages: pytorch-extension\n",
            "  Created temporary directory: /tmp/pip-record-pvx14zvw\n",
            "    Running setup.py install for pytorch-extension: started\n",
            "    Running setup.py install for pytorch-extension: finished with status 'done'\n",
            "  Removing source in /tmp/pip-install-af4fxbbx/pytorch-extension\n",
            "Successfully installed pytorch-extension-0.1\n",
            "Cleaning up...\n",
            "Removed build tracker '/tmp/pip-req-tracker-0j422ky8'\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/pip/_internal/commands/install.py:283: UserWarning: Disabling all use of wheels due to the use of --build-options / --global-options / --install-options.\n",
            "  cmdoptions.check_install_build_global(options)\n",
            "    Running command python setup.py egg_info\n",
            "    torch.__version__  =  1.4.0\n",
            "    running egg_info\n",
            "    creating /tmp/pip-install-af4fxbbx/pytorch-extension/pip-egg-info/pytorch_extension.egg-info\n",
            "    writing /tmp/pip-install-af4fxbbx/pytorch-extension/pip-egg-info/pytorch_extension.egg-info/PKG-INFO\n",
            "    writing dependency_links to /tmp/pip-install-af4fxbbx/pytorch-extension/pip-egg-info/pytorch_extension.egg-info/dependency_links.txt\n",
            "    writing top-level names to /tmp/pip-install-af4fxbbx/pytorch-extension/pip-egg-info/pytorch_extension.egg-info/top_level.txt\n",
            "    writing manifest file '/tmp/pip-install-af4fxbbx/pytorch-extension/pip-egg-info/pytorch_extension.egg-info/SOURCES.txt'\n",
            "    reading manifest file '/tmp/pip-install-af4fxbbx/pytorch-extension/pip-egg-info/pytorch_extension.egg-info/SOURCES.txt'\n",
            "    writing manifest file '/tmp/pip-install-af4fxbbx/pytorch-extension/pip-egg-info/pytorch_extension.egg-info/SOURCES.txt'\n",
            "    /tmp/pip-install-af4fxbbx/pytorch-extension/setup.py:46: UserWarning: Option --pyprof not specified. Not installing PyProf dependencies!\n",
            "      warnings.warn(\"Option --pyprof not specified. Not installing PyProf dependencies!\")\n",
            "    Running command /usr/bin/python3 -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '\"'\"'/tmp/pip-install-af4fxbbx/pytorch-extension/setup.py'\"'\"'; __file__='\"'\"'/tmp/pip-install-af4fxbbx/pytorch-extension/setup.py'\"'\"';f=getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__);code=f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' --cpp_ext --cuda_ext install --record /tmp/pip-record-pvx14zvw/install-record.txt --single-version-externally-managed --compile\n",
            "    torch.__version__  =  1.4.0\n",
            "    /tmp/pip-install-af4fxbbx/pytorch-extension/setup.py:46: UserWarning: Option --pyprof not specified. Not installing PyProf dependencies!\n",
            "      warnings.warn(\"Option --pyprof not specified. Not installing PyProf dependencies!\")\n",
            "\n",
            "    Compiling cuda extensions with\n",
            "    nvcc: NVIDIA (R) Cuda compiler driver\n",
            "    Copyright (c) 2005-2019 NVIDIA Corporation\n",
            "    Built on Sun_Jul_28_19:07:16_PDT_2019\n",
            "    Cuda compilation tools, release 10.1, V10.1.243\n",
            "    from /usr/local/cuda-10.1/bin\n",
            "\n",
            "    running install\n",
            "    running build\n",
            "    running build_py\n",
            "    creating build\n",
            "    creating build/lib.linux-x86_64-3.6\n",
            "    creating build/lib.linux-x86_64-3.6/apex\n",
            "    copying apex/__init__.py -> build/lib.linux-x86_64-3.6/apex\n",
            "    creating build/lib.linux-x86_64-3.6/apex/normalization\n",
            "    copying apex/normalization/fused_layer_norm.py -> build/lib.linux-x86_64-3.6/apex/normalization\n",
            "    copying apex/normalization/__init__.py -> build/lib.linux-x86_64-3.6/apex/normalization\n",
            "    creating build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/handle.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/utils.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/_amp_state.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/_process_optimizer.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/amp.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/opt.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/wrap.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/__version__.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/_initialize.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/rnn_compat.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/__init__.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/compat.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/frontend.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    copying apex/amp/scaler.py -> build/lib.linux-x86_64-3.6/apex/amp\n",
            "    creating build/lib.linux-x86_64-3.6/apex/optimizers\n",
            "    copying apex/optimizers/fused_lamb.py -> build/lib.linux-x86_64-3.6/apex/optimizers\n",
            "    copying apex/optimizers/fused_sgd.py -> build/lib.linux-x86_64-3.6/apex/optimizers\n",
            "    copying apex/optimizers/fused_adam.py -> build/lib.linux-x86_64-3.6/apex/optimizers\n",
            "    copying apex/optimizers/fused_novograd.py -> build/lib.linux-x86_64-3.6/apex/optimizers\n",
            "    copying apex/optimizers/__init__.py -> build/lib.linux-x86_64-3.6/apex/optimizers\n",
            "    creating build/lib.linux-x86_64-3.6/apex/contrib\n",
            "    copying apex/contrib/__init__.py -> build/lib.linux-x86_64-3.6/apex/contrib\n",
            "    creating build/lib.linux-x86_64-3.6/apex/RNN\n",
            "    copying apex/RNN/cells.py -> build/lib.linux-x86_64-3.6/apex/RNN\n",
            "    copying apex/RNN/RNNBackend.py -> build/lib.linux-x86_64-3.6/apex/RNN\n",
            "    copying apex/RNN/models.py -> build/lib.linux-x86_64-3.6/apex/RNN\n",
            "    copying apex/RNN/__init__.py -> build/lib.linux-x86_64-3.6/apex/RNN\n",
            "    creating build/lib.linux-x86_64-3.6/apex/pyprof\n",
            "    copying apex/pyprof/__init__.py -> build/lib.linux-x86_64-3.6/apex/pyprof\n",
            "    creating build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/multiproc.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/distributed.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/optimized_sync_batchnorm.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/optimized_sync_batchnorm_kernel.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/LARC.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/sync_batchnorm.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/__init__.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    copying apex/parallel/sync_batchnorm_kernel.py -> build/lib.linux-x86_64-3.6/apex/parallel\n",
            "    creating build/lib.linux-x86_64-3.6/apex/reparameterization\n",
            "    copying apex/reparameterization/weight_norm.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\n",
            "    copying apex/reparameterization/__init__.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\n",
            "    copying apex/reparameterization/reparameterization.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\n",
            "    creating build/lib.linux-x86_64-3.6/apex/fp16_utils\n",
            "    copying apex/fp16_utils/fp16util.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\n",
            "    copying apex/fp16_utils/fp16_optimizer.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\n",
            "    copying apex/fp16_utils/loss_scaler.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\n",
            "    copying apex/fp16_utils/__init__.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\n",
            "    creating build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\n",
            "    copying apex/multi_tensor_apply/multi_tensor_apply.py -> build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\n",
            "    copying apex/multi_tensor_apply/__init__.py -> build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\n",
            "    creating build/lib.linux-x86_64-3.6/apex/amp/lists\n",
            "    copying apex/amp/lists/tensor_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\n",
            "    copying apex/amp/lists/torch_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\n",
            "    copying apex/amp/lists/functional_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\n",
            "    copying apex/amp/lists/__init__.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\n",
            "    creating build/lib.linux-x86_64-3.6/apex/contrib/groupbn\n",
            "    copying apex/contrib/groupbn/batch_norm.py -> build/lib.linux-x86_64-3.6/apex/contrib/groupbn\n",
            "    copying apex/contrib/groupbn/__init__.py -> build/lib.linux-x86_64-3.6/apex/contrib/groupbn\n",
            "    creating build/lib.linux-x86_64-3.6/apex/contrib/optimizers\n",
            "    copying apex/contrib/optimizers/fused_sgd.py -> build/lib.linux-x86_64-3.6/apex/contrib/optimizers\n",
            "    copying apex/contrib/optimizers/fused_adam.py -> build/lib.linux-x86_64-3.6/apex/contrib/optimizers\n",
            "    copying apex/contrib/optimizers/fp16_optimizer.py -> build/lib.linux-x86_64-3.6/apex/contrib/optimizers\n",
            "    copying apex/contrib/optimizers/__init__.py -> build/lib.linux-x86_64-3.6/apex/contrib/optimizers\n",
            "    creating build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn\n",
            "    copying apex/contrib/multihead_attn/self_multihead_attn.py -> build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn\n",
            "    copying apex/contrib/multihead_attn/fast_encdec_multihead_attn_func.py -> build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn\n",
            "    copying apex/contrib/multihead_attn/fast_encdec_multihead_attn_norm_add_func.py -> build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn\n",
            "    copying apex/contrib/multihead_attn/encdec_multihead_attn_func.py -> build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn\n",
            "    copying apex/contrib/multihead_attn/fast_self_multihead_attn_func.py -> build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn\n",
            "    copying apex/contrib/multihead_attn/fast_self_multihead_attn_norm_add_func.py -> build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn\n",
            "    copying apex/contrib/multihead_attn/encdec_multihead_attn.py -> build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn\n",
            "    copying apex/contrib/multihead_attn/self_multihead_attn_func.py -> build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn\n",
            "    copying apex/contrib/multihead_attn/__init__.py -> build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn\n",
            "    creating build/lib.linux-x86_64-3.6/apex/contrib/xentropy\n",
            "    copying apex/contrib/xentropy/softmax_xentropy.py -> build/lib.linux-x86_64-3.6/apex/contrib/xentropy\n",
            "    copying apex/contrib/xentropy/__init__.py -> build/lib.linux-x86_64-3.6/apex/contrib/xentropy\n",
            "    creating build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/blas.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/output.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/data.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/pooling.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/randomSample.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/conv.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/softmax.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/reduction.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/embedding.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/misc.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/recurrentCell.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/loss.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/pointwise.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/index_slice_join_mutate.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/dropout.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/usage.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/normalization.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/prof.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/base.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/optim.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/convert.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/__main__.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/linear.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/activation.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/__init__.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    copying apex/pyprof/prof/utility.py -> build/lib.linux-x86_64-3.6/apex/pyprof/prof\n",
            "    creating build/lib.linux-x86_64-3.6/apex/pyprof/nvtx\n",
            "    copying apex/pyprof/nvtx/nvmarker.py -> build/lib.linux-x86_64-3.6/apex/pyprof/nvtx\n",
            "    copying apex/pyprof/nvtx/__init__.py -> build/lib.linux-x86_64-3.6/apex/pyprof/nvtx\n",
            "    creating build/lib.linux-x86_64-3.6/apex/pyprof/parse\n",
            "    copying apex/pyprof/parse/nvvp.py -> build/lib.linux-x86_64-3.6/apex/pyprof/parse\n",
            "    copying apex/pyprof/parse/kernel.py -> build/lib.linux-x86_64-3.6/apex/pyprof/parse\n",
            "    copying apex/pyprof/parse/db.py -> build/lib.linux-x86_64-3.6/apex/pyprof/parse\n",
            "    copying apex/pyprof/parse/__main__.py -> build/lib.linux-x86_64-3.6/apex/pyprof/parse\n",
            "    copying apex/pyprof/parse/__init__.py -> build/lib.linux-x86_64-3.6/apex/pyprof/parse\n",
            "    copying apex/pyprof/parse/parse.py -> build/lib.linux-x86_64-3.6/apex/pyprof/parse\n",
            "    running build_ext\n",
            "    building 'apex_C' extension\n",
            "    creating build/temp.linux-x86_64-3.6\n",
            "    creating build/temp.linux-x86_64-3.6/csrc\n",
            "    x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/include/python3.6m -c csrc/flatten_unflatten.cpp -o build/temp.linux-x86_64-3.6/csrc/flatten_unflatten.o -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=apex_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/csrc/flatten_unflatten.o -o build/lib.linux-x86_64-3.6/apex_C.cpython-36m-x86_64-linux-gnu.so\n",
            "    building 'amp_C' extension\n",
            "    x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda-10.1/include -I/usr/include/python3.6m -c csrc/amp_C_frontend.cpp -o build/temp.linux-x86_64-3.6/csrc/amp_C_frontend.o -O3 -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    /usr/local/cuda-10.1/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda-10.1/include -I/usr/include/python3.6m -c csrc/multi_tensor_sgd_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_sgd_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_60,code=sm_60 -std=c++11\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/cuda-10.1/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda-10.1/include -I/usr/include/python3.6m -c csrc/multi_tensor_scale_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_scale_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_60,code=sm_60 -std=c++11\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/cuda-10.1/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda-10.1/include -I/usr/include/python3.6m -c csrc/multi_tensor_axpby_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_axpby_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_60,code=sm_60 -std=c++11\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/cuda-10.1/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda-10.1/include -I/usr/include/python3.6m -c csrc/multi_tensor_l2norm_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_l2norm_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_60,code=sm_60 -std=c++11\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/cuda-10.1/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda-10.1/include -I/usr/include/python3.6m -c csrc/multi_tensor_lamb_stage_1.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_lamb_stage_1.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_60,code=sm_60 -std=c++11\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/cuda-10.1/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda-10.1/include -I/usr/include/python3.6m -c csrc/multi_tensor_lamb_stage_2.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_lamb_stage_2.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_60,code=sm_60 -std=c++11\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/cuda-10.1/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda-10.1/include -I/usr/include/python3.6m -c csrc/multi_tensor_adam.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_adam.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_60,code=sm_60 -std=c++11\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/cuda-10.1/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda-10.1/include -I/usr/include/python3.6m -c csrc/multi_tensor_novograd.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_novograd.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_60,code=sm_60 -std=c++11\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/cuda-10.1/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda-10.1/include -I/usr/include/python3.6m -c csrc/multi_tensor_lamb.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_lamb.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_60,code=sm_60 -std=c++11\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/csrc/amp_C_frontend.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_sgd_kernel.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_scale_kernel.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_axpby_kernel.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_l2norm_kernel.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_lamb_stage_1.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_lamb_stage_2.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_adam.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_novograd.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_lamb.o -L/usr/local/cuda-10.1/lib64 -lcudart -o build/lib.linux-x86_64-3.6/amp_C.cpython-36m-x86_64-linux-gnu.so\n",
            "    building 'syncbn' extension\n",
            "    x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda-10.1/include -I/usr/include/python3.6m -c csrc/syncbn.cpp -o build/temp.linux-x86_64-3.6/csrc/syncbn.o -O3 -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=syncbn -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    /usr/local/cuda-10.1/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda-10.1/include -I/usr/include/python3.6m -c csrc/welford.cu -o build/temp.linux-x86_64-3.6/csrc/welford.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -O3 -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=syncbn -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_60,code=sm_60 -std=c++11\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/csrc/syncbn.o build/temp.linux-x86_64-3.6/csrc/welford.o -L/usr/local/cuda-10.1/lib64 -lcudart -o build/lib.linux-x86_64-3.6/syncbn.cpython-36m-x86_64-linux-gnu.so\n",
            "    building 'fused_layer_norm_cuda' extension\n",
            "    x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda-10.1/include -I/usr/include/python3.6m -c csrc/layer_norm_cuda.cpp -o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda.o -O3 -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_layer_norm_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
            "    /usr/local/cuda-10.1/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda-10.1/include -I/usr/include/python3.6m -c csrc/layer_norm_cuda_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -maxrregcount=50 -O3 --use_fast_math -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_layer_norm_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_60,code=sm_60 -std=c++11\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    /usr/local/lib/python3.6/dist-packages/torch/include/c10/core/TensorTypeSet.h(44): warning: integer conversion resulted in a change of sign\n",
            "\n",
            "    x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda.o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda_kernel.o -L/usr/local/cuda-10.1/lib64 -lcudart -o build/lib.linux-x86_64-3.6/fused_layer_norm_cuda.cpython-36m-x86_64-linux-gnu.so\n",
            "    running install_lib\n",
            "    copying build/lib.linux-x86_64-3.6/syncbn.cpython-36m-x86_64-linux-gnu.so -> /usr/local/lib/python3.6/dist-packages\n",
            "    copying build/lib.linux-x86_64-3.6/apex_C.cpython-36m-x86_64-linux-gnu.so -> /usr/local/lib/python3.6/dist-packages\n",
            "    copying build/lib.linux-x86_64-3.6/fused_layer_norm_cuda.cpython-36m-x86_64-linux-gnu.so -> /usr/local/lib/python3.6/dist-packages\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/normalization\n",
            "    copying build/lib.linux-x86_64-3.6/apex/normalization/fused_layer_norm.py -> /usr/local/lib/python3.6/dist-packages/apex/normalization\n",
            "    copying build/lib.linux-x86_64-3.6/apex/normalization/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/normalization\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/handle.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/utils.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/_amp_state.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/amp/lists\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/tensor_overrides.py -> /usr/local/lib/python3.6/dist-packages/apex/amp/lists\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/torch_overrides.py -> /usr/local/lib/python3.6/dist-packages/apex/amp/lists\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/functional_overrides.py -> /usr/local/lib/python3.6/dist-packages/apex/amp/lists\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/amp/lists\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/_process_optimizer.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/amp.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/opt.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/wrap.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/__version__.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/_initialize.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/rnn_compat.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/compat.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/frontend.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    copying build/lib.linux-x86_64-3.6/apex/amp/scaler.py -> /usr/local/lib/python3.6/dist-packages/apex/amp\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/optimizers\n",
            "    copying build/lib.linux-x86_64-3.6/apex/optimizers/fused_lamb.py -> /usr/local/lib/python3.6/dist-packages/apex/optimizers\n",
            "    copying build/lib.linux-x86_64-3.6/apex/optimizers/fused_sgd.py -> /usr/local/lib/python3.6/dist-packages/apex/optimizers\n",
            "    copying build/lib.linux-x86_64-3.6/apex/optimizers/fused_adam.py -> /usr/local/lib/python3.6/dist-packages/apex/optimizers\n",
            "    copying build/lib.linux-x86_64-3.6/apex/optimizers/fused_novograd.py -> /usr/local/lib/python3.6/dist-packages/apex/optimizers\n",
            "    copying build/lib.linux-x86_64-3.6/apex/optimizers/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/optimizers\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/contrib\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/contrib/groupbn\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/groupbn/batch_norm.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/groupbn\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/groupbn/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/groupbn\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/contrib/optimizers\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/optimizers/fused_sgd.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/optimizers\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/optimizers/fused_adam.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/optimizers\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/optimizers/fp16_optimizer.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/optimizers\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/optimizers/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/optimizers\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn/self_multihead_attn.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn/fast_encdec_multihead_attn_func.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn/fast_encdec_multihead_attn_norm_add_func.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn/encdec_multihead_attn_func.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn/fast_self_multihead_attn_func.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn/fast_self_multihead_attn_norm_add_func.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn/encdec_multihead_attn.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn/self_multihead_attn_func.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/multihead_attn/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/contrib/xentropy\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/xentropy/softmax_xentropy.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/xentropy\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/xentropy/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib/xentropy\n",
            "    copying build/lib.linux-x86_64-3.6/apex/contrib/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/contrib\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/RNN\n",
            "    copying build/lib.linux-x86_64-3.6/apex/RNN/cells.py -> /usr/local/lib/python3.6/dist-packages/apex/RNN\n",
            "    copying build/lib.linux-x86_64-3.6/apex/RNN/RNNBackend.py -> /usr/local/lib/python3.6/dist-packages/apex/RNN\n",
            "    copying build/lib.linux-x86_64-3.6/apex/RNN/models.py -> /usr/local/lib/python3.6/dist-packages/apex/RNN\n",
            "    copying build/lib.linux-x86_64-3.6/apex/RNN/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/RNN\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/pyprof\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/blas.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/output.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/data.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/pooling.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/randomSample.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/conv.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/softmax.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/reduction.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/embedding.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/misc.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/recurrentCell.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/loss.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/pointwise.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/index_slice_join_mutate.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/dropout.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/usage.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/normalization.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/prof.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/base.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/optim.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/convert.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/__main__.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/linear.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/activation.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/prof/utility.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/pyprof/nvtx\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/nvtx/nvmarker.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/nvtx\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/nvtx/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/nvtx\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/parse/nvvp.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/parse/kernel.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/parse/db.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/parse/__main__.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/parse/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse\n",
            "    copying build/lib.linux-x86_64-3.6/apex/pyprof/parse/parse.py -> /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/multiproc.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/distributed.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/optimized_sync_batchnorm.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/optimized_sync_batchnorm_kernel.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/LARC.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/sync_batchnorm.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    copying build/lib.linux-x86_64-3.6/apex/parallel/sync_batchnorm_kernel.py -> /usr/local/lib/python3.6/dist-packages/apex/parallel\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/reparameterization\n",
            "    copying build/lib.linux-x86_64-3.6/apex/reparameterization/weight_norm.py -> /usr/local/lib/python3.6/dist-packages/apex/reparameterization\n",
            "    copying build/lib.linux-x86_64-3.6/apex/reparameterization/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/reparameterization\n",
            "    copying build/lib.linux-x86_64-3.6/apex/reparameterization/reparameterization.py -> /usr/local/lib/python3.6/dist-packages/apex/reparameterization\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/fp16_utils\n",
            "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/fp16util.py -> /usr/local/lib/python3.6/dist-packages/apex/fp16_utils\n",
            "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/fp16_optimizer.py -> /usr/local/lib/python3.6/dist-packages/apex/fp16_utils\n",
            "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/loss_scaler.py -> /usr/local/lib/python3.6/dist-packages/apex/fp16_utils\n",
            "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/fp16_utils\n",
            "    creating /usr/local/lib/python3.6/dist-packages/apex/multi_tensor_apply\n",
            "    copying build/lib.linux-x86_64-3.6/apex/multi_tensor_apply/multi_tensor_apply.py -> /usr/local/lib/python3.6/dist-packages/apex/multi_tensor_apply\n",
            "    copying build/lib.linux-x86_64-3.6/apex/multi_tensor_apply/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex/multi_tensor_apply\n",
            "    copying build/lib.linux-x86_64-3.6/apex/__init__.py -> /usr/local/lib/python3.6/dist-packages/apex\n",
            "    copying build/lib.linux-x86_64-3.6/amp_C.cpython-36m-x86_64-linux-gnu.so -> /usr/local/lib/python3.6/dist-packages\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/normalization/fused_layer_norm.py to fused_layer_norm.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/normalization/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/handle.py to handle.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/utils.py to utils.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/_amp_state.py to _amp_state.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/lists/tensor_overrides.py to tensor_overrides.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/lists/torch_overrides.py to torch_overrides.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/lists/functional_overrides.py to functional_overrides.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/lists/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/_process_optimizer.py to _process_optimizer.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/amp.py to amp.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/opt.py to opt.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/wrap.py to wrap.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/__version__.py to __version__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/_initialize.py to _initialize.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/rnn_compat.py to rnn_compat.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/compat.py to compat.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/frontend.py to frontend.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/amp/scaler.py to scaler.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/optimizers/fused_lamb.py to fused_lamb.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/optimizers/fused_sgd.py to fused_sgd.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/optimizers/fused_adam.py to fused_adam.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/optimizers/fused_novograd.py to fused_novograd.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/optimizers/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/groupbn/batch_norm.py to batch_norm.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/groupbn/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/optimizers/fused_sgd.py to fused_sgd.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/optimizers/fused_adam.py to fused_adam.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/optimizers/fp16_optimizer.py to fp16_optimizer.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/optimizers/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn/self_multihead_attn.py to self_multihead_attn.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn/fast_encdec_multihead_attn_func.py to fast_encdec_multihead_attn_func.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn/fast_encdec_multihead_attn_norm_add_func.py to fast_encdec_multihead_attn_norm_add_func.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn/encdec_multihead_attn_func.py to encdec_multihead_attn_func.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn/fast_self_multihead_attn_func.py to fast_self_multihead_attn_func.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn/fast_self_multihead_attn_norm_add_func.py to fast_self_multihead_attn_norm_add_func.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn/encdec_multihead_attn.py to encdec_multihead_attn.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn/self_multihead_attn_func.py to self_multihead_attn_func.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/multihead_attn/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/xentropy/softmax_xentropy.py to softmax_xentropy.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/xentropy/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/contrib/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/RNN/cells.py to cells.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/RNN/RNNBackend.py to RNNBackend.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/RNN/models.py to models.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/RNN/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/blas.py to blas.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/output.py to output.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/data.py to data.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/pooling.py to pooling.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/randomSample.py to randomSample.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/conv.py to conv.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/softmax.py to softmax.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/reduction.py to reduction.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/embedding.py to embedding.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/misc.py to misc.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/recurrentCell.py to recurrentCell.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/loss.py to loss.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/pointwise.py to pointwise.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/index_slice_join_mutate.py to index_slice_join_mutate.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/dropout.py to dropout.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/usage.py to usage.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/normalization.py to normalization.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/prof.py to prof.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/base.py to base.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/optim.py to optim.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/convert.py to convert.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/__main__.py to __main__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/linear.py to linear.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/activation.py to activation.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/prof/utility.py to utility.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/nvtx/nvmarker.py to nvmarker.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/nvtx/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse/nvvp.py to nvvp.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse/kernel.py to kernel.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse/db.py to db.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse/__main__.py to __main__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/pyprof/parse/parse.py to parse.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/multiproc.py to multiproc.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/distributed.py to distributed.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/optimized_sync_batchnorm.py to optimized_sync_batchnorm.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/optimized_sync_batchnorm_kernel.py to optimized_sync_batchnorm_kernel.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/LARC.py to LARC.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/sync_batchnorm.py to sync_batchnorm.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/parallel/sync_batchnorm_kernel.py to sync_batchnorm_kernel.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/reparameterization/weight_norm.py to weight_norm.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/reparameterization/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/reparameterization/reparameterization.py to reparameterization.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/fp16_utils/fp16util.py to fp16util.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/fp16_utils/fp16_optimizer.py to fp16_optimizer.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/fp16_utils/loss_scaler.py to loss_scaler.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/fp16_utils/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/multi_tensor_apply/multi_tensor_apply.py to multi_tensor_apply.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/multi_tensor_apply/__init__.py to __init__.cpython-36.pyc\n",
            "    byte-compiling /usr/local/lib/python3.6/dist-packages/apex/__init__.py to __init__.cpython-36.pyc\n",
            "    running install_egg_info\n",
            "    running egg_info\n",
            "    writing pytorch_extension.egg-info/PKG-INFO\n",
            "    writing dependency_links to pytorch_extension.egg-info/dependency_links.txt\n",
            "    writing top-level names to pytorch_extension.egg-info/top_level.txt\n",
            "    reading manifest file 'pytorch_extension.egg-info/SOURCES.txt'\n",
            "    writing manifest file 'pytorch_extension.egg-info/SOURCES.txt'\n",
            "    Copying pytorch_extension.egg-info to /usr/local/lib/python3.6/dist-packages/pytorch_extension-0.1-py3.6.egg-info\n",
            "    running install_scripts\n",
            "    writing list of installed files to '/tmp/pip-record-pvx14zvw/install-record.txt'\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dF_NSzaKPjs2",
        "colab_type": "code",
        "outputId": "88770b47-bca5-4ce0-cdd7-d21f2668d2fb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 379
        }
      },
      "source": [
        "from text2text.text_generator import TextGenerator\n",
        "qg = TextGenerator(output_type=\"question\")\n",
        "\n",
        "notre_dame_str = \"As at most other universities, Notre Dame's students run a number of news media outlets. The nine student - run outlets include three newspapers, both a radio and television station, and several magazines and journals. Begun as a one - page journal in September 1876, the Scholastic magazine is issued twice monthly and claims to be the oldest continuous collegiate publication in the United States. The other magazine, The Juggler, is released twice a year and focuses on student literature and artwork. The Dome yearbook is published annually. The newspapers have varying publication interests, with The Observer published daily and mainly reporting university and other news, and staffed by students from both Notre Dame and Saint Mary's College. Unlike Scholastic and The Dome, The Observer is an independent publication and does not have a faculty advisor or any editorial oversight from the University. In 1987, when some students believed that The Observer began to show a conservative bias, a liberal newspaper, Common Sense was published. Likewise, in 2003, when other students believed that the paper showed a liberal bias, the conservative paper Irish Rover went into production. Neither paper is published as often as The Observer; however, all three are distributed to all students. Finally, in Spring 2008 an undergraduate journal for political science research, Beyond Politics, made its debut.\"\n",
        "\n",
        "bacteria_str = \"Bacteria are a type of biological cell. They constitute a large domain of prokaryotic microorganisms. Typically a few micrometres in length, bacteria have a number of shapes, ranging from spheres to rods and spirals. Bacteria were among the first life forms to appear on Earth, and are present in most of its habitats.\"\n",
        "\n",
        "bio_str = \"Biology is the science that studies life. What exactly is life? This may sound like a silly question with an obvious answer, but it is not easy to define life. For example, a branch of biology called virology studies viruses, which exhibit some of the characteristics of living entities but lack others. It turns out that although viruses can attack living organisms, cause diseases, and even reproduce, they do not meet the criteria that biologists use to define life.\"\n",
        "\n",
        "qg.predict([\n",
        "            bio_str,\n",
        "            bio_str,\n",
        "            bio_str,\n",
        "            bio_str,\n",
        "            bio_str,\n",
        "            \"I will go to school today to take my math exam.\",\n",
        "            \"I will go to school today to take my math exam.\",\n",
        "            \"Tomorrow is my cousin's birthday. He will turn 24 years old.\",\n",
        "            notre_dame_str,\n",
        "            bacteria_str,\n",
        "            bacteria_str,\n",
        "            bacteria_str,\n",
        "            \"I will go to school today to take my math exam. [SEP] school\",\n",
        "            \"I will go to school today to take my math exam. [SEP] exam\",\n",
        "            \"I will go to school today to take my math exam. [SEP] math\",\n",
        "          ])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 213450/213450 [00:00<00:00, 5157139.09B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "***** Recover model: %s ***** qg_model.bin\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 1242874899/1242874899 [00:17<00:00, 69480397.84B/s]\n",
            "100%|██████████| 1/1 [00:04<00:00,  4.64s/it]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('What is biology the science that studies?', 'life'),\n",
              " ('What is the study of life?', 'studies'),\n",
              " ('What would you find the question \" life \"?', 'sound'),\n",
              " ('What can viruses do to living organisms?', 'attack'),\n",
              " ('What is the study of life?', 'studies'),\n",
              " ('Where will I go to to take my math exam?', 'school'),\n",
              " ('Where will I go to to take my math exam?', 'school'),\n",
              " (\"What will my cousin's birthday?\", 'turn'),\n",
              " ('What type of oversight does The Observer not have?', 'editorial'),\n",
              " ('What shape can bacteria be found in?', 'rods'),\n",
              " ('What is the typical length of bacteria?', 'micrometres'),\n",
              " ('What is the typical length of bacteria?', 'micrometres'),\n",
              " ('Where will I go to to take my math exam?', 'school'),\n",
              " ('What will I take after school?', 'exam'),\n",
              " ('What exam will I take?', 'math')]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JRBEPBFBvBqe",
        "colab_type": "code",
        "outputId": "6d9d0201-df9d-420c-f53e-375f8bbee2f8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145
        }
      },
      "source": [
        "sg = TextGenerator(output_type=\"summary\")\n",
        "sg.predict([notre_dame_str, bacteria_str, bio_str])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "enable fp16 with amp\n",
            "***** Recover model: %s ***** cnndm_model.bin\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 1/1 [00:13<00:00, 13.28s/it]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[\"Notre Dame's students run nine student - run outlets . [X_SEP] Scholastic magazine claims to be the oldest continuous collegiate publication in the United States . [X_SEP] The Observer is an independent publication .\",\n",
              " 'Bacteria were among the first life forms to appear on Earth .',\n",
              " 'biology is the science that studies life .']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    }
  ]
}