indictrans_fairseq_inference.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AI4Bharat/indicTrans/blob/main/indictrans_fairseq_inference.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P0uptOB6U7GW",
        "outputId": "988c867e-76ee-4a54-a232-e69abbc5c3db"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/testing\n"
          ]
        }
      ],
      "source": [
        "# create a seperate folder to store everything\n",
        "!mkdir testing\n",
        "%cd testing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kQFRiLtSalzt",
        "outputId": "03070c7c-8299-46bf-de56-df09c3213a3f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'indicTrans'...\n",
            "remote: Enumerating objects: 398, done.\u001b[K\n",
            "remote: Counting objects: 100% (398/398), done.\u001b[K\n",
            "remote: Compressing objects: 100% (267/267), done.\u001b[K\n",
            "remote: Total 398 (delta 231), reused 251 (delta 126), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (398/398), 1.41 MiB | 6.82 MiB/s, done.\n",
            "Resolving deltas: 100% (231/231), done.\n",
            "/content/testing/indicTrans\n",
            "Cloning into 'indic_nlp_library'...\n",
            "remote: Enumerating objects: 1325, done.\u001b[K\n",
            "remote: Counting objects: 100% (147/147), done.\u001b[K\n",
            "remote: Compressing objects: 100% (103/103), done.\u001b[K\n",
            "remote: Total 1325 (delta 84), reused 89 (delta 41), pack-reused 1178\u001b[K\n",
            "Receiving objects: 100% (1325/1325), 9.57 MiB | 7.40 MiB/s, done.\n",
            "Resolving deltas: 100% (688/688), done.\n",
            "Cloning into 'indic_nlp_resources'...\n",
            "remote: Enumerating objects: 133, done.\u001b[K\n",
            "remote: Counting objects: 100% (7/7), done.\u001b[K\n",
            "remote: Compressing objects: 100% (7/7), done.\u001b[K\n",
            "remote: Total 133 (delta 0), reused 2 (delta 0), pack-reused 126\u001b[K\n",
            "Receiving objects: 100% (133/133), 149.77 MiB | 23.46 MiB/s, done.\n",
            "Resolving deltas: 100% (51/51), done.\n",
            "Cloning into 'subword-nmt'...\n",
            "remote: Enumerating objects: 580, done.\u001b[K\n",
            "remote: Counting objects: 100% (4/4), done.\u001b[K\n",
            "remote: Compressing objects: 100% (4/4), done.\u001b[K\n",
            "remote: Total 580 (delta 0), reused 0 (delta 0), pack-reused 576\u001b[K\n",
            "Receiving objects: 100% (580/580), 237.41 KiB | 1.57 MiB/s, done.\n",
            "Resolving deltas: 100% (349/349), done.\n",
            "/content/testing\n"
          ]
        }
      ],
      "source": [
        "# clone the repo for running evaluation\n",
        "!git clone https://github.com/AI4Bharat/indicTrans.git\n",
        "%cd indicTrans\n",
        "# clone requirements repositories\n",
        "!git clone https://github.com/anoopkunchukuttan/indic_nlp_library.git\n",
        "!git clone https://github.com/anoopkunchukuttan/indic_nlp_resources.git\n",
        "!git clone https://github.com/rsennrich/subword-nmt.git\n",
        "%cd .."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FHUQGCACVvVf",
        "outputId": "67c7c3a0-f8bf-46a2-8214-e36556df989b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 3.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (1.1.5)\n",
            "Collecting mock\n",
            "  Downloading https://files.pythonhosted.org/packages/5c/03/b7e605db4a57c0f6fba744b11ef3ddf4ddebcada35022927a2b5fc623fdf/mock-4.0.3-py3-none-any.whl\n",
            "Collecting sacrebleu\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/57/0c7ca4e31a126189dab99c19951910bd081dea5bbd25f24b77107750eae7/sacrebleu-1.5.1-py3-none-any.whl (54kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 8.3MB/s \n",
            "\u001b[?25hCollecting tensorboardX\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/07/84/46421bd3e0e89a92682b1a38b40efc22dafb6d8e3d947e4ceefd4a5fabc7/tensorboardX-2.2-py2.py3-none-any.whl (120kB)\n",
            "\u001b[K     |████████████████████████████████| 122kB 35.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyarrow in /usr/local/lib/python3.7/dist-packages (3.0.0)\n",
            "Collecting indic-nlp-library\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/84/d4/495bb43b88a2a6d04b09c29fc5115f24872af74cd8317fe84026abd4ddb1/indic_nlp_library-0.81-py3-none-any.whl (40kB)\n",
            "\u001b[K     |████████████████████████████████| 40kB 5.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from sacremoses) (4.41.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses) (7.1.2)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from sacremoses) (2019.12.20)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.0.1)\n",
            "Requirement already satisfied: numpy>=1.15.4 in /usr/local/lib/python3.7/dist-packages (from pandas) (1.19.5)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas) (2018.9)\n",
            "Collecting portalocker==2.0.0\n",
            "  Downloading https://files.pythonhosted.org/packages/89/a6/3814b7107e0788040870e8825eebf214d72166adf656ba7d4bf14759a06a/portalocker-2.0.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorboardX) (3.12.4)\n",
            "Collecting morfessor\n",
            "  Downloading https://files.pythonhosted.org/packages/39/e6/7afea30be2ee4d29ce9de0fa53acbb033163615f849515c0b1956ad074ee/Morfessor-2.0.6-py3-none-any.whl\n",
            "Collecting sphinx-rtd-theme\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ac/24/2475e8f83519b54b2148d4a56eb1111f9cec630d088c3ffc214492c12107/sphinx_rtd_theme-0.5.2-py2.py3-none-any.whl (9.1MB)\n",
            "\u001b[K     |████████████████████████████████| 9.2MB 28.0MB/s \n",
            "\u001b[?25hCollecting sphinx-argparse\n",
            "  Downloading https://files.pythonhosted.org/packages/06/2b/dfad6a1831c3aeeae25d8d3d417224684befbf45e10c7f2141631616a6ed/sphinx-argparse-0.2.5.tar.gz\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.8.0->tensorboardX) (57.0.0)\n",
            "Collecting docutils<0.17\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/81/44/8a15e45ffa96e6cf82956dd8d7af9e666357e16b0d93b253903475ee947f/docutils-0.16-py2.py3-none-any.whl (548kB)\n",
            "\u001b[K     |████████████████████████████████| 552kB 30.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: sphinx in /usr/local/lib/python3.7/dist-packages (from sphinx-rtd-theme->indic-nlp-library) (1.8.5)\n",
            "Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (2.6.1)\n",
            "Requirement already satisfied: requests>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (2.23.0)\n",
            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (2.1.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (20.9)\n",
            "Requirement already satisfied: imagesize in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (1.2.0)\n",
            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (0.7.12)\n",
            "Requirement already satisfied: sphinxcontrib-websupport in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (1.2.4)\n",
            "Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (2.9.1)\n",
            "Requirement already satisfied: Jinja2>=2.3 in /usr/local/lib/python3.7/dist-packages (from sphinx->sphinx-rtd-theme->indic-nlp-library) (2.11.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx->sphinx-rtd-theme->indic-nlp-library) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx->sphinx-rtd-theme->indic-nlp-library) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx->sphinx-rtd-theme->indic-nlp-library) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx->sphinx-rtd-theme->indic-nlp-library) (2.10)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->sphinx->sphinx-rtd-theme->indic-nlp-library) (2.4.7)\n",
            "Requirement already satisfied: sphinxcontrib-serializinghtml in /usr/local/lib/python3.7/dist-packages (from sphinxcontrib-websupport->sphinx->sphinx-rtd-theme->indic-nlp-library) (1.1.4)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2>=2.3->sphinx->sphinx-rtd-theme->indic-nlp-library) (2.0.1)\n",
            "Building wheels for collected packages: sphinx-argparse\n",
            "  Building wheel for sphinx-argparse (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sphinx-argparse: filename=sphinx_argparse-0.2.5-cp37-none-any.whl size=11552 sha256=d8804d903bcf829240052e806acb7c6051e0c240bddf22ef8bd4e4bd2abdfbac\n",
            "  Stored in directory: /root/.cache/pip/wheels/2a/18/1b/4990a1859da4edc77ab312bc2986c08d2733fb5713d06e44f5\n",
            "Successfully built sphinx-argparse\n",
            "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "Installing collected packages: sacremoses, mock, portalocker, sacrebleu, tensorboardX, morfessor, docutils, sphinx-rtd-theme, sphinx-argparse, indic-nlp-library\n",
            "  Found existing installation: docutils 0.17.1\n",
            "    Uninstalling docutils-0.17.1:\n",
            "      Successfully uninstalled docutils-0.17.1\n",
            "Successfully installed docutils-0.16 indic-nlp-library-0.81 mock-4.0.3 morfessor-2.0.6 portalocker-2.0.0 sacrebleu-1.5.1 sacremoses-0.0.45 sphinx-argparse-0.2.5 sphinx-rtd-theme-0.5.2 tensorboardX-2.2\n",
            "Cloning into 'fairseq'...\n",
            "remote: Enumerating objects: 28243, done.\u001b[K\n",
            "remote: Counting objects: 100% (62/62), done.\u001b[K\n",
            "remote: Compressing objects: 100% (39/39), done.\u001b[K\n",
            "remote: Total 28243 (delta 29), reused 44 (delta 22), pack-reused 28181\u001b[K\n",
            "Receiving objects: 100% (28243/28243), 11.83 MiB | 8.53 MiB/s, done.\n",
            "Resolving deltas: 100% (21233/21233), done.\n",
            "/content/testing/fairseq\n",
            "Obtaining file:///content/testing/fairseq\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: sacrebleu>=1.4.12 in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+2fd9d8a) (1.5.1)\n",
            "Collecting hydra-core<1.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/52/e3/fbd70dd0d3ce4d1d75c22d56c0c9f895cfa7ed6587a9ffb821d6812d6a60/hydra_core-1.0.6-py3-none-any.whl (123kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 4.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+2fd9d8a) (2019.12.20)\n",
            "Requirement already satisfied: cython in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+2fd9d8a) (0.29.23)\n",
            "Requirement already satisfied: cffi in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+2fd9d8a) (1.14.5)\n",
            "Collecting omegaconf<2.1\n",
            "  Downloading https://files.pythonhosted.org/packages/d0/eb/9d63ce09dd8aa85767c65668d5414958ea29648a0eec80a4a7d311ec2684/omegaconf-2.0.6-py3-none-any.whl\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+2fd9d8a) (1.8.1+cu101)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+2fd9d8a) (4.41.1)\n",
            "Requirement already satisfied: numpy; python_version >= \"3.7\" in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+2fd9d8a) (1.19.5)\n",
            "Requirement already satisfied: portalocker==2.0.0 in /usr/local/lib/python3.7/dist-packages (from sacrebleu>=1.4.12->fairseq==1.0.0a0+2fd9d8a) (2.0.0)\n",
            "Requirement already satisfied: importlib-resources; python_version < \"3.9\" in /usr/local/lib/python3.7/dist-packages (from hydra-core<1.1->fairseq==1.0.0a0+2fd9d8a) (5.1.3)\n",
            "Collecting antlr4-python3-runtime==4.8\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/56/02/789a0bddf9c9b31b14c3e79ec22b9656185a803dc31c15f006f9855ece0d/antlr4-python3-runtime-4.8.tar.gz (112kB)\n",
            "\u001b[K     |████████████████████████████████| 112kB 17.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi->fairseq==1.0.0a0+2fd9d8a) (2.20)\n",
            "Collecting PyYAML>=5.1.*\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7a/a5/393c087efdc78091afa2af9f1378762f9821c9c1d7a22c5753fb5ac5f97a/PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636kB)\n",
            "\u001b[K     |████████████████████████████████| 645kB 14.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from omegaconf<2.1->fairseq==1.0.0a0+2fd9d8a) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-resources; python_version < \"3.9\"->hydra-core<1.1->fairseq==1.0.0a0+2fd9d8a) (3.4.1)\n",
            "Building wheels for collected packages: antlr4-python3-runtime\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.8-cp37-none-any.whl size=141231 sha256=f9207fa94682c5ba5daa722d4103f4c9eb131c8dd86870ae9cf43f7df7a90154\n",
            "  Stored in directory: /root/.cache/pip/wheels/e3/e2/fa/b78480b448b8579ddf393bebd3f47ee23aa84c89b6a78285c8\n",
            "Successfully built antlr4-python3-runtime\n",
            "Installing collected packages: PyYAML, omegaconf, antlr4-python3-runtime, hydra-core, fairseq\n",
            "  Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "  Running setup.py develop for fairseq\n",
            "Successfully installed PyYAML-5.4.1 antlr4-python3-runtime-4.8 fairseq hydra-core-1.0.6 omegaconf-2.0.6\n",
            "/content/testing\n"
          ]
        }
      ],
      "source": [
        "# Install the necessary libraries\n",
        "!pip install sacremoses pandas mock sacrebleu tensorboardX pyarrow indic-nlp-library\n",
        "# Install fairseq from source\n",
        "!git clone https://github.com/pytorch/fairseq.git\n",
        "%cd fairseq\n",
        "# !git checkout da9eaba12d82b9bfc1442f0e2c6fc1b895f4d35d\n",
        "!pip install ./\n",
        "! pip install xformers\n",
        "%cd .."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "L6SGeKxq1XPI"
      },
      "outputs": [],
      "source": [
        "# add fairseq folder to python path\n",
        "import os\n",
        "os.environ['PYTHONPATH'] += \":/content/fairseq/\"\n",
        "# sanity check to see if fairseq is installed\n",
        "from fairseq import checkpoint_utils, distributed_utils, options, tasks, utils"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kKA8afhBawO5",
        "outputId": "d346f462-d5d4-43a0-c29b-90aaab2fb4d2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "--2021-06-09 15:06:00--  https://storage.googleapis.com/samanantar-public/V0.2/models/indic-en.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 64.233.188.128, 64.233.189.128, 108.177.97.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|64.233.188.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4551079075 (4.2G) [application/zip]\n",
            "Saving to: ‘indic-en.zip’\n",
            "\n",
            "indic-en.zip        100%[===================>]   4.24G  49.9MB/s    in 1m 47s  \n",
            "\n",
            "2021-06-09 15:07:48 (40.5 MB/s) - ‘indic-en.zip’ saved [4551079075/4551079075]\n",
            "\n",
            "Archive:  indic-en.zip\n",
            "   creating: indic-en/\n",
            "   creating: indic-en/vocab/\n",
            "  inflating: indic-en/vocab/bpe_codes.32k.SRC  \n",
            "  inflating: indic-en/vocab/vocab.SRC  \n",
            "  inflating: indic-en/vocab/vocab.TGT  \n",
            "  inflating: indic-en/vocab/bpe_codes.32k.TGT  \n",
            "   creating: indic-en/final_bin/\n",
            "  inflating: indic-en/final_bin/dict.TGT.txt  \n",
            "  inflating: indic-en/final_bin/dict.SRC.txt  \n",
            "   creating: indic-en/model/\n",
            "  inflating: indic-en/model/checkpoint_best.pt  \n",
            "--2021-06-09 15:09:51--  https://storage.googleapis.com/samanantar-public/V0.2/models/en-indic.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.204.128, 64.233.188.128, 64.233.189.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.204.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4609212103 (4.3G) [application/zip]\n",
            "Saving to: ‘en-indic.zip’\n",
            "\n",
            "en-indic.zip        100%[===================>]   4.29G  33.8MB/s    in 1m 51s  \n",
            "\n",
            "2021-06-09 15:11:44 (39.5 MB/s) - ‘en-indic.zip’ saved [4609212103/4609212103]\n",
            "\n",
            "Archive:  en-indic.zip\n",
            "   creating: en-indic/\n",
            "   creating: en-indic/vocab/\n",
            "  inflating: en-indic/vocab/bpe_codes.32k.SRC  \n",
            "  inflating: en-indic/vocab/vocab.SRC  \n",
            "  inflating: en-indic/vocab/vocab.TGT  \n",
            "  inflating: en-indic/vocab/bpe_codes.32k.TGT  \n",
            "   creating: en-indic/final_bin/\n",
            "  inflating: en-indic/final_bin/dict.TGT.txt  \n",
            "  inflating: en-indic/final_bin/dict.SRC.txt  \n",
            "   creating: en-indic/model/\n",
            "  inflating: en-indic/model/checkpoint_best.pt  \n",
            "--2021-06-09 15:14:11--  https://storage.googleapis.com/samanantar-public/models/m2m.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.23.128, 74.125.203.128, 74.125.204.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.23.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4081990185 (3.8G) [application/zip]\n",
            "Saving to: ‘m2m.zip’\n",
            "\n",
            "m2m.zip             100%[===================>]   3.80G  41.5MB/s    in 96s     \n",
            "\n",
            "2021-06-09 15:15:48 (40.4 MB/s) - ‘m2m.zip’ saved [4081990185/4081990185]\n",
            "\n",
            "Archive:  m2m.zip\n",
            "   creating: m2m/\n",
            "   creating: m2m/vocab/\n",
            "  inflating: m2m/vocab/vocab.SRC     \n",
            "  inflating: m2m/vocab/vocab.TGT     \n",
            "  inflating: m2m/vocab/bpe_codes.32k.SRC_TGT  \n",
            "   creating: m2m/final_bin/\n",
            "  inflating: m2m/final_bin/dict.TGT.txt  \n",
            "  inflating: m2m/final_bin/dict.SRC.txt  \n",
            "   creating: m2m/model/\n",
            "  inflating: m2m/model/checkpoint_best.pt  \n",
            "/content/testing/indicTrans\n"
          ]
        }
      ],
      "source": [
        "# download the indictrans model\n",
        "\n",
        "\n",
        "# downloading the indic-en model\n",
        "!wget https://ai4b-public-nlu-nlg.objectstore.e2enetworks.net/indic2en.zip\n",
        "!unzip indic2en.zip\n",
        "\n",
        "# downloading the en-indic model\n",
        "# !wget https://ai4b-public-nlu-nlg.objectstore.e2enetworks.net/en2indic.zip\n",
        "# !unzip en2indic.zip\n",
        "\n",
        "# # downloading the indic-indic model\n",
        "# !wget https://ai4b-public-nlu-nlg.objectstore.e2enetworks.net/m2m.zip\n",
        "# !unzip m2m.zip\n",
        "\n",
        "%cd indicTrans"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Lg1sQFfyWJli"
      },
      "outputs": [],
      "source": [
        "# creating a text file and adding en sentences we can use for testing the model\n",
        "!touch en_sentences.txt\n",
        "!echo 'This bicycle is too small for you !!' >> en_sentences.txt\n",
        "!echo \"I will directly meet you at the airport.\" >> en_sentences.txt\n",
        "!echo 'If COVID-19 is spreading in your community, stay safe by taking some simple precautions, such as physical distancing, wearing a mask, keeping rooms well ventilated, avoiding crowds, cleaning your hands, and coughing into a bent elbow or tissue' >> en_sentences.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fLg9BWAGWvLU",
        "outputId": "f3ca6f65-9a39-4d80-c25d-88806daf3e7b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Wed Jun 9 15:18:01 UTC 2021\n",
            "Applying normalization and script conversion\n",
            "100% 3/3 [00:00<00:00, 71.78it/s]\n",
            "Number of sentences in input: 3\n",
            "Applying BPE\n",
            "Decoding\n",
            "Extracting translations, script conversion and detokenization\n",
            "Translation completed\n"
          ]
        }
      ],
      "source": [
        "# joint_translate takes src_file, output_fname, src_lang, tgt_lang, model_folder as inputs\n",
        "# src_file -> input text file to be translated\n",
        "# output_fname -> name of the output file (will get created) containing the model predictions\n",
        "# src_lang -> source lang code of the input text ( in this case we are using en-indic model and hence src_lang would be 'en')\n",
        "# tgt_lang -> target lang code of the input text ( tgt lang for en-indic model would be any of the 11 indic langs we trained on:\n",
        "#              as, bn, hi, gu, kn, ml, mr, or, pa, ta, te)\n",
        "# supported languages are:\n",
        "#              as - assamese, bn - bengali, gu - gujarathi, hi - hindi, kn - kannada, \n",
        "#              ml - malayalam, mr - marathi, or - oriya, pa - punjabi, ta - tamil, te - telugu\n",
        "\n",
        "# model_dir -> the directory containing the model and the vocab files\n",
        "\n",
        "# Note: if the translation is taking a lot of time, please tune the buffer_size and batch_size parameter for fairseq-interactive defined inside this joint_translate script\n",
        "\n",
        "\n",
        "# here we are translating the english sentences to tamil\n",
        "!bash joint_translate.sh en_sentences.txt ta_outputs.txt 'en' 'ta' '../en-indic'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8QzkBCgeGZiH",
        "outputId": "c150360c-6d01-4689-8c2e-9bdd0eba1504"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "இந்த சைக்கிள் உங்களுக்கு மிகவும் சிறியது!\n",
            "விமான நிலையத்தில் உங்களை நேரில் சந்திக்கிறேன்.\n",
            "உங்கள் சமூகத்தில் கோவிட்-19 பரவுகிறது என்றால், சில எளிய முன்னெச்சரிக்கை நடவடிக்கைகளான, தனி நபர் இடைவெளி, முகக்கவசம் அணிதல், அறைகளை நன்கு காற்றோட்டமாக வைத்திருத்தல், கூட்டத்தைத் தவிர்த்தல், கைகளைக் கழுவுதல், முழங்கை அல்லது திசுக்களில் இருமல் போன்றவற்றை மேற்கொள்வதன் மூலம் பாதுகாப்பாக இருங்கள்.\n"
          ]
        }
      ],
      "source": [
        "!cat ta_outputs.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c4v9BmbZao5d",
        "outputId": "6efac2a3-5f79-4e72-821b-bc80702a7fa8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Wed Jun 9 15:21:31 UTC 2021\n",
            "Applying normalization and script conversion\n",
            "100% 3/3 [00:00<00:00, 88.59it/s]\n",
            "Number of sentences in input: 3\n",
            "Applying BPE\n",
            "Decoding\n",
            "Extracting translations, script conversion and detokenization\n",
            "Translation completed\n"
          ]
        }
      ],
      "source": [
        "# Similarly, we can translate the english sentences to hindi\n",
        "!bash joint_translate.sh en_sentences.txt hi_outputs.txt 'en' 'hi' '../en-indic'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pNNzyR_LfqIr",
        "outputId": "095b9532-e76a-4451-dec9-4862566a4288"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "यह साइकिल तुम्हारे लिए बहुत छोटी है!\n",
            "मैं आपसे एयरपोर्ट पर ही मिलने वाला हूं।\n",
            "यदि आपके समुदाय में कोविड-19 फैल रहा है, तो कुछ सरल सावधानियां बरतें, जैसे शारीरिक दूरी बनाए रखना, मास्क पहनना, कमरों को अच्छी तरह से हवादार रखना, भीड़ से बचना, अपने हाथों को साफ करना और कोहनी या ऊतक को मोड़कर खांसते हुए सुरक्षित रहें\n"
          ]
        }
      ],
      "source": [
        "!cat hi_outputs.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PzjbDLBtaol9"
      },
      "outputs": [],
      "source": [
        "# creating a text file and adding hi sentences we can use for testing the model\n",
        "!touch hi_sentences.txt\n",
        "!echo 'तुम आज सुबह यहाँ क्यों आए?' >> hi_sentences.txt\n",
        "!echo \"मेरे परिवार में हर कोई जल्दी उठता है।\" >> hi_sentences.txt\n",
        "!echo ' स्वास्थ्य और परिवार कल्याण मंत्रालय द्वारा प्रदान की गई जानकारी और सलाह को सावधानी व सही तरीके से पालन कर वायरस के स्थानीय प्रसार को रोका जा सकता है।' >> hi_sentences.txt\n",
        "\n",
        "!touch ta_sentences.txt\n",
        "!echo 'அவனுக்கு நம்மைப் தெரியும் என்று தோன்றுகிறது' >> ta_sentences.txt\n",
        "!echo \"இது எங்கே இருக்கு என்று என்னால் கண்டுபிடிக்க முடியவில்லை.\" >> ta_sentences.txt\n",
        "!echo 'உங்களுக்கு உங்கள் அருகில் இருக்கும் ஒருவருக்கோ இத்தகைய அறிகுறிகள் தென்பட்டால், வீட்டிலேயே இருப்பது, கொரோனா வைரஸ் தொற்று பிறருக்கு வராமல் தடுக்க உதவும்.' >> ta_sentences.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5uaOmKb8gmeN",
        "outputId": "951bbdf9-61d0-4703-a8df-0c3fcb4e5bb3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Wed Jun 9 15:24:43 UTC 2021\n",
            "Applying normalization and script conversion\n",
            "100% 3/3 [00:00<00:00, 74.90it/s]\n",
            "Number of sentences in input: 3\n",
            "Applying BPE\n",
            "Decoding\n",
            "Extracting translations, script conversion and detokenization\n",
            "Translation completed\n"
          ]
        }
      ],
      "source": [
        "# here we are translating the english sentences to hindi\n",
        "!bash joint_translate.sh hi_sentences.txt en_outputs.txt 'hi' 'en' '../indic-en'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iLD7WPqmlSnC",
        "outputId": "359050fa-6d35-4055-a9c5-13a15322c59e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Why did you come here this morning?\n",
            "Everyone in my family gets up early.\n",
            "The local spread of the virus can be curbed by following the information and advice provided by the Ministry of Health and Family Welfare in a careful and correct manner.\n"
          ]
        }
      ],
      "source": [
        "! cat en_outputs.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O3mJyj-QljWz",
        "outputId": "1c0420e5-4b80-41d9-f09e-2fdff79bc7bd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Wed Jun 9 15:28:05 UTC 2021\n",
            "Applying normalization and script conversion\n",
            "100% 3/3 [00:00<00:00, 72.92it/s]\n",
            "Number of sentences in input: 3\n",
            "Applying BPE\n",
            "Decoding\n",
            "Extracting translations, script conversion and detokenization\n",
            "Translation completed\n"
          ]
        }
      ],
      "source": [
        "# here we are translating the english sentences to tamil\n",
        "!bash joint_translate.sh ta_sentences.txt en_outputs.txt 'ta' 'en' '../indic-en'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GapEJESiloD8",
        "outputId": "dc8b2a8c-4f36-4bf9-d517-6826aa65da57"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "He seems to know us.\n",
            "I couldnt find it anywhere.\n",
            "If someone in your neighbourhood develops these symptoms, staying at home can help prevent the spread of the coronavirus infection.\n"
          ]
        }
      ],
      "source": [
        "! cat en_outputs.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ckfW2P6abcB3"
      },
      "outputs": [],
      "source": [
        "# we just rename the m2m_joint_vocab file here as joint_translate uses bpe_codes.32k.SRC\n",
        "mv ../m2m/vocab/bpe_codes.32k.SRC_TGT ../m2m/vocab/bpe_codes.32k.SRC"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H-3vPdCqSWoK",
        "outputId": "d5a80c59-cc89-4910-a9ce-7317fac6bf8d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Wed Jun 9 15:39:26 UTC 2021\n",
            "Applying normalization and script conversion\n",
            "100% 3/3 [00:00<00:00, 63.53it/s]\n",
            "Number of sentences in input: 3\n",
            "Applying BPE\n",
            "Decoding\n",
            "Extracting translations, script conversion and detokenization\n",
            "Translation completed\n"
          ]
        }
      ],
      "source": [
        "# here we are using the indic2indic model for translating the hindi sentences to tamil\n",
        "!bash joint_translate.sh hi_sentences.txt ta_outputs.txt 'hi' 'ta' '../m2m'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "22yPo78Zb_oR",
        "outputId": "4df17e93-9029-4020-8deb-0dbaf8bb0b27"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "तुम आज सुबह यहाँ क्यों आए?\n",
            "मेरे परिवार में हर कोई जल्दी उठता है।\n",
            " स्वास्थ्य और परिवार कल्याण मंत्रालय द्वारा प्रदान की गई जानकारी और सलाह को सावधानी व सही तरीके से पालन कर वायरस के स्थानीय प्रसार को रोका जा सकता है।\n"
          ]
        }
      ],
      "source": [
        " ! cat hi_sentences.txt # the hindi inputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "onnfzTDESg2I",
        "outputId": "1bc600d4-d3ff-40fa-d258-7d1c876bd49c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "ஏன் இன்று காலையில் வந்தீர்கள்?\n",
            "எனது குடும்பத்தில் உள்ள ஒவ்வொருவரும் விரைவில் எழுவார்கள்.\n",
            "மத்திய சுகாதாரம் மற்றும் குடும்ப நல அமைச்சகத்தின் அறிவுறுத்தல்கள் மற்றும் தகவல்களைப் பின்பற்றுவதன் மூலம், உள்ளூர் அளவில் வைரஸ் பரவுவதைத் தடுக்க முடியும்.\n"
          ]
        }
      ],
      "source": [
        "! cat ta_outputs.txt # the tamil outputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5klOcwi8SjGS",
        "outputId": "bc4e47fa-ee1d-4da2-85ea-f7900cae7b48"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Wed Jun 9 15:45:53 UTC 2021\n",
            "Applying normalization and script conversion\n",
            "100% 3/3 [00:00<00:00, 82.25it/s]\n",
            "Number of sentences in input: 3\n",
            "Applying BPE\n",
            "Decoding\n",
            "Extracting translations, script conversion and detokenization\n",
            "Translation completed\n"
          ]
        }
      ],
      "source": [
        "# here we are using the indic2indic model for translating the hindi sentences to tamil (same as above with reversing the direction)\n",
        "!bash joint_translate.sh ta_sentences.txt hi_outputs.txt 'ta' 'hi' '../m2m'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4ifZhGkKc6oo",
        "outputId": "a0112e2b-a54b-48ad-e3ae-a3d84c6d097e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "அவனுக்கு நம்மைப் தெரியும் என்று தோன்றுகிறது\n",
            "இது எங்கே இருக்கு என்று என்னால் கண்டுபிடிக்க முடியவில்லை.\n",
            "உங்களுக்கு உங்கள் அருகில் இருக்கும் ஒருவருக்கோ இத்தகைய அறிகுறிகள் தென்பட்டால், வீட்டிலேயே இருப்பது, கொரோனா வைரஸ் தொற்று பிறருக்கு வராமல் தடுக்க உதவும்.\n"
          ]
        }
      ],
      "source": [
        "! cat ta_sentences.txt # the tamil inputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v0x0YrWYSwwK",
        "outputId": "4c37d699-5b8e-4ae7-9724-953d7e165035"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "ऐसा लगता है कि वह हमें जानता है।\n",
            "मुझे पता नहीं था कि यह कहां है।\n",
            "अगर आपके आस-पास के किसी व्यक्ति में ऐसे लक्षण दिखाई देते हैं, तो घर पर रहने से कोरोना वायरस को फैलने से रोकने में मदद मिलेगी।\n"
          ]
        }
      ],
      "source": [
        "! cat hi_outputs.txt   # the hi outputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-xcnDOc4gNKC"
      },
      "outputs": [],
      "source": [
        "# to compute bleu scores for the predicitions with a reference file, use the following command\n",
        "\n",
        "# bash compute_bleu.sh pred_fname ref_fname src_lang tgt_lang\n",
        "# arguments:\n",
        "# pred_fname: file that contains model predictions\n",
        "# ref_fname: file that contains references\n",
        "# src_lang and tgt_lang : the source and target language"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9YK2BdwvrUgI"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "include_colab_link": true,
      "name": "indictrans_fairseq_inference.ipynb",
      "provenance": []
    },
    "interpreter": {
      "hash": "3c7d4130300118f0c7487d576c6841c0dbbdeec039e1e658ac9b107412a09af0"
    },
    "kernelspec": {
      "display_name": "Python 3.7.7 64-bit",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": ""
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}