MyProjectTest2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 124
    },
    "colab_type": "code",
    "id": "LLWbySujDwpR",
    "outputId": "bddf9455-3a91-431e-e325-9ee1b425270a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /gdrive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/gdrive', force_remount=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "Ny-TYeggD5Hj",
    "outputId": "5b63cef8-2375-4d3a-c06a-7979c9e8a446"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0-rc3\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 679
    },
    "colab_type": "code",
    "id": "GjLqEbteD71D",
    "outputId": "82072cec-30cd-4ed9-fe8a-e7d8eb08cef8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tensorflow==1.13.1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/77/63/a9fa76de8dffe7455304c4ed635be4aa9c0bacef6e0633d87d5f54530c5c/tensorflow-1.13.1-cp36-cp36m-manylinux1_x86_64.whl (92.5MB)\n",
      "\u001b[K     |████████████████████████████████| 92.5MB 61kB/s \n",
      "\u001b[?25hRequirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (3.10.0)\n",
      "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (0.8.1)\n",
      "Requirement already satisfied: absl-py>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (0.9.0)\n",
      "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.12.0)\n",
      "Collecting tensorboard<1.14.0,>=1.13.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/39/bdd75b08a6fba41f098b6cb091b9e8c7a80e1b4d679a581a0ccd17b10373/tensorboard-1.13.1-py3-none-any.whl (3.2MB)\n",
      "\u001b[K     |████████████████████████████████| 3.2MB 39.3MB/s \n",
      "\u001b[?25hRequirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.1.0)\n",
      "Collecting tensorflow-estimator<1.14.0rc0,>=1.13.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bb/48/13f49fc3fa0fdf916aa1419013bb8f2ad09674c275b4046d5ee669a46873/tensorflow_estimator-1.13.0-py2.py3-none-any.whl (367kB)\n",
      "\u001b[K     |████████████████████████████████| 368kB 48.2MB/s \n",
      "\u001b[?25hRequirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.28.1)\n",
      "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (0.3.3)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.1.0)\n",
      "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.18.2)\n",
      "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (0.34.2)\n",
      "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.13.1) (1.0.8)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow==1.13.1) (46.1.3)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow==1.13.1) (3.2.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow==1.13.1) (1.0.1)\n",
      "Collecting mock>=2.0.0\n",
      "  Downloading https://files.pythonhosted.org/packages/cd/74/d72daf8dff5b6566db857cfd088907bb0355f5dd2914c4b3ef065c790735/mock-4.0.2-py3-none-any.whl\n",
      "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow==1.13.1) (2.10.0)\n",
      "Installing collected packages: tensorboard, mock, tensorflow-estimator, tensorflow\n",
      "  Found existing installation: tensorboard 2.2.0\n",
      "    Uninstalling tensorboard-2.2.0:\n",
      "      Successfully uninstalled tensorboard-2.2.0\n",
      "  Found existing installation: tensorflow-estimator 2.2.0rc0\n",
      "    Uninstalling tensorflow-estimator-2.2.0rc0:\n",
      "      Successfully uninstalled tensorflow-estimator-2.2.0rc0\n",
      "  Found existing installation: tensorflow 2.2.0rc3\n",
      "    Uninstalling tensorflow-2.2.0rc3:\n",
      "      Successfully uninstalled tensorflow-2.2.0rc3\n",
      "Successfully installed mock-4.0.2 tensorboard-1.13.1 tensorflow-1.13.1 tensorflow-estimator-1.13.0\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow==1.13.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" \n",
    "Remark\n",
    "\n",
    "BERT model does not publish source code for learning a new WordPiece vocabulary because they  implemented in C++ with\n",
    "dependencies on Google's internal libraries.They recommended that  the following libraries for learning vocabularies of other\n",
    "languages. However, they warned that these libraries are not compatible with BERT tokenization.py library.\n",
    "-Google's SentencePiece library\n",
    "-tensor2tensor's WordPiece generation script\n",
    "-Rico Sennrich's Byte Pair Encoding library\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "\n",
    "Create Pretraining data\n",
    "I  created vocabulary file using https://github.com/kwonmha/bert-vocab-builder. \n",
    "This link built wordpiece(subword) vocabulary compatible for Google Research's BERT.\n",
    "Use  myanmar vocabulary file  created by above mention link.\n",
    "Use myanWordTest.txt for pretraining data.\n",
    "max_seq_length=128\n",
    "I found many [UNK](unknown_token) because it's not in the dictionary(vocabulary file) when finished running of \n",
    "create_pretraining_data.py(BERT code).\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "6oGUoQ9J3hSB",
    "outputId": "c1ce31ca-81e4-49cb-9666-ef9463323bd8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "INFO:tensorflow:*** Reading from input files ***\n",
      "INFO:tensorflow:*** Reading from input files ***\n",
      "INFO:tensorflow:  /gdrive/My Drive/Colab Notebooks/codeTest/myaninput/myanWordTest.txt\n",
      "INFO:tensorflow:  /gdrive/My Drive/Colab Notebooks/codeTest/myaninput/myanWordTest.txt\n",
      "INFO:tensorflow:*** Writing to output files ***\n",
      "INFO:tensorflow:*** Writing to output files ***\n",
      "INFO:tensorflow:  /gdrive/My Drive/Colab Notebooks/codeTest/output/tf_examples.tfrecord\n",
      "INFO:tensorflow:  /gdrive/My Drive/Colab Notebooks/codeTest/output/tf_examples.tfrecord\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] domain [UNK] က se ##co ##nd level [MASK] [UNK] ရ ဘယ ##ဘ ##က [UNK] [UNK] [UNK] ရ [UNK] တယ ။ [MASK] [UNK] [UNK] [UNK] [UNK] က [UNK] [UNK] [UNK] [UNK] [UNK] ၏ [UNK] [UNK] ##ย ဤ [MASK] က [UNK] [UNK] [MASK] [MASK] ၊ [UNK] [UNK] [UNK] [MASK] တစ ##က ##မတ ##မတ [UNK] ၍ [UNK] [UNK] [UNK] သည ။ Com [UNK] အမည [UNK] [UNK] [SEP] ဒ [UNK] လ [UNK] [UNK] တယ ။ ထ [UNK] က [UNK] မ [UNK] ခ [UNK] [UNK] [MASK] [UNK] [MASK] ရ သည ။ [MASK] ၊ [UNK] [MASK] [UNK] နင [UNK] [MASK] မ [UNK] [UNK] [UNK] ထက ရ သည ။ [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [MASK] ၊ [MASK] [UNK] က ပ [UNK] ဆ [UNK] [UNK] ၊ [MASK] [UNK] [UNK] မ [UNK] ၊ [MASK] [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] domain [UNK] က se ##co ##nd level [MASK] [UNK] ရ ဘယ ##ဘ ##က [UNK] [UNK] [UNK] ရ [UNK] တယ ။ [MASK] [UNK] [UNK] [UNK] [UNK] က [UNK] [UNK] [UNK] [UNK] [UNK] ၏ [UNK] [UNK] ##ย ဤ [MASK] က [UNK] [UNK] [MASK] [MASK] ၊ [UNK] [UNK] [UNK] [MASK] တစ ##က ##မတ ##မတ [UNK] ၍ [UNK] [UNK] [UNK] သည ။ Com [UNK] အမည [UNK] [UNK] [SEP] ဒ [UNK] လ [UNK] [UNK] တယ ။ ထ [UNK] က [UNK] မ [UNK] ခ [UNK] [UNK] [MASK] [UNK] [MASK] ရ သည ။ [MASK] ၊ [UNK] [MASK] [UNK] နင [UNK] [MASK] မ [UNK] [UNK] [UNK] ထက ရ သည ။ [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [MASK] ၊ [MASK] [UNK] က ပ [UNK] ဆ [UNK] [UNK] ၊ [MASK] [UNK] [UNK] မ [UNK] ၊ [MASK] [SEP]\n",
      "INFO:tensorflow:input_ids: 3 356 2 7 1008 499 607 535 5 2 12 108 368 169 2 2 2 12 2 57 18 5 2 2 2 2 7 2 2 2 2 2 26 2 2 1166 100 5 7 2 2 5 5 23 2 2 2 5 30 169 1361 1361 2 33 2 2 2 17 18 1575 2 104 2 2 4 25 2 15 2 2 57 18 20 2 7 2 8 2 16 2 2 5 2 5 12 17 18 5 23 2 5 2 319 2 5 8 2 2 2 101 12 17 18 2 2 5 2 2 2 5 23 5 2 7 9 2 22 2 2 23 5 2 2 8 2 23 5 4\n",
      "INFO:tensorflow:input_ids: 3 356 2 7 1008 499 607 535 5 2 12 108 368 169 2 2 2 12 2 57 18 5 2 2 2 2 7 2 2 2 2 2 26 2 2 1166 100 5 7 2 2 5 5 23 2 2 2 5 30 169 1361 1361 2 33 2 2 2 17 18 1575 2 104 2 2 4 25 2 15 2 2 57 18 20 2 7 2 8 2 16 2 2 5 2 5 12 17 18 5 23 2 5 2 319 2 5 8 2 2 2 101 12 17 18 2 2 5 2 2 2 5 23 5 2 7 9 2 22 2 2 23 5 2 2 8 2 23 5 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 8 21 35 37 41 42 47 59 81 83 87 90 94 105 109 111 115 120 126 0\n",
      "INFO:tensorflow:masked_lm_positions: 8 21 35 37 41 42 47 59 81 83 87 90 94 105 109 111 115 120 126 0\n",
      "INFO:tensorflow:masked_lm_ids: 356 30 2 2 33 2 319 2 2 2 2 23 11 95 2 2 2 2 248 0\n",
      "INFO:tensorflow:masked_lm_ids: 356 30 2 2 33 2 319 2 2 2 2 23 11 95 2 2 2 2 248 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] ၂ ခ မ တစ ##ဆ ##င [UNK] [UNK] [MASK] တစ [MASK] [UNK] အတ ##က [UNK] [UNK] သည [MASK] [UNK] [MASK] [MASK] နင အသက ##အ ##ရ ##ယ အလက [UNK] [UNK] [UNK] သည [MASK] ၁၈၆ [MASK] ခန ##စ တင [UNK] က [UNK] [UNK] က [UNK] ခ [UNK] သည [MASK] ၆ လက ##မ [UNK] ပင မ ရ [UNK] [UNK] [UNK] က [UNK] ၏ [UNK] နင မက [MASK] ##ဆ ##က [SEP] ထ [UNK] ထ [MASK] ##ပ [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] က [UNK] [MASK] [UNK] [MASK] [UNK] [UNK] အမည [MASK] [UNK] က အသ ##အ [MASK] အမည [UNK] [UNK] [UNK] ခ [UNK] [UNK] သည ။ ၁၉၇၈ ခန [MASK] [MASK] [UNK] ခ [MASK] [UNK] အရ [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] ခ [MASK] ။ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] ၂ ခ မ တစ ##ဆ ##င [UNK] [UNK] [MASK] တစ [MASK] [UNK] အတ ##က [UNK] [UNK] သည [MASK] [UNK] [MASK] [MASK] နင အသက ##အ ##ရ ##ယ အလက [UNK] [UNK] [UNK] သည [MASK] ၁၈၆ [MASK] ခန ##စ တင [UNK] က [UNK] [UNK] က [UNK] ခ [UNK] သည [MASK] ၆ လက ##မ [UNK] ပင မ ရ [UNK] [UNK] [UNK] က [UNK] ၏ [UNK] နင မက [MASK] ##ဆ ##က [SEP] ထ [UNK] ထ [MASK] ##ပ [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] က [UNK] [MASK] [UNK] [MASK] [UNK] [UNK] အမည [MASK] [UNK] က အသ ##အ [MASK] အမည [UNK] [UNK] [UNK] ခ [UNK] [UNK] သည ။ ၁၉၇၈ ခန [MASK] [MASK] [UNK] ခ [MASK] [UNK] အရ [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] ခ [MASK] ။ [SEP]\n",
      "INFO:tensorflow:input_ids: 3 2 160 16 8 30 433 177 2 2 5 30 5 2 36 169 2 2 17 5 2 5 5 319 170 412 223 278 252 2 2 2 17 5 841 5 82 199 78 2 7 2 2 7 2 16 2 17 5 237 49 243 2 50 8 12 2 2 2 7 2 26 2 319 107 5 433 169 4 20 2 20 5 207 2 2 2 2 2 2 7 2 5 2 5 2 2 104 5 2 7 53 412 5 104 2 2 2 16 2 2 17 18 567 82 5 5 2 16 5 2 39 2 2 2 2 2 2 2 2 2 2 2 2 16 5 18 4\n",
      "INFO:tensorflow:input_ids: 3 2 160 16 8 30 433 177 2 2 5 30 5 2 36 169 2 2 17 5 2 5 5 319 170 412 223 278 252 2 2 2 17 5 841 5 82 199 78 2 7 2 2 7 2 16 2 17 5 237 49 243 2 50 8 12 2 2 2 7 2 26 2 319 107 5 433 169 4 20 2 20 5 207 2 2 2 2 2 2 7 2 5 2 5 2 2 104 5 2 7 53 412 5 104 2 2 2 16 2 2 17 18 567 82 5 5 2 16 5 2 39 2 2 2 2 2 2 2 2 2 2 2 2 16 5 18 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 10 12 19 21 22 33 35 42 48 65 72 82 84 88 93 105 106 109 125 0\n",
      "INFO:tensorflow:masked_lm_positions: 10 12 19 21 22 33 35 42 48 65 72 82 84 88 93 105 106 109 125 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 2 18 17 2 18 180 2 18 169 413 33 2 2 1361 199 78 2 17 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 2 18 17 2 18 180 2 18 169 413 33 2 2 1361 199 78 2 17 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] [UNK] ##- [UNK] [UNK] ၌ မ ဝက ##ဝ သည လ က သ ကယ သ [UNK] ရန [UNK] မ တက ##ခ ##က [UNK] ။ [UNK] [UNK] ရန နင နင ##င [MASK] ကန ##ထ ##တ ##လ ##ပ ##မ [UNK] [UNK] [UNK] [UNK] ၊ ထက ##က [MASK] [UNK] စသည [MASK] [MASK] ထ နင ##င [UNK] ၏ မတတ ##က [MASK] [UNK] [UNK] ၏ ။ [SEP] [MASK] တင [UNK] [UNK] ခ ရ [MASK] ။ [UNK] က [UNK] နင [UNK] ။ ၁၈၆ ##၄ ခန ##စ ၊ [UNK] ဗက ##က ##န ##တ [MASK] ##က ##သ ##လ [MASK] [UNK] ခ [UNK] [UNK] [UNK] [UNK] တစ [UNK] [UNK] [UNK] ၊ [UNK] သည [UNK] တင [MASK] [UNK] [UNK] [MASK] [UNK] တစ [UNK] ဒဿန သည ။ [UNK] သည [MASK] ၏ ခန ##စ လ [MASK] သန ##တ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] [UNK] ##- [UNK] [UNK] ၌ မ ဝက ##ဝ သည လ က သ ကယ သ [UNK] ရန [UNK] မ တက ##ခ ##က [UNK] ။ [UNK] [UNK] ရန နင နင ##င [MASK] ကန ##ထ ##တ ##လ ##ပ ##မ [UNK] [UNK] [UNK] [UNK] ၊ ထက ##က [MASK] [UNK] စသည [MASK] [MASK] ထ နင ##င [UNK] ၏ မတတ ##က [MASK] [UNK] [UNK] ၏ ။ [SEP] [MASK] တင [UNK] [UNK] ခ ရ [MASK] ။ [UNK] က [UNK] နင [UNK] ။ ၁၈၆ ##၄ ခန ##စ ၊ [UNK] ဗက ##က ##န ##တ [MASK] ##က ##သ ##လ [MASK] [UNK] ခ [UNK] [UNK] [UNK] [UNK] တစ [UNK] [UNK] [UNK] ၊ [UNK] သည [UNK] တင [MASK] [UNK] [UNK] [MASK] [UNK] တစ [UNK] ဒဿန သည ။ [UNK] သည [MASK] ၏ ခန ##စ လ [MASK] သန ##တ [SEP]\n",
      "INFO:tensorflow:input_ids: 3 2 2 1635 2 2 47 8 122 164 17 15 7 13 213 13 2 41 2 8 60 369 169 2 18 2 2 41 319 319 177 5 97 413 235 367 207 243 2 2 2 2 23 101 169 5 2 143 5 5 20 319 177 2 26 1492 169 5 2 2 26 18 4 5 78 2 2 16 12 5 18 2 7 2 319 2 18 841 232 82 199 23 2 387 169 217 235 5 169 332 367 5 2 16 2 2 2 2 30 2 2 2 23 2 17 2 78 5 2 2 5 2 30 2 307 17 18 2 17 5 26 82 199 15 5 70 235 4\n",
      "INFO:tensorflow:input_ids: 3 2 2 1635 2 2 47 8 122 164 17 15 7 13 213 13 2 41 2 8 60 369 169 2 18 2 2 41 319 319 177 5 97 413 235 367 207 243 2 2 2 2 23 101 169 5 2 143 5 5 20 319 177 2 26 1492 169 5 2 2 26 18 4 5 78 2 2 16 12 5 18 2 7 2 319 2 18 841 232 82 199 23 2 387 169 217 235 5 169 332 367 5 2 16 2 2 2 2 30 2 2 2 23 2 17 2 78 5 2 2 5 2 30 2 307 17 18 2 17 5 26 82 199 15 5 70 235 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 3 31 45 48 49 55 57 63 69 72 74 87 91 107 110 114 119 124 125 0\n",
      "INFO:tensorflow:masked_lm_positions: 3 31 45 48 49 55 57 63 69 72 74 87 91 107 110 114 119 124 125 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 26 217 11 2 117 2 2 17 7 319 169 78 2 23 2 2 2 70 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 26 217 11 2 117 2 2 17 7 319 169 78 2 23 2 2 2 70 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] တ သည [UNK] သ [UNK] [UNK] [UNK] [MASK] [UNK] [MASK] [UNK] နင ##င [UNK] [UNK] [UNK] မရ ။ [UNK] သ တ [UNK] [UNK] [UNK] သည [UNK] [MASK] [UNK] တ ၌ [UNK] [UNK] [UNK] [UNK] သ တ ၏ [UNK] [UNK] [MASK] အရပ ##ရပ တ [MASK] [UNK] တ [UNK] [UNK] [UNK] [MASK] ဟ [UNK] [MASK] ၍ [UNK] [UNK] [MASK] [UNK] တ [UNK] ဖန ##ပ [SEP] [UNK] က [MASK] [UNK] လ [UNK] သည ၊ [UNK] [MASK] [UNK] ၏ [UNK] [UNK] နင လတ [MASK] ##ပ ##ခ ##င [UNK] က [UNK] ၍ [UNK] [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] တင [UNK] [UNK] ၊ [UNK] [UNK] နင [UNK] [MASK] [MASK] အတ ##က ၊ [UNK] [UNK] [UNK] ရန ##$ [UNK] ၊ [MASK] က [UNK] [UNK] သည [UNK] [MASK] [UNK] [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] တ သည [UNK] သ [UNK] [UNK] [UNK] [MASK] [UNK] [MASK] [UNK] နင ##င [UNK] [UNK] [UNK] မရ ။ [UNK] သ တ [UNK] [UNK] [UNK] သည [UNK] [MASK] [UNK] တ ၌ [UNK] [UNK] [UNK] [UNK] သ တ ၏ [UNK] [UNK] [MASK] အရပ ##ရပ တ [MASK] [UNK] တ [UNK] [UNK] [UNK] [MASK] ဟ [UNK] [MASK] ၍ [UNK] [UNK] [MASK] [UNK] တ [UNK] ဖန ##ပ [SEP] [UNK] က [MASK] [UNK] လ [UNK] သည ၊ [UNK] [MASK] [UNK] ၏ [UNK] [UNK] နင လတ [MASK] ##ပ ##ခ ##င [UNK] က [UNK] ၍ [UNK] [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] တင [UNK] [UNK] ၊ [UNK] [UNK] နင [UNK] [MASK] [MASK] အတ ##က ၊ [UNK] [UNK] [UNK] ရန ##$ [UNK] ၊ [MASK] က [UNK] [UNK] သည [UNK] [MASK] [UNK] [SEP]\n",
      "INFO:tensorflow:input_ids: 3 2 11 17 2 13 2 2 2 5 2 5 2 319 177 2 2 2 165 18 2 13 11 2 2 2 17 2 5 2 11 47 2 2 2 2 13 11 26 2 2 5 176 1360 11 5 2 11 2 2 2 5 29 2 5 33 2 2 5 2 11 2 161 207 4 2 7 5 2 15 2 17 23 2 5 2 26 2 2 319 222 5 207 369 177 2 7 2 33 2 2 2 2 2 5 2 2 2 2 78 2 2 23 2 2 319 2 5 5 36 169 23 2 2 2 41 1743 2 23 5 7 2 2 17 2 5 2 4\n",
      "INFO:tensorflow:input_ids: 3 2 11 17 2 13 2 2 2 5 2 5 2 319 177 2 2 2 165 18 2 13 11 2 2 2 17 2 5 2 11 47 2 2 2 2 13 11 26 2 2 5 176 1360 11 5 2 11 2 2 2 5 29 2 5 33 2 2 5 2 11 2 161 207 4 2 7 5 2 15 2 17 23 2 5 2 26 2 2 319 222 5 207 369 177 2 7 2 33 2 2 2 2 2 5 2 2 2 2 78 2 2 23 2 2 319 2 5 5 36 169 23 2 2 2 41 1743 2 23 5 7 2 2 17 2 5 2 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 9 11 18 28 41 45 51 54 58 67 69 74 81 94 107 108 116 119 125 0\n",
      "INFO:tensorflow:masked_lm_positions: 9 11 18 28 41 45 51 54 58 67 69 74 81 94 107 108 116 119 125 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 2 17 33 2 7 17 2 2 2 15 13 367 23 2 11 2 2 2 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 2 17 33 2 7 17 2 2 2 15 13 367 23 2 11 2 2 2 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ##re ##ta ##ce ##ou ##s te ##rti ##ary [UNK] [UNK] [MASK] အသက ##ရ ##င [UNK] ခ [MASK] [UNK] တ ၏ တစ ခ [UNK] [UNK] အဆက ##အ [MASK] ##ယ [UNK] [MASK] ယ ##ဆ [UNK] သည ။ [UNK] သည ၄. [MASK] ##ည [UNK] [MASK] သင တ သည ဧည ##သ ##ည [UNK] [UNK] သင [MASK] သည ဧည ##သ ##ည [MASK] [MASK] [UNK] [UNK] ၏ [MASK] [UNK] တစ [UNK] [UNK] [UNK] နင ပတ ##သ ##က ၍ [UNK] သက ##သ ထ [UNK] [UNK] နင ပတ ##သ ##က ၍ [UNK] [UNK] အဖ သည [UNK] ၎ [UNK] [MASK] [MASK] ဖဆပလ [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [SEP] [UNK] [MASK] သည [UNK] [UNK] [UNK] [UNK] [UNK] ၍ ထ [UNK] [UNK] က [UNK] ခ [UNK] [UNK] [MASK] [UNK] ၁၃၀ ခ [UNK] [MASK] ။ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ##re ##ta ##ce ##ou ##s te ##rti ##ary [UNK] [UNK] [MASK] အသက ##ရ ##င [UNK] ခ [MASK] [UNK] တ ၏ တစ ခ [UNK] [UNK] အဆက ##အ [MASK] ##ယ [UNK] [MASK] ယ ##ဆ [UNK] သည ။ [UNK] သည ၄. [MASK] ##ည [UNK] [MASK] သင တ သည ဧည ##သ ##ည [UNK] [UNK] သင [MASK] သည ဧည ##သ ##ည [MASK] [MASK] [UNK] [UNK] ၏ [MASK] [UNK] တစ [UNK] [UNK] [UNK] နင ပတ ##သ ##က ၍ [UNK] သက ##သ ထ [UNK] [UNK] နင ပတ ##သ ##က ၍ [UNK] [UNK] အဖ သည [UNK] ၎ [UNK] [MASK] [MASK] ဖဆပလ [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [SEP] [UNK] [MASK] သည [UNK] [UNK] [UNK] [UNK] [UNK] ၍ ထ [UNK] [UNK] က [UNK] ခ [UNK] [UNK] [MASK] [UNK] ၁၃၀ ခ [UNK] [MASK] ။ [SEP]\n",
      "INFO:tensorflow:input_ids: 3 525 473 527 822 171 1004 1182 1066 2 2 5 170 223 177 2 16 5 2 11 26 30 16 2 2 327 412 5 278 2 5 24 433 2 17 18 2 17 1079 5 329 2 5 76 11 17 470 332 329 2 2 76 5 17 470 332 329 5 5 2 2 26 5 2 30 2 2 2 319 89 332 169 33 2 67 332 20 2 2 319 89 332 169 33 2 2 43 17 2 1878 2 5 5 305 2 2 2 5 2 2 2 2 4 2 5 17 2 2 2 2 2 33 20 2 2 7 2 16 2 2 5 2 791 16 2 5 18 4\n",
      "INFO:tensorflow:input_ids: 3 525 473 527 822 171 1004 1182 1066 2 2 5 170 223 177 2 16 5 2 11 26 30 16 2 2 327 412 5 278 2 5 24 433 2 17 18 2 17 1079 5 329 2 5 76 11 17 470 332 329 2 2 76 5 17 470 332 329 5 5 2 2 26 5 2 30 2 2 2 319 89 332 169 33 2 67 332 20 2 2 319 89 332 169 33 2 2 43 17 2 1878 2 5 5 305 2 2 2 5 2 2 2 2 4 2 5 17 2 2 2 2 2 33 20 2 2 7 2 16 2 2 5 2 791 16 2 5 18 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 11 17 27 30 38 39 42 52 57 58 62 86 91 92 97 104 120 122 125 0\n",
      "INFO:tensorflow:masked_lm_positions: 11 17 27 30 38 39 42 52 57 58 62 86 91 92 97 104 120 122 125 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 2 217 2 470 332 2 11 109 7 18 319 11 319 17 2 2 2 17 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 2 217 2 470 332 2 11 109 7 18 319 11 319 17 2 2 2 17 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] နစ [UNK] တင [UNK] [UNK] [UNK] [MASK] [UNK] တင [UNK] [UNK] [UNK] [UNK] ခ ခ [MASK] [MASK] ။ [SEP] [UNK] [MASK] [MASK] က [UNK] တင [UNK] [UNK] ဘရ ##င [UNK] [UNK] မတ ##ယ [UNK] သည [MASK] [MASK] [UNK] အသစ က [UNK] သမ ##ဟ ##တ [UNK] ဟ [UNK] ခ သည ။ အန ##ဒ ၏ [MASK] [UNK] ၊ [UNK] နင [UNK] ၏ [UNK] [UNK] ၊ [UNK] တ သည [UNK] တ [MASK] [UNK] ၄၁ [UNK] [UNK] က [UNK] [UNK] [UNK] [UNK] ##? တစ နစ [UNK] [UNK] [MASK] [MASK] [UNK] ။ [UNK] [UNK] [UNK] ရ [UNK] ၊ [UNK] [UNK] [UNK] [UNK] ရ [UNK] [MASK] ။ [UNK] [UNK] [MASK] [UNK] နင [MASK] [UNK] [UNK] ရ [UNK] [UNK] က [MASK] [UNK] [UNK] မ [UNK] ဘ မ [UNK] နင [UNK] [UNK] ။ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] နစ [UNK] တင [UNK] [UNK] [UNK] [MASK] [UNK] တင [UNK] [UNK] [UNK] [UNK] ခ ခ [MASK] [MASK] ။ [SEP] [UNK] [MASK] [MASK] က [UNK] တင [UNK] [UNK] ဘရ ##င [UNK] [UNK] မတ ##ယ [UNK] သည [MASK] [MASK] [UNK] အသစ က [UNK] သမ ##ဟ ##တ [UNK] ဟ [UNK] ခ သည ။ အန ##ဒ ၏ [MASK] [UNK] ၊ [UNK] နင [UNK] ၏ [UNK] [UNK] ၊ [UNK] တ သည [UNK] တ [MASK] [UNK] ၄၁ [UNK] [UNK] က [UNK] [UNK] [UNK] [UNK] ##? တစ နစ [UNK] [UNK] [MASK] [MASK] [UNK] ။ [UNK] [UNK] [UNK] ရ [UNK] ၊ [UNK] [UNK] [UNK] [UNK] ရ [UNK] [MASK] ။ [UNK] [UNK] [MASK] [UNK] နင [MASK] [UNK] [UNK] ရ [UNK] [UNK] က [MASK] [UNK] [UNK] မ [UNK] ဘ မ [UNK] နင [UNK] [UNK] ။ [SEP]\n",
      "INFO:tensorflow:input_ids: 3 2 148 2 78 2 2 2 5 2 78 2 2 2 2 16 16 5 5 18 4 2 5 5 7 2 78 2 2 489 177 2 2 112 278 2 17 5 5 2 256 7 2 75 466 235 2 29 2 16 17 18 48 346 26 5 2 23 2 319 2 26 2 2 23 2 11 17 2 11 5 2 898 2 2 7 2 2 2 2 1634 30 148 2 2 5 5 2 18 2 2 2 12 2 23 2 2 2 2 12 2 5 18 2 2 5 2 319 5 2 2 12 2 2 7 5 2 2 8 2 28 8 2 319 2 2 18 4\n",
      "INFO:tensorflow:input_ids: 3 2 148 2 78 2 2 2 5 2 78 2 2 2 2 16 16 5 5 18 4 2 5 5 7 2 78 2 2 489 177 2 2 112 278 2 17 5 5 2 256 7 2 75 466 235 2 29 2 16 17 18 48 346 26 5 2 23 2 319 2 26 2 2 23 2 11 17 2 11 5 2 898 2 2 7 2 2 2 2 1634 30 148 2 2 5 5 2 18 2 2 2 12 2 23 2 2 2 2 12 2 5 18 2 2 5 2 319 5 2 2 12 2 2 7 5 2 2 8 2 28 8 2 319 2 2 18 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 8 17 18 22 23 37 38 55 70 72 80 82 85 86 101 105 106 108 115 0\n",
      "INFO:tensorflow:masked_lm_positions: 8 17 18 22 23 37 38 55 70 72 80 82 85 86 101 105 106 108 115 0\n",
      "INFO:tensorflow:masked_lm_ids: 26 12 17 2 2 18 100 2 8 2 2 148 8 2 26 17 2 256 2 0\n",
      "INFO:tensorflow:masked_lm_ids: 26 12 17 2 2 18 100 2 8 2 2 148 8 2 26 17 2 256 2 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] က ကန ##က ##က [UNK] သည [UNK] စစ ##တ သတ [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] သည ။ [MASK] [MASK] [UNK] [UNK] [UNK] ။ ပထမ [UNK] မန [UNK] က [UNK] [UNK] [UNK] ၌ [UNK] တစ [UNK] တင [MASK] [UNK] [MASK] [MASK] ၏ [UNK] [UNK] သည အစဉ မနက ##န ရ [MASK] ဟ [UNK] [UNK] ၏ ။ [UNK] ၁၉၇၄ [UNK] [MASK] ရ [UNK] ၊ [UNK] [UNK] [MASK] [UNK] [UNK] ၊ [UNK] [UNK] ၊ [UNK] [MASK] [MASK] [MASK] [UNK] ဆင နင [UNK] ၊ [UNK] ၊ [MASK] ##က ##က နင [UNK] [UNK] [UNK] က [UNK] ပစ သည ။ ပလ ##တ သည [MASK] [UNK] [MASK] ခ နင လန [SEP] ဘ ##လ ##ဂ ##တ ##စ က အသက ၁၃ နစ [UNK] တင l ##a ##ke ##si ##de [UNK] ၌ စတင [MASK] သည ။ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] က ကန ##က ##က [UNK] သည [UNK] စစ ##တ သတ [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] သည ။ [MASK] [MASK] [UNK] [UNK] [UNK] ။ ပထမ [UNK] မန [UNK] က [UNK] [UNK] [UNK] ၌ [UNK] တစ [UNK] တင [MASK] [UNK] [MASK] [MASK] ၏ [UNK] [UNK] သည အစဉ မနက ##န ရ [MASK] ဟ [UNK] [UNK] ၏ ။ [UNK] ၁၉၇၄ [UNK] [MASK] ရ [UNK] ၊ [UNK] [UNK] [MASK] [UNK] [UNK] ၊ [UNK] [UNK] ၊ [UNK] [MASK] [MASK] [MASK] [UNK] ဆင နင [UNK] ၊ [UNK] ၊ [MASK] ##က ##က နင [UNK] [UNK] [UNK] က [UNK] ပစ သည ။ ပလ ##တ သည [MASK] [UNK] [MASK] ခ နင လန [SEP] ဘ ##လ ##ဂ ##တ ##စ က အသက ၁၃ နစ [UNK] တင l ##a ##ke ##si ##de [UNK] ၌ စတင [MASK] သည ။ [SEP]\n",
      "INFO:tensorflow:input_ids: 3 7 97 169 169 2 17 2 58 235 59 2 2 5 2 2 2 17 18 5 5 2 2 2 18 193 2 87 2 7 2 2 2 47 2 30 2 78 5 2 5 5 26 2 2 17 218 457 217 12 5 29 2 2 26 18 2 617 2 5 12 2 23 2 2 5 2 2 23 2 2 23 2 5 5 5 2 83 319 2 23 2 23 5 169 169 319 2 2 2 7 2 121 17 18 261 235 17 5 2 5 16 319 202 4 28 367 324 235 199 7 170 290 148 2 78 863 259 1202 683 403 2 47 128 5 17 18 4\n",
      "INFO:tensorflow:input_ids: 3 7 97 169 169 2 17 2 58 235 59 2 2 5 2 2 2 17 18 5 5 2 2 2 18 193 2 87 2 7 2 2 2 47 2 30 2 78 5 2 5 5 26 2 2 17 218 457 217 12 5 29 2 2 26 18 2 617 2 5 12 2 23 2 2 5 2 2 23 2 2 23 2 5 5 5 2 83 319 2 23 2 23 5 169 169 319 2 2 2 7 2 121 17 18 261 235 17 5 2 5 16 319 202 4 28 367 324 235 199 7 170 290 148 2 78 863 259 1202 683 403 2 47 128 5 17 18 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 10 13 19 20 26 38 40 41 50 59 65 73 74 75 77 83 98 100 124 0\n",
      "INFO:tensorflow:masked_lm_positions: 10 13 19 20 26 38 40 41 50 59 65 73 74 75 77 83 98 100 124 0\n",
      "INFO:tensorflow:masked_lm_ids: 207 13 25 2 2 2 2 11 51 2 23 2 23 2 2 72 20 249 1261 0\n",
      "INFO:tensorflow:masked_lm_ids: 207 13 25 2 2 2 2 11 51 2 23 2 23 2 2 72 20 249 1261 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ရန ##က ##န မ [UNK] သ [UNK] [MASK] [UNK] နင သည ။ [SEP] [UNK] [UNK] ၍ အန ##ဒ ##ယ ##န ##င ##င တင [UNK] ရ ##ys သည ။ [MASK] [UNK] တင တက [MASK] ##တ [UNK] သည [MASK] အရန [MASK] [UNK] [UNK] [UNK] [UNK] သည ဟ ထင ရ [MASK] [UNK] [UNK] လ သည ဟ ဆ [UNK] သည ။ ဒယ လ တတ တ လ ရ [UNK] [UNK] ။ [UNK] က ဒ [UNK] [UNK] [UNK] [UNK] [MASK] က ဆပ ##င က တ တ ည ##ပ [MASK] [UNK] ။ [UNK] [MASK] [UNK] အလ ##န [MASK] [MASK] ။ n ##x x ##m [UNK] ၏ [UNK] [UNK] သမ ##ဟ ##တ လက ##သ ##စ ဂန ##ထ [MASK] ##င [UNK] ကစ ##စ ရပ [UNK] က [UNK] [UNK] [MASK] [UNK] ရန [UNK] တစ ခ စ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ရန ##က ##န မ [UNK] သ [UNK] [MASK] [UNK] နင သည ။ [SEP] [UNK] [UNK] ၍ အန ##ဒ ##ယ ##န ##င ##င တင [UNK] ရ ##ys သည ။ [MASK] [UNK] တင တက [MASK] ##တ [UNK] သည [MASK] အရန [MASK] [UNK] [UNK] [UNK] [UNK] သည ဟ ထင ရ [MASK] [UNK] [UNK] လ သည ဟ ဆ [UNK] သည ။ ဒယ လ တတ တ လ ရ [UNK] [UNK] ။ [UNK] က ဒ [UNK] [UNK] [UNK] [UNK] [MASK] က ဆပ ##င က တ တ ည ##ပ [MASK] [UNK] ။ [UNK] [MASK] [UNK] အလ ##န [MASK] [MASK] ။ n ##x x ##m [UNK] ၏ [UNK] [UNK] သမ ##ဟ ##တ လက ##သ ##စ ဂန ##ထ [MASK] ##င [UNK] ကစ ##စ ရပ [UNK] က [UNK] [UNK] [MASK] [UNK] ရန [UNK] တစ ခ စ [SEP]\n",
      "INFO:tensorflow:input_ids: 3 41 169 217 8 2 13 2 5 2 319 17 18 4 2 2 33 48 346 278 217 177 177 78 2 12 1037 17 18 5 2 78 60 5 235 2 17 5 981 5 2 2 2 2 17 29 124 12 5 2 2 15 17 29 22 2 17 18 388 15 92 11 15 12 2 2 18 2 7 25 2 2 2 2 5 7 552 177 7 11 11 27 207 5 2 18 2 5 2 52 217 5 5 18 862 447 1873 402 2 26 2 2 75 466 235 49 332 199 322 413 5 177 2 267 199 72 2 7 2 2 5 2 41 2 30 16 14 4\n",
      "INFO:tensorflow:input_ids: 3 41 169 217 8 2 13 2 5 2 319 17 18 4 2 2 33 48 346 278 217 177 177 78 2 12 1037 17 18 5 2 78 60 5 235 2 17 5 981 5 2 2 2 2 17 29 124 12 5 2 2 15 17 29 22 2 17 18 388 15 92 11 15 12 2 2 18 2 7 25 2 2 2 2 5 7 552 177 7 11 11 27 207 5 2 18 2 5 2 52 217 5 5 18 862 447 1873 402 2 26 2 2 75 466 235 49 332 199 322 413 5 177 2 267 199 72 2 7 2 2 5 2 41 2 30 16 14 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 8 26 29 33 37 39 48 58 74 83 87 91 92 98 103 106 110 114 120 0\n",
      "INFO:tensorflow:masked_lm_positions: 8 26 29 33 37 39 48 58 74 83 87 91 92 98 103 106 110 114 120 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 2 2 369 2 2 2 2 2 2 17 2 17 2 466 332 164 199 2 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 2 2 369 2 2 2 2 2 2 17 2 17 2 466 332 164 199 2 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] အရ [MASK] က ၂၀၀၅ ခန ##စ [UNK] ၅ ရက တင [UNK] mi ##ch ##a ##el ##e . b ##ro ##w ##n [MASK] [UNK] ch ##ad ##tr ##u ##ji [MASK] ##o နင [UNK] d ##a ##vi ##dr ##ab ##in ##o ##wi ##t ##z တ မ [UNK] [UNK] ရ ခ သည ။ [SEP] [UNK] [UNK] နင ဖဆပလ [MASK] ၏ [UNK] [UNK] ထ [MASK] မ ဟတ [UNK] ။ [UNK] [MASK] ယ [UNK] [UNK] [MASK] [MASK] ဒတ ##ယ [UNK] [UNK] [UNK] [UNK] က [UNK] [UNK] [UNK] ၌ [UNK] တစ [MASK] သည [UNK] [UNK] တစ [UNK] နင [UNK] [MASK] တစ ခ ထက [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] ထ [UNK] [MASK] [UNK] [UNK] သည [MASK] [UNK] နင [UNK] [UNK] [MASK] ရ [MASK] [MASK] က [UNK] [UNK] သည [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] အရ [MASK] က ၂၀၀၅ ခန ##စ [UNK] ၅ ရက တင [UNK] mi ##ch ##a ##el ##e . b ##ro ##w ##n [MASK] [UNK] ch ##ad ##tr ##u ##ji [MASK] ##o နင [UNK] d ##a ##vi ##dr ##ab ##in ##o ##wi ##t ##z တ မ [UNK] [UNK] ရ ခ သည ။ [SEP] [UNK] [UNK] နင ဖဆပလ [MASK] ၏ [UNK] [UNK] ထ [MASK] မ ဟတ [UNK] ။ [UNK] [MASK] ယ [UNK] [UNK] [MASK] [MASK] ဒတ ##ယ [UNK] [UNK] [UNK] [UNK] က [UNK] [UNK] [UNK] ၌ [UNK] တစ [MASK] သည [UNK] [UNK] တစ [UNK] နင [UNK] [MASK] တစ ခ ထက [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] ထ [UNK] [MASK] [UNK] [UNK] သည [MASK] [UNK] နင [UNK] [UNK] [MASK] ရ [MASK] [MASK] က [UNK] [UNK] သည [SEP]\n",
      "INFO:tensorflow:input_ids: 3 39 5 7 613 82 199 2 196 80 78 2 1524 526 259 728 195 510 644 650 396 224 5 2 1302 1439 1381 337 1056 5 316 319 2 1803 259 769 1431 1440 420 316 1039 255 680 11 8 2 2 12 16 17 18 4 2 2 319 305 5 26 2 2 20 5 8 983 2 18 2 5 24 2 2 5 5 459 278 2 2 2 2 7 2 2 2 47 2 30 5 17 2 2 30 2 319 2 5 30 16 101 2 5 2 2 2 2 2 2 2 2 20 2 5 2 2 17 5 2 319 2 2 5 12 5 5 7 2 2 17 4\n",
      "INFO:tensorflow:input_ids: 3 39 5 7 613 82 199 2 196 80 78 2 1524 526 259 728 195 510 644 650 396 224 5 2 1302 1439 1381 337 1056 5 316 319 2 1803 259 769 1431 1440 420 316 1039 255 680 11 8 2 2 12 16 17 18 4 2 2 319 305 5 26 2 2 20 5 8 983 2 18 2 5 24 2 2 5 5 459 278 2 2 2 2 7 2 2 2 47 2 30 5 17 2 2 30 2 319 2 5 30 16 101 2 5 2 2 2 2 2 2 2 2 20 2 5 2 2 17 5 2 319 2 2 5 12 5 5 7 2 2 17 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 2 10 18 22 29 56 61 67 71 72 86 88 94 99 110 114 119 121 122 0\n",
      "INFO:tensorflow:masked_lm_positions: 2 10 18 22 29 56 61 67 71 72 86 88 94 99 110 114 119 121 122 0\n",
      "INFO:tensorflow:masked_lm_ids: 199 78 644 23 562 11 149 7 2 18 2 2 33 2 26 2 2 51 2 0\n",
      "INFO:tensorflow:masked_lm_ids: 199 78 644 23 562 11 149 7 2 18 2 2 33 2 26 2 2 51 2 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] [UNK] [UNK] မ ခန [MASK] မင ခန [UNK] သည ။ [UNK] [UNK] [UNK] [UNK] [MASK] ။ [SEP] က [UNK] [UNK] [UNK] ။ [UNK] [MASK] ၏ [MASK] ##လ ##န ##ရ တင မ [UNK] မ [UNK] [UNK] ။ [UNK] သည [UNK] ၊ [UNK] တ မ [UNK] ရန ##က ##န တင [UNK] ခ [UNK] [UNK] [UNK] [UNK] [UNK] [MASK] [MASK] အရ ##စ တင [UNK] d ##ys ##no ##mi ##a အမည ရ [UNK] [UNK] ၁ [UNK] ရ [UNK] ၂၀၀၅ ခန ##စ [MASK] ၁၀ ရက တင [UNK] [UNK] ရ ခ သည ။ [UNK] [UNK] ၄ မင [UNK] ##ac [UNK] [MASK] [UNK] [UNK] တင [UNK] သ ၁ [UNK] [UNK] [UNK] က [MASK] မ [UNK] [MASK] ၉ [UNK] အကန ##ခ ၍ [UNK] [MASK] [UNK] [MASK] ##ဝ [UNK] နစ တင [UNK] [MASK] [UNK] [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] [UNK] [UNK] မ ခန [MASK] မင ခန [UNK] သည ။ [UNK] [UNK] [UNK] [UNK] [MASK] ။ [SEP] က [UNK] [UNK] [UNK] ။ [UNK] [MASK] ၏ [MASK] ##လ ##န ##ရ တင မ [UNK] မ [UNK] [UNK] ။ [UNK] သည [UNK] ၊ [UNK] တ မ [UNK] ရန ##က ##န တင [UNK] ခ [UNK] [UNK] [UNK] [UNK] [UNK] [MASK] [MASK] အရ ##စ တင [UNK] d ##ys ##no ##mi ##a အမည ရ [UNK] [UNK] ၁ [UNK] ရ [UNK] ၂၀၀၅ ခန ##စ [MASK] ၁၀ ရက တင [UNK] [UNK] ရ ခ သည ။ [UNK] [UNK] ၄ မင [UNK] ##ac [UNK] [MASK] [UNK] [UNK] တင [UNK] သ ၁ [UNK] [UNK] [UNK] က [MASK] မ [UNK] [MASK] ၉ [UNK] အကန ##ခ ၍ [UNK] [MASK] [UNK] [MASK] ##ဝ [UNK] နစ တင [UNK] [MASK] [UNK] [SEP]\n",
      "INFO:tensorflow:input_ids: 3 2 2 2 8 82 5 79 82 2 17 18 2 2 2 2 5 18 4 7 2 2 2 18 2 5 26 5 367 217 223 78 8 2 8 2 2 18 2 17 2 23 2 11 8 2 41 169 217 78 2 16 2 2 2 2 2 5 5 39 199 78 2 1803 1037 685 651 259 104 12 2 2 184 2 12 2 613 82 199 5 281 80 78 2 2 12 16 17 18 2 2 190 79 2 826 2 5 2 2 78 2 13 184 2 2 2 7 5 8 2 5 275 2 744 369 33 2 5 2 5 164 2 148 78 2 5 2 4\n",
      "INFO:tensorflow:input_ids: 3 2 2 2 8 82 5 79 82 2 17 18 2 2 2 2 5 18 4 7 2 2 2 18 2 5 26 5 367 217 223 78 8 2 8 2 2 18 2 17 2 23 2 11 8 2 41 169 217 78 2 16 2 2 2 2 2 5 5 39 199 78 2 1803 1037 685 651 259 104 12 2 2 184 2 12 2 613 82 199 5 281 80 78 2 2 12 16 17 18 2 2 190 79 2 826 2 5 2 2 78 2 13 184 2 2 2 7 5 8 2 5 275 2 744 369 33 2 5 2 5 164 2 148 78 2 5 2 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 4 6 16 25 27 57 58 79 86 88 94 96 102 103 107 110 117 119 125 0\n",
      "INFO:tensorflow:masked_lm_positions: 4 6 16 25 27 57 58 79 86 88 94 96 102 103 107 110 117 119 125 0\n",
      "INFO:tensorflow:masked_lm_ids: 8 199 17 11 22 17 18 2 16 18 12 319 2 2 2 2 16 569 16 0\n",
      "INFO:tensorflow:masked_lm_ids: 8 199 17 11 22 17 18 2 16 18 12 319 2 2 2 2 16 569 16 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [MASK] ၏ [UNK] [UNK] [UNK] ကသ ရ [MASK] [UNK] ၏ ။ [UNK] လန ခ [UNK] [UNK] ဆယ စနစ [UNK] တင [UNK] [MASK] [UNK] နစ နင ##င [UNK] [UNK] ခ [MASK] ။ [MASK] [UNK] [UNK] [UNK] တ [UNK] ရ [UNK] [UNK] ။ ပက ##ဆ [MASK] ##တ [MASK] ခ လက ရ တယ ။ [MASK] [MASK] တင လ ##ထ က သ နင ##အ [MASK] [UNK] ရန [UNK] နင [MASK] သည ။ [UNK] တင [UNK] [UNK] [UNK] တပ ##ဖ [UNK] [UNK] [UNK] 人 ။ [UNK] [UNK] [UNK] [UNK] [UNK] ။ မမ တ နင [MASK] တင မမ [MASK] [UNK] သခင [UNK] [SEP] [UNK] [MASK] [UNK] ၍ [UNK] [MASK] [UNK] နင [UNK] ပတ ##လ ##ည တင [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] က [MASK] ##က [UNK] မတမ နင သည [MASK] [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [MASK] ၏ [UNK] [UNK] [UNK] ကသ ရ [MASK] [UNK] ၏ ။ [UNK] လန ခ [UNK] [UNK] ဆယ စနစ [UNK] တင [UNK] [MASK] [UNK] နစ နင ##င [UNK] [UNK] ခ [MASK] ။ [MASK] [UNK] [UNK] [UNK] တ [UNK] ရ [UNK] [UNK] ။ ပက ##ဆ [MASK] ##တ [MASK] ခ လက ရ တယ ။ [MASK] [MASK] တင လ ##ထ က သ နင ##အ [MASK] [UNK] ရန [UNK] နင [MASK] သည ။ [UNK] တင [UNK] [UNK] [UNK] တပ ##ဖ [UNK] [UNK] [UNK] 人 ။ [UNK] [UNK] [UNK] [UNK] [UNK] ။ မမ တ နင [MASK] တင မမ [MASK] [UNK] သခင [UNK] [SEP] [UNK] [MASK] [UNK] ၍ [UNK] [MASK] [UNK] နင [UNK] ပတ ##လ ##ည တင [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] က [MASK] ##က [UNK] မတမ နင သည [MASK] [SEP]\n",
      "INFO:tensorflow:input_ids: 3 5 26 2 2 2 149 12 5 2 26 18 2 202 16 2 2 135 127 2 78 2 5 2 148 319 177 2 2 16 5 18 5 2 2 2 11 2 12 2 2 18 315 433 5 235 5 16 49 12 57 18 5 5 78 15 413 7 13 319 412 5 2 41 2 319 5 17 18 2 78 2 2 2 88 523 2 2 2 1800 18 2 2 2 2 2 18 382 11 319 5 78 382 5 2 137 2 4 2 5 2 33 2 5 2 319 2 89 367 329 78 2 2 2 2 2 2 2 2 7 5 169 2 1269 319 17 5 4\n",
      "INFO:tensorflow:input_ids: 3 5 26 2 2 2 149 12 5 2 26 18 2 202 16 2 2 135 127 2 78 2 5 2 148 319 177 2 2 16 5 18 5 2 2 2 11 2 12 2 2 18 315 433 5 235 5 16 49 12 57 18 5 5 78 15 413 7 13 319 412 5 2 41 2 319 5 17 18 2 78 2 2 2 88 523 2 2 2 1800 18 2 2 2 2 2 18 382 11 319 5 78 382 5 2 137 2 4 2 5 2 33 2 5 2 319 2 89 367 329 78 2 2 2 2 2 2 2 2 7 5 169 2 1269 319 17 5 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 1 8 22 30 32 44 46 48 52 53 61 66 79 90 93 99 103 120 126 0\n",
      "INFO:tensorflow:masked_lm_positions: 1 8 22 30 32 44 46 48 52 53 61 66 79 90 93 99 103 120 126 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 2 2 17 2 412 2 49 100 2 235 16 17 177 11 78 2 111 18 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 2 2 17 2 412 2 49 100 2 235 16 17 177 11 78 2 111 18 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] [UNK] တင [UNK] [UNK] မ [UNK] ၊ [UNK] ကစ [MASK] [UNK] [MASK] [UNK] သစ တည ရန အတ ##က [UNK] [MASK] [UNK] ခ သည ။ [UNK] တ ၏ [UNK] [UNK] က [MASK] [UNK] မ [UNK] မ [UNK] နစ [UNK] [UNK] က [UNK] [UNK] [UNK] [UNK] တင [UNK] ရ [MASK] သည ။ [UNK] ၊ [UNK] [UNK] ရ [UNK] ၊ ခမည [UNK] တ နင [UNK] [MASK] [MASK] [UNK] [UNK] [UNK] သည [MASK] [UNK] ထ [UNK] [MASK] က [UNK] ရစ ခ နင [UNK] ဝတ ##ထ ##တ [UNK] [MASK] [UNK] မ [UNK] [UNK] ည [UNK] နင [UNK] [MASK] ##ထ ၁၂၃ [UNK] [MASK] [UNK] သည ။ ##op [UNK] အန တယ [SEP] မင [UNK] [UNK] လက ရ [UNK] [UNK] [UNK] [UNK] [MASK] သည ဆင [UNK] z နင [UNK] တင ရ သည ။ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] [UNK] တင [UNK] [UNK] မ [UNK] ၊ [UNK] ကစ [MASK] [UNK] [MASK] [UNK] သစ တည ရန အတ ##က [UNK] [MASK] [UNK] ခ သည ။ [UNK] တ ၏ [UNK] [UNK] က [MASK] [UNK] မ [UNK] မ [UNK] နစ [UNK] [UNK] က [UNK] [UNK] [UNK] [UNK] တင [UNK] ရ [MASK] သည ။ [UNK] ၊ [UNK] [UNK] ရ [UNK] ၊ ခမည [UNK] တ နင [UNK] [MASK] [MASK] [UNK] [UNK] [UNK] သည [MASK] [UNK] ထ [UNK] [MASK] က [UNK] ရစ ခ နင [UNK] ဝတ ##ထ ##တ [UNK] [MASK] [UNK] မ [UNK] [UNK] ည [UNK] နင [UNK] [MASK] ##ထ ၁၂၃ [UNK] [MASK] [UNK] သည ။ ##op [UNK] အန တယ [SEP] မင [UNK] [UNK] လက ရ [UNK] [UNK] [UNK] [UNK] [MASK] သည ဆင [UNK] z နင [UNK] တင ရ သည ။ [SEP]\n",
      "INFO:tensorflow:input_ids: 3 2 2 78 2 2 8 2 23 2 267 5 2 5 2 91 55 41 36 169 2 5 2 16 17 18 2 11 26 2 2 7 5 2 8 2 8 2 148 2 2 7 2 2 2 2 78 2 12 5 17 18 2 23 2 2 12 2 23 1286 2 11 319 2 5 5 2 2 2 17 5 2 20 2 5 7 2 144 16 319 2 109 413 235 2 5 2 8 2 2 27 2 319 2 5 413 1469 2 5 2 17 18 954 2 48 57 4 79 2 2 49 12 2 2 2 2 5 17 83 2 1512 319 2 78 12 17 18 4\n",
      "INFO:tensorflow:input_ids: 3 2 2 78 2 2 8 2 23 2 267 5 2 5 2 91 55 41 36 169 2 5 2 16 17 18 2 11 26 2 2 7 5 2 8 2 8 2 148 2 2 7 2 2 2 2 78 2 12 5 17 18 2 23 2 2 12 2 23 1286 2 11 319 2 5 5 2 2 2 17 5 2 20 2 5 7 2 144 16 319 2 109 413 235 2 5 2 8 2 2 27 2 319 2 5 413 1469 2 5 2 17 18 954 2 48 57 4 79 2 2 49 12 2 2 2 2 5 17 83 2 1512 319 2 78 12 17 18 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 11 13 21 32 49 56 59 64 65 70 74 81 85 94 96 98 102 116 120 0\n",
      "INFO:tensorflow:masked_lm_positions: 11 13 21 32 49 56 59 64 65 70 74 81 85 94 96 98 102 116 120 0\n",
      "INFO:tensorflow:masked_lm_ids: 199 2 7 2 12 12 87 2 13 18 2 109 23 109 235 11 2 319 2 0\n",
      "INFO:tensorflow:masked_lm_ids: 199 2 7 2 12 12 87 2 13 18 2 109 23 109 235 11 2 319 2 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] က [UNK] [UNK] န ရ [MASK] [UNK] [MASK] [UNK] [UNK] တကဗ [UNK] သရ [UNK] [UNK] [UNK] [UNK] ၅၅၀ , ၀ ##၀၀ [UNK] အထ [UNK] [UNK] [UNK] [UNK] သည ။ [UNK] က [UNK] [UNK] [UNK] [UNK] ။ [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] တ [UNK] ၀ [MASK] ##၁ အမတ က [MASK] [UNK] [UNK] [UNK] တယ ။ [SEP] အရ [MASK] [UNK] st [MASK] ##ar [MASK] ##nd [UNK] [UNK] ၏ [UNK] က ၃၀ [UNK] [UNK] [MASK] [UNK] [UNK] မည [MASK] [UNK] [UNK] [MASK] ပမ [MASK] [MASK] [UNK] သ [UNK] [UNK] မည [UNK] သည ဟ ဆ [UNK] သည ။ [UNK] [UNK] ရ [UNK] [UNK] တင [MASK] [UNK] [UNK] [UNK] တင [UNK] ၁၁၂ [UNK] ၍ ပ ##ထ ##လ ##ပ [UNK] [UNK] ဘရ [MASK] ၏ [UNK] ပ နင တ [UNK] [UNK] [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] က [UNK] [UNK] န ရ [MASK] [UNK] [MASK] [UNK] [UNK] တကဗ [UNK] သရ [UNK] [UNK] [UNK] [UNK] ၅၅၀ , ၀ ##၀၀ [UNK] အထ [UNK] [UNK] [UNK] [UNK] သည ။ [UNK] က [UNK] [UNK] [UNK] [UNK] ။ [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] တ [UNK] ၀ [MASK] ##၁ အမတ က [MASK] [UNK] [UNK] [UNK] တယ ။ [SEP] အရ [MASK] [UNK] st [MASK] ##ar [MASK] ##nd [UNK] [UNK] ၏ [UNK] က ၃၀ [UNK] [UNK] [MASK] [UNK] [UNK] မည [MASK] [UNK] [UNK] [MASK] ပမ [MASK] [MASK] [UNK] သ [UNK] [UNK] မည [UNK] သည ဟ ဆ [UNK] သည ။ [UNK] [UNK] ရ [UNK] [UNK] တင [MASK] [UNK] [UNK] [UNK] တင [UNK] ၁၁၂ [UNK] ၍ ပ ##ထ ##လ ##ပ [UNK] [UNK] ဘရ [MASK] ၏ [UNK] ပ နင တ [UNK] [UNK] [SEP]\n",
      "INFO:tensorflow:input_ids: 3 2 7 2 2 10 12 5 2 5 2 2 999 2 304 2 2 2 2 1445 336 700 522 2 62 2 2 2 2 17 18 2 7 2 2 2 2 18 5 2 2 2 5 2 11 2 700 5 240 436 7 5 2 2 2 57 18 4 39 5 2 931 5 656 5 607 2 2 26 2 7 351 2 2 5 2 2 51 5 2 2 5 208 5 5 2 13 2 2 51 2 17 29 22 2 17 18 2 2 12 2 2 78 5 2 2 2 78 2 1473 2 33 9 413 367 207 2 2 489 5 26 2 9 319 11 2 2 4\n",
      "INFO:tensorflow:input_ids: 3 2 7 2 2 10 12 5 2 5 2 2 999 2 304 2 2 2 2 1445 336 700 522 2 62 2 2 2 2 17 18 2 7 2 2 2 2 18 5 2 2 2 5 2 11 2 700 5 240 436 7 5 2 2 2 57 18 4 39 5 2 931 5 656 5 607 2 2 26 2 7 351 2 2 5 2 2 51 5 2 2 5 208 5 5 2 13 2 2 51 2 17 29 22 2 17 18 2 2 12 2 2 78 5 2 2 2 78 2 1473 2 33 9 413 367 207 2 2 489 5 26 2 9 319 11 2 2 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 7 9 12 30 38 42 47 51 59 62 64 74 78 81 83 84 103 109 119 0\n",
      "INFO:tensorflow:masked_lm_positions: 7 9 12 30 38 42 47 51 59 62 64 74 78 81 83 84 103 109 119 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 18 62 18 220 7 180 2 2 1063 1039 2 2 17 2 2 2 2 177 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 18 62 18 220 7 180 2 2 1063 1039 2 2 17 2 2 2 2 177 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] ၊ [UNK] တ ၌ စ ##ယ ##စ ##ဗ ##ဟ ##သ ##တ ရ [UNK] [UNK] [UNK] [UNK] ဘ သက ##သ [UNK] တင ရ [MASK] [MASK] [MASK] ##တ [UNK] [UNK] [MASK] ##3 [UNK] သပ ##ပ နင [UNK] [UNK] ၌ စ ##ယ ##စ ##ဗ ##ဟ [MASK] [MASK] ရ [MASK] ရည ##ရ ##ယ [UNK] [MASK] [MASK] ။ [UNK] သ ၏ [UNK] [UNK] အစဉ [UNK] [MASK] [UNK] [SEP] ခန ##စ တင [UNK] သည ပထဝ [UNK] နင [UNK] [UNK] [UNK] [UNK] [UNK] ၎ [UNK] တ ၏ [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] ရ [UNK] မ [UNK] သ [UNK] ခ [MASK] မ စတင [UNK] [MASK] [UNK] [UNK] [UNK] သည ။ [UNK] သ ရ လတ ##လ ##ပ [MASK] ##င က မ [MASK] [UNK] ဆ ရင လတ ##လ ##တ ##လ ##ပ ##လ ##ပ [UNK] [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] ၊ [UNK] တ ၌ စ ##ယ ##စ ##ဗ ##ဟ ##သ ##တ ရ [UNK] [UNK] [UNK] [UNK] ဘ သက ##သ [UNK] တင ရ [MASK] [MASK] [MASK] ##တ [UNK] [UNK] [MASK] ##3 [UNK] သပ ##ပ နင [UNK] [UNK] ၌ စ ##ယ ##စ ##ဗ ##ဟ [MASK] [MASK] ရ [MASK] ရည ##ရ ##ယ [UNK] [MASK] [MASK] ။ [UNK] သ ၏ [UNK] [UNK] အစဉ [UNK] [MASK] [UNK] [SEP] ခန ##စ တင [UNK] သည ပထဝ [UNK] နင [UNK] [UNK] [UNK] [UNK] [UNK] ၎ [UNK] တ ၏ [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] ရ [UNK] မ [UNK] သ [UNK] ခ [MASK] မ စတင [UNK] [MASK] [UNK] [UNK] [UNK] သည ။ [UNK] သ ရ လတ ##လ ##ပ [MASK] ##င က မ [MASK] [UNK] ဆ ရင လတ ##လ ##တ ##လ ##ပ ##လ ##ပ [UNK] [SEP]\n",
      "INFO:tensorflow:input_ids: 3 2 23 2 11 47 14 278 199 373 466 332 235 12 2 2 2 2 28 67 332 2 78 12 5 5 5 235 2 2 5 1154 2 192 207 319 2 2 47 14 278 199 373 466 5 5 12 5 145 223 278 2 5 5 18 2 13 26 2 2 218 2 5 2 4 82 199 78 2 17 548 2 319 2 2 2 2 2 1878 2 11 26 2 5 2 2 2 2 12 2 8 2 13 2 16 5 8 128 2 5 2 2 2 17 18 2 13 12 222 367 207 5 177 7 8 5 2 22 56 222 367 235 367 207 367 207 2 4\n",
      "INFO:tensorflow:input_ids: 3 2 23 2 11 47 14 278 199 373 466 332 235 12 2 2 2 2 28 67 332 2 78 12 5 5 5 235 2 2 5 1154 2 192 207 319 2 2 47 14 278 199 373 466 5 5 12 5 145 223 278 2 5 5 18 2 13 26 2 2 218 2 5 2 4 82 199 78 2 17 548 2 319 2 2 2 2 2 1878 2 11 26 2 5 2 2 2 2 12 2 8 2 13 2 16 5 8 128 2 5 2 2 2 17 18 2 13 12 222 367 207 5 177 7 8 5 2 22 56 222 367 235 367 207 367 207 2 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 14 24 25 26 30 31 34 44 45 47 52 53 60 62 83 95 99 111 115 0\n",
      "INFO:tensorflow:masked_lm_positions: 14 24 25 26 30 31 34 44 45 47 52 53 60 62 83 95 99 111 115 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 2 172 332 11 17 207 332 235 41 2 17 218 2 2 2 2 369 2 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 2 172 332 11 17 207 332 235 41 2 17 218 2 2 2 2 369 2 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] မ [MASK] ၍ ၄ [UNK] [UNK] ၄ [UNK] [UNK] [UNK] [MASK] [MASK] ရ သည ။ [MASK] သည ကလ ##သ ##မ ##ဂ ##ဂ အဖ ##ဝ ##င [MASK] ##င [UNK] သည ။ [UNK] [MASK] [UNK] [MASK] ၂၆ [UNK] ၁၉၁ ##၇ တင [UNK] ၌ [UNK] [MASK] တ [MASK] [UNK] ခ သည [MASK] [UNK] ၂၀၀၆ ခန ##စ [UNK] ၁၃ ရက တင [UNK] မ [UNK] ၏ အမည [MASK] ယ ၍ အရ ##စ ဟ [UNK] [UNK] ခ [SEP] [UNK] တစ [UNK] [UNK] တ st ##e ##ve j ##ob ##s [UNK] ကယ ##လ ##န [MASK] [UNK] အသက ၅၆ နစ [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] ခ ရ [UNK] [UNK] [UNK] တယ ။ [UNK] ၆၀ [UNK] [MASK] ။ [UNK] [UNK] သည [UNK] [UNK] က [UNK] တစ ခ [MASK] တစ [MASK] [UNK] [UNK] သည [MASK] [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] မ [MASK] ၍ ၄ [UNK] [UNK] ၄ [UNK] [UNK] [UNK] [MASK] [MASK] ရ သည ။ [MASK] သည ကလ ##သ ##မ ##ဂ ##ဂ အဖ ##ဝ ##င [MASK] ##င [UNK] သည ။ [UNK] [MASK] [UNK] [MASK] ၂၆ [UNK] ၁၉၁ ##၇ တင [UNK] ၌ [UNK] [MASK] တ [MASK] [UNK] ခ သည [MASK] [UNK] ၂၀၀၆ ခန ##စ [UNK] ၁၃ ရက တင [UNK] မ [UNK] ၏ အမည [MASK] ယ ၍ အရ ##စ ဟ [UNK] [UNK] ခ [SEP] [UNK] တစ [UNK] [UNK] တ st ##e ##ve j ##ob ##s [UNK] ကယ ##လ ##န [MASK] [UNK] အသက ၅၆ နစ [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] ခ ရ [UNK] [UNK] [UNK] တယ ။ [UNK] ၆၀ [UNK] [MASK] ။ [UNK] [UNK] သည [UNK] [UNK] က [UNK] တစ ခ [MASK] တစ [MASK] [UNK] [UNK] သည [MASK] [SEP]\n",
      "INFO:tensorflow:input_ids: 3 8 5 33 190 2 2 190 2 2 2 5 5 12 17 18 5 17 114 332 243 324 324 43 164 177 5 177 2 17 18 2 5 2 5 414 2 569 231 78 2 47 2 5 11 5 2 16 17 5 2 405 82 199 2 290 80 78 2 8 2 26 104 5 24 33 39 199 29 2 2 16 4 2 30 2 2 11 931 195 584 1863 1049 171 2 213 367 217 5 2 170 966 148 2 2 2 5 2 2 16 12 2 2 2 57 18 2 1443 2 5 18 2 2 17 2 2 7 2 30 16 5 30 5 2 2 17 5 4\n",
      "INFO:tensorflow:input_ids: 3 8 5 33 190 2 2 190 2 2 2 5 5 12 17 18 5 17 114 332 243 324 324 43 164 177 5 177 2 17 18 2 5 2 5 414 2 569 231 78 2 47 2 5 11 5 2 16 17 5 2 405 82 199 2 290 80 78 2 8 2 26 104 5 24 33 39 199 29 2 2 16 4 2 30 2 2 11 931 195 584 1863 1049 171 2 213 367 217 5 2 170 966 148 2 2 2 5 2 2 16 12 2 2 2 57 18 2 1443 2 5 18 2 2 17 2 2 7 2 30 16 5 30 5 2 2 17 5 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 2 11 12 16 26 32 34 43 45 49 50 63 88 96 107 109 120 122 126 0\n",
      "INFO:tensorflow:masked_lm_positions: 2 11 12 16 26 32 34 43 45 49 50 63 88 96 107 109 120 122 126 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 78 2 2 319 114 2 2 7 18 2 7 2 7 2 57 319 16 18 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 78 2 2 319 114 2 2 7 18 2 7 2 7 2 57 319 16 18 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] အတတ [UNK] ရ [UNK] က [UNK] က ၅ [MASK] [MASK] [UNK] [MASK] [MASK] [UNK] [UNK] ။ [UNK] က [UNK] [UNK] ခ သ [UNK] [UNK] [UNK] မ [UNK] [UNK] ပင [UNK] သည ဟ ယ ##ဆ နင [UNK] သည [MASK] တတ ##ယ ##က [MASK] [MASK] ##ဏ သည ယခ [UNK] တင [MASK] နင [UNK] [UNK] [MASK] တင [UNK] တင [UNK] [UNK] စဉ [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] အဓမ [MASK] [UNK] ခ သည က [UNK] [UNK] [UNK] [UNK] အမ [MASK] ##ဆ ##င ရ [UNK] [MASK] သည ။ [UNK] [MASK] ထက [UNK] မ [UNK] ဘ [UNK] က [UNK] ဝယ ဟ [UNK] [UNK] [MASK] [UNK] ထက [UNK] ၍ [UNK] က [UNK] [UNK] ဟ [UNK] [UNK] သည [MASK] [SEP] [MASK] [UNK] သည ။ ဒ [UNK] က [UNK] [UNK] ရ [UNK] [UNK] ။ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] အတတ [UNK] ရ [UNK] က [UNK] က ၅ [MASK] [MASK] [UNK] [MASK] [MASK] [UNK] [UNK] ။ [UNK] က [UNK] [UNK] ခ သ [UNK] [UNK] [UNK] မ [UNK] [UNK] ပင [UNK] သည ဟ ယ ##ဆ နင [UNK] သည [MASK] တတ ##ယ ##က [MASK] [MASK] ##ဏ သည ယခ [UNK] တင [MASK] နင [UNK] [UNK] [MASK] တင [UNK] တင [UNK] [UNK] စဉ [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] အဓမ [MASK] [UNK] ခ သည က [UNK] [UNK] [UNK] [UNK] အမ [MASK] ##ဆ ##င ရ [UNK] [MASK] သည ။ [UNK] [MASK] ထက [UNK] မ [UNK] ဘ [UNK] က [UNK] ဝယ ဟ [UNK] [UNK] [MASK] [UNK] ထက [UNK] ၍ [UNK] က [UNK] [UNK] ဟ [UNK] [UNK] သည [MASK] [SEP] [MASK] [UNK] သည ။ ဒ [UNK] က [UNK] [UNK] ရ [UNK] [UNK] ။ [SEP]\n",
      "INFO:tensorflow:input_ids: 3 456 2 12 2 7 2 7 196 5 5 2 5 5 2 2 18 2 7 2 2 16 13 2 2 2 8 2 2 50 2 17 29 24 433 319 2 17 5 92 278 169 5 5 419 17 141 2 78 5 319 2 2 5 78 2 78 2 2 96 2 5 2 2 2 2 1480 5 2 16 17 7 2 2 2 2 35 5 433 177 12 2 5 17 18 2 5 101 2 8 2 28 2 7 2 163 29 2 2 5 2 101 2 33 2 7 2 2 29 2 2 17 5 4 5 2 17 18 25 2 7 2 2 12 2 2 18 4\n",
      "INFO:tensorflow:input_ids: 3 456 2 12 2 7 2 7 196 5 5 2 5 5 2 2 18 2 7 2 2 16 13 2 2 2 8 2 2 50 2 17 29 24 433 319 2 17 5 92 278 169 5 5 419 17 141 2 78 5 319 2 2 5 78 2 78 2 2 96 2 5 2 2 2 2 1480 5 2 16 17 7 2 2 2 2 35 5 433 177 12 2 5 17 18 2 5 101 2 8 2 28 2 7 2 163 29 2 2 5 2 101 2 33 2 7 2 2 29 2 2 17 5 4 5 2 17 18 25 2 7 2 2 12 2 2 18 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 9 10 12 13 35 38 42 43 49 53 61 67 77 82 86 95 99 112 114 0\n",
      "INFO:tensorflow:masked_lm_positions: 9 10 12 13 35 38 42 43 49 53 61 67 77 82 86 95 99 112 114 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 2 15 12 319 18 243 207 2 2 7 243 56 12 2 2 2 18 2 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 2 15 12 319 18 243 207 2 2 7 243 56 12 2 2 2 18 2 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] ဏ သည ။ [UNK] [UNK] တယ အယ [MASK] တယ ။ [UNK] နင [UNK] တ သည [UNK] တင တရ ##တ [UNK] အဖ ##ဝ ##င [UNK] အတ [MASK] [MASK] [MASK] ##ဝ ##င [UNK] [UNK] တဗ ##က [UNK] [MASK] မ [UNK] [UNK] သည ။ [MASK] [UNK] [UNK] ထက [UNK] [UNK] [UNK] တ [UNK] ပမ [MASK] [UNK] [UNK] [UNK] အမည တ အဖ [UNK] [MASK] [UNK] [MASK] [MASK] က ပင ယ ၍ ၊ [MASK] [UNK] သည ။ [UNK] [UNK] [UNK] သည [UNK] [UNK] [UNK] [UNK] ၏ [UNK] [UNK] က [UNK] [UNK] သည [MASK] ထ [MASK] နစ ခ [UNK] [UNK] ၊ [UNK] [UNK] [UNK] ၍ ရ နင [UNK] [SEP] con [MASK] ##ct ##or တစ ခ က စသ တစ ခ [UNK] သ [UNK] [UNK] ၉၀ လည သည [UNK] [MASK] [UNK] သည ။ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] ဏ သည ။ [UNK] [UNK] တယ အယ [MASK] တယ ။ [UNK] နင [UNK] တ သည [UNK] တင တရ ##တ [UNK] အဖ ##ဝ ##င [UNK] အတ [MASK] [MASK] [MASK] ##ဝ ##င [UNK] [UNK] တဗ ##က [UNK] [MASK] မ [UNK] [UNK] သည ။ [MASK] [UNK] [UNK] ထက [UNK] [UNK] [UNK] တ [UNK] ပမ [MASK] [UNK] [UNK] [UNK] အမည တ အဖ [UNK] [MASK] [UNK] [MASK] [MASK] က ပင ယ ၍ ၊ [MASK] [UNK] သည ။ [UNK] [UNK] [UNK] သည [UNK] [UNK] [UNK] [UNK] ၏ [UNK] [UNK] က [UNK] [UNK] သည [MASK] ထ [MASK] နစ ခ [UNK] [UNK] ၊ [UNK] [UNK] [UNK] ၍ ရ နင [UNK] [SEP] con [MASK] ##ct ##or တစ ခ က စသ တစ ခ [UNK] သ [UNK] [UNK] ၉၀ လည သည [UNK] [MASK] [UNK] သည ။ [SEP]\n",
      "INFO:tensorflow:input_ids: 3 2 102 17 18 2 2 57 138 5 57 18 2 319 2 11 17 2 78 68 235 2 43 164 177 2 36 5 5 5 164 177 2 2 460 169 2 5 8 2 2 17 18 5 2 2 101 2 2 2 11 2 208 5 2 2 2 104 11 43 2 5 2 5 5 7 50 24 33 23 5 2 17 18 2 2 2 17 2 2 2 2 26 2 2 7 2 2 17 5 20 5 148 16 2 2 23 2 2 2 33 12 319 2 4 1124 5 689 385 30 16 7 262 30 16 2 13 2 2 963 34 17 2 5 2 17 18 4\n",
      "INFO:tensorflow:input_ids: 3 2 102 17 18 2 2 57 138 5 57 18 2 319 2 11 17 2 78 68 235 2 43 164 177 2 36 5 5 5 164 177 2 2 460 169 2 5 8 2 2 17 18 5 2 2 101 2 2 2 11 2 208 5 2 2 2 104 11 43 2 5 2 5 5 7 50 24 33 23 5 2 17 18 2 2 2 17 2 2 2 2 26 2 2 7 2 2 17 5 20 5 148 16 2 2 23 2 2 2 33 12 319 2 4 1124 5 689 385 30 16 7 262 30 16 2 13 2 2 963 34 17 2 5 2 17 18 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 2 8 9 27 28 29 37 43 53 61 63 64 70 85 89 91 106 112 123 0\n",
      "INFO:tensorflow:masked_lm_positions: 2 8 9 27 28 29 37 43 53 61 63 64 70 85 89 91 106 112 123 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 10 11 169 60 207 7 2 2 2 2 11 2 7 18 2 498 2 2 0\n",
      "INFO:tensorflow:masked_lm_ids: 2 10 11 169 60 207 7 2 2 2 2 11 2 7 18 2 498 2 2 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] ။ [UNK] [UNK] k ##ing ##ed ##ward ##me ##mo ##ra ##l [UNK] [MASK] [MASK] [UNK] [UNK] ခ သည ။ [UNK] သည [UNK] [UNK] [UNK] [UNK] [UNK] သည ။ [UNK] [MASK] ခ ခ အလ ##ပ လပ နင ရန [MASK] ##က စက မ [UNK] နင ##$ အတ ##က [UNK] [MASK] [UNK] [UNK] ပရ ##ဂ ##ရ ##မ [MASK] ခ လ ##အ ##ပ [UNK] [UNK] သည [SEP] နင [MASK] ။ mo ထ [MASK] ကက [UNK] လ ၏ [UNK] [UNK] သည [UNK] သက ##သ [UNK] [UNK] [UNK] [MASK] ။ အပင [UNK] [UNK] [UNK] သည [MASK] [UNK] နင [UNK] တ [UNK] [UNK] [UNK] ၍ [UNK] [MASK] [UNK] [MASK] ။ ဤ [UNK] [MASK] [UNK] အက ##တ ##မ [UNK] သည [UNK] တစ ခ [UNK] နင [UNK] [UNK] [UNK] [UNK] မန [MASK] က [UNK] [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] ။ [UNK] [UNK] k ##ing ##ed ##ward ##me ##mo ##ra ##l [UNK] [MASK] [MASK] [UNK] [UNK] ခ သည ။ [UNK] သည [UNK] [UNK] [UNK] [UNK] [UNK] သည ။ [UNK] [MASK] ခ ခ အလ ##ပ လပ နင ရန [MASK] ##က စက မ [UNK] နင ##$ အတ ##က [UNK] [MASK] [UNK] [UNK] ပရ ##ဂ ##ရ ##မ [MASK] ခ လ ##အ ##ပ [UNK] [UNK] သည [SEP] နင [MASK] ။ mo ထ [MASK] ကက [UNK] လ ၏ [UNK] [UNK] သည [UNK] သက ##သ [UNK] [UNK] [UNK] [MASK] ။ အပင [UNK] [UNK] [UNK] သည [MASK] [UNK] နင [UNK] တ [UNK] [UNK] [UNK] ၍ [UNK] [MASK] [UNK] [MASK] ။ ဤ [UNK] [MASK] [UNK] အက ##တ ##မ [UNK] သည [UNK] တစ ခ [UNK] နင [UNK] [UNK] [UNK] [UNK] မန [MASK] က [UNK] [SEP]\n",
      "INFO:tensorflow:input_ids: 3 2 18 2 2 1855 563 687 1372 652 1403 467 314 2 5 5 2 2 16 17 18 2 17 2 2 2 2 2 17 18 2 5 16 16 52 207 119 319 41 5 169 63 8 2 319 1743 36 169 2 5 2 2 156 324 223 243 5 16 15 412 207 2 2 17 4 319 5 18 1523 20 5 300 2 15 26 2 2 17 2 67 332 2 2 2 5 18 140 2 2 2 17 5 2 319 2 11 2 2 2 33 2 5 2 5 18 100 2 5 2 38 235 243 2 17 2 30 16 2 319 2 2 2 2 87 5 7 2 4\n",
      "INFO:tensorflow:input_ids: 3 2 18 2 2 1855 563 687 1372 652 1403 467 314 2 5 5 2 2 16 17 18 2 17 2 2 2 2 2 17 18 2 5 16 16 52 207 119 319 41 5 169 63 8 2 319 1743 36 169 2 5 2 2 156 324 223 243 5 16 15 412 207 2 2 17 4 319 5 18 1523 20 5 300 2 15 26 2 2 17 2 67 332 2 2 2 5 18 140 2 2 2 17 5 2 319 2 11 2 2 2 33 2 5 2 5 18 100 2 5 2 38 235 243 2 17 2 30 16 2 319 2 2 2 2 87 5 7 2 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 14 15 30 31 39 45 49 56 66 68 70 83 84 91 101 103 107 114 124 0\n",
      "INFO:tensorflow:masked_lm_positions: 14 15 30 31 39 45 49 56 66 68 70 83 84 91 101 103 107 114 124 0\n",
      "INFO:tensorflow:masked_lm_ids: 8 135 2 30 36 41 2 30 106 2 2 2 17 18 77 17 2 2 2 0\n",
      "INFO:tensorflow:masked_lm_ids: 8 135 2 30 36 41 2 30 106 2 2 2 17 18 77 17 2 2 2 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] [UNK] တင ၁၉၉၇ [UNK] [UNK] [UNK] ၏ ရက ##ခ ##တ ##မ [UNK] ၊ ၂၀၀၃ [MASK] ##စ [UNK] [UNK] ၊ [UNK] ၁၁ ရက တက ##ခ [MASK] ##မ [UNK] [UNK] [MASK] ၏ [UNK] [MASK] ##ခ ##က မည [UNK] [UNK] က [UNK] [UNK] ခ သည ။ [MASK] [UNK] [MASK] က မ [UNK] [UNK] [MASK] က [UNK] [UNK] [UNK] နတ [UNK] သည ဟ ယ [MASK] ခ [UNK] [SEP] [UNK] [UNK] လ [UNK] [MASK] [MASK] ဆ [MASK] မည [MASK] [UNK] [UNK] [MASK] [UNK] က အလ ##န [MASK] [UNK] ရ [UNK] သ [UNK] မ [UNK] [UNK] [UNK] [UNK] ဟ အလ ##လ ##က [UNK] ခ ၏ ။ ကန အလယ တင [UNK] ၍ ကန [UNK] [MASK] [UNK] သက [MASK] [UNK] [UNK] ၌ တတ ##ဆ ##တ ၍ [UNK] ၌ လ [UNK] လတ [MASK] [MASK] [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] [UNK] တင ၁၉၉၇ [UNK] [UNK] [UNK] ၏ ရက ##ခ ##တ ##မ [UNK] ၊ ၂၀၀၃ [MASK] ##စ [UNK] [UNK] ၊ [UNK] ၁၁ ရက တက ##ခ [MASK] ##မ [UNK] [UNK] [MASK] ၏ [UNK] [MASK] ##ခ ##က မည [UNK] [UNK] က [UNK] [UNK] ခ သည ။ [MASK] [UNK] [MASK] က မ [UNK] [UNK] [MASK] က [UNK] [UNK] [UNK] နတ [UNK] သည ဟ ယ [MASK] ခ [UNK] [SEP] [UNK] [UNK] လ [UNK] [MASK] [MASK] ဆ [MASK] မည [MASK] [UNK] [UNK] [MASK] [UNK] က အလ ##န [MASK] [UNK] ရ [UNK] သ [UNK] မ [UNK] [UNK] [UNK] [UNK] ဟ အလ ##လ ##က [UNK] ခ ၏ ။ ကန အလယ တင [UNK] ၍ ကန [UNK] [MASK] [UNK] သက [MASK] [UNK] [UNK] ၌ တတ ##ဆ ##တ ၍ [UNK] ၌ လ [UNK] လတ [MASK] [MASK] [SEP]\n",
      "INFO:tensorflow:input_ids: 3 2 2 78 696 2 2 2 26 80 369 235 243 2 23 830 5 199 2 2 23 2 284 80 60 369 5 243 2 2 5 26 2 5 369 169 51 2 2 7 2 2 16 17 18 5 2 5 7 8 2 2 5 7 2 2 2 179 2 17 29 24 5 16 2 4 2 2 15 2 5 5 22 5 51 5 2 2 5 2 7 52 217 5 2 12 2 13 2 8 2 2 2 2 29 52 367 169 2 16 26 18 97 197 78 2 33 97 2 5 2 67 5 2 2 47 92 433 235 33 2 47 15 2 222 5 5 4\n",
      "INFO:tensorflow:input_ids: 3 2 2 78 696 2 2 2 26 80 369 235 243 2 23 830 5 199 2 2 23 2 284 80 60 369 5 243 2 2 5 26 2 5 369 169 51 2 2 7 2 2 16 17 18 5 2 5 7 8 2 2 5 7 2 2 2 179 2 17 29 24 5 16 2 4 2 2 15 2 5 5 22 5 51 5 2 2 5 2 7 52 217 5 2 12 2 13 2 8 2 2 2 2 29 52 367 169 2 16 26 18 97 197 78 2 33 97 2 5 2 67 5 2 2 47 92 433 235 33 2 47 15 2 222 5 5 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 16 26 30 33 45 47 52 62 70 71 73 75 78 83 109 112 123 125 126 0\n",
      "INFO:tensorflow:masked_lm_positions: 16 26 30 33 45 47 52 62 70 71 73 75 78 83 109 112 123 125 126 0\n",
      "INFO:tensorflow:masked_lm_ids: 82 169 43 60 2 2 11 433 17 29 12 2 17 2 78 332 2 235 18 0\n",
      "INFO:tensorflow:masked_lm_ids: 82 169 43 60 2 2 11 433 17 29 12 2 17 2 78 332 2 235 18 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] [MASK] ဘ ##ဘ ##စ နင [UNK] [UNK] [UNK] [MASK] [MASK] သည အန ##ဒ ##ယ [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] Bit ဟ [UNK] မ ဟတ [UNK] [UNK] [UNK] ခ သည ။ [UNK] [UNK] ဖ ဆ ရင [UNK] က [UNK] [UNK] [UNK] [UNK] သလ [MASK] [UNK] [UNK] ၊ [UNK] ၏ [UNK] ဟ [MASK] သည ။ [UNK] [MASK] [UNK] လပ ##ခ [MASK] [UNK] [UNK] ရ [SEP] [UNK] [UNK] ပင [UNK] [UNK] ၍ နတ [UNK] ရ သည ။ [UNK] [UNK] [UNK] ။ [UNK] [UNK] သည ဒတ ##ယ [UNK] [MASK] [UNK] [UNK] ရ [UNK] အခက ##အ ##ခ [MASK] နင [UNK] [UNK] မ [UNK] [UNK] ခ [UNK] သည [MASK] [UNK] သည [UNK] [UNK] [UNK] ရ [MASK] ရန [MASK] ##ရ ##ယ ၍ [UNK] [UNK] သည ။ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [UNK] [MASK] ဘ ##ဘ ##စ နင [UNK] [UNK] [UNK] [MASK] [MASK] သည အန ##ဒ ##ယ [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] Bit ဟ [UNK] မ ဟတ [UNK] [UNK] [UNK] ခ သည ။ [UNK] [UNK] ဖ ဆ ရင [UNK] က [UNK] [UNK] [UNK] [UNK] သလ [MASK] [UNK] [UNK] ၊ [UNK] ၏ [UNK] ဟ [MASK] သည ။ [UNK] [MASK] [UNK] လပ ##ခ [MASK] [UNK] [UNK] ရ [SEP] [UNK] [UNK] ပင [UNK] [UNK] ၍ နတ [UNK] ရ သည ။ [UNK] [UNK] [UNK] ။ [UNK] [UNK] သည ဒတ ##ယ [UNK] [MASK] [UNK] [UNK] ရ [UNK] အခက ##အ ##ခ [MASK] နင [UNK] [UNK] မ [UNK] [UNK] ခ [UNK] သည [MASK] [UNK] သည [UNK] [UNK] [UNK] ရ [MASK] ရန [MASK] ##ရ ##ယ ၍ [UNK] [UNK] သည ။ [SEP]\n",
      "INFO:tensorflow:input_ids: 3 2 5 28 368 199 319 2 2 2 5 5 17 48 346 278 5 2 2 2 5 2 2 2 5 2 2 1024 29 2 8 983 2 2 2 16 17 18 2 2 19 22 56 2 7 2 2 2 2 98 5 2 2 23 2 26 2 29 5 17 18 2 5 2 119 369 5 2 2 12 4 2 2 50 2 2 33 179 2 12 17 18 2 2 2 18 2 2 17 459 278 2 5 2 2 12 2 438 412 369 5 319 2 2 8 2 2 16 2 17 5 2 17 2 2 2 12 5 41 5 223 278 33 2 2 17 18 4\n",
      "INFO:tensorflow:input_ids: 3 2 5 28 368 199 319 2 2 2 5 5 17 48 346 278 5 2 2 2 5 2 2 2 5 2 2 1024 29 2 8 983 2 2 2 16 17 18 2 2 19 22 56 2 7 2 2 2 2 98 5 2 2 23 2 26 2 29 5 17 18 2 5 2 119 369 5 2 2 12 4 2 2 50 2 2 33 179 2 12 17 18 2 2 2 18 2 2 17 459 278 2 5 2 2 12 2 438 412 369 5 319 2 2 8 2 2 16 2 17 5 2 17 2 2 2 12 5 41 5 223 278 33 2 2 17 18 4\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 2 7 10 11 16 20 24 27 50 58 61 62 66 92 99 100 110 117 119 0\n",
      "INFO:tensorflow:masked_lm_positions: 2 7 10 11 16 20 24 27 50 58 61 62 66 92 99 100 110 117 119 0\n",
      "INFO:tensorflow:masked_lm_ids: 7 2 78 2 50 7 2 17 18 2 2 2 7 2 369 2 2 319 145 0\n",
      "INFO:tensorflow:masked_lm_ids: 7 2 78 2 50 7 2 17 18 2 2 2 7 2 369 2 2 319 145 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:Wrote 11962 total instances\n",
      "INFO:tensorflow:Wrote 11962 total instances\n"
     ]
    }
   ],
   "source": [
    "!python  '/gdrive/My Drive/Colab Notebooks/codeTest/bert/create_pretraining_data.py'  --input_file='/gdrive/My Drive/Colab Notebooks/codeTest/myaninput/myanWordTest.txt'  --output_file='/gdrive/My Drive/Colab Notebooks/codeTest/output/tf_examples.tfrecord' --vocab_file='/gdrive/My Drive/Colab Notebooks/codeTest/myan/vocab.txt' --do_lower_case=True  --max_seq_length=128 --max_predictions_per_seq=20  --masked_lm_prob=0.15  --random_seed=12345  --dupe_factor=5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Remark\n",
    "I got the following result after running run_pretraining.py(BERT code) according to above pretraining data\n",
    " Eval results *****\n",
    " global_step = 20\n",
    " loss = 5.81478\n",
    " masked_lm_accuracy = 0.4306579\n",
    " masked_lm_loss = 5.1212683\n",
    " next_sentence_accuracy = 0.49875\n",
    " next_sentence_loss = 0.6935117\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "9M9ZKd6P3h0F",
    "outputId": "0536caed-c7a5-4a8e-a6b5-a07767a174bb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "INFO:tensorflow:*** Input Files ***\n",
      "INFO:tensorflow:  /gdrive/My Drive/Colab Notebooks/codeTest/output/tf_examples.tfrecord\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7f432e73e9d8>) includes params argument, but params are not passed to Estimator.\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/gdrive/My Drive/Colab Notebooks/codeTest/pretraining_output', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f431cc585c0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n",
      "INFO:tensorflow:***** Running training *****\n",
      "INFO:tensorflow:  Batch size = 32\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /gdrive/My Drive/Colab Notebooks/codeTest/bert/run_pretraining.py:368: parallel_interleave (from tensorflow.contrib.data.python.ops.interleave_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.data.experimental.parallel_interleave(...)`.\n",
      "WARNING:tensorflow:From /gdrive/My Drive/Colab Notebooks/codeTest/bert/run_pretraining.py:385: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.data.experimental.map_and_batch(...)`.\n",
      "WARNING:tensorflow:From /gdrive/My Drive/Colab Notebooks/codeTest/bert/run_pretraining.py:400: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running train on CPU\n",
      "INFO:tensorflow:*** Features ***\n",
      "INFO:tensorflow:  name = input_ids, shape = (32, 128)\n",
      "INFO:tensorflow:  name = input_mask, shape = (32, 128)\n",
      "INFO:tensorflow:  name = masked_lm_ids, shape = (32, 20)\n",
      "INFO:tensorflow:  name = masked_lm_positions, shape = (32, 20)\n",
      "INFO:tensorflow:  name = masked_lm_weights, shape = (32, 20)\n",
      "INFO:tensorflow:  name = next_sentence_labels, shape = (32, 1)\n",
      "INFO:tensorflow:  name = segment_ids, shape = (32, 128)\n",
      "WARNING:tensorflow:From /gdrive/My Drive/Colab Notebooks/codeTest/bert/modeling.py:358: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /gdrive/My Drive/Colab Notebooks/codeTest/bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "INFO:tensorflow:**** Trainable Variables ****\n",
      "INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (12000, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = cls/predictions/transform/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = cls/predictions/transform/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = cls/predictions/output_bias:0, shape = (12000,)\n",
      "INFO:tensorflow:  name = cls/seq_relationship/output_weights:0, shape = (2, 768)\n",
      "INFO:tensorflow:  name = cls/seq_relationship/output_bias:0, shape = (2,)\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/learning_rate_decay_v2.py:321: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2020-04-19 14:44:34.055815: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2020-04-19 14:44:34.061212: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2020-04-19 14:44:34.061696: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x1a2cd60 executing computations on platform Host. Devices:\n",
      "2020-04-19 14:44:34.061790: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into /gdrive/My Drive/Colab Notebooks/codeTest/pretraining_output/model.ckpt.\n",
      "INFO:tensorflow:Saving checkpoints for 20 into /gdrive/My Drive/Colab Notebooks/codeTest/pretraining_output/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 6.1623797.\n",
      "INFO:tensorflow:training_loop marked as finished\n",
      "INFO:tensorflow:***** Running evaluation *****\n",
      "INFO:tensorflow:  Batch size = 8\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running eval on CPU\n",
      "INFO:tensorflow:*** Features ***\n",
      "INFO:tensorflow:  name = input_ids, shape = (8, 128)\n",
      "INFO:tensorflow:  name = input_mask, shape = (8, 128)\n",
      "INFO:tensorflow:  name = masked_lm_ids, shape = (8, 20)\n",
      "INFO:tensorflow:  name = masked_lm_positions, shape = (8, 20)\n",
      "INFO:tensorflow:  name = masked_lm_weights, shape = (8, 20)\n",
      "INFO:tensorflow:  name = next_sentence_labels, shape = (8, 1)\n",
      "INFO:tensorflow:  name = segment_ids, shape = (8, 128)\n",
      "INFO:tensorflow:**** Trainable Variables ****\n",
      "INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (12000, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768)\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = cls/predictions/transform/dense/kernel:0, shape = (768, 768)\n",
      "INFO:tensorflow:  name = cls/predictions/transform/dense/bias:0, shape = (768,)\n",
      "INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/beta:0, shape = (768,)\n",
      "INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/gamma:0, shape = (768,)\n",
      "INFO:tensorflow:  name = cls/predictions/output_bias:0, shape = (12000,)\n",
      "INFO:tensorflow:  name = cls/seq_relationship/output_weights:0, shape = (2, 768)\n",
      "INFO:tensorflow:  name = cls/seq_relationship/output_bias:0, shape = (2,)\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/metrics_impl.py:455: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2020-04-19T15:10:27Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from /gdrive/My Drive/Colab Notebooks/codeTest/pretraining_output/model.ckpt-20\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Evaluation [10/100]\n",
      "INFO:tensorflow:Evaluation [20/100]\n",
      "INFO:tensorflow:Evaluation [30/100]\n",
      "INFO:tensorflow:Evaluation [40/100]\n",
      "INFO:tensorflow:Evaluation [50/100]\n",
      "INFO:tensorflow:Evaluation [60/100]\n",
      "INFO:tensorflow:Evaluation [70/100]\n",
      "INFO:tensorflow:Evaluation [80/100]\n",
      "INFO:tensorflow:Evaluation [90/100]\n",
      "INFO:tensorflow:Evaluation [100/100]\n",
      "INFO:tensorflow:Finished evaluation at 2020-04-19-15:21:41\n",
      "INFO:tensorflow:Saving dict for global step 20: global_step = 20, loss = 5.81478, masked_lm_accuracy = 0.4306579, masked_lm_loss = 5.1212683, next_sentence_accuracy = 0.49875, next_sentence_loss = 0.6935117\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 20: /gdrive/My Drive/Colab Notebooks/codeTest/pretraining_output/model.ckpt-20\n",
      "INFO:tensorflow:evaluation_loop marked as finished\n",
      "INFO:tensorflow:***** Eval results *****\n",
      "INFO:tensorflow:  global_step = 20\n",
      "INFO:tensorflow:  loss = 5.81478\n",
      "INFO:tensorflow:  masked_lm_accuracy = 0.4306579\n",
      "INFO:tensorflow:  masked_lm_loss = 5.1212683\n",
      "INFO:tensorflow:  next_sentence_accuracy = 0.49875\n",
      "INFO:tensorflow:  next_sentence_loss = 0.6935117\n"
     ]
    }
   ],
   "source": [
    "!python '/gdrive/My Drive/Colab Notebooks/codeTest/bert/run_pretraining.py'   --input_file='/gdrive/My Drive/Colab Notebooks/codeTest/output/tf_examples.tfrecord'  --output_dir='/gdrive/My Drive/Colab Notebooks/codeTest/pretraining_output'  --do_train=True  --do_eval=True  --bert_config_file='/gdrive/My Drive/Colab Notebooks/codeTest/myan/bert_config.json' --train_batch_size=32 --max_seq_length=128 --max_predictions_per_seq=20 --num_train_steps=20 --num_warmup_steps=10 --learning_rate=2e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "p3zeWfA743Wn"
   },
   "outputs": [],
   "source": [
    "\n",
    "\"\"\"\n",
    "Remark\n",
    "I use vocabulary file and pre-trained checkpoints of of MultiLanguage BERT model.\n",
    "No found  [UNK](unknown_token) when using muitilanguage vocab file\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "zTozTFfwugw8",
    "outputId": "07780a44-aa53-4504-ac07-17ad6864cd29"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "INFO:tensorflow:*** Reading from input files ***\n",
      "INFO:tensorflow:*** Reading from input files ***\n",
      "INFO:tensorflow:  /gdrive/My Drive/Colab Notebooks/codeTest/myaninput/myanWordTest.txt\n",
      "INFO:tensorflow:  /gdrive/My Drive/Colab Notebooks/codeTest/myaninput/myanWordTest.txt\n",
      "INFO:tensorflow:*** Writing to output files ***\n",
      "INFO:tensorflow:*** Writing to output files ***\n",
      "INFO:tensorflow:  /gdrive/My Drive/Colab Notebooks/codeTest/multimyanoutput/tf_examples.tfrecord\n",
      "INFO:tensorflow:  /gdrive/My Drive/Colab Notebooks/codeTest/multimyanoutput/tf_examples.tfrecord\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ရ ##က ##က ##း န ##ေ စ ##ဉ အ [MASK] ##င Fevereiro သ ##ား မ ##ော [MASK] ##က ##ျ ##ော ##စ ##း မ [MASK] [MASK] ##မ ##သ ##မာ တ [MASK] ဥ [SEP] [MASK] ##သ ##ပ ##ပ natural sci ##em ##ces လ ##မ ##ရေး ##ရာ [MASK] ##ာ ##သာ ##ရ ##ပ human ##ities [MASK] ##င လ ##မ ##သ ##ပ ##ပ social sciences တ ##င [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ရ ##က ##က ##း န ##ေ စ ##ဉ အ [MASK] ##င Fevereiro သ ##ား မ ##ော [MASK] ##က ##ျ ##ော ##စ ##း မ [MASK] [MASK] ##မ ##သ ##မာ တ [MASK] ဥ [SEP] [MASK] ##သ ##ပ ##ပ natural sci ##em ##ces လ ##မ ##ရေး ##ရာ [MASK] ##ာ ##သာ ##ရ ##ပ human ##ities [MASK] ##င လ ##မ ##သ ##ပ ##ပ social sciences တ ##င [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1518 27205 27205 34673 1511 24189 1497 111478 1524 103 111473 75023 1521 35089 1516 80200 103 27205 111503 80200 111474 34673 1516 103 103 64529 111493 73335 1507 103 1526 102 103 111493 111489 111489 13409 58803 10451 14585 1519 64529 35626 56416 103 18378 108744 48081 111489 14179 17285 103 111473 1519 64529 111493 111489 111489 12142 21896 1507 111473 102\n",
      "INFO:tensorflow:input_ids: 101 1518 27205 27205 34673 1511 24189 1497 111478 1524 103 111473 75023 1521 35089 1516 80200 103 27205 111503 80200 111474 34673 1516 103 103 64529 111493 73335 1507 103 1526 102 103 111493 111489 111489 13409 58803 10451 14585 1519 64529 35626 56416 103 18378 108744 48081 111489 14179 17285 103 111473 1519 64529 111493 111489 111489 12142 21896 1507 111473 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 10 12 17 24 25 30 33 45 52 59 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 10 12 17 24 25 30 33 45 52 59 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111484 34673 111473 18378 1519 111474 34342 111492 1511 12142 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111484 34673 111473 18378 1519 111474 34342 111492 1511 12142 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ##တ ##ခ ##ျ ##က တ ##င မ ##ြ ##န [MASK] ဟ ရ ##ေး ##သား ရ ##ာ [MASK] ##င ' အ [MASK] ##ရ ' သ န ##ော ##က ##ပ ##တ သ [MASK] [SEP] ##ရ ##စ ##ယ ##ာန ##ဘ ##ာ ##သာ ၉ . ၂ % říše ဟ [MASK] ##ဒ [MASK] ##ာ ##သာ ၆ . ၃ % [MASK] က ##န ##ဖ ##ြ [MASK] [MASK] ##ပ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ##တ ##ခ ##ျ ##က တ ##င မ ##ြ ##န [MASK] ဟ ရ ##ေး ##သား ရ ##ာ [MASK] ##င ' အ [MASK] ##ရ ' သ န ##ော ##က ##ပ ##တ သ [MASK] [SEP] ##ရ ##စ ##ယ ##ာန ##ဘ ##ာ ##သာ ၉ . ၂ % říše ဟ [MASK] ##ဒ [MASK] ##ာ ##သာ ၆ . ၃ % [MASK] က ##န ##ဖ ##ြ [MASK] [MASK] ##ပ [SEP]\n",
      "INFO:tensorflow:input_ids: 101 111484 111470 111503 27205 1507 111473 1516 111504 111488 103 1522 1518 59512 69046 1518 18378 103 111473 112 1524 103 48081 112 1521 1511 80200 27205 111489 111484 1521 103 102 48081 111474 105942 98209 111492 18378 108744 1557 119 1550 110 77798 1522 103 111486 103 18378 108744 1554 119 1551 110 103 1492 111488 111490 111504 103 103 111489 102\n",
      "INFO:tensorflow:input_ids: 101 111484 111470 111503 27205 1507 111473 1516 111504 111488 103 1522 1518 59512 69046 1518 18378 103 111473 112 1524 103 48081 112 1521 1511 80200 27205 111489 111484 1521 103 102 48081 111474 105942 98209 111492 18378 108744 1557 119 1550 110 77798 1522 103 111486 103 18378 108744 1554 119 1551 110 103 1492 111488 111490 111504 103 103 111489 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 10 17 21 31 44 46 48 55 60 61 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 10 17 21 31 44 46 48 55 60 61 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 73335 111484 18378 111479 1558 111488 111492 1558 34673 48081 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 73335 111484 18378 111479 1558 111488 111492 1558 34673 48081 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ##ဉ ##က [MASK] ##ေး ##မ မ [MASK] [MASK] ၍ က ##မ ##ဘ ##ာ ပ ##ေ ##ါ တ ##င အ ##ဆ ##င [MASK] [MASK] ##န ##း [MASK] ##မ ##ြ ##င ##ဆ ##း [SEP] တ ##င လ [MASK] ##င မ ##ြ [MASK] ##း ##စ ##း က ##ျ ##မ ##း [MASK] ##ျ ##င ##မ က အ ##သ ##း ##ခ ##ျ က [MASK] တ ##က ##ခ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ##ဉ ##က [MASK] ##ေး ##မ မ [MASK] [MASK] ၍ က ##မ ##ဘ ##ာ ပ ##ေ ##ါ တ ##င အ ##ဆ ##င [MASK] [MASK] ##န ##း [MASK] ##မ ##ြ ##င ##ဆ ##း [SEP] တ ##င လ [MASK] ##င မ ##ြ [MASK] ##း ##စ ##း က ##ျ ##မ ##း [MASK] ##ျ ##င ##မ က အ ##သ ##း ##ခ ##ျ က [MASK] တ ##က ##ခ [SEP]\n",
      "INFO:tensorflow:input_ids: 101 111478 27205 103 59512 64529 1516 103 103 1561 1492 64529 111492 18378 1512 24189 50428 1507 111473 1524 111475 111473 103 103 111488 34673 103 64529 111504 111473 111475 34673 102 1507 111473 1519 103 111473 1516 111504 103 34673 111474 34673 1492 111503 64529 34673 103 111503 111473 64529 1492 1524 111493 34673 111470 111503 1492 103 1507 27205 111470 102\n",
      "INFO:tensorflow:input_ids: 101 111478 27205 103 59512 64529 1516 103 103 1561 1492 64529 111492 18378 1512 24189 50428 1507 111473 1524 111475 111473 103 103 111488 34673 103 64529 111504 111473 111475 34673 102 1507 111473 1519 103 111473 1516 111504 103 34673 111474 34673 1492 111503 64529 34673 103 111503 111473 64529 1492 1524 111493 34673 111470 111503 1492 103 1507 27205 111470 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 3 7 8 22 23 26 36 40 48 59 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 3 7 8 22 23 26 36 40 48 59 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111503 92751 1518 111496 111484 1524 1511 111473 27205 18378 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111503 92751 1518 111496 111484 1524 1511 111473 27205 18378 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [MASK] ##န ##စ ##စ ##တ ##ပ မ ##ျား ##калык ##င ##ရ ##ော ##က လ ##ာ [MASK] သ ##ည က အ ##င ##ဂ ##လ [MASK] န ##င အ ##မ [MASK] ##ရ ##က [SEP] ##ရ [MASK] ##ေ [MASK] ##က ##ြ [MASK] ##င ##း ၊ ဗ ##လ [MASK] ##ျ ##ပ ##န ##ေ ##ဝ [MASK] ##း ၏ တ ##ရား ##မ ##ဝ ##င သ ##ည အ ##ဖ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [MASK] ##န ##စ ##စ ##တ ##ပ မ ##ျား ##калык ##င ##ရ ##ော ##က လ ##ာ [MASK] သ ##ည က အ ##င ##ဂ ##လ [MASK] န ##င အ ##မ [MASK] ##ရ ##က [SEP] ##ရ [MASK] ##ေ [MASK] ##က ##ြ [MASK] ##င ##း ၊ ဗ ##လ [MASK] ##ျ ##ပ ##န ##ေ ##ဝ [MASK] ##း ၏ တ ##ရား ##မ ##ဝ ##င သ ##ည အ ##ဖ [SEP]\n",
      "INFO:tensorflow:input_ids: 101 103 111488 111474 111474 111484 111489 1516 92751 105052 111473 48081 80200 27205 1519 18378 103 1521 111479 1492 1524 111473 111471 41866 103 1511 111473 1524 64529 103 48081 27205 102 48081 103 24189 103 27205 111504 103 111473 34673 1558 1514 41866 103 111503 111489 111488 24189 34342 103 34673 1563 1507 98187 64529 34342 111473 1521 111479 1524 111490 102\n",
      "INFO:tensorflow:input_ids: 101 103 111488 111474 111474 111484 111489 1516 92751 105052 111473 48081 80200 27205 1519 18378 103 1521 111479 1492 1524 111473 111471 41866 103 1511 111473 1524 64529 103 48081 27205 102 48081 103 24189 103 27205 111504 103 111473 34673 1558 1514 41866 103 111503 111489 111488 24189 34342 103 34673 1563 1507 98187 64529 34342 111473 1521 111479 1524 111490 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 1 9 16 24 29 34 36 39 45 51 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 1 9 16 24 29 34 36 39 45 51 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111489 1520 1493 111489 24189 111474 41866 80200 111470 111473 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111489 1520 1493 111489 24189 111474 41866 80200 111470 111473 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ##င ဒ ##ေ ##သ တ ##စ ##ဝ ##က [MASK] ##ာ စ ##စ ##မ ##ြ ##ေ Landesliga [MASK] ##င က [MASK] [MASK] ##ြ [MASK] ##န ##ေ ##က ##ြ ##ော ##င ##း [MASK] [SEP] ##င ##း တ ##န ##င ##သာ ##ရ ##တ ##င ##း မ ##ြ ##တ ##ခ ##ရ ##င ၏ [MASK] ##ရ ##င ##ဝ ##န ရ ##း ##စ ##က ##ရာ မ ##ြ ဖ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ##င ဒ ##ေ ##သ တ ##စ ##ဝ ##က [MASK] ##ာ စ ##စ ##မ ##ြ ##ေ Landesliga [MASK] ##င က [MASK] [MASK] ##ြ [MASK] ##န ##ေ ##က ##ြ ##ော ##င ##း [MASK] [SEP] ##င ##း တ ##န ##င ##သာ ##ရ ##တ ##င ##း မ ##ြ ##တ ##ခ ##ရ ##င ၏ [MASK] ##ရ ##င ##ဝ ##န ရ ##း ##စ ##က ##ရာ မ ##ြ ဖ [SEP]\n",
      "INFO:tensorflow:input_ids: 101 111473 1509 24189 111493 1507 111474 34342 27205 103 18378 1497 111474 64529 111504 24189 100843 103 111473 1492 103 103 111504 103 111488 24189 27205 111504 80200 111473 34673 103 102 111473 34673 1507 111488 111473 108744 48081 111484 111473 34673 1516 111504 111484 111470 48081 111473 1563 103 48081 111473 34342 111488 1518 34673 111474 27205 56416 1516 111504 1513 102\n",
      "INFO:tensorflow:input_ids: 101 111473 1509 24189 111493 1507 111474 34342 27205 103 18378 1497 111474 64529 111504 24189 100843 103 111473 1492 103 103 111504 103 111488 24189 27205 111504 80200 111473 34673 103 102 111473 34673 1507 111488 111473 108744 48081 111484 111473 34673 1516 111504 111484 111470 48081 111473 1563 103 48081 111473 34342 111488 1518 34673 111474 27205 56416 1516 111504 1513 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 5 9 16 17 20 21 23 31 50 51 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 5 9 16 17 20 21 23 31 50 51 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 1507 1516 111489 111504 111493 1513 111474 1513 1493 48081 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 1507 1516 111489 111504 111493 1513 111474 1513 1493 48081 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [MASK] ##န ရ ပ ##ြ ##း ၊ ခ ##င ##ပ ##ြ ##မ ##န မ ရ ဘ န ##င ##င ##သား မ [MASK] သ ##မ ##ဟ [MASK] အ ##ခ ##ြား သ ##ော [SEP] ##း သ ##တ ##ေ ##သ ##န ##ဗ 1429 ##ဒ အ [MASK] လ ##ေ ##လ ##ာ ##ခ ##ြ ##င ##း ဖ [MASK] ##င [MASK] ခ [MASK] [MASK] အ ##ခ ##ါ က [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [MASK] ##န ရ ပ ##ြ ##း ၊ ခ ##င ##ပ ##ြ ##မ ##န မ ရ ဘ န ##င ##င ##သား မ [MASK] သ ##မ ##ဟ [MASK] အ ##ခ ##ြား သ ##ော [SEP] ##း သ ##တ ##ေ ##သ ##န ##ဗ 1429 ##ဒ အ [MASK] လ ##ေ ##လ ##ာ ##ခ ##ြ ##င ##း ဖ [MASK] ##င [MASK] ခ [MASK] [MASK] အ ##ခ ##ါ က [SEP]\n",
      "INFO:tensorflow:input_ids: 101 103 111488 1518 1512 111504 34673 1558 1493 111473 111489 111504 64529 111488 1516 1518 1515 1511 111473 111473 69046 1516 103 1521 64529 111494 103 1524 111470 67798 1521 80200 102 34673 1521 111484 24189 111493 111488 111491 80276 111486 1524 103 1519 24189 41866 18378 111470 111504 111473 34673 1513 103 111473 103 1493 103 103 1524 111470 50428 1492 102\n",
      "INFO:tensorflow:input_ids: 101 103 111488 1518 1512 111504 34673 1558 1493 111473 111489 111504 64529 111488 1516 1518 1515 1511 111473 111473 69046 1516 103 1521 64529 111494 103 1524 111470 67798 1521 80200 102 34673 1521 111484 24189 111493 111488 111491 80276 111486 1524 103 1519 24189 41866 18378 111470 111504 111473 34673 1513 103 111473 103 1493 103 103 1524 111470 50428 1492 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 1 13 22 26 40 43 53 55 57 58 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 1 13 22 26 40 43 53 55 57 58 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111493 111488 92751 111484 24189 48081 111504 1508 24189 111484 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111493 111488 92751 111484 24189 48081 111504 1508 24189 111484 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ##း [MASK] အ ##ခ ##န [MASK] [MASK] ##ရ မ ##ာ တ ##ော ##က ##တ ##က ##ြ ##း တ 剔 က ##ော ##င ဘ ##ယ [MASK] ##酥 ##ော [MASK] လ ##ာ မ [SEP] အ ##န ##က [MASK] ##က ##ဆ ##င ##ရာ ဂ Ce ##ဟ မ ##ျား ၏ ဂ ##ြ ##ဟ ##ရ ##လ မ ##ျား လ ##ည ##း အ ##ပ ##ါ ##အ ##ဝ [MASK] [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ##း [MASK] အ ##ခ ##န [MASK] [MASK] ##ရ မ ##ာ တ ##ော ##က ##တ ##က ##ြ ##း တ 剔 က ##ော ##င ဘ ##ယ [MASK] ##酥 ##ော [MASK] လ ##ာ မ [SEP] အ ##န ##က [MASK] ##က ##ဆ ##င ##ရာ ဂ Ce ##ဟ မ ##ျား ၏ ဂ ##ြ ##ဟ ##ရ ##လ မ ##ျား လ ##ည ##း အ ##ပ ##ါ ##အ ##ဝ [MASK] [SEP]\n",
      "INFO:tensorflow:input_ids: 101 34673 103 1524 111470 111488 103 103 48081 1516 18378 1507 80200 27205 111484 27205 111504 34673 1507 2571 1492 80200 111473 1515 105942 103 117635 80200 103 1519 18378 1516 102 1524 111488 27205 103 27205 111475 111473 56416 1494 13338 111494 1516 92751 1563 1494 111504 111494 48081 41866 1516 92751 1519 111479 34673 1524 111489 50428 111496 34342 103 102\n",
      "INFO:tensorflow:input_ids: 101 34673 103 1524 111470 111488 103 103 48081 1516 18378 1507 80200 27205 111484 27205 111504 34673 1507 2571 1492 80200 111473 1515 105942 103 117635 80200 103 1519 18378 1516 102 1524 111488 27205 103 27205 111475 111473 56416 1494 13338 111494 1516 92751 1563 1494 111504 111494 48081 41866 1516 92751 1519 111479 34673 1524 111489 50428 111496 34342 103 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 2 6 7 19 25 26 28 36 42 62 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 2 6 7 19 25 26 28 36 42 62 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 1521 34673 1511 111474 41866 1518 27205 1521 111504 111473 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 1521 34673 1511 111474 41866 1518 27205 1521 111504 111473 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ##င မ ##ြ [MASK] ##မာ ဆ ##က ##ဆ ##ရေး မ ##ာ ၁ ##၉ ##၈ [MASK] ပ ##ြ ##ည ##လ ##န န [MASK] ##င ##း ##ပ ##င ##း [MASK] [MASK] မ က [SEP] လ [MASK] ##ျ [MASK] တ ##င ##း တ ##င အ ##သ [MASK] ##ပ ##ြ [MASK] ##ျ ##က [MASK] သ ##ော အ ##က ##ခ ##ရာ ##စ ##ာ ##လ ##း မ ##ျား [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ##င မ ##ြ [MASK] ##မာ ဆ ##က ##ဆ ##ရေး မ ##ာ ၁ ##၉ ##၈ [MASK] ပ ##ြ ##ည ##လ ##န န [MASK] ##င ##း ##ပ ##င ##း [MASK] [MASK] မ က [SEP] လ [MASK] ##ျ [MASK] တ ##င ##း တ ##င အ ##သ [MASK] ##ပ ##ြ [MASK] ##ျ ##က [MASK] သ ##ော အ ##က ##ခ ##ရာ ##စ ##ာ ##လ ##း မ ##ျား [SEP]\n",
      "INFO:tensorflow:input_ids: 101 111473 1516 111504 103 73335 1498 27205 111475 35626 1516 18378 1549 42044 36145 103 1512 111504 111479 41866 111488 1511 103 111473 34673 111489 111473 34673 103 103 1516 1492 102 1519 103 111503 103 1507 111473 34673 1507 111473 1524 111493 103 111489 111504 103 111503 27205 103 1521 80200 1524 27205 111470 56416 111474 18378 41866 34673 1516 92751 102\n",
      "INFO:tensorflow:input_ids: 101 111473 1516 111504 103 73335 1498 27205 111475 35626 1516 18378 1549 42044 36145 103 1512 111504 111479 41866 111488 1511 103 111473 34673 111489 111473 34673 103 103 1516 1492 102 1519 103 111503 103 1507 111473 34673 1507 111473 1524 111493 103 111489 111504 103 111503 27205 103 1521 80200 1524 27205 111470 56416 111474 18378 41866 34673 1516 92751 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 4 15 22 28 29 34 36 44 47 50 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 4 15 22 28 29 34 36 44 47 50 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111488 34342 80200 1524 111485 64529 34673 34673 1519 1518 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111488 34342 80200 1524 111485 64529 34673 34673 1519 1518 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] မ ##ျ ##း သ ##ည ဂ ##ရ ##စ ##ဇ ##လ ##ဝ ##က ##ဝ မ ##ျ ##း န ##င တ ၍ ဝ ##က ##ဝ ##မ ##ည ##း မ ##ျ [MASK] ထ ##က [SEP] ##ေး ##ဝ ##ါ ##း ##ဘ ##း မ ##ျား [MASK] ##သ ##ည တ က [MASK] ##ည ##ဆ ##ော [MASK] [MASK] ##ော ရ ##ေး [MASK] ##ေ ##တ လ တ သ ##ည က [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] မ ##ျ ##း သ ##ည ဂ ##ရ ##စ ##ဇ ##လ ##ဝ ##က ##ဝ မ ##ျ ##း န ##င တ ၍ ဝ ##က ##ဝ ##မ ##ည ##း မ ##ျ [MASK] ထ ##က [SEP] ##ေး ##ဝ ##ါ ##း ##ဘ ##း မ ##ျား [MASK] ##သ ##ည တ က [MASK] ##ည ##ဆ ##ော [MASK] [MASK] ##ော ရ ##ေး [MASK] ##ေ ##တ လ တ သ ##ည က [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1516 111503 34673 1521 111479 1494 48081 111474 111476 41866 34342 27205 34342 1516 111503 34673 1511 111473 1507 1561 1520 27205 34342 64529 111479 34673 1516 111503 103 1508 27205 102 59512 34342 50428 34673 111492 34673 1516 92751 103 111493 111479 1507 1492 103 111479 111475 80200 103 103 80200 1518 59512 103 24189 111484 1519 1507 1521 111479 1492 102\n",
      "INFO:tensorflow:input_ids: 101 1516 111503 34673 1521 111479 1494 48081 111474 111476 41866 34342 27205 34342 1516 111503 34673 1511 111473 1507 1561 1520 27205 34342 64529 111479 34673 1516 111503 103 1508 27205 102 59512 34342 50428 34673 111492 34673 1516 92751 103 111493 111479 1507 1492 103 111479 111475 80200 103 103 80200 1518 59512 103 24189 111484 1519 1507 1521 111479 1492 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 16 29 39 41 46 47 50 51 55 62 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 16 29 39 41 46 47 50 51 55 62 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 34673 34673 1516 1497 1507 111479 27205 1521 1493 1492 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 34673 34673 1516 1497 1507 111479 27205 1521 1493 1492 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] မ မ ##င ##း တ ##စ ပ ##ါ ##း [MASK] ##ြ ##စ [MASK] ##ော သ [MASK] ##ဓ ##မ ##က်တွင် ##သော ##က [MASK] ##င ##း သ ##ည အ ##န ##ဒ ##ယ တ ##စ ##န ##င ##င လ ##း က သ ##မ ##း [MASK] [MASK] nej ##ေ ##ါ ##င ##း [SEP] မ ##န [MASK] ##စ ##ာ ပ ##ါ ပ ##ါ သ ##လ [MASK] ။ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] မ မ ##င ##း တ ##စ ပ ##ါ ##း [MASK] ##ြ ##စ [MASK] ##ော သ [MASK] ##ဓ ##မ ##က်တွင် ##သော ##က [MASK] ##င ##း သ ##ည အ ##န ##ဒ ##ယ တ ##စ ##န ##င ##င လ ##း က သ ##မ ##း [MASK] [MASK] nej ##ေ ##ါ ##င ##း [SEP] မ ##န [MASK] ##စ ##ာ ပ ##ါ ပ ##ါ သ ##လ [MASK] ။ [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1516 1516 111473 34673 1507 111474 1512 50428 34673 103 111504 111474 103 80200 1521 103 111487 64529 81074 17346 27205 103 111473 34673 1521 111479 1524 111488 111486 105942 1507 111474 111488 111473 111473 1519 34673 1492 1521 64529 34673 103 103 78128 24189 50428 111473 34673 102 1516 111488 103 111474 18378 1512 50428 1512 50428 1521 41866 103 1559 102\n",
      "INFO:tensorflow:input_ids: 101 1516 1516 111473 34673 1507 111474 1512 50428 34673 103 111504 111474 103 80200 1521 103 111487 64529 81074 17346 27205 103 111473 34673 1521 111479 1524 111488 111486 105942 1507 111474 111488 111473 111473 1519 34673 1492 1521 64529 34673 103 103 78128 24189 50428 111473 34673 102 1516 111488 103 111474 18378 1512 50428 1512 50428 1521 41866 103 1559 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 10 13 16 19 22 42 43 44 52 61 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 10 13 16 19 22 42 43 44 52 61 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 1513 1521 48081 73335 64529 111489 27205 1512 27205 35089 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 1513 1521 48081 73335 64529 111489 27205 1512 27205 35089 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [MASK] ##း အ [MASK] ##စ လ [MASK] ##း ထ ##ော ##င ##သ ##င ##း အ ##က ##ျ ##ဉ [MASK] ##ခ ##ျ ##ခ ဘ ##ဝ ရ ##ော [MASK] န ##ေ က ##ြ [SEP] ##ား သ ##ော ဒ [MASK] ##ပ ##ပ [MASK] ##ါ ##င ##း ၊ ဒ ##ြ ##ပ ##န ##ော မ [MASK] [MASK] ##င ဓ ##ာ ##တ လ ##မ ##း ##စ ##낼 မ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [MASK] ##း အ [MASK] ##စ လ [MASK] ##း ထ ##ော ##င ##သ ##င ##း အ ##က ##ျ ##ဉ [MASK] ##ခ ##ျ ##ခ ဘ ##ဝ ရ ##ော [MASK] န ##ေ က ##ြ [SEP] ##ား သ ##ော ဒ [MASK] ##ပ ##ပ [MASK] ##ါ ##င ##း ၊ ဒ ##ြ ##ပ ##န ##ော မ [MASK] [MASK] ##င ဓ ##ာ ##တ လ ##မ ##း ##စ ##낼 မ [SEP]\n",
      "INFO:tensorflow:input_ids: 101 103 34673 1524 103 111474 1519 103 34673 1508 80200 111473 111493 111473 34673 1524 27205 111503 111478 103 111470 111503 111470 1515 34342 1518 80200 103 1511 24189 1492 111504 102 35089 1521 80200 1509 103 111489 111489 103 50428 111473 34673 1558 1509 111504 111489 111488 80200 1516 103 103 111473 1510 18378 111484 1519 64529 34673 111474 118724 1516 102\n",
      "INFO:tensorflow:input_ids: 101 103 34673 1524 103 111474 1519 103 34673 1508 80200 111473 111493 111473 34673 1524 27205 111503 111478 103 111470 111503 111470 1515 34342 1518 80200 103 1511 24189 1492 111504 102 35089 1521 80200 1509 103 111489 111489 103 50428 111473 34673 1558 1509 111504 111489 111488 80200 1516 103 103 111473 1510 18378 111484 1519 64529 34673 111474 118724 1516 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 1 4 7 19 27 37 40 51 52 61 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 1 4 7 19 27 37 40 51 52 61 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 73335 26705 111479 34673 27205 111504 24189 92751 1507 111478 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 73335 26705 111479 34673 27205 111504 24189 92751 1507 111478 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ထ ##ирао [MASK] ##မ က မ တ ##ရ ##တ ##သ ##မ ##မ [MASK] ##န ##င ##င က အ ##ပ [MASK] ##ျ ##ပ သ [MASK] ။ [SEP] ##ေ က ဖ ##ြ ##စ စ ##ေ တ [MASK] ၏ ၊ သ ##လ [MASK] က ပ ##ား [MASK] ##ေ [MASK] ၊ လ ##ေ က န ##င ၏ ၊ အ ##ရ ##က [MASK] ##ည ရ ##ော ##င [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ထ ##ирао [MASK] ##မ က မ တ ##ရ ##တ ##သ ##မ ##မ [MASK] ##န ##င ##င က အ ##ပ [MASK] ##ျ ##ပ သ [MASK] ။ [SEP] ##ေ က ဖ ##ြ ##စ စ ##ေ တ [MASK] ၏ ၊ သ ##လ [MASK] က ပ ##ား [MASK] ##ေ [MASK] ၊ လ ##ေ က န ##င ၏ ၊ အ ##ရ ##က [MASK] ##ည ရ ##ော ##င [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1508 82894 103 64529 1492 1516 1507 48081 111484 111493 64529 64529 103 111488 111473 111473 1492 1524 111489 103 111503 111489 1521 103 1559 102 24189 1492 1513 111504 111474 1497 24189 1507 103 1563 1558 1521 41866 103 1492 1512 35089 103 24189 103 1558 1519 24189 1492 1511 111473 1563 1558 1524 48081 27205 103 111479 1518 80200 111473 102\n",
      "INFO:tensorflow:input_ids: 101 1508 82894 103 64529 1492 1516 1507 48081 111484 111493 64529 64529 103 111488 111473 111473 1492 1524 111489 103 111503 111489 1521 103 1559 102 24189 1492 1513 111504 111474 1497 24189 1507 103 1563 1558 1521 41866 103 1492 1512 35089 103 24189 103 1558 1519 24189 1492 1511 111473 1563 1558 1524 48081 27205 103 111479 1518 80200 111473 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 2 3 13 20 24 35 40 44 46 58 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 2 3 13 20 24 35 40 44 46 58 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111473 34342 111484 111470 111479 111484 111489 1497 1563 1521 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111473 34342 111484 111470 111479 111484 111489 1497 1563 1521 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ##ခ သ ##ည န [MASK] ##း ##ဇ ##လ ##န က ၁ ##၇ ##၆ ##၉ တ ##င ရ ##ော [MASK] ရ ခ ပ ##ြ ##း က [MASK] mas ##ရ ##း ##တ ##န ##း [MASK] ##စ ခ လ ##း က မ ##ြ ##ေ ##ပ ##ဆ ခ သ ##ည [SEP] ပ ##ြ ##skaper ##အ ##မ [MASK] ##င [MASK] မ ##ား န ##ေ တ ##ယ [MASK] [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ##ခ သ ##ည န [MASK] ##း ##ဇ ##လ ##န က ၁ ##၇ ##၆ ##၉ တ ##င ရ ##ော [MASK] ရ ခ ပ ##ြ ##း က [MASK] mas ##ရ ##း ##တ ##န ##း [MASK] ##စ ခ လ ##း က မ ##ြ ##ေ ##ပ ##ဆ ခ သ ##ည [SEP] ပ ##ြ ##skaper ##အ ##မ [MASK] ##င [MASK] မ ##ား န ##ေ တ ##ယ [MASK] [SEP]\n",
      "INFO:tensorflow:input_ids: 101 111470 1521 111479 1511 103 34673 111476 41866 111488 1492 1549 40402 37064 42044 1507 111473 1518 80200 103 1518 1493 1512 111504 34673 1492 103 11856 48081 34673 111484 111488 34673 103 111474 1493 1519 34673 1492 1516 111504 24189 111489 111475 1493 1521 111479 102 1512 111504 100611 111496 64529 103 111473 103 1516 35089 1511 24189 1507 105942 103 102\n",
      "INFO:tensorflow:input_ids: 101 111470 1521 111479 1511 103 34673 111476 41866 111488 1492 1549 40402 37064 42044 1507 111473 1518 80200 103 1518 1493 1512 111504 34673 1492 103 11856 48081 34673 111484 111488 34673 103 111474 1493 1519 34673 1492 1516 111504 24189 111489 111475 1493 1521 111479 102 1512 111504 100611 111496 64529 103 111473 103 1516 35089 1511 24189 1507 105942 103 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 5 19 26 27 33 36 50 53 55 62 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 5 19 26 27 33 36 50 53 55 62 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 105942 27205 64529 34673 1507 1519 111488 34673 24189 1559 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 105942 27205 64529 34673 1507 1519 111488 34673 24189 1559 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ည ##စ ##ပ [MASK] န ##ေ သ ##ော ရ ##ေ [MASK] တ ##င လ ##ည ##း က ##ြ ##ာ ##ရ ##ည အ ##သ ##က [MASK] ရ ##င တ [MASK] ခ відстані [SEP] [MASK] ##ာ ##ရ [MASK] ##စ ခ ပ ##ေ ##ါ ##င [MASK] [MASK] ##က လ ##ျ ##င မ ##ျ ##က ##စ ##တ ##စ ##ဆ ##း ဆ ##မ ##း ##တ ##င မ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ည ##စ ##ပ [MASK] န ##ေ သ ##ော ရ ##ေ [MASK] တ ##င လ ##ည ##း က ##ြ ##ာ ##ရ ##ည အ ##သ ##က [MASK] ရ ##င တ [MASK] ခ відстані [SEP] [MASK] ##ာ ##ရ [MASK] ##စ ခ ပ ##ေ ##ါ ##င [MASK] [MASK] ##က လ ##ျ ##င မ ##ျ ##က ##စ ##တ ##စ ##ဆ ##း ဆ ##မ ##း ##တ ##င မ [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1502 111474 111489 103 1511 24189 1521 80200 1518 24189 103 1507 111473 1519 111479 34673 1492 111504 18378 48081 111479 1524 111493 27205 103 1518 111473 1507 103 1493 22557 102 103 18378 48081 103 111474 1493 1512 24189 50428 111473 103 103 27205 1519 111503 111473 1516 111503 27205 111474 111484 111474 111475 34673 1498 64529 34673 111484 111473 1516 102\n",
      "INFO:tensorflow:input_ids: 101 1502 111474 111489 103 1511 24189 1521 80200 1518 24189 103 1507 111473 1519 111479 34673 1492 111504 18378 48081 111479 1524 111493 27205 103 1518 111473 1507 103 1493 22557 102 103 18378 48081 103 111474 1493 1512 24189 50428 111473 103 103 27205 1519 111503 111473 1516 111503 27205 111474 111484 111474 111475 34673 1498 64529 34673 111484 111473 1516 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 4 11 21 25 29 31 33 36 43 44 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 4 11 21 25 29 31 33 36 43 44 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111484 1508 111479 1516 111484 111503 111494 1511 34673 1519 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111484 1508 111479 1516 111484 111503 111494 1511 34673 1519 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:next_sentence_labels: 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ##ရ ##တ ##န ##ာ မ ##ျား [MASK] ##ည ##း ထ ##က ရ ရ ##ာ ၊ စ ##န [MASK] ##က [MASK] [MASK] [MASK] ##ေ ##သ မ ##ာ မ ##ဇ ##ဈ ##ပ ##ြ [SEP] ##င ##း ၊ က ##န ##း ##က ##ြ ##účastnil ##င ##း တ ဖ ##ြ ##င ရ [MASK] ##က [MASK] န ##င အ ##ခ ##ြား မ ##ြ [MASK] ##မာ ##ပ [MASK] [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ##ရ ##တ ##န ##ာ မ ##ျား [MASK] ##ည ##း ထ ##က ရ ရ ##ာ ၊ စ ##န [MASK] ##က [MASK] [MASK] [MASK] ##ေ ##သ မ ##ာ မ ##ဇ ##ဈ ##ပ ##ြ [SEP] ##င ##း ၊ က ##န ##း ##က ##ြ ##účastnil ##င ##း တ ဖ ##ြ ##င ရ [MASK] ##က [MASK] န ##င အ ##ခ ##ြား မ ##ြ [MASK] ##မာ ##ပ [MASK] [SEP]\n",
      "INFO:tensorflow:input_ids: 101 48081 111484 111488 18378 1516 92751 103 111479 34673 1508 27205 1518 1518 18378 1558 1497 111488 103 27205 103 103 103 24189 111493 1516 18378 1516 111476 111477 111489 111504 102 111473 34673 1558 1492 111488 34673 27205 111504 85135 111473 34673 1507 1513 111504 111473 1518 103 27205 103 1511 111473 1524 111470 67798 1516 111504 103 73335 111489 103 102\n",
      "INFO:tensorflow:input_ids: 101 48081 111484 111488 18378 1516 92751 103 111479 34673 1508 27205 1518 1518 18378 1558 1497 111488 103 27205 103 103 103 24189 111493 1516 18378 1516 111476 111477 111489 111504 102 111473 34673 1558 1492 111488 34673 27205 111504 85135 111473 34673 1507 1513 111504 111473 1518 103 27205 103 1511 111473 1524 111470 67798 1516 111504 103 73335 111489 103 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 7 18 20 21 22 41 49 51 59 62 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 7 18 20 21 22 41 49 51 59 62 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 1519 1508 1521 80200 1509 80200 111488 111488 111488 111504 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 1519 1508 1521 80200 1509 80200 111488 111488 111488 111504 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] [MASK] [MASK] ##မာ ##ရ ##တ တ ##င [MASK] ##င ##ဂ ##လ ##ပ [MASK] [MASK] ##န ##မာ [MASK] တ ##န [MASK] အ ##ထ ပ ##ည ##ာ ##သ ##င ##ယ ခ ၏ ။ [SEP] ##ဂ ၊ မ [MASK] ##ဏ ၊ အ ##ထ ##န ##း ၊ သ ##က ၊ ရ ##ာ ##ဇ ၊ အ ##ဓ [MASK] ##တ ဟ ၍ မ ##ဟ ##ာ ##ဘ ##တ ##အ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] [MASK] [MASK] ##မာ ##ရ ##တ တ ##င [MASK] ##င ##ဂ ##လ ##ပ [MASK] [MASK] ##န ##မာ [MASK] တ ##န [MASK] အ ##ထ ပ ##ည ##ာ ##သ ##င ##ယ ခ ၏ ။ [SEP] ##ဂ ၊ မ [MASK] ##ဏ ၊ အ ##ထ ##န ##း ၊ သ ##က ၊ ရ ##ာ ##ဇ ၊ အ ##ဓ [MASK] ##တ ဟ ၍ မ ##ဟ ##ာ ##ဘ ##တ ##အ [SEP]\n",
      "INFO:tensorflow:input_ids: 101 103 103 73335 48081 111484 1507 111473 103 111473 111471 41866 111489 103 103 111488 73335 103 1507 111488 103 1524 111485 1512 111479 18378 111493 111473 105942 1493 1563 1559 102 111471 1558 1516 103 111483 1558 1524 111485 111488 34673 1558 1521 27205 1558 1518 18378 111476 1558 1524 111487 103 111484 1522 1561 1516 111494 18378 111492 111484 111496 102\n",
      "INFO:tensorflow:input_ids: 101 103 103 73335 48081 111484 1507 111473 103 111473 111471 41866 111489 103 103 111488 73335 103 1507 111488 103 1524 111485 1512 111479 18378 111493 111473 105942 1493 1563 1559 102 111471 1558 1516 103 111483 1558 1524 111485 111488 34673 1558 1521 27205 1558 1518 18378 111476 1558 1524 111487 103 111484 1522 1561 1516 111494 18378 111492 111484 111496 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 1 2 8 13 14 17 20 36 53 60 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 1 2 8 13 14 17 20 36 53 60 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111488 1492 1524 1516 111504 1555 34673 48081 111489 111492 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111488 1492 1524 1516 111504 1555 34673 48081 111489 111492 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ခ ##န ##ဓ ##ာ ##က ##ယ ၌ သ ##ေး ##က [MASK] ##ော ##երել ##ျား ဖ ##ြ ##င သ ##ေး Деревня ##တ ပ က သ [MASK] ##ေ ##သ ##န [MASK] ##ြ ##ခ [SEP] ##င ##ဂ ##လ ##အ ##င ##ဒ ##း ##ယ ##န ##း အ ##သ ##င ##း [MASK] [MASK] ##င [MASK] ##တ ##င ##း [MASK] သ ##မ ##မ [MASK] က တ ##က ##ရ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ခ ##န ##ဓ ##ာ ##က ##ယ ၌ သ ##ေး ##က [MASK] ##ော ##երել ##ျား ဖ ##ြ ##င သ ##ေး Деревня ##တ ပ က သ [MASK] ##ေ ##သ ##န [MASK] ##ြ ##ခ [SEP] ##င ##ဂ ##လ ##အ ##င ##ဒ ##း ##ယ ##န ##း အ ##သ ##င ##း [MASK] [MASK] ##င [MASK] ##တ ##င ##း [MASK] သ ##မ ##မ [MASK] က တ ##က ##ရ [SEP]\n",
      "INFO:tensorflow:input_ids: 101 1493 111488 111487 18378 27205 105942 1560 1521 59512 27205 103 80200 89325 92751 1513 111504 111473 1521 59512 73401 111484 1512 1492 1521 103 24189 111493 111488 103 111504 111470 102 111473 111471 41866 111496 111473 111486 34673 105942 111488 34673 1524 111493 111473 34673 103 103 111473 103 111484 111473 34673 103 1521 64529 64529 103 1492 1507 27205 48081 102\n",
      "INFO:tensorflow:input_ids: 101 1493 111488 111487 18378 27205 105942 1560 1521 59512 27205 103 80200 89325 92751 1513 111504 111473 1521 59512 73401 111484 1512 1492 1521 103 24189 111493 111488 103 111504 111470 102 111473 111471 41866 111496 111473 111486 34673 105942 111488 34673 1524 111493 111473 34673 103 103 111473 103 111484 111473 34673 103 1521 64529 64529 103 1492 1507 27205 48081 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 11 13 20 25 29 47 48 50 54 58 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 11 13 20 25 29 47 48 50 54 58 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111504 1516 41866 111484 111489 111496 34342 34673 1516 111484 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111504 1516 41866 111484 111489 111496 34342 34673 1516 111484 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ##း တ အ ##ပ ##ါ ##အ ##ဝ ##င ရ [MASK] [MASK] ##ော [MASK] ##ပ တ ##င ပ ##ါ ##ဝ ##င ##ယ ##ဉ ##ပ ##ြ Bewegung ##မ ##ည ##သ အ ##ား ##လ [SEP] ##င ##း ##တ ##စ ##ပ ##ါ ##း ##သား အ [MASK] ##ေ ##ါ ##အ [MASK] ##ေ ##ါ မ ##ျား န [MASK] သ ##ပ [MASK] ##ပ ##ည [MASK] ဆ ##င ##ရာ အ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ##း တ အ ##ပ ##ါ ##အ ##ဝ ##င ရ [MASK] [MASK] ##ော [MASK] ##ပ တ ##င ပ ##ါ ##ဝ ##င ##ယ ##ဉ ##ပ ##ြ Bewegung ##မ ##ည ##သ အ ##ား ##လ [SEP] ##င ##း ##တ ##စ ##ပ ##ါ ##း ##သား အ [MASK] ##ေ ##ါ ##အ [MASK] ##ေ ##ါ မ ##ျား န [MASK] သ ##ပ [MASK] ##ပ ##ည [MASK] ဆ ##င ##ရာ အ [SEP]\n",
      "INFO:tensorflow:input_ids: 101 34673 1507 1524 111489 50428 111496 34342 111473 1518 103 103 80200 103 111489 1507 111473 1512 50428 34342 111473 105942 111478 111489 111504 39178 64529 111479 111493 1524 35089 41866 102 111473 34673 111484 111474 111489 50428 34673 69046 1524 103 24189 50428 111496 103 24189 50428 1516 92751 1511 103 1521 111489 103 111489 111479 103 1498 111473 56416 1524 102\n",
      "INFO:tensorflow:input_ids: 101 34673 1507 1524 111489 50428 111496 34342 111473 1518 103 103 80200 103 111489 1507 111473 1512 50428 34342 111473 105942 111478 111489 111504 39178 64529 111479 111493 1524 35089 41866 102 111473 34673 111484 111474 111489 50428 34673 69046 1524 103 24189 50428 111496 103 24189 50428 1516 92751 1511 103 1521 111489 103 111489 111479 103 1498 111473 56416 1524 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 10 11 13 23 25 42 46 52 55 58 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 10 11 13 23 25 42 46 52 55 58 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 59512 27205 27205 111489 111473 111470 34342 111473 111489 18378 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 59512 27205 27205 111489 111473 111470 34342 111473 111489 18378 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ##း ##န ##ည ##း ##ရ ##ပ ##ဝ ##န ##း ##ဒ ##ေ ##သ တ ##င မ အ [MASK] ##င [MASK] ##ျား [MASK] ##ာ က ##ြ ပ ##ြ ##း လ က ##ြ သ [SEP] ခ သ ##ော ##လ ##ည [MASK] [MASK] ##ော [MASK] [MASK] ##င ##း [MASK] ##င ပ ##ြ ##န ##လ [MASK] ပ ##ြ ##ပ ##ြ ##င ၍ မ ##ြ ##န ##န ##း [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ##း ##န ##ည ##း ##ရ ##ပ ##ဝ ##န ##း ##ဒ ##ေ ##သ တ ##င မ အ [MASK] ##င [MASK] ##ျား [MASK] ##ာ က ##ြ ပ ##ြ ##း လ က ##ြ သ [SEP] ခ သ ##ော ##လ ##ည [MASK] [MASK] ##ော [MASK] [MASK] ##င ##း [MASK] ##င ပ ##ြ ##န ##လ [MASK] ပ ##ြ ##ပ ##ြ ##င ၍ မ ##ြ ##န ##န ##း [SEP]\n",
      "INFO:tensorflow:input_ids: 101 34673 111488 111479 34673 48081 111489 34342 111488 34673 111486 24189 111493 1507 111473 1516 1524 103 111473 103 92751 103 18378 1492 111504 1512 111504 34673 1519 1492 111504 1521 102 1493 1521 80200 41866 111479 103 103 80200 103 103 111473 34673 103 111473 1512 111504 111488 41866 103 1512 111504 111489 111504 111473 1561 1516 111504 111488 111488 34673 102\n",
      "INFO:tensorflow:input_ids: 101 34673 111488 111479 34673 48081 111489 34342 111488 34673 111486 24189 111493 1507 111473 1516 1524 103 111473 103 92751 103 18378 1492 111504 1512 111504 34673 1519 1492 111504 1521 102 1493 1521 80200 41866 111479 103 103 80200 103 103 111473 34673 103 111473 1512 111504 111488 41866 103 1512 111504 111489 111504 111473 1561 1516 111504 111488 111488 34673 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 8 17 19 21 38 39 41 42 45 51 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 8 17 19 21 38 39 41 42 45 51 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111488 111489 1516 1516 34673 1511 27205 111489 1507 111479 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 111488 111489 1516 1516 34673 1511 27205 111489 1507 111479 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:tokens: [CLS] ##ေ [MASK] ##ဖ [MASK] ##င မ [MASK] အ ##ဖ ##ြ ##စ အ ##လ ##ယ [MASK] ##လ ##တ ##လ ##မ ##း ##စ ##ဉ ##သ ##မာ ##း မ ##ျား [MASK] မ ##မ ယ [SEP] တ [MASK] ##ျ ##း ##တ အ ##မ [MASK] ##စ ##ာ ##ရ ##င ##း [MASK] က တ ##စ [MASK] ဖ ##ြ ##စ န ##င သ ##လ iso က အ ##ခ ##ြ [SEP]\n",
      "INFO:tensorflow:tokens: [CLS] ##ေ [MASK] ##ဖ [MASK] ##င မ [MASK] အ ##ဖ ##ြ ##စ အ ##လ ##ယ [MASK] ##လ ##တ ##လ ##မ ##း ##စ ##ဉ ##သ ##မာ ##း မ ##ျား [MASK] မ ##မ ယ [SEP] တ [MASK] ##ျ ##း ##တ အ ##မ [MASK] ##စ ##ာ ##ရ ##င ##း [MASK] က တ ##စ [MASK] ဖ ##ြ ##စ န ##င သ ##လ iso က အ ##ခ ##ြ [SEP]\n",
      "INFO:tensorflow:input_ids: 101 24189 103 111490 103 111473 1516 103 1524 111490 111504 111474 1524 41866 105942 103 41866 111484 41866 64529 34673 111474 111478 111493 73335 34673 1516 92751 103 1516 64529 1517 102 1507 103 111503 34673 111484 1524 64529 103 111474 18378 48081 111473 34673 103 1492 1507 111474 103 1513 111504 111474 1511 111473 1521 41866 93455 1492 1524 111470 111504 102\n",
      "INFO:tensorflow:input_ids: 101 24189 103 111490 103 111473 1516 103 1524 111490 111504 111474 1524 41866 105942 103 41866 111484 41866 64529 34673 111474 111478 111493 73335 34673 1516 92751 103 1516 64529 1517 102 1507 103 111503 34673 111484 1524 64529 103 111474 18378 48081 111473 34673 103 1492 1507 111474 103 1513 111504 111474 1511 111473 1521 41866 93455 1492 1524 111470 111504 102\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "INFO:tensorflow:masked_lm_positions: 2 4 7 15 28 34 40 46 50 62 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_positions: 2 4 7 15 28 34 40 46 50 62 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 1524 34342 92751 111496 1522 1516 111479 1508 1493 111504 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_ids: 1524 34342 92751 111496 1522 1516 111479 1508 1493 111504 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:masked_lm_weights: 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:next_sentence_labels: 1\n",
      "INFO:tensorflow:Wrote 41845 total instances\n",
      "INFO:tensorflow:Wrote 41845 total instances\n"
     ]
    }
   ],
   "source": [
    "!python  '/gdrive/My Drive/Colab Notebooks/codeTest/bert/create_pretraining_data.py'  --input_file='/gdrive/My Drive/Colab Notebooks/codeTest/myaninput/myanWordTest.txt'  --output_file='/gdrive/My Drive/Colab Notebooks/codeTest/multimyanoutput/tf_examples.tfrecord' --vocab_file='/gdrive/My Drive/Colab Notebooks/codeTest/multilan/vocab.txt' --do_lower_case=True  --max_seq_length=64 --max_predictions_per_seq=20  --masked_lm_prob=0.15  --random_seed=12345  --dupe_factor=5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" \n",
    "Remark\n",
    "I got the following result after running run_pretraining.py when using  pre-trained checkpoints of of MultiLanguage BERT model.\n",
    "***** Eval results *****\n",
    " global_step = 20\n",
    " loss = 2.9664276\n",
    " masked_lm_accuracy = 0.39133173\n",
    " masked_lm_loss = 2.4433851\n",
    " next_sentence_accuracy = 0.74625\n",
    " next_sentence_loss = 0.5230467\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "ylZVDIJkv4pW",
    "outputId": "52a128fc-73de-4121-dc4f-5199a9371ef2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "INFO:tensorflow:*** Input Files ***\n",
      "INFO:tensorflow:  /gdrive/My Drive/Colab Notebooks/codeTest/multimyanoutput/tf_examples.tfrecord\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7f88a5fec950>) includes params argument, but params are not passed to Estimator.\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/gdrive/My Drive/Colab Notebooks/codeTest/multipretraining_myan', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f88944f65c0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n",
      "INFO:tensorflow:***** Running training *****\n",
      "INFO:tensorflow:  Batch size = 32\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /gdrive/My Drive/Colab Notebooks/codeTest/bert/run_pretraining.py:368: parallel_interleave (from tensorflow.contrib.data.python.ops.interleave_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.data.experimental.parallel_interleave(...)`.\n",
      "WARNING:tensorflow:From /gdrive/My Drive/Colab Notebooks/codeTest/bert/run_pretraining.py:385: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.data.experimental.map_and_batch(...)`.\n",
      "WARNING:tensorflow:From /gdrive/My Drive/Colab Notebooks/codeTest/bert/run_pretraining.py:400: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running train on CPU\n",
      "INFO:tensorflow:*** Features ***\n",
      "INFO:tensorflow:  name = input_ids, shape = (32, 64)\n",
      "INFO:tensorflow:  name = input_mask, shape = (32, 64)\n",
      "INFO:tensorflow:  name = masked_lm_ids, shape = (32, 20)\n",
      "INFO:tensorflow:  name = masked_lm_positions, shape = (32, 20)\n",
      "INFO:tensorflow:  name = masked_lm_weights, shape = (32, 20)\n",
      "INFO:tensorflow:  name = next_sentence_labels, shape = (32, 1)\n",
      "INFO:tensorflow:  name = segment_ids, shape = (32, 64)\n",
      "WARNING:tensorflow:From /gdrive/My Drive/Colab Notebooks/codeTest/bert/modeling.py:358: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /gdrive/My Drive/Colab Notebooks/codeTest/bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "INFO:tensorflow:**** Trainable Variables ****\n",
      "INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (119547, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = cls/predictions/transform/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = cls/predictions/transform/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = cls/predictions/output_bias:0, shape = (119547,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = cls/seq_relationship/output_weights:0, shape = (2, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = cls/seq_relationship/output_bias:0, shape = (2,), *INIT_FROM_CKPT*\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/learning_rate_decay_v2.py:321: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2020-04-20 15:34:10.187322: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2020-04-20 15:34:10.191001: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2020-04-20 15:34:10.191325: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x2660d60 executing computations on platform Host. Devices:\n",
      "2020-04-20 15:34:10.191367: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into /gdrive/My Drive/Colab Notebooks/codeTest/multipretraining_myan/model.ckpt.\n",
      "INFO:tensorflow:Saving checkpoints for 20 into /gdrive/My Drive/Colab Notebooks/codeTest/multipretraining_myan/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 3.1210353.\n",
      "INFO:tensorflow:training_loop marked as finished\n",
      "INFO:tensorflow:***** Running evaluation *****\n",
      "INFO:tensorflow:  Batch size = 8\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running eval on CPU\n",
      "INFO:tensorflow:*** Features ***\n",
      "INFO:tensorflow:  name = input_ids, shape = (8, 64)\n",
      "INFO:tensorflow:  name = input_mask, shape = (8, 64)\n",
      "INFO:tensorflow:  name = masked_lm_ids, shape = (8, 20)\n",
      "INFO:tensorflow:  name = masked_lm_positions, shape = (8, 20)\n",
      "INFO:tensorflow:  name = masked_lm_weights, shape = (8, 20)\n",
      "INFO:tensorflow:  name = next_sentence_labels, shape = (8, 1)\n",
      "INFO:tensorflow:  name = segment_ids, shape = (8, 64)\n",
      "INFO:tensorflow:**** Trainable Variables ****\n",
      "INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (119547, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = cls/predictions/transform/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = cls/predictions/transform/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = cls/predictions/transform/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = cls/predictions/output_bias:0, shape = (119547,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = cls/seq_relationship/output_weights:0, shape = (2, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = cls/seq_relationship/output_bias:0, shape = (2,), *INIT_FROM_CKPT*\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/metrics_impl.py:455: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2020-04-20T15:54:09Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from /gdrive/My Drive/Colab Notebooks/codeTest/multipretraining_myan/model.ckpt-20\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Evaluation [10/100]\n",
      "INFO:tensorflow:Evaluation [20/100]\n",
      "INFO:tensorflow:Evaluation [30/100]\n",
      "INFO:tensorflow:Evaluation [40/100]\n",
      "INFO:tensorflow:Evaluation [50/100]\n",
      "INFO:tensorflow:Evaluation [60/100]\n",
      "INFO:tensorflow:Evaluation [70/100]\n",
      "INFO:tensorflow:Evaluation [80/100]\n",
      "INFO:tensorflow:Evaluation [90/100]\n",
      "INFO:tensorflow:Evaluation [100/100]\n",
      "INFO:tensorflow:Finished evaluation at 2020-04-20-16:02:35\n",
      "INFO:tensorflow:Saving dict for global step 20: global_step = 20, loss = 2.9664276, masked_lm_accuracy = 0.39133173, masked_lm_loss = 2.4433851, next_sentence_accuracy = 0.74625, next_sentence_loss = 0.5230467\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 20: /gdrive/My Drive/Colab Notebooks/codeTest/multipretraining_myan/model.ckpt-20\n",
      "INFO:tensorflow:evaluation_loop marked as finished\n",
      "INFO:tensorflow:***** Eval results *****\n",
      "INFO:tensorflow:  global_step = 20\n",
      "INFO:tensorflow:  loss = 2.9664276\n",
      "INFO:tensorflow:  masked_lm_accuracy = 0.39133173\n",
      "INFO:tensorflow:  masked_lm_loss = 2.4433851\n",
      "INFO:tensorflow:  next_sentence_accuracy = 0.74625\n",
      "INFO:tensorflow:  next_sentence_loss = 0.5230467\n"
     ]
    }
   ],
   "source": [
    "!python '/gdrive/My Drive/Colab Notebooks/codeTest/bert/run_pretraining.py'   --input_file='/gdrive/My Drive/Colab Notebooks/codeTest/multimyanoutput/tf_examples.tfrecord'  --output_dir='/gdrive/My Drive/Colab Notebooks/codeTest/multipretraining_myan'  --do_train=True  --do_eval=True  --bert_config_file='/gdrive/My Drive/Colab Notebooks/codeTest/multilan/bert_config.json'  --init_checkpoint='/gdrive/My Drive/Colab Notebooks/codeTest/multilan/bert_model.ckpt' --train_batch_size=32 --max_seq_length=64 --max_predictions_per_seq=20 --num_train_steps=20 --num_warmup_steps=10 --learning_rate=2e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pHBAqetpEHQC"
   },
   "outputs": [],
   "source": [
    "#!jupyter notebook --NotebookApp.iopub_data_rate_limit=10000000000"
   ]
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "collapsed_sections": [],
   "name": "SumyProject.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}