Bantu_Language_Modeling Final.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Bantu Language Modeling\n",
        "\n"
      ],
      "metadata": {
        "id": "rgCT4sQCc448"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data Loading and preprocessing"
      ],
      "metadata": {
        "id": "TZyFgFg-dEU3"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "OG7R-7bVc1T8"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "\n",
        "path_sw = r\"/content/sw-train.txt\"\n",
        "path_kw = r\"/content/cwe-train.txt\"\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.utils import to_categorical\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "class Preprocessing:\n",
        "\n",
        "    def __init__(self, data:list = None) -> None:\n",
        "        self.data = data\n",
        "        self.vocab = None\n",
        "        self.encoder = None\n",
        "        self.decoder = None\n",
        "\n",
        "\n",
        "    def clean_data(self):\n",
        "      self.data[0] = self.data[0].replace(\"\\n\", \" \")\n",
        "\n",
        "\n",
        "    def read_file(self, path):\n",
        "\n",
        "      '''\n",
        "      https://colab.research.google.com/drive/1pFL6pHFsG6QAl0th99mIKsPaUJ84w6gN?usp=sharing#scrollTo=XM3TiINWVhyf      \n",
        "      '''\n",
        "\n",
        "      self.data = open(path).readlines()\n",
        "      \n",
        "\n",
        "    def set_vocab_encoder_decoder(self):\n",
        "\n",
        "      '''\n",
        "      set the vocab and encoder/decoder for the dataset\n",
        "      https://colab.research.google.com/drive/16_OYcypTu0OtA1eOT6Zjg65atzXFX9GH#scrollTo=DCf7rn8bX3Ws      \n",
        "      '''\n",
        "\n",
        "\n",
        "      self.vocab = sorted(list(set(self.data[0])))\n",
        "      self.encoder = dict((c,i) for i,c in enumerate(self.vocab))\n",
        "      self.decoder = dict((i,c) for i,c in enumerate(self.vocab))\n",
        "\n",
        "    def set_X_y(self, maxlen=40, steps = 2, sample_size:tuple = tuple()):\n",
        "\n",
        "      '''\n",
        "      cut the text in semi-redundant sequences of maxlen characters. \n",
        "      if sample_size specified, choose subset of that size.\n",
        "\n",
        "      '''\n",
        "      \n",
        "\n",
        "      print(sample_size[0], sample_size[1])\n",
        "      if len(sample_size) == 1:\n",
        "        self.data[0] = self.data[0][:sample_size[0]]\n",
        "      elif len(sample_size) == 2:\n",
        "        self.data[0] = self.data[0][sample_size[0]:sample_size[1]]\n",
        "      else:\n",
        "        self.data[0] = self.data[0]\n",
        "\n",
        "      maxlen = 40\n",
        "      steps = 2\n",
        "\n",
        "      sentences = []\n",
        "      next_chars = []\n",
        "\n",
        "      # Append sentences to X\n",
        "      # Append the next character the sentence precedes as the target value\n",
        "\n",
        "      for i in range(0, len(self.data[0]) - maxlen, steps):\n",
        "          sentences.append(self.data[0][i : i + maxlen])\n",
        "          next_chars.append(self.data[0][i + maxlen])\n",
        "\n",
        "      print(\"Number of sequences:\", len(sentences))\n",
        "\n",
        "      x = np.zeros((len(sentences), maxlen, len(self.vocab)), dtype=float)\n",
        "      y = np.zeros((len(sentences), len(self.vocab)), dtype=float)\n",
        "\n",
        "      # encode the values and features      \n",
        "      for i, sentence in enumerate(sentences):\n",
        "          for t, char in enumerate(sentence):\n",
        "              x[i, t, self.encoder[char]] = 1\n",
        "\n",
        "          y[i, self.encoder[next_chars[i]]] = 1\n",
        "\n",
        "      print(x.shape, y.shape)\n",
        "        \n",
        "      return x, y\n",
        "\n"
      ],
      "metadata": {
        "id": "7bNi25V-FLBE"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Model Definitions"
      ],
      "metadata": {
        "id": "c3RjloF4WfIb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Embedding, Flatten, Dropout, \\\n",
        "                                    LSTM, Bidirectional\n",
        "from tensorflow.keras import Model,Input\n",
        "from tensorflow.keras.optimizers import Adam, RMSprop\n",
        "from sklearn.metrics import accuracy_score\n",
        "from tensorflow.keras.models import save_model, load_model\n",
        "import time\n",
        "\n",
        "class Model:\n",
        "  \n",
        "  def __init__(self, X_train, X_test, y_train, y_test, input_dim, output_dim, input_length):\n",
        "    self.model = None\n",
        "    self.X_train, self.X_test, self.y_train, self.y_test = X_train, X_test, y_train, y_test\n",
        "    self.y_pred = None\n",
        "    self.input_dim, self.output_dim, self.input_length = input_dim, output_dim, input_length\n",
        "    self.batch_size = 64\n",
        "    self.epochs = 50\n",
        "    self.history = None\n",
        "    self.model_path = None\n",
        "\n",
        "  def build_model_variant1(self, shape=()):\n",
        "\n",
        "    '''\n",
        "    A simple Bidirectional LSTM model\n",
        "    '''\n",
        "\n",
        "    self.model = Sequential([\n",
        "        \n",
        "      Input(shape=(self.input_length, self.input_dim)),\n",
        "      Bidirectional(LSTM(64,return_sequences=False)),\n",
        "      Dense(self.input_dim, activation=\"softmax\"),\n",
        "\n",
        "    ])\n",
        "\n",
        "    optimizer = RMSprop(learning_rate=1e-2, decay=1e-6)\n",
        "    self.model.compile(loss='categorical_crossentropy', optimizer=optimizer)\n",
        "\n",
        "    self.model.summary() \n",
        "\n",
        "  def build_model_variant2(self, shape=()):\n",
        "\n",
        "    '''\n",
        "    A single LSTM layer model\n",
        "    '''\n",
        "\n",
        "    self.model = Sequential([\n",
        "        \n",
        "      Input(shape=(self.input_length, self.input_dim)),\n",
        "      LSTM(128,return_sequences=False),\n",
        "      Dense(self.input_dim, activation=\"softmax\"),\n",
        "\n",
        "    ])\n",
        "\n",
        "    optimizer = RMSprop(learning_rate=1e-2, decay=1e-6)\n",
        "\n",
        "    self.model.compile(loss = 'categorical_crossentropy', optimizer = optimizer)\n",
        "\n",
        "    self.model.summary()  \n",
        "\n",
        "  def model_fit(self, batch_size=None, epochs = None, name=\"\"):\n",
        "\n",
        "    if batch_size is not None:\n",
        "      self.batch_size = batch_size\n",
        "\n",
        "    if epochs is not None:\n",
        "      self.epochs = epochs\n",
        "\n",
        "    # save model history locally\n",
        "    # https://stackoverflow.com/questions/47843265/how-can-i-get-a-keras-models-history-after-loading-it-from-a-file-in-python\n",
        "    from keras.callbacks import CSVLogger\n",
        "    tagger = str(time.time())[0:10]\n",
        "    csv_logger = CSVLogger(name + \"_\" + tagger + '_training.log', separator=',', append=False)\n",
        "\n",
        "    self.history = self.model.fit(self.X_train, self.y_train, \\\n",
        "                                  batch_size=self.batch_size, \\\n",
        "                                  epochs=self.epochs, \\\n",
        "                                  validation_split=0.15, \\\n",
        "                                  callbacks=[csv_logger],\n",
        "                                  verbose=2)\n",
        "    # save model\n",
        "    path = os.getcwd() + \"/\" + name + \"_\" + tagger + \"_loss_\" + \\\n",
        "          str(self.history.history[\"val_loss\"][-1])[0:6].replace(\".\", \"_\") + \\\n",
        "          \"_model.h5\"\n",
        "    self.model_path = path\n",
        "    save_model(self.model, path)\n",
        "\n",
        "\n",
        "  def model_predict(self, X_test = None, y_test = None):\n",
        "    if X_test is not None:\n",
        "      self.X_test = X_test\n",
        "\n",
        "    if y_test is not None:\n",
        "      self.y_test = y_test\n",
        "\n",
        "    self.y_pred = self.model.predict(self.X_test)\n",
        "\n",
        "    for y in self.y_pred:\n",
        "      for ind, val in enumerate(y):\n",
        "        if val == max(y):\n",
        "          y[ind] = 1.0\n",
        "        else:\n",
        "          y[ind] = 0.0\n",
        "\n",
        "\n",
        "  def model_evaluate(self):\n",
        "    \n",
        "    for y in self.y_pred:\n",
        "      for ind, val in enumerate(y):\n",
        "        if val == max(y):\n",
        "          y[ind] = 1.0\n",
        "        else:\n",
        "          y[ind] = 0.0\n",
        "\n",
        "    return accuracy_score(self.y_test, self.y_pred)\n",
        "\n",
        "\n",
        "  def load_model(self, path):\n",
        "    self.model = load_model(path)"
      ],
      "metadata": {
        "id": "4XARHAd11_15"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluate"
      ],
      "metadata": {
        "id": "uLwCHmLDDVW1"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Swahili"
      ],
      "metadata": {
        "id": "wspwNklTCRi_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from time import sleep\n",
        "\n",
        "# SWAHILI\n",
        "s_p = Preprocessing()\n",
        "s_p.read_file(path_sw)\n",
        "s_p.clean_data()\n",
        "s_p.set_vocab_encoder_decoder()\n",
        "\n",
        "input_dim = len(s_p.vocab)\n",
        "inputlen = emb_dim = 40\n",
        "\n",
        "data_len = len(s_p.data[0])\n",
        "\n",
        "counter = 0\n",
        "\n",
        "# Train in stages to prevent session crashing for a total of 20 epochs\n",
        "while counter < 20:\n",
        "  sleep(60)\n",
        "  start = int((data_len / 160) * counter)\n",
        "  end = int(((data_len / 160) * (counter+1)) - 1)\n",
        "\n",
        "  s_p.read_file(path_sw)\n",
        "  s_p.clean_data()\n",
        "\n",
        "  sw_X_train, sw_X_test = s_p.set_X_y(sample_size = (start, end))\n",
        "\n",
        "  input_dim = len(s_p.vocab)\n",
        "  inputlen = emb_dim = 40\n",
        "\n",
        "  sw_X_train, sw_X_test, sw_y_train, sw_y_test = train_test_split(sw_X_train, sw_X_test, test_size=0.1, random_state = 42)\n",
        "\n",
        "  if counter == 0:\n",
        "    s_m = Model(sw_X_train, sw_X_test, sw_y_train, sw_y_test, input_dim, emb_dim, inputlen)\n",
        "    s_m.build_model_variant2()\n",
        "    print('Counter ', counter)\n",
        "\n",
        "  else:\n",
        "    s_m.load_model(model_path)\n",
        "\n",
        "  counter += 1\n",
        "  optimizer = RMSprop(learning_rate=1e-2, decay=1e-6)\n",
        "  s_m.model.compile(loss = 'categorical_crossentropy', optimizer = optimizer)\n",
        "\n",
        "  s_m.model_fit(batch_size = 256, epochs = 1, name=\"sw\")\n",
        "  model_path = s_m.model_path\n",
        "  print(model_path)\n",
        "\n",
        "\n",
        "\n",
        "# # sm = make_model(path_sw)\n",
        "# # km = make_model(path_kw)\n"
      ],
      "metadata": {
        "id": "OIh63N6qkcOW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4ac74ec0-cfc4-4345-d9b0-3376dd33547c"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0 245380\n",
            "Number of sequences: 122670\n",
            "(122670, 40, 48) (122670, 48)\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " lstm (LSTM)                 (None, 128)               90624     \n",
            "                                                                 \n",
            " dense (Dense)               (None, 48)                6192      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 96,816\n",
            "Trainable params: 96,816\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Counter  0\n",
            "367/367 - 10s - loss: 2.3028 - val_loss: 2.0554 - 10s/epoch - 27ms/step\n",
            "/content/sw_1667446371_loss_2_0554_model.h5\n",
            "245381 490762\n",
            "Number of sequences: 122671\n",
            "(122671, 40, 48) (122671, 48)\n",
            "367/367 - 5s - loss: 1.9699 - val_loss: 1.9198 - 5s/epoch - 14ms/step\n",
            "/content/sw_1667446446_loss_1_9197_model.h5\n",
            "490763 736143\n",
            "Number of sequences: 122670\n",
            "(122670, 40, 48) (122670, 48)\n",
            "367/367 - 5s - loss: 1.8372 - val_loss: 1.8565 - 5s/epoch - 13ms/step\n",
            "/content/sw_1667446518_loss_1_8564_model.h5\n",
            "736144 981525\n",
            "Number of sequences: 122671\n",
            "(122671, 40, 48) (122671, 48)\n",
            "367/367 - 5s - loss: 1.7446 - val_loss: 1.8177 - 5s/epoch - 14ms/step\n",
            "/content/sw_1667446591_loss_1_8177_model.h5\n",
            "981526 1226907\n",
            "Number of sequences: 122671\n",
            "(122671, 40, 48) (122671, 48)\n",
            "367/367 - 5s - loss: 1.6751 - val_loss: 1.8053 - 5s/epoch - 13ms/step\n",
            "/content/sw_1667446661_loss_1_8052_model.h5\n",
            "1226908 1472288\n",
            "Number of sequences: 122670\n",
            "(122670, 40, 48) (122670, 48)\n",
            "367/367 - 5s - loss: 1.6190 - val_loss: 1.7925 - 5s/epoch - 13ms/step\n",
            "/content/sw_1667446731_loss_1_7925_model.h5\n",
            "1472289 1717670\n",
            "Number of sequences: 122671\n",
            "(122671, 40, 48) (122671, 48)\n",
            "367/367 - 6s - loss: 1.5788 - val_loss: 1.8095 - 6s/epoch - 17ms/step\n",
            "/content/sw_1667446803_loss_1_8095_model.h5\n",
            "1717671 1963052\n",
            "Number of sequences: 122671\n",
            "(122671, 40, 48) (122671, 48)\n",
            "367/367 - 5s - loss: 1.5384 - val_loss: 1.8142 - 5s/epoch - 13ms/step\n",
            "/content/sw_1667446875_loss_1_8142_model.h5\n",
            "1963053 2208433\n",
            "Number of sequences: 122670\n",
            "(122670, 40, 48) (122670, 48)\n",
            "367/367 - 5s - loss: 1.5076 - val_loss: 1.8146 - 5s/epoch - 13ms/step\n",
            "/content/sw_1667446947_loss_1_8145_model.h5\n",
            "2208434 2453815\n",
            "Number of sequences: 122671\n",
            "(122671, 40, 48) (122671, 48)\n",
            "367/367 - 5s - loss: 1.4796 - val_loss: 1.8461 - 5s/epoch - 13ms/step\n",
            "/content/sw_1667447020_loss_1_8461_model.h5\n",
            "2453816 2699197\n",
            "Number of sequences: 122671\n",
            "(122671, 40, 48) (122671, 48)\n",
            "367/367 - 5s - loss: 1.4548 - val_loss: 1.8344 - 5s/epoch - 15ms/step\n",
            "/content/sw_1667447091_loss_1_8343_model.h5\n",
            "2699198 2944578\n",
            "Number of sequences: 122670\n",
            "(122670, 40, 48) (122670, 48)\n",
            "367/367 - 5s - loss: 1.4360 - val_loss: 1.8543 - 5s/epoch - 13ms/step\n",
            "/content/sw_1667447164_loss_1_8542_model.h5\n",
            "2944579 3189960\n",
            "Number of sequences: 122671\n",
            "(122671, 40, 48) (122671, 48)\n",
            "367/367 - 5s - loss: 1.4201 - val_loss: 1.8699 - 5s/epoch - 13ms/step\n",
            "/content/sw_1667447235_loss_1_8698_model.h5\n",
            "3189961 3435341\n",
            "Number of sequences: 122670\n",
            "(122670, 40, 48) (122670, 48)\n",
            "367/367 - 7s - loss: 1.4040 - val_loss: 1.8921 - 7s/epoch - 18ms/step\n",
            "/content/sw_1667447307_loss_1_8920_model.h5\n",
            "3435342 3680723\n",
            "Number of sequences: 122671\n",
            "(122671, 40, 48) (122671, 48)\n",
            "367/367 - 5s - loss: 1.3946 - val_loss: 1.8987 - 5s/epoch - 14ms/step\n",
            "/content/sw_1667447380_loss_1_8987_model.h5\n",
            "3680724 3926105\n",
            "Number of sequences: 122671\n",
            "(122671, 40, 48) (122671, 48)\n",
            "367/367 - 5s - loss: 1.3814 - val_loss: 1.8985 - 5s/epoch - 13ms/step\n",
            "/content/sw_1667447449_loss_1_8985_model.h5\n",
            "3926106 4171486\n",
            "Number of sequences: 122670\n",
            "(122670, 40, 48) (122670, 48)\n",
            "367/367 - 6s - loss: 1.3700 - val_loss: 1.9225 - 6s/epoch - 16ms/step\n",
            "/content/sw_1667447522_loss_1_9225_model.h5\n",
            "4171487 4416868\n",
            "Number of sequences: 122671\n",
            "(122671, 40, 48) (122671, 48)\n",
            "367/367 - 5s - loss: 1.3590 - val_loss: 1.9171 - 5s/epoch - 13ms/step\n",
            "/content/sw_1667447596_loss_1_9170_model.h5\n",
            "4416869 4662250\n",
            "Number of sequences: 122671\n",
            "(122671, 40, 48) (122671, 48)\n",
            "367/367 - 5s - loss: 1.3527 - val_loss: 1.9273 - 5s/epoch - 13ms/step\n",
            "/content/sw_1667447668_loss_1_9273_model.h5\n",
            "4662251 4907631\n",
            "Number of sequences: 122670\n",
            "(122670, 40, 48) (122670, 48)\n",
            "367/367 - 5s - loss: 1.3448 - val_loss: 1.9363 - 5s/epoch - 14ms/step\n",
            "/content/sw_1667447741_loss_1_9363_model.h5\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Kwere"
      ],
      "metadata": {
        "id": "BakMv4-xCFnl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# KWERE\n",
        "\n",
        "k_p = Preprocessing()\n",
        "k_p.read_file(path_kw)\n",
        "k_p.clean_data()\n",
        "k_p.set_vocab_encoder_decoder()\n",
        "\n",
        "print('Data ingested')\n",
        "\n",
        "kw_X, kw_y = k_p.set_X_y(sample_size = (0,600000))\n",
        "\n",
        "print('X, y set')\n",
        "\n",
        "input_dim = len(k_p.vocab)\n",
        "inputlen = emb_dim = 40\n",
        "\n",
        "kw_X_train, kw_X_test, kw_y_train, kw_y_test = train_test_split(kw_X, kw_y, test_size=0.2, random_state = 42)\n",
        "k_m = Model(kw_X_train, kw_X_test, kw_y_train, kw_y_test, input_dim, emb_dim, inputlen)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "En7p3cH5kjj-",
        "outputId": "29b09e5b-1af7-4f09-852e-0e180009ebd0"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Data ingested\n",
            "Number of sequences: 301696\n",
            "(301696, 40, 31) (301696, 31)\n",
            "X, y set\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "k_m.build_model_variant2()\n",
        "k_m.model_fit(batch_size = 512, epochs = 15, name=\"kw\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_HRrPqL_cXIO",
        "outputId": "ebcbeb09-d6a8-4e28-fe15-4e0e7ca46a37"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " lstm (LSTM)                 (None, 128)               81920     \n",
            "                                                                 \n",
            " dense (Dense)               (None, 31)                3999      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 85,919\n",
            "Trainable params: 85,919\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/15\n",
            "401/401 - 11s - loss: 1.7854 - val_loss: 1.4575 - 11s/epoch - 29ms/step\n",
            "Epoch 2/15\n",
            "401/401 - 3s - loss: 1.3314 - val_loss: 1.2814 - 3s/epoch - 8ms/step\n",
            "Epoch 3/15\n",
            "401/401 - 3s - loss: 1.2069 - val_loss: 1.2229 - 3s/epoch - 8ms/step\n",
            "Epoch 4/15\n",
            "401/401 - 3s - loss: 1.1437 - val_loss: 1.1833 - 3s/epoch - 8ms/step\n",
            "Epoch 5/15\n",
            "401/401 - 3s - loss: 1.1027 - val_loss: 1.1627 - 3s/epoch - 8ms/step\n",
            "Epoch 6/15\n",
            "401/401 - 3s - loss: 1.0713 - val_loss: 1.1546 - 3s/epoch - 9ms/step\n",
            "Epoch 7/15\n",
            "401/401 - 4s - loss: 1.0486 - val_loss: 1.1538 - 4s/epoch - 9ms/step\n",
            "Epoch 8/15\n",
            "401/401 - 3s - loss: 1.0298 - val_loss: 1.1524 - 3s/epoch - 9ms/step\n",
            "Epoch 9/15\n",
            "401/401 - 3s - loss: 1.0420 - val_loss: 1.1526 - 3s/epoch - 9ms/step\n",
            "Epoch 10/15\n",
            "401/401 - 3s - loss: 1.0100 - val_loss: 1.1519 - 3s/epoch - 8ms/step\n",
            "Epoch 11/15\n",
            "401/401 - 3s - loss: 0.9983 - val_loss: 1.1605 - 3s/epoch - 9ms/step\n",
            "Epoch 12/15\n",
            "401/401 - 4s - loss: 0.9868 - val_loss: 1.1512 - 4s/epoch - 9ms/step\n",
            "Epoch 13/15\n",
            "401/401 - 3s - loss: 0.9779 - val_loss: 1.1573 - 3s/epoch - 9ms/step\n",
            "Epoch 14/15\n",
            "401/401 - 3s - loss: 0.9708 - val_loss: 1.1464 - 3s/epoch - 8ms/step\n",
            "Epoch 15/15\n",
            "401/401 - 3s - loss: 0.9617 - val_loss: 1.1495 - 3s/epoch - 8ms/step\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "k_m.model_predict()\n",
        "k_m.model_evaluate()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Sc5j_PSiCeUH",
        "outputId": "914c8e00-9274-48c3-b16d-6a25f5519c6c"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1886/1886 [==============================] - 6s 3ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.6457739476300961"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Cross-Val Loss Evaluation"
      ],
      "metadata": {
        "id": "DEKLqJ_SrM0i"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from math import log2\n",
        "\n",
        "def evaluate_one(lang, model, vocab, encoder):\n",
        "\n",
        "  '''\n",
        "  calculate cross-entropy loss\n",
        "  iteratively read the test file's characters, calculate loss of each\n",
        "  and subtract from total loss\n",
        "  '''\n",
        "\n",
        "  testfile = open(lang+'-test.txt', 'r')\n",
        "  \n",
        "  max_history = 30 #max sentence length\n",
        "  history = []\n",
        "  loss_anything_goes = 0\n",
        "  count = 0\n",
        "\n",
        "  while True:\n",
        "    c = testfile.read(1)\n",
        "\n",
        "    if c=='\\n':\n",
        "      continue\n",
        "\n",
        "    if not c:\n",
        "      break\n",
        "\n",
        "    count += 1\n",
        "\n",
        "    loss_anything_goes -= log2(predict_next_proba(c, model, vocab, encoder, history))\n",
        "\n",
        "    if len(history) == max_history:\n",
        "      history.pop(0)\n",
        "\n",
        "    history.append(c)\n",
        "\n",
        "  return [loss_anything_goes/count]\n",
        "\n",
        "\n",
        "def predict_next_proba(c, model, vocab, encoder, history):\n",
        "  '''\n",
        "  returns the probability of the expected character\n",
        "  '''\n",
        "\n",
        "  inputlen = 40\n",
        "  x_test = np.zeros((1, inputlen, len(vocab)),dtype=float)\n",
        "\n",
        "  # create a matrix of current sequence of characters,\n",
        "  for ind,val in enumerate(history[:-1]):   \n",
        "      x_test[0, ind, encoder[val]] = 1\n",
        "\n",
        "  # pass our x into the model and return a prediction matrix\n",
        "  y_pred = model.predict(x_test, verbose=0)[0]\n",
        "\n",
        "  # return the computed probability of our character\n",
        "  proba = y_pred[encoder[c]]\n",
        "\n",
        "  return proba\n"
      ],
      "metadata": {
        "id": "NiNdNecvKep7"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "swahili_loss = evaluate_one(\"sw\", s_m.model,s_p.vocab, s_p.encoder)"
      ],
      "metadata": {
        "id": "e3RhP_eZm5Op"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Swahili Cross Entropy Loss: \", swahili_loss)"
      ],
      "metadata": {
        "id": "9PKmItpqRigy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "kwere_loss = evaluate_one(\"cwe\", k_m.model,k_p.vocab, k_p.encoder)"
      ],
      "metadata": {
        "id": "uSYKGdb_mUFy"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Kwere Cross Entropy Loss: \", kwere_loss)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2Iqh9isxE3hX",
        "outputId": "aa789b28-b668-41d1-d5af-83caa617a5f1"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Kwere Cross Entropy Loss:  [4.701911114505756]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "EKWDofcMSAI9"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}