1. Language Models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "22669d1c",
   "metadata": {},
   "source": [
    "<div style=\"width: 100%; overflow: hidden;\">\n",
    "    <div style=\"width: 150px; float: left;\"> <img src=\"https://raw.githubusercontent.com/DataForScience/Networks/master/data/D4Sci_logo_ball.png\" alt=\"Data For Science, Inc\" align=\"left\" border=\"0\" width=150px> </div>\n",
    "    <div style=\"float: left; margin-left: 10px;\"> <h1>ChatGPT And Friends</h1>\n",
    "<h1>Language Models</h1>\n",
    "        <p>Bruno Gonçalves<br/>\n",
    "        <a href=\"http://www.data4sci.com/\">www.data4sci.com</a><br/>\n",
    "            @bgoncalves, @data4sci</p></div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d1fc4f66",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter, defaultdict\n",
    "import random\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import reuters\n",
    "from nltk import bigrams, trigrams\n",
    "\n",
    "import tqdm as tq\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "import watermark\n",
    "\n",
    "%load_ext watermark\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d792a75",
   "metadata": {},
   "source": [
    "We start by printing out the versions of the libraries we're using for future reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6055871b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python implementation: CPython\n",
      "Python version       : 3.11.7\n",
      "IPython version      : 8.12.3\n",
      "\n",
      "Compiler    : Clang 14.0.6 \n",
      "OS          : Darwin\n",
      "Release     : 23.6.0\n",
      "Machine     : arm64\n",
      "Processor   : arm\n",
      "CPU cores   : 16\n",
      "Architecture: 64bit\n",
      "\n",
      "Git hash: b44900a26f10de8fbaf559b307e69185828b77b4\n",
      "\n",
      "pandas    : 2.2.3\n",
      "json      : 2.0.9\n",
      "numpy     : 1.26.4\n",
      "tqdm      : 4.66.4\n",
      "matplotlib: 3.8.0\n",
      "nltk      : 3.8.1\n",
      "watermark : 2.4.3\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%watermark -n -v -m -g -iv"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9691540",
   "metadata": {},
   "source": [
    "Load default figure style"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ad3ac23b",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.style.use('d4sci.mplstyle')\n",
    "colors = plt.rcParams['axes.prop_cycle'].by_key()['color']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3fc24c3",
   "metadata": {},
   "source": [
    "# Segmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "59806e02",
   "metadata": {},
   "outputs": [],
   "source": [
    "Pword = pd.read_csv('data/count_1w.txt.gz', sep='\\t', header=None, index_col=0)\n",
    "norm = float(Pword.sum().iloc[0])\n",
    "Pword/=norm\n",
    "\n",
    "pword_dict = defaultdict(lambda: 1/norm)\n",
    "pword_dict.update(dict(Pword.reset_index().values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f426dd71",
   "metadata": {},
   "outputs": [],
   "source": [
    "def Pwords(words):\n",
    "    \"Probability of words, assuming each word is independent of others.\"\n",
    "    return np.prod([float(pword_dict[w]) for w in words])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "02af6e34",
   "metadata": {},
   "outputs": [],
   "source": [
    "def splits(text, verbose=False):\n",
    "    s = [(text[:i], text[i:]) \n",
    "            for i in range(1, len(text)+1)]\n",
    "    \n",
    "    if verbose:\n",
    "        print(s)\n",
    "    \n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3992fa7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def segment(text):\n",
    "    \"Return a list of words that is the most probable segmentation of text.\"\n",
    "    if not text: \n",
    "        return []\n",
    "    else:\n",
    "        candidates = ([first] + segment(rest) \n",
    "                      for (first, rest) in splits(text))\n",
    "        return max(candidates, key=Pwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c510f389",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['choose', 'spain']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "segment('choosespain')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b2067e7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.3415578949797063e-08"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Pwords(['choose', 'spain'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "85032de7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.9488948824561327e-10"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Pwords(['chooses', 'pain'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25ae0c91",
   "metadata": {},
   "source": [
    "# \"Small\" Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "aab26a4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = defaultdict(lambda: defaultdict(lambda: 0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "186540ac",
   "metadata": {},
   "source": [
    "We start by counting number of trigram co-occurrences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b5c07727",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(None, None, 'the'),\n",
       " (None, 'the', 'united'),\n",
       " ('the', 'united', 'states'),\n",
       " ('united', 'states', 'of'),\n",
       " ('states', 'of', 'america'),\n",
       " ('of', 'america', None),\n",
       " ('america', None, None)]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(trigrams([\"the\", \"united\", \"states\", \"of\", \"america\"], pad_right=True, pad_left=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20e88276",
   "metadata": {},
   "source": [
    "$$P\\left(w_3|w_1,w_2\\right)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a3fd0f46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e540ea89245849b2b35f2f2db682a243",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/54716 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for sentence in tqdm(reuters.sents(), total=54_716):\n",
    "    for w1, w2, w3 in trigrams(sentence, pad_right=True, pad_left=True):\n",
    "        context = (w1, w2)\n",
    "        model[context][w3] += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f7fb5c9",
   "metadata": {},
   "source": [
    "And normalizing the probabilities for each bigram. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "07e018fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "for context in model:\n",
    "    total_count = float(sum(model[context].values()))\n",
    "\n",
    "    for w3 in model[context]:\n",
    "        model[context][w3] /= total_count"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ee90e61",
   "metadata": {},
   "source": [
    "Our language model is just a weighted mapping between each bigram and the possible next words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1ea827ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(<function __main__.<lambda>.<locals>.<lambda>()>,\n",
       "            {'.': 0.10950080515297907,\n",
       "             'where': 0.001610305958132045,\n",
       "             'and': 0.11594202898550725,\n",
       "             'in': 0.01610305958132045,\n",
       "             'citizen': 0.001610305958132045,\n",
       "             'could': 0.008051529790660225,\n",
       "             'is': 0.03864734299516908,\n",
       "             'has': 0.04669887278582931,\n",
       "             'took': 0.00322061191626409,\n",
       "             ',': 0.15780998389694043,\n",
       "             'from': 0.00644122383252818,\n",
       "             'decided': 0.001610305958132045,\n",
       "             'agreed': 0.00644122383252818,\n",
       "             'should': 0.014492753623188406,\n",
       "             '.\"': 0.004830917874396135,\n",
       "             'had': 0.00966183574879227,\n",
       "             \"'\": 0.008051529790660225,\n",
       "             'it': 0.00322061191626409,\n",
       "             'imposes': 0.00322061191626409,\n",
       "             'despite': 0.001610305958132045,\n",
       "             ',\"': 0.014492753623188406,\n",
       "             'markets': 0.001610305958132045,\n",
       "             'imported': 0.001610305958132045,\n",
       "             'requires': 0.001610305958132045,\n",
       "             'did': 0.004830917874396135,\n",
       "             'open': 0.001610305958132045,\n",
       "             'would': 0.030595813204508857,\n",
       "             ')': 0.00322061191626409,\n",
       "             'to': 0.04830917874396135,\n",
       "             'will': 0.0322061191626409,\n",
       "             'of': 0.004830917874396135,\n",
       "             'included': 0.001610305958132045,\n",
       "             'House': 0.001610305958132045,\n",
       "             'made': 0.001610305958132045,\n",
       "             'successfully': 0.001610305958132045,\n",
       "             'itself': 0.001610305958132045,\n",
       "             'influence': 0.001610305958132045,\n",
       "             'into': 0.00322061191626409,\n",
       "             'charged': 0.001610305958132045,\n",
       "             'asked': 0.001610305958132045,\n",
       "             'said': 0.00966183574879227,\n",
       "             'expects': 0.001610305958132045,\n",
       "             'failed': 0.001610305958132045,\n",
       "             'have': 0.00644122383252818,\n",
       "             'outside': 0.001610305958132045,\n",
       "             'aluminum': 0.001610305958132045,\n",
       "             'responded': 0.001610305958132045,\n",
       "             'was': 0.011272141706924315,\n",
       "             'disclosed': 0.001610305958132045,\n",
       "             'government': 0.001610305958132045,\n",
       "             'if': 0.004830917874396135,\n",
       "             'for': 0.008051529790660225,\n",
       "             'opposes': 0.001610305958132045,\n",
       "             'wants': 0.00644122383252818,\n",
       "             'warned': 0.001610305958132045,\n",
       "             'appears': 0.00322061191626409,\n",
       "             'now': 0.001610305958132045,\n",
       "             ').\"': 0.001610305958132045,\n",
       "             'apparently': 0.001610305958132045,\n",
       "             'on': 0.008051529790660225,\n",
       "             'may': 0.00644122383252818,\n",
       "             'remains': 0.00322061191626409,\n",
       "             'with': 0.00322061191626409,\n",
       "             'invoked': 0.001610305958132045,\n",
       "             '-': 0.00644122383252818,\n",
       "             'forces': 0.001610305958132045,\n",
       "             'vessels': 0.001610305958132045,\n",
       "             'that': 0.00322061191626409,\n",
       "             'earlier': 0.001610305958132045,\n",
       "             'faces': 0.001610305958132045,\n",
       "             'attack': 0.001610305958132045,\n",
       "             'sold': 0.001610305958132045,\n",
       "             'which': 0.001610305958132045,\n",
       "             'Treasury': 0.001610305958132045,\n",
       "             'were': 0.00644122383252818,\n",
       "             'Lines': 0.008051529790660225,\n",
       "             'fell': 0.004830917874396135,\n",
       "             'against': 0.001610305958132045,\n",
       "             'but': 0.00644122383252818,\n",
       "             'are': 0.00322061191626409,\n",
       "             'ports': 0.001610305958132045,\n",
       "             'plans': 0.001610305958132045,\n",
       "             'over': 0.00644122383252818,\n",
       "             'cutting': 0.001610305958132045,\n",
       "             'does': 0.00322061191626409,\n",
       "             'last': 0.00644122383252818,\n",
       "             'when': 0.001610305958132045,\n",
       "             'must': 0.011272141706924315,\n",
       "             'might': 0.001610305958132045,\n",
       "             'during': 0.001610305958132045,\n",
       "             'lets': 0.001610305958132045,\n",
       "             'went': 0.001610305958132045,\n",
       "             '80': 0.00322061191626409,\n",
       "             'department': 0.001610305958132045,\n",
       "             'sees': 0.001610305958132045,\n",
       "             'not': 0.001610305958132045,\n",
       "             'defended': 0.001610305958132045,\n",
       "             'as': 0.001610305958132045,\n",
       "             'ready': 0.001610305958132045,\n",
       "             'retaliate': 0.001610305958132045,\n",
       "             'while': 0.00322061191626409,\n",
       "             'can': 0.001610305958132045,\n",
       "             'totaled': 0.001610305958132045,\n",
       "             'section': 0.001610305958132045,\n",
       "             '\"': 0.00322061191626409,\n",
       "             'comes': 0.001610305958132045,\n",
       "             'conducts': 0.001610305958132045,\n",
       "             'dropped': 0.00322061191626409,\n",
       "             'makes': 0.001610305958132045,\n",
       "             'prohibiting': 0.001610305958132045,\n",
       "             'hit': 0.001610305958132045,\n",
       "             '88': 0.001610305958132045,\n",
       "             'include': 0.00322061191626409,\n",
       "             '11': 0.001610305958132045,\n",
       "             'by': 0.00322061191626409,\n",
       "             'overly': 0.001610305958132045,\n",
       "             'believes': 0.001610305958132045,\n",
       "             'at': 0.00322061191626409,\n",
       "             'increased': 0.00322061191626409,\n",
       "             'lost': 0.001610305958132045,\n",
       "             'replaced': 0.001610305958132045,\n",
       "             'alone': 0.001610305958132045,\n",
       "             'than': 0.001610305958132045,\n",
       "             'compared': 0.001610305958132045,\n",
       "             'released': 0.001610305958132045,\n",
       "             'face': 0.001610305958132045,\n",
       "             '48': 0.001610305958132045,\n",
       "             'subsidies': 0.001610305958132045,\n",
       "             'subsidizes': 0.001610305958132045,\n",
       "             'wanted': 0.00322061191626409,\n",
       "             'Bankruptcy': 0.001610305958132045,\n",
       "             'also': 0.001610305958132045,\n",
       "             'or': 0.001610305958132045,\n",
       "             'waived': 0.001610305958132045,\n",
       "             'uses': 0.00322061191626409,\n",
       "             '...': 0.00322061191626409,\n",
       "             'whose': 0.001610305958132045,\n",
       "             'cut': 0.001610305958132045,\n",
       "             'closed': 0.001610305958132045,\n",
       "             '--': 0.001610305958132045})"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model[(\"United\", \"States\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6ec2a2f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(<function __main__.<lambda>.<locals>.<lambda>()>,\n",
       "            {',': 0.21428571428571427,\n",
       "             'and': 0.21428571428571427,\n",
       "             'blender': 0.07142857142857142,\n",
       "             ')': 0.14285714285714285,\n",
       "             'company': 0.07142857142857142,\n",
       "             'operations': 0.07142857142857142,\n",
       "             'assets': 0.07142857142857142,\n",
       "             'Ltd': 0.07142857142857142,\n",
       "             '.': 0.07142857142857142})"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model[(\"United\", \"Kingdom\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2defd4b",
   "metadata": {},
   "source": [
    "This is all we need to generate new text staring from a bigram prompt. We must simply perform a random walk on this weighted graph starting from an initial prompt:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c9965dd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_sentence_from_prompt(prompt, zero_temperature=False):\n",
    "    text = [*prompt]\n",
    "\n",
    "    # Dont impose any fixed sentence length\n",
    "    while True:\n",
    "        # the current not we're in is just the one that accounts\n",
    "        # for the last two words in the text\n",
    "        context = tuple(text[-2:])\n",
    "\n",
    "        # We extract the list of possible next words and their probabilities\n",
    "        words = []\n",
    "        probs = []\n",
    "\n",
    "        for word, prob in model[context].items():\n",
    "            words.append(word)\n",
    "            probs.append(prob)\n",
    "\n",
    "        # Choose one word proportionally to each probability\n",
    "        selection = np.random.multinomial(1, probs)\n",
    "        \n",
    "        # Check which one was chosen\n",
    "        if zero_temperature:\n",
    "            pos = np.argmax(probs) # Temperature = 0\n",
    "        else:\n",
    "            pos = np.argmax(selection) # Temperature = 1\n",
    "            \n",
    "        # Check which one was chosen\n",
    "        pos = np.argmax(selection)\n",
    "\n",
    "        word = words[pos]\n",
    "\n",
    "        # Append the new word to our runnning text\n",
    "        text.append(word)\n",
    "\n",
    "        # Stop when we hit two None tokens in a row, that represnet the end of a sentence\n",
    "        if text[-2:] == [None, None]:\n",
    "            break\n",
    "        \n",
    "        # Make sure we don't run forever\n",
    "        if len(text) > 100:\n",
    "            break\n",
    "                \n",
    "    return \" \".join([t for t in text if t])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6304c653",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'United States may lift its sanctions on some Japanese electronic goods from abroad , when it was lower .'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_sentence_from_prompt(('United', 'States'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21ee83f7",
   "metadata": {},
   "source": [
    "United States => as\n",
    "\n",
    "United States as -> States as => well\n",
    "\n",
    "United States as well -> as well => as\n",
    "\n",
    "United States as well as -> well as => Amsterdam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "628014e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Today the government expects the momentum will be marketed independently under their present names and management .'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_sentence_from_prompt(('Today', 'the'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "83d72976",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'financial markets and to branches of Citizens Heritage Bank of Japan said in separate statements the new common shares .'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_sentence_from_prompt(('financial', 'markets'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bad7542",
   "metadata": {},
   "source": [
    "<center>\n",
    "     <img src=\"https://raw.githubusercontent.com/DataForScience/Networks/master/data/D4Sci_logo_full.png\" alt=\"Data For Science, Inc\" align=\"center\" border=\"0\" width=300px> \n",
    "</center>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}