1. Generative AI .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8c5b8f35",
   "metadata": {},
   "source": [
    "<div style=\"width: 100%; overflow: hidden;\">\n",
    "    <div style=\"width: 150px; float: left;\"> <img src=\"https://raw.githubusercontent.com/DataForScience/Networks/master/data/D4Sci_logo_ball.png\" alt=\"Data For Science, Inc\" align=\"left\" border=\"0\" width=150px> </div>\n",
    "    <div style=\"float: left; margin-left: 10px;\"> <h1>Generative AI with OpenAI API</h1>\n",
    "<h1>Basic Concepts</h1>\n",
    "        <p>Bruno Gonçalves<br/>\n",
    "        <a href=\"http://www.data4sci.com/\">www.data4sci.com</a><br/>\n",
    "            @bgoncalves, @data4sci</p></div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9c7f668d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter, defaultdict\n",
    "from pprint import pprint\n",
    "import random\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "import openai\n",
    "from openai import OpenAI\n",
    "\n",
    "import tiktoken\n",
    "import nltk\n",
    "from nltk.corpus import reuters\n",
    "from nltk import bigrams, trigrams\n",
    "\n",
    "from ipywidgets import interact\n",
    "\n",
    "import os\n",
    "import gzip\n",
    "\n",
    "import tqdm as tq\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "import watermark\n",
    "\n",
    "%load_ext watermark\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27564a0f",
   "metadata": {},
   "source": [
    "We start by printing out the versions of the libraries we're using for future reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f6792721",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python implementation: CPython\n",
      "Python version       : 3.11.7\n",
      "IPython version      : 8.12.3\n",
      "\n",
      "Compiler    : Clang 14.0.6 \n",
      "OS          : Darwin\n",
      "Release     : 23.5.0\n",
      "Machine     : arm64\n",
      "Processor   : arm\n",
      "CPU cores   : 16\n",
      "Architecture: 64bit\n",
      "\n",
      "Git hash: 2f98fe4b5aabf6921b7e3b8fba71f74cb0ebead8\n",
      "\n",
      "matplotlib: 3.8.0\n",
      "tiktoken  : 0.7.0\n",
      "watermark : 2.4.3\n",
      "numpy     : 1.26.4\n",
      "openai    : 1.30.5\n",
      "json      : 2.0.9\n",
      "nltk      : 3.8.1\n",
      "pandas    : 2.1.4\n",
      "tqdm      : 4.66.4\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%watermark -n -v -m -g -iv"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa6f7ae7",
   "metadata": {},
   "source": [
    "Load default figure style"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cb0a9744",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.style.use('d4sci.mplstyle')\n",
    "colors = plt.rcParams['axes.prop_cycle'].by_key()['color']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ff7fa7f",
   "metadata": {},
   "source": [
    "# Encodings"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "579769db",
   "metadata": {},
   "source": [
    "tiktoken supports several types of encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "87e0c41d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gpt2', 'r50k_base', 'p50k_base', 'p50k_edit', 'cl100k_base', 'o200k_base']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tiktoken.list_encoding_names()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41a7a408",
   "metadata": {},
   "source": [
    "Encodings can by loaded by name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "94e90252",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Encoding 'cl100k_base'>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoding = tiktoken.get_encoding(\"cl100k_base\")\n",
    "encoding"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33b0e313",
   "metadata": {},
   "source": [
    "or by specifying the name of the model we are using"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8893a2e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Encoding 'o200k_base'>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoding = tiktoken.encoding_for_model(\"gpt-4o\")\n",
    "encoding"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2056806",
   "metadata": {},
   "source": [
    "After loading, we can use the encoding object to tokenize text by using the __encode()__ method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "67ad8761",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[83, 8251, 2488, 382, 2212, 0]\n"
     ]
    }
   ],
   "source": [
    "encoded_text = encoding.encode(\"tiktoken is great!\")\n",
    "print(encoded_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aac5eccd",
   "metadata": {},
   "source": [
    "which returns numerical IDs for each of the tokens. Numerical IDs can be converted back to the original text using __decode()__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fed60838",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tiktoken is great!'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoding.decode(encoded_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b13b7e9",
   "metadata": {},
   "source": [
    "Tokens can be individual letters, characters, or even full words. To convert indivudal numerical IDs to tokens, we should use __decode_single_token_bytes()__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "90a56583",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "83\t->\tb't'\n",
      "8251\t->\tb'ikt'\n",
      "2488\t->\tb'oken'\n",
      "382\t->\tb' is'\n",
      "2212\t->\tb' great'\n",
      "0\t->\tb'!'\n"
     ]
    }
   ],
   "source": [
    "for token in encoded_text:\n",
    "    print('%s\\t->\\t%s' % (token, encoding.decode_single_token_bytes(token)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81c22b2e",
   "metadata": {},
   "source": [
    "The number of tokens generated depend on the specific encoding used. This is particularly noticeable in long words, so let's take the longest english word as an example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cc730637",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "gpt2: 15 tokens\n",
      "Token IDs: [79, 25668, 261, 25955, 859, 2500, 1416, 404, 873, 41896, 709, 349, 5171, 36221, 42960]\n",
      "Token Text: [b'p', b'neum', b'on', b'oult', b'ram', b'icro', b'sc', b'op', b'ics', b'ilic', b'ov', b'ol', b'can', b'ocon', b'iosis']\n",
      "\n",
      "r50k_base: 15 tokens\n",
      "Token IDs: [79, 25668, 261, 25955, 859, 2500, 1416, 404, 873, 41896, 709, 349, 5171, 36221, 42960]\n",
      "Token Text: [b'p', b'neum', b'on', b'oult', b'ram', b'icro', b'sc', b'op', b'ics', b'ilic', b'ov', b'ol', b'can', b'ocon', b'iosis']\n",
      "\n",
      "p50k_base: 15 tokens\n",
      "Token IDs: [79, 25668, 261, 25955, 859, 2500, 1416, 404, 873, 41896, 709, 349, 5171, 36221, 42960]\n",
      "Token Text: [b'p', b'neum', b'on', b'oult', b'ram', b'icro', b'sc', b'op', b'ics', b'ilic', b'ov', b'ol', b'can', b'ocon', b'iosis']\n",
      "\n",
      "p50k_edit: 15 tokens\n",
      "Token IDs: [79, 25668, 261, 25955, 859, 2500, 1416, 404, 873, 41896, 709, 349, 5171, 36221, 42960]\n",
      "Token Text: [b'p', b'neum', b'on', b'oult', b'ram', b'icro', b'sc', b'op', b'ics', b'ilic', b'ov', b'ol', b'can', b'ocon', b'iosis']\n",
      "\n",
      "cl100k_base: 17 tokens\n",
      "Token IDs: [79, 818, 372, 263, 11206, 99040, 2823, 2445, 454, 1233, 321, 292, 869, 337, 69377, 444, 91260]\n",
      "Token Text: [b'p', b'ne', b'um', b'on', b'oul', b'tram', b'icro', b'sc', b'op', b'ics', b'il', b'ic', b'ov', b'ol', b'cano', b'con', b'iosis']\n",
      "\n",
      "o200k_base: 15 tokens\n",
      "Token IDs: [79, 611, 394, 263, 9826, 371, 26169, 2199, 47750, 1541, 112176, 47186, 6929, 29452, 156038]\n",
      "Token Text: [b'p', b'ne', b'um', b'on', b'oul', b'tr', b'amic', b'ros', b'cop', b'ics', b'ilic', b'ovol', b'can', b'ocon', b'iosis']\n"
     ]
    }
   ],
   "source": [
    "example_string = \"pneumonoultramicroscopicsilicovolcanoconiosis\"\n",
    "\n",
    "for encoding_name in tiktoken.list_encoding_names():\n",
    "    encoding = tiktoken.get_encoding(encoding_name)\n",
    "    encoded_text = encoding.encode(example_string)\n",
    "    num_tokens = len(encoded_text)\n",
    "    token_text = [encoding.decode_single_token_bytes(token) for token in encoded_text]\n",
    "    print()\n",
    "    print(f\"{encoding_name}: {num_tokens} tokens\")\n",
    "    print(f\"Token IDs: {encoded_text}\")\n",
    "    print(f\"Token Text: {token_text}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4208fc0",
   "metadata": {},
   "source": [
    "Encodings are capable of handling a large number of languages and character sets. Let's take Japanese for example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3c43edb0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "gpt2: 14 tokens\n",
      "Token IDs: [2515, 232, 45739, 243, 37955, 33768, 98, 2515, 232, 1792, 223, 30640, 30201, 29557]\n",
      "Token Text: [b'\\xe3\\x81', b'\\x8a', b'\\xe8\\xaa', b'\\x95', b'\\xe7\\x94\\x9f', b'\\xe6\\x97', b'\\xa5', b'\\xe3\\x81', b'\\x8a', b'\\xe3\\x82', b'\\x81', b'\\xe3\\x81\\xa7', b'\\xe3\\x81\\xa8', b'\\xe3\\x81\\x86']\n",
      "\n",
      "r50k_base: 14 tokens\n",
      "Token IDs: [2515, 232, 45739, 243, 37955, 33768, 98, 2515, 232, 1792, 223, 30640, 30201, 29557]\n",
      "Token Text: [b'\\xe3\\x81', b'\\x8a', b'\\xe8\\xaa', b'\\x95', b'\\xe7\\x94\\x9f', b'\\xe6\\x97', b'\\xa5', b'\\xe3\\x81', b'\\x8a', b'\\xe3\\x82', b'\\x81', b'\\xe3\\x81\\xa7', b'\\xe3\\x81\\xa8', b'\\xe3\\x81\\x86']\n",
      "\n",
      "p50k_base: 14 tokens\n",
      "Token IDs: [2515, 232, 45739, 243, 37955, 33768, 98, 2515, 232, 1792, 223, 30640, 30201, 29557]\n",
      "Token Text: [b'\\xe3\\x81', b'\\x8a', b'\\xe8\\xaa', b'\\x95', b'\\xe7\\x94\\x9f', b'\\xe6\\x97', b'\\xa5', b'\\xe3\\x81', b'\\x8a', b'\\xe3\\x82', b'\\x81', b'\\xe3\\x81\\xa7', b'\\xe3\\x81\\xa8', b'\\xe3\\x81\\x86']\n",
      "\n",
      "p50k_edit: 14 tokens\n",
      "Token IDs: [2515, 232, 45739, 243, 37955, 33768, 98, 2515, 232, 1792, 223, 30640, 30201, 29557]\n",
      "Token Text: [b'\\xe3\\x81', b'\\x8a', b'\\xe8\\xaa', b'\\x95', b'\\xe7\\x94\\x9f', b'\\xe6\\x97', b'\\xa5', b'\\xe3\\x81', b'\\x8a', b'\\xe3\\x82', b'\\x81', b'\\xe3\\x81\\xa7', b'\\xe3\\x81\\xa8', b'\\xe3\\x81\\x86']\n",
      "\n",
      "cl100k_base: 9 tokens\n",
      "Token IDs: [33334, 45918, 243, 21990, 9080, 33334, 62004, 16556, 78699]\n",
      "Token Text: [b'\\xe3\\x81\\x8a', b'\\xe8\\xaa', b'\\x95', b'\\xe7\\x94\\x9f', b'\\xe6\\x97\\xa5', b'\\xe3\\x81\\x8a', b'\\xe3\\x82\\x81', b'\\xe3\\x81\\xa7', b'\\xe3\\x81\\xa8\\xe3\\x81\\x86']\n",
      "\n",
      "o200k_base: 8 tokens\n",
      "Token IDs: [8930, 9697, 243, 128225, 8930, 17693, 4344, 48669]\n",
      "Token Text: [b'\\xe3\\x81\\x8a', b'\\xe8\\xaa', b'\\x95', b'\\xe7\\x94\\x9f\\xe6\\x97\\xa5', b'\\xe3\\x81\\x8a', b'\\xe3\\x82\\x81', b'\\xe3\\x81\\xa7', b'\\xe3\\x81\\xa8\\xe3\\x81\\x86']\n"
     ]
    }
   ],
   "source": [
    "example_string = \"お誕生日おめでとう\"\n",
    "\n",
    "for encoding_name in tiktoken.list_encoding_names():\n",
    "    encoding = tiktoken.get_encoding(encoding_name)\n",
    "    encoded_text = encoding.encode(example_string)\n",
    "    num_tokens = len(encoded_text)\n",
    "    token_text = [encoding.decode_single_token_bytes(token) for token in encoded_text]\n",
    "    print()\n",
    "    print(f\"{encoding_name}: {num_tokens} tokens\")\n",
    "    print(f\"Token IDs: {encoded_text}\")\n",
    "    print(f\"Token Text: {token_text}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b72e72be",
   "metadata": {},
   "source": [
    "Here we are seeing the unicode representation of the text, but we can easily recover the original Kanji and Hiragana text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b952a41f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "お誕生日おめでとう\n"
     ]
    }
   ],
   "source": [
    "print(b\"\".join(token_text).decode())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10f5e6ec",
   "metadata": {},
   "source": [
    "# Counting tokens for API calls\n",
    "\n",
    "We can use tiktoken to count how many tokens our API calls are going to consume. Naturally, this depends on the language model used. Based on https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d4a49a44",
   "metadata": {},
   "outputs": [],
   "source": [
    "def num_tokens_from_messages(messages, model=\"gpt-3.5-turbo-0613\"):\n",
    "    \"\"\"Return the number of tokens used by a list of messages.\"\"\"\n",
    "    try:\n",
    "        encoding = tiktoken.encoding_for_model(model)\n",
    "    except KeyError:\n",
    "        print(\"Warning: model not found. Using cl100k_base encoding.\")\n",
    "        encoding = tiktoken.get_encoding(\"cl100k_base\")\n",
    "    if model in {\n",
    "        \"gpt-3.5-turbo-0613\",\n",
    "        \"gpt-3.5-turbo-16k-0613\",\n",
    "        \"gpt-4-0314\",\n",
    "        \"gpt-4-32k-0314\",\n",
    "        \"gpt-4-0613\",\n",
    "        \"gpt-4-32k-0613\",\n",
    "        }:\n",
    "        tokens_per_message = 3\n",
    "        tokens_per_name = 1\n",
    "    elif model == \"gpt-3.5-turbo-0301\":\n",
    "        tokens_per_message = 4  # every message follows <|start|>{role/name}\\n{content}<|end|>\\n\n",
    "        tokens_per_name = -1  # if there's a name, the role is omitted\n",
    "    elif \"gpt-3.5-turbo\" in model:\n",
    "        print(\"Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.\")\n",
    "        return num_tokens_from_messages(messages, model=\"gpt-3.5-turbo-0613\")\n",
    "    elif \"gpt-4\" in model:\n",
    "        print(\"Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.\")\n",
    "        return num_tokens_from_messages(messages, model=\"gpt-4-0613\")\n",
    "    else:\n",
    "        raise NotImplementedError(\n",
    "            f\"\"\"num_tokens_from_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.\"\"\"\n",
    "        )\n",
    "    num_tokens = 0\n",
    "    for message in messages:\n",
    "        num_tokens += tokens_per_message\n",
    "        for key, value in message.items():\n",
    "            n_tokens = len(encoding.encode(value))\n",
    "            num_tokens += n_tokens\n",
    "            if key == \"name\":\n",
    "                num_tokens += tokens_per_name\n",
    "    num_tokens += 3  # every reply is primed with <|start|>assistant<|message|>\n",
    "    return num_tokens\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ef66026",
   "metadata": {},
   "source": [
    "Instatiate the client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3b987819",
   "metadata": {},
   "outputs": [],
   "source": [
    "client = OpenAI()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6115921d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpt-3.5-turbo-0301\n",
      "127 prompt tokens counted by num_tokens_from_messages().\n",
      "127 prompt tokens counted by the OpenAI API.\n",
      "\n",
      "gpt-3.5-turbo-0613\n",
      "129 prompt tokens counted by num_tokens_from_messages().\n",
      "129 prompt tokens counted by the OpenAI API.\n",
      "\n",
      "gpt-3.5-turbo\n",
      "Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.\n",
      "129 prompt tokens counted by num_tokens_from_messages().\n",
      "129 prompt tokens counted by the OpenAI API.\n",
      "\n",
      "gpt-4-0613\n",
      "129 prompt tokens counted by num_tokens_from_messages().\n",
      "129 prompt tokens counted by the OpenAI API.\n",
      "\n",
      "gpt-4\n",
      "Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.\n",
      "129 prompt tokens counted by num_tokens_from_messages().\n",
      "129 prompt tokens counted by the OpenAI API.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "example_messages = [\n",
    "    {\n",
    "        \"role\": \"system\",\n",
    "        \"content\": \"You are a helpful, pattern-following assistant that translates corporate jargon into plain English.\",\n",
    "    },\n",
    "    {\n",
    "        \"role\": \"system\",\n",
    "        \"name\": \"example_user\",\n",
    "        \"content\": \"New synergies will help drive top-line growth.\",\n",
    "    },\n",
    "    {\n",
    "        \"role\": \"system\",\n",
    "        \"name\": \"example_assistant\",\n",
    "        \"content\": \"Things working well together will increase revenue.\",\n",
    "    },\n",
    "    {\n",
    "        \"role\": \"system\",\n",
    "        \"name\": \"example_user\",\n",
    "        \"content\": \"Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.\",\n",
    "    },\n",
    "    {\n",
    "        \"role\": \"system\",\n",
    "        \"name\": \"example_assistant\",\n",
    "        \"content\": \"Let's talk later when we're less busy about how to do better.\",\n",
    "    },\n",
    "    {\n",
    "        \"role\": \"user\",\n",
    "        \"content\": \"This late pivot means we don't have time to boil the ocean for the client deliverable.\",\n",
    "    },\n",
    "]\n",
    "\n",
    "for model in [\n",
    "    \"gpt-3.5-turbo-0301\",\n",
    "    \"gpt-3.5-turbo-0613\",\n",
    "    \"gpt-3.5-turbo\",\n",
    "    \"gpt-4-0613\",\n",
    "    \"gpt-4\",\n",
    "    ]:\n",
    "    print(model)\n",
    "    # example token count from the function defined above\n",
    "    print(f\"{num_tokens_from_messages(example_messages, model)} prompt tokens counted by num_tokens_from_messages().\")\n",
    "    # example token count from the OpenAI API\n",
    "    response = client.chat.completions.create(\n",
    "        model=model,\n",
    "        messages=example_messages,\n",
    "        temperature=0,\n",
    "        max_tokens=1,  # we're only counting input tokens here, so let's not waste tokens on the output\n",
    "    )\n",
    "    print(f'{response.usage.prompt_tokens} prompt tokens counted by the OpenAI API.')\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd931d1e",
   "metadata": {},
   "source": [
    "# \"Small\" Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0c14a9de",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = defaultdict(lambda: defaultdict(lambda: 0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48da6c45",
   "metadata": {},
   "source": [
    "We start by counting number of trigram co-occurrences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "23effa64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23b3f2a24b8d400e8a1a3b1acefcd827",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/54711 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for sentence in tqdm(reuters.sents(), total=54_716):\n",
    "    for w1, w2, w3 in trigrams(sentence, pad_right=True, pad_left=True):\n",
    "        bigram = (w1, w2)\n",
    "        model[bigram][w3] += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19e98c99",
   "metadata": {},
   "source": [
    "And normalizing the probabilities for each bigram. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4e7773ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "for bigram in model:\n",
    "    total_count = float(sum(model[bigram].values()))\n",
    "\n",
    "    for w3 in model[bigram]:\n",
    "        model[bigram][w3] /= total_count"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "671668b3",
   "metadata": {},
   "source": [
    "Our language model is just a weighted mapping between each bigram and the possible next words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "51ec728c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(<function __main__.<lambda>.<locals>.<lambda>()>,\n",
       "            {'States': 0.880672268907563,\n",
       "             'Kingdom': 0.011764705882352941,\n",
       "             'Arab': 0.052100840336134456,\n",
       "             'Permanent': 0.0016806722689075631,\n",
       "             'Steelworkers': 0.0033613445378151263,\n",
       "             'Nations': 0.025210084033613446,\n",
       "             'Coconut': 0.0067226890756302525,\n",
       "             'State': 0.0033613445378151263,\n",
       "             'Democratic': 0.0016806722689075631,\n",
       "             'Food': 0.008403361344537815,\n",
       "             'Automobile': 0.0016806722689075631,\n",
       "             'acquisition': 0.0016806722689075631,\n",
       "             'Rubber': 0.0016806722689075631})"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model[(\"the\", \"United\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0d02460a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(<function __main__.<lambda>.<locals>.<lambda>()>,\n",
       "            {',': 0.21428571428571427,\n",
       "             'and': 0.21428571428571427,\n",
       "             'blender': 0.07142857142857142,\n",
       "             ')': 0.14285714285714285,\n",
       "             'company': 0.07142857142857142,\n",
       "             'operations': 0.07142857142857142,\n",
       "             'assets': 0.07142857142857142,\n",
       "             'Ltd': 0.07142857142857142,\n",
       "             '.': 0.07142857142857142})"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model[(\"United\", \"Kingdom\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "877ef978",
   "metadata": {},
   "source": [
    "This is all we need to generate new text staring from a bigram prompt. We must simply perform a random walk on this weighted graph starting from an initial prompt:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "21476aa4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_sentence_from_prompt(prompt, zero_temperature=False):\n",
    "    text = [*prompt]\n",
    "\n",
    "    # Dont impose any fixed sentence length\n",
    "    while True:\n",
    "        # the current not we're in is just the one that accounts\n",
    "        # for the last two words in the text\n",
    "        bigram = tuple(text[-2:])\n",
    "\n",
    "        # We extract the list of possible next words and their probabilities\n",
    "        words = []\n",
    "        probs = []\n",
    "\n",
    "        for word, prob in model[bigram].items():\n",
    "            words.append(word)\n",
    "            probs.append(prob)\n",
    "\n",
    "        # Choose one word proportionally to each probability\n",
    "        selection = np.random.multinomial(1, probs)\n",
    "        \n",
    "        # Check which one was chosen\n",
    "        if zero_temperature:\n",
    "            pos = np.argmax(probs) # Temperature = 0\n",
    "        else:\n",
    "            pos = np.argmax(selection) # Temperature = 1\n",
    "\n",
    "        word = words[pos]\n",
    "\n",
    "        # Append the new word to our runnning text\n",
    "        text.append(word)\n",
    "\n",
    "        # Wtop when we hit two None tokens in a row, that represnet the end of a sentence\n",
    "        if text[-2:] == [None, None]:\n",
    "            break\n",
    "        \n",
    "        # Make sure we don't run forever\n",
    "        if zero_temperature and len(text) > 100:\n",
    "            break\n",
    "                \n",
    "    return \" \".join([t for t in text if t])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ad73e98f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'United States for edible and non - rounded rate of 35 pct corporate income tax treaty with the adjustment process in this south - Atlantic states , plus contingent payments .'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_sentence_from_prompt(('United', 'States'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d5ee68b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'today the company at this moment because the price of oil market last year , against 2 . 4 mln dlrs on March 11 , FED SAYS'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_sentence_from_prompt(('today', 'the'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "af8fb2d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'financial markets which can be allocated , an official said .'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_sentence_from_prompt(('financial', 'markets'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1cbd82a",
   "metadata": {},
   "source": [
    "<center>\n",
    "     <img src=\"https://raw.githubusercontent.com/DataForScience/Networks/master/data/D4Sci_logo_full.png\" alt=\"Data For Science, Inc\" align=\"center\" border=\"0\" width=300px> \n",
    "</center>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}