news_RNN_language_model.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aburkov/theLMbook/blob/main/news_RNN_language_model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# RNN-based language model\n",
        "\n",
        "## Utility functions and classes\n",
        "\n",
        "In the cell below, we import the dependencies and define the utility functions and the model class:"
      ],
      "metadata": {
        "id": "4k_iGNa_JVHp"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "yy0zjL_2ouOU"
      },
      "outputs": [],
      "source": [
        "# Import required libraries\n",
        "import os               # For file and path operations (check_file_exists, extract_dataset)\n",
        "import urllib.request   # For downloading dataset files from URLs\n",
        "import tarfile          # For extracting .tar.gz dataset archives\n",
        "import torch            # Main PyTorch library for tensor operations and deep learning\n",
        "import torch.nn as nn   # Neural network modules, layers, and utilities\n",
        "from torch.utils.data import DataLoader, IterableDataset  # For efficient data loading and streaming\n",
        "import random           # For setting random seeds in reproducibility\n",
        "from tqdm import tqdm   # For progress bars in training and evaluation\n",
        "import math             # For computing perplexity using exp()\n",
        "import re               # For preprocessing text (replacing numbers with placeholders)\n",
        "from transformers import AutoTokenizer # For loading a pre-trained tokenizer\n",
        "\n",
        "# ----------------------------\n",
        "# Utility Functions\n",
        "# ----------------------------\n",
        "\n",
        "def set_seed(seed):\n",
        "    \"\"\"\n",
        "    Sets random seeds for reproducibility across different Python libraries.\n",
        "    This ensures that random operations give the same results across runs.\n",
        "\n",
        "    Args:\n",
        "        seed (int): Seed value for random number generation\n",
        "    \"\"\"\n",
        "    # Set seed for Python's built-in random module\n",
        "    random.seed(seed)\n",
        "    # Set seed for PyTorch's CPU random number generator\n",
        "    torch.manual_seed(seed)\n",
        "    # Set seed for PyTorch's GPU random number generator\n",
        "    torch.cuda.manual_seed_all(seed)\n",
        "    # Requests cuDNN to use deterministic algorithms when possible\n",
        "    # Note: This may impact performance and might not guarantee determinism in all cases\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    # Disables cuDNN's auto-tuner which finds the best algorithm for your specific input size\n",
        "    # Ensures consistent behavior but might be slower as it doesn't optimize for input sizes\n",
        "    torch.backends.cudnn.benchmark = False\n",
        "\n",
        "class IterableTextDataset(IterableDataset):\n",
        "    \"\"\"\n",
        "    An iterable dataset for processing text data in a memory-efficient way.\n",
        "    Instead of loading all data into memory, it streams data from disk.\n",
        "    Inherits from PyTorch's IterableDataset for streaming support.\n",
        "\n",
        "    Args:\n",
        "        file_path (str): Path to the text file containing sentences\n",
        "        tokenizer: Tokenizer object for converting text to tokens\n",
        "        max_length (int): Maximum sequence length to process (default: 30)\n",
        "    \"\"\"\n",
        "    def __init__(self, file_path, tokenizer, max_length=30):\n",
        "        # Store file path for reading data\n",
        "        self.file_path = file_path\n",
        "        # Store tokenizer for text processing\n",
        "        self.tokenizer = tokenizer\n",
        "        # Set maximum sequence length to truncate long sequences\n",
        "        self.max_length = max_length\n",
        "        self._count_sentences()\n",
        "\n",
        "    def __iter__(self):\n",
        "        \"\"\"\n",
        "        Creates an iterator over the dataset.\n",
        "        This method is called when iterating over the dataset.\n",
        "\n",
        "        Yields:\n",
        "            tuple: (input_sequence, target_sequence) pairs for language modeling\n",
        "                  input_sequence is the sequence up to the last token\n",
        "                  target_sequence is the sequence shifted one position right\n",
        "        \"\"\"\n",
        "        # Open file in read mode with UTF-8 encoding\n",
        "        with open(self.file_path, 'r', encoding=\"utf-8\") as f:\n",
        "            # Process each line (sentence) in the file\n",
        "            for line in f:\n",
        "                # Remove leading/trailing whitespace\n",
        "                sentence = line.strip()\n",
        "                # Replace all numbers with ### placeholder\n",
        "                # This reduces vocabulary size and helps model generalize\n",
        "                sentence = re.sub(r\"\\d+\", \"###\", sentence)\n",
        "\n",
        "                # Convert sentence to token IDs\n",
        "                encoded_sentence = self.tokenizer.encode(\n",
        "                    sentence,\n",
        "                    max_length=self.max_length,\n",
        "                    truncation=True\n",
        "                )\n",
        "\n",
        "                # Only use sequences with at least 2 tokens\n",
        "                # (need at least one input and one target token)\n",
        "                if len(encoded_sentence) >= 2:\n",
        "                    # Input is all tokens except last\n",
        "                    input_seq = encoded_sentence[:-1]\n",
        "                    # Target is all tokens except first\n",
        "                    target_seq = encoded_sentence[1:]\n",
        "                    # Convert to PyTorch tensors and yield\n",
        "                    yield torch.tensor(input_seq, dtype=torch.long), torch.tensor(target_seq, dtype=torch.long)\n",
        "    def __len__(self):\n",
        "        return self._num_sentences\n",
        "\n",
        "    def _count_sentences(self):\n",
        "        print(f\"Counting sentences in {self.file_path}...\")\n",
        "        with open(self.file_path, 'r', encoding=\"utf-8\") as f:\n",
        "            self._num_sentences = sum(1 for _ in f)\n",
        "        print(f\"Found {self._num_sentences} sentences in {self.file_path}.\")\n",
        "\n",
        "## ----------------------------\n",
        "## Download and prepare data\n",
        "## ----------------------------\n",
        "\n",
        "def create_collate_fn(tokenizer):\n",
        "    \"\"\"\n",
        "    Creates a collate function for batching sequences of different lengths.\n",
        "    This function pads shorter sequences to match the longest sequence in the batch.\n",
        "\n",
        "    Args:\n",
        "        tokenizer: Tokenizer object containing padding token information\n",
        "\n",
        "    Returns:\n",
        "        function: Collate function that handles padding in batches\n",
        "    \"\"\"\n",
        "    def collate_fn(batch):\n",
        "        # Separate inputs and targets from batch\n",
        "        input_seqs, target_seqs = zip(*batch)\n",
        "        # Get padding token ID from tokenizer\n",
        "        pad_index = tokenizer.pad_token_id\n",
        "        # Pad input sequences to same length\n",
        "        input_padded = nn.utils.rnn.pad_sequence(input_seqs, batch_first=True, padding_value=pad_index)\n",
        "        # Pad target sequences to same length\n",
        "        target_padded = nn.utils.rnn.pad_sequence(target_seqs, batch_first=True, padding_value=pad_index)\n",
        "        return input_padded, target_padded\n",
        "    return collate_fn\n",
        "\n",
        "def check_file_exists(filename):\n",
        "    \"\"\"\n",
        "    Checks if a file exists in the current directory.\n",
        "    Args:\n",
        "        filename (str): Name of the file to check\n",
        "    Returns:\n",
        "        bool: True if file exists, False otherwise\n",
        "    \"\"\"\n",
        "    return os.path.exists(filename)\n",
        "\n",
        "def download_file(url):\n",
        "    \"\"\"\n",
        "    Downloads a file from the given URL if it doesn't exist locally.\n",
        "    Uses a custom User-Agent to help prevent download blocks.\n",
        "\n",
        "    Args:\n",
        "        url (str): URL of the file to download\n",
        "    Returns:\n",
        "        str: Name of the downloaded file (\"news.tar.gz\")\n",
        "    \"\"\"\n",
        "    # Always use news.tar.gz as the filename, regardless of URL\n",
        "    filename = \"news.tar.gz\"\n",
        "\n",
        "    if not check_file_exists(filename):\n",
        "        print(f\"Downloading dataset from {url}...\")\n",
        "        req = urllib.request.Request(\n",
        "            url,\n",
        "            headers={\"User-Agent\": \"Mozilla/5.0\"}\n",
        "        )\n",
        "        with urllib.request.urlopen(req) as response:\n",
        "            with open(filename, \"wb\") as out_file:\n",
        "                out_file.write(response.read())\n",
        "        print(\"Download completed.\")\n",
        "    else:\n",
        "        print(f\"{filename} already downloaded.\")\n",
        "    return filename\n",
        "\n",
        "def is_within_directory(directory, target):\n",
        "    \"\"\"\n",
        "    Checks if a target path is within a specified directory by comparing absolute paths.\n",
        "\n",
        "    Args:\n",
        "        directory (str): Base directory path\n",
        "        target (str): Target path to check\n",
        "    Returns:\n",
        "        bool: True if target's absolute path starts with directory's absolute path\n",
        "    \"\"\"\n",
        "    abs_directory = os.path.abspath(directory)\n",
        "    abs_target = os.path.abspath(target)\n",
        "    prefix = os.path.commonprefix([abs_directory, abs_target])\n",
        "    return prefix == abs_directory\n",
        "\n",
        "def extract_dataset(filename):\n",
        "    \"\"\"\n",
        "    Extracts train.txt and test.txt from the downloaded archive.\n",
        "    Includes debug information about archive contents.\n",
        "\n",
        "    Args:\n",
        "        filename (str): Name of the archive file\n",
        "    Returns:\n",
        "        tuple: Paths to extracted train and test files\n",
        "    \"\"\"\n",
        "    data_dir = os.path.join(os.path.dirname(filename), \"news\")\n",
        "    train_path = os.path.join(data_dir, \"train.txt\")\n",
        "    test_path = os.path.join(data_dir, \"test.txt\")\n",
        "\n",
        "    if check_file_exists(train_path) and check_file_exists(test_path):\n",
        "        print(\"Data files already extracted.\")\n",
        "        return train_path, test_path\n",
        "\n",
        "    print(\"\\nListing archive contents:\")\n",
        "    with tarfile.open(filename, \"r:gz\") as tar:\n",
        "        for member in tar.getmembers():\n",
        "            print(f\"Archive member: {member.name}\")\n",
        "\n",
        "        print(\"\\nExtracting files...\")\n",
        "        # Extract to current directory first\n",
        "        tar.extractall('.')\n",
        "\n",
        "    if not (check_file_exists(train_path) and check_file_exists(test_path)):\n",
        "        raise FileNotFoundError(f\"Required files not found in the archive. Please check the paths above.\")\n",
        "\n",
        "    print(\"Extraction completed.\")\n",
        "    return train_path, test_path\n",
        "\n",
        "def create_datasets(train_file, test_file, tokenizer, max_length=30):\n",
        "    \"\"\"\n",
        "    Creates IterableTextDataset objects for training and testing.\n",
        "    These datasets will stream data from disk instead of loading it all into memory.\n",
        "\n",
        "    Args:\n",
        "        train_file (str): Path to training data file\n",
        "        test_file (str): Path to test data file\n",
        "        tokenizer: Tokenizer object for text processing\n",
        "\n",
        "    Returns:\n",
        "        tuple: (train_dataset, test_dataset) - Dataset objects for training and testing\n",
        "    \"\"\"\n",
        "    # Create training dataset\n",
        "    train_dataset = IterableTextDataset(train_file, tokenizer, max_length)\n",
        "    # Create test dataset\n",
        "    test_dataset = IterableTextDataset(test_file, tokenizer, max_length)\n",
        "\n",
        "    # Print dataset sizes\n",
        "    print(f\"Training sentences: {len(train_dataset)}\")\n",
        "    print(f\"Test sentences: {len(test_dataset)}\")\n",
        "\n",
        "    return train_dataset, test_dataset\n",
        "\n",
        "def create_dataloaders(train_dataset, test_dataset, batch_size, collate_fn):\n",
        "    \"\"\"\n",
        "    Creates DataLoader objects for efficient data iteration.\n",
        "\n",
        "    Args:\n",
        "        train_dataset: Training dataset\n",
        "        test_dataset: Test dataset\n",
        "        batch_size (int): Number of sequences per batch\n",
        "        collate_fn: Function to handle padding and batch creation\n",
        "\n",
        "    Returns:\n",
        "        tuple: (train_dataloader, test_dataloader) - DataLoader objects for\n",
        "               iterating over batches of data with proper padding\n",
        "    \"\"\"\n",
        "    # Create training data loader\n",
        "    train_dataloader = DataLoader(\n",
        "        train_dataset,\n",
        "        batch_size=batch_size,\n",
        "        collate_fn=collate_fn,    # Function to handle padding\n",
        "        num_workers=0             # Number of worker processes (0 = single process)\n",
        "    )\n",
        "    # Create test data loader\n",
        "    test_dataloader = DataLoader(\n",
        "        test_dataset,\n",
        "        batch_size=batch_size,\n",
        "        collate_fn=collate_fn,\n",
        "        num_workers=0\n",
        "    )\n",
        "    return train_dataloader, test_dataloader\n",
        "\n",
        "def download_and_prepare_data(url, batch_size, tokenizer, max_length=30):\n",
        "    \"\"\"\n",
        "    Main function to handle the complete data preparation pipeline.\n",
        "    Downloads data, extracts it, and creates necessary dataset objects.\n",
        "\n",
        "    Args:\n",
        "        url (str): URL where the dataset archive can be downloaded\n",
        "        batch_size (int): Batch size for data loading\n",
        "        tokenizer: Tokenizer object for text processing\n",
        "        max_length (int): Maximum sequence length for tokenization (default: 30)\n",
        "\n",
        "    Returns:\n",
        "        tuple: (train_dataloader, test_dataloader) - Ready-to-use data loaders\n",
        "    \"\"\"\n",
        "    # Step 1: Download dataset archive from URL\n",
        "    filename = download_file(url)\n",
        "\n",
        "    # Step 2: Extract training and test files from archive\n",
        "    train_file, test_file = extract_dataset(filename)\n",
        "\n",
        "    # Step 3: Create dataset objects for streaming data\n",
        "    train_dataset, test_dataset = create_datasets(train_file, test_file, tokenizer, max_length)\n",
        "\n",
        "    # Step 4: Create function to handle batch creation\n",
        "    collate_fn = create_collate_fn(tokenizer)\n",
        "\n",
        "    # Step 5: Create and return data loaders\n",
        "    return create_dataloaders(train_dataset, test_dataset, batch_size, collate_fn)\n",
        "\n",
        "def compute_loss_and_perplexity(model, dataloader, tokenizer, criterion, device, max_sentences=1000):\n",
        "    \"\"\"\n",
        "    Evaluates model performance by computing loss and perplexity on data.\n",
        "\n",
        "    Args:\n",
        "        model (nn.Module): The language model to evaluate\n",
        "        dataloader (DataLoader): Data loader containing batched sequences\n",
        "        tokenizer: Tokenizer for handling special tokens like padding\n",
        "        criterion: Loss function (usually CrossEntropyLoss)\n",
        "        device: Device to run computation on (cuda/cpu)\n",
        "        max_sentences (int): Maximum number of sentences to evaluate (default: 1000)\n",
        "                           Limits evaluation to a subset for faster validation\n",
        "\n",
        "    Returns:\n",
        "        tuple: (average_loss, perplexity)\n",
        "               - average_loss: Mean loss per token (excluding padding)\n",
        "               - perplexity: exp(average_loss), lower is better\n",
        "    \"\"\"\n",
        "    # Set model to evaluation mode (disables dropout, etc.)\n",
        "    model.eval()\n",
        "\n",
        "    # Initialize counters for loss calculation\n",
        "    total_loss = 0.0          # Accumulator for total loss across all batches\n",
        "    total_tokens = 0          # Counter for total number of tokens (excluding padding)\n",
        "    sentences_processed = 0    # Counter for number of sentences processed\n",
        "\n",
        "    # Disable gradient computation for efficiency\n",
        "    with torch.no_grad():\n",
        "        # Iterate through data with progress bar\n",
        "        for input_seq, target_seq in tqdm(dataloader, desc=\"Evaluating\", leave=False):\n",
        "            # Move input and target sequences to specified device\n",
        "            input_seq = input_seq.to(device)      # Shape: (batch_size, seq_len)\n",
        "            target_seq = target_seq.to(device)    # Shape: (batch_size, seq_len)\n",
        "\n",
        "            # Get current batch size (might be smaller for last batch)\n",
        "            batch_size_current = input_seq.size(0)\n",
        "\n",
        "            # Forward pass through the model\n",
        "            logits = model(input_seq)             # Shape: (batch_size, seq_len, vocab_size)\n",
        "\n",
        "            # Reshape logits and target for loss calculation\n",
        "            logits = logits.reshape(-1, logits.size(-1))  # Shape: (batch_size * seq_len, vocab_size)\n",
        "            target = target_seq.reshape(-1)              # Shape: (batch_size * seq_len)\n",
        "\n",
        "            # Create mask to exclude padding tokens\n",
        "            mask = target != tokenizer.pad_token_id\n",
        "\n",
        "            # Compute loss only on non-padded tokens\n",
        "            loss = criterion(logits[mask], target[mask])\n",
        "\n",
        "            # Update counters\n",
        "            loss_value = loss.item() * mask.sum().item()  # Total loss for this batch\n",
        "            total_loss += loss_value                      # Accumulate batch loss\n",
        "            total_tokens += mask.sum().item()             # Count non-padding tokens\n",
        "\n",
        "            # Update sentence counter and check if we've reached maximum\n",
        "            sentences_processed += batch_size_current\n",
        "            if sentences_processed >= max_sentences:\n",
        "                break\n",
        "\n",
        "    # Calculate final metrics\n",
        "    average_loss = total_loss / total_tokens           # Normalize loss by number of tokens\n",
        "    perplexity = math.exp(average_loss)               # Convert loss to perplexity\n",
        "\n",
        "    return average_loss, perplexity\n",
        "\n",
        "def perform_model_evaluation(model, test_dataloader, criterion, tokenizer, device, contexts):\n",
        "    \"\"\"\n",
        "    Perform evaluation of the model including loss calculation, perplexity, and text generation.\n",
        "\n",
        "    Args:\n",
        "        model: The neural network model\n",
        "        test_dataloader: DataLoader containing test/validation data\n",
        "        criterion: Loss function\n",
        "        tokenizer: Tokenizer for text generation\n",
        "        device: Device to run computations on (cuda/cpu)\n",
        "        contexts: List of context strings for text generation\n",
        "\n",
        "    Returns:\n",
        "        tuple: (average_loss, perplexity)\n",
        "    \"\"\"\n",
        "    # Switch to evaluation mode\n",
        "    model.eval()\n",
        "\n",
        "    # Compute metrics\n",
        "    average_loss, perplexity = compute_loss_and_perplexity(\n",
        "        model, test_dataloader, tokenizer, criterion, device, max_sentences=1000\n",
        "    )\n",
        "\n",
        "    print(f\"Validation Average Loss: {average_loss:.4f}, Perplexity: {perplexity:.2f}\")\n",
        "\n",
        "    # Generate text using the contexts\n",
        "    print(\"Generating text based on contexts using generate_text:\\n\")\n",
        "    for context in contexts:\n",
        "        generated_text = generate_text(\n",
        "            model=model,          # The loaded language model\n",
        "            start_string=context, # Context to continue\n",
        "            tokenizer=tokenizer,  # Tokenizer for text conversion\n",
        "            device=device,        # CPU or GPU device\n",
        "            max_length=50         # Maximum length of generated sequence\n",
        "        )\n",
        "        print(f\"\\nContext: {context}\")\n",
        "        print(f\"\\nGenerated text: {generated_text}\\n\")\n",
        "\n",
        "    return average_loss, perplexity\n",
        "\n",
        "def generate_text(model, start_string, tokenizer, device, max_length=50):\n",
        "    \"\"\"\n",
        "    Generates text continuation from a given start string using greedy decoding.\n",
        "    This method always chooses the most likely next token.\n",
        "\n",
        "    Args:\n",
        "        model (nn.Module): Trained language model\n",
        "        start_string (str): Initial text to continue from\n",
        "        tokenizer: Tokenizer for text processing\n",
        "        device: Device to run generation on (cuda/cpu)\n",
        "        max_length (int): Maximum length of generated sequence\n",
        "\n",
        "    Returns:\n",
        "        str: Generated text continuation\n",
        "    \"\"\"\n",
        "    # Set model to evaluation mode\n",
        "    model.eval()\n",
        "\n",
        "    # Convert start string to token ids and move to device\n",
        "    # return_tensors=\"pt\" returns PyTorch tensor instead of list\n",
        "    tokens = tokenizer.encode(start_string, return_tensors=\"pt\", max_length=max_length, truncation=True).to(device)\n",
        "\n",
        "    # Initialize generated sequence with input tokens\n",
        "    generated = tokens\n",
        "\n",
        "    # Generate new tokens one at a time\n",
        "    for _ in range(max_length):\n",
        "        # Get model's predictions\n",
        "        output = model(generated)                    # Shape: (1, seq_len, vocab_size)\n",
        "        # Get logits for the next token (last position)\n",
        "        next_token_logits = output[0, -1, :]        # Shape: (vocab_size)\n",
        "\n",
        "        # Choose token with highest probability (greedy decoding)\n",
        "        # unsqueeze twice to match expected shape (1, 1)\n",
        "        next_token_id = torch.argmax(next_token_logits, dim=-1).unsqueeze(0).unsqueeze(0)\n",
        "\n",
        "        # Add new token to generated sequence\n",
        "        generated = torch.cat((generated, next_token_id), dim=1)\n",
        "\n",
        "        # Stop if end of sequence token is generated\n",
        "        if next_token_id.item() == tokenizer.eos_token_id:\n",
        "            break\n",
        "\n",
        "    # Convert token ids back to text\n",
        "    generated_text = tokenizer.decode(generated.squeeze().tolist())\n",
        "    return generated_text\n",
        "\n",
        "def save_model(model, tokenizer, file_prefix):\n",
        "    model_state = {\n",
        "        \"state_dict\": model.state_dict(),\n",
        "        \"vocab_size\": model.vocab_size,\n",
        "        \"emb_dim\": model.emb_dim,\n",
        "        \"num_layers\": model.num_layers,\n",
        "        \"pad_index\": model.pad_index,\n",
        "        \"training\": model.training  # Save training state\n",
        "    }\n",
        "\n",
        "    torch.save(model_state, f\"{file_prefix}_model.pth\")\n",
        "    tokenizer.save_pretrained(f\"{file_prefix}_tokenizer\")\n",
        "\n",
        "def load_model(file_prefix):\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "    # Load state dict to the correct device first\n",
        "    model_state = torch.load(f\"{file_prefix}_model.pth\", map_location=device, weights_only=True)\n",
        "\n",
        "    # Create model and move it to device before loading state dict\n",
        "    model = RecurrentLanguageModel(\n",
        "        model_state[\"vocab_size\"],\n",
        "        model_state[\"emb_dim\"],\n",
        "        model_state[\"num_layers\"],\n",
        "        model_state[\"pad_index\"]\n",
        "    ).to(device)\n",
        "\n",
        "    # Load state dict after model is on correct device\n",
        "    model.load_state_dict(model_state[\"state_dict\"])\n",
        "\n",
        "    # Keep model on device\n",
        "    model.eval()  # Set to evaluation mode\n",
        "\n",
        "    tokenizer = AutoTokenizer.from_pretrained(f\"{file_prefix}_tokenizer\")\n",
        "    return model, tokenizer\n",
        "\n",
        "def get_hyperparameters():\n",
        "    \"\"\"\n",
        "    Returns default hyperparameters for model training.\n",
        "\n",
        "    Returns:\n",
        "        tuple: (emb_dim, num_layers, batch_size, learning_rate, num_epochs)\n",
        "    \"\"\"\n",
        "    emb_dim = 128         # Embedding dimension\n",
        "    num_layers = 2        # Number of RNN layers\n",
        "    batch_size = 128      # Training batch size\n",
        "    learning_rate = 0.001 # Learning rate for optimization\n",
        "    num_epochs = 1        # Number of training epochs\n",
        "    context_size = 30     # Maximum input sequence length\n",
        "    return emb_dim, num_layers, batch_size, learning_rate, num_epochs, context_size"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model Classes\n",
        "\n",
        "RNN language model classes and the initialization method:"
      ],
      "metadata": {
        "id": "SrxY7I8Bml9c"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# ----------------------------\n",
        "# Recurrent Language Model Class\n",
        "# ----------------------------\n",
        "\n",
        "def initialize_weights(model):\n",
        "    \"\"\"\n",
        "    Initializes model weights using Xavier uniform initialization for multi-dimensional\n",
        "    parameters and uniform initialization for biases and other 1D parameters.\n",
        "\n",
        "    Args:\n",
        "        model (nn.Module): PyTorch model whose weights need to be initialized\n",
        "    \"\"\"\n",
        "    # Loop through all named parameters in the model\n",
        "    for name, param in model.named_parameters():\n",
        "        # Check if parameter has more than 1 dimension (e.g., weight matrices)\n",
        "        if param.dim() > 1:\n",
        "            # Use Xavier uniform initialization for weight matrices\n",
        "            # This helps prevent vanishing/exploding gradients by keeping the variance constant\n",
        "            nn.init.xavier_uniform_(param)\n",
        "        else:\n",
        "            # For 1D parameters (like biases), use simple uniform initialization\n",
        "            nn.init.uniform_(param)\n",
        "\n",
        "class ElmanRNNUnit(nn.Module):\n",
        "    \"\"\"\n",
        "    Implementation of a single Elman RNN unit (a simple recurrent neural network cell).\n",
        "    This is the basic building block of our RNN that processes one time step of input.\n",
        "\n",
        "    Args:\n",
        "        emb_dim (int): Dimension of the embedding/hidden state vectors\n",
        "    \"\"\"\n",
        "    def __init__(self, emb_dim):\n",
        "        super(ElmanRNNUnit, self).__init__()\n",
        "        # Hidden-to-hidden weight matrix: transforms previous hidden state\n",
        "        # Shape: (emb_dim, emb_dim)\n",
        "        self.Uh = nn.Parameter(torch.rand(emb_dim, emb_dim))\n",
        "\n",
        "        # Input-to-hidden weight matrix: transforms current input\n",
        "        # Shape: (emb_dim, emb_dim)\n",
        "        self.Wh = nn.Parameter(torch.rand(emb_dim, emb_dim))\n",
        "\n",
        "        # Bias term added to the sum of transformations\n",
        "        # Shape: (emb_dim,)\n",
        "        self.b = nn.Parameter(torch.rand(emb_dim))\n",
        "\n",
        "    def forward(self, x, h):\n",
        "        \"\"\"\n",
        "        Computes one step of the RNN unit.\n",
        "\n",
        "        Args:\n",
        "            x (torch.Tensor): Current input tensor of shape (batch_size, emb_dim)\n",
        "            h (torch.Tensor): Previous hidden state of shape (batch_size, emb_dim)\n",
        "\n",
        "        Returns:\n",
        "            torch.Tensor: New hidden state of shape (batch_size, emb_dim)\n",
        "\n",
        "        The formula implemented is: h_new = tanh(x @ Wh + h @ Uh + b)\n",
        "        where @ represents matrix multiplication\n",
        "        \"\"\"\n",
        "        # 1. Transform current input: x @ Wh\n",
        "        input_transform = x @ self.Wh\n",
        "\n",
        "        # 2. Transform previous hidden state: h @ Uh\n",
        "        hidden_transform = h @ self.Uh\n",
        "\n",
        "        # 3. Add both transformations and bias\n",
        "        # 4. Apply tanh activation function to get new hidden state\n",
        "        # tanh squashes values to range (-1, 1), helping prevent exploding gradients\n",
        "        return torch.tanh(input_transform + hidden_transform + self.b)\n",
        "\n",
        "class ElmanRNN(nn.Module):\n",
        "    \"\"\"\n",
        "    Multi-layer Elman RNN implementation that processes entire sequences.\n",
        "    Stacks multiple RNN units to create a deeper network that can learn more complex patterns.\n",
        "\n",
        "    Args:\n",
        "        emb_dim (int): Dimension of embeddings and hidden states\n",
        "        num_layers (int): Number of stacked RNN layers\n",
        "    \"\"\"\n",
        "    def __init__(self, emb_dim, num_layers):\n",
        "        super().__init__()\n",
        "        self.emb_dim = emb_dim\n",
        "        self.num_layers = num_layers\n",
        "\n",
        "        # Create a list of RNN units, one for each layer\n",
        "        # ModuleList is used so PyTorch tracks all parameters\n",
        "        self.rnn_units = nn.ModuleList(\n",
        "            [ElmanRNNUnit(emb_dim) for _ in range(num_layers)]\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        Processes input sequence through all RNN layers.\n",
        "\n",
        "        Args:\n",
        "            x (torch.Tensor): Input tensor of shape (batch_size, seq_len, emb_dim)\n",
        "\n",
        "        Returns:\n",
        "            torch.Tensor: Output tensor of shape (batch_size, seq_len, emb_dim)\n",
        "        \"\"\"\n",
        "        # Get dimensions from input tensor\n",
        "        batch_size, seq_len, emb_dim = x.size()\n",
        "\n",
        "        # Initialize hidden states for each layer with zeros\n",
        "        # Each hidden state has shape (batch_size, emb_dim)\n",
        "        h_prev = [\n",
        "            torch.zeros(batch_size, emb_dim, device=x.device)\n",
        "            for _ in range(self.num_layers)\n",
        "        ]\n",
        "\n",
        "        # Will store outputs for each time step\n",
        "        outputs = []\n",
        "\n",
        "        # Process each time step\n",
        "        for t in range(seq_len):\n",
        "            # Get input for current time step\n",
        "            input_t = x[:, t]\n",
        "\n",
        "            # Process through each layer\n",
        "            for l, rnn_unit in enumerate(self.rnn_units):\n",
        "                # Compute new hidden state for this layer\n",
        "                h_new = rnn_unit(input_t, h_prev[l])\n",
        "\n",
        "                # Update hidden state for this layer\n",
        "                h_prev[l] = h_new\n",
        "\n",
        "                # Output of this layer becomes input to next layer\n",
        "                input_t = h_new\n",
        "\n",
        "            # Add final layer's output to results\n",
        "            outputs.append(input_t)\n",
        "\n",
        "        # Stack all time steps' outputs into a single tensor\n",
        "        # Shape: (batch_size, seq_len, emb_dim)\n",
        "        return torch.stack(outputs, dim=1)\n",
        "\n",
        "class RecurrentLanguageModel(nn.Module):\n",
        "    \"\"\"\n",
        "    Complete language model implementation combining embedding layer,\n",
        "    multi-layer RNN, and output projection layer.\n",
        "\n",
        "    The model architecture is:\n",
        "    1. Input tokens -> Embedding Layer -> Embedded Vectors\n",
        "    2. Embedded Vectors -> RNN Layers -> Context Vectors\n",
        "    3. Context Vectors -> Linear Layer -> Vocabulary Predictions\n",
        "\n",
        "    Args:\n",
        "        vocab_size (int): Size of the vocabulary (number of unique tokens)\n",
        "        emb_dim (int): Dimension of embeddings and hidden states\n",
        "        num_layers (int): Number of RNN layers\n",
        "        pad_index (int): Index used for padding tokens\n",
        "    \"\"\"\n",
        "    def __init__(self, vocab_size, emb_dim, num_layers, pad_index):\n",
        "        super().__init__()\n",
        "\n",
        "        # Save model parameters\n",
        "        self.vocab_size = vocab_size\n",
        "        self.emb_dim = emb_dim\n",
        "        self.num_layers = num_layers\n",
        "        self.pad_index = pad_index\n",
        "\n",
        "        # Embedding layer: converts token indices to dense vectors\n",
        "        # pad_index tokens will be mapped to zero vectors\n",
        "        self.embedding = nn.Embedding(vocab_size, emb_dim, pad_index)\n",
        "\n",
        "        # RNN layers for processing sequences\n",
        "        self.rnn = ElmanRNN(\n",
        "            emb_dim=emb_dim,\n",
        "            num_layers=num_layers\n",
        "        )\n",
        "\n",
        "        # Final linear layer to convert RNN outputs to vocabulary predictions\n",
        "        # Output size is vocab_size to get logits for each possible token\n",
        "        self.fc = nn.Linear(emb_dim, vocab_size)\n",
        "\n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        Processes input sequences through the entire model.\n",
        "\n",
        "        Args:\n",
        "            x (torch.Tensor): Input tensor of token indices\n",
        "                            Shape: (batch_size, seq_len)\n",
        "\n",
        "        Returns:\n",
        "            torch.Tensor: Output logits for next token prediction\n",
        "                         Shape: (batch_size, seq_len, vocab_size)\n",
        "\n",
        "        Process:\n",
        "        1. Convert token indices to embeddings\n",
        "        2. Process embeddings through RNN layers\n",
        "        3. Project RNN outputs to vocabulary size\n",
        "        \"\"\"\n",
        "        # Convert token indices to embeddings\n",
        "        # Shape: (batch_size, seq_len) -> (batch_size, seq_len, emb_dim)\n",
        "        embeddings = self.embedding(x)\n",
        "\n",
        "        # Process through RNN layers\n",
        "        # Shape: (batch_size, seq_len, emb_dim) -> (batch_size, seq_len, emb_dim)\n",
        "        rnn_output = self.rnn(embeddings)\n",
        "\n",
        "        # Project to vocabulary size to get logits\n",
        "        # Shape: (batch_size, seq_len, emb_dim) -> (batch_size, seq_len, vocab_size)\n",
        "        logits = self.fc(rnn_output)\n",
        "\n",
        "        return logits"
      ],
      "metadata": {
        "id": "A82GEwnWmorj"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training the language model\n",
        "\n",
        "In the cell below, we load the data, train, and save the language model:"
      ],
      "metadata": {
        "id": "xcutQEMSJg77"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "s61ovCwawq3f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "42f9c201196846ecaaaca13424066f31",
            "7dc8266e8e9349f59617784563bcfb0d",
            "e396c523cfa14d6ab7582345428d7e2b",
            "63c9afad68224b13a7f345826757d74d",
            "d1abdf9e1dbc4c2db3a83717d52052a7",
            "dbe62517041c478f89fd4f7031684997",
            "c573226f95d34166833c410c0559047e",
            "5f7faf15a50545bf971902428f788aba",
            "539e601e729a4c7d81b48e17a50b6ab2",
            "0873e85b65a447199d4e506731d2f60e",
            "d5208cb6e46f4c19a893568e94fe471b",
            "e468ae9b5ee146eb97c30388eabeddfe",
            "0294c86e78d0416ca6f74294f93a0c57",
            "8b193f31f5714da18bcc046cbafc807f",
            "5b8f89c68dd34744946be5a9aa7cf1a2",
            "4e93cc7e12834e6da060dad9f1cc93b5",
            "53c0255664eb44e68fa0e93769512a67",
            "44fdec6de91f47889673265c2ef80ab3",
            "f388c603cafa4060b01fae139c814d4d",
            "646a033463044d61af033e64c263f5e7",
            "c9161c25556e467bb05d0f4d866bf467",
            "fe1150d2657c4d88aa81ca6548832860",
            "45aa4c9ffccc484698af6b51d6c6324a",
            "b6f4495af1f64a35877716f00f40c33c",
            "17d7b1b82bf340b4af2cef3b9791168e",
            "67f97fe065834d8a8d1a57cfe83e77a6",
            "3e6a18e429554400b8b1c480cfd129aa",
            "74d48d183a5e41cd8cc52f939bb2b842",
            "2819c169ef5f4239bcd655771ffc11ab",
            "93b8236e001341fc89a5556d0c0f4233",
            "f32e37cf305d4abcad893cdbd08e0c6a",
            "1e243fa6c99142a0a7f45e3d3fb1d96f",
            "3445fee928cc41f483d07bf946046bdc",
            "b2d13951116649d3b1a9c6403c442b43",
            "767362af7c494617a8ccde641fff124f",
            "0c68a7b351b2486abb2b65d794d15aec",
            "1e611591b22c46d7b5977c2ecdde02f2",
            "258b534bd0894450b59b2eb7ab0905ec",
            "6806c2567a274957a9335f46c505db15",
            "2789503992c04d6991db2606f580f230",
            "3f7221ac1912480ca7054b64f8a87a7f",
            "9989d9d05d264f3da894bcf7c84b6f94",
            "67cb7426300947ebbe9087ee81b3f4db",
            "bb2b55614d5b4e78b40e625084776bb8",
            "3aaa2099e38142de9854b22a7595a88e",
            "d67374a248974744b9bfd2922acf3e88",
            "ab2f98969f734e399e224069c23a79b0",
            "56408789191d4ad3a0043fa816e03529",
            "8b7ab57a33184571914ff60ba032a20d",
            "cdf2acfd3e304fc492378c0dee8a8946",
            "cd3342d14a5e4832843aaa9830bdee4d",
            "0629dca9bbd2478eb5e1b6791540024c",
            "a279be615d464c269d508c0c0fa33ebd",
            "2022b6b16bb041f3b1d961033eda33a3",
            "ef8ec7ac921849ed8c4814536b24b61b"
          ]
        },
        "outputId": "eda4a309-f19a-4c45-cb0f-ef66ad7af4b7"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/3.98k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "42f9c201196846ecaaaca13424066f31"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.model:   0%|          | 0.00/500k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e468ae9b5ee146eb97c30388eabeddfe"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.84M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "45aa4c9ffccc484698af6b51d6c6324a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "added_tokens.json:   0%|          | 0.00/306 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b2d13951116649d3b1a9c6403c442b43"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/665 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3aaa2099e38142de9854b22a7595a88e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading dataset from https://www.thelmbook.com/data/news...\n",
            "Download completed.\n",
            "\n",
            "Listing archive contents:\n",
            "Archive member: news\n",
            "Archive member: news/train.txt\n",
            "Archive member: news/test.txt\n",
            "\n",
            "Extracting files...\n",
            "Extraction completed.\n",
            "Counting sentences in news/train.txt...\n",
            "Found 22034911 sentences in news/train.txt.\n",
            "Counting sentences in news/test.txt...\n",
            "Found 449693 sentences in news/test.txt.\n",
            "Training sentences: 22034911\n",
            "Test sentences: 449693\n",
            "Total trainable parameters: 8292619\n",
            "\n",
            "Starting Epoch 1/1, Model in training mode: True\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   1%|          | 1561/172148 [01:13<2:12:15, 21.50it/s, loss=5.7836]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 6.3757\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 2/3514 [00:00<03:25, 17.05it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:12, 26.39it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 5.5840, Perplexity: 266.13\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow , the ##-year-old , who was a lot of the world . '' he said . 's the first time . '' he said . 's the first time , the first time of the time , I 've been a lot of\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York , ## , was a lot of the world . '' he said . 's the first time . '' he said . 's the first time , the first time of the time , I 've been a lot of the world . '' he said\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane , the first-year-old , who was a lot of the world . '' he said . 's the first time . '' he said . 's the first time , the first time of the time , I 've been a lot of\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   1%|          | 1566/172148 [01:17<17:54:50,  2.65it/s, loss=5.7444]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President was a lot of the world . '' he said . 's the first time . '' he said . 's the first time , 's the first time . '' he said . 's the first time . '' he said . 's the\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   2%|▏         | 3123/172148 [02:33<2:15:54, 20.73it/s, loss=5.4569]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 5.4135\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.30it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 5.2107, Perplexity: 183.23\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's mother , who was a `` to be able to be a good way . '' he said . 's the time . '' he said . 's the time . '' he said . 's the time . '' he said . 's\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , who was a `` to be able to be a good way . '' he said . 's the time . '' he said . 's the time . '' he said . 's the time . '' he said . 's the time\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane , the ##-year-old was a `` to be able to be a good way . '' he said . 's the time . '' he said . 's the time . '' he said . 's the time . '' he said .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   2%|▏         | 3129/172148 [02:34<7:10:52,  6.54it/s, loss=5.2555]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President , ## , said : 'We are not a good way . '' he said . 's the time . '' he said . 's the time . '' he said . 's the time . '' he said . 's the time . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   3%|▎         | 4688/172148 [03:50<2:10:57, 21.31it/s, loss=5.1191]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 5.1565\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:43, 33.76it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 5.0319, Perplexity: 153.22\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's mother , who was a key of the most of the most of the most of the most of the most of the most of the most of the most of the most of the most of the most of the most of the most of the most\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who was a second-time . 's a lot of people . 's a lot of people . '' he said . 's a lot of people . 's a lot of . '' . 's a time . '' said\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was a free kick in the #### . ' I 'm not going to be a bit of the right . '' . 's a time . '' said the . 's the time . '' said the . 's the time . '' said the\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   3%|▎         | 4691/172148 [03:51<9:02:25,  5.15it/s, loss=5.0833]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the UK , the ##-year-old was a `` <rare> '' . 's the right . '' he said . 's a lot of people . 's a lot of . '' . 's a time . '' said the\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   4%|▎         | 6250/172148 [05:08<2:15:19, 20.43it/s, loss=4.9857]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 5.0092\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:49, 31.95it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.9154, Perplexity: 136.38\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's mother , who was a ##-year-old . ' I 'm not going to be a lot of people . '' said the . 's . '' said the ##-year-old was a `` <rare> ' , and\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York , the ##-year-old was a `` <rare> ' , and the family 's mother , who was a ##-year-old . ' I 'm not going to be a lot of people . '' said the . 's\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was a very good thing . ' '' he said . 's the time . 's the time . 's the time . 's the time . 's the time . 's the time . 's the time . 's the time\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   4%|▎         | 6256/172148 [05:10<7:06:47,  6.48it/s, loss=5.0330]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President Barack Obama said the ##-year-old was a `` <rare> ' , '' said the former <rare> , said : ' I 'm not going to be a lot of people . '' said the . 's . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   5%|▍         | 7814/172148 [06:26<2:12:34, 20.66it/s, loss=4.8549]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.9085\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:46, 32.88it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.8310, Perplexity: 125.34\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's first-degree murder , was a `` <rare> '' . 's the end of the day . '' said . '' he said . 's the time . '' said . 's ) . '' he said . 's the\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , the former presidential candidate , who was a `` <rare> '' . 's the end of the day . '' said . '' he said . 's the time . '' said . 's ) . '' he said . '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was a very good way to the . 's ] . '' he said . 's the time . '' said . 's ) . '' he said . 's the time . '' said . 's ) . '' he said . 's\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   5%|▍         | 7817/172148 [06:28<9:00:10,  5.07it/s, loss=4.7741]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President Barack Obama has been a `` to be a good way to get a lot of people who are not going to be a good way . '' . 's . '' said the . 's ] . '' he said . 's the time\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   5%|▌         | 9375/172148 [07:44<2:10:48, 20.74it/s, loss=4.8292]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.8369\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.80it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.7661, Perplexity: 117.47\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's first-team-class-year-old was also charged with murdering a few months ago . 's a . ' '' he said . 's the time . 's ] . 's a . '' . 's ] .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the ##-year-old was found in the first half of the first time . 's a . ' '' he said . 's the time . 's ] . 's a . '' . 's ] . 's\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane of the <rare> , the world 's most important thing you can be able to be able to be able to get out of the world . '' said . 's the right . 's ] . 's a . '' . 's\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   5%|▌         | 9380/172148 [07:46<7:29:20,  6.04it/s, loss=4.8661]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the National Assembly of the National Institute of Defence , which is also known as a result of the first time . 's a . ' '' he said . 's the time . 's ] . 's a . '' . 's\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   6%|▋         | 10940/172148 [09:02<2:11:45, 20.39it/s, loss=4.7031]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.7787\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:51, 31.39it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.7127, Perplexity: 111.36\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow is the first time in the first time in the ####s , which is the first time in the ####s , which is the first time in the ####s . 's the . '' . 's the . '' . 's the . ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the ##-year-old was a `` serious condition '' . 's the . '' . 's the . '' . 's the . '' . 's the . '' . 's the . '' . 's the .\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a very good way to get the . '' . 's the . '' . 's the . '' . 's the . '' . 's the . '' . 's the . '' . 's the . '' . 's the\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   6%|▋         | 10943/172148 [09:04<9:14:56,  4.84it/s, loss=4.7860]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President Barack Obama said the ##-year-old was a `` serious condition '' . 's the . '' . 's the . '' . 's the . '' . 's the . '' . 's the . '' . 's\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   7%|▋         | 12502/172148 [10:20<2:07:22, 20.89it/s, loss=4.8597]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.7274\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:44, 33.46it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.6745, Perplexity: 107.18\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's official said the report is not the case . '' . 's ] . ' '' he said . '' said . 's the . '' . 's Office . '' said . 's the . '' . 's Office . '' said\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the ##-year-old was a `` unprecedented '' of the country 's most-elected . '' . 's ] . ' '' he said . '' said . 's the . '' . 's Office\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane , which is the most popularity of the world 's most of the world . '' . 's ] . ' '' he said . '' said . 's the . '' . 's Office . '' said . 's the . '' .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   7%|▋         | 12508/172148 [10:22<6:51:21,  6.47it/s, loss=4.7940]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the city of the city of the city of the city of the city of the city of the city of the city of the city . '' . 's ] . ' '' he said . '' said . 's the . '' . 's\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   8%|▊         | 14066/172148 [11:38<2:07:20, 20.69it/s, loss=4.7040]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.6893\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.42it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.6457, Perplexity: 104.14\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's most important thing is the most important thing that is the most important thing . '' ' said the . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who has been charged with murdering the death of the ##-year-old son , who was born in the ####s . ' '' he said . ' '' he said . ' '' he said . ' '' he said . '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane of the <rare> , the U.S. military has been linked to the U.S. military . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   8%|▊         | 14069/172148 [11:40<8:40:37,  5.06it/s, loss=4.7456]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's death was not the first time to be a `` <rare> '' . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   9%|▉         | 15628/172148 [12:57<2:06:15, 20.66it/s, loss=4.5647]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.6595\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:45, 33.34it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.6143, Perplexity: 100.92\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's most popular director , who is the most popular in the world , and the other side of the world . '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the ##-year-old was a `` unfortunate decision . '' ' said the company said . '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' ''\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was found in the area of the city 's most popular in the world . '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:   9%|▉         | 15634/172148 [12:58<6:35:52,  6.59it/s, loss=4.5966]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. government has been accused of the death of the ##-year-old . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  10%|▉         | 17190/172148 [14:15<2:07:29, 20.26it/s, loss=4.6015]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.6339\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.44it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.5910, Perplexity: 98.59\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's largest city of the country , and the U.S. military spokesman said . 'S ] . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New Yorkers have been a very good way to get the way . '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a big-day location . '' . ' ] is a good job . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  10%|▉         | 17195/172148 [14:17<7:16:32,  5.92it/s, loss=4.5845]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's newborn was a `` unacceptable '' . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  11%|█         | 18755/172148 [15:33<2:05:17, 20.40it/s, loss=4.5637]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.6104\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.22it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.5709, Perplexity: 96.63\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's first-choice-born man was arrested in #### , was found in the ####s , and the city of the city of the city was in the middle of the country . '' . ' '' he said . ' '' he said . '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the government was `` not to be a good thing . '' . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the UK , and the UK is the first time in the UK . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  11%|█         | 18758/172148 [15:34<8:22:59,  5.08it/s, loss=4.5959]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government was `` not to be a good thing . '' . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  12%|█▏        | 20316/172148 [16:51<2:03:36, 20.47it/s, loss=4.5015]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.5863\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.51it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.5529, Perplexity: 94.91\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's deaths were not immediately clear . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who is the most important thing that is the most important thing that is the most important thing . '' 's ] . '' 's ] . ' '' he said . ' '' he said . ' '' he said . ' '' he\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane of the most common sense of the most important thing . '' 's ] , '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  12%|█▏        | 20322/172148 [16:52<6:29:31,  6.50it/s, loss=4.5711]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's death was a `` significant problem '' . '' 's ] , ' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  13%|█▎        | 21880/172148 [18:09<2:01:32, 20.61it/s, loss=4.6180]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.5733\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:49, 31.95it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.5302, Perplexity: 92.78\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's most popularity of the world 's largest , and the country 's most importantly , the country 's most important thing . '' ) . '' ) . '' . ' '' he said . '' ) . '' said . '' )\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the couple had been a `` <rare> '' . '' he said . '' ) . '' said . '' ) . '' said . '' ) . '' said . '' ) . '' said . '' ) . '' said . '' )\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the UK . '' the report said . '' ) . '' . ' '' he said . '' ) . '' said . '' ) . '' said . '' ) . '' said . '' ) . '' said . '' ) .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  13%|█▎        | 21885/172148 [18:10<6:52:04,  6.08it/s, loss=4.5331]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. military has been in the UK and the UK 's most important thing . '' ) . '' . '' ) . '' said . '' ) . '' said . '' ) . '' said . '' ) . '' said .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  14%|█▎        | 23442/172148 [19:27<2:00:30, 20.57it/s, loss=4.5115]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.5587\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:51, 31.39it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.5235, Perplexity: 92.16\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has said the government 's decision to be `` a very serious '' . '' said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council has been a major role in the public that the government has been `` not to be able to do . '' . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said .\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was found in the area . 's the club . '' ) . '' said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  14%|█▎        | 23448/172148 [19:29<6:13:06,  6.64it/s, loss=4.5962]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government 's decision to be `` a very serious '' . '' said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  15%|█▍        | 25005/172148 [20:45<1:57:41, 20.84it/s, loss=4.5187]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.5378\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.72it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.5127, Perplexity: 91.16\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been criticised by the U.S. District Court in #### . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the ##-year-old was arrested on the scene of the incident . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' ''\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane of the city 's eastern town of the city of the city of the city of the city . '' ) said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  15%|█▍        | 25011/172148 [20:47<6:13:11,  6.57it/s, loss=4.5142]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President was the first time of the attack . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  15%|█▌        | 26569/172148 [22:04<1:57:51, 20.59it/s, loss=4.6281]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.5264\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:49, 31.92it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.4918, Perplexity: 89.29\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major role in the first time in the ####s . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City : The ##-year-old man who was a ##-year-old girl who was a ##-year-old girl who was a member of the family . ' '' he said . ' '' he said . ' '' he said .\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane of the city 's body was found in the area . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  15%|█▌        | 26575/172148 [22:05<6:18:08,  6.42it/s, loss=4.5934]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government would be a `` unacceptable '' . '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  16%|█▋        | 28131/172148 [23:21<1:57:09, 20.49it/s, loss=4.4456]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.5144\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.48it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.4849, Perplexity: 88.67\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been in the country 's most popular destination for the United States , and the United States , the United States , said : ‘ The incident was a very good person . '' said . ' '' he said . ' '' he said . ' ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who is the first time in the United States , said : ' I 'm not going to be a good player . '' said . ' '' he said . ' '' he said . ' '' he said . ' '' he said .\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane of the country is the most popular in the world , and the world 's largest . 'S . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  16%|█▋        | 28137/172148 [23:23<6:11:39,  6.46it/s, loss=4.3693]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government has been `` not to be a lot of people to do . '' . ' '' said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  17%|█▋        | 29695/172148 [24:39<2:05:49, 18.87it/s, loss=4.2938]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.5016\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:53, 30.99it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.4723, Perplexity: 87.56\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's death , the report said . 'Donnell said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who has been charged with the murder of the incident . ' '' said the .##m . 'F . ' '' said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was found in the water and thefts . 'Fotland is the first time . '' said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  17%|█▋        | 29699/172148 [24:41<8:24:51,  4.70it/s, loss=4.4051]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. military has been killed in the attack . ' '' said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' '' he said . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  18%|█▊        | 31257/172148 [25:58<1:53:26, 20.70it/s, loss=4.5722]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4976\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.75it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.4633, Perplexity: 86.77\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been in the UK and the UK 's most recent ##-year-old is the first time in the Premier League . ' '' he said . ' '' he said . ' '' she said . ' '' she said . ' '' she said\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's ##-year-old son was arrested in the attack , and the ##-year-old was a ##-year-old girl who was a ##-year-old girl who was a ##-year-old girl who was\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane of the ##-year-old was a ##-year-old girl who was a ##-year-old girl who was a ##-year-old girl who was a ##-year-old girl who was a ##-year-old\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  18%|█▊        | 31263/172148 [25:59<6:00:01,  6.52it/s, loss=4.4452]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. military has been in the UK and the UK 's most recent ##-year-old has been in the UK . ' '' he said . ' '' he said . ' '' she said . ' '' she said .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  19%|█▉        | 32821/172148 [27:16<1:52:02, 20.73it/s, loss=4.3727]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4855\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.65it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.4544, Perplexity: 86.01\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been criticized by the government to the U.S. Embassy in #### . ' '' said the .### . ' '' he said . ' '' said the .### . ' '' he said . ' ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , ## , was found in the area . 's the .### . ' '' he said . ' '' said the .### . ' '' he said . ' '' said the .### . ' '' he said . ' ''\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the most popular destination for the first time , and the most popular players have been in the world . '' 's first lady . ' '' he said . ' '' said the .### . ' '' he said . ' '' said the .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  19%|█▉        | 32826/172148 [27:17<6:20:18,  6.11it/s, loss=4.4859]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's newborns , the U.S. Embassy in #### , and the U.S. Embassy in #### . 's the . ' '' he said . ' '' said the .### . ' '' he said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  20%|█▉        | 34385/172148 [28:34<1:53:14, 20.28it/s, loss=4.5607]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4781\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:51, 31.61it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.4410, Perplexity: 84.86\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with murder and murdered the death of the death of the death of the .### . ' '' said the .##.# . ' '' said the .### . ' '' said the statement . ' '' said the .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City : The ##-year-old was shot in the attacking half . '## . ' '' said the .##m . ' '' said the statement . ' '' said the .### . ' '' said the statement . ' '' said\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane of the ##-year-old was found in the area of the city of the city of the city . 's . '' ) said . ' '' said the .### . ' '' said the statement . ' '' said the .###\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  20%|█▉        | 34388/172148 [28:36<7:34:33,  5.05it/s, loss=4.2880]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. Embassy in the UK , the U.S. Embassy in the UK , the U.S. Embassy in #### , the ##-year-old , who has been charged with murder and\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  21%|██        | 35947/172148 [29:52<1:57:44, 19.28it/s, loss=4.4307]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4688\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<01:57, 29.88it/s]\u001b[A\n",
            "Evaluating:   0%|          | 7/3514 [00:00<01:51, 31.51it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.4416, Perplexity: 84.91\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's office said the law is not the case . ' '' said the .### , '' said the company . ' '' said the company . ' '' said the company . ' '' said the company . ' '' said the company . ' ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City : The ##-year-old woman was arrested in the attack , which was found dead in the city of the city . ' '' said the .### , '' said the U.S. . ' '' said the .### ,\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane of the ship 's body was found in the water . '## . '' 's not . ' '' said the .### , '' said the .### , '' said the company . ' '' said the company . ' '' said the\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  21%|██        | 35952/172148 [29:54<6:27:33,  5.86it/s, loss=4.4555]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's ##-year-old was a member of the U.S. Embassy in the city of the city . ' '' said the .### , '' said the U.S. . ' '' said the company . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  22%|██▏       | 37511/172148 [31:11<1:47:16, 20.92it/s, loss=4.3672]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4635\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.80it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.4315, Perplexity: 84.06\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's first lady , who was a ##-year-old daughter , said : ' I 'm not going to be a good friend . ' '' said the .### . ' '' said the .### . ' '' said the .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who is the first of the largest city of the city , and the UK is the largest of the largest . 'S . ' '' said the report . ' I 'm not going to be a good friend . ' '' said the\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane of the disease is the only way to the . ' '' said the company . ' '' said the .### . ' '' said the . ' I 'm not going to be a good friend . ' '' said the .### . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  22%|██▏       | 37514/172148 [31:12<7:24:15,  5.05it/s, loss=4.5374]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's name was the first time that the government has been a `` significant '' of the `` <rare> '' of the <rare> . ' '' said the company said . ' I 'm not going to be a good job . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  23%|██▎       | 39072/172148 [32:28<1:49:30, 20.25it/s, loss=4.4933]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4565\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:51, 31.57it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.5218, Perplexity: 92.00\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with murder and the death of the death of the death of the death of the death of the death of the death of the death of the death of the death of the death of the death of the death of the death of the death\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York 's most recent interview with the U.S. military in the UK and the U.S. military in the United States , where the death of the death of the death of the death of the .### , and the victim of a\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was found in the area , which was found in the area of the area . ' '' said the .### , '' he said . ' I 'm not going to be a very difficult time to do it . '' 's a very good\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  23%|██▎       | 39077/172148 [32:30<6:15:06,  5.91it/s, loss=4.4784]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been released in the case of the case , but the police were not immediately returned to the hospital . ' ' I 'm not going to be a very difficult time to do it . '' 's a very good friend . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  24%|██▎       | 40636/172148 [33:47<1:44:56, 20.89it/s, loss=4.3749]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4505\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:16, 25.68it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:05, 27.87it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.4200, Perplexity: 83.10\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's office said the government would be `` a very difficult time '' . ' '' said the . ' I 'm not going to do . '' 's said . ' I 'm not going to do . ' '' said the .###\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City : The ##-year-old was arrested on suspicion of murdering a woman who was arrested on suspicion of murdering a woman who was arrested on suspicion of murdering a woman who was arrested on suspicion of murdering a\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane in the UK are the largest city of the city . '' ' said the . 'S . ' '' said . ' I 'm not going to do . ' '' said the .### . ' '' said the statement . ' I 'm\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  24%|██▎       | 40641/172148 [33:49<6:11:27,  5.90it/s, loss=4.2400]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President was the first time of the attack . ' '' said the .### . ' '' said the statement . ' '' said the statement . ' I 'm not going to do . ' '' said the .### . ' '' said the statement\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  25%|██▍       | 42199/172148 [35:06<1:47:09, 20.21it/s, loss=4.3259]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4474\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:49, 32.14it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.4173, Perplexity: 82.87\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a member of the House of Commons in #### . ' I 'm not going to do . '' ' said the .##am . ' '' said . ' I 'm not going to do . '' ' said the .##am .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York Times : ' I 'm not going to be a good job . '' 's a .### . ' '' said . ' I 'm not going to do . '' ' said the .##am . ' '' said . ' I '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane , the ##-year-old , who was arrested in #### , was arrested in #### . ' I 'm not going to do . ' '' said the .##am . ' '' said . ' I 'm not going to do . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  25%|██▍       | 42205/172148 [35:07<5:29:24,  6.57it/s, loss=4.5252]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's most recent cases are not the first time in the UK . ' I 'm not going to do . '' ' said the .##am . ' '' said . ' I 'm not going to do . '' ' said the .##\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  25%|██▌       | 43763/172148 [36:24<1:44:22, 20.50it/s, loss=4.5171]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4414\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:49, 32.03it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.4110, Perplexity: 82.35\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a very good idea . '' 's said . ' I 'm not going to be a very good job . '' 's said . ' I 'm not going to be a very good job . '' 's said . ' I\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who is a professor of the U.S. government to be able to provide a new deal with the government 's government to be able to make a new deal with the government . ' '' said the company . ' I 'm\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane of the disease is a very rare condition , and it 's not a problem . '' 's said . ' I 'm not going to be a very good job . '' 's said . ' I 'm not going to be a very\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  25%|██▌       | 43766/172148 [36:26<7:07:09,  5.01it/s, loss=4.3998]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's family have been charged with a child in the case . ' I 'm not going to be a very good job . '' 's said . ' I 'm not going to be a very good job . '' 's said . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  26%|██▋       | 45326/172148 [37:43<1:41:11, 20.89it/s, loss=4.3642]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4341\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:52, 31.31it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3983, Perplexity: 81.31\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's first minister , who has been in the United States , which is the first time in the United States . 's . '' 's a .### . 'D# . ' '' said the . ' I 'm not going to\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's first-round tie was a second goal in the second half . ' I 'm not going to be a good job . '' 's a good job . ' '' said the .### . ' I 'm not going to\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a very small number of people who have been killed . 's the first time . '' 's a .### . 'Dowell said . ' I 'm not going to be a good thing . '' 's a .##\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  26%|██▋       | 45329/172148 [37:45<6:58:30,  5.05it/s, loss=4.5039]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's new lawyers have been in the case , which is the first time in the United States . 's the .### . 'S.# . ) . '' 's a . ' I think it 's a good thing\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  27%|██▋       | 46889/172148 [39:02<1:41:23, 20.59it/s, loss=4.5328]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4303\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.79it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.4014, Perplexity: 81.56\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been a member of the state 's government , which is the most important thing to do with the . '' . ' '' said . ' I 'm not going to do . '' 's said . ' I 'm not\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who has been a member of the House of Commons in #### . ' '' said the .### . ' '' said the statement . ' '' said the .##am . ' '' said . ' I 'm not going to do\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a huge amount of time . '' 's a . ' '' said . ' I 'm not going to do . '' 's said . ' I 'm not going to do . '' 's said . ' I 'm not going\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  27%|██▋       | 46892/172148 [39:03<6:57:11,  5.00it/s, loss=4.4631]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government has been a `` unacceptable '' of the `` <rare> '' of the `` <rare> '' of the `` <rare> '' of the `` <rare> '' of the `` <rare> '' of\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  28%|██▊       | 48452/172148 [40:20<1:40:04, 20.60it/s, loss=4.4368]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4254\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.46it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3907, Perplexity: 80.69\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow , who has been charged with the death of the U.S. military . ' '' said the .##m . ' '' said the .##am . ' '' said the .##am . ' '' said the .##am . ' ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , who was a member of the National Institute of Technology , and the U.S. military . '' 's said . ' I 'm not going to be a very good job . ' '' said the .##am . ' '' said\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane of the plane was found in the area . 's the .### . 'S. . ' '' said the .##m . ' '' said the .##m . ' '' said the .##am . ' '' said the .##\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  28%|██▊       | 48457/172148 [40:22<5:48:55,  5.91it/s, loss=4.3742]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government would be `` a very serious '' of the case . ' '' said the .##am . ' '' said the .##am . ' '' said the .##am . ' '' said the .##am . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  29%|██▉       | 50014/172148 [41:38<1:43:30, 19.66it/s, loss=4.3985]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4214\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:46, 33.08it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3863, Perplexity: 80.35\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of the murder of the ##-year-old , who was arrested on suspicion of murder and was arrested on suspicion of murder . ' ' I 'm not going to do . ' '' said the .### . '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who has been accused of the murder of the ##-year-old , who was arrested on suspicion of murder and was arrested on suspicion of murder . ' ' I 'm not going to do . ' '' said the .\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is also known to be a popularity of the world 's most popular destination for the world . '' 's said . '' 's a great-grandmother . ' ) . '' 's a great man . ' '' said the .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  29%|██▉       | 50019/172148 [41:40<5:34:40,  6.08it/s, loss=4.3058]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been accused of being a member of the country 's most recent history . '' . 'Fort Office said . '' 's a very good thing . ' '' said . ' I 'm not going to do . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  30%|██▉       | 51577/172148 [42:56<1:35:48, 20.97it/s, loss=4.4827]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4175\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:45, 33.23it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3941, Perplexity: 80.98\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a member of the country 's most recent years . '' 's said . ' '' said the . ' I 'm not going to do . '' 's said . ' I 'm not going to be a . ' I '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , the chief executive of the U.S. Army , said : 'The police are not going to be a very good guy . '' 's said . ' I 'm not going to be a . ' I 'm not\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane of the attack is the largest of the world , and the country is the most common . '' . ' '' said . ' I 'm not going to be a . '' . ' '' said . ' I 'm not going to be a .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  30%|██▉       | 51583/172148 [42:58<5:04:59,  6.59it/s, loss=4.5257]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been charged with the murder of the . 'Fort Office said . ' I 'm not going to be a . '' . ' '' said . ' I 'm not going to be a . ' I 'm not going\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  31%|███       | 53141/172148 [44:14<1:35:27, 20.78it/s, loss=4.4079]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4105\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:45, 33.16it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3859, Perplexity: 80.31\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a member of the country 's government . ' '' said the company . 'Fox . ' '' said the .### . 'Fort . '' 's said . ' I 'm not going to do . ' '' said\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City manager , who has been charged with murder and attempted murder . ' ' I 'm not going to do . ' '' said the .### . 'Fort . '' 's said . ' I 'm not going to do . '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is to be a great place in the world . '' 's said . ' '' said the .### . 'Fort . '' 's said . ' I 'm not going to do . ' '' said the .### . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  31%|███       | 53144/172148 [44:15<6:31:16,  5.07it/s, loss=4.3846]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been charged with murder . ' '' said the .### . 'Fort . '' 's said . ' I 'm not going to do . ' '' said the .### . 'Fort . '' 's\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  32%|███▏      | 54702/172148 [45:31<1:31:37, 21.37it/s, loss=4.4164]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4119\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:44, 33.46it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3771, Perplexity: 79.61\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of the death penalty . ' '' said the .### . ' '' said the company . 's . ' '' said the company . 's . ' '' said the company . 's . ' '' said the company . '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who has been charged with murdering a child , and the family said . ' I 'm not going to be able to get the best friend . '' 's a .### . ' '' said the company . 's .\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was found in the area , which was found in the area , which was found in the area , which was found in the area , which was found in the area . 's the .### . ' '' said the company . 's .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  32%|███▏      | 54708/172148 [45:33<4:57:32,  6.58it/s, loss=4.2807]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's decision to be a `` tough '' of the president 's decision to be a good job . '' 's not . ' '' said the .### . ' '' said the .### . ' '' said the company . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  33%|███▎      | 56267/172148 [46:49<1:33:20, 20.69it/s, loss=4.3546]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4035\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:49, 31.92it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3719, Perplexity: 79.19\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with murder and a ##-year-old man was shot dead . ' I was . ' I 'm not going to be a very good job . ' '' said the .### . 'S . ' '' said . '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City : The ##-year-old was arrested in connection with the death of the death of the .### . 'S. . ' '' said the company said . ' I 'm not going to be a very good job . ' ''\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a very difficult time for the future . '' 's official said . ' I 'm not going to be a very good job . ' '' said the .### . 'S . ' '' said . ' I 'm not going to\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  33%|███▎      | 56270/172148 [46:50<6:28:04,  4.98it/s, loss=4.4099]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been charged with murder and a ##-year-old man was shot dead . ' I was . ' I 'm not going to be a very good job . ' '' said the .### . 'S . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  34%|███▎      | 57830/172148 [48:06<1:31:26, 20.84it/s, loss=4.3800]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4031\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:51, 31.61it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3674, Perplexity: 78.84\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major role in the attacking half . ' '' said the .### . '' #### #### . ) . '' said . '' 's a .### . '' said . '' 'Suchy said . '' 's a\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who has been in the UK , and the United States has been in the country . ' '' said the .### . '' #### #### . ) . '' said . '' 's a .### . '' said . '' '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a major problem , but the government has been in the country . '' 's a . '' said . '' said . '' 's a .### . '' said . '' 'Suchy said . '' 's a . '' said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  34%|███▎      | 57835/172148 [48:08<5:21:12,  5.93it/s, loss=4.3439]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's lawyers have been in the country 's largest city in the city of the city of the city . 'Fort . '' 's most . '' said the .### . '' #### #### . ) . '' said . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  34%|███▍      | 59391/172148 [49:23<1:29:18, 21.04it/s, loss=4.4734]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.4007\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.56it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3672, Perplexity: 78.82\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's ##-year-old , who was a ##-year-old son of a ##-year-old girl , who was a ##-year-old son of a woman who was a ##-year-old son of a woman\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who has been charged with murder and a police officer who was a `` tough '' of the `` unacceptable '' of the `` unacceptable '' of the `` unacceptable '' of the `` American '' . '' . '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is expected to be a third-party agreement with the United States . '' 's said . 'Fort . '' said the company . 's . '' said the company . 's . '' said the company . 's . '' said the\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  35%|███▍      | 59397/172148 [49:25<4:47:20,  6.54it/s, loss=4.3757]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been charged with a ##-year-old , who was a `` tougher '' to be a `` tough '' of the `` unacceptable '' of the `` American '' . '' . 'Fort . '' said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  35%|███▌      | 60954/172148 [50:41<1:28:17, 20.99it/s, loss=4.3208]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3974\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.79it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3711, Perplexity: 79.13\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow is not the first time in the UK . ' '' said the .### . ' '' said the .### . ' '' said the .### . ' '' said the .### . ' '' said the .### . '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City : The ##-year-old was arrested on suspicion of murder and was arrested . ' I was a very good friend . '' 's said . '' said the .##pm . ' '' said . '' said . '' said the .\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the world . '' 's said . '' said the .### . ' '' said the .### . ' '' said the .### . ' '' said the .### . ' '' said the .##\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  35%|███▌      | 60960/172148 [50:43<4:43:38,  6.53it/s, loss=4.4582]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's lawyers are not the first time in the UK . ' '' said the .### . ' '' said the .### . ' '' said the .### . ' '' said the .### . ' '' said the\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  36%|███▋      | 62517/172148 [51:59<1:33:09, 19.62it/s, loss=4.5049]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3918\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.59it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3683, Perplexity: 78.91\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` significant step '' for the first time in the case . ' '' said the .### . '' said . ' I 'm not going to do . '' 's home . ' '' said the .### . ' ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's first-half-year-old was arrested in the attack , and the ##-year-old was arrested in the attack . ' I 'm not going to be a good job . '' 's home . ' '' said the\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the largest , the largest city of the world . '' 's official said . '' said the .### . '' . 'S. . '' . 'S. . ' '' said . ' I 'm not going to do . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  36%|███▋      | 62523/172148 [52:01<4:40:59,  6.50it/s, loss=4.2597]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's lawyers said the government has been `` not the most important thing '' . ' '' said the .### . '' said . ' I 'm not going to do . '' 's home . ' '' said the .###\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  37%|███▋      | 64082/172148 [53:17<1:31:14, 19.74it/s, loss=4.4122]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3873\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:50, 31.76it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3658, Perplexity: 78.72\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` great example of the issue of the government , '' he said . 'We are not going to be a very good job . '' 's not . ' '' said the .### . 'S.D . 'S.\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's first-half-year-old was a ##-year-old man who was a ##-year-old man , who was a ##-year-old man , who was a ##-year-old man , who was\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a major problem in the UK , and the UK 's economy . '' '' said . 'Fort . '' said . 'Fort . '' said . 'Fort . '' said . 'Fort . '' said . 'Fort\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  37%|███▋      | 64087/172148 [53:19<4:55:23,  6.10it/s, loss=4.4445]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been in the case , and the government has been in the case . ' '' said the company . 'Fort . '' said . 'Fort . '' said . 'Fort . '' said . 'Fort . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  38%|███▊      | 65645/172148 [54:35<1:26:46, 20.45it/s, loss=4.3692]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3863\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.41it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3605, Perplexity: 78.30\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been in the country 's largest city in the UK . ' '' said the .### . '' ) said . '' said the .### . '' . 'S ) . '' said the .### . 'S. . ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who has been in the country , but the government has not been able to use the money to be able to use the money . ' '' said the .### . 'S. . '' . 'S ) . '' said the\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is not the first time in the country . '' 's official said . '' said the .### . '' . 'S ) . '' said the .### . 'S. . '' . 'S ) . '' said the .##\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  38%|███▊      | 65648/172148 [54:36<5:55:09,  5.00it/s, loss=4.4611]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President was also charged with murder and a police officer . 's said . '' said the .### . 'S. . '' . 'S ) . '' said the .### . 'S. . '' . 'S ) . '' said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  39%|███▉      | 67207/172148 [55:54<1:27:21, 20.02it/s, loss=4.3660]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3825\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:49, 32.15it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3598, Perplexity: 78.24\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's first minister , who has been accused of being charged with murder . ' ' I 'm not going to be able to get to the floor . ' '' said the .##pm . ' '' said . ' I 'm not going to\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who has been accused of being charged with murder . ' ' I 'm not going to be able to get to the floor . ' '' said the .##pm . ' '' said . ' I 'm not going to be able\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the area , and the ##-year-old was killed . ' '' said the .##pm . ' '' said the .##pm . ' '' said . ' I 'm not going to be able to get to\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  39%|███▉      | 67212/172148 [55:55<5:06:06,  5.71it/s, loss=4.4699]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the country 's government has been accused of being a `` very serious issue '' . ' '' said the .##pm . ' '' said . ' I 'm not going to be able to get to the floor . ' '' said the .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  40%|███▉      | 68771/172148 [57:12<1:26:42, 19.87it/s, loss=4.5108]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3829\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:52, 31.27it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3553, Perplexity: 77.89\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has said the government has not yet been released . ' '' said the .##pm . ' '' said . ' I 'm not going to do . ' '' said the .##pm . ' '' said . ' I 'm\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's first-half years ago , he said : ' I 'm not going to be a little bit . ' '' said the .##-caliber pistol . ' I 'm not going to be a little bit . ' ''\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is to be a huge amount of time . ' '' said the .##-caliber gunfire . ' '' said the .##-caliber gunfire . ' '' said the .##-caliber gunfire . ' '' said the .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  40%|███▉      | 68776/172148 [57:14<4:39:53,  6.16it/s, loss=4.4126]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President was a ##-year-old woman who was a ##-year-old woman , who was arrested in #### , was arrested in connection with the death of the ##-year-old woman who was arrested in the ##-year-old ,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  41%|████      | 70332/172148 [58:31<1:22:10, 20.65it/s, loss=4.3449]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3789\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:50, 31.65it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3459, Perplexity: 77.16\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` significant '' of the country 's most recent incidents , including the <rare> , and the most important thing . '' said . '' said . ' I said . '' said . ' I said . ' I think it '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council has been accused of being a `` significant '' of the `` significant '' of the `` <rare> '' of the `` <rare> '' of the `` <rare> '' of the `` <rare> '' of the `` <rare>\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the area , and the ##-year-old was found in the area . 's said . ' I said . '' said . ' I said . ' I think it 's a very good thing . '' 's\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  41%|████      | 70338/172148 [58:33<4:17:48,  6.58it/s, loss=4.3671]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President was a `` significant '' of the `` significant '' of the `` <rare> '' of the `` <rare> '' of the `` <rare> '' of the `` <rare> '' of the `` <rare> '' of the `` <rare\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  42%|████▏     | 71896/172148 [59:49<1:21:05, 20.60it/s, loss=4.3304]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3780\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.49it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3457, Perplexity: 77.14\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major role in the country 's largest city . '' 's official . '' said . '' said the .### .### .### .### . ) . '' 's a .### .### .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-# draw with the goalkeeper , but the club is a good job . ' '' said the .### .### .### .### . ) . '' 's a .### .### .\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time the world 's largest city of the country . '' 's a .### .### .### .### . ) . '' 's a .### .### .### .### .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  42%|████▏     | 71902/172148 [59:51<4:15:51,  6.53it/s, loss=4.3059]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office is a `` good-class '' and `` unacceptable '' of the `` <rare> '' -- the .### . '' said . '' said the .### .### .### .### . ) .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  43%|████▎     | 73460/172148 [1:01:08<1:21:17, 20.23it/s, loss=4.3886]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3763\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:46, 32.91it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3463, Perplexity: 77.19\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of the attackers . ' '' said the incident . ' '' said the incident . ' '' said the incident . ' '' said the incident . ' '' said the incident . ' '' said the incident . ' '' said the incident .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , the ##-year-old , was a former president of the United States . ' '' said the . 'Following of the . ' '' said the . 'Following of the . ' '' said the . 'Follow\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is expected to be a major role in the first time . '' 's a . 'Furbart , '' he said . ' '' said the . 'Following of the incident . ' '' said the . 'Following of\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  43%|████▎     | 73463/172148 [1:01:10<5:27:31,  5.02it/s, loss=4.2916]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been accused of the attackers . ' '' said the incident . ' '' said the incident . ' '' said the incident . ' '' said the incident . ' '' said the incident . ' '' said the incident . ' '' said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  44%|████▎     | 75022/172148 [1:02:27<1:19:00, 20.49it/s, loss=4.3844]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3729\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:11, 26.63it/s]\u001b[A\n",
            "Evaluating:   0%|          | 7/3514 [00:00<01:56, 30.08it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3500, Perplexity: 77.48\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been criticised for the first time in the country 's government . ' '' said the .### . '' ) said . ' '' said the .### . '' ) said . ' '' said the .### . '' ) said\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council has been criticised for the first time in the case . ' '' said the .### . '' ) said . ' '' said the .### . '' ) said . ' '' said the .### . '' ) said .\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a major threat to the country . ' '' said the .### . '' ) said . ' '' said the .### . '' ) said . ' '' said the .### . '' ) said . ' '' said the .##\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  44%|████▎     | 75028/172148 [1:02:29<4:06:43,  6.56it/s, loss=4.5674]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been criticised for the first time in the case . ' '' said the .### . '' ) said . ' '' said the .### . '' ) said . ' '' said the .### . '' ) said . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  44%|████▍     | 76586/172148 [1:03:45<1:21:39, 19.51it/s, loss=4.2767]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3711\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:54, 30.76it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3427, Perplexity: 76.92\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major factor in the United States , and the United States , the United States , and the United States , and the United States , the United States , said the government has been in the UK . ' '' said . ' I 'm\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's first-ever-minute video shows the video of the video . ' I 'm not going to be a very good friend . ' '' said . ' I 'm not going to be a very good friend . ' '' said\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the world , and the United States is the largest of the world 's largest cities . '' . ) . '' 's a .### .### .### .### .### . ) . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  44%|████▍     | 76590/172148 [1:03:47<5:12:35,  5.10it/s, loss=4.4532]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. government is not the first time in the United States , and the United States , the United States , and the United States , and the United States , the United States , said the government has been in the UK . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  45%|████▌     | 78148/172148 [1:05:04<1:19:53, 19.61it/s, loss=4.3380]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3747\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:49, 32.14it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3410, Perplexity: 76.79\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's office said the government 's decision to make a new deal of the new rules . ' '' said the company 's website . 'F . ) . ' I 'm not going to do . ' '' said the .### .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the ##-year-old was arrested in #### , but he was arrested in #### . ' I was a very good man . ' '' said the .### . 'Fort . '' said . ' I think it 's\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the UK , which is the largest of the largest population of ## % of the population . '' 's official . 'F.C. , . 'Fort . '' said . 'Fort . '' said . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  45%|████▌     | 78153/172148 [1:05:06<4:29:42,  5.81it/s, loss=4.4432]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been a `` very difficult time '' . ' '' said the .### . 'Fort . '' said . ) . ' I 'm not going to do . ' '' said the .### . 'Fort .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  46%|████▋     | 79712/172148 [1:06:23<1:20:55, 19.04it/s, loss=4.3885]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3651\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:49, 32.03it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3445, Perplexity: 77.05\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been criticised for the first time in the ####s . ' '' he said . ' I 'm not going to be a little bit . ' '' said the .##pm . '' ) said . ' I 'm not\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who has been a major role in the United States . ' '' said the .### . ' '' said the .### . 'Fort . '' said the .### . 'Fort . '' said the .##\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is to be seen in the city of the city . ' '' said the .### . ' '' said the .### . 'Fort . '' said the .### . 'Fort . '' said the .### . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  46%|████▋     | 79717/172148 [1:06:24<4:22:19,  5.87it/s, loss=4.3442]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been criticised for the `` austerity '' of the `` <rare> '' . ' '' said the .### . ' '' said the .### . 'Fort . '' said the .### . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  47%|████▋     | 81274/172148 [1:07:41<1:14:24, 20.35it/s, loss=4.3220]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3678\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:52, 31.34it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3365, Perplexity: 76.44\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been criticised for the first time in the ####s . '' said the .### . '' said . '' said the .##pm . '' said . '' said the .##pm . '' said . '' said the .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's first lady , who was a ##-year-old girl who was a ##-year-old girl , who was arrested in #### , was arrested in connection with the death of the death of the .##am . '' said .\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the United States . 's the same time . '' said the .### . '' said . '' said the .##pm . '' said . '' said the .##pm . '' said . '' said the .##\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  47%|████▋     | 81279/172148 [1:07:43<4:13:44,  5.97it/s, loss=4.3751]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government would not be allowed to pay for the money . ' '' said the .### . '' said . '' said the .##pm . '' said . '' said the .##pm . '' said . '' said the\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  48%|████▊     | 82837/172148 [1:09:00<1:12:36, 20.50it/s, loss=4.3338]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3635\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:50, 31.82it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3440, Perplexity: 77.02\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` significant step in the country '' . ' '' said the .##-caliber gun . ' '' said the .##-caliber gun . ' '' said the .##-## . 'Fort . '' said . '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , ## , was arrested and charged with murder and a police officer . ' I was a very good person . ' '' said . ' I was a very good person . ' '' said . ' I 'm not sure . ' '' said the\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is to be a major problem . '' 's a . 'Fort . '' said . 'Following . '' said the .##-caliber gun . ' '' said the .##-caliber gun . ' '' said the .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  48%|████▊     | 82842/172148 [1:09:01<4:06:17,  6.04it/s, loss=4.3178]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been a `` significant step in the country '' . ' '' said the .##-caliber gun . ' '' said the .##-caliber gun . ' '' said the .##-## . 'Fort . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  49%|████▉     | 84399/172148 [1:10:18<1:13:49, 19.81it/s, loss=4.4841]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3607\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<01:58, 29.70it/s]\u001b[A\n",
            "Evaluating:   0%|          | 7/3514 [00:00<01:52, 31.18it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3337, Perplexity: 76.22\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with murder and a criminal investigation . ' '' said the .### .### ) . '' ) said . ' I 'm not going to do . ' '' said the .### .### ) . '' ) said\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the ##-year-old was arrested in the hospital , and the police were found in the area . ' '' said the .### . ' '' said the .### . 'F ) , '' he said . ' ''\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the country , and the country 's most recent cases were reported . ' '' said the .### . '' ) said . ' I 'm not going to do . ' '' said the .### .###\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  49%|████▉     | 84404/172148 [1:10:20<4:20:24,  5.62it/s, loss=4.3879]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been charged with murder and a criminal investigation . ' '' said the .### .### ) . '' ) said . ' I 'm not going to do . ' '' said the .### .### ) . '' ) said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  50%|████▉     | 85964/172148 [1:11:37<1:10:47, 20.29it/s, loss=4.3207]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3601\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:03, 28.39it/s]\u001b[A\n",
            "Evaluating:   0%|          | 7/3514 [00:00<01:55, 30.28it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3315, Perplexity: 76.06\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of the alleged attack on the death of the ##-year-old . 'Forta . '' said . 'Furbart , which is the first time , the first time , the president 's office said the\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's ##-year-old , who was a former presidential candidate , who has been accused of being a `` very serious '' of the `` terrorist '' of the Islamic State . '' . 'Forta said . ' I\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the ####s , which is the first time of the ####s , which is the first time of the ####s , which is the first time in the ####s , which is the first time in the ####s , which\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  50%|████▉     | 85969/172148 [1:11:39<4:03:56,  5.89it/s, loss=4.3711]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. military has been accused of the attack , which is the first of the most recent years , the president said . ' I 'm not going to be able to get the money . ' '' said the . 'Science\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  51%|█████     | 87525/172148 [1:12:55<1:10:13, 20.08it/s, loss=4.3941]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3601\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:54, 30.65it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3288, Perplexity: 75.85\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been a `` very important thing '' . ' '' said the .### . ' '' said the .### . ' '' said the .### . ' '' said the .### . ' '' said the .##\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-# win over the World Cup . 's win . '' 's the World Cup . ' '' said the .### . ' '' said the .### . ' '' said the .### . ' '' said\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the UK , and the ##-year-old has been in the UK . '##s . '' 's the World Cup . ' '' said the .### . ' '' said the .### . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  51%|█████     | 87531/172148 [1:12:56<3:39:40,  6.42it/s, loss=4.4521]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. Embassy in the U.S. Embassy in the U.S. Embassy in the U.S. Embassy in the U.S. Embassy in the U.S\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  52%|█████▏    | 89089/172148 [1:14:13<1:07:58, 20.36it/s, loss=4.3759]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3584\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.50it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3299, Perplexity: 75.94\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's first-half-year-old was a ##-year-old , who has been a ##-year-old , who has been a ##-year-old , who has been a ##-year-old , who has been\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's ##-year-old , who has been a ##-year-old , who has been a ##-year-old , who has been a ##-year-old , who has been a ##-year-old , who\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane , the U.S. military , and the U.S. military has been killed in the attack . '## . '' said . 'Bothers and the .### . '' ) said . ' I 'm not going to be\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  52%|█████▏    | 89094/172148 [1:14:15<3:50:07,  6.02it/s, loss=4.3973]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been criticised for the first time in the UK . '## . '' said . 'Bothers and the .### . '' ) said . ' I 'm not going to be a good job . ' '' said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  53%|█████▎    | 90651/172148 [1:15:32<1:08:26, 19.84it/s, loss=4.3013]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3566\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:01, 28.79it/s]\u001b[A\n",
            "Evaluating:   0%|          | 7/3514 [00:00<01:55, 30.48it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3224, Perplexity: 75.37\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's first-largest city of the country is the first time in the country . '' 's a .##-caliber and a .##-caliber and a bit of a good day . '' ' I said . '' ' I\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-#-# win over the Champions League . '' ' I 'm not going to be a good player . '' 's a .##-caliber and a bit of a good day . '' ' I said . ''\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time the attack was a `` very serious '' and the case . ' '' said the incident . ' '' said the incident . ' I was a very good friend . '' 's a .##-caliber handgun . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  53%|█████▎    | 90657/172148 [1:15:34<3:46:02,  6.01it/s, loss=4.4102]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the country 's largest group of the country 's largest city of the country is the first time . '' 's a .##-caliber and a .##-caliber and a bit of a good day . '' ' I said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  54%|█████▎    | 92216/172148 [1:16:51<1:05:38, 20.29it/s, loss=4.3260]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3561\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.23it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3267, Perplexity: 75.70\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been a `` very difficult time '' . ' '' said the .### . '' ) said . '' 's a .##-caliber and right footed shot from the right side of the box is saved in the bottom\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-#-# win over the World Cup in #### . ' '' said the .### . '' ) . ' '' said the .### . '' ) . 'Fort . '' 's office said . '' said\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the city of the city . '' 's office said . '' said . 'Fort . '' said . 'Fort . '' said . 'Fort . '' said . 'Fort . '' said . 'F\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  54%|█████▎    | 92219/172148 [1:16:53<4:27:43,  4.98it/s, loss=4.4005]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office is a `` very difficult time '' . ' '' said the .### . '' ) said . '' 's a .##-caliber and right footed shot from the right side of the box is saved in the bottom right\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  54%|█████▍    | 93777/172148 [1:18:10<1:03:09, 20.68it/s, loss=4.3673]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3531\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:51, 31.44it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3332, Perplexity: 76.19\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been in the midst of a `` significant impact on the ground '' . ' '' said the .### . '' said . '' said . '' said . '' said . '' said . '' said . '' said . '' said .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who has been in the UK , and the US government has been given a number of people to pay for the UK . ' '' said the .### . '' said . '' said . '' said . '' said . '' said .\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the ####s . '' #### #### #### . '' #### #### #### . '' ) said . '' 's a source of the <rare> . '' said . '' said . '' said . '' said . '' said . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  54%|█████▍    | 93782/172148 [1:18:12<3:34:54,  6.08it/s, loss=4.2236]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office is a `` significant '' of the `` <rare> '' -- a `` <rare> '' -- a `` <rare> '' -- a `` <rare> '' -- a `` <rare> '' -- a `` <rare> ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  55%|█████▌    | 95342/172148 [1:19:28<1:02:05, 20.62it/s, loss=4.2964]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3505\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:49, 32.13it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3276, Perplexity: 75.76\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been in the UK to be a `` significant '' of the country 's most recent financial crisis . '' 's a .### , '' he said . ' I 'm sure it 's a good thing . '' '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , ## , #### , and the ##-year-old woman was found dead in the ##th minute , and the ##-year-old was found dead in the attack . ' I was in the middle of the day . ' '' said\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane , the ##-year-old , who was killed in the attack , and the police said . ' I 'm not sure . ' '' said the .### . '' said . 'Farms said . ' '' said the .##\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  55%|█████▌    | 95345/172148 [1:19:30<4:16:58,  4.98it/s, loss=4.2796]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. government has been in the country 's most recent financial crisis . '' 's a .### , '' he said . ' I 'm sure it 's a good thing . '' 's a .###\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  56%|█████▋    | 96904/172148 [1:20:47<1:02:09, 20.18it/s, loss=4.3469]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3521\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.50it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3213, Perplexity: 75.29\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with murder and was not a member of the family . ' '' said the .### . '' said . 'Furbart , '' said the .### . '' said . ) . '' 's a .###\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the ##-year-old was killed in the attack . '##s . '' 's a .### . '' said . ) . '' 's a .### . '' said . 'Furbart , ''\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the UK . '##s . '' said the .### . '' said . ) . '' 's a .### . '' said . 'Furbart , '' said the .### . '' said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  56%|█████▋    | 96910/172148 [1:20:49<3:08:04,  6.67it/s, loss=4.2564]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the UK 's most recent report , which is the first time in the UK , the UK government said . '### . '' said . 'Furbart , '' said the .### . '' said . ) . '' '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  57%|█████▋    | 98467/172148 [1:22:04<1:00:13, 20.39it/s, loss=4.2648]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3502\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:46, 32.88it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3235, Perplexity: 75.45\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's official statement said : 'The ICC 's decision is not the first time . ' '' said the .### . '' said . 'Fort . '' 's official website . 'Five . 'Fort . '' said\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City officials said the ##-year-old was arrested in connection with the death of the ##-year-old . ' I was a very good friend . ' '' said the .### . ' I 'm not going to do . '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the country . ' '' said the .### . '' said . 'Fort . '' said . 'Five . ' '' said the .### . ' '' said the .### . ' '' said .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  57%|█████▋    | 98473/172148 [1:22:06<3:10:58,  6.43it/s, loss=4.3644]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government 's `` unacceptable '' of the `` unacceptable '' of the case is not the first time . ' '' said the .### . '' said . 'Fort . '' 's official website .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  58%|█████▊    | 100030/172148 [1:23:22<58:38, 20.50it/s, loss=4.2603]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3464\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:45, 33.22it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3186, Perplexity: 75.08\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with murder and murder , and the ##-year-old man was arrested in connection with the incident . ' I was a man . ' '' said the .### . '' said . ' I 'm not sure . ' ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who have been charged with murder and murder , and the ##-year-old man was arrested in connection with the incident . ' I was a man . ' '' said the .### . '' said . ' I 'm not\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the UK , and the ##-year-old is the most expensive . '' #### .### ) # .### . '' ) said . '' ' I 'm not sure . ' '' he said . ' I\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  58%|█████▊    | 100033/172148 [1:23:24<3:58:04,  5.05it/s, loss=4.4058]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been criticised for the first time in the UK . ' '' said the company 's `` <rare> '' -- a . '' said . ' I 'm not sure . ' '' said the .### . '' said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  59%|█████▉    | 101592/172148 [1:24:41<57:04, 20.60it/s, loss=4.3285]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3461\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:49, 32.11it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3133, Perplexity: 74.69\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` significant step in the way '' . '' 's the news . ' ) . '' ' I 'm not going to be a good person . '' 's funny . ' ) . '' ' I 'm not going to\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who has been a `` good idea '' . ' '' he said . ' I 'm not going to be a bit of a bit of a bit of a bit of a bit of a bit of a bit of a bit of a\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the city of the city of the city . 'Fifseyside . '' ) said . '' 's a source . ' ) . '' 's a very high school . '' ) . ' I 'm not\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  59%|█████▉    | 101598/172148 [1:24:42<3:03:25,  6.41it/s, loss=4.4346]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been a `` significant step in the way '' . '' 's the news . ' ) . '' ' I 'm not going to be a good person . '' 's funny . ' ) . '' ' I 'm not going to\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  60%|█████▉    | 103155/172148 [1:25:59<56:18, 20.42it/s, loss=4.3693]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3509\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.22it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3144, Perplexity: 74.77\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very important step '' . ' '' said the .### . '' ) . '' ) . ' I 'm not going to be able to get the best wishes . '' ' I 'm not going to be able to get\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council has been charged with the murder of the murder of the murder of the . ' I 'm not going to be able to get the chance to get a lot of people . '' 's naked . ' I 'm not going to\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the ####s . '' #### . ) # .### ) # . '' ) . '' ) . '' ) . ' I 'm not going to be able to get the chance to get a lot of money . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  60%|█████▉    | 103161/172148 [1:26:01<2:58:08,  6.45it/s, loss=4.3607]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. military , the U.S. military , the U.S. military , the U.S. military , and the U.S. military . '' ) said . '' 's a . '' ) said .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  61%|██████    | 104719/172148 [1:27:18<54:49, 20.50it/s, loss=4.2933]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3441\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.53it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3180, Perplexity: 75.04\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been in the same way of the .### . '' ) said . '' 's a big part of the game . '' 's nudge . ' '' he said . ' I 'm not going to be a good friend .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York 's ##-year-old , who was a ##-year-old man , who was a ##-year-old man , who was arrested in the hospital , and the police were killed . ' I 'm not going to be a\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was found in the city of the city . 'Farms said . 's a lot of people . '' 's a very high school . '' ) . ' I 'm not going to be a big . '' ) . ' I '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  61%|██████    | 104725/172148 [1:27:20<2:53:11,  6.49it/s, loss=4.2774]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been criticised by the government 's office . ' '' said the company 's chief executive of the United States . 's the same time . '' 's not . ' '' said the .##-caliber pig\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  62%|██████▏   | 106282/172148 [1:28:37<54:32, 20.13it/s, loss=4.2497]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3437\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.76it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3153, Perplexity: 74.84\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very serious '' of the `` terrorist '' of the `` terrorist '' of the `` terrorist '' of the `` terrorist '' of the `` terrorist '' of the `` terrorist '' of the `` terrorist '' of\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's first-half-year-old was shot in the attacking half . ' I was a bit of a good friend . ' '' said the .### . ' '' said . ' I 'm not sure what I 'm\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the first time , the first time in the world 's top-flight match was a first-half-year-old 's first goal of the goal . ' I was going to be a bit of a good\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  62%|██████▏   | 106288/172148 [1:28:39<2:49:10,  6.49it/s, loss=4.2938]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been accused of being a `` very serious '' of the `` terrorist '' of the `` terrorist '' of the `` terrorist '' of the `` terrorist '' of the `` terrorist '' of the `` terrorist '' of\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  63%|██████▎   | 107845/172148 [1:29:56<53:23, 20.08it/s, loss=4.3404]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3450\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.22it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3130, Perplexity: 74.66\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with conspiracy to commit suicide . ' . ' '' she said . ' I 'm not going to be a very good thing . '' ' I said . '' ' I 'm not sure . ' '' he said .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who has been charged with conspiracy to commit suicide . ' . ' '' she said . ' I 'm not going to be a very good thing . '' ' I said . '' ' I 'm not sure . '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time of the crash , the police said . ' I 'm not going to be a very good thing . '' ' I said . '' ' I 'm not sure . ' '' he said . ' I 'm not going to\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  63%|██████▎   | 107851/172148 [1:29:58<2:53:48,  6.17it/s, loss=4.3235]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. military has been charged with conspiracy to commit suicide . ' . ' '' said the company 's name . ' '' said the company 's name . ' '' said the company 's name . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  64%|██████▎   | 109407/172148 [1:31:14<51:20, 20.36it/s, loss=4.1687]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3431\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:50, 31.77it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3025, Perplexity: 73.88\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been criticised for the first time in the ####s . ' '' said the .### . '' ) . '' ) . '' ' I said . '' ' I said . '' ' I was a very good friend . '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who has been charged with murdering a ##-year-old , who was arrested in the hospital , where he was arrested in the hospital . ' I was a very good man . '' ' I said . '' ' I was a\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a very difficult time for the future . '' 's time . '' ' I said . '' ' I said . '' ' I was a very good friend . ' '' said the .### . '' ) . ' I 'm not going\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  64%|██████▎   | 109413/172148 [1:31:16<2:42:44,  6.42it/s, loss=4.3373]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the country 's largest cities are expected to be a major boost for the country . ' '' said the company 's website . ) . '' ) . '' ' I 'm not going to be a good thing . '' ' I said .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  64%|██████▍   | 110970/172148 [1:32:33<50:40, 20.12it/s, loss=4.2438]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3389\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:02, 28.70it/s]\u001b[A\n",
            "Evaluating:   0%|          | 7/3514 [00:00<01:55, 30.39it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3107, Perplexity: 74.49\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very serious issue '' of the `` <rare> '' of the `` <rare> '' of the `` <rare> '' of the `` <rare> '' of the `` <rare> '' of the `` <rare> ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , the ##-year-old , who was born in the village of the city , and the same . ' '' said the .### . '' ) . '' ' I 've got a lot of people . ' '' said the .\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the same as the first time in the world . '' 's not . '' said . '' ' I think it 's a good thing . '' ' I 've got a lot of money . ' '' said the .### . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  64%|██████▍   | 110975/172148 [1:32:35<3:07:24,  5.44it/s, loss=4.1974]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. military has been a `` very serious '' of the government 's government . '' 's not clear whether the government is not the first time . '' 's not clear whether the government is not the first time . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  65%|██████▌   | 112534/172148 [1:33:52<49:38, 20.02it/s, loss=4.3836]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3408\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.32it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3095, Perplexity: 74.40\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow , the U.S. military said the government would not be able to use the new rules . ' '' said the .##-caliber rifle . ' '' said the .##-caliber rifle . ' I was a man who\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the ##-year-old was arrested in connection with the death of the ##-year-old man , who was arrested in connection with the death of the ##-year-old . ' I was a man who was a man\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the area , the U.S. military said . 's a lot of people . '' 's a time . ' '' said the .##pm . '' ) . ' I 'm a man who was a man\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  65%|██████▌   | 112539/172148 [1:33:54<2:46:08,  5.98it/s, loss=4.3834]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. military has been arrested in connection with the deaths of the U.S. and the U.S. military . '' 's the court heard . ' I 'm not going to be a man . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  66%|██████▋   | 114098/172148 [1:35:11<46:46, 20.68it/s, loss=4.3274]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3418\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.53it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3198, Perplexity: 75.17\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's official statement said : 'The other was a very good idea , '' he said . ' I 'm not going to be a very good friend . ' '' said the .##-caliber gun . ' '' he said . ' I\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City : The ##-year-old was a ##-year-old girl who was a ##-year-old girl , who was a member of the family , who was a member of the family , who was a member of the family ,\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the ####s , and the ##-year-old was shot dead . ' I was a very good man . ' '' said the .##-caliber gun . ' I 'm not going to be a very good\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  66%|██████▋   | 114101/172148 [1:35:12<3:12:32,  5.02it/s, loss=4.2517]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been a `` very serious '' of the country 's government to be a `` very good '' and a `` very good '' and a `` very good '' and a `` very good '' and a `` very good '' and a `` very good ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  67%|██████▋   | 115661/172148 [1:36:30<46:12, 20.37it/s, loss=4.3790]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3388\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.74it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3015, Perplexity: 73.81\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's presidential candidate , who is the first of the most popular presidential candidate . ' '' said the .### . '' ) . '' ' I 've got to be able to play . '' ' I 'm not going to be\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's ##-year-old , who was ## , and the ##-year-old was arrested on suspicion of murder . ' ' I 'm not going to be a very good . ' '' I was . ' '' she said\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the ####s , and the ##-year-old was in the top ## . ' I 've been a good player . ' '' he said . ' I 'm not going to be able to play . '' '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  67%|██████▋   | 115664/172148 [1:36:31<3:05:12,  5.08it/s, loss=4.3209]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been charged with murder and a ##-year-old man . ' ' I 'm not going to be a very good . ' '' I 've been told . ' I 'm not going to be a very good player . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  68%|██████▊   | 117222/172148 [1:37:48<45:41, 20.04it/s, loss=4.3882]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3369\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:50, 31.80it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3054, Perplexity: 74.10\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's first ministerial candidate has been charged with the murder of the ####s . ' I said . '' said the .### . '' ) said . ' I 'm not going to be a very good thing . ' '' said the .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the ##-year-old was a member of the country 's largest city in the country . '' 's not . '' said . '' said . ' I said . '' said . ' I was going to be a very good\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the city of the city of the city . ' '' said the .### ) . '' said . ' '' said the .### ) . '' said . ' I 've got to be a very good thing .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  68%|██████▊   | 117228/172148 [1:37:50<2:22:08,  6.44it/s, loss=4.4126]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. military is a `` very difficult time '' to be a member of the country 's most recent . '' . ) # . '' ) said . ' I 'm not going to be a very good thing . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  69%|██████▉   | 118786/172148 [1:39:07<44:41, 19.90it/s, loss=4.3110]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3370\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:52, 31.11it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3036, Perplexity: 73.96\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very serious '' of the government 's decision . '' 's not . '' said . '' ) . '' said . '' ) . '' said . '' ) . '' said . '' said . '' ) . '' said . ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's ##-year-old was a ##-year-old man who was arrested in #### . ' I was a bit of a bit of a bit of a lot of time . '' ' I was a bit of a bit of a\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane season was a ##-year-old man . ' I was a bit of a bit of a bit of a lot of the time . '' ' I 'm not going to be a good job . ' '' said the .### . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  69%|██████▉   | 118791/172148 [1:39:09<2:40:18,  5.55it/s, loss=4.3990]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. military has been a `` very serious '' of the government 's decision . '' 's not . '' said . '' ) . '' said . '' ) . '' said . '' ) . '' said . '' said .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  70%|██████▉   | 120348/172148 [1:40:26<42:50, 20.15it/s, loss=4.3366]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3337\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:51, 31.41it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3014, Perplexity: 73.80\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's official statement said . '' ) said . '' ' I 'm not sure . ' '' he said . ' I 'm not sure . ' '' he said . ' I 'm not sure . ' '' she said . ' I '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-#-# #-# win over the ##th minute , but the goal is to be able to get the ball to the bottom of the table . '' ' I 'm not sure . '' ' I think it '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the world , and the ##-year-old was a 'tremont ' . '' ) said . ' I 'm not going to be a very good thing . ' '' said . ' I 'm not sure\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  70%|██████▉   | 120354/172148 [1:40:28<2:13:46,  6.45it/s, loss=4.3852]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the ##-year-old was a 'very untrue ' . ' '' said . ' I 'm not sure . ' '' he said . ' I 'm not sure . ' '' she said . ' I 'm\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  71%|███████   | 121913/172148 [1:41:45<41:35, 20.13it/s, loss=4.2976]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3319\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:50, 31.86it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2999, Perplexity: 73.69\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's official statement said . ' I 'm not going to be a very important thing . ' '' said . ' I 'm not going to be a very nice man . ' '' said . ' I 'm not going to be a very\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , which is the first time in the UK , which is the most common problem . ' '' said the company said . ' I 'm not going to be a very important thing . ' '' said . ' I 'm not going to\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the world , and the world 's largest <rare> , the company said . ' I 'm not going to be a very important thing . ' '' said . ' I 'm not going to be a very nice\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  71%|███████   | 121916/172148 [1:41:47<2:47:08,  5.01it/s, loss=4.3987]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government is not the only way to make a new deal . '' 's not the same way . '' ' I 'm not going to be a good friend . ' '' said . ' I 'm not going to be\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  72%|███████▏  | 123475/172148 [1:43:04<41:31, 19.53it/s, loss=4.2147]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3322\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:52, 31.08it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2978, Perplexity: 73.54\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very serious '' of the government 's decision to be a `` very serious '' of the government 's decision to be a `` very serious '' of the government 's decision to be a `` very serious '' of the government '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York , the ##-year-old , who was a member of the family , said the man was killed in the attack . ' '' said the .## . '' ) . '' ' I 've got a lot of people . '' 's a\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the city , and the ##-year-old was found in the city of the city . ' '' said the .## '' . ' '' said . '' ' I 'm a friend of the family . '' ' I\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  72%|███████▏  | 123480/172148 [1:43:06<2:27:50,  5.49it/s, loss=4.3273]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. government is not the first time in the country . '' 's not . '' said the .## '' . '' ) . '' ' I 've got a lot of people . '' 's a very good friend .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  73%|███████▎  | 125037/172148 [1:44:23<38:23, 20.45it/s, loss=4.3047]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3301\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:46, 32.92it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3017, Perplexity: 73.83\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's first-half-year-old , who was a ##-year-old man who was a ##-year-old girl who was a ##-year-old girl , who was born in #### , was found guilty of murdering\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , who have been arrested in connection with the death of the death of the .## . '' ) . '' ' I said . '' ' I 'm not sure what I 'm going to be a good person . ' '' said .\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane , the ##-year-old , who was born in the ####s , and the ##-year-old was born in #### . ' I 'm not sure what I 'm going to be a good player . '' ' I think I\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  73%|███████▎  | 125042/172148 [1:44:25<2:10:53,  6.00it/s, loss=4.3341]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office is not the first time in the world . '' 's report . '' ) said . '' 's a lot of people who are going to be a good person . '' 's n't . ' '' said . ' I\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  74%|███████▎  | 126602/172148 [1:45:42<37:34, 20.20it/s, loss=4.3640]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3347\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.69it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2953, Perplexity: 73.35\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's first-half-year-old was a ##-year-old man who was shot dead in the attack . ' I was a bit of a bad foul . ' '' he said . ' I 'm not going to be a\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said : 'The first time I was going to be a little bit of the way I 've got to be able to get the ball . '' ' I 'm not going to be a good player . ' '' he said . '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time in the world , and the most important thing to do . '' 's not the first time . ' '' he said . ' I 'm not going to be a little bit . ' '' he said . ' I 'm\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  74%|███████▎  | 126605/172148 [1:45:44<2:31:57,  5.00it/s, loss=4.4825]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. government is not the first time in the United States . '' 's not the first time . '' 's `` The Situation Room '' . ' '' he said . ' I 'm not going to be a little\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  74%|███████▍  | 128164/172148 [1:47:01<35:51, 20.44it/s, loss=4.2922]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3320\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:46, 33.06it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3186, Perplexity: 75.09\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with murder and was arrested on suspicion of murdering a ##-year-old girl . ' I was a very good man . ' '' said the .### . '' ) . ' I 'm not going to be a\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the `` <rare> '' is the first time , the company has been in the UK . '## . '' ' I 'm a fan of the world . '' ' I 'm a fan of the world . '' ' I\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane season , which is the first time in the ####s , and the ##-year-old is a member of the world 's largest in the world , and the most popular tourist group , which is the first time in the ####s ,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  74%|███████▍  | 128170/172148 [1:47:02<1:50:38,  6.62it/s, loss=4.2988]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been charged with murder and was arrested on suspicion of murdering a ##-year-old girl . ' I was a very good man . ' '' said the .### . '' ) . ' I 'm not going to be a\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  75%|███████▌  | 129727/172148 [1:48:20<34:37, 20.42it/s, loss=4.3756]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3311\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.23it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2982, Perplexity: 73.57\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's chief executive of the United Nations said the ##-year-old was killed in the attack . ' '' said the .### . '' ) . '' ) . '' ) . '' ' I 've got a lot of people . ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City officials said the ##-year-old was a `` very good '' and he was a very good friend . ' '' said the .### . '' ) . '' ) . '' ) . '' ' I 've got a lot of people\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the ####s , and the ##-year-old was found dead in the car . ' '' said the .### . '' ) . '' ) . '' ) . '' ' I 've got a lot of people\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  75%|███████▌  | 129733/172148 [1:48:22<1:49:09,  6.48it/s, loss=4.3863]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the country 's parliamentary committee will be able to provide a `` thorough investigation '' . '' 's report . '' ) . '' ' I told me . ' I 'm not sure what I 'm going to be a very good job\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  76%|███████▋  | 131289/172148 [1:49:39<33:33, 20.30it/s, loss=4.3057]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3251\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:50, 31.88it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2960, Perplexity: 73.41\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's office , which is the first of the most popular , and the most popularity of the world 's most popular destinations are the most popular to the world . '' 's most of the world . '' 's most of the world\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council has been criticised for the first time in the ####s , and the ##-year-old was a `` very good idea '' . ' '' he said . '' ' I think I 'm not going to be a very good friend\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the ####s , and the ##-year-old was a ##-year-old girl , who was arrested in #### , and the ##-year-old was arrested in connection with the death of the ##-year\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  76%|███████▋  | 131295/172148 [1:49:41<1:47:50,  6.31it/s, loss=4.4463]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been criticised for the first time in the ####s , and the ##-year-old was a `` very good idea '' . ' '' he said . '' ' I think I 'm not going to be a very good friend . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  77%|███████▋  | 132852/172148 [1:50:58<32:04, 20.41it/s, loss=4.2561]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3326\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<01:59, 29.40it/s]\u001b[A\n",
            "Evaluating:   0%|          | 7/3514 [00:00<01:52, 31.06it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2987, Perplexity: 73.60\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very good place '' in the country . '' 's narcotics , '' he said . '' ' I 'm not sure what 's going to happen . '' ' I 'm not sure what 's going to\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York , the ##-year-old , who has been a great player , and he has been a good player . ' '' he said . ' I 'm not going to be a good player . ' '' he said . ' I 'm not\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a very good idea . '' 's a great player . ' '' said the .### . '' ) . '' ' I 'm going to be a good player . ' '' said the .### . '' ) . '' ' I\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  77%|███████▋  | 132858/172148 [1:51:00<1:42:11,  6.41it/s, loss=4.3477]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been a `` very good idea '' to be a good idea . '' ' I 'm going to be a good player . ' '' said the .### . '' ) . '' ' I 'm going to be a good player . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  78%|███████▊  | 134415/172148 [1:52:17<31:17, 20.09it/s, loss=4.4530]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3265\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:12, 26.44it/s]\u001b[A\n",
            "Evaluating:   0%|          | 7/3514 [00:00<02:00, 29.13it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2989, Perplexity: 73.62\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with murder and a ##-year-old boy who was a `` very dangerous person '' . ' '' said the .### . '' ) . '' ' I said . ' I 'm not going to be a little bit .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-# win over the World Cup . ' I 've got to get back to the ground . '' ' I 'm sure he 's a good player . '' ' I think it 's a good player . ' ''\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a very difficult time for the first time . '' 's a lot of people . '' ' I think it 's a good thing . '' ' I said . ' I 'm not going to be a little bit . ' '' said the\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  78%|███████▊  | 134421/172148 [1:52:18<1:40:30,  6.26it/s, loss=4.2934]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been charged with the murder of the ##-year-old boy who was killed in the attack . ' I was a bit of a bad foul . ' '' said . ' I 'm not going to be a little bit\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  79%|███████▉  | 135978/172148 [1:53:35<30:18, 19.89it/s, loss=4.3501]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3274\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.31it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2953, Perplexity: 73.36\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with murder and a ##-year-old man who was arrested on suspicion of murder and was arrested on suspicion of murder . ' ' I was a very nice man . ' '' said . ' I was a very nice man\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's ##-year-old , who has been in the world , and the ##-year-old was born in #### . ' I was a very nice man . ' '' said the .### . '' ) . '' ' I\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a long-term solution , and the .### , '' said the .## '' of the #### World War . ' '' said . ' I 've been in the world . '' ' I said . ' I 'm not going to\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  79%|███████▉  | 135983/172148 [1:53:37<1:42:24,  5.89it/s, loss=4.3034]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the National Assembly , which is the most important thing for the country 's most recent . '' #### #### #### #### .### , #### . '' ) . '' said . ' I 've been in the world . '' ' I 'm\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  80%|███████▉  | 137543/172148 [1:54:54<28:26, 20.27it/s, loss=4.2844]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3285\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.42it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2956, Perplexity: 73.38\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very good '' of the country 's most recent .##-caliber rifle . ' '' said the .##pm . '' ) . '' said . ' I 've been able to get a good time . ' ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the ##-year-old was found guilty of murdering ##-year-old daughter , who was found guilty of murdering ##-year-old daughter , who was found guilty of murdering ##-year-old daughter ,\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is the first time the first time in the world is the most commonplace . ' '' said the .##-caliber rifle . ' '' said the .##pm . '' ) . '' said . ' I 've been able to get\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  80%|███████▉  | 137546/172148 [1:54:56<1:54:21,  5.04it/s, loss=4.3947]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office is the first time in the United States , which is the first time in the United States . '' 's . '' said . '' ' I said . '' ' I said . ' I 've been able to get a good time\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  81%|████████  | 139106/172148 [1:56:13<26:12, 21.01it/s, loss=4.4562]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3237\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:14, 26.08it/s]\u001b[A\n",
            "Evaluating:   0%|          | 7/3514 [00:00<02:00, 29.05it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2957, Perplexity: 73.38\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very good '' of the president 's speech , '' he said . '' ' I 'm not sure . '' ' I think it was a very good thing . '' ' I think it was a great thing . '' ' I\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's ##-year-old was arrested in the hospital after the attack was found in the area . ' ' I was a very good man . '' ' I was a bit of a little bit . ' '' said the .##-cal\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane of the crash occurred in the city of the city . '' 's . '' ) . '' ' I 'm not sure . '' ' I think it was a very good thing . '' ' I think it was a very good thing . '' '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  81%|████████  | 139111/172148 [1:56:15<1:31:58,  5.99it/s, loss=4.2655]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office is a very good man , '' he said . '' ' I 'm not sure . '' ' I think it was a very good thing . '' ' I think it was a very good thing . '' ' I think it was a\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  82%|████████▏ | 140667/172148 [1:57:32<26:06, 20.10it/s, loss=4.3837]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3247\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:51, 31.61it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2875, Perplexity: 72.79\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with the murder of the ##-year-old , who was arrested in the hospital . ' ' I was in the house . ' '' said . ' I was a man who was a man who was a man who was a member\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the ##-year-old was arrested in the hospital , where he was arrested in the hospital . ' . ' '' said the .## . '' ) . ' I was a very good friend . ' '' said . ' I was\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a huge increase in the number of people in the country . ' . '' ) . '' ' I think I 'm not going to be able to get the right to be able to get the best to the first time . ' '' he said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  82%|████████▏ | 140673/172148 [1:57:34<1:20:58,  6.48it/s, loss=4.2623]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. government is not the first time in the country . '' 's . ' '' said . ' I 'm not sure what I 'm not going to be a very good thing . ' '' said . ' I '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  83%|████████▎ | 142230/172148 [1:58:52<24:41, 20.19it/s, loss=4.3806]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3265\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:50, 31.89it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.3570, Perplexity: 78.03\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very dangerous '' and that he would be a very good idea . '' 's a very good player . '' ' I 'm not going to be a bit of a very good team . 's a very good player . ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-# win over the last ## . ' I 've got to be a good player . '' ' I 'm not going to be a bit of a bit of a bit of a bit of a bit of a bit of\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a long time , the government has been a `` very dangerous '' to the government . '' 's not the case . '' ' I 'm not going to be a very good idea . '' 's a very good player . '' ' I\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  83%|████████▎ | 142235/172148 [1:58:54<1:23:09,  5.99it/s, loss=4.4508]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been a `` very strong '' to the government . '' 's not the president , '' he said . '' ) . '' ' I said . '' ' I said . '' ' I do n't think I 'm going to be a\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  84%|████████▎ | 143794/172148 [2:00:11<23:20, 20.25it/s, loss=4.3707]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3268\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:48, 32.24it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2968, Perplexity: 73.47\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a 'happy ' of the 'deal ' by the . ' I was in the room . ' '' he said . ' I 'm not sure . ' '' he said . ' I was a very good friend . ' ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-# win over the world , but the ##-year-old was a ##-year-old man who was a ##-year-old girl , who was a ##-year-old girl , who was found guilty\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a long time , but it is not a problem . ' '' said the .### ) . '' ' I 've got to be a very good thing . ' '' said . ' I was a very good friend . ' '' said .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  84%|████████▎ | 143799/172148 [2:00:13<1:19:27,  5.95it/s, loss=4.5398]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been a 'most important role in the first time , but the government has been in the case . '' 's not the case . ' '' he said . ' I 'm not sure . ' '' he said . ' I was a very\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  84%|████████▍ | 145358/172148 [2:01:30<21:57, 20.33it/s, loss=4.2614]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3223\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:50, 31.88it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2903, Perplexity: 72.99\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's official statement said : `` The U.S. '' of the U.S. military . '' 's `` <rare> '' . ' '' said the .### ) . '' ' I 'm not sure what I 'm\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council has been charged with murdering a ##-year-old girl , who was arrested in connection with the incident . ' ' I 'm not sure what I 'm not going to do . '' ' I said . ' I 'm\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane is a long-term solution to the .### . '' ) . '' ' I said . ' I 'm not sure how to do it . ' '' he said . ' I 'm not sure how to do it . ' '' she\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  84%|████████▍ | 145361/172148 [2:01:32<1:29:25,  4.99it/s, loss=4.4544]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. military has been a `` very serious '' of the country 's most recent years . '' 's not the first time . ' '' he said . ' I 'm not sure how to do it . ' '' he\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  85%|████████▌ | 146921/172148 [2:02:50<20:35, 20.42it/s, loss=4.3030]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3227\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<01:57, 30.00it/s]\u001b[A\n",
            "Evaluating:   0%|          | 7/3514 [00:00<01:51, 31.34it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2991, Perplexity: 73.63\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very good opportunity to get the best to the team . '' ' I think it 's a good thing . '' ' I think I 'm not sure what I 'm going to be a good player . ' '' he said\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the ##-year-old was a ##-year-old boy who was a woman who was a woman who was a woman who was a woman who was a woman who was a woman who was a woman who was a woman who\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was a major problem in the region . '' 's a source . ) . '' said . '' ' I said . '' ' I told him . ' I 'm not sure what I was going to be a bit . ' '' said . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  85%|████████▌ | 146924/172148 [2:02:51<1:23:50,  5.01it/s, loss=4.2996]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office , which is the first time of the year , and the ##-year-old was in the midst of a ##-year-old man who was a ##-year-old boy who was shot dead by a police officer .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  86%|████████▋ | 148484/172148 [2:04:08<19:26, 20.28it/s, loss=4.2752]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3204\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:50, 31.74it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2960, Perplexity: 73.40\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very sadistic '' of the president 's speech . '' 's `` The <rare> '' . ' '' said the .### . '' ) . '' said . ' I 'm a friend who was a little girl\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-# win over the world . ' '' he said . ' I 'm not going to be a little bit . ' '' said the .### . ' '' said the .### . ' '' said . '' '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was discovered in the area . ' '' said the .### . '' ) . '' said . '' ' I said . ' I 'm not going to be a little bit . ' '' said the .## . '' ) . '' said .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  86%|████████▋ | 148487/172148 [2:04:10<1:17:53,  5.06it/s, loss=4.4657]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been a `` very sadistic '' of the president 's speech . '' 's `` The <rare> '' . ' '' said the .### . '' ) . '' said . ' I 'm a friend who was\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  87%|████████▋ | 150047/172148 [2:05:26<17:36, 20.92it/s, loss=4.4097]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3193\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.51it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2945, Perplexity: 73.30\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been accused of the attacking the Islamic State . ' '' said the .##-caliber handgun . ' '' said the .##pm . ' ) . '' ' I 'm a friend of the . ' I\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's ##-year-old , who was a ##-year-old , who was a ##-year-old , who was a ##-year-old , who was a ##-year-old , who was a ##-\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was found in the area , and the ##-year-old was killed . ' . ' '' said the .##pm . ' ) . '' ' I 'm a friend of the . ' I was a bit of a good job . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  87%|████████▋ | 150050/172148 [2:05:28<1:12:59,  5.05it/s, loss=4.2847]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office is not the first time . '' 's `` The right '' . ' '' said the .##pm . ' ) . '' ' I 'm a friend of the . ' I was a bit of a good job . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  88%|████████▊ | 151610/172148 [2:06:45<16:48, 20.36it/s, loss=4.3501]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3186\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:46, 32.81it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2950, Perplexity: 73.33\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been accused of being a `` very good '' of the country 's most powerful . '' . '' said . '' ' I said . ' I was a very good friend . ' '' said . ' I was a very good friend . ' ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's official website said : 'The whole thing is that I 've been doing . ' I was a very good friend . ' '' said . ' I was a very good friend . ' '' said . ' I was a very good friend\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the UK , and the ##-year-old was arrested in connection with the incident . ' I was a man . ' '' said . ' I was a very good friend . ' '' said . ' I was a very\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  88%|████████▊ | 151613/172148 [2:06:47<1:07:08,  5.10it/s, loss=4.2721]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. government has been accused of the attacking the government 's government to be able to control the government 's government to be able to control the government 's government to be able to control the government 's government to\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  89%|████████▉ | 153172/172148 [2:08:04<15:27, 20.46it/s, loss=4.2498]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3202\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<02:10, 26.93it/s]\u001b[A\n",
            "Evaluating:   0%|          | 6/3514 [00:00<02:02, 28.55it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2971, Perplexity: 73.48\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's government has been accused of being a `` very dangerous '' of the country 's largest . '' '' said the .## '' . '' ) . '' said . '' ' I was a member of the school . ' '' said the .##\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , the ##-year-old , who has been in the world , and the most popular destination for the world 's biggest-ever-generation of the world 's biggest-ever-generation is the most popular destination for the\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the area , the ##-year-old said . 's not a case of the case . '' ' I was a member of the school . ' '' said the .## p.m. , the .## .\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  89%|████████▉ | 153178/172148 [2:08:05<49:25,  6.40it/s, loss=4.4326]  "
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President was also accused of the `` serious threat '' to be a `` very dangerous '' of the country 's largest . '' '' said the .## '' . '' ) . '' said . '' ' I was a member of the school . ' '' said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  90%|████████▉ | 154735/172148 [2:09:22<14:16, 20.33it/s, loss=4.1536]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3179\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.68it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2869, Perplexity: 72.74\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major issue of the country 's most recent years . '' 's not . '' said . '' ' I said . ' I was a bit of a bit of a lot of people . ' '' said the .## . '' )\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council , which is the first time in the UK , which is expected to be a key part of the country 's most recent years . '' 's not . ' '' said the .### . '' ) . '' ' I was a\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the crash , which was found in the area . ' '' said the police officer . ' I was a woman . ' '' said . ' I was a woman . ' '' said . ' I was a woman . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  90%|████████▉ | 154741/172148 [2:09:24<44:58,  6.45it/s, loss=4.3694]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been charged with the death of the two-year-old , who was killed in the crash , which was found in the village of the city of the city of the city of the city of the city of the city of the city of the\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  91%|█████████ | 156299/172148 [2:10:40<13:05, 20.18it/s, loss=4.3814]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3203\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:46, 33.00it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2888, Perplexity: 72.88\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very good opportunity to be a good player . '' ' I 'm not sure what I 'm going to be a little bit . ' '' said the .### ) . '' ' I told him . ' I was a\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's ##-year-old was arrested in #### after the deaths of the ##-year-old . ' I was a little bit . ' '' said . ' I 'm not sure what I 'm going to do . '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the world , and the ##-year-old was in the first half . ' I was a bit of a good job . ' '' said the .### . ' I 'm not going to be a little bit\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  91%|█████████ | 156302/172148 [2:10:42<52:22,  5.04it/s, loss=4.3695]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the United States has been in the UK , which is expected to be a third of the most expensive . 's #.# million . '' ) . '' said . ' I 'm not sure what I 'm going to do . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  92%|█████████▏| 157861/172148 [2:11:58<11:54, 20.00it/s, loss=4.3152]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3167\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:49, 32.10it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2851, Perplexity: 72.61\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with the murder of the murder of the murder of the murder of the murder of the woman . ' ' I was a man . ' '' said . ' I 've got a lot of people . ' '' said the . ' I\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-# draw with the United States is the first to be a great player in the Premier League . ' '' he said . ' I 've got a lot of the players . '' ' I 've got a lot of the\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was found in the area , and the ##-year-old was killed in the crash . ' ' I 've got a lot of people . ' '' said the . ' I 'm not sure . ' '' he said . ' I '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  92%|█████████▏| 157866/172148 [2:12:00<40:32,  5.87it/s, loss=4.2367]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been charged with the murder of the murder of the murder of the murder of the murder of the woman . ' ' I was a man . ' '' said . ' I 've got a lot of people . ' '' said the\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  93%|█████████▎| 159424/172148 [2:13:16<10:25, 20.34it/s, loss=4.2477]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3144\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:46, 32.82it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2908, Perplexity: 73.02\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very important '' of the country 's government . '' 's not the same . '' ) . '' said . ' I was a very good friend . ' '' said . ' I was a very good friend . ' '' said\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , the ##-year-old , who is a member of the United States . 's the same . '' ) . '' ' I 'm not sure what happened . ' '' she said . ' I was a very good friend . '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the UK , the ##-year-old , who is the first of the most famous man . ' '' said the .##pm . '' ) . '' ' I 'm not sure what happened . ' '' she said\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  93%|█████████▎| 159430/172148 [2:13:18<32:31,  6.52it/s, loss=4.2765]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office is a `` very important '' of the American government . '' 's not the same . '' ' I 'm not sure what happened . ' '' she said . ' I was a very good friend . ' '' said . ' I\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  94%|█████████▎| 160987/172148 [2:14:35<09:03, 20.53it/s, loss=4.4473]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3161\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 3/3514 [00:00<01:57, 29.95it/s]\u001b[A\n",
            "Evaluating:   0%|          | 7/3514 [00:00<01:50, 31.78it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2853, Perplexity: 72.63\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's official statement said . 'Given the . '' ) . '' said . ' I 'm not going to be a little bit . ' '' she said . ' I 'm not going to be a little bit . ' '' she said\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-# win over the World Cup . ' I 'm going to be a good player . '' ' I 'm going to be a good player . '' ' I 'm going to be a good player . '' ' I\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was discovered in the city of the city . ' '' I said . ' '' she said . ' I 'm not going to be a little bit . ' '' she said . ' I 'm not going to be a little bit . ' ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  94%|█████████▎| 160993/172148 [2:14:37<28:38,  6.49it/s, loss=4.2262]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been a `` very good opportunity '' to the United States . '' 's a news conference . ' '' he said . ' I 'm not going to be a little bit . ' '' she said . ' I 'm not going to be\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  94%|█████████▍| 162550/172148 [2:15:53<07:49, 20.45it/s, loss=4.3133]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3144\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.51it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2890, Perplexity: 72.89\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a major factor in the attacking government . '' 's a statement . ) . '' ' I 'm sure . ' '' he said . ' I 'm not sure . ' '' he said . ' I 'm not sure .\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-# draw with the ##-year-old , who was a ##-year-old , who was a ##-year-old , who was a ##-year-old , who was arrested in the hospital , and\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was found in the water , and the ##-year-old was found in the area . ' '' I 'm a friend of the family and friends . ' '' said . ' I 'm a friend of the family and I was n'\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  94%|█████████▍| 162556/172148 [2:15:55<24:36,  6.50it/s, loss=4.3957]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. government is not the only one of the most important people . '' 's a .### ) . '' ' I 'm sure . ' '' said the .### ) . '' ' I 'm a friend\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  95%|█████████▌| 164114/172148 [2:17:11<06:32, 20.46it/s, loss=4.2926]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3151\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:47, 32.78it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2917, Perplexity: 73.09\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's official statement said : 'We are not going to be able to get the money to be able to do it . '' ' I told him . ' I 'm not sure what happened . ' '' she said . '' ' I told him\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council has been accused of being a `` very serious threat '' to be a . '' ) . '' ' I told the BBC . ' I 'm not sure what happened . ' '' she said . '' ' I told him . ' I '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was found in the city of the city of the city of the city of the city of the city of the city of the city of the city of the city of the city of the city of the city of the city of the city of the city\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  95%|█████████▌| 164117/172148 [2:17:12<26:26,  5.06it/s, loss=4.3839]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President of the U.S. military has been accused of being a `` very serious threat '' to be a `` very serious threat '' . '' ' I told him . ' I 'm not sure what happened . ' '' she said . '' ' I\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  96%|█████████▌| 165676/172148 [2:18:29<05:15, 20.51it/s, loss=4.2406]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3164\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:50, 31.83it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2904, Perplexity: 72.99\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very good '' of the presidential election . '' . '' ) . '' said . '' ' I 'm a friend . '' . ) . '' ) . '' ' I 'm a friend . '' . ) . '' )\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City , the ##-year-old , who was a ##-year-old man who was a ##-year-old man who was a ##-year-old man who was shot dead by a police officer . ' ' I was a\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was found in the area . ' '' said the .### ) . '' . ) # . '' ) . ' I 'm a friend . '' . ) . '' ) . '' ' I 'm a friend . '' . ) . ''\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  96%|█████████▌| 165681/172148 [2:18:30<18:25,  5.85it/s, loss=4.3360]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government has not been able to identify the suspects . ' '' said the .## . '' ) . '' said . '' ' I 'm a friend . '' . ) . '' ) . '' ' I 'm a\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  97%|█████████▋| 167239/172148 [2:19:48<04:11, 19.51it/s, loss=4.2733]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3129\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:52, 31.22it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2897, Perplexity: 72.95\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's official statement said : 'We are not aware of the situation . '' ' I said . ' '' she said . ' I 'm not sure what happened . ' '' she said . ' I 'm not sure what happened . ' ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Council said the `` <rare> '' of the `` <rare> '' of the `` <rare> '' of the `` <rare> '' of the `` <rare> '' of the <rare> , which is the first time . ''\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the past year . ' '' he said . ' '' he said . ' I 'm not sure what happened . ' '' she said . ' I 'm not sure what happened . ' '' she said . ' I '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  97%|█████████▋| 167244/172148 [2:19:49<14:13,  5.75it/s, loss=4.2491]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President has been a `` very strong '' of the president 's decision to end the . '' ) of the first time . ' '' he said . ' I 'm not sure what happened . ' '' she said . ' I 'm not sure what\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  98%|█████████▊| 168803/172148 [2:21:06<02:46, 20.06it/s, loss=4.3676]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3152\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:52, 31.07it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2865, Perplexity: 72.71\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with the murder of a ##-year-old woman who was arrested in connection with the death of the death of the death of the death of the death of the death of the death of the death of the death of the death of\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-# #-# #-# #-# #-# #-# #-# #-# #-# #-# #-# #-# #-# #-# #-# #-\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time of the year , and the ##-year-old was in the UK . ' ' I 'm not sure what I 'm going to be a little bit . ' '' said . ' I 'm not sure what I\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  98%|█████████▊| 168806/172148 [2:21:08<11:05,  5.02it/s, loss=4.2422]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office said the government has been `` very concerned '' by the government . '' 's . ' ) . '' ' I said . ' I was a very good friend . ' '' said . ' I was a friend of the couple . '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  99%|█████████▉| 170364/172148 [2:22:24<01:26, 20.72it/s, loss=4.3556]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3114\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:46, 32.92it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2846, Perplexity: 72.57\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow 's first-handed attacker was the first of the deadliest attack on the island of the city . '' ' I 'm a friend . '' ' I 'm a friend . ' '' said . '' ' I 'm a friend\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-# win over the World Cup . '' ' I 've got to be a bit more . '' ' I 've got to be a bit more . '' ' I 've got to be a bit more . '' '\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was found in the area , and the plane was found in the area . ' '' he said . ' '' she said . ' I 'm not sure what I 'm not going to be a bit of a good job . '' ' I '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1:  99%|█████████▉| 170370/172148 [2:22:25<04:29,  6.59it/s, loss=4.2799]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's government has been in the country 's government and the government 's government to be a `` very dangerous '' . '' '' said . '' ) . '' ' I 'm not sure what I 'm going to be a very good thing\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1: 100%|█████████▉| 171927/172148 [2:23:42<00:10, 20.38it/s, loss=4.2463]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After 200064 examples, Average Loss: 4.3117\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Evaluating:   0%|          | 0/3514 [00:00<?, ?it/s]\u001b[A\n",
            "Evaluating:   0%|          | 4/3514 [00:00<01:52, 31.15it/s]\u001b[A\n",
            "                                                            \u001b[A"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2843, Perplexity: 72.55\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been charged with the murder of the death of the ##-year-old . ' I was a very good friend . ' '' said . ' I was a friend . ' '' said . ' I was a friend . ' '' said . '\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City Mayor Michael Bloomberg said the attack was `` very concerned '' by the president 's government . '' ' I said . '' ' I said . ' I was a little bit of the car . ' '' she said . ' I was a\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was the first time in the UK , and the ##-year-old is the first time in the UK . ' '' he said . ' I 'm not going to be a good time . ' '' she said . ' I was a friend\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1: 100%|█████████▉| 171933/172148 [2:23:44<00:32,  6.53it/s, loss=4.2707]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office is not the first time to be able to take the lead . '' ' I 'm not sure what 's going to happen . '' ' I said . '' ' I was a friend . ' '' said . ' I was a\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 1/1: 100%|██████████| 172148/172148 [2:23:54<00:00, 19.94it/s, loss=4.3168]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Epoch 1/1, Average Loss: 4.3072\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": []
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2824, Perplexity: 72.41\n",
            "\n",
            "Generating text based on contexts using generate_text:\n",
            "\n",
            "\n",
            "Context: Moscow\n",
            "\n",
            "Generated text: Moscow has been a `` very serious '' to the government 's decision . '' 's . '' ) . '' ) . '' ' I 'm not sure what 's wrong . ' '' ) . '' ) . ' '' she said . ' ''\n",
            "\n",
            "Context: New York\n",
            "\n",
            "Generated text: New York City 's #-#-# win over the world 's top ## . ' '' he said . ' '' he said . ' '' he said . ' '' I 'm not sure ] . ' '' she said . ' '' she said\n",
            "\n",
            "Context: The hurricane\n",
            "\n",
            "Generated text: The hurricane was found in the water , and the ##-year-old was found dead in the car . ' '' . ' '' ) . '' ' I 'm not sure what 's happening . '' ) . ' '' she said . ' '' she\n",
            "\n",
            "Context: The President\n",
            "\n",
            "Generated text: The President 's office has been criticised for the first time in the past . '' ' I 'm not sure what 's happening . '' ' I 'm not sure what I 'm not going to be a little bit . ' '' he said\n"
          ]
        }
      ],
      "source": [
        "# ----------------------------\n",
        "# Main training loop for a Recurrent Neural Network Language Model\n",
        "# This script handles the entire training process including data loading,\n",
        "# model training, validation, and text generation\n",
        "# ----------------------------\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    # Initialize random seeds to ensure reproducible results\n",
        "    set_seed(42)\n",
        "\n",
        "    # Check for CUDA-capable GPU and set the device accordingly\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "    # Retrieve model architecture and training hyperparameters from configuration\n",
        "    # emb_dim: dimensionality of token embeddings and hidden states\n",
        "    # num_layers: number of recurrent layers in the model\n",
        "    # batch_size: mini-batch size\n",
        "    # learning_rate: step size for optimizer updates\n",
        "    # num_epochs: number of complete passes through the training dataset\n",
        "    # context_size: maximum input sequence length\n",
        "    emb_dim, num_layers, batch_size, learning_rate, num_epochs, context_size = get_hyperparameters()\n",
        "\n",
        "    # Initialize the tokenizer using Microsoft's Phi-3.5-mini model\n",
        "    tokenizer = AutoTokenizer.from_pretrained(\"microsoft/Phi-3.5-mini-instruct\")\n",
        "\n",
        "    # Get the size of the vocabulary that the model needs to handle\n",
        "    vocab_size = len(tokenizer)\n",
        "\n",
        "    # Download the news dataset and create DataLoader objects for training and testing\n",
        "    # DataLoaders handle batching and shuffling\n",
        "    data_url = \"https://www.thelmbook.com/data/news\"\n",
        "    train_dataloader, test_dataloader = download_and_prepare_data(data_url, batch_size, tokenizer, context_size)\n",
        "\n",
        "    # Initialize the RNN language model with specified architecture parameters\n",
        "    # vocab_size: determines output layer dimensionality\n",
        "    # emb_dim: size of word embeddings and hidden states\n",
        "    # num_layers: number of RNN layers\n",
        "    # pad_token_id: special token ID used for padding shorter sequences\n",
        "    model = RecurrentLanguageModel(vocab_size, emb_dim, num_layers, tokenizer.pad_token_id)\n",
        "\n",
        "    # Move the model to GPU if available\n",
        "    model.to(device)\n",
        "\n",
        "    # Initialize model weights using custom initialization scheme\n",
        "    # This is important for stable training of deep neural networks\n",
        "    initialize_weights(model)\n",
        "\n",
        "    # Calculate and display the total number of trainable parameters in the model\n",
        "    total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "    print(f\"Total trainable parameters: {total_params}\\n\")\n",
        "\n",
        "    # Initialize the loss function (Cross Entropy) for training\n",
        "    # ignore_index=pad_token_id ensures that padding tokens don't contribute to the loss\n",
        "    # This prevents the model from learning to predict padding tokens\n",
        "    criterion = nn.CrossEntropyLoss(ignore_index=tokenizer.pad_token_id)\n",
        "\n",
        "    # Initialize the AdamW optimizer with specified learning rate\n",
        "    optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n",
        "\n",
        "    # Set evaluation interval (number of examples after which to perform validation)\n",
        "    # 200,000 examples provides a good balance between training time and monitoring frequency\n",
        "    eval_interval = 200_000\n",
        "    examples_processed = 0  # Counter for tracking progress toward next evaluation\n",
        "\n",
        "    # Define test contexts for generating sample text during evaluation\n",
        "    contexts = [\n",
        "        \"Moscow\",\n",
        "        \"New York\",\n",
        "        \"The hurricane\",\n",
        "        \"The President\"\n",
        "    ]\n",
        "\n",
        "    # Main training loop - iterate through specified number of epochs\n",
        "    for epoch in range(num_epochs):\n",
        "        # Set model to training mode\n",
        "        model.train()\n",
        "        print(f\"Starting Epoch {epoch+1}/{num_epochs}, Model in training mode: {model.training}\")\n",
        "\n",
        "        # Initialize tracking variables for this epoch\n",
        "        total_loss = 0.0      # Accumulator for loss across all batches\n",
        "        total_tokens = 0      # Counter for actual tokens processed (excluding padding)\n",
        "        # Create progress bar for monitoring training progress\n",
        "        progress_bar = tqdm(train_dataloader, desc=f\"Epoch {epoch+1}/{num_epochs}\")\n",
        "\n",
        "        # Iterate through batches in the training data\n",
        "        for batch_idx, (input_seq, target_seq) in enumerate(progress_bar):\n",
        "            # Move input and target sequences to GPU if available\n",
        "            input_seq = input_seq.to(device)\n",
        "            target_seq = target_seq.to(device)\n",
        "            # Get current batch dimensions for reshaping operations\n",
        "            batch_size_current, seq_len = input_seq.shape\n",
        "\n",
        "            # Clear gradients from previous batch\n",
        "            # This is necessary as PyTorch accumulates gradients by default\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            # Forward pass: get model predictions for this batch\n",
        "            # output shape: (batch_size, seq_len, vocab_size)\n",
        "            output = model(input_seq)\n",
        "\n",
        "            # Reshape output and target tensors for loss computation\n",
        "            # - output: reshape to (batch_size * seq_len, vocab_size) for CrossEntropyLoss\n",
        "            # - target: reshape to (batch_size * seq_len) to match CrossEntropyLoss requirements\n",
        "            output = output.reshape(batch_size_current * seq_len, vocab_size)\n",
        "            target = target_seq.reshape(batch_size_current * seq_len)\n",
        "\n",
        "            # Count number of non-padding tokens in target\n",
        "            # This is needed because to calculate the average loss for multiple batches we need to divide the total loss\n",
        "            # by the number of tokens in these batches, but criterion(output, target) returns the average loss per token in a batch.\n",
        "            # So, we will multiply the loss per token by the number of tokens to get the loss per batch\n",
        "            non_padding_token_count = (target != tokenizer.pad_token_id).sum().item()\n",
        "\n",
        "            # Compute loss between model predictions and actual targets\n",
        "            loss = criterion(output, target)\n",
        "            # Backward pass: compute gradients of loss with respect to model parameters\n",
        "            loss.backward()\n",
        "            # Update model parameters using calculated gradients\n",
        "            optimizer.step()\n",
        "\n",
        "            # Calculate actual loss value for this batch\n",
        "            # Multiply the loss per token by number of non-padding tokens to get total loss for the batch\n",
        "            loss_value = loss.item() * non_padding_token_count\n",
        "            # Accumulate total loss for epoch statistics\n",
        "            total_loss += loss_value\n",
        "            # Accumulate total number of actual tokens processed\n",
        "            total_tokens += non_padding_token_count\n",
        "            # Increment counter for examples processed\n",
        "            examples_processed += batch_size_current\n",
        "\n",
        "            # Update progress bar with current batch loss\n",
        "            progress_bar.set_postfix({'loss': f\"{loss.item():.4f}\"})\n",
        "\n",
        "            # Periodic evaluation after processing specified number of examples\n",
        "            if examples_processed >= eval_interval:\n",
        "                # Calculate average loss over the last eval_interval examples\n",
        "                avg_loss = total_loss / total_tokens\n",
        "                print(f\"\\nAfter {examples_processed} examples, Average Loss: {avg_loss:.4f}\")\n",
        "\n",
        "                # Switch to evaluation mode\n",
        "                model.eval()\n",
        "\n",
        "                # Compute validation metrics\n",
        "                # average_loss: mean loss on validation set\n",
        "                # perplexity: exponential of average loss, lower is better\n",
        "                # sentences_processed: number of validation sequences evaluated\n",
        "                average_loss, perplexity = compute_loss_and_perplexity(\n",
        "                    model, test_dataloader, tokenizer, criterion, device, max_sentences=1000\n",
        "                )\n",
        "                print(f\"Validation Average Loss: {average_loss:.4f}, Perplexity: {perplexity:.2f}\")\n",
        "\n",
        "                # Generate sample texts to qualitatively assess model performance\n",
        "                print(\"Generating text based on contexts using generate_text:\\n\")\n",
        "                for context in contexts:\n",
        "                    # Generate text continuation for each test context\n",
        "                    generated_text = generate_text(\n",
        "                        model=model,          # The loaded language model\n",
        "                        start_string=context, # Context to continue\n",
        "                        tokenizer=tokenizer,  # Tokenizer for text conversion\n",
        "                        device=device,        # CPU or GPU device\n",
        "                        max_length=50         # Maximum length of generated sequence\n",
        "                    )\n",
        "                    print(f\"\\nContext: {context}\")\n",
        "                    print(f\"\\nGenerated text: {generated_text}\")\n",
        "\n",
        "                # Switch back to training mode for continued training\n",
        "                model.train()\n",
        "                # Reset counters for next evaluation interval\n",
        "                examples_processed = 0\n",
        "                total_loss = 0.0\n",
        "                total_tokens = 0\n",
        "\n",
        "        # End-of-epoch reporting\n",
        "        if total_tokens > 0:\n",
        "            # Calculate and display average loss for the epoch\n",
        "            avg_loss = total_loss / total_tokens\n",
        "            print(f\"\\nEpoch {epoch+1}/{num_epochs}, Average Loss: {avg_loss:.4f}\")\n",
        "        else:\n",
        "            # Handle edge case where no tokens were processed\n",
        "            print(f\"\\nEpoch {epoch+1}/{num_epochs} completed.\")\n",
        "\n",
        "        # Perform end-of-epoch validation\n",
        "        model.eval()\n",
        "        average_loss, perplexity = compute_loss_and_perplexity(\n",
        "            model, test_dataloader, tokenizer, criterion, device, max_sentences=1000\n",
        "        )\n",
        "        print(f\"Validation Average Loss: {average_loss:.4f}, Perplexity: {perplexity:.2f}\\n\")\n",
        "\n",
        "        print(\"Generating text based on contexts using generate_text:\\n\")\n",
        "        for context in contexts:\n",
        "            # Generate text continuation for each test context\n",
        "            generated_text = generate_text(\n",
        "                model=model,\n",
        "                start_string=context,\n",
        "                tokenizer=tokenizer,\n",
        "                device=device,\n",
        "                max_length=50  # Limit generation to 50 tokens for brevity\n",
        "            )\n",
        "            print(f\"\\nContext: {context}\")\n",
        "            print(f\"\\nGenerated text: {generated_text}\")\n",
        "\n",
        "        # Reset to training mode for next epoch\n",
        "        model.train()\n",
        "\n",
        "    # Save the trained model and tokenizer for later use\n",
        "    # This includes model architecture, weights, and tokenizer configuration\n",
        "    model_name = \"RNN_LM\"\n",
        "    save_model(model, tokenizer, model_name)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Testing the model\n",
        "\n",
        "In the cell below, we load and test the language model:"
      ],
      "metadata": {
        "id": "JoSevu4VJ-Rm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# ----------------------------\n",
        "# Model tests\n",
        "# ----------------------------\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "\n",
        "    model_name = \"RNN_LM\"\n",
        "\n",
        "    # Load the previously saved model and tokenizer from disk\n",
        "    # This recreates the exact model state from after training\n",
        "    model, tokenizer = load_model(model_name)\n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    average_loss, perplexity = compute_loss_and_perplexity(\n",
        "        model, test_dataloader, tokenizer, criterion, device, max_sentences=1000\n",
        "    )\n",
        "    print(f\"Validation Average Loss: {average_loss:.4f}, Perplexity: {perplexity:.2f}\\n\")\n",
        "\n",
        "    # Print header for test section\n",
        "    print(\"Testing the model:\\n\")\n",
        "\n",
        "    # Define a list of test prompts to evaluate model performance\n",
        "    contexts = [\n",
        "        \"Moscow\",\n",
        "        \"New York\",\n",
        "        \"A hurricane\",\n",
        "        \"The President\"\n",
        "    ]\n",
        "\n",
        "    # Iterate through each test prompt and generate text\n",
        "    for context in contexts:\n",
        "        # Generate text using greedy decoding (most likely tokens)\n",
        "        generated_text = generate_text(\n",
        "            model=model,          # The loaded language model\n",
        "            start_string=context, # Context to continue\n",
        "            tokenizer=tokenizer,  # Tokenizer for text conversion\n",
        "            device=device,        # CPU or GPU device\n",
        "            max_length=50         # Maximum length of generated sequence\n",
        "        )\n",
        "\n",
        "        # Print the original prompt and model's response\n",
        "        print(f\"\\nPrompt: {context}\")\n",
        "        print(f\"\\nGenerated response: {generated_text}\")"
      ],
      "metadata": {
        "id": "7JmWY6HACeCd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "40713443-023d-44e8-dfa1-dfd696c55923"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": []
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation Average Loss: 4.2824, Perplexity: 72.41\n",
            "\n",
            "Testing the model:\n",
            "\n",
            "\n",
            "Prompt: Moscow\n",
            "\n",
            "Generated response: Moscow has been a `` very serious '' to the government 's decision . '' 's . '' ) . '' ) . '' ' I 'm not sure what 's wrong . ' '' ) . '' ) . ' '' she said . ' ''\n",
            "\n",
            "Prompt: New York\n",
            "\n",
            "Generated response: New York City 's #-#-# win over the world 's top ## . ' '' he said . ' '' he said . ' '' he said . ' '' I 'm not sure ] . ' '' she said . ' '' she said\n",
            "\n",
            "Prompt: A hurricane\n",
            "\n",
            "Generated response: A hurricane season , the ##-year-old was found dead in the car . ' '' . ' '' ) . '' ' I 'm not sure what 's ] . ' '' she said . ' '' she said . ' '' she said . '\n",
            "\n",
            "Prompt: The President\n",
            "\n",
            "Generated response: The President 's office has been criticised for the first time in the past . '' ' I 'm not sure what 's happening . '' ' I 'm not sure what I 'm not going to be a little bit . ' '' he said\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import runtime\n",
        "runtime.unassign()"
      ],
      "metadata": {
        "id": "N3ypCXJmxubs"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "A100",
      "provenance": [],
      "authorship_tag": "ABX9TyMPC1n4wS9vN0DsFOnDmB6b",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "42f9c201196846ecaaaca13424066f31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7dc8266e8e9349f59617784563bcfb0d",
              "IPY_MODEL_e396c523cfa14d6ab7582345428d7e2b",
              "IPY_MODEL_63c9afad68224b13a7f345826757d74d"
            ],
            "layout": "IPY_MODEL_d1abdf9e1dbc4c2db3a83717d52052a7"
          }
        },
        "7dc8266e8e9349f59617784563bcfb0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dbe62517041c478f89fd4f7031684997",
            "placeholder": "​",
            "style": "IPY_MODEL_c573226f95d34166833c410c0559047e",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "e396c523cfa14d6ab7582345428d7e2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5f7faf15a50545bf971902428f788aba",
            "max": 3984,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_539e601e729a4c7d81b48e17a50b6ab2",
            "value": 3984
          }
        },
        "63c9afad68224b13a7f345826757d74d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0873e85b65a447199d4e506731d2f60e",
            "placeholder": "​",
            "style": "IPY_MODEL_d5208cb6e46f4c19a893568e94fe471b",
            "value": " 3.98k/3.98k [00:00&lt;00:00, 302kB/s]"
          }
        },
        "d1abdf9e1dbc4c2db3a83717d52052a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dbe62517041c478f89fd4f7031684997": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c573226f95d34166833c410c0559047e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5f7faf15a50545bf971902428f788aba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "539e601e729a4c7d81b48e17a50b6ab2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0873e85b65a447199d4e506731d2f60e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d5208cb6e46f4c19a893568e94fe471b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e468ae9b5ee146eb97c30388eabeddfe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0294c86e78d0416ca6f74294f93a0c57",
              "IPY_MODEL_8b193f31f5714da18bcc046cbafc807f",
              "IPY_MODEL_5b8f89c68dd34744946be5a9aa7cf1a2"
            ],
            "layout": "IPY_MODEL_4e93cc7e12834e6da060dad9f1cc93b5"
          }
        },
        "0294c86e78d0416ca6f74294f93a0c57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_53c0255664eb44e68fa0e93769512a67",
            "placeholder": "​",
            "style": "IPY_MODEL_44fdec6de91f47889673265c2ef80ab3",
            "value": "tokenizer.model: 100%"
          }
        },
        "8b193f31f5714da18bcc046cbafc807f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f388c603cafa4060b01fae139c814d4d",
            "max": 499723,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_646a033463044d61af033e64c263f5e7",
            "value": 499723
          }
        },
        "5b8f89c68dd34744946be5a9aa7cf1a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c9161c25556e467bb05d0f4d866bf467",
            "placeholder": "​",
            "style": "IPY_MODEL_fe1150d2657c4d88aa81ca6548832860",
            "value": " 500k/500k [00:00&lt;00:00, 23.0MB/s]"
          }
        },
        "4e93cc7e12834e6da060dad9f1cc93b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "53c0255664eb44e68fa0e93769512a67": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "44fdec6de91f47889673265c2ef80ab3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f388c603cafa4060b01fae139c814d4d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "646a033463044d61af033e64c263f5e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c9161c25556e467bb05d0f4d866bf467": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fe1150d2657c4d88aa81ca6548832860": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "45aa4c9ffccc484698af6b51d6c6324a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b6f4495af1f64a35877716f00f40c33c",
              "IPY_MODEL_17d7b1b82bf340b4af2cef3b9791168e",
              "IPY_MODEL_67f97fe065834d8a8d1a57cfe83e77a6"
            ],
            "layout": "IPY_MODEL_3e6a18e429554400b8b1c480cfd129aa"
          }
        },
        "b6f4495af1f64a35877716f00f40c33c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_74d48d183a5e41cd8cc52f939bb2b842",
            "placeholder": "​",
            "style": "IPY_MODEL_2819c169ef5f4239bcd655771ffc11ab",
            "value": "tokenizer.json: 100%"
          }
        },
        "17d7b1b82bf340b4af2cef3b9791168e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_93b8236e001341fc89a5556d0c0f4233",
            "max": 1844408,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f32e37cf305d4abcad893cdbd08e0c6a",
            "value": 1844408
          }
        },
        "67f97fe065834d8a8d1a57cfe83e77a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1e243fa6c99142a0a7f45e3d3fb1d96f",
            "placeholder": "​",
            "style": "IPY_MODEL_3445fee928cc41f483d07bf946046bdc",
            "value": " 1.84M/1.84M [00:00&lt;00:00, 32.7MB/s]"
          }
        },
        "3e6a18e429554400b8b1c480cfd129aa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "74d48d183a5e41cd8cc52f939bb2b842": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2819c169ef5f4239bcd655771ffc11ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "93b8236e001341fc89a5556d0c0f4233": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f32e37cf305d4abcad893cdbd08e0c6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1e243fa6c99142a0a7f45e3d3fb1d96f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3445fee928cc41f483d07bf946046bdc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b2d13951116649d3b1a9c6403c442b43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_767362af7c494617a8ccde641fff124f",
              "IPY_MODEL_0c68a7b351b2486abb2b65d794d15aec",
              "IPY_MODEL_1e611591b22c46d7b5977c2ecdde02f2"
            ],
            "layout": "IPY_MODEL_258b534bd0894450b59b2eb7ab0905ec"
          }
        },
        "767362af7c494617a8ccde641fff124f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6806c2567a274957a9335f46c505db15",
            "placeholder": "​",
            "style": "IPY_MODEL_2789503992c04d6991db2606f580f230",
            "value": "added_tokens.json: 100%"
          }
        },
        "0c68a7b351b2486abb2b65d794d15aec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3f7221ac1912480ca7054b64f8a87a7f",
            "max": 306,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9989d9d05d264f3da894bcf7c84b6f94",
            "value": 306
          }
        },
        "1e611591b22c46d7b5977c2ecdde02f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_67cb7426300947ebbe9087ee81b3f4db",
            "placeholder": "​",
            "style": "IPY_MODEL_bb2b55614d5b4e78b40e625084776bb8",
            "value": " 306/306 [00:00&lt;00:00, 25.8kB/s]"
          }
        },
        "258b534bd0894450b59b2eb7ab0905ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6806c2567a274957a9335f46c505db15": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2789503992c04d6991db2606f580f230": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3f7221ac1912480ca7054b64f8a87a7f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9989d9d05d264f3da894bcf7c84b6f94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "67cb7426300947ebbe9087ee81b3f4db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb2b55614d5b4e78b40e625084776bb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3aaa2099e38142de9854b22a7595a88e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d67374a248974744b9bfd2922acf3e88",
              "IPY_MODEL_ab2f98969f734e399e224069c23a79b0",
              "IPY_MODEL_56408789191d4ad3a0043fa816e03529"
            ],
            "layout": "IPY_MODEL_8b7ab57a33184571914ff60ba032a20d"
          }
        },
        "d67374a248974744b9bfd2922acf3e88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cdf2acfd3e304fc492378c0dee8a8946",
            "placeholder": "​",
            "style": "IPY_MODEL_cd3342d14a5e4832843aaa9830bdee4d",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "ab2f98969f734e399e224069c23a79b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0629dca9bbd2478eb5e1b6791540024c",
            "max": 665,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a279be615d464c269d508c0c0fa33ebd",
            "value": 665
          }
        },
        "56408789191d4ad3a0043fa816e03529": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2022b6b16bb041f3b1d961033eda33a3",
            "placeholder": "​",
            "style": "IPY_MODEL_ef8ec7ac921849ed8c4814536b24b61b",
            "value": " 665/665 [00:00&lt;00:00, 52.8kB/s]"
          }
        },
        "8b7ab57a33184571914ff60ba032a20d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cdf2acfd3e304fc492378c0dee8a8946": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cd3342d14a5e4832843aaa9830bdee4d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0629dca9bbd2478eb5e1b6791540024c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a279be615d464c269d508c0c0fa33ebd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2022b6b16bb041f3b1d961033eda33a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ef8ec7ac921849ed8c4814536b24b61b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}