BackTranslation_Data_Augmentation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a9db9b60-9cc0-43ea-bea7-d2b6c4ccc405",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in c:\\users\\carmine\\anaconda3\\lib\\site-packages (4.46.3)\n",
      "Requirement already satisfied: torch in c:\\users\\carmine\\anaconda3\\lib\\site-packages (2.5.1)\n",
      "Requirement already satisfied: pandas in c:\\users\\carmine\\anaconda3\\lib\\site-packages (2.2.2)\n",
      "Requirement already satisfied: tqdm in c:\\users\\carmine\\anaconda3\\lib\\site-packages (4.66.5)\n",
      "Collecting sentencepiece\n",
      "  Downloading sentencepiece-0.2.0-cp312-cp312-win_amd64.whl.metadata (8.3 kB)\n",
      "Collecting sacremoses\n",
      "  Downloading sacremoses-0.1.1-py3-none-any.whl.metadata (8.3 kB)\n",
      "Requirement already satisfied: filelock in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from transformers) (3.13.1)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from transformers) (0.26.3)\n",
      "Requirement already satisfied: numpy>=1.17 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from transformers) (1.26.4)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from transformers) (24.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from transformers) (6.0.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from transformers) (2024.9.11)\n",
      "Requirement already satisfied: requests in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from transformers) (2.32.3)\n",
      "Requirement already satisfied: tokenizers<0.21,>=0.20 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from transformers) (0.20.3)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from transformers) (0.4.5)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from torch) (4.11.0)\n",
      "Requirement already satisfied: setuptools in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from torch) (75.1.0)\n",
      "Requirement already satisfied: sympy==1.13.1 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from torch) (1.13.1)\n",
      "Requirement already satisfied: networkx in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from torch) (3.3)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from torch) (3.1.4)\n",
      "Requirement already satisfied: fsspec in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from torch) (2024.6.1)\n",
      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from sympy==1.13.1->torch) (1.3.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from pandas) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from pandas) (2023.3)\n",
      "Requirement already satisfied: colorama in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from tqdm) (0.4.6)\n",
      "Requirement already satisfied: click in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from sacremoses) (8.1.7)\n",
      "Requirement already satisfied: joblib in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from sacremoses) (1.4.2)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from jinja2->torch) (2.1.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from requests->transformers) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from requests->transformers) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from requests->transformers) (2.2.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\carmine\\anaconda3\\lib\\site-packages (from requests->transformers) (2024.8.30)\n",
      "Downloading sentencepiece-0.2.0-cp312-cp312-win_amd64.whl (991 kB)\n",
      "   ---------------------------------------- 0.0/992.0 kB ? eta -:--:--\n",
      "   --------------------- ------------------ 524.3/992.0 kB 8.2 MB/s eta 0:00:01\n",
      "   ---------------------------------------- 992.0/992.0 kB 9.4 MB/s eta 0:00:00\n",
      "Downloading sacremoses-0.1.1-py3-none-any.whl (897 kB)\n",
      "   ---------------------------------------- 0.0/897.5 kB ? eta -:--:--\n",
      "   --------------------------------------- 897.5/897.5 kB 20.5 MB/s eta 0:00:00\n",
      "Installing collected packages: sentencepiece, sacremoses\n",
      "Successfully installed sacremoses-0.1.1 sentencepiece-0.2.0\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers torch pandas tqdm sentencepiece sacremoses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d7ec46fa-304b-4186-adcf-64483119ca56",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from transformers import pipeline, AutoModelForSeq2SeqLM, AutoTokenizer\n",
    "import torch\n",
    "from tqdm import tqdm\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "464f2fef-f2ef-4eea-9bf0-bb0bd0c52bbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_translation_pipeline(model_name, device=0):\n",
    "    \"\"\"Create a translation pipeline with specified model and device.\"\"\"\n",
    "    return pipeline(\"translation\", model=model_name, device=device)\n",
    "\n",
    "def chunk_text(text, max_length=512):\n",
    "    \"\"\"Split text into chunks that won't exceed model's max token limit.\"\"\"\n",
    "    words = text.split()\n",
    "    chunks = []\n",
    "    current_chunk = []\n",
    "    current_length = 0\n",
    "    \n",
    "    for word in words:\n",
    "        # Rough estimate of token length\n",
    "        word_length = len(word) + 1  # +1 for space\n",
    "        if current_length + word_length > max_length:\n",
    "            chunks.append(' '.join(current_chunk))\n",
    "            current_chunk = [word]\n",
    "            current_length = word_length\n",
    "        else:\n",
    "            current_chunk.append(word)\n",
    "            current_length += word_length\n",
    "            \n",
    "    if current_chunk:\n",
    "        chunks.append(' '.join(current_chunk))\n",
    "    return chunks\n",
    "\n",
    "def back_translate(text, en_to_mid_pipeline, mid_to_en_pipeline, intermediate_lang=\"fr\", max_length=512):\n",
    "    \"\"\"Perform back-translation on a single text.\"\"\"\n",
    "    try:\n",
    "        # Split into chunks\n",
    "        chunks = chunk_text(text, max_length)\n",
    "        translated_chunks = []\n",
    "        \n",
    "        for chunk in chunks:\n",
    "            # Translate to intermediate language\n",
    "            mid_translation = en_to_mid_pipeline(chunk)[0]['translation_text']\n",
    "            \n",
    "            # Translate back to English\n",
    "            back_translation = mid_to_en_pipeline(mid_translation)[0]['translation_text']\n",
    "            translated_chunks.append(back_translation)\n",
    "            \n",
    "        # Combine chunks\n",
    "        return ' '.join(translated_chunks)\n",
    "    except Exception as e:\n",
    "        print(f\"Error in back translation: {str(e)}\")\n",
    "        return text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9ad0ec3a-28d1-4c26-93c8-d1ed43829b21",
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment_dataset_with_backtranslation(df, batch_size=64, intermediate_langs=[\"fr\", \"de\", \"es\"], \n",
    "                                       device=0, max_samples=None):\n",
    "    \"\"\"\n",
    "    Augment a dataset using back-translation through multiple intermediate languages.\n",
    "    \n",
    "    Args:\n",
    "        df: DataFrame containing 'review' column\n",
    "        batch_size: Number of samples to process at once (increased for GPU)\n",
    "        intermediate_langs: List of intermediate languages to use\n",
    "        device: Device to run translations on (0 for first GPU)\n",
    "        max_samples: Maximum number of samples to process (None for all)\n",
    "    \n",
    "    Returns:\n",
    "        DataFrame with original and augmented reviews\n",
    "    \"\"\"\n",
    "    # Verify CUDA is available and being used\n",
    "    print(f\"CUDA available: {torch.cuda.is_available()}\")\n",
    "    print(f\"Current device: {torch.cuda.current_device()}\")\n",
    "    print(f\"Device name: {torch.cuda.get_device_name(0)}\")\n",
    "    \n",
    "    augmented_reviews = []\n",
    "    \n",
    "    if max_samples:\n",
    "        df = df.head(max_samples)\n",
    "        \n",
    "    for lang in intermediate_langs:\n",
    "        print(f\"\\nPerforming back-translation through {lang}\")\n",
    "        \n",
    "        # Initialize translation pipelines\n",
    "        en_to_mid_model = f\"Helsinki-NLP/opus-mt-en-{lang}\"\n",
    "        mid_to_en_model = f\"Helsinki-NLP/opus-mt-{lang}-en\"\n",
    "        \n",
    "        en_to_mid_pipeline = create_translation_pipeline(en_to_mid_model, device)\n",
    "        mid_to_en_pipeline = create_translation_pipeline(mid_to_en_model, device)\n",
    "        \n",
    "        # Process in batches\n",
    "        for i in tqdm(range(0, len(df), batch_size)):\n",
    "            batch = df.iloc[i:i+batch_size]\n",
    "            \n",
    "            # Back-translate each review in the batch\n",
    "            translated = []\n",
    "            for text in batch['review']:\n",
    "                augmented = back_translate(text, en_to_mid_pipeline, mid_to_en_pipeline, \n",
    "                                        intermediate_lang=lang)\n",
    "                translated.append(augmented)\n",
    "                \n",
    "                # Reduced delay since GPU can handle faster processing\n",
    "                time.sleep(0.05)\n",
    "            \n",
    "            augmented_reviews.extend(translated)\n",
    "            \n",
    "            # Clear GPU cache periodically\n",
    "            if i % (batch_size * 10) == 0:\n",
    "                torch.cuda.empty_cache()\n",
    "            \n",
    "        print(f\"Completed back-translation through {lang}\")\n",
    "        \n",
    "        # Clear GPU cache after each language\n",
    "        torch.cuda.empty_cache()\n",
    "    \n",
    "    # Create new dataframe with original and augmented reviews\n",
    "    augmented_df = pd.DataFrame({\n",
    "        'review': augmented_reviews,\n",
    "        'sentiment': df['sentiment'].tolist() * len(intermediate_langs)\n",
    "    })\n",
    "    \n",
    "    return augmented_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "683aefa4-7b0e-4adc-a2ad-4ffa10664072",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_augmented_data(original_df, augmented_df, output_file):\n",
    "    \"\"\"Save augmented dataset to CSV.\"\"\"\n",
    "    augmented_df.to_csv(output_file, index=False)\n",
    "    print(f\"Saved augmented dataset with {len(augmented_df)} samples to {output_file}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "172ae69b-0861-4766-a192-d57084885589",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read original data\n",
    "imdb_data = pd.read_csv(\"../input/imdb_dataset.csv\")\n",
    "    \n",
    "# Create train/test split\n",
    "train_data = imdb_data.iloc[:40000]\n",
    "test_data = imdb_data.iloc[40000:]\n",
    "    \n",
    "# Augment training data\n",
    "augmented_train = augment_dataset_with_backtranslation(\n",
    "    train_data,\n",
    "    batch_size=256,  # Increased batch size for GPU\n",
    "    intermediate_langs=[\"fr\", \"de\", \"es\"],\n",
    "    device=0,  # Use first GPU\n",
    "    max_samples=100  # For testing, remove for full dataset\n",
    ")\n",
    "\n",
    "save_augmented_data(train_data, augmented_train, \"backtranslated_train_reviews.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}