examples/tweet_eval-hate-dedup.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting gen-dedup\n",
      "  Downloading gen_dedup-0.0.1-py3-none-any.whl.metadata (2.9 kB)\n",
      "Collecting torch (from gen-dedup)\n",
      "  Downloading torch-2.4.1-cp311-cp311-manylinux1_x86_64.whl.metadata (26 kB)\n",
      "Requirement already satisfied: tqdm in /data/lxm/miniconda3/lib/python3.11/site-packages (from gen-dedup) (4.65.0)\n",
      "Collecting datasets (from gen-dedup)\n",
      "  Downloading datasets-3.0.1-py3-none-any.whl.metadata (20 kB)\n",
      "Collecting transformers (from gen-dedup)\n",
      "  Downloading transformers-4.45.2-py3-none-any.whl.metadata (44 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.4/44.4 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting filelock (from datasets->gen-dedup)\n",
      "  Downloading filelock-3.16.1-py3-none-any.whl.metadata (2.9 kB)\n",
      "Collecting numpy>=1.17 (from datasets->gen-dedup)\n",
      "  Downloading numpy-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.9/60.9 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting pyarrow>=15.0.0 (from datasets->gen-dedup)\n",
      "  Downloading pyarrow-17.0.0-cp311-cp311-manylinux_2_28_x86_64.whl.metadata (3.3 kB)\n",
      "Collecting dill<0.3.9,>=0.3.0 (from datasets->gen-dedup)\n",
      "  Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
      "Collecting pandas (from datasets->gen-dedup)\n",
      "  Downloading pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (89 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.9/89.9 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting requests>=2.32.2 (from datasets->gen-dedup)\n",
      "  Using cached requests-2.32.3-py3-none-any.whl.metadata (4.6 kB)\n",
      "Collecting tqdm (from gen-dedup)\n",
      "  Using cached tqdm-4.66.5-py3-none-any.whl.metadata (57 kB)\n",
      "Collecting xxhash (from datasets->gen-dedup)\n",
      "  Downloading xxhash-3.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
      "Collecting multiprocess (from datasets->gen-dedup)\n",
      "  Downloading multiprocess-0.70.17-py311-none-any.whl.metadata (7.2 kB)\n",
      "Collecting fsspec<=2024.6.1,>=2023.1.0 (from fsspec[http]<=2024.6.1,>=2023.1.0->datasets->gen-dedup)\n",
      "  Downloading fsspec-2024.6.1-py3-none-any.whl.metadata (11 kB)\n",
      "Collecting aiohttp (from datasets->gen-dedup)\n",
      "  Downloading aiohttp-3.10.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.6 kB)\n",
      "Collecting huggingface-hub>=0.22.0 (from datasets->gen-dedup)\n",
      "  Downloading huggingface_hub-0.25.1-py3-none-any.whl.metadata (13 kB)\n",
      "Requirement already satisfied: packaging in /data/lxm/miniconda3/lib/python3.11/site-packages (from datasets->gen-dedup) (23.1)\n",
      "Collecting pyyaml>=5.1 (from datasets->gen-dedup)\n",
      "  Downloading PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (2.1 kB)\n",
      "Collecting typing-extensions>=4.8.0 (from torch->gen-dedup)\n",
      "  Using cached typing_extensions-4.12.2-py3-none-any.whl.metadata (3.0 kB)\n",
      "Collecting sympy (from torch->gen-dedup)\n",
      "  Downloading sympy-1.13.3-py3-none-any.whl.metadata (12 kB)\n",
      "Collecting networkx (from torch->gen-dedup)\n",
      "  Downloading networkx-3.3-py3-none-any.whl.metadata (5.1 kB)\n",
      "Collecting jinja2 (from torch->gen-dedup)\n",
      "  Downloading jinja2-3.1.4-py3-none-any.whl.metadata (2.6 kB)\n",
      "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch->gen-dedup)\n",
      "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch->gen-dedup)\n",
      "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch->gen-dedup)\n",
      "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
      "Collecting nvidia-cudnn-cu12==9.1.0.70 (from torch->gen-dedup)\n",
      "  Downloading nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
      "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch->gen-dedup)\n",
      "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch->gen-dedup)\n",
      "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-curand-cu12==10.3.2.106 (from torch->gen-dedup)\n",
      "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch->gen-dedup)\n",
      "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
      "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch->gen-dedup)\n",
      "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
      "Collecting nvidia-nccl-cu12==2.20.5 (from torch->gen-dedup)\n",
      "  Downloading nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl.metadata (1.8 kB)\n",
      "Collecting nvidia-nvtx-cu12==12.1.105 (from torch->gen-dedup)\n",
      "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\n",
      "Collecting triton==3.0.0 (from torch->gen-dedup)\n",
      "  Downloading triton-3.0.0-1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.3 kB)\n",
      "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch->gen-dedup)\n",
      "  Downloading nvidia_nvjitlink_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting regex!=2019.12.17 (from transformers->gen-dedup)\n",
      "  Downloading regex-2024.9.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (40 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.5/40.5 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting safetensors>=0.4.1 (from transformers->gen-dedup)\n",
      "  Downloading safetensors-0.4.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.8 kB)\n",
      "Collecting tokenizers<0.21,>=0.20 (from transformers->gen-dedup)\n",
      "  Downloading tokenizers-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n",
      "Collecting aiohappyeyeballs>=2.3.0 (from aiohttp->datasets->gen-dedup)\n",
      "  Downloading aiohappyeyeballs-2.4.3-py3-none-any.whl.metadata (6.1 kB)\n",
      "Collecting aiosignal>=1.1.2 (from aiohttp->datasets->gen-dedup)\n",
      "  Downloading aiosignal-1.3.1-py3-none-any.whl.metadata (4.0 kB)\n",
      "Collecting attrs>=17.3.0 (from aiohttp->datasets->gen-dedup)\n",
      "  Downloading attrs-24.2.0-py3-none-any.whl.metadata (11 kB)\n",
      "Collecting frozenlist>=1.1.1 (from aiohttp->datasets->gen-dedup)\n",
      "  Downloading frozenlist-1.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
      "Collecting multidict<7.0,>=4.5 (from aiohttp->datasets->gen-dedup)\n",
      "  Downloading multidict-6.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.0 kB)\n",
      "Collecting yarl<2.0,>=1.12.0 (from aiohttp->datasets->gen-dedup)\n",
      "  Downloading yarl-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (50 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.8/50.8 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: charset-normalizer<4,>=2 in /data/lxm/miniconda3/lib/python3.11/site-packages (from requests>=2.32.2->datasets->gen-dedup) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /data/lxm/miniconda3/lib/python3.11/site-packages (from requests>=2.32.2->datasets->gen-dedup) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /data/lxm/miniconda3/lib/python3.11/site-packages (from requests>=2.32.2->datasets->gen-dedup) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /data/lxm/miniconda3/lib/python3.11/site-packages (from requests>=2.32.2->datasets->gen-dedup) (2023.7.22)\n",
      "Collecting MarkupSafe>=2.0 (from jinja2->torch->gen-dedup)\n",
      "  Downloading MarkupSafe-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.0 kB)\n",
      "INFO: pip is looking at multiple versions of multiprocess to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting multiprocess (from datasets->gen-dedup)\n",
      "  Downloading multiprocess-0.70.16-py311-none-any.whl.metadata (7.2 kB)\n",
      "Collecting python-dateutil>=2.8.2 (from pandas->datasets->gen-dedup)\n",
      "  Downloading python_dateutil-2.9.0.post0-py2.py3-none-any.whl.metadata (8.4 kB)\n",
      "Collecting pytz>=2020.1 (from pandas->datasets->gen-dedup)\n",
      "  Downloading pytz-2024.2-py2.py3-none-any.whl.metadata (22 kB)\n",
      "Collecting tzdata>=2022.7 (from pandas->datasets->gen-dedup)\n",
      "  Downloading tzdata-2024.2-py2.py3-none-any.whl.metadata (1.4 kB)\n",
      "Collecting mpmath<1.4,>=1.1.0 (from sympy->torch->gen-dedup)\n",
      "  Downloading mpmath-1.3.0-py3-none-any.whl.metadata (8.6 kB)\n",
      "Collecting six>=1.5 (from python-dateutil>=2.8.2->pandas->datasets->gen-dedup)\n",
      "  Downloading six-1.16.0-py2.py3-none-any.whl.metadata (1.8 kB)\n",
      "Downloading gen_dedup-0.0.1-py3-none-any.whl (9.8 kB)\n",
      "Downloading datasets-3.0.1-py3-none-any.whl (471 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m471.6/471.6 kB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hUsing cached tqdm-4.66.5-py3-none-any.whl (78 kB)\n",
      "Downloading torch-2.4.1-cp311-cp311-manylinux1_x86_64.whl (797.1 MB)\n",
      "\u001b[2K   \u001b[91m━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.8/797.1 MB\u001b[0m \u001b[31m32.6 MB/s\u001b[0m eta \u001b[36m0:00:23\u001b[0m^C\n",
      "\u001b[2K   \u001b[91m━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.1/797.1 MB\u001b[0m \u001b[31m33.4 MB/s\u001b[0m eta \u001b[36m0:00:22\u001b[0m\n",
      "\u001b[?25h\u001b[31mERROR: Operation cancelled by user\u001b[0m\u001b[31m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install gen-dedup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "os.environ['WANDB_MODE'] = 'disabled'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "# 1. load dataset\n",
    "ds = load_dataset('cardiffnlp/tweet_eval', 'hate', split='train')\n",
    "ds = ds.select_columns(['text'])\n",
    "ds = ds.rename_column('text', 'sentence')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keybert import KeyBERT\n",
    "\n",
    "# 2. generate two keywords. Other keyword extraction models can also be used, such as tfidf, LLMs, etc.\n",
    "keybert = KeyBERT()\n",
    "ds = ds.map(lambda x: {'labels': \" \".join([k[0] for k in keybert.extract_keywords(x['sentence'].lower())[:2]]), 'sentence': x['sentence'].lower()})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'sentence': '@user nice new signage. are you not concerned by beatlemania -style hysterical crowds crongregating on you…', 'labels': 'signage crowds'}\n"
     ]
    }
   ],
   "source": [
    "for obj in ds:\n",
    "    print(obj)\n",
    "    break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba76410b93064277ab6d9fc72c3faae4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/9000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Disabling the wandb service is deprecated as of version 0.18.0 and will be removed in future versions. \n",
      "You're using a T5TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='282' max='282' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [282/282 00:24, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>7.133300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>5.306700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>4.604000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>3.911500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>3.619200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>3.565100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70</td>\n",
       "      <td>3.116100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>2.804100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90</td>\n",
       "      <td>2.914800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>2.558200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>110</td>\n",
       "      <td>2.602200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>120</td>\n",
       "      <td>2.538500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>130</td>\n",
       "      <td>2.495600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>140</td>\n",
       "      <td>2.574800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>2.554800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>160</td>\n",
       "      <td>2.406200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>170</td>\n",
       "      <td>2.213400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>180</td>\n",
       "      <td>2.353200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>190</td>\n",
       "      <td>2.294100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>2.198100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>210</td>\n",
       "      <td>2.248100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>220</td>\n",
       "      <td>2.073000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>230</td>\n",
       "      <td>2.192600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>240</td>\n",
       "      <td>2.088600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>250</td>\n",
       "      <td>2.003400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>260</td>\n",
       "      <td>2.145400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>270</td>\n",
       "      <td>2.144700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>280</td>\n",
       "      <td>2.252100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from gen_dedup import GenDedup\n",
    "\n",
    "gd = GenDedup('google/flan-t5-small')\n",
    "gd.fit(ds, output_dir='./hate-dedup')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/9000 [00:00<?, ?it/s]/data/lxm/miniconda3/envs/py310/lib/python3.10/site-packages/transformers/generation/utils.py:1273: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n",
      "100%|██████████| 9000/9000 [07:55<00:00, 18.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 duplicate text detected!\n",
      "8900 nonduplicate text detected!\n",
      "Duplicate text has been saved to hate-dedup/duplicate.jsonl\n",
      "Nonduplicate text has been saved to hate-dedup/nonduplicate.jsonl\n"
     ]
    }
   ],
   "source": [
    "gd.deduplicate('hate-dedup', max_label_words=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py310",
   "language": "python",
   "name": "py310"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}