simplification_finetune_decoder-LMs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BO7MEGbb6mtB"
   },
   "source": [
    "# Simplification experiments\n",
    "This code was used for finetuning all decoder LMs for the simplification task and their evaluation on this task.\n",
    "\n",
    "## Part 1. Finetuning decoder LMs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Xyhc5yrzR75j",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# %%bash\n",
    "# git clone https://github.com/huggingface/transformers\n",
    "# cd transformers\n",
    "# pip install .\n",
    "\n",
    "#!pip install transformers==4.22.2\n",
    "#!pip install pytest\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python run_clm.py \\\n",
    "    --model_name_or_path sberbank-ai/rugpt3small_based_on_gpt2 \\\n",
    "    --train_file ./data/simplification/train.txt \\\n",
    "    --validation_file ./data/simplification/dev.txt \\\n",
    "    --per_device_train_batch_size 2 \\\n",
    "    --per_device_eval_batch_size 2 \\\n",
    "    --block_size 512 \\\n",
    "    --dataset_config_name plain_text \\\n",
    "    --do_train \\\n",
    "    --output_dir models/simplification/RuGPT3-small"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WqwZiumW8WbZ"
   },
   "source": [
    "## Part 2. Decoder LMs evaluation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1QY3mE2NXYIT"
   },
   "outputs": [],
   "source": [
    "from transformers import GPT2LMHeadModel, AutoTokenizer\n",
    "model_name = 'RuGPT3-small'\n",
    "mymodel = GPT2LMHeadModel.from_pretrained('./models/simplification/' + model_name).cuda()\n",
    "mytokenizer = AutoTokenizer.from_pretrained('./models/simplification/' + model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install evaluate\n",
    "#!pip pip install bert_score\n",
    "import tqdm\n",
    "import torch\n",
    "import pandas as pd\n",
    "from evaluate import load\n",
    "import numpy as np\n",
    "\n",
    "sari = load(\"sari\")\n",
    "bertscore = load(\"bertscore\")\n",
    "def add_eos_to_examples(example):\n",
    "    return '</s>%s</s> ==> ' % (example.strip())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Public"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "do_sample = False\n",
    "top_p=None\n",
    "repetition_penalty=None\n",
    "top_k = None\n",
    "num_beams = 5\n",
    "temperature = None\n",
    "data_answers = pd.read_csv(\"./data/simplification/public_test_sents.csv\", sep=\",\")\n",
    "sources = []\n",
    "answers = []\n",
    "ans_for_q = []\n",
    "for source, ans in zip(data_answers[\"INPUT:source\"], data_answers[\"OUTPUT:output\"]):\n",
    "    if source not in sources:\n",
    "        if ans_for_q:\n",
    "            answers.append(ans_for_q)\n",
    "        ans_for_q = []\n",
    "        sources.append(source)\n",
    "        ans_for_q.append(ans)\n",
    "    else:\n",
    "        ans_for_q.append(ans)\n",
    "answers.append(ans_for_q)\n",
    "\n",
    "\n",
    "path_to_file = \"./results/simplification/simplification_\"+model_name.replace('/','_')+'_public.txt'\n",
    "with open(path_to_file, \"w\") as out_file:\n",
    "    for text in tqdm.tqdm(sources):\n",
    "        formatted_text = add_eos_to_examples(text)\n",
    "        input_ids = mytokenizer.encode(formatted_text, return_tensors=\"pt\").cuda()\n",
    "        length = input_ids.detach().cpu().numpy().shape[1]\n",
    "        with torch.no_grad():\n",
    "            out = mymodel.generate(input_ids.cuda(),\n",
    "                                do_sample = do_sample, \n",
    "                                max_length=2 * length + 10,\n",
    "                                num_beams = num_beams)\n",
    "        dec = list(map(mytokenizer.decode, out))[0].split('==>')[1].split('</s>')[0].strip()\n",
    "        out_file.write(dec.replace('\\n','')+'\\n')\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path_to_file) as inf:\n",
    "    predictions = [i.strip().replace('\\n','') for i in inf.readlines()]\n",
    "    \n",
    "print(len(answers), len(predictions), len(sources))\n",
    "print(answers[20], predictions[20], sources[20])\n",
    "\n",
    "\n",
    "print(model_name)\n",
    "print('public results')\n",
    "results = bertscore.compute(predictions=predictions, references=sources, lang=\"ru\")\n",
    "print('BertScore', np.mean(results[\"f1\"]))\n",
    "results = sari.compute(predictions=predictions, sources=sources, references=answers)\n",
    "print('Sari', np.mean(results[\"sari\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Private"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "do_sample = False\n",
    "top_p=None\n",
    "repetition_penalty=None\n",
    "top_k = None\n",
    "num_beams = 5\n",
    "temperature = None\n",
    "data_answers = pd.read_csv(\"./data/simplification/hidden_test_sents.csv\", sep=\",\")\n",
    "sources = []\n",
    "answers = []\n",
    "ans_for_q = []\n",
    "for source, ans in zip(data_answers[\"INPUT:source\"], data_answers[\"OUTPUT:output\"]):\n",
    "    if source not in sources:\n",
    "        if ans_for_q:\n",
    "            answers.append(ans_for_q)\n",
    "        ans_for_q = []\n",
    "        sources.append(source)\n",
    "        ans_for_q.append(ans)\n",
    "    else:\n",
    "        ans_for_q.append(ans)\n",
    "answers.append(ans_for_q)\n",
    "\n",
    "\n",
    "path_to_file = \"./results/simplification/simplification_\"+model_name.replace('/','_')+'_hidden.txt'\n",
    "with open(path_to_file, \"w\") as out_file:\n",
    "    for text in tqdm.tqdm(sources):\n",
    "        formatted_text = add_eos_to_examples(text)\n",
    "        input_ids = mytokenizer.encode(formatted_text, return_tensors=\"pt\").cuda()\n",
    "        with torch.no_grad():\n",
    "            out = mymodel.generate(input_ids.cuda(),\n",
    "                                do_sample = do_sample, \n",
    "                                max_length=2 * length + 10,\n",
    "                                num_beams = num_beams)\n",
    "        dec = list(map(mytokenizer.decode, out))[0].split('==>')[1].split('</s>')[0].strip()\n",
    "        out_file.write(dec.replace('\\n','')+'\\n')\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path_to_file) as inf:\n",
    "    predictions = [i.strip().replace('\\n','') for i in inf.readlines()]\n",
    "    \n",
    "print(len(answers), len(predictions), len(sources))\n",
    "print(answers[20], predictions[20], sources[20])\n",
    "\n",
    "print(model_name)\n",
    "print('public results')\n",
    "results = bertscore.compute(predictions=predictions, references=sources, lang=\"ru\")\n",
    "print('BertScore', np.mean(results[\"f1\"]))\n",
    "results = sari.compute(predictions=predictions, sources=sources, references=answers)\n",
    "print('Sari', np.mean(results[\"sari\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "RuGPT3FinetuneHF.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}