feedback-prize-annotated.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "befa5412",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:29:18.282317Z",
     "iopub.status.busy": "2023-03-13T05:29:18.281798Z",
     "iopub.status.idle": "2023-03-13T05:29:54.207915Z",
     "shell.execute_reply": "2023-03-13T05:29:54.206750Z"
    },
    "papermill": {
     "duration": 35.949417,
     "end_time": "2023-03-13T05:29:54.210610",
     "exception": false,
     "start_time": "2023-03-13T05:29:18.261193",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing /kaggle/input/pip-wheels-feedback/transformers-4.20.1-py3-none-any.whl\r\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers==4.20.1) (2021.11.10)\r\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /opt/conda/lib/python3.7/site-packages (from transformers==4.20.1) (0.12.1)\r\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.7/site-packages (from transformers==4.20.1) (4.64.0)\r\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.7/site-packages (from transformers==4.20.1) (21.3)\r\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from transformers==4.20.1) (3.6.0)\r\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from transformers==4.20.1) (2.27.1)\r\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /opt/conda/lib/python3.7/site-packages (from transformers==4.20.1) (0.8.1)\r\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.7/site-packages (from transformers==4.20.1) (6.0)\r\n",
      "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from transformers==4.20.1) (4.12.0)\r\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.7/site-packages (from transformers==4.20.1) (1.21.6)\r\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.7/site-packages (from huggingface-hub<1.0,>=0.1.0->transformers==4.20.1) (4.1.1)\r\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging>=20.0->transformers==4.20.1) (3.0.9)\r\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->transformers==4.20.1) (3.8.0)\r\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.7/site-packages (from requests->transformers==4.20.1) (2.0.12)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->transformers==4.20.1) (2022.6.15)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->transformers==4.20.1) (3.3)\r\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->transformers==4.20.1) (1.26.9)\r\n",
      "Installing collected packages: transformers\r\n",
      "  Attempting uninstall: transformers\r\n",
      "    Found existing installation: transformers 4.18.0\r\n",
      "    Uninstalling transformers-4.18.0:\r\n",
      "      Successfully uninstalled transformers-4.18.0\r\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",
      "allennlp 2.9.3 requires transformers<4.19,>=4.1, but you have transformers 4.20.1 which is incompatible.\u001b[0m\u001b[31m\r\n",
      "\u001b[0mSuccessfully installed transformers-4.20.1\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m\u001b[33mWARNING: There was an error checking the latest version of pip.\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install ../input/pip-wheels-feedback/transformers-4.20.1-py3-none-any.whl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "184fbab1",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-13T05:29:54.236924Z",
     "iopub.status.busy": "2023-03-13T05:29:54.236062Z",
     "iopub.status.idle": "2023-03-13T05:29:56.526186Z",
     "shell.execute_reply": "2023-03-13T05:29:56.525189Z"
    },
    "papermill": {
     "duration": 2.306259,
     "end_time": "2023-03-13T05:29:56.529013",
     "exception": false,
     "start_time": "2023-03-13T05:29:54.222754",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import torch\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, AutoConfig, AutoModel\n",
    "import os\n",
    "import json\n",
    "import shutil\n",
    "from types import SimpleNamespace  \n",
    "import yaml\n",
    "import multiprocessing as mp\n",
    "from tqdm.auto import tqdm\n",
    "import os\n",
    "import gc\n",
    "\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "894cca51",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-03-12T02:30:40.331879Z",
     "iopub.status.idle": "2023-03-12T02:30:40.333436Z",
     "shell.execute_reply": "2023-03-12T02:30:40.333176Z",
     "shell.execute_reply.started": "2023-03-12T02:30:40.333134Z"
    },
    "papermill": {
     "duration": 0.012268,
     "end_time": "2023-03-13T05:29:56.554126",
     "exception": false,
     "start_time": "2023-03-13T05:29:56.541858",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "706aebae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:29:56.578449Z",
     "iopub.status.busy": "2023-03-13T05:29:56.577926Z",
     "iopub.status.idle": "2023-03-13T05:29:59.870695Z",
     "shell.execute_reply": "2023-03-13T05:29:59.869713Z"
    },
    "papermill": {
     "duration": 3.307689,
     "end_time": "2023-03-13T05:29:59.873211",
     "exception": false,
     "start_time": "2023-03-13T05:29:56.565522",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "N_CORES = mp.cpu_count()\n",
    "\n",
    "ID_SAMPLE = 0.01\n",
    "\n",
    "N_SPLITS = 25\n",
    "\n",
    "NUM_MODELS = 3\n",
    "\n",
    "LABEL_MEANS = np.array([0.57056984, 0.25366517, 0.17576499])\n",
    "\n",
    "LABEL_MEANS[0] *= 1.025\n",
    "LABEL_MEANS[1] *= 0.875\n",
    "LABEL_MEANS = LABEL_MEANS / LABEL_MEANS.sum()\n",
    "\n",
    "sample_submission = pd.read_csv(\"../input/feedback-prize-effectiveness/sample_submission.csv\")\n",
    "sample = False\n",
    "\n",
    "if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n",
    "    data_folder = \"test\"\n",
    "    df = pd.read_csv(\"../input/feedback-prize-effectiveness/test.csv\")\n",
    "    CALC_SCORE = False\n",
    "    \n",
    "else:\n",
    "    data_folder = \"train\"\n",
    "    df = pd.read_csv(\"../input/feedback-prize-2021/train.csv\") #(\"../input/feedback-prize-effectiveness/train.csv\")\n",
    "    df.loc[df.discourse_id == 1623258656795, \"discourse_text\"] = \"This whole thing is point less how they have us in here for two days im missing my education. We could have finished this in one day and had the rest of the week to get back on the track of learning. I've missed both days of weight lifting, algebra, and my world history that i do not want to fail again! If their are any people actually gonna sit down and take the time to read this then\\n\\nDO NOT DO THIS NEXT YEAR\\n\\n.\\n\\nThey are giving us cold lunches. ham and cheese and an apple, I am 16 years old and my body needs proper food. I wouldnt be complaining if they served actual breakfast. but because of Michelle Obama and her healthy diet rule they surve us 1 poptart in the moring. How does the school board expect us to last from 7:05-12:15 on a pop tart? then expect us to get A's, we are more focused on lunch than anything else. I am about done so if you have the time to read this even though this does not count. Bring PROPER_NAME a big Mac from mc donalds, SCHOOL_NAME, (idk area code but its in LOCATION_NAME)       \\xa0    \"\n",
    "    df = df[df.discourse_id != 1623258656795]\n",
    "                \n",
    "    df = df.drop(\n",
    "            columns=['discourse_start', 'discourse_end', 'discourse_type_num', 'predictionstring']).rename(\n",
    "            columns={\"id\": \"essay_id\"}).reindex(\n",
    "            columns=['discourse_id', 'essay_id', 'discourse_text', 'discourse_type']\n",
    "            )\n",
    "   \n",
    "    \n",
    "    df[\"discourse_type_essay\"] = df.groupby(\"essay_id\")[\"discourse_type\"].transform(lambda x: \" \".join(x)).values\n",
    "\n",
    "    ids = df.essay_id.unique()\n",
    "    ids_splits = np.array_split(ids, N_SPLITS)\n",
    "    dfs = []\n",
    "    for split in ids_splits:\n",
    "            df_split = df[df.essay_id.isin(split)]\n",
    "            df_split = df_split.reset_index(drop=True)\n",
    "            dfs.append(df_split)\n",
    "    \n",
    "    CALC_SCORE = False\n",
    "    \n",
    "    if sample:\n",
    "        # take only 1% of essays (41) for analysis\n",
    "        ids = df.essay_id.unique()\n",
    "        np.random.seed(1337)\n",
    "        val_ids = np.random.choice(ids, size=int(ID_SAMPLE*len(ids)), replace=False)\n",
    "        df = df[df.essay_id.isin(val_ids)]\n",
    "        df = df.reset_index(drop=True)\n",
    "        CALC_SCORE = True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "62fd4d96",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:29:59.899029Z",
     "iopub.status.busy": "2023-03-13T05:29:59.898091Z",
     "iopub.status.idle": "2023-03-13T05:29:59.906365Z",
     "shell.execute_reply": "2023-03-13T05:29:59.905415Z"
    },
    "papermill": {
     "duration": 0.023055,
     "end_time": "2023-03-13T05:29:59.908492",
     "exception": false,
     "start_time": "2023-03-13T05:29:59.885437",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "CURR_SPLIT = 4\n",
    "\n",
    "df = dfs[CURR_SPLIT - 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d16bf414",
   "metadata": {
    "papermill": {
     "duration": 0.011262,
     "end_time": "2023-03-13T05:29:59.931504",
     "exception": false,
     "start_time": "2023-03-13T05:29:59.920242",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Feature engineering\n",
    "1. map full essay texts to discourse element\n",
    "2. contain essay length and original order in variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5712635b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:29:59.955461Z",
     "iopub.status.busy": "2023-03-13T05:29:59.955180Z",
     "iopub.status.idle": "2023-03-13T05:31:20.350488Z",
     "shell.execute_reply": "2023-03-13T05:31:20.349465Z"
    },
    "papermill": {
     "duration": 80.409991,
     "end_time": "2023-03-13T05:31:20.352857",
     "exception": false,
     "start_time": "2023-03-13T05:29:59.942866",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15594/15594 [01:19<00:00, 195.36it/s]\n"
     ]
    }
   ],
   "source": [
    "from glob import glob\n",
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "\n",
    "essay_texts = {}\n",
    "for fname in tqdm(glob(f\"../input/feedback-prize-2021/{data_folder}/*.txt\")):\n",
    "    with open(fname) as f:\n",
    "        lines = f.read()\n",
    "        \n",
    "    essay_texts[fname.split(\"/\")[-1][:-4]] = lines\n",
    "\n",
    "df[\"essay_text\"] = df.essay_id.map(essay_texts)\n",
    "\n",
    "del essay_texts\n",
    "gc.collect()\n",
    "\n",
    "df[\"count\"] = df[\"essay_text\"].apply(lambda x: len(x))\n",
    "df[\"orig_order\"] = range(len(df))\n",
    "df = df.sort_values([\"count\", \"essay_id\", \"orig_order\"], ascending=True).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9dbb651d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:31:20.453713Z",
     "iopub.status.busy": "2023-03-13T05:31:20.451927Z",
     "iopub.status.idle": "2023-03-13T05:31:20.458437Z",
     "shell.execute_reply": "2023-03-13T05:31:20.457568Z"
    },
    "papermill": {
     "duration": 0.057745,
     "end_time": "2023-03-13T05:31:20.460472",
     "exception": false,
     "start_time": "2023-03-13T05:31:20.402727",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "initial_df = df.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98715ac8",
   "metadata": {
    "papermill": {
     "duration": 0.048675,
     "end_time": "2023-03-13T05:31:20.557523",
     "exception": false,
     "start_time": "2023-03-13T05:31:20.508848",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Essay Group Model\n",
    "\n",
    "FeedbackDataset initializes a pre-processed dataset: \n",
    "\n",
    "One sample includes one essay. We start by adding a list of all types of the discourses in the essay with a SEP token and then we mark the individual discourses with custom START and END tokens. We then run this sample through the backbone, and pool between START and END tokens for each discourse. The input batch size is always 1, and this gets transformed to a batch size that depends on the number of discourses within the essay. These pooled embeddings then run through a final linear layer predicting the class.\n",
    "> Lead Position Claim Evidence Counterclaim Rebuttal Evidence Counterclaim Concluding Statement [SEP]  [START] Hi, i'm Isaac, i'm going to be writing about how this face on Mars is a natural landform or if there is life on Mars that made it. The story is about how NASA took a picture of Mars and a face was seen on the planet. NASA doesn't know if the landform was created by life on Mars, or if it is just a natural landform. [END]   [START] On my perspective, I think that the face is a natural landform because I dont think that there is any life on Mars. In these next few paragraphs, I'll be talking about how I think that is is a natural landform [END] … more text follows here\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "df351660",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:31:20.656595Z",
     "iopub.status.busy": "2023-03-13T05:31:20.656256Z",
     "iopub.status.idle": "2023-03-13T05:31:20.688440Z",
     "shell.execute_reply": "2023-03-13T05:31:20.687355Z"
    },
    "papermill": {
     "duration": 0.085089,
     "end_time": "2023-03-13T05:31:20.690629",
     "exception": false,
     "start_time": "2023-03-13T05:31:20.605540",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch\n",
    "import collections\n",
    "\n",
    "class FeedbackDataset(Dataset):\n",
    "\n",
    "    def __init__(self, df, mode, cfg):\n",
    "        self.df = df.reset_index(drop=True)\n",
    "        self.mode = mode\n",
    "        self.cfg = cfg\n",
    "        \n",
    "        self.tokenizer = AutoTokenizer.from_pretrained(cfg.architecture.cache_dir)\n",
    "        if self.tokenizer.pad_token is None:\n",
    "            self.tokenizer.pad_token = self.tokenizer.eos_token\n",
    "\n",
    "        if self.tokenizer.sep_token is None:\n",
    "            self.tokenizer.sep_token = \" \"\n",
    "            \n",
    "        if hasattr(cfg.dataset, \"separator\") and len(cfg.dataset.separator):\n",
    "            self.cfg._tokenizer_sep_token = cfg.dataset.separator\n",
    "        else:\n",
    "            self.cfg._tokenizer_sep_token = self.tokenizer.sep_token\n",
    "                                                       \n",
    "        self.text = self.get_texts(self.df, self.cfg, self.tokenizer.sep_token)\n",
    "        \n",
    "        if self.cfg.tokenizer.lowercase:\n",
    "            self.df[\"essay_text\"] = self.df[\"essay_text\"].str.lower()\n",
    "            self.df[\"discourse_text\"] = self.df[\"discourse_text\"].str.lower()\n",
    "\n",
    "        if self.cfg.dataset.group_discourse:\n",
    "            grps = self.df.groupby(\"essay_id\", sort=False)\n",
    "            self.grp_texts = []\n",
    "            \n",
    "            s = 0\n",
    "\n",
    "            for grp in grps.groups:\n",
    "                g = grps.get_group(grp)\n",
    "                t = g.essay_text.values[0]\n",
    "                \n",
    "                # mark the individual discourses with custom START and END tokens\n",
    "                end = 0\n",
    "                for j in range(len(g)):\n",
    "                    # find start idx by characters for a discourse element\n",
    "                    d = g.discourse_text.values[j]\n",
    "                    start = t[end:].find(d.strip()) \n",
    "                    if start == -1:\n",
    "                        print(\"ERROR\")\n",
    "                    # update next element's start and end idx\n",
    "                    start = start + end\n",
    "                    end = start + len(d.strip())\n",
    "                    # main version: do not specify type of each discourse\n",
    "                    # add auxiliary loss to help regularizing the training\n",
    "                    if self.cfg.architecture.aux_type:\n",
    "                        t = t[:start] + f\" [START] \" + t[start:end] + \" [END] \" + t[end:]\n",
    "                    # sub-approach: mark the type of each discourse\n",
    "                    elif self.cfg.architecture.use_type:\n",
    "                        t = t[:start] + f\" [START_{g.discourse_type.values[j]}]  \" + t[start:end] + f\" [END_{g.discourse_type.values[j]}] \" + t[end:] \n",
    "                    else:\n",
    "                        t = t[:start] + f\" [START] {g.discourse_type.values[j]} \" + t[start:end] + \" [END] \" + t[end:] \n",
    "                \n",
    "                # Add the prefix of a list of all types of the discourses in the essay with a SEP token\n",
    "                if hasattr(self.cfg.dataset, \"add_group_types\") and self.cfg.dataset.add_group_types:\n",
    "                    t = \" \".join(g.discourse_type.values) + f\" {self.cfg._tokenizer_sep_token} \" + t\n",
    "                        \n",
    "                self.grp_texts.append(t)\n",
    "\n",
    "                s += len(g)\n",
    "\n",
    "            if self.cfg.dataset.group_discourse:\n",
    "                \n",
    "                self.cfg._tokenizer_start_token_id = []\n",
    "                self.cfg._tokenizer_end_token_id = []\n",
    "\n",
    "                if self.cfg.architecture.use_type:\n",
    "                    for type in sorted(self.df.discourse_type.unique()):\n",
    "                        self.tokenizer.add_tokens([f\"[START_{type}]\"], special_tokens=True)\n",
    "                        self.cfg._tokenizer_start_token_id.append(self.tokenizer.encode(f\"[START_{type}]\")[1])\n",
    "                    \n",
    "                    for type in sorted(self.df.discourse_type.unique()):\n",
    "                        self.tokenizer.add_tokens([f\"[END_{type}]\"], special_tokens=True)\n",
    "                        self.cfg._tokenizer_end_token_id.append(self.tokenizer.encode(f\"[END_{type}]\")[1])\n",
    "\n",
    "                else:\n",
    "                    self.tokenizer.add_tokens([\"[START]\", \"[END]\"], special_tokens=True)\n",
    "                    self.cfg._tokenizer_start_token_id.append(self.tokenizer.encode(f\"[START]\")[1])\n",
    "                    self.cfg._tokenizer_end_token_id.append(self.tokenizer.encode(f\"[END]]\")[1])\n",
    "\n",
    "                print(self.cfg._tokenizer_start_token_id)\n",
    "                print(self.cfg._tokenizer_end_token_id)\n",
    "\n",
    "            if hasattr(self.cfg.tokenizer, \"add_newline_token\") and self.cfg.tokenizer.add_newline_token:\n",
    "                self.tokenizer.add_tokens([f\"\\n\"], special_tokens=True)\n",
    "\n",
    "            self.cfg._tokenizer_size = len(self.tokenizer)\n",
    "            \n",
    "    def __len__(self):\n",
    "        if self.cfg.dataset.group_discourse:\n",
    "            return len(self.grp_texts)\n",
    "        else:\n",
    "            return len(self.df)\n",
    "        \n",
    "    @staticmethod\n",
    "    def collate_fn(batch):\n",
    "        elem = batch[0]\n",
    "\n",
    "        ret = {}\n",
    "        for key in elem:\n",
    "            if key in {\"target\", \"weight\"}:\n",
    "                ret[key] = [d[key].float() for d in batch]\n",
    "            elif key in {\"target_aux\"}:\n",
    "\n",
    "                ret[key] = [d[key].float() for d in batch]\n",
    "            else:\n",
    "                ret[key] = torch.stack([d[key] for d in batch], 0)\n",
    "        return ret\n",
    "            \n",
    "    def batch_to_device(batch, device):\n",
    "\n",
    "        if isinstance(batch, torch.Tensor):\n",
    "            return batch.to(device)\n",
    "        elif isinstance(batch, collections.abc.Mapping):\n",
    "            return {\n",
    "                key: FeedbackDataset.batch_to_device(value, device)\n",
    "                for key, value in batch.items()\n",
    "            }\n",
    "        elif isinstance(batch, collections.abc.Sequence):\n",
    "            return [FeedbackDataset.batch_to_device(value, device) for value in batch]\n",
    "        else:\n",
    "            raise ValueError(f\"Can not move {type(batch)} to device.\")\n",
    "    \n",
    "    @staticmethod\n",
    "    def _lowercase(sample):\n",
    "        if isinstance(sample, str):\n",
    "            return sample.lower()\n",
    "        elif isinstance(sample, Iterable):\n",
    "            return [x.lower() for x in sample]\n",
    "    \n",
    "    def get_texts(cls, df, cfg, separator):\n",
    "        if separator is None:\n",
    "            if hasattr(cfg.dataset, \"separator\") and len(cfg.dataset.separator):\n",
    "                separator = cfg.dataset.separator\n",
    "            else:\n",
    "                separator = getattr(cfg, \"_tokenizer_sep_token\", \"<SEPARATOR>\")\n",
    "\n",
    "        lowercase = hasattr(cfg, \"tokenizer\") and cfg.tokenizer.lowercase\n",
    "        if isinstance(cfg.dataset.text_column, str):\n",
    "            texts = df[cfg.dataset.text_column].astype(str)\n",
    "            if lowercase:\n",
    "                texts = texts.apply(cls._lowercase)\n",
    "            texts = texts.values\n",
    "        else:\n",
    "            columns = list(cfg.dataset.text_column)\n",
    "            join_str = f\" {separator} \"\n",
    "            texts = df[columns].astype(str)\n",
    "            if lowercase:\n",
    "                texts = texts.apply(cls._lowercase)\n",
    "            texts = texts.apply(lambda x: join_str.join(x), axis=1).values\n",
    "\n",
    "        return texts\n",
    "        \n",
    "    def _read_data(self, idx, sample):\n",
    "\n",
    "        if self.cfg.dataset.group_discourse:\n",
    "            text = self.grp_texts[idx]\n",
    "        else:\n",
    "            text = self.text[idx]\n",
    "\n",
    "        if idx == 0:\n",
    "            print(text)\n",
    "            \n",
    "        sample.update(self.encode(text))\n",
    "        return sample\n",
    "    \n",
    "    def encode(self, text):\n",
    "        sample = dict()\n",
    "        encodings = self.tokenizer(\n",
    "            text,\n",
    "            return_tensors=\"pt\",\n",
    "            padding=\"max_length\",\n",
    "            truncation=True,\n",
    "            max_length=self.cfg.tokenizer.max_length,\n",
    "        )\n",
    "        sample[\"input_ids\"] = encodings[\"input_ids\"][0]\n",
    "        sample[\"attention_mask\"] = encodings[\"attention_mask\"][0]\n",
    "        return sample\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        sample = dict()\n",
    "            \n",
    "        sample = self._read_data(idx=idx, sample=sample)\n",
    "\n",
    "        return sample"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f319a4a6",
   "metadata": {
    "papermill": {
     "duration": 0.048947,
     "end_time": "2023-03-13T05:31:20.788593",
     "exception": false,
     "start_time": "2023-03-13T05:31:20.739646",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Pooling each discourse\n",
    "The main idea of this approach is to feed a full essay into the model, and pool each discourse separately and then feed it through the final linear layer for prediction.\n",
    "\n",
    "#### `NLPAllclsTokenPooling`\n",
    "NLPAllclsTokenPooling implements the \"CLS token pooling\" method, which is a common technique used in NLP tasks involving classification. In this method, the output representation of the first token (usually the \"CLS\" token) is used as a fixed-size representation of the entire input sequence. The method involves identifying the position of the \"CLS\" token in the input sequence and computing the average of the hidden states of all the tokens up to that position.\n",
    "\n",
    "The forward method of `NLPAllclsTokenPooling` takes three arguments: `x`, which is a tensor of shape [batch_size, seq_len, hidden_size] representing the hidden states of each token in the input sequence; `attention_mask`, which is a tensor of shape [batch_size, seq_len] containing the attention mask for the input sequence; and `input_ids`, which is a tensor of shape [batch_size, seq_len] containing the input token IDs.\n",
    "\n",
    "The method first checks if `cfg.dataset.group_discourse` is False, in which case it performs token pooling by creating an attention mask based on the CLS and SEP tokens and computing the sum of the hidden states weighted by the attention mask. The result is divided by the sum of the attention mask.\n",
    "\n",
    "If `cfg.dataset.group_discourse` is True, the method performs pooling separately for each group of tokens between start and end tokens. It first finds the indices of the start and end tokens for each group, and then computes the mean of the hidden states for each group, concatenating the result for each group. The output is a list of tensors, where each tensor represents the pooled hidden states for each group in the input sequence.\n",
    "\n",
    "#### `GeMText`\n",
    "On the other hand, GeMText implements the \"Generalized Mean Pooling\" method, which is a more general pooling method that allows for flexible pooling of information across feature dimensions. GeMText computes a generalized mean over the input features along a specified dimension, with the exponent of the mean being a learnable parameter. GeMText is often used as an alternative to average or max pooling, as it can provide better performance in certain scenarios.\n",
    "\n",
    "`GeMText` defines a PyTorch module, which stands for Generalized Mean pooling for Text. Its forward method performs a generalized mean pooling operation on the input tensor `x` and returns the pooled output tensor, using the `p` parameter to determine the degree of pooling.\n",
    "\n",
    "i.e. The input batch size is always 1, and this gets transformed to a batch size that depends on the number of discourses within the essay. These pooled embeddings then run through a final linear layer predicting the class.\n",
    "\n",
    "#### `FeedbackModel`\n",
    "\n",
    "This class defines a feedback model used for classification tasks. The model uses a pre-trained transformer backbone obtained from Hugging Face's `AutoModel` and applies a pooling method from `NLPPoolings` to obtain a fixed-length feature vector from the hidden states of the transformer. The feature vector is then passed through a linear layer to obtain the final logits. The `get_features` method is used to extract the features for the input batch, while the `forward` method is used for inference. The `calculate_loss` parameter is used to determine whether to calculate the loss during training or just return the logits during inference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "05c5ecc1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:31:20.886018Z",
     "iopub.status.busy": "2023-03-13T05:31:20.885661Z",
     "iopub.status.idle": "2023-03-13T05:31:20.907178Z",
     "shell.execute_reply": "2023-03-13T05:31:20.906232Z"
    },
    "papermill": {
     "duration": 0.073147,
     "end_time": "2023-03-13T05:31:20.909280",
     "exception": false,
     "start_time": "2023-03-13T05:31:20.836133",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from torch import nn\n",
    "from torch.nn.parameter import Parameter\n",
    "class NLPAllclsTokenPooling(nn.Module):\n",
    "\n",
    "    def __init__(self, dim, cfg):\n",
    "        super(NLPAllclsTokenPooling, self).__init__()\n",
    "\n",
    "        self.dim = dim\n",
    "        self.feat_mult = 1\n",
    "        if cfg.dataset.group_discourse:\n",
    "            self.feat_mult = 3\n",
    "\n",
    "    def forward(self, x, attention_mask, input_ids, cfg):\n",
    "\n",
    "        if not cfg.dataset.group_discourse:\n",
    "            input_ids_expanded = input_ids.clone().unsqueeze(-1).expand(x.shape)\n",
    "            attention_mask_expanded = torch.zeros_like(input_ids_expanded)\n",
    "\n",
    "            attention_mask_expanded[(input_ids_expanded == cfg._tokenizer_cls_token_id) | (input_ids_expanded == cfg._tokenizer_sep_token_id)] = 1\n",
    "\n",
    "            sum_features = (x * attention_mask_expanded).sum(self.dim)\n",
    "            ret = sum_features / attention_mask_expanded.sum(self.dim).clip(min=1e-8)\n",
    "\n",
    "        else:\n",
    "            ret = []\n",
    "\n",
    "            for j in range(x.shape[0]):\n",
    "\n",
    "\n",
    "                idx0 = torch.where((input_ids[j] >= min(cfg._tokenizer_start_token_id)) & (input_ids[j] <= max(cfg._tokenizer_start_token_id)))[0]\n",
    "                idx1 = torch.where((input_ids[j] >= min(cfg._tokenizer_end_token_id)) & (input_ids[j] <= max(cfg._tokenizer_end_token_id)))[0]\n",
    "\n",
    "                xx = []\n",
    "                for jj in range(len(idx0)):\n",
    "                    xx0 = x[j, idx0[jj]]\n",
    "                    xx1 = x[j, idx1[jj]]\n",
    "                    xx2 = x[j, idx0[jj]+1:idx1[jj]].mean(dim=0)\n",
    "                    xxx = torch.cat([xx0, xx1, xx2]).unsqueeze(0)\n",
    "                    xx.append(xxx)\n",
    "                xx = torch.cat(xx)\n",
    "                ret.append(xx)\n",
    "        \n",
    "        return ret\n",
    "\n",
    "class GeMText(nn.Module):\n",
    "    def __init__(self, dim, cfg, p=3, eps=1e-6):\n",
    "        super(GeMText, self).__init__()\n",
    "        self.dim = dim\n",
    "        self.p = Parameter(torch.ones(1) * p)\n",
    "        self.eps = eps\n",
    "        self.feat_mult = 1\n",
    "\n",
    "    def forward(self, x, attention_mask, input_ids, cfg):\n",
    "        attention_mask_expanded = attention_mask.unsqueeze(-1).expand(x.shape)\n",
    "        x = (x.clamp(min=self.eps) * attention_mask_expanded).pow(self.p).sum(self.dim)\n",
    "        ret = x / attention_mask_expanded.sum(self.dim).clip(min=self.eps)\n",
    "        ret = ret.pow(1 / self.p)\n",
    "        return ret\n",
    "    \n",
    "class NLPPoolings:\n",
    "    _poolings = {\n",
    "        \"All [CLS] token\": NLPAllclsTokenPooling,\n",
    "        \"GeM\": GeMText\n",
    "    }\n",
    "    @classmethod\n",
    "    def get(cls, name):\n",
    "        return cls._poolings.get(name)\n",
    "\n",
    "class FeedbackModel(nn.Module):\n",
    "\n",
    "\n",
    "    def __init__(self, cfg):\n",
    "\n",
    "        super(FeedbackModel, self).__init__()\n",
    "\n",
    "        self.cfg = cfg\n",
    "        self.n_classes = 3\n",
    "        config = AutoConfig.from_pretrained(cfg.architecture.cache_dir)\n",
    "        self.backbone = AutoModel.from_config(config)\n",
    "    \n",
    "        self.backbone.pooler = None\n",
    "        \n",
    "        if self.cfg.dataset.group_discourse:\n",
    "            self.backbone.resize_token_embeddings(cfg._tokenizer_size)\n",
    "        \n",
    "        self.pooling = NLPPoolings.get(self.cfg.architecture.pool)\n",
    "        self.pooling = self.pooling(dim=1, cfg=cfg)  # init pooling and pool over token dimension\n",
    "        \n",
    "        self.head = nn.Linear(self.backbone.config.hidden_size*self.pooling.feat_mult, self.n_classes)\n",
    "\n",
    "    def get_features(self, batch):\n",
    "        attention_mask = batch[\"attention_mask\"]\n",
    "        input_ids = batch[\"input_ids\"]\n",
    "\n",
    "        x = self.backbone(\n",
    "            input_ids=input_ids, attention_mask=attention_mask\n",
    "        ).last_hidden_state\n",
    "\n",
    "        x = self.pooling(x, attention_mask, input_ids, cfg=self.cfg)\n",
    "\n",
    "        if self.cfg.dataset.group_discourse:\n",
    "            x = torch.cat(x)\n",
    "        \n",
    "        if self.cfg.architecture.dropout > 0.0:\n",
    "            x = F.dropout(x, p=self.cfg.architecture.dropout, training=self.training)\n",
    "\n",
    "        return x\n",
    "\n",
    "    def forward(self, batch, calculate_loss=False):\n",
    "        \n",
    "        idx = int(torch.where(batch[\"attention_mask\"] == 1)[1].max())\n",
    "        idx += 1\n",
    "        batch[\"attention_mask\"] = batch[\"attention_mask\"][:, :idx]\n",
    "        batch[\"input_ids\"] = batch[\"input_ids\"][:, :idx]\n",
    "        \n",
    "        x = self.get_features(batch)\n",
    "                \n",
    "        logits = self.head(x)\n",
    "        outputs = {}\n",
    "\n",
    "        outputs[\"logits\"] = logits\n",
    "\n",
    "        return outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6234cbe2",
   "metadata": {
    "papermill": {
     "duration": 0.047699,
     "end_time": "2023-03-13T05:31:21.005082",
     "exception": false,
     "start_time": "2023-03-13T05:31:20.957383",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Backbone\n",
    "\n",
    "For backbones, we could only get deberta-(v3)-large to work."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8aefcfce",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:31:21.106841Z",
     "iopub.status.busy": "2023-03-13T05:31:21.106480Z",
     "iopub.status.idle": "2023-03-13T05:31:22.066392Z",
     "shell.execute_reply": "2023-03-13T05:31:22.065162Z"
    },
    "papermill": {
     "duration": 1.014477,
     "end_time": "2023-03-13T05:31:22.069034",
     "exception": false,
     "start_time": "2023-03-13T05:31:21.054557",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import AutoConfig, AutoModel, PreTrainedModel\n",
    "\n",
    "def create_nlp_backbone(\n",
    "    cfg, model_class=AutoModel, remove_pooling_layer=False\n",
    "):\n",
    "\n",
    "    config = AutoConfig.from_pretrained(\n",
    "        cfg[\"backbone\"], cache_dir=cfg[\"cache_dir\"]\n",
    "    )\n",
    "\n",
    "    kwargs = dict(add_pooling_layer=False) if remove_pooling_layer else dict()\n",
    "    \n",
    "    try:\n",
    "        backbone = model_class.from_config(config, **kwargs)\n",
    "    except TypeError:\n",
    "        backbone = model_class.from_config(config)\n",
    "\n",
    "    return backbone\n",
    "\n",
    "def glorot_uniform(parameter):\n",
    "    nn.init.xavier_uniform_(parameter.data, gain=1.0)\n",
    "\n",
    "\n",
    "class NBMEHead(nn.Module):\n",
    "    def __init__(self, input_dim, output_dim):\n",
    "        super(NBMEHead, self).__init__()\n",
    "        self.dropout1 = nn.Dropout(0.1)\n",
    "        self.dropout2 = nn.Dropout(0.2)\n",
    "        self.dropout3 = nn.Dropout(0.3)\n",
    "        self.dropout4 = nn.Dropout(0.4)\n",
    "        self.dropout5 = nn.Dropout(0.5)\n",
    "        self.classifier = nn.Linear(input_dim, output_dim)\n",
    "        glorot_uniform(self.classifier.weight)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # x is B x S x C\n",
    "        logits1 = self.classifier(self.dropout1(x))\n",
    "        logits2 = self.classifier(self.dropout2(x))\n",
    "        logits3 = self.classifier(self.dropout3(x))\n",
    "        logits4 = self.classifier(self.dropout4(x))\n",
    "        logits5 = self.classifier(self.dropout5(x))\n",
    "\n",
    "        logits = ((logits1 + logits2 + logits3 + logits4 + logits5) / 5)\n",
    "\n",
    "        return logits\n",
    "\n",
    "class ModelYauhen(nn.Module):\n",
    "    def __init__(self, cfg):\n",
    "        super(ModelYauhen, self).__init__()\n",
    "\n",
    "        self.cfg = cfg\n",
    "        self.n_classes = 3\n",
    "        self.backbone = create_nlp_backbone(\n",
    "            self.cfg,\n",
    "            model_class=AutoModel,\n",
    "            remove_pooling_layer=False,\n",
    "        )\n",
    "        self.head = nn.Linear(self.backbone.config.hidden_size, 3)\n",
    "        if self.cfg[\"add_wide_dropout\"]:\n",
    "            self.token_type_head = NBMEHead(self.backbone.config.hidden_size, 3)\n",
    "\n",
    "    def forward(self, batch, calculate_loss=True):\n",
    "        outputs = {}\n",
    "        \n",
    "        idx = int(torch.where(batch[\"attention_mask\"] == 1)[1].max()) \n",
    "        idx += 1\n",
    "        batch[\"attention_mask\"] = batch[\"attention_mask\"][:, :idx]\n",
    "        batch[\"input_ids\"] = batch[\"input_ids\"][:, :idx]\n",
    "        batch[\"word_start_mask\"] = batch[\"word_start_mask\"][:, :idx]\n",
    "        batch[\"word_ids\"] = batch[\"word_ids\"][:, :idx]\n",
    "\n",
    "        outputs[\"word_start_mask\"] = batch[\"word_start_mask\"]\n",
    "\n",
    "        x = self.backbone(\n",
    "            input_ids=batch[\"input_ids\"], attention_mask=batch[\"attention_mask\"]\n",
    "        ).last_hidden_state\n",
    "\n",
    "        for obs_id in range(x.size()[0]):\n",
    "            for w_id in range(int(torch.max(batch[\"word_ids\"][obs_id]).item()) + 1):\n",
    "                chunk_mask = batch[\"word_ids\"][obs_id] == w_id\n",
    "                chunk_logits = x[obs_id] * chunk_mask.unsqueeze(-1)\n",
    "                chunk_logits = chunk_logits.sum(dim=0) / chunk_mask.sum()\n",
    "                x[obs_id][chunk_mask] = chunk_logits\n",
    "\n",
    "        if self.cfg[\"add_wide_dropout\"]:\n",
    "            logits = self.token_type_head(x)\n",
    "        else:\n",
    "            logits = self.head(x)\n",
    "        outputs[\"logits\"] = logits      \n",
    "\n",
    "        return outputs\n",
    "    \n",
    "    \n",
    "from torch.utils.data import Dataset\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "\n",
    "class FeedbackDatasetYauhen(Dataset):\n",
    "    @staticmethod\n",
    "    def _lowercase(sample):\n",
    "        if isinstance(sample, str):\n",
    "            return sample.lower()\n",
    "        elif isinstance(sample, Iterable):\n",
    "            return [x.lower() for x in sample]\n",
    "\n",
    "    def __init__(self, df, cfg):\n",
    "        self.cfg = cfg\n",
    "        self.df = df\n",
    "\n",
    "        self.tokenizer = AutoTokenizer.from_pretrained(\n",
    "            self.cfg[\"backbone\"],\n",
    "            add_prefix_space=True,\n",
    "            use_fast=True,\n",
    "            cache_dir=self.cfg[\"cache_dir\"],\n",
    "        )\n",
    "\n",
    "        self.text = self.get_texts(self.df, self.cfg, self.tokenizer.sep_token)\n",
    "        self.labels = self.df[\"tokens\"].values\n",
    "\n",
    "    @classmethod\n",
    "    def get_texts(cls, df, cfg, separator=None):\n",
    "        texts = df[cfg[\"text_column\"]].values\n",
    "\n",
    "        if cfg[\"lowercase\"]:\n",
    "            texts = [cls._lowercase(x) for x in texts]\n",
    "\n",
    "        return texts\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        sample = dict()\n",
    "            \n",
    "        text = self.text[idx]\n",
    "        \n",
    "        if \"deberta-v3\" in self.cfg[\"backbone\"]:\n",
    "            text = [x.replace(\"\\n\", \"[NL_HYDRO]\") for x in list(text)]\n",
    "            text = [x if not x.isspace() else \"[SP_HYDRO]\" * len(x) for x in text]\n",
    "            tokenizer_input = [text]\n",
    "            raise ValueError(f\"BES {text}\")\n",
    "        else:\n",
    "            if \"add_types\" in self.cfg and self.cfg[\"add_types\"]:\n",
    "                tokenizer_input = [x if x_idx > 0 else x + self.tokenizer.sep_token for x_idx, x in enumerate(list(text))]\n",
    "            else:\n",
    "                tokenizer_input = [list(text)]\n",
    "\n",
    "        encodings = self.tokenizer(\n",
    "            tokenizer_input,\n",
    "            return_tensors=\"pt\",\n",
    "            padding=\"max_length\",\n",
    "            truncation=True,\n",
    "            max_length=2048,\n",
    "            is_split_into_words=True,\n",
    "        )\n",
    "\n",
    "        sample[\"input_ids\"] = encodings[\"input_ids\"][0]\n",
    "        sample[\"attention_mask\"] = encodings[\"attention_mask\"][0]\n",
    "\n",
    "        word_ids = encodings.word_ids(0)\n",
    "        word_ids = [-1 if x is None else x for x in word_ids]\n",
    "        sample[\"word_ids\"] = torch.tensor(word_ids)\n",
    "\n",
    "        word_start_mask = []\n",
    "        lab_idx = -1\n",
    "        for i, word in enumerate(word_ids):\n",
    "            word_start = word > -1 and (i == 0 or word_ids[i - 1] != word)\n",
    "            if word_start:\n",
    "                lab_idx += 1\n",
    "                if self.labels[idx][lab_idx] != 1:\n",
    "                    word_start_mask.append(True)\n",
    "                    continue\n",
    "\n",
    "            word_start_mask.append(False)\n",
    "\n",
    "        sample[\"word_start_mask\"] = torch.tensor(word_start_mask)\n",
    "\n",
    "        return sample\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b24467ff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:31:22.177615Z",
     "iopub.status.busy": "2023-03-13T05:31:22.177242Z",
     "iopub.status.idle": "2023-03-13T05:31:22.186500Z",
     "shell.execute_reply": "2023-03-13T05:31:22.185584Z"
    },
    "papermill": {
     "duration": 0.065814,
     "end_time": "2023-03-13T05:31:22.188722",
     "exception": false,
     "start_time": "2023-03-13T05:31:22.122908",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test = df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f4964034",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:31:22.289841Z",
     "iopub.status.busy": "2023-03-13T05:31:22.289554Z",
     "iopub.status.idle": "2023-03-13T05:31:23.008489Z",
     "shell.execute_reply": "2023-03-13T05:31:23.007112Z"
    },
    "papermill": {
     "duration": 0.772226,
     "end_time": "2023-03-13T05:31:23.011084",
     "exception": false,
     "start_time": "2023-03-13T05:31:22.238858",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 624/624 [00:00<00:00, 901.89it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "all_obs = []\n",
    "\n",
    "for name, gr in tqdm(test.groupby(\"essay_id\", sort=False)):\n",
    "    essay_text_start_end = gr.essay_text.values[0]\n",
    "    token_labels = []\n",
    "    token_obs = []\n",
    "    end_pos = 0\n",
    "    \n",
    "    for idx, row in gr.reset_index(drop=True).iterrows():\n",
    "        target_text = row[\"discourse_type\"] + \" \" + row[\"discourse_text\"].strip()\n",
    "        \n",
    "        essay_text_start_end = essay_text_start_end[:end_pos] + essay_text_start_end[end_pos:].replace(row[\"discourse_text\"].strip(), target_text, 1)\n",
    "        \n",
    "        start_pos = essay_text_start_end[end_pos:].find(target_text)\n",
    "        if start_pos == -1:\n",
    "            print(row)\n",
    "            continue\n",
    "        start_pos += end_pos\n",
    "        \n",
    "        if idx == 0 and start_pos > 0:\n",
    "            token_labels.append(1)\n",
    "            token_obs.append(essay_text_start_end[:start_pos])\n",
    "        \n",
    "        if start_pos > end_pos and end_pos > 0:\n",
    "            token_labels.append(1)\n",
    "            token_obs.append(essay_text_start_end[end_pos:start_pos])\n",
    "  \n",
    "        end_pos = start_pos + len(target_text)\n",
    "        token_labels.append(0)\n",
    "        token_obs.append(essay_text_start_end[start_pos: end_pos])\n",
    "            \n",
    "        if idx == len(gr) - 1 and end_pos < len(essay_text_start_end):\n",
    "            token_labels.append(1)\n",
    "            token_obs.append(essay_text_start_end[end_pos:])\n",
    "            \n",
    "    if len(token_labels) != len(token_obs):\n",
    "        raise ValueError()\n",
    "            \n",
    "    all_obs.append((name, token_labels, token_obs))\n",
    "\n",
    "tt = pd.DataFrame(all_obs, columns=[\"essay_id\", \"tokens\", \"essay_text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6a831d7b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:31:23.115380Z",
     "iopub.status.busy": "2023-03-13T05:31:23.115069Z",
     "iopub.status.idle": "2023-03-13T05:31:23.851136Z",
     "shell.execute_reply": "2023-03-13T05:31:23.850037Z"
    },
    "papermill": {
     "duration": 0.790673,
     "end_time": "2023-03-13T05:31:23.853654",
     "exception": false,
     "start_time": "2023-03-13T05:31:23.062981",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 624/624 [00:00<00:00, 878.14it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "all_obs = []\n",
    "\n",
    "for name, gr in tqdm(test.groupby(\"essay_id\", sort=False)):\n",
    "    essay_text_start_end = gr.essay_text.values[0]\n",
    "    token_labels = []\n",
    "    token_obs = []\n",
    "    end_pos = 0\n",
    "    \n",
    "    token_obs.append(\" \".join(gr.discourse_type.to_list()))\n",
    "    token_labels.append(1)\n",
    "    \n",
    "    for idx, row in gr.reset_index(drop=True).iterrows():\n",
    "        target_text = row[\"discourse_type\"] + \" \" + row[\"discourse_text\"].strip()\n",
    "        \n",
    "        essay_text_start_end = essay_text_start_end[:end_pos] + essay_text_start_end[end_pos:].replace(row[\"discourse_text\"].strip(), target_text, 1)\n",
    "        \n",
    "        start_pos = essay_text_start_end[end_pos:].find(target_text)\n",
    "        if start_pos == -1:\n",
    "            print(row)\n",
    "            continue\n",
    "        start_pos += end_pos\n",
    "        \n",
    "        if idx == 0 and start_pos > 0:\n",
    "            token_labels.append(1)\n",
    "            token_obs.append(essay_text_start_end[:start_pos])\n",
    "        \n",
    "        if start_pos > end_pos and end_pos > 0:\n",
    "            token_labels.append(1)\n",
    "            token_obs.append(essay_text_start_end[end_pos:start_pos])\n",
    "  \n",
    "        end_pos = start_pos + len(target_text)\n",
    "        token_labels.append(0)\n",
    "        token_obs.append(essay_text_start_end[start_pos: end_pos])\n",
    "            \n",
    "        if idx == len(gr) - 1 and end_pos < len(essay_text_start_end):\n",
    "            token_labels.append(1)\n",
    "            token_obs.append(essay_text_start_end[end_pos:])\n",
    "            \n",
    "    if len(token_labels) != len(token_obs):\n",
    "        raise ValueError()\n",
    "            \n",
    "    all_obs.append((name, token_labels, token_obs))\n",
    "\n",
    "tt_v2 = pd.DataFrame(all_obs, columns=[\"essay_id\", \"tokens\", \"essay_text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a3315443",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:31:23.953717Z",
     "iopub.status.busy": "2023-03-13T05:31:23.952830Z",
     "iopub.status.idle": "2023-03-13T05:31:23.970914Z",
     "shell.execute_reply": "2023-03-13T05:31:23.970034Z"
    },
    "papermill": {
     "duration": 0.06939,
     "end_time": "2023-03-13T05:31:23.972863",
     "exception": false,
     "start_time": "2023-03-13T05:31:23.903473",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def run_predictions_philipp(exp_name, df, BS=1, num_models=NUM_MODELS):\n",
    "    \n",
    "    cfg = yaml.safe_load(open(f\"../input/{exp_name}/cfg.yaml\").read())\n",
    "    for k,v in cfg.items():\n",
    "        if type(v) == dict:\n",
    "            cfg[k] = SimpleNamespace(**v)\n",
    "    cfg = SimpleNamespace(**cfg)\n",
    "\n",
    "    if cfg.architecture.backbone == 'microsoft/deberta-v3-large':\n",
    "        cfg.architecture.cache_dir = \"../input/deberta-v3-large/\"\n",
    "    elif cfg.architecture.backbone == 'microsoft/deberta-v3-small':\n",
    "        cfg.architecture.cache_dir = \"../input/deberta-v3-small/\"\n",
    "    elif cfg.architecture.backbone == 'microsoft/deberta-v3-base':\n",
    "        cfg.architecture.cache_dir = \"../input/deberta-v3-lbase/\"\n",
    "\n",
    "    ds = FeedbackDataset(df.iloc[:], mode=\"test\", cfg=cfg)\n",
    "    \n",
    "    preds_all = []\n",
    "    for fold in range(num_models):\n",
    "        print(f\"running model {fold}\")\n",
    "        \n",
    "        model = FeedbackModel(cfg).to(\"cuda\").eval()\n",
    "    \n",
    "        d = torch.load(f\"../input/{exp_name}/checkpoint-fold{fold}.pth\", map_location=\"cpu\")\n",
    "\n",
    "        model_weights = d[\"model\"]\n",
    "        model_weights = {k.replace(\"module.\", \"\"): v for k, v in model_weights.items()}\n",
    "        \n",
    "        for k in list(model_weights.keys()):\n",
    "            if \"aux\" in k or \"loss_fn\" in k:\n",
    "                del model_weights[k]\n",
    "\n",
    "        model.load_state_dict(collections.OrderedDict(model_weights), strict=True)\n",
    "        \n",
    "        del d\n",
    "        del model_weights\n",
    "        gc.collect() \n",
    "    \n",
    "        batch_size = BS\n",
    "        dl = DataLoader(ds, shuffle=False, batch_size = batch_size, num_workers = N_CORES)\n",
    "\n",
    "        with torch.no_grad():    \n",
    "            preds = []\n",
    "            for batch in tqdm(dl):\n",
    "\n",
    "                batch = FeedbackDataset.batch_to_device(batch, \"cuda\")\n",
    "                out = model(batch)\n",
    "                preds.append(out[\"logits\"].float().softmax(dim=1).detach().cpu().numpy())\n",
    "\n",
    "        preds_all.append(np.concatenate(preds, axis=0))\n",
    "        \n",
    "        del model\n",
    "        del dl\n",
    "        gc.collect()\n",
    "        \n",
    "    del ds\n",
    "    \n",
    "    \n",
    "    preds = np.mean(preds_all, axis=0)\n",
    "    \n",
    "    return preds\n",
    "\n",
    "def run_predictions_yauhen(all_cfgs, df, yauhen_batch_size=1):\n",
    "    ds = FeedbackDatasetYauhen(df=df, cfg=all_cfgs[0])\n",
    "        \n",
    "    preds_all = []\n",
    "\n",
    "    for params in all_cfgs:\n",
    "\n",
    "        model = ModelYauhen(params).to(\"cuda\").eval()\n",
    "\n",
    "        d = torch.load(params[\"path\"], map_location=\"cpu\")\n",
    "\n",
    "        model_weights = d[\"model\"]\n",
    "        model_weights = {k.replace(\"module.\", \"\"): v for k, v in model_weights.items()}\n",
    "        model.load_state_dict(collections.OrderedDict(model_weights), strict=True)\n",
    "        \n",
    "        del d\n",
    "        del model_weights\n",
    "        gc.collect() \n",
    "    \n",
    "        dl = DataLoader(ds, shuffle=False, batch_size = yauhen_batch_size, num_workers = N_CORES)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            \n",
    "            preds = []\n",
    "            for batch in tqdm(dl):\n",
    "                texts = {\n",
    "                key: value.to(\"cuda\")\n",
    "                for key, value in batch.items()\n",
    "            }\n",
    "                output = model.forward(texts, calculate_loss=False)\n",
    "\n",
    "                val = (\n",
    "                        torch.softmax(output[\"logits\"][output[\"word_start_mask\"]], dim=1).detach().cpu().numpy()\n",
    "                    )\n",
    "\n",
    "                preds.append(val)\n",
    "\n",
    "        preds_all.append(np.concatenate(preds, axis=0))\n",
    "        \n",
    "        del model\n",
    "        del dl\n",
    "        gc.collect()\n",
    "        \n",
    "    del ds\n",
    "    \n",
    "    preds = np.mean(preds_all, axis=0)\n",
    "    \n",
    "    return preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0e64faad",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:31:24.073350Z",
     "iopub.status.busy": "2023-03-13T05:31:24.073002Z",
     "iopub.status.idle": "2023-03-13T05:31:24.077409Z",
     "shell.execute_reply": "2023-03-13T05:31:24.076366Z"
    },
    "papermill": {
     "duration": 0.057055,
     "end_time": "2023-03-13T05:31:24.079319",
     "exception": false,
     "start_time": "2023-03-13T05:31:24.022264",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "preds = []\n",
    "weights = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "faf206eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:31:24.178414Z",
     "iopub.status.busy": "2023-03-13T05:31:24.177608Z",
     "iopub.status.idle": "2023-03-13T05:31:24.183358Z",
     "shell.execute_reply": "2023-03-13T05:31:24.182522Z"
    },
    "papermill": {
     "duration": 0.057128,
     "end_time": "2023-03-13T05:31:24.185383",
     "exception": false,
     "start_time": "2023-03-13T05:31:24.128255",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def scale_probs(pp_single):\n",
    "    pp = pp_single.copy()\n",
    "\n",
    "    for _ in range(100):\n",
    "\n",
    "        pp = pp * (LABEL_MEANS.reshape(1,3) / pp.mean(axis=0))\n",
    "\n",
    "        pp = pp / pp.sum(axis=1, keepdims=True)\n",
    "        \n",
    "    return pp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8390b5c",
   "metadata": {
    "papermill": {
     "duration": 0.048398,
     "end_time": "2023-03-13T05:31:24.281941",
     "exception": false,
     "start_time": "2023-03-13T05:31:24.233543",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Further models\n",
    "For diversity, we added the following models with minor impact to our solution:\n",
    "\n",
    "Simple Deberta classification on Discourse input only\n",
    "Bag-of-words LightGBM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4816a8bc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:31:24.382631Z",
     "iopub.status.busy": "2023-03-13T05:31:24.382268Z",
     "iopub.status.idle": "2023-03-13T05:31:37.466824Z",
     "shell.execute_reply": "2023-03-13T05:31:37.465586Z"
    },
    "papermill": {
     "duration": 13.138177,
     "end_time": "2023-03-13T05:31:37.469536",
     "exception": false,
     "start_time": "2023-03-13T05:31:24.331359",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6988, 6612)\n",
      "[0.43257522 0.52945943 0.03796536]\n"
     ]
    }
   ],
   "source": [
    "import lightgbm\n",
    "def get_features(df, vects):\n",
    "    vect_discourse = vects[\"vect_discourse\"]\n",
    "    X = vect_discourse.transform(df[\"discourse_text\"]).A\n",
    "\n",
    "    X = np.concatenate([X, X.sum(axis=1).reshape(-1,1)], axis=1)\n",
    "\n",
    "    vect_essay = vects[\"vect_essay\"]\n",
    "    XX = vect_essay.transform(df[\"essay_text\"]).A\n",
    "    XX = np.concatenate([XX, XX.sum(axis=1).reshape(-1,1)], axis=1)\n",
    "    X = np.concatenate([X, XX], axis=1)\n",
    "\n",
    "    vect_type = vects[\"vect_type\"]\n",
    "    X = np.concatenate([X, vect_type.transform(df[\"discourse_type\"]).A], axis=1)\n",
    "\n",
    "    vect_type_essay = vects[\"vect_type_essay\"]\n",
    "    X = np.concatenate([X, vect_type_essay.transform(df[\"discourse_type_essay\"]).A], axis=1)\n",
    "\n",
    "    f = \"rel_rank\"\n",
    "    X = np.concatenate([X, df[f].values.reshape(-1,1)], axis=1)\n",
    "    print(X.shape)\n",
    "\n",
    "    return X\n",
    "\n",
    "def run_predictions_lgb(exp_name, df):\n",
    "    df[\"rank\"] = df.groupby(\"essay_id\")[\"discourse_type\"].transform(lambda x: np.arange(len(x))).values\n",
    "    df[\"length\"] = df.groupby(\"essay_id\")[\"discourse_type\"].transform(lambda x: len(x)).values\n",
    "    df[\"rel_rank\"] = df[\"rank\"] / df[\"length\"]\n",
    "    \n",
    "    pps = []\n",
    "    for fold in [-1]:\n",
    "    \n",
    "        vects = pd.read_pickle(f\"../input/{exp_name}/fold{fold}/vectorizers.p\")\n",
    "\n",
    "        X = get_features(df, vects)\n",
    "\n",
    "        clf = lightgbm.Booster(model_file=f\"../input/{exp_name}/fold{fold}/model_seed0.txt\")\n",
    "\n",
    "        preds = clf.predict(X)\n",
    "        pps.append(preds)\n",
    "    preds = np.mean(pps, axis=0)\n",
    "    \n",
    "    print(preds.mean(axis=0))\n",
    "    \n",
    "    return preds\n",
    "\n",
    "preds.append(scale_probs(run_predictions_lgb(\"lgb-v0\", df)))\n",
    "weights.append(1.131471)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2aad3b8",
   "metadata": {
    "papermill": {
     "duration": 0.049653,
     "end_time": "2023-03-13T05:31:37.569855",
     "exception": false,
     "start_time": "2023-03-13T05:31:37.520202",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Ensembling\n",
    "For ensembling different models we resorted to directly optimizing the blending weights between the models. Interestingly, we also had several models with negative weights, but this worked for us both on CV as well as LB."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "75e5e6f7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:31:37.673688Z",
     "iopub.status.busy": "2023-03-13T05:31:37.673290Z",
     "iopub.status.idle": "2023-03-13T05:49:18.464489Z",
     "shell.execute_reply": "2023-03-13T05:49:18.463361Z"
    },
    "papermill": {
     "duration": 1060.846487,
     "end_time": "2023-03-13T05:49:18.466868",
     "exception": false,
     "start_time": "2023-03-13T05:31:37.620381",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 156/156 [02:59<00:00,  1.15s/it]\n",
      "100%|██████████| 156/156 [03:07<00:00,  1.20s/it]\n",
      "100%|██████████| 156/156 [03:13<00:00,  1.24s/it]\n",
      "100%|██████████| 156/156 [03:13<00:00,  1.24s/it]\n",
      "100%|██████████| 156/156 [03:13<00:00,  1.24s/it]\n"
     ]
    }
   ],
   "source": [
    "cfg_1 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/pseudo-75-datasets-v4/olivine-spaniel-ff/checkpoint-fold0.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "      }\n",
    "\n",
    "cfg_2 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/pseudo-75-datasets-v4/olivine-spaniel-ff/checkpoint-fold1.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "      }\n",
    "\n",
    "cfg_3 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/pseudo-75-datasets-v4/olivine-spaniel-ff/checkpoint-fold2.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "      }\n",
    "\n",
    "cfg_4 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/pseudo-75-datasets-v4/olivine-spaniel-ff/checkpoint-fold3.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "      }\n",
    "\n",
    "cfg_5 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/pseudo-75-datasets-v4/olivine-spaniel-ff/checkpoint-fold4.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "      }\n",
    "\n",
    "all_cfgs = [cfg_1, cfg_2, cfg_3, cfg_4, cfg_5]\n",
    "\n",
    "preds.append(scale_probs(run_predictions_yauhen(all_cfgs, tt, yauhen_batch_size=4)))\n",
    "weights.append(-1.795161)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8b2b6c04",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T05:49:18.691555Z",
     "iopub.status.busy": "2023-03-13T05:49:18.691174Z",
     "iopub.status.idle": "2023-03-13T06:00:39.121407Z",
     "shell.execute_reply": "2023-03-13T06:00:39.120235Z"
    },
    "papermill": {
     "duration": 680.56668,
     "end_time": "2023-03-13T06:00:39.124425",
     "exception": false,
     "start_time": "2023-03-13T05:49:18.557745",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 624/624 [03:23<00:00,  3.06it/s]\n",
      "100%|██████████| 624/624 [03:23<00:00,  3.06it/s]\n",
      "100%|██████████| 624/624 [03:23<00:00,  3.06it/s]\n"
     ]
    }
   ],
   "source": [
    "cfg_1 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/meteoric-bettong-v2-ff/checkpoint-fold0.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "      }\n",
    "\n",
    "cfg_2 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/meteoric-bettong-v2-ff/checkpoint-fold1.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "      }\n",
    "\n",
    "cfg_3 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/meteoric-bettong-v2-ff/checkpoint-fold2.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "      }\n",
    "\n",
    "all_cfgs = [cfg_1, cfg_2, cfg_3]\n",
    "\n",
    "preds.append(scale_probs(run_predictions_yauhen(all_cfgs, tt)))\n",
    "weights.append(-0.455578)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4e28de9",
   "metadata": {
    "papermill": {
     "duration": 0.277815,
     "end_time": "2023-03-13T06:00:39.713401",
     "exception": false,
     "start_time": "2023-03-13T06:00:39.435586",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Cross validation\n",
    "\n",
    "For splitting the folds, we just used an efficiency-stratified split on essays.\n",
    "\n",
    "As the data is small to medium size and the metric is log loss, the scores can vary between different runs. This is typical for deep learning models as they are quite dependent on the seed at hand that influences weight initializations, batching, or augmentations. Yet, this means one should not judge model performance on single seeds, and it is better to always evaluate on multiple seeds.\n",
    "\n",
    "Given that model training was quite fast, we thus only relied on checking blends of 3 seeds for each model. Also, single model scores did not correlate well here with their ability to blend into larger ensembles. So a better individual model could have quite a worse performance in the blend, diversity really mattered here. Consequently, we also always checked models in the blend, even if they did not seem too promising on an individual basis. Similar to how we checked CV, we then always subbed a blend of 3 seeds of models trained on the full data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "287189d9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T06:00:40.133831Z",
     "iopub.status.busy": "2023-03-13T06:00:40.133441Z",
     "iopub.status.idle": "2023-03-13T06:22:47.970534Z",
     "shell.execute_reply": "2023-03-13T06:22:47.969515Z"
    },
    "papermill": {
     "duration": 1328.027183,
     "end_time": "2023-03-13T06:22:47.972860",
     "exception": false,
     "start_time": "2023-03-13T06:00:39.945677",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 156/156 [03:19<00:00,  1.28s/it]\n",
      "100%|██████████| 156/156 [03:19<00:00,  1.28s/it]\n",
      "100%|██████████| 156/156 [03:19<00:00,  1.28s/it]\n",
      "100%|██████████| 156/156 [03:19<00:00,  1.28s/it]\n",
      "100%|██████████| 156/156 [03:19<00:00,  1.28s/it]\n",
      "100%|██████████| 156/156 [03:19<00:00,  1.28s/it]\n"
     ]
    }
   ],
   "source": [
    "cfg_1 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/saffron-rook-ff/checkpoint_fold_0.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "cfg_2 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/saffron-rook-ff/checkpoint_fold_1.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "cfg_3 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/saffron-rook-ff/checkpoint_fold_2.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "all_cfgs = [cfg_1, cfg_2, cfg_3]\n",
    "\n",
    "seed_1 = run_predictions_yauhen(all_cfgs, tt_v2, yauhen_batch_size=4)\n",
    "\n",
    "cfg_1 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/saffron-rook-v2-ff/checkpoint_fold_0.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "cfg_2 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/saffron-rook-v2-ff/checkpoint_fold_1.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "cfg_3 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/saffron-rook-v2-ff/checkpoint_fold_2.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "all_cfgs = [cfg_1, cfg_2, cfg_3]\n",
    "\n",
    "seed_2 = run_predictions_yauhen(all_cfgs, tt_v2, yauhen_batch_size=4)\n",
    "\n",
    "seeds = (seed_1 + seed_2) / 2\n",
    "\n",
    "preds.append(scale_probs(seeds))\n",
    "weights.append(0.639652)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f2cbff42",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T06:22:48.518870Z",
     "iopub.status.busy": "2023-03-13T06:22:48.518502Z",
     "iopub.status.idle": "2023-03-13T06:59:06.586349Z",
     "shell.execute_reply": "2023-03-13T06:59:06.584755Z"
    },
    "papermill": {
     "duration": 2178.321964,
     "end_time": "2023-03-13T06:59:06.591080",
     "exception": false,
     "start_time": "2023-03-13T06:22:48.269116",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 156/156 [06:34<00:00,  2.53s/it]\n",
      "100%|██████████| 156/156 [06:34<00:00,  2.53s/it]\n",
      "100%|██████████| 156/156 [06:34<00:00,  2.53s/it]\n",
      "100%|██████████| 156/156 [06:34<00:00,  2.53s/it]\n",
      "100%|██████████| 156/156 [06:34<00:00,  2.53s/it]\n"
     ]
    }
   ],
   "source": [
    "cfg_1 = {\"backbone\": \"../input/debertaxlarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertaxlarge\",\n",
    "       \"path\": \"../input/big-ocelot-ff/checkpoint_fold_0.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "cfg_2 = {\"backbone\": \"../input/debertaxlarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertaxlarge\",\n",
    "       \"path\": \"../input/big-ocelot-ff/checkpoint_fold_1.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "cfg_3 = {\"backbone\": \"../input/debertaxlarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertaxlarge\",\n",
    "       \"path\": \"../input/big-ocelot-ff/checkpoint_fold_2.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "cfg_4 = {\"backbone\": \"../input/debertaxlarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertaxlarge\",\n",
    "       \"path\": \"../input/big-ocelot-ff-v3/checkpoint-fold0.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "cfg_5 = {\"backbone\": \"../input/debertaxlarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertaxlarge\",\n",
    "       \"path\": \"../input/big-ocelot-ff-v3/checkpoint-fold1.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "all_cfgs = [cfg_1, cfg_2, cfg_3, cfg_4, cfg_5]\n",
    "\n",
    "preds.append(scale_probs(run_predictions_yauhen(all_cfgs, tt_v2, yauhen_batch_size=4)))\n",
    "weights.append(1.586749)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6ccb4f37",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T06:59:07.313683Z",
     "iopub.status.busy": "2023-03-13T06:59:07.313300Z",
     "iopub.status.idle": "2023-03-13T07:20:40.526347Z",
     "shell.execute_reply": "2023-03-13T07:20:40.525174Z"
    },
    "papermill": {
     "duration": 1293.579846,
     "end_time": "2023-03-13T07:20:40.528714",
     "exception": false,
     "start_time": "2023-03-13T06:59:06.948868",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 156/156 [03:13<00:00,  1.24s/it]\n",
      "100%|██████████| 156/156 [03:13<00:00,  1.24s/it]\n",
      "100%|██████████| 156/156 [03:13<00:00,  1.24s/it]\n",
      "100%|██████████| 156/156 [03:13<00:00,  1.24s/it]\n",
      "100%|██████████| 156/156 [03:13<00:00,  1.24s/it]\n",
      "100%|██████████| 156/156 [03:13<00:00,  1.24s/it]\n"
     ]
    }
   ],
   "source": [
    "cfg_1 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/shrewd-rook-3ep-ff/checkpoint-fold0.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "      }\n",
    "\n",
    "cfg_2 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/shrewd-rook-3ep-ff/checkpoint-fold1.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "      }\n",
    "\n",
    "cfg_3 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/shrewd-rook-3ep-ff/checkpoint-fold2.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "      }\n",
    "\n",
    "cfg_4 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/shrewd-rook-3ep-ff-v2/checkpoint-fold0.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "      }\n",
    "\n",
    "cfg_5 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/shrewd-rook-3ep-ff-v2/checkpoint-fold1.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "      }\n",
    "\n",
    "cfg_6 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/shrewd-rook-3ep-ff-v2/checkpoint-fold2.pth\",\n",
    "         \"add_wide_dropout\": False,\n",
    "      }\n",
    "\n",
    "all_cfgs = [cfg_1, cfg_2, cfg_3, cfg_4, cfg_5, cfg_6]\n",
    "\n",
    "preds.append(scale_probs(run_predictions_yauhen(all_cfgs, tt, yauhen_batch_size=4)))\n",
    "weights.append(0.983297)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "5835e849",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T07:20:41.218592Z",
     "iopub.status.busy": "2023-03-13T07:20:41.217821Z",
     "iopub.status.idle": "2023-03-13T07:42:57.585544Z",
     "shell.execute_reply": "2023-03-13T07:42:57.584441Z"
    },
    "papermill": {
     "duration": 1336.687428,
     "end_time": "2023-03-13T07:42:57.588018",
     "exception": false,
     "start_time": "2023-03-13T07:20:40.900590",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 156/156 [03:19<00:00,  1.28s/it]\n",
      "100%|██████████| 156/156 [03:19<00:00,  1.28s/it]\n",
      "100%|██████████| 156/156 [03:19<00:00,  1.28s/it]\n",
      "100%|██████████| 156/156 [03:19<00:00,  1.28s/it]\n",
      "100%|██████████| 156/156 [03:19<00:00,  1.28s/it]\n",
      "100%|██████████| 156/156 [03:19<00:00,  1.28s/it]\n"
     ]
    }
   ],
   "source": [
    "cfg_1 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/conscious-uakari-ff/checkpoint-fold0.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "cfg_2 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/conscious-uakari-ff/checkpoint-fold1.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "cfg_3 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/conscious-uakari-ff/checkpoint-fold2.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "cfg_4 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/conscious-uakari-ff-v2/checkpoint-fold0.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "cfg_5 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/conscious-uakari-ff-v2/checkpoint-fold1.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "cfg_6 = {\"backbone\": \"../input/debertalarge\",\n",
    "       \"lowercase\": False,\n",
    "       \"text_column\": \"essay_text\",\n",
    "       \"cache_dir\": \"../input/debertalarge\",\n",
    "       \"path\": \"../input/conscious-uakari-ff-v2/checkpoint-fold2.pth\",\n",
    "         \"add_wide_dropout\": True,\n",
    "         \"add_types\": True,\n",
    "      }\n",
    "\n",
    "all_cfgs = [cfg_1, cfg_2, cfg_3, cfg_4, cfg_5, cfg_6]\n",
    "\n",
    "\n",
    "preds.append(scale_probs(run_predictions_yauhen(all_cfgs, tt_v2, yauhen_batch_size=4)))\n",
    "weights.append(3.707194)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "aba4a09e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T07:42:58.461158Z",
     "iopub.status.busy": "2023-03-13T07:42:58.460760Z",
     "iopub.status.idle": "2023-03-13T07:59:25.898492Z",
     "shell.execute_reply": "2023-03-13T07:59:25.897408Z"
    },
    "papermill": {
     "duration": 987.827804,
     "end_time": "2023-03-13T07:59:25.900803",
     "exception": false,
     "start_time": "2023-03-13T07:42:58.072999",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/transformers/convert_slow_tokenizer.py:435: UserWarning: The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.\n",
      "  \"The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/874 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead [SEP] Lead Position Claim Evidence [SEP] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 874/874 [05:05<00:00,  2.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/874 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead [SEP] Lead Position Claim Evidence [SEP] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 874/874 [05:04<00:00,  2.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/874 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead [SEP] Lead Position Claim Evidence [SEP] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 874/874 [05:04<00:00,  2.87it/s]\n"
     ]
    }
   ],
   "source": [
    "preds.append(scale_probs(run_predictions_philipp(\"valiant-degu-ff-2\", df, BS=8)))\n",
    "weights.append(0.590190)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "bfac1a4a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T07:59:27.081818Z",
     "iopub.status.busy": "2023-03-13T07:59:27.081366Z",
     "iopub.status.idle": "2023-03-13T08:19:27.103880Z",
     "shell.execute_reply": "2023-03-13T08:19:27.102787Z"
    },
    "papermill": {
     "duration": 1200.671318,
     "end_time": "2023-03-13T08:19:27.106436",
     "exception": false,
     "start_time": "2023-03-13T07:59:26.435118",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128001]\n",
      "[128002]\n",
      "running model 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] Lead In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] Position I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] Claim like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Evidence Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:23<00:00,  2.61s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] Lead In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] Position I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] Claim like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Evidence Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:23<00:00,  2.61s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] Lead In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] Position I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] Claim like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Evidence Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:24<00:00,  2.62s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] Lead In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] Position I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] Claim like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Evidence Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:24<00:00,  2.62s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] Lead In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] Position I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] Claim like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Evidence Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:24<00:00,  2.62s/it]\n"
     ]
    }
   ],
   "source": [
    "preds.append(scale_probs(run_predictions_philipp(\"axiomatic-vulture-ff-v2\", df, BS=8, num_models=5)))\n",
    "weights.append(0.964377)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a75e9087",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T08:19:28.188384Z",
     "iopub.status.busy": "2023-03-13T08:19:28.188017Z",
     "iopub.status.idle": "2023-03-13T08:31:26.016505Z",
     "shell.execute_reply": "2023-03-13T08:31:26.015369Z"
    },
    "papermill": {
     "duration": 718.373161,
     "end_time": "2023-03-13T08:31:26.019322",
     "exception": false,
     "start_time": "2023-03-13T08:19:27.646161",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128001]\n",
      "[128002]\n",
      "running model 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/624 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 624/624 [03:31<00:00,  2.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/624 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 624/624 [03:31<00:00,  2.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/624 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 624/624 [03:31<00:00,  2.95it/s]\n"
     ]
    }
   ],
   "source": [
    "preds.append(scale_probs(run_predictions_philipp(\"smart-bumblebee-ff\", df)))\n",
    "weights.append(0.366988)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ca63fe0c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T08:31:27.291302Z",
     "iopub.status.busy": "2023-03-13T08:31:27.290881Z",
     "iopub.status.idle": "2023-03-13T08:49:38.670073Z",
     "shell.execute_reply": "2023-03-13T08:49:38.661306Z"
    },
    "papermill": {
     "duration": 1092.015221,
     "end_time": "2023-03-13T08:49:38.674208",
     "exception": false,
     "start_time": "2023-03-13T08:31:26.658987",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128001]\n",
      "[128002]\n",
      "running model 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:12<00:00,  2.47s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:12<00:00,  2.47s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:10<00:00,  2.44s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:12<00:00,  2.47s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:12<00:00,  2.47s/it]\n"
     ]
    }
   ],
   "source": [
    "preds.append(scale_probs(run_predictions_philipp(\"awesome-rose-ff-v2\", df, BS=8, num_models=5)))\n",
    "weights.append(1.162001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d86994b1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T08:49:40.611874Z",
     "iopub.status.busy": "2023-03-13T08:49:40.611488Z",
     "iopub.status.idle": "2023-03-13T09:00:58.860552Z",
     "shell.execute_reply": "2023-03-13T09:00:58.859252Z"
    },
    "papermill": {
     "duration": 679.387039,
     "end_time": "2023-03-13T09:00:59.208759",
     "exception": false,
     "start_time": "2023-03-13T08:49:39.821720",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128001]\n",
      "[128002]\n",
      "running model 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:19<00:00,  2.56s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:19<00:00,  2.56s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:19<00:00,  2.56s/it]\n"
     ]
    }
   ],
   "source": [
    "preds.append(scale_probs(run_predictions_philipp(\"honest-apple-ff\", df, BS=8)))\n",
    "weights.append(0.543224)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "8788b77f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:01:00.634304Z",
     "iopub.status.busy": "2023-03-13T09:01:00.633850Z",
     "iopub.status.idle": "2023-03-13T09:12:27.479780Z",
     "shell.execute_reply": "2023-03-13T09:12:27.478482Z"
    },
    "papermill": {
     "duration": 687.945104,
     "end_time": "2023-03-13T09:12:27.834046",
     "exception": false,
     "start_time": "2023-03-13T09:00:59.888942",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128001]\n",
      "[128002]\n",
      "running model 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:19<00:00,  2.56s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:19<00:00,  2.56s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:19<00:00,  2.56s/it]\n"
     ]
    }
   ],
   "source": [
    "preds.append(scale_probs(run_predictions_philipp(\"funky-funk-ff\", df, BS=8)))\n",
    "weights.append(1.455657)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b1e90970",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:12:29.259574Z",
     "iopub.status.busy": "2023-03-13T09:12:29.259183Z",
     "iopub.status.idle": "2023-03-13T09:31:26.591924Z",
     "shell.execute_reply": "2023-03-13T09:31:26.590678Z"
    },
    "papermill": {
     "duration": 1138.543857,
     "end_time": "2023-03-13T09:31:27.057635",
     "exception": false,
     "start_time": "2023-03-13T09:12:28.513778",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128001]\n",
      "[128002]\n",
      "running model 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:19<00:00,  2.56s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:20<00:00,  2.56s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:20<00:00,  2.57s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:19<00:00,  2.56s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [03:19<00:00,  2.56s/it]\n"
     ]
    }
   ],
   "source": [
    "preds.append(scale_probs(run_predictions_philipp(\"lame-flame-ff-v2\", df, BS=8, num_models=5)))\n",
    "weights.append(1.981731)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e3046f74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:31:28.529569Z",
     "iopub.status.busy": "2023-03-13T09:31:28.529193Z",
     "iopub.status.idle": "2023-03-13T09:33:48.213119Z",
     "shell.execute_reply": "2023-03-13T09:33:48.212025Z"
    },
    "papermill": {
     "duration": 140.428026,
     "end_time": "2023-03-13T09:33:48.215709",
     "exception": false,
     "start_time": "2023-03-13T09:31:27.787683",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[128001]\n",
      "[128002]\n",
      "running model 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [00:36<00:00,  2.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [00:36<00:00,  2.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lead Position Claim Evidence [SEP]  [START] In my opinion as a student: I don't agree at this type of method but the same time is good for students and teachers to learnd everyday, life is like a school we never stop learning something everyday and acquire new expirence and skills in our grammer. [END]   [START] I should go neither everybody need their will to vote what they want to do or spend their summer break as they like. [END] \n",
      "\n",
      " [START] like you see is their break and is not fair to obligate a student to do a project with out a vote to give them a choice to pick if they want to do it or not. [END]   [START] Example; like the elections a President can't be a President with out votes, the country pick their own candidates and the citizens give the vote to the person they pick best for their country\n",
      "\n",
      "Most likely everything have a vote in our community fuction nothing move with out a vote to people is fair to vote,\n",
      "\n",
      "that how it works [END]  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [00:36<00:00,  2.16it/s]\n"
     ]
    }
   ],
   "source": [
    "preds.append(scale_probs(run_predictions_philipp(\"pastel-frog-ff\", df, BS=8)))\n",
    "weights.append(-1.005743)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "3ef6caeb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:33:49.701152Z",
     "iopub.status.busy": "2023-03-13T09:33:49.700763Z",
     "iopub.status.idle": "2023-03-13T09:33:49.706921Z",
     "shell.execute_reply": "2023-03-13T09:33:49.706020Z"
    },
    "papermill": {
     "duration": 0.792617,
     "end_time": "2023-03-13T09:33:49.709096",
     "exception": false,
     "start_time": "2023-03-13T09:33:48.916479",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "preds_orig = np.array(preds).copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "9530d414",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:33:51.181154Z",
     "iopub.status.busy": "2023-03-13T09:33:51.180759Z",
     "iopub.status.idle": "2023-03-13T09:33:51.203310Z",
     "shell.execute_reply": "2023-03-13T09:33:51.202018Z"
    },
    "papermill": {
     "duration": 0.762364,
     "end_time": "2023-03-13T09:33:51.205536",
     "exception": false,
     "start_time": "2023-03-13T09:33:50.443172",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_ind_models = df.copy()\n",
    "\n",
    "for model_idx in range(len(preds)):\n",
    "    df_ind_models[f\"Adequate_{model_idx}\"] = preds[model_idx][:,0]\n",
    "    df_ind_models[f\"Effective_{model_idx}\"] = preds[model_idx][:,1]\n",
    "    df_ind_models[f\"Ineffective_{model_idx}\"] = preds[model_idx][:,2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "93974278",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:33:52.872903Z",
     "iopub.status.busy": "2023-03-13T09:33:52.871803Z",
     "iopub.status.idle": "2023-03-13T09:33:52.878951Z",
     "shell.execute_reply": "2023-03-13T09:33:52.878010Z"
    },
    "papermill": {
     "duration": 0.743401,
     "end_time": "2023-03-13T09:33:52.881030",
     "exception": false,
     "start_time": "2023-03-13T09:33:52.137629",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "preds = np.average(preds, weights=weights, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "91051083",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:33:54.391299Z",
     "iopub.status.busy": "2023-03-13T09:33:54.390207Z",
     "iopub.status.idle": "2023-03-13T09:33:54.396794Z",
     "shell.execute_reply": "2023-03-13T09:33:54.395745Z"
    },
    "papermill": {
     "duration": 0.811855,
     "end_time": "2023-03-13T09:33:54.399133",
     "exception": false,
     "start_time": "2023-03-13T09:33:53.587278",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "assert len(preds) == len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ecce0dbc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:33:55.817110Z",
     "iopub.status.busy": "2023-03-13T09:33:55.816747Z",
     "iopub.status.idle": "2023-03-13T09:33:55.881260Z",
     "shell.execute_reply": "2023-03-13T09:33:55.880301Z"
    },
    "papermill": {
     "duration": 0.77522,
     "end_time": "2023-03-13T09:33:55.883557",
     "exception": false,
     "start_time": "2023-03-13T09:33:55.108337",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pp = preds.copy()\n",
    "\n",
    "eps = 0.0001\n",
    "pp = pp.clip(eps, 1 - eps)\n",
    "pp = pp / pp.sum(axis=1, keepdims=True)\n",
    "\n",
    "pp = scale_probs(pp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "cdde0dbb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:33:57.369801Z",
     "iopub.status.busy": "2023-03-13T09:33:57.369449Z",
     "iopub.status.idle": "2023-03-13T09:33:57.376536Z",
     "shell.execute_reply": "2023-03-13T09:33:57.375539Z"
    },
    "papermill": {
     "duration": 0.708625,
     "end_time": "2023-03-13T09:33:57.378633",
     "exception": false,
     "start_time": "2023-03-13T09:33:56.670008",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df[\"Adequate\"] = pp[:, 0] \n",
    "df[\"Effective\"] = pp[:, 1] \n",
    "df[\"Ineffective\"] = pp[:, 2] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "0ba06680",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:33:58.866941Z",
     "iopub.status.busy": "2023-03-13T09:33:58.866575Z",
     "iopub.status.idle": "2023-03-13T09:33:58.873735Z",
     "shell.execute_reply": "2023-03-13T09:33:58.872748Z"
    },
    "papermill": {
     "duration": 0.712776,
     "end_time": "2023-03-13T09:33:58.876013",
     "exception": false,
     "start_time": "2023-03-13T09:33:58.163237",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_ind_models[\"Adequate\"] = pp[:, 0] \n",
    "df_ind_models[\"Effective\"] = pp[:, 1] \n",
    "df_ind_models[\"Ineffective\"] = pp[:, 2] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "45544b3c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:34:00.371405Z",
     "iopub.status.busy": "2023-03-13T09:34:00.370809Z",
     "iopub.status.idle": "2023-03-13T09:34:00.375393Z",
     "shell.execute_reply": "2023-03-13T09:34:00.374438Z"
    },
    "papermill": {
     "duration": 0.802165,
     "end_time": "2023-03-13T09:34:00.377430",
     "exception": false,
     "start_time": "2023-03-13T09:33:59.575265",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "orig_preds = pp.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05fbf306",
   "metadata": {
    "papermill": {
     "duration": 0.734061,
     "end_time": "2023-03-13T09:34:01.818132",
     "exception": false,
     "start_time": "2023-03-13T09:34:01.084071",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 2nd level models\n",
    "We additionally trained several 2nd level models to further improve our predictions.\n",
    "\n",
    "LightGBMs\n",
    "For the 2nd level LightGBM model we took the weighted ensemble prediction, together with individual models predictions, and generated some aggregate features based on the whole essay. We trained 2 LightGBM versions with different features and parameters.\n",
    "\n",
    "Neural networks\n",
    "We tuned two types of neural networks here. The first takes the weighted ensemble prediction, as well as an average across the essay and across the type within an essay for each of the three target columns as input and trains a three-layer DNN. The second one takes the same features, but on an individual model basis and then uses a three-layer Conv1d with average pooling afterwards.\n",
    "\n",
    "All together, 2nd level models were consistently bringing us about 0.003-0.005 points on CV and the leaderboard throughout the competition."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "16327bcc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:34:03.509908Z",
     "iopub.status.busy": "2023-03-13T09:34:03.509548Z",
     "iopub.status.idle": "2023-03-13T09:34:03.515778Z",
     "shell.execute_reply": "2023-03-13T09:34:03.514847Z"
    },
    "papermill": {
     "duration": 0.889455,
     "end_time": "2023-03-13T09:34:03.517825",
     "exception": false,
     "start_time": "2023-03-13T09:34:02.628370",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if CALC_SCORE:\n",
    "    from sklearn.metrics import log_loss\n",
    "    \n",
    "    label_cols = [\"Adequate\", \"Effective\", \"Ineffective\"]\n",
    "    \n",
    "    y = np.zeros_like(preds)\n",
    "    \n",
    "    for ii, jj in enumerate([label_cols.index(x) for x in df[\"discourse_effectiveness\"].values]):\n",
    "        y[ii,jj] = 1\n",
    "        \n",
    "    print(log_loss(y, pp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "503d889a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:34:05.037928Z",
     "iopub.status.busy": "2023-03-13T09:34:05.037534Z",
     "iopub.status.idle": "2023-03-13T09:34:05.045665Z",
     "shell.execute_reply": "2023-03-13T09:34:05.044702Z"
    },
    "papermill": {
     "duration": 0.796142,
     "end_time": "2023-03-13T09:34:05.048354",
     "exception": false,
     "start_time": "2023-03-13T09:34:04.252212",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "label_cols = [\"Adequate\", \"Effective\", \"Ineffective\"]\n",
    "oof_cols = []\n",
    "for j, l in enumerate(label_cols):\n",
    "\n",
    "    df[f\"oof_{l}\"] = pp[:,j]\n",
    "    oof_cols.append(f\"oof_{l}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a35348f3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:34:06.464038Z",
     "iopub.status.busy": "2023-03-13T09:34:06.463615Z",
     "iopub.status.idle": "2023-03-13T09:34:17.306184Z",
     "shell.execute_reply": "2023-03-13T09:34:17.304942Z"
    },
    "papermill": {
     "duration": 11.554055,
     "end_time": "2023-03-13T09:34:17.308761",
     "exception": false,
     "start_time": "2023-03-13T09:34:05.754706",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model ../input/feedback-nn-v8-blend151-ff/checkpoint_fold2_seed0.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 68.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model ../input/feedback-nn-v8-blend151-ff/checkpoint_fold1_seed0.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 69.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model ../input/feedback-nn-v8-blend151-ff/checkpoint_fold4_seed0.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 70.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model ../input/feedback-nn-v8-blend151-ff/checkpoint_fold3_seed0.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 57.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model ../input/feedback-nn-v8-blend151-ff/checkpoint_fold0_seed0.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:02<00:00, 52.60it/s]\n"
     ]
    }
   ],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import torch\n",
    "\n",
    "import torch.nn as nn\n",
    "\n",
    "class FeedbackStackerModel(nn.Module):\n",
    "    def __init__(self, n_features):\n",
    "        super(FeedbackStackerModel, self).__init__()\n",
    "        \n",
    "        self.sizes = [256, 128, 64]\n",
    "        \n",
    "        self.features = nn.Sequential(\n",
    "            nn.utils.weight_norm(nn.Linear(n_features, self.sizes[0])),\n",
    "            nn.PReLU(),\n",
    "            nn.Linear(self.sizes[0], self.sizes[1]),\n",
    "            nn.PReLU(),\n",
    "            nn.Linear(self.sizes[1], self.sizes[2]),\n",
    "            nn.PReLU(),\n",
    "        )\n",
    "        \n",
    "        self.head = nn.Linear(self.sizes[-1], 3)\n",
    "        \n",
    "        self.loss_fn = nn.CrossEntropyLoss()\n",
    "        \n",
    "    def forward(self, x, y):\n",
    "        \n",
    "        x = self.features(x)\n",
    "        x = self.head(x)\n",
    "        \n",
    "        \n",
    "        output = {}\n",
    "        \n",
    "        output[\"logits\"] = x\n",
    "        \n",
    "        if self.training:\n",
    "            output[\"loss\"] = self.loss_fn(x, y.argmax(dim=1))\n",
    "        \n",
    "        return output\n",
    "\n",
    "class FeedbackStackerDataset(Dataset):\n",
    "\n",
    "    def __init__(self, df, mode):\n",
    "        self.df = df.copy().reset_index(drop=True)\n",
    "        self.mode = mode\n",
    "\n",
    "        self.feature_cols = oof_cols.copy()\n",
    "        self.label_cols = label_cols.copy()\n",
    "        \n",
    "        df = self.df\n",
    "        \n",
    "        df[f\"len\"] = df.groupby(\"essay_id\")[f\"discourse_id\"].transform(\"count\") / 10\n",
    "        self.feature_cols.append(f\"len\")\n",
    "        \n",
    "        for j, l in enumerate(label_cols):\n",
    "            df[f\"oof_{l}_mean\"] = df.groupby(\"essay_id\")[f\"oof_{l}\"].transform(\"mean\")\n",
    "            self.feature_cols.append(f\"oof_{l}_mean\")\n",
    "            \n",
    "            df[f\"oof_{l}_t_mean\"] = df.groupby([\"essay_id\", \"discourse_type\"])[f\"oof_{l}\"].transform(\"mean\")\n",
    "            self.feature_cols.append(f\"oof_{l}_t_mean\")\n",
    "\n",
    "        self.num_features = len(self.feature_cols)\n",
    "\n",
    "        self.X = self.df[self.feature_cols].values\n",
    "        self.y = self.df[self.label_cols].values\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        row = self.df.iloc[idx]\n",
    "        \n",
    "        X = self.X[idx]\n",
    "        y = self.y[idx]\n",
    "        \n",
    "        return torch.FloatTensor(X), torch.FloatTensor(y)\n",
    "\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.df.shape[0]\n",
    "    \n",
    "ds = FeedbackStackerDataset(df.copy(), mode=\"val\")\n",
    "ds[0][0].shape\n",
    "\n",
    "def run_nn_stacker(exp_name, df, BS=64):\n",
    "\n",
    "\n",
    "    ds = FeedbackStackerDataset(df.iloc[:].copy(), mode=\"test\")\n",
    "    \n",
    "    checkpoints = glob(f\"../input/{exp_name}/*.pth\")\n",
    "    \n",
    "    preds_all = []\n",
    "    for checkpoint in checkpoints:\n",
    "        print(f\"running model {checkpoint}\")\n",
    "        \n",
    "        model = FeedbackStackerModel(n_features=ds.num_features).to(\"cuda\").eval()\n",
    "    \n",
    "        model_weights = torch.load(checkpoint, map_location=\"cpu\")\n",
    "\n",
    "        model.load_state_dict(collections.OrderedDict(model_weights), strict=True)\n",
    "        \n",
    "        del model_weights\n",
    "        gc.collect() \n",
    "    \n",
    "        batch_size = BS\n",
    "        dl = DataLoader(ds, shuffle=False, batch_size = batch_size, num_workers = N_CORES)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            preds = []\n",
    "            for batch in tqdm(dl):\n",
    "\n",
    "                data = [x.to(\"cuda\") for x in batch]\n",
    "                inputs, target = data\n",
    "                out = model(inputs, target)\n",
    "                preds.append(out[\"logits\"].float().softmax(dim=1).detach().cpu().numpy())\n",
    "\n",
    "        preds_all.append(np.concatenate(preds, axis=0))\n",
    "        \n",
    "        del model\n",
    "        del dl\n",
    "        gc.collect()\n",
    "        \n",
    "    del ds\n",
    "    \n",
    "    \n",
    "    preds = np.mean(preds_all, axis=0)\n",
    "    \n",
    "    return preds\n",
    "\n",
    "nn_stacker_preds_1 = run_nn_stacker(\"feedback-nn-v8-blend151-ff\", df, BS=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "197e4a14",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:34:18.823300Z",
     "iopub.status.busy": "2023-03-13T09:34:18.822905Z",
     "iopub.status.idle": "2023-03-13T09:34:36.764720Z",
     "shell.execute_reply": "2023-03-13T09:34:36.763553Z"
    },
    "papermill": {
     "duration": 18.663154,
     "end_time": "2023-03-13T09:34:36.767255",
     "exception": false,
     "start_time": "2023-03-13T09:34:18.104101",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6988, 10, 15)\n",
      "(6988, 10, 15)\n",
      "running model ../input/feedback-nn-v11-blend151-ff/checkpoint_fold2_seed0.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:08<00:00, 13.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model ../input/feedback-nn-v11-blend151-ff/checkpoint_fold1_seed0.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 66.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model ../input/feedback-nn-v11-blend151-ff/checkpoint_fold4_seed0.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 65.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model ../input/feedback-nn-v11-blend151-ff/checkpoint_fold3_seed0.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 63.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running model ../input/feedback-nn-v11-blend151-ff/checkpoint_fold0_seed0.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 57.15it/s]\n"
     ]
    }
   ],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import torch\n",
    "\n",
    "import torch.nn as nn\n",
    "\n",
    "class FeedbackStackerDataset(Dataset):\n",
    "\n",
    "    def __init__(self, df, mode):\n",
    "        self.df = df.copy()\n",
    "        self.mode = mode\n",
    "\n",
    "        self.label_cols = label_cols.copy()\n",
    "        \n",
    "        p = [p[self.df.index.values] for p in preds_orig.copy()]\n",
    "        p = np.stack(p)\n",
    "        \n",
    "        df = self.df\n",
    "        \n",
    "        X = []\n",
    "        for j in range(p.shape[0]):\n",
    "            cols = []\n",
    "            for jj, l in enumerate(label_cols):\n",
    "\n",
    "                df[f\"oof_{l}\"] = p[j,:,jj]\n",
    "                cols.append(f\"oof_{l}\")\n",
    "                \n",
    "                df[f\"oof_{l}_mean\"] = df.groupby(\"essay_id\")[f\"oof_{l}\"].transform(\"mean\")\n",
    "                cols.append(f\"oof_{l}_mean\")\n",
    "\n",
    "                df[f\"oof_{l}_t_mean\"] = df.groupby([\"essay_id\", \"discourse_type\"])[f\"oof_{l}\"].transform(\"mean\")\n",
    "                cols.append(f\"oof_{l}_t_mean\")\n",
    "                \n",
    "            df[f\"len\"] = df.groupby(\"essay_id\")[f\"discourse_id\"].transform(\"count\") / 10\n",
    "            cols.append(f\"len\")\n",
    "        \n",
    "            \n",
    "            X.append(df[cols].values)\n",
    "         \n",
    "        X = np.stack(X).transpose(1,2,0)\n",
    "        print(X.shape)\n",
    "        \n",
    "        self.num_features = X.shape[1]\n",
    "\n",
    "        self.X = X\n",
    "        self.y = self.df[self.label_cols].values\n",
    "\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        row = self.df.iloc[idx]\n",
    "        \n",
    "        X = self.X[idx]\n",
    "        y = self.y[idx]\n",
    "        \n",
    "        return torch.FloatTensor(X), torch.FloatTensor(y)\n",
    "\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.df.shape[0]\n",
    "\n",
    "class FeedbackStackerModel(nn.Module):\n",
    "    def __init__(self, n_features):\n",
    "        super(FeedbackStackerModel, self).__init__()\n",
    "        \n",
    "        self.sizes = [256, 128, 64]\n",
    "        \n",
    "        layers = []\n",
    "        for j,s in enumerate(self.sizes):\n",
    "            if j == 0:\n",
    "                layers.append(nn.Conv1d(n_features, s, 1))\n",
    "            else:\n",
    "                layers.append(nn.Conv1d(self.sizes[j-1], s, 1))\n",
    "            layers.append(nn.PReLU())\n",
    "            layers.append(nn.Dropout(0.2))\n",
    "        \n",
    "        self.features = nn.Sequential(*layers)\n",
    "        self.head = nn.Linear(self.sizes[-1], 3)\n",
    "        \n",
    "        self.loss_fn = nn.CrossEntropyLoss()\n",
    "        \n",
    "    def forward(self, x, y):\n",
    "        x = self.features(x)\n",
    "        x = x.mean(dim=2)\n",
    "        x = self.head(x)\n",
    "        \n",
    "        output = {}\n",
    "        \n",
    "        output[\"logits\"] = x\n",
    "        \n",
    "        if self.training:\n",
    "            output[\"loss\"] = self.loss_fn(x, y.argmax(dim=1))\n",
    "        \n",
    "        return output\n",
    "    \n",
    "ds = FeedbackStackerDataset(df.copy(), mode=\"val\")\n",
    "ds[0][0].shape\n",
    "\n",
    "nn_stacker_preds_2 = run_nn_stacker(\"feedback-nn-v11-blend151-ff\", df, BS=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "82c0f896",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:34:38.292729Z",
     "iopub.status.busy": "2023-03-13T09:34:38.292322Z",
     "iopub.status.idle": "2023-03-13T09:34:38.298013Z",
     "shell.execute_reply": "2023-03-13T09:34:38.297042Z"
    },
    "papermill": {
     "duration": 0.723664,
     "end_time": "2023-03-13T09:34:38.300081",
     "exception": false,
     "start_time": "2023-03-13T09:34:37.576417",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def gen_x(values):\n",
    "    range = 1\n",
    "    return np.histogram(np.clip(values, 0.001, 0.999*range), bins=3, density=True, range=(0,range))[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "d79b97ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:34:39.812855Z",
     "iopub.status.busy": "2023-03-13T09:34:39.812460Z",
     "iopub.status.idle": "2023-03-13T09:34:41.360773Z",
     "shell.execute_reply": "2023-03-13T09:34:41.359774Z"
    },
    "papermill": {
     "duration": 2.35084,
     "end_time": "2023-03-13T09:34:41.363212",
     "exception": false,
     "start_time": "2023-03-13T09:34:39.012372",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 624/624 [00:01<00:00, 466.29it/s]\n"
     ]
    }
   ],
   "source": [
    "all_groups = []\n",
    "\n",
    "gb = df.groupby('essay_id', sort=False)\n",
    "for name, group in tqdm(gb):\n",
    "    group[\"n_types\"] = group.discourse_type.nunique()\n",
    "    for class_name in [\"Adequate\", \"Effective\", \"Ineffective\"]:\n",
    "        if class_name in [\"Adequate\", \"Effective\"]:\n",
    "            continue\n",
    "        for idx, val in enumerate(gen_x(group[class_name].values)):\n",
    "            group[f\"{class_name}_bin_{idx}\"] = val \n",
    "        group[f\"mean_{class_name}\"] = group[class_name].mean()    \n",
    "\n",
    "    all_groups.append(group)\n",
    "\n",
    "df = pd.concat(all_groups).reset_index(drop=True)\n",
    "\n",
    "disc_types_mapping = {'Lead': 0,\n",
    "'Position': 1,\n",
    "'Claim': 2,\n",
    "'Evidence': 3,\n",
    "'Counterclaim': 4,\n",
    "'Rebuttal': 5,\n",
    "'Concluding Statement': 6}\n",
    "df[\"len_disc\"] = df.discourse_text.str.len()\n",
    "\n",
    "df[\"discourse_type\"] = df[\"discourse_type\"].map(disc_types_mapping)\n",
    "\n",
    "df[\"paragraph_cnt\"] = df.essay_text.map(lambda x: len(x.split(\"\\n\\n\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "e01f0242",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:34:42.807001Z",
     "iopub.status.busy": "2023-03-13T09:34:42.806581Z",
     "iopub.status.idle": "2023-03-13T09:34:43.394163Z",
     "shell.execute_reply": "2023-03-13T09:34:43.392759Z"
    },
    "papermill": {
     "duration": 1.31122,
     "end_time": "2023-03-13T09:34:43.396309",
     "exception": false,
     "start_time": "2023-03-13T09:34:42.085089",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n"
     ]
    }
   ],
   "source": [
    "import lightgbm\n",
    "\n",
    "lgb_stacker_preds = []\n",
    "\n",
    "for fold in range(5):\n",
    "    print(fold)\n",
    "    gbm = lightgbm.Booster(model_file=f\"../input/lightgbm151/model_fold_{fold}.txt\")\n",
    "    valid_pred = gbm.predict(df[['discourse_type', 'Adequate', 'Effective', 'Ineffective', 'n_types',\n",
    "       'Ineffective_bin_0', 'Ineffective_bin_1', 'Ineffective_bin_2',\n",
    "       'mean_Ineffective', 'len_disc', 'paragraph_cnt']])\n",
    "    lgb_stacker_preds.append(valid_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "bb7dee0e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:34:45.009919Z",
     "iopub.status.busy": "2023-03-13T09:34:45.009553Z",
     "iopub.status.idle": "2023-03-13T09:34:45.015197Z",
     "shell.execute_reply": "2023-03-13T09:34:45.014222Z"
    },
    "papermill": {
     "duration": 0.764943,
     "end_time": "2023-03-13T09:34:45.017273",
     "exception": false,
     "start_time": "2023-03-13T09:34:44.252330",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "lgb_stacker_preds = np.array(lgb_stacker_preds).mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "e956b94b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:34:46.584970Z",
     "iopub.status.busy": "2023-03-13T09:34:46.584546Z",
     "iopub.status.idle": "2023-03-13T09:34:59.867914Z",
     "shell.execute_reply": "2023-03-13T09:34:59.866909Z"
    },
    "papermill": {
     "duration": 14.104836,
     "end_time": "2023-03-13T09:34:59.870511",
     "exception": false,
     "start_time": "2023-03-13T09:34:45.765675",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 624/624 [00:12<00:00, 49.35it/s]\n"
     ]
    }
   ],
   "source": [
    "all_groups = []\n",
    "\n",
    "gb = df_ind_models.groupby('essay_id', sort=False)\n",
    "for name, group in tqdm(gb):\n",
    "    group[\"n_types\"] = group.discourse_type.nunique()\n",
    "\n",
    "    for class_name in [\"Adequate\", \"Effective\", \"Ineffective\"]:\n",
    "        if class_name in [\"Adequate\", \"Effective\"]:\n",
    "            continue\n",
    "\n",
    "        for idx, val in enumerate(gen_x(group[class_name].values)):\n",
    "            group[f\"{class_name}_bin_{idx}\"] = val\n",
    "\n",
    "        group[f\"mean_{class_name}\"] = group[class_name].mean()\n",
    "        group[f\"max_{class_name}\"] = group[class_name].max()\n",
    "\n",
    "    for class_name in [f\"Ineffective_{i}\" for i in range(15)]:\n",
    "        group[f\"mean_{class_name}\"] = group[class_name].mean()\n",
    "\n",
    "    for class_name in [f\"Effective_{i}\" for i in range(15)]:\n",
    "        group[f\"mean_{class_name}\"] = group[class_name].mean()\n",
    "\n",
    "    all_groups.append(group)\n",
    "\n",
    "df_ind_models = pd.concat(all_groups).reset_index(drop=True)\n",
    "\n",
    "df_ind_models[\"paragraph_cnt\"] = df_ind_models.essay_text.map(lambda x: len(x.split(\"\\n\\n\")))\n",
    "\n",
    "disc_types_mapping = {'Lead': 0,\n",
    "'Position': 1,\n",
    "'Claim': 2,\n",
    "'Evidence': 3,\n",
    "'Counterclaim': 4,\n",
    "'Rebuttal': 5,\n",
    "'Concluding Statement': 6}\n",
    "\n",
    "df_ind_models[\"discourse_type\"] = df_ind_models[\"discourse_type\"].map(disc_types_mapping)\n",
    "\n",
    "for i in range(15):\n",
    "    df_ind_models = df_ind_models.drop([f'Adequate_{i}'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "39f67405",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:35:01.401102Z",
     "iopub.status.busy": "2023-03-13T09:35:01.400699Z",
     "iopub.status.idle": "2023-03-13T09:35:02.070155Z",
     "shell.execute_reply": "2023-03-13T09:35:02.068900Z"
    },
    "papermill": {
     "duration": 1.483228,
     "end_time": "2023-03-13T09:35:02.072278",
     "exception": false,
     "start_time": "2023-03-13T09:35:00.589050",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n"
     ]
    }
   ],
   "source": [
    "import lightgbm\n",
    "\n",
    "lgb_stacker_preds_2 = []\n",
    "\n",
    "for fold in range(5):\n",
    "    print(fold)\n",
    "    gbm = lightgbm.Booster(model_file=f\"../input/lightgbm151v2/model_fold_{fold}.txt\")\n",
    "    valid_pred = gbm.predict(df_ind_models[['discourse_type', 'Adequate', 'Effective', 'Ineffective', 'Effective_0',\n",
    "       'Ineffective_0', 'Effective_1', 'Ineffective_1', 'Effective_2',\n",
    "       'Ineffective_2', 'Effective_3', 'Ineffective_3', 'Effective_4',\n",
    "       'Ineffective_4', 'Effective_5', 'Ineffective_5', 'Effective_6',\n",
    "       'Ineffective_6', 'Effective_7', 'Ineffective_7', 'Effective_8',\n",
    "       'Ineffective_8', 'Effective_9', 'Ineffective_9', 'Effective_10',\n",
    "       'Ineffective_10', 'Effective_11', 'Ineffective_11', 'Effective_12',\n",
    "       'Ineffective_12', 'Effective_13', 'Ineffective_13', 'Effective_14',\n",
    "       'Ineffective_14', 'n_types', 'Ineffective_bin_0', 'Ineffective_bin_1',\n",
    "       'Ineffective_bin_2', 'mean_Ineffective', 'max_Ineffective',\n",
    "       'mean_Ineffective_0', 'mean_Ineffective_1', 'mean_Ineffective_2',\n",
    "       'mean_Ineffective_3', 'mean_Ineffective_4', 'mean_Ineffective_5',\n",
    "       'mean_Ineffective_6', 'mean_Ineffective_7', 'mean_Ineffective_8',\n",
    "       'mean_Ineffective_9', 'mean_Ineffective_10', 'mean_Ineffective_11',\n",
    "       'mean_Ineffective_12', 'mean_Ineffective_13', 'mean_Ineffective_14',\n",
    "       'mean_Effective_0', 'mean_Effective_1', 'mean_Effective_2',\n",
    "       'mean_Effective_3', 'mean_Effective_4', 'mean_Effective_5',\n",
    "       'mean_Effective_6', 'mean_Effective_7', 'mean_Effective_8',\n",
    "       'mean_Effective_9', 'mean_Effective_10', 'mean_Effective_11',\n",
    "       'mean_Effective_12', 'mean_Effective_13', 'mean_Effective_14',\n",
    "       'paragraph_cnt']])\n",
    "    lgb_stacker_preds_2.append(valid_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "ea870b0d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:35:03.525174Z",
     "iopub.status.busy": "2023-03-13T09:35:03.524728Z",
     "iopub.status.idle": "2023-03-13T09:35:03.530750Z",
     "shell.execute_reply": "2023-03-13T09:35:03.529720Z"
    },
    "papermill": {
     "duration": 0.732866,
     "end_time": "2023-03-13T09:35:03.532774",
     "exception": false,
     "start_time": "2023-03-13T09:35:02.799908",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "lgb_stacker_preds_2 = np.array(lgb_stacker_preds_2).mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "2b6d1b11",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:35:05.119467Z",
     "iopub.status.busy": "2023-03-13T09:35:05.119089Z",
     "iopub.status.idle": "2023-03-13T09:35:05.126106Z",
     "shell.execute_reply": "2023-03-13T09:35:05.125049Z"
    },
    "papermill": {
     "duration": 0.747866,
     "end_time": "2023-03-13T09:35:05.128175",
     "exception": false,
     "start_time": "2023-03-13T09:35:04.380309",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_preds = [\n",
    "    orig_preds.copy(),\n",
    "    lgb_stacker_preds.copy(),\n",
    "    nn_stacker_preds_1.copy(),\n",
    "    nn_stacker_preds_2.copy(),\n",
    "    lgb_stacker_preds_2.copy(),\n",
    "]\n",
    "\n",
    "all_preds = np.average(all_preds, axis=0, weights=[2.17532521, 0.96247677, 1.15351147, 0.62746974, 0.47835051])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "bdea7b7d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:35:06.654463Z",
     "iopub.status.busy": "2023-03-13T09:35:06.654086Z",
     "iopub.status.idle": "2023-03-13T09:35:06.661343Z",
     "shell.execute_reply": "2023-03-13T09:35:06.660238Z"
    },
    "papermill": {
     "duration": 0.816054,
     "end_time": "2023-03-13T09:35:06.663541",
     "exception": false,
     "start_time": "2023-03-13T09:35:05.847487",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df[\"Adequate\"] = all_preds[:, 0]\n",
    "df[\"Effective\"] = all_preds[:, 1]\n",
    "df[\"Ineffective\"] = all_preds[:, 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "537b94b3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:35:08.203295Z",
     "iopub.status.busy": "2023-03-13T09:35:08.202765Z",
     "iopub.status.idle": "2023-03-13T09:35:08.254554Z",
     "shell.execute_reply": "2023-03-13T09:35:08.253481Z"
    },
    "papermill": {
     "duration": 0.824339,
     "end_time": "2023-03-13T09:35:08.257208",
     "exception": false,
     "start_time": "2023-03-13T09:35:07.432869",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df[['discourse_id', 'Ineffective', 'Adequate', 'Effective']].to_csv(f\"submission{CURR_SPLIT}.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "6bfe339b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-13T09:35:09.945675Z",
     "iopub.status.busy": "2023-03-13T09:35:09.945304Z",
     "iopub.status.idle": "2023-03-13T09:35:09.951495Z",
     "shell.execute_reply": "2023-03-13T09:35:09.950521Z"
    },
    "papermill": {
     "duration": 0.729264,
     "end_time": "2023-03-13T09:35:09.953544",
     "exception": false,
     "start_time": "2023-03-13T09:35:09.224280",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if CALC_SCORE:\n",
    "    from sklearn.metrics import log_loss\n",
    "    \n",
    "    label_cols = [\"Adequate\", \"Effective\", \"Ineffective\"]\n",
    "    \n",
    "    y = np.zeros_like(all_preds)\n",
    "    \n",
    "    for ii, jj in enumerate([label_cols.index(x) for x in df[\"discourse_effectiveness\"].values]):\n",
    "        y[ii,jj] = 1\n",
    "        \n",
    "    print(log_loss(y, all_preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a51624bd",
   "metadata": {
    "papermill": {
     "duration": 0.800646,
     "end_time": "2023-03-13T09:35:11.486118",
     "exception": false,
     "start_time": "2023-03-13T09:35:10.685472",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70daa4ea",
   "metadata": {
    "papermill": {
     "duration": 0.809665,
     "end_time": "2023-03-13T09:35:13.022714",
     "exception": false,
     "start_time": "2023-03-13T09:35:12.213049",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f3d5c60",
   "metadata": {
    "papermill": {
     "duration": 0.717797,
     "end_time": "2023-03-13T09:35:14.500444",
     "exception": false,
     "start_time": "2023-03-13T09:35:13.782647",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 14768.638339,
   "end_time": "2023-03-13T09:35:18.747254",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-13T05:29:10.108915",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}