dev-notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A bunch of experimenting here. Take a look at the actual scripts to get up to speed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "germanquad = load_dataset(\"deepset/germanquad\")\n",
    "germanquad = germanquad[\"train\"]\n",
    "germanquad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "germanquad[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "germanquad[\"context\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "germandpr = load_dataset(\"deepset/germandpr\")\n",
    "germandpr = germandpr[\"train\"]\n",
    "germandpr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "germandpr['positive_ctxs'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create sets of the 'context' fields in both datasets\n",
    "germanquad_contexts = set([item['context'] for item in germanquad])\n",
    "germandpr_contexts = set([item['positive_ctxs']['text'][0] for item in germandpr])\n",
    "print(len(germanquad_contexts), len(germandpr_contexts))\n",
    "\n",
    "# Find the intersection of the two sets\n",
    "overlap = germanquad_contexts & germandpr_contexts\n",
    "\n",
    "# The size of the intersection set is the number of overlapping contexts\n",
    "overlap_size = len(overlap)\n",
    "\n",
    "# Print the result\n",
    "print(f'There are {overlap_size} overlapping contexts between the two datasets.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(germanquad), len(germandpr))\n",
    "# Create sets of the 'question' fields in both datasets\n",
    "germanquad_questions = set([item['question'] for item in germanquad])\n",
    "germandpr_questions = set([item['question'] for item in germandpr])\n",
    "print(len(germanquad_questions), len(germandpr_questions))\n",
    "\n",
    "# Find the intersection of the two sets\n",
    "overlap = germanquad_questions & germandpr_questions\n",
    "\n",
    "# The size of the intersection set is the number of overlapping questions\n",
    "overlap_size = len(overlap)\n",
    "\n",
    "# Print the result\n",
    "print(f'There are {overlap_size} overlapping questions between the two datasets.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "# Get all questions from both datasets\n",
    "germanquad_questions = [item['question'] for item in germanquad]\n",
    "germandpr_questions = [item['question'] for item in germandpr]\n",
    "\n",
    "# Count the occurrences of each question\n",
    "germanquad_question_counts = Counter(germanquad_questions)\n",
    "germandpr_question_counts = Counter(germandpr_questions)\n",
    "\n",
    "# Get the questions that appear more than once\n",
    "duplicate_germanquad_questions = [question for question, count in germanquad_question_counts.items() if count > 1]\n",
    "duplicate_germandpr_questions = [question for question, count in germandpr_question_counts.items() if count > 1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Format the duplicate questions as a string\n",
    "duplicate_germanquad_questions_str = \"\\n\".join(duplicate_germanquad_questions)\n",
    "duplicate_germandpr_questions_str = \"\\n\".join(duplicate_germandpr_questions)\n",
    "\n",
    "# Print the duplicate questions\n",
    "print(\"Duplicate questions in germanquad:\\n\", duplicate_germanquad_questions_str)\n",
    "print(\"Duplicate questions in germandpr:\\n\", duplicate_germandpr_questions_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create sets of the 'answers' fields in both datasets\n",
    "germanquad_answers = set([item['answers']['text'][0] for item in germanquad])\n",
    "germandpr_answers = set([item['answers'][0] for item in germandpr])\n",
    "print(len(germanquad_answers), len(germandpr_answers))\n",
    "\n",
    "# Find the intersection of the two sets\n",
    "overlap = germanquad_answers & germandpr_answers\n",
    "\n",
    "# The size of the intersection set is the number of overlapping answers\n",
    "overlap_size = len(overlap)\n",
    "\n",
    "# Print the result\n",
    "print(f'There are {overlap_size} overlapping answers between the two datasets.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "germanquad[168]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "germandpr_example = germandpr[0]\n",
    "germandpr_example.keys(), germandpr_example['positive_ctxs'].keys()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "germandpr_titles = set([item['positive_ctxs']['title'][0] for item in germandpr])\n",
    "'Portugal' in germandpr_titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "portugal = germandpr.filter(lambda example: 'Portugal' in example['positive_ctxs']['title'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(portugal[0]['positive_ctxs']['text'][0])\n",
    "portugal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len('Die EPRDF bildete eine neue Regierung und erlaubte die Unabhängigkeit Eritreas. Diese wurde nach einer durch die UN überwachten Volksabstimmung am 24. Mai 1993 erklärt, bei der 99,83 Prozent der Teilnehmer für die Unabhängigkeit stimmten. Dieser Tag ist seither Nationalfeiertag Eritreas.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "germandpr_example['answers'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "germandpr_example['positive_ctxs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "germandpr = load_dataset(\"deepset/germandpr\")[\"train\"]\n",
    "# check for Portugal and Utrecht\n",
    "portugal = germandpr.filter(lambda example: 'Portugal' in example['positive_ctxs']['title'][0])\n",
    "print(len(portugal))\n",
    "utrecht = germandpr.filter(lambda example: 'Bahnhof Utrecht Centraal' in example['answers'][0])\n",
    "print(len(utrecht))\n",
    "oklahoma = germandpr.filter(lambda example: 'Wie heißt das Stadion der Oklahoma City Thunder?' in example['question'])\n",
    "print(len(oklahoma))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# typo in Portugal context\n",
    "germandpr = germandpr.filter(lambda example: 'Portugal' not in example['positive_ctxs']['title'][0])\n",
    "# Utrecht duplicate\n",
    "germandpr = germandpr.filter(lambda example: 'Bahnhof Utrecht Centraal' not in example['answers'][0])\n",
    "# Oklahoma near duplicate\n",
    "germandpr = germandpr.filter(lambda example: 'Wie heißt das Stadion der Oklahoma City Thunder?' not in example['question'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Among multiple identical positive contexts, keep only the one with the longest answer.\"\"\"\n",
    "from collections import defaultdict\n",
    "\n",
    "def get_longest_answer(answer_list):\n",
    "    return max(answer_list, key=lambda answer: len(answer))\n",
    "\n",
    "positive_ctxs_dict = defaultdict(list)\n",
    "\n",
    "\n",
    "for item in germandpr:\n",
    "    positive_ctxs_dict[item['positive_ctxs']['text'][0]].append(item)\n",
    "\n",
    "germandpr_subset = [get_longest_answer(answer_list) for answer_list in positive_ctxs_dict.values()]\n",
    "len(germandpr_subset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check for Portugal and Utrecht\n",
    "portugal = germandpr.filter(lambda example: 'Portugal' in example['positive_ctxs']['title'][0])\n",
    "print(len(portugal))\n",
    "utrecht = germandpr.filter(lambda example: 'Bahnhof Utrecht Centraal' in example['answers'][0])\n",
    "print(len(utrecht))\n",
    "oklahoma = germandpr.filter(lambda example: 'Wie heißt das Stadion der Oklahoma City Thunder?' in example['question'])\n",
    "print(len(oklahoma))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "# Get all questions and answers from the subset\n",
    "germandpr_subset_questions = [item['question'] for item in germandpr_subset]\n",
    "germandpr_subset_answers = [item['answers'][0] for item in germandpr_subset]\n",
    "\n",
    "# Count the occurrences of each question and answer\n",
    "germandpr_subset_question_counts = Counter(germandpr_subset_questions)\n",
    "germandpr_subset_answer_counts = Counter(germandpr_subset_answers)\n",
    "\n",
    "# Get the questions and answers that appear more than once\n",
    "duplicate_germandpr_subset_questions = [question for question, count in germandpr_subset_question_counts.items() if count > 1]\n",
    "duplicate_germandpr_subset_answers = [answer for answer, count in germandpr_subset_answer_counts.items() if count > 1]\n",
    "\n",
    "# Format the duplicate questions and answers as a string\n",
    "duplicate_germandpr_subset_questions_str = \"\\n\".join(duplicate_germandpr_subset_questions)\n",
    "duplicate_germandpr_subset_answers_str = \"\\n\".join(duplicate_germandpr_subset_answers)\n",
    "\n",
    "# Print the duplicate questions and answers\n",
    "print(\"Duplicate questions in germandpr_subset:\\n\", duplicate_germandpr_subset_questions_str)\n",
    "print(\"Duplicate answers in germandpr_subset:\\n\", duplicate_germandpr_subset_answers_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "germandpr_subset_df = pd.DataFrame(germandpr_subset)\n",
    "germandpr_subset_df['positive_ctxs'] = germandpr_subset_df['positive_ctxs'].apply(lambda x: x['text'][0])\n",
    "\n",
    "answer_lengths = germandpr_subset_df['answers'].apply(lambda x: len(x[0]))\n",
    "answer_lengths.describe()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the duplicate questions and answers\n",
    "duplicate_questions = germandpr_subset_df[germandpr_subset_df.duplicated(['question'], keep=False)]\n",
    "duplicate_answers = germandpr_subset_df[germandpr_subset_df.duplicated(['answers'], keep=False)]\n",
    "\n",
    "# Sort the DataFrames so that duplicates are on subsequent rows\n",
    "duplicate_questions = duplicate_questions.sort_values('question')\n",
    "duplicate_answers = duplicate_answers.sort_values('answers')\n",
    "\n",
    "# print(duplicate_questions.iloc[0, 2]['text'][0])\n",
    "# print(len(duplicate_questions.iloc[0, 2]['text'][0]))\n",
    "# print(duplicate_questions.iloc[1, 2]['text'][0])\n",
    "# print(len(duplicate_questions.iloc[1, 2]['text'][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "longest_answer = max(germandpr, key=lambda x: len(x['answers'][0]))['answers'][0]\n",
    "shortest_answer = min(germandpr, key=lambda x: len(x['answers'][0]))['answers'][0]\n",
    "sorted_answers = sorted(germandpr, key=lambda x: len(x['answers'][0]))\n",
    "median_answer = sorted_answers[len(sorted_answers) // 2]['answers'][0]\n",
    "print(longest_answer)\n",
    "print(shortest_answer)\n",
    "print(median_answer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "q_for_shortest_answer = min(germandpr, key=lambda x: len(x['answers'][0]))['question']\n",
    "q_for_shortest_answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "longest_answer = max(germandpr_subset, key=lambda x: len(x['answers'][0]))['answers'][0]\n",
    "shortest_answer = min(germandpr_subset, key=lambda x: len(x['answers'][0]))['answers'][0]\n",
    "sorted_answers = sorted(germandpr_subset, key=lambda x: len(x['answers'][0]))\n",
    "median_answer = sorted_answers[len(sorted_answers) // 2]['answers'][0]\n",
    "print(longest_answer)\n",
    "print(shortest_answer)\n",
    "print(median_answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_count = 0\n",
    "for example in germandpr:\n",
    "    if example[\"answers\"][0].isdigit():\n",
    "        num_count += 1\n",
    "print(f'The answer text is a digit in {num_count} examples in germandpr.')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "!python -m spacy download de_core_news_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the German model\n",
    "nlp = spacy.load('de_core_news_sm')\n",
    "\n",
    "for example in germandpr_subset[:20]:\n",
    "    doc = nlp(example['answers'][0])\n",
    "    \n",
    "    # Extract entities\n",
    "    for entity in doc.ents:\n",
    "        print(entity.text, entity.label_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_first_element_single_word(lst):\n",
    "    first = lst[0].strip()\n",
    "    if lst and isinstance(first, str):\n",
    "        return len(first.split()) == 1\n",
    "    return False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "# python -m spacy download de_core_news_lg\n",
    "nlp = spacy.load('de_core_news_lg')\n",
    "\n",
    "def nlp_is_first_element_single_word(lst):\n",
    "    first = lst[0].strip()\n",
    "    if lst and isinstance(first, str):\n",
    "        doc = nlp(first)\n",
    "        return len(doc) == 1\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_headings(split_text):\n",
    "    if split_text[0] == \"\":\n",
    "        split_text.pop(0)\n",
    "    if split_text[0].count(\"=\") >= 2:\n",
    "        split_text.pop(0)\n",
    "    return split_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "example = germandpr_subset[0]\n",
    "example[\"hard_negative_ctxs\"][\"text\"][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exception_answers = [\"Knicklichter\", \"Goodluck Jonathan\", \"Beim Danner-Verfahren fließt eine Glasschmelze als Band auf einen schräg nach unten geneigten, rotierenden keramischen Hohlzylinder \"]\n",
    "# use conventional splitting and nlp to remove titels and headings\n",
    "for example in germandpr_subset:\n",
    "    # print(example)\n",
    "    positive_ctxs = example['positive_ctxs']['text']\n",
    "    negative_ctxs = example['hard_negative_ctxs']['text']\n",
    "    contexts = positive_ctxs + negative_ctxs\n",
    "    for ctx in contexts:\n",
    "        ctx_split = ctx.split(\"\\n\")\n",
    "        if not is_first_element_single_word(ctx_split):\n",
    "            print(ctx_split)\n",
    "            raise ValueError(\"First element is not a single word\")\n",
    "        ctx_split.pop(0)    # remove title\n",
    "        ctx_split = remove_headings(ctx_split)\n",
    "        if nlp_is_first_element_single_word(ctx_split):\n",
    "            print(ctx)\n",
    "            print(ctx_split)\n",
    "            print(example['question'])\n",
    "            print(example['positive_ctxs']['text'])\n",
    "            print(example['answers'])\n",
    "            if example[\"answers\"][0] in exception_answers:\n",
    "                # remove them too?\n",
    "                continue\n",
    "            raise ValueError(\"First sentence is a single word\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}