_functions_rev2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3f95259b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import pdfplumber\n",
    "import spacy\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f5c62ccd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    text = re.sub(r\"\\s+\", \" \", text)\n",
    "    words = {\n",
    "        #'Soll-/Istwerte': 'Sollwerte, Istwerte',\n",
    "        ' Œ': '-', \n",
    "        '\\n':'',\n",
    "        #'[\\[].*?[\\]]': '',\n",
    "        #'[!\"#%\\'()*+,-./:;<=>?@\\[\\]^_`{|}~1234567890’\"\"′‘\\\\\\]':'', \n",
    "        #' +': ' ',\n",
    "        ' •': ''\n",
    "    }\n",
    "    for k, w in words.items():\n",
    "        text = text.replace(k, w)\n",
    "\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "88673236",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text_2(text):\n",
    "    words = {\n",
    "\t'..':'', '▶':'-', '- ':'', # 1 and 2 in 0,1,4,5. 3 in 4.\n",
    "#\t'>':':' \n",
    "    }\n",
    "    for k, w in words.items():\n",
    "        text = text.replace(k, w)\n",
    "\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dda7b908",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text_3(text): # combination of clean_text and clean_text_2\n",
    "    text = re.sub(r\"\\s+\", \" \", text)\n",
    "    words = {\n",
    "        ' Œ': '-', \n",
    "        '\\n':'',\n",
    "        ' •': '',\n",
    "        '..':'', '▶':'-', '- ':'', '¼':'`',\n",
    "    }\n",
    "    for k, w in words.items():\n",
    "        text = text.replace(k, w)\n",
    "\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "35d46762",
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_correct_sentence_german(sent):\n",
    "    if len(sent) <= 3:\n",
    "#        print(\"  >too few words\")\n",
    "        return False\n",
    "\n",
    "    has_verb = False\n",
    "    has_subject = False\n",
    "    for token in sent:\n",
    "        # this will be different for English\n",
    "        if token.pos_ == \"VERB\" or token.tag_ in [\"VAFIN\", \"VMFIN\"]:\n",
    "            has_verb = True\n",
    "        elif token.dep_ == \"sb\":\n",
    "            has_subject = True\n",
    "\n",
    "    is_correct = has_verb and has_subject\n",
    "#    if not is_correct:\n",
    "#        if not has_verb:\n",
    "#            print(\"  >has no verb\")\n",
    "#        if not has_subject:\n",
    "#            print(\"  >has no subject\")\n",
    "\n",
    "    return is_correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b3053994-2cc6-457a-a18b-55c7e53b41b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def document_processing(document, page_start, page_end):\n",
    "    \n",
    "    def check_bboxes(word, table_bbox):\n",
    "        \"\"\"\n",
    "        Check whether word is inside a table bbox.\n",
    "        \"\"\"\n",
    "        l = word['x0'], word['top'], word['x1'], word['bottom']\n",
    "        r = table_bbox\n",
    "        return l[0] > r[0] and l[1] > r[1] and l[2] < r[2] and l[3] < r[3]\n",
    "\n",
    "    tables = []\n",
    "    counter = 0\n",
    "    raw_text_cleaned = ''\n",
    "    TOI_start = page_start\n",
    "    TOI_end = page_end\n",
    "\n",
    "    with pdfplumber.open(document) as pdf:\n",
    "        for page in pdf.pages[TOI_start-1:TOI_end]:        \n",
    "            tables.append(page.find_tables())\n",
    "            table_bboxes = [i.bbox for i in tables[counter]]\n",
    "            tables[counter] = [{'table': [[clean_text_3(x) if x is not None else x for x in item] for item in i.extract()], 'doctop': i.bbox[1]} for i in tables[counter]]\n",
    "            non_table_words = [word for word in page.extract_words() if not any(\n",
    "                [check_bboxes(word, table_bbox) for table_bbox in table_bboxes])]\n",
    "    \n",
    "            for cluster in pdfplumber.utils.cluster_objects(non_table_words+tables[counter], 'doctop', tolerance=5):\n",
    "                if 'text' in cluster[0]:\n",
    "                    text = (' '.join([i['text'] for i in cluster]))+' '\n",
    "                    text = clean_text_3(text)\n",
    "                    raw_text_cleaned += text\n",
    "            counter+=1\n",
    "    d = dict();\n",
    "    d['text'] = raw_text_cleaned\n",
    "    d['tables'] = [item for item in tables if item!= []]\n",
    "    \n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "83deff72-f6dc-4d42-ba9b-ff2cc5f86bbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentence_catcher(text, phrase):\n",
    "\n",
    "    nlp = spacy.load(\"en_core_web_lg\")   \n",
    "    doc = nlp(text)\n",
    "    \n",
    "    sentence_spans = []\n",
    "    for sent in doc.sents:\n",
    "        sentence_spans.append(sent)\n",
    "\n",
    "    phraseParts = phrase.split()\n",
    "    list_matched_sents = []\n",
    "    \n",
    "    if phrase == 'tooling adaptor':\n",
    "        phraseRegex = r'(?=.*\\b(' + phraseParts[0] + r'|tool)\\b)(?=.*\\b(' + phraseParts[1] + r'|adaptors)\\b)(?!.*\\b(assembly|assemblies|assembly`s)\\b)'\n",
    "        \n",
    "        counter=0\n",
    "        for item in sentence_spans:\n",
    "            if re.search(phraseRegex, item.text, re.IGNORECASE) or re.search(phraseRegex, item.text, re.IGNORECASE) or re.search(phraseRegex, item.text, re.IGNORECASE):\n",
    "                #print(counter,\"\\n\",item)\n",
    "                counter+=1\n",
    "                list_matched_sents.append(item)\n",
    "\n",
    "        # filtration for Tooling Adaptor sents\n",
    "        list_matched_sents[0] = list_matched_sents[0][4:]\n",
    "        list_matched_sents[5] = list_matched_sents[5][2:]\n",
    "        list_matched_sents[6] = list_matched_sents[6][4:]\n",
    "        list_matched_sents = list_matched_sents[:-1]\n",
    "\n",
    "    if phrase == 'robot adaptor':\n",
    "        phraseRegex = r'(?=.*\\b(' + phraseParts[0] + r')\\b)(?=.*\\b(' + phraseParts[1] + r'|adaptors)\\b)(?!.*\\b(assembly|assemblies|assembly`s)\\b)'\n",
    "        \n",
    "        counter=0\n",
    "        for item in sentence_spans:\n",
    "            if re.search(phraseRegex, item.text, re.IGNORECASE) or re.search(phraseRegex, item.text, re.IGNORECASE) or re.search(phraseRegex, item.text, re.IGNORECASE):\n",
    "                #print(counter,\"\\n\",item)\n",
    "                counter+=1\n",
    "                list_matched_sents.append(item)\n",
    "\n",
    "        # filtration for Robot Adaptor sents\n",
    "        list_matched_sents[0] = list_matched_sents[0][17:]\n",
    "        del list_matched_sents[2]\n",
    "        list_matched_sents[5] = list_matched_sents[5][23:]\n",
    "        list_matched_sents[7] = list_matched_sents[7][2:]\n",
    "        del list_matched_sents[4]\n",
    "        del list_matched_sents[5]\n",
    "        del list_matched_sents[6]\n",
    "\n",
    "    return list_matched_sents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b50781e2-683e-45d5-bf4f-83c9c74d6ddd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def ontology_transformer(sentence):\n",
    "        \n",
    "    def decision_maker(sent):\n",
    "        \n",
    "        wordsInSents = {}\n",
    "        for token in sent:\n",
    "            if spacy.explain(token.pos_) in ('noun', 'pronoun', 'proper noun'):\n",
    "                wordsInSents[token.text] = 'object'\n",
    "            if spacy.explain(token.pos_) in ('particle', 'auxiliary', 'verb'):\n",
    "                wordsInSents[token.text] = 'action'\n",
    "            if spacy.explain(token.pos_) in ('adposition', 'subordinating conjunction', 'coordinating conjunction'):\n",
    "                wordsInSents[token.text] = 'positioning'\n",
    "            if spacy.explain(token.pos_) in ('determiner', 'adverb', 'adjective', 'numeral'):\n",
    "                wordsInSents[token.text] = 'determinant'\n",
    "        \n",
    "        dummy = {}\n",
    "        removals = []\n",
    "        for i in range(1,len(wordsInSents)):\n",
    "            if list(wordsInSents.values())[i] != list(wordsInSents.values())[i-1]:\n",
    "                dummy[list(wordsInSents.keys())[i-1]] = list(wordsInSents.values())[i-1]\n",
    "            else:\n",
    "                dummy[list(wordsInSents.keys())[i-1]+' '+list(wordsInSents.keys())[i]] = list(wordsInSents.values())[i]\n",
    "                removals.append(i)\n",
    "        \n",
    "        if list(wordsInSents.values())[-1] == list(wordsInSents.values())[-2]:\n",
    "            dummy[list(wordsInSents.keys())[-2]+' '+list(wordsInSents.keys())[-1]] = list(wordsInSents.values())[-1]\n",
    "            dummy.pop(list(wordsInSents.keys())[-2], None)\n",
    "        else:\n",
    "            dummy[list(wordsInSents.keys())[-1]] = list(wordsInSents.values())[-1]\n",
    "        \n",
    "        for k in removals:\n",
    "            dummy.pop(list(wordsInSents.keys())[k], None)\n",
    "        \n",
    "        wordsInSents = dummy\n",
    "        \n",
    "        return wordsInSents\n",
    "\n",
    "    list_of_wordsInSent = []\n",
    "    for item in sentence:\n",
    "        list_of_wordsInSent.append(decision_maker(item))\n",
    "\n",
    "    return list_of_wordsInSent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5972d1b3-5c1c-45b0-92c0-4caf63cd1e90",
   "metadata": {},
   "outputs": [],
   "source": [
    "def phrase_namer(phrase):\n",
    "    naming = []\n",
    "    for i in range(len(phrase.split())):\n",
    "        naming.append(phrase.split()[i])\n",
    "        if i == len(phrase.split())-1:\n",
    "            break\n",
    "        else:\n",
    "            naming.append('-')\n",
    "    return ''.join(naming)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}