Vocabs_process_rdfscomments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ampligraph\n",
    "from ampligraph.datasets import load_from_ntriples, load_from_rdf\n",
    "from ampligraph.evaluation import train_test_split_no_unseen\n",
    "from ampligraph.latent_features import ComplEx, DistMult, TransE, save_model, restore_model\n",
    "from ampligraph.evaluation import evaluate_performance, mrr_score, hits_at_n_score, mr_score\n",
    "\n",
    "from rdflib.util import guess_format\n",
    "from rdflib import Graph, URIRef, ConjunctiveGraph, Literal, BNode\n",
    "from rdflib.namespace import RDF, OWL, RDFS\n",
    "import rdflib\n",
    "\n",
    "import requests\n",
    "import urllib.request\n",
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy import stats\n",
    "\n",
    "import pickle\n",
    "\n",
    "np.set_printoptions(threshold=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vocabPrefix</th>\n",
       "      <th>domain</th>\n",
       "      <th>uri version</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SAN</td>\n",
       "      <td>IoT</td>\n",
       "      <td>https://lov.linkeddata.es/dataset/lov/vocabs/S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SAN</td>\n",
       "      <td>Support</td>\n",
       "      <td>https://lov.linkeddata.es/dataset/lov/vocabs/S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>acco</td>\n",
       "      <td>eBusiness</td>\n",
       "      <td>https://lov.linkeddata.es/dataset/lov/vocabs/a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>acl</td>\n",
       "      <td>API</td>\n",
       "      <td>https://lov.linkeddata.es/dataset/lov/vocabs/a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>acm</td>\n",
       "      <td>Catalogs</td>\n",
       "      <td>https://lov.linkeddata.es/dataset/lov/vocabs/a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  vocabPrefix     domain                                        uri version\n",
       "0         SAN        IoT  https://lov.linkeddata.es/dataset/lov/vocabs/S...\n",
       "1         SAN    Support  https://lov.linkeddata.es/dataset/lov/vocabs/S...\n",
       "2        acco  eBusiness  https://lov.linkeddata.es/dataset/lov/vocabs/a...\n",
       "3         acl        API  https://lov.linkeddata.es/dataset/lov/vocabs/a...\n",
       "4         acm   Catalogs  https://lov.linkeddata.es/dataset/lov/vocabs/a..."
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_graphs = pd.read_csv(\"lov_uri-domain-version - lov_uri-domain-version.csv\", sep=\",\")\n",
    "all_graphs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vocabPrefix</th>\n",
       "      <th>uri version</th>\n",
       "      <th>domain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>661</td>\n",
       "      <td>661</td>\n",
       "      <td>661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>661</td>\n",
       "      <td>660</td>\n",
       "      <td>121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>rami</td>\n",
       "      <td>https://lov.linkeddata.es/dataset/lov/vocabs/l...</td>\n",
       "      <td>Metadata</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       vocabPrefix                                        uri version  \\\n",
       "count          661                                                661   \n",
       "unique         661                                                660   \n",
       "top           rami  https://lov.linkeddata.es/dataset/lov/vocabs/l...   \n",
       "freq             1                                                  2   \n",
       "\n",
       "          domain  \n",
       "count        661  \n",
       "unique       121  \n",
       "top     Metadata  \n",
       "freq          40  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grouped = all_graphs.groupby([\"vocabPrefix\", \"uri version\"])[\"domain\"].apply(lambda x: ','.join(x)).reset_index()\n",
    "grouped.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "661\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['SAN', 'acco', 'acl', ..., 'xhv', 'xkos', 'zbwext'], dtype=object)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y = grouped.domain.values\n",
    "vocabs = grouped.vocabPrefix.values\n",
    "Urls = grouped.iloc[:,1].values\n",
    "print(len(Urls))\n",
    "#print(Urls)\n",
    "vocabs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The triplets containing these predicates have to be removed (they are for metadata or are not relevant information)\n",
    "TEXT_SUFFIXES = [\"comment\", \"description\", \"label\", \"definition\"]\n",
    "#TEXT_SUFFIXES = [\"comment\", \"description\", \"definition\"]\n",
    "#TEXT_SUFFIXES = [\"comment\", \"description\"]\n",
    "\n",
    "# From a rdflib graph, concatenate all rdfs:comments in a string\n",
    "def process(rdflib_graph):\n",
    "    full_text = \"\"\n",
    "    \n",
    "    for s,p,o in rdflib_graph:\n",
    "        # remove literals\n",
    "        #if type(o) != rdflib.term.Literal:\n",
    "        suf = rdflib_graph.compute_qname(p)[2]\n",
    "        if suf in TEXT_SUFFIXES:\n",
    "            text = str(o)\n",
    "            if len(text) > 0 and text[-1] != \".\":\n",
    "                text += \".\"\n",
    "            full_text += text\n",
    "            full_text += \" \"\n",
    "        \n",
    "        #print(str(o))\n",
    "        #print(text)\n",
    "            \n",
    "    return full_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "at line 2179 of <>:\n",
      "Bad syntax (objectList expected) at ^ in:\n",
      "\"...b'lGlassBottle ,\\n    container:640mlPlasticBottle ,\\n    contai'^b'ner:1_8lPlasticBottle ;\\n  rdfs:seeAlso <http://ko.wikipedia.'...\" https://lov.linkeddata.es/dataset/lov/vocabs/bevon/versions/2015-07-23.n3\n",
      "at line 39 of <>:\n",
      "Bad syntax (objectList expected) at ^ in:\n",
      "\"...b's4:Concept;\\n    ns4:prefLabel \"Regio\"@nl;\\n    ns4:definition'^b\" '''De gemeenten in Nederland zijn onderverdeeld in wijken e\"...\" https://lov.linkeddata.es/dataset/lov/vocabs/cbs/versions/2018-05-02.n3\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n"
     ]
    }
   ],
   "source": [
    "all_texts = []\n",
    "inds_to_remove = []\n",
    "\n",
    "for i,(url,vocab) in enumerate(zip(Urls, vocabs)):\n",
    "    if i % 100 == 0:\n",
    "        print(i)\n",
    "    # Load graphs from url\n",
    "    try:\n",
    "#         Download the file from `url`, save it in a temporary directory and get the\n",
    "#         path to it (e.g. '/tmp/tmpb48zma.txt') in the `file_name` variable:\n",
    "#         file_name, headers = urllib.request.urlretrieve(url)\n",
    "        \n",
    "        # Download the graph file from url, if the file does not exist\n",
    "        path = \"vocabs/\" + vocab\n",
    "        if not(os.path.isfile(path)):\n",
    "            urllib.request.urlretrieve(url, path)\n",
    "            \n",
    "        \n",
    "        # Load rdflib graph from n3 file\n",
    "        g = Graph()\n",
    "        data = open(path, \"rb\")\n",
    "        g.parse(data, format=\"n3\")\n",
    "        \n",
    "        # Extract all text from graph\n",
    "        text = process(g)\n",
    "        all_texts.append(text)\n",
    "    except Exception as e:\n",
    "        # If graph cannot be loaded, refresh arrays of vocabs and tags\n",
    "        inds_to_remove.append(i)\n",
    "        print(e, url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove unloaded graphs in labels arrays\n",
    "vocabs2 = np.delete(vocabs,inds_to_remove,0)\n",
    "Y2 = np.delete(Y,inds_to_remove,0)\n",
    "assert len(all_texts) == len(vocabs2) == len(Y2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=659, minmax=(0, 1002586), mean=15896.468892261002, variance=3533422744.006249, skewness=12.460220705192233, kurtosis=186.43158015783698)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_texts_length = [len(st) for st in all_texts]\n",
    "stats.describe(all_texts_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([201, 248, 269]),)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inds_len0 = np.where(np.array(all_texts_length) == 0)\n",
    "inds_len0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gci inno keys\n"
     ]
    }
   ],
   "source": [
    "# 3 vocabs n'ont pas de textes dans leur n3. inno est un fichier vide et gci et keys n'ont pas de commentaires/labels\n",
    "print(vocabs2[201],vocabs2[248],vocabs2[269])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Delete vocabs with no text at all\n",
    "for ind in inds_len0[0]:\n",
    "    del all_texts[ind]\n",
    "vocabs3 = np.delete(vocabs2,inds_len0[0],0)\n",
    "Y3 = np.delete(Y2,inds_len0[0],0)\n",
    "\n",
    "assert len(all_texts) == len(vocabs3) == len(Y3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_multilabel(Y):\n",
    "    Y_out = []\n",
    "    for y in Y:\n",
    "        labels = y.split(\",\")\n",
    "        Y_out.append(labels)\n",
    "    return Y_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_final = process_multilabel(Y3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "aos\n",
      "Simple Skin Color Albino. Gender (Generic). Blue Gray Iris (5). Has Thigh Measurement in Meters. Blond. Von Luschan Skin Color 22. Caribbean. Light Blue Iris (1c). Unknown. White. Frau. Mediterranean European/Hispanic. Dark Blond (Human Hair Atlas). Von Luschan Skin Color 10. Has Thigh Measurement in Inches. Has Sex. Not Available. Blue. Freckles. White. Gender Unknown. Medium to Dark Red Brown (Human Hair Atlas). htmlColor. Blue. Yellow Brown and Brown Green (8). Has Over Arm Measurement in Inches. Von Luschan Skin Color 31. Von Luschan Skin Color 7. Fair. Medium to Dark Gray Brown, Black (Human Hair Atlas). Dark to Opaque Brown (Human Hair Atlas). Medium. Grey. Sandy. Medium to Dark Red (Human Hair Atlas). Gray (6). Has Pant Length Measurement. Brown (ICCS #4). Light Brown with Yellow Specks (ICCS #3). Dyed Hair Color. Green. White and Black Caribbean. Natural Hair Color. Simple Eye Color Reference. Pétrequin Eye Color Reference. Grey/Gray. Has Inseam Measurement in Meters. Brown. Green. Bald. Has Pant Length Measurement in Meters. Woman. has Sexual Aversion. Wilde Eye Color Reference. Any other ethnic group. Black. Light Blue Iris (2a). Strict F/M coding of Gender types, with restriction. Any other Mixed ethnic background. Dark Grey. Von Luschan Skin Color 17. African. Dark Blue Iris (2b). Nebraska DMV Standards. Has Neck Measurement. Simple Skin Color Black. Grey (Riddell). Purple. Strict F/M coding of Gender types, with restriction. Femme. Dark Brown and Brown Black (16). Green with Yellow Specks (ICCS #2). has Appearance. Pakistani. Von Luschan Skin Color 33. Light to Medium Gray Brown (Human Hair Atlas). Light Brown. Black. Mann. Black. This skin color is the same as number 14. . Has Band Measurement. Has Waist Measurement in Inches. Riddell Color Reference. Red. Roux (Pétrequin). Female Body Measurement. No Brown. Riddell Eye Color Reference. Von Luschan Skin Color 27. Non-sexual. has Height Measurement in Meters. Pétrequin Eye Color Reference. Natural Person. Bald. Red. Gray (Human Hair Atlas). Brown. Von Luschan Skin Color 15. Fitzpatrick Scale Type. Has Chest Measurement. Blue (Wilde). Unknown. Has Knee Length Measurement in Meters. Von Luschan Skin Color 14. Arab person. Man. Traces of Brown. Observation. Blue (Riddell). Has Band Measurement in Inches. Ogle, Robert R, Fox, Michelle J. Atlas of human hair microscopic characteristics, CRC Press, 1999. Predominantly heterosexual, only incidentally homosexual. Blue with Brown Specks (ICCS #2). Light to Medium Brown (Human Hair Atlas). Riddell Color Reference. Sandy. S. Hazel. Woman. Green. Black (Wilde). Dark. William Robert Wilde. An Essay on the malformations and congenital diseases of the organs of sight, John Churchill, 1862. Femme. Simple Skin Color White. Bleus (Pétrequin). Has Waist Measurement in Meters. Von Luschan Skin Color 32. Dark Blue. Blonde / Strawberry. Black (Pétrequin). Albino. Brown (ICCS #5). Black, Dark Gray Brown to Opaque Gray Brown (Human Hair Atlas). Grieve Morant Eye Color Reference. Predominantly homosexual, only incidentally heterosexual. Has Chest Measurement in Meters. Gray (Wilde). Dark Brown and Brown Black (15). Light Red (Human Hair Atlas). Man. Man. Blonde. Bruns (Pétrequin). Has Inseam Measurement in Inches. XX. Woman. Yellow Brown and Brown Green (10). Light intermediate. Simple eye color reference. Light. Medium to Dark Brown (Human Hair Atlas). Grey / Gray. Sex Inconnu. Riddell Spots Eye Color. UK CensusCodes. Has Band Measurement in Meters. Brown. Gray. EyeColorNebraskaDMV. Blue with Yellow Specks (ICCS #1). Chinese. Hazel. Has Thigh Measurement. White. Has Neck Measurement in Meters. Dichromatic. Maroon. Very Dark. Self Reported. Green with Brown Specks (ICCS #3). Has Over Arm Measurement. Still more Brown. Predominantly heterosexual, but more than incidentally homosexual. Brown. Wilde Eye Color Reference. Green with Yellow Specks (ICCS #3). SimplebGender. Light Brown with Brown Specks (ICCS #3). Hair Color. Riddell General Eye Color. Red (Human Hair Atlas). Medium to Dark Golden Brown (Human Hair Atlas). Light Blue Iris. Hazel (Wilde). Von Luschan Skin Color 2. Has Pant Length Measurement in Inches. Simple Skin Color Brown. Blue (Pétrequin). Any other Asian ethnic background. Von Luschan Skin Color 20. Dark Golden Brown (Human Hair Atlas). Von Luschan Skin Color 16. Brown. Gris (Pétrequin). has Entertaiment Preference. Light Gray Brown (Human Hair Atlas). No color (Riddell). This ontology is meant to record individual's sex, gender identities as well as their preferences, desires and relationships. Green. FBI Eye Color Reference - Essentially the same as used by the RCMP. Von Luschan Skin Color 21. Mann. Von Luschan Skin Color 30. Dark Brown and Brown Black (12). Riddell Diffuse Eye Color. Dark. has Entertaiment Aversion. FBI Standards. Dark Brown and Brown Black (14). Simple Skin Color Olive. Green (Riddell). Has Arm Measurement in Meters. has Romantic Aversion. More Brown (J). Von Luschan Skin Color 5. Von Luschan Skin Color 34. Mann. Gray with Yellow Specks (ICCS #1). Black. Golden Brown (Human Hair Atlas). Homme. Homme. HairColorAtlas. Has Gender. Riddell Eye Color Reference. The Riddell standard makes use of a three part scale that marks up the main colour of the eye, the diffuse color of the eye and any colour spots within the eye. Yellow Brown and Brown Green (7). Black. Red Brown (Human Hair Atlas). Dark intermediate. Very Fair. Beige with a brown tint. Femme. Gray (Pétrequin). White person, northern European type. FBI Eye Color Reference. Dark Red Brown (Human Hair Atlas). Von Luschan Skin Color 35. Von Luschan Skin Color 28. Chinese, Japanese, or South-East Asian person. Mediterranean. Dark Brown and Brown Black (13). Light Blond to Medium (Human Hair Atlas). Has Height Measurement in Inches. Has Body Measurement. Height Measurement. Von Luschan Skin Color 11. Orange. Skin Color. Von Luschan Skin Color 13. Medium to Dark Blond (Human Hair Atlas). Gray Iris (4b). Male Body Measurement. RCMP Standards. J. Petrequin. Sur les diverses couleurs de l'iris et leurs proportions dens nos climats.. Annales D'oculistique 10, Florent Cunier, 1843. Von Luschan Skin Color 8. Frau. Strawberry Red. Blue. Green with Brown Specks (ICCS #2). Red/Auburn. Has Neck Measurement in Inches. Gray with Brown Specks (ICCS #1). Has Knee Length Measurement in Inches. White and Asian. Has Arm Measurement. Von Luschan Skin Color 12. Light Blue Iris (1a). XY. Von Luschan Skin Color 3. Red. Tan (Riddell). Brown (Pétrequin). Any other White ethnic background. Blue with Yellow Specks (ICCS #2). Von Luschan Skin Color 23. Has Over Arm Measurement in Meters. Black. British. Light Golden Brown (Human Hair Atlas). Light Blue. Origin unknown. Yellow (Riddell). Exclusively homosexual. Unknown. Most Brown (K). Has Knee Length Measurement. J. M. Seddon, C. R. Sahagian, R. J. Glynn, R. D. Sperduto, E. S. Gragoudas. Evaluation of an iris color classification system. The Eye Disorders Case-Control Study Group.. Investigative Ophthalmology & Visual Science 31(8), August, 1990. Von Luschan Skin Color 4. GRIEVE, J., MORANT, G. M.. RECORDS OF EYE COLOURS FOR BRITISH POPULATIONS AND A DESCRIPTION OF A NEW EYE-COLOUR SCALE. Annals of Eugenics 13(1), Blackwell Publishing Ltd, 1946. Chocolate (Riddell). Includes partially gray. Simple Skin Color. NebraskaDMVHairColor. Exclusively heterosexual. Brown. RIDDELL, W. J. B.. STUDIES IN THE CLASSIFICATION OF EYE COLOUR. Annals of Eugenics 11(1), Blackwell Publishing Ltd, 1941. Mann. Iris Color Classification System. Irish. More Blue or Grey than Brown. Blue Gray Iris (3). Has Hips Measurement in Inches. Has Hips Measurement. Von Luschan Skin Color 19. Von Luschan Skin Color 29. RCMP Hair Color. Dark brown. Von Luschan Skin Color 6. Has Inseam Measurement. Pink. Sex (ISO/IEC 5218). Simple F/M coding of Gender types. Blue with Brown Specks (ICCS #1). Brown (Human Hair Atlas). Sex, Genders, Preferences and Desires Ontology. Light Blond (Human Hair Atlas). White. Gray. Brown. Pink. This skin color is the same as number 13. . FBI Hair Color.  Grieve Morant Eye Color Reference. Has Arm Measurement in Inches. Von Luschan Skin Color 1. This ontology is meant to record individual's sex, gender identities as well as their preferences, desires and relationships. Simple F/M coding of Gender types. Von Luschan Skin Color 25. Gray Iris (4a). Yellow Brown and Brown Green (9). Eye Color. Light to Medium Golden Brown (Human Hair Atlas). Opaque Brown (Human Hair Atlas). Light Grey. Skin Color. Bald. Von Luschan Skin Color 36. Grey (Pétrequin). Not Stated. White and Black African. Light Red Brown (Human Hair Atlas). Yellow Brown and Brown Green (11). Least Brown. Brown (Wilde). African/Afro-Caribbean person. Most Brown. Has Chest Measurement in Inches. Iris Color Classification System. Has Hips Measurement in Meters. Frau. Dark-skinned European. More Brown Than Blue or Grey. Blond (Human Hair Atlas). Von Luschan Skin Color 24. Von Luschan Skin Color 9. Gray with Yellow Specks (ICCS #2). Gender (Strict). Indian, Pakistani, Nepalese, Maldivian, Sri Lankan, Bangladeshi, or any other (South) Asian person. Gray with Brown Specks (ICCS #2). Homme. Dark Red (Human Hair Atlas). has Sexual Preference. Gray Brown (Human Hair Atlas). White (Human Hair Atlas). Has Waist Measurement. Noirs (Pétrequin). Indian. EyeColorMartinSchultz. Other Color(Human Hair Atlas). Von Luschan Skin Color 18. Kinsey Scale. Any other Black ethnic background. More Brown. Light to Medium Red (Human Hair Atlas). Predominantly homosexual, but more than incidentally heterosexual. Olive Skin. Mediterranean Caucasian. Gray. Light to Medium Red Brown (Human Hair Atlas). Blue. Dark (Wilde). Light Blue Iris (1b). UK IC-Codes. Equally heterosexual and homosexual. Bangladeshi. Von Luschan Skin Color 26. White. Red (Pétrequin). Light-skinned European. Pure Brown. has Romantic Preference. Light Brown (Human Hair Atlas). \n"
     ]
    }
   ],
   "source": [
    "print(vocabs3[18])\n",
    "print(all_texts[18])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "# Save data for classification\n",
    "with open('DATA.pkl', 'wb') as handle:\n",
    "    pickle.dump((all_texts, Y_final, vocabs3), handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "############# PROCESS NEW VOCABS ################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['UsabilityOntology.rdf',\n",
       " 'cultural-event.owl',\n",
       " 'munc.owl',\n",
       " 'catalogue.owl',\n",
       " 'terms.ttl',\n",
       " 'denotative-description.owl',\n",
       " 'context-description.owl',\n",
       " 'ontology.ttl',\n",
       " 'location.owl',\n",
       " 'core.owl',\n",
       " 'vir.ttl',\n",
       " 'arco.owl']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "dir_path = \"./new_vocabs\"\n",
    "files = os.listdir(dir_path)\n",
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = []\n",
    "vocabNames = []\n",
    "for f in files:\n",
    "    try:\n",
    "        path = dir_path + \"/\" + f\n",
    "\n",
    "        g = Graph()\n",
    "        data = open(path, \"rb\")\n",
    "        if f[-3:] == \"ttl\":\n",
    "            g.parse(file=data, format=\"n3\")\n",
    "        else:\n",
    "            g.parse(file=data)\n",
    "        \n",
    "        # Extract all text from graph\n",
    "        text = process(g)\n",
    "        texts.append(text)\n",
    "        vocabNames.append(f)\n",
    "        \n",
    "    except Exception as e:\n",
    "        print(e, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['UsabilityOntology.rdf',\n",
       " 'cultural-event.owl',\n",
       " 'munc.owl',\n",
       " 'catalogue.owl',\n",
       " 'terms.ttl',\n",
       " 'denotative-description.owl',\n",
       " 'context-description.owl',\n",
       " 'ontology.ttl',\n",
       " 'location.owl',\n",
       " 'core.owl',\n",
       " 'vir.ttl',\n",
       " 'arco.owl']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocabNames\n",
    "#texts[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save data for classification\n",
    "with open('newDATA.pkl', 'wb') as handle:\n",
    "    pickle.dump((texts, vocabNames), handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}