data_scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1238,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from Bio import Entrez\n",
    "from Bio import Medline\n",
    "from collections import defaultdict\n",
    "\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import requests\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1240,
   "metadata": {},
   "outputs": [],
   "source": [
    "Entrez.email = \"dn070017@gmail.com\"\n",
    "while(1):\n",
    "    try:\n",
    "        result = Entrez.read(Entrez.esearch(db=\"pubmed\", retmax=10, term=\"Kazi Alam\"))\n",
    "        break\n",
    "    except:\n",
    "        print('connection failed')\n",
    "        time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1241,
   "metadata": {},
   "outputs": [],
   "source": [
    "roster = defaultdict(dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1242,
   "metadata": {},
   "outputs": [],
   "source": [
    "def order_title(title):\n",
    "    if title in [\"Student\"]:\n",
    "        return 0\n",
    "    if title in [\"Engineer\", \"Laboratory\", \"Others\", \"Researcher\"]:\n",
    "        return 1\n",
    "    if title in [\"PhD\"]:\n",
    "        return 2\n",
    "    if title in [\"Postdoc\"]:\n",
    "        return 3\n",
    "    if title in [\"Professor\", \"Head of Group\"]:\n",
    "        return 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1243,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unify_title(title):\n",
    "    match = re.search('group leader|head of group', title.lower())\n",
    "    if match:\n",
    "        return 'Head of Group'\n",
    "    match = re.search('professor|lecturer', title.lower())\n",
    "    if match:\n",
    "        return 'Professor'\n",
    "    match = re.search('postdoctoral|post-doctoral|postdoc|post', title.lower())\n",
    "    if match:\n",
    "        return 'Postdoc'\n",
    "    match = re.search('doctoral|phd|ph\\.d', title.lower())\n",
    "    if match:\n",
    "        return 'PhD'\n",
    "    match = re.search('msc|bsc|master|bachelor|erasmus|undvp|trainee|student|junior fellow', title.lower())\n",
    "    if match:\n",
    "        return 'Student'\n",
    "    match = re.search('administrat|office|director|personal assistant', title.lower())\n",
    "    if match:\n",
    "        return 'Administration'\n",
    "    match = re.search('advis|consult', title.lower())\n",
    "    if match:\n",
    "        return 'Consultant'\n",
    "    match = re.search('coordinator|project assistant|project manager', title.lower())\n",
    "    if match:\n",
    "        return 'Coordinator'\n",
    "    match = re.search('laboratory|lab assistant', title.lower())\n",
    "    if match:\n",
    "        return 'Laboratory'\n",
    "    match = re.search('IT', title)\n",
    "    if match:\n",
    "        return 'IT'\n",
    "    match = re.search('information', title.lower())\n",
    "    if match:\n",
    "        return 'IT'\n",
    "    match = re.search('engineer|bioinformatician|technician', title.lower())\n",
    "    if match:\n",
    "        return 'Engineer'\n",
    "    match = re.search('research|academic employee|scientist|computational biologist', title.lower())\n",
    "    if match:\n",
    "        return 'Researcher'\n",
    "    return 'Others'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NCMM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1244,
   "metadata": {},
   "outputs": [],
   "source": [
    "for p in range(1, 5):\n",
    "    url = \"https://www.med.uio.no/ncmm/english/people/?page={}&u-page={}\".format(p, p)\n",
    "    resp = requests.get(url)\n",
    "    resp.encoding = 'utf-8'\n",
    "    soup = BeautifulSoup(resp.text, 'html.parser')\n",
    "    tds = soup.find_all(\"td\", class_=\"vrtx-person-listing-name\")\n",
    "    for td in tds:\n",
    "        i = 2\n",
    "        if len(td.findChildren()) == 2:\n",
    "            i = 0\n",
    "        if td.findChildren()[i].text not in roster:\n",
    "            name = td.findChildren()[i].text.split(',')\n",
    "            name = name[1][1:] + ' ' + name[0]\n",
    "            roster[name]['Name'] = name\n",
    "            roster[name]['Title'] = unify_title(td.span.text)\n",
    "            roster[name]['Institution'] = 'NCMM'\n",
    "            titles.add(unify_title(td.span.text))\n",
    "        else:\n",
    "            print('Duplicated names in roster of NCMM')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1245,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups = [\"esguerra\", \"mathelier\", \"gozen\", \"morth\", 'haapaniemi', 'kuijjer', 'sekulic', 'lopez-aviles', 'staerk', 'luecke']\n",
    "for g in groups:\n",
    "    url = \"https://www.med.uio.no/ncmm/english/groups/{}-group\".format(g)\n",
    "    resp = requests.get(url)\n",
    "    resp.encoding = 'utf-8'\n",
    "    soup = BeautifulSoup(resp.text, 'html.parser')\n",
    "    div = soup.find_all(\"div\", class_=\"vrtx-box-content\")\n",
    "    soup = BeautifulSoup(str(div), 'html.parser')\n",
    "    a_list = soup.find_all(\"a\")\n",
    "    for i, a in enumerate(a_list):\n",
    "        if i == 0:\n",
    "            l = a.text\n",
    "        if i == len(a_list) - 1:\n",
    "            continue\n",
    "        if a.text in roster:\n",
    "            if 'Group' not in roster[a.text]:\n",
    "                roster[a.text]['Group'] = set()\n",
    "            roster[a.text]['Group'].add(l)\n",
    "        else:\n",
    "            print(a.text + 'is not in NCMM dictionary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1246,
   "metadata": {},
   "outputs": [],
   "source": [
    "roster['Rafael Riudavets Puig']['Group'] = ['Anthony Mathelier']\n",
    "roster['Kazi Alam']['Group'] = ['Johannes Landskron']\n",
    "roster['Alexandra Gade']['Group'] = ['Johannes Landskron']\n",
    "roster['Silvia Espada Burriel']['Group'] = ['Sandra Lopez Aviles']\n",
    "roster['Flore Kersten']['Group'] = ['Hartmut Luecke']\n",
    "roster['Shixiong Wang']['Group'] = ['Antoni Hurtado Rodriguez']\n",
    "\n",
    "roster['Johannes Landskron']['Title'] = 'Head of Group'\n",
    "roster['Emma Haapaniemi']['Title'] = 'Head of Group'\n",
    "roster['Janna Saarela']['Title'] = 'Head of Group'\n",
    "roster['Hartmut Luecke']['Title'] = 'Head of Group'\n",
    "\n",
    "roster['Ahmad Ali Ahmad']['Name'] = 'Ali Ahmad'\n",
    "roster['Jaime Abraham Castro Mondragón']['Name'] = 'Jaime Castro Mondragón'\n",
    "roster['Camila Vicencio Esguerra']['Name'] = 'Camila Esguerra'\n",
    "roster['Kinga Aurelia Gawel']['Name'] = 'Kinga Gawel'\n",
    "roster['João Paulo Ribeiro Proença Santana']['Name'] = 'João Santana'\n",
    "roster['Antoni Hurtado Rodriguez']['Name'] = 'Antoni Rodriguez'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FIMM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1247,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups = ['Ganna', 'Groop', 'Hennah', 'Kaprio', 'Latvala', 'Ollikainen', 'Palotie', 'Pirinen', 'Ripatti',\n",
    "          'Saarela', 'Tukiainen', 'Vuoksimaa', 'Widen', 'Aittokallio', 'Heckman', 'Horvath', 'Kallioniemi',\n",
    "          'Lundin', 'Verschuren', 'Wennerberg']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1248,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sailalitha Bollepalli belongs to multiple groups in FIMM\n",
      "Milla Kibble belongs to multiple groups in FIMM\n",
      "Mia Urjansson belongs to multiple groups in FIMM\n",
      "Christian Benner  belongs to multiple groups in FIMM\n",
      "Sini Kerminen belongs to multiple groups in FIMM\n",
      "Janna Saarelacan be found in NCMM\n",
      "Eero Vuoksimaacan be found in NCMM\n",
      "Anna Cichonska belongs to multiple groups in FIMM\n",
      "Heidi Arling-Tripepi belongs to multiple groups in FIMM\n",
      "Piia  Mikkonen belongs to multiple groups in FIMM\n",
      "Heidi Arling-Tripepi belongs to multiple groups in FIMM\n",
      "Jie Bao belongs to multiple groups in FIMM\n",
      "Nora Nordström belongs to multiple groups in FIMM\n",
      "Sarang Talwelkar belongs to multiple groups in FIMM\n"
     ]
    }
   ],
   "source": [
    "for g in groups:\n",
    "    if g == \"Vuoksimaa\":\n",
    "        url = \"https://www.fimm.fi/en/research/human-genomics/cognitive-and-brain-aging\"\n",
    "    else:\n",
    "        url = \"https://www.fimm.fi/en/research/groups/{}\".format(g.lower())\n",
    "    resp = requests.get(url)\n",
    "    resp.encoding = 'utf-8'\n",
    "    soup = BeautifulSoup(resp.text, 'html.parser')\n",
    "    intro = soup.find_all(\"div\", class_=\"intro\")\n",
    "    tbody = soup.find_all(\"tbody\")\n",
    "    \n",
    "    soup = BeautifulSoup(str(intro), 'html.parser')\n",
    "    leader = soup.find_all(\"h2\")[0].text\n",
    "    \n",
    "    if leader in roster:\n",
    "        print(leader + \"can be found in NCMM\")\n",
    "    else:\n",
    "        roster[leader]['Name'] = leader\n",
    "        roster[leader]['Title'] = 'Head of Group'\n",
    "        roster[leader]['Group'] = set()\n",
    "        roster[leader]['Group'].add(leader)\n",
    "        roster[leader]['Institution'] = 'FIMM'\n",
    "    \n",
    "    soup = BeautifulSoup(str(tbody), 'html.parser')\n",
    "    name = soup.find_all(\"div\", class_=\"show-info--name\")\n",
    "    position = soup.find_all(\"div\", class_=\"field-job-title\")\n",
    "    for n, p in zip(name, position):\n",
    "        match = re.search('^\\s+(.+?)\\s+$', p.text)\n",
    "        if match:\n",
    "            title = match.group(1)\n",
    "        else:\n",
    "            title = p.text\n",
    "        if n.text == '':\n",
    "            continue\n",
    "        if n.text in roster:\n",
    "            print(n.text + \" belongs to multiple groups in FIMM\")\n",
    "            roster[n.text]['Group'].add(leader)\n",
    "        else:\n",
    "            roster[n.text]['Name'] = n.text\n",
    "            roster[n.text]['Title'] = unify_title(title)\n",
    "            if 'Group' not in roster[n.text]:\n",
    "                roster[n.text]['Group'] = set()\n",
    "            roster[n.text]['Group'].add(leader)\n",
    "            roster[n.text]['Institution'] = 'FIMM'\n",
    "        titles.add(unify_title(title))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DANDRITE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1249,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups = [\"nissen\", \"nykjaer\", \"Jensen\", \"philipsborn\", \"denham\", \"kvitsiani\", \"yonehara\", \n",
    "          \"nabavi\", \"tomonori-takeuch\", \"hanne-poulsen\", \"magnus-kjaergaard\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1250,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hanne Poulsen belongs to multiple groups in DANDRITE\n",
      "Monica Dahlstrup Sietam belongs to multiple groups in DANDRITE\n",
      "Magnus Kjærgaard belongs to multiple groups in DANDRITE\n"
     ]
    }
   ],
   "source": [
    "for g in groups:\n",
    "    if g in [\"tomonori-takeuch\", \"hanne-poulsen\", \"magnus-kjaergaard\"]:\n",
    "        url = \"http://dandrite.au.dk/people/team-leaders/{}/team-members/\".format(g.lower())\n",
    "    else:\n",
    "        url = \"http://dandrite.au.dk/people/group-leaders/{}-group/group-members/\".format(g.lower())\n",
    "    resp = requests.get(url)\n",
    "    resp.encoding = 'utf-8'\n",
    "    soup = BeautifulSoup(resp.text, 'html.parser')\n",
    "    divs = soup.find_all(\"div\", class_=\"vcard pure-simple-person-single\")\n",
    "    leader = \"\"\n",
    "    for i, div in enumerate(divs):\n",
    "        soup = BeautifulSoup(str(div), \"html.parser\")\n",
    "        first_name = soup.find(\"span\", class_=\"given-name\").text\n",
    "        last_name = soup.find(\"span\", class_=\"family-name\").text\n",
    "        title = soup.find(\"span\", class_=\"title\")\n",
    "        if title is not None:\n",
    "            title = title.text\n",
    "        else:\n",
    "            title = \"\"\n",
    "        name = first_name + \" \" + last_name\n",
    "        if i == 0:\n",
    "            leader = name\n",
    "        \n",
    "        if name in roster:\n",
    "            print(name + \" belongs to multiple groups in DANDRITE\")\n",
    "            roster[name]['Group'].add(leader)\n",
    "        else:\n",
    "            roster[name]['Name'] = name\n",
    "            if i == 0:\n",
    "                roster[name]['Title'] = \"Head of Group\"\n",
    "            else:\n",
    "                roster[name]['Title'] = title\n",
    "            if 'Group' not in roster[name]:\n",
    "                roster[name]['Group'] = set()\n",
    "            roster[name]['Group'].add(leader)\n",
    "            roster[name]['Institution'] = 'DANDRITE'\n",
    "        \n",
    "        titles.add(unify_title(title))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MIMS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1251,
   "metadata": {},
   "outputs": [],
   "source": [
    "mims = pd.read_csv('./MIMS.csv', sep=',', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1252,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Yngve Östberg belongs to multiple groups in MIMS\n",
      "Lalitha Tadala belongs to multiple groups in MIMS\n"
     ]
    }
   ],
   "source": [
    "for i, d in mims.iterrows():\n",
    "    name = d[0]\n",
    "    title = d[1]\n",
    "    leader = d[2]\n",
    "    if name in roster:\n",
    "        print(name + \" belongs to multiple groups in MIMS\")\n",
    "        roster[name]['Group'].add(leader)\n",
    "    else:\n",
    "        roster[name]['Name'] = name\n",
    "        if 'Group' not in roster[name]:\n",
    "            roster[name]['Group'] = set()\n",
    "        roster[name]['Group'].add(leader)\n",
    "        roster[name]['Title'] = unify_title(title)\n",
    "        roster[name]['Institution'] = 'MIMS'\n",
    "        titles.add(unify_title(title))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fetch Publication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1254,
   "metadata": {},
   "outputs": [],
   "source": [
    "for n in roster.keys():\n",
    "    while(True):\n",
    "        try:\n",
    "            result = Entrez.read(Entrez.esearch(db=\"pubmed\", retmax=50, term=n + \"[AUTH]\"))\n",
    "            roster[n]['Publications'] = (set(result[\"IdList\"]))\n",
    "            time.sleep(0.5)\n",
    "            break\n",
    "        except:\n",
    "            time.sleep(0.5)\n",
    "            print('retry ' + n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1255,
   "metadata": {},
   "outputs": [],
   "source": [
    "publications = dict()\n",
    "for n in roster.keys():\n",
    "    publications[n] = roster[n]['Publications']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1189,
   "metadata": {},
   "outputs": [],
   "source": [
    "for n in roster.keys():\n",
    "    roster[n]['Publications'] = publications[n]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Connection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1277,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = list(roster.keys())\n",
    "links = np.zeros((len(names), len(names)))\n",
    "for i in range(0, len(names)):\n",
    "    a = names[i]\n",
    "    for j in range(0, len(names)):\n",
    "        if i == j:\n",
    "            continue\n",
    "        link = 0\n",
    "        b = names[j]\n",
    "        if 'Group' in roster[b] and a in roster[b]['Group']:\n",
    "            link += 1\n",
    "        link += len(roster[a]['Publications'] & roster[b]['Publications'])\n",
    "        links[i, j] = np.max([link, links[i, j]])\n",
    "        links[j, i] = np.max([link, links[j, i]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Write Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1278,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_node = set()\n",
    "json_obj = defaultdict(list)\n",
    "for i in range(links.shape[0]):\n",
    "    for j in range(i + 1, links.shape[0]):\n",
    "        if links[i, j] == 0:\n",
    "            continue\n",
    "        if unify_title(roster[names[i]][\"Title\"]) in [\"IT\", \"Administration\", \"Coordinator\", \"Consultant\"]:\n",
    "            continue\n",
    "        if unify_title(roster[names[j]][\"Title\"]) in [\"IT\", \"Administration\", \"Coordinator\", \"Consultant\"]:\n",
    "            continue\n",
    "        data = {\n",
    "            \"source\": roster[names[i]]['Name'],\n",
    "            \"target\": roster[names[j]]['Name'],\n",
    "            \"value\": links[i, j],\n",
    "        }\n",
    "        is_node.add(roster[names[i]]['Name'])\n",
    "        is_node.add(roster[names[j]]['Name'])\n",
    "        json_obj[\"links\"].append(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1279,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, n in enumerate(names):\n",
    "    if np.sum(links[i, :]) + np.sum(links[:, i]) == 0:\n",
    "        continue\n",
    "    if unify_title(roster[n][\"Title\"]) in [\"IT\", \"Administration\", \"Coordinator\", \"Consultant\"]:\n",
    "        continue\n",
    "    if roster[n]['Name'] not in is_node:\n",
    "        continue\n",
    "    if \"Institution\" not in roster[n]:\n",
    "        ins = roster[n][\"Insitution\"]\n",
    "    else:\n",
    "        ins = roster[n][\"Institution\"]\n",
    "    data = {\n",
    "        \"id\": roster[n][\"Name\"],\n",
    "        \"title\": order_title(unify_title(roster[n][\"Title\"])),\n",
    "        \"institution\": ins\n",
    "    }\n",
    "    titles.add(unify_title(roster[n]['Title']))\n",
    "    json_obj[\"nodes\"].append(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1280,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./dataset/roster.json', 'w') as file:\n",
    "    json.dump(json_obj, file)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}