mygene, gene wiki url check.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gene Wiki wikipedia URLs in MyGene.info coverage checker\n",
    "\n",
    "After retrieving Wikipedia urls for AD genes for the Gene Wiki Review series and manually inspecting the AD genes without Wikipedia urls available in MyGene.info, it became apparent that the Wikipedia url annotation data in MyGene.info may not be up-to-date, this is to check what is missing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import mygene\n",
    "import pandas\n",
    "import json\n",
    "import urllib.request\n",
    "import requests\n",
    "from collections import OrderedDict\n",
    "from pandas import read_csv\n",
    "\n",
    "mg = mygene.MyGeneInfo()\n",
    "\n",
    "infilepath = 'data/'\n",
    "exppath = 'results/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "###############################################################################\n",
    "## This module uses mygene.info python wrapper to obtain gene wikipedia \n",
    "## titles from entrez ids.\n",
    "###############################################################################\n",
    "\n",
    "def check_wiki_titles_for_geneids (genelist):\n",
    "    print('obtaining wikipedia page titles for gene ids')\n",
    "    entrezlist = mg.querymany(genelist, scopes='entrezgene', fields=('wikipedia.url_stub','symbol'), as_dataframe=True)\n",
    "    ## Split the returned dataframe into two lists (one with a result, one without)\n",
    "    entrezlist['wikipedia'] = entrezlist['wikipedia'].fillna('None')\n",
    "    no_response = entrezlist.loc[entrezlist['wikipedia']=='None']\n",
    "    url_found = entrezlist.loc[entrezlist['wikipedia']!='None']\n",
    "    ## For gene ids that returned a Wikipedia stub, clean up result for querying Wikipedia for Wikipedia information\n",
    "    url_found['wikipedia'] = url_found['wikipedia'].apply(lambda x: x['url_stub'])\n",
    "    return(url_found,no_response)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run the sparql query to retrieve all human genes with English Wikipedia URLs. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "query completed\n"
     ]
    }
   ],
   "source": [
    "\n",
    "url = 'https://query.wikidata.org/sparql'\n",
    "query = \"\"\"\n",
    "SELECT \n",
    "  ?gene ?geneid ?protein ?article\n",
    "WHERE {\n",
    "  ?gene wdt:P31 wd:Q7187;  #Find genes\n",
    "        wdt:P703 wd:Q15978631. #limit to humans\n",
    "  ?article schema:about ?gene. #limit to genes with corresponding English Wikipedia articles\n",
    "  ?article schema:inLanguage \"en\".\n",
    "  ?article schema:isPartOf <https://en.wikipedia.org/>\n",
    "  OPTIONAL { ?gene wdt:P688 ?protein } #Get associated proteins\n",
    "  OPTIONAL { ?gene wdt:P351 ?geneid } #Get the geneid\n",
    "  }\n",
    "\"\"\"\n",
    "r = requests.get(url, params = {'format': 'json', 'query': query})\n",
    "data = r.json()\n",
    "print(\"query completed\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clean up the results of the SPARQL query. The SPARQL query will also pull any proteins that are encoded by the genes. Since the mapping of gene to protein isn't always 1:1, there maybe duplicate gene entries corresponding to different proteins or single gene entries corresponding to the same protein."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16223\n"
     ]
    }
   ],
   "source": [
    "genes = []\n",
    "for item in data['results']['bindings']:\n",
    "    genes.append(OrderedDict({\n",
    "        'gene_uri': item['gene']['value'],\n",
    "        'gene_id': item['geneid']['value'] \n",
    "            if 'geneid' in item else None,\n",
    "        'protein_uri': item['protein']['value'] \n",
    "            if 'protein' in item else None,\n",
    "        'wiki_uri': item['article']['value'] \n",
    "            if 'article' in item else None}))\n",
    "\n",
    "wikidata_genes_uri = pandas.DataFrame(genes)\n",
    "wikidata_genes_uri['genes_wdid'] = wikidata_genes_uri['gene_uri'].astype(str).str.replace(\"http://www.wikidata.org/entity/\",\"\")\n",
    "wikidata_genes_uri['protein_wdid'] = wikidata_genes_uri['protein_uri'].astype(str).str.replace(\"http://www.wikidata.org/entity/\",\"\")\n",
    "wikidata_genes_uri['wiki_stub'] = wikidata_genes_uri['wiki_uri'].astype(str).str.replace(\"https://en.wikipedia.org/wiki/\",\"\")\n",
    "wikidata_genes_uri.head()\n",
    "print(len(wikidata_genes_uri))\n",
    "\n",
    "#wikidata_genes_uri.to_csv(exppath+'wikidata_genes_uri.tsv',sep='\\t',header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pull all the unique gene ids for human genes with Wikipedia pages."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   gene_id   ids\n",
      "0     2312  2312\n",
      "1     5925  5925\n"
     ]
    }
   ],
   "source": [
    "genedf = wikidata_genes_uri[['gene_id']].fillna(-1).astype(int)\n",
    "genedf.drop_duplicates('gene_id', keep='first', inplace=True)\n",
    "genedf['ids'] = genedf['gene_id'].astype(str)\n",
    "\n",
    "genelist = genedf['ids'].tolist()\n",
    "#print(wikidata_genes_uri.head(n=2))\n",
    "print(genedf[0:2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use MyGene.info to find the Wikipedia urls for those genes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "obtaining wikipedia page titles for gene ids\n",
      "querying 1-1000...done.\n",
      "querying 1001-2000...done.\n",
      "querying 2001-3000...done.\n",
      "querying 3001-4000...done.\n",
      "querying 4001-5000...done.\n",
      "querying 5001-6000...done.\n",
      "querying 6001-7000...done.\n",
      "querying 7001-8000...done.\n",
      "querying 8001-9000...done.\n",
      "querying 9001-10000...done.\n",
      "querying 10001-11000...done.\n",
      "querying 11001-12000...done.\n",
      "querying 12001-12430...done.\n",
      "Finished.\n",
      "11 input query terms found no hit:\n",
      "\t['-1', '653365', '6025', '9193', '11217', '353293', '117153', '544437', '387281', '8142', '107984557\n",
      "Pass \"returnall=True\" to return complete lists of duplicate or missing query terms.\n",
      "Length of original gene list:  12430 length of list with urls in mygene:  10855\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ginger\\Anaconda3\\envs\\gw_reviews\\lib\\site-packages\\ipykernel\\__main__.py:14: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "url_found,no_response = check_wiki_titles_for_geneids(genelist)\n",
    "print('Length of original gene list: ',len(genelist), \n",
    "      'length of list with urls in mygene: ',len(url_found))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Upon manual inspection, the 10 gene ids in the not found list appear to be genes which have been discontinued or otherwise withdrawn and potentially replaced. In other words, these may be genes which should be marked in Wikidata as deprecated in some manner.\n",
    "\n",
    "Comparing hte list of unique genes in the original list (12429) and the list returned from mygene (10855), it looks like there are about 1574 genes which are in Wikidata that have associated Wikipedia URLs, which are not found in MyGene.info.\n",
    "\n",
    "To identify the the genes with Wikipedia URLS according to Wikidata, but no URLS according to MyGene, do a merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  gene_id                                gene_uri  \\\n",
      "0    2312  http://www.wikidata.org/entity/Q410688   \n",
      "1    5925   http://www.wikidata.org/entity/Q40108   \n",
      "\n",
      "                                protein_uri  \\\n",
      "0  http://www.wikidata.org/entity/Q21201832   \n",
      "1  http://www.wikidata.org/entity/Q21111463   \n",
      "\n",
      "                                            wiki_uri genes_wdid protein_wdid  \\\n",
      "0            https://en.wikipedia.org/wiki/Filaggrin    Q410688    Q21201832   \n",
      "1  https://en.wikipedia.org/wiki/Retinoblastoma_p...     Q40108    Q21111463   \n",
      "\n",
      "                wiki_stub  ids query  _score notfound symbol wikipedia  \n",
      "0               Filaggrin  NaN   NaN     NaN      NaN    NaN       NaN  \n",
      "1  Retinoblastoma_protein  NaN   NaN     NaN      NaN    NaN       NaN  \n"
     ]
    }
   ],
   "source": [
    "mygene_results = url_found.reset_index()\n",
    "mygene_to_merge = mygene_results.rename(columns={'_id':'ids'})\n",
    "mygene_to_merge['ids'] = mygene_to_merge['ids'].astype(str)\n",
    "\n",
    "gene_compare = wikidata_genes_uri.merge(genedf.merge(mygene_to_merge, on='ids',how='left'), on='gene_id', how='left')\n",
    "print(gene_compare.head(n=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filter for genes where the MyGene.info failed to retrieve a url."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16223\n",
      "  gene_id                                gene_uri  \\\n",
      "0    2312  http://www.wikidata.org/entity/Q410688   \n",
      "1    5925   http://www.wikidata.org/entity/Q40108   \n",
      "\n",
      "                                protein_uri  \\\n",
      "0  http://www.wikidata.org/entity/Q21201832   \n",
      "1  http://www.wikidata.org/entity/Q21111463   \n",
      "\n",
      "                                            wiki_uri genes_wdid protein_wdid  \\\n",
      "0            https://en.wikipedia.org/wiki/Filaggrin    Q410688    Q21201832   \n",
      "1  https://en.wikipedia.org/wiki/Retinoblastoma_p...     Q40108    Q21111463   \n",
      "\n",
      "                wiki_stub  ids query  _score notfound symbol wikipedia  \n",
      "0               Filaggrin  NaN   NaN     NaN      NaN    NaN       NaN  \n",
      "1  Retinoblastoma_protein  NaN   NaN     NaN      NaN    NaN       NaN  \n"
     ]
    }
   ],
   "source": [
    "missing_from_mygene = gene_compare.loc[gene_compare['wikipedia'].isnull()]\n",
    "print(len(missing_from_mygene))\n",
    "print(missing_from_mygene.head(n=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}