2.TCGA-process.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert downloaded TCGA datasets into sample × gene matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read sample information\n",
    "\n",
    "This file contains sample information. See the [online documentation](https://genome-cancer.soe.ucsc.edu/proj/site/xena/datapages/?dataset=TCGA.PANCAN.sampleMap/PANCAN_clinicalMatrix&host=https://tcga.xenahubs.net) for `PANCAN_clinicalMatrix`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12811, 40)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = os.path.join('download', 'PANCAN_clinicalMatrix.tsv.bz2')\n",
    "clinmat_df = (\n",
    "    pandas.read_table(path)\n",
    "    .rename(columns={'sampleID': 'sample_id'})\n",
    ")\n",
    "# Check that no sample_ids are duplicated\n",
    "assert not clinmat_df.sample_id.duplicated().any()\n",
    "clinmat_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Primary Tumor                                      10593\n",
       "Solid Tissue Normal                                 1475\n",
       "Metastatic                                           396\n",
       "Primary Blood Derived Cancer - Peripheral Blood      200\n",
       "Recurrent Tumor                                       56\n",
       "Additional - New Primary                              11\n",
       "Additional Metastatic                                  1\n",
       "Name: sample_type, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Types of samples\n",
    "clinmat_df.sample_type.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read mutation data\n",
    "\n",
    "This file contains mutation data (which mutations each sample contains) See the [online documentation](https://genome-cancer.soe.ucsc.edu/proj/site/xena/datapages/?dataset=TCGA.PANCAN.sampleMap/PANCAN_mutation&host=https://tcga.xenahubs.net) for `PANCAN_mutation`. Note that duplicate mutation rows, which [occur](https://groups.google.com/d/msg/ucsc-cancer-genomics-browser/eg6nJOFSefw/Z0BM6pU9BAAJ \"Message on the Xena Browser Google Group\") for samples that were sequenced multiple times, are filtered."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample_id</th>\n",
       "      <th>chr</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>reference</th>\n",
       "      <th>alt</th>\n",
       "      <th>gene</th>\n",
       "      <th>effect</th>\n",
       "      <th>DNA_VAF</th>\n",
       "      <th>RNA_VAF</th>\n",
       "      <th>Amino_Acid_Change</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TCGA-D8-A1J8-01</td>\n",
       "      <td>chr10</td>\n",
       "      <td>52587953</td>\n",
       "      <td>52587953</td>\n",
       "      <td>C</td>\n",
       "      <td>A</td>\n",
       "      <td>A1CF</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>p.R236I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TCGA-BH-A0HP-01</td>\n",
       "      <td>chr10</td>\n",
       "      <td>52595854</td>\n",
       "      <td>52595854</td>\n",
       "      <td>G</td>\n",
       "      <td>A</td>\n",
       "      <td>A1CF</td>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>p.A195V</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         sample_id    chr     start       end reference alt  gene  \\\n",
       "0  TCGA-D8-A1J8-01  chr10  52587953  52587953         C   A  A1CF   \n",
       "1  TCGA-BH-A0HP-01  chr10  52595854  52595854         G   A  A1CF   \n",
       "\n",
       "              effect  DNA_VAF  RNA_VAF Amino_Acid_Change  \n",
       "0  Missense_Mutation      NaN      NaN           p.R236I  \n",
       "1  Missense_Mutation      NaN      NaN           p.A195V  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = os.path.join('download', 'PANCAN_mutation.tsv.bz2')\n",
    "snp_mutation_df = (\n",
    "    pandas.read_table(path)\n",
    "    .rename(columns={'sample': 'sample_id'})\n",
    "    .drop_duplicates()\n",
    ")\n",
    "snp_mutation_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8510"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of samples with at least one mutation\n",
    "snp_mutation_df.sample_id.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>effect</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Missense_Mutation</td>\n",
       "      <td>1132319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Silent</td>\n",
       "      <td>474679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Nonsense_Mutation</td>\n",
       "      <td>87104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RNA</td>\n",
       "      <td>75134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Frame_Shift_Del</td>\n",
       "      <td>46991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Splice_Site</td>\n",
       "      <td>46477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Frame_Shift_Ins</td>\n",
       "      <td>21657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>In_Frame_Del</td>\n",
       "      <td>10663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Translation_Start_Site</td>\n",
       "      <td>3437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>In_Frame_Ins</td>\n",
       "      <td>2685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Nonstop_Mutation</td>\n",
       "      <td>1370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>3'UTR</td>\n",
       "      <td>211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Intron</td>\n",
       "      <td>149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>5'UTR</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>IGR</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>5'Flank</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     index   effect\n",
       "0        Missense_Mutation  1132319\n",
       "1                   Silent   474679\n",
       "2        Nonsense_Mutation    87104\n",
       "3                      RNA    75134\n",
       "4          Frame_Shift_Del    46991\n",
       "5              Splice_Site    46477\n",
       "6          Frame_Shift_Ins    21657\n",
       "7             In_Frame_Del    10663\n",
       "8   Translation_Start_Site     3437\n",
       "9             In_Frame_Ins     2685\n",
       "10        Nonstop_Mutation     1370\n",
       "11                   3'UTR      211\n",
       "12                  Intron      149\n",
       "13                   5'UTR       28\n",
       "14                     IGR       16\n",
       "15                 5'Flank        1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mutations counts by type\n",
    "snp_mutation_df.effect.value_counts().reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert SNP mutations to gene mutations\n",
    "\n",
    "The next cell specifies which mutations to preserve as gene-affecting, which were chosen according to the red & blue [mutation effects in Xena](http://xena.ucsc.edu/how-we-characterize-mutations/)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mutations = {\n",
    "    'Frame_Shift_Del',\n",
    "    'Frame_Shift_Ins',\n",
    "    'In_Frame_Del',\n",
    "    'In_Frame_Ins',\n",
    "    'Missense_Mutation',\n",
    "    'Nonsense_Mutation',\n",
    "    'Nonstop_Mutation',\n",
    "    'RNA',\n",
    "    'Splice_Site',\n",
    "    'Translation_Start_Site',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\"3'UTR\", \"5'Flank\", \"5'UTR\", 'IGR', 'Intron', 'Silent'}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mutations effects that were observed but nut included\n",
    "set(snp_mutation_df.effect.unique()) - mutations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample_id</th>\n",
       "      <th>chr</th>\n",
       "      <th>gene</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TCGA-02-0003-01</td>\n",
       "      <td>chr1</td>\n",
       "      <td>TPM3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TCGA-02-0003-01</td>\n",
       "      <td>chr10</td>\n",
       "      <td>TACC2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         sample_id    chr   gene  count\n",
       "0  TCGA-02-0003-01   chr1   TPM3      1\n",
       "1  TCGA-02-0003-01  chr10  TACC2      1"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_mutation_df = (snp_mutation_df\n",
    "    .query(\"effect in @mutations\")\n",
    "    .groupby(['sample_id', 'chr', 'gene'])\n",
    "    .apply(len)\n",
    "    .reset_index()\n",
    "    .rename(columns={0: 'count'})\n",
    ")\n",
    "\n",
    "gene_mutation_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, map combination of chromosome/gene symbol to Entrez ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entrez_id</th>\n",
       "      <th>chr</th>\n",
       "      <th>symbol</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>chr19</td>\n",
       "      <td>A1BG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>chr12</td>\n",
       "      <td>A2M</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   entrez_id    chr symbol\n",
       "0          1  chr19   A1BG\n",
       "1          2  chr12    A2M"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Retrieve chr/gene symbol to entrez_id mapping\n",
    "path = os.path.join('mapping', 'PANCAN-mutation', 'PANCAN-mutation-gene-map.tsv')\n",
    "mutation_map_df = pandas.read_table(path)\n",
    "mutation_map_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample_id</th>\n",
       "      <th>chr</th>\n",
       "      <th>gene</th>\n",
       "      <th>count</th>\n",
       "      <th>entrez_id</th>\n",
       "      <th>symbol</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TCGA-02-0003-01</td>\n",
       "      <td>chr1</td>\n",
       "      <td>TPM3</td>\n",
       "      <td>1</td>\n",
       "      <td>7170</td>\n",
       "      <td>TPM3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TCGA-17-Z015-01</td>\n",
       "      <td>chr1</td>\n",
       "      <td>TPM3</td>\n",
       "      <td>1</td>\n",
       "      <td>7170</td>\n",
       "      <td>TPM3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         sample_id   chr  gene  count  entrez_id symbol\n",
       "0  TCGA-02-0003-01  chr1  TPM3      1       7170   TPM3\n",
       "1  TCGA-17-Z015-01  chr1  TPM3      1       7170   TPM3"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# merge with mapping df to yield column with entrez_id\n",
    "# inner join will drop mutations that are not mapped\n",
    "gene_mutation_df = pandas.merge(gene_mutation_df, mutation_map_df, left_on = ['chr', 'gene'], right_on = ['chr', 'symbol'], how='inner')\n",
    "\n",
    "gene_mutation_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8507, 21940)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a sample (rows) by gene (columns) matrix of mutation status\n",
    "\n",
    "gene_mutation_mat_df = (gene_mutation_df\n",
    "    .pivot_table(index='sample_id', columns='entrez_id', values='count', fill_value=0)\n",
    "    .astype(bool).astype(int)\n",
    ")\n",
    "gene_mutation_mat_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.67% sample-gene pairs are mutated'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'{:.2%} sample-gene pairs are mutated'.format(\n",
    "    gene_mutation_mat_df.stack().mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>gene</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TP53</td>\n",
       "      <td>2992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TTN</td>\n",
       "      <td>2465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MUC16</td>\n",
       "      <td>1518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PIK3CA</td>\n",
       "      <td>1024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CSMD3</td>\n",
       "      <td>989</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    index  gene\n",
       "0    TP53  2992\n",
       "1     TTN  2465\n",
       "2   MUC16  1518\n",
       "3  PIK3CA  1024\n",
       "4   CSMD3   989"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Top mutated genes\n",
    "gene_mutation_df.gene.value_counts().reset_index().head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>sample_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TCGA-IB-7651-01</td>\n",
       "      <td>8364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TCGA-FW-A3R5-06</td>\n",
       "      <td>7661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TCGA-AP-A0LM-01</td>\n",
       "      <td>7030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TCGA-AP-A059-01</td>\n",
       "      <td>6479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TCGA-B5-A0JY-01</td>\n",
       "      <td>6151</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             index  sample_id\n",
       "0  TCGA-IB-7651-01       8364\n",
       "1  TCGA-FW-A3R5-06       7661\n",
       "2  TCGA-AP-A0LM-01       7030\n",
       "3  TCGA-AP-A059-01       6479\n",
       "4  TCGA-B5-A0JY-01       6151"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Top mutated samples\n",
    "gene_mutation_df.sample_id.value_counts().reset_index().head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read gene expression data\n",
    "\n",
    "This file contains gene expression data from RNA-Sequencing. See the [online documentation](https://genome-cancer.soe.ucsc.edu/proj/site/xena/datapages/?dataset=TCGA.PANCAN.sampleMap/HiSeqV2&host=https://tcga.xenahubs.net) for `HiSeqV2`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Read the gene × sample dataset\n",
    "path = os.path.join('download', 'HiSeqV2.tsv.bz2')\n",
    "expr_df = pandas.read_table(path, index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Retrieve symbol to gene mapping for HiSeqV2\n",
    "path = os.path.join('mapping', 'HiSeqV2-genes', 'HiSeqV2-gene-map.tsv')\n",
    "gene_map_df = pandas.read_table(path)\n",
    "symbol_to_entrez = dict(zip(gene_map_df.symbol, gene_map_df.entrez_gene_id))\n",
    "\n",
    "# Check that there aren't any unmapped symbols\n",
    "unmapped_symbols = set(expr_df.index) - set(symbol_to_entrez)\n",
    "unmapped_symbols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10459, 20530)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Process the dataset\n",
    "expr_df = (expr_df\n",
    "    # Convert gene symbols to entrez gene ids\n",
    "    .rename(index=symbol_to_entrez)\n",
    "    # Transpose so the data is sample × gene\n",
    "    .transpose()\n",
    "    # Sort rows and columns\n",
    "    .sort_index(axis='rows')\n",
    "    .sort_index(axis='columns')\n",
    ")\n",
    "\n",
    "expr_df.index.rename('sample_id', inplace=True)\n",
    "\n",
    "expr_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Sample</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>12</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TCGA-02-0047-01</th>\n",
       "      <td>6.9774</td>\n",
       "      <td>15.0538</td>\n",
       "      <td>5.3559</td>\n",
       "      <td>3.5171</td>\n",
       "      <td>14.9601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-0055-01</th>\n",
       "      <td>8.6177</td>\n",
       "      <td>15.3879</td>\n",
       "      <td>6.5382</td>\n",
       "      <td>1.0849</td>\n",
       "      <td>14.5426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-2483-01</th>\n",
       "      <td>8.0920</td>\n",
       "      <td>14.3622</td>\n",
       "      <td>5.1023</td>\n",
       "      <td>2.4967</td>\n",
       "      <td>13.3549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-2485-01</th>\n",
       "      <td>6.4084</td>\n",
       "      <td>12.9292</td>\n",
       "      <td>5.9968</td>\n",
       "      <td>4.3923</td>\n",
       "      <td>14.8337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-2486-01</th>\n",
       "      <td>6.7716</td>\n",
       "      <td>15.3224</td>\n",
       "      <td>6.3377</td>\n",
       "      <td>2.2199</td>\n",
       "      <td>16.7832</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Sample               1        2       9       10       12\n",
       "sample_id                                                \n",
       "TCGA-02-0047-01  6.9774  15.0538  5.3559  3.5171  14.9601\n",
       "TCGA-02-0055-01  8.6177  15.3879  6.5382  1.0849  14.5426\n",
       "TCGA-02-2483-01  8.0920  14.3622  5.1023  2.4967  13.3549\n",
       "TCGA-02-2485-01  6.4084  12.9292  5.9968  4.3923  14.8337\n",
       "TCGA-02-2486-01  6.7716  15.3224  6.3377  2.2199  16.7832"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Peak at the data matrix\n",
    "expr_df.iloc[:5, :5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Integrate expression and mutation data\n",
    "\n",
    "Find samples with both mutation and expression data. We assume that if a sample was not in `PANCAN_mutation`, it was not assayed for mutation. Hence, zero-mutation cancers are excluded even if they have mutation data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7705"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_ids = list(gene_mutation_mat_df.index & expr_df.index)\n",
    "len(sample_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Filter expression (x) and mutation (y) matrices for common samples\n",
    "x_df = expr_df.loc[sample_ids, :]\n",
    "y_df = gene_mutation_mat_df.loc[sample_ids, :]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Export matrices to TSVs\n",
    "\n",
    "Matrices are saved as sample × gene TSVs. Subsetted matrices are also exported to allow users to quickly explore small portions of the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def sample_df(df, nrows=None, ncols=None, row_seed=0, col_seed=0):\n",
    "    \"\"\"Randomly subset a dataframe, preserving row and column order.\"\"\"\n",
    "    if nrows is None:\n",
    "        nrows = len(df)\n",
    "    if ncols is None:\n",
    "        ncols = len(df.columns)\n",
    "    return (df\n",
    "        .sample(n=nrows, random_state=row_seed, axis='rows')\n",
    "        .sample(n=ncols, random_state=col_seed, axis='columns')\n",
    "        .sort_index(axis='rows')\n",
    "        .sort_index(axis='columns')\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tsv_args = {'sep': '\\t', 'float_format': '%.3g'}\n",
    "\n",
    "for df, name in (x_df, 'expression-matrix'), (y_df, 'mutation-matrix'):\n",
    "\n",
    "    # Save full dataset\n",
    "    path = os.path.join('data', name + '.tsv.bz2')\n",
    "    df.to_csv(path, **tsv_args, compression='bz2')\n",
    "    \n",
    "    # Save subsetted datasets\n",
    "    for sample, nrows, ncols in ('small', 50, 15), ('all-samples', None, 15), ('all-genes', 50, None):\n",
    "        path = os.path.join('data', 'subset', '{}-{}.tsv'.format(name, sample))\n",
    "        sample_df(df, nrows=nrows, ncols=ncols).to_csv(path, **tsv_args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}