perturbagen_and_related_signatures_metadata_processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Author:** Benoît BAILLIF\n",
    "\n",
    "**Purpose:**\n",
    "- Merge GSE92742 and GSE70138 perturbagen and signature info  \n",
    "- Select only compounds (not genetic perturbagens)  \n",
    "- Process data (canonize SMILES RDKIT, harmonize dose) \n",
    "\n",
    "**Input:**\n",
    "- data/raw/\n",
    " - GSE92742_Broad_LINCS_pert_info.txt\n",
    " - GSE92742_Broad_LINCS_sig_info.txt\n",
    " - GSE70138_Broad_LINCS_pert_info.txt\n",
    " - GSE70138_Broad_LINCS_sig_info_2017-03-06.txt\n",
    " \n",
    "**Output:** \n",
    "- data/processed/\n",
    " - cmp_info_cmap.csv\n",
    " - sig_info_cmap_cmp.csv in processed data directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy  as np\n",
    "import pandas as pd\n",
    "from rdkit import Chem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dose_to_micromolar(dose_string) :\n",
    "    \"\"\"Convert a dose (string) in micromolar if needed\n",
    "    Args:\n",
    "        dose_string (str) : dose expressed as a string, containing generally a value and a unit\n",
    "        \n",
    "    Returns:\n",
    "        float : converted dose\n",
    "    \"\"\"\n",
    "    value_unit = dose_string.split()\n",
    "    value = float(value_unit[0])\n",
    "    unit = value_unit[1]\n",
    "        \n",
    "    if (unit == 'nM') :\n",
    "        value = value / 1000\n",
    "            \n",
    "    return value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_data_directory = 'data/raw/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "GSE92742_pert_info_file_name = 'GSE92742_Broad_LINCS_pert_info.txt'\n",
    "GSE92742_pert_info_path = raw_data_directory + GSE92742_pert_info_file_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "GSE92742_sig_info_file_name = 'GSE92742_Broad_LINCS_sig_info.txt'\n",
    "GSE92742_sig_info_path = raw_data_directory + GSE92742_sig_info_file_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "GSE70138_pert_info_file_name = 'GSE70138_Broad_LINCS_pert_info.txt'\n",
    "GSE70138_pert_info_path = raw_data_directory + GSE70138_pert_info_file_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "GSE70138_sig_info_file_name = 'GSE70138_Broad_LINCS_sig_info_2017-03-06.txt'\n",
    "GSE70138_sig_info_path = raw_data_directory + GSE70138_sig_info_file_name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_data_directory = 'data/processed/'\n",
    "if not os.path.exists(processed_data_directory) :\n",
    "    os.mkdir(processed_data_directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmp_info_cmap_path = processed_data_directory + 'cmp_info_cmap.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sig_info_cmap_path = processed_data_directory + 'sig_info_cmap.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load GSE92742 perturbagen info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "pert_info_GSE92742 = pd.read_csv(GSE92742_pert_info_path, sep='\\t', dtype='str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of perturbagens in GSE92742 : 51383 (controls included)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pert_id</th>\n",
       "      <th>pert_iname</th>\n",
       "      <th>pert_type</th>\n",
       "      <th>is_touchstone</th>\n",
       "      <th>inchi_key_prefix</th>\n",
       "      <th>inchi_key</th>\n",
       "      <th>canonical_smiles</th>\n",
       "      <th>pubchem_cid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>56582</td>\n",
       "      <td>AKT2</td>\n",
       "      <td>trt_oe</td>\n",
       "      <td>0</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5981</td>\n",
       "      <td>HSF1</td>\n",
       "      <td>trt_oe</td>\n",
       "      <td>0</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7150</td>\n",
       "      <td>NFE2L2</td>\n",
       "      <td>trt_oe</td>\n",
       "      <td>0</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ABL1_G2A</td>\n",
       "      <td>ABL1</td>\n",
       "      <td>trt_oe.mut</td>\n",
       "      <td>0</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ABL1_T315I</td>\n",
       "      <td>ABL1</td>\n",
       "      <td>trt_oe.mut</td>\n",
       "      <td>0</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      pert_id pert_iname   pert_type is_touchstone inchi_key_prefix inchi_key  \\\n",
       "0       56582       AKT2      trt_oe             0             -666      -666   \n",
       "1        5981       HSF1      trt_oe             0             -666      -666   \n",
       "2        7150     NFE2L2      trt_oe             0             -666      -666   \n",
       "3    ABL1_G2A       ABL1  trt_oe.mut             0             -666      -666   \n",
       "4  ABL1_T315I       ABL1  trt_oe.mut             0             -666      -666   \n",
       "\n",
       "  canonical_smiles pubchem_cid  \n",
       "0             -666        -666  \n",
       "1             -666        -666  \n",
       "2             -666        -666  \n",
       "3             -666        -666  \n",
       "4             -666        -666  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_pert_GSE92742 = pert_info_GSE92742.shape[0]\n",
    "print('Number of perturbagens in GSE92742 : ' + str(n_pert_GSE92742) + \n",
    "      ' (controls included)')\n",
    "\n",
    "pert_info_GSE92742.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop inchi_key_prefix because is contained in inchi_key\n",
    "pert_info_GSE92742 = pert_info_GSE92742.drop('inchi_key_prefix', axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load GSE70138 Perturbagen info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "pert_info_GSE70138 = pd.read_csv(GSE70138_pert_info_path, sep='\\t', dtype='str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of perturbagens in GSE70138 : 2170 (controls excluded)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pert_id</th>\n",
       "      <th>canonical_smiles</th>\n",
       "      <th>inchi_key</th>\n",
       "      <th>pert_iname</th>\n",
       "      <th>pert_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BRD-K70792160</td>\n",
       "      <td>CCN(CC)CCCCN1c2ccccc2Oc2ccc(Cl)cc12</td>\n",
       "      <td>GYBXAGDWMCJZJK-UHFFFAOYSA-N</td>\n",
       "      <td>10-DEBC</td>\n",
       "      <td>trt_cp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BRD-K68552125</td>\n",
       "      <td>CCCCCCCCCCCCCC(=O)O[C@@H]1[C@@H](C)[C@]2(O)[C@...</td>\n",
       "      <td>PHEDXBVPIONUQT-RGYGYFBISA-N</td>\n",
       "      <td>phorbol-myristate-acetate</td>\n",
       "      <td>trt_cp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BRD-K92301463</td>\n",
       "      <td>CCCCC(C)(C)[C@H](O)\\C=C\\[C@H]1[C@H](O)CC(=O)[C...</td>\n",
       "      <td>QAOBBBBDJSWHMU-WMBBNPMCSA-N</td>\n",
       "      <td>16,16-dimethylprostaglandin-e2</td>\n",
       "      <td>trt_cp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BRD-A29731977</td>\n",
       "      <td>CCCCCC(=O)O[C@@]1(CCC2C3CCC4=CC(=O)CC[C@]4(C)C...</td>\n",
       "      <td>DOMWKUIIPQCAJU-JKPPDDDBSA-N</td>\n",
       "      <td>17-hydroxyprogesterone-caproate</td>\n",
       "      <td>trt_cp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>BRD-K07954936</td>\n",
       "      <td>OC(=O)CCCC[C@@H]1SC[C@@H]2NC(=N)N[C@H]12</td>\n",
       "      <td>WWVANQJRLPIHNS-ZKWXMUAHSA-N</td>\n",
       "      <td>2-iminobiotin</td>\n",
       "      <td>trt_cp</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         pert_id                                   canonical_smiles  \\\n",
       "0  BRD-K70792160                CCN(CC)CCCCN1c2ccccc2Oc2ccc(Cl)cc12   \n",
       "1  BRD-K68552125  CCCCCCCCCCCCCC(=O)O[C@@H]1[C@@H](C)[C@]2(O)[C@...   \n",
       "2  BRD-K92301463  CCCCC(C)(C)[C@H](O)\\C=C\\[C@H]1[C@H](O)CC(=O)[C...   \n",
       "3  BRD-A29731977  CCCCCC(=O)O[C@@]1(CCC2C3CCC4=CC(=O)CC[C@]4(C)C...   \n",
       "4  BRD-K07954936           OC(=O)CCCC[C@@H]1SC[C@@H]2NC(=N)N[C@H]12   \n",
       "\n",
       "                     inchi_key                       pert_iname pert_type  \n",
       "0  GYBXAGDWMCJZJK-UHFFFAOYSA-N                          10-DEBC    trt_cp  \n",
       "1  PHEDXBVPIONUQT-RGYGYFBISA-N        phorbol-myristate-acetate    trt_cp  \n",
       "2  QAOBBBBDJSWHMU-WMBBNPMCSA-N   16,16-dimethylprostaglandin-e2    trt_cp  \n",
       "3  DOMWKUIIPQCAJU-JKPPDDDBSA-N  17-hydroxyprogesterone-caproate    trt_cp  \n",
       "4  WWVANQJRLPIHNS-ZKWXMUAHSA-N                    2-iminobiotin    trt_cp  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_pert_GSE70138 = pert_info_GSE70138.shape[0]\n",
    "print('Number of perturbagens in GSE70138 : ' \n",
    "      + str(n_pert_GSE70138) + ' (controls excluded)')\n",
    "\n",
    "pert_info_GSE70138.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compile the 2 GEO perturbagen info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "pert_info_cmap = pd.concat([pert_info_GSE92742, pert_info_GSE70138])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop duplicate perturbagen, based on pert_id. The first occurence is conserved, the GSE92742 one\n",
    "# containing the pubchem_cid information\n",
    "pert_info_cmap = pert_info_cmap.drop_duplicates(subset='pert_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of perturbagens in compiled GEO datasets : 52641\n"
     ]
    }
   ],
   "source": [
    "n_pert_cmap = pert_info_cmap.shape[0]\n",
    "print('Total number of perturbagens in compiled GEO datasets : ' + str(n_pert_cmap))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pert_id</th>\n",
       "      <th>pert_iname</th>\n",
       "      <th>pert_type</th>\n",
       "      <th>is_touchstone</th>\n",
       "      <th>inchi_key</th>\n",
       "      <th>canonical_smiles</th>\n",
       "      <th>pubchem_cid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>56582</td>\n",
       "      <td>AKT2</td>\n",
       "      <td>trt_oe</td>\n",
       "      <td>0</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5981</td>\n",
       "      <td>HSF1</td>\n",
       "      <td>trt_oe</td>\n",
       "      <td>0</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7150</td>\n",
       "      <td>NFE2L2</td>\n",
       "      <td>trt_oe</td>\n",
       "      <td>0</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ABL1_G2A</td>\n",
       "      <td>ABL1</td>\n",
       "      <td>trt_oe.mut</td>\n",
       "      <td>0</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ABL1_T315I</td>\n",
       "      <td>ABL1</td>\n",
       "      <td>trt_oe.mut</td>\n",
       "      <td>0</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "      <td>-666</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      pert_id pert_iname   pert_type is_touchstone inchi_key canonical_smiles  \\\n",
       "0       56582       AKT2      trt_oe             0      -666             -666   \n",
       "1        5981       HSF1      trt_oe             0      -666             -666   \n",
       "2        7150     NFE2L2      trt_oe             0      -666             -666   \n",
       "3    ABL1_G2A       ABL1  trt_oe.mut             0      -666             -666   \n",
       "4  ABL1_T315I       ABL1  trt_oe.mut             0      -666             -666   \n",
       "\n",
       "  pubchem_cid  \n",
       "0        -666  \n",
       "1        -666  \n",
       "2        -666  \n",
       "3        -666  \n",
       "4        -666  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pert_info_cmap.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "trt_cp             21299\n",
       "trt_sh             18493\n",
       "trt_sh.cgs          4345\n",
       "trt_sh.css          3807\n",
       "trt_oe              3492\n",
       "trt_lig              622\n",
       "trt_xpr              353\n",
       "trt_oe.mut           135\n",
       "ctl_vector            80\n",
       "ctl_vector.cns         8\n",
       "ctl_vehicle            3\n",
       "ctl_vehicle.cns        2\n",
       "ctl_untrt              1\n",
       "ctl_untrt.cns          1\n",
       "Name: pert_type, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pert_info_cmap.pert_type.value_counts(dropna=False)\n",
    "# More info on : https://clue.io/connectopedia/perturbagen_types_and_controls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      29806\n",
       "1      21577\n",
       "NaN     1258\n",
       "Name: is_touchstone, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pert_info_cmap['is_touchstone'].value_counts(dropna=False)\n",
    "# More info on : https://clue.io/connectopedia/the_touchstone_dataset\n",
    "# NaN values corresponds to GSE70138, as they did not present this distinction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Select compounds only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We select compound with the pert_type, but note that DMSO vehicule are under the category ctl_vehicule\n",
    "cmp_info_cmap = pert_info_cmap[pert_info_cmap['pert_type'] == 'trt_cp'].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of compounds in CMap : 21299\n",
      "+ 1 (DMSO vehicule) = 21300\n"
     ]
    }
   ],
   "source": [
    "n_cmp_cmap = cmp_info_cmap.shape[0]\n",
    "print('Number of compounds in CMap : ' + str(n_cmp_cmap))\n",
    "print('+ 1 (DMSO vehicule) = ' + str(n_cmp_cmap + 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pert_type is trt_cp for all, so this column is not relevant anymore\n",
    "cmp_info_cmap = cmp_info_cmap.drop('pert_type', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "incorrect_smiles = cmp_info_cmap['canonical_smiles'].isin(['-666', 'restricted'])\n",
    "cmp_info_cmap = cmp_info_cmap[~incorrect_smiles]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make RDKIT canonized SMILES\n",
    "cmp_info_cmap['canonical_smiles'] = cmp_info_cmap['canonical_smiles'].apply(Chem.MolFromSmiles)\n",
    "cmp_info_cmap['canonical_smiles'] = cmp_info_cmap['canonical_smiles'].apply(Chem.MolToSmiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pert_id</th>\n",
       "      <th>pert_iname</th>\n",
       "      <th>is_touchstone</th>\n",
       "      <th>inchi_key</th>\n",
       "      <th>canonical_smiles</th>\n",
       "      <th>pubchem_cid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>BRD-A00100033</td>\n",
       "      <td>nifurtimox</td>\n",
       "      <td>1</td>\n",
       "      <td>ARFHIAQFJWUCFH-UHFFFAOYSA-N</td>\n",
       "      <td>CC1CS(=O)(=O)CCN1N=Cc1ccc([N+](=O)[O-])o1</td>\n",
       "      <td>6842999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>BRD-A00150179</td>\n",
       "      <td>5-hydroxytryptophan</td>\n",
       "      <td>0</td>\n",
       "      <td>QSHLMQDRPXXYEE-UHFFFAOYSA-N</td>\n",
       "      <td>NC(Cc1c[nH]c2cccc(O)c12)C(=O)O</td>\n",
       "      <td>589768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>BRD-A00267231</td>\n",
       "      <td>hemado</td>\n",
       "      <td>1</td>\n",
       "      <td>KOCIMZNSNPOGOP-UHFFFAOYSA-N</td>\n",
       "      <td>CCCCC#Cc1nc(NC)c2ncn(C3OC(CO)C(O)C3O)c2n1</td>\n",
       "      <td>4043357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>BRD-A00420644</td>\n",
       "      <td>SA-3676</td>\n",
       "      <td>0</td>\n",
       "      <td>ASCBUEVCEVGOFP-UHFFFAOYSA-N</td>\n",
       "      <td>CCN1c2ccccc2NC2N=C(OC)C(c3ccccc3)C21</td>\n",
       "      <td>2853908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>BRD-A00474148</td>\n",
       "      <td>BRD-A00474148</td>\n",
       "      <td>0</td>\n",
       "      <td>RCGAUPRLRFZAMS-UHFFFAOYSA-N</td>\n",
       "      <td>O=C1Cc2cc([S+](=O)([O-])N3CCN(c4ccc(O)cc4)CC3)...</td>\n",
       "      <td>44825297</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           pert_id           pert_iname is_touchstone  \\\n",
       "98   BRD-A00100033           nifurtimox             1   \n",
       "99   BRD-A00150179  5-hydroxytryptophan             0   \n",
       "100  BRD-A00267231               hemado             1   \n",
       "101  BRD-A00420644              SA-3676             0   \n",
       "102  BRD-A00474148        BRD-A00474148             0   \n",
       "\n",
       "                       inchi_key  \\\n",
       "98   ARFHIAQFJWUCFH-UHFFFAOYSA-N   \n",
       "99   QSHLMQDRPXXYEE-UHFFFAOYSA-N   \n",
       "100  KOCIMZNSNPOGOP-UHFFFAOYSA-N   \n",
       "101  ASCBUEVCEVGOFP-UHFFFAOYSA-N   \n",
       "102  RCGAUPRLRFZAMS-UHFFFAOYSA-N   \n",
       "\n",
       "                                      canonical_smiles pubchem_cid  \n",
       "98           CC1CS(=O)(=O)CCN1N=Cc1ccc([N+](=O)[O-])o1     6842999  \n",
       "99                      NC(Cc1c[nH]c2cccc(O)c12)C(=O)O      589768  \n",
       "100          CCCCC#Cc1nc(NC)c2ncn(C3OC(CO)C(O)C3O)c2n1     4043357  \n",
       "101               CCN1c2ccccc2NC2N=C(OC)C(c3ccccc3)C21     2853908  \n",
       "102  O=C1Cc2cc([S+](=O)([O-])N3CCN(c4ccc(O)cc4)CC3)...    44825297  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cmp_info_cmap.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of compounds in CMap with correct structure: 21220\n",
      "So we have 79 incorrect compounds\n"
     ]
    }
   ],
   "source": [
    "n_correct_cmp_cmap = cmp_info_cmap.shape[0]\n",
    "print('Number of compounds in CMap with correct structure: ' + str(n_correct_cmp_cmap))\n",
    "print('So we have ' + str(n_cmp_cmap - n_correct_cmp_cmap) + ' incorrect compounds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmap_correct_cmp_list = cmp_info_cmap['pert_id'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load GSE92742 signature info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "sig_info_GSE92742 = pd.read_csv(GSE92742_sig_info_path, sep='\\t', dtype='str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of signatures in GSE92742 : 473647 (controls included)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sig_id</th>\n",
       "      <th>pert_id</th>\n",
       "      <th>pert_iname</th>\n",
       "      <th>pert_type</th>\n",
       "      <th>cell_id</th>\n",
       "      <th>pert_dose</th>\n",
       "      <th>pert_dose_unit</th>\n",
       "      <th>pert_idose</th>\n",
       "      <th>pert_time</th>\n",
       "      <th>pert_time_unit</th>\n",
       "      <th>pert_itime</th>\n",
       "      <th>distil_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AML001_CD34_24H:A05</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>ctl_vehicle</td>\n",
       "      <td>CD34</td>\n",
       "      <td>0.1</td>\n",
       "      <td>%</td>\n",
       "      <td>0.1 %</td>\n",
       "      <td>24</td>\n",
       "      <td>h</td>\n",
       "      <td>24 h</td>\n",
       "      <td>AML001_CD34_24H_X1_F1B10:A05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AML001_CD34_24H:A06</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>ctl_vehicle</td>\n",
       "      <td>CD34</td>\n",
       "      <td>0.1</td>\n",
       "      <td>%</td>\n",
       "      <td>0.1 %</td>\n",
       "      <td>24</td>\n",
       "      <td>h</td>\n",
       "      <td>24 h</td>\n",
       "      <td>AML001_CD34_24H_X3_F1B10:A06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AML001_CD34_24H:B05</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>ctl_vehicle</td>\n",
       "      <td>CD34</td>\n",
       "      <td>0.1</td>\n",
       "      <td>%</td>\n",
       "      <td>0.1 %</td>\n",
       "      <td>24</td>\n",
       "      <td>h</td>\n",
       "      <td>24 h</td>\n",
       "      <td>AML001_CD34_24H_X1_F1B10:B05|AML001_CD34_24H_X...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AML001_CD34_24H:B06</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>ctl_vehicle</td>\n",
       "      <td>CD34</td>\n",
       "      <td>0.1</td>\n",
       "      <td>%</td>\n",
       "      <td>0.1 %</td>\n",
       "      <td>24</td>\n",
       "      <td>h</td>\n",
       "      <td>24 h</td>\n",
       "      <td>AML001_CD34_24H_X3_F1B10:B06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AML001_CD34_24H:BRD-A03772856:0.37037</td>\n",
       "      <td>BRD-A03772856</td>\n",
       "      <td>BRD-A03772856</td>\n",
       "      <td>trt_cp</td>\n",
       "      <td>CD34</td>\n",
       "      <td>0.37037</td>\n",
       "      <td>µM</td>\n",
       "      <td>500 nM</td>\n",
       "      <td>24</td>\n",
       "      <td>h</td>\n",
       "      <td>24 h</td>\n",
       "      <td>AML001_CD34_24H_X1_F1B10:J04|AML001_CD34_24H_X...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  sig_id        pert_id     pert_iname  \\\n",
       "0                    AML001_CD34_24H:A05           DMSO           DMSO   \n",
       "1                    AML001_CD34_24H:A06           DMSO           DMSO   \n",
       "2                    AML001_CD34_24H:B05           DMSO           DMSO   \n",
       "3                    AML001_CD34_24H:B06           DMSO           DMSO   \n",
       "4  AML001_CD34_24H:BRD-A03772856:0.37037  BRD-A03772856  BRD-A03772856   \n",
       "\n",
       "     pert_type cell_id pert_dose pert_dose_unit pert_idose pert_time  \\\n",
       "0  ctl_vehicle    CD34       0.1              %      0.1 %        24   \n",
       "1  ctl_vehicle    CD34       0.1              %      0.1 %        24   \n",
       "2  ctl_vehicle    CD34       0.1              %      0.1 %        24   \n",
       "3  ctl_vehicle    CD34       0.1              %      0.1 %        24   \n",
       "4       trt_cp    CD34   0.37037             µM     500 nM        24   \n",
       "\n",
       "  pert_time_unit pert_itime                                          distil_id  \n",
       "0              h       24 h                       AML001_CD34_24H_X1_F1B10:A05  \n",
       "1              h       24 h                       AML001_CD34_24H_X3_F1B10:A06  \n",
       "2              h       24 h  AML001_CD34_24H_X1_F1B10:B05|AML001_CD34_24H_X...  \n",
       "3              h       24 h                       AML001_CD34_24H_X3_F1B10:B06  \n",
       "4              h       24 h  AML001_CD34_24H_X1_F1B10:J04|AML001_CD34_24H_X...  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_sig_GSE92742 = sig_info_GSE92742.shape[0]\n",
    "print('Number of signatures in GSE92742 : ' + str(n_sig_GSE92742) + \n",
    "      ' (controls included)')\n",
    "\n",
    "sig_info_GSE92742.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add a new GEO column to stock the origin of the signatures\n",
    "sig_info_GSE92742['GEO'] = 'GSE92742'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load GSE70138 signature info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "sig_info_GSE70138 = pd.read_csv(GSE70138_sig_info_path, sep='\\t', dtype='str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of signatures in GSE70138 : 118050 (controls excluded)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sig_id</th>\n",
       "      <th>pert_id</th>\n",
       "      <th>pert_iname</th>\n",
       "      <th>pert_type</th>\n",
       "      <th>cell_id</th>\n",
       "      <th>pert_idose</th>\n",
       "      <th>pert_itime</th>\n",
       "      <th>distil_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LJP005_A375_24H:A03</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>ctl_vehicle</td>\n",
       "      <td>A375</td>\n",
       "      <td>-666</td>\n",
       "      <td>24 h</td>\n",
       "      <td>LJP005_A375_24H_X1_B19:A03|LJP005_A375_24H_X2_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LJP005_A375_24H:A04</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>ctl_vehicle</td>\n",
       "      <td>A375</td>\n",
       "      <td>-666</td>\n",
       "      <td>24 h</td>\n",
       "      <td>LJP005_A375_24H_X1_B19:A04|LJP005_A375_24H_X2_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LJP005_A375_24H:A05</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>ctl_vehicle</td>\n",
       "      <td>A375</td>\n",
       "      <td>-666</td>\n",
       "      <td>24 h</td>\n",
       "      <td>LJP005_A375_24H_X1_B19:A05|LJP005_A375_24H_X2_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LJP005_A375_24H:A06</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>ctl_vehicle</td>\n",
       "      <td>A375</td>\n",
       "      <td>-666</td>\n",
       "      <td>24 h</td>\n",
       "      <td>LJP005_A375_24H_X1_B19:A06|LJP005_A375_24H_X2_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LJP005_A375_24H:A07</td>\n",
       "      <td>BRD-K76908866</td>\n",
       "      <td>CP-724714</td>\n",
       "      <td>trt_cp</td>\n",
       "      <td>A375</td>\n",
       "      <td>10.0 um</td>\n",
       "      <td>24 h</td>\n",
       "      <td>LJP005_A375_24H_X1_B19:A07|LJP005_A375_24H_X2_...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                sig_id        pert_id pert_iname    pert_type cell_id  \\\n",
       "0  LJP005_A375_24H:A03           DMSO       DMSO  ctl_vehicle    A375   \n",
       "1  LJP005_A375_24H:A04           DMSO       DMSO  ctl_vehicle    A375   \n",
       "2  LJP005_A375_24H:A05           DMSO       DMSO  ctl_vehicle    A375   \n",
       "3  LJP005_A375_24H:A06           DMSO       DMSO  ctl_vehicle    A375   \n",
       "4  LJP005_A375_24H:A07  BRD-K76908866  CP-724714       trt_cp    A375   \n",
       "\n",
       "  pert_idose pert_itime                                          distil_id  \n",
       "0       -666       24 h  LJP005_A375_24H_X1_B19:A03|LJP005_A375_24H_X2_...  \n",
       "1       -666       24 h  LJP005_A375_24H_X1_B19:A04|LJP005_A375_24H_X2_...  \n",
       "2       -666       24 h  LJP005_A375_24H_X1_B19:A05|LJP005_A375_24H_X2_...  \n",
       "3       -666       24 h  LJP005_A375_24H_X1_B19:A06|LJP005_A375_24H_X2_...  \n",
       "4    10.0 um       24 h  LJP005_A375_24H_X1_B19:A07|LJP005_A375_24H_X2_...  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_sig_GSE70138 = sig_info_GSE70138.shape[0]\n",
    "print('Number of signatures in GSE70138 : ' + str(n_sig_GSE70138) + \n",
    "      ' (controls excluded)')\n",
    "\n",
    "sig_info_GSE70138.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "sig_info_GSE70138['GEO'] = 'GSE70138'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merge the 2 GEO signature info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "sig_info_cmap = pd.concat([sig_info_GSE92742, sig_info_GSE70138], sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of signatures in both GEO : 591697\n"
     ]
    }
   ],
   "source": [
    "n_sig_cmap = sig_info_cmap.shape[0]\n",
    "print('Number of signatures in both GEO : ' + str(n_sig_cmap))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Select signatures from compounds only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_compound = sig_info_cmap['pert_type'] == 'trt_cp'\n",
    "is_dmso = sig_info_cmap['pert_id'] == 'DMSO'\n",
    "sig_info_cmap_cmp = sig_info_cmap[is_compound | is_dmso]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of compound (+ DMSO vehicule) signatures in cmap : 333273\n"
     ]
    }
   ],
   "source": [
    "n_sig_cmp = sig_info_cmap_cmp.shape[0]\n",
    "print('Total number of compound (+ DMSO vehicule) signatures in cmap : ' + str(n_sig_cmp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "sig_info_cmap_cmp = sig_info_cmap[sig_info_cmap['pert_id'].isin(cmap_correct_cmp_list)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove irrelevant columns : pert_idose and pert_itime combine these data ; pert_type because all trt_cp\n",
    "columns_to_drop = ['pert_dose', 'pert_dose_unit', 'pert_time', 'pert_time_unit', 'pert_type']\n",
    "sig_info_cmap_cmp = sig_info_cmap_cmp.drop(columns_to_drop, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Curate concentration and time info : set respectively to µM and h, using only float in pert_idose\n",
    "sig_info_cmap_cmp['pert_idose'] = sig_info_cmap_cmp['pert_idose'].apply(dose_to_micromolar)\n",
    "sig_info_cmap_cmp['pert_itime'] = sig_info_cmap_cmp['pert_itime'].apply(lambda s : int(s.split()[0]))\n",
    "sig_info_cmap_cmp = sig_info_cmap_cmp.rename({'pert_idose' : 'pert_idose (µM)',\n",
    "                                             'pert_itime' : 'pert_itime (h)'}, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sig_id</th>\n",
       "      <th>pert_id</th>\n",
       "      <th>pert_iname</th>\n",
       "      <th>cell_id</th>\n",
       "      <th>pert_idose (µM)</th>\n",
       "      <th>pert_itime (h)</th>\n",
       "      <th>distil_id</th>\n",
       "      <th>GEO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AML001_CD34_24H:BRD-A03772856:0.37037</td>\n",
       "      <td>BRD-A03772856</td>\n",
       "      <td>BRD-A03772856</td>\n",
       "      <td>CD34</td>\n",
       "      <td>0.5</td>\n",
       "      <td>24</td>\n",
       "      <td>AML001_CD34_24H_X1_F1B10:J04|AML001_CD34_24H_X...</td>\n",
       "      <td>GSE92742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AML001_CD34_24H:BRD-A03772856:1.11111</td>\n",
       "      <td>BRD-A03772856</td>\n",
       "      <td>BRD-A03772856</td>\n",
       "      <td>CD34</td>\n",
       "      <td>1.0</td>\n",
       "      <td>24</td>\n",
       "      <td>AML001_CD34_24H_X1_F1B10:J03|AML001_CD34_24H_X...</td>\n",
       "      <td>GSE92742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AML001_CD34_24H:BRD-A03772856:10</td>\n",
       "      <td>BRD-A03772856</td>\n",
       "      <td>BRD-A03772856</td>\n",
       "      <td>CD34</td>\n",
       "      <td>10.0</td>\n",
       "      <td>24</td>\n",
       "      <td>AML001_CD34_24H_X1_F1B10:I03|AML001_CD34_24H_X...</td>\n",
       "      <td>GSE92742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AML001_CD34_24H:BRD-A03772856:3.33333</td>\n",
       "      <td>BRD-A03772856</td>\n",
       "      <td>BRD-A03772856</td>\n",
       "      <td>CD34</td>\n",
       "      <td>3.0</td>\n",
       "      <td>24</td>\n",
       "      <td>AML001_CD34_24H_X1_F1B10:I04|AML001_CD34_24H_X...</td>\n",
       "      <td>GSE92742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AML001_CD34_24H:BRD-A19037878:1.11111</td>\n",
       "      <td>BRD-A19037878</td>\n",
       "      <td>trichostatin-a</td>\n",
       "      <td>CD34</td>\n",
       "      <td>1.0</td>\n",
       "      <td>24</td>\n",
       "      <td>AML001_CD34_24H_X1_F1B10:F05|AML001_CD34_24H_X...</td>\n",
       "      <td>GSE92742</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  sig_id        pert_id      pert_iname  \\\n",
       "4  AML001_CD34_24H:BRD-A03772856:0.37037  BRD-A03772856   BRD-A03772856   \n",
       "5  AML001_CD34_24H:BRD-A03772856:1.11111  BRD-A03772856   BRD-A03772856   \n",
       "6       AML001_CD34_24H:BRD-A03772856:10  BRD-A03772856   BRD-A03772856   \n",
       "7  AML001_CD34_24H:BRD-A03772856:3.33333  BRD-A03772856   BRD-A03772856   \n",
       "8  AML001_CD34_24H:BRD-A19037878:1.11111  BRD-A19037878  trichostatin-a   \n",
       "\n",
       "  cell_id  pert_idose (µM)  pert_itime (h)  \\\n",
       "4    CD34              0.5              24   \n",
       "5    CD34              1.0              24   \n",
       "6    CD34             10.0              24   \n",
       "7    CD34              3.0              24   \n",
       "8    CD34              1.0              24   \n",
       "\n",
       "                                           distil_id       GEO  \n",
       "4  AML001_CD34_24H_X1_F1B10:J04|AML001_CD34_24H_X...  GSE92742  \n",
       "5  AML001_CD34_24H_X1_F1B10:J03|AML001_CD34_24H_X...  GSE92742  \n",
       "6  AML001_CD34_24H_X1_F1B10:I03|AML001_CD34_24H_X...  GSE92742  \n",
       "7  AML001_CD34_24H_X1_F1B10:I04|AML001_CD34_24H_X...  GSE92742  \n",
       "8  AML001_CD34_24H_X1_F1B10:F05|AML001_CD34_24H_X...  GSE92742  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sig_info_cmap_cmp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of correct compound signatures in cmap : 310114\n"
     ]
    }
   ],
   "source": [
    "n_sig_cmap_cmp = sig_info_cmap_cmp.shape[0]\n",
    "print('Total number of correct compound signatures in cmap : ' + str(n_sig_cmap_cmp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "sig_info_cmap_cmp.to_csv(sig_info_cmap_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MCF7      41608\n",
       "PC3       34724\n",
       "VCAP      34674\n",
       "A375      27352\n",
       "HT29      26236\n",
       "A549      23483\n",
       "HA1E      20982\n",
       "YAPC      10061\n",
       "HELA      10057\n",
       "HCC515    10024\n",
       "Name: cell_id, dtype: int64"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sig_info_cmap_cmp['cell_id'].value_counts()[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.00    139056\n",
       "5.00      46741\n",
       "0.04      17578\n",
       "0.37      17546\n",
       "1.11      17527\n",
       "          ...  \n",
       "4.00          6\n",
       "4.50          6\n",
       "8.00          6\n",
       "2.50          6\n",
       "1.70          6\n",
       "Name: pert_idose (µM), Length: 95, dtype: int64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sig_info_cmap_cmp['pert_idose (µM)'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24    209613\n",
       "6      95577\n",
       "3       4866\n",
       "48        58\n",
       "Name: pert_itime (h), dtype: int64"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sig_info_cmap_cmp['pert_itime (h)'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cell line choice for downstream analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We wanted to choose first the the 9 core cell lines presented in the original L1000 paper\n",
    "# https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5990023/\n",
    "core_cell_lines = ['MCF7', 'PC3', 'A549', 'A375', 'HEPG2', 'VCAP', 'HCC515', 'HT29', 'HA1E']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "concentration_is_10um = sig_info_cmap_cmp['pert_idose (µM)'] == 10\n",
    "time_is_24h = sig_info_cmap_cmp['pert_itime (h)'] == 24\n",
    "cell_line_is_core = sig_info_cmap_cmp['cell_id'].isin(core_cell_lines)\n",
    "sig_info_10uM_24h = sig_info_cmap_cmp[concentration_is_10um & time_is_24h & cell_line_is_core]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PC3       10314\n",
       "MCF7      10195\n",
       "VCAP       7554\n",
       "A549       6069\n",
       "HA1E       4400\n",
       "A375       3962\n",
       "HT29       3605\n",
       "HCC515     2098\n",
       "HEPG2       544\n",
       "Name: cell_id, dtype: int64"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sig_info_10uM_24h['cell_id'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# HepG2 has a low number of 10 µM and 24 h signatures, we decided to remove it from the downstream analysis\n",
    "sig_info_10uM_24h = sig_info_10uM_24h[sig_info_10uM_24h['cell_id'] != 'HEPG2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 9035 compounds profiled at 10 µM and 24 h in the 8 selected cell lines\n"
     ]
    }
   ],
   "source": [
    "used_compounds = sig_info_10uM_24h['pert_id'].unique()\n",
    "n_used_compounds = used_compounds.shape[0]\n",
    "print('There are ' + str(n_used_compounds) + ' compounds profiled at 10 µM and 24 h in the 8 selected cell lines')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmp_info_cmap['used_compound'] = cmp_info_cmap['pert_id'].isin(used_compounds).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmp_info_cmap.to_csv(cmp_info_cmap_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_sig_info = sig_info_10uM_24h.drop_duplicates(subset=['pert_id', 'cell_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PC3       8071\n",
       "MCF7      7546\n",
       "VCAP      6365\n",
       "A549      5267\n",
       "HA1E      3646\n",
       "A375      3525\n",
       "HT29      3192\n",
       "HCC515    1932\n",
       "Name: cell_id, dtype: int64"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_sig_info['cell_id'].value_counts()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}