datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "ROOT = '/mnt/c/Users/sergio/Desktop/TFG'\n",
    "TCIA_ROOT = f'{ROOT}/TCIA'\n",
    "GM_ROOT = f'{ROOT}/GM'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "TCIA = f'{TCIA_ROOT}/participants.csv'\n",
    "GM = f'{GM_ROOT}/GM-BRATS+NORM/participants.tsv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "# negative = wildtype\n",
    "# positive = mutated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [],
   "source": [
    "def idh_status_to_label(idh_status):\n",
    "    idh_status = idh_status.lower()\n",
    "    if 'mutated' in idh_status or 'idh' in idh_status or 'positivo' in idh_status:\n",
    "        return 1  # 'mutated'\n",
    "    elif 'wildtype' in idh_status or 'negativo' in idh_status:\n",
    "        return 0  # 'wildtype'\n",
    "    else:\n",
    "        return ValueError(idh_status)\n",
    "\n",
    "def codeletion_to_label(codeletion):\n",
    "    if pd.isna(codeletion):\n",
    "        return codeletion\n",
    "    codeletion = codeletion.lower()\n",
    "    if 'deletion' in codeletion or 'presente' in codeletion:\n",
    "        return 1  # 'co-deleted'\n",
    "    elif 'intact' in codeletion or 'ausente' in codeletion:\n",
    "        return 0  # 'intact'\n",
    "    else:\n",
    "        raise ValueError(codeletion)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **TCIA**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "501\n"
     ]
    }
   ],
   "source": [
    "tcia_df = pd.read_csv(TCIA, sep=';')\n",
    "print(len(tcia_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>participant_id</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>who_cns_grade</th>\n",
       "      <th>histologic_subtype</th>\n",
       "      <th>idh_status</th>\n",
       "      <th>codeletion_1p19q_status</th>\n",
       "      <th>database</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sub-0004</td>\n",
       "      <td>M</td>\n",
       "      <td>66</td>\n",
       "      <td>4</td>\n",
       "      <td>Glioblastoma</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TCIA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sub-0005</td>\n",
       "      <td>F</td>\n",
       "      <td>80</td>\n",
       "      <td>4</td>\n",
       "      <td>Glioblastoma</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TCIA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  participant_id sex  age  who_cns_grade histologic_subtype  idh_status  \\\n",
       "0       sub-0004   M   66              4       Glioblastoma           0   \n",
       "1       sub-0005   F   80              4       Glioblastoma           0   \n",
       "\n",
       "   codeletion_1p19q_status database  \n",
       "0                      NaN     TCIA  \n",
       "1                      NaN     TCIA  "
      ]
     },
     "execution_count": 285,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tcia_processed = tcia_df[['participant_id', 'sex', 'age', 'who_cns_grade', 'diagnosis', 'idh_status', 'codeletion_1p19q_status']]\n",
    "# tcia_processed = tcia_processed[~tcia_processed['codeletion_1p19q_status'].isnull()].copy()\n",
    "\n",
    "tcia_processed = tcia_processed.rename(columns={'diagnosis': 'histologic_subtype'})\n",
    "tcia_processed = tcia_processed.drop(tcia_processed[tcia_processed['participant_id'] == 'sub-138'].index, axis=0)  # not present\n",
    "tcia_processed = tcia_processed.drop(tcia_processed[tcia_processed['participant_id'] == 'sub-175'].index, axis=0)  # not present\n",
    "tcia_processed = tcia_processed.drop(tcia_processed[tcia_processed['participant_id'] == 'sub-181'].index, axis=0)  # not present\n",
    "tcia_processed = tcia_processed.drop(tcia_processed[tcia_processed['participant_id'] == 'sub-278'].index, axis=0)  # not present\n",
    "tcia_processed = tcia_processed.drop(tcia_processed[tcia_processed['participant_id'] == 'sub-289'].index, axis=0)  # not present\n",
    "tcia_processed = tcia_processed.drop(tcia_processed[tcia_processed['participant_id'] == 'sub-315'].index, axis=0)  # not present\n",
    "tcia_processed = tcia_processed.drop(tcia_processed[tcia_processed['participant_id'] == 'sub-541'].index, axis=0)  # corrupted segmentation\n",
    "tcia_processed['participant_id'] = tcia_processed['participant_id'].str.replace(r'sub-(\\d{1,3})$', lambda x: 'sub-' + x.group(1).zfill(4), regex=True)\n",
    "tcia_processed['histologic_subtype'] = tcia_processed['histologic_subtype'].str.split(',').str[0]\n",
    "tcia_processed['idh_status'] = tcia_processed['idh_status'].apply(idh_status_to_label)\n",
    "tcia_processed['codeletion_1p19q_status'] = tcia_processed['codeletion_1p19q_status'].apply(codeletion_to_label)\n",
    "tcia_processed['database'] = ['TCIA'] * len(tcia_processed)\n",
    "tcia_processed = tcia_processed.reset_index(drop=True)\n",
    "tcia_processed.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "494\n"
     ]
    }
   ],
   "source": [
    "print(len(tcia_processed))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **Gregorio Marañón**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42\n"
     ]
    }
   ],
   "source": [
    "gm_df = pd.read_csv(GM, sep='\\t', encoding='latin1')\n",
    "print(len(gm_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>participant_id</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>who_cns_grade</th>\n",
       "      <th>histologic_subtype</th>\n",
       "      <th>idh_status</th>\n",
       "      <th>codeletion_1p19q_status</th>\n",
       "      <th>database</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sub-0000</td>\n",
       "      <td>M</td>\n",
       "      <td>50</td>\n",
       "      <td>4</td>\n",
       "      <td>Glioblastoma</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sub-0002</td>\n",
       "      <td>F</td>\n",
       "      <td>57</td>\n",
       "      <td>4</td>\n",
       "      <td>Glioblastoma</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GM</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  participant_id sex  age  who_cns_grade histologic_subtype  idh_status  \\\n",
       "0       sub-0000   M   50              4       Glioblastoma           0   \n",
       "1       sub-0002   F   57              4       Glioblastoma           0   \n",
       "\n",
       "   codeletion_1p19q_status database  \n",
       "0                      0.0       GM  \n",
       "1                      0.0       GM  "
      ]
     },
     "execution_count": 287,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gm_processed = gm_df[['participant_id', 'sex', 'age', 'who_cns_grade', 'histologic_subtype', 'idh_status', 'codeletion_1p19q_status']]\n",
    "# gm_processed = gm_processed[~gm_processed['codeletion_1p19q_status'].isnull()].copy()\n",
    "\n",
    "gm_processed = gm_processed.drop(gm_processed[gm_processed['participant_id'] == 'sub-0008'].index)\n",
    "gm_processed['histologic_subtype'] = gm_processed['histologic_subtype'].str.title()\n",
    "gm_processed['histologic_subtype'] = gm_processed['histologic_subtype'].apply(lambda x: x.replace('Astrocitoma', 'Astrocytoma'))\n",
    "gm_processed = gm_processed[gm_processed['histologic_subtype'] != 'Glioma Difuso Linea Media']                                 \n",
    "gm_processed['idh_status'] = gm_processed['idh_status'].apply(idh_status_to_label)\n",
    "gm_processed['codeletion_1p19q_status'] = gm_processed['codeletion_1p19q_status'].apply(codeletion_to_label)\n",
    "gm_processed['database'] = ['GM'] * len(gm_processed)\n",
    "gm_processed = gm_processed.reset_index(drop=True)\n",
    "gm_processed.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sub-0001', 'sub-0008'}"
      ]
     },
     "execution_count": 288,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(gm_df['participant_id']).difference(gm_processed['participant_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n"
     ]
    }
   ],
   "source": [
    "print(len(gm_processed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>participant_id</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>who_cns_grade</th>\n",
       "      <th>histologic_subtype</th>\n",
       "      <th>idh_status</th>\n",
       "      <th>codeletion_1p19q_status</th>\n",
       "      <th>database</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>sub-0004</td>\n",
       "      <td>M</td>\n",
       "      <td>66</td>\n",
       "      <td>4</td>\n",
       "      <td>Glioblastoma</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TCIA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>sub-0005</td>\n",
       "      <td>F</td>\n",
       "      <td>80</td>\n",
       "      <td>4</td>\n",
       "      <td>Glioblastoma</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TCIA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index participant_id sex  age  who_cns_grade histologic_subtype  \\\n",
       "0      0       sub-0004   M   66              4       Glioblastoma   \n",
       "1      1       sub-0005   F   80              4       Glioblastoma   \n",
       "\n",
       "   idh_status  codeletion_1p19q_status database  \n",
       "0           0                      NaN     TCIA  \n",
       "1           0                      NaN     TCIA  "
      ]
     },
     "execution_count": 290,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df = pd.concat([tcia_processed, gm_processed], axis=0).reset_index(drop=True)\n",
    "final_df.insert(0, 'index', final_df.index)\n",
    "final_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.to_csv(f'{ROOT}/participants.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}