kits19/h5_data_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "import h5py\n",
    "import src.starter.utils as starter\n",
    "import src.starter.visualize as vis\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"data_interpolated_stats.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "max_val = data['max_val'].quantile(0.95)\n",
    "min_val = data['min_val'].quantile(0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stage='train'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"{}_data_interpolated_stats.csv\".format(stage))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "crops = h5py.File(\"{}_interpolated_crops.hdf5\".format(stage), \"w\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "SIZE = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "def normalize(volume):\n",
    "    # Scale to values between 0 and 1\n",
    "    mxval = max_val\n",
    "    mnval = min_val\n",
    "    im_volume = (volume - mnval)/max(mxval - mnval, 1e-3)\n",
    "    return im_volume"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "def positions(im_shape, crop_shape, pads):\n",
    "        zi, xi, yi = im_shape\n",
    "        zc, xc, yc = crop_shape\n",
    "        zp, xp, yp = pads\n",
    "        for z in [z for z in range(0, zi, zp) if z + zc <= zi]:\n",
    "            for x in [x for x in range(0, xi, xp) if x + xc <= xi]:\n",
    "                for y in [y for y in range(0, yi, yp) if y + yc <= yi]:\n",
    "                    yield (z, x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "pycharm": {},
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35e52125482149cf9bb3585319afd13a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=42), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# 5 for local run. replace with len(data) for cloud\n",
    "crops_data = []\n",
    "for i in tqdm.tqdm_notebook(range(len(data))):\n",
    "    row = data.iloc[i]\n",
    "    case_id, z, x, y = row['case_id'], row['num_slices'], row['height'], row['width']\n",
    "    im, mask = starter.load_case(case_id)\n",
    "    im, mask = im.get_data(), mask.get_data()\n",
    "    if im.shape[0] % SIZE == 0:\n",
    "        z_pad_size = 0\n",
    "    else:\n",
    "        z_pad_size = SIZE - im.shape[0] % SIZE \n",
    "    \n",
    "    if im.shape[1] % SIZE == 0:\n",
    "        x_pad_size = 0\n",
    "    else:\n",
    "        x_pad_size = SIZE - im.shape[1] % SIZE \n",
    "        \n",
    "    if im.shape[2] % SIZE == 0:\n",
    "        y_pad_size = 0\n",
    "    else:\n",
    "        y_pad_size = SIZE - im.shape[2] % SIZE \n",
    "        \n",
    "    im = normalize(im)\n",
    "    im = np.pad(im, ((z_pad_size, 0), (x_pad_size, 0), (y_pad_size, 0)), 'constant')\n",
    "    mask = np.pad(mask, ((z_pad_size, 0), (x_pad_size, 0), (y_pad_size, 0)), 'constant')\n",
    "    for position in positions((int(z + z_pad_size) , int(x + x_pad_size), int(y + y_pad_size)), (64, 64, 64), (32, 32, 32)):\n",
    "        z, x, y = position\n",
    "        crop = mask[z:z+64, x:x+64,y:y+64]\n",
    "        kid_size = np.sum(crop == 1)\n",
    "        tumor_size = np.sum(crop == 2)\n",
    "        crops_data.append([case_id, position, 64, 32, kid_size, tumor_size])\n",
    "    crops.create_dataset(case_id, data=np.array([im, mask]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "crops_data = pd.DataFrame(data=crops_data, columns=[\"case_id\", \"position\", \"window\", \"padding\", \"kid_size\", \"tumor_size\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "crops.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "crops_data.to_csv(\"val_interpolated_crops.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}