process_pretrained_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import torch\n",
    "import numpy as np\n",
    "import wfdb\n",
    "from tqdm import tqdm\n",
    "from skimage.transform import resize\n",
    "from joblib import dump\n",
    "\n",
    "def normalize_signal(signal):\n",
    "    # Calculate mean and standard deviation\n",
    "    mean = np.mean(signal, axis=0)\n",
    "    std = np.std(signal, axis=0)\n",
    "    \n",
    "    # Avoid division by zero by adding a small epsilon where std is 0\n",
    "    std_safe = np.where(std == 0, 1e-10, std)  # Replace 0 std with a small value\n",
    "    \n",
    "    # Normalize the signal\n",
    "    normalized_signal = (signal - mean) / std_safe\n",
    "    \n",
    "    return normalized_signal\n",
    "\n",
    "def interpolate_signal(signal):\n",
    "    signal_df = pd.DataFrame(signal)\n",
    "    signal_df.interpolate(method='linear', axis=0, inplace=True, limit_direction='both')\n",
    "    signal_corrected = signal_df.to_numpy()\n",
    "    return signal_corrected\n",
    "\n",
    "def load_ecg_data(df, base_path):\n",
    "    full_paths = base_path + df['path']\n",
    "    loaded_ecg_data = []\n",
    "\n",
    "    for f in tqdm(full_paths, desc=\"Loading ECG data\"):\n",
    "        wave_array, meta = wfdb.rdsamp(f)\n",
    "        # Interpolate and preprocess the signal\n",
    "        wave_array = interpolate_signal(wave_array)\n",
    "        \n",
    "        # Reshape the signal into multiple channels\n",
    "        num_channels = meta[\"n_sig\"]\n",
    "        num_samples_per_channel = wave_array.size // num_channels\n",
    "        \n",
    "        if wave_array.size % num_channels == 0:\n",
    "            wave_array = wave_array.reshape(num_samples_per_channel, num_channels)\n",
    "            \n",
    "            # Process the signal further if needed\n",
    "            wave_array = normalize_signal(wave_array)\n",
    "            \n",
    "            wave_array = wave_array.reshape(1, -1)\n",
    "            \n",
    "            loaded_ecg_data.append(wave_array)\n",
    "            \n",
    "        else:\n",
    "            print(f\"Unexpected data size in {f}. Skipping file.\")\n",
    "\n",
    "    # Assuming all arrays are of the same shape after flattening\n",
    "    if loaded_ecg_data:\n",
    "        loaded_ecg_data_np = np.stack(loaded_ecg_data, axis=0)\n",
    "        return torch.tensor(loaded_ecg_data_np, dtype=torch.float32)\n",
    "    else:\n",
    "        return torch.empty(0)  # Return an empty tensor if no data\n",
    "\n",
    "\n",
    "# Define parameters\n",
    "base_path = 'Z:/tale2/Shared/Mohammod/mimic-iv-ecg-diagnostic-electrocardiogram-matched-subset-1.0/'\n",
    "Y = pd.read_csv(base_path + 'final_mimic_ecg_metadata.csv')\n",
    "\n",
    "# Load ECG data\n",
    "X_ecg_loaded = load_ecg_data(Y, base_path)\n",
    "\n",
    "\n",
    "# Save the processed data\n",
    "torch.save(X_ecg_loaded, 'data_feature/ecg_features_tensor.pt')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import skimage.io\n",
    "import skimage.transform\n",
    "import torch\n",
    "from PIL import Image\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "def preprocess_image(image_path, resize_dim=(224, 224)):\n",
    "    print(\"Processing:\", image_path)\n",
    "    # Read image with skimage\n",
    "    image = skimage.io.imread(image_path)\n",
    "\n",
    "    # Ensure image is grayscale, if it's not, convert to grayscale\n",
    "    if len(image.shape) == 3:\n",
    "        image = skimage.color.rgb2gray(image)\n",
    "    \n",
    "    # Resize image\n",
    "    image = skimage.transform.resize(image, resize_dim, mode='constant', anti_aliasing=True)\n",
    "    \n",
    "    # Normalize image for the encoder model\n",
    "    # Assuming the model expects pixel values in [0, 1], adjust if necessary\n",
    "    image = (image * 255).astype(np.uint8)  # Convert to uint8\n",
    "    normalize = transforms.Normalize(mean=[0.5], std=[0.5])  # Adjust mean and std as per your model's training\n",
    "    transform = transforms.Compose([\n",
    "        transforms.ToPILImage(),\n",
    "        transforms.ToTensor(),\n",
    "        normalize\n",
    "    ])\n",
    "    image = transform(image)\n",
    "\n",
    "    return image\n",
    "\n",
    "def process_images(root, csv_file):\n",
    "    cases = pd.read_csv(os.path.join(root, csv_file))\n",
    "    all_images = []\n",
    "\n",
    "    for idx, row in cases.iterrows():\n",
    "        image_path = os.path.join(root, row['file_path'])\n",
    "        image = preprocess_image(image_path)\n",
    "        all_images.append(image)\n",
    "\n",
    "    # Convert list of tensors to 4D tensor (batch, channels, height, width)\n",
    "    all_images = torch.stack(all_images)\n",
    "\n",
    "    return all_images\n",
    "\n",
    "# Usage\n",
    "root = 'Z:/tale2/Shared/Mohammod/physionet.org/files/mimic-cxr-jpg/2.0.0'\n",
    "csv_file = 'final_mimic_cxr_metadata.csv'\n",
    "X_image_tensor = process_images(root, csv_file)\n",
    "torch.save(X_image_tensor, 'data_feature/encoder_image_tensor.pt')\n"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "2db00650db45e86c"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}