From 887ed665ff7cbbfc752ec395639492a4dbeac124 Mon Sep 17 00:00:00 2001 From: michaelgroeger Date: Wed, 5 Apr 2023 15:56:08 +0200 Subject: [PATCH 1/5] Improve documentation --- README.md | 30 +++++++- environment.yaml | 186 ----------------------------------------------- requirements.txt | 17 +++++ 3 files changed, 45 insertions(+), 188 deletions(-) delete mode 100644 environment.yaml create mode 100644 requirements.txt diff --git a/README.md b/README.md index 499f49a..852c33e 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,44 @@ # BoxShrink: From Bounding Boxes to Segmentation Masks + This is the repository for the corresponding MICCAI-MILLanD Workshop 2022 paper [BoxShrink: From Bounding Boxes to Segmentation Masks](https://arxiv.org/abs/2208.03142) -## Introduction +## Overview + We present two algorithms how one can process bounding boxes to pseudo segmentation masks in a binary class segmenetation setting: + 1. rapidBoxshrink: Works by a simple thresholding and overlapping strategy between the initial bounding box and the generated superpixels. This algorithm will reject superpixels that don't overlap to a certain percentage with the bounding box and then run a F-CRF on the pseudomask. ![rapidBoxshrink-Overview](/images/rapidBoxshrink_overview.png) + 2. robustBoxshrink: Compares the superpixels on the boundary with the mean foreground and background embedding of the training dataset. Those whose cosine distance is closer to the background embedding are being rejected. Finally, a F-CRF is being run on the pseudomask. ![robustBoxshrink-Overview](/images/robustBoxshrink_Overview.png) +## Installation + +Please follow these steps to install the environment. + +```zsh +# Creates new conda environment +conda create -n boxshrink python=3.10.8 ipython +# Activates conda environment +conda activate boxshrink +# Makes script folder callable as module in python scripts +conda install conda-build +conda develop ./scripts +# Installs dependencies +pip install -r requirements.txt +# Make conda available in jupyter notebook +conda install ipykernel +python -m ipykernel install --user --name=boxshrink +``` + ## Usage + Please check the config file in `scripts/config` to set paths and hyperparameters. Please have a look at the notebook files if you want to generate bounding boxes from masks, run rapidBoxshrink or robustBoxshrink. After you generated the masks feel free to use them as training input as shown in `train.ipynb`. Have fun! ## Citation + If you use this work please cite: -``` + +```latex @inproceedings{groger2022boxshrink, title={BoxShrink: From Bounding Boxes to Segmentation Masks}, author={Gröger, Michael and Borisov, Vadim and Kasneci, Gjergji}, diff --git a/environment.yaml b/environment.yaml deleted file mode 100644 index 2bc5ac4..0000000 --- a/environment.yaml +++ /dev/null @@ -1,186 +0,0 @@ -name: boxshrink -channels: - - conda-forge - - defaults -dependencies: - - appnope=0.1.2=py38hecd8cb5_1001 - - asttokens=2.0.5=pyhd3eb1b0_0 - - backcall=0.2.0=pyhd3eb1b0_0 - - beautifulsoup4=4.11.1=py38hecd8cb5_0 - - brotlipy=0.7.0=py38h9ed2024_1003 - - bzip2=1.0.8=h1de35cc_0 - - ca-certificates=2022.07.19=hecd8cb5_0 - - cctools=949.0.1=h9abeeb2_25 - - cctools_osx-64=949.0.1=hc7db93f_25 - - certifi=2022.9.24=py38hecd8cb5_0 - - cffi=1.15.1=py38hc55c11b_0 - - cfgv=3.3.1=pyhd8ed1ab_0 - - chardet=4.0.0=py38hecd8cb5_1003 - - conda=22.9.0=py38hecd8cb5_0 - - conda-build=3.22.0=py38hecd8cb5_0 - - conda-package-handling=1.9.0=py38hca72f7f_0 - - cryptography=37.0.1=py38hf6deb26_0 - - debugpy=1.5.1=py38he9d5cce_0 - - decorator=5.1.1=pyhd3eb1b0_0 - - entrypoints=0.4=py38hecd8cb5_0 - - executing=0.8.3=pyhd3eb1b0_0 - - filelock=3.8.0=pyhd8ed1ab_0 - - glob2=0.7=pyhd3eb1b0_0 - - icu=58.2=h0a44026_3 - - identify=2.5.5=pyhd8ed1ab_0 - - idna=3.3=pyhd3eb1b0_0 - - ipykernel=6.9.1=py38hecd8cb5_0 - - ipython=8.4.0=py38hecd8cb5_0 - - jedi=0.18.1=py38hecd8cb5_1 - - jupyter_client=7.3.4=py38hecd8cb5_0 - - jupyter_core=4.10.0=py38hecd8cb5_0 - - ld64=530=h20443b4_25 - - ld64_osx-64=530=h70f3046_25 - - ldid=2.1.2=h2d21305_2 - - libarchive=3.6.1=he336d3b_0 - - libcxx=12.0.0=h2f01273_0 - - libffi=3.3=hb1e8313_2 - - libiconv=1.16=hca72f7f_2 - - liblief=0.11.5=he9d5cce_1 - - libllvm14=14.0.6=he552d86_0 - - libsodium=1.0.18=h1de35cc_0 - - libxml2=2.9.14=hbf8cd5e_0 - - lz4-c=1.9.3=h23ab428_1 - - matplotlib-inline=0.1.2=pyhd3eb1b0_2 - - ncurses=6.3=hca72f7f_3 - - nest-asyncio=1.5.5=py38hecd8cb5_0 - - nodeenv=1.7.0=pyhd8ed1ab_0 - - openssl=1.1.1q=hca72f7f_0 - - parso=0.8.3=pyhd3eb1b0_0 - - patch=2.7.6=h1de35cc_1001 - - pexpect=4.8.0=pyhd3eb1b0_3 - - pickleshare=0.7.5=pyhd3eb1b0_1003 - - pip=22.1.2=py38hecd8cb5_0 - - pkginfo=1.8.2=pyhd3eb1b0_0 - - platformdirs=2.5.2=pyhd8ed1ab_1 - - pre-commit=2.20.0=py38h50d1736_0 - - prompt-toolkit=3.0.20=pyhd3eb1b0_0 - - psutil=5.9.0=py38hca72f7f_0 - - ptyprocess=0.7.0=pyhd3eb1b0_2 - - pure_eval=0.2.2=pyhd3eb1b0_0 - - py-lief=0.11.5=py38he9d5cce_1 - - pycosat=0.6.3=py38h1de35cc_1 - - pycparser=2.21=pyhd8ed1ab_0 - - pygments=2.11.2=pyhd3eb1b0_0 - - pyopenssl=22.0.0=pyhd3eb1b0_0 - - pysocks=1.7.1=py38_1 - - python=3.8.13=hdfd78df_0 - - python-dateutil=2.8.2=pyhd3eb1b0_0 - - python-libarchive-c=2.9=pyhd3eb1b0_1 - - python_abi=3.8=2_cp38 - - pyyaml=6.0=py38hed1de0f_4 - - pyzmq=23.2.0=py38he9d5cce_0 - - readline=8.1.2=hca72f7f_1 - - requests=2.28.1=py38hecd8cb5_0 - - ripgrep=13.0.0=hc2228c6_0 - - ruamel_yaml=0.15.100=py38h9ed2024_0 - - setuptools=63.4.1=py38hecd8cb5_0 - - six=1.16.0=pyhd3eb1b0_1 - - sqlite=3.39.2=h707629a_0 - - stack_data=0.2.0=pyhd3eb1b0_0 - - tapi=1000.10.8=ha1b3eb9_0 - - tk=8.6.12=h5d9f67b_0 - - toml=0.10.2=pyhd8ed1ab_0 - - toolz=0.11.2=pyhd3eb1b0_0 - - tornado=6.1=py38h9ed2024_0 - - traitlets=5.1.1=pyhd3eb1b0_0 - - ukkonen=1.0.1=py38h12bbefe_1 - - virtualenv=20.16.5=py38h50d1736_0 - - wcwidth=0.2.5=pyhd3eb1b0_0 - - wheel=0.37.1=pyhd3eb1b0_0 - - xz=5.2.5=hca72f7f_1 - - yaml=0.2.5=h0d85af4_2 - - zeromq=4.3.4=h23ab428_0 - - zlib=1.2.12=h4dc903c_2 - - zstd=1.5.2=hcb37349_0 - - pip: - - alembic==1.8.1 - - attrs==22.1.0 - - black==22.8.0 - - charset-normalizer==2.1.1 - - click==8.0.2 - - cloudpickle==2.1.0 - - colorama==0.4.5 - - cpplint==1.6.1 - - cycler==0.11.0 - - cython==0.29.32 - - databricks-cli==0.17.3 - - distlib==0.3.6 - - docker==5.0.3 - - efficientnet-pytorch==0.7.1 - - flask==2.2.2 - - fonttools==4.37.1 - - gitdb==4.0.9 - - gitpython==3.1.27 - - grad-cam==1.4.5 - - greenlet==1.1.3 - - gunicorn==20.1.0 - - imageio==2.21.2 - - importlib-metadata==4.12.0 - - importlib-resources==5.9.0 - - iniconfig==1.1.1 - - install==1.3.5 - - itsdangerous==2.1.2 - - jinja2==3.1.2 - - joblib==1.2.0 - - kiwisolver==1.4.4 - - mako==1.2.2 - - markupsafe==2.1.1 - - matplotlib==3.5.3 - - mlflow==1.28.0 - - munch==2.5.0 - - mypy-extensions==0.4.3 - - networkx==2.8.6 - - numpy==1.23.2 - - oauthlib==3.2.0 - - opencv-python==4.6.0.66 - - opencv-python-headless==4.5.2.52 - - packaging==21.3 - - pandas==1.4.3 - - pathspec==0.10.1 - - pillow==9.2.0 - - pluggy==1.0.0 - - pretrainedmodels==0.7.4 - - prometheus-client==0.14.1 - - prometheus-flask-exporter==0.20.3 - - protobuf==4.21.5 - - py==1.11.0 - - pydensecrf==1.0rc2 - - pyjwt==2.4.0 - - pyngrok==5.1.0 - - pyparsing==3.0.9 - - pytest==7.1.3 - - pytz==2022.2.1 - - pywavelets==1.3.0 - - querystring-parser==1.2.4 - - scikit-image==0.19.3 - - scikit-learn==1.1.2 - - scipy==1.9.1 - - segmentation-models-pytorch==0.3.0 - - sklearn==0.0 - - smmap==5.0.0 - - soupsieve==2.3.2.post1 - - sqlalchemy==1.4.40 - - sqlparse==0.4.2 - - tabulate==0.8.10 - - threadpoolctl==3.1.0 - - tifffile==2022.8.12 - - tiffile==2018.10.18 - - timm==0.4.12 - - tomli==2.0.1 - - torch==1.12.1 - - torchmetrics==0.9.3 - - torchvision==0.13.1 - - tqdm==4.64.0 - - ttach==0.0.3 - - typing-extensions==4.3.0 - - urllib3==1.26.12 - - websocket-client==1.4.0 - - werkzeug==2.2.2 - - zipp==3.8.1 -prefix: /Users/michaelgroeger/miniconda3/envs/boxshrink diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..227bebc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,17 @@ +# Automatically generated by https://github.com/damnever/pigar. +grad-cam==1.4.6 +matplotlib==3.6.2 +numpy==1.23.4 +opencv-python==4.7.0.72 +pandas==1.5.2 +Pillow==9.2.0 +pytorch-text-crf==0.1 +scikit-image==0.20.0 +scikit-learn==1.2.1 +segmentation-models-pytorch==0.3.2 +tifffile==2023.3.21 +torch==1.12.1 +torchmetrics==0.11.0 +torchvision==0.13.1 +tqdm==4.64.1 +git+https://github.com/lucasb-eyer/pydensecrf.git \ No newline at end of file From 441922d6b26253bb59f7062fd40fc6f6b519de4f Mon Sep 17 00:00:00 2001 From: michaelgroeger Date: Wed, 5 Apr 2023 15:56:35 +0200 Subject: [PATCH 2/5] Adjust jaccard to newest parameter requirments --- scripts/crf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/crf.py b/scripts/crf.py index 5a65b65..f4f54fe 100644 --- a/scripts/crf.py +++ b/scripts/crf.py @@ -20,7 +20,7 @@ RGB_STD, ) -jaccard_crf = JaccardIndex(num_classes=len(CLASSES), average="none", ignore_index=0).to( +jaccard_crf = JaccardIndex(num_classes=len(CLASSES), average="none", ignore_index=0, task="binary").to( DEVICE ) From 8767f745ac9bcecb444f75124e3a892577bd2830 Mon Sep 17 00:00:00 2001 From: michaelgroeger Date: Wed, 5 Apr 2023 15:56:49 +0200 Subject: [PATCH 3/5] improve notebook --- robust_boxshrink.ipynb | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/robust_boxshrink.ipynb b/robust_boxshrink.ipynb index 02ab968..a39e827 100644 --- a/robust_boxshrink.ipynb +++ b/robust_boxshrink.ipynb @@ -2,18 +2,9 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/michaelgroeger/miniconda3/envs/boxshrink/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], + "outputs": [], "source": [ "from sklearn.model_selection import train_test_split\n", "from scripts.tools import return_files_in_directory, human_sort\n", @@ -23,12 +14,12 @@ "from torchvision.models import resnet50\n", "from scripts.dataset import Colonoscopy_Dataset\n", "from torch.utils.data import DataLoader\n", - "from scripts.embeddings import ResnetFeatureExtractor,generate_embedding_masks_for_dataset" + "from scripts.embeddings import ResnetFeatureExtractor, generate_embedding_masks_for_dataset" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -37,10 +28,8 @@ "# Ensure files are in correct order\n", "human_sort(image_files)\n", "human_sort(box_files)\n", - "X_train, X_test, y_train, y_test = train_test_split(image_files, box_files, test_size=0.1, random_state=1)\n", - "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.11111, random_state=1) # 0.1111 x 0.9 = 0.1\n", "\n", - "dataset = Colonoscopy_Dataset(X_val, y_val)\n", + "dataset = Colonoscopy_Dataset(image_files, box_files)\n", "data_loader = DataLoader(dataset, batch_size=64, shuffle=False, num_workers=0)\n", "\n", "TESTING_DIR = DATA_DIR + \"/testing/robust_boxshrink\"\n", @@ -54,7 +43,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -67,17 +56,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Generating Embedding Masks: 100%|██████████| 1/1 [13:15<00:00, 795.02s/batch]\n" - ] - } - ], + "outputs": [], "source": [ "resnet = resnet50(weights=\"ResNet50_Weights.IMAGENET1K_V2\")\n", "resnet.eval()\n", From de9659eda0a835f8ce63a2222603f833ed2b5ac3 Mon Sep 17 00:00:00 2001 From: michaelgroeger Date: Wed, 5 Apr 2023 15:56:57 +0200 Subject: [PATCH 4/5] refactoring --- scripts/embeddings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/embeddings.py b/scripts/embeddings.py index a191ca1..80ddf3b 100644 --- a/scripts/embeddings.py +++ b/scripts/embeddings.py @@ -393,7 +393,7 @@ def generate_embedding_masks_for_dataset( with tqdm( data_loader, unit="batch", desc="Generating Embedding Masks" ) as tepoch: - for train_inputs, train_labels, train_org_images in tepoch: + for _, train_labels, train_org_images in tepoch: for idx in range(train_labels.shape[0]): mask_name = dataset.X[filecounter] pseudomask = get_embedding_mask_or_box( From 7e706622979980e3564f26c23782d166e157979f Mon Sep 17 00:00:00 2001 From: michaelgroeger Date: Wed, 5 Apr 2023 15:57:08 +0200 Subject: [PATCH 5/5] delete unrelevant files --- scripts/voc_legacy/label_colors.py | 28 -- scripts/voc_legacy/voc_tools.py | 509 ----------------------------- 2 files changed, 537 deletions(-) delete mode 100644 scripts/voc_legacy/label_colors.py delete mode 100644 scripts/voc_legacy/voc_tools.py diff --git a/scripts/voc_legacy/label_colors.py b/scripts/voc_legacy/label_colors.py deleted file mode 100644 index f5925f6..0000000 --- a/scripts/voc_legacy/label_colors.py +++ /dev/null @@ -1,28 +0,0 @@ -import numpy as np - -# Used to map colors of mask from PascalVOC to class index -label_colors = np.array( - [ - (0, 0, 0), # Class 0 - (192, 128, 128), # Class 1 - (192, 0, 0), # Class 2 - (0, 64, 128), # Class 3 - (64, 0, 0), # Class 4 - (128, 64, 0), # Class 5 - (0, 192, 0), # Class 6 - (128, 128, 0), # Class 7 - (128, 128, 128), # Class 8 - (0, 128, 0), # Class 9 - (0, 0, 128), # Class 10 - (128, 192, 0), # Class 11 - (64, 128, 0), # Class 12 - (192, 0, 128), # Class 13 - (64, 0, 128), # Class 14 - (128, 0, 0), # Class 15 - (0, 128, 128), # Class 16 - (0, 64, 0), # Class 17 - (64, 128, 128), # Class 18 - (192, 128, 0), # Class 19 - (128, 0, 128), # Class 20 - ] -) diff --git a/scripts/voc_legacy/voc_tools.py b/scripts/voc_legacy/voc_tools.py deleted file mode 100644 index 7516009..0000000 --- a/scripts/voc_legacy/voc_tools.py +++ /dev/null @@ -1,509 +0,0 @@ -import os -import random -import re -import shutil -import xml.etree.ElementTree as ET -from os.path import isfile, join - -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -import torch -from bs4 import BeautifulSoup -from PIL import Image -from sklearn.model_selection import train_test_split -from tqdm import tqdm - -############################################################################################################################# -# This is a collection of scripts we wrote for our first experiments on PascalVOC. # -# We later needed to switch the datasets because we were lacking the resources for a dataset of the size of PascalVOC. # -# We still publish it here since some people might find them useful and we uses some of the scripts in the Boxshrink paper. # -############################################################################################################################# - - -def get_value_of_tag(filepath, tag): - """Check for file if condition is fullfilled - - Args: - filepath (string): path to file holding annotation data - tag (string)): tag to look for - condition (string, integer, float): Condition the tag should fullfill - - Returns: - [string or None]: None if condition is not fullfilled, string if condition is met - """ - # Reading the data inside the xml - # file to a variable under the name - # data - with open(filepath, "r") as f: - annotation_data = f.read() - - soup = BeautifulSoup(annotation_data, features="lxml") - return str(soup.find(tag).string) - - -# Function to get only segmented, only unsegmented or all images -> Create dataset -def create_dataset(xml_files, tag, condition): - """Creates a dataset based on a tag and a condition of that tag. Can be used to train on subsets of the - PascalVOC dataset. - - Args: - xml_files (list): list of path to files holding annotation data - tag (string): tag to look for - condition (string, integer, float): condition the tag should fullfill - - Returns: - [list]: dataset holding the image paths - """ - # create imageset - return ( - [ - xml_files[file] - for file in tqdm(range(len(xml_files)), desc="Creating dataset") - if get_value_of_tag(xml_files[file], tag) == str(condition) - ], - ) - - -# Function to get image & segmentation sets -def get_image_and_segmentation_sets(dataset, image_path, segmentation_path): - """Takes as input path to xml annotation files. Will return two lists with path to - Images and segmentation files. - - Args: - dataset (list): List holding path information to xml annotations - image_path (list): List holding path information to image data - segmentation_path (list): List holding path information to segmentation data - - Returns: - image_set: List with image paths - segmentation_set: List with segmentation paths - """ - image_set = [] - segmentation_set = [] - for index in tqdm(range(len(dataset)), desc="Building image and segmentation sets"): - annotation = dataset[index][0] - image_name = annotation.split("/")[-1].replace("xml", "jpg") - segmentation_name = annotation.split("/")[-1].replace("xml", "png") - image_set.append(image_path + "/" + image_name) - segmentation_set.append(segmentation_path + "/" + segmentation_name) - return image_set, segmentation_set - - -# Function to create validation, train, test splits -def train_test_val_split( - image_set, - segmentation_set, - test_size=0.20, - validation_size=0.10, - seed=42, - clean_validation=False, - clean_segmentation_path=None, -): - """Create train, test, validation splits of initial datasets. - - Args: - image_set (list): List holding path information to images - segmentation_set (list): List holding path information to segmentations - test_size (Float): Float between 0.0 - 1.0 to determine test data size - validation_size (Float): Float between 0.0 - 1.0 to determine the validation dataset size - seed (int, optional): Set seed for sampling. Defaults to 42. - clean_validation (bool, optional): Whether to use clean human made segmentation or pseudolabels. Defaults to False. - clean_segmentation_path (String, optional): If you wish to use human labels then provide path. Defaults to None. - - Returns: - [type]: [description] - """ - # Turn percentage into concrete number of samples to draw - random.seed(seed) - validation_size = round(len(image_set) * validation_size) - validation_indexes = random.sample(range(len(image_set)), validation_size) - X_val = [image_set[i] for i in validation_indexes] - y_val = [segmentation_set[i] for i in validation_indexes] - - # Drop images and segmentations separated for validation - image_set = [image for image in image_set if image not in X_val] - segmentation_set = [ - segmentation for segmentation in segmentation_set if segmentation not in y_val - ] - # Rebuild path to clean annotations - if clean_validation == True: - y_val = [ - clean_segmentation_path - + "/" - + re.split("/", segmentation)[-1].replace("jpg", "png") - for segmentation in y_val - ] - X_train, X_test, y_train, y_test = train_test_split( - image_set, segmentation_set, test_size=test_size, random_state=seed - ) - return (X_train, y_train, X_test, y_test, X_val, y_val) - - -def export_dataset(dataset, output_path): - """Export created dataset. Will be data: data: [path_to_img_1, ..., path_to_img_n] - - Args: - dataset (list): list of paths - output_path (string) - """ - data = {"data": dataset} - df = pd.DataFrame(data) - df.to_csv(output_path, index=False) - - -def import_dataset(input_path): - """Import dataset create with export dataset function. - - Args: - input_path (string) - - Returns: - List - """ - df = pd.read_csv(input_path) - return df.values.tolist() - - -def check_box_coordinates(box_coordinate): - """Catch case where box coordinate is of type float in PascalVOC xml annotation file and convert to int. - - Args: - box_coordinate (string): Box coordinate field vale from xml file. - - Returns: - int: Box coordinate as integer - """ - if "." in box_coordinate: - return int(float(box_coordinate)) - else: - return int(box_coordinate) - - -def get_bounding_boxes(xml_file: str): - """Returns bounding box information from PascalVOC xml annotation file - - Args: - xml_file (str): path to file holding annotation information - - Returns: - list: List with bounding box information - """ - tree = ET.parse(xml_file) - root = tree.getroot() - - list_with_all_boxes = [] - - for boxes in root.iter("object"): - - filename = root.find("filename").text - - ymin, xmin, ymax, xmax = None, None, None, None - classname = str(boxes.find("name").text) - ymin = check_box_coordinates(boxes.find("bndbox/ymin").text) - xmin = check_box_coordinates(boxes.find("bndbox/xmin").text) - ymax = check_box_coordinates(boxes.find("bndbox/ymax").text) - xmax = check_box_coordinates(boxes.find("bndbox/xmax").text) - list_with_single_boxes = [xmin, ymin, xmax, ymax] - list_with_all_boxes.append({classname: list_with_single_boxes}) - - return filename, list_with_all_boxes - - -def create_class_color_code_csv( - dataset, output_path=None, background=False, return_dataframe=False -): - """Creates a csv that has all unique categories in the given dataset and - generates color codes for each of the categories based on random numbers. - - Args: - dataset (list): list holding path information to annotation files - output_path (string): path to save the csv to, please also provide the name of the csv e.g. path/to/csv.csv - background (boolean): whether to include a background class or not - return_dataframe (boolean): whether to return a dataframe or not - """ - # Get categories - categories = [] - for file in tqdm(range(len(dataset)), desc="Collecting Categories"): - category = get_value_of_tag(dataset[file], "name") - if category not in categories: - categories.append(category) - # Generate color codes - # set seed to ensure reproducibility - np.random.seed(0) - red = [int(np.random.randint(0, 255, 1)) for category in categories] - green = [int(np.random.randint(0, 255, 1)) for category in categories] - blue = [int(np.random.randint(0, 255, 1)) for category in categories] - # Create class indexes - indexes = [index for index in range(len(categories))] - # Check whether to add background class or not - if background == True: - categories.insert(0, "background") - red.insert(0, 0) - green.insert(0, 0) - blue.insert(0, 0) - indexes = [index + 1 for index in indexes] - indexes.insert(0, 0) - # Build dataframe - data = { - "index": indexes, - "category": categories, - "red": red, - "green": green, - "blue": blue, - } - df = pd.DataFrame(data) - if return_dataframe == False: - df.to_csv(output_path, index=False) - else: - return df - - -# Function to generate masks from bounding boxes -def generate_mask_from_box( - dataset, - category_data, - output_path, -): - """Generate weak segmentation masks from dataset. - - Args: - dataset (List): holding paths to images - category_data (str): Path to file holding class to color mapping - output_path (str): where to save the masks - """ - category_data = pd.read_csv(category_data) - for element in tqdm(range(len(dataset)), desc="Creating Masks"): - image_width, image_height = ( - int(get_value_of_tag(dataset[element][0], "width")), - int(get_value_of_tag(dataset[element][0], "height")), - ) - channels = 3 - # Create mask - mask = np.zeros([image_height, image_width, channels], dtype=np.uint8) - # Get all boxes present on the image - boxes = get_bounding_boxes(dataset[element][0]) - for box in boxes[1]: - # Get category - category = list(box.keys())[0] - # Get color code - red, green, blue = ( - np.uint8(category_data[category_data["category"] == category]["red"]), - np.uint8(category_data[category_data["category"] == category]["green"]), - np.uint8(category_data[category_data["category"] == category]["blue"]), - ) - # get coordinates - ymin, xmin, ymax, xmax = ( - box[category][0], - box[category][1], - box[category][2], - box[category][3], - ) - if ymax != image_height: - ymax = ymax + 1 - if xmax != image_width: - xmax = xmax + 1 - mask[xmin:xmax, ymin:ymax] = [red[0], green[0], blue[0]] - mask = Image.fromarray(mask, mode="RGB") - output_path_mask = ( - output_path + "/" + get_value_of_tag(dataset[element][0], "filename") - ).replace("jpg", "png") - mask.save(output_path_mask, quality=100, subsampling=0) - - -def visualize_mask(image_path, path_to_masks): - """Will return an overlay of the image and its mask - - Args: - image_path (string): path to image you want to visualize - path_to_masks (string): path to masks - """ - # Ground truth image - background = Image.open(image_path) - # Generate path to mask - image_path = re.split(r"/", image_path) - image_name = image_path[-1] - mask_name = image_name.replace("jpg", "png") - mask_path = path_to_masks + "/" + mask_name - # load mask - overlay = Image.open(mask_path) - # Ensure same encoding - background = background.convert("RGBA") - overlay = overlay.convert("RGBA") - # Create new image from overlap and make overlay 50 % transparent - new_img = Image.blend(background, overlay, 0.5) - new_img.show() - - -def resize_images_in_folder( - path_to_folder, path_to_output_folder, new_x_res, new_y_res -): - """ - - Args: - path_to_folder (string): Folder holding original images - path_to_output_folder (string): where to save resized images - new_x_res (int): New max x resolution - new_y_res (int): New max y resolution - """ - files = [f for f in os.listdir(path_to_folder) if isfile(join(path_to_folder, f))] - for file in tqdm(range(len(files)), desc="Rescaling images: "): - # open image - img = Image.open(join(path_to_folder, files[file])) - # resize image - img = img.resize((new_x_res, new_y_res), resample=Image.NEAREST) - # save image in outputpath - img.save(join(path_to_output_folder, files[file]), quality=100, subsampling=0) - - -def get_smallest_imagesize_in_folder(path_to_folder): - """Returns smallest side of images in folder. - - Args: - path_to_folder (string): path to folder - - Returns: - int: shortest side of images in folder - """ - files = [f for f in os.listdir(path_to_folder) if isfile(join(path_to_folder, f))] - sizes = np.array([]) - for file in tqdm(range(len(files)), desc="Getting smallest side: "): - # open image - size_image = Image.open(join(path_to_folder, files[file])).size - # Convert image size to numpy array - sizes = np.append(sizes, size_image[0], size_image[1]) - # Return smallest value - return int(sizes.min()) - - -# All delivered segmentation masks have a white boundary which one might want to eliminate -# the color code is [224, 224, 192] -def drop_color_in_image( - path, - output_path, - color_code=[224, 224, 192], -): - """sets pixels of a certain value to zero - - Args: - path (string): path to image - output_path (string): where to save altered image - color_code (list, optional): RGB value to be set to zero. Defaults to [224, 224, 192]. - """ - # open image - img = Image.open(path) - # Convert to RGB - img = img.convert("RGB") - # Convert to numpy - img_np = np.array(img) - # Change color code to black - img_np[img_np[:, :, :] == color_code] = 0 - # Generate image name - img_name = re.split("/", path)[-1] - # Export image - new_img = Image.fromarray(img_np) - path = os.path.join(output_path, img_name) - new_img.save(path, quality=100, subsampling=0) - - -def export_visualize(path, **images): - """Plots image e.g. img, img+ground truth, img+prediction in one line - and exports it. - - Args: - path (string): where to save the image - """ - n = len(images) - plt.figure(figsize=(16, 5)) - for i, (name, image) in enumerate(images.items()): - plt.subplot(1, n, i + 1) - plt.xticks([]) - plt.yticks([]) - plt.title(" ".join(name.split("_")).title()) - plt.imshow(image) - plot = plt.gcf() - plot.savefig(path) - - -def IoU(image, gt, model, eps=0.0000000001, mode="mean"): - """Calculation of IoU metric. The function checks if we have a batchsize of 1 - then the IoU of the single image with the ground truth mask is returned. If the batchsize is - bigger then the IoU over the batch is calculated. - - Args: - gt (tensor): ground truth class mask - pred (tensor): prediction made by the model. - eps (float, optional): Epsilon to avoid zero division error. Defaults to 0.0000000001. - mode (string, optional): Whether to return the entire tensor or the mean. Defaults to mean. Can be 'mean' - or 'tensor' - - Returns: - [float]: IoU score - """ - batchsize = gt.shape[0] - ious = [] - with torch.no_grad(): - if gt.shape[0] > 1: - pass - # Unfold the images and calculate prediction for each of them - for i in range(batchsize): - current_gt = gt[i, :, :] - current_pred = torch.argmax( - model(image[i, :, :].unsqueeze(0)), dim=1, keepdim=True - )[:, -1, :, :] - intersection = torch.logical_and(current_gt, current_pred) + eps - union = torch.logical_or(current_gt, current_pred) + eps - iou_score = torch.sum(intersection) / torch.sum(union) - ious.append(iou_score.item()) - if mode == "mean": - return torch.tensor(ious, dtype=torch.float64).mean() - if mode == "tensor": - return torch.tensor(ious, dtype=torch.float64) - else: - return None - else: - # Calculate IoU for batchsize = 1 - pred = torch.argmax(model(image), dim=1, keepdim=True)[:, -1, :, :] - intersection = torch.logical_and(gt, pred) + eps - union = torch.logical_or(gt, pred) + eps - iou_score = torch.sum(intersection) / torch.sum(union) - return iou_score - - -# What directories to expect -def copy_images_to_directories(root, directories, datasets, select_subset_of_images): - """Copy images from datasets into new directory to keep the original data separated from any - data with which experiments were done. Also to separate images into the representative dataset folder. - - Args: - root (string): Root directory where the new datasets folders should be created in - directories (list): List of names for the new directories. e.g. ["X_train", "y_train", ...] - datasets ([type]): [description] - select_subset_of_images ([type]): [description] - """ - select_subset_of_images = select_subset_of_images - for index in range(len(directories)): - # Build new path where to copy the images to - new_path = root + "/" + directories[index] - # If the new path has already images from earlier experiments, delete those - if os.path.exists(new_path): - shutil.rmtree(new_path) - # create folder to save new copied images - os.mkdir(new_path) - # go through each dataset - dataset = datasets[index] - # Scenario where we only want some images to be copied and not the entire dataset - if select_subset_of_images == 0: - for i in tqdm( - range(len(datasets[index])), desc=f"Copy images to {directories[index]}" - ): - new_image_path = new_path + "/" + dataset[i].split("/")[-1] - shutil.copy(dataset[i], new_image_path) - else: - # Scenario where we want to work with the entire dataset - for i in tqdm( - range(select_subset_of_images), - desc=f"Copy images to {directories[index]}", - ): - new_image_path = new_path + "/" + dataset[i].split("/")[-1] - shutil.copy(dataset[i], new_image_path)