eval_metrics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4f600eb3",
   "metadata": {},
   "source": [
    "# Evaluation Metrics\n",
    "\n",
    "In this notebook, we run following evaluation metrics as described in our paper: \n",
    "1. LPIPS (Learned Perceptual Image Patch Similarity, implementation from https://github.com/richzhang/PerceptualSimilarity)\n",
    "2. MSE (Mean Squared Error)\n",
    "3. LMSE (Local Mean Squared Error, implementation from https://github.com/davidstutz/grosse2009-intrinsic-images)\n",
    "4. FID50k (Frechet Inception Distance, implementation from https://github.com/GaParmar/clean-fid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1decb07c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import lpips\n",
    "import os\n",
    "from PIL import Image \n",
    "import torch \n",
    "from rewrite_utils import renormalize, show\n",
    "import glob \n",
    "from cleanfid import fid\n",
    "import numpy as np\n",
    "\n",
    "def pilim(img): \n",
    "    return renormalize.from_image(img)\n",
    "\n",
    "lpips_metric = lpips.LPIPS(net='alex')\n",
    "mse_metric = torch.nn.MSELoss()\n",
    "mask = np.ones(truth.shape) #mask for LMSE\n",
    "\n",
    "#local mean squared error from Intrinsic Image Algorithms (repo linked above)\n",
    "def local_error(correct, estimate, mask, window_size, window_shift):\n",
    "    \"\"\"Returns the sum of the local sum-squared-errors, where the estimate may\n",
    "    be rescaled within each local region to minimize the error. The windows are\n",
    "    window_size x window_size, and they are spaced by window_shift.\"\"\"\n",
    "    M, N = correct.shape[1:]\n",
    "    ssq = total = 0.\n",
    "    for i in range(0, M - window_size + 1, window_shift):\n",
    "        for j in range(0, N - window_size + 1, window_shift):\n",
    "            correct_curr = correct[:, i:i+window_size, j:j+window_size]\n",
    "            estimate_curr = estimate[:, i:i+window_size, j:j+window_size]\n",
    "            mask_curr = mask[:, i:i+window_size, j:j+window_size]\n",
    "            ssq += ssq_error(correct_curr, estimate_curr, mask_curr)\n",
    "        total += np.sum(mask_curr * correct_curr**2)\n",
    "    assert ~np.isnan(ssq/total)\n",
    "    return ssq / total\n",
    "\n",
    "def ssq_error(correct, estimate, mask):\n",
    "    \"\"\"Compute the sum-squared-error for an image, where the estimate is\n",
    "    multiplied by a scalar which minimizes the error. Sums over all pixels\n",
    "    where mask is True. If the inputs are color, each color channel can be\n",
    "    rescaled independently.\"\"\"\n",
    "    assert correct.ndim == 3\n",
    "    if np.sum(estimate**2 * mask) > 1e-5:\n",
    "        alpha = np.sum(correct * estimate * mask) / np.sum(estimate**2 * mask)\n",
    "    else:\n",
    "        alpha = 0.\n",
    "    return np.sum(mask * (correct - alpha*estimate) ** 2)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b78c66d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_path = \"evaldata/ada_lonoff\" #path to folder of generated images\n",
    "truth_path = \"datasets/lonoff_light_all/test_B/\" #path to folder of ground truth (LONOFF dataset images)\n",
    "\n",
    "lpips_all = []\n",
    "mse_all = []\n",
    "lmse_all = []\n",
    "\n",
    "#iterate through each image in LONOFF\n",
    "i = 0\n",
    "for img_path in os.listdir(truth_path): \n",
    "    i+=1\n",
    "    print(f'{i} {img_path}')\n",
    "    \n",
    "    #reshape all images to be 256 x 256 pixels\n",
    "    w, h = (256, 256)\n",
    "    truth = pilim(Image.open(os.path.join(truth_path, img_path)).resize((h, w), Image.BILINEAR))\n",
    "    \n",
    "    \n",
    "    lpips_temp = []\n",
    "    mse_temp = []\n",
    "    lmse_temp = []\n",
    "    \n",
    "    #compute metrics in comparison to each of the 3 test images relit to different intensities\n",
    "    img_path = img_path[4:-4]\n",
    "    for path in glob.glob(os.path.join(test_path, f'{img_path}_output_image*')): \n",
    "        test = pilim(Image.open(os.path.join(path)).resize((h, w), Image.BILINEAR))\n",
    "        lpips_temp.append(torch.squeeze(lpips_metric(test, truth)).item())\n",
    "        mse_temp.append(torch.squeeze(mse_metric(test, truth)).item())\n",
    "        lmse_temp.append(local_error(truth.numpy(), test.numpy(), mask, 20, 10))\n",
    "    \n",
    "    #use the best out of three relit versions to calculate the final metric\n",
    "    lpips_all.append(min(lpips_temp))    \n",
    "    mse_all.append(min(mse_temp))   \n",
    "    lmse_all.append(min(lmse_temp))\n",
    "     \n",
    "\n",
    "lpips_avg = sum(lpips_all)/len(lpips_all)\n",
    "mse_avg = sum(mse_all)/len(mse_all)\n",
    "lmse_avg = sum(lmse_all)/len(lmse_all)\n",
    "\n",
    "print(f'lpips: {lpips_avg}')\n",
    "print(f'mse: {mse_avg}')\n",
    "print(f'lmse: {lmse_avg}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb304778",
   "metadata": {},
   "outputs": [],
   "source": [
    "#compute FID50k \n",
    "\n",
    "test_folder = 'evaldata/e4e_50k' #path to folder of 50k generated images\n",
    "truth_folder = 'evaldata/real_bedrooms_50k'#path to folder of 50k real images of bedrooms \n",
    "fid_score = fid.compute_fid(test_folder,  truth_folder, mode='clean')\n",
    "print(f'FID50k: {fid_score}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}