Prepare challenge (#8)

* challenge evaluation script

* challenge set zip

* updates

* adding attributes

* model small

* formatting

.gitattributes

@@ -0,0 +1,2 @@
+dataset/challenge_set.zip filter=lfs diff=lfs merge=lfs -text
+logs/sviewds_public_baseline/model_best.pkl filter=lfs diff=lfs merge=lfs -text

README.md

@@ -24,13 +24,15 @@ This repo is based on the [Pytorch Project Template](https://github.com/L1aoXing
     - [Training the segmentation model](#training-the-segmentation-model)
     - [Evaluate the camera calibration model](#evaluate-the-camera-calibration-model)
 
+  - [CHALLENGE set](#challeng-eset)
+
   - [Submission format](#submission-format)
 
 - [Acknowledgments](#acknowledgments)
 
 # In a Nutshell
 
-The purpose of this challenge is to predict the camera calibration parameters from a single frame of a basketball game. Participants have access to a dataset of 728 pairs of images and camera calibration parameters. By default these pairs are divided in train (548), val (96) and test (84) splits. Note that this test split is different from the one on which the challenge participants will be evaluated on. Therefore, all the 728 examples can be used for the training purpose.
+The purpose of this challenge is to predict the camera calibration parameters from a single frame of a basketball game. Participants have access to a dataset of 728 pairs of images and camera calibration parameters. By default these pairs are divided in train (480), val (164) and test (84) splits. Note that this test split is different from the one on which the challenge participants will be evaluated on. Therefore, all the 728 examples can be used for the training purpose.
 
 Participants are encouraged to explore different methods to predict the camera calibration parameters. However, a baseline will be provided as described in the [In Details](#in-details) section.
 
@@ -146,6 +148,31 @@ You can now submit the `predictions.json` on EvalAI for the `Test` phase and ver
 
 When the challenge set will be released, you will need to set `DATASETS.RUN_METRICS` as `False` and generate the prediction only.
 
+# CHALLENGE set
+
+This section explains how to generate the `predictions.json` file for the CHALLENGE set.
+
+First of all, unzip the file in `dataset/challenge_set.zip` as:
+
+```bash
+unzip dataset/challenge_set.zip -d .
+```
+
+You now have the images in the CHALLENGE folder. For convenience, the images have been generated of size `[960, 540]` (`INPUT.MULTIPLICATIVE_FACTOR: 2`). The relative evaluation script will consider this resolution.
+
+To run the inference on these images you will need to modify your config file as:
+
+- `DATASETS.TEST: "challenge"`
+- `DATASETS.RUN_METRICS: False`
+
+The config file `configs/eval_challenge.yml` is provided as an example. Then run:
+
+```python
+python tools/evaluate_net.py --config_file configs/eval_challenge.yml
+```
+
+This will create the `predictions.json` file needed to be updated in [EvalAI](https://eval.ai/web/challenges/challenge-page/1687/overview).
+
 ## Submission format
 
 The submission format is a single `json` file containing a list of dicts. Each dict should contain all the camera parameters `T`, `K`, `kc`, `R`, `C`, `P`, `Pinv`, `Kinv`. Note that the evaluation script retrieves the camera parameters from the Projection Matrix `P`. See the class [calib3d.Calib](https://github.com/ispgroupucl/calib3d/blob/b20694a42a3e043b157dcd9b363833184cc3fcdc/calib3d/calib.py#L155). Please consider that the evaluation script follows the list of images provided: an empty dict will be replaced by a diagonal homography (see `run_metrics` in `engine/example_evaluation.py`).

configs/eval_challenge.yml

@@ -0,0 +1,39 @@
+MODEL:
+  ARCHITECTURE: "DeepLabv3"
+  NUM_CLASSES: 21
+  LOSS: "cross_entropy"
+  SEGMENTATION_LOSS: True
+  LOSS_FUNCTION: "loss_fn_seg"
+  LOSS_WEIGHT_BACKGROUND: 0.5
+
+INPUT:
+  PIXEL_MEAN: [0.485, 0.456, 0.406]
+  PIXEL_STD: [0.229, 0.224, 0.225]
+  GENERATED_VIEW_SIZE: [480, 270]
+  MULTIPLICATIVE_FACTOR: 2
+  TRANSFORMS: False
+
+SOLVER:
+  OPTIMIZER_NAME: "AdamW"
+  AMSGRAD: True
+  BASE_LR: 0.001
+  WEIGHT_DECAY: 0.0001
+  IMS_PER_BATCH: 14
+  WARMUP_ITERS: 3
+  STEPS: (40000,)
+
+TEST:
+  IMS_PER_BATCH: 1
+  WEIGHT: "logs/sviewds_public_baseline/model_best.pkl"
+
+DATASETS:
+  TRAIN: "sviewds"
+  TEST: "challenge"
+  EVALUATION: True
+  EVAL_ON: "test"
+  RUN_METRICS: False
+
+DATALOADER:
+  NUM_WORKERS: 8
+
+OUTPUT_DIR: "logs/sviewds_public_baseline"

configs/eval_sviewds_public_baseline.yml

@@ -24,7 +24,7 @@ SOLVER:
 
 TEST:
   IMS_PER_BATCH: 1
-  WEIGHT: "logs/sviewds_public_baseline/model_checkpoint_XXX.pt"
+  WEIGHT: "logs/sviewds_public_baseline/model_best.pkl"
 
 DATASETS:
   TRAIN: "sviewds"

configs/train_sviewds_public_baseline.yml

@@ -10,7 +10,7 @@ INPUT:
   PIXEL_MEAN: [0.485, 0.456, 0.406]
   PIXEL_STD: [0.229, 0.224, 0.225]
   GENERATED_VIEW_SIZE: [480, 270]
-  MULTIPLICATIVE_FACTOR: 2
+  MULTIPLICATIVE_FACTOR: 1
   TRANSFORMS: False
 
 SOLVER:
@@ -24,7 +24,7 @@ SOLVER:
 
 TEST:
   IMS_PER_BATCH: 5
-  WEIGHT: "logs/sviewds_public_baseline/model_checkpoint_XXX.pt"
+  WEIGHT: "logs/sviewds_public_baseline/model_best.pkl"
 
 DATASETS:
   TRAIN: "sviewds"

data/build.py

@@ -17,11 +17,16 @@
 from yacs.config import CfgNode
 
 from .datasets.mnist import MNIST
-from .datasets.viewds import VIEWDS
+from .datasets.viewds import VIEWDS, CHALLENGE
 from .datasets.viewds import SVIEWDS, GenerateSViewDS
 from .transforms import build_transforms
 
-DATASETS = {"mnist": MNIST, "viewds": VIEWDS, "sviewds": SVIEWDS}
+DATASETS = {
+    "mnist": MNIST,
+    "viewds": VIEWDS,
+    "sviewds": SVIEWDS,
+    "challenge": CHALLENGE,
+}
 
 
 def build_dataset(
@@ -44,6 +49,8 @@ def build_dataset(
         "transform": transforms,
         "download": False,
     }
+    if cfg.DATASETS.TEST == "challenge":
+        return CHALLENGE("CHALLENGE", transform=transforms)
     if cfg.DATASETS.TRAIN == "sviewds":
         width, height = (
             cfg.INPUT.MULTIPLICATIVE_FACTOR * cfg.INPUT.GENERATED_VIEW_SIZE[0],

data/datasets/__init__.py

@@ -3,4 +3,3 @@
 @author:  sherlock
 @contact: [email protected]
 """
-

data/datasets/mnist.py

@@ -5,4 +5,3 @@
 """
 
 from torchvision.datasets import MNIST
-

data/datasets/viewds.py

@@ -136,6 +136,40 @@ def __getitem__(self, index):
         return img, y
 
 
+class CHALLENGE(torch.utils.data.Dataset):
+    """The Challenge dataset that returns transformed images."""
+
+    def __init__(
+        self,
+        root: str,
+        transform: Optional[Callable] = None,
+    ) -> None:
+        """_summary_
+
+        Args:
+            path (_type_): _description_
+        """
+        self.list_IDs = os.listdir(root)
+        self.path = root
+        self.transform = transform
+
+    def __len__(self):
+        "Denotes the total number of samples"
+        return len(self.list_IDs)
+
+    def __getitem__(self, index):
+        "Generates one sample of data"
+        # Select sample
+        fname = self.list_IDs[index]
+
+        # Load data
+        img = Image.open(os.path.join(self.path, f"{fname}"))
+        if self.transform is not None:
+            img = self.transform(img)
+
+        return img, []
+
+
 class GenerateSViewDS:
     def __init__(
         self,

data/transforms/build.py

@@ -23,9 +23,7 @@ def build_transforms(cfg, is_train=True):
         mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD
     )
     if cfg.INPUT.TRANSFORMS == False:
-        transform = T.Compose(
-            [T.ToTensor(), normalize_transform]
-        )
+        transform = T.Compose([T.ToTensor(), normalize_transform])
         return transform
     if is_train:
         transform = T.Compose(

data/transforms/transforms.py

@@ -9,7 +9,7 @@
 
 
 class RandomErasing(object):
-    """ Randomly selects a rectangle region in an image and erases its pixels.
+    """Randomly selects a rectangle region in an image and erases its pixels.
         'Random Erasing Data Augmentation' by Zhong et al.
         See https://arxiv.org/pdf/1708.04896.pdf
     Args:
@@ -20,7 +20,14 @@ class RandomErasing(object):
          mean: Erasing value.
     """
 
-    def __init__(self, probability=0.5, sl=0.02, sh=0.4, r1=0.3, mean=(0.4914, 0.4822, 0.4465)):
+    def __init__(
+        self,
+        probability=0.5,
+        sl=0.02,
+        sh=0.4,
+        r1=0.3,
+        mean=(0.4914, 0.4822, 0.4465),
+    ):
         self.probability = probability
         self.mean = mean
         self.sl = sl
@@ -45,11 +52,11 @@ def __call__(self, img):
                 x1 = random.randint(0, img.size()[1] - h)
                 y1 = random.randint(0, img.size()[2] - w)
                 if img.size()[0] == 3:
-                    img[0, x1:x1 + h, y1:y1 + w] = self.mean[0]
-                    img[1, x1:x1 + h, y1:y1 + w] = self.mean[1]
-                    img[2, x1:x1 + h, y1:y1 + w] = self.mean[2]
+                    img[0, x1 : x1 + h, y1 : y1 + w] = self.mean[0]
+                    img[1, x1 : x1 + h, y1 : y1 + w] = self.mean[1]
+                    img[2, x1 : x1 + h, y1 : y1 + w] = self.mean[2]
                 else:
-                    img[0, x1:x1 + h, y1:y1 + w] = self.mean[0]
+                    img[0, x1 : x1 + h, y1 : y1 + w] = self.mean[0]
                 return img
 
         return img

engine/example_evaluation.py

@@ -78,8 +78,10 @@ def run_metrics(
 
     accuracy = 0
     mse_ = []
+    len_set = 0
 
     for dat, gt in zip(obj, obj_gt[1:]):
+        len_set += 1
         predicted_P = dat.get("P", None)
         if not predicted_P:
             calib = Calib.from_P(
@@ -104,6 +106,7 @@ def run_metrics(
         y = calib_gt.project_2D_to_3D(test_2d_ponts, Z=0)
 
         mse_.append(np.sqrt(np.square(np.subtract(y_pred, y)).mean()))
+    accuracy /= len_set
     print(f"Accuracy: {accuracy}, discounted MSE: {np.mean(mse_)} cm")
 
 
@@ -135,6 +138,7 @@ def __init__(self, cfg) -> None:
         )
         self.test_2d_ponts = TEST_2D_POINTS * [[self.width], [self.height]]
         self.dumpable_list = []
+        self.is_challenge = cfg.DATASETS.TEST == "challenge"
 
     def __call__(self, x, y, y_pred):
         points2d, points3d = find_intersections(
@@ -145,11 +149,6 @@ def __call__(self, x, y, y_pred):
         calib = compute_camera_model(
             points2d, points3d, (self.height, self.width)
         )
-        calib_gt = Calib.from_P(
-            np.squeeze(y["calib"].cpu().numpy().astype(np.float32)),
-            width=self.width,
-            height=self.height,
-        )
         data = {
             "numper of points2d": len(points2d),
         }
@@ -158,6 +157,13 @@ def __call__(self, x, y, y_pred):
         self.dumpable_list.append(data)
 
         y_pred = calib.project_2D_to_3D(self.test_2d_ponts, Z=0)
+        if self.is_challenge:
+            return torch.as_tensor(y_pred)
+        calib_gt = Calib.from_P(
+            np.squeeze(y["calib"].cpu().numpy().astype(np.float32)),
+            width=self.width,
+            height=self.height,
+        )
         y = calib_gt.project_2D_to_3D(self.test_2d_ponts, Z=0)
 
         return (torch.as_tensor(y_pred), torch.as_tensor(y))
@@ -177,19 +183,23 @@ def evaluation(cfg, model, val_loader):
     logger.info("Start evaluation")
     cm = confusion_matrix.ConfusionMatrix(num_classes=21)
     camera_transform = CameraTransform(cfg)
+    metrics = {"mse": MeanAbsoluteError()}
+    if cfg.DATASETS.TEST == "challenge":
+        metrics = None
 
     evaluator = create_supervised_evaluator(
         model,
-        metrics={"mse": MeanAbsoluteError()},
+        metrics=metrics,
         device=device,
         output_transform=camera_transform,
     )
 
     # adding handlers using `evaluator.on` decorator API
+
     @evaluator.on(Events.EPOCH_COMPLETED)
     def print_validation_results(engine):
         metrics = evaluator.state.metrics
-        mse = metrics["mse"]
+        mse = metrics["mse"] if cfg.DATASETS.TEST == "sviewds" else 0.0
         logger.info(
             "Camera Evaluation Overall Results - MSE: {:.3f}".format(mse)
         )