Crop Mask Integration

benchmarks/deep_learning.py

@@ -4,7 +4,7 @@
 import json
 
 from cropharvest.datasets import CropHarvest
-from cropharvest.utils import DATAFOLDER_PATH
+from cropharvest.config import DATAFOLDER_PATH
 from cropharvest.engineer import TestInstance
 
 from config import (

benchmarks/dl/maml.py

@@ -3,7 +3,6 @@
 import dill
 import warnings
 from random import shuffle, random
-from collections import defaultdict
 
 import torch
 from torch import nn
@@ -19,10 +18,10 @@
 
 from cropharvest.datasets import CropHarvest, CropHarvestLabels, Task
 from cropharvest import countries
-from cropharvest.config import TEST_DATASETS, TEST_REGIONS
+from cropharvest.config import TEST_COUNTRIES_TO_CROPS
 from cropharvest.utils import NoDataForBoundingBoxError
 
-from typing import Dict, Tuple, Optional, List, DefaultDict
+from typing import Dict, Tuple, Optional, List
 
 
 class TrainDataLoader:
@@ -363,18 +362,11 @@ def _make_tasks(
     ) -> Tuple[Dict[str, CropHarvest], Dict[str, CropHarvest]]:
         labels = CropHarvestLabels(self.root)
 
-        # remove any test regions, and collect the countries / crops
-        test_countries_to_crops: DefaultDict[str, List[str]] = defaultdict(list)
-
-        # reshuffle the test_regions dict so its a little easier to
-        # manipulate in this function
-        for identifier, _ in TEST_REGIONS.items():
-            country, crop, _, _ = identifier.split("_")
-            test_countries_to_crops[country].append(crop)
-
         label_to_task: Dict[str, CropHarvest] = {}
 
-        countries_to_ignore = [country for country, _ in TEST_DATASETS.items() if crop is None]
+        countries_to_ignore = [
+            country for country, crop in TEST_COUNTRIES_TO_CROPS.items() if crop is not None
+        ]
 
         for country in tqdm(countries.get_countries()):
             if country in countries_to_ignore:
@@ -394,8 +386,8 @@ def _make_tasks(
                     label_to_task[task.id] = task
 
                 for label in labels.classes_in_bbox(country_bbox):
-                    if country in test_countries_to_crops:
-                        if label in test_countries_to_crops[country]:
+                    if country in TEST_COUNTRIES_TO_CROPS:
+                        if label in TEST_COUNTRIES_TO_CROPS[country]:
                             continue
                     try:
                         task = CropHarvest(

benchmarks/random_forest.py

@@ -3,7 +3,7 @@
 from sklearn.ensemble import RandomForestClassifier
 
 from cropharvest.datasets import CropHarvest
-from cropharvest.utils import DATAFOLDER_PATH
+from cropharvest.config import DATAFOLDER_PATH
 from cropharvest.engineer import TestInstance
 
 from config import SHUFFLE_SEEDS, DATASET_TO_SIZES, RANDOM_FOREST

cropharvest/boundingbox.py

@@ -0,0 +1,77 @@
+from dataclasses import dataclass
+from pathlib import Path
+from shapely.geometry import Polygon
+from math import sin, cos, radians
+from typing import List, Tuple
+import re
+
+from typing import Optional
+
+
+@dataclass
+class BBox:
+
+    min_lat: float
+    max_lat: float
+    min_lon: float
+    max_lon: float
+
+    name: Optional[str] = None
+
+    def __post_init__(self):
+        if self.max_lon < self.min_lon:
+            raise ValueError("max_lon should be larger than min_lon")
+        if self.max_lat < self.min_lat:
+            raise ValueError("max_lat should be larger than min_lat")
+
+        self.url = (
+            f"http://bboxfinder.com/#{self.min_lat},{self.min_lon},{self.max_lat},{self.max_lon}"
+        )
+
+    def contains(self, lat: float, lon: float) -> bool:
+        return (
+            (lat >= self.min_lat)
+            & (lat <= self.max_lat)
+            & (lon >= self.min_lon)
+            & (lon <= self.max_lon)
+        )
+
+    def contains_bbox(self, bbox: "BBox") -> bool:
+        return (
+            (bbox.min_lat >= self.min_lat)
+            & (bbox.max_lat <= self.max_lat)
+            & (bbox.min_lon >= self.min_lon)
+            & (bbox.max_lon <= self.max_lon)
+        )
+
+    @property
+    def three_dimensional_points(self) -> List[float]:
+        r"""
+        If we are passing the central latitude and longitude to
+        an ML model, we want it to know the extremes are close together.
+        Mapping them to 3d space allows us to do that
+        """
+        lat, lon = self.get_centre(in_radians=True)
+        return [cos(lat) * cos(lon), cos(lat) * sin(lon), sin(lat)]
+
+    def get_centre(self, in_radians: bool = True) -> Tuple[float, float]:
+
+        # roughly calculate the centres
+        lat = self.min_lat + ((self.max_lat - self.min_lat) / 2)
+        lon = self.min_lon + ((self.max_lon - self.min_lon) / 2)
+        if in_radians:
+            return radians(lat), radians(lon)
+        else:
+            return lat, lon
+
+    @classmethod
+    def polygon_to_bbox(cls, polygon: Polygon, name: Optional[str] = None):
+        (min_lon, min_lat, max_lon, max_lat) = polygon.bounds
+        return cls(min_lat, max_lat, min_lon, max_lon, name)
+
+    @classmethod
+    def from_eo_tif_file(cls, path: Path) -> "BBox":
+        decimals_in_p = re.findall(r"=-?\d*\.?\d*", path.stem)
+        coords = [float(d[1:]) for d in decimals_in_p[0:4]]
+        bbox = cls(min_lat=coords[0], min_lon=coords[1], max_lat=coords[2], max_lon=coords[3])
+        return bbox

cropharvest/columns.py

@@ -18,7 +18,7 @@ def date_columns(cls) -> List[str]:
 
 class RequiredColumns(Columns):
 
-    INDEX = "index"
+    INDEX = "dataset_index"
     IS_CROP = "is_crop"
     LAT = "lat"
     LON = "lon"
@@ -42,3 +42,15 @@ class NullableColumns(Columns):
     @classmethod
     def date_columns(cls) -> List[str]:
         return [cls.HARVEST_DATE, cls.PLANTING_DATE]
+
+
+class EngColumns:
+    """
+    Some columns uniquely created & used by the labels
+    as loaded by the Engineer
+    """
+
+    FEATURES_FILENAME = "features_filename"
+    FEATURES_PATH = "features_path"
+    EXISTS = "feature_exists"
+    TIF_FILEPATHS = "tif_path"

cropharvest/config.py

@@ -1,4 +1,7 @@
-from .countries import BBox
+from pathlib import Path
+from collections import defaultdict
+
+from .boundingbox import BBox
 
 from typing import Dict
 
@@ -22,6 +25,14 @@
 FEATURES_DIR = "features"
 TEST_FEATURES_DIR = "test_features"
 
+# These values describe the structure of the data folder
+DATAFOLDER_PATH = Path(__file__).parent.parent / "data"
+EO_FILEPATH = DATAFOLDER_PATH / "eo_data"
+TEST_EO_FILEPATH = DATAFOLDER_PATH / "test_eo_data"
+FEATURES_FILEPATH = DATAFOLDER_PATH / FEATURES_DIR
+ARRAYS_FILEPATH = FEATURES_FILEPATH / "arrays"
+TEST_FEATURES_FILEPATH = DATAFOLDER_PATH / TEST_FEATURES_DIR
+
 # the default seed is useful because it also seeds the deterministic
 # shuffling algorithm we use (in cropharvest.utils.deterministic_shuffle)
 # so fixing this ensures the evaluation sets consist of the same data no matter
@@ -47,3 +58,18 @@
 }
 
 TEST_DATASETS = {"Togo": "togo-eval"}
+
+
+def test_countries_to_crops():
+    output_dict = defaultdict(list)
+    for identifier, _ in TEST_REGIONS.items():
+        country, crop, _, _ = identifier.split("_")
+        output_dict[country].append(crop)
+
+    for country, _ in TEST_DATASETS.items():
+        output_dict[country].append(None)
+
+    return output_dict
+
+
+TEST_COUNTRIES_TO_CROPS = test_countries_to_crops()

cropharvest/countries.py

@@ -1,75 +1,12 @@
-from dataclasses import dataclass
 import geopandas
 from shapely.geometry import Polygon, MultiPolygon
-from math import sin, cos, radians
-from typing import List, Tuple
+from typing import List
 from pathlib import Path
 
-from typing import Optional
+from cropharvest.boundingbox import BBox
 
-COUNTRY_SHAPEFILE = geopandas.read_file(str(Path(__file__).parent / "country_shapefile"))
-
-
-@dataclass
-class BBox:
-
-    min_lat: float
-    max_lat: float
-    min_lon: float
-    max_lon: float
-
-    name: Optional[str] = None
-
-    def __post_init__(self):
-        if self.max_lon < self.min_lon:
-            raise ValueError("max_lon should be larger than min_lon")
-        if self.max_lat < self.min_lat:
-            raise ValueError("max_lat should be larger than min_lat")
-
-        self.url = (
-            f"http://bboxfinder.com/#{self.min_lat},{self.min_lon},{self.max_lat},{self.max_lon}"
-        )
 
-    def contains(self, lat: float, lon: float) -> bool:
-        return (
-            (lat >= self.min_lat)
-            & (lat <= self.max_lat)
-            & (lon >= self.min_lon)
-            & (lon <= self.max_lon)
-        )
-
-    def contains_bbox(self, bbox) -> bool:
-        return (
-            (bbox.min_lat >= self.min_lat)
-            & (bbox.max_lat <= self.max_lat)
-            & (bbox.min_lon >= self.min_lon)
-            & (bbox.max_lon <= self.max_lon)
-        )
-
-    @property
-    def three_dimensional_points(self) -> List[float]:
-        r"""
-        If we are passing the central latitude and longitude to
-        an ML model, we want it to know the extremes are close together.
-        Mapping them to 3d space allows us to do that
-        """
-        lat, lon = self.get_centre(in_radians=True)
-        return [cos(lat) * cos(lon), cos(lat) * sin(lon), sin(lat)]
-
-    def get_centre(self, in_radians: bool = True) -> Tuple[float, float]:
-
-        # roughly calculate the centres
-        lat = self.min_lat + ((self.max_lat - self.min_lat) / 2)
-        lon = self.min_lon + ((self.max_lon - self.min_lon) / 2)
-        if in_radians:
-            return radians(lat), radians(lon)
-        else:
-            return lat, lon
-
-    @classmethod
-    def polygon_to_bbox(cls, polygon: Polygon, name: Optional[str] = None):
-        (min_lon, min_lat, max_lon, max_lat) = polygon.bounds
-        return cls(min_lat, max_lat, min_lon, max_lon, name)
+COUNTRY_SHAPEFILE = geopandas.read_file(str(Path(__file__).parent / "country_shapefile"))
 
 
 def get_country_bbox(country_name: str) -> List[BBox]: