Skip to content

Commit

Permalink
Burn Scar example for Sentinel-2 NRT (#89)
Browse files Browse the repository at this point in the history
* Add a dNBR example

* Finish thumbnail work and some broader fixes.

* Add GM indexing

* Local dev environment simple dNBR

* Implements 3-band nDBR transform with example

* Resolve linting errors, update burnscar example configuration

* Remove empty section

Co-authored-by: Alex Leith <[email protected]>
  • Loading branch information
james-blitzm and alexgleith authored May 5, 2021
1 parent 43f3622 commit a695f82
Show file tree
Hide file tree
Showing 8 changed files with 275 additions and 11 deletions.
42 changes: 41 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,42 @@ index:
datacube dataset add --ignore-lineage --confirm-ignore-lineage \
s3://dea-public-data/baseline/ga_ls8c_ard_3/098/073/2020/07/19/ga_ls8c_ard_3-1-0_098073_2020-07-19_final.odc-metadata.yaml \
s3://dea-public-data/baseline/ga_ls5t_ard_3/108/083/2010/10/02/ga_ls5t_ard_3-0-0_108083_2010-10-02_final.odc-metadata.yaml \
s3://dea-public-data/baseline/ga_ls7e_ard_3/100/075/2003/10/15/ga_ls7e_ard_3-0-0_100075_2003-10-15_final.odc-metadata.yaml
s3://dea-public-data/baseline/ga_ls7e_ard_3/100/075/2003/10/15/ga_ls7e_ard_3-0-0_100075_2003-10-15_final.odc-metadata.yaml \
s3://dea-public-data/baseline/ga_ls8c_ard_3/091/089/2019/01/20/ga_ls8c_ard_3-0-0_091089_2019-01-20_final.odc-metadata.yaml

index-geomedian:
docker-compose exec alchemist \
bash -c "\
datacube product add https://data.dea.ga.gov.au/geomedian-australia/v2.1.0/product-definition.yaml;\
s3-to-dc --no-sign-request 's3://dea-public-data/geomedian-australia/v2.1.0/L8/**/*.yaml' ls8_nbart_geomedian_annual\
"

metadata-s2-nrt:
docker-compose exec alchemist \
datacube metadata add \
https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/nrt/sentinel/eo_s2_nrt.odc-type.yaml

product-s2-nrt:
docker-compose exec alchemist \
datacube product add \
https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/nrt/sentinel/s2_nrt.products.yaml

product-s2a_ard_granule:
docker-compose exec alchemist \
datacube product add \
https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/ga_s2_ard_nbar/ga_s2_ard_nbar_granule.yaml

metadata-eo_plus:
docker-compose exec alchemist \
datacube metadata add \
https://raw.githubusercontent.com/opendatacube/datacube-dataset-config/master/metadata_types/eo_plus.odc-type.yaml

index-s2-nrt:
docker-compose exec alchemist \
datacube dataset add --ignore-lineage --confirm-ignore-lineage \
s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2021-05-02/S2B_OPER_MSI_ARD_TL_VGS1_20210502T033357_A021693_T51KYA_N03.00/ARD-METADATA.yaml \
s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2021-02-05/S2A_OPER_MSI_ARD_TL_VGS1_20210205T055002_A029372_T50HMK_N02.09/ARD-METADATA.yaml \
s3://dea-public-data/baseline/s2a_ard_granule/2021-02-05/S2A_OPER_MSI_ARD_TL_VGS1_20210205T055002_A029372_T50HMK_N02.09/eo3-ARD-METADATA.yaml

# Landsat 8, 7 and 5 respectively
THREE_SCENES=600645a5-5256-4632-a13d-fa13d1c11a8f 8b215983-ae1b-45bd-ad63-7245248bd41b 3fda2741-e810-4d3e-a54a-279fc3cd795f
Expand All @@ -80,6 +115,11 @@ fc-one:
datacube-alchemist run-one --config-file ./examples/c3_config_fc.yaml \
--uuid 600645a5-5256-4632-a13d-fa13d1c11a8f

dnbr-one:
docker-compose exec alchemist \
datacube-alchemist run-one --config-file ./examples/c3_config_dnbr_3band.yaml \
--uuid 28eb72e3-415a-4beb-a47f-a8ca779dda07

wofs-one-of-each:
docker-compose exec alchemist \
bash -c \
Expand Down
4 changes: 2 additions & 2 deletions datacube_alchemist/_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def _configure_logger():

def _write_thumbnail(task: AlchemistTask, dataset_assembler: DatasetAssembler):
if task.settings.output.preview_image is not None:
dataset_assembler.write_thumbnail(*task.settings.output.preview_image)
dataset_assembler.write_thumbnail(**task.settings.output.preview_image)
elif task.settings.output.preview_image_singleband is not None:
dataset_assembler.write_thumbnail_singleband(
**task.settings.output.preview_image_singleband
Expand Down Expand Up @@ -183,7 +183,7 @@ def _guess_region_code(ds: Dataset) -> str:
# The region code is 50JPP.
tile_match = RE_TILE_REGION_CODE.match(ds.metadata_doc["tile_id"])
if not tile_match:
raise ValueError("No region code for dataset {}".format(d.id))
raise ValueError("No region code for dataset {}".format(ds.id))
return tile_match.group(1)


Expand Down
8 changes: 3 additions & 5 deletions datacube_alchemist/settings.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
from typing import Optional, Mapping, List, Sequence, Any, Union
from typing import Optional, Mapping, Sequence, Any, Union

import attr
import cattr
import numpy as np
from rasterio.enums import Resampling

from datacube.model import Dataset
Expand Down Expand Up @@ -34,12 +33,11 @@ def _convert_write_data_settings(settings):
@attr.s(auto_attribs=True)
class OutputSettings:
location: str
dtype: np.dtype
nodata: int # type depends on dtype
write_data_settings: Optional[Mapping[str, str]] = attr.ib(
converter=_convert_write_data_settings
)
preview_image: Optional[List[str]] = None
nodata: Optional[int] = None
preview_image: Optional[Mapping[Any, Any]] = None
preview_image_singleband: Optional[Mapping[Any, Any]] = None
metadata: Optional[Mapping[str, str]] = None
properties: Optional[Mapping[str, str]] = None
Expand Down
138 changes: 138 additions & 0 deletions datacube_alchemist/transforms.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
from typing import Dict

import numpy
from datacube import Datacube
from datacube.virtual import Measurement, Transformation
from xarray import Dataset, merge


class DeltaNBR(Transformation):
"""Return NBR"""

def __init__(self):
self.output_measurements = {
"dnbr": {"name": "dnbr", "dtype": "float", "nodata": -9999, "units": ""}
}

def measurements(self, input_measurements) -> Dict[str, Measurement]:
return self.output_measurements

def compute(self, data) -> Dataset:
base_year = data.time.dt.year.values[0] - 1
if base_year == 2021:
base_year = 2020
if base_year == 2012:
base_year = 2013

dc = Datacube()
gm_data = dc.load(
product="ls8_nbart_geomedian_annual",
time=str(base_year),
like=data.geobox,
measurements=["nir", "swir2"],
)

pre = (gm_data.nir - gm_data.swir2) / (gm_data.nir + gm_data.swir2)
data = merge([data, {"pre": pre.isel(time=0, drop=True)}])

data["post"] = (data.nir - data.swir2) / (data.nir + data.swir2)
data["dnbr"] = data.pre - data.post

data["dnbr"] = data.dnbr.where(data.nir != -999).astype(numpy.single)

data = data.drop(["nir", "swir2", "pre", "post"])

return data


class DeltaNBR_3band(Transformation):
"""Return 3-Band NBR"""

def __init__(self):
self.output_measurements = {
"delta_nbr": {
"name": "dnbr",
"dtype": "float",
"nodata": -9999,
"units": "",
},
"delta_bsi": {
"name": "bsi",
"dtype": "float",
"nodata": -9999,
"units": "",
},
"delta_ndvi": {
"name": "ndvi",
"dtype": "float",
"nodata": -9999,
"units": "",
},
}

def measurements(self, input_measurements) -> Dict[str, Measurement]:
return self.output_measurements

def compute(self, data) -> Dataset:

"""
Implementation ported from https://github.com/daleroberts/nrt-predict/blob/main/nrtmodels/burnscar.py#L39
"""

gm_base_year = data.time.dt.year.values[0] - 1
if gm_base_year == 2021:
gm_base_year = 2020
if gm_base_year == 2012:
gm_base_year = 2013

dc = Datacube()
gm_data = dc.load(
product="ls8_nbart_geomedian_annual",
time=str(gm_base_year),
like=data.geobox,
measurements=["blue", "red", "nir", "swir2"], # B02, B04, B08, B11
)

# Delta Normalised Burn Ratio (dNBR) = (B08 - B11)/(B08 + B11)
pre_nbr = (gm_data.nir - gm_data.swir2) / (gm_data.nir + gm_data.swir2)
data = merge([data, {"pre_nbr": pre_nbr.isel(time=0, drop=True)}])
data["post_nbr"] = (data.nir - data.swir2) / (data.nir + data.swir2)
data["delta_nbr"] = data.pre_nbr - data.post_nbr
data["delta_nbr"] = data.delta_nbr.where(data.nir != -999).astype(numpy.single)

# Burn Scar Index (BSI) = ((B11 + B04) - (B08 - B02)) / ((B11 + B04) + (B08 - B02))
pre_bsi = ((gm_data.swir2 + gm_data.red) - (gm_data.nir - gm_data.blue)) / (
(gm_data.swir2 + gm_data.red) + (gm_data.nir - gm_data.blue)
)
data = merge([data, {"pre_bsi": pre_bsi.isel(time=0, drop=True)}])
data["post_bsi"] = ((data.swir2 + data.red) - (data.nir - data.blue)) / (
(data.swir2 + data.red) + (data.nir - data.blue)
)
data["delta_bsi"] = data.pre_bsi - data.post_bsi
data["delta_bsi"] = data.delta_bsi.where(data.nir != -999).astype(numpy.single)

# Normalized Difference Vegetation Index (NDVI) = (B08 - B04)/(B08 + B04)
pre_ndvi = (gm_data.nir - gm_data.red) / (gm_data.nir + gm_data.red)
data = merge([data, {"pre_ndvi": pre_ndvi.isel(time=0, drop=True)}])
data["post_ndvi"] = (data.nir - data.red) / (data.nir + data.red)
data["delta_ndvi"] = data.post_ndvi - data.pre_ndvi
data["delta_ndvi"] = data.delta_ndvi.where(data.nir != -999).astype(
numpy.single
)

data = data.drop(
[
"nir",
"swir2",
"red",
"blue",
"post_nbr",
"pre_nbr",
"pre_bsi",
"post_bsi",
"pre_ndvi",
"post_ndvi",
]
)

return data
43 changes: 43 additions & 0 deletions examples/c3_config_dnbr.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
specification:
products:
- s2a_nrt_granule
- s2b_nrt_granule
measurements: ['nbart_nir_1', 'nbart_red_edge_1']
measurement_renames:
nbart_nir_1: nir
nbart_red_edge_1: swir2

aws_unsigned: True
transform: datacube_alchemist.transforms.DeltaNBR
transform_args:

override_product_family: ard
basis: nbart_nir_1

output:
location: /tmp/alchemist
preview_image:
red: dnbr
green: dnbr
blue: dnbr
static_stretch: [-1, 1]
explorer_url: https://explorer.dev.dea.ga.gov.au
write_data_settings:
overview_resampling: average
reference_source_dataset: True
write_stac: True
inherit_geometry: True

metadata:
product_family: dnbr
producer: ga.gov.au
dataset_version: 0.0.2
collection_number: 3
naming_conventions: dea_c3
properties:
dea:dataset_maturity: interim

processing:
dask_chunks:
x: -1
y: 4096
44 changes: 44 additions & 0 deletions examples/c3_config_dnbr_3band.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
specification:
products:
- s2a_nrt_granule
- s2b_nrt_granule
measurements: ['nbart_nir_1', 'nbart_red_edge_1', 'nbart_blue', 'nbart_red']
measurement_renames:
nbart_nir_1: nir
nbart_red_edge_1: swir2
nbart_blue: blue
nbart_red: red

aws_unsigned: True
transform: datacube_alchemist.transforms.DeltaNBR_3band

override_product_family: ard
basis: nbart_nir_1

output:
location: /tmp/alchemist
preview_image:
red: delta_nbr
green: delta_ndvi
blue: delta_bsi
static_stretch: [-1, 1]
explorer_url: https://explorer.dev.dea.ga.gov.au
write_data_settings:
overview_resampling: average
reference_source_dataset: True
write_stac: True
inherit_geometry: True

metadata:
product_family: bs
producer: ga.gov.au
dataset_version: 1.6.0
collection_number: 3
naming_conventions: dea_c3
properties:
dea:dataset_maturity: interim

processing:
dask_chunks:
x: -1
y: 4096
6 changes: 4 additions & 2 deletions examples/c3_config_fc.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,10 @@ specification:
output:
location: /tmp/alchemist
nodata: 255
dtype: uint8
preview_image: [bs, pv, npv]
preview_image:
red: bs
green: pv
blue: pv
explorer_url: https://explorer.dev.dea.ga.gov.au
write_data_settings:
overview_resampling: average
Expand Down
1 change: 0 additions & 1 deletion examples/c3_config_wo.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ specification:
output:
location: /tmp/alchemist
nodata: 1
dtype: uint8
explorer_url: https://explorer.dev.dea.ga.gov.au
preview_image_singleband:
measurement: water
Expand Down

0 comments on commit a695f82

Please sign in to comment.