Update the dockerfile to fix FC install, also upgrade to Python 3.8

Dockerfile

@@ -1,52 +1,59 @@
-FROM opendatacube/geobase:wheels as env_builder
 ARG py_env_path=/env
+ARG V_BASE=3.3.0
 
-COPY requirements.txt /tmp
-RUN env-build-tool new /tmp/requirements.txt ${py_env_path}
+FROM opendatacube/geobase-builder:${V_BASE} as env_builder
+ENV LC_ALL=C.UTF-8
+ARG py_env_path
 
-ENV PATH=${py_env_path}/bin:$PATH
+COPY requirements.txt constraints.txt /tmp/
+RUN env-build-tool new /tmp/requirements.txt /tmp/constraints.txt ${py_env_path}
 
 # Copy source code and install it
 RUN mkdir -p /code
 WORKDIR /code
 ADD . /code
 
-RUN pip install --use-feature=2020-resolver --extra-index-url="https://packages.dea.ga.gov.au" .
+ENV PATH="${py_env_path}/bin:${PATH}"
+RUN pip install --extra-index-url="https://packages.dea.ga.gov.au" -c /tmp/constraints.txt .
 
 # Make sure it's working first
 RUN datacube-alchemist --version
 
 # Build the production runner stage from here
-FROM opendatacube/geobase:runner
+FROM opendatacube/geobase-runner:${V_BASE}
+ARG py_env_path=/env
+
+# Environment can be whatever is supported by setup.py
+# so, either deployment, test
+ARG ENVIRONMENT=deployment
+RUN echo "Environment is: $ENVIRONMENT"
 
 ENV LC_ALL=C.UTF-8 \
     DEBIAN_FRONTEND=noninteractive \
     SHELL=bash
 
 RUN apt-get update \
     && apt-get install -y \
-    git vim nano fish wget postgresql \
+    git vim nano fish wget postgresql libgfortran5 \
     && rm -rf /var/lib/apt/lists/*
 
-COPY --from=env_builder /env /env
-ENV PATH=/env/bin:$PATH
-
-# Environment can be whatever is supported by setup.py
-# so, either deployment, test
-ARG ENVIRONMENT=deployment
-RUN echo "Environment is: $ENVIRONMENT"
+COPY --from=env_builder $py_env_path $py_env_path
+ENV PATH="${py_env_path}/bin:${PATH}"
 
 # Set up a nice workdir, and only copy the things we care about in
 ENV APPDIR=/code
 RUN mkdir -p $APPDIR
 WORKDIR $APPDIR
 ADD . $APPDIR
 
+RUN datacube-alchemist --version
+
 # These ENVIRONMENT flags make this a bit complex, but basically, if we are in dev
 # then we want to link the source (with the -e flag) and if we're in prod, we
 # want to delete the stuff in the /code folder to keep it simple.
 RUN if [ "$ENVIRONMENT" = "deployment" ] ; then rm -rf $APPDIR ; \
-    else pip install --extra-index-url="https://packages.dea.ga.gov.au" --editable .[$ENVIRONMENT] ; \
+    else pip install --extra-index-url="https://packages.dea.ga.gov.au" \
+    -c /code/constraints.txt --editable .[$ENVIRONMENT] ; \
     fi
 
 RUN datacube-alchemist --version

Makefile

@@ -164,6 +164,14 @@ wo-africa-one:
 		datacube-alchemist run-one --config-file ./examples/wofs_ls.alchemist.yaml \
 		--uuid 1f88087d-0da6-55be-aafb-5e370520e405
 
+fc-africa-one:
+	docker-compose exec \
+		--env AWS_DEFAULT_REGION=af-south-1 \
+		--env AWS_S3_ENDPOINT=s3.af-south-1.amazonaws.com \
+		alchemist \
+		datacube-alchemist run-one --config-file ./examples/fc_ls.alchemist.yaml \
+		--uuid 1f88087d-0da6-55be-aafb-5e370520e405
+
 THREE_AFRICA=1f88087d-0da6-55be-aafb-5e370520e405 272c298f-03e3-5a08-a584-41a0a3c3cb95 834d56e2-7465-5980-a6af-615ef0f67e28
 
 wo-africa-three:
@@ -175,6 +183,15 @@ wo-africa-three:
 			xargs -n1 datacube-alchemist run-one --config-file ./examples/wofs_ls.alchemist.yaml --uuid \
 			"
 
+fc-africa-three:
+	docker-compose exec \
+		--env AWS_DEFAULT_REGION=af-south-1 \
+		--env AWS_S3_ENDPOINT=s3.af-south-1.amazonaws.com \
+		alchemist bash -c\
+			"echo '${THREE_AFRICA}' | \
+			xargs -n1 datacube-alchemist run-one --config-file ./examples/fc_ls.alchemist.yaml --uuid \
+			"
+
 # Queue testing
 wofs-to-queue:
 	docker-compose exec alchemist \

constraints.txt

@@ -0,0 +1,104 @@
+affine==2.3.0
+aiobotocore==1.2.0
+aiohttp==3.7.3
+aioitertools==0.7.1
+astropy==4.1
+async-timeout==3.0.1
+attrs==20.3.0
+awscli==1.18.212
+azure-core==1.13.0
+azure-storage-blob==12.8.0
+boto3==1.16.52
+botocore==1.19.52
+Bottleneck==1.3.2
+cachetools==4.2.1
+certifi==2020.12.5
+cffi==1.14.5
+cftime==1.3.1
+chardet==3.0.4
+ciso8601==2.1.3
+click==7.1.2
+click-plugins==1.1.1
+cligj==0.7.1
+cloudpickle==1.6.0
+colorama==0.4.3
+contextvars==2.4
+cryptography==3.4.7
+cycler==0.10.0
+dask==2021.5.0
+dask-image==0.5.0
+dataclasses==0.8
+decorator==4.4.2
+distributed==2021.5.0
+docutils==0.15.2
+HeapDict==1.0.1
+idna==2.10
+idna-ssl==1.1.0
+imageio==2.9.0
+immutables==0.14
+importlib-metadata==3.4.0
+isodate==0.6.0
+Jinja2==2.11.2
+jmespath==0.10.0
+jsonschema==3.2.0
+kiwisolver==1.3.1
+lark-parser==0.11.1
+lmdb==1.0.0
+lxml==4.6.2
+MarkupSafe==1.1.1
+matplotlib==3.3.3
+morecantile==2.1.0
+msgpack==1.0.2
+msrest==0.6.21
+multidict==5.1.0
+netCDF4==1.5.4
+networkx==2.5
+numexpr==2.7.2
+numpy==1.20.3
+oauthlib==3.1.0
+pandas==1.1.5
+Pillow==8.1.0
+PIMS==0.5
+psutil==5.8.0
+psycopg2==2.8.6
+pyasn1==0.4.8
+pycparser==2.20
+pydantic==1.8.1
+pyparsing==2.4.7
+pyproj==2.6.1.post1
+pyrsistent==0.17.3
+pystac==0.5.4
+python-dateutil==2.7.5
+pytz==2020.5
+PyWavelets==1.1.1
+PyYAML==5.3.1
+rasterio==1.2.2
+requests==2.25.1
+requests-oauthlib==1.3.0
+rio-cogeo==2.1.4
+rsa==4.5
+s3transfer==0.3.4
+sat-search==0.3.0
+sat-stac==0.4.1
+scikit-image==0.17.2
+scipy==1.5.4
+Shapely==1.7.1
+six==1.15.0
+slicerator==1.0.0
+snuggs==1.4.7
+sortedcontainers==2.3.0
+SQLAlchemy==1.3.22
+tblib==1.7.0
+thredds-crawler==1.5.4
+tifffile==2020.9.3
+toolz==0.11.1
+tornado==6.1
+tqdm==4.56.0
+typing-extensions==3.7.4.3
+urllib3==1.25.11
+wrapt==1.12.1
+xarray==0.16.1
+yarl==1.6.3
+zict==2.0.0
+zipp==3.4.0
+zstandard==0.15.1

examples/c3_config_fc.yaml

@@ -45,7 +45,7 @@ output:
   preview_image:
     red: bs
     green: pv
-    blue: pv
+    blue: npv
   explorer_url: https://explorer.dev.dea.ga.gov.au
   write_data_settings:
     overview_resampling: average

examples/fc_ls.alchemist.yaml

@@ -0,0 +1,70 @@
+specification:
+  products:
+    - ls8_sr
+    - ls7_sr
+    - ls5_sr
+  measurements: ['green', 'red', 'nir', 'swir_1', 'swir_2']
+  measurement_renames:
+    swir_1: swir1
+    swir_2: swir2
+  resampling:
+    pixel_quality: nearest
+    "*": bilinear
+  basis: green
+
+  aws_unsigned: True
+  transform: fc.virtualproduct.FractionalCover
+  transform_url: 'https://github.com/GeoscienceAustralia/fc/'
+  override_product_family: ard
+
+  transform_args_per_product:
+    ls5_sr:
+      c2_scaling: True
+    ls7_sr:
+      c2_scaling: True
+    ls8_sr:
+      c2_scaling: True
+      regression_coefficients:
+        blue:
+          - 4.1
+          - 0.97470
+        green:
+          - 28.9
+          - 0.99779
+        red:
+          - 27.4
+          - 1.00446
+        nir:
+          - 0.4
+          - 0.98906
+        swir1:
+          - 25.6
+          - 0.99467
+        swir2:
+          - -32.7
+          - 1.02551
+
+output:
+  explorer_url: https://explorer.digitalearth.africa/products/fc_ls/extents
+  location: /tmp/alchemist
+  nodata: 255
+  preview_image:
+    red: bs
+    green: pv
+    blue: npv
+  write_data_settings:
+    overview_resampling: nearest
+  reference_source_dataset: True
+  write_stac: True
+  inherit_geometry: True
+
+  metadata:
+    product_family: fc
+    producer: digitalearthafrica.org
+    dataset_version: 0.0.1
+    naming_conventions: deafrica
+
+processing:
+  dask_chunks:
+    x: 4096
+    y: 4096

requirements.txt

@@ -1,13 +1,13 @@
 --extra-index-url="https://packages.dea.ga.gov.au"
-aiobotocore[awscli]==1.1.2
-boto3==1.14.44
-botocore==1.17.44
-# Pinned for production DEA run
-fc==1.3.2
+aiobotocore[awscli]
+boto3
+botocore
+fc==1.3.4
 fsspec==0.8.7
 odc-apps-cloud
 odc-apps-dc-tools
 odc-aws
 odc-index
 wofs==1.6.3
-xarray==0.16.1
+xarray
+numpy

setup.cfg

@@ -35,6 +35,7 @@ install_requires =
     odc-aws
     odc-index
     awscli
+    h5py
 #include_package_data = True
 
 [options.packages.find]