Merge cloud libs into one package (#331)

README.md

@@ -23,9 +23,10 @@ Full list of libraries, and install instructions:
 - `odc.stac` STAC to ODC conversion tools
 - `odc.dscache` experimental key-value store where `key=UUID`, `value=Dataset`
 - `odc.io` common IO utilities, used by apps mainly
-- `odc.aws` AWS/S3 utilities, used by apps mainly
-- `odc.aio` faster concurrent fetching from S3 with async, used by apps
-- `odc.{thredds,azure}` internal libs for cloud IO
+- `odc-cloud[ASYNC,AZURE,THREDDS]` cloud crawling support package
+  - `odc.aws` AWS/S3 utilities, used by apps mainly
+  - `odc.aio` faster concurrent fetching from S3 with async, used by apps `odc-cloud[ASYNC]`
+  - `odc.{thredds,azure}` internal libs for cloud IO `odc-cloud[THREDDS,AZURE]`
 
 Pre-release of these libraries is on PyPI now, so can be installed with `pip`
 "the normal way". Most recent development versions of `odc-tools` packages are
@@ -38,8 +39,7 @@ pip install --extra-index-url="https://packages.dea.ga.gov.au" \
   odc-stats \
   odc-algo \
   odc-io \
-  odc-aws \
-  odc-aio \
+  odc-cloud[ASYNC] \
   odc-dscache
 ```
 

apps/cloud/setup.cfg

@@ -19,14 +19,13 @@ packages = find_namespace:
 python_requires = >=3.6
 tests_require = pytest
 install_requires =
-    odc_aws
-    odc_io
-    odc_aio
+    odc-cloud[ASYNC]
+    odc-io
     click
 
 [options.extras_require]
-THREDDS = odc-thredds
-AZURE = odc-azure
+THREDDS = odc-cloud[THREDDS]
+AZURE = odc-cloud[AZURE]
 GCP = google-cloud-storage
 
 [options.entry_points]

apps/dc_tools/setup.cfg

@@ -31,14 +31,18 @@ install_requires =
     datacube
     odc_stac
     odc_io
-    odc_aws
-    odc_aio
+    odc-cloud
+    odc-cloud[ASYNC]
     pystac>=1.0.0
 
 [options.extras_require]
-tests = pytest; deepdiff; pytest-depends
-AZURE = odc_azure
-THREDDS = odc_thredds
+tests =
+   pytest
+   deepdiff
+   pytest-depends
+
+AZURE = odc-cloud[AZURE]
+THREDDS = odc-cloud[THREDDS]
 
 [options.entry_points]
 console_scripts =

libs/algo/setup.cfg

@@ -36,7 +36,7 @@ install_requires =
 hdstats = hdstats>=0.1.7.post5
 s3 =
     boto3
-    odc-aws
+    odc-cloud
 
 [options.packages.find]
 include =

libs/cloud/README.md

@@ -0,0 +1,24 @@
+odc.cloud
+=========
+
+Various cloud access helper methods.
+
+This package ships several namespaces:
+
+- `odc.aws.` S3 and other AWS resources
+- `odc.aio.` S3 but async, depends (needs `[ASYNC]` feature flag, pulls in `aiobotocore`)
+- `odc.azure.` Azure blob storage traversal helpers (needs `[AZURE]` feature flag)
+- `odc.thredds.` THREDDS traversal helper method (needs `[THREDDS]` feature flag)
+
+
+Installation
+------------
+
+```
+pip install odc-cloud
+```
+
+Usage
+-----
+
+TODO

libs/aio/odc/aio/__init__.py → libs/cloud/odc/aio/__init__.py

@@ -1,4 +1,4 @@
-from ._version import __version__
+from ..cloud._version import __version__
 import logging
 
 import botocore

libs/aws/odc/aws/__init__.py → libs/cloud/odc/aws/__init__.py

@@ -1,7 +1,7 @@
 """
 Helper methods for working with AWS
 """
-from ._version import __version__
+from ..cloud._version import __version__
 import os
 import botocore
 import botocore.session

libs/azure/odc/azure/__init__.py → libs/cloud/odc/azure/__init__.py

@@ -1,6 +1,6 @@
 """Azure blob storage crawling and YAML fetching utilities
 """
-from ._version import __version__
+from ..cloud._version import __version__
 from azure.storage.blob import ContainerClient, BlobClient
 from typing import List, Tuple, Optional
 from multiprocessing.dummy import Pool as ThreadPool

libs/thredds/odc/thredds/__init__.py → libs/cloud/odc/thredds/__init__.py

@@ -1,6 +1,6 @@
 """Thredds crawling and YAML fetching utilities
 """
-from ._version import __version__
+from ..cloud._version import __version__
 from thredds_crawler.crawl import Crawl
 import requests
 from urllib.parse import urlparse

libs/aws/setup.cfg → libs/cloud/setup.cfg

@@ -1,7 +1,7 @@
 [metadata]
-name = odc-aws
+name = odc-cloud
 description = Various AWS helper methods
-version = attr: odc.aws._version.__version__
+version = attr: odc.cloud._version.__version__
 author = Open Data Cube
 author_email =
 maintainer = Open Data Cube
@@ -22,6 +22,15 @@ install_requires =
     botocore
     boto3
 
+[options.extras_require]
+THREDDS = 
+    thredds_crawler
+    requests
+
+AZURE = azure-storage-blob 
+ASYNC = aiobotocore[boto3]>=1.0
+
+
 [options.packages.find]
 include =
   odc*

libs/stats/setup.cfg

@@ -25,9 +25,8 @@ install_requires =
     datacube
     distributed
     numpy
-    odc_aio
+    odc-cloud[ASYNC]
     odc_algo
-    odc_aws
     odc_dscache
     odc_io
     odc_stac

staging/dnsup/setup.cfg

@@ -19,7 +19,7 @@ packages = find_namespace:
 python_requires = >=3.6
 tests_require = pytest
 install_requires =
-    odc_aws
+    odc-cloud
 
 [options.entry_points]
 console_scripts =