diff --git a/README.md b/README.md index 6882d33c0..6aa7c2587 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,10 @@ Full list of libraries, and install instructions: - `odc.stac` STAC to ODC conversion tools - `odc.dscache` experimental key-value store where `key=UUID`, `value=Dataset` - `odc.io` common IO utilities, used by apps mainly -- `odc.aws` AWS/S3 utilities, used by apps mainly -- `odc.aio` faster concurrent fetching from S3 with async, used by apps -- `odc.{thredds,azure}` internal libs for cloud IO +- `odc-cloud[ASYNC,AZURE,THREDDS]` cloud crawling support package + - `odc.aws` AWS/S3 utilities, used by apps mainly + - `odc.aio` faster concurrent fetching from S3 with async, used by apps `odc-cloud[ASYNC]` + - `odc.{thredds,azure}` internal libs for cloud IO `odc-cloud[THREDDS,AZURE]` Pre-release of these libraries is on PyPI now, so can be installed with `pip` "the normal way". Most recent development versions of `odc-tools` packages are @@ -38,8 +39,7 @@ pip install --extra-index-url="https://packages.dea.ga.gov.au" \ odc-stats \ odc-algo \ odc-io \ - odc-aws \ - odc-aio \ + odc-cloud[ASYNC] \ odc-dscache ``` diff --git a/apps/cloud/setup.cfg b/apps/cloud/setup.cfg index d97135615..ef7df4b4a 100644 --- a/apps/cloud/setup.cfg +++ b/apps/cloud/setup.cfg @@ -19,14 +19,13 @@ packages = find_namespace: python_requires = >=3.6 tests_require = pytest install_requires = - odc_aws - odc_io - odc_aio + odc-cloud[ASYNC] + odc-io click [options.extras_require] -THREDDS = odc-thredds -AZURE = odc-azure +THREDDS = odc-cloud[THREDDS] +AZURE = odc-cloud[AZURE] GCP = google-cloud-storage [options.entry_points] diff --git a/apps/dc_tools/setup.cfg b/apps/dc_tools/setup.cfg index 0189c759f..cf938b54b 100644 --- a/apps/dc_tools/setup.cfg +++ b/apps/dc_tools/setup.cfg @@ -31,14 +31,18 @@ install_requires = datacube odc_stac odc_io - odc_aws - odc_aio + odc-cloud + odc-cloud[ASYNC] pystac>=1.0.0 [options.extras_require] -tests = pytest; deepdiff; pytest-depends -AZURE = odc_azure -THREDDS = odc_thredds +tests = + pytest + deepdiff + pytest-depends + +AZURE = odc-cloud[AZURE] +THREDDS = odc-cloud[THREDDS] [options.entry_points] console_scripts = diff --git a/libs/aio/README.md b/libs/aio/README.md deleted file mode 100644 index f56b275a9..000000000 --- a/libs/aio/README.md +++ /dev/null @@ -1,34 +0,0 @@ -odc.aio -======= - -Async IO (from S3). - -Installation ------------- - -``` -pip install odc-aio -``` - -Usage ------ - -TODO - -Development (Venv) ------------------ - -TODO - -Development (Conda) -------------------- - -Create a conda environment and install this library in editable mode. This assumes -the [conda-forge](https://conda-forge.org/) channel is added. - -```bash -conda create --name odc_tools datacube -conda activate odc_tools -cd libs/aio/ -pip install -e . -``` diff --git a/libs/aio/odc/aio/_version.py b/libs/aio/odc/aio/_version.py deleted file mode 100644 index bbd912be9..000000000 --- a/libs/aio/odc/aio/_version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.2.0a1" diff --git a/libs/aio/setup.cfg b/libs/aio/setup.cfg deleted file mode 100644 index b65d99b28..000000000 --- a/libs/aio/setup.cfg +++ /dev/null @@ -1,28 +0,0 @@ -[metadata] -name = odc-aio -description = Async IO (from S3) -version = attr: odc.aio._version.__version__ -author = Open Data Cube -author_email = -maintainer = Open Data Cube -maintainer_email = -long_description_content_type = text/markdown -long_description = file: README.md -platforms = any -license = Apache License 2.0 -url = https://github.com/opendatacube/odc-tools/ - -[options] -include_package_data = true -zip_safe = false -packages = find_namespace: -python_requires = >=3.6 -tests_require = pytest -install_requires = - aiobotocore>=1.0 - botocore - odc_aws - -[options.packages.find] -include = - odc* diff --git a/libs/algo/setup.cfg b/libs/algo/setup.cfg index 77a9ecd12..e0d8a0ea8 100644 --- a/libs/algo/setup.cfg +++ b/libs/algo/setup.cfg @@ -36,7 +36,7 @@ install_requires = hdstats = hdstats>=0.1.7.post5 s3 = boto3 - odc-aws + odc-cloud [options.packages.find] include = diff --git a/libs/aws/README.md b/libs/aws/README.md deleted file mode 100644 index c3b9e17dc..000000000 --- a/libs/aws/README.md +++ /dev/null @@ -1,18 +0,0 @@ -odc.aws -======= - -Various AWS helper methods. - - -Installation ------------- - -``` -pip install odc-aws -``` - - -Usage ------ - -TODO diff --git a/libs/aws/pyproject.toml b/libs/aws/pyproject.toml deleted file mode 100644 index 3607e0fbd..000000000 --- a/libs/aws/pyproject.toml +++ /dev/null @@ -1,3 +0,0 @@ -[build-system] -requires = ["setuptools>=51.0.0", "wheel"] -build-backend = "setuptools.build_meta" diff --git a/libs/aws/setup.py b/libs/aws/setup.py deleted file mode 100644 index 606849326..000000000 --- a/libs/aws/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from setuptools import setup - -setup() diff --git a/libs/azure/odc/azure/_version.py b/libs/azure/odc/azure/_version.py deleted file mode 100644 index 3e7164d5a..000000000 --- a/libs/azure/odc/azure/_version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.2.0a" diff --git a/libs/azure/pyproject.toml b/libs/azure/pyproject.toml deleted file mode 100644 index 3607e0fbd..000000000 --- a/libs/azure/pyproject.toml +++ /dev/null @@ -1,3 +0,0 @@ -[build-system] -requires = ["setuptools>=51.0.0", "wheel"] -build-backend = "setuptools.build_meta" diff --git a/libs/azure/setup.cfg b/libs/azure/setup.cfg deleted file mode 100644 index 95d2a1e07..000000000 --- a/libs/azure/setup.cfg +++ /dev/null @@ -1,26 +0,0 @@ -[metadata] -name = odc-azure -description = Various Azure helper methods (experimental) -version = attr: odc.azure._version.__version__ -author = Open Data Cube -author_email = -maintainer = Open Data Cube -maintainer_email = -long_description_content_type = text/markdown -long_description = file: README.md -platforms = any -license = Apache License 2.0 -url = https://github.com/opendatacube/odc-tools/ - -[options] -include_package_data = true -zip_safe = false -packages = find_namespace: -python_requires = >=3.6 -tests_require = pytest -install_requires = - azure-storage-blob - -[options.packages.find] -include = - odc* diff --git a/libs/azure/setup.py b/libs/azure/setup.py deleted file mode 100644 index 606849326..000000000 --- a/libs/azure/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from setuptools import setup - -setup() diff --git a/libs/cloud/README.md b/libs/cloud/README.md new file mode 100644 index 000000000..14d027424 --- /dev/null +++ b/libs/cloud/README.md @@ -0,0 +1,24 @@ +odc.cloud +========= + +Various cloud access helper methods. + +This package ships several namespaces: + +- `odc.aws.` S3 and other AWS resources +- `odc.aio.` S3 but async, depends (needs `[ASYNC]` feature flag, pulls in `aiobotocore`) +- `odc.azure.` Azure blob storage traversal helpers (needs `[AZURE]` feature flag) +- `odc.thredds.` THREDDS traversal helper method (needs `[THREDDS]` feature flag) + + +Installation +------------ + +``` +pip install odc-cloud +``` + +Usage +----- + +TODO diff --git a/libs/aio/odc/aio/__init__.py b/libs/cloud/odc/aio/__init__.py similarity index 99% rename from libs/aio/odc/aio/__init__.py rename to libs/cloud/odc/aio/__init__.py index e8dd556b9..ae99a44ff 100644 --- a/libs/aio/odc/aio/__init__.py +++ b/libs/cloud/odc/aio/__init__.py @@ -1,4 +1,4 @@ -from ._version import __version__ +from ..cloud._version import __version__ import logging import botocore diff --git a/libs/aws/odc/aws/__init__.py b/libs/cloud/odc/aws/__init__.py similarity index 99% rename from libs/aws/odc/aws/__init__.py rename to libs/cloud/odc/aws/__init__.py index 8a59f9000..04b65f625 100644 --- a/libs/aws/odc/aws/__init__.py +++ b/libs/cloud/odc/aws/__init__.py @@ -1,7 +1,7 @@ """ Helper methods for working with AWS """ -from ._version import __version__ +from ..cloud._version import __version__ import os import botocore import botocore.session diff --git a/libs/aws/odc/aws/_find.py b/libs/cloud/odc/aws/_find.py similarity index 100% rename from libs/aws/odc/aws/_find.py rename to libs/cloud/odc/aws/_find.py diff --git a/libs/aws/odc/aws/dns.py b/libs/cloud/odc/aws/dns.py similarity index 100% rename from libs/aws/odc/aws/dns.py rename to libs/cloud/odc/aws/dns.py diff --git a/libs/aws/odc/aws/inventory.py b/libs/cloud/odc/aws/inventory.py similarity index 100% rename from libs/aws/odc/aws/inventory.py rename to libs/cloud/odc/aws/inventory.py diff --git a/libs/aws/odc/aws/misc.py b/libs/cloud/odc/aws/misc.py similarity index 100% rename from libs/aws/odc/aws/misc.py rename to libs/cloud/odc/aws/misc.py diff --git a/libs/aws/odc/aws/queue.py b/libs/cloud/odc/aws/queue.py similarity index 100% rename from libs/aws/odc/aws/queue.py rename to libs/cloud/odc/aws/queue.py diff --git a/libs/azure/odc/azure/__init__.py b/libs/cloud/odc/azure/__init__.py similarity index 98% rename from libs/azure/odc/azure/__init__.py rename to libs/cloud/odc/azure/__init__.py index fab5438e7..7a91d38e6 100644 --- a/libs/azure/odc/azure/__init__.py +++ b/libs/cloud/odc/azure/__init__.py @@ -1,6 +1,6 @@ """Azure blob storage crawling and YAML fetching utilities """ -from ._version import __version__ +from ..cloud._version import __version__ from azure.storage.blob import ContainerClient, BlobClient from typing import List, Tuple, Optional from multiprocessing.dummy import Pool as ThreadPool diff --git a/libs/aws/odc/aws/_version.py b/libs/cloud/odc/cloud/_version.py similarity index 100% rename from libs/aws/odc/aws/_version.py rename to libs/cloud/odc/cloud/_version.py diff --git a/libs/aio/odc/ppt/__init__.py b/libs/cloud/odc/ppt/__init__.py similarity index 100% rename from libs/aio/odc/ppt/__init__.py rename to libs/cloud/odc/ppt/__init__.py diff --git a/libs/aio/odc/ppt/async_thread.py b/libs/cloud/odc/ppt/async_thread.py similarity index 100% rename from libs/aio/odc/ppt/async_thread.py rename to libs/cloud/odc/ppt/async_thread.py diff --git a/libs/thredds/odc/thredds/__init__.py b/libs/cloud/odc/thredds/__init__.py similarity index 98% rename from libs/thredds/odc/thredds/__init__.py rename to libs/cloud/odc/thredds/__init__.py index 9ed15978b..0e10a318c 100644 --- a/libs/thredds/odc/thredds/__init__.py +++ b/libs/cloud/odc/thredds/__init__.py @@ -1,6 +1,6 @@ """Thredds crawling and YAML fetching utilities """ -from ._version import __version__ +from ..cloud._version import __version__ from thredds_crawler.crawl import Crawl import requests from urllib.parse import urlparse diff --git a/libs/aio/pyproject.toml b/libs/cloud/pyproject.toml similarity index 100% rename from libs/aio/pyproject.toml rename to libs/cloud/pyproject.toml diff --git a/libs/aws/setup.cfg b/libs/cloud/setup.cfg similarity index 72% rename from libs/aws/setup.cfg rename to libs/cloud/setup.cfg index 1b5ed02c4..7fb69529e 100644 --- a/libs/aws/setup.cfg +++ b/libs/cloud/setup.cfg @@ -1,7 +1,7 @@ [metadata] -name = odc-aws +name = odc-cloud description = Various AWS helper methods -version = attr: odc.aws._version.__version__ +version = attr: odc.cloud._version.__version__ author = Open Data Cube author_email = maintainer = Open Data Cube @@ -22,6 +22,15 @@ install_requires = botocore boto3 +[options.extras_require] +THREDDS = + thredds_crawler + requests + +AZURE = azure-storage-blob +ASYNC = aiobotocore[boto3]>=1.0 + + [options.packages.find] include = odc* diff --git a/libs/aio/setup.py b/libs/cloud/setup.py similarity index 100% rename from libs/aio/setup.py rename to libs/cloud/setup.py diff --git a/libs/aws/tests/test_aws.py b/libs/cloud/tests/test_aws.py similarity index 100% rename from libs/aws/tests/test_aws.py rename to libs/cloud/tests/test_aws.py diff --git a/libs/azure/tests/test_azure.py b/libs/cloud/tests/test_azure.py similarity index 100% rename from libs/azure/tests/test_azure.py rename to libs/cloud/tests/test_azure.py diff --git a/libs/thredds/tests/test_thredds.py b/libs/cloud/tests/test_thredds.py similarity index 100% rename from libs/thredds/tests/test_thredds.py rename to libs/cloud/tests/test_thredds.py diff --git a/libs/stats/setup.cfg b/libs/stats/setup.cfg index d69fc8d8a..769a49abf 100644 --- a/libs/stats/setup.cfg +++ b/libs/stats/setup.cfg @@ -25,9 +25,8 @@ install_requires = datacube distributed numpy - odc_aio + odc-cloud[ASYNC] odc_algo - odc_aws odc_dscache odc_io odc_stac diff --git a/libs/thredds/odc/thredds/_version.py b/libs/thredds/odc/thredds/_version.py deleted file mode 100644 index bbd912be9..000000000 --- a/libs/thredds/odc/thredds/_version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.2.0a1" diff --git a/libs/thredds/pyproject.toml b/libs/thredds/pyproject.toml deleted file mode 100644 index 3607e0fbd..000000000 --- a/libs/thredds/pyproject.toml +++ /dev/null @@ -1,3 +0,0 @@ -[build-system] -requires = ["setuptools>=51.0.0", "wheel"] -build-backend = "setuptools.build_meta" diff --git a/libs/thredds/setup.cfg b/libs/thredds/setup.cfg deleted file mode 100644 index abb7b52b7..000000000 --- a/libs/thredds/setup.cfg +++ /dev/null @@ -1,27 +0,0 @@ -[metadata] -name = odc-thredds -description = Various Thredds helper methods (for NCI) -version = attr: odc.thredds._version.__version__ -author = Open Data Cube -author_email = -maintainer = Open Data Cube -maintainer_email = -long_description_content_type = text/markdown -long_description = file: README.md -platforms = any -license = Apache License 2.0 -url = https://github.com/opendatacube/odc-tools/ - -[options] -include_package_data = true -zip_safe = false -packages = find_namespace: -python_requires = >=3.6 -tests_require = pytest -install_requires = - thredds_crawler - requests - -[options.packages.find] -include = - odc* diff --git a/libs/thredds/setup.py b/libs/thredds/setup.py deleted file mode 100644 index 606849326..000000000 --- a/libs/thredds/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from setuptools import setup - -setup() diff --git a/staging/dnsup/setup.cfg b/staging/dnsup/setup.cfg index 044d5318f..dbcb7c9f0 100644 --- a/staging/dnsup/setup.cfg +++ b/staging/dnsup/setup.cfg @@ -19,7 +19,7 @@ packages = find_namespace: python_requires = >=3.6 tests_require = pytest install_requires = - odc_aws + odc-cloud [options.entry_points] console_scripts =