diff --git a/audb/core/api.py b/audb/core/api.py index e5f8f4cc..c00d6102 100644 --- a/audb/core/api.py +++ b/audb/core/api.py @@ -37,10 +37,10 @@ def available( Examples: >>> df = audb.available(only_latest=True) >>> df.loc[["air", "emodb"]] - backend host repository version + backend host repository version name - air artifactory https://audeering.jfrog.io/artifactory data-public 1.4.2 - emodb artifactory https://audeering.jfrog.io/artifactory data-public 1.4.1 + air minio s3.dualstack.eu-north-1.amazonaws.com audb-public 1.4.2 + emodb minio s3.dualstack.eu-north-1.amazonaws.com audb-public 1.4.1 """ # noqa: E501 databases = [] @@ -48,7 +48,7 @@ def available( try: backend_interface = repository.create_backend_interface() with backend_interface.backend as backend: - if repository.backend == "artifactory": + if repository.backend == "artifactory": # pragma: nocover # avoid backend_interface.ls('/') # which is very slow on Artifactory # see https://github.com/audeering/audbackend/issues/132 @@ -550,7 +550,7 @@ def repository( Examples: >>> audb.repository("emodb", "1.4.1") - Repository('data-public', 'https://audeering.jfrog.io/artifactory', 'artifactory') + Repository('audb-public', 's3.dualstack.eu-north-1.amazonaws.com', 'minio') """ # noqa: E501 if not versions(name): @@ -579,7 +579,7 @@ def versions( try: backend_interface = repository.create_backend_interface() with backend_interface.backend as backend: - if repository.backend == "artifactory": + if repository.backend == "artifactory": # pragma: nocover # Do not use `backend_interface.versions()` on Artifactory, # as calling `backend_interface.ls()` is slow on Artifactory, # see https://github.com/devopshq/artifactory/issues/423. @@ -597,7 +597,7 @@ def versions( header = p.joinpath(f"db-{version}.yaml") if header.exists(): vs.extend([version]) - except Exception: # pragma: nocover + except Exception: # Might happen, # if a database is not available on the backend, # or we don't have read permissions. diff --git a/audb/core/conftest.py b/audb/core/conftest.py index 6309d876..95e6d6f0 100644 --- a/audb/core/conftest.py +++ b/audb/core/conftest.py @@ -39,7 +39,7 @@ def cache(tmpdir_factory): @pytest.fixture(autouse=True) def public_repository(doctest_namespace): - r"""Provide access to the public Artifactory repository. + r"""Provide access to the public repository. Some tests in the docstrings need access to the emodb database. As all the unit tests defined under ``tests/*`` @@ -54,9 +54,9 @@ def public_repository(doctest_namespace): """ audb.config.REPOSITORIES = [ audb.Repository( - name="data-public", - host="https://audeering.jfrog.io/artifactory", - backend="artifactory", + name="audb-public", + host="s3.dualstack.eu-north-1.amazonaws.com", + backend="minio", ), ] doctest_namespace["audb"] = audb diff --git a/audb/core/etc/audb.yaml b/audb/core/etc/audb.yaml index ab47be34..bac83e9c 100644 --- a/audb/core/etc/audb.yaml +++ b/audb/core/etc/audb.yaml @@ -1,9 +1,9 @@ cache_root: ~/audb shared_cache_root: /data/audb repositories: - - name: data-public - backend: artifactory - host: https://audeering.jfrog.io/artifactory + - name: audb-public + backend: minio + host: s3.dualstack.eu-north-1.amazonaws.com - name: data-local backend: file-system host: ~/audb-host diff --git a/audb/core/repository.py b/audb/core/repository.py index 47563fa0..49b96acc 100644 --- a/audb/core/repository.py +++ b/audb/core/repository.py @@ -27,10 +27,15 @@ class Repository: """ - backend_registry = { + backends = { "file-system": audbackend.backend.FileSystem, - "artifactory": audbackend.backend.Artifactory, + "minio": audbackend.backend.Minio, } + + if hasattr(audbackend.backend, "Artifactory"): + backends["artifactory"] = audbackend.backend.Artifactory + + backend_registry = backends r"""Backend registry. Holds mapping between registered backend names, @@ -76,8 +81,7 @@ def create_backend_interface(self) -> typing.Type[audbackend.interface.Base]: r"""Create backend interface to access repository. When :attr:`Repository.backend` equals ``artifactory``, - it creates an instance of :class:`audbackend.backend.Artifactory` - and wraps an :class:`audbackend.interface.Maven` interface + it wraps an :class:`audbackend.interface.Maven` interface around it. The files will then be stored with the following structure on the Artifactory backend @@ -89,12 +93,11 @@ def create_backend_interface(self) -> typing.Type[audbackend.interface.Base]: emodb/media/.../1.0.0/... <-- media files emodb/meta/.../1.0.0/... <-- tables - When :attr:`Repository.backend` equals ``file-system``, - it creates an instance of :class:`audbackend.backend.FileSystem` - and wraps an :class:`audbackend.interface.Versioned` interface + Otherwise, + it wraps an :class:`audbackend.interface.Versioned` interface around it. The files will then be stored - with the following structure on the Artifactory backend + with the following structure on the backend (shown by the example of version 1.0.0 of the emodb dataset):: emodb/1.0.0/db.yaml <-- header diff --git a/pyproject.toml b/pyproject.toml index 653f2e54..1eab7c0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ classifiers = [ ] requires-python = '>=3.9' dependencies = [ - 'audbackend[artifactory] >=2.0.0', + 'audbackend[all] @ git+https://github.com/audeering/audbackend.git@minio', 'audeer >=2.1.0', 'audformat >=1.2.0', 'audiofile >=1.0.0', diff --git a/tests/conftest.py b/tests/conftest.py index fc7052f9..cea6dedb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,10 @@ import audb +PUBLIC_HOST = "s3.dualstack.eu-north-1.amazonaws.com" +PUBLIC_BACKEND = "minio" + + if platform.system() == "Darwin": # Avoid multi-threading on MacOS runner, # as it might fail from time to time @@ -185,25 +189,21 @@ def persistent_repository(tmpdir_factory): @pytest.fixture(scope="module", autouse=False) def private_and_public_repository(): - r"""Private and public repository on Artifactory. + r"""Private and public repository. Configure the following repositories: - * data-private: repo on public Artifactory without access - * data-public: repo on public Artifactory with anonymous access - * data-public2: repo on public Artifactory with anonymous access + * audb-private: repo without access + * audb-public: repo with anonymous access Note, that the order of the repos is important. audb will visit the repos in the given order until it finds the requested database. """ - host = "https://audeering.jfrog.io/artifactory" - backend = "artifactory" current_repositories = audb.config.REPOSITORIES audb.config.REPOSITORIES = [ - audb.Repository("data-private", host, backend), - audb.Repository("data-public", host, backend), - audb.Repository("data-public2", host, backend), + audb.Repository("audb-private", PUBLIC_HOST, PUBLIC_BACKEND), + audb.Repository("audb-public", PUBLIC_HOST, PUBLIC_BACKEND), ] yield repository @@ -213,23 +213,21 @@ def private_and_public_repository(): @pytest.fixture(scope="module", autouse=False) def non_existing_repository(): - r"""Non-existing repository on Artifactory. + r"""Non-existing repository. Configure the following repositories: * non-existing: non-exsiting repo on public Artifactory - * data-public: repo on public Artifactory with anonymous access + * audb-public: repo on public Artifactory with anonymous access Note, that the order of the repos is important. audb will visit the repos in the given order until it finds the requested database. """ - host = "https://audeering.jfrog.io/artifactory" - backend = "artifactory" current_repositories = audb.config.REPOSITORIES audb.config.REPOSITORIES = [ - audb.Repository("non-existing", host, backend), - audb.Repository("data-public", host, backend), + audb.Repository("non-existing", PUBLIC_HOST, PUBLIC_BACKEND), + audb.Repository("audb-public", PUBLIC_HOST, PUBLIC_BACKEND), ] yield repository