diff --git a/.github/workflows/ci_action.yml b/.github/workflows/ci_action.yml index cecd3633..9bf3302a 100644 --- a/.github/workflows/ci_action.yml +++ b/.github/workflows/ci_action.yml @@ -108,7 +108,7 @@ jobs: pytest -m "not sh_integration and not aws_integration" - name: Upload code coverage - if: ${{ matrix.full_test_suite }} + if: ${{ matrix.full_test_suite && github.event_name == 'push' }} uses: codecov/codecov-action@v2 with: files: coverage.xml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3a4c2167..44b000bb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: language_version: python3 - repo: https://github.com/pycqa/isort - rev: 5.11.4 + rev: 5.12.0 hooks: - id: isort name: isort (python) @@ -46,7 +46,7 @@ repos: - flake8-typing-imports - repo: https://github.com/nbQA-dev/nbQA - rev: 1.6.0 + rev: 1.6.1 hooks: - id: nbqa-black - id: nbqa-isort diff --git a/requirements-dev.txt b/requirements-dev.txt index c5b400cf..bf806516 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,6 +5,7 @@ codecov fs moto mypy>=0.990 +pandas pre-commit pylint>=2.14.0 pytest>=4.0.0 diff --git a/sentinelhub/_version.py b/sentinelhub/_version.py index f338eae5..fa213ace 100644 --- a/sentinelhub/_version.py +++ b/sentinelhub/_version.py @@ -1,3 +1,3 @@ """Version of the sentinelhub package.""" -__version__ = "3.8.1" +__version__ = "3.8.2" diff --git a/sentinelhub/api/batch/process.py b/sentinelhub/api/batch/process.py index 447f563e..052c04db 100644 --- a/sentinelhub/api/batch/process.py +++ b/sentinelhub/api/batch/process.py @@ -14,6 +14,7 @@ from ...constants import RequestType from ...data_collections import DataCollection +from ...exceptions import deprecated_function from ...geometry import CRS, BBox, Geometry from ...types import Json, JsonDict from ..base import BaseCollection, SentinelHubFeatureIterator @@ -358,6 +359,7 @@ def get_tile(self, batch_request: BatchRequestType, tile_id: Optional[int]) -> J url = self._get_tiles_url(request_id, tile_id=tile_id) return self.client.get_json_dict(url, use_session=True) + @deprecated_function(message_suffix="The service endpoint will be removed soon. Please use `restart_job` instead.") def reprocess_tile(self, batch_request: BatchRequestType, tile_id: Optional[int]) -> Json: """Reprocess a single failed tile diff --git a/sentinelhub/constants.py b/sentinelhub/constants.py index e4b660d6..569fa027 100644 --- a/sentinelhub/constants.py +++ b/sentinelhub/constants.py @@ -16,9 +16,10 @@ from aenum import extend_enum from ._version import __version__ -from .exceptions import SHUserWarning +from .exceptions import SHUserWarning, deprecated_class +@deprecated_class() class PackageProps: """Class for obtaining package properties. Currently, it supports obtaining package version.""" @@ -115,7 +116,7 @@ def _parse_crs(value: object) -> object: """ if isinstance(value, dict) and "init" in value: value = value["init"] - if isinstance(value, pyproj.CRS): + if hasattr(value, "to_epsg"): if value == CRSMeta._UNSUPPORTED_CRS: message = ( "sentinelhub-py supports only WGS 84 coordinate reference system with " @@ -148,7 +149,7 @@ def _parse_crs(value: object) -> object: value = match.group("code") if value.upper() == "CRS84": return "4326" - return value.lower().strip("epsg: ") + return value.lower().replace("epsg:", "").strip() return value @@ -451,4 +452,4 @@ class SHConstants: """ LATEST = "latest" - HEADERS = {"User-Agent": f"sentinelhub-py/v{PackageProps.get_version()}"} + HEADERS = {"User-Agent": f"sentinelhub-py/v{__version__}"} diff --git a/sentinelhub/download/client.py b/sentinelhub/download/client.py index 173c9b4b..e489998e 100644 --- a/sentinelhub/download/client.py +++ b/sentinelhub/download/client.py @@ -4,10 +4,10 @@ import json import logging import os -import sys import warnings -from concurrent.futures import Future, ThreadPoolExecutor, as_completed -from typing import Any, List, Optional, Union, overload +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import nullcontext +from typing import Any, Iterable, List, Optional, Union from xml.etree import ElementTree import requests @@ -19,6 +19,7 @@ DownloadFailedException, HashedNameCollisionException, MissingDataInRequestException, + SHDeprecationWarning, SHRuntimeWarning, ) from ..io_utils import read_data @@ -56,77 +57,60 @@ def __init__( self.config = config or SHConfig() - @overload def download( self, - download_requests: DownloadRequest, - max_threads: Optional[int] = None, - decode_data: bool = True, - show_progress: bool = False, - ) -> Any: - ... - - @overload - def download( - self, - download_requests: List[DownloadRequest], + download_requests: Iterable[DownloadRequest], max_threads: Optional[int] = None, decode_data: bool = True, show_progress: bool = False, ) -> List[Any]: - ... - - def download( - self, - download_requests: Union[DownloadRequest, List[DownloadRequest]], - max_threads: Optional[int] = None, - decode_data: bool = True, - show_progress: bool = False, - ) -> Union[List[Any], Any]: """Download one or multiple requests, provided as a request list. - :param download_requests: A list of requests or a single request to be executed. + :param download_requests: A list of requests to be executed. :param max_threads: Maximum number of threads to be used for download in parallel. The default is `max_threads=None` which will use the number of processors on the system multiplied by 5. :param decode_data: If `True` it will decode data otherwise it will return it in form of a `DownloadResponse` objects which contain binary data and response metadata. :param show_progress: Whether a progress bar should be displayed while downloading - :return: A list of results or a single result, depending on input parameter `download_requests` + :return: A list of results """ - downloads = [download_requests] if isinstance(download_requests, DownloadRequest) else download_requests + if isinstance(download_requests, DownloadRequest): + warnings.warn( + ( + "The parameter `download_requests` should be a sequence of requests. In future versions download of" + " single requests will only be supported if provided as a singelton tuple or list." + ), + category=SHDeprecationWarning, + ) + requests_list: List[DownloadRequest] = [download_requests] + else: + requests_list = list(download_requests) - data_list = [None] * len(downloads) + results = [None] * len(requests_list) single_download_method = self._single_download_decoded if decode_data else self._single_download + with ThreadPoolExecutor(max_workers=max_threads) as executor: - download_list = [executor.submit(single_download_method, request) for request in downloads] + download_list = [executor.submit(single_download_method, request) for request in requests_list] future_order = {future: i for i, future in enumerate(download_list)} - # Consider using tqdm.contrib.concurrent.thread_map in the future - if show_progress: - with tqdm(total=len(download_list)) as pbar: - for future in as_completed(download_list): - data_list[future_order[future]] = self._process_download_future(future) - pbar.update(1) - else: + progress_context = tqdm(total=len(download_list)) if show_progress else nullcontext() + with progress_context as progress_bar: for future in as_completed(download_list): - data_list[future_order[future]] = self._process_download_future(future) + try: + results[future_order[future]] = future.result() + except DownloadFailedException as download_exception: + if self.raise_download_errors: + raise download_exception + + warnings.warn(str(download_exception), category=SHRuntimeWarning) + + if progress_bar: + progress_bar.update(1) if isinstance(download_requests, DownloadRequest): - return data_list[0] - return data_list - - def _process_download_future(self, future: Future) -> Any: - """Unpacks the future and correctly handles exceptions""" - try: - return future.result() - except DownloadFailedException as download_exception: - if self.raise_download_errors: - traceback = sys.exc_info()[2] - raise download_exception.with_traceback(traceback) - - warnings.warn(str(download_exception), category=SHRuntimeWarning) - return None + return results[0] # type: ignore[return-value] # will be removed in future version + return results def _single_download_decoded(self, request: DownloadRequest) -> Any: """Downloads a response and decodes it into data. By decoding a single response""" diff --git a/sentinelhub/download/handlers.py b/sentinelhub/download/handlers.py index a2e5a956..a3ba7a3c 100644 --- a/sentinelhub/download/handlers.py +++ b/sentinelhub/download/handlers.py @@ -1,6 +1,7 @@ """ Module implementing error handlers which can occur during download procedure """ +import functools import logging import sys import time @@ -36,6 +37,7 @@ class _HasConfig(Protocol): def fail_user_errors(download_func: Callable[[Self, DownloadRequest], T]) -> Callable[[Self, DownloadRequest], T]: """Decorator function for handling user errors""" + @functools.wraps(download_func) def new_download_func(self: Self, request: DownloadRequest) -> T: try: return download_func(self, request) @@ -58,14 +60,17 @@ def retry_temporary_errors( """Decorator function for handling server and connection errors""" backoff_coefficient = 3 + @functools.wraps(download_func) def new_download_func(self: SelfWithConfig, request: DownloadRequest) -> T: download_attempts = self.config.max_download_attempts sleep_time = self.config.download_sleep_time - for attempt_num in range(download_attempts): + for attempt_idx in range(download_attempts): try: return download_func(self, request) + except requests.RequestException as exception: + attempts_left = download_attempts - (attempt_idx + 1) if not ( _is_temporary_problem(exception) or ( @@ -75,14 +80,14 @@ def new_download_func(self: SelfWithConfig, request: DownloadRequest) -> T: ): raise exception from exception - if attempt_num == download_attempts - 1: + if attempts_left <= 0: message = _create_download_failed_message(exception, request.url) raise DownloadFailedException(message, request_exception=exception) from exception LOGGER.debug( "Download attempt failed: %s\n%d attempts left, will retry in %ds", exception, - download_attempts - attempt_num - 1, + attempts_left, sleep_time, ) time.sleep(sleep_time) @@ -98,6 +103,7 @@ def new_download_func(self: SelfWithConfig, request: DownloadRequest) -> T: def fail_missing_file(download_func: Callable[[Self, DownloadRequest], T]) -> Callable[[Self, DownloadRequest], T]: """A decorator for raising an error if a file is missing""" + @functools.wraps(download_func) def new_download_func(self: Self, request: DownloadRequest) -> T: try: return download_func(self, request) diff --git a/sentinelhub/download/rate_limit.py b/sentinelhub/download/rate_limit.py index 9867a0cc..f59df436 100644 --- a/sentinelhub/download/rate_limit.py +++ b/sentinelhub/download/rate_limit.py @@ -3,7 +3,6 @@ """ import time from enum import Enum -from typing import Union from ..types import JsonDict @@ -52,8 +51,7 @@ def register_next(self) -> float: def update(self, headers: dict) -> None: """Update the next possible download time if the service has responded with the rate limit""" - retry_after: float - retry_after = int(headers.get(self.RETRY_HEADER, 0)) + retry_after: float = round(headers.get(self.RETRY_HEADER, 0)) retry_after = retry_after / 1000 if retry_after: @@ -63,7 +61,7 @@ def update(self, headers: dict) -> None: class PolicyBucket: """A class representing Sentinel Hub policy bucket""" - def __init__(self, policy_type: Union[str, PolicyType], policy_payload: JsonDict): + def __init__(self, policy_type: PolicyType, policy_payload: JsonDict): """ :param policy_type: A type of policy :param policy_payload: A dictionary of policy parameters @@ -77,7 +75,7 @@ def __init__(self, policy_type: Union[str, PolicyType], policy_payload: JsonDict # The following is the same as if we would interpret samplingPeriod string self.refill_per_second = 10**9 / policy_payload["nanosBetweenRefills"] - self._content = self.capacity + self.content = self.capacity def __repr__(self) -> str: """Representation of the bucket content""" @@ -86,16 +84,6 @@ def __repr__(self) -> str: f"refill_period={self.refill_period}, refill_per_second={self.refill_per_second})" ) - @property - def content(self) -> float: - """Variable `content` can be accessed as a property""" - return self._content - - @content.setter - def content(self, value: float) -> None: - """Variable `content` can be modified by external classes""" - self._content = value - def count_cost_per_second(self, elapsed_time: float, new_content: float) -> float: """Calculates the cost per second for the bucket given the elapsed time and the new content. @@ -121,9 +109,7 @@ def get_wait_time( expected_content = max(self.content + elapsed_time * self.refill_per_second - overall_completed_cost, 0) if self.is_fixed(): - if expected_content < cost_per_request: - return -1 - return 0 + return -1 if expected_content < cost_per_request else 0 return max(cost_per_request - expected_content + buffer_cost, 0) / self.refill_per_second diff --git a/sentinelhub/testing_utils.py b/sentinelhub/testing_utils.py index 0ec1826d..f44872fa 100644 --- a/sentinelhub/testing_utils.py +++ b/sentinelhub/testing_utils.py @@ -2,7 +2,7 @@ Utility tools for writing unit tests for packages which rely on `sentinelhub-py` """ import os -from typing import Any, Callable, Dict, Optional, Tuple +from typing import Any, Callable, Dict, Optional, Tuple, Union import numpy as np from pytest import approx @@ -24,7 +24,7 @@ def get_output_folder(current_file: str) -> str: def test_numpy_data( data: Optional[np.ndarray] = None, exp_shape: Optional[Tuple[int, ...]] = None, - exp_dtype: Optional[np.dtype] = None, + exp_dtype: Union[None, type, np.dtype] = None, exp_min: Optional[float] = None, exp_max: Optional[float] = None, exp_mean: Optional[float] = None, @@ -43,7 +43,7 @@ def test_numpy_data( def assert_statistics_match( data: np.ndarray, exp_shape: Optional[Tuple[int, ...]] = None, - exp_dtype: Optional[np.dtype] = None, + exp_dtype: Union[None, type, np.dtype] = None, exp_min: Optional[float] = None, exp_max: Optional[float] = None, exp_mean: Optional[float] = None, diff --git a/tests/TestInputs/batch_stat_failed_results.json b/tests/TestInputs/batch_stat_failed_results.json new file mode 100644 index 00000000..ef76d002 --- /dev/null +++ b/tests/TestInputs/batch_stat_failed_results.json @@ -0,0 +1,113 @@ +[ + { + "id": 3, + "identifier": "SI21.FOI.6620269001", + "response": null + }, + { + "id": 4, + "identifier": "SI21.FOI.6681475001", + "error": { + "type": "EXECUTION_ERROR" + } + }, + { + "id": 5, + "identifier": "SI21.FOI.6828863001", + "response": { + "data": [ + { + "interval": { + "from": "2020-06-09T00:00:00Z", + "to": "2020-06-10T00:00:00Z" + }, + "error": { + "type": "EXECUTION_ERROR" + } + }, + { + "interval": { + "from": "2020-06-11T00:00:00Z", + "to": "2020-06-12T00:00:00Z" + }, + "outputs": { + "ndvi": { + "bands": { + "B0": { + "stats": { + "min": 0.28503936529159546, + "max": 0.6191503405570984, + "mean": 0.41071487963199604, + "stDev": 0.06737269465911691, + "sampleCount": 48, + "noDataCount": 0, + "percentiles": { + "50.0": 0.4020618498325348 + } + }, + "histogram": { + "bins": [ + { + "lowEdge": -1.0, + "highEdge": -0.8, + "count": 0 + }, + { + "lowEdge": -0.8, + "highEdge": -0.6, + "count": 0 + }, + { + "lowEdge": -0.6, + "highEdge": -0.399999999999999, + "count": 0 + }, + { + "lowEdge": -0.3999999999999999, + "highEdge": -0.19999999999999996, + "count": 0 + }, + { + "lowEdge": -0.19999999999999996, + "highEdge": 0.0, + "count": 0 + }, + { + "lowEdge": 0.0, + "highEdge": 0.20000000000000018, + "count": 0 + }, + { + "lowEdge": 0.20000000000000018, + "highEdge": 0.40000000000000013, + "count": 24 + }, + { + "lowEdge": 0.40000000000000013, + "highEdge": 0.6000000000000001, + "count": 23 + }, + { + "lowEdge": 0.6000000000000001, + "highEdge": 0.8, + "count": 1 + }, + { + "lowEdge": 0.8, + "highEdge": 1.0, + "count": 0 + } + ], + "overflowCount": 0, + "underflowCount": 0 + } + } + } + } + } + } + ], + "status": "OK" + } + } +] diff --git a/tests/TestInputs/batch_stat_results.json b/tests/TestInputs/batch_stat_results.json new file mode 100644 index 00000000..63481ba9 --- /dev/null +++ b/tests/TestInputs/batch_stat_results.json @@ -0,0 +1,338 @@ +[ + { + "id": 1, + "identifier": "SI21.FOI.7059004001", + "response": { + "data": [ + { + "interval": { + "from": "2020-06-09T00:00:00Z", + "to": "2020-06-10T00:00:00Z" + }, + "outputs": { + "ndvi": { + "bands": { + "B0": { + "stats": { + "min": "NaN", + "max": "NaN", + "mean": "NaN", + "stDev": "NaN", + "sampleCount": 112, + "noDataCount": 112 + }, + "histogram": { + "bins": [ + { + "lowEdge": -1.0, + "highEdge": -0.8, + "count": 0 + }, + { + "lowEdge": -0.8, + "highEdge": -0.6, + "count": 0 + }, + { + "lowEdge": -0.6, + "highEdge": -0.3999999999999999, + "count": 0 + }, + { + "lowEdge": -0.3999999999999999, + "highEdge": -0.19999999999999996, + "count": 0 + }, + { + "lowEdge": -0.19999999999999996, + "highEdge": 0.0, + "count": 0 + }, + { + "lowEdge": 0.0, + "highEdge": 0.20000000000000018, + "count": 0 + }, + { + "lowEdge": 0.20000000000000018, + "highEdge": 0.40000000000000013, + "count": 0 + }, + { + "lowEdge": 0.40000000000000013, + "highEdge": 0.6000000000000001, + "count": 0 + }, + { + "lowEdge": 0.6000000000000001, + "highEdge": 0.8, + "count": 0 + }, + { + "lowEdge": 0.8, + "highEdge": 1.0, + "count": 0 + } + ], + "overflowCount": 0, + "underflowCount": 0 + } + } + } + } + } + }, + { + "interval": { + "from": "2020-06-11T00:00:00Z", + "to": "2020-06-12T00:00:00Z" + }, + "outputs": { + "ndvi": { + "bands": { + "B0": { + "stats": { + "min": 0.23480869829654694, + "max": 0.7734549045562744, + "mean": 0.38401383599010064, + "stDev": 0.11153442042832748, + "sampleCount": 112, + "noDataCount": 3, + "percentiles": { + "50.0": 0.3445783257484436 + } + }, + "histogram": { + "bins": [ + { + "lowEdge": -1.0, + "highEdge": -0.8, + "count": 0 + }, + { + "lowEdge": -0.8, + "highEdge": -0.6, + "count": 0 + }, + { + "lowEdge": -0.6, + "highEdge": -0.3999999999999999, + "count": 0 + }, + { + "lowEdge": -0.3999999999999999, + "highEdge": -0.19999999999999996, + "count": 0 + }, + { + "lowEdge": -0.19999999999999996, + "highEdge": 0.0, + "count": 0 + }, + { + "lowEdge": 0.0, + "highEdge": 0.20000000000000018, + "count": 0 + }, + { + "lowEdge": 0.20000000000000018, + "highEdge": 0.40000000000000013, + "count": 68 + }, + { + "lowEdge": 0.40000000000000013, + "highEdge": 0.6000000000000001, + "count": 35 + }, + { + "lowEdge": 0.6000000000000001, + "highEdge": 0.8, + "count": 6 + }, + { + "lowEdge": 0.8, + "highEdge": 1.0, + "count": 0 + } + ], + "overflowCount": 0, + "underflowCount": 0 + } + } + } + } + } + } + ], + "status": "OK" + } + }, + { + "id": 2, + "identifier": "SI21.FOI.7059009001", + "response": { + "data": [ + { + "interval": { + "from": "2020-06-09T00:00:00Z", + "to": "2020-06-10T00:00:00Z" + }, + "outputs": { + "ndvi": { + "bands": { + "B0": { + "stats": { + "min": "NaN", + "max": "NaN", + "mean": "NaN", + "stDev": "NaN", + "sampleCount": 48, + "noDataCount": 48 + }, + "histogram": { + "bins": [ + { + "lowEdge": -1.0, + "highEdge": -0.8, + "count": 0 + }, + { + "lowEdge": -0.8, + "highEdge": -0.6, + "count": 0 + }, + { + "lowEdge": -0.6, + "highEdge": -0.3999999999999999, + "count": 0 + }, + { + "lowEdge": -0.3999999999999999, + "highEdge": -0.19999999999999996, + "count": 0 + }, + { + "lowEdge": -0.19999999999999996, + "highEdge": 0.0, + "count": 0 + }, + { + "lowEdge": 0.0, + "highEdge": 0.20000000000000018, + "count": 0 + }, + { + "lowEdge": 0.20000000000000018, + "highEdge": 0.40000000000000013, + "count": 0 + }, + { + "lowEdge": 0.40000000000000013, + "highEdge": 0.6000000000000001, + "count": 0 + }, + { + "lowEdge": 0.6000000000000001, + "highEdge": 0.8, + "count": 0 + }, + { + "lowEdge": 0.8, + "highEdge": 1.0, + "count": 0 + } + ], + "overflowCount": 0, + "underflowCount": 0 + } + } + } + } + } + }, + { + "interval": { + "from": "2020-06-11T00:00:00Z", + "to": "2020-06-12T00:00:00Z" + }, + "outputs": { + "ndvi": { + "bands": { + "B0": { + "stats": { + "min": 0.28503936529159546, + "max": 0.6191503405570984, + "mean": 0.41071487963199604, + "stDev": 0.06737269465911691, + "sampleCount": 48, + "noDataCount": 0, + "percentiles": { + "50.0": 0.4020618498325348 + } + }, + "histogram": { + "bins": [ + { + "lowEdge": -1.0, + "highEdge": -0.8, + "count": 0 + }, + { + "lowEdge": -0.8, + "highEdge": -0.6, + "count": 0 + }, + { + "lowEdge": -0.6, + "highEdge": -0.3999999999999999, + "count": 0 + }, + { + "lowEdge": -0.3999999999999999, + "highEdge": -0.19999999999999996, + "count": 0 + }, + { + "lowEdge": -0.19999999999999996, + "highEdge": 0.0, + "count": 0 + }, + { + "lowEdge": 0.0, + "highEdge": 0.20000000000000018, + "count": 0 + }, + { + "lowEdge": 0.20000000000000018, + "highEdge": 0.40000000000000013, + "count": 24 + }, + { + "lowEdge": 0.40000000000000013, + "highEdge": 0.6000000000000001, + "count": 23 + }, + { + "lowEdge": 0.6000000000000001, + "highEdge": 0.8, + "count": 1 + }, + { + "lowEdge": 0.8, + "highEdge": 1.0, + "count": 0 + } + ], + "overflowCount": 0, + "underflowCount": 0 + } + } + } + } + } + } + ], + "status": "OK" + } + } +] diff --git a/tests/download/test_client.py b/tests/download/test_client.py index ef9aec40..9d671451 100644 --- a/tests/download/test_client.py +++ b/tests/download/test_client.py @@ -16,7 +16,7 @@ def download_request_fixture(output_folder: str) -> DownloadRequest: return DownloadRequest( url="https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/1/C/CV/2017/1/14/0/tileInfo.json", headers={"Content-Type": MimeType.JSON.get_string()}, - data_type="json", + data_type=MimeType.JSON, save_response=True, data_folder=output_folder, filename=None, @@ -24,28 +24,22 @@ def download_request_fixture(output_folder: str) -> DownloadRequest: ) -def test_single_download(download_request: DownloadRequest) -> None: +@pytest.mark.parametrize("num_requests", [0, 1, 2]) +@pytest.mark.parametrize("decode", [True, False]) +def test_download_return_values(num_requests: int, decode: bool, download_request: DownloadRequest) -> None: client = DownloadClient(redownload=False) + requests = [copy.copy(download_request) for _ in range(num_requests)] - result = client.download(download_request) + results = client.download(requests, decode_data=decode) - assert isinstance(result, dict) + expected_element_type = dict if decode else DownloadResponse + assert isinstance(results, list) and len(results) == len(requests) + assert all(isinstance(result, expected_element_type) for result in results) - request_path, response_path = download_request.get_storage_paths() - assert os.path.isfile(request_path) - assert os.path.isfile(response_path) - - -def test_download_without_decode_data(download_request: DownloadRequest) -> None: - client = DownloadClient(redownload=False) - - response = client.download(download_request, decode_data=False) - assert isinstance(response, DownloadResponse) - - responses = client.download([download_request, download_request], decode_data=False) - assert isinstance(responses, list) - assert len(responses) == 2 - assert all(isinstance(resp, DownloadResponse) for resp in responses) + for req in requests: + request_path, response_path = req.get_storage_paths() + assert request_path is not None and os.path.isfile(request_path) + assert response_path is not None and os.path.isfile(response_path) def test_download_with_custom_filename(download_request: DownloadRequest) -> None: @@ -56,7 +50,7 @@ def test_download_with_custom_filename(download_request: DownloadRequest) -> Non client = DownloadClient(redownload=False) for _ in range(3): - client.download(download_request) + client.download([download_request]) request_path, response_path = download_request.get_storage_paths() assert request_path is None @@ -65,7 +59,7 @@ def test_download_with_custom_filename(download_request: DownloadRequest) -> Non @pytest.mark.parametrize("show_progress", [True, False]) -def test_multiple_downloads(download_request: DownloadRequest, show_progress: bool) -> None: +def test_download_with_different_options(download_request: DownloadRequest, show_progress: bool) -> None: client = DownloadClient(redownload=True, raise_download_errors=False) request2 = copy.deepcopy(download_request) @@ -89,23 +83,27 @@ def test_hash_collision(download_request: DownloadRequest) -> None: # Give all requests same hash download_request.get_hashed_name = lambda: "same_hash" - request2 = copy.deepcopy(download_request) - request3 = copy.deepcopy(download_request) - request3.post_values = {"zero": 0} + structurally_same_request = copy.deepcopy(download_request) + structurally_different_request = copy.deepcopy(download_request) + structurally_different_request.post_values = {"zero": 0} - client.download(download_request) - client.download(request2) + client.download([download_request]) + client.download([structurally_same_request]) with pytest.raises(HashedNameCollisionException): - client.download(request3) + client.download([structurally_different_request]) + + +def test_check_cached_request_is_matching(download_request: DownloadRequest) -> None: + """Checks that when the request is saved (and loaded) the method correctly recognizes it's matching. + Ensures no issues with false detections if the jsonification changes values.""" + client = DownloadClient() - # Check that there are no issues with re-loading - request4 = copy.deepcopy(download_request) - request4.post_values = {"transformed-when-saved": [(1, 2)]} + download_request.post_values = {"transformed-when-saved": [(1, 2)]} - request_path, _ = request4.get_storage_paths() - request_info = request4.get_request_params(include_metadata=True) + request_path, _ = download_request.get_storage_paths() + request_info = download_request.get_request_params(include_metadata=True) write_data(request_path, request_info, data_format=MimeType.JSON) # Copied from download client # pylint: disable=protected-access - client._check_cached_request_is_matching(request4, request_path) + client._check_cached_request_is_matching(download_request, request_path) diff --git a/tests/download/test_sentinelhub_statistical_client.py b/tests/download/test_sentinelhub_statistical_client.py index d77e7ad4..ca89f32e 100644 --- a/tests/download/test_sentinelhub_statistical_client.py +++ b/tests/download/test_sentinelhub_statistical_client.py @@ -45,9 +45,9 @@ def test_statistical_client_download_per_interval(download_request: DownloadRequ ], ) - data = client.download(download_request) + data = client.download([download_request]) - assert data == { + assert data[0] == { "data": [ {"interval": {"from": "2020-01-05", "to": "2020-01-05"}, "outputs": 0}, {"interval": {"from": "2020-01-10", "to": "2020-01-10"}, "error": {"type": "BAD_REQUEST"}}, @@ -80,5 +80,5 @@ def test_statistical_client_runs_out_of_retries(download_request: DownloadReques ) with pytest.raises(DownloadFailedException) as exception_info: - client.download(download_request) + client.download([download_request]) assert str(exception_info.value) == "No more interval retries available, download unsuccessful" diff --git a/tests/test_data_utils.py b/tests/test_data_utils.py new file mode 100644 index 00000000..882733ec --- /dev/null +++ b/tests/test_data_utils.py @@ -0,0 +1,35 @@ +import datetime as dt +import os + +import numpy as np +import pytest + +from sentinelhub import read_data +from sentinelhub.data_utils import get_failed_statistical_requests, statistical_to_dataframe + +column_type_pairs = [ + (float, ["ndvi_B0_min", "ndvi_B0_max", "ndvi_B0_mean", "ndvi_B0_stDev", "ndvi_B0_percentiles_50.0"]), + (np.int64, ["ndvi_B0_sampleCount", "ndvi_B0_noDataCount"]), + (list, ["ndvi_B0_bins", "ndvi_B0_counts"]), + (dt.date, ["interval_from", "interval_to"]), + (str, ["identifier"]), +] + + +def test_statistical_to_dataframe(input_folder: str) -> None: + batch_stat_results_path = os.path.join(input_folder, "batch_stat_results.json") + batch_stat_results = read_data(batch_stat_results_path) + df = statistical_to_dataframe(batch_stat_results) + assert len(set(df["identifier"])) == 2, "Wrong number of polygons" + assert len(df.columns) == 12, "Wrong number of columns" + assert len(df) == 2, "Wrong number of valid rows" + for data_type, columns in column_type_pairs: + assert all(isinstance(df[column].iloc[0], data_type) for column in columns), "Wrong data type of columns" + + +@pytest.mark.skip("get_failed_statistical_requests function needs to be fixed") +def test_get_failed_statistical_requests(input_folder: str) -> None: + batch_stat_failed_results_path = os.path.join(input_folder, "batch_stat_failed_results.json") + batch_stat_failed_results = read_data(batch_stat_failed_results_path) + failed_requests = get_failed_statistical_requests(batch_stat_failed_results) + assert len(failed_requests) == 1