Skip to content

Commit

Permalink
[PT-5294] Improve stac resilience (#611)
Browse files Browse the repository at this point in the history
* Simplify stac client handling and drop unneeded token passing

* Add resilience for stac metadata retrieval

* Reduce duplication

* Version bump and CHANGELOG.md update
  • Loading branch information
javidq authored May 23, 2024
1 parent a1ccfab commit 3b5cdb7
Show file tree
Hide file tree
Showing 15 changed files with 171 additions and 232 deletions.
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,16 @@ You can check your current version with the following command:

For more information, see [UP42 Python package description](https://pypi.org/project/up42-py/).

## 1.0.3a1

**May 23, 2024**
- Added tenacity as dependency.
- Added resilience on `asset::stac_info` and `asset::stac_items`
- Dropped pystac client subclassing
- Cleaned up fixtures
- Improved test coverage
- Dropped unneeded exposure of token


## 1.0.2

Expand Down
19 changes: 17 additions & 2 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "up42-py"
version = "1.0.2"
version = "1.0.3a1"
description = "Python SDK for UP42, the geospatial marketplace and developer platform."
authors = ["UP42 GmbH <[email protected]>"]
license = "https://github.com/up42/up42-py/blob/master/LICENSE"
Expand All @@ -24,6 +24,7 @@ geojson = "3.1.0"
geopandas = "^0.13.2"
pystac-client = "^0.7.2"
pyproj = "^3.6.1"
tenacity = "^8.3.0"


[tool.poetry.dev-dependencies]
Expand Down
60 changes: 0 additions & 60 deletions tests/fixtures/fixtures_asset.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
import datetime
import pathlib

import pystac
import pystac_client
import pytest

from up42 import asset
Expand All @@ -16,32 +13,6 @@ def _asset_mock(auth_mock, requests_mock):
url_asset_info = f"{constants.API_HOST}/v2/assets/{constants.ASSET_ID}/metadata"
requests_mock.get(url=url_asset_info, json=constants.JSON_ASSET)

mock_item_collection = pystac.ItemCollection(
items=[
pystac.Item(
id="test",
geometry=None,
properties={},
bbox=None,
datetime=datetime.datetime.now(),
)
]
)

url_asset_stac_info = f"{constants.API_HOST}/v2/assets/stac/search"

requests_mock.post(
url_asset_stac_info,
[
{"json": constants.JSON_STORAGE_STAC},
{"json": constants.JSON_STORAGE_STAC},
{"json": mock_item_collection.to_dict()},
],
)

# asset stac item
requests_mock.get(url=constants.URL_STAC_CATALOG, json=constants.JSON_STAC_CATALOG_RESPONSE)

# asset update
updated_json_asset = constants.JSON_ASSET.copy()
updated_json_asset["title"] = "some_other_title"
Expand All @@ -54,37 +25,6 @@ def _asset_mock(auth_mock, requests_mock):
json={"url": constants.DOWNLOAD_URL},
)

# stac_info url
mock_client = pystac_client.CollectionClient(
id="up42-storage",
description="UP42 Storage STAC API",
extra_fields={"up42-system:asset_id": constants.ASSET_ID},
extent=pystac.Extent(
spatial=pystac.SpatialExtent(
bboxes=[
[
13.3783333333333,
52.4976111111112,
13.3844444444445,
52.5017222222223,
]
]
),
temporal=pystac.TemporalExtent(
intervals=[
[
datetime.datetime(2021, 5, 31),
datetime.datetime(2021, 5, 31),
]
]
),
),
)
requests_mock.get(
url=f"{constants.API_HOST}/v2/assets/stac/collections/{constants.STAC_COLLECTION_ID}",
json=mock_client.to_dict(),
)

return asset.Asset(auth=auth_mock, asset_id=constants.ASSET_ID)


Expand Down
87 changes: 10 additions & 77 deletions tests/fixtures/fixtures_globals.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@

WEBHOOK_ID = "123"

URL_STAC_CATALOG = "https://api.up42.com/v2/assets/stac/"
URL_STAC_SEARCH = "https://api.up42.com/v2/assets/stac/search"

JSON_ASSET = {
"accountId": "69353acb-f942-423f-8f32-11d6d67caa77",
"createdAt": "2022-12-07T14:25:34.968Z",
Expand Down Expand Up @@ -91,25 +94,7 @@
"empty": False,
}

JSON_STORAGE_STAC = {
"links": [
{
"href": "https://api.up42.com/v2/assets/stac/",
"rel": "root",
"type": "application/json",
},
{
"href": "https://api.up42.com/v2/assets/stac/search",
"rel": "next",
"type": "application/json",
"body": {
"sortby": [{"field": "bbox", "direction": "desc"}],
"filter": {},
"token": "next:12345",
},
"method": "POST",
},
],
STAC_SEARCH_RESPONSE = {
"type": "FeatureCollection",
"features": [
{
Expand All @@ -132,9 +117,10 @@
"type": "application/json",
},
{
"href": "https://api.up42.com/v2/assets/stac/",
"href": URL_STAC_CATALOG,
"rel": "root",
"type": "application/json",
"title": "UP42 Storage",
},
],
"stac_extensions": [
Expand Down Expand Up @@ -181,7 +167,7 @@
}


PYSTAC_MOCK_CLIENT = mock_pystac_client = {
STAC_CATALOG_RESPONSE = {
"conformsTo": [
"https://api.stacspec.org/v1.0.0-rc.1/item-search#sort",
"https://api.stacspec.org/v1.0.0-rc.1/collections",
Expand All @@ -201,12 +187,12 @@
],
"links": [
{
"href": "https://api.up42.com/v2/assets/stac/",
"href": URL_STAC_CATALOG,
"rel": "self",
"type": "application/json",
},
{
"href": "https://api.up42.com/v2/assets/stac/",
"href": URL_STAC_CATALOG,
"rel": "root",
"type": "application/json",
},
Expand All @@ -216,7 +202,7 @@
"type": "application/json",
},
{
"href": "https://api.up42.com/v2/assets/stac/search",
"href": URL_STAC_SEARCH,
"rel": "search",
"type": "application/json",
"method": "POST",
Expand Down Expand Up @@ -285,56 +271,3 @@
}

JSON_BALANCE = {"data": {"balance": 10693}}


URL_STAC_CATALOG = "https://api.up42.com/v2/assets/stac"

JSON_STAC_CATALOG_RESPONSE = {
"type": "Catalog",
"id": "up42-storage",
"stac_version": "1.0.0",
"description": "UP42 Storage STAC API",
"links": [
{
"rel": "root",
"href": "https://api.up42.com/v2/assets/stac",
"type": "application/json",
"title": "UP42 Storage",
},
{
"rel": "data",
"href": "https://api.up42.com/v2/assets/stac/collections",
"type": "application/json",
},
{
"rel": "search",
"href": "https://api.up42.com/v2/assets/stac/search",
"type": "application/json",
"method": "POST",
},
{
"rel": "self",
"href": "https://api.up42.com/v2/assets/stac",
"type": "application/json",
},
],
"stac_extensions": [],
"conformsTo": [
"https://api.stacspec.org/v1.0.0-rc.1/collections",
"https://api.stacspec.org/v1.0.0-rc.1/core",
"http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/features-filter",
"http://www.opengis.net/spec/ogcapi-features-4/1.0/conf/simpletx",
"https://api.stacspec.org/v1.0.0-rc.1/item-search#filter",
"http://www.opengis.net/spec/cql2/1.0/conf/cql2-text",
"https://api.stacspec.org/v1.0.0-rc.1/item-search",
"https://api.stacspec.org/v1.0.0-rc.1/ogcapi-features/extensions/transaction",
"http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core",
"http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/filter",
"http://www.opengis.net/spec/cql2/1.0/conf/basic-cql2",
"https://api.stacspec.org/v1.0.0-rc.1/ogcapi-features",
"http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson",
"https://api.stacspec.org/v1.0.0-rc.1/item-search#sort",
"http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30",
],
"title": "UP42 Storage",
}
6 changes: 2 additions & 4 deletions tests/fixtures/fixtures_storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,14 @@
@pytest.fixture()
def storage_mock(auth_mock, requests_mock):
# pystac client authentication
url_pystac_client = f"{constants.API_HOST}/v2/assets/stac"
requests_mock.get(url=url_pystac_client, json=constants.PYSTAC_MOCK_CLIENT)
requests_mock.get(url=constants.URL_STAC_CATALOG, json=constants.STAC_CATALOG_RESPONSE)

# assets
url_storage_assets = f"{constants.API_HOST}/v2/assets"
requests_mock.get(url=url_storage_assets, json=constants.JSON_ASSETS)

# storage stac
url_storage_stac = f"{constants.API_HOST}/v2/assets/stac/search"
requests_mock.post(url=url_storage_stac, json=constants.JSON_STORAGE_STAC)
requests_mock.post(url=constants.URL_STAC_SEARCH, json=constants.STAC_SEARCH_RESPONSE)

# asset info
url_asset_info = f"{constants.API_HOST}/v2/assets/{constants.ASSET_ID}/metadata"
Expand Down
3 changes: 0 additions & 3 deletions tests/http/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,9 @@ def test_should_create_if_only_one_source_is_given(
detected_settings: List[Optional[Dict]],
):
detect_settings = mock.MagicMock(side_effect=detected_settings)
access_token = "some-token"
retrieve = mock.sentinel.some_object
detect_retriever = mock.MagicMock(return_value=retrieve)
auth = mock.MagicMock()
auth.token.access_token = access_token
create_auth = mock.MagicMock(return_value=auth)
session = mock.MagicMock()
create_session = mock.MagicMock(return_value=session)
Expand All @@ -43,7 +41,6 @@ def test_should_create_if_only_one_source_is_given(
create_auth=create_auth,
create_session=create_session,
)
assert result.token == access_token
assert result.auth == auth
assert result.session == session
detect_settings.assert_has_calls([mock.call(source) for source in sources])
Expand Down
Loading

0 comments on commit 3b5cdb7

Please sign in to comment.