From 2a9b840782c4a62cbb40f921a725f6a91e8aa6b2 Mon Sep 17 00:00:00 2001 From: Luke Yang Date: Thu, 25 Apr 2024 19:43:30 -0400 Subject: [PATCH 1/2] BACK-2749: support svg image uri in opensea parser --- docs/changelog.md | 4 ++ docs/index.md | 2 +- .../metadata/parsers/collection/artblocks.py | 2 +- offchain/metadata/parsers/collection/ens.py | 2 +- offchain/metadata/parsers/schema/opensea.py | 7 +++ pyproject.toml | 2 +- .../parsers/test_default_catchall_parser.py | 2 +- tests/metadata/parsers/test_opensea_parser.py | 45 ++++++++++++++++++- 8 files changed, 59 insertions(+), 7 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index d1776e1..cdd2bf8 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,9 @@ # Changelog +## v0.3.3 + +- Fix an issue in `OpenseaParser` where the plain-text svg wouldn't be recognized as valid image uri + ## v0.3.2 - Fix an issue in `DataURIAdapter` where plain-text json data uri would get ignored diff --git a/docs/index.md b/docs/index.md index 8630f6b..4b22746 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,6 @@ # Getting Started -Documentation for version: **v0.3.2** +Documentation for version: **v0.3.3** ## Overview diff --git a/offchain/metadata/parsers/collection/artblocks.py b/offchain/metadata/parsers/collection/artblocks.py index a8068f5..118f537 100644 --- a/offchain/metadata/parsers/collection/artblocks.py +++ b/offchain/metadata/parsers/collection/artblocks.py @@ -224,7 +224,7 @@ def get_additional_fields(self, raw_data: dict) -> list[MetadataField]: # type: return additional_fields - def parse_traits(self, raw_data: dict) -> Optional[list[Attribute]]: # type: ignore[type-arg] # noqa: E501 + def parse_traits(self, raw_data: dict) -> list[Attribute]: # type: ignore[type-arg] # noqa: E501 traits = raw_data.get("traits") if not traits or not isinstance(traits, list): return # type: ignore[return-value] diff --git a/offchain/metadata/parsers/collection/ens.py b/offchain/metadata/parsers/collection/ens.py index 9110898..b9f92b7 100644 --- a/offchain/metadata/parsers/collection/ens.py +++ b/offchain/metadata/parsers/collection/ens.py @@ -58,7 +58,7 @@ def get_additional_fields(self, raw_data: dict) -> list[MetadataField]: # type: ) return additional_fields - def parse_attributes(self, raw_data: dict) -> Optional[list[Attribute]]: # type: ignore[type-arg] # noqa: E501 + def parse_attributes(self, raw_data: dict) -> list[Attribute]: # type: ignore[type-arg] # noqa: E501 attributes = raw_data.get("attributes") if not attributes or not isinstance(attributes, list): return # type: ignore[return-value] diff --git a/offchain/metadata/parsers/schema/opensea.py b/offchain/metadata/parsers/schema/opensea.py index 5a207ce..fb27bf1 100644 --- a/offchain/metadata/parsers/schema/opensea.py +++ b/offchain/metadata/parsers/schema/opensea.py @@ -1,3 +1,4 @@ +import base64 from typing import Optional from offchain.metadata.models.metadata import ( @@ -90,6 +91,9 @@ def parse_metadata(self, token: Token, raw_data: dict, *args, **kwargs) -> Optio image = None image_uri = raw_data.get("image") or raw_data.get("image_data") if image_uri: + if image_uri.startswith(""] readme = "README.md" diff --git a/tests/metadata/parsers/test_default_catchall_parser.py b/tests/metadata/parsers/test_default_catchall_parser.py index ae0e694..39b0d5f 100644 --- a/tests/metadata/parsers/test_default_catchall_parser.py +++ b/tests/metadata/parsers/test_default_catchall_parser.py @@ -14,7 +14,7 @@ class TestDefaultCatchallParser: token = Token( chain_identifier="ETHEREUM-MAINNET", collection_address="0x74cb086a1611cc9ca672f458b7742dd4159ac9db", - token_id="80071", + token_id=80071, uri="https://api.dego.finance/gego-token-v2/80071", ) raw_data = { diff --git a/tests/metadata/parsers/test_opensea_parser.py b/tests/metadata/parsers/test_opensea_parser.py index 6fe4b55..f7c00fb 100644 --- a/tests/metadata/parsers/test_opensea_parser.py +++ b/tests/metadata/parsers/test_opensea_parser.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock import pytest +import base64 from offchain.metadata.adapters.ipfs import IPFSAdapter from offchain.metadata.fetchers.metadata_fetcher import MetadataFetcher @@ -12,7 +13,6 @@ Metadata, MetadataField, MetadataFieldType, - MetadataStandard, ) from offchain.metadata.models.token import Token from offchain.metadata.parsers.schema.opensea import OpenseaParser @@ -22,7 +22,7 @@ class TestOpenseaParser: token = Token( chain_identifier="ETHEREUM-MAINNET", collection_address="0x5180db8f5c931aae63c74266b211f580155ecac8", - token_id="1", + token_id=1, uri="ipfs://QmSr3vdMuP2fSxWD7S26KzzBWcAN1eNhm4hk1qaR3x3vmj/1.json", ) @@ -229,3 +229,44 @@ async def test_opensea_parser_gen_parses_metadata(self, raw_crypto_coven_metadat token=self.token, raw_data=raw_crypto_coven_metadata ) assert metadata + + @pytest.mark.asyncio + async def test_opensea_parser_parses_token_with_xml_image(self): + parser = OpenseaParser(fetcher=MetadataFetcher()) # type: ignore[abstract] + token = Token( + chain_identifier="BASE-MAINNET", + collection_address="0x00000000001594c61dd8a6804da9ab58ed2483ce", + token_id=91107139416293979998100172630436458595092238971, + uri="https://metadata.nfts2me.com/api/ownerTokenURI/8453/91107139416293979998100172630436458595092238971/574759207385280074438303243253258373278259074888/10000/", + ) + raw_data = { + "name": "NFTs2Me Collection Owner - drako", + "description": "Represents **Ownership of the NFTs2Me Collection** with address '[0x0fF562Ab42325222cF72971d32ED9CDF373b927B](https://0x0fF562Ab42325222cF72971d32ED9CDF373b927B_8453.nfts2.me/)'.\n\nTransferring this NFT implies changing the owner of the collection, as well as who will receive 100% of the profits from primary and secondary sales.\n\n[NFTs2Me](https://nfts2me.com/) is a showcase of unique digital creations from talented creators who have used the tool to generate their own NFT projects. These projects range from digital art and collectibles to gaming items and more, all with the added value of being verified on the blockchain. With a wide range of styles and themes, the [NFTs2Me](https://nfts2me.com/) tool offers something for every fan of the growing NFT space.", + "image_data": ' COLLECTION • drako COLLECTION • drako OWNER • 0x64Ad181f69466bD4D15076aC1d33a22c6Cc9d748 OWNER • 0x64Ad181f69466bD4D15076aC1d33a22c6Cc9d748 ', + "background_color": "#FFFFFF", + "attributes": [ + {"trait_type": "Collection Name", "value": "drako"}, + { + "trait_type": "Collection Address", + "value": "0x0fF562Ab42325222cF72971d32ED9CDF373b927B", + }, + { + "trait_type": "Owner Address", + "value": "0x64Ad181f69466bD4D15076aC1d33a22c6Cc9d748", + }, + {"display_type": "number", "trait_type": "Revenue", "value": 100}, + ], + "external_url": "https://0x0fF562Ab42325222cF72971d32ED9CDF373b927B_8453.nfts2.me/", + } + metadata = await parser._gen_parse_metadata_impl(token=token, raw_data=raw_data) + svg_encoded = base64.b64encode( + raw_data.get("image_data").encode("utf-8") + ).decode("utf-8") + expected_image_uri = f"data:image/svg+xml;base64,{svg_encoded}" + assert metadata + assert metadata.image == MediaDetails( + size=3256, + sha256=None, + uri=expected_image_uri, + mime_type="image/svg+xml", + ) From 87faddd4880a4f00ea5ea7eb50ccc6e2f7975839 Mon Sep 17 00:00:00 2001 From: Luke Yang Date: Thu, 25 Apr 2024 21:14:37 -0400 Subject: [PATCH 2/2] BACK-2735: DefaultCatchallParser skips unparsible raw data --- docs/changelog.md | 1 + offchain/metadata/parsers/catchall/default_catchall.py | 5 ++++- tests/metadata/parsers/test_default_catchall_parser.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index cdd2bf8..8876290 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,7 @@ ## v0.3.3 - Fix an issue in `OpenseaParser` where the plain-text svg wouldn't be recognized as valid image uri +- Add check in `DefaultCatchallParser` to require that `raw_data` be a `dict` ## v0.3.2 diff --git a/offchain/metadata/parsers/catchall/default_catchall.py b/offchain/metadata/parsers/catchall/default_catchall.py index 43f9fa4..0bdda1e 100644 --- a/offchain/metadata/parsers/catchall/default_catchall.py +++ b/offchain/metadata/parsers/catchall/default_catchall.py @@ -242,4 +242,7 @@ def should_parse_token( # type: ignore[no-untyped-def] Returns: bool: whether or not the collection parser handles this token. """ - return bool(token.uri and raw_data) + + if raw_data is not None and not isinstance(raw_data, dict): + logger.info("DefaultCatchallParser skips token {token} due to invalid raw data") + return bool(token.uri and raw_data is not None and isinstance(raw_data, dict)) diff --git a/tests/metadata/parsers/test_default_catchall_parser.py b/tests/metadata/parsers/test_default_catchall_parser.py index 39b0d5f..dcdef25 100644 --- a/tests/metadata/parsers/test_default_catchall_parser.py +++ b/tests/metadata/parsers/test_default_catchall_parser.py @@ -5,7 +5,7 @@ import pytest from offchain.metadata.fetchers.metadata_fetcher import MetadataFetcher -from offchain.metadata.models.metadata import Metadata, MetadataStandard +from offchain.metadata.models.metadata import Metadata from offchain.metadata.models.token import Token from offchain.metadata.parsers.catchall.default_catchall import DefaultCatchallParser