make things async

offchain/metadata/fetchers/metadata_fetcher.py

@@ -63,11 +63,6 @@ def _head(self, uri: str):  # type: ignore[no-untyped-def]
     def _get(self, uri: str):  # type: ignore[no-untyped-def]
         return self.sess.get(uri, timeout=self.timeout, allow_redirects=True)
 
-    async def _gen_head(self, uri: str):  # type: ignore[no-untyped-def]
-        return await self.async_sess.head(
-            uri, timeout=self.timeout, follow_redirects=True
-        )
-
     async def _gen(self, uri: str) -> httpx.Response:
         from offchain.metadata.pipelines.metadata_pipeline import (
             DEFAULT_ADAPTER_CONFIGS,
@@ -86,7 +81,9 @@ async def _gen(self, uri: str) -> httpx.Response:
                 return await adapter.gen_send(
                     url=uri, timeout=self.timeout, sess=self.async_sess
                 )
-        return await self.async_sess.get(uri, timeout=self.timeout)
+        return await self.async_sess.get(
+            uri, timeout=self.timeout, follow_redirects=True
+        )
 
     def fetch_mime_type_and_size(self, uri: str) -> Tuple[str, int]:
         """Fetch the mime type and size of the content at a given uri.
@@ -126,10 +123,11 @@ async def gen_fetch_mime_type_and_size(self, uri: str) -> Tuple[str, int]:
             tuple[str, int]: mime type and size
         """
         try:
-            res = await self._gen_head(uri)
-            # For any error status, try a get
-            if 300 <= res.status_code < 600:
-                res = self._gen(uri)
+            # try skip head request
+            # res = await self._gen_head(uri)
+            # # For any error status, try a get
+            # if 300 <= res.status_code < 600:
+            res = await self._gen(uri)
             res.raise_for_status()
             headers = res.headers
             size = headers.get("content-length", 0)

offchain/metadata/parsers/base_parser.py

@@ -1,3 +1,4 @@
+from abc import abstractmethod
 from typing import Optional, Protocol
 
 import pytest
@@ -52,6 +53,7 @@ def should_parse_token(  # type: ignore[no-untyped-def]
         """
         pass
 
+    @abstractmethod
     async def _gen_parse_metadata_impl(  # type: ignore[no-untyped-def]
         self, token: Token, raw_data: dict, *args, **kwargs  # type: ignore[type-arg]
     ):

offchain/metadata/parsers/collection/artblocks.py

@@ -1,3 +1,4 @@
+import asyncio
 from typing import Optional
 
 from offchain.constants.addresses import CollectionAddress
@@ -248,6 +249,41 @@ def get_image(self, raw_data: dict) -> Optional[MediaDetails]:  # type: ignore[r
             except Exception:
                 pass
 
+    async def gen_image(self, raw_data: dict) -> Optional[MediaDetails]:  # type: ignore[return, type-arg]  # noqa: E501
+        image_uri = raw_data.get("image")
+        if image_uri:
+            image = MediaDetails(uri=image_uri, size=None, sha256=None, mime_type=None)
+            try:
+                content_type, size = await self.fetcher.gen_fetch_mime_type_and_size(
+                    image_uri
+                )
+                image.mime_type = content_type
+                image.size = size
+                return image
+            except Exception:
+                pass
+
+    async def _gen_parse_metadata_impl(
+        self, token: Token, raw_data: dict, *args, **kwargs
+    ):
+        token.uri = f"https://api.artblocks.io/token/{token.token_id}"
+        raw_data, mime_type_and_size = await asyncio.gather(
+            self.fetcher.gen_fetch_content(token.uri),
+            self.fetcher.gen_fetch_mime_type_and_size(token.uri),
+        )
+        mime_type, _ = mime_type_and_size
+        image = await self.gen_image(raw_data=raw_data)
+        return Metadata(
+            token=token,
+            raw_data=raw_data,
+            attributes=self.parse_traits(raw_data),
+            name=raw_data.get("name"),
+            description=raw_data.get("description"),
+            mime_type=mime_type,
+            image=image,
+            additional_fields=self.get_additional_fields(raw_data=raw_data),
+        )
+
     def parse_metadata(self, token: Token, raw_data: dict, *args, **kwargs) -> Optional[Metadata]:  # type: ignore[no-untyped-def, type-arg]  # noqa: E501
         token.uri = f"https://api.artblocks.io/token/{token.token_id}"
 

offchain/metadata/parsers/collection/autoglyphs.py

@@ -1,13 +1,14 @@
+import asyncio
 from typing import Optional
 
 from offchain.constants.addresses import CollectionAddress
 from offchain.metadata.constants.autoglyphs import get_symbol_by_index
 from offchain.metadata.models.metadata import (
-    Metadata,
     Attribute,
+    MediaDetails,
+    Metadata,
     MetadataField,
     MetadataFieldType,
-    MediaDetails,
 )
 from offchain.metadata.models.token import Token
 from offchain.metadata.parsers.collection.collection_parser import CollectionParser
@@ -49,6 +50,19 @@ def get_symbol_scheme(self, index: int) -> Optional[int]:
 
         return results[0]
 
+    async def gen_symbol_scheme(self, index: int) -> Optional[int]:
+        results = await self.contract_caller.rpc.async_reader.gen_call_single_function_single_address_many_args(
+            address=ADDRESS,
+            function_sig="symbolScheme(uint256)",
+            return_type=["uint8"],
+            args=[[index]],
+        )
+
+        if len(results) < 1:
+            return None
+
+        return results[0]
+
     def get_raw_content(self, index: int) -> Optional[str]:
         results = self.contract_caller.single_address_single_fn_many_args(
             address=ADDRESS,
@@ -62,6 +76,19 @@ def get_raw_content(self, index: int) -> Optional[str]:
 
         return results[0]
 
+    async def gen_raw_content(self, index: int) -> Optional[str]:
+        results = await self.contract_caller.rpc.async_reader.gen_call_single_function_single_address_many_args(
+            address=ADDRESS,
+            function_sig="draw(uint256)",
+            return_type=["string"],
+            args=[[index]],
+        )
+
+        if len(results) < 1:
+            return None
+
+        return results[0]
+
     def get_image_details(self, raw_data: dict) -> Optional[MediaDetails]:  # type: ignore[type-arg]  # noqa: E501
         image_uri = raw_data.get("image")
         details = MediaDetails(uri=image_uri, size=None, sha256=None, mime_type=None)
@@ -73,13 +100,31 @@ def get_image_details(self, raw_data: dict) -> Optional[MediaDetails]:  # type:
             pass
         return details
 
+    async def gen_image_details(self, raw_data: dict) -> Optional[MediaDetails]:  # type: ignore[type-arg]  # noqa: E501
+        image_uri = raw_data.get("image")
+        details = MediaDetails(uri=image_uri, size=None, sha256=None, mime_type=None)
+        try:
+            content_type, size = await self.fetcher.gen_fetch_mime_type_and_size(image_uri)  # type: ignore[arg-type]  # noqa: E501
+            details.mime_type = content_type
+            details.size = size
+        except Exception:
+            pass
+        return details
+
     def get_content_details(self, index: int) -> Optional[MediaDetails]:
         raw_uri = self.get_raw_content(index)
         content_uri = self.encode_uri_data(raw_uri)  # type: ignore[arg-type]
         return MediaDetails(
             uri=content_uri, size=None, sha256=None, mime_type="text/plain"
         )  # noqa: E501
 
+    async def gen_content_details(self, index: int) -> Optional[MediaDetails]:
+        raw_uri = await self.gen_raw_content(index)
+        content_uri = self.encode_uri_data(raw_uri)  # type: ignore[arg-type]
+        return MediaDetails(
+            uri=content_uri, size=None, sha256=None, mime_type="text/plain"
+        )  # noqa: E501
+
     def parse_additional_fields(self, raw_data: dict) -> list[MetadataField]:  # type: ignore[type-arg]  # noqa: E501
         additional_fields = []
         if (external_url := raw_data.get("external_url")) is not None:
@@ -119,6 +164,22 @@ def parse_attributes(self, token_id: int) -> list[Attribute]:
 
         return attributes
 
+    async def gen_parse_attributes(self, token_id: int) -> list[Attribute]:
+        attributes = []
+
+        scheme = get_symbol_by_index(await self.gen_symbol_scheme(token_id))  # type: ignore[arg-type]  # noqa: E501
+        if scheme is None:
+            scheme = "Unknown"
+
+        scheme_attribute = Attribute(
+            trait_type="Symbol Scheme",
+            value=scheme,
+            display_type=None,
+        )
+        attributes.append(scheme_attribute)
+
+        return attributes
+
     def parse_metadata(self, token: Token, raw_data: dict, *args, **kwargs) -> Metadata:  # type: ignore[no-untyped-def, type-arg]  # noqa: E501
         raw_data = self.create_raw_data(token.token_id)
 
@@ -133,3 +194,22 @@ def parse_metadata(self, token: Token, raw_data: dict, *args, **kwargs) -> Metad
             content=self.get_content_details(token.token_id),
             additional_fields=self.parse_additional_fields(raw_data),
         )
+
+    async def _gen_parse_metadata_impl(self, token: Token, raw_data: dict, *args, **kwargs) -> Metadata:  # type: ignore[no-untyped-def, type-arg]  # noqa: E501
+        raw_data = self.create_raw_data(token.token_id)
+        attributes, image, content = await asyncio.gather(
+            self.gen_parse_attributes(token.token_id),
+            self.gen_image_details(raw_data),
+            self.gen_content_details(token.token_id),
+        )
+        return Metadata(
+            token=token,
+            raw_data=raw_data,
+            attributes=attributes,
+            name=raw_data.get("name"),
+            description=raw_data.get("description"),
+            mime_type="application/json",
+            image=image,
+            content=content,
+            additional_fields=self.parse_additional_fields(raw_data),
+        )

offchain/metadata/parsers/collection/chainrunners.py

@@ -1,11 +1,13 @@
+import asyncio
 from typing import Optional
 
 from offchain.constants.addresses import CollectionAddress
 from offchain.metadata.models.metadata import Metadata
 from offchain.metadata.models.token import Token
-from offchain.metadata.parsers.collection.collection_parser import CollectionParser
 from offchain.metadata.parsers.catchall.default_catchall import DefaultCatchallParser
+from offchain.metadata.parsers.collection.collection_parser import CollectionParser
 from offchain.metadata.registries.parser_registry import ParserRegistry
+from offchain.utils.utils import nullthrows
 
 ADDRESS = CollectionAddress.CHAINRUNNERS
 
@@ -27,10 +29,36 @@ def get_dna(self, token_id: int) -> Optional[int]:
 
         return results[0]
 
+    async def gen_dna(self, token_id: int) -> Optional[int]:
+        results = await self.contract_caller.rpc.async_reader.gen_call_single_function_single_address_many_args(
+            address=ADDRESS,
+            function_sig="getDna(uint256)",
+            return_type=["uint256"],
+            args=[[token_id]],
+        )
+
+        if len(results) < 1:
+            return None
+
+        return results[0]
+
     def parse_metadata(self, token: Token, raw_data: Optional[dict], *args, **kwargs) -> Optional[Metadata]:  # type: ignore[no-untyped-def, type-arg]  # noqa: E501
         if token.uri is None or raw_data is None:
             dna = self.get_dna(token.token_id)
             token.uri = f"https://api.chainrunners.xyz/tokens/metadata/{token.token_id}?dna={dna}"
             raw_data = self.fetcher.fetch_content(token.uri)  # type: ignore[assignment]
 
         return DefaultCatchallParser(self.fetcher).parse_metadata(token=token, raw_data=raw_data)  # type: ignore[arg-type]  # noqa: E501
+
+    async def _gen_parse_metadata_impl(self, token: Token, raw_data: Optional[dict], *args, **kwargs) -> Optional[Metadata]:  # type: ignore[no-untyped-def, type-arg]  # noqa: E501
+        if token.uri is None or raw_data is None:
+            if token.uri:
+                dna, raw_data = await asyncio.gather(
+                    self.gen_dna(token.token_id),
+                    self.fetcher.gen_fetch_content(nullthrows(token.uri)),
+                )
+            else:
+                dna = None
+            token.uri = f"https://api.chainrunners.xyz/tokens/metadata/{token.token_id}?dna={dna}"
+
+        return await DefaultCatchallParser(self.fetcher).gen_parse_metadata(token=token, raw_data=raw_data)  # type: ignore[arg-type]  # noqa: E501

offchain/metadata/parsers/collection/decentraland.py

@@ -3,8 +3,8 @@
 from offchain.constants.addresses import CollectionAddress
 from offchain.metadata.models.metadata import Metadata, MetadataField, MetadataFieldType
 from offchain.metadata.models.token import Token
-from offchain.metadata.parsers.collection.collection_parser import CollectionParser
 from offchain.metadata.parsers.catchall.default_catchall import DefaultCatchallParser
+from offchain.metadata.parsers.collection.collection_parser import CollectionParser
 from offchain.metadata.registries.parser_registry import ParserRegistry
 
 
@@ -58,3 +58,14 @@ def parse_metadata(self, token: Token, raw_data: Optional[dict], *args, **kwargs
         metadata.mime_type = "application/json"  # type: ignore[union-attr]
 
         return metadata
+
+    async def _gen_parse_metadata_impl(self, token: Token, raw_data: Optional[dict], *args, **kwargs) -> Optional[Metadata]:  # type: ignore[no-untyped-def, type-arg]  # noqa: E501
+        if token.uri is None or raw_data is None:
+            token.uri = f"https://api.decentraland.org/v2/contracts/{token.collection_address.lower()}/tokens/{token.token_id}"
+            raw_data = await self.fetcher.gen_fetch_content(token.uri)  # type: ignore[assignment]
+
+        metadata = await DefaultCatchallParser(self.fetcher).gen_parse_metadata(token=token, raw_data=raw_data)  # type: ignore[arg-type]  # noqa: E501
+        metadata.additional_fields = self.parse_additional_fields(raw_data)  # type: ignore[arg-type, union-attr]  # noqa: E501
+        metadata.mime_type = "application/json"  # type: ignore[union-attr]
+
+        return metadata

offchain/metadata/parsers/collection/ens.py

@@ -1,3 +1,4 @@
+import asyncio
 from typing import Optional
 
 from offchain.constants.addresses import CollectionAddress
@@ -115,3 +116,58 @@ def parse_metadata(self, token: Token, raw_data: dict, *args, **kwargs) -> Optio
             content=self.get_background_image(raw_data=raw_data),
             additional_fields=self.get_additional_fields(raw_data=raw_data),
         )
+
+    async def gen_image(self, raw_data: dict) -> Optional[MediaDetails]:  # type: ignore[return, type-arg]  # noqa: E501
+        image_uri = raw_data.get("image_url") or raw_data.get("image")
+        if image_uri:
+            image = MediaDetails(uri=image_uri, size=None, sha256=None, mime_type=None)
+            try:
+                content_type, size = await self.fetcher.gen_fetch_mime_type_and_size(
+                    image_uri
+                )
+                image.mime_type = content_type
+                image.size = size
+                return image
+            except Exception:
+                pass
+
+    async def gen_background_image(self, raw_data: dict) -> Optional[MediaDetails]:  # type: ignore[return, type-arg]  # noqa: E501
+        bg_image_uri = raw_data.get("background_image")
+        if bg_image_uri:
+            image = MediaDetails(
+                uri=bg_image_uri, size=None, sha256=None, mime_type=None
+            )  # noqa: E501
+            try:
+                content_type, size = await self.fetcher.gen_fetch_mime_type_and_size(
+                    bg_image_uri
+                )
+                image.mime_type = content_type
+                image.size = size
+                return image
+            except Exception:
+                pass
+
+    async def _gen_parse_metadata_impl(self, token: Token, raw_data: dict, *args, **kwargs) -> Optional[Metadata]:  # type: ignore[no-untyped-def, type-arg]  # noqa: E501
+        ens_chain_name = self.make_ens_chain_name(token.chain_identifier)
+
+        token.uri = f"https://metadata.ens.domains/{ens_chain_name}/{token.collection_address.lower()}/{token.token_id}/"
+        raw_data, mime_type_and_size = await asyncio.gather(
+            self.fetcher.gen_fetch_content(token.uri),
+            self.fetcher.gen_fetch_mime_type_and_size(token.uri),
+        )
+        mime_type, _ = mime_type_and_size
+        image, background_image = await asyncio.gather(
+            self.gen_image(raw_data=raw_data),
+            self.gen_background_image(raw_data=raw_data),
+        )
+        return Metadata(
+            token=token,
+            raw_data=raw_data,
+            attributes=self.parse_attributes(raw_data),
+            name=raw_data.get("name"),
+            description=raw_data.get("description"),
+            mime_type=mime_type,
+            image=image,
+            content=background_image,
+            additional_fields=self.get_additional_fields(raw_data=raw_data),
+        )

offchain/metadata/parsers/collection/foundation.py

@@ -3,8 +3,8 @@
 from offchain.constants.addresses import CollectionAddress
 from offchain.metadata.models.metadata import Metadata
 from offchain.metadata.models.token import Token
-from offchain.metadata.parsers.collection.collection_parser import CollectionParser
 from offchain.metadata.parsers.catchall.default_catchall import DefaultCatchallParser
+from offchain.metadata.parsers.collection.collection_parser import CollectionParser
 from offchain.metadata.registries.parser_registry import ParserRegistry
 
 
@@ -22,3 +22,14 @@ def parse_metadata(self, token: Token, raw_data: Optional[dict], *args, **kwargs
             metadata.content.mime_type = "model/gltf-binary"  # type: ignore[union-attr]
 
         return metadata
+
+    async def _gen_parse_metadata_impl(self, token: Token, raw_data: Optional[dict], *args, **kwargs) -> Optional[Metadata]:  # type: ignore[no-untyped-def, type-arg]  # noqa: E501
+        if token.uri is None or raw_data is None:
+            token.uri = f"https://api.foundation.app/opensea/{token.token_id}"
+            raw_data = await self.fetcher.gen_fetch_content(token.uri)  # type: ignore[assignment]
+        metadata = await DefaultCatchallParser(self.fetcher).gen_parse_metadata(token=token, raw_data=raw_data)  # type: ignore[arg-type]  # noqa: E501
+        metadata.standard = None  # type: ignore[union-attr]
+        if metadata.content.uri.endswith("glb"):  # type: ignore[union-attr]
+            metadata.content.mime_type = "model/gltf-binary"  # type: ignore[union-attr]
+
+        return metadata