Fixes data collection for market liquidity removal by using subgraph instead of filtering on-chain events

packages/packages.json

@@ -3,10 +3,10 @@
         "contract/valory/conditional_tokens/0.1.0": "bafybeibpzhbknohnx5mp653ky5betjfiz3ed3agj554oecsszqq4uxe4qi",
         "contract/valory/fpmm_deterministic_factory/0.1.0": "bafybeif7hgwamqm6iljhtr32ap6eazn5yhmxrtlvicmpo746gv2s4tyzqq",
         "contract/valory/realtio/0.1.0": "bafybeigsciwpawqiyxtk7klpnx6vyaz4nic5s6afl4eahab5ub4hftu4du",
-        "skill/valory/market_creation_manager_abci/0.1.0": "bafybeibmavneaxzrtu2uxnaxmyc627rvetsc5dpibn7dtaevlkrzcv7scu",
-        "skill/valory/market_maker_abci/0.1.0": "bafybeihrilzqkjs2urebi3a5yle2s44fy72pahqfuvhtyn5lnzfzi5xolu",
-        "agent/valory/market_maker/0.1.0": "bafybeihjpzqk6msveftuxgvfcqxytvmisynjdwtjsdal24awgpskqvyta4",
-        "service/valory/market_maker/0.1.0": "bafybeibhlutt7qo3lcpek3poqzpheyjqf5roar2h5k63hc2z76olefh5kq",
+        "skill/valory/market_creation_manager_abci/0.1.0": "bafybeihsc2xenplve2nro63gr7ekerjlcuggxmlihykjzplb3gc6v6fqwy",
+        "skill/valory/market_maker_abci/0.1.0": "bafybeiaj3u2zigypkqazu5p63eysnc33cmapsm7lvnjdwfnsvaogxzf7dq",
+        "agent/valory/market_maker/0.1.0": "bafybeifqlmv5e3nxh4a6uhaamvht3hvm2v5xilbapcbedpcztziuem33xq",
+        "service/valory/market_maker/0.1.0": "bafybeib5zgmrtquhyq6zfnnyikwcddzo3okrrivwhdmzftxlcofmebtrru",
         "contract/valory/wxdai/0.1.0": "bafybeidsp4dptfuxatr75vksdruzvqyhinj4nwiw5f3brzrewldprxuwg4",
         "skill/valory/market_validation_abci/0.1.0": "bafybeifyfwynotmzg6wypbzbwsfnrplhjhumfx32dm2j3ddasp7ygrfoqa",
         "contract/valory/fpmm/0.1.0": "bafybeihoczslmqbfpx57tmjps663r2ib7ycc3dfa4smdhufb4wbnidewtq"

packages/valory/agents/market_maker/aea-config.yaml

@@ -37,9 +37,9 @@ protocols:
 skills:
 - valory/abstract_abci:0.1.0:bafybeiamr3cwetayz6lr72buhq6iviabtboj4hhpq47tlumimxolvvqjcy
 - valory/abstract_round_abci:0.1.0:bafybeiaseziuvbzh3trjggl5hx2tv3fduqrpiwiyksudaf6nvuxmwrg74i
-- valory/market_maker_abci:0.1.0:bafybeihrilzqkjs2urebi3a5yle2s44fy72pahqfuvhtyn5lnzfzi5xolu
+- valory/market_maker_abci:0.1.0:bafybeiaj3u2zigypkqazu5p63eysnc33cmapsm7lvnjdwfnsvaogxzf7dq
 - valory/registration_abci:0.1.0:bafybeibafy7qmcvvzaykimy3mbqdwtcuajmbghqilaa7zwkh6wla33cfce
-- valory/market_creation_manager_abci:0.1.0:bafybeibmavneaxzrtu2uxnaxmyc627rvetsc5dpibn7dtaevlkrzcv7scu
+- valory/market_creation_manager_abci:0.1.0:bafybeihsc2xenplve2nro63gr7ekerjlcuggxmlihykjzplb3gc6v6fqwy
 - valory/market_validation_abci:0.1.0:bafybeifyfwynotmzg6wypbzbwsfnrplhjhumfx32dm2j3ddasp7ygrfoqa
 - valory/reset_pause_abci:0.1.0:bafybeialcwck7fahrr23jckv5qjwg3cdq4ai2ihyjsofnbj44jzyl4cjmm
 - valory/termination_abci:0.1.0:bafybeifqsogqiar4yook5bu3j6z66dbdcizey7dr3e5oxeocdjijvfbaja

packages/valory/services/market_maker/service.yaml

@@ -7,7 +7,7 @@ license: Apache-2.0
 fingerprint:
   README.md: bafybeibwz3af6326msp4h3kqehijvmyhaytvyfbo3o2npc2w4b6zrg6pfq
 fingerprint_ignore_patterns: []
-agent: valory/market_maker:0.1.0:bafybeihjpzqk6msveftuxgvfcqxytvmisynjdwtjsdal24awgpskqvyta4
+agent: valory/market_maker:0.1.0:bafybeifqlmv5e3nxh4a6uhaamvht3hvm2v5xilbapcbedpcztziuem33xq
 number_of_agents: 1
 deployment:
   agent:

packages/valory/skills/market_creation_manager_abci/behaviours.py

@@ -23,6 +23,7 @@
 import json
 import random
 from abc import ABC
+from string import Template
 from typing import Any, Dict, Generator, List, Optional, Set, Tuple, Type, cast
 
 from packages.valory.connections.openai.connection import (
@@ -112,6 +113,35 @@
 
 ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"
 
+CONDITIONS_QUERY = Template(
+    """{
+        conditions(where:{id_in:$conditions}){
+        id,
+        oracle,
+        questionId,
+        outcomeSlotCount
+    }
+}
+"""
+)
+
+QUESTIONS_QUERY = Template(
+    """{
+        questions(where:{id_in:$conditions}){
+        id,
+        openingTimestamp
+    }
+}"""
+)
+
+
+def to_content(query: str) -> bytes:
+    """Convert the given query string to payload content, i.e., add it under a `queries` key and convert it to bytes."""
+    finalized_query = {"query": query}
+    encoded_query = json.dumps(finalized_query, sort_keys=True).encode("utf-8")
+
+    return encoded_query
+
 
 def parse_date_timestring(string: str) -> Optional[datetime.datetime]:
     """Parse and return a datetime string."""
@@ -364,11 +394,12 @@ def get_markets_to_removal_ts(
         condition_id_to_market = {}
         for market in markets:
             for condition_id in market["condition_ids"]:
-                condition_id_to_market[condition_id] = market
+                condition_id_to_market[f"0x{condition_id.hex()}"] = market
         condition_ids = list(condition_id_to_market.keys())
         condition_preparations = yield from self._get_condition_preparation_events(
-            condition_ids, from_block
+            condition_ids=condition_ids
         )
+        self.context.logger.info(f"Conditions: {condition_preparations}")
         if condition_preparations is None:
             # something went wrong
             return None
@@ -390,7 +421,8 @@ def get_markets_to_removal_ts(
         question_ids = [
             preparation["question_id"] for preparation in condition_preparations
         ]
-        questions = yield from self._get_questions(question_ids, from_block)
+        self.context.logger.info(f"Questions: {question_ids}")
+        questions = yield from self._get_questions(question_ids=question_ids)
         if questions is None:
             # something went wrong
             return None
@@ -474,51 +506,36 @@ def _get_markets_with_funds(
         return markets_with_liq
 
     def _get_condition_preparation_events(
-        self,
-        condition_ids: List[bytes],
-        from_block: int,
+        self, condition_ids: List[str]
     ) -> Generator[None, None, Optional[List[Dict[str, Any]]]]:
         """Get condition preparation events."""
-        contract_api_response = yield from self.get_contract_api_response(
-            performative=ContractApiMessage.Performative.GET_STATE,
-            contract_address=self.params.conditional_tokens_contract,
-            contract_id=str(ConditionalTokensContract.contract_id),
-            contract_callable="get_condition_preparation_events",
-            condition_ids=condition_ids,
-            from_block=from_block,
-        )
-        if contract_api_response.performative != ContractApiMessage.Performative.STATE:
-            self.context.logger.error(
-                f"Failed to get condition preparation events: {contract_api_response}"
-            )
-            return None
-        condition_preparation_events = cast(
-            Optional[List[Dict[str, Any]]],
-            contract_api_response.state.body.get("data", []),
+        query = CONDITIONS_QUERY.substitute(conditions=json.dumps(condition_ids))
+        response = yield from self.get_http_response(
+            content=to_content(query),
+            **self.context.omen_subgraph.get_spec(),
         )
+        data = json.loads(response.body.decode())
+        condition_preparation_events = data["data"]["conditions"]
+        for condition in condition_preparation_events:
+            condition["condition_id"] = condition.pop("id")
+            condition["question_id"] = condition.pop("questionId")
+            condition["outcome_slot_count"] = condition.pop("outcomeSlotCount")
         return condition_preparation_events
 
     def _get_questions(
-        self, question_ids: List[str], from_block: int
+        self, question_ids: List[str]
     ) -> Generator[None, None, Optional[List[Dict[str, Any]]]]:
         """Get question ids."""
-        contract_api_response = yield from self.get_contract_api_response(
-            performative=ContractApiMessage.Performative.GET_STATE,
-            contract_address=self.params.realitio_contract,
-            contract_id=str(RealtioContract.contract_id),
-            contract_callable="get_question_events",
-            question_ids=question_ids,
-            from_block=from_block,
-        )
-        if contract_api_response.performative != ContractApiMessage.Performative.STATE:
-            self.context.logger.error(
-                f"Failed to get question: {contract_api_response}"
-            )
-            return None
-        questions = cast(
-            Optional[List[Dict[str, Any]]],
-            contract_api_response.state.body.get("data", []),
+        query = QUESTIONS_QUERY.substitute(conditions=json.dumps(question_ids))
+        response = yield from self.get_http_response(
+            content=to_content(query),
+            **self.context.omen_subgraph.get_spec(),
         )
+        data = json.loads(response.body.decode())
+        questions = data["data"]["questions"]
+        for condition in questions:
+            condition["question_id"] = condition.pop("id")
+            condition["opening_ts"] = int(condition.pop("openingTimestamp"))
         return questions
 
 

packages/valory/skills/market_creation_manager_abci/models.py

@@ -115,5 +115,9 @@ class RandomnessApi(ApiSpecs):
     """A model for randomness api specifications."""
 
 
+class OmenSubgraph(ApiSpecs):
+    """A model that wraps ApiSpecs for the OMEN's subgraph specifications."""
+
+
 Requests = BaseRequests
 BenchmarkTool = BaseBenchmarkTool

packages/valory/skills/market_creation_manager_abci/skill.yaml

@@ -8,11 +8,11 @@ aea_version: '>=1.0.0, <2.0.0'
 fingerprint:
   .gitignore: bafybeihdfdezgtr3s2lzq5y3oaitfrdy4u4pehionju2bdez35wcjgqx6y
   __init__.py: bafybeibkyjt4covc4yhd22aw7kav45zozk3exdv344emt3lilln64soaxm
-  behaviours.py: bafybeidbfmqndd5ig5gv4jkce7bt7qynm2rxwpp75agvqfwl3tuhinuxye
+  behaviours.py: bafybeig65mpjl52jn6diprmg7bzted2m3xsa3hdfrvktf7qxhnoq4rsxca
   dialogues.py: bafybeicmaufkl7vdomnfciv7lw4536ssld7x4uemdapuhsyvfpd4ncibza
   fsm_specification.yaml: bafybeiay4aepnfdkcsriajxqige65bsszrujin57riubflm7dyw2keapoi
   handlers.py: bafybeietxjfli2i57kb7heoy772rcq2znusl36gg7jjj5g3pddw7egny3q
-  models.py: bafybeidpprylojhopdp4aoxwxri2kp7nmv5f3acdok4n5c6ak3a5wnn5cu
+  models.py: bafybeia226bfyqfnzkkp24wmabaof4umi7lqnjtmhb63goszboqk2krqsm
   payloads.py: bafybeib57gmbwstx646qxvxxl62n6of4vo2ei2nl32ogo5vhi3ukz6ldqe
   rounds.py: bafybeifhj72msvgtp2upzawe4ynyagir4quexvr3lwz4clob3f3b4pqi6i
   tests/__init__.py: bafybeihfxvqnyfly72tbxnnnglshcilm2kanihqnjiasvcz3ec3csw32ti
@@ -172,6 +172,18 @@ models:
       retries: 5
       url: https://drand.cloudflare.com/public/latest
     class_name: RandomnessApi
+  omen_subgraph:
+    args:
+      api_id: omen
+      headers:
+        Content-Type: application/json
+      method: POST
+      parameters: {}
+      response_key: data
+      response_type: list
+      retries: 5
+      url: https://api.thegraph.com/subgraphs/name/protofire/omen-xdai
+    class_name: OmenSubgraph
   requests:
     args: {}
     class_name: Requests

packages/valory/skills/market_maker_abci/models.py

@@ -29,6 +29,9 @@
 from packages.valory.skills.market_creation_manager_abci.models import (
     MarketCreationManagerParams,
 )
+from packages.valory.skills.market_creation_manager_abci.models import (
+    OmenSubgraph as BaseOmenSubgraph,
+)
 from packages.valory.skills.market_creation_manager_abci.models import (
     RandomnessApi as MarketCreationManagerRandomnessApi,
 )
@@ -47,6 +50,7 @@
 Requests = BaseRequests
 BenchmarkTool = BaseBenchmarkTool
 RandomnessApi = MarketCreationManagerRandomnessApi
+OmenSubgraph = BaseOmenSubgraph
 
 
 class SharedState(BaseSharedState):

packages/valory/skills/market_maker_abci/skill.yaml

@@ -13,7 +13,7 @@ fingerprint:
   dialogues.py: bafybeicintyylxt4nd5gcufh3rehbxr5jvdn6un7wvaoel7hvj3dmurlpy
   fsm_specification.yaml: bafybeicvtoez7ahmyl25h66edgjywx5ymxtqnyexzq73ouyu2p6smi3uoa
   handlers.py: bafybeidlgouig5odju36dkjl5vdqx6mqrukaoyn2chnl5dv2ca6owqlecm
-  models.py: bafybeieagk5wihiorhd3nzqgyq7g735ww2bltg7olpkzmstxu2mjkeuazu
+  models.py: bafybeie5hchh34fbyoshkujhdonvv4ye2knivw2xlqboqyr6ooa5egqjri
   tests/__init__.py: bafybeig7noiiz2y2b2scjeaalby56ft24eqj4o433blm2oze2pu3lljp44
 fingerprint_ignore_patterns: []
 connections: []
@@ -23,7 +23,7 @@ skills:
 - valory/abstract_round_abci:0.1.0:bafybeiaseziuvbzh3trjggl5hx2tv3fduqrpiwiyksudaf6nvuxmwrg74i
 - valory/registration_abci:0.1.0:bafybeibafy7qmcvvzaykimy3mbqdwtcuajmbghqilaa7zwkh6wla33cfce
 - valory/reset_pause_abci:0.1.0:bafybeialcwck7fahrr23jckv5qjwg3cdq4ai2ihyjsofnbj44jzyl4cjmm
-- valory/market_creation_manager_abci:0.1.0:bafybeibmavneaxzrtu2uxnaxmyc627rvetsc5dpibn7dtaevlkrzcv7scu
+- valory/market_creation_manager_abci:0.1.0:bafybeihsc2xenplve2nro63gr7ekerjlcuggxmlihykjzplb3gc6v6fqwy
 - valory/market_validation_abci:0.1.0:bafybeifyfwynotmzg6wypbzbwsfnrplhjhumfx32dm2j3ddasp7ygrfoqa
 - valory/termination_abci:0.1.0:bafybeifqsogqiar4yook5bu3j6z66dbdcizey7dr3e5oxeocdjijvfbaja
 - valory/transaction_settlement_abci:0.1.0:bafybeiacwr7p4nhhufoey7uz2jqkegrlykdrmc7mm3rzkvh2mslu66gyle
@@ -169,6 +169,18 @@ models:
       retries: 5
       url: https://drand.cloudflare.com/public/latest
     class_name: RandomnessApi
+  omen_subgraph:
+    args:
+      api_id: omen
+      headers:
+        Content-Type: application/json
+      method: POST
+      parameters: {}
+      response_key: data
+      response_type: list
+      retries: 5
+      url: https://api.thegraph.com/subgraphs/name/protofire/omen-xdai
+    class_name: OmenSubgraph
   requests:
     args: {}
     class_name: Requests