From d1569e595f9453971d1d6b1027109df4afcc75a7 Mon Sep 17 00:00:00 2001 From: angrybayblade Date: Wed, 2 Aug 2023 14:45:55 +0000 Subject: [PATCH] fix: data collection for market liqudity removal --- packages/packages.json | 8 +- .../agents/market_maker/aea-config.yaml | 4 +- .../valory/services/market_maker/service.yaml | 2 +- .../behaviours.py | 95 +++++++++++-------- .../market_creation_manager_abci/models.py | 4 + .../market_creation_manager_abci/skill.yaml | 16 +++- .../valory/skills/market_maker_abci/models.py | 4 + .../skills/market_maker_abci/skill.yaml | 16 +++- 8 files changed, 99 insertions(+), 50 deletions(-) diff --git a/packages/packages.json b/packages/packages.json index 2475c61..b6c67f1 100644 --- a/packages/packages.json +++ b/packages/packages.json @@ -3,10 +3,10 @@ "contract/valory/conditional_tokens/0.1.0": "bafybeibpzhbknohnx5mp653ky5betjfiz3ed3agj554oecsszqq4uxe4qi", "contract/valory/fpmm_deterministic_factory/0.1.0": "bafybeif7hgwamqm6iljhtr32ap6eazn5yhmxrtlvicmpo746gv2s4tyzqq", "contract/valory/realtio/0.1.0": "bafybeigsciwpawqiyxtk7klpnx6vyaz4nic5s6afl4eahab5ub4hftu4du", - "skill/valory/market_creation_manager_abci/0.1.0": "bafybeibmavneaxzrtu2uxnaxmyc627rvetsc5dpibn7dtaevlkrzcv7scu", - "skill/valory/market_maker_abci/0.1.0": "bafybeihrilzqkjs2urebi3a5yle2s44fy72pahqfuvhtyn5lnzfzi5xolu", - "agent/valory/market_maker/0.1.0": "bafybeihjpzqk6msveftuxgvfcqxytvmisynjdwtjsdal24awgpskqvyta4", - "service/valory/market_maker/0.1.0": "bafybeibhlutt7qo3lcpek3poqzpheyjqf5roar2h5k63hc2z76olefh5kq", + "skill/valory/market_creation_manager_abci/0.1.0": "bafybeihsc2xenplve2nro63gr7ekerjlcuggxmlihykjzplb3gc6v6fqwy", + "skill/valory/market_maker_abci/0.1.0": "bafybeiaj3u2zigypkqazu5p63eysnc33cmapsm7lvnjdwfnsvaogxzf7dq", + "agent/valory/market_maker/0.1.0": "bafybeifqlmv5e3nxh4a6uhaamvht3hvm2v5xilbapcbedpcztziuem33xq", + "service/valory/market_maker/0.1.0": "bafybeib5zgmrtquhyq6zfnnyikwcddzo3okrrivwhdmzftxlcofmebtrru", "contract/valory/wxdai/0.1.0": "bafybeidsp4dptfuxatr75vksdruzvqyhinj4nwiw5f3brzrewldprxuwg4", "skill/valory/market_validation_abci/0.1.0": "bafybeifyfwynotmzg6wypbzbwsfnrplhjhumfx32dm2j3ddasp7ygrfoqa", "contract/valory/fpmm/0.1.0": "bafybeihoczslmqbfpx57tmjps663r2ib7ycc3dfa4smdhufb4wbnidewtq" diff --git a/packages/valory/agents/market_maker/aea-config.yaml b/packages/valory/agents/market_maker/aea-config.yaml index 30f1c3d..7e91af2 100644 --- a/packages/valory/agents/market_maker/aea-config.yaml +++ b/packages/valory/agents/market_maker/aea-config.yaml @@ -37,9 +37,9 @@ protocols: skills: - valory/abstract_abci:0.1.0:bafybeiamr3cwetayz6lr72buhq6iviabtboj4hhpq47tlumimxolvvqjcy - valory/abstract_round_abci:0.1.0:bafybeiaseziuvbzh3trjggl5hx2tv3fduqrpiwiyksudaf6nvuxmwrg74i -- valory/market_maker_abci:0.1.0:bafybeihrilzqkjs2urebi3a5yle2s44fy72pahqfuvhtyn5lnzfzi5xolu +- valory/market_maker_abci:0.1.0:bafybeiaj3u2zigypkqazu5p63eysnc33cmapsm7lvnjdwfnsvaogxzf7dq - valory/registration_abci:0.1.0:bafybeibafy7qmcvvzaykimy3mbqdwtcuajmbghqilaa7zwkh6wla33cfce -- valory/market_creation_manager_abci:0.1.0:bafybeibmavneaxzrtu2uxnaxmyc627rvetsc5dpibn7dtaevlkrzcv7scu +- valory/market_creation_manager_abci:0.1.0:bafybeihsc2xenplve2nro63gr7ekerjlcuggxmlihykjzplb3gc6v6fqwy - valory/market_validation_abci:0.1.0:bafybeifyfwynotmzg6wypbzbwsfnrplhjhumfx32dm2j3ddasp7ygrfoqa - valory/reset_pause_abci:0.1.0:bafybeialcwck7fahrr23jckv5qjwg3cdq4ai2ihyjsofnbj44jzyl4cjmm - valory/termination_abci:0.1.0:bafybeifqsogqiar4yook5bu3j6z66dbdcizey7dr3e5oxeocdjijvfbaja diff --git a/packages/valory/services/market_maker/service.yaml b/packages/valory/services/market_maker/service.yaml index d0c8e5e..345eb5b 100644 --- a/packages/valory/services/market_maker/service.yaml +++ b/packages/valory/services/market_maker/service.yaml @@ -7,7 +7,7 @@ license: Apache-2.0 fingerprint: README.md: bafybeibwz3af6326msp4h3kqehijvmyhaytvyfbo3o2npc2w4b6zrg6pfq fingerprint_ignore_patterns: [] -agent: valory/market_maker:0.1.0:bafybeihjpzqk6msveftuxgvfcqxytvmisynjdwtjsdal24awgpskqvyta4 +agent: valory/market_maker:0.1.0:bafybeifqlmv5e3nxh4a6uhaamvht3hvm2v5xilbapcbedpcztziuem33xq number_of_agents: 1 deployment: agent: diff --git a/packages/valory/skills/market_creation_manager_abci/behaviours.py b/packages/valory/skills/market_creation_manager_abci/behaviours.py index 998ae45..1d3a517 100644 --- a/packages/valory/skills/market_creation_manager_abci/behaviours.py +++ b/packages/valory/skills/market_creation_manager_abci/behaviours.py @@ -23,6 +23,7 @@ import json import random from abc import ABC +from string import Template from typing import Any, Dict, Generator, List, Optional, Set, Tuple, Type, cast from packages.valory.connections.openai.connection import ( @@ -112,6 +113,35 @@ ZERO_ADDRESS = "0x0000000000000000000000000000000000000000" +CONDITIONS_QUERY = Template( + """{ + conditions(where:{id_in:$conditions}){ + id, + oracle, + questionId, + outcomeSlotCount + } +} +""" +) + +QUESTIONS_QUERY = Template( + """{ + questions(where:{id_in:$conditions}){ + id, + openingTimestamp + } +}""" +) + + +def to_content(query: str) -> bytes: + """Convert the given query string to payload content, i.e., add it under a `queries` key and convert it to bytes.""" + finalized_query = {"query": query} + encoded_query = json.dumps(finalized_query, sort_keys=True).encode("utf-8") + + return encoded_query + def parse_date_timestring(string: str) -> Optional[datetime.datetime]: """Parse and return a datetime string.""" @@ -364,11 +394,12 @@ def get_markets_to_removal_ts( condition_id_to_market = {} for market in markets: for condition_id in market["condition_ids"]: - condition_id_to_market[condition_id] = market + condition_id_to_market[f"0x{condition_id.hex()}"] = market condition_ids = list(condition_id_to_market.keys()) condition_preparations = yield from self._get_condition_preparation_events( - condition_ids, from_block + condition_ids=condition_ids ) + self.context.logger.info(f"Conditions: {condition_preparations}") if condition_preparations is None: # something went wrong return None @@ -390,7 +421,8 @@ def get_markets_to_removal_ts( question_ids = [ preparation["question_id"] for preparation in condition_preparations ] - questions = yield from self._get_questions(question_ids, from_block) + self.context.logger.info(f"Questions: {question_ids}") + questions = yield from self._get_questions(question_ids=question_ids) if questions is None: # something went wrong return None @@ -474,51 +506,36 @@ def _get_markets_with_funds( return markets_with_liq def _get_condition_preparation_events( - self, - condition_ids: List[bytes], - from_block: int, + self, condition_ids: List[str] ) -> Generator[None, None, Optional[List[Dict[str, Any]]]]: """Get condition preparation events.""" - contract_api_response = yield from self.get_contract_api_response( - performative=ContractApiMessage.Performative.GET_STATE, - contract_address=self.params.conditional_tokens_contract, - contract_id=str(ConditionalTokensContract.contract_id), - contract_callable="get_condition_preparation_events", - condition_ids=condition_ids, - from_block=from_block, - ) - if contract_api_response.performative != ContractApiMessage.Performative.STATE: - self.context.logger.error( - f"Failed to get condition preparation events: {contract_api_response}" - ) - return None - condition_preparation_events = cast( - Optional[List[Dict[str, Any]]], - contract_api_response.state.body.get("data", []), + query = CONDITIONS_QUERY.substitute(conditions=json.dumps(condition_ids)) + response = yield from self.get_http_response( + content=to_content(query), + **self.context.omen_subgraph.get_spec(), ) + data = json.loads(response.body.decode()) + condition_preparation_events = data["data"]["conditions"] + for condition in condition_preparation_events: + condition["condition_id"] = condition.pop("id") + condition["question_id"] = condition.pop("questionId") + condition["outcome_slot_count"] = condition.pop("outcomeSlotCount") return condition_preparation_events def _get_questions( - self, question_ids: List[str], from_block: int + self, question_ids: List[str] ) -> Generator[None, None, Optional[List[Dict[str, Any]]]]: """Get question ids.""" - contract_api_response = yield from self.get_contract_api_response( - performative=ContractApiMessage.Performative.GET_STATE, - contract_address=self.params.realitio_contract, - contract_id=str(RealtioContract.contract_id), - contract_callable="get_question_events", - question_ids=question_ids, - from_block=from_block, - ) - if contract_api_response.performative != ContractApiMessage.Performative.STATE: - self.context.logger.error( - f"Failed to get question: {contract_api_response}" - ) - return None - questions = cast( - Optional[List[Dict[str, Any]]], - contract_api_response.state.body.get("data", []), + query = QUESTIONS_QUERY.substitute(conditions=json.dumps(question_ids)) + response = yield from self.get_http_response( + content=to_content(query), + **self.context.omen_subgraph.get_spec(), ) + data = json.loads(response.body.decode()) + questions = data["data"]["questions"] + for condition in questions: + condition["question_id"] = condition.pop("id") + condition["opening_ts"] = int(condition.pop("openingTimestamp")) return questions diff --git a/packages/valory/skills/market_creation_manager_abci/models.py b/packages/valory/skills/market_creation_manager_abci/models.py index a28b77c..2d84048 100644 --- a/packages/valory/skills/market_creation_manager_abci/models.py +++ b/packages/valory/skills/market_creation_manager_abci/models.py @@ -115,5 +115,9 @@ class RandomnessApi(ApiSpecs): """A model for randomness api specifications.""" +class OmenSubgraph(ApiSpecs): + """A model that wraps ApiSpecs for the OMEN's subgraph specifications.""" + + Requests = BaseRequests BenchmarkTool = BaseBenchmarkTool diff --git a/packages/valory/skills/market_creation_manager_abci/skill.yaml b/packages/valory/skills/market_creation_manager_abci/skill.yaml index 0da126a..2307276 100644 --- a/packages/valory/skills/market_creation_manager_abci/skill.yaml +++ b/packages/valory/skills/market_creation_manager_abci/skill.yaml @@ -8,11 +8,11 @@ aea_version: '>=1.0.0, <2.0.0' fingerprint: .gitignore: bafybeihdfdezgtr3s2lzq5y3oaitfrdy4u4pehionju2bdez35wcjgqx6y __init__.py: bafybeibkyjt4covc4yhd22aw7kav45zozk3exdv344emt3lilln64soaxm - behaviours.py: bafybeidbfmqndd5ig5gv4jkce7bt7qynm2rxwpp75agvqfwl3tuhinuxye + behaviours.py: bafybeig65mpjl52jn6diprmg7bzted2m3xsa3hdfrvktf7qxhnoq4rsxca dialogues.py: bafybeicmaufkl7vdomnfciv7lw4536ssld7x4uemdapuhsyvfpd4ncibza fsm_specification.yaml: bafybeiay4aepnfdkcsriajxqige65bsszrujin57riubflm7dyw2keapoi handlers.py: bafybeietxjfli2i57kb7heoy772rcq2znusl36gg7jjj5g3pddw7egny3q - models.py: bafybeidpprylojhopdp4aoxwxri2kp7nmv5f3acdok4n5c6ak3a5wnn5cu + models.py: bafybeia226bfyqfnzkkp24wmabaof4umi7lqnjtmhb63goszboqk2krqsm payloads.py: bafybeib57gmbwstx646qxvxxl62n6of4vo2ei2nl32ogo5vhi3ukz6ldqe rounds.py: bafybeifhj72msvgtp2upzawe4ynyagir4quexvr3lwz4clob3f3b4pqi6i tests/__init__.py: bafybeihfxvqnyfly72tbxnnnglshcilm2kanihqnjiasvcz3ec3csw32ti @@ -172,6 +172,18 @@ models: retries: 5 url: https://drand.cloudflare.com/public/latest class_name: RandomnessApi + omen_subgraph: + args: + api_id: omen + headers: + Content-Type: application/json + method: POST + parameters: {} + response_key: data + response_type: list + retries: 5 + url: https://api.thegraph.com/subgraphs/name/protofire/omen-xdai + class_name: OmenSubgraph requests: args: {} class_name: Requests diff --git a/packages/valory/skills/market_maker_abci/models.py b/packages/valory/skills/market_maker_abci/models.py index 6875513..5cd4ceb 100644 --- a/packages/valory/skills/market_maker_abci/models.py +++ b/packages/valory/skills/market_maker_abci/models.py @@ -29,6 +29,9 @@ from packages.valory.skills.market_creation_manager_abci.models import ( MarketCreationManagerParams, ) +from packages.valory.skills.market_creation_manager_abci.models import ( + OmenSubgraph as BaseOmenSubgraph, +) from packages.valory.skills.market_creation_manager_abci.models import ( RandomnessApi as MarketCreationManagerRandomnessApi, ) @@ -47,6 +50,7 @@ Requests = BaseRequests BenchmarkTool = BaseBenchmarkTool RandomnessApi = MarketCreationManagerRandomnessApi +OmenSubgraph = BaseOmenSubgraph class SharedState(BaseSharedState): diff --git a/packages/valory/skills/market_maker_abci/skill.yaml b/packages/valory/skills/market_maker_abci/skill.yaml index b3441fe..2671f56 100644 --- a/packages/valory/skills/market_maker_abci/skill.yaml +++ b/packages/valory/skills/market_maker_abci/skill.yaml @@ -13,7 +13,7 @@ fingerprint: dialogues.py: bafybeicintyylxt4nd5gcufh3rehbxr5jvdn6un7wvaoel7hvj3dmurlpy fsm_specification.yaml: bafybeicvtoez7ahmyl25h66edgjywx5ymxtqnyexzq73ouyu2p6smi3uoa handlers.py: bafybeidlgouig5odju36dkjl5vdqx6mqrukaoyn2chnl5dv2ca6owqlecm - models.py: bafybeieagk5wihiorhd3nzqgyq7g735ww2bltg7olpkzmstxu2mjkeuazu + models.py: bafybeie5hchh34fbyoshkujhdonvv4ye2knivw2xlqboqyr6ooa5egqjri tests/__init__.py: bafybeig7noiiz2y2b2scjeaalby56ft24eqj4o433blm2oze2pu3lljp44 fingerprint_ignore_patterns: [] connections: [] @@ -23,7 +23,7 @@ skills: - valory/abstract_round_abci:0.1.0:bafybeiaseziuvbzh3trjggl5hx2tv3fduqrpiwiyksudaf6nvuxmwrg74i - valory/registration_abci:0.1.0:bafybeibafy7qmcvvzaykimy3mbqdwtcuajmbghqilaa7zwkh6wla33cfce - valory/reset_pause_abci:0.1.0:bafybeialcwck7fahrr23jckv5qjwg3cdq4ai2ihyjsofnbj44jzyl4cjmm -- valory/market_creation_manager_abci:0.1.0:bafybeibmavneaxzrtu2uxnaxmyc627rvetsc5dpibn7dtaevlkrzcv7scu +- valory/market_creation_manager_abci:0.1.0:bafybeihsc2xenplve2nro63gr7ekerjlcuggxmlihykjzplb3gc6v6fqwy - valory/market_validation_abci:0.1.0:bafybeifyfwynotmzg6wypbzbwsfnrplhjhumfx32dm2j3ddasp7ygrfoqa - valory/termination_abci:0.1.0:bafybeifqsogqiar4yook5bu3j6z66dbdcizey7dr3e5oxeocdjijvfbaja - valory/transaction_settlement_abci:0.1.0:bafybeiacwr7p4nhhufoey7uz2jqkegrlykdrmc7mm3rzkvh2mslu66gyle @@ -169,6 +169,18 @@ models: retries: 5 url: https://drand.cloudflare.com/public/latest class_name: RandomnessApi + omen_subgraph: + args: + api_id: omen + headers: + Content-Type: application/json + method: POST + parameters: {} + response_key: data + response_type: list + retries: 5 + url: https://api.thegraph.com/subgraphs/name/protofire/omen-xdai + class_name: OmenSubgraph requests: args: {} class_name: Requests