Merge pull request #121 from valory-xyz/feat/story_market_creator

Story-based market creation

market_approval_server/market_approval_server.py

@@ -31,14 +31,15 @@
     Usage for server running in http mode (replace by https if applies).
 
     - Service API:
-        curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "MARKET_ID", ...}' -k http://127.0.0.1:5000/propose_market
-        curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "MARKET_ID"}' -k http://127.0.0.1:5000/process_market
+        curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "market_id", ...}' -k http://127.0.0.1:5000/propose_market
+        curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "market_id"}' -k http://127.0.0.1:5000/process_market
         curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -k http://127.0.0.1:5000/get_process_random_approved_market
-        curl -X PUT -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "MARKET_ID", ...}' -k http://127.0.0.1:5000/update_market
+        curl -X PUT -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "market_id", ...}' -k http://127.0.0.1:5000/update_market
+        curl -X PUT -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "market_id", "new_id": "new_market_id"}' -k http://127.0.0.1:5000/update_market_id
 
     - User API
-        curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "MARKET_ID"}' -k http://127.0.0.1:5000/approve_market
-        curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "MARKET_ID"}' -k http://127.0.0.1:5000/reject_market
+        curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "market_id"}' -k http://127.0.0.1:5000/approve_market
+        curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "market_id"}' -k http://127.0.0.1:5000/reject_market
 
         curl -X DELETE -H "Authorization: YOUR_API_KEY" -k http://127.0.0.1:5000/clear_proposed_markets
         curl -X DELETE -H "Authorization: YOUR_API_KEY" -k http://127.0.0.1:5000/clear_approved_markets
@@ -52,6 +53,7 @@
 import logging
 import os
 import secrets
+import sys
 import uuid
 from datetime import datetime
 from enum import Enum
@@ -65,7 +67,7 @@
 app = Flask(__name__)
 CORS(app)
 
-CONFIG_FILE = "server_config.json"
+CONFIG_FILE = os.getenv("MARKET_APPROVAL_SERVER_CONFIG_FILE", "server_config.json")
 LOG_FILE = "market_approval_server.log"
 CERT_FILE = "server_cert.pem"
 KEY_FILE = "server_key.pem"
@@ -93,20 +95,28 @@ class MarketState(str, Enum):
 api_keys: Dict[str, str] = {}
 
 
+def get_databases() -> Dict[str, Dict[str, Any]]:
+    """Returns all databases into a single object."""
+
+    return {
+        "proposed_markets": proposed_markets,
+        "approved_markets": approved_markets,
+        "rejected_markets": rejected_markets,
+        "processed_markets": processed_markets,
+    }
+
+
 def load_config() -> None:
     """Loads the configuration from a JSON file."""
     global proposed_markets, approved_markets, rejected_markets, processed_markets, api_keys  # pylint: disable=global-statement
     try:
         with open(CONFIG_FILE, "r", encoding="utf-8") as f:
+            logger.info("Using config file: %s", CONFIG_FILE)
             data = json.load(f)
     except FileNotFoundError:
         # If the file is not found, set the dictionaries to empty
-        proposed_markets = {}
-        approved_markets = {}
-        rejected_markets = {}
-        processed_markets = {}
-        api_keys = DEFAULT_API_KEYS
-        save_config()
+        logger.info("FileNotFoundError: %s", CONFIG_FILE)
+        sys.exit(1)
     else:
         # If the file is found, set the dictionaries to the loaded data
         proposed_markets = data.get("proposed_markets", {})
@@ -174,6 +184,55 @@ def get_markets() -> Tuple[Response, int]:
         return jsonify({"error": str(e)}), 500
 
 
+@app.route("/proposed_market/<market_id>", methods=["GET"])
+@app.route("/approved_market/<market_id>", methods=["GET"])
+@app.route("/rejected_market/<market_id>", methods=["GET"])
+@app.route("/processed_market/<market_id>", methods=["GET"])
+def get_market_by_id(market_id: str) -> Tuple[Response, int]:
+    """Retrieve a specific market by its ID from the corresponding market database."""
+    try:
+        endpoint = request.path.split("/")[1]
+
+        if endpoint == "proposed_market":
+            markets = proposed_markets
+        elif endpoint == "approved_market":
+            markets = approved_markets
+        elif endpoint == "rejected_market":
+            markets = rejected_markets
+        elif endpoint == "processed_market":
+            markets = processed_markets
+        else:
+            return jsonify({"error": "Invalid endpoint."}), 404
+
+        if market_id in markets:
+            market = markets[market_id]
+            return jsonify(market), 200
+
+        return (
+            jsonify({"error": f"Market ID {market_id} not found in {endpoint}s."}),
+            404,
+        )
+    except Exception as e:  # pylint: disable=broad-except
+        return jsonify({"error": str(e)}), 500
+
+
+@app.route("/market/<market_id>", methods=["GET"])
+def get_market_by_id_all_databases(market_id: str) -> Tuple[Response, int]:
+    """Retrieve a market by its ID from any database."""
+    try:
+        for _, db in get_databases().items():
+            if market_id in db:
+                return jsonify(db[market_id]), 200
+
+        return (
+            jsonify({"error": f"Market ID {market_id!r} not found in any database."}),
+            404,
+        )
+
+    except Exception as e:  # pylint: disable=broad-except
+        return jsonify({"error": str(e)}), 500
+
+
 @app.route("/clear_proposed_markets", methods=["DELETE"])
 @app.route("/clear_approved_markets", methods=["DELETE"])
 @app.route("/clear_rejected_markets", methods=["DELETE"])
@@ -240,6 +299,8 @@ def propose_market() -> Tuple[Response, int]:
             market["id"] = str(uuid.uuid4())
 
         market_id = str(market["id"])
+        market_id = market_id.lower()
+        market["id"] = market_id
 
         if any(
             market_id in db
@@ -304,6 +365,7 @@ def move_market() -> Tuple[Response, int]:
             return jsonify({"error": "Invalid JSON format. Missing id."}), 400
 
         market_id = data["id"]
+        market_id = market_id.lower()
 
         if market_id not in move_from:
             return jsonify({"error": f"Market ID {market_id} not found."}), 404
@@ -332,7 +394,7 @@ def get_random_approved_market() -> Tuple[Response, int]:
             return (
                 jsonify({"info": "No approved markets available."}),
                 204,
-            )  # No content, json will be ignored by the server
+            )
 
         market_id = secrets.choice(list(approved_markets.keys()))
         market = approved_markets[market_id]
@@ -360,6 +422,7 @@ def update_market() -> Tuple[Response, int]:
             return jsonify({"error": "Invalid JSON format. Missing id."}), 400
 
         market_id = market["id"]
+        market_id = market_id.lower()
 
         # Check if the market exists in any of the databases
         databases = [
@@ -387,7 +450,72 @@ def update_market() -> Tuple[Response, int]:
             )
 
         return (
-            jsonify({"error": f"Market ID {market_id} not found in any database."}),
+            jsonify({"error": f"Market ID {market_id} not found in the database."}),
+            404,
+        )
+
+    except Exception as e:  # pylint: disable=broad-except
+        return jsonify({"error": str(e)}), 500
+
+
+@app.route("/update_market_id", methods=["PUT"])
+def update_market_id() -> (
+    Tuple[Response, int]
+):  # pylint: disable=too-many-return-statements
+    """Update the market ID in any of the databases if the market exists."""
+    try:
+        api_key = request.headers.get("Authorization")
+        if not check_api_key(api_key):
+            return jsonify({"error": "Unauthorized access. Invalid API key."}), 401
+
+        data = request.get_json()
+        current_market_id = data.get("id")
+        new_market_id = data.get("new_id")
+
+        if not current_market_id:
+            return jsonify({"error": "'id' is required."}), 400
+        if not new_market_id:
+            return jsonify({"error": "'new_id' is required."}), 400
+        if current_market_id == new_market_id:
+            return jsonify({"error": "'id' is equal to 'new_id' in the request."}), 409
+
+        # The next line is intentionally commented to allow fixing uppercase market_ids externally.
+        # current_market_id = current_market_id.lower()  # noqa
+        new_market_id = new_market_id.lower()
+
+        databases = get_databases()
+
+        if any(new_market_id in db for _, db in databases.items()):
+            return (
+                jsonify(
+                    {
+                        "error": f"Market ID {new_market_id} already exists in database. Try using a different ID."
+                    }
+                ),
+                409,
+            )
+
+        for db_name, db in databases.items():
+            if current_market_id in db:
+                db[new_market_id] = db.pop(current_market_id)
+                db[new_market_id]["id"] = new_market_id
+                db[new_market_id]["utc_timestamp_updated"] = int(
+                    datetime.utcnow().timestamp()
+                )
+                save_config()
+                return (
+                    jsonify(
+                        {
+                            "message": f"Market ID {current_market_id!r} successfully changed to {new_market_id!r} in {db_name}."
+                        }
+                    ),
+                    200,
+                )
+
+        return (
+            jsonify(
+                {"error": f"Market ID {current_market_id!r} not found in the database."}
+            ),
             404,
         )
 

market_approval_server/templates/index.html

@@ -24,30 +24,47 @@ <h2>View the server databases</h2>
         <li><a href="/all_markets">All markets</a></li>
     </ul>
 
+    <h2>View a single market by <code>market_id</code> endpoints</h2>
+    <p>Requires to know the <code>market_id</code>:</p>
+    <ul>
+        <li><code>/proposed_market/market_id</code>: Marked in "Proposed markets" database</a></li>
+        <li><code>/approved_market/market_id</code>: Marked in "Approved markets" database</a></li>
+        <li><code>/rejected_market/market_id</code>: Marked in "Rejected markets" database</a></li>
+        <li><code>/processed_market/market_id</code>: Marked in "Processed markets" database</a></li>
+        <li><code>/market/market_id</code>: Marked in any database</a></li>
+    </ul>
+
+    <h2>Note on <code>market_id</code>s</h2>
+    <p>All <code>market_id</code>s must be lowercase. The server will always convert received market IDs to lowercase before storing or modifying the IDs of any market on the database.</p>
+
     <h2>Main interaction methods</h2>
 
     <ul>
         <li>Approve a market:
-            <pre>curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "MARKET_ID"}' -k {{ server_ip }}/approve_market</pre>
+            <pre>curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "market_id"}' -k {{ server_ip }}/approve_market</pre>
         </li>
 
         <li>Reject a market:
-            <pre>curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "MARKET_ID"}' -k {{ server_ip }}/reject_market</pre>
+            <pre>curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "market_id"}' -k {{ server_ip }}/reject_market</pre>
         </li>
     </ul>
 
     <h2>Other methods</h2>
     <ul>
         <li>Propose a market:
-            <pre>curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "MARKET_ID", ...}' -k {{ server_ip }}/propose_market</pre>
+            <pre>curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "market_id", ...}' -k {{ server_ip }}/propose_market</pre>
         </li>
 
         <li>Mark a market as processed:
-            <pre>curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "MARKET_ID"}' -k {{ server_ip }}/process_market</pre>
+            <pre>curl -X POST -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "market_id"}' -k {{ server_ip }}/process_market</pre>
         </li>
 
         <li>Update a market:
-            <pre>curl -X PUT -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "MARKET_ID", ...}' -k {{ server_ip }}/update_market</pre>
+            <pre>curl -X PUT -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "market_id", ...}' -k {{ server_ip }}/update_market</pre>
+        </li>
+
+        <li>Update a market id:
+            <pre>curl -X PUT -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{"id": "market_id", "new_id": "new_market_id"}' -k {{ server_ip }}/update_market_id</pre>
         </li>
 
         <li>Get a random accepted market and mark it as processed:

packages/packages.json

@@ -3,10 +3,10 @@
         "contract/valory/fpmm_deterministic_factory/0.1.0": "bafybeigjfuahxhb2y4q5ueayry55boaojcryoamwr6fshgayf5s762vpl4",
         "contract/valory/wxdai/0.1.0": "bafybeidalocwbhmbto6ii6adldtpcughtdt6j3v4tv36utevjk2wrdyqie",
         "contract/valory/fpmm/0.1.0": "bafybeiai2ruj27nnglvn7yc5atojyojo3fkmofw6wrjgz2ybps2uwdizx4",
-        "skill/valory/market_creation_manager_abci/0.1.0": "bafybeidgpctdcbxubjdlqzcrzr2qwtuocv236ngtjduabkmja7qvqr3fri",
-        "skill/valory/market_maker_abci/0.1.0": "bafybeietyfyi6qphajffab37bueo4seb2fdtptak6sneszk7ww7fjhwr5y",
-        "agent/valory/market_maker/0.1.0": "bafybeibbnx4azoeyuh2zsd6ebxvv2k2y3yvmuhfl4776p5pz4tzjb5uyuu",
-        "service/valory/market_maker/0.1.0": "bafybeiftf5j3vu7tab5uxfsidxrneayacfjbrgwejt5wk3qoyn4llgpkji"
+        "skill/valory/market_creation_manager_abci/0.1.0": "bafybeibjvymcbydezmteqymyepunrgpgd4trrobvnp5ympqf6fz7lvhgjq",
+        "skill/valory/market_maker_abci/0.1.0": "bafybeibnllwu777bvc5lbztp2jroxpoyscehhv6ci2low4vp24mafbdimu",
+        "agent/valory/market_maker/0.1.0": "bafybeifehge2tytznohl6pjt5o5vhgpsujrelu4aioevvo626efgfc55w4",
+        "service/valory/market_maker/0.1.0": "bafybeidrl3nvgcw46xbxte4lplx2o5tf6p7yq3vyaphhss667hj5d62efq"
     },
     "third_party": {
         "protocol/valory/contract_api/1.0.0": "bafybeidgu7o5llh26xp3u3ebq3yluull5lupiyeu6iooi2xyymdrgnzq5i",

packages/valory/agents/market_maker/aea-config.yaml

@@ -38,9 +38,9 @@ protocols:
 skills:
 - valory/abstract_abci:0.1.0:bafybeieh4ei3qdelmacnm7vwq57phoewgumr3udvxt6pybmuggwc3yk65q
 - valory/abstract_round_abci:0.1.0:bafybeiar2yhzxacfe3qqamqhaihtlcimquwedffctw55sowx6rac3cm3ui
-- valory/market_maker_abci:0.1.0:bafybeietyfyi6qphajffab37bueo4seb2fdtptak6sneszk7ww7fjhwr5y
+- valory/market_maker_abci:0.1.0:bafybeibnllwu777bvc5lbztp2jroxpoyscehhv6ci2low4vp24mafbdimu
 - valory/registration_abci:0.1.0:bafybeieu7vq3pyns4t5ty6u3sbmpkd7yznpg3rmqifoz3jhy7pmqyg3w6q
-- valory/market_creation_manager_abci:0.1.0:bafybeidgpctdcbxubjdlqzcrzr2qwtuocv236ngtjduabkmja7qvqr3fri
+- valory/market_creation_manager_abci:0.1.0:bafybeibjvymcbydezmteqymyepunrgpgd4trrobvnp5ympqf6fz7lvhgjq
 - valory/reset_pause_abci:0.1.0:bafybeiameewywqigpupy3u2iwnkfczeiiucue74x2l5lbge74rmw6bgaie
 - valory/termination_abci:0.1.0:bafybeif2zim2de356eo3sipkmoev5emwadpqqzk3huwqarywh4tmqt3vzq
 - valory/transaction_settlement_abci:0.1.0:bafybeic3tccdjypuge2lewtlgprwkbb53lhgsgn7oiwzyrcrrptrbeyote
@@ -82,6 +82,10 @@ dependencies:
     version: ==1.52.0
   open-aea-test-autonomy:
     version: ==0.14.12
+  gql:
+    version: ==3.5.0
+  requests-toolbelt:
+    version: ==1.0.0
 default_connection: null
 ---
 public_id: valory/abci:0.1.0
@@ -172,16 +176,17 @@ models:
       initial_funds: ${float:1.0}
       market_timeout: ${int:1}
       min_market_proposal_interval_seconds: ${int:7200}
-      news_sources: ${list:["bbc-news","bbc-sport","abc-news","cnn","google-news","reuters","usa-today","breitbart-news","the-verge","techradar"]}
+      news_sources: ${list:["bbc-news","bbc-sport","abc-news","cnn","reuters","usa-today","breitbart-news","the-verge","techradar"]}
       event_offset_start_days: ${int:4}
       event_offset_end_days: ${int:7}
       market_proposal_round_timeout_seconds_per_day: ${int:45}
+      max_markets_per_story: ${int:5}
       realitio_contract: ${str:0x79e32aE03fb27B07C89c0c568F80287C01ca2E57}
       realitio_oracle_proxy_contract: ${str:0xab16d643ba051c11962da645f74632d3130c81e2}
       conditional_tokens_contract: ${str:0xCeAfDD6bc0bEF976fdCd1112955828E00543c0Ce}
       fpmm_deterministic_factory_contract: ${str:0x9083A2B699c0a4AD06F63580BDE2635d26a3eeF0}
       collateral_tokens_contract: ${str:0xe91d153e0b41518a2ce8dd3d7944fa863463a97d}
-      arbitrator_contract: ${str:0xe40dd83a262da3f56976038f1554fe541fa75ecd}
+      arbitrator_contract: ${str:0x29f39de98d750eb77b5fafb31b2837f079fce222}
       multisend_address: ${str:0xA238CBeb142c10Ef7Ad8442C6D1f9E89e07e7761}
       multisend_batch_size: ${int:5}
       ipfs_address: ${str:https://gateway.autonolas.tech/ipfs/}
@@ -215,6 +220,8 @@ models:
       light_slash_unit_amount: ${int:5000000000000000}
       serious_slash_unit_amount: ${int:8000000000000000}
       xdai_threshold: ${int:1000000000000000000}
+      serper_api_key: ${str:serper_api_key}
+      subgraph_api_key: ${str:subgraph_api_key}
       google_api_key: ${str:google_api_key}
       google_engine_id: ${str:google_engine_id}
       openai_api_key: ${str:openai_api_key}