From ba25664e65a281d98ac638243295a2427b602771 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 7 Nov 2024 17:11:07 -0800 Subject: [PATCH 001/119] add a websockets router --- openbb_platform/dev_install.py | 1 + .../extensions/websockets/README.md | 0 .../websockets/openbb_websockets/__init__.py | 1 + .../websockets/openbb_websockets/broadcast.py | 233 +++ .../websockets/openbb_websockets/client.py | 675 +++++++ .../websockets/openbb_websockets/helpers.py | 238 +++ .../websockets/openbb_websockets/listen.py | 157 ++ .../websockets/openbb_websockets/models.py | 98 + .../openbb_websockets/websockets_router.py | 428 +++++ .../extensions/websockets/poetry.lock | 1618 +++++++++++++++++ .../extensions/websockets/pyproject.toml | 19 + .../providers/fmp/openbb_fmp/__init__.py | 2 + .../openbb_fmp/models/websocket_connection.py | 198 ++ .../fmp/openbb_fmp/utils/websocket_client.py | 190 ++ .../tiingo/openbb_tiingo/__init__.py | 2 + .../models/websocket_connection.py | 288 +++ .../openbb_tiingo/utils/websocket_client.py | 278 +++ openbb_platform/pyproject.toml | 3 + 18 files changed, 4429 insertions(+) create mode 100644 openbb_platform/extensions/websockets/README.md create mode 100644 openbb_platform/extensions/websockets/openbb_websockets/__init__.py create mode 100644 openbb_platform/extensions/websockets/openbb_websockets/broadcast.py create mode 100644 openbb_platform/extensions/websockets/openbb_websockets/client.py create mode 100644 openbb_platform/extensions/websockets/openbb_websockets/helpers.py create mode 100644 openbb_platform/extensions/websockets/openbb_websockets/listen.py create mode 100644 openbb_platform/extensions/websockets/openbb_websockets/models.py create mode 100644 openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py create mode 100644 openbb_platform/extensions/websockets/poetry.lock create mode 100644 openbb_platform/extensions/websockets/pyproject.toml create mode 100644 openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py create mode 100644 openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py create mode 100644 openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py create mode 100644 openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py diff --git a/openbb_platform/dev_install.py b/openbb_platform/dev_install.py index 258d41fa9bd3..d3f540c6b26e 100644 --- a/openbb_platform/dev_install.py +++ b/openbb_platform/dev_install.py @@ -71,6 +71,7 @@ openbb-econometrics = { path = "./extensions/econometrics", optional = true, develop = true } openbb-quantitative = { path = "./extensions/quantitative", optional = true, develop = true } openbb-technical = { path = "./extensions/technical", optional = true, develop = true } +openbb-websockets = { path = "./extensions/websockets", optional = true, develop = true } """ diff --git a/openbb_platform/extensions/websockets/README.md b/openbb_platform/extensions/websockets/README.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openbb_platform/extensions/websockets/openbb_websockets/__init__.py b/openbb_platform/extensions/websockets/openbb_websockets/__init__.py new file mode 100644 index 000000000000..50da7c837624 --- /dev/null +++ b/openbb_platform/extensions/websockets/openbb_websockets/__init__.py @@ -0,0 +1 @@ +"""OpenBB WebSockets Router Extension.""" diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py new file mode 100644 index 000000000000..9235a6b4bdf1 --- /dev/null +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -0,0 +1,233 @@ +import asyncio +import json +import sys +from typing import Optional + +import uvicorn +from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from starlette.websockets import WebSocketState + +from openbb_websockets.helpers import get_logger, parse_kwargs + +connected_clients = set() + +kwargs = parse_kwargs() + +HOST = kwargs.pop("host", None) or "localhost" +PORT = kwargs.pop("port", None) or 6666 +PORT = int(PORT) + +RESULTS_FILE = kwargs.pop("results_file", None) +TABLE_NAME = kwargs.pop("table_name", None) or "records" +SLEEP_TIME = kwargs.pop("sleep_time", None) or 0.25 +AUTH_TOKEN = kwargs.pop("auth_token", None) + +app = FastAPI() + + +@app.websocket("/") +async def websocket_endpoint( # noqa: PLR0915 + websocket: WebSocket, auth_token: Optional[str] = None +): + + broadcast_server = BroadcastServer( + RESULTS_FILE, + TABLE_NAME, + SLEEP_TIME, + str(AUTH_TOKEN), + ) + auth_token = str(auth_token) + + if ( + broadcast_server.auth_token is not None + and auth_token != broadcast_server.auth_token + ): + await websocket.accept() + await websocket.send_text( + "ERROR: Invalid authentication token. Could not connect to the broadcast." + ) + broadcast_server.logger.error( + "ERROR: Invalid authentication token passed by a client connecting." + ) + await websocket.close(code=1008, reason="Invalid authentication token") + return + + await websocket.accept() + + if RESULTS_FILE is None: + raise ValueError("Results file path is required for WebSocket server.") + + broadcast_server.websocket = websocket + connected_clients.add(broadcast_server) + + stream_task = asyncio.create_task(broadcast_server.stream_results()) + try: + await websocket.receive_text() + + except WebSocketDisconnect: + pass + except Exception as e: + broadcast_server.logger.error(f"Unexpected error: {e}") + pass + finally: + if broadcast_server in connected_clients: + connected_clients.remove(broadcast_server) + stream_task.cancel() + try: + await stream_task + except asyncio.CancelledError: + broadcast_server.logger.info("Stream task cancelled") + except Exception as e: + broadcast_server.logger.error(f"Error while cancelling stream task: {e}") + if websocket.client_state != WebSocketState.DISCONNECTED: + try: + await websocket.close() + except RuntimeError as e: + broadcast_server.logger.error(f"Error while closing websocket: {e}") + + +class BroadcastServer: + """Stream new results from a continuously written SQLite database. + + Not intended to be used directly, it is initialized by the server app when it accepts a new connection. + It is responsible for reading the results database and sending new messages to the connected client(s). + """ + + def __init__( + self, + results_file, + table_name, + sleep_time: float = 0.25, + auth_token: Optional[str] = None, + ): + + self.results_file = results_file + self.table_name = table_name + self.logger = get_logger("openbb.websocket.broadcast_server") + self.sleep_time = sleep_time + self.auth_token = auth_token + self._app = app + self.websocket = None + + async def stream_results(self): # noqa: PLR0915 + """Continuously read the database and send new messages as JSON via WebSocket.""" + import sqlite3 # noqa + from pathlib import Path + from openbb_core.app.model.abstract.error import OpenBBError + + file_path = Path(self.results_file).absolute() + last_id = 0 + + if not file_path.exists(): + self.logger.error(f"Results file not found: {file_path}") + return + else: + conn = sqlite3.connect(self.results_file) + cursor = conn.cursor() + cursor.execute(f"SELECT MAX(id) FROM {self.table_name}") # noqa:S608 + last_id = cursor.fetchone()[0] or 0 + conn.close() + + try: + while True: + try: + if file_path.exists(): + conn = sqlite3.connect(self.results_file) + cursor = conn.cursor() + cursor.execute( + f"SELECT * FROM {self.table_name} WHERE id > ?", # noqa:S608 + (last_id,), + ) + rows = cursor.fetchall() + conn.close() + + if rows: + for row in rows: + index, message = row + await self.broadcast(json.dumps(json.loads(message))) + last_id = max(row[0] for row in rows) + else: + self.logger.error(f"Results file not found: {file_path}") + break + + await asyncio.sleep(self.sleep_time) + except KeyboardInterrupt: + self.logger.info("\nResults stream cancelled.") + break + except sqlite3.OperationalError as e: + if "no such table" in str(e): + self.logger.error( + "Results file was removed by the parent process." + ) + break + else: + raise OpenBBError(e) from e + except asyncio.CancelledError: + break + except WebSocketDisconnect: + pass + except Exception as e: + self.logger.error(f"Unexpected error: {e}") + finally: + return + + async def broadcast(self, message: str): + """Broadcast a message to all connected connected clients.""" + disconnected_clients = set() + for client in connected_clients.copy(): + try: + await client.websocket.send_text(message) + except WebSocketDisconnect: + disconnected_clients.add(client) + except Exception as e: + self.logger.error(f"Unexpected error: {e}") + disconnected_clients.add(client) + # Remove disconnected connected clients + for client in disconnected_clients: + connected_clients.remove(client) + + def start_app(self, host: str = "127.0.0.1", port: int = 6666, **kwargs): + uvicorn.run( + self._app, + host=host, + port=port, + **kwargs, + ) + + +def create_broadcast_server( + results_file: str, + table_name: str, + sleep_time: float = 0.25, + auth_token: Optional[str] = None, + **kwargs, +): + return BroadcastServer(results_file, table_name, sleep_time, auth_token) + + +def main(): + broadcast_server = create_broadcast_server( + RESULTS_FILE, + TABLE_NAME, + SLEEP_TIME, + str(AUTH_TOKEN), + ) + + try: + broadcast_server.start_app( + host=HOST, + port=PORT, + **kwargs, + ) + except TypeError as e: + broadcast_server.logger.error( + f"Invalid keyword argument passed to unvicorn. -> {e.args[0]}\n" + ) + except KeyboardInterrupt: + broadcast_server.logger.info("Broadcast server terminated.") + finally: + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py new file mode 100644 index 000000000000..3ddad5092e4c --- /dev/null +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -0,0 +1,675 @@ +"""WebSocket Client module for interacting with a provider websocket in a non-blocking pattern.""" + +# pylint: disable=too-many-statements +# flake8: noqa: PLR0915 +import logging +from typing import TYPE_CHECKING, Literal, Optional + +if TYPE_CHECKING: + from openbb_core.provider.abstract.data import Data + + +class WebSocketClient: + """Client for interacting with a websocket server in a non-blocking pattern. + + Parameters + ---------- + name : str + Name to assign the WebSocket connection. Used to identify and manage multiple instances. + module : str + The Python module for the provider server connection. Runs in a separate thread. + Example: 'openbb_fmp.websockets.server'. Pass additional keyword arguments by including kwargs. + symbol : Optional[str] + The symbol(s) requested to subscribe. Enter multiple symbols separated by commas without spaces. + limit : Optional[int] + The limit of records to hold in memory. Once the limit is reached, the oldest records are removed. + Default is None. + results_file : Optional[str] + Absolute path to the file for continuous writing. By default, a temporary file is created. + table_name : Optional[str] + SQL table name to store serialized data messages. By default, 'records'. + save_results : bool + Whether to persist the results after the main Python session ends. Default is False. + data_model : Optional[Data] + Pydantic data model to validate the results before storing them in the database. + Also used to deserialize the results from the database. + auth_token : Optional[str] + The authentication token to use for the WebSocket connection. Default is None. + Only used for API and Python application endpoints. + logger : Optional[logging.Logger] + The logger instance to use this connection. By default, a new logger is created. + kwargs : dict + Additional keyword arguments to pass to the target module. + + Properties + ---------- + symbol : str + Symbol(s) requested to subscribe. + module : str + Path to the provider connection script. + is_running : bool + Check if the provider connection is running. + is_broadcasting : bool + Check if the broadcast server is running. + broadcast_address : str + URI address for the results broadcast server. + results : list + All stored results from the provider's WebSocket stream. The results are stored in a SQLite database. + Set the 'limit' property to cap the number of stored records. + Clear the results by deleting the property. e.g., del client.results + transformed_results : list + Deserialize the records from the results file using the provided data model, if available. + + Methods + ------- + connect + Connect to the provider WebSocket stream. + disconnect + Disconnect from the provider WebSocket. + subscribe + Subscribe to a new symbol or list of symbols. + unsubscribe + Unsubscribe from a symbol or list of symbols. + start_broadcasting + Start the broadcast server to stream results over a network connection. + stop_broadcasting + Stop the broadcast server and disconnect all reading clients. + send_message + Send a message to the WebSocket process. + """ + + def __init__( # noqa: PLR0913 + self, + name: str, + module: str, + symbol: Optional[str] = None, + limit: Optional[int] = None, + results_file: Optional[str] = None, + table_name: Optional[str] = None, + save_results: bool = False, + data_model: Optional["Data"] = None, + auth_token: Optional[str] = None, + logger: Optional[logging.Logger] = None, + **kwargs, + ): + """Initialize the WebSocketClient class.""" + # pylint: disable=import-outside-toplevel + import asyncio # noqa + import atexit + import tempfile + import threading + from aiosqlite import DatabaseError + from queue import Queue + from pathlib import Path + from openbb_websockets.helpers import get_logger + + self.name = name + self.module = module.replace(".py", "") + self.results_file = results_file if results_file else None + self.table_name = table_name if table_name else "records" + self._limit = limit + self.data_model = data_model + self._auth_token = auth_token + self._symbol = symbol + self._kwargs = ( + [f"{k}={str(v).strip().replace(" ", "_")}" for k, v in kwargs.items()] + if kwargs + else None + ) + + self._process = None + self._psutil_process = None + self._thread = None + self._log_thread = None + self._provider_message_queue = Queue() + self._stop_log_thread_event = threading.Event() + self._stop_broadcasting_event = threading.Event() + self._broadcast_address = None + self._broadcast_process = None + self._psutil_broadcast_process = None + self._broadcast_thread = None + self._broadcast_log_thread = None + self._broadcast_message_queue = Queue() + + if not results_file: + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + pass + temp_file_path = temp_file.name + self.results_path = Path(temp_file_path).absolute() + self.results_file = temp_file_path + + self.results_path = Path(self.results_file).absolute() + self.save_results = save_results + self.logger = logger if logger else get_logger("openbb.websocket.client") + + atexit.register(self._atexit) + + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + try: + if loop.is_running(): + loop.create_task(self._setup_database()) + else: + asyncio.run(self._setup_database()) + except DatabaseError as e: + self.logger.error("Error setting up the SQLite database and table: %s", e) + + def _atexit(self): + """Clean up the WebSocket client processes at exit.""" + # pylint: disable=import-outside-toplevel + import os + + if self.is_running: + self.disconnect() + if self.is_broadcasting: + self.stop_broadcasting() + if self.save_results: + self.logger.info("Websocket results saved to, %s\n", self.results_file) + if os.path.exists(self.results_file): + os.remove(self.results_file) + + async def _setup_database(self): + """Set up the SQLite database and table.""" + # pylint: disable=import-outside-toplevel + from openbb_websockets.helpers import setup_database + + return await setup_database(self.results_path, self.table_name) + + def _log_provider_output(self, output_queue): + """Log output from the provider server queue.""" + # pylint: disable=import-outside-toplevel + import queue # noqa + import sys + from openbb_websockets.helpers import clean_message + + while not self._stop_log_thread_event.is_set(): + try: + output = output_queue.get(timeout=1) + if output: + output = clean_message(output) + output = output + "\n" + sys.stdout.write(output + "\n") + sys.stdout.flush() + except queue.Empty: + continue + + def _log_broadcast_output(self, output_queue): + """Log output from the broadcast server queue.""" + # pylint: disable=import-outside-toplevel + import queue # noqa + import sys + from openbb_websockets.helpers import clean_message + + while not self._stop_broadcasting_event.is_set(): + try: + output = output_queue.get(timeout=1) + + if output and "Uvicorn running" in output: + address = ( + output.split("Uvicorn running on ")[-1] + .strip() + .replace(" (Press CTRL+C to quit)", "") + .replace("http", "ws") + ) + output = "INFO: " + f"Stream results from {address}" + self._broadcast_address = address + + if output and "Started server process" in output: + output = None + + if output and "Waiting for application startup." in output: + output = None + + if output and "Application startup complete." in output: + output = None + + if output: + if "ERROR:" in output: + output = output.replace("ERROR:", "BROADCAST ERROR:") + "\n" + if "INFO:" in output: + output = output.replace("INFO:", "BROADCAST INFO:") + "\n" + output = output[0] if isinstance(output, tuple) else output + output = clean_message(output) + sys.stdout.write(output + "\n") + sys.stdout.flush() + except queue.Empty: + continue + + def connect(self): + """Connect to the provider WebSocket.""" + # pylint: disable=import-outside-toplevel + import json # noqa + import os + import queue + import subprocess + import threading + import psutil + + if self.is_running: + self.logger.info("Provider connection already running.") + return + + symbol = self.symbol + + if not symbol: + self.logger.info("No subscribed symbols.") + return + + command = self.module.copy() + command.extend([f"symbol={symbol}"]) + command.extend([f"results_file={self.results_file}"]) + command.extend([f"table_name={self.table_name}"]), + + if self.limit: + command.extend([f"limit={self.limit}"]) + + if self._kwargs: + for kwarg in self._kwargs: + if kwarg not in command: + command.extend([kwarg]) + + self._process = subprocess.Popen( # noqa + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdin=subprocess.PIPE, + env=os.environ, + text=True, + bufsize=1, + ) + self._psutil_process = psutil.Process(self._process.pid) + + log_output_queue = queue.Queue() + self._thread = threading.Thread( + target=non_blocking_websocket, + args=( + self, + log_output_queue, + self._provider_message_queue, + ), + ) + self._thread.daemon = True + self._thread.start() + + self._log_thread = threading.Thread( + target=self._log_provider_output, + args=(log_output_queue,), + ) + self._log_thread.daemon = True + self._log_thread.start() + + if not self.is_running: + self.logger.error("The provider server failed to start.") + + def send_message( + self, message, target: Literal["provider", "broadcast"] = "provider" + ): + """Send a message to the WebSocket process.""" + if target == "provider": + self._provider_message_queue.put(message) + read_message_queue(self, self._provider_message_queue) + elif target == "broadcast": + self._broadcast_message_queue.put(message) + read_message_queue(self, self._broadcast_message_queue, target="broadcast") + + def disconnect(self): + """Disconnect from the provider WebSocket.""" + self._stop_log_thread_event.set() + if self._process is None or self.is_running is False: + self.logger.info("Not connected to the provider WebSocket.") + return + if ( + self._psutil_process is not None + and hasattr(self._psutil_process, "is_running") + and self._psutil_process.is_running() + ): + self._psutil_process.kill() + self._process.wait() + self._thread.join() + self._log_thread.join() + self._stop_log_thread_event.clear() + self.logger.info("Disconnected from the provider WebSocket.") + return + + def subscribe(self, symbol): + """Subscribe to a new symbol or list of symbols.""" + # pylint: disable=import-outside-toplevel + import json + + ticker = symbol if isinstance(symbol, list) else symbol.split(",") + msg = {"event": "subscribe", "symbol": ticker} + self.send_message(json.dumps(msg)) + old_symbols = self.symbol.split(",") + new_symbols = list(set(old_symbols + ticker)) + self._symbol = ",".join(new_symbols) + + def unsubscribe(self, symbol): + """Unsubscribe from a symbol or list of symbols.""" + # pylint: disable=import-outside-toplevel + import json + + if not self.symbol: + self.logger.info("No subscribed symbols.") + return + + ticker = symbol if isinstance(symbol, list) else symbol.split(",") + msg = {"event": "unsubscribe", "symbol": ticker} + self.send_message(json.dumps(msg)) + old_symbols = self.symbol.split(",") + new_symbols = list(set(old_symbols) - set(ticker)) + self._symbol = ",".join(new_symbols) + + @property + def is_running(self): + """Check if the provider connection is running.""" + if hasattr(self._psutil_process, "is_running"): + return self._psutil_process.is_running() + return False + + @property + def is_broadcasting(self): + """Check if the broadcast server is running.""" + if hasattr(self._psutil_broadcast_process, "is_running"): + return self._psutil_broadcast_process.is_running() + return False + + @property + def results(self): + """Retrieve the raw results dumped by the WebSocket stream.""" + # pylint: disable=import-outside-toplevel + import json # noqa + import sqlite3 + + output: list = [] + file_path = self.results_path + if file_path.exists(): + with sqlite3.connect(file_path) as conn: + cursor = conn.execute(f"SELECT * FROM {self.table_name}") # noqa + for row in cursor: + index, message = row + output.append(json.loads(message)) + + return output + + self.logger.info("No results found in %s", self.results_file) + + return [] + + @results.deleter + def results(self): + """Clear results stored from the WebSocket stream.""" + # pylint: disable=import-outside-toplevel + import asyncio + import sqlite3 + + try: + with sqlite3.connect(self.results_path) as conn: + conn.execute(f"DELETE FROM {self.table_name}") # noqa + conn.commit() + + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + loop.run_until_complete(self._setup_database()) + self.logger.info( + "Results cleared from table %s in %s", + self.table_name, + self.results_file, + ) + except Exception as e: + self.logger.error("Error clearing results: %s", e) + + @property + def module(self): + """Path to the provider connection script.""" + return self._module + + @module.setter + def module(self, module): + """Set the path to the provider connection script.""" + # pylint: disable=import-outside-toplevel + import sys + + self._module = [ + sys.executable, + "-m", + module, + ] + + @property + def symbol(self): + """Symbol(s) requested to subscribe.""" + return self._symbol + + @property + def limit(self): + """Get the limit of records to hold in memory.""" + return self._limit + + @limit.setter + def limit(self, limit): + """Set the limit of records to hold in memory.""" + self._limit = limit + + @property + def broadcast_address(self): + """Get the WebSocket broadcast address.""" + return ( + self._broadcast_address + if self._broadcast_address and self.is_broadcasting + else None + ) + + def start_broadcasting( + self, + host: str = "127.0.0.1", + port: int = 6666, + **kwargs, + ): + """Broadcast results over a network connection.""" + # pylint: disable=import-outside-toplevel + import os # noqa + import subprocess + import sys + import threading + import psutil + import queue + from openbb_platform_api.utils.api import check_port + + if ( + self._broadcast_process is not None + and self._broadcast_process.poll() is None + ): + self.logger.info( + f"WebSocket broadcast already running on: {self._broadcast_address}" + ) + return + + open_port = check_port(host, port) + + if open_port != port: + msg = f"Port {port} is already in use. Using {open_port} instead." + self.logger.warning(msg) + + command = [ + sys.executable, + "-m", + "openbb_websockets.broadcast", + f"host={host}", + f"port={open_port}", + f"results_file={self.results_file}", + f"table_name={self.table_name}", + f"auth_token={self._auth_token}", + ] + if kwargs: + for kwarg in kwargs: + command.extend([f"{kwarg}={kwargs[kwarg]}"]) + + self._broadcast_process = subprocess.Popen( # noqa + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdin=subprocess.PIPE, + env=os.environ, + text=True, + bufsize=1, + ) + self._psutil_broadcast_process = psutil.Process(self._broadcast_process.pid) + output_queue = queue.Queue() + self._broadcast_thread = threading.Thread( + target=non_blocking_broadcast, + args=( + self, + output_queue, + self._broadcast_message_queue, + ), + ) + self._broadcast_thread.daemon = True + self._broadcast_thread.start() + + self._broadcast_log_thread = threading.Thread( + target=self._log_broadcast_output, + args=(output_queue,), + ) + self._broadcast_log_thread.daemon = True + self._broadcast_log_thread.start() + + if not self.is_broadcasting: + self.logger.error( + "The broadcast server failed to start on: %s", + self._broadcast_address, + ) + + def stop_broadcasting(self): + """Stop the broadcast server.""" + broadcast_address = self._broadcast_address + self._stop_broadcasting_event.set() + if self._broadcast_process is None or self.is_broadcasting is False: + self.logger.info("Not currently broadcasting.") + return + if ( + self._psutil_broadcast_process is not None + and hasattr(self._psutil_broadcast_process, "is_running") + and self._psutil_broadcast_process.is_running() + ): + self._psutil_broadcast_process.kill() + if broadcast_address: + self.logger.info("Stopped broadcasting to: %s", broadcast_address) + + self._broadcast_process.wait() + self._broadcast_thread.join() + self._broadcast_log_thread.join() + self._broadcast_process = None + self._psutil_broadcast_process = None + self._broadcast_address = None + self._stop_broadcasting_event.clear() + return + + @property + def transformed_results(self): + """Deserialize the records from the results file.""" + # pylint: disable=import-outside-toplevel + import json + + if not self.data_model: + raise NotImplementedError("No model provided to transform the results.") + + return [self.data_model.model_validate(json.loads(d)) for d in self.results] + + def __repr__(self): + """Return the WebSocketClient representation.""" + return ( + f"WebSocketClient(module={self.module}, symbol={self.symbol}, " + f"is_running={self.is_running}, provider_pid: " + f"{self._psutil_process.pid if self._psutil_process else ''}, is_broadcasting={self.is_broadcasting}, " + f"broadcast_address={self.broadcast_address}, " + f"broadcast_pid: {self._psutil_broadcast_process.pid if self._psutil_broadcast_process else ''}, " + f"results_file={self.results_file}, table_name={self.table_name}, " + f"save_results={self.save_results})" + ) + + +def non_blocking_websocket(client, output_queue, provider_message_queue): + """Communicate with the threaded process.""" + try: + while not client._stop_log_thread_event.is_set(): + while not provider_message_queue.empty(): + read_message_queue(client, provider_message_queue) + output = client._process.stdout.readline() + if output == "" and client._process.poll() is not None: + break + if output: + output_queue.put(output.strip()) + + except Exception as e: + raise e from e + client.logger.error(f"Error in non_blocking_websocket: {e}") + finally: + client._process.stdout.close() + client._process.wait() + + +def send_message( + client, message, target: Literal["provider", "broadcast"] = "provider" +): + """Send a message to the WebSocket process.""" + try: + if target == "provider": + if client._process and client._process.stdin: + client._process.stdin.write(message + "\n") + client._process.stdin.flush() + else: + client.logger.error("Provider process is not running.") + elif target == "broadcast": + if client._broadcast_process and client._broadcast_process.stdin: + client._broadcast_process.stdin.write(message + "\n") + client._broadcast_process.stdin.flush() + else: + client.logger.error("Broadcast process is not running.") + except Exception as e: + client.logger.error(f"Error sending message to WebSocket process: {e}") + + +def read_message_queue( + client, message_queue, target: Literal["provider", "broadcast"] = "provider" +): + """Read messages from the queue and send them to the WebSocket process.""" + while not message_queue.empty(): + try: + if target == "provider": + while not client._stop_log_thread_event.is_set(): + message = message_queue.get(timeout=1) + if message: + send_message(client, message, target="provider") + elif target == "broadcast": + while not client._stop_broadcasting_event.is_set(): + message = message_queue.get(timeout=1) + if message: + send_message(client, message, target="broadcast") + except Exception as e: + err = f"Error reading message queue: {e.args[0]} -> {message}" + client.logger.error(err) + finally: + break + + +def non_blocking_broadcast(client, output_queue, broadcast_message_queue): + """Continuously read the output from the broadcast process and log it to the main thread.""" + try: + while not client._stop_broadcasting_event.is_set(): + while not broadcast_message_queue.empty(): + read_message_queue(client, broadcast_message_queue, target="broadcast") + + output = client._broadcast_process.stdout.readline() + if output == "" and client._broadcast_process.poll() is not None: + break + if output: + output_queue.put(output.strip()) + except Exception as e: + client.logger.error(f"Error in non_blocking_broadcast: {e}") + finally: + client._broadcast_process.stdout.close() + client._broadcast_process.wait() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py new file mode 100644 index 000000000000..beb9bec5dbc4 --- /dev/null +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -0,0 +1,238 @@ +"""WebSockets helpers.""" + +import logging +import re +import sys +from typing import Optional + +from openbb_core.app.model.abstract.error import OpenBBError +from openbb_core.provider.utils.errors import UnauthorizedError + +AUTH_TOKEN_FILTER = re.compile( + r"(auth_token=)([^&]*)", + re.IGNORECASE | re.MULTILINE, +) + +connected_clients: dict = {} + + +def clean_message(message: str) -> str: + """Clean the message.""" + return AUTH_TOKEN_FILTER.sub(r"\1********", message) + + +def get_logger(name, level=logging.INFO): + """Get a logger instance.""" + # pylint: disable=import-outside-toplevel + import logging + import uuid + + logger = logging.getLogger(f"{name}-{uuid.uuid4()}") + handler = logging.StreamHandler() + handler.setLevel(level) + formatter = logging.Formatter("%(message)s\n") + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(level) + return logger + + +async def get_status(name: str) -> dict: + """Get the status of a client.""" + if name not in connected_clients: + raise OpenBBError(f"Client {name} not connected.") + client = connected_clients[name] + provider_pid = client._psutil_process.pid if client.is_running else None + broadcast_pid = ( + client._psutil_broadcast_process.pid if client.is_broadcasting else None + ) + status = { + "name": client.name, + "auth_required": client._auth_token is not None, + "subscribed_symbols": client.symbol, + "is_running": client.is_running, + "provider_pid": provider_pid, + "is_broadcasting": client.is_broadcasting, + "broadcast_address": client.broadcast_address, + "broadcast_pid": broadcast_pid, + "results_file": client.results_file, + "table_name": client.table_name, + "save_results": client.save_results, + } + return status + + +async def check_auth(name: str, auth_token: Optional[str] = None) -> bool: + """Check the auth token.""" + if name not in connected_clients: + raise OpenBBError(f"Client {name} not connected.") + client = connected_clients[name] + if client._auth_token is None: + return True + if auth_token is None: + raise UnauthorizedError(f"Client authorization token is required for {name}.") + if auth_token != client._auth_token: + raise UnauthorizedError(f"Invalid client authorization token for {name}.") + return True + + +def handle_termination_signal(logger): + """Handle termination signals to ensure graceful shutdown.""" + # pylint: disable=import-outside-toplevel + import sys + + logger.info( + "PROVIDER INFO: Termination signal received. WebSocket connection closed." + ) + sys.exit(0) + + +def parse_kwargs(): + """Parse command line keyword arguments.""" + # pylint: disable=import-outside-toplevel + import sys + + args = sys.argv[1:].copy() + _kwargs: dict = {} + for i, arg in enumerate(args): + if "=" in arg: + key, value = arg.split("=") + _kwargs[key] = value + elif arg.startswith("--"): + key = arg[2:] + if i + 1 < len(args) and not args[i + 1].startswith("--"): + value = args[i + 1] + if isinstance(value, str) and value.lower() in ["false", "true"]: + value = value.lower() == "true" + elif isinstance(value, str) and value.lower() == "none": + value = None + _kwargs[key] = value + else: + _kwargs[key] = True + + return _kwargs + + +async def setup_database(results_path, table_name): + # pylint: disable=import-outside-toplevel + import os # noqa + import aiosqlite + + async with aiosqlite.connect(results_path) as conn: + if os.path.exists(results_path): + try: + await conn.execute("SELECT name FROM sqlite_master WHERE type='table';") + except aiosqlite.DatabaseError: + os.remove(results_path) + + async with aiosqlite.connect(results_path) as conn: + await conn.execute( + f""" + CREATE TABLE IF NOT EXISTS {table_name} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message TEXT + ) + """ + ) + await conn.commit() + + +async def write_to_db(message, results_path, table_name, limit): + """Write the WebSocket message to the SQLite database.""" + # pylint: disable=import-outside-toplevel + import json # noqa + import aiosqlite + + conn = await aiosqlite.connect(results_path) + await conn.execute( + f"INSERT INTO {table_name} (message) VALUES (?)", # noqa + (json.dumps(message),), + ) + await conn.commit() + records = await conn.execute(f"SELECT COUNT(*) FROM {table_name}") # noqa + count = (await records.fetchone())[0] + count = await conn.execute(f"SELECT COUNT(*) FROM {table_name}") # noqa + current_count = int((await count.fetchone())[0]) + limit = 0 if limit is None else int(limit) + + if current_count > limit and limit != 0: + await conn.execute( + f""" + DELETE FROM {table_name} + WHERE id IN ( + SELECT id FROM {table_name} + ORDER BY id DESC + LIMIT -1 OFFSET ? + ) + """, # noqa: S608 + (limit,), + ) + + await conn.commit() + await conn.close() + + +class StdOutSink: + """Filter stdout for PII.""" + + def write(self, message): + """Write to stdout.""" + cleaned_message = AUTH_TOKEN_FILTER.sub(r"\1********", message) + if cleaned_message != message: + cleaned_message = f"{cleaned_message}\n" + sys.__stdout__.write(cleaned_message) + + def flush(self): + """Flush stdout.""" + sys.__stdout__.flush() + + +class AuthTokenFilter(logging.Formatter): + """Custom logging formatter to filter auth tokens.""" + + def format(self, record): + original_message = super().format(record) + cleaned_message = AUTH_TOKEN_FILTER.sub(r"\1********", original_message) + return cleaned_message + + +class MessageQueue: + def __init__(self, max_size: int = 1000, max_retries=5, backoff_factor=0.5): + """Initialize the MessageQueue.""" + # pylint: disable=import-outside-toplevel + from asyncio import Queue + + self.queue = Queue(maxsize=max_size) + self.max_retries = max_retries + self.backoff_factor = backoff_factor + self.logger = get_logger("openbb.websocket.queue") + + async def dequeue(self): + return await self.queue.get() + + async def enqueue(self, message): + """Enqueue a message.""" + # pylint: disable=import-outside-toplevel + from asyncio import sleep + from queue import Full + + retries = 0 + while retries < self.max_retries: + try: + await self.queue.put(message) + return + except Full: + retries += 1 + msg = f"Queue is full. Retrying {retries}/{self.max_retries}..." + self.logger.warning(msg) + await sleep(self.backoff_factor * retries) + self.logger.error("Failed to enqueue message after maximum retries.") + + async def process_queue(self, handler): + while True: + message = await self.queue.get() + await self._process_message(message, handler) + self.queue.task_done() + + async def _process_message(self, message, handler): + await handler(message) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/listen.py b/openbb_platform/extensions/websockets/openbb_websockets/listen.py new file mode 100644 index 000000000000..e16f96817a64 --- /dev/null +++ b/openbb_platform/extensions/websockets/openbb_websockets/listen.py @@ -0,0 +1,157 @@ +"""Convenience tool for listening to raw broadcast streams outside of the main application thread.""" + + +class Listener: + """WebSocket broadcast listener. Not intended to be initialized directly, use the 'listen' function.""" + + def __init__(self, **kwargs): + """Initialize the Listener. All keyword arguments are passed directly to websockets.connect.""" + + self.loop = None + self.websocket = None + self.current_task = None + self.kwargs = {} + if kwargs: + self.kwargs = kwargs + + async def listen(self, url, **kwargs): # noqa: PLR0915 + """Listen for WebSocket messages.""" + # pylint: disable=import-outside-toplevel + import asyncio # noqa + import json + import websockets + from openbb_core.app.model.abstract.error import OpenBBError + from openbb_core.provider.utils.errors import UnauthorizedError + from openbb_websockets.helpers import clean_message, get_logger + from websockets.exceptions import InvalidStatusCode + + kwargs = kwargs or {} + + if self.kwargs: + for k, v in self.kwargs.items(): + if k not in kwargs: + kwargs[k] = v + + self.logger = get_logger(url) + url = url.replace("http", "ws") + + if url.startswith("localhost"): + url = url.replace("localhost", "ws://localhost") + + if url[0].isdigit(): + url = f"ws://{url}" + + try: + while True: + try: + async with websockets.connect(url, **kwargs) as websocket: + self.websocket = websocket + url = clean_message(url) + self.logger.info( + f"\nListening for messages from {clean_message(url)}" + ) + for handler in self.logger.handlers: + handler.flush() + async for message in websocket: + if ( + isinstance(message, str) + and "Invalid authentication token" in message + ): + raise UnauthorizedError(message) + self.logger.info(json.loads(message)) + for handler in self.logger.handlers: + handler.flush() + except UnauthorizedError as error: + self.logger.error(error) + break + except (KeyboardInterrupt, asyncio.CancelledError): + self.logger.info("Disconnected from server.") + break + except ( + websockets.ConnectionClosedError, + asyncio.IncompleteReadError, + ): + self.logger.error( + f"The process hosting {clean_message(url)} was terminated." + ) + break + except websockets.exceptions.InvalidURI as error: + self.logger.error(f"Invalid URI -> {error}") + break + except InvalidStatusCode as error: + self.logger.error(f"Invalid status code -> {error}") + break + except OSError as error: + if "Multiple exceptions" in str(error): + err = str(error).split("Multiple exceptions:")[1].strip() + err = err.split("[")[-1].strip().replace("]", ":") + self.logger.error( + f"An error occurred while attempting to connect to: {clean_message(url)} -> {err}" + ) + else: + self.logger.error( + f"An error occurred while attempting to connect to: {clean_message(url)} -> {error}" + ) + break + except Exception as error: + self.logger.error(f"An unexpected error occurred: {error}") + raise OpenBBError(error) from error + finally: + if self.websocket: + await self.websocket.close() + + def stop(self): + if self.current_task: + self.current_task.cancel() + self.loop.run_until_complete(self.current_task) + if self.websocket: + self.loop.run_until_complete(self.websocket.close()) + if not self.loop.is_closed(): + self.loop.stop() + + async def start_listening(self, url, **kwargs): + # pylint: disable=import-outside-toplevel + import asyncio + import contextlib + + self.current_task = self.loop.create_task(self.listen(url, **kwargs)) + with contextlib.suppress(asyncio.CancelledError): + await self.current_task + + def run(self, url, **kwargs): + # pylint: disable=import-outside-toplevel + import asyncio + + try: + self.loop = asyncio.get_running_loop() + except RuntimeError: + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + try: + self.loop.run_until_complete(self.start_listening(url, **kwargs)) + except KeyboardInterrupt: + self.logger.info("\nWebSocket listener terminated.") + finally: + self.stop() + + +def listen(url, **kwargs): + """Listen for WebSocket messages from a given URL. This function is blocking. + + Parameters + ---------- + url : str + The WebSocket URL to connect to. + kwargs : dict + Additional keyword arguments passed directly to websockets.connect + """ + # pylint: disable=import-outside-toplevel + from openbb_core.app.model.abstract.error import OpenBBError + + try: + listener = Listener(**kwargs) + listener.run(url, **kwargs) + except Exception as e: + raise OpenBBError(e) from e + finally: + return diff --git a/openbb_platform/extensions/websockets/openbb_websockets/models.py b/openbb_platform/extensions/websockets/openbb_websockets/models.py new file mode 100644 index 000000000000..a631ba0cf8f9 --- /dev/null +++ b/openbb_platform/extensions/websockets/openbb_websockets/models.py @@ -0,0 +1,98 @@ +"""WebSockets models.""" + +from datetime import datetime +from typing import Any, Optional + +from openbb_core.provider.abstract.data import Data +from openbb_core.provider.abstract.query_params import QueryParams +from openbb_core.provider.utils.descriptions import ( + DATA_DESCRIPTIONS, + QUERY_DESCRIPTIONS, +) +from pydantic import ConfigDict, Field, field_validator + +from openbb_websockets.client import WebSocketClient + + +class WebSocketQueryParams(QueryParams): + """Query parameters for WebSocket connection creation.""" + + symbol: str = Field( + description=QUERY_DESCRIPTIONS.get("symbol", ""), + ) + name: str = Field( + description="Name to assign the client connection.", + ) + auth_token: Optional[str] = Field( + default=None, + description="Authentication token for API access control of the client, not related to the provider credentials.", + ) + results_file: Optional[str] = Field( + default=None, + description="Absolute path to the file for continuous writing. By default, a temporary file is created.", + ) + save_results: bool = Field( + default=False, + description="Whether to save the results after the session ends.", + ) + table_name: str = Field( + default="records", + description="Name of the SQL table to write the results to.", + ) + limit: Optional[int] = Field( + default=1000, + description="Maximum number of newest records to keep in the database." + + " If None, all records are kept, which can be memory-intensive.", + ) + sleep_time: float = Field( + default=0.25, + description="Time to sleep between checking for new records in the database from the broadcast server." + + " The default is 0.25 seconds.", + ) + broadcast_host: str = Field( + default="127.0.0.1", + description="IP address to bind the broadcast server to.", + ) + broadcast_port: int = Field( + default=6666, + description="Port to bind the broadcast server to.", + ) + start_broadcast: bool = Field( + default=True, + description="Whether to start the broadcast server." + + " Set to False if system or network conditions do not allow it." + + " Can be started manually with the 'start_broadcasting' method.", + ) + + +class WebSocketData(Data): + """WebSocket data model.""" + + date: datetime = Field( + description=DATA_DESCRIPTIONS.get("date", ""), + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + + +class WebSocketConnection(Data): + """Data model for returning WebSocketClient from the Provider Interface.""" + + __model_config__ = ConfigDict( + extra="forbid", + ) + + client: Any = Field( + description="Instance of WebSocketClient class initialized by a provider Fetcher." + + " The client is used to communicate with the provider's data stream." + + " It is not returned to the user, but is handled by the router for API access.", + exclude=True, + ) + + @field_validator("client", mode="before", check_fields=False) + def _validate_client(cls, v): + """Validate the client.""" + if not isinstance(v, WebSocketClient): + raise ValueError("Client must be an instance of WebSocketClient.") + return v diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py new file mode 100644 index 000000000000..18bbbb85bc15 --- /dev/null +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -0,0 +1,428 @@ +"""Websockets Router.""" + +# pylint: disable=unused-argument + +import asyncio +import sys +from typing import Any, Optional + +from openbb_core.app.model.abstract.error import OpenBBError +from openbb_core.app.model.command_context import CommandContext +from openbb_core.app.model.example import APIEx +from openbb_core.app.model.obbject import OBBject +from openbb_core.app.provider_interface import ( + ExtraParams, + ProviderChoices, + StandardParams, +) +from openbb_core.app.query import Query +from openbb_core.app.router import Router +from openbb_core.provider.utils.errors import EmptyDataError + +from openbb_websockets.helpers import ( + StdOutSink, + check_auth, + connected_clients, + get_status, +) + +router = Router("", description="WebSockets Router") +sys.stdout = StdOutSink() + + +@router.command( + model="WebSocketConnection", +) +async def create_connection( + cc: CommandContext, + provider_choices: ProviderChoices, + standard_params: StandardParams, + extra_params: ExtraParams, +) -> OBBject: + """Create a new provider websocket connection.""" + name = extra_params.name + if name in connected_clients: + broadcast_address = connected_clients[name].broadcast_address + is_running = connected_clients[name].is_running + if broadcast_address or is_running: + raise OpenBBError( + f"Client {name} already connected! Broadcasting to: {broadcast_address}" + ) + raise OpenBBError(f"Client {name} already connected but not running.") + del name + + obbject = await OBBject.from_query(Query(**locals())) + client = obbject.results.client + + await asyncio.sleep(1) + + if not client.is_running: + client._atexit() + raise OpenBBError("Client failed to connect.") + + if hasattr(extra_params, "start_broadcast") and extra_params.start_broadcast: + client.start_broadcasting() + + client_name = client.name + connected_clients[client_name] = client + results = await get_status(client_name) + + obbject.results = results + + return obbject + + +@router.command( + methods=["GET"], +) +async def get_results(name: str, auth_token: Optional[str] = None) -> OBBject: + """Get all recorded results from a client connection. + + Parameters + ---------- + name : str + The name of the client. + auth_token : Optional[str] + The client's authorization token. + + Returns + ------- + list[Data] + The recorded results from the client. + """ + + if not await check_auth(name, auth_token): + raise OpenBBError("Error finding client.") + client = connected_clients[name] + if not client.results: + raise EmptyDataError(f"No results recorded for client {name}.") + try: + return OBBject(results=client.transformed_results) + except NotImplementedError: + return OBBject(results=client.results) + + +@router.command( + methods=["GET"], +) +async def clear_results(name: str, auth_token: Optional[str] = None) -> OBBject[str]: + """Clear all stored results from a client connection. Does not stop the client or broadcast. + + Parameters + ---------- + name : str + The name of the client. + auth_token : Optional[str] + The client's authorization token. + + Returns + ------- + str + The number of results cleared from the client. + """ + if not await check_auth(name, auth_token): + raise OpenBBError("Error finding client.") + client = connected_clients[name] + n_before = len(client.results) + del client.results + return OBBject(results=f"{n_before} results cleared from {name}.") + + +@router.command( + methods=["GET"], +) +async def subscribe( + name: str, symbol: str, auth_token: Optional[str] = None +) -> OBBject[str]: + """Subscribe to a new symbol. + + Parameters + ---------- + name : str + The name of the client. + symbol : str + The symbol to subscribe to. + auth_token : Optional[str] + The client's authorization token. + + Returns + ------- + str + The message that the client subscribed to the symbol. + """ + if not await check_auth(name, auth_token): + raise OpenBBError("Error finding client.") + client = connected_clients[name] + symbols = client.symbol.split(",") + if symbols and symbol in symbols: + raise OpenBBError(f"Client {name} already subscribed to {symbol}.") + client.subscribe(symbol) + # await asyncio.sleep(2) + if client.is_running: + return OBBject(results=f"Added {symbol} to client {name} connection.") + client.logger.error( + f"Client {name} failed to subscribe to {symbol} and is not running." + ) + + +@router.command( + methods=["GET"], +) +async def unsubscribe( + name: str, symbol: str, auth_token: Optional[str] = None +) -> OBBject[str]: + """Unsubscribe to a symbol. + + Parameters + ---------- + name : str + The name of the client. + symbol : str + The symbol to unsubscribe from. + auth_token : Optional[str] + The client's authorization token. + + Returns + ------- + str + The message that the client unsubscribed from the symbol. + """ + if not await check_auth(name, auth_token): + raise OpenBBError("Error finding client.") + client = connected_clients[name] + symbols = client.symbol.split(",") + if symbol not in symbols: + raise OpenBBError(f"Client {name} not subscribed to {symbol}.") + client.unsubscribe(symbol) + # await asyncio.sleep(2) + if client.is_running: + return OBBject(results=f"Client {name} unsubscribed to {symbol}.") + client.logger.error( + f"Client {name} failed to unsubscribe to {symbol} and is not running." + ) + + +@router.command( + methods=["GET"], +) +async def get_client_status(name: str = "all") -> OBBject[list[dict]]: + """Get the status of a client, or all client connections. + + Parameters + ---------- + name : str + The name of the client. Default is "all". + + Returns + ------- + list[dict] + The status of the client(s). + """ + if not connected_clients: + raise OpenBBError("No active connections.") + if name == "all": + connections = [ + await get_status(client.name) for client in connected_clients.values() + ] + else: + connections = [await get_status(name)] + return OBBject(results=connections) + + +@router.command( + methods=["GET"], + include_in_schema=False, +) +async def get_client(name: str, auth_token: Optional[str] = None) -> OBBject: + """Get an open client connection object. This endpoint is only available from the Python interface. + + Parameters + ---------- + name : str + The name of the client. + auth_token : Optional[str] + The client's authorization token. + + Returns + ------- + WebSocketClient + The provider client connection object. + """ + if not await check_auth(name, auth_token): + raise OpenBBError("Error finding client.") + client = connected_clients[name] + return OBBject(results=client) + + +@router.command( + methods=["GET"], +) +async def stop_connection(name: str, auth_token: Optional[str] = None) -> OBBject[str]: + """Stop a the connection to the provider's websocket. Does not stop the broadcast server. + + Parameters + ---------- + name : str + The name of the client. + auth_token : Optional[str] + The client's authorization token. + + Returns + ------- + str + The message that the provider connection was stopped. + """ + if not await check_auth(name, auth_token): + raise OpenBBError("Error finding client.") + client = connected_clients[name] + client.disconnect() + return OBBject( + results=f"Client {name} connection to the provider's websocket was stopped." + ) + + +@router.command( + methods=["GET"], +) +async def restart_connection( + name: str, auth_token: Optional[str] = None +) -> OBBject[str]: + """Restart a websocket connection. + + Parameters + ---------- + name : str + The name of the client. + auth_token : Optional[str] + The client's authorization token. + + Returns + ------- + str + The message that the client connection was restarted. + """ + if name not in connected_clients: + raise OpenBBError(f"No active client named, {name}. Use create_connection.") + if not await check_auth(name, auth_token): + raise OpenBBError("Error finding client.") + client = connected_clients[name] + client.connect() + return OBBject(results=f"Client {name} connection was restarted.") + + +@router.command( + methods=["GET"], +) +async def stop_broadcasting( + name: str, auth_token: Optional[str] = None +) -> OBBject[str]: + """Stop the broadcast server. + + Parameters + ---------- + name : str + The name of the client. + auth_token : Optional[str] + The client's authorization token. + + Returns + ------- + str + The message that the client stopped broadcasting to the address. + """ + if name not in connected_clients: + raise OpenBBError(f"Client {name} not connected.") + + if not await check_auth(name, auth_token): + raise OpenBBError("Error finding client.") + + client = connected_clients[name] + + if not client.is_broadcasting: + raise OpenBBError(f"Client {name} not broadcasting.") + + old_address = client.broadcast_address + client.stop_broadcasting() + + if not client.is_running: + client._atexit() + del connected_clients[name] + return OBBject( + results=f"Client {name} stopped broadcasting and was not running, client removed." + ) + + return OBBject(results=f"Client {name} stopped broadcasting to: {old_address}") + + +@router.command( + methods=["GET"], +) +async def start_broadcasting( + name: str, + auth_token: Optional[str] = None, + host: str = "127.0.0.1", + port: int = 6666, + uvicorn_kwargs: Optional[dict[str, Any]] = None, +) -> OBBject[str]: + """Start broadcasting from a websocket. + + Parameters + ---------- + name : str + The name of the client. + auth_token : Optional[str] + The client's authorization token. + host : str + The host address to broadcast to. Default is 127.0.0.1" + port : int + The port to broadcast to. Default is 6666. + uvicorn_kwargs : Optional[dict[str, Any]] + Additional keyword arguments for passing directly to the uvicorn server. + + Returns + ------- + str + The message that the client started broadcasting. + """ + if name not in connected_clients: + raise OpenBBError(f"Client {name} not connected.") + if not await check_auth(name, auth_token): + raise OpenBBError("Error finding client.") + client = connected_clients[name] + kwargs = uvicorn_kwargs if uvicorn_kwargs else {} + client.start_broadcasting(host=host, port=port, **kwargs) + + await asyncio.sleep(2) + if not client.is_broadcasting: + raise OpenBBError(f"Client {name} failed to broadcast.") + return OBBject( + results=f"Client {name} started broadcasting to {client.broadcast_address}." + ) + + +@router.command( + methods=["GET"], +) +async def kill(name: str, auth_token: Optional[str] = None) -> OBBject[str]: + """Kills a client. + + Parameters + ---------- + name : str + The name of the client. + auth_token : Optional[str] + The client's authorization token. + + Returns + ------- + str + The message that the client was killed. + """ + if not connected_clients: + raise OpenBBError("No connections to kill.") + elif name and name not in connected_clients: + raise OpenBBError(f"Client {name} not connected.") + client = connected_clients[name] + client._atexit() + del connected_clients[name] + return OBBject(results=f"Clients {name} killed.") diff --git a/openbb_platform/extensions/websockets/poetry.lock b/openbb_platform/extensions/websockets/poetry.lock new file mode 100644 index 000000000000..88faa6559e70 --- /dev/null +++ b/openbb_platform/extensions/websockets/poetry.lock @@ -0,0 +1,1618 @@ +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. + +[[package]] +name = "aiohappyeyeballs" +version = "2.4.3" +description = "Happy Eyeballs for asyncio" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiohappyeyeballs-2.4.3-py3-none-any.whl", hash = "sha256:8a7a83727b2756f394ab2895ea0765a0a8c475e3c71e98d43d76f22b4b435572"}, + {file = "aiohappyeyeballs-2.4.3.tar.gz", hash = "sha256:75cf88a15106a5002a8eb1dab212525c00d1f4c0fa96e551c9fbe6f09a621586"}, +] + +[[package]] +name = "aiohttp" +version = "3.10.10" +description = "Async http client/server framework (asyncio)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiohttp-3.10.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:be7443669ae9c016b71f402e43208e13ddf00912f47f623ee5994e12fc7d4b3f"}, + {file = "aiohttp-3.10.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7b06b7843929e41a94ea09eb1ce3927865387e3e23ebe108e0d0d09b08d25be9"}, + {file = "aiohttp-3.10.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:333cf6cf8e65f6a1e06e9eb3e643a0c515bb850d470902274239fea02033e9a8"}, + {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:274cfa632350225ce3fdeb318c23b4a10ec25c0e2c880eff951a3842cf358ac1"}, + {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9e5e4a85bdb56d224f412d9c98ae4cbd032cc4f3161818f692cd81766eee65a"}, + {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b606353da03edcc71130b52388d25f9a30a126e04caef1fd637e31683033abd"}, + {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab5a5a0c7a7991d90446a198689c0535be89bbd6b410a1f9a66688f0880ec026"}, + {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:578a4b875af3e0daaf1ac6fa983d93e0bbfec3ead753b6d6f33d467100cdc67b"}, + {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8105fd8a890df77b76dd3054cddf01a879fc13e8af576805d667e0fa0224c35d"}, + {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3bcd391d083f636c06a68715e69467963d1f9600f85ef556ea82e9ef25f043f7"}, + {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fbc6264158392bad9df19537e872d476f7c57adf718944cc1e4495cbabf38e2a"}, + {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e48d5021a84d341bcaf95c8460b152cfbad770d28e5fe14a768988c461b821bc"}, + {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2609e9ab08474702cc67b7702dbb8a80e392c54613ebe80db7e8dbdb79837c68"}, + {file = "aiohttp-3.10.10-cp310-cp310-win32.whl", hash = "sha256:84afcdea18eda514c25bc68b9af2a2b1adea7c08899175a51fe7c4fb6d551257"}, + {file = "aiohttp-3.10.10-cp310-cp310-win_amd64.whl", hash = "sha256:9c72109213eb9d3874f7ac8c0c5fa90e072d678e117d9061c06e30c85b4cf0e6"}, + {file = "aiohttp-3.10.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c30a0eafc89d28e7f959281b58198a9fa5e99405f716c0289b7892ca345fe45f"}, + {file = "aiohttp-3.10.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:258c5dd01afc10015866114e210fb7365f0d02d9d059c3c3415382ab633fcbcb"}, + {file = "aiohttp-3.10.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:15ecd889a709b0080f02721255b3f80bb261c2293d3c748151274dfea93ac871"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3935f82f6f4a3820270842e90456ebad3af15810cf65932bd24da4463bc0a4c"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:413251f6fcf552a33c981c4709a6bba37b12710982fec8e558ae944bfb2abd38"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1720b4f14c78a3089562b8875b53e36b51c97c51adc53325a69b79b4b48ebcb"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:679abe5d3858b33c2cf74faec299fda60ea9de62916e8b67e625d65bf069a3b7"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:79019094f87c9fb44f8d769e41dbb664d6e8fcfd62f665ccce36762deaa0e911"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe2fb38c2ed905a2582948e2de560675e9dfbee94c6d5ccdb1301c6d0a5bf092"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a3f00003de6eba42d6e94fabb4125600d6e484846dbf90ea8e48a800430cc142"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1bbb122c557a16fafc10354b9d99ebf2f2808a660d78202f10ba9d50786384b9"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:30ca7c3b94708a9d7ae76ff281b2f47d8eaf2579cd05971b5dc681db8caac6e1"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:df9270660711670e68803107d55c2b5949c2e0f2e4896da176e1ecfc068b974a"}, + {file = "aiohttp-3.10.10-cp311-cp311-win32.whl", hash = "sha256:aafc8ee9b742ce75044ae9a4d3e60e3d918d15a4c2e08a6c3c3e38fa59b92d94"}, + {file = "aiohttp-3.10.10-cp311-cp311-win_amd64.whl", hash = "sha256:362f641f9071e5f3ee6f8e7d37d5ed0d95aae656adf4ef578313ee585b585959"}, + {file = "aiohttp-3.10.10-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9294bbb581f92770e6ed5c19559e1e99255e4ca604a22c5c6397b2f9dd3ee42c"}, + {file = "aiohttp-3.10.10-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a8fa23fe62c436ccf23ff930149c047f060c7126eae3ccea005f0483f27b2e28"}, + {file = "aiohttp-3.10.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5c6a5b8c7926ba5d8545c7dd22961a107526562da31a7a32fa2456baf040939f"}, + {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:007ec22fbc573e5eb2fb7dec4198ef8f6bf2fe4ce20020798b2eb5d0abda6138"}, + {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9627cc1a10c8c409b5822a92d57a77f383b554463d1884008e051c32ab1b3742"}, + {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:50edbcad60d8f0e3eccc68da67f37268b5144ecc34d59f27a02f9611c1d4eec7"}, + {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a45d85cf20b5e0d0aa5a8dca27cce8eddef3292bc29d72dcad1641f4ed50aa16"}, + {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b00807e2605f16e1e198f33a53ce3c4523114059b0c09c337209ae55e3823a8"}, + {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f2d4324a98062be0525d16f768a03e0bbb3b9fe301ceee99611dc9a7953124e6"}, + {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:438cd072f75bb6612f2aca29f8bd7cdf6e35e8f160bc312e49fbecab77c99e3a"}, + {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:baa42524a82f75303f714108fea528ccacf0386af429b69fff141ffef1c534f9"}, + {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a7d8d14fe962153fc681f6366bdec33d4356f98a3e3567782aac1b6e0e40109a"}, + {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c1277cd707c465cd09572a774559a3cc7c7a28802eb3a2a9472588f062097205"}, + {file = "aiohttp-3.10.10-cp312-cp312-win32.whl", hash = "sha256:59bb3c54aa420521dc4ce3cc2c3fe2ad82adf7b09403fa1f48ae45c0cbde6628"}, + {file = "aiohttp-3.10.10-cp312-cp312-win_amd64.whl", hash = "sha256:0e1b370d8007c4ae31ee6db7f9a2fe801a42b146cec80a86766e7ad5c4a259cf"}, + {file = "aiohttp-3.10.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ad7593bb24b2ab09e65e8a1d385606f0f47c65b5a2ae6c551db67d6653e78c28"}, + {file = "aiohttp-3.10.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1eb89d3d29adaf533588f209768a9c02e44e4baf832b08118749c5fad191781d"}, + {file = "aiohttp-3.10.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3fe407bf93533a6fa82dece0e74dbcaaf5d684e5a51862887f9eaebe6372cd79"}, + {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50aed5155f819873d23520919e16703fc8925e509abbb1a1491b0087d1cd969e"}, + {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4f05e9727ce409358baa615dbeb9b969db94324a79b5a5cea45d39bdb01d82e6"}, + {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dffb610a30d643983aeb185ce134f97f290f8935f0abccdd32c77bed9388b42"}, + {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa6658732517ddabe22c9036479eabce6036655ba87a0224c612e1ae6af2087e"}, + {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:741a46d58677d8c733175d7e5aa618d277cd9d880301a380fd296975a9cdd7bc"}, + {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e00e3505cd80440f6c98c6d69269dcc2a119f86ad0a9fd70bccc59504bebd68a"}, + {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ffe595f10566f8276b76dc3a11ae4bb7eba1aac8ddd75811736a15b0d5311414"}, + {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:bdfcf6443637c148c4e1a20c48c566aa694fa5e288d34b20fcdc58507882fed3"}, + {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d183cf9c797a5291e8301790ed6d053480ed94070637bfaad914dd38b0981f67"}, + {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:77abf6665ae54000b98b3c742bc6ea1d1fb31c394bcabf8b5d2c1ac3ebfe7f3b"}, + {file = "aiohttp-3.10.10-cp313-cp313-win32.whl", hash = "sha256:4470c73c12cd9109db8277287d11f9dd98f77fc54155fc71a7738a83ffcc8ea8"}, + {file = "aiohttp-3.10.10-cp313-cp313-win_amd64.whl", hash = "sha256:486f7aabfa292719a2753c016cc3a8f8172965cabb3ea2e7f7436c7f5a22a151"}, + {file = "aiohttp-3.10.10-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:1b66ccafef7336a1e1f0e389901f60c1d920102315a56df85e49552308fc0486"}, + {file = "aiohttp-3.10.10-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:acd48d5b80ee80f9432a165c0ac8cbf9253eaddb6113269a5e18699b33958dbb"}, + {file = "aiohttp-3.10.10-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3455522392fb15ff549d92fbf4b73b559d5e43dc522588f7eb3e54c3f38beee7"}, + {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45c3b868724137f713a38376fef8120c166d1eadd50da1855c112fe97954aed8"}, + {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:da1dee8948d2137bb51fbb8a53cce6b1bcc86003c6b42565f008438b806cccd8"}, + {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c5ce2ce7c997e1971b7184ee37deb6ea9922ef5163c6ee5aa3c274b05f9e12fa"}, + {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28529e08fde6f12eba8677f5a8608500ed33c086f974de68cc65ab218713a59d"}, + {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f7db54c7914cc99d901d93a34704833568d86c20925b2762f9fa779f9cd2e70f"}, + {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:03a42ac7895406220124c88911ebee31ba8b2d24c98507f4a8bf826b2937c7f2"}, + {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:7e338c0523d024fad378b376a79faff37fafb3c001872a618cde1d322400a572"}, + {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:038f514fe39e235e9fef6717fbf944057bfa24f9b3db9ee551a7ecf584b5b480"}, + {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:64f6c17757251e2b8d885d728b6433d9d970573586a78b78ba8929b0f41d045a"}, + {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:93429602396f3383a797a2a70e5f1de5df8e35535d7806c9f91df06f297e109b"}, + {file = "aiohttp-3.10.10-cp38-cp38-win32.whl", hash = "sha256:c823bc3971c44ab93e611ab1a46b1eafeae474c0c844aff4b7474287b75fe49c"}, + {file = "aiohttp-3.10.10-cp38-cp38-win_amd64.whl", hash = "sha256:54ca74df1be3c7ca1cf7f4c971c79c2daf48d9aa65dea1a662ae18926f5bc8ce"}, + {file = "aiohttp-3.10.10-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:01948b1d570f83ee7bbf5a60ea2375a89dfb09fd419170e7f5af029510033d24"}, + {file = "aiohttp-3.10.10-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9fc1500fd2a952c5c8e3b29aaf7e3cc6e27e9cfc0a8819b3bce48cc1b849e4cc"}, + {file = "aiohttp-3.10.10-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f614ab0c76397661b90b6851a030004dac502e48260ea10f2441abd2207fbcc7"}, + {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00819de9e45d42584bed046314c40ea7e9aea95411b38971082cad449392b08c"}, + {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05646ebe6b94cc93407b3bf34b9eb26c20722384d068eb7339de802154d61bc5"}, + {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:998f3bd3cfc95e9424a6acd7840cbdd39e45bc09ef87533c006f94ac47296090"}, + {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9010c31cd6fa59438da4e58a7f19e4753f7f264300cd152e7f90d4602449762"}, + {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ea7ffc6d6d6f8a11e6f40091a1040995cdff02cfc9ba4c2f30a516cb2633554"}, + {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ef9c33cc5cbca35808f6c74be11eb7f5f6b14d2311be84a15b594bd3e58b5527"}, + {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ce0cdc074d540265bfeb31336e678b4e37316849d13b308607efa527e981f5c2"}, + {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:597a079284b7ee65ee102bc3a6ea226a37d2b96d0418cc9047490f231dc09fe8"}, + {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:7789050d9e5d0c309c706953e5e8876e38662d57d45f936902e176d19f1c58ab"}, + {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:e7f8b04d83483577fd9200461b057c9f14ced334dcb053090cea1da9c8321a91"}, + {file = "aiohttp-3.10.10-cp39-cp39-win32.whl", hash = "sha256:c02a30b904282777d872266b87b20ed8cc0d1501855e27f831320f471d54d983"}, + {file = "aiohttp-3.10.10-cp39-cp39-win_amd64.whl", hash = "sha256:edfe3341033a6b53a5c522c802deb2079eee5cbfbb0af032a55064bd65c73a23"}, + {file = "aiohttp-3.10.10.tar.gz", hash = "sha256:0631dd7c9f0822cc61c88586ca76d5b5ada26538097d0f1df510b082bad3411a"}, +] + +[package.dependencies] +aiohappyeyeballs = ">=2.3.0" +aiosignal = ">=1.1.2" +async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} +attrs = ">=17.3.0" +frozenlist = ">=1.1.1" +multidict = ">=4.5,<7.0" +yarl = ">=1.12.0,<2.0" + +[package.extras] +speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"] + +[[package]] +name = "aiosignal" +version = "1.3.1" +description = "aiosignal: a list of registered asynchronous callbacks" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] + +[package.dependencies] +frozenlist = ">=1.1.0" + +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] + +[[package]] +name = "anyio" +version = "4.6.2.post1" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.9" +files = [ + {file = "anyio-4.6.2.post1-py3-none-any.whl", hash = "sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d"}, + {file = "anyio-4.6.2.post1.tar.gz", hash = "sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" +typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} + +[package.extras] +doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21.0b1)"] +trio = ["trio (>=0.26.1)"] + +[[package]] +name = "async-timeout" +version = "4.0.3" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + +[[package]] +name = "attrs" +version = "24.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, + {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, +] + +[package.extras] +benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "certifi" +version = "2024.8.30" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-win32.whl", hash = "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-win32.whl", hash = "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-win32.whl", hash = "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-win32.whl", hash = "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-win32.whl", hash = "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca"}, + {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, + {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, +] + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.2" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "fastapi" +version = "0.115.4" +description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fastapi-0.115.4-py3-none-any.whl", hash = "sha256:0b504a063ffb3cf96a5e27dc1bc32c80ca743a2528574f9cdc77daa2d31b4742"}, + {file = "fastapi-0.115.4.tar.gz", hash = "sha256:db653475586b091cb8b2fec2ac54a680ac6a158e07406e1abae31679e8826349"}, +] + +[package.dependencies] +pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0" +starlette = ">=0.40.0,<0.42.0" +typing-extensions = ">=4.8.0" + +[package.extras] +all = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.5)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.7)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] +standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.5)", "httpx (>=0.23.0)", "jinja2 (>=2.11.2)", "python-multipart (>=0.0.7)", "uvicorn[standard] (>=0.12.0)"] + +[[package]] +name = "frozenlist" +version = "1.5.0" +description = "A list-like structure which implements collections.abc.MutableSequence" +optional = false +python-versions = ">=3.8" +files = [ + {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5b6a66c18b5b9dd261ca98dffcb826a525334b2f29e7caa54e182255c5f6a65a"}, + {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d1b3eb7b05ea246510b43a7e53ed1653e55c2121019a97e60cad7efb881a97bb"}, + {file = "frozenlist-1.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15538c0cbf0e4fa11d1e3a71f823524b0c46299aed6e10ebb4c2089abd8c3bec"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e79225373c317ff1e35f210dd5f1344ff31066ba8067c307ab60254cd3a78ad5"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9272fa73ca71266702c4c3e2d4a28553ea03418e591e377a03b8e3659d94fa76"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:498524025a5b8ba81695761d78c8dd7382ac0b052f34e66939c42df860b8ff17"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92b5278ed9d50fe610185ecd23c55d8b307d75ca18e94c0e7de328089ac5dcba"}, + {file = "frozenlist-1.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f3c8c1dacd037df16e85227bac13cca58c30da836c6f936ba1df0c05d046d8d"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f2ac49a9bedb996086057b75bf93538240538c6d9b38e57c82d51f75a73409d2"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e66cc454f97053b79c2ab09c17fbe3c825ea6b4de20baf1be28919460dd7877f"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:5a3ba5f9a0dfed20337d3e966dc359784c9f96503674c2faf015f7fe8e96798c"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6321899477db90bdeb9299ac3627a6a53c7399c8cd58d25da094007402b039ab"}, + {file = "frozenlist-1.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:76e4753701248476e6286f2ef492af900ea67d9706a0155335a40ea21bf3b2f5"}, + {file = "frozenlist-1.5.0-cp310-cp310-win32.whl", hash = "sha256:977701c081c0241d0955c9586ffdd9ce44f7a7795df39b9151cd9a6fd0ce4cfb"}, + {file = "frozenlist-1.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:189f03b53e64144f90990d29a27ec4f7997d91ed3d01b51fa39d2dbe77540fd4"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fd74520371c3c4175142d02a976aee0b4cb4a7cc912a60586ffd8d5929979b30"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2f3f7a0fbc219fb4455264cae4d9f01ad41ae6ee8524500f381de64ffaa077d5"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f47c9c9028f55a04ac254346e92977bf0f166c483c74b4232bee19a6697e4778"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0996c66760924da6e88922756d99b47512a71cfd45215f3570bf1e0b694c206a"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2fe128eb4edeabe11896cb6af88fca5346059f6c8d807e3b910069f39157869"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a8ea951bbb6cacd492e3948b8da8c502a3f814f5d20935aae74b5df2b19cf3d"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de537c11e4aa01d37db0d403b57bd6f0546e71a82347a97c6a9f0dcc532b3a45"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c2623347b933fcb9095841f1cc5d4ff0b278addd743e0e966cb3d460278840d"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cee6798eaf8b1416ef6909b06f7dc04b60755206bddc599f52232606e18179d3"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f5f9da7f5dbc00a604fe74aa02ae7c98bcede8a3b8b9666f9f86fc13993bc71a"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:90646abbc7a5d5c7c19461d2e3eeb76eb0b204919e6ece342feb6032c9325ae9"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bdac3c7d9b705d253b2ce370fde941836a5f8b3c5c2b8fd70940a3ea3af7f4f2"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03d33c2ddbc1816237a67f66336616416e2bbb6beb306e5f890f2eb22b959cdf"}, + {file = "frozenlist-1.5.0-cp311-cp311-win32.whl", hash = "sha256:237f6b23ee0f44066219dae14c70ae38a63f0440ce6750f868ee08775073f942"}, + {file = "frozenlist-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:0cc974cc93d32c42e7b0f6cf242a6bd941c57c61b618e78b6c0a96cb72788c1d"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:31115ba75889723431aa9a4e77d5f398f5cf976eea3bdf61749731f62d4a4a21"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7437601c4d89d070eac8323f121fcf25f88674627505334654fd027b091db09d"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7948140d9f8ece1745be806f2bfdf390127cf1a763b925c4a805c603df5e697e"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feeb64bc9bcc6b45c6311c9e9b99406660a9c05ca8a5b30d14a78555088b0b3a"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:683173d371daad49cffb8309779e886e59c2f369430ad28fe715f66d08d4ab1a"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7d57d8f702221405a9d9b40f9da8ac2e4a1a8b5285aac6100f3393675f0a85ee"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30c72000fbcc35b129cb09956836c7d7abf78ab5416595e4857d1cae8d6251a6"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:000a77d6034fbad9b6bb880f7ec073027908f1b40254b5d6f26210d2dab1240e"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5d7f5a50342475962eb18b740f3beecc685a15b52c91f7d975257e13e029eca9"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:87f724d055eb4785d9be84e9ebf0f24e392ddfad00b3fe036e43f489fafc9039"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6e9080bb2fb195a046e5177f10d9d82b8a204c0736a97a153c2466127de87784"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9b93d7aaa36c966fa42efcaf716e6b3900438632a626fb09c049f6a2f09fc631"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:52ef692a4bc60a6dd57f507429636c2af8b6046db8b31b18dac02cbc8f507f7f"}, + {file = "frozenlist-1.5.0-cp312-cp312-win32.whl", hash = "sha256:29d94c256679247b33a3dc96cce0f93cbc69c23bf75ff715919332fdbb6a32b8"}, + {file = "frozenlist-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:8969190d709e7c48ea386db202d708eb94bdb29207a1f269bab1196ce0dcca1f"}, + {file = "frozenlist-1.5.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:7a1a048f9215c90973402e26c01d1cff8a209e1f1b53f72b95c13db61b00f953"}, + {file = "frozenlist-1.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dd47a5181ce5fcb463b5d9e17ecfdb02b678cca31280639255ce9d0e5aa67af0"}, + {file = "frozenlist-1.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1431d60b36d15cda188ea222033eec8e0eab488f39a272461f2e6d9e1a8e63c2"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6482a5851f5d72767fbd0e507e80737f9c8646ae7fd303def99bfe813f76cf7f"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:44c49271a937625619e862baacbd037a7ef86dd1ee215afc298a417ff3270608"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:12f78f98c2f1c2429d42e6a485f433722b0061d5c0b0139efa64f396efb5886b"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce3aa154c452d2467487765e3adc730a8c153af77ad84096bc19ce19a2400840"}, + {file = "frozenlist-1.5.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b7dc0c4338e6b8b091e8faf0db3168a37101943e687f373dce00959583f7439"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:45e0896250900b5aa25180f9aec243e84e92ac84bd4a74d9ad4138ef3f5c97de"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:561eb1c9579d495fddb6da8959fd2a1fca2c6d060d4113f5844b433fc02f2641"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:df6e2f325bfee1f49f81aaac97d2aa757c7646534a06f8f577ce184afe2f0a9e"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:140228863501b44b809fb39ec56b5d4071f4d0aa6d216c19cbb08b8c5a7eadb9"}, + {file = "frozenlist-1.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7707a25d6a77f5d27ea7dc7d1fc608aa0a478193823f88511ef5e6b8a48f9d03"}, + {file = "frozenlist-1.5.0-cp313-cp313-win32.whl", hash = "sha256:31a9ac2b38ab9b5a8933b693db4939764ad3f299fcaa931a3e605bc3460e693c"}, + {file = "frozenlist-1.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:11aabdd62b8b9c4b84081a3c246506d1cddd2dd93ff0ad53ede5defec7886b28"}, + {file = "frozenlist-1.5.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:dd94994fc91a6177bfaafd7d9fd951bc8689b0a98168aa26b5f543868548d3ca"}, + {file = "frozenlist-1.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2d0da8bbec082bf6bf18345b180958775363588678f64998c2b7609e34719b10"}, + {file = "frozenlist-1.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:73f2e31ea8dd7df61a359b731716018c2be196e5bb3b74ddba107f694fbd7604"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:828afae9f17e6de596825cf4228ff28fbdf6065974e5ac1410cecc22f699d2b3"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1577515d35ed5649d52ab4319db757bb881ce3b2b796d7283e6634d99ace307"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2150cc6305a2c2ab33299453e2968611dacb970d2283a14955923062c8d00b10"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a72b7a6e3cd2725eff67cd64c8f13335ee18fc3c7befc05aed043d24c7b9ccb9"}, + {file = "frozenlist-1.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c16d2fa63e0800723139137d667e1056bee1a1cf7965153d2d104b62855e9b99"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:17dcc32fc7bda7ce5875435003220a457bcfa34ab7924a49a1c19f55b6ee185c"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:97160e245ea33d8609cd2b8fd997c850b56db147a304a262abc2b3be021a9171"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:f1e6540b7fa044eee0bb5111ada694cf3dc15f2b0347ca125ee9ca984d5e9e6e"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:91d6c171862df0a6c61479d9724f22efb6109111017c87567cfeb7b5d1449fdf"}, + {file = "frozenlist-1.5.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c1fac3e2ace2eb1052e9f7c7db480818371134410e1f5c55d65e8f3ac6d1407e"}, + {file = "frozenlist-1.5.0-cp38-cp38-win32.whl", hash = "sha256:b97f7b575ab4a8af9b7bc1d2ef7f29d3afee2226bd03ca3875c16451ad5a7723"}, + {file = "frozenlist-1.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:374ca2dabdccad8e2a76d40b1d037f5bd16824933bf7bcea3e59c891fd4a0923"}, + {file = "frozenlist-1.5.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9bbcdfaf4af7ce002694a4e10a0159d5a8d20056a12b05b45cea944a4953f972"}, + {file = "frozenlist-1.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1893f948bf6681733aaccf36c5232c231e3b5166d607c5fa77773611df6dc336"}, + {file = "frozenlist-1.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2b5e23253bb709ef57a8e95e6ae48daa9ac5f265637529e4ce6b003a37b2621f"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f253985bb515ecd89629db13cb58d702035ecd8cfbca7d7a7e29a0e6d39af5f"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04a5c6babd5e8fb7d3c871dc8b321166b80e41b637c31a995ed844a6139942b6"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9fe0f1c29ba24ba6ff6abf688cb0b7cf1efab6b6aa6adc55441773c252f7411"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:226d72559fa19babe2ccd920273e767c96a49b9d3d38badd7c91a0fdeda8ea08"}, + {file = "frozenlist-1.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15b731db116ab3aedec558573c1a5eec78822b32292fe4f2f0345b7f697745c2"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:366d8f93e3edfe5a918c874702f78faac300209a4d5bf38352b2c1bdc07a766d"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1b96af8c582b94d381a1c1f51ffaedeb77c821c690ea5f01da3d70a487dd0a9b"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:c03eff4a41bd4e38415cbed054bbaff4a075b093e2394b6915dca34a40d1e38b"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:50cf5e7ee9b98f22bdecbabf3800ae78ddcc26e4a435515fc72d97903e8488e0"}, + {file = "frozenlist-1.5.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1e76bfbc72353269c44e0bc2cfe171900fbf7f722ad74c9a7b638052afe6a00c"}, + {file = "frozenlist-1.5.0-cp39-cp39-win32.whl", hash = "sha256:666534d15ba8f0fda3f53969117383d5dc021266b3c1a42c9ec4855e4b58b9d3"}, + {file = "frozenlist-1.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:5c28f4b5dbef8a0d8aad0d4de24d1e9e981728628afaf4ea0792f5d0939372f0"}, + {file = "frozenlist-1.5.0-py3-none-any.whl", hash = "sha256:d994863bba198a4a518b467bb971c56e1db3f180a25c6cf7bb1949c267f748c3"}, + {file = "frozenlist-1.5.0.tar.gz", hash = "sha256:81d5af29e61b9c8348e876d442253723928dce6433e0e76cd925cd83f1b4b817"}, +] + +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "html5lib" +version = "1.1" +description = "HTML parser based on the WHATWG HTML specification" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d"}, + {file = "html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f"}, +] + +[package.dependencies] +six = ">=1.9" +webencodings = "*" + +[package.extras] +all = ["chardet (>=2.2)", "genshi", "lxml"] +chardet = ["chardet (>=2.2)"] +genshi = ["genshi"] +lxml = ["lxml"] + +[[package]] +name = "idna" +version = "3.10" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.6" +files = [ + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, +] + +[package.extras] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + +[[package]] +name = "importlib-metadata" +version = "8.5.0" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b"}, + {file = "importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7"}, +] + +[package.dependencies] +zipp = ">=3.20" + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +perf = ["ipython"] +test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] +type = ["pytest-mypy"] + +[[package]] +name = "monotonic" +version = "1.6" +description = "An implementation of time.monotonic() for Python 2 & < 3.3" +optional = false +python-versions = "*" +files = [ + {file = "monotonic-1.6-py2.py3-none-any.whl", hash = "sha256:68687e19a14f11f26d140dd5c86f3dba4bf5df58003000ed467e0e2a69bca96c"}, + {file = "monotonic-1.6.tar.gz", hash = "sha256:3a55207bcfed53ddd5c5bae174524062935efed17792e9de2ad0205ce9ad63f7"}, +] + +[[package]] +name = "multidict" +version = "6.1.0" +description = "multidict implementation" +optional = false +python-versions = ">=3.8" +files = [ + {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3380252550e372e8511d49481bd836264c009adb826b23fefcc5dd3c69692f60"}, + {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:99f826cbf970077383d7de805c0681799491cb939c25450b9b5b3ced03ca99f1"}, + {file = "multidict-6.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a114d03b938376557927ab23f1e950827c3b893ccb94b62fd95d430fd0e5cf53"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1c416351ee6271b2f49b56ad7f308072f6f44b37118d69c2cad94f3fa8a40d5"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b5d83030255983181005e6cfbac1617ce9746b219bc2aad52201ad121226581"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3e97b5e938051226dc025ec80980c285b053ffb1e25a3db2a3aa3bc046bf7f56"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d618649d4e70ac6efcbba75be98b26ef5078faad23592f9b51ca492953012429"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10524ebd769727ac77ef2278390fb0068d83f3acb7773792a5080f2b0abf7748"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ff3827aef427c89a25cc96ded1759271a93603aba9fb977a6d264648ebf989db"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:06809f4f0f7ab7ea2cabf9caca7d79c22c0758b58a71f9d32943ae13c7ace056"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f179dee3b863ab1c59580ff60f9d99f632f34ccb38bf67a33ec6b3ecadd0fd76"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:aaed8b0562be4a0876ee3b6946f6869b7bcdb571a5d1496683505944e268b160"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3c8b88a2ccf5493b6c8da9076fb151ba106960a2df90c2633f342f120751a9e7"}, + {file = "multidict-6.1.0-cp310-cp310-win32.whl", hash = "sha256:4a9cb68166a34117d6646c0023c7b759bf197bee5ad4272f420a0141d7eb03a0"}, + {file = "multidict-6.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:20b9b5fbe0b88d0bdef2012ef7dee867f874b72528cf1d08f1d59b0e3850129d"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3efe2c2cb5763f2f1b275ad2bf7a287d3f7ebbef35648a9726e3b69284a4f3d6"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7053d3b0353a8b9de430a4f4b4268ac9a4fb3481af37dfe49825bf45ca24156"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27e5fc84ccef8dfaabb09d82b7d179c7cf1a3fbc8a966f8274fcb4ab2eb4cadb"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e2b90b43e696f25c62656389d32236e049568b39320e2735d51f08fd362761b"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d83a047959d38a7ff552ff94be767b7fd79b831ad1cd9920662db05fec24fe72"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1a9dd711d0877a1ece3d2e4fea11a8e75741ca21954c919406b44e7cf971304"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec2abea24d98246b94913b76a125e855eb5c434f7c46546046372fe60f666351"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4867cafcbc6585e4b678876c489b9273b13e9fff9f6d6d66add5e15d11d926cb"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b48204e8d955c47c55b72779802b219a39acc3ee3d0116d5080c388970b76e3"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d8fff389528cad1618fb4b26b95550327495462cd745d879a8c7c2115248e399"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a7a9541cd308eed5e30318430a9c74d2132e9a8cb46b901326272d780bf2d423"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:da1758c76f50c39a2efd5e9859ce7d776317eb1dd34317c8152ac9251fc574a3"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c943a53e9186688b45b323602298ab727d8865d8c9ee0b17f8d62d14b56f0753"}, + {file = "multidict-6.1.0-cp311-cp311-win32.whl", hash = "sha256:90f8717cb649eea3504091e640a1b8568faad18bd4b9fcd692853a04475a4b80"}, + {file = "multidict-6.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:82176036e65644a6cc5bd619f65f6f19781e8ec2e5330f51aa9ada7504cc1926"}, + {file = "multidict-6.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b04772ed465fa3cc947db808fa306d79b43e896beb677a56fb2347ca1a49c1fa"}, + {file = "multidict-6.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6180c0ae073bddeb5a97a38c03f30c233e0a4d39cd86166251617d1bbd0af436"}, + {file = "multidict-6.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:071120490b47aa997cca00666923a83f02c7fbb44f71cf7f136df753f7fa8761"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50b3a2710631848991d0bf7de077502e8994c804bb805aeb2925a981de58ec2e"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b58c621844d55e71c1b7f7c498ce5aa6985d743a1a59034c57a905b3f153c1ef"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55b6d90641869892caa9ca42ff913f7ff1c5ece06474fbd32fb2cf6834726c95"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b820514bfc0b98a30e3d85462084779900347e4d49267f747ff54060cc33925"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10a9b09aba0c5b48c53761b7c720aaaf7cf236d5fe394cd399c7ba662d5f9966"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1e16bf3e5fc9f44632affb159d30a437bfe286ce9e02754759be5536b169b305"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:76f364861c3bfc98cbbcbd402d83454ed9e01a5224bb3a28bf70002a230f73e2"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:820c661588bd01a0aa62a1283f20d2be4281b086f80dad9e955e690c75fb54a2"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0e5f362e895bc5b9e67fe6e4ded2492d8124bdf817827f33c5b46c2fe3ffaca6"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3ec660d19bbc671e3a6443325f07263be452c453ac9e512f5eb935e7d4ac28b3"}, + {file = "multidict-6.1.0-cp312-cp312-win32.whl", hash = "sha256:58130ecf8f7b8112cdb841486404f1282b9c86ccb30d3519faf301b2e5659133"}, + {file = "multidict-6.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:188215fc0aafb8e03341995e7c4797860181562380f81ed0a87ff455b70bf1f1"}, + {file = "multidict-6.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d569388c381b24671589335a3be6e1d45546c2988c2ebe30fdcada8457a31008"}, + {file = "multidict-6.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:052e10d2d37810b99cc170b785945421141bf7bb7d2f8799d431e7db229c385f"}, + {file = "multidict-6.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f90c822a402cb865e396a504f9fc8173ef34212a342d92e362ca498cad308e28"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b225d95519a5bf73860323e633a664b0d85ad3d5bede6d30d95b35d4dfe8805b"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:23bfd518810af7de1116313ebd9092cb9aa629beb12f6ed631ad53356ed6b86c"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c09fcfdccdd0b57867577b719c69e347a436b86cd83747f179dbf0cc0d4c1f3"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf6bea52ec97e95560af5ae576bdac3aa3aae0b6758c6efa115236d9e07dae44"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57feec87371dbb3520da6192213c7d6fc892d5589a93db548331954de8248fd2"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0c3f390dc53279cbc8ba976e5f8035eab997829066756d811616b652b00a23a3"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:59bfeae4b25ec05b34f1956eaa1cb38032282cd4dfabc5056d0a1ec4d696d3aa"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b2f59caeaf7632cc633b5cf6fc449372b83bbdf0da4ae04d5be36118e46cc0aa"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:37bb93b2178e02b7b618893990941900fd25b6b9ac0fa49931a40aecdf083fe4"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4e9f48f58c2c523d5a06faea47866cd35b32655c46b443f163d08c6d0ddb17d6"}, + {file = "multidict-6.1.0-cp313-cp313-win32.whl", hash = "sha256:3a37ffb35399029b45c6cc33640a92bef403c9fd388acce75cdc88f58bd19a81"}, + {file = "multidict-6.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:e9aa71e15d9d9beaad2c6b9319edcdc0a49a43ef5c0a4c8265ca9ee7d6c67774"}, + {file = "multidict-6.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:db7457bac39421addd0c8449933ac32d8042aae84a14911a757ae6ca3eef1392"}, + {file = "multidict-6.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d094ddec350a2fb899fec68d8353c78233debde9b7d8b4beeafa70825f1c281a"}, + {file = "multidict-6.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5845c1fd4866bb5dd3125d89b90e57ed3138241540897de748cdf19de8a2fca2"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9079dfc6a70abe341f521f78405b8949f96db48da98aeb43f9907f342f627cdc"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3914f5aaa0f36d5d60e8ece6a308ee1c9784cd75ec8151062614657a114c4478"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c08be4f460903e5a9d0f76818db3250f12e9c344e79314d1d570fc69d7f4eae4"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d093be959277cb7dee84b801eb1af388b6ad3ca6a6b6bf1ed7585895789d027d"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3702ea6872c5a2a4eeefa6ffd36b042e9773f05b1f37ae3ef7264b1163c2dcf6"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:2090f6a85cafc5b2db085124d752757c9d251548cedabe9bd31afe6363e0aff2"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:f67f217af4b1ff66c68a87318012de788dd95fcfeb24cc889011f4e1c7454dfd"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:189f652a87e876098bbc67b4da1049afb5f5dfbaa310dd67c594b01c10388db6"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:6bb5992037f7a9eff7991ebe4273ea7f51f1c1c511e6a2ce511d0e7bdb754492"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:ac10f4c2b9e770c4e393876e35a7046879d195cd123b4f116d299d442b335bcd"}, + {file = "multidict-6.1.0-cp38-cp38-win32.whl", hash = "sha256:e27bbb6d14416713a8bd7aaa1313c0fc8d44ee48d74497a0ff4c3a1b6ccb5167"}, + {file = "multidict-6.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:22f3105d4fb15c8f57ff3959a58fcab6ce36814486500cd7485651230ad4d4ef"}, + {file = "multidict-6.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4e18b656c5e844539d506a0a06432274d7bd52a7487e6828c63a63d69185626c"}, + {file = "multidict-6.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a185f876e69897a6f3325c3f19f26a297fa058c5e456bfcff8015e9a27e83ae1"}, + {file = "multidict-6.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ab7c4ceb38d91570a650dba194e1ca87c2b543488fe9309b4212694174fd539c"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e617fb6b0b6953fffd762669610c1c4ffd05632c138d61ac7e14ad187870669c"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16e5f4bf4e603eb1fdd5d8180f1a25f30056f22e55ce51fb3d6ad4ab29f7d96f"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c035da3f544b1882bac24115f3e2e8760f10a0107614fc9839fd232200b875"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:957cf8e4b6e123a9eea554fa7ebc85674674b713551de587eb318a2df3e00255"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:483a6aea59cb89904e1ceabd2b47368b5600fb7de78a6e4a2c2987b2d256cf30"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:87701f25a2352e5bf7454caa64757642734da9f6b11384c1f9d1a8e699758057"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:682b987361e5fd7a139ed565e30d81fd81e9629acc7d925a205366877d8c8657"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ce2186a7df133a9c895dea3331ddc5ddad42cdd0d1ea2f0a51e5d161e4762f28"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:9f636b730f7e8cb19feb87094949ba54ee5357440b9658b2a32a5ce4bce53972"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:73eae06aa53af2ea5270cc066dcaf02cc60d2994bbb2c4ef5764949257d10f43"}, + {file = "multidict-6.1.0-cp39-cp39-win32.whl", hash = "sha256:1ca0083e80e791cffc6efce7660ad24af66c8d4079d2a750b29001b53ff59ada"}, + {file = "multidict-6.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:aa466da5b15ccea564bdab9c89175c762bc12825f4659c11227f515cee76fa4a"}, + {file = "multidict-6.1.0-py3-none-any.whl", hash = "sha256:48e171e52d1c4d33888e529b999e5900356b9ae588c2f09a52dcefb158b27506"}, + {file = "multidict-6.1.0.tar.gz", hash = "sha256:22ae2ebf9b0c69d206c003e2f6a914ea33f0a932d4aa16f236afc049d9958f4a"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} + +[[package]] +name = "numpy" +version = "2.0.2" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-2.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51129a29dbe56f9ca83438b706e2e69a39892b5eda6cedcb6b0c9fdc9b0d3ece"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f15975dfec0cf2239224d80e32c3170b1d168335eaedee69da84fbe9f1f9cd04"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8c5713284ce4e282544c68d1c3b2c7161d38c256d2eefc93c1d683cf47683e66"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:becfae3ddd30736fe1889a37f1f580e245ba79a5855bff5f2a29cb3ccc22dd7b"}, + {file = "numpy-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2da5960c3cf0df7eafefd806d4e612c5e19358de82cb3c343631188991566ccd"}, + {file = "numpy-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:496f71341824ed9f3d2fd36cf3ac57ae2e0165c143b55c3a035ee219413f3318"}, + {file = "numpy-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a61ec659f68ae254e4d237816e33171497e978140353c0c2038d46e63282d0c8"}, + {file = "numpy-2.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d731a1c6116ba289c1e9ee714b08a8ff882944d4ad631fd411106a30f083c326"}, + {file = "numpy-2.0.2-cp310-cp310-win32.whl", hash = "sha256:984d96121c9f9616cd33fbd0618b7f08e0cfc9600a7ee1d6fd9b239186d19d97"}, + {file = "numpy-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7b0be4ef08607dd04da4092faee0b86607f111d5ae68036f16cc787e250a131"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:49ca4decb342d66018b01932139c0961a8f9ddc7589611158cb3c27cbcf76448"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:807ec44583fd708a21d4a11d94aedf2f4f3c3719035c76a2bbe1fe8e217bdc57"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8cafab480740e22f8d833acefed5cc87ce276f4ece12fdaa2e8903db2f82897a"}, + {file = "numpy-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15f476a45e6e5a3a79d8a14e62161d27ad897381fecfa4a09ed5322f2085669"}, + {file = "numpy-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e689d772146140a252c3a28501da66dfecd77490b498b168b501835041f951"}, + {file = "numpy-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9ea91dfb7c3d1c56a0e55657c0afb38cf1eeae4544c208dc465c3c9f3a7c09f9"}, + {file = "numpy-2.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c1c9307701fec8f3f7a1e6711f9089c06e6284b3afbbcd259f7791282d660a15"}, + {file = "numpy-2.0.2-cp311-cp311-win32.whl", hash = "sha256:a392a68bd329eafac5817e5aefeb39038c48b671afd242710b451e76090e81f4"}, + {file = "numpy-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:286cd40ce2b7d652a6f22efdfc6d1edf879440e53e76a75955bc0c826c7e64dc"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c"}, + {file = "numpy-2.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692"}, + {file = "numpy-2.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a"}, + {file = "numpy-2.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c"}, + {file = "numpy-2.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded"}, + {file = "numpy-2.0.2-cp312-cp312-win32.whl", hash = "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5"}, + {file = "numpy-2.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9059e10581ce4093f735ed23f3b9d283b9d517ff46009ddd485f1747eb22653c"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:423e89b23490805d2a5a96fe40ec507407b8ee786d66f7328be214f9679df6dd"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:2b2955fa6f11907cf7a70dab0d0755159bca87755e831e47932367fc8f2f2d0b"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:97032a27bd9d8988b9a97a8c4d2c9f2c15a81f61e2f21404d7e8ef00cb5be729"}, + {file = "numpy-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e795a8be3ddbac43274f18588329c72939870a16cae810c2b73461c40718ab1"}, + {file = "numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b258c385842546006213344c50655ff1555a9338e2e5e02a0756dc3e803dd"}, + {file = "numpy-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fec9451a7789926bcf7c2b8d187292c9f93ea30284802a0ab3f5be8ab36865d"}, + {file = "numpy-2.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9189427407d88ff25ecf8f12469d4d39d35bee1db5d39fc5c168c6f088a6956d"}, + {file = "numpy-2.0.2-cp39-cp39-win32.whl", hash = "sha256:905d16e0c60200656500c95b6b8dca5d109e23cb24abc701d41c02d74c6b3afa"}, + {file = "numpy-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:a3f4ab0caa7f053f6797fcd4e1e25caee367db3112ef2b6ef82d749530768c73"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7f0a0c6f12e07fa94133c8a67404322845220c06a9e80e85999afe727f7438b8"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-macosx_14_0_x86_64.whl", hash = "sha256:312950fdd060354350ed123c0e25a71327d3711584beaef30cdaa93320c392d4"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26df23238872200f63518dd2aa984cfca675d82469535dc7162dc2ee52d9dd5c"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a46288ec55ebbd58947d31d72be2c63cbf839f0a63b49cb755022310792a3385"}, + {file = "numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78"}, +] + +[[package]] +name = "numpy" +version = "2.1.2" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "numpy-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:30d53720b726ec36a7f88dc873f0eec8447fbc93d93a8f079dfac2629598d6ee"}, + {file = "numpy-2.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d3ca0a72dd8846eb6f7dfe8f19088060fcb76931ed592d29128e0219652884"}, + {file = "numpy-2.1.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:fc44e3c68ff00fd991b59092a54350e6e4911152682b4782f68070985aa9e648"}, + {file = "numpy-2.1.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:7c1c60328bd964b53f8b835df69ae8198659e2b9302ff9ebb7de4e5a5994db3d"}, + {file = "numpy-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cdb606a7478f9ad91c6283e238544451e3a95f30fb5467fbf715964341a8a86"}, + {file = "numpy-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d666cb72687559689e9906197e3bec7b736764df6a2e58ee265e360663e9baf7"}, + {file = "numpy-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6eef7a2dbd0abfb0d9eaf78b73017dbfd0b54051102ff4e6a7b2980d5ac1a03"}, + {file = "numpy-2.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:12edb90831ff481f7ef5f6bc6431a9d74dc0e5ff401559a71e5e4611d4f2d466"}, + {file = "numpy-2.1.2-cp310-cp310-win32.whl", hash = "sha256:a65acfdb9c6ebb8368490dbafe83c03c7e277b37e6857f0caeadbbc56e12f4fb"}, + {file = "numpy-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:860ec6e63e2c5c2ee5e9121808145c7bf86c96cca9ad396c0bd3e0f2798ccbe2"}, + {file = "numpy-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b42a1a511c81cc78cbc4539675713bbcf9d9c3913386243ceff0e9429ca892fe"}, + {file = "numpy-2.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:faa88bc527d0f097abdc2c663cddf37c05a1c2f113716601555249805cf573f1"}, + {file = "numpy-2.1.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:c82af4b2ddd2ee72d1fc0c6695048d457e00b3582ccde72d8a1c991b808bb20f"}, + {file = "numpy-2.1.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:13602b3174432a35b16c4cfb5de9a12d229727c3dd47a6ce35111f2ebdf66ff4"}, + {file = "numpy-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ebec5fd716c5a5b3d8dfcc439be82a8407b7b24b230d0ad28a81b61c2f4659a"}, + {file = "numpy-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2b49c3c0804e8ecb05d59af8386ec2f74877f7ca8fd9c1e00be2672e4d399b1"}, + {file = "numpy-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2cbba4b30bf31ddbe97f1c7205ef976909a93a66bb1583e983adbd155ba72ac2"}, + {file = "numpy-2.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8e00ea6fc82e8a804433d3e9cedaa1051a1422cb6e443011590c14d2dea59146"}, + {file = "numpy-2.1.2-cp311-cp311-win32.whl", hash = "sha256:5006b13a06e0b38d561fab5ccc37581f23c9511879be7693bd33c7cd15ca227c"}, + {file = "numpy-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:f1eb068ead09f4994dec71c24b2844f1e4e4e013b9629f812f292f04bd1510d9"}, + {file = "numpy-2.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d7bf0a4f9f15b32b5ba53147369e94296f5fffb783db5aacc1be15b4bf72f43b"}, + {file = "numpy-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b1d0fcae4f0949f215d4632be684a539859b295e2d0cb14f78ec231915d644db"}, + {file = "numpy-2.1.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f751ed0a2f250541e19dfca9f1eafa31a392c71c832b6bb9e113b10d050cb0f1"}, + {file = "numpy-2.1.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:bd33f82e95ba7ad632bc57837ee99dba3d7e006536200c4e9124089e1bf42426"}, + {file = "numpy-2.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b8cde4f11f0a975d1fd59373b32e2f5a562ade7cde4f85b7137f3de8fbb29a0"}, + {file = "numpy-2.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d95f286b8244b3649b477ac066c6906fbb2905f8ac19b170e2175d3d799f4df"}, + {file = "numpy-2.1.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ab4754d432e3ac42d33a269c8567413bdb541689b02d93788af4131018cbf366"}, + {file = "numpy-2.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e585c8ae871fd38ac50598f4763d73ec5497b0de9a0ab4ef5b69f01c6a046142"}, + {file = "numpy-2.1.2-cp312-cp312-win32.whl", hash = "sha256:9c6c754df29ce6a89ed23afb25550d1c2d5fdb9901d9c67a16e0b16eaf7e2550"}, + {file = "numpy-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:456e3b11cb79ac9946c822a56346ec80275eaf2950314b249b512896c0d2505e"}, + {file = "numpy-2.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a84498e0d0a1174f2b3ed769b67b656aa5460c92c9554039e11f20a05650f00d"}, + {file = "numpy-2.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4d6ec0d4222e8ffdab1744da2560f07856421b367928026fb540e1945f2eeeaf"}, + {file = "numpy-2.1.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:259ec80d54999cc34cd1eb8ded513cb053c3bf4829152a2e00de2371bd406f5e"}, + {file = "numpy-2.1.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:675c741d4739af2dc20cd6c6a5c4b7355c728167845e3c6b0e824e4e5d36a6c3"}, + {file = "numpy-2.1.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05b2d4e667895cc55e3ff2b56077e4c8a5604361fc21a042845ea3ad67465aa8"}, + {file = "numpy-2.1.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43cca367bf94a14aca50b89e9bc2061683116cfe864e56740e083392f533ce7a"}, + {file = "numpy-2.1.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:76322dcdb16fccf2ac56f99048af32259dcc488d9b7e25b51e5eca5147a3fb98"}, + {file = "numpy-2.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:32e16a03138cabe0cb28e1007ee82264296ac0983714094380b408097a418cfe"}, + {file = "numpy-2.1.2-cp313-cp313-win32.whl", hash = "sha256:242b39d00e4944431a3cd2db2f5377e15b5785920421993770cddb89992c3f3a"}, + {file = "numpy-2.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:f2ded8d9b6f68cc26f8425eda5d3877b47343e68ca23d0d0846f4d312ecaa445"}, + {file = "numpy-2.1.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2ffef621c14ebb0188a8633348504a35c13680d6da93ab5cb86f4e54b7e922b5"}, + {file = "numpy-2.1.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:ad369ed238b1959dfbade9018a740fb9392c5ac4f9b5173f420bd4f37ba1f7a0"}, + {file = "numpy-2.1.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:d82075752f40c0ddf57e6e02673a17f6cb0f8eb3f587f63ca1eaab5594da5b17"}, + {file = "numpy-2.1.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:1600068c262af1ca9580a527d43dc9d959b0b1d8e56f8a05d830eea39b7c8af6"}, + {file = "numpy-2.1.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a26ae94658d3ba3781d5e103ac07a876b3e9b29db53f68ed7df432fd033358a8"}, + {file = "numpy-2.1.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13311c2db4c5f7609b462bc0f43d3c465424d25c626d95040f073e30f7570e35"}, + {file = "numpy-2.1.2-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:2abbf905a0b568706391ec6fa15161fad0fb5d8b68d73c461b3c1bab6064dd62"}, + {file = "numpy-2.1.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ef444c57d664d35cac4e18c298c47d7b504c66b17c2ea91312e979fcfbdfb08a"}, + {file = "numpy-2.1.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:bdd407c40483463898b84490770199d5714dcc9dd9b792f6c6caccc523c00952"}, + {file = "numpy-2.1.2-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:da65fb46d4cbb75cb417cddf6ba5e7582eb7bb0b47db4b99c9fe5787ce5d91f5"}, + {file = "numpy-2.1.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c193d0b0238638e6fc5f10f1b074a6993cb13b0b431f64079a509d63d3aa8b7"}, + {file = "numpy-2.1.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a7d80b2e904faa63068ead63107189164ca443b42dd1930299e0d1cb041cec2e"}, + {file = "numpy-2.1.2.tar.gz", hash = "sha256:13532a088217fa624c99b843eeb54640de23b3414b14aa66d023805eb731066c"}, +] + +[[package]] +name = "openbb-core" +version = "1.3.5" +description = "OpenBB package with core functionality." +optional = false +python-versions = "<4.0,>=3.9" +files = [ + {file = "openbb_core-1.3.5-py3-none-any.whl", hash = "sha256:bb93b2343eea06faff2644e03fd6ee7a09c37392c5486d96021aa9ae137e1a90"}, + {file = "openbb_core-1.3.5.tar.gz", hash = "sha256:d8abed2a351a0bca1b02fdaba439574b452a7a812d72d5e059f2dbedab55bd19"}, +] + +[package.dependencies] +aiohttp = ">=3.10.4,<4.0.0" +fastapi = ">=0.115,<0.116" +html5lib = ">=1.1,<2.0" +importlib-metadata = ">=6.8.0" +pandas = ">=1.5.3" +posthog = ">=3.3.1,<4.0.0" +pydantic = ">=2.5.1,<3.0.0" +pyjwt = ">=2.8.0,<3.0.0" +python-dotenv = ">=1.0.0,<2.0.0" +python-multipart = ">=0.0.7,<0.0.8" +requests = ">=2.32.1,<3.0.0" +ruff = ">=0.7,<0.8" +uuid7 = ">=0.1.0,<0.2.0" +uvicorn = ">=0.32.0,<0.33.0" +websockets = ">=13.0,<14.0" + +[[package]] +name = "pandas" +version = "2.2.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"}, + {file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"}, + {file = "pandas-2.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d9c45366def9a3dd85a6454c0e7908f2b3b8e9c138f5dc38fed7ce720d8453ed"}, + {file = "pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57"}, + {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b8661b0238a69d7aafe156b7fa86c44b881387509653fdf857bebc5e4008ad42"}, + {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f"}, + {file = "pandas-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32"}, + {file = "pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"}, + {file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb"}, + {file = "pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a"}, + {file = "pandas-2.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39"}, + {file = "pandas-2.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30"}, + {file = "pandas-2.2.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cd6d7cc958a3910f934ea8dbdf17b2364827bb4dafc38ce6eef6bb3d65ff09c"}, + {file = "pandas-2.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c"}, + {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31d0ced62d4ea3e231a9f228366919a5ea0b07440d9d4dac345376fd8e1477ea"}, + {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761"}, + {file = "pandas-2.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e"}, + {file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +pyarrow = ["pyarrow (>=10.0.1)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + +[[package]] +name = "posthog" +version = "3.7.0" +description = "Integrate PostHog into any python application." +optional = false +python-versions = "*" +files = [ + {file = "posthog-3.7.0-py2.py3-none-any.whl", hash = "sha256:3555161c3a9557b5666f96d8e1f17f410ea0f07db56e399e336a1656d4e5c722"}, + {file = "posthog-3.7.0.tar.gz", hash = "sha256:b095d4354ba23f8b346ab5daed8ecfc5108772f922006982dfe8b2d29ebc6e0e"}, +] + +[package.dependencies] +backoff = ">=1.10.0" +monotonic = ">=1.5" +python-dateutil = ">2.1" +requests = ">=2.7,<3.0" +six = ">=1.5" + +[package.extras] +dev = ["black", "flake8", "flake8-print", "isort", "pre-commit"] +sentry = ["django", "sentry-sdk"] +test = ["coverage", "django", "flake8", "freezegun (==0.3.15)", "mock (>=2.0.0)", "pylint", "pytest", "pytest-timeout"] + +[[package]] +name = "propcache" +version = "0.2.0" +description = "Accelerated property cache" +optional = false +python-versions = ">=3.8" +files = [ + {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c5869b8fd70b81835a6f187c5fdbe67917a04d7e52b6e7cc4e5fe39d55c39d58"}, + {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:952e0d9d07609d9c5be361f33b0d6d650cd2bae393aabb11d9b719364521984b"}, + {file = "propcache-0.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:33ac8f098df0585c0b53009f039dfd913b38c1d2edafed0cedcc0c32a05aa110"}, + {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97e48e8875e6c13909c800fa344cd54cc4b2b0db1d5f911f840458a500fde2c2"}, + {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:388f3217649d6d59292b722d940d4d2e1e6a7003259eb835724092a1cca0203a"}, + {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f571aea50ba5623c308aa146eb650eebf7dbe0fd8c5d946e28343cb3b5aad577"}, + {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3dfafb44f7bb35c0c06eda6b2ab4bfd58f02729e7c4045e179f9a861b07c9850"}, + {file = "propcache-0.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3ebe9a75be7ab0b7da2464a77bb27febcb4fab46a34f9288f39d74833db7f61"}, + {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d2f0d0f976985f85dfb5f3d685697ef769faa6b71993b46b295cdbbd6be8cc37"}, + {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a3dc1a4b165283bd865e8f8cb5f0c64c05001e0718ed06250d8cac9bec115b48"}, + {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9e0f07b42d2a50c7dd2d8675d50f7343d998c64008f1da5fef888396b7f84630"}, + {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e63e3e1e0271f374ed489ff5ee73d4b6e7c60710e1f76af5f0e1a6117cd26394"}, + {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:56bb5c98f058a41bb58eead194b4db8c05b088c93d94d5161728515bd52b052b"}, + {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7665f04d0c7f26ff8bb534e1c65068409bf4687aa2534faf7104d7182debb336"}, + {file = "propcache-0.2.0-cp310-cp310-win32.whl", hash = "sha256:7cf18abf9764746b9c8704774d8b06714bcb0a63641518a3a89c7f85cc02c2ad"}, + {file = "propcache-0.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:cfac69017ef97db2438efb854edf24f5a29fd09a536ff3a992b75990720cdc99"}, + {file = "propcache-0.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:63f13bf09cc3336eb04a837490b8f332e0db41da66995c9fd1ba04552e516354"}, + {file = "propcache-0.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608cce1da6f2672a56b24a015b42db4ac612ee709f3d29f27a00c943d9e851de"}, + {file = "propcache-0.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:466c219deee4536fbc83c08d09115249db301550625c7fef1c5563a584c9bc87"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc2db02409338bf36590aa985a461b2c96fce91f8e7e0f14c50c5fcc4f229016"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a6ed8db0a556343d566a5c124ee483ae113acc9a557a807d439bcecc44e7dfbb"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:91997d9cb4a325b60d4e3f20967f8eb08dfcb32b22554d5ef78e6fd1dda743a2"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c7dde9e533c0a49d802b4f3f218fa9ad0a1ce21f2c2eb80d5216565202acab4"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffcad6c564fe6b9b8916c1aefbb37a362deebf9394bd2974e9d84232e3e08504"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:97a58a28bcf63284e8b4d7b460cbee1edaab24634e82059c7b8c09e65284f178"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:945db8ee295d3af9dbdbb698cce9bbc5c59b5c3fe328bbc4387f59a8a35f998d"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:39e104da444a34830751715f45ef9fc537475ba21b7f1f5b0f4d71a3b60d7fe2"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c5ecca8f9bab618340c8e848d340baf68bcd8ad90a8ecd7a4524a81c1764b3db"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:c436130cc779806bdf5d5fae0d848713105472b8566b75ff70048c47d3961c5b"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:191db28dc6dcd29d1a3e063c3be0b40688ed76434622c53a284e5427565bbd9b"}, + {file = "propcache-0.2.0-cp311-cp311-win32.whl", hash = "sha256:5f2564ec89058ee7c7989a7b719115bdfe2a2fb8e7a4543b8d1c0cc4cf6478c1"}, + {file = "propcache-0.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e2e54267980349b723cff366d1e29b138b9a60fa376664a157a342689553f71"}, + {file = "propcache-0.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2ee7606193fb267be4b2e3b32714f2d58cad27217638db98a60f9efb5efeccc2"}, + {file = "propcache-0.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:91ee8fc02ca52e24bcb77b234f22afc03288e1dafbb1f88fe24db308910c4ac7"}, + {file = "propcache-0.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2e900bad2a8456d00a113cad8c13343f3b1f327534e3589acc2219729237a2e8"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f52a68c21363c45297aca15561812d542f8fc683c85201df0bebe209e349f793"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e41d67757ff4fbc8ef2af99b338bfb955010444b92929e9e55a6d4dcc3c4f09"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a64e32f8bd94c105cc27f42d3b658902b5bcc947ece3c8fe7bc1b05982f60e89"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55346705687dbd7ef0d77883ab4f6fabc48232f587925bdaf95219bae072491e"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00181262b17e517df2cd85656fcd6b4e70946fe62cd625b9d74ac9977b64d8d9"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6994984550eaf25dd7fc7bd1b700ff45c894149341725bb4edc67f0ffa94efa4"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:56295eb1e5f3aecd516d91b00cfd8bf3a13991de5a479df9e27dd569ea23959c"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:439e76255daa0f8151d3cb325f6dd4a3e93043e6403e6491813bcaaaa8733887"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f6475a1b2ecb310c98c28d271a30df74f9dd436ee46d09236a6b750a7599ce57"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3444cdba6628accf384e349014084b1cacd866fbb88433cd9d279d90a54e0b23"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a9d9b4d0a9b38d1c391bb4ad24aa65f306c6f01b512e10a8a34a2dc5675d348"}, + {file = "propcache-0.2.0-cp312-cp312-win32.whl", hash = "sha256:69d3a98eebae99a420d4b28756c8ce6ea5a29291baf2dc9ff9414b42676f61d5"}, + {file = "propcache-0.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:ad9c9b99b05f163109466638bd30ada1722abb01bbb85c739c50b6dc11f92dc3"}, + {file = "propcache-0.2.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ecddc221a077a8132cf7c747d5352a15ed763b674c0448d811f408bf803d9ad7"}, + {file = "propcache-0.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0e53cb83fdd61cbd67202735e6a6687a7b491c8742dfc39c9e01e80354956763"}, + {file = "propcache-0.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92fe151145a990c22cbccf9ae15cae8ae9eddabfc949a219c9f667877e40853d"}, + {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6a21ef516d36909931a2967621eecb256018aeb11fc48656e3257e73e2e247a"}, + {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f88a4095e913f98988f5b338c1d4d5d07dbb0b6bad19892fd447484e483ba6b"}, + {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a5b3bb545ead161be780ee85a2b54fdf7092815995661947812dde94a40f6fb"}, + {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67aeb72e0f482709991aa91345a831d0b707d16b0257e8ef88a2ad246a7280bf"}, + {file = "propcache-0.2.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c997f8c44ec9b9b0bcbf2d422cc00a1d9b9c681f56efa6ca149a941e5560da2"}, + {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2a66df3d4992bc1d725b9aa803e8c5a66c010c65c741ad901e260ece77f58d2f"}, + {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3ebbcf2a07621f29638799828b8d8668c421bfb94c6cb04269130d8de4fb7136"}, + {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1235c01ddaa80da8235741e80815ce381c5267f96cc49b1477fdcf8c047ef325"}, + {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3947483a381259c06921612550867b37d22e1df6d6d7e8361264b6d037595f44"}, + {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d5bed7f9805cc29c780f3aee05de3262ee7ce1f47083cfe9f77471e9d6777e83"}, + {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e4a91d44379f45f5e540971d41e4626dacd7f01004826a18cb048e7da7e96544"}, + {file = "propcache-0.2.0-cp313-cp313-win32.whl", hash = "sha256:f902804113e032e2cdf8c71015651c97af6418363bea8d78dc0911d56c335032"}, + {file = "propcache-0.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8f188cfcc64fb1266f4684206c9de0e80f54622c3f22a910cbd200478aeae61e"}, + {file = "propcache-0.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:53d1bd3f979ed529f0805dd35ddaca330f80a9a6d90bc0121d2ff398f8ed8861"}, + {file = "propcache-0.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:83928404adf8fb3d26793665633ea79b7361efa0287dfbd372a7e74311d51ee6"}, + {file = "propcache-0.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:77a86c261679ea5f3896ec060be9dc8e365788248cc1e049632a1be682442063"}, + {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:218db2a3c297a3768c11a34812e63b3ac1c3234c3a086def9c0fee50d35add1f"}, + {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7735e82e3498c27bcb2d17cb65d62c14f1100b71723b68362872bca7d0913d90"}, + {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:20a617c776f520c3875cf4511e0d1db847a076d720714ae35ffe0df3e440be68"}, + {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67b69535c870670c9f9b14a75d28baa32221d06f6b6fa6f77a0a13c5a7b0a5b9"}, + {file = "propcache-0.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4569158070180c3855e9c0791c56be3ceeb192defa2cdf6a3f39e54319e56b89"}, + {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:db47514ffdbd91ccdc7e6f8407aac4ee94cc871b15b577c1c324236b013ddd04"}, + {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:2a60ad3e2553a74168d275a0ef35e8c0a965448ffbc3b300ab3a5bb9956c2162"}, + {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:662dd62358bdeaca0aee5761de8727cfd6861432e3bb828dc2a693aa0471a563"}, + {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:25a1f88b471b3bc911d18b935ecb7115dff3a192b6fef46f0bfaf71ff4f12418"}, + {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:f60f0ac7005b9f5a6091009b09a419ace1610e163fa5deaba5ce3484341840e7"}, + {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:74acd6e291f885678631b7ebc85d2d4aec458dd849b8c841b57ef04047833bed"}, + {file = "propcache-0.2.0-cp38-cp38-win32.whl", hash = "sha256:d9b6ddac6408194e934002a69bcaadbc88c10b5f38fb9307779d1c629181815d"}, + {file = "propcache-0.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:676135dcf3262c9c5081cc8f19ad55c8a64e3f7282a21266d05544450bffc3a5"}, + {file = "propcache-0.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:25c8d773a62ce0451b020c7b29a35cfbc05de8b291163a7a0f3b7904f27253e6"}, + {file = "propcache-0.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:375a12d7556d462dc64d70475a9ee5982465fbb3d2b364f16b86ba9135793638"}, + {file = "propcache-0.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1ec43d76b9677637a89d6ab86e1fef70d739217fefa208c65352ecf0282be957"}, + {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f45eec587dafd4b2d41ac189c2156461ebd0c1082d2fe7013571598abb8505d1"}, + {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc092ba439d91df90aea38168e11f75c655880c12782facf5cf9c00f3d42b562"}, + {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa1076244f54bb76e65e22cb6910365779d5c3d71d1f18b275f1dfc7b0d71b4d"}, + {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:682a7c79a2fbf40f5dbb1eb6bfe2cd865376deeac65acf9beb607505dced9e12"}, + {file = "propcache-0.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e40876731f99b6f3c897b66b803c9e1c07a989b366c6b5b475fafd1f7ba3fb8"}, + {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:363ea8cd3c5cb6679f1c2f5f1f9669587361c062e4899fce56758efa928728f8"}, + {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:140fbf08ab3588b3468932974a9331aff43c0ab8a2ec2c608b6d7d1756dbb6cb"}, + {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e70fac33e8b4ac63dfc4c956fd7d85a0b1139adcfc0d964ce288b7c527537fea"}, + {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:b33d7a286c0dc1a15f5fc864cc48ae92a846df287ceac2dd499926c3801054a6"}, + {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f6d5749fdd33d90e34c2efb174c7e236829147a2713334d708746e94c4bde40d"}, + {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:22aa8f2272d81d9317ff5756bb108021a056805ce63dd3630e27d042c8092798"}, + {file = "propcache-0.2.0-cp39-cp39-win32.whl", hash = "sha256:73e4b40ea0eda421b115248d7e79b59214411109a5bc47d0d48e4c73e3b8fcf9"}, + {file = "propcache-0.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:9517d5e9e0731957468c29dbfd0f976736a0e55afaea843726e887f36fe017df"}, + {file = "propcache-0.2.0-py3-none-any.whl", hash = "sha256:2ccc28197af5313706511fab3a8b66dcd6da067a1331372c82ea1cb74285e036"}, + {file = "propcache-0.2.0.tar.gz", hash = "sha256:df81779732feb9d01e5d513fad0122efb3d53bbc75f61b2a4f29a020bc985e70"}, +] + +[[package]] +name = "pydantic" +version = "2.9.2" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, + {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, +] + +[package.dependencies] +annotated-types = ">=0.6.0" +pydantic-core = "2.23.4" +typing-extensions = [ + {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, + {version = ">=4.6.1", markers = "python_version < \"3.13\""}, +] + +[package.extras] +email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata"] + +[[package]] +name = "pydantic-core" +version = "2.23.4" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.23.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b"}, + {file = "pydantic_core-2.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63e46b3169866bd62849936de036f901a9356e36376079b05efa83caeaa02ceb"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed1a53de42fbe34853ba90513cea21673481cd81ed1be739f7f2efb931b24916"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cfdd16ab5e59fc31b5e906d1a3f666571abc367598e3e02c83403acabc092e07"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255a8ef062cbf6674450e668482456abac99a5583bbafb73f9ad469540a3a232"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a7cd62e831afe623fbb7aabbb4fe583212115b3ef38a9f6b71869ba644624a2"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f09e2ff1f17c2b51f2bc76d1cc33da96298f0a036a137f5440ab3ec5360b624f"}, + {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e38e63e6f3d1cec5a27e0afe90a085af8b6806ee208b33030e65b6516353f1a3"}, + {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0dbd8dbed2085ed23b5c04afa29d8fd2771674223135dc9bc937f3c09284d071"}, + {file = "pydantic_core-2.23.4-cp310-none-win32.whl", hash = "sha256:6531b7ca5f951d663c339002e91aaebda765ec7d61b7d1e3991051906ddde119"}, + {file = "pydantic_core-2.23.4-cp310-none-win_amd64.whl", hash = "sha256:7c9129eb40958b3d4500fa2467e6a83356b3b61bfff1b414c7361d9220f9ae8f"}, + {file = "pydantic_core-2.23.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8"}, + {file = "pydantic_core-2.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b"}, + {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0"}, + {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64"}, + {file = "pydantic_core-2.23.4-cp311-none-win32.whl", hash = "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f"}, + {file = "pydantic_core-2.23.4-cp311-none-win_amd64.whl", hash = "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3"}, + {file = "pydantic_core-2.23.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231"}, + {file = "pydantic_core-2.23.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126"}, + {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e"}, + {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24"}, + {file = "pydantic_core-2.23.4-cp312-none-win32.whl", hash = "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84"}, + {file = "pydantic_core-2.23.4-cp312-none-win_amd64.whl", hash = "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9"}, + {file = "pydantic_core-2.23.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7530e201d10d7d14abce4fb54cfe5b94a0aefc87da539d0346a484ead376c3cc"}, + {file = "pydantic_core-2.23.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df933278128ea1cd77772673c73954e53a1c95a4fdf41eef97c2b779271bd0bd"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cb3da3fd1b6a5d0279a01877713dbda118a2a4fc6f0d821a57da2e464793f05"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c6dcb030aefb668a2b7009c85b27f90e51e6a3b4d5c9bc4c57631292015b0d"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:696dd8d674d6ce621ab9d45b205df149399e4bb9aa34102c970b721554828510"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2971bb5ffe72cc0f555c13e19b23c85b654dd2a8f7ab493c262071377bfce9f6"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8394d940e5d400d04cad4f75c0598665cbb81aecefaca82ca85bd28264af7f9b"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dff76e0602ca7d4cdaacc1ac4c005e0ce0dcfe095d5b5259163a80d3a10d327"}, + {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7d32706badfe136888bdea71c0def994644e09fff0bfe47441deaed8e96fdbc6"}, + {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ed541d70698978a20eb63d8c5d72f2cc6d7079d9d90f6b50bad07826f1320f5f"}, + {file = "pydantic_core-2.23.4-cp313-none-win32.whl", hash = "sha256:3d5639516376dce1940ea36edf408c554475369f5da2abd45d44621cb616f769"}, + {file = "pydantic_core-2.23.4-cp313-none-win_amd64.whl", hash = "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5"}, + {file = "pydantic_core-2.23.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d4488a93b071c04dc20f5cecc3631fc78b9789dd72483ba15d423b5b3689b555"}, + {file = "pydantic_core-2.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:81965a16b675b35e1d09dd14df53f190f9129c0202356ed44ab2728b1c905658"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffa2ebd4c8530079140dd2d7f794a9d9a73cbb8e9d59ffe24c63436efa8f271"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:61817945f2fe7d166e75fbfb28004034b48e44878177fc54d81688e7b85a3665"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29d2c342c4bc01b88402d60189f3df065fb0dda3654744d5a165a5288a657368"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e11661ce0fd30a6790e8bcdf263b9ec5988e95e63cf901972107efc49218b13"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d18368b137c6295db49ce7218b1a9ba15c5bc254c96d7c9f9e924a9bc7825ad"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec4e55f79b1c4ffb2eecd8a0cfba9955a2588497d96851f4c8f99aa4a1d39b12"}, + {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:374a5e5049eda9e0a44c696c7ade3ff355f06b1fe0bb945ea3cac2bc336478a2"}, + {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5c364564d17da23db1106787675fc7af45f2f7b58b4173bfdd105564e132e6fb"}, + {file = "pydantic_core-2.23.4-cp38-none-win32.whl", hash = "sha256:d7a80d21d613eec45e3d41eb22f8f94ddc758a6c4720842dc74c0581f54993d6"}, + {file = "pydantic_core-2.23.4-cp38-none-win_amd64.whl", hash = "sha256:5f5ff8d839f4566a474a969508fe1c5e59c31c80d9e140566f9a37bba7b8d556"}, + {file = "pydantic_core-2.23.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a4fa4fc04dff799089689f4fd502ce7d59de529fc2f40a2c8836886c03e0175a"}, + {file = "pydantic_core-2.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a7df63886be5e270da67e0966cf4afbae86069501d35c8c1b3b6c168f42cb36"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcedcd19a557e182628afa1d553c3895a9f825b936415d0dbd3cd0bbcfd29b4b"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f54b118ce5de9ac21c363d9b3caa6c800341e8c47a508787e5868c6b79c9323"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d2f57d3e1379a9525c5ab067b27dbb8a0642fb5d454e17a9ac434f9ce523e3"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de6d1d1b9e5101508cb37ab0d972357cac5235f5c6533d1071964c47139257df"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1278e0d324f6908e872730c9102b0112477a7f7cf88b308e4fc36ce1bdb6d58c"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a6b5099eeec78827553827f4c6b8615978bb4b6a88e5d9b93eddf8bb6790f55"}, + {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e55541f756f9b3ee346b840103f32779c695a19826a4c442b7954550a0972040"}, + {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a5c7ba8ffb6d6f8f2ab08743be203654bb1aaa8c9dcb09f82ddd34eadb695605"}, + {file = "pydantic_core-2.23.4-cp39-none-win32.whl", hash = "sha256:37b0fe330e4a58d3c58b24d91d1eb102aeec675a3db4c292ec3928ecd892a9a6"}, + {file = "pydantic_core-2.23.4-cp39-none-win_amd64.whl", hash = "sha256:1498bec4c05c9c787bde9125cfdcc63a41004ff167f495063191b863399b1a29"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f455ee30a9d61d3e1a15abd5068827773d6e4dc513e795f380cdd59932c782d5"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1e90d2e3bd2c3863d48525d297cd143fe541be8bbf6f579504b9712cb6b643ec"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e203fdf807ac7e12ab59ca2bfcabb38c7cf0b33c41efeb00f8e5da1d86af480"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e08277a400de01bc72436a0ccd02bdf596631411f592ad985dcee21445bd0068"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f220b0eea5965dec25480b6333c788fb72ce5f9129e8759ef876a1d805d00801"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d06b0c8da4f16d1d1e352134427cb194a0a6e19ad5db9161bf32b2113409e728"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ba1a0996f6c2773bd83e63f18914c1de3c9dd26d55f4ac302a7efe93fb8e7433"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:9a5bce9d23aac8f0cf0836ecfc033896aa8443b501c58d0602dbfd5bd5b37753"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:78ddaaa81421a29574a682b3179d4cf9e6d405a09b99d93ddcf7e5239c742e21"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:883a91b5dd7d26492ff2f04f40fbb652de40fcc0afe07e8129e8ae779c2110eb"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88ad334a15b32a791ea935af224b9de1bf99bcd62fabf745d5f3442199d86d59"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233710f069d251feb12a56da21e14cca67994eab08362207785cf8c598e74577"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19442362866a753485ba5e4be408964644dd6a09123d9416c54cd49171f50744"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:624e278a7d29b6445e4e813af92af37820fafb6dcc55c012c834f9e26f9aaaef"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f5ef8f42bec47f21d07668a043f077d507e5bf4e668d5c6dfe6aaba89de1a5b8"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:aea443fffa9fbe3af1a9ba721a87f926fe548d32cab71d188a6ede77d0ff244e"}, + {file = "pydantic_core-2.23.4.tar.gz", hash = "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "pyjwt" +version = "2.9.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, + {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-dotenv" +version = "1.0.1" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, + {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + +[[package]] +name = "python-multipart" +version = "0.0.7" +description = "A streaming multipart parser for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "python_multipart-0.0.7-py3-none-any.whl", hash = "sha256:b1fef9a53b74c795e2347daac8c54b252d9e0df9c619712691c1cc8021bd3c49"}, + {file = "python_multipart-0.0.7.tar.gz", hash = "sha256:288a6c39b06596c1b988bb6794c6fbc80e6c369e35e5062637df256bee0c9af9"}, +] + +[package.extras] +dev = ["atomicwrites (==1.2.1)", "attrs (==19.2.0)", "coverage (==6.5.0)", "hatch", "invoke (==2.2.0)", "more-itertools (==4.3.0)", "pbr (==4.3.0)", "pluggy (==1.0.0)", "py (==1.11.0)", "pytest (==7.2.0)", "pytest-cov (==4.0.0)", "pytest-timeout (==2.1.0)", "pyyaml (==5.1)"] + +[[package]] +name = "pytz" +version = "2024.2" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, + {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, +] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "ruff" +version = "0.7.1" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.7.1-py3-none-linux_armv6l.whl", hash = "sha256:cb1bc5ed9403daa7da05475d615739cc0212e861b7306f314379d958592aaa89"}, + {file = "ruff-0.7.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:27c1c52a8d199a257ff1e5582d078eab7145129aa02721815ca8fa4f9612dc35"}, + {file = "ruff-0.7.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:588a34e1ef2ea55b4ddfec26bbe76bc866e92523d8c6cdec5e8aceefeff02d99"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94fc32f9cdf72dc75c451e5f072758b118ab8100727168a3df58502b43a599ca"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:985818742b833bffa543a84d1cc11b5e6871de1b4e0ac3060a59a2bae3969250"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32f1e8a192e261366c702c5fb2ece9f68d26625f198a25c408861c16dc2dea9c"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:699085bf05819588551b11751eff33e9ca58b1b86a6843e1b082a7de40da1565"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:344cc2b0814047dc8c3a8ff2cd1f3d808bb23c6658db830d25147339d9bf9ea7"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4316bbf69d5a859cc937890c7ac7a6551252b6a01b1d2c97e8fc96e45a7c8b4a"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79d3af9dca4c56043e738a4d6dd1e9444b6d6c10598ac52d146e331eb155a8ad"}, + {file = "ruff-0.7.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c5c121b46abde94a505175524e51891f829414e093cd8326d6e741ecfc0a9112"}, + {file = "ruff-0.7.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8422104078324ea250886954e48f1373a8fe7de59283d747c3a7eca050b4e378"}, + {file = "ruff-0.7.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:56aad830af8a9db644e80098fe4984a948e2b6fc2e73891538f43bbe478461b8"}, + {file = "ruff-0.7.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:658304f02f68d3a83c998ad8bf91f9b4f53e93e5412b8f2388359d55869727fd"}, + {file = "ruff-0.7.1-py3-none-win32.whl", hash = "sha256:b517a2011333eb7ce2d402652ecaa0ac1a30c114fbbd55c6b8ee466a7f600ee9"}, + {file = "ruff-0.7.1-py3-none-win_amd64.whl", hash = "sha256:f38c41fcde1728736b4eb2b18850f6d1e3eedd9678c914dede554a70d5241307"}, + {file = "ruff-0.7.1-py3-none-win_arm64.whl", hash = "sha256:19aa200ec824c0f36d0c9114c8ec0087082021732979a359d6f3c390a6ff2a37"}, + {file = "ruff-0.7.1.tar.gz", hash = "sha256:9d8a41d4aa2dad1575adb98a82870cf5db5f76b2938cf2206c22c940034a36f4"}, +] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + +[[package]] +name = "starlette" +version = "0.41.2" +description = "The little ASGI library that shines." +optional = false +python-versions = ">=3.8" +files = [ + {file = "starlette-0.41.2-py3-none-any.whl", hash = "sha256:fbc189474b4731cf30fcef52f18a8d070e3f3b46c6a04c97579e85e6ffca942d"}, + {file = "starlette-0.41.2.tar.gz", hash = "sha256:9834fd799d1a87fd346deb76158668cfa0b0d56f85caefe8268e2d97c3468b62"}, +] + +[package.dependencies] +anyio = ">=3.4.0,<5" +typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} + +[package.extras] +full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.7)", "pyyaml"] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[[package]] +name = "tzdata" +version = "2024.2" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, + {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, +] + +[[package]] +name = "urllib3" +version = "2.2.3" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, + {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "uuid7" +version = "0.1.0" +description = "UUID version 7, generating time-sorted UUIDs with 200ns time resolution and 48 bits of randomness" +optional = false +python-versions = ">=3.7" +files = [ + {file = "uuid7-0.1.0-py2.py3-none-any.whl", hash = "sha256:5e259bb63c8cb4aded5927ff41b444a80d0c7124e8a0ced7cf44efa1f5cccf61"}, + {file = "uuid7-0.1.0.tar.gz", hash = "sha256:8c57aa32ee7456d3cc68c95c4530bc571646defac01895cfc73545449894a63c"}, +] + +[[package]] +name = "uvicorn" +version = "0.32.0" +description = "The lightning-fast ASGI server." +optional = false +python-versions = ">=3.8" +files = [ + {file = "uvicorn-0.32.0-py3-none-any.whl", hash = "sha256:60b8f3a5ac027dcd31448f411ced12b5ef452c646f76f02f8cc3f25d8d26fd82"}, + {file = "uvicorn-0.32.0.tar.gz", hash = "sha256:f78b36b143c16f54ccdb8190d0a26b5f1901fe5a3c777e1ab29f26391af8551e"}, +] + +[package.dependencies] +click = ">=7.0" +h11 = ">=0.8" +typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} + +[package.extras] +standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] + +[[package]] +name = "webencodings" +version = "0.5.1" +description = "Character encoding aliases for legacy web content" +optional = false +python-versions = "*" +files = [ + {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, + {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, +] + +[[package]] +name = "websockets" +version = "13.1" +description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "websockets-13.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f48c749857f8fb598fb890a75f540e3221d0976ed0bf879cf3c7eef34151acee"}, + {file = "websockets-13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c7e72ce6bda6fb9409cc1e8164dd41d7c91466fb599eb047cfda72fe758a34a7"}, + {file = "websockets-13.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f779498eeec470295a2b1a5d97aa1bc9814ecd25e1eb637bd9d1c73a327387f6"}, + {file = "websockets-13.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4676df3fe46956fbb0437d8800cd5f2b6d41143b6e7e842e60554398432cf29b"}, + {file = "websockets-13.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7affedeb43a70351bb811dadf49493c9cfd1ed94c9c70095fd177e9cc1541fa"}, + {file = "websockets-13.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1971e62d2caa443e57588e1d82d15f663b29ff9dfe7446d9964a4b6f12c1e700"}, + {file = "websockets-13.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5f2e75431f8dc4a47f31565a6e1355fb4f2ecaa99d6b89737527ea917066e26c"}, + {file = "websockets-13.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:58cf7e75dbf7e566088b07e36ea2e3e2bd5676e22216e4cad108d4df4a7402a0"}, + {file = "websockets-13.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c90d6dec6be2c7d03378a574de87af9b1efea77d0c52a8301dd831ece938452f"}, + {file = "websockets-13.1-cp310-cp310-win32.whl", hash = "sha256:730f42125ccb14602f455155084f978bd9e8e57e89b569b4d7f0f0c17a448ffe"}, + {file = "websockets-13.1-cp310-cp310-win_amd64.whl", hash = "sha256:5993260f483d05a9737073be197371940c01b257cc45ae3f1d5d7adb371b266a"}, + {file = "websockets-13.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:61fc0dfcda609cda0fc9fe7977694c0c59cf9d749fbb17f4e9483929e3c48a19"}, + {file = "websockets-13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ceec59f59d092c5007e815def4ebb80c2de330e9588e101cf8bd94c143ec78a5"}, + {file = "websockets-13.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c1dca61c6db1166c48b95198c0b7d9c990b30c756fc2923cc66f68d17dc558fd"}, + {file = "websockets-13.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:308e20f22c2c77f3f39caca508e765f8725020b84aa963474e18c59accbf4c02"}, + {file = "websockets-13.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62d516c325e6540e8a57b94abefc3459d7dab8ce52ac75c96cad5549e187e3a7"}, + {file = "websockets-13.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87c6e35319b46b99e168eb98472d6c7d8634ee37750d7693656dc766395df096"}, + {file = "websockets-13.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5f9fee94ebafbc3117c30be1844ed01a3b177bb6e39088bc6b2fa1dc15572084"}, + {file = "websockets-13.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7c1e90228c2f5cdde263253fa5db63e6653f1c00e7ec64108065a0b9713fa1b3"}, + {file = "websockets-13.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6548f29b0e401eea2b967b2fdc1c7c7b5ebb3eeb470ed23a54cd45ef078a0db9"}, + {file = "websockets-13.1-cp311-cp311-win32.whl", hash = "sha256:c11d4d16e133f6df8916cc5b7e3e96ee4c44c936717d684a94f48f82edb7c92f"}, + {file = "websockets-13.1-cp311-cp311-win_amd64.whl", hash = "sha256:d04f13a1d75cb2b8382bdc16ae6fa58c97337253826dfe136195b7f89f661557"}, + {file = "websockets-13.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9d75baf00138f80b48f1eac72ad1535aac0b6461265a0bcad391fc5aba875cfc"}, + {file = "websockets-13.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9b6f347deb3dcfbfde1c20baa21c2ac0751afaa73e64e5b693bb2b848efeaa49"}, + {file = "websockets-13.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de58647e3f9c42f13f90ac7e5f58900c80a39019848c5547bc691693098ae1bd"}, + {file = "websockets-13.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1b54689e38d1279a51d11e3467dd2f3a50f5f2e879012ce8f2d6943f00e83f0"}, + {file = "websockets-13.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf1781ef73c073e6b0f90af841aaf98501f975d306bbf6221683dd594ccc52b6"}, + {file = "websockets-13.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d23b88b9388ed85c6faf0e74d8dec4f4d3baf3ecf20a65a47b836d56260d4b9"}, + {file = "websockets-13.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3c78383585f47ccb0fcf186dcb8a43f5438bd7d8f47d69e0b56f71bf431a0a68"}, + {file = "websockets-13.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d6d300f8ec35c24025ceb9b9019ae9040c1ab2f01cddc2bcc0b518af31c75c14"}, + {file = "websockets-13.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a9dcaf8b0cc72a392760bb8755922c03e17a5a54e08cca58e8b74f6902b433cf"}, + {file = "websockets-13.1-cp312-cp312-win32.whl", hash = "sha256:2f85cf4f2a1ba8f602298a853cec8526c2ca42a9a4b947ec236eaedb8f2dc80c"}, + {file = "websockets-13.1-cp312-cp312-win_amd64.whl", hash = "sha256:38377f8b0cdeee97c552d20cf1865695fcd56aba155ad1b4ca8779a5b6ef4ac3"}, + {file = "websockets-13.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a9ab1e71d3d2e54a0aa646ab6d4eebfaa5f416fe78dfe4da2839525dc5d765c6"}, + {file = "websockets-13.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b9d7439d7fab4dce00570bb906875734df13d9faa4b48e261c440a5fec6d9708"}, + {file = "websockets-13.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:327b74e915cf13c5931334c61e1a41040e365d380f812513a255aa804b183418"}, + {file = "websockets-13.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:325b1ccdbf5e5725fdcb1b0e9ad4d2545056479d0eee392c291c1bf76206435a"}, + {file = "websockets-13.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:346bee67a65f189e0e33f520f253d5147ab76ae42493804319b5716e46dddf0f"}, + {file = "websockets-13.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91a0fa841646320ec0d3accdff5b757b06e2e5c86ba32af2e0815c96c7a603c5"}, + {file = "websockets-13.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18503d2c5f3943e93819238bf20df71982d193f73dcecd26c94514f417f6b135"}, + {file = "websockets-13.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:a9cd1af7e18e5221d2878378fbc287a14cd527fdd5939ed56a18df8a31136bb2"}, + {file = "websockets-13.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:70c5be9f416aa72aab7a2a76c90ae0a4fe2755c1816c153c1a2bcc3333ce4ce6"}, + {file = "websockets-13.1-cp313-cp313-win32.whl", hash = "sha256:624459daabeb310d3815b276c1adef475b3e6804abaf2d9d2c061c319f7f187d"}, + {file = "websockets-13.1-cp313-cp313-win_amd64.whl", hash = "sha256:c518e84bb59c2baae725accd355c8dc517b4a3ed8db88b4bc93c78dae2974bf2"}, + {file = "websockets-13.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:c7934fd0e920e70468e676fe7f1b7261c1efa0d6c037c6722278ca0228ad9d0d"}, + {file = "websockets-13.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:149e622dc48c10ccc3d2760e5f36753db9cacf3ad7bc7bbbfd7d9c819e286f23"}, + {file = "websockets-13.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a569eb1b05d72f9bce2ebd28a1ce2054311b66677fcd46cf36204ad23acead8c"}, + {file = "websockets-13.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95df24ca1e1bd93bbca51d94dd049a984609687cb2fb08a7f2c56ac84e9816ea"}, + {file = "websockets-13.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8dbb1bf0c0a4ae8b40bdc9be7f644e2f3fb4e8a9aca7145bfa510d4a374eeb7"}, + {file = "websockets-13.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:035233b7531fb92a76beefcbf479504db8c72eb3bff41da55aecce3a0f729e54"}, + {file = "websockets-13.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:e4450fc83a3df53dec45922b576e91e94f5578d06436871dce3a6be38e40f5db"}, + {file = "websockets-13.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:463e1c6ec853202dd3657f156123d6b4dad0c546ea2e2e38be2b3f7c5b8e7295"}, + {file = "websockets-13.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6d6855bbe70119872c05107e38fbc7f96b1d8cb047d95c2c50869a46c65a8e96"}, + {file = "websockets-13.1-cp38-cp38-win32.whl", hash = "sha256:204e5107f43095012b00f1451374693267adbb832d29966a01ecc4ce1db26faf"}, + {file = "websockets-13.1-cp38-cp38-win_amd64.whl", hash = "sha256:485307243237328c022bc908b90e4457d0daa8b5cf4b3723fd3c4a8012fce4c6"}, + {file = "websockets-13.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9b37c184f8b976f0c0a231a5f3d6efe10807d41ccbe4488df8c74174805eea7d"}, + {file = "websockets-13.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:163e7277e1a0bd9fb3c8842a71661ad19c6aa7bb3d6678dc7f89b17fbcc4aeb7"}, + {file = "websockets-13.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4b889dbd1342820cc210ba44307cf75ae5f2f96226c0038094455a96e64fb07a"}, + {file = "websockets-13.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:586a356928692c1fed0eca68b4d1c2cbbd1ca2acf2ac7e7ebd3b9052582deefa"}, + {file = "websockets-13.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7bd6abf1e070a6b72bfeb71049d6ad286852e285f146682bf30d0296f5fbadfa"}, + {file = "websockets-13.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2aad13a200e5934f5a6767492fb07151e1de1d6079c003ab31e1823733ae79"}, + {file = "websockets-13.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:df01aea34b6e9e33572c35cd16bae5a47785e7d5c8cb2b54b2acdb9678315a17"}, + {file = "websockets-13.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e54affdeb21026329fb0744ad187cf812f7d3c2aa702a5edb562b325191fcab6"}, + {file = "websockets-13.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9ef8aa8bdbac47f4968a5d66462a2a0935d044bf35c0e5a8af152d58516dbeb5"}, + {file = "websockets-13.1-cp39-cp39-win32.whl", hash = "sha256:deeb929efe52bed518f6eb2ddc00cc496366a14c726005726ad62c2dd9017a3c"}, + {file = "websockets-13.1-cp39-cp39-win_amd64.whl", hash = "sha256:7c65ffa900e7cc958cd088b9a9157a8141c991f8c53d11087e6fb7277a03f81d"}, + {file = "websockets-13.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5dd6da9bec02735931fccec99d97c29f47cc61f644264eb995ad6c0c27667238"}, + {file = "websockets-13.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:2510c09d8e8df777177ee3d40cd35450dc169a81e747455cc4197e63f7e7bfe5"}, + {file = "websockets-13.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1c3cf67185543730888b20682fb186fc8d0fa6f07ccc3ef4390831ab4b388d9"}, + {file = "websockets-13.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bcc03c8b72267e97b49149e4863d57c2d77f13fae12066622dc78fe322490fe6"}, + {file = "websockets-13.1-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:004280a140f220c812e65f36944a9ca92d766b6cc4560be652a0a3883a79ed8a"}, + {file = "websockets-13.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e2620453c075abeb0daa949a292e19f56de518988e079c36478bacf9546ced23"}, + {file = "websockets-13.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9156c45750b37337f7b0b00e6248991a047be4aa44554c9886fe6bdd605aab3b"}, + {file = "websockets-13.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:80c421e07973a89fbdd93e6f2003c17d20b69010458d3a8e37fb47874bd67d51"}, + {file = "websockets-13.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82d0ba76371769d6a4e56f7e83bb8e81846d17a6190971e38b5de108bde9b0d7"}, + {file = "websockets-13.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e9875a0143f07d74dc5e1ded1c4581f0d9f7ab86c78994e2ed9e95050073c94d"}, + {file = "websockets-13.1-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a11e38ad8922c7961447f35c7b17bffa15de4d17c70abd07bfbe12d6faa3e027"}, + {file = "websockets-13.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:4059f790b6ae8768471cddb65d3c4fe4792b0ab48e154c9f0a04cefaabcd5978"}, + {file = "websockets-13.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:25c35bf84bf7c7369d247f0b8cfa157f989862c49104c5cf85cb5436a641d93e"}, + {file = "websockets-13.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:83f91d8a9bb404b8c2c41a707ac7f7f75b9442a0a876df295de27251a856ad09"}, + {file = "websockets-13.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a43cfdcddd07f4ca2b1afb459824dd3c6d53a51410636a2c7fc97b9a8cf4842"}, + {file = "websockets-13.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48a2ef1381632a2f0cb4efeff34efa97901c9fbc118e01951ad7cfc10601a9bb"}, + {file = "websockets-13.1-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:459bf774c754c35dbb487360b12c5727adab887f1622b8aed5755880a21c4a20"}, + {file = "websockets-13.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:95858ca14a9f6fa8413d29e0a585b31b278388aa775b8a81fa24830123874678"}, + {file = "websockets-13.1-py3-none-any.whl", hash = "sha256:a9a396a6ad26130cdae92ae10c36af09d9bfe6cafe69670fd3b6da9b07b4044f"}, + {file = "websockets-13.1.tar.gz", hash = "sha256:a3b3366087c1bc0a2795111edcadddb8b3b59509d5db5d7ea3fdd69f954a8878"}, +] + +[[package]] +name = "yarl" +version = "1.17.1" +description = "Yet another URL library" +optional = false +python-versions = ">=3.9" +files = [ + {file = "yarl-1.17.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1794853124e2f663f0ea54efb0340b457f08d40a1cef78edfa086576179c91"}, + {file = "yarl-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fbea1751729afe607d84acfd01efd95e3b31db148a181a441984ce9b3d3469da"}, + {file = "yarl-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8ee427208c675f1b6e344a1f89376a9613fc30b52646a04ac0c1f6587c7e46ec"}, + {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b74ff4767d3ef47ffe0cd1d89379dc4d828d4873e5528976ced3b44fe5b0a21"}, + {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:62a91aefff3d11bf60e5956d340eb507a983a7ec802b19072bb989ce120cd948"}, + {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:846dd2e1243407133d3195d2d7e4ceefcaa5f5bf7278f0a9bda00967e6326b04"}, + {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e844be8d536afa129366d9af76ed7cb8dfefec99f5f1c9e4f8ae542279a6dc3"}, + {file = "yarl-1.17.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc7c92c1baa629cb03ecb0c3d12564f172218fb1739f54bf5f3881844daadc6d"}, + {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ae3476e934b9d714aa8000d2e4c01eb2590eee10b9d8cd03e7983ad65dfbfcba"}, + {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c7e177c619342e407415d4f35dec63d2d134d951e24b5166afcdfd1362828e17"}, + {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:64cc6e97f14cf8a275d79c5002281f3040c12e2e4220623b5759ea7f9868d6a5"}, + {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:84c063af19ef5130084db70ada40ce63a84f6c1ef4d3dbc34e5e8c4febb20822"}, + {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:482c122b72e3c5ec98f11457aeb436ae4aecca75de19b3d1de7cf88bc40db82f"}, + {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:380e6c38ef692b8fd5a0f6d1fa8774d81ebc08cfbd624b1bca62a4d4af2f9931"}, + {file = "yarl-1.17.1-cp310-cp310-win32.whl", hash = "sha256:16bca6678a83657dd48df84b51bd56a6c6bd401853aef6d09dc2506a78484c7b"}, + {file = "yarl-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:561c87fea99545ef7d692403c110b2f99dced6dff93056d6e04384ad3bc46243"}, + {file = "yarl-1.17.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cbad927ea8ed814622305d842c93412cb47bd39a496ed0f96bfd42b922b4a217"}, + {file = "yarl-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fca4b4307ebe9c3ec77a084da3a9d1999d164693d16492ca2b64594340999988"}, + {file = "yarl-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff5c6771c7e3511a06555afa317879b7db8d640137ba55d6ab0d0c50425cab75"}, + {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b29beab10211a746f9846baa39275e80034e065460d99eb51e45c9a9495bcca"}, + {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a52a1ffdd824fb1835272e125385c32fd8b17fbdefeedcb4d543cc23b332d74"}, + {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:58c8e9620eb82a189c6c40cb6b59b4e35b2ee68b1f2afa6597732a2b467d7e8f"}, + {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d216e5d9b8749563c7f2c6f7a0831057ec844c68b4c11cb10fc62d4fd373c26d"}, + {file = "yarl-1.17.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:881764d610e3269964fc4bb3c19bb6fce55422828e152b885609ec176b41cf11"}, + {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8c79e9d7e3d8a32d4824250a9c6401194fb4c2ad9a0cec8f6a96e09a582c2cc0"}, + {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:299f11b44d8d3a588234adbe01112126010bd96d9139c3ba7b3badd9829261c3"}, + {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:cc7d768260f4ba4ea01741c1b5fe3d3a6c70eb91c87f4c8761bbcce5181beafe"}, + {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:de599af166970d6a61accde358ec9ded821234cbbc8c6413acfec06056b8e860"}, + {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2b24ec55fad43e476905eceaf14f41f6478780b870eda5d08b4d6de9a60b65b4"}, + {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9fb815155aac6bfa8d86184079652c9715c812d506b22cfa369196ef4e99d1b4"}, + {file = "yarl-1.17.1-cp311-cp311-win32.whl", hash = "sha256:7615058aabad54416ddac99ade09a5510cf77039a3b903e94e8922f25ed203d7"}, + {file = "yarl-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:14bc88baa44e1f84164a392827b5defb4fa8e56b93fecac3d15315e7c8e5d8b3"}, + {file = "yarl-1.17.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:327828786da2006085a4d1feb2594de6f6d26f8af48b81eb1ae950c788d97f61"}, + {file = "yarl-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cc353841428d56b683a123a813e6a686e07026d6b1c5757970a877195f880c2d"}, + {file = "yarl-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c73df5b6e8fabe2ddb74876fb82d9dd44cbace0ca12e8861ce9155ad3c886139"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdff5e0995522706c53078f531fb586f56de9c4c81c243865dd5c66c132c3b5"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:06157fb3c58f2736a5e47c8fcbe1afc8b5de6fb28b14d25574af9e62150fcaac"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1654ec814b18be1af2c857aa9000de7a601400bd4c9ca24629b18486c2e35463"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f6595c852ca544aaeeb32d357e62c9c780eac69dcd34e40cae7b55bc4fb1147"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:459e81c2fb920b5f5df744262d1498ec2c8081acdcfe18181da44c50f51312f7"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e48cdb8226644e2fbd0bdb0a0f87906a3db07087f4de77a1b1b1ccfd9e93685"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d9b6b28a57feb51605d6ae5e61a9044a31742db557a3b851a74c13bc61de5172"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e594b22688d5747b06e957f1ef822060cb5cb35b493066e33ceac0cf882188b7"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5f236cb5999ccd23a0ab1bd219cfe0ee3e1c1b65aaf6dd3320e972f7ec3a39da"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a2a64e62c7a0edd07c1c917b0586655f3362d2c2d37d474db1a509efb96fea1c"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d0eea830b591dbc68e030c86a9569826145df485b2b4554874b07fea1275a199"}, + {file = "yarl-1.17.1-cp312-cp312-win32.whl", hash = "sha256:46ddf6e0b975cd680eb83318aa1d321cb2bf8d288d50f1754526230fcf59ba96"}, + {file = "yarl-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:117ed8b3732528a1e41af3aa6d4e08483c2f0f2e3d3d7dca7cf538b3516d93df"}, + {file = "yarl-1.17.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5d1d42556b063d579cae59e37a38c61f4402b47d70c29f0ef15cee1acaa64488"}, + {file = "yarl-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c0167540094838ee9093ef6cc2c69d0074bbf84a432b4995835e8e5a0d984374"}, + {file = "yarl-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2f0a6423295a0d282d00e8701fe763eeefba8037e984ad5de44aa349002562ac"}, + {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5b078134f48552c4d9527db2f7da0b5359abd49393cdf9794017baec7506170"}, + {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d401f07261dc5aa36c2e4efc308548f6ae943bfff20fcadb0a07517a26b196d8"}, + {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5f1ac7359e17efe0b6e5fec21de34145caef22b260e978336f325d5c84e6938"}, + {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f63d176a81555984e91f2c84c2a574a61cab7111cc907e176f0f01538e9ff6e"}, + {file = "yarl-1.17.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e275792097c9f7e80741c36de3b61917aebecc08a67ae62899b074566ff8556"}, + {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:81713b70bea5c1386dc2f32a8f0dab4148a2928c7495c808c541ee0aae614d67"}, + {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:aa46dce75078fceaf7cecac5817422febb4355fbdda440db55206e3bd288cfb8"}, + {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1ce36ded585f45b1e9bb36d0ae94765c6608b43bd2e7f5f88079f7a85c61a4d3"}, + {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2d374d70fdc36f5863b84e54775452f68639bc862918602d028f89310a034ab0"}, + {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2d9f0606baaec5dd54cb99667fcf85183a7477f3766fbddbe3f385e7fc253299"}, + {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b0341e6d9a0c0e3cdc65857ef518bb05b410dbd70d749a0d33ac0f39e81a4258"}, + {file = "yarl-1.17.1-cp313-cp313-win32.whl", hash = "sha256:2e7ba4c9377e48fb7b20dedbd473cbcbc13e72e1826917c185157a137dac9df2"}, + {file = "yarl-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:949681f68e0e3c25377462be4b658500e85ca24323d9619fdc41f68d46a1ffda"}, + {file = "yarl-1.17.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8994b29c462de9a8fce2d591028b986dbbe1b32f3ad600b2d3e1c482c93abad6"}, + {file = "yarl-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f9cbfbc5faca235fbdf531b93aa0f9f005ec7d267d9d738761a4d42b744ea159"}, + {file = "yarl-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b40d1bf6e6f74f7c0a567a9e5e778bbd4699d1d3d2c0fe46f4b717eef9e96b95"}, + {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5efe0661b9fcd6246f27957f6ae1c0eb29bc60552820f01e970b4996e016004"}, + {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5c4804e4039f487e942c13381e6c27b4b4e66066d94ef1fae3f6ba8b953f383"}, + {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5d6a6c9602fd4598fa07e0389e19fe199ae96449008d8304bf5d47cb745462e"}, + {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f4c9156c4d1eb490fe374fb294deeb7bc7eaccda50e23775b2354b6a6739934"}, + {file = "yarl-1.17.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6324274b4e0e2fa1b3eccb25997b1c9ed134ff61d296448ab8269f5ac068c4c"}, + {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d8a8b74d843c2638f3864a17d97a4acda58e40d3e44b6303b8cc3d3c44ae2d29"}, + {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:7fac95714b09da9278a0b52e492466f773cfe37651cf467a83a1b659be24bf71"}, + {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:c180ac742a083e109c1a18151f4dd8675f32679985a1c750d2ff806796165b55"}, + {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:578d00c9b7fccfa1745a44f4eddfdc99d723d157dad26764538fbdda37209857"}, + {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:1a3b91c44efa29e6c8ef8a9a2b583347998e2ba52c5d8280dbd5919c02dfc3b5"}, + {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a7ac5b4984c468ce4f4a553df281450df0a34aefae02e58d77a0847be8d1e11f"}, + {file = "yarl-1.17.1-cp39-cp39-win32.whl", hash = "sha256:7294e38f9aa2e9f05f765b28ffdc5d81378508ce6dadbe93f6d464a8c9594473"}, + {file = "yarl-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:eb6dce402734575e1a8cc0bb1509afca508a400a57ce13d306ea2c663bad1138"}, + {file = "yarl-1.17.1-py3-none-any.whl", hash = "sha256:f1790a4b1e8e8e028c391175433b9c8122c39b46e1663228158e61e6f915bf06"}, + {file = "yarl-1.17.1.tar.gz", hash = "sha256:067a63fcfda82da6b198fa73079b1ca40b7c9b7994995b6ee38acda728b64d47"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" +propcache = ">=0.2.0" + +[[package]] +name = "zipp" +version = "3.20.2" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, + {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "891d55f4c0fd00c315fb2a7f25dfcbfbe0991a9861b1aa66f5378eb82f614063" diff --git a/openbb_platform/extensions/websockets/pyproject.toml b/openbb_platform/extensions/websockets/pyproject.toml new file mode 100644 index 000000000000..62f5ca59fe06 --- /dev/null +++ b/openbb_platform/extensions/websockets/pyproject.toml @@ -0,0 +1,19 @@ +[tool.poetry] +name = "openbb-websockets" +version = "1.0.0b" +description = "Websockets extension for OpenBB" +authors = ["OpenBB Team "] +license = "AGPL-3.0-only" +readme = "README.md" +packages = [{ include = "openbb_websockets" }] + +[tool.poetry.dependencies] +python = "^3.9" +openbb-core = "^1.3.5" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.plugins."openbb_core_extension"] +websockets = "openbb_websockets.websockets_router:router" diff --git a/openbb_platform/providers/fmp/openbb_fmp/__init__.py b/openbb_platform/providers/fmp/openbb_fmp/__init__.py index 3d85d3ed4509..c2531817f93d 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/__init__.py +++ b/openbb_platform/providers/fmp/openbb_fmp/__init__.py @@ -63,6 +63,7 @@ from openbb_fmp.models.risk_premium import FMPRiskPremiumFetcher from openbb_fmp.models.share_statistics import FMPShareStatisticsFetcher from openbb_fmp.models.treasury_rates import FMPTreasuryRatesFetcher +from openbb_fmp.models.websocket_connection import FMPWebSocketFetcher from openbb_fmp.models.world_news import FMPWorldNewsFetcher from openbb_fmp.models.yield_curve import FMPYieldCurveFetcher @@ -137,6 +138,7 @@ "TreasuryRates": FMPTreasuryRatesFetcher, "WorldNews": FMPWorldNewsFetcher, "EtfHistorical": FMPEquityHistoricalFetcher, + "WebSocketConnection": FMPWebSocketFetcher, "YieldCurve": FMPYieldCurveFetcher, }, repr_name="Financial Modeling Prep (FMP)", diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py new file mode 100644 index 000000000000..193990d280c8 --- /dev/null +++ b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py @@ -0,0 +1,198 @@ +"""FMP WebSocket model.""" + +from datetime import datetime +from typing import Any, Literal, Optional + +from openbb_core.app.model.abstract.error import OpenBBError +from openbb_core.provider.abstract.fetcher import Fetcher +from openbb_websockets.client import WebSocketClient +from openbb_websockets.models import ( + WebSocketConnection, + WebSocketData, + WebSocketQueryParams, +) +from pydantic import Field, field_validator + +URL_MAP = { + "stock": "wss://websockets.financialmodelingprep.com", + "fx": "wss://forex.financialmodelingprep.com", + "crypto": "wss://crypto.financialmodelingprep.com", +} + + +class FmpWebSocketQueryParams(WebSocketQueryParams): + """FMP WebSocket query parameters.""" + + __json_schema_extra__ = { + "symbol": {"multiple_items_allowed": True}, + "asset_type": { + "multiple_items_allowed": False, + "choices": ["stock", "fx", "crypto"], + }, + } + + symbol: str = Field( + description="The symbol(s) of the asset to fetch data for.", + ) + asset_type: Literal["stock", "fx", "crypto"] = Field( + default="crypto", + description="The asset type, required for the provider URI.", + ) + + +class FmpWebSocketData(WebSocketData): + """FMP WebSocket data model.""" + + __alias_dict__ = { + "symbol": "s", + "date": "t", + "exchange": "e", + "type": "type", + "bid_size": "bs", + "bid_price": "bp", + "ask_size": "as", + "ask_price": "ap", + "last_price": "lp", + "last_size": "ls", + } + + symbol: str = Field( + description="The symbol of the asset.", + ) + date: datetime = Field( + description="The datetime of the data.", + ) + exchange: Optional[str] = Field( + default=None, + description="The exchange of the data.", + ) + type: Literal["quote", "trade", "break"] = Field( + description="The type of data.", + ) + bid_price: Optional[float] = Field( + default=None, + description="The price of the bid.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + bid_size: Optional[float] = Field( + default=None, + description="The size of the bid.", + ) + ask_size: Optional[float] = Field( + default=None, + description="The size of the ask.", + ) + ask_price: Optional[float] = Field( + default=None, + description="The price of the ask.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + last_price: Optional[float] = Field( + default=None, + description="The last trade price.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + last_size: Optional[float] = Field( + default=None, + description="The size of the trade.", + ) + + @field_validator("type", mode="before", check_fields=False) + def _valiidate_data_type(cls, v): + """Validate the data type.""" + return ( + "quote" if v == "Q" else "trade" if v == "T" else "break" if v == "B" else v + ) + + @field_validator("date", mode="before", check_fields=False) + def _validate_date(cls, v): + """Validate the date.""" + # pylint: disable=import-outside-toplevel + from pytz import timezone + + if isinstance(v, str): + return datetime.fromisoformat(v) + try: + return datetime.fromtimestamp(v / 1000) + except Exception: + if isinstance(v, (int, float)): + # Check if the timestamp is in nanoseconds and convert to seconds + if v > 1e12: + v = v / 1e9 # Convert nanoseconds to seconds + dt = datetime.fromtimestamp(v) + dt = timezone("America/New_York").localize(dt) + return dt + return v + + +class FmpWebSocketConnection(WebSocketConnection): + """FMP WebSocket connection model.""" + + +class FMPWebSocketFetcher(Fetcher[FmpWebSocketQueryParams, FmpWebSocketConnection]): + """FMP WebSocket model.""" + + @staticmethod + def transform_query(params: dict[str, Any]) -> FmpWebSocketQueryParams: + """Transform the query parameters.""" + return FmpWebSocketQueryParams(**params) + + @staticmethod + def extract_data( + query: FmpWebSocketQueryParams, + credentials: Optional[dict[str, str]], + **kwargs: Any, + ) -> WebSocketClient: + """Extract data from the WebSocket.""" + # pylint: disable=import-outside-toplevel + import time + + api_key = credentials.get("fmp_api_key") if credentials else "" + url = URL_MAP[query.asset_type] + + symbol = query.symbol.lower() + + kwargs = { + "url": url, + "api_key": api_key, + } + + client = WebSocketClient( + name=query.name, + module="openbb_fmp.utils.websocket_client", + symbol=symbol, + limit=query.limit, + results_file=query.results_file, + table_name=query.table_name, + save_results=query.save_results, + data_model=FmpWebSocketData, + sleep_time=query.sleep_time, + broadcast_host=query.broadcast_host, + broadcast_port=query.broadcast_port, + auth_token=query.auth_token, + **kwargs, + ) + + try: + client.connect() + + except Exception as e: # pylint: disable=broad-except + client.disconnect() + raise OpenBBError(e) from e + + time.sleep(1) + + if client.is_running: + return client + + client.disconnect() + raise OpenBBError("Failed to connect to the WebSocket.") + + @staticmethod + def transform_data( + data: WebSocketClient, + query: FmpWebSocketQueryParams, + **kwargs: Any, + ) -> FmpWebSocketConnection: + """Return the client as an instance of Data.""" + return FmpWebSocketConnection(client=data) diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py new file mode 100644 index 000000000000..125dc076a365 --- /dev/null +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -0,0 +1,190 @@ +"""FMP WebSocket server.""" + +import asyncio +import json +import os +import signal +import sys + +import websockets +import websockets.exceptions +from openbb_fmp.models.websocket_connection import FmpWebSocketData +from openbb_websockets.helpers import ( + MessageQueue, + get_logger, + handle_termination_signal, + parse_kwargs, + write_to_db, +) + +queue = MessageQueue() +command_queue = MessageQueue() + + +# Create a logger instance. +logger = get_logger("openbb.websocket.fmp") +kwargs = parse_kwargs() + + +async def login(websocket, api_key): + login_event = { + "event": "login", + "data": { + "apiKey": api_key, + }, + } + try: + await websocket.send(json.dumps(login_event)) + response = await websocket.recv() + if json.loads(response).get("message") == "Unauthorized": + logger.error( + "PROVIDER ERROR: Account not authorized." + " Please check that the API key is entered correctly and is entitled to access." + ) + sys.exit(1) + msg = json.loads(response).get("message") + logger.info("PROVIDER INFO: %s", msg) + except Exception as e: + logger.error("PROVIDER ERROR: %s", e.args[0]) + sys.exit(1) + + +async def subscribe(websocket, symbol, event): + """Subscribe or unsubscribe to a symbol.""" + ticker = symbol.split(",") if isinstance(symbol, str) else symbol + subscribe_event = { + "event": event, + "data": { + "ticker": ticker, + }, + } + try: + await websocket.send(json.dumps(subscribe_event)) + except Exception as e: + msg = f"PROVIDER ERROR: {e}" + logger.error(msg) + + +async def read_stdin_and_queue_commands(): + """Read from stdin and queue commands.""" + while True: + line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline) + sys.stdin.flush() + + if not line: + break + + try: + command = json.loads(line.strip()) + await command_queue.enqueue(command) + except json.JSONDecodeError: + logger.error("Invalid JSON received from stdin") + + +async def process_message(message, results_path, table_name, limit): + result = {} + message = json.loads(message) + if message.get("event") != "heartbeat": + if message.get("event") in ["login", "subscribe", "unsubscribe"]: + msg = f"PROVIDER INFO: {message.get("message")}" + logger.info(msg) + return None + try: + result = FmpWebSocketData.model_validate(message).model_dump_json( + exclude_none=True, exclude_unset=True + ) + except Exception as e: + msg = f"PROVIDER ERROR: Error validating data: {e}" + logger.error(msg) + return None + if result: + await write_to_db(result, results_path, table_name, limit) + return + + +async def connect_and_stream(url, symbol, api_key, results_path, table_name, limit): + """Connect to the WebSocket and stream data to file.""" + + handler_task = asyncio.create_task( + queue.process_queue( + lambda message: process_message(message, results_path, table_name, limit) + ) + ) + + stdin_task = asyncio.create_task(read_stdin_and_queue_commands()) + + try: + async with websockets.connect(url) as websocket: + await login(websocket, api_key) + await subscribe(websocket, symbol, "subscribe") + + while True: + ws_task = asyncio.create_task(websocket.recv()) + cmd_task = asyncio.create_task(command_queue.dequeue()) + + done, pending = await asyncio.wait( + [ws_task, cmd_task], return_when=asyncio.FIRST_COMPLETED + ) + for task in pending: + task.cancel() + + for task in done: + if task == ws_task: + message = task.result() + await queue.enqueue(message) + elif task == cmd_task: + command = task.result() + symbol = command.get("symbol") + event = command.get("event") + if symbol and event: + await subscribe(websocket, symbol, event) + + except websockets.ConnectionClosed: + logger.info("PROVIDER INFO: The WebSocket connection was closed.") + + except websockets.WebSocketException as e: + logger.error(e) + sys.exit(1) + + except Exception as e: + msg = f"PROVIDER ERROR: Unexpected error -> {e}" + logger.error(msg) + sys.exit(1) + + finally: + handler_task.cancel() + stdin_task.cancel() + await asyncio.gather(handler_task, stdin_task, return_exceptions=True) + sys.exit(0) + + +if __name__ == "__main__": + try: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + for sig in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(sig, handle_termination_signal, logger) + + asyncio.run_coroutine_threadsafe( + connect_and_stream( + kwargs["url"], + kwargs["symbol"], + kwargs["api_key"], + os.path.abspath(kwargs["results_file"]), + kwargs["table_name"], + kwargs.get("limit", None), + ), + loop, + ) + loop.run_forever() + + except (KeyboardInterrupt, websockets.ConnectionClosed): + logger.error("PROVIDER ERROR: WebSocket connection closed") + + except Exception as e: # pylint: disable=broad-except + msg = f"PROVIDER ERROR: {e.args[0]}" + logger.error(msg) + + finally: + sys.exit(0) diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/__init__.py b/openbb_platform/providers/tiingo/openbb_tiingo/__init__.py index 029f57a13c47..9df73581ca96 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/__init__.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/__init__.py @@ -6,6 +6,7 @@ from openbb_tiingo.models.currency_historical import TiingoCurrencyHistoricalFetcher from openbb_tiingo.models.equity_historical import TiingoEquityHistoricalFetcher from openbb_tiingo.models.trailing_dividend_yield import TiingoTrailingDivYieldFetcher +from openbb_tiingo.models.websocket_connection import TiingoWebSocketFetcher from openbb_tiingo.models.world_news import TiingoWorldNewsFetcher tiingo_provider = Provider( @@ -22,6 +23,7 @@ "CryptoHistorical": TiingoCryptoHistoricalFetcher, "CurrencyHistorical": TiingoCurrencyHistoricalFetcher, "TrailingDividendYield": TiingoTrailingDivYieldFetcher, + "WebSocketConnection": TiingoWebSocketFetcher, }, repr_name="Tiingo", ) diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py new file mode 100644 index 000000000000..7291d1cd24ce --- /dev/null +++ b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py @@ -0,0 +1,288 @@ +"""Tiingo WebSocket model.""" + +from datetime import datetime +from typing import Any, Literal, Optional + +from openbb_core.app.model.abstract.error import OpenBBError +from openbb_core.provider.abstract.fetcher import Fetcher +from openbb_core.provider.utils.descriptions import ( + QUERY_DESCRIPTIONS, +) +from openbb_websockets.client import WebSocketClient +from openbb_websockets.models import ( + WebSocketConnection, + WebSocketData, + WebSocketQueryParams, +) +from pydantic import Field, field_validator + +URL_MAP = { + "stock": "wss://api.tiingo.com/iex", + "fx": "wss://api.tiingo.com/fx", + "crypto": "wss://api.tiingo.com/crypto", +} + +# These are the data array order of definitions. +IEX_FIELDS = [ + "type", + "date", + "timestamp", + "symbol", + "bid_size", + "bid_price", + "mid_price", + "ask_price", + "ask_size", + "last_price", + "last_size", + "halted", + "after_hours", + "sweep_order", + "oddlot", + "nms_rule", +] +FX_FIELDS = [ + "type", + "symbol", + "date", + "bid_size", + "bid_price", + "mid_price", + "ask_price", + "ask_size", + "ask_price", +] +CRYPTO_TRADE_FIELDS = [ + "type", + "symbol", + "date", + "exchange", + "last_size", + "last_price", +] +CRYPTO_QUOTE_FIELDS = [ + "type", + "symbol", + "date", + "exchange", + "bid_size", + "bid_price", + "mid_price", + "ask_size", + "ask_price", +] + + +class TiingoWebSocketQueryParams(WebSocketQueryParams): + """Tiingo WebSocket query parameters.""" + + __json_schema_extra__ = { + "symbol": {"multiple_items_allowed": True}, + "asset_type": { + "multiple_items_allowed": False, + "choices": ["stock", "fx", "crypto"], + }, + "feed": { + "multiple_items_allowed": False, + "choices": ["trade", "trade_and_quote"], + }, + } + + symbol: str = Field( + description=QUERY_DESCRIPTIONS.get("symbol", "") + "Use '*' for all symbols.", + ) + asset_type: Literal["stock", "fx", "crypto"] = Field( + default="crypto", + description="The asset type for the feed.", + ) + feed: Literal["trade", "trade_and_quote"] = Field( + default="trade_and_quote", + description="The type of data feed to subscribe to. FX only supports quote.", + ) + + +class TiingoWebSocketData(WebSocketData): + """Tiingo WebSocket data model.""" + + timestamp: Optional[int] = Field( + default=None, + description="Nanoseconds since POSIX time UTC.", + ) + type: Literal["quote", "trade", "break"] = Field( + description="The type of data.", + ) + exchange: Optional[str] = Field( + default=None, + description="The exchange of the data. Only for crypto.", + ) + bid_size: Optional[float] = Field( + default=None, + description="The size of the bid.", + ) + bid_price: Optional[float] = Field( + default=None, + description="The price of the bid.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + mid_price: Optional[float] = Field( + default=None, + description="The mid price.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + ask_price: Optional[float] = Field( + default=None, + description="The price of the ask.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + ask_size: Optional[float] = Field( + default=None, + description="The size of the ask.", + ) + last_price: Optional[float] = Field( + default=None, + description="The last trade price.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + last_size: Optional[float] = Field( + default=None, + description="The size of the trade.", + ) + halted: Optional[bool] = Field( + default=None, + description="If the asset is halted. Only for stock.", + ) + after_hours: Optional[bool] = Field( + default=None, + description="If the data is after hours. Only for stock.", + ) + sweep_order: Optional[bool] = Field( + default=None, + description="If the order is an intermarket sweep order. Only for stock.", + ) + oddlot: Optional[bool] = Field( + default=None, + description="If the order is an oddlot. Only for stock.", + ) + nms_rule: Optional[bool] = Field( + default=None, + description="True if the order is not subject to NMS Rule 611. Only for stock.", + ) + + @field_validator("type", mode="before", check_fields=False) + def _valiidate_data_type(cls, v): + """Validate the data type.""" + return ( + "quote" if v == "Q" else "trade" if v == "T" else "break" if v == "B" else v + ) + + @field_validator("date", mode="before", check_fields=False) + def _validate_date(cls, v): + """Validate the date.""" + # pylint: disable=import-outside-toplevel + from pytz import timezone + + if isinstance(v, str): + return datetime.fromisoformat(v) + try: + return datetime.fromtimestamp(v / 1000) + except Exception: + if isinstance(v, (int, float)): + # Check if the timestamp is in nanoseconds and convert to seconds + if v > 1e12: + v = v / 1e9 # Convert nanoseconds to seconds + dt = datetime.fromtimestamp(v) + dt = timezone("America/New_York").localize(dt) + return dt + return v + + +class TiingoWebSocketConnection(WebSocketConnection): + """Tiingo WebSocket connection model.""" + + +class TiingoWebSocketFetcher( + Fetcher[TiingoWebSocketQueryParams, TiingoWebSocketConnection] +): + """Tiingo WebSocket model.""" + + @staticmethod + def transform_query(params: dict[str, Any]) -> TiingoWebSocketQueryParams: + """Transform the query parameters.""" + asset_type = params.get("asset_type") + feed = params.get("feed") + + if asset_type == "fx" and feed == "trade": + raise ValueError("FX only supports quote feed.") + + return TiingoWebSocketQueryParams(**params) + + @staticmethod + async def aextract_data( + query: TiingoWebSocketQueryParams, + credentials: Optional[dict[str, str]], + **kwargs: Any, + ) -> WebSocketClient: + """Initiailze the WebSocketClient and connect.""" + # pylint: disable=import-outside-toplevel + from asyncio import sleep + + api_key = credentials.get("tiingo_token") if credentials else "" + url = URL_MAP[query.asset_type] + threshold_level = ( + 5 + if query.asset_type == "fx" or query.feed == "trade" + else ( + 2 + if query.asset_type == "crypto" and query.feed == "trade_and_quote" + else 0 + ) + ) + + symbol = query.symbol.lower() + + kwargs = { + "url": url, + "api_key": api_key, + "threshold_level": threshold_level, + } + + client = WebSocketClient( + name=query.name, + module="openbb_tiingo.utils.websocket_client", + symbol=symbol.lower(), + limit=query.limit, + results_file=query.results_file, + table_name=query.table_name, + save_results=query.save_results, + data_model=TiingoWebSocketData, + sleep_time=query.sleep_time, + broadcast_host=query.broadcast_host, + broadcast_port=query.broadcast_port, + auth_token=query.auth_token, + **kwargs, + ) + + try: + client.connect() + # Unhandled exceptions are caught and raised as OpenBBError + except Exception as e: # pylint: disable=broad-except + client.disconnect() + raise OpenBBError(e) from e + + # Wait for the connection to be established before returning. + await sleep(2) + + if client.is_running: + return client + + client.disconnect() + raise OpenBBError("Failed to connect to the WebSocket.") + + @staticmethod + def transform_data( + data: WebSocketClient, + query: TiingoWebSocketQueryParams, + **kwargs: Any, + ) -> TiingoWebSocketConnection: + """Return the client as an instance of Data.""" + return TiingoWebSocketConnection(client=data) diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py new file mode 100644 index 000000000000..5c31928890b3 --- /dev/null +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -0,0 +1,278 @@ +"""FMP WebSocket server.""" + +import asyncio +import json +import os +import signal +import sys + +import websockets +from openbb_core.provider.utils.errors import UnauthorizedError +from openbb_tiingo.models.websocket_connection import TiingoWebSocketData +from openbb_websockets.helpers import ( + MessageQueue, + get_logger, + handle_termination_signal, + parse_kwargs, + write_to_db, +) + +# These are the data array definitions. +IEX_FIELDS = [ + "type", + "date", + "timestamp", + "symbol", + "bid_size", + "bid_price", + "mid_price", + "ask_price", + "ask_size", + "last_price", + "last_size", + "halted", + "after_hours", + "sweep_order", + "oddlot", + "nms_rule", +] +FX_FIELDS = [ + "type", + "symbol", + "date", + "bid_size", + "bid_price", + "mid_price", + "ask_price", + "ask_size", + "ask_price", +] +CRYPTO_TRADE_FIELDS = [ + "type", + "symbol", + "date", + "exchange", + "last_size", + "last_price", +] +CRYPTO_QUOTE_FIELDS = [ + "type", + "symbol", + "date", + "exchange", + "bid_size", + "bid_price", + "mid_price", + "ask_size", + "ask_price", +] +subscription_id = None +queue = MessageQueue() +logger = get_logger("openbb.websocket.tiingo") +kwargs = parse_kwargs() + + +# Subscribe and unsubscribe events are handled in a separate connection using the subscription_id set by the login event. +async def update_symbols(symbol, event): + """Update the symbols to subscribe to.""" + url = kwargs["url"] + + if not subscription_id: + logger.error( + "PROVIDER ERROR: Must be assigned a subscription ID to update symbols. Try logging in." + ) + return + + update_event = { + "eventName": event, + "authorization": kwargs["api_key"], + "eventData": { + "subscriptionId": subscription_id, + "tickers": symbol, + }, + } + + async with websockets.connect(url) as websocket: + await websocket.send(json.dumps(update_event)) + response = await websocket.recv() + message = json.loads(response) + if message.get("response", {}).get("code") != 200: + logger.error(f"PROVIDER ERROR: {message}") + else: + msg = ( + f"PROVIDER INFO: {message.get('response', {}).get('message')}. " + f"Subscribed to symbols: {message.get('data', {}).get('tickers')}" + ) + logger.info(msg) + + +async def read_stdin_and_update_symbols(): + """Read from stdin and update symbols.""" + while True: + line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline) + sys.stdin.flush() + + if not line: + break + + line = json.loads(line.strip()) + + if line: + symbol = line.get("symbol") + event = line.get("event") + await update_symbols(symbol, event) + + +async def process_message(message, results_path, table_name, limit): + result = {} + data_message = {} + message = json.loads(message) + msg = "" + if message.get("messageType") == "E": + response = message.get("response", {}) + msg = f"PROVIDER ERROR: {response.get('code')}: {response.get('message')}" + logger.error(msg) + sys.exit(1) + elif message.get("messageType") == "I": + response = message.get("response", {}) + + if response.get("code") != 200: + msg = ( + f"PROVIDER ERROR: {response.get('code')}: {response.get('message')}" + ) + logger.error(msg) + raise UnauthorizedError(msg) + + if response.get("code") == 200: + msg = f"PROVIDER INFO: Authorization: {response.get('message')}" + logger.info(msg) + if message.get("data", {}).get("subscriptionId"): + global subscription_id + + subscription_id = message["data"]["subscriptionId"] + + if "tickers" in message.get("data", {}): + tickers = message["data"]["tickers"] + threshold_level = message["data"].get("thresholdLevel") + msg = f"PROVIDER INFO: Subscribed to {tickers} with threshold level {threshold_level}" + logger.info(msg) + elif message.get("messageType") == "A": + data = message.get("data", []) + service = message.get("service") + if service == "iex": + data_message = {IEX_FIELDS[i]: data[i] for i in range(len(data))} + elif service == "fx": + data_message = {FX_FIELDS[i]: data[i] for i in range(len(data))} + elif service == "crypto_data": + if data[0] == "T": + data_message = { + CRYPTO_TRADE_FIELDS[i]: data[i] for i in range(len(data)) + } + elif data[0] == "Q": + data_message = { + CRYPTO_QUOTE_FIELDS[i]: data[i] for i in range(len(data)) + } + else: + return + + try: + result = TiingoWebSocketData.model_validate(data_message).model_dump_json( + exclude_none=True, exclude_unset=True + ) + except Exception as e: + msg = f"PROVIDER ERROR: Error validating data: {e}" + logger.error(msg) + return + if result: + await write_to_db(result, results_path, table_name, limit) + return + + +async def connect_and_stream( + url, symbol, threshold_level, api_key, results_path, table_name, limit +): + """Connect to the WebSocket and stream data to file.""" + + handler_task = asyncio.create_task( + queue.process_queue( + lambda message: process_message(message, results_path, table_name, limit) + ) + ) + + stdin_task = asyncio.create_task(read_stdin_and_update_symbols()) + + if isinstance(symbol, str): + ticker = symbol.lower().split(",") + + subscribe_event = { + "eventName": "subscribe", + "authorization": api_key, + "eventData": {"thresholdLevel": threshold_level, "tickers": ticker}, + } + try: + async with websockets.connect( + url, ping_interval=20, ping_timeout=20, max_queue=1000 + ) as websocket: + logger.info("PROVIDER INFO: WebSocket connection established.") + await websocket.send(json.dumps(subscribe_event)) + while True: + message = await websocket.recv() + await queue.enqueue(message) + + except websockets.ConnectionClosed as e: + msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e.reason}" + logger.info(msg) + # Attempt to reopen the connection + await asyncio.sleep(5) + await connect_and_stream( + url, symbol, threshold_level, api_key, results_path, table_name, limit + ) + + except websockets.WebSocketException as e: + logger.error(e) + sys.exit(1) + + except Exception as e: + msg = f"PROVIDER ERROR: Unexpected error -> {e}" + logger.error(msg) + sys.exit(1) + + finally: + handler_task.cancel() + await handler_task + stdin_task.cancel() + await stdin_task + sys.exit(0) + + +if __name__ == "__main__": + try: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + for sig in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(sig, handle_termination_signal, logger) + + asyncio.run_coroutine_threadsafe( + connect_and_stream( + kwargs["url"], + kwargs["symbol"], + kwargs["threshold_level"], + kwargs["api_key"], + os.path.abspath(kwargs["results_file"]), + kwargs["table_name"], + kwargs.get("limit", None), + ), + loop, + ) + loop.run_forever() + + except (KeyboardInterrupt, websockets.ConnectionClosed): + logger.error("PROVIDER ERROR: WebSocket connection closed") + + except Exception as e: # pylint: disable=broad-except + msg = f"PROVIDER ERROR: {e.args[0]}" + logger.error(msg) + + finally: + sys.exit(0) diff --git a/openbb_platform/pyproject.toml b/openbb_platform/pyproject.toml index f2102d1d220f..a7666d2b7823 100644 --- a/openbb_platform/pyproject.toml +++ b/openbb_platform/pyproject.toml @@ -61,6 +61,7 @@ openbb-charting = { version = "^2.2.4", optional = true } openbb-econometrics = { version = "^1.4.4", optional = true } openbb-quantitative = { version = "^1.3.4", optional = true } openbb-technical = { version = "^1.3.4", optional = true } +openbb-websockets = { version = "^1.3.4", optional = true } [tool.poetry.extras] alpha_vantage = ["openbb-alpha-vantage"] @@ -80,6 +81,7 @@ stockgrid = ["openbb-stockgrid"] technical = ["openbb-technical"] tmx = ["openbb-tmx"] tradier = ["openbb-tradier"] +websockets = ["openbb-websockets"] wsj = ["openbb-wsj"] @@ -101,6 +103,7 @@ all = [ "openbb-technical", "openbb-tmx", "openbb-tradier", + "openbb-websockets", "openbb-wsj", ] From 70c47668cf282be579270adfa3be1213d7fc00f7 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 7 Nov 2024 17:56:00 -0800 Subject: [PATCH 002/119] setup_db as non-async --- .../websockets/openbb_websockets/client.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 3ddad5092e4c..b3df39c6dcbf 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -170,12 +170,12 @@ def _atexit(self): if os.path.exists(self.results_file): os.remove(self.results_file) - async def _setup_database(self): + def _setup_database(self): """Set up the SQLite database and table.""" # pylint: disable=import-outside-toplevel from openbb_websockets.helpers import setup_database - return await setup_database(self.results_path, self.table_name) + return setup_database(self.results_path, self.table_name) def _log_provider_output(self, output_queue): """Log output from the provider server queue.""" @@ -401,19 +401,13 @@ def results(self): def results(self): """Clear results stored from the WebSocket stream.""" # pylint: disable=import-outside-toplevel - import asyncio import sqlite3 try: with sqlite3.connect(self.results_path) as conn: conn.execute(f"DELETE FROM {self.table_name}") # noqa conn.commit() - - try: - loop = asyncio.get_event_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - loop.run_until_complete(self._setup_database()) + self._setup_database() self.logger.info( "Results cleared from table %s in %s", self.table_name, From 7089dc70a87b5ae291bcbc137b4d408f9d9885bc Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 7 Nov 2024 17:57:51 -0800 Subject: [PATCH 003/119] fix the other reference --- .../extensions/websockets/openbb_websockets/client.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index b3df39c6dcbf..3c81970bcc73 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -145,14 +145,7 @@ def __init__( # noqa: PLR0913 atexit.register(self._atexit) try: - loop = asyncio.get_event_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - try: - if loop.is_running(): - loop.create_task(self._setup_database()) - else: - asyncio.run(self._setup_database()) + self._setup_database() except DatabaseError as e: self.logger.error("Error setting up the SQLite database and table: %s", e) From 291ba508de39d7e20f97e092ea13f866e123d324 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 7 Nov 2024 18:32:13 -0800 Subject: [PATCH 004/119] do it this way instead --- .../websockets/openbb_websockets/client.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 3c81970bcc73..fef517c2c478 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -145,7 +145,14 @@ def __init__( # noqa: PLR0913 atexit.register(self._atexit) try: - self._setup_database() + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + try: + if loop.is_running(): + loop.create_task(self._setup_database()) + else: + asyncio.run(self._setup_database()) except DatabaseError as e: self.logger.error("Error setting up the SQLite database and table: %s", e) @@ -163,12 +170,12 @@ def _atexit(self): if os.path.exists(self.results_file): os.remove(self.results_file) - def _setup_database(self): + async def _setup_database(self): """Set up the SQLite database and table.""" # pylint: disable=import-outside-toplevel from openbb_websockets.helpers import setup_database - return setup_database(self.results_path, self.table_name) + return await setup_database(self.results_path, self.table_name) def _log_provider_output(self, output_queue): """Log output from the provider server queue.""" @@ -394,13 +401,15 @@ def results(self): def results(self): """Clear results stored from the WebSocket stream.""" # pylint: disable=import-outside-toplevel + import asyncio import sqlite3 try: with sqlite3.connect(self.results_path) as conn: conn.execute(f"DELETE FROM {self.table_name}") # noqa conn.commit() - self._setup_database() + + asyncio.create_task(self._setup_database()) self.logger.info( "Results cleared from table %s in %s", self.table_name, From 5a99083e4a94e5435c3f138eb077d1639ecffcc2 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 7 Nov 2024 18:59:08 -0800 Subject: [PATCH 005/119] deleter --- .../websockets/openbb_websockets/client.py | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index fef517c2c478..fa852598a81b 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -146,7 +146,7 @@ def __init__( # noqa: PLR0913 try: loop = asyncio.get_event_loop() - except RuntimeError: + except (RuntimeError, RuntimeWarning): loop = asyncio.new_event_loop() try: if loop.is_running(): @@ -401,15 +401,36 @@ def results(self): def results(self): """Clear results stored from the WebSocket stream.""" # pylint: disable=import-outside-toplevel + import sqlite3 # noqa import asyncio - import sqlite3 + import threading + + def run_in_new_loop(): + """Run setup in new event loop.""" + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(self._setup_database()) + finally: + loop.close() + + def run_in_thread(): + """Run setup in separate thread.""" + thread = threading.Thread(target=run_in_new_loop) + thread.start() + thread.join() try: with sqlite3.connect(self.results_path) as conn: conn.execute(f"DELETE FROM {self.table_name}") # noqa conn.commit() - asyncio.create_task(self._setup_database()) + try: + loop = asyncio.get_running_loop() # noqa + run_in_thread() + except RuntimeError: + run_in_new_loop() + self.logger.info( "Results cleared from table %s in %s", self.table_name, From 0a2264d9bb1bb842ba9acbadb2bb66df399dc0ba Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 7 Nov 2024 19:04:53 -0800 Subject: [PATCH 006/119] lint --- .../extensions/websockets/openbb_websockets/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index fa852598a81b..cc4b79e6ef28 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -112,7 +112,7 @@ def __init__( # noqa: PLR0913 self._auth_token = auth_token self._symbol = symbol self._kwargs = ( - [f"{k}={str(v).strip().replace(" ", "_")}" for k, v in kwargs.items()] + [f"{k}={str(v).strip().replace(' ', '_')}" for k, v in kwargs.items()] if kwargs else None ) From e2294604522213989c4bd4093d2f2bf4bf258b55 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 7 Nov 2024 19:12:03 -0800 Subject: [PATCH 007/119] inner quote as single --- .../providers/fmp/openbb_fmp/utils/websocket_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py index 125dc076a365..6d50c573045d 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -86,7 +86,7 @@ async def process_message(message, results_path, table_name, limit): message = json.loads(message) if message.get("event") != "heartbeat": if message.get("event") in ["login", "subscribe", "unsubscribe"]: - msg = f"PROVIDER INFO: {message.get("message")}" + msg = f"PROVIDER INFO: {message.get('message')}" logger.info(msg) return None try: From f9d00c1d1586a99e56e0d69132d0ab18264d492d Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 9 Nov 2024 00:09:35 -0800 Subject: [PATCH 008/119] add most of polygon and some general updates --- .../websockets/openbb_websockets/client.py | 8 + .../websockets/openbb_websockets/helpers.py | 7 + .../websockets/openbb_websockets/models.py | 29 +- .../fmp/openbb_fmp/utils/websocket_client.py | 7 +- .../polygon/openbb_polygon/__init__.py | 2 + .../models/websocket_connection.py | 870 ++++++++++++++++++ .../polygon/openbb_polygon/utils/constants.py | 186 ++++ .../openbb_polygon/utils/websocket_client.py | 239 +++++ 8 files changed, 1342 insertions(+), 6 deletions(-) create mode 100644 openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py create mode 100644 openbb_platform/providers/polygon/openbb_polygon/utils/constants.py create mode 100644 openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index cc4b79e6ef28..3f042a67b0a3 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -188,6 +188,14 @@ def _log_provider_output(self, output_queue): try: output = output_queue.get(timeout=1) if output: + if ( + "server rejected" in output.lower() + or "PROVIDER ERROR" in output + ): + self._stop_log_thread_event.set() + self._psutil_process.kill() + self.logger.error(output) + break output = clean_message(output) output = output + "\n" sys.stdout.write(output + "\n") diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index beb9bec5dbc4..675e43e2cc61 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -90,6 +90,7 @@ def handle_termination_signal(logger): def parse_kwargs(): """Parse command line keyword arguments.""" # pylint: disable=import-outside-toplevel + import json import sys args = sys.argv[1:].copy() @@ -97,11 +98,17 @@ def parse_kwargs(): for i, arg in enumerate(args): if "=" in arg: key, value = arg.split("=") + + if key == "connect_kwargs": + value = {} if value == "None" else json.loads(value) + _kwargs[key] = value elif arg.startswith("--"): key = arg[2:] + if i + 1 < len(args) and not args[i + 1].startswith("--"): value = args[i + 1] + if isinstance(value, str) and value.lower() in ["false", "true"]: value = value.lower() == "true" elif isinstance(value, str) and value.lower() == "none": diff --git a/openbb_platform/extensions/websockets/openbb_websockets/models.py b/openbb_platform/extensions/websockets/openbb_websockets/models.py index a631ba0cf8f9..e50833f09613 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/models.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/models.py @@ -3,6 +3,7 @@ from datetime import datetime from typing import Any, Optional +from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.abstract.data import Data from openbb_core.provider.abstract.query_params import QueryParams from openbb_core.provider.utils.descriptions import ( @@ -58,11 +59,37 @@ class WebSocketQueryParams(QueryParams): description="Port to bind the broadcast server to.", ) start_broadcast: bool = Field( - default=True, + default=False, description="Whether to start the broadcast server." + " Set to False if system or network conditions do not allow it." + " Can be started manually with the 'start_broadcasting' method.", ) + connect_kwargs: Optional[Any] = Field( + default=None, + description="A formatted dictionary, or serialized JSON string, of keyword arguments to pass" + + " directly to websockets.connect().", + ) + + @field_validator("connect_kwargs", mode="before", check_fields=False) + @classmethod + def _validate_connect_kwargs(cls, v): + """Validate the connect_kwargs format.""" + # pylint: disable=import-outside-toplevel + import json + + if isinstance(v, str): + try: + v = json.loads(v) + except json.JSONDecodeError as e: + raise OpenBBError( + f"Invalid JSON format for 'connect_kwargs': {e}" + ) from e + if v is not None and not isinstance(v, dict): + raise OpenBBError( + "Invalid 'connect_kwargs' format. Must be a dictionary or serialized JSON string." + ) + + return json.dumps(v, separators=(",", ":")) class WebSocketData(Data): diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py index 6d50c573045d..bbcdf57ff871 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -17,13 +17,10 @@ write_to_db, ) -queue = MessageQueue() -command_queue = MessageQueue() - - -# Create a logger instance. logger = get_logger("openbb.websocket.fmp") kwargs = parse_kwargs() +queue = MessageQueue(max_size=kwargs.get("limit", 1000)) +command_queue = MessageQueue() async def login(websocket, api_key): diff --git a/openbb_platform/providers/polygon/openbb_polygon/__init__.py b/openbb_platform/providers/polygon/openbb_polygon/__init__.py index c8de4baf5660..1bad20eb46ca 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/__init__.py +++ b/openbb_platform/providers/polygon/openbb_polygon/__init__.py @@ -15,6 +15,7 @@ PolygonIndexHistoricalFetcher, ) from openbb_polygon.models.market_snapshots import PolygonMarketSnapshotsFetcher +from openbb_polygon.models.websocket_connection import PolygonWebSocketFetcher polygon_provider = Provider( name="polygon", @@ -38,6 +39,7 @@ "IndexHistorical": PolygonIndexHistoricalFetcher, "MarketIndices": PolygonIndexHistoricalFetcher, "MarketSnapshots": PolygonMarketSnapshotsFetcher, + "WebSocketConnection": PolygonWebSocketFetcher, }, repr_name="Polygon.io", deprecated_credentials={"API_POLYGON_KEY": "polygon_api_key"}, diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py new file mode 100644 index 000000000000..2c24d7359d7f --- /dev/null +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -0,0 +1,870 @@ +"""Polygon WebSocket Connection Model.""" + +from datetime import datetime +from typing import Any, Literal, Optional + +from openbb_core.app.model.abstract.error import OpenBBError +from openbb_core.provider.abstract.data import Data +from openbb_core.provider.abstract.fetcher import Fetcher +from openbb_core.provider.utils.descriptions import ( + DATA_DESCRIPTIONS, + QUERY_DESCRIPTIONS, +) +from openbb_polygon.utils.constants import ( + CRYPTO_EXCHANGE_MAP, + FX_EXCHANGE_MAP, + STOCK_EXCHANGE_MAP, + STOCK_QUOTE_CONDITIONS, + STOCK_QUOTE_INDICATORS, + STOCK_TRADE_CONDITIONS, +) +from openbb_polygon.utils.helpers import map_tape +from openbb_websockets.client import WebSocketClient +from openbb_websockets.models import ( + WebSocketConnection, + WebSocketData, + WebSocketQueryParams, +) +from pydantic import ConfigDict, Field, field_validator, model_validator + +URL_MAP = { + "stock": "wss://socket.polygon.io/stocks", + "stock_delayed": "wss://delayed.polygon.io/stocks", + "fx": "wss://socket.polygon.io/forex", + "crypto": "wss://socket.polygon.io/crypto", +} + +ASSET_CHOICES = ["stock", "stock_delayed", "fx", "crypto"] + +FEED_MAP = { + "crypto": { + "aggs_min": "XA", + "aggs_sec": "XAS", + "trade": "XT", + "quote": "XQ", + "l2": "XL2", + "fmv": "FMV", + }, + "fx": { + "aggs_min": "CA", + "aggs_sec": "CAS", + "quote": "C", + "fmv": "FMV", + }, + "stock": { + "aggs_min": "AM", + "aggs_sec": "AS", + "trade": "T", + "quote": "Q", + "fmv": "FMV", + }, + "stock_delayed": { + "aggs_min": "AM", + "aggs_sec": "AS", + "trade": "T", + "quote": "Q", + "fmv": "FMV", + }, +} + + +def validate_date(cls, v): + """Validate the date.""" + # pylint: disable=import-outside-toplevel + from pytz import timezone + + try: + dt = datetime.utcfromtimestamp(v / 1000).replace(tzinfo=timezone("UTC")) + dt = dt.astimezone(timezone("America/New_York")) + return dt + except Exception: + if isinstance(v, (int, float)): + # Check if the timestamp is in nanoseconds and convert to seconds + if v > 1e12: + v = v / 1e9 # Convert nanoseconds to seconds + dt = datetime.fromtimestamp(v, tz=timezone("UTC")) + dt = dt.astimezone(timezone("America/New_York")) + return dt + + +class PolygonWebSocketQueryParams(WebSocketQueryParams): + """Polygon WebSocket query parameters.""" + + __json_schema_extra__ = { + "symbol": {"multiple_items_allowed": True}, + "asset_type": { + "multiple_items_allowed": False, + "choices": ASSET_CHOICES, + }, + } + + symbol: str = Field( + description=QUERY_DESCRIPTIONS.get("symbol", ""), + ) + asset_type: Literal["stock", "stock_delayed", "fx", "crypto"] = Field( + default="crypto", + description="The asset type associated with the symbol(s)." + + " Choose from: stock, stock_delayed, fx, crypto.", + ) + feed: Literal["aggs_min", "aggs_sec", "trade", "quote", "l2"] = Field( + default="aggs_sec", + description="The feed type to subscribe to. Choose from: aggs_min, aggs_sec, trade, quote, l2." + + "l2 is only available for crypto.", + ) + + @model_validator(mode="before") + @classmethod + def _validate_feed(cls, values): + """Validate the feed.""" + feed = values.get("feed") + asset_type = values.get("asset_type") + if asset_type == "fx" and feed in ["trade", "l2"]: + raise ValueError("FX does not support the trade or l2 feeds.") + if asset_type in ["stock", "stock_delayed"] and feed == "l2": + raise ValueError("Stock does not support the l2 feed.") + if asset_type == "index" and feed in ["trade", "quote", "l2", "fmv"]: + raise ValueError( + "Index does not support the trade, quote, l2, or fmv feeds." + ) + + return values + + +class PolygonCryptoAggsWebSocketData(WebSocketData): + """Polygon Crypto Aggregates WebSocket data model.""" + + __alias_dict__ = { + "type": "ev", + "symbol": "pair", + "date": "e", + "open": "o", + "high": "h", + "low": "l", + "close": "c", + "vwap": "vw", + "volume": "v", + "avg_size": "z", + } + + type: str = Field( + description="The type of data.", + ) + date: datetime = Field( + description="The start of the aggregate window.", + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + open: float = Field( + description=DATA_DESCRIPTIONS.get("open", ""), + json_schema_extra={"x-unit_measurement": "currency"}, + ) + high: float = Field( + description=DATA_DESCRIPTIONS.get("high", ""), + json_schema_extra={"x-unit_measurement": "currency"}, + ) + low: float = Field( + description=DATA_DESCRIPTIONS.get("low", ""), + json_schema_extra={"x-unit_measurement": "currency"}, + ) + close: float = Field( + description=DATA_DESCRIPTIONS.get("close", ""), + json_schema_extra={"x-unit_measurement": "currency"}, + ) + vwap: float = Field( + description=DATA_DESCRIPTIONS.get("vwap", ""), + ) + volume: float = Field( + description=DATA_DESCRIPTIONS.get("volume", ""), + ) + avg_size: Optional[float] = Field( + default=None, + description="The average trade size for the aggregate window.", + ) + + @field_validator("date", mode="before", check_fields=False) + @classmethod + def _validate_date(cls, v): + """Validate the date.""" + return validate_date(cls, v) + + @model_validator(mode="before") + @classmethod + def _validate_model(cls, values): + """Validate the model.""" + _ = values.pop("s", None) + if values.get("z") and values["z"] == 0 or not values.get("z"): + _ = values.pop("z", None) + return values + + +class PolygonCryptoTradeWebSocketData(WebSocketData): + """Polygon Crypto WebSocket data model.""" + + __alias_dict__ = { + "type": "ev", + "symbol": "pair", + "date": "t", + "exchange": "x", + "price": "p", + "size": "s", + "conditions": "c", + "received_at": "r", + } + + type: str = Field( + description="The type of data.", + ) + date: datetime = Field( + description=DATA_DESCRIPTIONS.get("date", ""), + ) + received_at: datetime = Field( + description="The time the data was received by Polygon.", + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + exchange: str = Field( + default=None, + description="The exchange of the data.", + ) + conditions: Optional[str] = Field( + default=None, + description="The conditions of the trade. Either sellside or buyside, if available.", + ) + price: float = Field( + description="The price of the trade.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + size: float = Field( + description="The size of the trade.", + ) + + @field_validator("conditions", mode="before", check_fields=False) + @classmethod + def _validate_conditions(cls, v): + """Validate the conditions.""" + if v is None or isinstance(v, list) and v[0] == 0: + return None + elif isinstance(v, list) and v[0] == 1: + return "sellside" + elif isinstance(v, list) and v[0] == 2: + return "buyside" + return str(v) + + @field_validator("date", "received_at", mode="before", check_fields=False) + @classmethod + def _validate_date(cls, v): + """Validate the date.""" + return validate_date(cls, v) + + @field_validator("exchange", mode="before", check_fields=False) + @classmethod + def _validate_exchange(cls, v): + """Validate the exchange.""" + return CRYPTO_EXCHANGE_MAP.get(v, str(v)) + + @model_validator(mode="before") + def _validate_model(cls, values): + """Validate the model.""" + _ = values.pop("i", None) + return values + + +class PolygonCryptoQuoteWebSocketData(WebSocketData): + """Polygon Crypto Quote WebSocket data model.""" + + __alias_dict__ = { + "type": "ev", + "symbol": "pair", + "date": "t", + "exchange": "x", + "bid": "bp", + "bid_size": "bs", + "ask": "ap", + "ask_size": "as", + "received_at": "r", + } + + type: str = Field( + description="The type of data.", + ) + date: datetime = Field( + description=DATA_DESCRIPTIONS.get("date", ""), + ) + received_at: datetime = Field( + description="The time the data was received by Polygon.", + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + exchange: str = Field( + default=None, + description="The exchange of the data.", + ) + bid_size: float = Field( + description="The size of the bid.", + ) + bid: float = Field( + description="The bid price.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + ask: float = Field( + description="The ask price.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + ask_size: float = Field( + description="The size of the ask.", + ) + + @field_validator("date", "received_at", mode="before", check_fields=False) + @classmethod + def _validate_date(cls, v): + """Validate the date.""" + return validate_date(cls, v) + + @field_validator("exchange", mode="before", check_fields=False) + @classmethod + def _validate_exchange(cls, v): + """Validate the exchange.""" + return CRYPTO_EXCHANGE_MAP.get(v, str(v)) + + @model_validator(mode="before") + @classmethod + def _validate_model(cls, values): + """Validate the model.""" + lp = values.pop("lp", None) + ls = values.pop("ls", None) + if lp: + values["last_price"] = lp + if ls: + values["last_size"] = ls + + return values + + +class PolygonCryptoL2WebSocketData(WebSocketData): + """Polygon Crypto L2 WebSocket data model.""" + + __alias_dict__ = { + "type": "ev", + "symbol": "pair", + "date": "t", + "exchange": "x", + "bid": "b", + "ask": "a", + "received_at": "r", + } + + type: str = Field( + description="The type of data.", + ) + date: datetime = Field( + description=DATA_DESCRIPTIONS.get("date", ""), + ) + received_at: datetime = Field( + description="The time the data was received by Polygon.", + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + exchange: str = Field( + default=None, + description="The exchange of the data.", + ) + bid: list[list[float]] = Field( + description="An array of bid prices, where each entry contains two elements:" + + " the first is the bid price, and the second is the size, with a maximum depth of 100.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + ask: list[list[float]] = Field( + description="An array of ask prices, where each entry contains two elements:" + + " the first is the ask price, and the second is the size, with a maximum depth of 100.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + + @field_validator("date", "received_at", mode="before", check_fields=False) + @classmethod + def _validate_date(cls, v): + """Validate the date.""" + return validate_date(cls, v) + + @field_validator("exchange", mode="before", check_fields=False) + @classmethod + def _validate_exchange(cls, v): + """Validate the exchange.""" + return CRYPTO_EXCHANGE_MAP.get(v, str(v)) + + +class PolygonFXQuoteWebSocketData(WebSocketData): + """Polygon FX Quote WebSocket data model.""" + + __alias_dict__ = { + "date": "t", + "type": "ev", + "symbol": "p", + "exchange": "x", + "ask": "a", + "bid": "b", + } + + type: str = Field( + description="The type of data.", + ) + date: datetime = Field( + description=DATA_DESCRIPTIONS.get("date", ""), + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + exchange: str = Field( + default=None, + description="The exchange of the data.", + ) + bid: float = Field( + description="The bid price.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + ask: float = Field( + description="The ask price.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + + @field_validator("date", mode="before", check_fields=False) + @classmethod + def _validate_date(cls, v): + """Validate the date.""" + return validate_date(cls, v) + + @field_validator("exchange", mode="before", check_fields=False) + @classmethod + def _validate_exchange(cls, v): + """Validate the exchange.""" + return FX_EXCHANGE_MAP.get(v, str(v)) + + +class PolygonStockAggsWebSocketData(WebSocketData): + """Polygon Stock Aggregates WebSocket data model.""" + + __alias_dict__ = { + "type": "ev", + "symbol": "sym", + "date": "e", + "day_open": "op", + "day_volume": "av", + "open": "o", + "high": "h", + "low": "l", + "close": "c", + "vwap": "vw", + "day_vwap": "a", + "volume": "v", + "avg_size": "z", + } + + type: str = Field( + description="The type of data.", + ) + date: datetime = Field( + description="The start of the aggregate window.", + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + day_open: float = Field( + description="Today's official opening price.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + open: float = Field( + description=DATA_DESCRIPTIONS.get("open", "") + + " For the current aggregate window.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + high: float = Field( + description=DATA_DESCRIPTIONS.get("high", "") + + " For the current aggregate window.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + low: float = Field( + description=DATA_DESCRIPTIONS.get("low", "") + + " For the current aggregate window.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + close: float = Field( + description=DATA_DESCRIPTIONS.get("close", "") + + " For the current aggregate window.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + vwap: float = Field( + description=DATA_DESCRIPTIONS.get("vwap", "") + + " For the current aggregate window.", + ) + day_vwap: float = Field( + description="Today's volume weighted average price.", + ) + volume: float = Field( + description=DATA_DESCRIPTIONS.get("volume", "") + + " For the current aggregate window.", + ) + day_volume: float = Field( + description="Today's accumulated volume.", + ) + avg_size: Optional[float] = Field( + default=None, + description="The average trade size for the aggregate window.", + ) + + @field_validator("date", mode="before", check_fields=False) + @classmethod + def _validate_date(cls, v): + """Validate the date.""" + return validate_date(cls, v) + + @model_validator(mode="before") + @classmethod + def _validate_model(cls, values): + """Validate the model.""" + _ = values.pop("s", None) + _ = values.pop("otc", None) + if values.get("z") and values["z"] == 0 or not values.get("z"): + _ = values.pop("z", None) + return values + + +class PolygonStockTradeWebSocketData(WebSocketData): + """Polygon Stock Trade WebSocket data model.""" + + __alias_dict__ = { + "type": "ev", + "symbol": "sym", + "date": "t", + "exchange": "x", + "trf_id": "trfi", + "tape": "z", + "price": "p", + "conditions": "c", + } + + type: str = Field( + description="The type of data.", + ) + date: datetime = Field( + description="The start of the aggregate window.", + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + price: float = Field( + description="The price of the trade.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + exchange: str = Field( + description="The exchange where the trade originated.", + ) + tape: str = Field( + description="The tape where the trade occurred.", + ) + conditions: Optional[str] = Field( + default=None, + description="The conditions of the trade.", + ) + trf_id: Optional[str] = Field( + default=None, + description="The ID for the Trade Reporting Facility where the trade took place.", + ) + trf_timestamp: Optional[datetime] = Field( + default=None, + description="The timestamp of when the trade reporting facility received this trade.", + ) + + @field_validator("date", "trf_timestamp", mode="before", check_fields=False) + @classmethod + def _validate_date(cls, v): + """Validate the date.""" + return validate_date(cls, v) + + @field_validator("tape", mode="before", check_fields=False) + @classmethod + def _validate_tape(cls, v): + """Validate the tape.""" + return map_tape(v) + + @field_validator("exchange", mode="before", check_fields=False) + @classmethod + def _validate_exchange(cls, v): + """Validate the exchange.""" + return STOCK_EXCHANGE_MAP.get(v, str(v)) + + @field_validator("conditions", mode="before", check_fields=False) + @classmethod + def _validate_conditions(cls, v): + """Validate the conditions.""" + if v is None or not v: + return None + new_conditions: list = [] + if isinstance(v, list): + for c in v: + new_conditions.append(STOCK_TRADE_CONDITIONS.get(c, str(c))) + elif isinstance(v, int): + new_conditions.append(STOCK_TRADE_CONDITIONS.get(v, str(v))) + + if not new_conditions: + return None + return "; ".join(new_conditions) + + @model_validator(mode="before") + @classmethod + def _validate_model(cls, values): + """Validate the model.""" + _ = values.pop("i", None) + _ = values.pop("q", None) + return values + + +class PolygonStockQuoteWebSocketData(WebSocketData): + """Polygon Stock Quote WebSocket data model.""" + + __alias_dict__ = { + "type": "ev", + "symbol": "sym", + "date": "t", + "bid_exchange": "bx", + "bid_size": "bs", + "bid": "bp", + "ask": "ap", + "ask_size": "as", + "ask_exchange": "ax", + "tape": "z", + "condition": "c", + "indicators": "i", + } + + type: str = Field( + description="The type of data.", + ) + date: datetime = Field( + description="The start of the aggregate window.", + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + bid_exchange: str = Field( + description="The exchange where the bid originated.", + ) + bid_size: float = Field( + description="The size of the bid.", + ) + bid: float = Field( + description="The bid price.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + ask: float = Field( + description="The ask price.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + ask_size: float = Field( + description="The size of the ask.", + ) + ask_exchange: str = Field( + description="The exchange where the ask originated.", + ) + tape: str = Field( + description="The tape where the quote occurred.", + ) + condition: Optional[str] = Field( + default=None, + description="The condition of the quote.", + ) + indicators: Optional[str] = Field( + default=None, + description="The indicators of the quote.", + ) + + @field_validator("date", mode="before", check_fields=False) + @classmethod + def _validate_date(cls, v): + """Validate the date.""" + return validate_date(cls, v) + + @field_validator("bid_exchange", "ask_exchange", mode="before", check_fields=False) + @classmethod + def _validate_exchange(cls, v): + """Validate the exchange.""" + return STOCK_EXCHANGE_MAP.get(v, str(v)) + + @field_validator("tape", mode="before", check_fields=False) + @classmethod + def _validate_tape(cls, v): + """Validate the tape.""" + return map_tape(v) + + @field_validator("condition", mode="before", check_fields=False) + @classmethod + def _validate_condition(cls, v): + """Validate the condition.""" + return STOCK_QUOTE_CONDITIONS.get(v, str(v)) + + @field_validator("indicators", mode="before", check_fields=False) + @classmethod + def _validate_indicators(cls, v): + """Validate the indicators.""" + if v is None or not v: + return None + new_indicators: list = [] + if isinstance(v, list): + for c in v: + new_indicators.append(STOCK_QUOTE_INDICATORS.get(c, str(c))) + elif isinstance(v, int): + new_indicators.append(STOCK_QUOTE_INDICATORS.get(v, str(v))) + + if not new_indicators: + return None + return "; ".join(new_indicators) + + @model_validator(mode="before") + @classmethod + def _validate_model(cls, values): + """Validate the model.""" + _ = values.pop("q", None) + return values + + +class PolygonFairMarketValueData(WebSocketData): + """Polygon Fair Market Value WebSocket Data.""" + + __alias_dict__ = { + "type": "ev", + "symbol": "sym", + "date": "t", + "fair_market_value": "fmv", + } + + type: str = Field( + description="The type of data.", + ) + date: datetime = Field( + description=DATA_DESCRIPTIONS.get("date", ""), + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + fair_market_value: float = Field( + description="Polygon proprietary algorithm determining real-time, accurate," + + " fair market value of a tradable security.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + + +MODEL_MAP = { + "XT": PolygonCryptoTradeWebSocketData, + "XQ": PolygonCryptoQuoteWebSocketData, + "XL2": PolygonCryptoL2WebSocketData, + "XA": PolygonCryptoAggsWebSocketData, + "XAS": PolygonCryptoAggsWebSocketData, + "FMV": PolygonFairMarketValueData, + "CA": PolygonCryptoAggsWebSocketData, + "CAS": PolygonCryptoAggsWebSocketData, + "C": PolygonFXQuoteWebSocketData, + "AM": PolygonStockAggsWebSocketData, + "AS": PolygonStockAggsWebSocketData, + "T": PolygonStockTradeWebSocketData, + "Q": PolygonStockQuoteWebSocketData, +} + + +class PolygonWebSocketData(Data): + """Polygon WebSocket data model.""" + + # model_config = ConfigDict( + # extra="allow", + # populate_by_alias=True, + # arbitrary_types_allowed=True, + # validate_default=False, + # frozen=False, + # strict=False, + # ) + + def __new__(cls, **data): + """Create new instance of appropriate model type.""" + model = MODEL_MAP.get(data.get("ev")) or MODEL_MAP.get(data.get("type")) + if not model: + return super().__new__(cls) + + return model.model_validate(data) + + +class PolygonWebSocketConnection(WebSocketConnection): + """Polygon WebSocket connection model.""" + + +class PolygonWebSocketFetcher( + Fetcher[PolygonWebSocketQueryParams, PolygonWebSocketConnection] +): + """Polygon WebSocket Fetcher.""" + + @staticmethod + def transform_query(params: dict[str, Any]) -> PolygonWebSocketQueryParams: + """Transform the query parameters.""" + return PolygonWebSocketQueryParams(**params) + + @staticmethod + def extract_data( + query: PolygonWebSocketQueryParams, + credentials: Optional[dict[str, str]], + **kwargs: Any, + ) -> WebSocketClient: + """Extract data from the WebSocket.""" + # pylint: disable=import-outside-toplevel + import time + + api_key = credentials.get("polygon_api_key") if credentials else "" + url = URL_MAP[query.asset_type] + + symbol = query.symbol.upper() + + kwargs = { + "url": url, + "asset_type": query.asset_type, + "feed": query.feed, + "api_key": api_key, + "connect_kwargs": query.connect_kwargs, + } + + client = WebSocketClient( + name=query.name, + module="openbb_polygon.utils.websocket_client", + symbol=symbol, + limit=query.limit, + results_file=query.results_file, + table_name=query.table_name, + save_results=query.save_results, + data_model=PolygonWebSocketData, + sleep_time=query.sleep_time, + broadcast_host=query.broadcast_host, + broadcast_port=query.broadcast_port, + auth_token=query.auth_token, + **kwargs, + ) + + try: + client.connect() + + except Exception as e: # pylint: disable=broad-except + client.disconnect() + raise OpenBBError(e) from e + + time.sleep(1) + + if client.is_running: + return client + + raise OpenBBError("Failed to connect to the WebSocket.") + + @staticmethod + def transform_data( + data: WebSocketClient, + query: PolygonWebSocketQueryParams, + **kwargs: Any, + ) -> PolygonWebSocketConnection: + """Return the client as an instance of Data.""" + return PolygonWebSocketConnection(client=data) diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py b/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py new file mode 100644 index 000000000000..4922e6eaf5fc --- /dev/null +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py @@ -0,0 +1,186 @@ +"""Polygon Constants.""" + +CRYPTO_EXCHANGE_MAP = { + 1: "Coinbase", + 2: "Bitfinex", + 6: "Bitstamp", + 10: "Binance", + 23: "Kraken", +} + +FX_EXCHANGE_MAP = { + 48: "Currency Banks 1", +} + +STOCK_EXCHANGE_MAP = { + 1: "XNYS", + 2: "XNAS", + 3: "XNYS", + 4: "FINR", + 5: "XNAS", + 6: "XNAS", + 7: "XCBO", + 8: "XCBO", + 9: "XNYS", + 10: "XNYS", + 11: "XNYS", + 12: "XNAS", + 13: "XNYS", + 14: "LTSE", + 15: "IEXG", + 16: "XCBO", + 17: "XNAS", + 18: "XCBO", + 19: "XCBO", + 20: "MIHI", + 21: "MEMX", + 62: "FINR", +} + +STOCK_TRADE_CONDITIONS = { + 0: "Regular Trade", + 1: "Acquisition", + 2: "Average Price Trade", + 3: "Automatic Execution", + 4: "Bunched Trade", + 5: "Bunched Sold Trade", + 6: "CAP Election", + 7: "Cash Sale", + 8: "Closing Prints", + 9: "Cross Trade", + 10: "Derivatively Priced", + 11: "Distribution", + 12: "Form T/Extended Hours", + 13: "Extended Hours (Sold Out Of Sequence)", + 14: "Intermarket Sweep", + 15: "Market Center Official Close", + 16: "Market Center Official Open", + 17: "Market Center Opening Trade", + 18: "Market Center Reopening Trade", + 19: "Market Center Closing Trade", + 20: "Next Day", + 21: "Price Variation Trade", + 22: "Prior Reference Price", + 23: "Rule 155 Trade (AMEX)", + 24: "Rule 127 (NYSE Only)", + 25: "Opening Prints", + 27: "Stopped Stock (Regular Trade)", + 28: "Re-Opening Prints", + 29: "Seller", + 30: "Sold Last", + 31: "Sold Last and Stopped Stock", + 32: "Sold (Out Of Sequence)", + 33: "Sold (Out of Sequence) and Stopped Stock", + 34: "Split Trade", + 35: "Stock Option", + 36: "Yellow Flag Regular Trade", + 37: "Odd Lot Trade", + 38: "Corrected Consolidated Close (per listing market)", + 41: "Trade Thru Exempt", + 52: "Contingent Trade", + 53: "Qualified Contingent Trade", + 55: "Opening Reopening Trade Detail", + 57: "Short Sale Restriction Activated", + 58: "Short Sale Restriction Continued", + 59: "Short Sale Restriction Deactivated", + 60: "Short Sale Restriction In Effect", + 62: "Financial Status - Bankrupt", + 63: "Financial Status - Deficient", + 64: "Financial Status - Delinquent", + 65: "Financial Status - Bankrupt and Deficient", + 66: "Financial Status - Bankrupt and Delinquent", + 67: "Financial Status - Deficient and Delinquent", + 68: "Financial Status - Deficient, Delinquent, and Bankrupt", + 69: "Financial Status - Liquidation", + 70: "Financial Status - Creations Suspended", + 71: "Financial Status - Redemptions Suspended", +} + +STOCK_QUOTE_CONDITIONS = { + 0: "Regular", + 1: "RegularTwoSidedOpen", + 2: "RegularOneSidedOpen", + 3: "SlowAsk", + 4: "SlowBid", + 5: "SlowBidAsk", + 6: "SlowDueLRPBid", + 7: "SlowDueLRPAsk", + 8: "SlowDueNYSELRP", + 9: "SlowDueSetSlowListBidAsk", + 10: "ManualAskAutomatedBid", + 11: "ManualBidAutomatedAsk", + 12: "ManualBidAndAsk", + 13: "Opening", + 14: "Closing", + 15: "Closed", + 16: "Resume", + 17: "FastTrading", + 18: "TradingRangeIndication", + 19: "MarketMakerQuotesClosed", + 20: "NonFirm", + 21: "NewsDissemination", + 22: "OrderInflux", + 23: "OrderImbalance", + 24: "DueToRelatedSecurityNewsDissemination", + 25: "DueToRelatedSecurityNewsPending", + 26: "AdditionalInformation", + 27: "NewsPending", + 28: "AdditionalInformationDueToRelatedSecurity", + 29: "DueToRelatedSecurity", + 30: "InViewOfCommon", + 31: "EquipmentChangeover", + 32: "NoOpenNoResponse", + 33: "SubPennyTrading", + 34: "AutomatedBidNoOfferNoBid", + 35: "LULDPriceBand", + 36: "MarketWideCircuitBreakerLevel1", + 37: "MarketWideCircuitBreakerLevel2", + 38: "MarketWideCircuitBreakerLevel3", + 39: "RepublishedLULDPriceBand", + 40: "OnDemandAuction", + 41: "CashOnlySettlement", + 42: "NextDaySettlement", + 43: "LULDTradingPause", + 71: "SlowDuelRPBidAsk", + 80: "Cancel", + 81: "Corrected Price", + 82: "SIPGenerated", + 83: "Unknown", + 84: "Crossed Market", + 85: "Locked Market", + 86: "Depth On Offer Side", + 87: "Depth On Bid Side", + 88: "Depth On Bid And Offer", + 89: "Pre Opening Indication", + 90: "Syndicate Bid", + 91: "Pre Syndicate Bid", + 92: "Penalty Bid", +} + +STOCK_QUOTE_INDICATORS = { + 601: "NBBO_NO_CHANGE", + 602: "NBBO_QUOTE_IS_NBBO", + 603: "NBBO_NO_BB_NO_BO", + 604: "NBBO_BB_BO_SHORT_APPENDAGE", + 605: "NBBO_BB_BO_LONG_APPENDAGE", + 621: "HELD_TRADE_NOT_LAST_SALE_AND_NOT_ON_CONSOLIDATED", + 622: "HELD_TRADE_LAST_SALE_BUT_NOT_ON_CONSOLIDATED", + 623: "HELD_TRADE_LAST_SALE_AND_ON_CONSOLIDATED", + 501: "RETAIL_INTEREST_ON_BID", + 502: "RETAIL_INTEREST_ON_ASK", + 503: "RETAIL_INTEREST_ON_BID_AND_ASK", + 504: "FINRA_BBO_NO_CHANGE", + 505: "FINRA_BBO_DOES_NOT_EXIST", + 506: "FINRA_BB_BO_EXECUTABLE", + 507: "FINRA_BB_BELOW_LOWER_BAND", + 508: "FINRA_BO_ABOVE_UPPER_BAND", + 509: "FINRA_BB_BELOW_LOWER_BAND_BO_ABOVE_UPPER_BAND", + 901: "CTA_NOT_DUE_TO_RELATED_SECURITY", + 902: "CTA_DUE_TO_RELATED_SECURITY", + 903: "CTA_NOT_IN_VIEW_OF_COMMON", + 904: "CTA_IN_VIEW_OF_COMMON", + 905: "CTA_PRICE_INDICATOR", + 906: "CTA_NEW_PRICE_INDICATOR", + 907: "CTA_CORRECTED_PRICE_INDICATION", + 908: "CTA_CANCELLED_MARKET_IMBALANCE_PRICE_TRADING_RANGE_INDICATION", +} diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py new file mode 100644 index 000000000000..993b86eb6e62 --- /dev/null +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -0,0 +1,239 @@ +"""Polygon WebSocket server.""" + +import asyncio +import json +import os +import signal +import sys + +import websockets +import websockets.exceptions +from openbb_polygon.models.websocket_connection import ( + FEED_MAP, + PolygonWebSocketData, +) +from openbb_websockets.helpers import ( + MessageQueue, + get_logger, + handle_termination_signal, + parse_kwargs, + write_to_db, +) + +logger = get_logger("openbb.websocket.polygon") +queue = MessageQueue() +command_queue = MessageQueue() + +kwargs = parse_kwargs() +CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) +FEED = kwargs.pop("feed", None) +ASSET_TYPE = kwargs.pop("asset_type", None) + + +async def handle_symbol(symbol): + """Handle the symbol and map it to the correct format.""" + symbols = symbol.split(",") if isinstance(symbol, str) else symbol + new_symbols: list = [] + feed = FEED_MAP.get(ASSET_TYPE, {}).get(FEED) + for s in symbols: + if s == "*": + new_symbols.append(f"{feed}.*") + continue + ticker = s.upper() + if ASSET_TYPE == "crypto" and "-" not in ticker and ticker != "*": + ticker = ticker[:3] + "-" + ticker[3:] + elif ASSET_TYPE == "fx" and "/" not in ticker and ticker != "*": + ticker = ticker[:3] + "/" + ticker[3:] + elif ASSET_TYPE == "fx" and "-" in ticker: + ticker = ticker.replace("-", "/") + + if ticker and "." not in ticker and not ticker.startswith(feed): + ticker = f"{feed}.{ticker}" + new_symbols.append(ticker) + + return ",".join(new_symbols) + + +async def login(websocket, api_key): + login_event = f'{{"action":"auth","params":"{api_key}"}}' + try: + await websocket.send(login_event) + res = await websocket.recv() + response = json.loads(res) + messages = response if isinstance(response, list) else [response] + for msg in messages: + if msg.get("status") == "connected": + logger.info("PROVIDER INFO: %s", msg.get("message")) + continue + if msg.get("status") != "auth_success": + err = f"PROVIDER ERROR: {msg.get('status')} -> {msg.get('message')}" + logger.error(err) + sys.exit(1) + logger.info("PROVIDER INFO: %s", msg.get("message")) + except Exception as e: + logger.error("PROVIDER ERROR: %s", e.args[0]) + sys.exit(1) + + +async def subscribe(websocket, symbol, event): + """Subscribe or unsubscribe to a symbol.""" + ticker = await handle_symbol(symbol) + subscribe_event = f'{{"action":"{event}","params":"{ticker}"}}' + try: + await websocket.send(subscribe_event) + except Exception as e: + msg = f"PROVIDER ERROR: {e}" + logger.error(msg) + + +async def read_stdin_and_queue_commands(): + """Read from stdin and queue commands.""" + while True: + line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline) + sys.stdin.flush() + + if not line: + break + + try: + command = json.loads(line.strip()) + await command_queue.enqueue(command) + except json.JSONDecodeError: + logger.error("Invalid JSON received from stdin") + + +async def process_message(message, results_path, table_name, limit): + """Process the WebSocket message.""" + messages = message if isinstance(message, list) else [message] + for msg in messages: + if "status" in msg or "message" in msg: + if "status" in msg and msg["status"] == "error": + err = msg.get("message") + raise websockets.WebSocketException(err) + if "message" in msg and msg.get("message"): + logger.info("PROVIDER INFO: %s", msg.get("message")) + elif msg and "ev" in msg and "status" not in msg: + try: + result = PolygonWebSocketData(**msg).model_dump_json( + exclude_none=True, exclude_unset=True + ) + except Exception as e: + err = f"PROVIDER ERROR: Error validating data: {e}" + logger.error(err) + return None + if result: + await write_to_db(result, results_path, table_name, limit) + + +async def connect_and_stream(url, symbol, api_key, results_path, table_name, limit): + """Connect to the WebSocket and stream data to file.""" + + handler_task = asyncio.create_task( + queue.process_queue( + lambda message: process_message(message, results_path, table_name, limit) + ) + ) + + stdin_task = asyncio.create_task(read_stdin_and_queue_commands()) + + try: + connect_kwargs = CONNECT_KWARGS.copy() + if "ping_timeout" not in connect_kwargs: + connect_kwargs["ping_timeout"] = None + if "close_timeout" not in connect_kwargs: + connect_kwargs["close_timeout"] = None + + try: + async with websockets.connect(url, **connect_kwargs) as websocket: + await login(websocket, api_key) + response = await websocket.recv() + messages = json.loads(response) + await process_message(messages, results_path, table_name, limit) + await subscribe(websocket, symbol, "subscribe") + response = await websocket.recv() + messages = json.loads(response) + await process_message(messages, results_path, table_name, limit) + while True: + ws_task = asyncio.create_task(websocket.recv()) + cmd_task = asyncio.create_task(command_queue.dequeue()) + + done, pending = await asyncio.wait( + [ws_task, cmd_task], return_when=asyncio.FIRST_COMPLETED + ) + for task in pending: + task.cancel() + + for task in done: + if task == ws_task: + messages = task.result() + await asyncio.shield(queue.enqueue(json.loads(messages))) + elif task == cmd_task: + command = task.result() + symbol = command.get("symbol") + event = command.get("event") + if symbol and event: + await subscribe(websocket, symbol, event) + except websockets.InvalidStatusCode as e: + if e.status_code == 404: + msg = f"PROVIDER ERROR: {e}" + logger.error(msg) + sys.exit(1) + else: + raise + + except websockets.ConnectionClosed as e: + msg = f"PROVIDER INFO: The WebSocket connection was closed -> {str(e)}" + logger.info(msg) + # Attempt to reopen the connection + logger.info("PROVIDER INFO: Attempting to reconnect after five seconds.") + await asyncio.sleep(5) + await connect_and_stream(url, symbol, api_key, results_path, table_name, limit) + + except websockets.WebSocketException as e: + msg = f"PROVIDER ERROR: WebSocketException -> {e}" + logger.error(msg) + sys.exit(1) + + except Exception as e: + msg = f"PROVIDER ERROR: Unexpected error -> {e}" + logger.error(msg) + sys.exit(1) + + finally: + handler_task.cancel() + stdin_task.cancel() + await asyncio.gather(handler_task, stdin_task, return_exceptions=True) + sys.exit(0) + + +if __name__ == "__main__": + try: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.set_exception_handler(lambda loop, context: None) + + for sig in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(sig, handle_termination_signal, logger) + + asyncio.run_coroutine_threadsafe( + connect_and_stream( + kwargs["url"], + kwargs["symbol"], + kwargs["api_key"], + os.path.abspath(kwargs["results_file"]), + kwargs["table_name"], + kwargs.get("limit", None), + ), + loop, + ) + loop.run_forever() + + except (KeyboardInterrupt, websockets.ConnectionClosed): + logger.error("PROVIDER ERROR: WebSocket connection closed") + + except Exception as e: # pylint: disable=broad-except + msg = f"PROVIDER ERROR: {e.args[0]}" + logger.error(msg) + + finally: + sys.exit(0) From 2c317a346bc14e8ef847eced871e5b8441adc098 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 9 Nov 2024 00:10:04 -0800 Subject: [PATCH 009/119] didn't add that file --- .../openbb_polygon/models/websocket_connection.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 2c24d7359d7f..5972f42cea6b 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -775,15 +775,6 @@ class PolygonFairMarketValueData(WebSocketData): class PolygonWebSocketData(Data): """Polygon WebSocket data model.""" - # model_config = ConfigDict( - # extra="allow", - # populate_by_alias=True, - # arbitrary_types_allowed=True, - # validate_default=False, - # frozen=False, - # strict=False, - # ) - def __new__(cls, **data): """Create new instance of appropriate model type.""" model = MODEL_MAP.get(data.get("ev")) or MODEL_MAP.get(data.get("type")) From b53a636f43ee882150f653e11a0a4e11769fccb5 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 9 Nov 2024 13:44:19 -0800 Subject: [PATCH 010/119] add some exception handling --- .../websockets/openbb_websockets/client.py | 92 ++++++++++--------- .../websockets/openbb_websockets/models.py | 44 +++++++++ .../openbb_websockets/websockets_router.py | 11 ++- .../models/websocket_connection.py | 8 +- .../openbb_polygon/utils/websocket_client.py | 2 +- 5 files changed, 105 insertions(+), 52 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 3f042a67b0a3..ff909bbb0319 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -3,7 +3,7 @@ # pylint: disable=too-many-statements # flake8: noqa: PLR0915 import logging -from typing import TYPE_CHECKING, Literal, Optional +from typing import TYPE_CHECKING, Literal, Optional, Union if TYPE_CHECKING: from openbb_core.provider.abstract.data import Data @@ -23,7 +23,7 @@ class WebSocketClient: The symbol(s) requested to subscribe. Enter multiple symbols separated by commas without spaces. limit : Optional[int] The limit of records to hold in memory. Once the limit is reached, the oldest records are removed. - Default is None. + Default is 300. Set to None to keep all records. results_file : Optional[str] Absolute path to the file for continuous writing. By default, a temporary file is created. table_name : Optional[str] @@ -37,9 +37,11 @@ class WebSocketClient: The authentication token to use for the WebSocket connection. Default is None. Only used for API and Python application endpoints. logger : Optional[logging.Logger] - The logger instance to use this connection. By default, a new logger is created. - kwargs : dict - Additional keyword arguments to pass to the target module. + The pre-configured logger instance to use for this connection. By default, a new logger is created. + kwargs : Optional[dict] + Additional keyword arguments to pass to the target provider module. Keywords and values must not contain spaces. + To pass items to 'websocket.connect()', include them in the 'kwargs' dictionary as, + {'connect_kwargs': {'key': 'value'}}. Properties ---------- @@ -48,9 +50,9 @@ class WebSocketClient: module : str Path to the provider connection script. is_running : bool - Check if the provider connection is running. + Check if the provider connection process is running. is_broadcasting : bool - Check if the broadcast server is running. + Check if the broadcast server process is running. broadcast_address : str URI address for the results broadcast server. results : list @@ -83,7 +85,7 @@ def __init__( # noqa: PLR0913 name: str, module: str, symbol: Optional[str] = None, - limit: Optional[int] = None, + limit: Optional[int] = 300, results_file: Optional[str] = None, table_name: Optional[str] = None, save_results: bool = False, @@ -91,7 +93,7 @@ def __init__( # noqa: PLR0913 auth_token: Optional[str] = None, logger: Optional[logging.Logger] = None, **kwargs, - ): + ) -> None: """Initialize the WebSocketClient class.""" # pylint: disable=import-outside-toplevel import asyncio # noqa @@ -130,6 +132,7 @@ def __init__( # noqa: PLR0913 self._broadcast_thread = None self._broadcast_log_thread = None self._broadcast_message_queue = Queue() + self._exception = None if not results_file: with tempfile.NamedTemporaryFile(delete=False) as temp_file: @@ -144,6 +147,8 @@ def __init__( # noqa: PLR0913 atexit.register(self._atexit) + # Set up the SQLite database and table. + # Loop handling is for when the class is used directly instead of from the app or API. try: loop = asyncio.get_event_loop() except (RuntimeError, RuntimeWarning): @@ -156,7 +161,7 @@ def __init__( # noqa: PLR0913 except DatabaseError as e: self.logger.error("Error setting up the SQLite database and table: %s", e) - def _atexit(self): + def _atexit(self) -> None: """Clean up the WebSocket client processes at exit.""" # pylint: disable=import-outside-toplevel import os @@ -170,14 +175,14 @@ def _atexit(self): if os.path.exists(self.results_file): os.remove(self.results_file) - async def _setup_database(self): + async def _setup_database(self) -> None: """Set up the SQLite database and table.""" # pylint: disable=import-outside-toplevel from openbb_websockets.helpers import setup_database return await setup_database(self.results_path, self.table_name) - def _log_provider_output(self, output_queue): + def _log_provider_output(self, output_queue) -> None: """Log output from the provider server queue.""" # pylint: disable=import-outside-toplevel import queue # noqa @@ -191,11 +196,13 @@ def _log_provider_output(self, output_queue): if ( "server rejected" in output.lower() or "PROVIDER ERROR" in output + or "Unexpected error" in output ): - self._stop_log_thread_event.set() - self._psutil_process.kill() + err = ChildProcessError(output) + self._exception = err self.logger.error(output) break + output = clean_message(output) output = output + "\n" sys.stdout.write(output + "\n") @@ -203,7 +210,7 @@ def _log_provider_output(self, output_queue): except queue.Empty: continue - def _log_broadcast_output(self, output_queue): + def _log_broadcast_output(self, output_queue) -> None: """Log output from the broadcast server queue.""" # pylint: disable=import-outside-toplevel import queue # noqa @@ -245,7 +252,7 @@ def _log_broadcast_output(self, output_queue): except queue.Empty: continue - def connect(self): + def connect(self) -> None: """Connect to the provider WebSocket.""" # pylint: disable=import-outside-toplevel import json # noqa @@ -313,7 +320,7 @@ def connect(self): def send_message( self, message, target: Literal["provider", "broadcast"] = "provider" - ): + ) -> None: """Send a message to the WebSocket process.""" if target == "provider": self._provider_message_queue.put(message) @@ -322,7 +329,7 @@ def send_message( self._broadcast_message_queue.put(message) read_message_queue(self, self._broadcast_message_queue, target="broadcast") - def disconnect(self): + def disconnect(self) -> None: """Disconnect from the provider WebSocket.""" self._stop_log_thread_event.set() if self._process is None or self.is_running is False: @@ -339,9 +346,11 @@ def disconnect(self): self._log_thread.join() self._stop_log_thread_event.clear() self.logger.info("Disconnected from the provider WebSocket.") + if hasattr(self, "_exception") and self._exception: + raise self._exception return - def subscribe(self, symbol): + def subscribe(self, symbol) -> None: """Subscribe to a new symbol or list of symbols.""" # pylint: disable=import-outside-toplevel import json @@ -353,7 +362,7 @@ def subscribe(self, symbol): new_symbols = list(set(old_symbols + ticker)) self._symbol = ",".join(new_symbols) - def unsubscribe(self, symbol): + def unsubscribe(self, symbol) -> None: """Unsubscribe from a symbol or list of symbols.""" # pylint: disable=import-outside-toplevel import json @@ -370,22 +379,22 @@ def unsubscribe(self, symbol): self._symbol = ",".join(new_symbols) @property - def is_running(self): + def is_running(self) -> bool: """Check if the provider connection is running.""" if hasattr(self._psutil_process, "is_running"): return self._psutil_process.is_running() return False @property - def is_broadcasting(self): + def is_broadcasting(self) -> bool: """Check if the broadcast server is running.""" if hasattr(self._psutil_broadcast_process, "is_running"): return self._psutil_broadcast_process.is_running() return False @property - def results(self): - """Retrieve the raw results dumped by the WebSocket stream.""" + def results(self) -> Union[list[dict], None]: + """Retrieve the deserialized results from the results file.""" # pylint: disable=import-outside-toplevel import json # noqa import sqlite3 @@ -397,13 +406,14 @@ def results(self): cursor = conn.execute(f"SELECT * FROM {self.table_name}") # noqa for row in cursor: index, message = row - output.append(json.loads(message)) + output.append(json.loads(json.loads(message))) + if output: return output self.logger.info("No results found in %s", self.results_file) - return [] + return @results.deleter def results(self): @@ -448,7 +458,7 @@ def run_in_thread(): self.logger.error("Error clearing results: %s", e) @property - def module(self): + def module(self) -> str: """Path to the provider connection script.""" return self._module @@ -465,12 +475,12 @@ def module(self, module): ] @property - def symbol(self): + def symbol(self) -> str: """Symbol(s) requested to subscribe.""" return self._symbol @property - def limit(self): + def limit(self) -> Union[int, None]: """Get the limit of records to hold in memory.""" return self._limit @@ -480,7 +490,7 @@ def limit(self, limit): self._limit = limit @property - def broadcast_address(self): + def broadcast_address(self) -> Union[str, None]: """Get the WebSocket broadcast address.""" return ( self._broadcast_address @@ -493,7 +503,7 @@ def start_broadcasting( host: str = "127.0.0.1", port: int = 6666, **kwargs, - ): + ) -> None: """Broadcast results over a network connection.""" # pylint: disable=import-outside-toplevel import os # noqa @@ -594,15 +604,11 @@ def stop_broadcasting(self): return @property - def transformed_results(self): - """Deserialize the records from the results file.""" - # pylint: disable=import-outside-toplevel - import json - + def transformed_results(self) -> list["Data"]: + """Model validated records from the results file.""" if not self.data_model: raise NotImplementedError("No model provided to transform the results.") - - return [self.data_model.model_validate(json.loads(d)) for d in self.results] + return [self.data_model.model_validate(d) for d in self.results] def __repr__(self): """Return the WebSocketClient representation.""" @@ -617,7 +623,7 @@ def __repr__(self): ) -def non_blocking_websocket(client, output_queue, provider_message_queue): +def non_blocking_websocket(client, output_queue, provider_message_queue) -> None: """Communicate with the threaded process.""" try: while not client._stop_log_thread_event.is_set(): @@ -639,8 +645,8 @@ def non_blocking_websocket(client, output_queue, provider_message_queue): def send_message( client, message, target: Literal["provider", "broadcast"] = "provider" -): - """Send a message to the WebSocket process.""" +) -> None: + """Send a message to the WebSocketConnection process.""" try: if target == "provider": if client._process and client._process.stdin: @@ -661,7 +667,7 @@ def send_message( def read_message_queue( client, message_queue, target: Literal["provider", "broadcast"] = "provider" ): - """Read messages from the queue and send them to the WebSocket process.""" + """Read messages from the queue and send them to the WebSocketConnection process.""" while not message_queue.empty(): try: if target == "provider": @@ -681,7 +687,7 @@ def read_message_queue( break -def non_blocking_broadcast(client, output_queue, broadcast_message_queue): +def non_blocking_broadcast(client, output_queue, broadcast_message_queue) -> None: """Continuously read the output from the broadcast process and log it to the main thread.""" try: while not client._stop_broadcasting_event.is_set(): diff --git a/openbb_platform/extensions/websockets/openbb_websockets/models.py b/openbb_platform/extensions/websockets/openbb_websockets/models.py index e50833f09613..19d59769d5cc 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/models.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/models.py @@ -123,3 +123,47 @@ def _validate_client(cls, v): if not isinstance(v, WebSocketClient): raise ValueError("Client must be an instance of WebSocketClient.") return v + + +class WebSocketConnectionStatus(Data): + """Data model for WebSocketConnection status information.""" + + name: str = Field( + description="Name assigned to the client connection.", + ) + auth_required: bool = Field( + description="True when 'auth_token' is supplied at initialization." + " When True, interactions with the client from the Python or API" + + " endpoints requires it to be supplied as a query parameter.", + ) + subscribed_symbols: str = Field( + description="Symbols subscribed to by the client connection.", + ) + is_running: bool = Field( + description="Whether the client connection is running.", + ) + provider_pid: Optional[int] = Field( + default=None, + description="Process ID of the provider connection.", + ) + is_broadcasting: bool = Field( + description="Whether the client connection is broadcasting.", + ) + broadcast_address: Optional[str] = Field( + default=None, + description="URI to the broadcast server.", + ) + broadcast_pid: Optional[int] = Field( + default=None, + description="Process ID of the broadcast server.", + ) + results_file: Optional[str] = Field( + description="Absolute path to the file for continuous writing.", + ) + table_name: Optional[str] = Field( + default=None, + description="Name of the SQL table to write the results to.", + ) + save_results: bool = Field( + description="Whether to save the results after the session ends.", + ) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index 18bbbb85bc15..29d5c0446e8b 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -25,6 +25,7 @@ connected_clients, get_status, ) +from openbb_websockets.models import WebSocketConnectionStatus router = Router("", description="WebSockets Router") sys.stdout = StdOutSink() @@ -38,7 +39,7 @@ async def create_connection( provider_choices: ProviderChoices, standard_params: StandardParams, extra_params: ExtraParams, -) -> OBBject: +) -> OBBject[WebSocketConnectionStatus]: """Create a new provider websocket connection.""" name = extra_params.name if name in connected_clients: @@ -67,7 +68,7 @@ async def create_connection( connected_clients[client_name] = client results = await get_status(client_name) - obbject.results = results + obbject.results = WebSocketConnectionStatus(**results) return obbject @@ -205,7 +206,9 @@ async def unsubscribe( @router.command( methods=["GET"], ) -async def get_client_status(name: str = "all") -> OBBject[list[dict]]: +async def get_client_status( + name: str = "all", +) -> OBBject[list[WebSocketConnectionStatus]]: """Get the status of a client, or all client connections. Parameters @@ -226,7 +229,7 @@ async def get_client_status(name: str = "all") -> OBBject[list[dict]]: ] else: connections = [await get_status(name)] - return OBBject(results=connections) + return OBBject(results=[WebSocketConnectionStatus(**d) for d in connections]) @router.command( diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 5972f42cea6b..4753f560062e 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -193,8 +193,6 @@ def _validate_date(cls, v): def _validate_model(cls, values): """Validate the model.""" _ = values.pop("s", None) - if values.get("z") and values["z"] == 0 or not values.get("z"): - _ = values.pop("z", None) return values @@ -839,13 +837,15 @@ def extract_data( try: client.connect() - - except Exception as e: # pylint: disable=broad-except + except Exception as e: client.disconnect() raise OpenBBError(e) from e time.sleep(1) + if client._exception: + raise client._exception from client._exception + if client.is_running: return client diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index 993b86eb6e62..a8b8bbce56b2 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -195,7 +195,7 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim sys.exit(1) except Exception as e: - msg = f"PROVIDER ERROR: Unexpected error -> {e}" + msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e.__str__()}" logger.error(msg) sys.exit(1) From f1ac74bd6631f9005debdf19f2868f651144252a Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 9 Nov 2024 17:40:02 -0800 Subject: [PATCH 011/119] handle ValidationError --- .../websockets/openbb_websockets/client.py | 26 +++++++++++- .../websockets/openbb_websockets/helpers.py | 9 ++++ .../openbb_websockets/websockets_router.py | 6 ++- .../models/websocket_connection.py | 2 +- .../openbb_polygon/utils/websocket_client.py | 41 +++++++++++++++---- 5 files changed, 72 insertions(+), 12 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index ff909bbb0319..1c43612da300 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -185,14 +185,35 @@ async def _setup_database(self) -> None: def _log_provider_output(self, output_queue) -> None: """Log output from the provider server queue.""" # pylint: disable=import-outside-toplevel - import queue # noqa + import json # noqa + import queue import sys from openbb_websockets.helpers import clean_message + from pydantic import ValidationError while not self._stop_log_thread_event.is_set(): try: output = output_queue.get(timeout=1) if output: + if "ValidationError" in output: + self._psutil_process.kill() + self._process.wait() + self._thread.join() + title, errors = output.split(" -> ")[-1].split(": ") + line_errors = json.loads(errors.strip()) + err = ValidationError.from_exception_data( + title=title.strip(), line_errors=line_errors + ) + self._exception = err + msg = ( + "PROVIDER ERROR: Disconnecting because a ValidatonError was raised" + + " by the provider while processing data." + + f"\n\n{str(err)}\n" + ) + sys.stdout.write(msg + "\n") + sys.stdout.flush() + break + if ( "server rejected" in output.lower() or "PROVIDER ERROR" in output @@ -201,6 +222,9 @@ def _log_provider_output(self, output_queue) -> None: err = ChildProcessError(output) self._exception = err self.logger.error(output) + self._psutil_process.kill() + self._process.wait() + self._thread.join() break output = clean_message(output) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 675e43e2cc61..21539d192087 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -7,6 +7,7 @@ from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.utils.errors import UnauthorizedError +from pydantic import ValidationError AUTH_TOKEN_FILTER = re.compile( r"(auth_token=)([^&]*)", @@ -34,9 +35,17 @@ def get_logger(name, level=logging.INFO): handler.setFormatter(formatter) logger.addHandler(handler) logger.setLevel(level) + return logger +def handle_validation_error(logger: logging.Logger, error: ValidationError): + """Log and raise a Pydantic ValidationError from a provider connection.""" + err = f"{error.__class__.__name__} -> {error.title}: {str(error.json())}" + logger.error(err) + raise error from error + + async def get_status(name: str) -> dict: """Get the status of a client.""" if name not in connected_clients: diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index 29d5c0446e8b..4e0b43f987db 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -58,7 +58,11 @@ async def create_connection( await asyncio.sleep(1) if not client.is_running: - client._atexit() + if client._exception: + exc = getattr(client, "_exception", None) + delattr(client, "_exception") + client._atexit() + raise OpenBBError(exc) raise OpenBBError("Client failed to connect.") if hasattr(extra_params, "start_broadcast") and extra_params.start_broadcast: diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 4753f560062e..8c4ac14a3e65 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -25,7 +25,7 @@ WebSocketData, WebSocketQueryParams, ) -from pydantic import ConfigDict, Field, field_validator, model_validator +from pydantic import Field, field_validator, model_validator URL_MAP = { "stock": "wss://socket.polygon.io/stocks", diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index a8b8bbce56b2..ba8feeee0e15 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -16,9 +16,11 @@ MessageQueue, get_logger, handle_termination_signal, + handle_validation_error, parse_kwargs, write_to_db, ) +from pydantic import ValidationError logger = get_logger("openbb.websocket.polygon") queue = MessageQueue() @@ -36,19 +38,38 @@ async def handle_symbol(symbol): new_symbols: list = [] feed = FEED_MAP.get(ASSET_TYPE, {}).get(FEED) for s in symbols: + if "." in s: + _check = s.split(".")[0] + if _check not in list(FEED_MAP.get(ASSET_TYPE, {}).values()): + logger.error( + "PROVIDER INFO: Invalid feed, %s, for asset type, %s", + _check, + ASSET_TYPE, + ) + continue + if s == "*": new_symbols.append(f"{feed}.*") continue ticker = s.upper() - if ASSET_TYPE == "crypto" and "-" not in ticker and ticker != "*": + if ticker and "." not in ticker and not ticker.startswith(feed): + ticker = f"{feed}.{ticker}" + elif ( + ASSET_TYPE == "crypto" + and "." not in ticker + and "-" not in ticker + and ticker != "*" + ): ticker = ticker[:3] + "-" + ticker[3:] - elif ASSET_TYPE == "fx" and "/" not in ticker and ticker != "*": + elif ( + ASSET_TYPE == "fx" + and "." not in ticker + and "/" not in ticker + and ticker != "*" + ): ticker = ticker[:3] + "/" + ticker[3:] elif ASSET_TYPE == "fx" and "-" in ticker: ticker = ticker.replace("-", "/") - - if ticker and "." not in ticker and not ticker.startswith(feed): - ticker = f"{feed}.{ticker}" new_symbols.append(ticker) return ",".join(new_symbols) @@ -117,10 +138,12 @@ async def process_message(message, results_path, table_name, limit): result = PolygonWebSocketData(**msg).model_dump_json( exclude_none=True, exclude_unset=True ) - except Exception as e: - err = f"PROVIDER ERROR: Error validating data: {e}" - logger.error(err) - return None + except ValidationError as e: + try: + handle_validation_error(logger, e) + except ValidationError: + raise e from e + if result: await write_to_db(result, results_path, table_name, limit) From 402a71f3e6598e87952aa5c99f1a1dfe71c86800 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 9 Nov 2024 17:48:50 -0800 Subject: [PATCH 012/119] use sys.stdout.write instead of logger.error for unexpected error --- .../websockets/openbb_websockets/client.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 1c43612da300..b045aec18966 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -217,14 +217,15 @@ def _log_provider_output(self, output_queue) -> None: if ( "server rejected" in output.lower() or "PROVIDER ERROR" in output - or "Unexpected error" in output + or "unexpected error" in output.lower() ): - err = ChildProcessError(output) - self._exception = err - self.logger.error(output) self._psutil_process.kill() self._process.wait() self._thread.join() + err = ChildProcessError(output) + self._exception = err + sys.stdout.write(msg + "\n") + sys.stdout.flush() break output = clean_message(output) @@ -265,9 +266,9 @@ def _log_broadcast_output(self, output_queue) -> None: output = None if output: - if "ERROR:" in output: + if output.startswith("ERROR:"): output = output.replace("ERROR:", "BROADCAST ERROR:") + "\n" - if "INFO:" in output: + elif output.startswith("INFO:"): output = output.replace("INFO:", "BROADCAST INFO:") + "\n" output = output[0] if isinstance(output, tuple) else output output = clean_message(output) From d748e815aaf46409bc814ce5286a4744a759c99e Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 9 Nov 2024 22:43:00 -0800 Subject: [PATCH 013/119] add polygon indices and send message to broadcast server from main client --- .../websockets/openbb_websockets/broadcast.py | 47 +++++-- .../websockets/openbb_websockets/client.py | 7 +- .../models/websocket_connection.py | 129 +++++++++++++++++- .../openbb_polygon/utils/websocket_client.py | 40 +++--- 4 files changed, 187 insertions(+), 36 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py index 9235a6b4bdf1..29bc7b739b31 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -25,6 +25,31 @@ app = FastAPI() +async def read_stdin(broadcast_server): + """Read from stdin.""" + while True: + line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline) + sys.stdin.flush() + + if not line: + break + + try: + command = ( + json.loads(line.strip()) + if line.strip().startswith("{") or line.strip().startswith("[") + else line.strip() + ) + msg = ( + "BROADCAST INFO: Message received from parent process and relayed to active listeners ->" + + f" {json.dumps(command)}" + ) + await broadcast_server.broadcast(json.dumps(command)) + broadcast_server.logger.info(msg) + except json.JSONDecodeError: + broadcast_server.logger.error("Invalid JSON received from stdin") + + @app.websocket("/") async def websocket_endpoint( # noqa: PLR0915 websocket: WebSocket, auth_token: Optional[str] = None @@ -61,29 +86,35 @@ async def websocket_endpoint( # noqa: PLR0915 connected_clients.add(broadcast_server) stream_task = asyncio.create_task(broadcast_server.stream_results()) + stdin_task = asyncio.create_task(read_stdin(broadcast_server)) try: await websocket.receive_text() except WebSocketDisconnect: pass except Exception as e: - broadcast_server.logger.error(f"Unexpected error: {e}") + msg = f"Unexpected error: {e.__class__.__name__} -> {e}" + broadcast_server.logger.error(msg) pass finally: if broadcast_server in connected_clients: connected_clients.remove(broadcast_server) stream_task.cancel() + stdin_task.cancel() try: await stream_task + await stdin_task except asyncio.CancelledError: broadcast_server.logger.info("Stream task cancelled") except Exception as e: - broadcast_server.logger.error(f"Error while cancelling stream task: {e}") + msg = f"Unexpected error while cancelling stream task: {e.__class__.__name__} -> {e}" + broadcast_server.logger.error(msg) if websocket.client_state != WebSocketState.DISCONNECTED: try: await websocket.close() except RuntimeError as e: - broadcast_server.logger.error(f"Error while closing websocket: {e}") + msg = f"Unexpected error while closing websocket: {e.__class__.__name__} -> {e}" + broadcast_server.logger.error(msg) class BroadcastServer: @@ -147,7 +178,7 @@ async def stream_results(self): # noqa: PLR0915 await self.broadcast(json.dumps(json.loads(message))) last_id = max(row[0] for row in rows) else: - self.logger.error(f"Results file not found: {file_path}") + self.logger.error("Results file not found: %s", str(file_path)) break await asyncio.sleep(self.sleep_time) @@ -167,7 +198,8 @@ async def stream_results(self): # noqa: PLR0915 except WebSocketDisconnect: pass except Exception as e: - self.logger.error(f"Unexpected error: {e}") + msg = f"Unexpected error: {e.__class__.__name__} -> {e}" + self.logger.error(msg) finally: return @@ -220,9 +252,8 @@ def main(): **kwargs, ) except TypeError as e: - broadcast_server.logger.error( - f"Invalid keyword argument passed to unvicorn. -> {e.args[0]}\n" - ) + msg = f"Invalid keyword argument passed to unvicorn. -> {e.args[0]}\n" + broadcast_server.logger.error(msg) except KeyboardInterrupt: broadcast_server.logger.info("Broadcast server terminated.") finally: diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index b045aec18966..743d58d51c72 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -224,7 +224,7 @@ def _log_provider_output(self, output_queue) -> None: self._thread.join() err = ChildProcessError(output) self._exception = err - sys.stdout.write(msg + "\n") + sys.stdout.write(output + "\n") sys.stdout.flush() break @@ -672,6 +672,11 @@ def send_message( client, message, target: Literal["provider", "broadcast"] = "provider" ) -> None: """Send a message to the WebSocketConnection process.""" + # pylint: disable=import-outside-toplevel + import json + + if isinstance(message, (dict, list)): + message = json.dumps(message) try: if target == "provider": if client._process and client._process.stdin: diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 8c4ac14a3e65..f7d8d06a07bd 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -30,11 +30,13 @@ URL_MAP = { "stock": "wss://socket.polygon.io/stocks", "stock_delayed": "wss://delayed.polygon.io/stocks", + "index": "wss://socket.polygon.io/indices", + "index_delayed": "wss://delayed.polygon.io/indices", "fx": "wss://socket.polygon.io/forex", "crypto": "wss://socket.polygon.io/crypto", } -ASSET_CHOICES = ["stock", "stock_delayed", "fx", "crypto"] +ASSET_CHOICES = ["stock", "stock_delayed", "fx", "crypto", "index", "index_delayed"] FEED_MAP = { "crypto": { @@ -65,6 +67,16 @@ "quote": "Q", "fmv": "FMV", }, + "index": { + "aggs_min": "AM", + "aggs_sec": "AS", + "value": "V", + }, + "index_delayed": { + "aggs_min": "AM", + "aggs_sec": "AS", + "value": "V", + }, } @@ -101,7 +113,9 @@ class PolygonWebSocketQueryParams(WebSocketQueryParams): symbol: str = Field( description=QUERY_DESCRIPTIONS.get("symbol", ""), ) - asset_type: Literal["stock", "stock_delayed", "fx", "crypto"] = Field( + asset_type: Literal[ + "stock", "stock_delayed", "fx", "crypto", "index", "index_delayed" + ] = Field( default="crypto", description="The asset type associated with the symbol(s)." + " Choose from: stock, stock_delayed, fx, crypto.", @@ -150,7 +164,8 @@ class PolygonCryptoAggsWebSocketData(WebSocketData): description="The type of data.", ) date: datetime = Field( - description="The start of the aggregate window.", + description=DATA_DESCRIPTIONS.get("date", "") + + "The end of the aggregate window.", ) symbol: str = Field( description=DATA_DESCRIPTIONS.get("symbol", ""), @@ -464,7 +479,8 @@ class PolygonStockAggsWebSocketData(WebSocketData): description="The type of data.", ) date: datetime = Field( - description="The start of the aggregate window.", + description=DATA_DESCRIPTIONS.get("date", "") + + "The end of the aggregate window.", ) symbol: str = Field( description=DATA_DESCRIPTIONS.get("symbol", ""), @@ -547,7 +563,8 @@ class PolygonStockTradeWebSocketData(WebSocketData): description="The type of data.", ) date: datetime = Field( - description="The start of the aggregate window.", + description=DATA_DESCRIPTIONS.get("date", "") + + "The end of the aggregate window.", ) symbol: str = Field( description=DATA_DESCRIPTIONS.get("symbol", ""), @@ -641,7 +658,8 @@ class PolygonStockQuoteWebSocketData(WebSocketData): description="The type of data.", ) date: datetime = Field( - description="The start of the aggregate window.", + description=DATA_DESCRIPTIONS.get("date", "") + + "The end of the aggregate window.", ) symbol: str = Field( description=DATA_DESCRIPTIONS.get("symbol", ""), @@ -727,6 +745,94 @@ def _validate_model(cls, values): return values +class PolygonIndexAggsWebSocketData(WebSocketData): + """Polygon Index Aggregates WebSocket data model.""" + + __alias_dict__ = { + "type": "ev", + "symbol": "sym", + "date": "e", + "day_open": "op", + "open": "o", + "high": "h", + "low": "l", + "close": "c", + } + + type: str = Field( + description="The type of data.", + ) + date: datetime = Field( + description=DATA_DESCRIPTIONS.get("date", "") + + "The end of the aggregate window.", + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + day_open: float = Field( + description="Today's official opening level.", + ) + open: float = Field( + description=DATA_DESCRIPTIONS.get("open", "") + + " For the current aggregate window.", + ) + high: float = Field( + description=DATA_DESCRIPTIONS.get("high", "") + + " For the current aggregate window.", + ) + low: float = Field( + description=DATA_DESCRIPTIONS.get("low", "") + + " For the current aggregate window.", + ) + close: float = Field( + description=DATA_DESCRIPTIONS.get("close", "") + + " For the current aggregate window.", + ) + + @field_validator("date", mode="before", check_fields=False) + @classmethod + def _validate_date(cls, v): + """Validate the date.""" + return validate_date(cls, v) + + @model_validator(mode="before") + @classmethod + def _validate_model(cls, values): + """Validate the model.""" + _ = values.pop("s", None) + return values + + +class PolygonIndexValueWebSocketData(WebSocketData): + """Polygon Index Value WebSocket data model.""" + + __alias_dict__ = { + "type": "ev", + "symbol": "T", + "date": "t", + "value": "val", + } + + type: str = Field( + description="The type of data.", + ) + date: datetime = Field( + description=DATA_DESCRIPTIONS.get("date", ""), + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + value: float = Field( + description="The value of the index.", + ) + + @field_validator("date", mode="before", check_fields=False) + @classmethod + def _validate_date(cls, v): + """Validate the date.""" + return validate_date(cls, v) + + class PolygonFairMarketValueData(WebSocketData): """Polygon Fair Market Value WebSocket Data.""" @@ -767,6 +873,8 @@ class PolygonFairMarketValueData(WebSocketData): "AS": PolygonStockAggsWebSocketData, "T": PolygonStockTradeWebSocketData, "Q": PolygonStockQuoteWebSocketData, + "A": PolygonIndexAggsWebSocketData, + "V": PolygonIndexValueWebSocketData, } @@ -775,7 +883,14 @@ class PolygonWebSocketData(Data): def __new__(cls, **data): """Create new instance of appropriate model type.""" - model = MODEL_MAP.get(data.get("ev")) or MODEL_MAP.get(data.get("type")) + index_symbol = data.get("sym", "").startswith("I:") or data.get( + "symbol", "" + ).startswith("I:") + model = ( + MODEL_MAP["A"] + if index_symbol + else MODEL_MAP.get(data.get("ev")) or MODEL_MAP.get(data.get("type")) + ) if not model: return super().__new__(cls) diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index ba8feeee0e15..39cbf0dd6a7f 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -38,6 +38,11 @@ async def handle_symbol(symbol): new_symbols: list = [] feed = FEED_MAP.get(ASSET_TYPE, {}).get(FEED) for s in symbols: + + if s == "*": + new_symbols.append(f"{feed}.*") + continue + if "." in s: _check = s.split(".")[0] if _check not in list(FEED_MAP.get(ASSET_TYPE, {}).values()): @@ -48,28 +53,21 @@ async def handle_symbol(symbol): ) continue - if s == "*": - new_symbols.append(f"{feed}.*") - continue ticker = s.upper() - if ticker and "." not in ticker and not ticker.startswith(feed): + + if ticker and "." not in ticker: ticker = f"{feed}.{ticker}" - elif ( - ASSET_TYPE == "crypto" - and "." not in ticker - and "-" not in ticker - and ticker != "*" - ): + + if ASSET_TYPE == "crypto" and "-" not in ticker and ticker != "*": ticker = ticker[:3] + "-" + ticker[3:] - elif ( - ASSET_TYPE == "fx" - and "." not in ticker - and "/" not in ticker - and ticker != "*" - ): + elif ASSET_TYPE == "fx" and "/" not in ticker and ticker != "*": ticker = ticker[:3] + "/" + ticker[3:] elif ASSET_TYPE == "fx" and "-" in ticker: ticker = ticker.replace("-", "/") + elif ASSET_TYPE == "index" and ":" not in ticker and ticker != "*": + _feed, _ticker = ticker.split(".") if "." in ticker else (feed, ticker) + ticker = f"{_feed}.I:{_ticker}" + new_symbols.append(ticker) return ",".join(new_symbols) @@ -87,12 +85,12 @@ async def login(websocket, api_key): logger.info("PROVIDER INFO: %s", msg.get("message")) continue if msg.get("status") != "auth_success": - err = f"PROVIDER ERROR: {msg.get('status')} -> {msg.get('message')}" + err = f"PROVIDER ERROR: {msg.get('status')} -> {msg.get('message')}" logger.error(err) sys.exit(1) logger.info("PROVIDER INFO: %s", msg.get("message")) except Exception as e: - logger.error("PROVIDER ERROR: %s", e.args[0]) + logger.error("PROVIDER ERROR: %s -> %s", e.__class__.__name__, e.args[0]) sys.exit(1) @@ -103,7 +101,7 @@ async def subscribe(websocket, symbol, event): try: await websocket.send(subscribe_event) except Exception as e: - msg = f"PROVIDER ERROR: {e}" + msg = f"PROVIDER ERROR: {e.__class__.__name__} -> {e}" logger.error(msg) @@ -146,6 +144,8 @@ async def process_message(message, results_path, table_name, limit): if result: await write_to_db(result, results_path, table_name, limit) + else: + logger.info("PROVIDER INFO: %s", msg) async def connect_and_stream(url, symbol, api_key, results_path, table_name, limit): @@ -255,7 +255,7 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim logger.error("PROVIDER ERROR: WebSocket connection closed") except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: {e.args[0]}" + msg = f"PROVIDER ERROR: {e.__class__.__name__} -> {e}" logger.error(msg) finally: From 82b706a41e324d02b4554d700ca723305f57b503 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sun, 10 Nov 2024 12:21:31 -0800 Subject: [PATCH 014/119] handle UnauthorizedError and map polygon options feeds --- .../websockets/openbb_websockets/client.py | 12 + .../openbb_websockets/websockets_router.py | 4 +- .../models/websocket_connection.py | 254 +++++++++++++++++- .../polygon/openbb_polygon/utils/constants.py | 60 +++++ .../openbb_polygon/utils/websocket_client.py | 48 +++- 5 files changed, 360 insertions(+), 18 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 743d58d51c72..714b175125ad 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -188,6 +188,7 @@ def _log_provider_output(self, output_queue) -> None: import json # noqa import queue import sys + from openbb_core.provider.utils.errors import UnauthorizedError from openbb_websockets.helpers import clean_message from pydantic import ValidationError @@ -195,6 +196,17 @@ def _log_provider_output(self, output_queue) -> None: try: output = output_queue.get(timeout=1) if output: + # Handle raised exceptions from the provider connection thread. + if "UnauthorizedError" in output: + self._psutil_process.kill() + self._process.wait() + self._thread.join() + err = UnauthorizedError(output) + self._exception = err + sys.stdout.write(output + "\n") + sys.stdout.flush() + break + if "ValidationError" in output: self._psutil_process.kill() self._process.wait() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index 4e0b43f987db..91fd66967cda 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -17,7 +17,7 @@ ) from openbb_core.app.query import Query from openbb_core.app.router import Router -from openbb_core.provider.utils.errors import EmptyDataError +from openbb_core.provider.utils.errors import EmptyDataError, UnauthorizedError from openbb_websockets.helpers import ( StdOutSink, @@ -62,6 +62,8 @@ async def create_connection( exc = getattr(client, "_exception", None) delattr(client, "_exception") client._atexit() + if isinstance(exc, UnauthorizedError): + raise exc raise OpenBBError(exc) raise OpenBBError("Client failed to connect.") diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index f7d8d06a07bd..5aa96ed83623 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -1,5 +1,7 @@ """Polygon WebSocket Connection Model.""" +# pylint: disable=unused-argument, too-many-lines + from datetime import datetime from typing import Any, Literal, Optional @@ -13,6 +15,8 @@ from openbb_polygon.utils.constants import ( CRYPTO_EXCHANGE_MAP, FX_EXCHANGE_MAP, + OPTIONS_EXCHANGE_MAP, + OPTIONS_TRADE_CONDITIONS, STOCK_EXCHANGE_MAP, STOCK_QUOTE_CONDITIONS, STOCK_QUOTE_INDICATORS, @@ -30,13 +34,24 @@ URL_MAP = { "stock": "wss://socket.polygon.io/stocks", "stock_delayed": "wss://delayed.polygon.io/stocks", - "index": "wss://socket.polygon.io/indices", - "index_delayed": "wss://delayed.polygon.io/indices", + "options": "wss://socket.polygon.io/options", + "options_delayed": "wss://delayed.polygon.io/options", "fx": "wss://socket.polygon.io/forex", "crypto": "wss://socket.polygon.io/crypto", + "index": "wss://socket.polygon.io/indices", + "index_delayed": "wss://delayed.polygon.io/indices", } -ASSET_CHOICES = ["stock", "stock_delayed", "fx", "crypto", "index", "index_delayed"] +ASSET_CHOICES = [ + "stock", + "stock_delayed", + "options", + "options_delayed", + "fx", + "crypto", + "index", + "index_delayed", +] FEED_MAP = { "crypto": { @@ -77,6 +92,20 @@ "aggs_sec": "AS", "value": "V", }, + "options": { + "aggs_min": "AM", + "aggs_sec": "A", + "trade": "T", + "quote": "Q", + "fmv": "FMV", + }, + "options_delayed": { + "aggs_min": "AM", + "aggs_sec": "A", + "trade": "T", + "quote": "Q", + "fmv": "FMV", + }, } @@ -111,10 +140,20 @@ class PolygonWebSocketQueryParams(WebSocketQueryParams): } symbol: str = Field( - description=QUERY_DESCRIPTIONS.get("symbol", ""), + description=QUERY_DESCRIPTIONS.get("symbol", "") + + " All feeds, except Options, support the wildcard symbol, '*', to subscribe to all symbols." + + " For Options, the OCC contract symbol is used to subscribe up to 1000 individual contracts" + + " per connection." ) asset_type: Literal[ - "stock", "stock_delayed", "fx", "crypto", "index", "index_delayed" + "stock", + "stock_delayed", + "options", + "options_delayed", + "fx", + "crypto", + "index", + "index_delayed", ] = Field( default="crypto", description="The asset type associated with the symbol(s)." @@ -833,6 +872,146 @@ def _validate_date(cls, v): return validate_date(cls, v) +class PolygonOptionsTradeWebSocketData(WebSocketData): + """Polygon Options Trade WebSocket data model.""" + + __alias_dict__ = { + "type": "ev", + "symbol": "sym", + "date": "t", + "exchange": "x", + "price": "p", + "size": "s", + "conditions": "c", + } + + type: str = Field( + description="The type of data.", + ) + date: datetime = Field( + description=DATA_DESCRIPTIONS.get("date", ""), + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + price: float = Field( + description="The price of the trade.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + size: float = Field( + description="The size of the trade.", + ) + exchange: str = Field( + description="The exchange where the trade originated.", + ) + conditions: Optional[str] = Field( + default=None, + description="The conditions of the trade.", + ) + + @field_validator("date", mode="before", check_fields=False) + @classmethod + def _validate_date(cls, v): + """Validate the date.""" + return validate_date(cls, v) + + @field_validator("exchange", mode="before", check_fields=False) + @classmethod + def _validate_exchange(cls, v): + """Validate the exchange.""" + return OPTIONS_EXCHANGE_MAP.get(v, str(v)) + + @field_validator("conditions", mode="before", check_fields=False) + @classmethod + def _validate_conditions(cls, v): + """Validate the conditions.""" + if v is None or not v: + return None + new_conditions: list = [] + if isinstance(v, list): + for c in v: + new_conditions.append(OPTIONS_TRADE_CONDITIONS.get(c, str(c))) + elif isinstance(v, int): + new_conditions.append(OPTIONS_TRADE_CONDITIONS.get(v, str(v))) + + if not new_conditions: + return None + return "; ".join(new_conditions) + + @model_validator(mode="before") + @classmethod + def _validate_model(cls, values): + """Validate the model.""" + _ = values.pop("q", None) + return values + + +class PolygonOptionsQuoteWebSocketData(WebSocketData): + """Polygon Options Quote WebSocket data model.""" + + __alias_dict__ = { + "type": "ev", + "symbol": "sym", + "date": "t", + "bid_exchange": "bx", + "bid_size": "bs", + "bid": "bp", + "ask": "ap", + "ask_size": "as", + "ask_exchange": "ax", + } + + type: str = Field( + description="The type of data.", + ) + date: datetime = Field( + description=DATA_DESCRIPTIONS.get("date", "") + + "The end of the aggregate window.", + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + bid_exchange: str = Field( + description="The exchange where the bid originated.", + ) + bid_size: float = Field( + description="The size of the bid.", + ) + bid: float = Field( + description="The bid price.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + ask: float = Field( + description="The ask price.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + ask_size: float = Field( + description="The size of the ask.", + ) + ask_exchange: str = Field( + description="The exchange where the ask originated.", + ) + + @field_validator("date", mode="before", check_fields=False) + @classmethod + def _validate_date(cls, v): + """Validate the date.""" + return validate_date(cls, v) + + @field_validator("bid_exchange", "ask_exchange", mode="before", check_fields=False) + @classmethod + def _validate_exchange(cls, v): + """Validate the exchange.""" + return OPTIONS_EXCHANGE_MAP.get(v, str(v)) + + @model_validator(mode="before") + @classmethod + def _validate_model(cls, values): + """Validate the model.""" + _ = values.pop("q", None) + return values + + class PolygonFairMarketValueData(WebSocketData): """Polygon Fair Market Value WebSocket Data.""" @@ -877,20 +1056,73 @@ class PolygonFairMarketValueData(WebSocketData): "V": PolygonIndexValueWebSocketData, } +OPTIONS_MODEL_MAP = { + "AM": PolygonStockAggsWebSocketData, + "A": PolygonStockAggsWebSocketData, + "T": PolygonOptionsTradeWebSocketData, + "Q": PolygonOptionsQuoteWebSocketData, + "FMV": PolygonFairMarketValueData, +} + class PolygonWebSocketData(Data): - """Polygon WebSocket data model.""" + """Polygon WebSocket data model. This model is used to identify the appropriate model for the data. + The model is determined based on the type of data received from the WebSocket. + Some asset feeds share common data structures with other asset feeds - for example, FX and Crypto aggregates. + + Stock + ----- + - Aggs: AS, AM - PolygonStockAggsWebSocketData + - Trade: T - PolygonStockTradeWebSocketData + - Quote: Q - PolygonStockQuoteWebSocketData + - Fair Market Value: FMV - PolygonFairMarketValueData + + Options + ------- + - Aggs: A, AM - PolygonStockAggsWebSocketData + - Trade: T - PolygonOptionsTradeWebSocketData + - Quote: Q - PolygonOptionsQuoteWebSocketData + - Fair Market Value: FMV - PolygonFairMarketValueData + + Index + ----- + - Aggs: A, AM - PolygonIndexAggsWebSocketData + - Value: V - PolygonIndexValueWebSocketData + + Crypto + ------ + - Aggs: XAS, XA - PolygonCryptoAggsWebSocketData + - Trade: XT - PolygonCryptoTradeWebSocketData + - Quote: XQ - PolygonCryptoQuoteWebSocketData + - L2: XL2 - PolygonCryptoL2WebSocketData + - Fair Market Value: FMV - PolygonFairMarketValueData + + FX + -- + - Aggs: CAS, CA - PolygonCryptoAggsWebSocketData + - Quote: C - PolygonFXQuoteWebSocketData + - Fair Market Value: FMV - PolygonFairMarketValueData + """ def __new__(cls, **data): """Create new instance of appropriate model type.""" index_symbol = data.get("sym", "").startswith("I:") or data.get( "symbol", "" ).startswith("I:") - model = ( - MODEL_MAP["A"] - if index_symbol - else MODEL_MAP.get(data.get("ev")) or MODEL_MAP.get(data.get("type")) - ) + options_symbol = data.get("sym", "").startswith("O:") or data.get( + "symbol", "" + ).startswith("O:") + + if options_symbol: + model = OPTIONS_MODEL_MAP.get(data.get("ev")) or OPTIONS_MODEL_MAP.get( + data.get("type") + ) + else: + model = ( + MODEL_MAP["A"] + if index_symbol + else MODEL_MAP.get(data.get("ev")) or MODEL_MAP.get(data.get("type")) + ) if not model: return super().__new__(cls) diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py b/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py index 4922e6eaf5fc..9b827c091161 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py @@ -184,3 +184,63 @@ 907: "CTA_CORRECTED_PRICE_INDICATION", 908: "CTA_CANCELLED_MARKET_IMBALANCE_PRICE_TRADING_RANGE_INDICATION", } + +OPTIONS_EXCHANGE_MAP = { + 300: "XNYS", + 301: "XBOX", + 302: "XCBO", + 303: "MIHI", + 304: "XCBO", + 307: "GEMX", + 308: "XISX", + 309: "XISX", + 310: "XISX", + 312: "MIHI", + 313: "XNYS", + 314: "OPRA", + 315: "MIHI", + 316: "XNAS", + 318: "MIHI", + 319: "XNAS", + 320: "MEMX", + 322: "XCBO", + 323: "XNAS", + 325: "XCBO", +} + + +OPTIONS_TRADE_CONDITIONS = { + 201: "Canceled", + 202: "Late and Out Of Sequence", + 203: "Last and Canceled", + 204: "Late", + 205: "Opening Trade and Canceled", + 206: "Opening Trade, Late, and Out Of Sequence", + 207: "Only Trade and Canceled", + 208: "Opening Trade and Late", + 209: "Automatic Execution", + 210: "Reopening Trade", + 219: "Intermarket Sweep Order", + 227: "Single Leg Auction Non ISO", + 228: "Single Leg Auction ISO", + 229: "Single Leg Cross Non ISO", + 230: "Single Leg Cross ISO", + 231: "Single Leg Floor Trade", + 232: "Multi Leg auto-electronic trade", + 233: "Multi Leg Auction", + 234: "Multi Leg Cross", + 235: "Multi Leg floor trade", + 236: "Multi Leg auto-electronic trade against single leg(s)", + 237: "Stock Options Auction", + 238: "Multi Leg Auction against single leg(s)", + 239: "Multi Leg floor trade against single leg(s)", + 240: "Stock Options auto-electronic trade", + 241: "Stock Options Cross", + 242: "Stock Options floor trade", + 243: "Stock Options auto-electronic trade against single leg(s)", + 244: "Stock Options Auction against single leg(s)", + 245: "Stock Options floor trade against single leg(s)", + 246: "Multi Leg Floor Trade of Proprietary Products", + 247: "Multilateral Compression Trade of Proprietary Products", + 248: "Extended Hours Trade", +} diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index 39cbf0dd6a7f..f7248f3cd5ed 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -8,6 +8,7 @@ import websockets import websockets.exceptions +from openbb_core.provider.utils.errors import UnauthorizedError from openbb_polygon.models.websocket_connection import ( FEED_MAP, PolygonWebSocketData, @@ -39,6 +40,12 @@ async def handle_symbol(symbol): feed = FEED_MAP.get(ASSET_TYPE, {}).get(FEED) for s in symbols: + if ASSET_TYPE == "options" and "*" in s: + logger.error( + "PROVIDER INFO: Options symbols do not support wildcards." + ) + continue + if s == "*": new_symbols.append(f"{feed}.*") continue @@ -64,9 +71,16 @@ async def handle_symbol(symbol): ticker = ticker[:3] + "/" + ticker[3:] elif ASSET_TYPE == "fx" and "-" in ticker: ticker = ticker.replace("-", "/") - elif ASSET_TYPE == "index" and ":" not in ticker and ticker != "*": + elif ( + ASSET_TYPE in ["index", "index_delayed"] + and ":" not in ticker + and ticker != "*" + ): _feed, _ticker = ticker.split(".") if "." in ticker else (feed, ticker) ticker = f"{_feed}.I:{_ticker}" + elif ASSET_TYPE in ["options", "options_delayed"] and ":" not in ticker: + _feed, _ticker = ticker.split(".") if "." in ticker else (feed, ticker) + ticker = f"{_feed}.O:{_ticker}" new_symbols.append(ticker) @@ -85,9 +99,12 @@ async def login(websocket, api_key): logger.info("PROVIDER INFO: %s", msg.get("message")) continue if msg.get("status") != "auth_success": - err = f"PROVIDER ERROR: {msg.get('status')} -> {msg.get('message')}" + err = ( + f"UnauthorizedError -> {msg.get('status')} -> {msg.get('message')}" + ) logger.error(err) sys.exit(1) + raise UnauthorizedError(f"{msg.get('status')} -> {msg.get('message')}") logger.info("PROVIDER INFO: %s", msg.get("message")) except Exception as e: logger.error("PROVIDER ERROR: %s -> %s", e.__class__.__name__, e.args[0]) @@ -125,6 +142,12 @@ async def process_message(message, results_path, table_name, limit): """Process the WebSocket message.""" messages = message if isinstance(message, list) else [message] for msg in messages: + if "Your plan doesn't include websocket access" in msg.get("message"): + err = f"UnauthorizedError -> {msg.get('message')}" + logger.error(err) + sys.exit(1) + raise UnauthorizedError(msg.get("message")) + if "status" in msg or "message" in msg: if "status" in msg and msg["status"] == "error": err = msg.get("message") @@ -198,14 +221,27 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim await subscribe(websocket, symbol, event) except websockets.InvalidStatusCode as e: if e.status_code == 404: - msg = f"PROVIDER ERROR: {e}" + msg = f"PROVIDER ERROR: {e.__str__()}" logger.error(msg) sys.exit(1) else: raise + except websockets.InvalidURI as e: + msg = f"PROVIDER ERROR: {e.__str__()}" + logger.error(msg) + sys.exit(1) + + except websockets.ConnectionClosedOK as e: + msg = ( + f"PROVIDER INFO: The WebSocket connection was closed -> {e.__str__()}" + ) + logger.info(msg) + sys.exit(0) except websockets.ConnectionClosed as e: - msg = f"PROVIDER INFO: The WebSocket connection was closed -> {str(e)}" + msg = ( + f"PROVIDER INFO: The WebSocket connection was closed -> {e.__str__()}" + ) logger.info(msg) # Attempt to reopen the connection logger.info("PROVIDER INFO: Attempting to reconnect after five seconds.") @@ -213,12 +249,12 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim await connect_and_stream(url, symbol, api_key, results_path, table_name, limit) except websockets.WebSocketException as e: - msg = f"PROVIDER ERROR: WebSocketException -> {e}" + msg = f"PROVIDER ERROR: WebSocketException -> {e.__str__()}" logger.error(msg) sys.exit(1) except Exception as e: - msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e.__str__()}" + msg = f"Unexpected error -> {e.__class__.__name__}: {e.__str__()}" logger.error(msg) sys.exit(1) From 2489128b67567e2b096290ed7ce705dca680bcc5 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sun, 10 Nov 2024 12:45:17 -0800 Subject: [PATCH 015/119] clear exceptions atexit --- .../extensions/websockets/openbb_websockets/client.py | 2 ++ .../websockets/openbb_websockets/websockets_router.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 714b175125ad..e5b909185851 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -166,6 +166,8 @@ def _atexit(self) -> None: # pylint: disable=import-outside-toplevel import os + self._exception = None + if self.is_running: self.disconnect() if self.is_broadcasting: diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index 91fd66967cda..e75e7d9457f0 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -60,7 +60,6 @@ async def create_connection( if not client.is_running: if client._exception: exc = getattr(client, "_exception", None) - delattr(client, "_exception") client._atexit() if isinstance(exc, UnauthorizedError): raise exc @@ -159,14 +158,19 @@ async def subscribe( """ if not await check_auth(name, auth_token): raise OpenBBError("Error finding client.") + client = connected_clients[name] symbols = client.symbol.split(",") + if symbols and symbol in symbols: raise OpenBBError(f"Client {name} already subscribed to {symbol}.") + client.subscribe(symbol) - # await asyncio.sleep(2) + await asyncio.sleep(1) + if client.is_running: return OBBject(results=f"Added {symbol} to client {name} connection.") + client.logger.error( f"Client {name} failed to subscribe to {symbol} and is not running." ) From 1e82e1e08f2c95ca23e75fb8fdab9dc93dfcb4bd Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sun, 10 Nov 2024 13:08:40 -0800 Subject: [PATCH 016/119] polygon symbol handling edge case --- .../extensions/websockets/openbb_websockets/client.py | 3 ++- .../websockets/openbb_websockets/websockets_router.py | 6 +++--- .../polygon/openbb_polygon/utils/websocket_client.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index e5b909185851..85a0ee8c1c2e 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -705,7 +705,8 @@ def send_message( else: client.logger.error("Broadcast process is not running.") except Exception as e: - client.logger.error(f"Error sending message to WebSocket process: {e}") + msg = f"Error sending message to WebSocket process: {e.__class__.__name__} -> {e.__str__()}" + client.logger.error(msg) def read_message_queue( diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index e75e7d9457f0..bd061c032b71 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -57,9 +57,9 @@ async def create_connection( await asyncio.sleep(1) - if not client.is_running: - if client._exception: - exc = getattr(client, "_exception", None) + if not client.is_running or client._exception: + exc = getattr(client, "_exception", None) + if exc: client._atexit() if isinstance(exc, UnauthorizedError): raise exc diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index f7248f3cd5ed..f0c21e379d2d 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -65,9 +65,9 @@ async def handle_symbol(symbol): if ticker and "." not in ticker: ticker = f"{feed}.{ticker}" - if ASSET_TYPE == "crypto" and "-" not in ticker and ticker != "*": + if ASSET_TYPE == "crypto" and "-" not in ticker and "*" not in ticker: ticker = ticker[:3] + "-" + ticker[3:] - elif ASSET_TYPE == "fx" and "/" not in ticker and ticker != "*": + elif ASSET_TYPE == "fx" and "/" not in ticker and "*" not in ticker: ticker = ticker[:3] + "/" + ticker[3:] elif ASSET_TYPE == "fx" and "-" in ticker: ticker = ticker.replace("-", "/") From 632342072a730377a74814d52025e90eb8b802da Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sun, 10 Nov 2024 21:46:00 -0800 Subject: [PATCH 017/119] add symbol error handling and move symbol to provider models for custom docstrings. --- .../websockets/openbb_websockets/client.py | 28 +++++- .../websockets/openbb_websockets/models.py | 3 - .../openbb_websockets/websockets_router.py | 6 +- .../models/websocket_connection.py | 85 +++++++++++++++---- .../openbb_polygon/utils/websocket_client.py | 82 ++++++++++-------- 5 files changed, 145 insertions(+), 59 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 85a0ee8c1c2e..234a553e2d12 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -185,7 +185,7 @@ async def _setup_database(self) -> None: return await setup_database(self.results_path, self.table_name) def _log_provider_output(self, output_queue) -> None: - """Log output from the provider server queue.""" + """Log output from the provider logger, handling exceptions, errors, and messages that are not data.""" # pylint: disable=import-outside-toplevel import json # noqa import queue @@ -198,7 +198,7 @@ def _log_provider_output(self, output_queue) -> None: try: output = output_queue.get(timeout=1) if output: - # Handle raised exceptions from the provider connection thread. + # Handle raised exceptions from the provider connection thread, killing the process if required. if "UnauthorizedError" in output: self._psutil_process.kill() self._process.wait() @@ -241,6 +241,11 @@ def _log_provider_output(self, output_queue) -> None: sys.stdout.write(output + "\n") sys.stdout.flush() break + # We don't kill the process on SymbolError, but raise the exception in the main thread instead. + if "SymbolError" in output: + err = ValueError(output) + self._exception = err + continue output = clean_message(output) output = output + "\n" @@ -296,10 +301,11 @@ def connect(self) -> None: # pylint: disable=import-outside-toplevel import json # noqa import os + import psutil import queue import subprocess import threading - import psutil + import time if self.is_running: self.logger.info("Provider connection already running.") @@ -354,6 +360,13 @@ def connect(self) -> None: self._log_thread.daemon = True self._log_thread.start() + time.sleep(0.75) + + if self._exception is not None: + exc = getattr(self, "_exception", None) + self._exception = None + raise exc + if not self.is_running: self.logger.error("The provider server failed to start.") @@ -392,11 +405,18 @@ def disconnect(self) -> None: def subscribe(self, symbol) -> None: """Subscribe to a new symbol or list of symbols.""" # pylint: disable=import-outside-toplevel - import json + import json # noqa + import time + from openbb_core.app.model.abstract.error import OpenBBError ticker = symbol if isinstance(symbol, list) else symbol.split(",") msg = {"event": "subscribe", "symbol": ticker} self.send_message(json.dumps(msg)) + time.sleep(0.1) + if self._exception: + exc = getattr(self, "_exception", None) + self._exception = None + raise OpenBBError(exc) old_symbols = self.symbol.split(",") new_symbols = list(set(old_symbols + ticker)) self._symbol = ",".join(new_symbols) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/models.py b/openbb_platform/extensions/websockets/openbb_websockets/models.py index 19d59769d5cc..48ca4266595e 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/models.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/models.py @@ -18,9 +18,6 @@ class WebSocketQueryParams(QueryParams): """Query parameters for WebSocket connection creation.""" - symbol: str = Field( - description=QUERY_DESCRIPTIONS.get("symbol", ""), - ) name: str = Field( description="Name to assign the client connection.", ) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index bd061c032b71..1b9473436d66 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -165,8 +165,10 @@ async def subscribe( if symbols and symbol in symbols: raise OpenBBError(f"Client {name} already subscribed to {symbol}.") - client.subscribe(symbol) - await asyncio.sleep(1) + try: + client.subscribe(symbol) + except OpenBBError as e: + raise e from e if client.is_running: return OBBject(results=f"Added {symbol} to client {name} connection.") diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 5aa96ed83623..7cc4830b3e47 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -140,10 +140,52 @@ class PolygonWebSocketQueryParams(WebSocketQueryParams): } symbol: str = Field( - description=QUERY_DESCRIPTIONS.get("symbol", "") - + " All feeds, except Options, support the wildcard symbol, '*', to subscribe to all symbols." - + " For Options, the OCC contract symbol is used to subscribe up to 1000 individual contracts" - + " per connection." + description="Polygon symbol to get data for." + + " All feeds, except Options, support the wildcard symbol, '*', for all symbols." + + "\n Options symbols are the OCC contract symbol and support up to 1000 individual contracts" + + " per connection. Crypto and FX symbols should be entered as a pair, i.e., 'BTCUSD', 'JPYUSD'." + + "\n Multiple feeds can be subscribed to - i.e, aggs and quote - by formatting the symbol" + + " with prefixes described below. No prefix required for symbols within the 'feed' parameter." + + " All subscribed symbols must be from the same 'asset_type' for a single connection." + + """ \n + Stock + ----- + - aggs_min: AM. + - aggs_sec: AS. + - trade: T. + - quote: Q. + - fmv: FMV. + + Options + ------- + - aggs_min: AM.O: + - aggs_sec: A.O: + - trade: T.O: + - quote: Q.O: + - fmv: FMV.O: + + Index + ----- + - aggs_min: AM.I: + - aggs_sec: A.I: + - value: V.I: + + Crypto + ------ + - aggs_min: XA. + - aggs_sec: XAS. + - trade: XT. + - quote: XQ. + - l2: XL2. + - fmv: FMV. + + FX + -- + - aggs_min: CA. + - aggs_sec: CAS. + - quote: C. + - fmv: FMV. + \n\n""" ) asset_type: Literal[ "stock", @@ -159,10 +201,12 @@ class PolygonWebSocketQueryParams(WebSocketQueryParams): description="The asset type associated with the symbol(s)." + " Choose from: stock, stock_delayed, fx, crypto.", ) - feed: Literal["aggs_min", "aggs_sec", "trade", "quote", "l2"] = Field( - default="aggs_sec", - description="The feed type to subscribe to. Choose from: aggs_min, aggs_sec, trade, quote, l2." - + "l2 is only available for crypto.", + feed: Literal["aggs_min", "aggs_sec", "trade", "quote", "l2", "fmv", "value"] = ( + Field( + default="aggs_sec", + description="The feed type to subscribe to. Choose from: aggs_min, aggs_sec, trade, quote, l2, fmv, value" + + "l2 is only available for crypto. value is only available for index.", + ) ) @model_validator(mode="before") @@ -171,15 +215,26 @@ def _validate_feed(cls, values): """Validate the feed.""" feed = values.get("feed") asset_type = values.get("asset_type") - if asset_type == "fx" and feed in ["trade", "l2"]: - raise ValueError("FX does not support the trade or l2 feeds.") - if asset_type in ["stock", "stock_delayed"] and feed == "l2": - raise ValueError("Stock does not support the l2 feed.") - if asset_type == "index" and feed in ["trade", "quote", "l2", "fmv"]: + if asset_type == "fx" and feed in ["trade", "l2", "value"]: + raise ValueError(f"FX does not support the {feed} feed.") + if asset_type in [ + "stock", + "stock_delayed", + "options", + "options_delayed", + ] and feed in ["l2", "value"]: raise ValueError( - "Index does not support the trade, quote, l2, or fmv feeds." + f"Asset type, {asset_type}, does not support the {feed} feed." ) - + if asset_type in ["index", "index_delayed"] and feed in [ + "trade", + "quote", + "l2", + "fmv", + ]: + raise ValueError(f"Index does not support the {feed} feed.") + if asset_type == "crypto" and feed == "value": + raise ValueError(f"Crypto does not support the {feed} feed.") return values diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index f0c21e379d2d..74b2e428c39e 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -8,7 +8,6 @@ import websockets import websockets.exceptions -from openbb_core.provider.utils.errors import UnauthorizedError from openbb_polygon.models.websocket_connection import ( FEED_MAP, PolygonWebSocketData, @@ -40,10 +39,11 @@ async def handle_symbol(symbol): feed = FEED_MAP.get(ASSET_TYPE, {}).get(FEED) for s in symbols: - if ASSET_TYPE == "options" and "*" in s: - logger.error( - "PROVIDER INFO: Options symbols do not support wildcards." + if ASSET_TYPE in ["options", "options_delayed"] and "*" in s: + symbol_error = ( + f"SymbolError -> {symbol}: Options symbols do not support wildcards." ) + logger.error(symbol_error) continue if s == "*": @@ -53,12 +53,9 @@ async def handle_symbol(symbol): if "." in s: _check = s.split(".")[0] if _check not in list(FEED_MAP.get(ASSET_TYPE, {}).values()): - logger.error( - "PROVIDER INFO: Invalid feed, %s, for asset type, %s", - _check, - ASSET_TYPE, + raise ValueError( + f"SymbolError -> Invalid feed, {_check}, for asset type, {ASSET_TYPE}" ) - continue ticker = s.upper() @@ -66,7 +63,7 @@ async def handle_symbol(symbol): ticker = f"{feed}.{ticker}" if ASSET_TYPE == "crypto" and "-" not in ticker and "*" not in ticker: - ticker = ticker[:3] + "-" + ticker[3:] + ticker = ticker[:-3] + "-" + ticker[-3:] elif ASSET_TYPE == "fx" and "/" not in ticker and "*" not in ticker: ticker = ticker[:3] + "/" + ticker[3:] elif ASSET_TYPE == "fx" and "-" in ticker: @@ -74,7 +71,7 @@ async def handle_symbol(symbol): elif ( ASSET_TYPE in ["index", "index_delayed"] and ":" not in ticker - and ticker != "*" + and "*" not in ticker ): _feed, _ticker = ticker.split(".") if "." in ticker else (feed, ticker) ticker = f"{_feed}.I:{_ticker}" @@ -98,13 +95,18 @@ async def login(websocket, api_key): if msg.get("status") == "connected": logger.info("PROVIDER INFO: %s", msg.get("message")) continue + if "Your plan doesn't include websocket access" in msg.get("message"): + err = f"UnauthorizedError -> {msg.get('message')}" + logger.error(err) + sys.exit(1) + break if msg.get("status") != "auth_success": err = ( f"UnauthorizedError -> {msg.get('status')} -> {msg.get('message')}" ) logger.error(err) sys.exit(1) - raise UnauthorizedError(f"{msg.get('status')} -> {msg.get('message')}") + break logger.info("PROVIDER INFO: %s", msg.get("message")) except Exception as e: logger.error("PROVIDER ERROR: %s -> %s", e.__class__.__name__, e.args[0]) @@ -113,7 +115,11 @@ async def login(websocket, api_key): async def subscribe(websocket, symbol, event): """Subscribe or unsubscribe to a symbol.""" - ticker = await handle_symbol(symbol) + try: + ticker = await handle_symbol(symbol) + except ValueError as e: + logger.error(e) + return subscribe_event = f'{{"action":"{event}","params":"{ticker}"}}' try: await websocket.send(subscribe_event) @@ -122,7 +128,7 @@ async def subscribe(websocket, symbol, event): logger.error(msg) -async def read_stdin_and_queue_commands(): +async def read_stdin(command_queue): """Read from stdin and queue commands.""" while True: line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline) @@ -138,21 +144,31 @@ async def read_stdin_and_queue_commands(): logger.error("Invalid JSON received from stdin") +async def process_stdin_queue(websocket): + """Process the command queue.""" + while True: + command = await command_queue.dequeue() + symbol = command.get("symbol") + event = command.get("event") + if symbol and event: + await subscribe(websocket, symbol, event) + + async def process_message(message, results_path, table_name, limit): """Process the WebSocket message.""" messages = message if isinstance(message, list) else [message] for msg in messages: - if "Your plan doesn't include websocket access" in msg.get("message"): - err = f"UnauthorizedError -> {msg.get('message')}" - logger.error(err) - sys.exit(1) - raise UnauthorizedError(msg.get("message")) - if "status" in msg or "message" in msg: if "status" in msg and msg["status"] == "error": err = msg.get("message") raise websockets.WebSocketException(err) if "message" in msg and msg.get("message"): + if "Your plan doesn't include websocket access" in msg.get("message"): + err = f"UnauthorizedError -> {msg.get('message')}" + logger.error(err) + sys.exit(1) + break + logger.info("PROVIDER INFO: %s", msg.get("message")) elif msg and "ev" in msg and "status" not in msg: try: @@ -179,9 +195,7 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim lambda message: process_message(message, results_path, table_name, limit) ) ) - - stdin_task = asyncio.create_task(read_stdin_and_queue_commands()) - + stdin_task = asyncio.create_task(read_stdin(command_queue)) try: connect_kwargs = CONNECT_KWARGS.copy() if "ping_timeout" not in connect_kwargs: @@ -191,6 +205,7 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim try: async with websockets.connect(url, **connect_kwargs) as websocket: + await login(websocket, api_key) response = await websocket.recv() messages = json.loads(response) @@ -200,25 +215,22 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim messages = json.loads(response) await process_message(messages, results_path, table_name, limit) while True: - ws_task = asyncio.create_task(websocket.recv()) - cmd_task = asyncio.create_task(command_queue.dequeue()) - + cmd_task = asyncio.create_task(process_stdin_queue(websocket)) + msg_task = asyncio.create_task(websocket.recv()) done, pending = await asyncio.wait( - [ws_task, cmd_task], return_when=asyncio.FIRST_COMPLETED + [cmd_task, msg_task], + return_when=asyncio.FIRST_COMPLETED, ) for task in pending: task.cancel() for task in done: - if task == ws_task: + if task == cmd_task: + await cmd_task + elif task == msg_task: messages = task.result() await asyncio.shield(queue.enqueue(json.loads(messages))) - elif task == cmd_task: - command = task.result() - symbol = command.get("symbol") - event = command.get("event") - if symbol and event: - await subscribe(websocket, symbol, event) + except websockets.InvalidStatusCode as e: if e.status_code == 404: msg = f"PROVIDER ERROR: {e.__str__()}" @@ -254,7 +266,7 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim sys.exit(1) except Exception as e: - msg = f"Unexpected error -> {e.__class__.__name__}: {e.__str__()}" + msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e.__str__()}" logger.error(msg) sys.exit(1) From 4fcb48ead684c2c1973683204c825efb3587ec3f Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 11 Nov 2024 10:43:08 -0800 Subject: [PATCH 018/119] fix fmp --- .../websockets/openbb_websockets/client.py | 22 ++-- .../openbb_websockets/websockets_router.py | 98 ++++++++------ .../extensions/websockets/pyproject.toml | 1 + .../openbb_fmp/models/websocket_connection.py | 21 +-- .../fmp/openbb_fmp/utils/websocket_client.py | 120 ++++++++++-------- .../models/websocket_connection.py | 12 +- 6 files changed, 158 insertions(+), 116 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 234a553e2d12..b9999f71ca87 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -199,6 +199,7 @@ def _log_provider_output(self, output_queue) -> None: output = output_queue.get(timeout=1) if output: # Handle raised exceptions from the provider connection thread, killing the process if required. + # UnauthorizedError should be raised by the parent thread, but we kill the process here. if "UnauthorizedError" in output: self._psutil_process.kill() self._process.wait() @@ -208,7 +209,9 @@ def _log_provider_output(self, output_queue) -> None: sys.stdout.write(output + "\n") sys.stdout.flush() break - + # ValidationError may occur after the provider connection is established. + # We write to stdout in case the exception can't be raised before the main function returns. + # We kill the connection here. if "ValidationError" in output: self._psutil_process.kill() self._process.wait() @@ -227,7 +230,14 @@ def _log_provider_output(self, output_queue) -> None: sys.stdout.write(msg + "\n") sys.stdout.flush() break - + # We don't kill the process on SymbolError, but raise the exception in the main thread instead. + # This is likely a subscribe event and the connection is already streaming. + if "SymbolError" in output: + err = ValueError(output) + self._exception = err + continue + # Other errors are logged to stdout and the process is killed. + # If the exception is raised by the parent thread, it will be treated as an unexpected error. if ( "server rejected" in output.lower() or "PROVIDER ERROR" in output @@ -241,11 +251,6 @@ def _log_provider_output(self, output_queue) -> None: sys.stdout.write(output + "\n") sys.stdout.flush() break - # We don't kill the process on SymbolError, but raise the exception in the main thread instead. - if "SymbolError" in output: - err = ValueError(output) - self._exception = err - continue output = clean_message(output) output = output + "\n" @@ -360,7 +365,8 @@ def connect(self) -> None: self._log_thread.daemon = True self._log_thread.start() - time.sleep(0.75) + # Give it some startup time to allow the connection to be establised and for exceptions to populate. + time.sleep(2) if self._exception is not None: exc = getattr(self, "_exception", None) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index 1b9473436d66..8e93f5d80a33 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -55,9 +55,7 @@ async def create_connection( obbject = await OBBject.from_query(Query(**locals())) client = obbject.results.client - await asyncio.sleep(1) - - if not client.is_running or client._exception: + if not client.is_running or client._exception is not None: exc = getattr(client, "_exception", None) if exc: client._atexit() @@ -139,7 +137,7 @@ async def clear_results(name: str, auth_token: Optional[str] = None) -> OBBject[ ) async def subscribe( name: str, symbol: str, auth_token: Optional[str] = None -) -> OBBject[str]: +) -> OBBject[WebSocketConnectionStatus]: """Subscribe to a new symbol. Parameters @@ -153,8 +151,8 @@ async def subscribe( Returns ------- - str - The message that the client subscribed to the symbol. + WebSocketConnectionStatus + The status of the client connection. """ if not await check_auth(name, auth_token): raise OpenBBError("Error finding client.") @@ -170,8 +168,10 @@ async def subscribe( except OpenBBError as e: raise e from e + status = await get_status(name) + if client.is_running: - return OBBject(results=f"Added {symbol} to client {name} connection.") + return OBBject(results=WebSocketConnectionStatus(**status)) client.logger.error( f"Client {name} failed to subscribe to {symbol} and is not running." @@ -183,7 +183,7 @@ async def subscribe( ) async def unsubscribe( name: str, symbol: str, auth_token: Optional[str] = None -) -> OBBject[str]: +) -> OBBject[WebSocketConnectionStatus]: """Unsubscribe to a symbol. Parameters @@ -197,22 +197,23 @@ async def unsubscribe( Returns ------- - str - The message that the client unsubscribed from the symbol. + WebSocketConnectionStatus + The status of the client connection. """ if not await check_auth(name, auth_token): raise OpenBBError("Error finding client.") + client = connected_clients[name] symbols = client.symbol.split(",") + if symbol not in symbols: raise OpenBBError(f"Client {name} not subscribed to {symbol}.") + client.unsubscribe(symbol) - # await asyncio.sleep(2) - if client.is_running: - return OBBject(results=f"Client {name} unsubscribed to {symbol}.") - client.logger.error( - f"Client {name} failed to unsubscribe to {symbol} and is not running." - ) + + status = await get_status(name) + + return OBBject(results=WebSocketConnectionStatus(**status)) @router.command( @@ -230,7 +231,7 @@ async def get_client_status( Returns ------- - list[dict] + list[WebSocketConnectionStatus] The status of the client(s). """ if not connected_clients: @@ -272,7 +273,9 @@ async def get_client(name: str, auth_token: Optional[str] = None) -> OBBject: @router.command( methods=["GET"], ) -async def stop_connection(name: str, auth_token: Optional[str] = None) -> OBBject[str]: +async def stop_connection( + name: str, auth_token: Optional[str] = None +) -> OBBject[WebSocketConnectionStatus]: """Stop a the connection to the provider's websocket. Does not stop the broadcast server. Parameters @@ -284,16 +287,16 @@ async def stop_connection(name: str, auth_token: Optional[str] = None) -> OBBjec Returns ------- - str - The message that the provider connection was stopped. + WebSocketConnectionStatus + The status of the client connection. """ if not await check_auth(name, auth_token): raise OpenBBError("Error finding client.") client = connected_clients[name] client.disconnect() - return OBBject( - results=f"Client {name} connection to the provider's websocket was stopped." - ) + status = await get_status(name) + + return OBBject(results=WebSocketConnectionStatus(**status)) @router.command( @@ -301,7 +304,7 @@ async def stop_connection(name: str, auth_token: Optional[str] = None) -> OBBjec ) async def restart_connection( name: str, auth_token: Optional[str] = None -) -> OBBject[str]: +) -> OBBject[WebSocketConnectionStatus]: """Restart a websocket connection. Parameters @@ -313,16 +316,23 @@ async def restart_connection( Returns ------- - str - The message that the client connection was restarted. + WebSocketConnectionStatus + The status of the client connection. """ if name not in connected_clients: raise OpenBBError(f"No active client named, {name}. Use create_connection.") if not await check_auth(name, auth_token): raise OpenBBError("Error finding client.") client = connected_clients[name] - client.connect() - return OBBject(results=f"Client {name} connection was restarted.") + + try: + client.connect() + except OpenBBError as e: + raise e from e + + status = await get_status(name) + + return OBBject(results=WebSocketConnectionStatus(**status)) @router.command( @@ -330,7 +340,7 @@ async def restart_connection( ) async def stop_broadcasting( name: str, auth_token: Optional[str] = None -) -> OBBject[str]: +) -> OBBject[WebSocketConnectionStatus]: """Stop the broadcast server. Parameters @@ -342,8 +352,8 @@ async def stop_broadcasting( Returns ------- - str - The message that the client stopped broadcasting to the address. + WebSocketConnectionStatus + The status of the client connection. """ if name not in connected_clients: raise OpenBBError(f"Client {name} not connected.") @@ -356,7 +366,6 @@ async def stop_broadcasting( if not client.is_broadcasting: raise OpenBBError(f"Client {name} not broadcasting.") - old_address = client.broadcast_address client.stop_broadcasting() if not client.is_running: @@ -366,7 +375,9 @@ async def stop_broadcasting( results=f"Client {name} stopped broadcasting and was not running, client removed." ) - return OBBject(results=f"Client {name} stopped broadcasting to: {old_address}") + status = await get_status(name) + + return OBBject(results=WebSocketConnectionStatus(**status)) @router.command( @@ -378,7 +389,7 @@ async def start_broadcasting( host: str = "127.0.0.1", port: int = 6666, uvicorn_kwargs: Optional[dict[str, Any]] = None, -) -> OBBject[str]: +) -> OBBject[WebSocketConnectionStatus]: """Start broadcasting from a websocket. Parameters @@ -396,23 +407,27 @@ async def start_broadcasting( Returns ------- - str - The message that the client started broadcasting. + WebSocketConnectionStatus + The status of the client connection. """ if name not in connected_clients: raise OpenBBError(f"Client {name} not connected.") + if not await check_auth(name, auth_token): raise OpenBBError("Error finding client.") + client = connected_clients[name] kwargs = uvicorn_kwargs if uvicorn_kwargs else {} client.start_broadcasting(host=host, port=port, **kwargs) await asyncio.sleep(2) + if not client.is_broadcasting: raise OpenBBError(f"Client {name} failed to broadcast.") - return OBBject( - results=f"Client {name} started broadcasting to {client.broadcast_address}." - ) + + status = await get_status(name) + + return OBBject(results=WebSocketConnectionStatus(**status)) @router.command( @@ -435,9 +450,12 @@ async def kill(name: str, auth_token: Optional[str] = None) -> OBBject[str]: """ if not connected_clients: raise OpenBBError("No connections to kill.") - elif name and name not in connected_clients: + + if name and name not in connected_clients: raise OpenBBError(f"Client {name} not connected.") + client = connected_clients[name] client._atexit() del connected_clients[name] + return OBBject(results=f"Clients {name} killed.") diff --git a/openbb_platform/extensions/websockets/pyproject.toml b/openbb_platform/extensions/websockets/pyproject.toml index 62f5ca59fe06..c53ca99fd173 100644 --- a/openbb_platform/extensions/websockets/pyproject.toml +++ b/openbb_platform/extensions/websockets/pyproject.toml @@ -10,6 +10,7 @@ packages = [{ include = "openbb_websockets" }] [tool.poetry.dependencies] python = "^3.9" openbb-core = "^1.3.5" +aiosqlite = "^0.20.0" [build-system] requires = ["poetry-core"] diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py index 193990d280c8..4a5b43c93dc3 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py @@ -32,7 +32,7 @@ class FmpWebSocketQueryParams(WebSocketQueryParams): } symbol: str = Field( - description="The symbol(s) of the asset to fetch data for.", + description="The FMP symbol to get data for.", ) asset_type: Literal["stock", "fx", "crypto"] = Field( default="crypto", @@ -138,14 +138,14 @@ def transform_query(params: dict[str, Any]) -> FmpWebSocketQueryParams: return FmpWebSocketQueryParams(**params) @staticmethod - def extract_data( + async def aextract_data( query: FmpWebSocketQueryParams, credentials: Optional[dict[str, str]], **kwargs: Any, ) -> WebSocketClient: """Extract data from the WebSocket.""" # pylint: disable=import-outside-toplevel - import time + import asyncio api_key = credentials.get("fmp_api_key") if credentials else "" url = URL_MAP[query.asset_type] @@ -155,6 +155,7 @@ def extract_data( kwargs = { "url": url, "api_key": api_key, + "connect_kwargs": query.connect_kwargs, } client = WebSocketClient( @@ -175,17 +176,17 @@ def extract_data( try: client.connect() - - except Exception as e: # pylint: disable=broad-except - client.disconnect() - raise OpenBBError(e) from e - - time.sleep(1) + await asyncio.sleep(2) + if client._exception: + raise client._exception + except OpenBBError as e: + if client.is_running: + client.disconnect() + raise e from e if client.is_running: return client - client.disconnect() raise OpenBBError("Failed to connect to the WebSocket.") @staticmethod diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py index bbcdf57ff871..957e6257d122 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -13,14 +13,17 @@ MessageQueue, get_logger, handle_termination_signal, + handle_validation_error, parse_kwargs, write_to_db, ) +from pydantic import ValidationError logger = get_logger("openbb.websocket.fmp") kwargs = parse_kwargs() -queue = MessageQueue(max_size=kwargs.get("limit", 1000)) +queue = MessageQueue() command_queue = MessageQueue() +CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) async def login(websocket, api_key): @@ -32,17 +35,21 @@ async def login(websocket, api_key): } try: await websocket.send(json.dumps(login_event)) + await asyncio.sleep(1) response = await websocket.recv() - if json.loads(response).get("message") == "Unauthorized": + message = json.loads(response) + if message.get("message") == "Unauthorized": logger.error( - "PROVIDER ERROR: Account not authorized." + "UnauthorizedError -> Account not authorized." " Please check that the API key is entered correctly and is entitled to access." ) sys.exit(1) - msg = json.loads(response).get("message") - logger.info("PROVIDER INFO: %s", msg) - except Exception as e: - logger.error("PROVIDER ERROR: %s", e.args[0]) + else: + msg = message.get("message") + logger.info("PROVIDER INFO: %s", msg) + except Exception as e: # pylint: disable=broad-except + msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e.__str__()}" + logger.error(msg) sys.exit(1) @@ -57,8 +64,9 @@ async def subscribe(websocket, symbol, event): } try: await websocket.send(json.dumps(subscribe_event)) - except Exception as e: - msg = f"PROVIDER ERROR: {e}" + await asyncio.sleep(1) + except Exception as e: # pylint: disable=broad-except + msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e.__str__()}" logger.error(msg) @@ -75,28 +83,43 @@ async def read_stdin_and_queue_commands(): command = json.loads(line.strip()) await command_queue.enqueue(command) except json.JSONDecodeError: - logger.error("Invalid JSON received from stdin") + logger.error("Invalid JSON received from stdin -> %s", line.strip()) + + +async def process_stdin_queue(websocket): + """Process the command queue.""" + while True: + command = await command_queue.dequeue() + symbol = command.get("symbol") + event = command.get("event") + if symbol and event: + await subscribe(websocket, symbol, event) async def process_message(message, results_path, table_name, limit): - result = {} - message = json.loads(message) + """Process the message and write to the database.""" + result: dict = {} + message = json.loads(message) if isinstance(message, str) else message if message.get("event") != "heartbeat": if message.get("event") in ["login", "subscribe", "unsubscribe"]: - msg = f"PROVIDER INFO: {message.get('message')}" - logger.info(msg) - return None - try: - result = FmpWebSocketData.model_validate(message).model_dump_json( - exclude_none=True, exclude_unset=True - ) - except Exception as e: - msg = f"PROVIDER ERROR: Error validating data: {e}" - logger.error(msg) - return None - if result: - await write_to_db(result, results_path, table_name, limit) - return + if "you are not authorized" in message.get("message", "").lower(): + msg = f"UnauthorizedError -> FMP Message: {message['message']}" + logger.error(msg) + else: + msg = f"PROVIDER INFO: {message.get('message')}" + logger.info(msg) + else: + try: + result = FmpWebSocketData.model_validate(message).model_dump_json( + exclude_none=True, exclude_unset=True + ) + except ValidationError as e: + try: + handle_validation_error(logger, e) + except ValidationError: + raise e from e + if result: + await write_to_db(result, results_path, table_name, limit) async def connect_and_stream(url, symbol, api_key, results_path, table_name, limit): @@ -111,30 +134,26 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim stdin_task = asyncio.create_task(read_stdin_and_queue_commands()) try: - async with websockets.connect(url) as websocket: - await login(websocket, api_key) - await subscribe(websocket, symbol, "subscribe") + websocket = await websockets.connect(url) + await login(websocket, api_key) + await subscribe(websocket, symbol, "subscribe") - while True: - ws_task = asyncio.create_task(websocket.recv()) - cmd_task = asyncio.create_task(command_queue.dequeue()) + while True: + ws_task = asyncio.create_task(websocket.recv()) + cmd_task = asyncio.create_task(process_stdin_queue(websocket)) - done, pending = await asyncio.wait( - [ws_task, cmd_task], return_when=asyncio.FIRST_COMPLETED - ) - for task in pending: - task.cancel() - - for task in done: - if task == ws_task: - message = task.result() - await queue.enqueue(message) - elif task == cmd_task: - command = task.result() - symbol = command.get("symbol") - event = command.get("event") - if symbol and event: - await subscribe(websocket, symbol, event) + done, pending = await asyncio.wait( + [ws_task, cmd_task], return_when=asyncio.FIRST_COMPLETED + ) + for task in pending: + task.cancel() + + for task in done: + if task == cmd_task: + await cmd_task + elif task == ws_task: + message = task.result() + await asyncio.shield(queue.enqueue(json.loads(message))) except websockets.ConnectionClosed: logger.info("PROVIDER INFO: The WebSocket connection was closed.") @@ -144,11 +163,12 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim sys.exit(1) except Exception as e: - msg = f"PROVIDER ERROR: Unexpected error -> {e}" + msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e.__str__()}" logger.error(msg) sys.exit(1) finally: + await websocket.close() handler_task.cancel() stdin_task.cancel() await asyncio.gather(handler_task, stdin_task, return_exceptions=True) @@ -180,7 +200,7 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim logger.error("PROVIDER ERROR: WebSocket connection closed") except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: {e.args[0]}" + msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e.__str__()}" logger.error(msg) finally: diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 7cc4830b3e47..0cea4380e14e 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -1205,9 +1205,6 @@ def extract_data( **kwargs: Any, ) -> WebSocketClient: """Extract data from the WebSocket.""" - # pylint: disable=import-outside-toplevel - import time - api_key = credentials.get("polygon_api_key") if credentials else "" url = URL_MAP[query.asset_type] @@ -1239,11 +1236,10 @@ def extract_data( try: client.connect() - except Exception as e: - client.disconnect() - raise OpenBBError(e) from e - - time.sleep(1) + except OpenBBError as e: + if client.is_running: + client.disconnect() + raise e from e if client._exception: raise client._exception from client._exception From 47f10ef44aa5968b38c0c07c99672755c0ab7486 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 11 Nov 2024 10:49:20 -0800 Subject: [PATCH 019/119] typo --- .../extensions/websockets/openbb_websockets/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index b9999f71ca87..a5ffca52aec5 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -365,7 +365,7 @@ def connect(self) -> None: self._log_thread.daemon = True self._log_thread.start() - # Give it some startup time to allow the connection to be establised and for exceptions to populate. + # Give it some startup time to allow the connection to be established and for exceptions to populate. time.sleep(2) if self._exception is not None: From cbbd16a2e94211eed43c751f7824f7781154256b Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 11 Nov 2024 11:20:22 -0800 Subject: [PATCH 020/119] run _setup_database in a thread --- .../websockets/openbb_websockets/client.py | 64 +++++++++---------- 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index a5ffca52aec5..0d416d3d6f60 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -147,17 +147,8 @@ def __init__( # noqa: PLR0913 atexit.register(self._atexit) - # Set up the SQLite database and table. - # Loop handling is for when the class is used directly instead of from the app or API. try: - loop = asyncio.get_event_loop() - except (RuntimeError, RuntimeWarning): - loop = asyncio.new_event_loop() - try: - if loop.is_running(): - loop.create_task(self._setup_database()) - else: - asyncio.run(self._setup_database()) + self._setup_database() except DatabaseError as e: self.logger.error("Error setting up the SQLite database and table: %s", e) @@ -177,12 +168,38 @@ def _atexit(self) -> None: if os.path.exists(self.results_file): os.remove(self.results_file) - async def _setup_database(self) -> None: + def _setup_database(self) -> None: """Set up the SQLite database and table.""" # pylint: disable=import-outside-toplevel + import asyncio # noqa + import threading from openbb_websockets.helpers import setup_database - return await setup_database(self.results_path, self.table_name) + def run_in_new_loop(): + """Run setup in new event loop.""" + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete( + setup_database(self.results_path, self.table_name) + ) + finally: + loop.close() + + def run_in_thread(): + """Run setup in separate thread.""" + thread = threading.Thread(target=run_in_new_loop) + thread.start() + thread.join() + + try: + try: + loop = asyncio.get_running_loop() # noqa + run_in_thread() + except RuntimeError: + run_in_new_loop() + finally: + return def _log_provider_output(self, output_queue) -> None: """Log output from the provider logger, handling exceptions, errors, and messages that are not data.""" @@ -485,34 +502,13 @@ def results(self): """Clear results stored from the WebSocket stream.""" # pylint: disable=import-outside-toplevel import sqlite3 # noqa - import asyncio - import threading - - def run_in_new_loop(): - """Run setup in new event loop.""" - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - loop.run_until_complete(self._setup_database()) - finally: - loop.close() - - def run_in_thread(): - """Run setup in separate thread.""" - thread = threading.Thread(target=run_in_new_loop) - thread.start() - thread.join() try: with sqlite3.connect(self.results_path) as conn: conn.execute(f"DELETE FROM {self.table_name}") # noqa conn.commit() - try: - loop = asyncio.get_running_loop() # noqa - run_in_thread() - except RuntimeError: - run_in_new_loop() + self._setup_database() self.logger.info( "Results cleared from table %s in %s", From 6b640df493377477fa97cbed141460ec2233e02b Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 11 Nov 2024 16:35:47 -0800 Subject: [PATCH 021/119] fix some tiingo weirdness --- .../models/websocket_connection.py | 67 +++++++++------ .../openbb_tiingo/utils/websocket_client.py | 82 +++++++++++-------- 2 files changed, 89 insertions(+), 60 deletions(-) diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py index 7291d1cd24ce..0aef53649aca 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py @@ -14,7 +14,7 @@ WebSocketData, WebSocketQueryParams, ) -from pydantic import Field, field_validator +from pydantic import Field, field_validator, model_validator URL_MAP = { "stock": "wss://api.tiingo.com/iex", @@ -89,25 +89,22 @@ class TiingoWebSocketQueryParams(WebSocketQueryParams): } symbol: str = Field( - description=QUERY_DESCRIPTIONS.get("symbol", "") + "Use '*' for all symbols.", + description=QUERY_DESCRIPTIONS.get("symbol", "") + " Use '*' for all symbols.", ) asset_type: Literal["stock", "fx", "crypto"] = Field( default="crypto", - description="The asset type for the feed.", + description="The asset type for the feed. Choices are 'stock', 'fx', or 'crypto'.", ) feed: Literal["trade", "trade_and_quote"] = Field( default="trade_and_quote", - description="The type of data feed to subscribe to. FX only supports quote.", + description="The type of data feed to subscribe to. FX only supports quote." + + " Choices are 'trade' or 'trade_and_quote'.", ) class TiingoWebSocketData(WebSocketData): """Tiingo WebSocket data model.""" - timestamp: Optional[int] = Field( - default=None, - description="Nanoseconds since POSIX time UTC.", - ) type: Literal["quote", "trade", "break"] = Field( description="The type of data.", ) @@ -175,25 +172,35 @@ def _valiidate_data_type(cls, v): "quote" if v == "Q" else "trade" if v == "T" else "break" if v == "B" else v ) - @field_validator("date", mode="before", check_fields=False) + @field_validator("date", "timestamp", mode="before", check_fields=False) def _validate_date(cls, v): """Validate the date.""" # pylint: disable=import-outside-toplevel + from pandas import to_datetime from pytz import timezone if isinstance(v, str): - return datetime.fromisoformat(v) - try: - return datetime.fromtimestamp(v / 1000) - except Exception: - if isinstance(v, (int, float)): - # Check if the timestamp is in nanoseconds and convert to seconds - if v > 1e12: - v = v / 1e9 # Convert nanoseconds to seconds - dt = datetime.fromtimestamp(v) - dt = timezone("America/New_York").localize(dt) - return dt - return v + dt = to_datetime(v, utc=True).tz_convert(timezone("America/New_York")) + else: + try: + dt = datetime.fromtimestamp(v / 1000) + except Exception: + if isinstance(v, (int, float)): + # Check if the timestamp is in nanoseconds and convert to seconds + if v > 1e12: + v = v / 1e9 # Convert nanoseconds to seconds + dt = datetime.fromtimestamp(v) + else: + dt = v + + return dt + + @model_validator(mode="before") + @classmethod + def _validate_model(cls, values): + """Validate the model.""" + + return values class TiingoWebSocketConnection(WebSocketConnection): @@ -244,6 +251,7 @@ async def aextract_data( "url": url, "api_key": api_key, "threshold_level": threshold_level, + "connect_kwargs": query.connect_kwargs, } client = WebSocketClient( @@ -264,13 +272,18 @@ async def aextract_data( try: client.connect() - # Unhandled exceptions are caught and raised as OpenBBError - except Exception as e: # pylint: disable=broad-except - client.disconnect() - raise OpenBBError(e) from e + except OpenBBError as e: + if client.is_running: + client.disconnect() + raise e from e + + await sleep(1) - # Wait for the connection to be established before returning. - await sleep(2) + if client._exception: + exc = getattr(client, "_exception", None) + client._exception = None + client._atexit() + raise OpenBBError(exc) if client.is_running: return client diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index 5c31928890b3..cf256c146a0e 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -13,9 +13,11 @@ MessageQueue, get_logger, handle_termination_signal, + handle_validation_error, parse_kwargs, write_to_db, ) +from pydantic import ValidationError # These are the data array definitions. IEX_FIELDS = [ @@ -66,10 +68,11 @@ "ask_size", "ask_price", ] -subscription_id = None +SUBSCRIPTION_ID = "" queue = MessageQueue() logger = get_logger("openbb.websocket.tiingo") kwargs = parse_kwargs() +CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) # Subscribe and unsubscribe events are handled in a separate connection using the subscription_id set by the login event. @@ -77,7 +80,7 @@ async def update_symbols(symbol, event): """Update the symbols to subscribe to.""" url = kwargs["url"] - if not subscription_id: + if not SUBSCRIPTION_ID: logger.error( "PROVIDER ERROR: Must be assigned a subscription ID to update symbols. Try logging in." ) @@ -87,7 +90,7 @@ async def update_symbols(symbol, event): "eventName": event, "authorization": kwargs["api_key"], "eventData": { - "subscriptionId": subscription_id, + "subscriptionId": SUBSCRIPTION_ID, "tickers": symbol, }, } @@ -96,13 +99,10 @@ async def update_symbols(symbol, event): await websocket.send(json.dumps(update_event)) response = await websocket.recv() message = json.loads(response) - if message.get("response", {}).get("code") != 200: - logger.error(f"PROVIDER ERROR: {message}") - else: - msg = ( - f"PROVIDER INFO: {message.get('response', {}).get('message')}. " - f"Subscribed to symbols: {message.get('data', {}).get('tickers')}" - ) + if "tickers" in message.get("data", {}): + tickers = message["data"]["tickers"] + threshold_level = message["data"].get("thresholdLevel") + msg = f"PROVIDER INFO: Subscribed to {tickers} with threshold level {threshold_level}" logger.info(msg) @@ -124,10 +124,11 @@ async def read_stdin_and_update_symbols(): async def process_message(message, results_path, table_name, limit): - result = {} - data_message = {} - message = json.loads(message) - msg = "" + """Process the message and write to the database.""" + result: dict = {} + data_message: dict = {} + message = message if isinstance(message, (dict, list)) else json.loads(message) + msg: str = "" if message.get("messageType") == "E": response = message.get("response", {}) msg = f"PROVIDER ERROR: {response.get('code')}: {response.get('message')}" @@ -147,20 +148,21 @@ async def process_message(message, results_path, table_name, limit): msg = f"PROVIDER INFO: Authorization: {response.get('message')}" logger.info(msg) if message.get("data", {}).get("subscriptionId"): - global subscription_id + global SUBSCRIPTION_ID + SUBSCRIPTION_ID = message["data"]["subscriptionId"] - subscription_id = message["data"]["subscriptionId"] + if "tickers" in response.get("data", {}): + tickers = message["data"]["tickers"] + threshold_level = message["data"].get("thresholdLevel") + msg = f"PROVIDER INFO: Subscribed to {tickers} with threshold level {threshold_level}" + logger.info(msg) - if "tickers" in message.get("data", {}): - tickers = message["data"]["tickers"] - threshold_level = message["data"].get("thresholdLevel") - msg = f"PROVIDER INFO: Subscribed to {tickers} with threshold level {threshold_level}" - logger.info(msg) elif message.get("messageType") == "A": data = message.get("data", []) service = message.get("service") if service == "iex": data_message = {IEX_FIELDS[i]: data[i] for i in range(len(data))} + _ = data_message.pop("timestamp", None) elif service == "fx": data_message = {FX_FIELDS[i]: data[i] for i in range(len(data))} elif service == "crypto_data": @@ -179,10 +181,12 @@ async def process_message(message, results_path, table_name, limit): result = TiingoWebSocketData.model_validate(data_message).model_dump_json( exclude_none=True, exclude_unset=True ) - except Exception as e: - msg = f"PROVIDER ERROR: Error validating data: {e}" - logger.error(msg) - return + except ValidationError as e: + try: + handle_validation_error(logger, e) + except ValidationError: + raise e from e + if result: await write_to_db(result, results_path, table_name, limit) return @@ -207,33 +211,45 @@ async def connect_and_stream( subscribe_event = { "eventName": "subscribe", "authorization": api_key, - "eventData": {"thresholdLevel": threshold_level, "tickers": ticker}, + "eventData": { + "thresholdLevel": threshold_level, + "tickers": ticker, + }, } + connect_kwargs = CONNECT_KWARGS.copy() + if "ping_timeout" not in connect_kwargs: + connect_kwargs["ping_timeout"] = None + if "close_timeout" not in connect_kwargs: + connect_kwargs["close_timeout"] = None + try: - async with websockets.connect( - url, ping_interval=20, ping_timeout=20, max_queue=1000 - ) as websocket: + async with websockets.connect(url, **connect_kwargs) as websocket: logger.info("PROVIDER INFO: WebSocket connection established.") await websocket.send(json.dumps(subscribe_event)) while True: message = await websocket.recv() await queue.enqueue(message) + except UnauthorizedError as e: + logger.error(str(e)) + sys.exit(1) + except websockets.ConnectionClosed as e: msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e.reason}" logger.info(msg) # Attempt to reopen the connection + logger.info("PROVIDER INFO: Attempting to reconnect after five seconds...") await asyncio.sleep(5) await connect_and_stream( url, symbol, threshold_level, api_key, results_path, table_name, limit ) except websockets.WebSocketException as e: - logger.error(e) + logger.error(str(e)) sys.exit(1) except Exception as e: - msg = f"PROVIDER ERROR: Unexpected error -> {e}" + msg = f"Unexpected error -> {e.__class__.__name__}: {e.__str__()}" logger.error(msg) sys.exit(1) @@ -268,10 +284,10 @@ async def connect_and_stream( loop.run_forever() except (KeyboardInterrupt, websockets.ConnectionClosed): - logger.error("PROVIDER ERROR: WebSocket connection closed") + logger.error("PROVIDER ERROR: WebSocket connection closed") except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: {e.args[0]}" + msg = f"Unexpected error -> {e.__class__.__name__}: {e.__str__()}" logger.error(msg) finally: From 8712e0569fe35e8cefd46702772d2cfee515e918 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 11 Nov 2024 16:59:57 -0800 Subject: [PATCH 022/119] raise exc as OpenBBError in client.connect --- .../extensions/websockets/openbb_websockets/client.py | 3 ++- .../openbb_polygon/models/websocket_connection.py | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 0d416d3d6f60..72a720b3ce5d 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -328,6 +328,7 @@ def connect(self) -> None: import subprocess import threading import time + from openbb_core.app.model.abstract.error import OpenBBError if self.is_running: self.logger.info("Provider connection already running.") @@ -388,7 +389,7 @@ def connect(self) -> None: if self._exception is not None: exc = getattr(self, "_exception", None) self._exception = None - raise exc + raise OpenBBError(exc) if not self.is_running: self.logger.error("The provider server failed to start.") diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 0cea4380e14e..c333ddfaf167 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -650,7 +650,9 @@ class PolygonStockTradeWebSocketData(WebSocketData): "trf_id": "trfi", "tape": "z", "price": "p", + "size": "s", "conditions": "c", + "trf_timestamp": "trft", } type: str = Field( @@ -658,7 +660,7 @@ class PolygonStockTradeWebSocketData(WebSocketData): ) date: datetime = Field( description=DATA_DESCRIPTIONS.get("date", "") - + "The end of the aggregate window.", + + "The SIP timestamp of the trade.", ) symbol: str = Field( description=DATA_DESCRIPTIONS.get("symbol", ""), @@ -667,6 +669,9 @@ class PolygonStockTradeWebSocketData(WebSocketData): description="The price of the trade.", json_schema_extra={"x-unit_measurement": "currency"}, ) + size: float = Field( + description="The size of the trade.", + ) exchange: str = Field( description="The exchange where the trade originated.", ) @@ -677,7 +682,7 @@ class PolygonStockTradeWebSocketData(WebSocketData): default=None, description="The conditions of the trade.", ) - trf_id: Optional[str] = Field( + trf_id: Optional[int] = Field( default=None, description="The ID for the Trade Reporting Facility where the trade took place.", ) From 95a1356c8c852bbf700a514c203bfb2885113a03 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 11 Nov 2024 17:18:16 -0800 Subject: [PATCH 023/119] trade size is optional --- .../openbb_polygon/models/websocket_connection.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index c333ddfaf167..989a135097fa 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -343,7 +343,8 @@ class PolygonCryptoTradeWebSocketData(WebSocketData): description="The price of the trade.", json_schema_extra={"x-unit_measurement": "currency"}, ) - size: float = Field( + size: Optional[float] = Field( + default=None, description="The size of the trade.", ) @@ -669,7 +670,8 @@ class PolygonStockTradeWebSocketData(WebSocketData): description="The price of the trade.", json_schema_extra={"x-unit_measurement": "currency"}, ) - size: float = Field( + size: Optional[float] = Field( + default=None, description="The size of the trade.", ) exchange: str = Field( @@ -958,7 +960,8 @@ class PolygonOptionsTradeWebSocketData(WebSocketData): description="The price of the trade.", json_schema_extra={"x-unit_measurement": "currency"}, ) - size: float = Field( + size: Optional[float] = Field( + default=None, description="The size of the trade.", ) exchange: str = Field( From 2e4d6930717b03baf2a85c793b917126dc30c469 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 11 Nov 2024 23:34:26 -0800 Subject: [PATCH 024/119] start some documentation --- .../extensions/websockets/README.md | 472 ++++++++++++++++++ .../websockets/openbb_websockets/broadcast.py | 2 +- .../openbb_websockets/websockets_router.py | 3 +- .../models/websocket_connection.py | 7 + .../openbb_polygon/utils/websocket_client.py | 2 +- .../openbb_tiingo/utils/websocket_client.py | 11 +- 6 files changed, 490 insertions(+), 7 deletions(-) diff --git a/openbb_platform/extensions/websockets/README.md b/openbb_platform/extensions/websockets/README.md index e69de29bb2d1..9f2895d6ef71 100644 --- a/openbb_platform/extensions/websockets/README.md +++ b/openbb_platform/extensions/websockets/README.md @@ -0,0 +1,472 @@ +# OpenBB WebSockets Toolkit + +At the application/API level, the user does not directly interact with the client, or provider stream. +Connections are established as background tasks, and there are not any direct methods for blocking the main thread and command line. + + +## Endpoints + +The extension creates a new router path from the application base - `obb.websockets`, api/v1/websockets for the API. + +Endpoints are for managing the life cycle of one or more provider websocket connections. + +```python +from openbb import obb + +obb.websockets +``` + +```sh +/websockets + clear_results + create_connection + get_client # Not included in API + get_client_status + get_results + kill + restart_connection + start_broadcasting + stop_broadcasting + stop_connection + subscribe + unsubscribe +``` + +> Except for, `get_results`, functions do not return the data or stream. Outputs will be a WebSocketConnectionStatus instance, or a string message. +> All functions, except `create_connection`, assume that a connection has already been establiehd and are referenced by parameters: +> +> |Parameter|Type | Required| Description | +> |:-------|:-----|:--------:|------------:| +> |name |String |Yes |The 'nane' assigned from `create_connection` | +> |auth_token |String |No |The 'auth_token' assigned, if any, from `create_connection` | +> +> Below is an explanation of each function, with `create_connection` representing the bulk of details. + +### create_connection + +All other endpoints require this to be used first. It is the only function mapping to the Provider Interface, and is used to establish a new connection. + +#### Standard Parameters + +|Parameter|Type | Required| Description | +|:-------|:-----|:--------:|------------:| +|provider |String |Yes |Name of the provider - i.e, `"polygon"`, `"fmp"`, `"tiingo"` | +|name |String |Yes |Name to assign the connection. This is the 'name' parameter in the other endpoints.| +|auth_token |String |No |When supplied, the same token must be passed for all future interactions with the connection, or to read from the broadcast server. | +|results_file |String |No |Absolute path to the file for continuous writing. Temp file is created by default. Unless 'save_results' is True, discarded on exit. | +|save_results |Boolean |No |Whether to persist the file after the session ends, default is `False` | +|table_name |String |No |Name of the SQL table to write the results to, consisting of an auto-increment ID and a serialized JSON string of the data. Default is `"records"`| +|limit |Integer |No |Maximum number of records to store in the 'results_file', set as `None` to retain all data messages. Default is `1000`| +|sleep_time |Float |No |Does not impact the provider connection. Time, in seconds, to sleep between checking for new records, default is `0.25` | +|broadcast_host |String |No |IP address for running the broadcast server, default is `"127.0.0.1"` | +|broadcast_port |Integer |No |Port number to bind the broadcasat server to, default is `6666` | +|start_broadcast |Boolean |No |Whether to start the broadcast server immediately, default is `False` | +|connect_kwargs |Dictionary |No |Keyword arguments to pass directly to `websockets.connect()` in the provider module. Also accepts a serialized JSON string dictionary. | + + +#### Provider-Specific Parameters + +Other parameters will be specific to the provider, but there may be common ground. Refer to the function's docstring for more detail. +The table below is not intended as a source of truth. + +|Parameter|Type | Required| Description | +|:-------|:-----|:--------:|------------:| +|symbol |String |Yes |The ticker symbol for the asset - i.e, `"aapl"`, `"usdjpy"`, `"dogeusd"`, `"btcusd,ethusd"`, `"*"`| +|asset_type |String |Yes |The asset type associated with the 'symbol'. Choices vary by provider, but typically include [`"stock"`, `"fx"`, `"crypto"`] | +|feed |String |No |The particular feed to subscribe to, if available. Choices vary by provider, but might include [`"trade"`, `"quote"`] | + +Availability will depend on the access level permitted by the provider's API key. + +#### Usage + +```python +conn = obb.websockets.create_connection(provider="tiingo", asset_type="crypto", symbol="*", feed="trade", start_broadcast=True) + +conn +``` + +```sh +PROVIDER INFO: WebSocket connection established. + +PROVIDER INFO: Authorization: Success + +BROADCAST INFO: Stream results from ws://127.0.0.1:6666 + +OBBject[T] + +id: 06732d37-fe11-744c-8000-072414ba1cdd +results: {'name': 'crypto_tiingo', 'auth_required': False, 'subscribed_symbols': '*... +provider: tiingo +warnings: None +chart: None +extra: {'metadata': {'arguments': {'provider_choices': {'provider': 'tiingo'}, 'sta... +``` + +```python +conn.results.model_dump() +``` + +```sh +{'name': 'crypto_tiingo', + 'auth_required': False, + 'subscribed_symbols': '*', + 'is_running': True, + 'provider_pid': 5810, + 'is_broadcasting': True, + 'broadcast_address': 'ws://127.0.0.1:6666', + 'broadcast_pid': 5813, + 'results_file': '/var/folders/kc/j2lm7bkd5dsfqqnvz259gm6c0000gn/T/tmpwb4jslbg', + 'table_name': 'records', + 'save_results': False} +``` + +All of the currently captured data can be dumped with the `get_results` endpoint. The return will be the typical data response object. + +```python +obb.websockets.get_results("crypto_tiingo").to_df().iloc[-5:] +``` + +| date | symbol | type | exchange | last_price | last_size | +|:---------------------------------|:---------|:-------|:-----------|-------------:|------------:| +| 2024-11-11 23:13:19.753398-05:00 | gfiusd | trade | gdax | 1.89012 | 257.12 | +| 2024-11-11 23:13:19.757000-05:00 | ondousdt | trade | mexc | 0.930851 | 3508.35 | +| 2024-11-11 23:13:19.760000-05:00 | neousdt | trade | huobi | 12.31 | 13.9489 | +| 2024-11-11 23:13:19.793594-05:00 | xrpusd | trade | gdax | 0.60433 | 4676.46 | +| 2024-11-11 23:13:19.819856-05:00 | xlmusd | trade | gdax | 0.11446 | 120.088 | + +#### Listen + +Listen to the stream by opening another terminal window and importing the `listen` function. + +> Using this function within the same session is not recommended because `ctrl-c` will stop the provider and broadcast servers without properly terminating the processes. When this happens, use the `kill` endpoint to finish the job. + + +```python +from openbb_websockets.listen import listen + +listen("ws://127.0.0.1:6666") +``` + +```sh +Listening for messages from ws://127.0.0.1:6666 + +{"date":"2024-11-11T23:51:33.083000-05:00","symbol":"klvusdt","type":"trade","exchange":"huobi","last_price":0.00239,"last_size":8367.4749} + +{"date":"2024-11-11T23:51:33.082000-05:00","symbol":"actsolusdt","type":"trade","exchange":"huobi","last_price":0.5837245604964619,"last_size":1070.2939999999999} +... +``` + +Opening a listener will notify the main thread: + +```sh +BROADCAST INFO: ('127.0.0.1', 59197) - "WebSocket /" [accepted] + +BROADCAST INFO: connection open + +BROADCAST INFO: connection closed +``` + +The provider connection can be stopped and restarted without disrupting the broadcast server. +The broadcast server can be terminated without stopping the provider connection. + + +### clear_results + +Clears the items written to `results_file`. The connection can be running or stopped and does not terminate writing or reading. + +#### Example + +```python +obb.websockets.clear_results("crypto_tiingo") +``` + +```sh +Results cleared from table records in /var/folders/kc/j2lm7bkd5dsfqqnvz259gm6c0000gn/T/tmpwb4jslbg + +OBBject[T] + +id: 06732ed2-a72c-758e-8000-b7943259f615 +results: 1001 results cleared from crypto_tiingo. +provider: None +warnings: None +chart: None +extra: {'metadata': {'arguments': {'provider_choices': {}, 'standard_params': {}, '... +``` + + +### get_client + +> Not available from the API. + +This returns the `WebSocketClient` object, and the provider client can be controlled directly as a Python object. Refer to the [Development](README.md#development) section for a detailed explanation of this class. + +#### Example + +```python +client = obb.websockets.get_client("crypto_tiingo").results +``` + +```sh +WebSocketClient(module=['/Users/someuser/miniconda3/envs/obb/bin/python', '-m', 'openbb_tiingo.utils.websocket_client'], symbol=*, is_running=True, provider_pid: 7125, is_broadcasting=True, broadcast_address=ws://127.0.0.1:6666, broadcast_pid: 7128, results_file=/var/folders/kc/j2lm7bkd5dsfqqnvz259gm6c0000gn/T/tmpwb4jslbg, table_name=records, save_results=False) +``` + +```python +print(client.is_running) +client.disconnect() +print(client.is_running) +``` + +```sh +True +Disconnected from the provider WebSocket. +False +``` + +### get_client_status + +Get the current status of an initialized WebSocketConnection. + +#### Example + +```python +obb.websockets.get_client_status("all").to_dict("records") +``` + +```sh +[{'name': 'crypto_tiingo', + 'auth_required': False, + 'subscribed_symbols': '*', + 'is_running': False, + 'provider_pid': None + 'is_broadcasting': True, + 'broadcast_address': 'ws://127.0.0.1:6666', + 'broadcast_pid': 7723, + 'results_file': '/var/folders/kc/j2lm7bkd5dsfqqnvz259gm6c0000gn/T/tmpup7zd_uu', + 'table_name': 'records', + 'save_results': False}, + {'name': 'fx_polygon', + 'auth_required': False, + 'subscribed_symbols': '*', + 'is_running': True, + 'provider_pid': 7773} + 'is_broadcasting': False, + 'broadcast_address': None, + 'broadcast_pid': None, + 'results_file': '/var/folders/kc/j2lm7bkd5dsfqqnvz259gm6c0000gn/T/tmpzs6of15g', + 'table_name': 'records', + 'save_results': False] +``` + +### get_results + +Get the captured records in the `results_file`. + +```python +obb.websockets.get_results("fx_polygon").to_dict("records")[-1] +``` + +```sh +{'date': Timestamp('2024-11-12 01:41:03-0500', tz='UTC-05:00'), + 'symbol': 'CAD/SGD', + 'type': 'C', + 'exchange': 'Currency Banks 1', + 'bid': 0.958360440227192, + 'ask': 0.958407631503548} +``` + +### kill + +Terminate a connection and all of its processes. + +#### Example + +```python +obb.websockets.kill("fx_polygon") +``` + +```sh +Disconnected from the provider WebSocket. + +OBBject[T] + +id: 06732fa3-1df8-7d82-8000-b492686a1b8b +results: Clients fx_polygon killed. +provider: None +warnings: None +chart: None +extra: {'metadata': {'arguments': {'provider_choices': {}, 'standard_params': {}, '... +``` + +### restart_connection + +Restart a connection after running `stop_connection`. + +#### Example + +```python +obb.websockets.restart_connection("crypto_tiingo").results.model_dump() +``` + +```sh +PROVIDER INFO: WebSocket connection established. + +PROVIDER INFO: Authorization: Success + +{'name': 'crypto_tiingo', + 'auth_required': False, + 'subscribed_symbols': '*', + 'is_running': True, + 'provider_pid': 7939, + 'is_broadcasting': True, + 'broadcast_address': 'ws://127.0.0.1:6666', + 'broadcast_pid': 7723, + 'results_file': '/var/folders/kc/j2lm7bkd5dsfqqnvz259gm6c0000gn/T/tmpup7zd_uu', + 'table_name': 'records', + 'save_results': False} +``` + +### start_broadcasting + +Start the broadcast server. + +#### Additional Parameters + +|Parameter|Type | Required| Description | +|:-------|:-----|:--------:|------------:| +|host |String |No |IP address to run the server over, default is `"127.0.0.1"` | +|port |Interger |No |Port to bind the server to, default is `6666` | +|uvicorn_kwargs| Dictionary |No |Additional keyword arguments to pass directly to `uvicorn.run()`. | + +#### Example + +```python +obb.websockets.start_broadcasting("crypto_tiingo").results +``` + +```sh +BROADCAST INFO: Stream results from ws://127.0.0.1:6666 + +WebSocketConnectionStatus(name=crypto_tiingo, auth_required=False, subscribed_symbols=*, is_running=True, provider_pid=7939, is_broadcasting=True, broadcast_address=ws://127.0.0.1:6666, broadcast_pid=8080, results_file=/var/folders/kc/j2lm7bkd5dsfqqnvz259gm6c0000gn/T/tmpup7zd_uu, table_name=records, save_results=False) +``` + +### stop_broadcasting + +Stop the broadcast server. + +#### Example + +```python +obb.websockets.stop_broadcasting("crypto_tiingo").results.model_dump() +``` + +```sh +Stopped broadcasting to: ws://127.0.0.1:6666 + +{'name': 'crypto_tiingo', + 'auth_required': False, + 'subscribed_symbols': '*', + 'is_running': True, + 'provider_pid': 7939, + 'is_broadcasting': False, + 'broadcast_address': None, + 'broadcast_pid': None, + 'results_file': '/var/folders/kc/j2lm7bkd5dsfqqnvz259gm6c0000gn/T/tmpup7zd_uu', + 'table_name': 'records', + 'save_results': False} +``` + +### stop_connection + +Stop the provider websocket connection. + +#### Example + +```python +obb.websockets.stop_connection("crypto_tiingo").results.model_dump() +``` + +```sh +Disconnected from the provider WebSocket. + +{'name': 'crypto_tiingo', + 'auth_required': False, + 'subscribed_symbols': '*', + 'is_running': False, + 'provider_pid': None, + 'is_broadcasting': False, + 'broadcast_address': None, + 'broadcast_pid': None, + 'results_file': '/var/folders/kc/j2lm7bkd5dsfqqnvz259gm6c0000gn/T/tmpup7zd_uu', + 'table_name': 'records', + 'save_results': False} +``` + +### subscribe + +Subscribe to a new symbol(s). Enter multiple symbols as a comma-seperated string. + +#### Example + +```python +obb.websockets.subscribe("fx_polygon", symbol="xauusd") +``` + +```sh +PROVIDER INFO: subscribed to: C.XAU/USD + +OBBject[T] + +id: 06733025-a43d-71ed-8000-981ec3cfb697 +results: {'name': 'fx_polygon', 'auth_required': False, 'subscribed_symbols': 'EURU... +provider: None +warnings: None +chart: None +extra: {'metadata': {'arguments': {'provider_choices': {}, 'standard_params': {}, '... +``` + +### unsubscribe + +Unsubscribe from a symbol(s) + +#### Example + +```python +obb.websockets.unsubscribe("fx_polygon", symbol="xauusd").results.model_dump() +``` + +```sh +PROVIDER INFO: unsubscribed to: C.XAU/USD + +{'name': 'fx_polygon', + 'auth_required': False, + 'subscribed_symbols': 'EURUSD', + 'is_running': True, + 'provider_pid': 8582, + 'is_broadcasting': False, + 'broadcast_address': None, + 'broadcast_pid': None, + 'results_file': '/var/folders/kc/j2lm7bkd5dsfqqnvz259gm6c0000gn/T/tmp1z70a3fw', + 'table_name': 'records', + 'save_results': False} +``` + + + +## Development + +The Python interface and Fast API endpoints are built with importable components that can be used independently of the application. + + +### WebSocketClient + +#### Import + +This is the client used for bidirectional communication with both, the provider connection, and, the broadcast server. + +```python +from openbb_websockets.client import WebSocketClient +``` + + + +...tbc \ No newline at end of file diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py index 29bc7b739b31..cabca40c8829 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -105,7 +105,7 @@ async def websocket_endpoint( # noqa: PLR0915 await stream_task await stdin_task except asyncio.CancelledError: - broadcast_server.logger.info("Stream task cancelled") + broadcast_server.logger.info("INFO: A listener task was cancelled.") except Exception as e: msg = f"Unexpected error while cancelling stream task: {e.__class__.__name__} -> {e}" broadcast_server.logger.error(msg) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index 8e93f5d80a33..d5317bea3733 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -66,6 +66,7 @@ async def create_connection( if hasattr(extra_params, "start_broadcast") and extra_params.start_broadcast: client.start_broadcasting() + await asyncio.sleep(1) client_name = client.name connected_clients[client_name] = client @@ -403,7 +404,7 @@ async def start_broadcasting( port : int The port to broadcast to. Default is 6666. uvicorn_kwargs : Optional[dict[str, Any]] - Additional keyword arguments for passing directly to the uvicorn server. + Additional keyword arguments to pass directly to `uvicorn.run()`. Returns ------- diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 989a135097fa..7aafbc6689ba 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -550,6 +550,13 @@ def _validate_exchange(cls, v): """Validate the exchange.""" return FX_EXCHANGE_MAP.get(v, str(v)) + @model_validator(mode="before") + @classmethod + def _validate_model(cls, values): + """Validate the model.""" + _ = values.pop("i", None) + return values + class PolygonStockAggsWebSocketData(WebSocketData): """Polygon Stock Aggregates WebSocket data model.""" diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index 74b2e428c39e..e22421abe2bd 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -65,7 +65,7 @@ async def handle_symbol(symbol): if ASSET_TYPE == "crypto" and "-" not in ticker and "*" not in ticker: ticker = ticker[:-3] + "-" + ticker[-3:] elif ASSET_TYPE == "fx" and "/" not in ticker and "*" not in ticker: - ticker = ticker[:3] + "/" + ticker[3:] + ticker = ticker[:-3] + "/" + ticker[-3:] elif ASSET_TYPE == "fx" and "-" in ticker: ticker = ticker.replace("-", "/") elif ( diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index cf256c146a0e..cc2e06841262 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -235,7 +235,9 @@ async def connect_and_stream( sys.exit(1) except websockets.ConnectionClosed as e: - msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e.reason}" + msg = ( + f"PROVIDER INFO: The WebSocket connection was closed -> {e.__str__()}" + ) logger.info(msg) # Attempt to reopen the connection logger.info("PROVIDER INFO: Attempting to reconnect after five seconds...") @@ -266,8 +268,7 @@ async def connect_and_stream( loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - for sig in (signal.SIGINT, signal.SIGTERM): - loop.add_signal_handler(sig, handle_termination_signal, logger) + loop.add_signal_handler(signal.SIGTERM, handle_termination_signal, logger) asyncio.run_coroutine_threadsafe( connect_and_stream( @@ -283,7 +284,7 @@ async def connect_and_stream( ) loop.run_forever() - except (KeyboardInterrupt, websockets.ConnectionClosed): + except websockets.ConnectionClosed: logger.error("PROVIDER ERROR: WebSocket connection closed") except Exception as e: # pylint: disable=broad-except @@ -291,4 +292,6 @@ async def connect_and_stream( logger.error(msg) finally: + loop.stop() + loop.close sys.exit(0) From b69cba831bf17a6a5ccfaeea7fc67996f27abcfe Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:35:41 -0800 Subject: [PATCH 025/119] make bid/ask optional in PolygonStockQuoteWebSocketData --- openbb_platform/extensions/websockets/README.md | 2 +- .../models/websocket_connection.py | 17 +++++++++-------- .../polygon/openbb_polygon/utils/constants.py | 2 ++ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/openbb_platform/extensions/websockets/README.md b/openbb_platform/extensions/websockets/README.md index 9f2895d6ef71..5d83dc925ff9 100644 --- a/openbb_platform/extensions/websockets/README.md +++ b/openbb_platform/extensions/websockets/README.md @@ -77,7 +77,7 @@ The table below is not intended as a source of truth. Availability will depend on the access level permitted by the provider's API key. -#### Usage +#### Example ```python conn = obb.websockets.create_connection(provider="tiingo", asset_type="crypto", symbol="*", feed="trade", start_broadcast=True) diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 7aafbc6689ba..d792f2606535 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -8,10 +8,7 @@ from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.abstract.data import Data from openbb_core.provider.abstract.fetcher import Fetcher -from openbb_core.provider.utils.descriptions import ( - DATA_DESCRIPTIONS, - QUERY_DESCRIPTIONS, -) +from openbb_core.provider.utils.descriptions import DATA_DESCRIPTIONS from openbb_polygon.utils.constants import ( CRYPTO_EXCHANGE_MAP, FX_EXCHANGE_MAP, @@ -775,18 +772,22 @@ class PolygonStockQuoteWebSocketData(WebSocketData): bid_exchange: str = Field( description="The exchange where the bid originated.", ) - bid_size: float = Field( + bid_size: Optional[float] = Field( + default=None, description="The size of the bid.", ) - bid: float = Field( + bid: Optional[float] = Field( + default=None, description="The bid price.", json_schema_extra={"x-unit_measurement": "currency"}, ) - ask: float = Field( + ask: Optional[float] = Field( + default=None, description="The ask price.", json_schema_extra={"x-unit_measurement": "currency"}, ) - ask_size: float = Field( + ask_size: Optional[float] = Field( + default=None, description="The size of the ask.", ) ask_exchange: str = Field( diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py b/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py index 9b827c091161..41ab54d99f13 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py @@ -155,9 +155,11 @@ 90: "Syndicate Bid", 91: "Pre Syndicate Bid", 92: "Penalty Bid", + 95: "CQS Generated", } STOCK_QUOTE_INDICATORS = { + 1: "EXCHANGE_ACQUISITION", 601: "NBBO_NO_CHANGE", 602: "NBBO_QUOTE_IS_NBBO", 603: "NBBO_NO_BB_NO_BO", From e93ab2ffd961e4336a7a649b91e70a41bcbd730a Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 12 Nov 2024 12:37:35 -0800 Subject: [PATCH 026/119] got some missing stock quote indicator definitions --- .../openbb_fmp/models/websocket_connection.py | 19 +++++++++------ .../polygon/openbb_polygon/utils/constants.py | 9 ++++++- .../models/websocket_connection.py | 5 ++++ .../openbb_tiingo/utils/websocket_client.py | 24 +++++++++---------- 4 files changed, 36 insertions(+), 21 deletions(-) diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py index 4a5b43c93dc3..68d662e4a16a 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py @@ -69,24 +69,24 @@ class FmpWebSocketData(WebSocketData): type: Literal["quote", "trade", "break"] = Field( description="The type of data.", ) - bid_price: Optional[float] = Field( - default=None, - description="The price of the bid.", - json_schema_extra={"x-unit_measurement": "currency"}, - ) bid_size: Optional[float] = Field( default=None, description="The size of the bid.", ) - ask_size: Optional[float] = Field( + bid_price: Optional[float] = Field( default=None, - description="The size of the ask.", + description="The price of the bid.", + json_schema_extra={"x-unit_measurement": "currency"}, ) ask_price: Optional[float] = Field( default=None, description="The price of the ask.", json_schema_extra={"x-unit_measurement": "currency"}, ) + ask_size: Optional[float] = Field( + default=None, + description="The size of the ask.", + ) last_price: Optional[float] = Field( default=None, description="The last trade price.", @@ -97,6 +97,11 @@ class FmpWebSocketData(WebSocketData): description="The size of the trade.", ) + @field_validator("symbol", mode="before") + def _validate_symbol(cls, v): + """Validate the symbol.""" + return v.upper() + @field_validator("type", mode="before", check_fields=False) def _valiidate_data_type(cls, v): """Validate the data type.""" diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py b/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py index 41ab54d99f13..5501c44ee13a 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/constants.py @@ -159,7 +159,14 @@ } STOCK_QUOTE_INDICATORS = { - 1: "EXCHANGE_ACQUISITION", + 1: "LULD_NBB_NBO_EXECUTABLE", + 22: "LULD_REPUBLISHED_LULD_PRICE_BAND", + 52: "FINANCIAL_STATUS_DEFICIENT", + 301: "SHORT_SALES_RESTRICTION_ACTIVATED", + 302: "SHORT_SALES_RESTRICTION_CONTINUED", + 303: "SHORT_SALES_RESTRICTION_DEACTIVATED", + 304: "SHORT_SALES_RESTRICTION_IN_EFFECT", + 305: "SHORT_SALES_RESTRICTION_MAX", 601: "NBBO_NO_CHANGE", 602: "NBBO_QUOTE_IS_NBBO", 603: "NBBO_NO_BB_NO_BO", diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py index 0aef53649aca..3b083645952d 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py @@ -165,6 +165,11 @@ class TiingoWebSocketData(WebSocketData): description="True if the order is not subject to NMS Rule 611. Only for stock.", ) + @field_validator("symbol", mode="before", check_fields=False) + def _validate_symbol(cls, v): + """Validate the symbol.""" + return v.upper() + @field_validator("type", mode="before", check_fields=False) def _valiidate_data_type(cls, v): """Validate the data type.""" diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index cc2e06841262..b93aa3287209 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -223,16 +223,17 @@ async def connect_and_stream( connect_kwargs["close_timeout"] = None try: - async with websockets.connect(url, **connect_kwargs) as websocket: - logger.info("PROVIDER INFO: WebSocket connection established.") - await websocket.send(json.dumps(subscribe_event)) - while True: - message = await websocket.recv() - await queue.enqueue(message) - - except UnauthorizedError as e: - logger.error(str(e)) - sys.exit(1) + try: + async with websockets.connect(url, **connect_kwargs) as websocket: + logger.info("PROVIDER INFO: WebSocket connection established.") + await websocket.send(json.dumps(subscribe_event)) + while True: + message = await websocket.recv() + await queue.enqueue(message) + + except UnauthorizedError as e: + logger.error(str(e)) + sys.exit(1) except websockets.ConnectionClosed as e: msg = ( @@ -284,9 +285,6 @@ async def connect_and_stream( ) loop.run_forever() - except websockets.ConnectionClosed: - logger.error("PROVIDER ERROR: WebSocket connection closed") - except Exception as e: # pylint: disable=broad-except msg = f"Unexpected error -> {e.__class__.__name__}: {e.__str__()}" logger.error(msg) From 8e6f776147549338288a1a513a5ef8011fb0bfe1 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:07:48 -0800 Subject: [PATCH 027/119] readme file with too much info --- .../extensions/websockets/README.md | 404 +++++++++++++++++- .../providers/fmp/openbb_fmp/__init__.py | 4 +- .../openbb_fmp/models/websocket_connection.py | 21 +- .../fmp/openbb_fmp/utils/websocket_client.py | 2 +- .../openbb_tiingo/utils/websocket_client.py | 2 +- 5 files changed, 410 insertions(+), 23 deletions(-) diff --git a/openbb_platform/extensions/websockets/README.md b/openbb_platform/extensions/websockets/README.md index 5d83dc925ff9..80f515fc1b42 100644 --- a/openbb_platform/extensions/websockets/README.md +++ b/openbb_platform/extensions/websockets/README.md @@ -454,19 +454,413 @@ PROVIDER INFO: unsubscribed to: C.XAU/USD ## Development -The Python interface and Fast API endpoints are built with importable components that can be used independently of the application. +### Provider Interface -### WebSocketClient +Providers can be added to the `create_connection` endpoint by following a slightly modified pattern. +This section outlines the adaptations, but does not contain any code for actually connecting to the provider's websocket. +For details on that part, go to [websocket_client](README.md###websocket_client) section below. -#### Import -This is the client used for bidirectional communication with both, the provider connection, and, the broadcast server. +Here, the Fetcher is used to start the provider client module (in a separate file) and return the client to the router, where it is intercepted and kept alive. + +> The provider client is not returned to the user, only its status. + +In the provider's "/models" folder, we need a file, `my_provider_websoccket_connection.py`, and it will layout nearly the same as any other provider model. + +We will create one additional model, `WebSocketConnection`, which has only one inherited field, 'client', and no other fields are permitted. This is what gets returned to the router. + +We also need another file, in the `utils` folder, `websocket_client.py`. + +Creating the QueryParams and Data models will be in the same style as all the other models, name it 'websocket_connection.py'. + +#### WebSocketQueryParams ```python +"""FMP WebSocket model.""" + +from datetime import datetime +from typing import Any, Literal, Optional + +from openbb_core.app.model.abstract.error import OpenBBError +from openbb_core.provider.abstract.fetcher import Fetcher from openbb_websockets.client import WebSocketClient +from openbb_websockets.models import ( + WebSocketConnection, + WebSocketData, + WebSocketQueryParams, +) +from pydantic import Field, field_validator + +URL_MAP = { + "stock": "wss://websockets.financialmodelingprep.com", + "fx": "wss://forex.financialmodelingprep.com", + "crypto": "wss://crypto.financialmodelingprep.com", +} + + +class FmpWebSocketQueryParams(WebSocketQueryParams): + """FMP WebSocket query parameters.""" + + __json_schema_extra__ = { + "symbol": {"multiple_items_allowed": True}, + "asset_type": { + "multiple_items_allowed": False, + "choices": ["stock", "fx", "crypto"], + }, + } + + symbol: str = Field( + description="The FMP symbol to get data for.", + ) + asset_type: Literal["stock", "fx", "crypto"] = Field( + default="crypto", + description="The asset type, required for the provider URI.", + ) +``` + +#### WebSocketData + +```python +class FmpWebSocketData(WebSocketData): + """FMP WebSocket data model.""" + + __alias_dict__ = { + "symbol": "s", + "date": "t", + "exchange": "e", + "type": "type", + "bid_size": "bs", + "bid_price": "bp", + "ask_size": "as", + "ask_price": "ap", + "last_price": "lp", + "last_size": "ls", + } + + exchange: Optional[str] = Field( + default=None, + description="The exchange of the data.", + ) + type: Literal["quote", "trade", "break"] = Field( + description="The type of data.", + ) + bid_size: Optional[float] = Field( + default=None, + description="The size of the bid.", + ) + bid_price: Optional[float] = Field( + default=None, + description="The price of the bid.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + ask_price: Optional[float] = Field( + default=None, + description="The price of the ask.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + ask_size: Optional[float] = Field( + default=None, + description="The size of the ask.", + ) + last_price: Optional[float] = Field( + default=None, + description="The last trade price.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + last_size: Optional[float] = Field( + default=None, + description="The size of the trade.", + ) + + @field_validator("symbol", mode="before") + def _validate_symbol(cls, v): + """Validate the symbol.""" + return v.upper() + + @field_validator("type", mode="before", check_fields=False) + def _valiidate_data_type(cls, v): + """Validate the data type.""" + return ( + "quote" if v == "Q" else "trade" if v == "T" else "break" if v == "B" else v + ) + + @field_validator("date", mode="before", check_fields=False) + def _validate_date(cls, v): + """Validate the date.""" + # pylint: disable=import-outside-toplevel + from pytz import timezone + + if isinstance(v, str): + dt = datetime.fromisoformat(v) + try: + dt = datetime.fromtimestamp(v / 1000) + except Exception: # pylint: disable=broad-except + if isinstance(v, (int, float)): + # Check if the timestamp is in nanoseconds and convert to seconds + if v > 1e12: + v = v / 1e9 # Convert nanoseconds to seconds + dt = datetime.fromtimestamp(v) + + return dt.astimezone(timezone("America/New_York")) +``` + +#### WebSocketConnection + +This model is what we return from the `FmpWebSocketFetcher`. + + +```python +class FmpWebSocketConnection(WebSocketConnection): + """FMP WebSocket connection model.""" +``` + +#### WebSocketFetcher + +This is where things diverge slightly. Instead of returning `FmpWebSocketData`, we will pass it to the client connection insteadd, for validating records as they are received. What gets returned by the Fetcher is the `WebSocketConnection`. + +```python +class FmpWebSocketFetcher(Fetcher[FmpWebSocketQueryParams, FmpWebSocketConnection]): + """FMP WebSocket model.""" + + @staticmethod + def transform_query(params: dict[str, Any]) -> FmpWebSocketQueryParams: + """Transform the query parameters.""" + return FmpWebSocketQueryParams(**params) + + @staticmethod + async def aextract_data( + query: FmpWebSocketQueryParams, + credentials: Optional[dict[str, str]], + **kwargs: Any, + ) -> WebSocketClient: + """Extract data from the WebSocket.""" + # pylint: disable=import-outside-toplevel + import asyncio + + api_key = credentials.get("fmp_api_key") if credentials else "" + url = URL_MAP[query.asset_type] + + symbol = query.symbol.lower() + + # Arrange a dictionary of parameters that will be passed to the client connection. + kwargs = { + "url": url, + "api_key": api_key, + "connect_kwargs": query.connect_kwargs, # Pass custom parameters to `websockets.connect()` + } + + # The object to be returned. Everything the provider client thread needs to know is in this instance. + client = WebSocketClient( + name=query.name, + module="openbb_fmp.utils.websocket_client", # This is the file with the client connection that gets run as a script. + symbol=symbol, + limit=query.limit, + results_file=query.results_file, + table_name=query.table_name, + save_results=query.save_results, + data_model=FmpWebSocketData, # WebSocketDataModel goes here. + sleep_time=query.sleep_time, + broadcast_host=query.broadcast_host, + broadcast_port=query.broadcast_port, + auth_token=query.auth_token, + **kwargs, + ) + + # Start the client thread, give it a moment to startup and check for exceptions. + try: + client.connect() + await asyncio.sleep(2) + # Exceptions are triggered from the stdout reader and are converted + # to a Python Exception that gets stored here. + # If an exception was caught and the connection failed, we catch it here. + # They may not have raised yet, and it will be checked again further down. + if client._exception: + raise client._exception + # Everything caught gets raised as an OpenBBError, we catch those. + except OpenBBError as e: + if client.is_running: + client.disconnect() + raise e from e + # Check if the process is still running before returning. + if client.is_running: + return client + + raise OpenBBError("Failed to connect to the WebSocket.") + + @staticmethod + def transform_data( + data: WebSocketClient, + query: FmpWebSocketQueryParams, + **kwargs: Any, + ) -> FmpWebSocketConnection: + """Return the client as an instance of Data.""" + # All we need to do here is return our client wrapped in the WebSocketConnection class. + return FmpWebSocketConnection(client=data) +``` + +#### Map To Router + +Map the new fetcher in the provider's `__init__.py` file by adding it to the `fetcher_dict`. + +```python +"WebSocketConnection": FmpWebSocketFetcher ``` +Assuming the communication with `websocket_client` is all in order, it will be ready-to-go as a `provider` to the `create_connection` endpoint. + +### websocket_client + +This is the file where all the action happens. It receives subscribe/unsubscribe events, writes records to the `results_file`, and returns info and error messages to the main application thread. + +Some components are importable, but variances between providers require some localized solutions. They will be similar, but not 100% repeatable. + +#### Imports: + +```python +import asyncio +import json +import os +import signal +import sys + +import websockets +import websockets.exceptions +from openbb_fmp.models.websocket_connection import FmpWebSocketData # Import the data model that was created in the 'websocket_connection' file. +from openbb_websockets.helpers import ( + MessageQueue, + get_logger, + handle_termination_signal, + handle_validation_error, + parse_kwargs, + write_to_db, +) +from pydantic import ValidationError +``` + +#### `parse_kwargs` + +This function converts the keyword arguments passed at the time of launch. It should be run at the top of the file, with the global constants. + +```python +kwargs = parse_kwargs() +``` + +The dictionary will have all the parameters needed to establish the connection, and the instructions for where to record the results. + + +#### `get_logger` + +This function creates a logger instance with a unique name, configured to the INFO level, with a new line break between messages. +The logger is used to communicate information and errors back to the main application. + +> Only pass non-data messages and errors to the logger. + +Create the logger after the import section. + +```python +logger = get_logger("openbb.websocket.fmp") # A UUID gets attached to the name so multiple instances of the script do not initialize the same logger. +``` + +#### `MessageQueue` + +This is an async Queue with an input for the message handler. Create a second instance if a separate queue is required for the subscibe events. + +Define your async message handler function, and create a task to run in the main event loop. + +```python + +# At the top with the `logger` +queue = MessageQueue() + + +# This goes right before the `websockets.connect` code. +handler_task = asyncio.create_task( + queue.process_queue( + lambda message: process_message(message, results_path, table_name, limit) + ) +) +``` + +The queue can also be dequeued manually. + +```python +message = await queue.dequeue() +``` + +#### `handle_validation_error` + +Before submitting the record to `write_to_db`, validate and transform the data with the WebSocketData that was created and imported. Use this function right before transmission, a failure will trigger a termination signal from the main application. + +```python +# code above confirms that the message being processed is a data message and not an info message or error. + + try: + result = FmpWebSocketData.model_validate(message).model_dump_json( + exclude_none=True, exclude_unset=True + ) + except ValidationError as e: + try: + handle_validation_error(logger, e) + except ValidationError: + raise e from e + if result: + await write_to_db(result, results_path, table_name, limit) + +``` + + +#### `write_to_db` + +This function is responsible for recording the data message to the `results_file`, and will be used in the message handler. + +The inputs are all positional arguments, and aside from `message`, are in the `kwargs` dictionary and were supplied during the initialization of `WebSocketClient` in the provider's Fetcher. + + +```python +results_path = os.path.abspath(kwargs.get("results_file")) +table_name = kwargs.get("table_name") +limit = kwargs.get("limit") + +await write_to_db(message, results_path, table_name, limit) +``` + +#### `handle_termination_signal` + +Simple function, that triggers `sys.exit(0)` with a message, for use in `loop.add_signal_handler`. + +```python +if __name__ == "__main__": + try: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + for sig in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(sig, handle_termination_signal, logger) + + # asyncio.run_coroutine_threadsafe(some_connect_and_stream_function, loop) + # loop.run_forever() +... +``` + +#### To-Build + +THe missing pieces that get created locally include: + +- Read `stdin` function for receiving subscribe/unsubscribe events while the connection is running. + - Messages to handle will always have the same format: `'{"event": "subscribe", "symbol": ticker}'` + - Converting for the symbology used by the provider needs to happen here. + - Implementation depends on the requirements of the provider - i.e, how to structure send events. + - Create the task before the initial `websockets.connect` block. + +- Initial login event, the `api_key` will be included in the `kwargs` dictionary, if required. + - This event might need to happen before a subscribe event, handle any custom messages before entering the `while True` block. + - `UnauthorizedError` is raised by sending a `logger.error()` that begins with "UnauthorizedError -> %s". +- Message Handler + - This is the handler task that reads the message queue and determines where to send the message, database or logger. + - If the message is a row of data, send it to `write_to_db`. Else, send it back to the main application via: + - `logger.info("PROVIDER INFO: %s", message.get('message'))` + - Raise the message as an unexpected error: + - `logger.error("Unexpected error -> %s", message.get('message'))` -...tbc \ No newline at end of file diff --git a/openbb_platform/providers/fmp/openbb_fmp/__init__.py b/openbb_platform/providers/fmp/openbb_fmp/__init__.py index c2531817f93d..93b225cd1a12 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/__init__.py +++ b/openbb_platform/providers/fmp/openbb_fmp/__init__.py @@ -63,7 +63,7 @@ from openbb_fmp.models.risk_premium import FMPRiskPremiumFetcher from openbb_fmp.models.share_statistics import FMPShareStatisticsFetcher from openbb_fmp.models.treasury_rates import FMPTreasuryRatesFetcher -from openbb_fmp.models.websocket_connection import FMPWebSocketFetcher +from openbb_fmp.models.websocket_connection import FmpWebSocketFetcher from openbb_fmp.models.world_news import FMPWorldNewsFetcher from openbb_fmp.models.yield_curve import FMPYieldCurveFetcher @@ -138,7 +138,7 @@ "TreasuryRates": FMPTreasuryRatesFetcher, "WorldNews": FMPWorldNewsFetcher, "EtfHistorical": FMPEquityHistoricalFetcher, - "WebSocketConnection": FMPWebSocketFetcher, + "WebSocketConnection": FmpWebSocketFetcher, "YieldCurve": FMPYieldCurveFetcher, }, repr_name="Financial Modeling Prep (FMP)", diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py index 68d662e4a16a..b157e572548a 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py @@ -36,7 +36,7 @@ class FmpWebSocketQueryParams(WebSocketQueryParams): ) asset_type: Literal["stock", "fx", "crypto"] = Field( default="crypto", - description="The asset type, required for the provider URI.", + description="The asset type associated with the symbol.", ) @@ -56,12 +56,6 @@ class FmpWebSocketData(WebSocketData): "last_size": "ls", } - symbol: str = Field( - description="The symbol of the asset.", - ) - date: datetime = Field( - description="The datetime of the data.", - ) exchange: Optional[str] = Field( default=None, description="The exchange of the data.", @@ -116,25 +110,24 @@ def _validate_date(cls, v): from pytz import timezone if isinstance(v, str): - return datetime.fromisoformat(v) + dt = datetime.fromisoformat(v) try: - return datetime.fromtimestamp(v / 1000) - except Exception: + dt = datetime.fromtimestamp(v / 1000) + except Exception: # pylint: disable=broad-except if isinstance(v, (int, float)): # Check if the timestamp is in nanoseconds and convert to seconds if v > 1e12: v = v / 1e9 # Convert nanoseconds to seconds dt = datetime.fromtimestamp(v) - dt = timezone("America/New_York").localize(dt) - return dt - return v + + return dt.astimezone(timezone("America/New_York")) class FmpWebSocketConnection(WebSocketConnection): """FMP WebSocket connection model.""" -class FMPWebSocketFetcher(Fetcher[FmpWebSocketQueryParams, FmpWebSocketConnection]): +class FmpWebSocketFetcher(Fetcher[FmpWebSocketQueryParams, FmpWebSocketConnection]): """FMP WebSocket model.""" @staticmethod diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py index 957e6257d122..d9f902c42ab3 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -134,7 +134,7 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim stdin_task = asyncio.create_task(read_stdin_and_queue_commands()) try: - websocket = await websockets.connect(url) + websocket = await websockets.connect(url, **CONNECT_KWARGS) await login(websocket, api_key) await subscribe(websocket, symbol, "subscribe") diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index b93aa3287209..50ca68c09c19 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -148,7 +148,7 @@ async def process_message(message, results_path, table_name, limit): msg = f"PROVIDER INFO: Authorization: {response.get('message')}" logger.info(msg) if message.get("data", {}).get("subscriptionId"): - global SUBSCRIPTION_ID + global SUBSCRIPTION_ID # noqa: PLW0603 SUBSCRIPTION_ID = message["data"]["subscriptionId"] if "tickers" in response.get("data", {}): From 34dc7b40d86be4a9a13615aa82cd39265fccc583 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:36:50 -0800 Subject: [PATCH 028/119] typo --- openbb_platform/extensions/websockets/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/extensions/websockets/README.md b/openbb_platform/extensions/websockets/README.md index 80f515fc1b42..b6b23247b001 100644 --- a/openbb_platform/extensions/websockets/README.md +++ b/openbb_platform/extensions/websockets/README.md @@ -845,7 +845,7 @@ if __name__ == "__main__": #### To-Build -THe missing pieces that get created locally include: +The missing pieces that get created locally include: - Read `stdin` function for receiving subscribe/unsubscribe events while the connection is running. - Messages to handle will always have the same format: `'{"event": "subscribe", "symbol": ticker}'` From 133cb7218f6bb4e84964f069573c6683dfea72f5 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:49:31 -0800 Subject: [PATCH 029/119] another typo --- openbb_platform/extensions/websockets/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/extensions/websockets/README.md b/openbb_platform/extensions/websockets/README.md index b6b23247b001..f1a400d5aa49 100644 --- a/openbb_platform/extensions/websockets/README.md +++ b/openbb_platform/extensions/websockets/README.md @@ -617,7 +617,7 @@ class FmpWebSocketConnection(WebSocketConnection): #### WebSocketFetcher -This is where things diverge slightly. Instead of returning `FmpWebSocketData`, we will pass it to the client connection insteadd, for validating records as they are received. What gets returned by the Fetcher is the `WebSocketConnection`. +This is where things diverge slightly. Instead of returning `FmpWebSocketData`, it gets passed to the client connection for validating records as they are received. What gets returned by the Fetcher is the `WebSocketConnection`. ```python class FmpWebSocketFetcher(Fetcher[FmpWebSocketQueryParams, FmpWebSocketConnection]): From 4f5a14b61587789839901187de57590ab1c08d51 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 13 Nov 2024 09:58:27 -0800 Subject: [PATCH 030/119] typo --- openbb_platform/extensions/websockets/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openbb_platform/extensions/websockets/README.md b/openbb_platform/extensions/websockets/README.md index f1a400d5aa49..1ac672498ce9 100644 --- a/openbb_platform/extensions/websockets/README.md +++ b/openbb_platform/extensions/websockets/README.md @@ -37,7 +37,7 @@ obb.websockets > > |Parameter|Type | Required| Description | > |:-------|:-----|:--------:|------------:| -> |name |String |Yes |The 'nane' assigned from `create_connection` | +> |name |String |Yes |The 'name' assigned from `create_connection` | > |auth_token |String |No |The 'auth_token' assigned, if any, from `create_connection` | > > Below is an explanation of each function, with `create_connection` representing the bulk of details. @@ -864,3 +864,4 @@ The missing pieces that get created locally include: - Raise the message as an unexpected error: - `logger.error("Unexpected error -> %s", message.get('message'))` +> With all the functions built, the file should run as a script where keyword arguments are formatted as `key=value`, with a space between each pair. From 31b05c0c15581b99aae6428befd88d966f993bf3 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 14 Nov 2024 19:38:20 -0800 Subject: [PATCH 031/119] fix discriminator tag issue --- .../websockets/openbb_websockets/helpers.py | 10 ++- .../websockets/openbb_websockets/models.py | 83 +++++++++++-------- .../openbb_websockets/websockets_router.py | 9 +- .../fmp/openbb_fmp/utils/websocket_client.py | 11 ++- .../models/websocket_connection.py | 2 +- 5 files changed, 68 insertions(+), 47 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 21539d192087..d8cedfcba97a 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -3,7 +3,7 @@ import logging import re import sys -from typing import Optional +from typing import Any, Optional from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.utils.errors import UnauthorizedError @@ -46,11 +46,13 @@ def handle_validation_error(logger: logging.Logger, error: ValidationError): raise error from error -async def get_status(name: str) -> dict: +async def get_status(name: Optional[str] = None, client: Optional[Any] = None) -> dict: """Get the status of a client.""" - if name not in connected_clients: + if name and name not in connected_clients: raise OpenBBError(f"Client {name} not connected.") - client = connected_clients[name] + if not name and not client: + raise OpenBBError("Either name or client must be provided.") + client = client if client else connected_clients[name] provider_pid = client._psutil_process.pid if client.is_running else None broadcast_pid = ( client._psutil_broadcast_process.pid if client.is_broadcasting else None diff --git a/openbb_platform/extensions/websockets/openbb_websockets/models.py b/openbb_platform/extensions/websockets/openbb_websockets/models.py index 48ca4266595e..8e53e72233ed 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/models.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/models.py @@ -8,9 +8,8 @@ from openbb_core.provider.abstract.query_params import QueryParams from openbb_core.provider.utils.descriptions import ( DATA_DESCRIPTIONS, - QUERY_DESCRIPTIONS, ) -from pydantic import ConfigDict, Field, field_validator +from pydantic import ConfigDict, Field, field_validator, model_validator from openbb_websockets.client import WebSocketClient @@ -89,39 +88,6 @@ def _validate_connect_kwargs(cls, v): return json.dumps(v, separators=(",", ":")) -class WebSocketData(Data): - """WebSocket data model.""" - - date: datetime = Field( - description=DATA_DESCRIPTIONS.get("date", ""), - ) - symbol: str = Field( - description=DATA_DESCRIPTIONS.get("symbol", ""), - ) - - -class WebSocketConnection(Data): - """Data model for returning WebSocketClient from the Provider Interface.""" - - __model_config__ = ConfigDict( - extra="forbid", - ) - - client: Any = Field( - description="Instance of WebSocketClient class initialized by a provider Fetcher." - + " The client is used to communicate with the provider's data stream." - + " It is not returned to the user, but is handled by the router for API access.", - exclude=True, - ) - - @field_validator("client", mode="before", check_fields=False) - def _validate_client(cls, v): - """Validate the client.""" - if not isinstance(v, WebSocketClient): - raise ValueError("Client must be an instance of WebSocketClient.") - return v - - class WebSocketConnectionStatus(Data): """Data model for WebSocketConnection status information.""" @@ -164,3 +130,50 @@ class WebSocketConnectionStatus(Data): save_results: bool = Field( description="Whether to save the results after the session ends.", ) + + +class WebSocketData(Data): + """WebSocket data model.""" + + date: datetime = Field( + description=DATA_DESCRIPTIONS.get("date", ""), + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + + +class WebSocketConnection(Data): + """Data model for returning WebSocketClient from the Provider Interface.""" + + __model_config__ = ConfigDict( + extra="forbid", + ) + + client: Optional[Any] = Field( + default=None, + description="Instance of WebSocketClient class initialized by a provider Fetcher." + + " The client is used to communicate with the provider's data stream." + + " It is not returned to the user, but is handled by the router for API access.", + exclude=True, + ) + status: Optional[WebSocketConnectionStatus] = Field( + default=None, + description="Status information for the WebSocket connection.", + ) + + @field_validator("client", mode="before", check_fields=False) + @classmethod + def _validate_client(cls, v): + """Validate the client.""" + if v and not isinstance(v, WebSocketClient): + raise ValueError("Client must be an instance of WebSocketClient.") + return v + + @model_validator(mode="before") + @classmethod + def _validate_inputs(cls, vaules): + """Validate the status.""" + if not vaules.get("status") and not vaules.get("client"): + raise ValueError("Cannot initialize empty.") + return vaules diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index d5317bea3733..5445e0e1b6a4 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -39,8 +39,9 @@ async def create_connection( provider_choices: ProviderChoices, standard_params: StandardParams, extra_params: ExtraParams, -) -> OBBject[WebSocketConnectionStatus]: +) -> OBBject: """Create a new provider websocket connection.""" + name = extra_params.name if name in connected_clients: broadcast_address = connected_clients[name].broadcast_address @@ -70,10 +71,8 @@ async def create_connection( client_name = client.name connected_clients[client_name] = client - results = await get_status(client_name) - - obbject.results = WebSocketConnectionStatus(**results) - + status = await get_status(client_name) + obbject.results.status = WebSocketConnectionStatus(**status) return obbject diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py index d9f902c42ab3..b54890874bc8 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -155,8 +155,15 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim message = task.result() await asyncio.shield(queue.enqueue(json.loads(message))) - except websockets.ConnectionClosed: - logger.info("PROVIDER INFO: The WebSocket connection was closed.") + except websockets.ConnectionClosed as e: + msg = ( + f"PROVIDER INFO: The WebSocket connection was closed -> {e.__str__()}" + ) + logger.info(msg) + # Attempt to reopen the connection + logger.info("PROVIDER INFO: Attempting to reconnect after five seconds.") + await asyncio.sleep(5) + await connect_and_stream(url, symbol, api_key, results_path, table_name, limit) except websockets.WebSocketException as e: logger.error(e) diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index d792f2606535..0960bb4850db 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -137,7 +137,7 @@ class PolygonWebSocketQueryParams(WebSocketQueryParams): } symbol: str = Field( - description="Polygon symbol to get data for." + description="\n Polygon symbol to get data for." + " All feeds, except Options, support the wildcard symbol, '*', for all symbols." + "\n Options symbols are the OCC contract symbol and support up to 1000 individual contracts" + " per connection. Crypto and FX symbols should be entered as a pair, i.e., 'BTCUSD', 'JPYUSD'." From b10d64034a68e8bc94dadd2b5dd41b5f770ed131 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 14 Nov 2024 22:50:04 -0800 Subject: [PATCH 032/119] add integration tests --- .../extensions/websockets/README.md | 8 +- .../integration/test_websockets_api.py | 376 +++++++++++++++++ .../integration/test_websockets_python.py | 388 ++++++++++++++++++ .../extensions/websockets/tests/__init__.py | 1 + 4 files changed, 772 insertions(+), 1 deletion(-) create mode 100644 openbb_platform/extensions/websockets/integration/test_websockets_api.py create mode 100644 openbb_platform/extensions/websockets/integration/test_websockets_python.py create mode 100644 openbb_platform/extensions/websockets/tests/__init__.py diff --git a/openbb_platform/extensions/websockets/README.md b/openbb_platform/extensions/websockets/README.md index 1ac672498ce9..b98a11e2aafc 100644 --- a/openbb_platform/extensions/websockets/README.md +++ b/openbb_platform/extensions/websockets/README.md @@ -607,8 +607,14 @@ class FmpWebSocketData(WebSocketData): #### WebSocketConnection -This model is what we return from the `FmpWebSocketFetcher`. +This model is what we return from the `FmpWebSocketFetcher`. The provider will inherit two fields, only the `client` needs to be included on output. No additional fields should be defined. +- client + - Instance of WebSocketClient, not returned to from the Router. +- status + - Leave empty, the Router fills this and returns it. + +The model will not accept additional fields. ```python class FmpWebSocketConnection(WebSocketConnection): diff --git a/openbb_platform/extensions/websockets/integration/test_websockets_api.py b/openbb_platform/extensions/websockets/integration/test_websockets_api.py new file mode 100644 index 000000000000..047ff50f2efa --- /dev/null +++ b/openbb_platform/extensions/websockets/integration/test_websockets_api.py @@ -0,0 +1,376 @@ +"""Test WebSockets API Integration.""" + +import base64 + +import pytest +import requests +from extensions.tests.conftest import parametrize +from openbb_core.env import Env +from openbb_core.provider.utils.helpers import get_querystring + + +@pytest.fixture(scope="session") +def headers(): + """Get the headers for the API request.""" + userpass = f"{Env().API_USERNAME}:{Env().API_PASSWORD}" + userpass_bytes = userpass.encode("ascii") + base64_bytes = base64.b64encode(userpass_bytes) + + return {"Authorization": f"Basic {base64_bytes.decode('ascii')}"} + + +# pylint: disable=redefined-outer-name + + +@parametrize( + "params", + [ + ( + { + "name": "test_fmp", + "provider": "fmp", + "symbol": "btcusd,dogeusd", + "asset_type": "crypto", + "auth_token": None, + "results_file": None, + "save_results": False, + "table_name": "records", + "limit": 10, + "sleep_time": 0.25, + "broadcast_host": "localhost", + "broadcast_port": 6666, + "start_broadcast": False, + "connect_kwargs": None, + } + ), + ( + { + "name": "test_tiingo", + "provider": "tiingo", + "symbol": "btcusd,dogeusd", + "asset_type": "crypto", + "feed": "trade_and_quote", + "auth_token": None, + "results_file": None, + "save_results": False, + "table_name": "records", + "limit": 10, + "sleep_time": 0.25, + "broadcast_host": "localhost", + "broadcast_port": 6666, + "start_broadcast": False, + "connect_kwargs": None, + } + ), + ( + { + "name": "test_polygon", + "provider": "polygon", + "symbol": "btcusd,dogeusd", + "asset_type": "crypto", + "feed": "quote", + "auth_token": None, + "results_file": None, + "save_results": False, + "table_name": "records", + "limit": 10, + "sleep_time": 0.25, + "broadcast_host": "localhost", + "broadcast_port": 6666, + "start_broadcast": False, + "connect_kwargs": None, + } + ), + ], +) +@pytest.mark.integration +def test_websockets_create_connection(params, headers): + """Test the websockets_create_connection endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/websockets/create_connection?{query_str}" + + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code == 200 + res = result.json()["results"] + assert isinstance(res, dict) + assert res.get("status", {}).get("is_running") + assert not res.get("status", {}).get("is_broadcasting") + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_get_results(params, headers): + """Test the websockets_get_results endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/websockets/get_results?{query_str}" + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code == 200 + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_clear_results(params, headers): + """Test the websockets_clear_results endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/websockets/clear_results?{query_str}" + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code == 200 + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "symbol": "ethusd", + "auth_token": None, + }, + { + "name": "test_tiingo", + "symbol": "ethusd", + "auth_token": None, + }, + { + "name": "test_polygon", + "symbol": "ethusd", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_subscribe(params, headers): + """Test the websockets_subscribe endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/websockets/subscribe?{query_str}" + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code == 200 + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + "host": None, + "port": None, + "uvicorn_kwargs": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + "host": None, + "port": None, + "uvicorn_kwargs": None, + }, + { + "name": "test_polygon", + "auth_token": None, + "host": None, + "port": None, + "uvicorn_kwargs": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_start_broadcasting(params, headers): + """Test the websockets_start_broadcasting endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/websockets/start_broadcasting?{query_str}" + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + "symbol": "ethusd", + }, + { + "name": "test_tiingo", + "auth_token": None, + "symbol": "ethusd", + }, + { + "name": "test_polygon", + "auth_token": None, + "symbol": "ethusd", + }, + ], +) +@pytest.mark.integration +def test_websockets_unsubscribe(params, headers): + """Test the websockets_unsubscribe endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/websockets/unsubscribe?{query_str}" + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code == 200 + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_stop_connection(params, headers): + """Test the websockets_stop_connection endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/websockets/stop_connection?{query_str}" + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code == 200 + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_restart_connection(params, headers): + """Test the websockets_restart_connection endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/websockets/restart_connection?{query_str}" + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code == 200 + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_stop_broadcasting(params, headers): + """Test the websockets_stop_broadcasting endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/websockets/stop_broadcasting?{query_str}" + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code == 200 + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_kill(params, headers): + """Test the websockets_kill endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/websockets/kill?{query_str}" + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code == 200 diff --git a/openbb_platform/extensions/websockets/integration/test_websockets_python.py b/openbb_platform/extensions/websockets/integration/test_websockets_python.py new file mode 100644 index 000000000000..5820715fa410 --- /dev/null +++ b/openbb_platform/extensions/websockets/integration/test_websockets_python.py @@ -0,0 +1,388 @@ +"""Test WebSockets Python Integration.""" + +# pylint: disable=redefined-outer-name, inconsistent-return-statements, import-outside-toplevel + +import pytest +from extensions.tests.conftest import parametrize +from openbb_core.app.model.obbject import OBBject +from openbb_websockets.models import WebSocketConnectionStatus + + +@pytest.fixture(scope="session") +def obb(pytestconfig): + """Fixture to setup obb.""" + + if pytestconfig.getoption("markexpr") != "not integration": + import openbb + + return openbb.obb + + +@parametrize( + "params", + [ + ( + { + "name": "test_fmp", + "provider": "fmp", + "symbol": "btcusd,dogeusd", + "asset_type": "crypto", + "auth_token": None, + "results_file": None, + "save_results": False, + "table_name": "records", + "limit": 10, + "sleep_time": 0.25, + "broadcast_host": "localhost", + "broadcast_port": 6666, + "start_broadcast": False, + "connect_kwargs": None, + } + ), + ( + { + "name": "test_tiingo", + "provider": "tiingo", + "symbol": "btcusd,dogeusd", + "asset_type": "crypto", + "feed": "trade_and_quote", + "auth_token": None, + "results_file": None, + "save_results": False, + "table_name": "records", + "limit": 10, + "sleep_time": 0.25, + "broadcast_host": "localhost", + "broadcast_port": 6666, + "start_broadcast": False, + "connect_kwargs": None, + } + ), + ( + { + "name": "test_polygon", + "provider": "polygon", + "symbol": "btcusd,dogeusd", + "asset_type": "crypto", + "feed": "quote", + "auth_token": None, + "results_file": None, + "save_results": False, + "table_name": "records", + "limit": 10, + "sleep_time": 0.25, + "broadcast_host": "localhost", + "broadcast_port": 6666, + "start_broadcast": False, + "connect_kwargs": None, + } + ), + ], +) +@pytest.mark.integration +def test_websockets_create_connection(params, obb): + """Test the websockets_create_connection endpoint.""" + params = {p: v for p, v in params.items() if v} + + result = obb.websockets.create_connection(**params) + assert result + assert isinstance(result, OBBject) + assert result.results is not None + assert isinstance(result.results.status, WebSocketConnectionStatus) + assert result.results.status.is_running is True + assert result.results.status.is_broadcasting is False + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_get_results(params, obb): + """Test the websockets_get_results endpoint.""" + params = {p: v for p, v in params.items() if v} + + result = obb.websockets.get_results(**params) + assert result + assert isinstance(result, OBBject) + assert result.results is not None + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_clear_results(params, obb): + """Test the websockets_clear_results endpoint.""" + params = {p: v for p, v in params.items() if v} + + result = obb.websockets.clear_results(**params) + assert result + assert isinstance(result, OBBject) + assert result.results is not None + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "symbol": "ethusd", + "auth_token": None, + }, + { + "name": "test_tiingo", + "symbol": "ethusd", + "auth_token": None, + }, + { + "name": "test_polygon", + "symbol": "ethusd", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_subscribe(params, obb): + """Test the websockets_subscribe endpoint.""" + params = {p: v for p, v in params.items() if v} + + result = obb.websockets.subscribe(**params) + assert result + assert isinstance(result, OBBject) + assert result.results is not None + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + "host": None, + "port": None, + "uvicorn_kwargs": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + "host": None, + "port": None, + "uvicorn_kwargs": None, + }, + { + "name": "test_polygon", + "auth_token": None, + "host": None, + "port": None, + "uvicorn_kwargs": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_start_broadcasting(params, obb): + """Test the websockets_start_broadcasting endpoint.""" + params = {p: v for p, v in params.items() if v} + + result = obb.websockets.start_broadcasting(**params) + assert result + assert isinstance(result, OBBject) + assert result.results is not None + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "symbol": "ethusd", + "auth_token": None, + }, + { + "name": "test_tiingo", + "symbol": "ethusd", + "auth_token": None, + }, + { + "name": "test_polygon", + "symbol": "ethusd", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_unsubscribe(params, obb): + """Test the websockets_unsubscribe endpoint.""" + params = {p: v for p, v in params.items() if v} + + result = obb.websockets.unsubscribe(**params) + assert result + assert isinstance(result, OBBject) + assert result.results is not None + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_get_client(params, obb): + """Test the websockets_get_client endpoint.""" + params = {p: v for p, v in params.items() if v} + + result = obb.websockets.get_client(**params) + assert result + assert isinstance(result, OBBject) + assert result.results is not None + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_stop_connection(params, obb): + """Test the websockets_stop_connection endpoint.""" + params = {p: v for p, v in params.items() if v} + + result = obb.websockets.stop_connection(**params) + assert result + assert isinstance(result, OBBject) + assert result.results is not None + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_restart_connection(params, obb): + """Test the websockets_restart_connection endpoint.""" + params = {p: v for p, v in params.items() if v} + + result = obb.websockets.restart_connection(**params) + assert result + assert isinstance(result, OBBject) + assert result.results is not None + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_stop_broadcasting(params, obb): + """Test the websockets_stop_broadcasting endpoint.""" + params = {p: v for p, v in params.items() if v} + + result = obb.websockets.stop_broadcasting(**params) + assert result + assert isinstance(result, OBBject) + assert result.results is not None + + +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_kill(params, obb): + """Test the websockets_kill endpoint.""" + params = {p: v for p, v in params.items() if v} + + result = obb.websockets.kill(**params) + assert result + assert isinstance(result, OBBject) + assert result.results is not None diff --git a/openbb_platform/extensions/websockets/tests/__init__.py b/openbb_platform/extensions/websockets/tests/__init__.py new file mode 100644 index 000000000000..b30092a1be6f --- /dev/null +++ b/openbb_platform/extensions/websockets/tests/__init__.py @@ -0,0 +1 @@ +"""WebSockets Extension Tests.""" From fc91aa1344add9e7b6dd9e09371f610c800e19b7 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 14 Nov 2024 23:04:17 -0800 Subject: [PATCH 033/119] some test params --- .../integration/test_websockets_api.py | 18 +++++++++--------- .../integration/test_websockets_python.py | 12 ++++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/openbb_platform/extensions/websockets/integration/test_websockets_api.py b/openbb_platform/extensions/websockets/integration/test_websockets_api.py index 047ff50f2efa..a01657cecb69 100644 --- a/openbb_platform/extensions/websockets/integration/test_websockets_api.py +++ b/openbb_platform/extensions/websockets/integration/test_websockets_api.py @@ -37,7 +37,7 @@ def headers(): "table_name": "records", "limit": 10, "sleep_time": 0.25, - "broadcast_host": "localhost", + "broadcast_host": "0.0.0.0", # noqa: S104 "broadcast_port": 6666, "start_broadcast": False, "connect_kwargs": None, @@ -56,7 +56,7 @@ def headers(): "table_name": "records", "limit": 10, "sleep_time": 0.25, - "broadcast_host": "localhost", + "broadcast_host": "0.0.0.0", # noqa: S104 "broadcast_port": 6666, "start_broadcast": False, "connect_kwargs": None, @@ -75,7 +75,7 @@ def headers(): "table_name": "records", "limit": 10, "sleep_time": 0.25, - "broadcast_host": "localhost", + "broadcast_host": "0.0.0.0", # noqa: S104 "broadcast_port": 6666, "start_broadcast": False, "connect_kwargs": None, @@ -196,22 +196,22 @@ def test_websockets_subscribe(params, headers): { "name": "test_fmp", "auth_token": None, - "host": None, - "port": None, + "host": "0.0.0.0", # noqa: S104 + "port": 6666, "uvicorn_kwargs": None, }, { "name": "test_tiingo", "auth_token": None, - "host": None, - "port": None, + "host": "0.0.0.0", # noqa: S104 + "port": 6667, "uvicorn_kwargs": None, }, { "name": "test_polygon", "auth_token": None, - "host": None, - "port": None, + "host": "0.0.0.0", # noqa: S104 + "port": 6668, "uvicorn_kwargs": None, }, ], diff --git a/openbb_platform/extensions/websockets/integration/test_websockets_python.py b/openbb_platform/extensions/websockets/integration/test_websockets_python.py index 5820715fa410..abd6f27740f6 100644 --- a/openbb_platform/extensions/websockets/integration/test_websockets_python.py +++ b/openbb_platform/extensions/websockets/integration/test_websockets_python.py @@ -33,7 +33,7 @@ def obb(pytestconfig): "table_name": "records", "limit": 10, "sleep_time": 0.25, - "broadcast_host": "localhost", + "broadcast_host": "0.0.0.0", # noqa: S104 "broadcast_port": 6666, "start_broadcast": False, "connect_kwargs": None, @@ -52,7 +52,7 @@ def obb(pytestconfig): "table_name": "records", "limit": 10, "sleep_time": 0.25, - "broadcast_host": "localhost", + "broadcast_host": "0.0.0.0", # noqa: S104 "broadcast_port": 6666, "start_broadcast": False, "connect_kwargs": None, @@ -71,7 +71,7 @@ def obb(pytestconfig): "table_name": "records", "limit": 10, "sleep_time": 0.25, - "broadcast_host": "localhost", + "broadcast_host": "0.0.0.0", # noqa: S104 "broadcast_port": 6666, "start_broadcast": False, "connect_kwargs": None, @@ -186,21 +186,21 @@ def test_websockets_subscribe(params, obb): { "name": "test_fmp", "auth_token": None, - "host": None, + "host": "0.0.0.0", # noqa: S104 "port": None, "uvicorn_kwargs": None, }, { "name": "test_tiingo", "auth_token": None, - "host": None, + "host": "0.0.0.0", # noqa: S104 "port": None, "uvicorn_kwargs": None, }, { "name": "test_polygon", "auth_token": None, - "host": None, + "host": "0.0.0.0", # noqa: S104 "port": None, "uvicorn_kwargs": None, }, From 3ca267fd5795459bf2b5a73b9186262a1f48942c Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Fri, 15 Nov 2024 11:36:21 -0800 Subject: [PATCH 034/119] docstring things --- .../extensions/websockets/README.md | 80 ++++++++++++++++++- .../websockets/openbb_websockets/client.py | 75 +++++++++-------- .../websockets/openbb_websockets/helpers.py | 2 +- .../openbb_websockets/websockets_router.py | 5 +- 4 files changed, 119 insertions(+), 43 deletions(-) diff --git a/openbb_platform/extensions/websockets/README.md b/openbb_platform/extensions/websockets/README.md index b98a11e2aafc..ee9c120a3cec 100644 --- a/openbb_platform/extensions/websockets/README.md +++ b/openbb_platform/extensions/websockets/README.md @@ -454,12 +454,11 @@ PROVIDER INFO: unsubscribed to: C.XAU/USD ## Development - ### Provider Interface Providers can be added to the `create_connection` endpoint by following a slightly modified pattern. This section outlines the adaptations, but does not contain any code for actually connecting to the provider's websocket. -For details on that part, go to [websocket_client](README.md###websocket_client) section below. +For details on that part, go to [websocket_client](README.md#websocket_client) section below. Here, the Fetcher is used to start the provider client module (in a separate file) and return the client to the router, where it is intercepted and kept alive. @@ -621,6 +620,83 @@ class FmpWebSocketConnection(WebSocketConnection): """FMP WebSocket connection model.""" ``` +#### WebSocketClient + +The `WebSocketClient` is the main class responsible for bidrectional communication between the provider, broadcast, and user. It handles the child processes and can be used as a standalone class. Pasted below is the docstring for the class. + +It can be imported to use as a standalone class, and this instance is the 'client' field in `WebSocketConnection` + +```python +from openbb_websockets.client import WebSocketClient +``` + +```console + Parameters + ---------- + name : str + Name to assign the WebSocket connection. Used to identify and manage multiple instances. + module : str + The Python module for the provider websocket_client module. Runs in a separate thread. + Example: 'openbb_fmp.utils.websocket_client'. Pass additional keyword arguments by including kwargs. + symbol : Optional[str] + The symbol(s) requested to subscribe. Enter multiple symbols separated by commas without spaces. + limit : Optional[int] + The limit of records to hold in memory. Once the limit is reached, the oldest records are removed. + Default is 1000. Set to None to keep all records. + results_file : Optional[str] + Absolute path to the file for continuous writing. By default, a temporary file is created. + table_name : Optional[str] + SQL table name to store serialized data messages. By default, 'records'. + save_results : bool + Whether to persist the results after the main Python session ends. Default is False. + data_model : Optional[Data] + Pydantic data model to validate the results before storing them in the database. + Also used to deserialize the results from the database. + auth_token : Optional[str] + The authentication token to use for the WebSocket connection. Default is None. + Only used for API and Python application endpoints. + logger : Optional[logging.Logger] + The pre-configured logger instance to use for this connection. By default, a new logger is created. + kwargs : Optional[dict] + Additional keyword arguments to pass to the target provider module. Keywords and values must not contain spaces. + To pass items to 'websocket.connect()', include them in the 'kwargs' dictionary as, + {'connect_kwargs': {'key': 'value'}}. + + Properties + ---------- + symbol : str + Symbol(s) requested to subscribe. + module : str + Path to the provider connection script. + is_running : bool + Check if the provider connection process is running. + is_broadcasting : bool + Check if the broadcast server process is running. + broadcast_address : str + URI address for the results broadcast server. + results : list[Data] + All stored results from the provider's WebSocket stream. + Results are stored in a SQLite database as a serialized JSON string, this property deserializes the results. + Clear the results by deleting the property. e.g., del client.results + + Methods + ------- + connect + Connect to the provider WebSocket stream. + disconnect + Disconnect from the provider WebSocket. + subscribe + Subscribe to a new symbol or list of symbols. + unsubscribe + Unsubscribe from a symbol or list of symbols. + start_broadcasting + Start the broadcast server to stream results over a network connection. + stop_broadcasting + Stop the broadcast server and disconnect all listening clients. + send_message + Send a message to the WebSocket process. Messages can be sent to "provider" or "broadcast" targets. +``` + #### WebSocketFetcher This is where things diverge slightly. Instead of returning `FmpWebSocketData`, it gets passed to the client connection for validating records as they are received. What gets returned by the Fetcher is the `WebSocketConnection`. diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 72a720b3ce5d..1aa7178ce502 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -3,7 +3,7 @@ # pylint: disable=too-many-statements # flake8: noqa: PLR0915 import logging -from typing import TYPE_CHECKING, Literal, Optional, Union +from typing import TYPE_CHECKING, Any, Literal, Optional, Union if TYPE_CHECKING: from openbb_core.provider.abstract.data import Data @@ -17,13 +17,13 @@ class WebSocketClient: name : str Name to assign the WebSocket connection. Used to identify and manage multiple instances. module : str - The Python module for the provider server connection. Runs in a separate thread. - Example: 'openbb_fmp.websockets.server'. Pass additional keyword arguments by including kwargs. + The Python module for the provider websocket_client module. Runs in a separate thread. + Example: 'openbb_fmp.utils.websocket_client'. Pass additional keyword arguments by including kwargs. symbol : Optional[str] The symbol(s) requested to subscribe. Enter multiple symbols separated by commas without spaces. limit : Optional[int] The limit of records to hold in memory. Once the limit is reached, the oldest records are removed. - Default is 300. Set to None to keep all records. + Default is 1000. Set to None to keep all records. results_file : Optional[str] Absolute path to the file for continuous writing. By default, a temporary file is created. table_name : Optional[str] @@ -55,12 +55,10 @@ class WebSocketClient: Check if the broadcast server process is running. broadcast_address : str URI address for the results broadcast server. - results : list - All stored results from the provider's WebSocket stream. The results are stored in a SQLite database. - Set the 'limit' property to cap the number of stored records. + results : list[Data] + All stored results from the provider's WebSocket stream. + Results are stored in a SQLite database as a serialized JSON string, this property deserializes the results. Clear the results by deleting the property. e.g., del client.results - transformed_results : list - Deserialize the records from the results file using the provided data model, if available. Methods ------- @@ -75,9 +73,9 @@ class WebSocketClient: start_broadcasting Start the broadcast server to stream results over a network connection. stop_broadcasting - Stop the broadcast server and disconnect all reading clients. + Stop the broadcast server and disconnect all listening clients. send_message - Send a message to the WebSocket process. + Send a message to the WebSocket process. Messages can be sent to "provider" or "broadcast" targets. """ def __init__( # noqa: PLR0913 @@ -85,7 +83,7 @@ def __init__( # noqa: PLR0913 name: str, module: str, symbol: Optional[str] = None, - limit: Optional[int] = 300, + limit: Optional[int] = 1000, results_file: Optional[str] = None, table_name: Optional[str] = None, save_results: bool = False, @@ -103,6 +101,7 @@ def __init__( # noqa: PLR0913 from aiosqlite import DatabaseError from queue import Queue from pathlib import Path + from openbb_core.app.model.abstract.error import OpenBBError from openbb_websockets.helpers import get_logger self.name = name @@ -119,20 +118,20 @@ def __init__( # noqa: PLR0913 else None ) - self._process = None - self._psutil_process = None - self._thread = None - self._log_thread = None - self._provider_message_queue = Queue() - self._stop_log_thread_event = threading.Event() - self._stop_broadcasting_event = threading.Event() - self._broadcast_address = None - self._broadcast_process = None - self._psutil_broadcast_process = None - self._broadcast_thread = None - self._broadcast_log_thread = None - self._broadcast_message_queue = Queue() - self._exception = None + self._process: Any = None + self._psutil_process: Any = None + self._thread: Any = None + self._log_thread: Any = None + self._provider_message_queue: Queue = Queue() + self._stop_log_thread_event: threading.Event = threading.Event() + self._stop_broadcasting_event: threading.Event = threading.Event() + self._broadcast_address: Any = None + self._broadcast_process: Any = None + self._psutil_broadcast_process: Any = None + self._broadcast_thread: Any = None + self._broadcast_log_thread: Any = None + self._broadcast_message_queue: Queue = Queue() + self._exception: Any = None if not results_file: with tempfile.NamedTemporaryFile(delete=False) as temp_file: @@ -150,7 +149,9 @@ def __init__( # noqa: PLR0913 try: self._setup_database() except DatabaseError as e: - self.logger.error("Error setting up the SQLite database and table: %s", e) + msg = f"Unexpected error setting up the SQLite database and table -> {e.__class___.__name__}: {e.__str__()}" + self.logger.error(msg) + self._exception = OpenBBError(msg) def _atexit(self) -> None: """Clean up the WebSocket client processes at exit.""" @@ -164,8 +165,8 @@ def _atexit(self) -> None: if self.is_broadcasting: self.stop_broadcasting() if self.save_results: - self.logger.info("Websocket results saved to, %s\n", self.results_file) - if os.path.exists(self.results_file): + self.logger.info("Websocket results saved to, %s\n", str(self.results_path)) + if os.path.exists(self.results_file) and not self.save_results: os.remove(self.results_file) def _setup_database(self) -> None: @@ -476,7 +477,7 @@ def is_broadcasting(self) -> bool: return False @property - def results(self) -> Union[list[dict], None]: + def results(self) -> Union[list[dict], list["Data"], None]: """Retrieve the deserialized results from the results file.""" # pylint: disable=import-outside-toplevel import json # noqa @@ -489,7 +490,12 @@ def results(self) -> Union[list[dict], None]: cursor = conn.execute(f"SELECT * FROM {self.table_name}") # noqa for row in cursor: index, message = row - output.append(json.loads(json.loads(message))) + if self.data_model: + output.append( + self.data_model.model_validate_json(json.loads(message)) + ) + else: + output.append(json.loads(json.loads(message))) if output: return output @@ -665,13 +671,6 @@ def stop_broadcasting(self): self._stop_broadcasting_event.clear() return - @property - def transformed_results(self) -> list["Data"]: - """Model validated records from the results file.""" - if not self.data_model: - raise NotImplementedError("No model provided to transform the results.") - return [self.data_model.model_validate(d) for d in self.results] - def __repr__(self): """Return the WebSocketClient representation.""" return ( diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index d8cedfcba97a..4dc8b1444046 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -41,7 +41,7 @@ def get_logger(name, level=logging.INFO): def handle_validation_error(logger: logging.Logger, error: ValidationError): """Log and raise a Pydantic ValidationError from a provider connection.""" - err = f"{error.__class__.__name__} -> {error.title}: {str(error.json())}" + err = f"{error.__class__.__name__} -> {error.title}: {error.json()}" logger.error(err) raise error from error diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index 5445e0e1b6a4..cd2a9c280344 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -18,6 +18,7 @@ from openbb_core.app.query import Query from openbb_core.app.router import Router from openbb_core.provider.utils.errors import EmptyDataError, UnauthorizedError +from pydantic import ValidationError from openbb_websockets.helpers import ( StdOutSink, @@ -101,9 +102,9 @@ async def get_results(name: str, auth_token: Optional[str] = None) -> OBBject: if not client.results: raise EmptyDataError(f"No results recorded for client {name}.") try: - return OBBject(results=client.transformed_results) - except NotImplementedError: return OBBject(results=client.results) + except ValidationError as e: + raise OpenBBError(e) from e @router.command( From ed2e6732512c21a7bd2d2fe424f821af99d110e0 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 18 Nov 2024 18:12:47 -0800 Subject: [PATCH 035/119] store key and auth_token as encrypted values --- .../websockets/openbb_websockets/broadcast.py | 22 ++++++- .../websockets/openbb_websockets/client.py | 60 ++++++++++++++++--- .../websockets/openbb_websockets/helpers.py | 32 +++++++++- 3 files changed, 102 insertions(+), 12 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py index cabca40c8829..3612a1f7dafb 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -65,7 +65,7 @@ async def websocket_endpoint( # noqa: PLR0915 if ( broadcast_server.auth_token is not None - and auth_token != broadcast_server.auth_token + and auth_token != broadcast_server._decrypt_value(broadcast_server.auth_token) ): await websocket.accept() await websocket.send_text( @@ -131,15 +131,33 @@ def __init__( sleep_time: float = 0.25, auth_token: Optional[str] = None, ): + # pylint: disable=import-outside-toplevel + import os self.results_file = results_file self.table_name = table_name self.logger = get_logger("openbb.websocket.broadcast_server") self.sleep_time = sleep_time - self.auth_token = auth_token self._app = app + self._key = os.urandom(32) + self._iv = os.urandom(16) + self.auth_token = self._encrypt_value(auth_token) if auth_token else None self.websocket = None + def _encrypt_value(self, value: str) -> str: + """Encrypt the value for storage.""" + # pylint: disable=import-outside-toplevel + from openbb_websockets.helpers import encrypt_value + + return encrypt_value(self._key, self._iv, value) + + def _decrypt_value(self, value: str) -> str: + """Decrypt the value for use.""" + # pylint: disable=import-outside-toplevel + from openbb_websockets.helpers import decrypt_value + + return decrypt_value(self._key, self._iv, value) + async def stream_results(self): # noqa: PLR0915 """Continuously read the database and send new messages as JSON via WebSocket.""" import sqlite3 # noqa diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 1aa7178ce502..1e412fbdb35c 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -2,6 +2,7 @@ # pylint: disable=too-many-statements # flake8: noqa: PLR0915 + import logging from typing import TYPE_CHECKING, Any, Literal, Optional, Union @@ -96,6 +97,7 @@ def __init__( # noqa: PLR0913 # pylint: disable=import-outside-toplevel import asyncio # noqa import atexit + import os import tempfile import threading from aiosqlite import DatabaseError @@ -110,13 +112,20 @@ def __init__( # noqa: PLR0913 self.table_name = table_name if table_name else "records" self._limit = limit self.data_model = data_model - self._auth_token = auth_token self._symbol = symbol - self._kwargs = ( - [f"{k}={str(v).strip().replace(' ', '_')}" for k, v in kwargs.items()] - if kwargs - else None - ) + self._key = os.urandom(32) + self._iv = os.urandom(16) + self._auth_token = self._encrypt_value(auth_token) if auth_token else None + # strings in kwargs are encrypted before storing in the class but unencrypted when passed to the provider module. + if kwargs: + for k, v in kwargs.items(): + if isinstance(v, str): + encrypted_value = self._encrypt_value(v) + kwargs[k] = encrypted_value + else: + kwargs[k] = v + + self._kwargs = kwargs if kwargs else {} self._process: Any = None self._psutil_process: Any = None @@ -153,6 +162,20 @@ def __init__( # noqa: PLR0913 self.logger.error(msg) self._exception = OpenBBError(msg) + def _encrypt_value(self, value): + """Encrypt a value before storing.""" + # pylint: disable=import-outside-toplevel + from openbb_websockets.helpers import encrypt_value + + return encrypt_value(self._key, self._iv, value) + + def _decrypt_value(self, encrypted_value): + """Decrypt the value for use.""" + # pylint: disable=import-outside-toplevel + from openbb_websockets.helpers import decrypt_value + + return decrypt_value(self._key, self._iv, encrypted_value) + def _atexit(self) -> None: """Clean up the WebSocket client processes at exit.""" # pylint: disable=import-outside-toplevel @@ -349,8 +372,23 @@ def connect(self) -> None: if self.limit: command.extend([f"limit={self.limit}"]) - if self._kwargs: - for kwarg in self._kwargs: + kwargs = self._kwargs.copy() + + if kwargs: + for k, v in kwargs.items(): + if isinstance(v, str): + unencrypted_value = self._decrypt_value(v) + kwargs[k] = unencrypted_value + else: + kwargs[k] = v + + _kwargs = ( + [f"{k}={str(v).strip().replace(' ', '_')}" for k, v in kwargs.items()] + if kwargs + else None + ) + + for kwarg in _kwargs: if kwarg not in command: command.extend([kwarg]) @@ -566,6 +604,10 @@ def broadcast_address(self) -> Union[str, None]: else None ) + def _get_auth_token(self): + """Get the authentication token.""" + return self._decrypt_value(self._auth_token) if self._auth_token else None + def start_broadcasting( self, host: str = "127.0.0.1", @@ -605,7 +647,7 @@ def start_broadcasting( f"port={open_port}", f"results_file={self.results_file}", f"table_name={self.table_name}", - f"auth_token={self._auth_token}", + f"auth_token={self._get_auth_token()}", ] if kwargs: for kwarg in kwargs: diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 4dc8b1444046..d86a3523a9fb 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -73,6 +73,36 @@ async def get_status(name: Optional[str] = None, client: Optional[Any] = None) - return status +def encrypt_value(key, iv, value): + """Encrypt a value before storing.""" + # pylint: disable=import-outside-toplevel + import base64 # noqa + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + + backend = default_backend() + cipher = Cipher(algorithms.AES(key), modes.CFB(iv), backend=backend) + encryptor = cipher.encryptor() + encrypted_value = encryptor.update(value.encode()) + encryptor.finalize() + return base64.b64encode(encrypted_value).decode() + + +def decrypt_value(key, iv, encrypted_value): + """Decrypt the value for use.""" + # pylint: disable=import-outside-toplevel + import base64 # noqa + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + + backend = default_backend() + cipher = Cipher(algorithms.AES(key), modes.CFB(iv), backend=backend) + decryptor = cipher.decryptor() + decrypted_value = ( + decryptor.update(base64.b64decode(encrypted_value)) + decryptor.finalize() + ) + return decrypted_value.decode() + + async def check_auth(name: str, auth_token: Optional[str] = None) -> bool: """Check the auth token.""" if name not in connected_clients: @@ -82,7 +112,7 @@ async def check_auth(name: str, auth_token: Optional[str] = None) -> bool: return True if auth_token is None: raise UnauthorizedError(f"Client authorization token is required for {name}.") - if auth_token != client._auth_token: + if auth_token != client._get_auth_token(): raise UnauthorizedError(f"Invalid client authorization token for {name}.") return True From 8337346cf5f9847e1ed8fd75d716f3dd74a2b876 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 18 Nov 2024 18:25:54 -0800 Subject: [PATCH 036/119] cleanup --- .../extensions/websockets/openbb_websockets/client.py | 6 +----- .../extensions/websockets/openbb_websockets/helpers.py | 9 +++++++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 1e412fbdb35c..671eb7fe545e 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -604,10 +604,6 @@ def broadcast_address(self) -> Union[str, None]: else None ) - def _get_auth_token(self): - """Get the authentication token.""" - return self._decrypt_value(self._auth_token) if self._auth_token else None - def start_broadcasting( self, host: str = "127.0.0.1", @@ -647,7 +643,7 @@ def start_broadcasting( f"port={open_port}", f"results_file={self.results_file}", f"table_name={self.table_name}", - f"auth_token={self._get_auth_token()}", + f"auth_token={self._decrypt_value(self._auth_token)}", ] if kwargs: for kwarg in kwargs: diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index d86a3523a9fb..78a5b2b8b396 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -2,7 +2,6 @@ import logging import re -import sys from typing import Any, Optional from openbb_core.app.model.abstract.error import OpenBBError @@ -112,7 +111,7 @@ async def check_auth(name: str, auth_token: Optional[str] = None) -> bool: return True if auth_token is None: raise UnauthorizedError(f"Client authorization token is required for {name}.") - if auth_token != client._get_auth_token(): + if auth_token != client._decrypt_value(client._auth_token): raise UnauthorizedError(f"Invalid client authorization token for {name}.") return True @@ -225,6 +224,9 @@ class StdOutSink: def write(self, message): """Write to stdout.""" + # pylint: disable=import-outside-toplevel + import sys + cleaned_message = AUTH_TOKEN_FILTER.sub(r"\1********", message) if cleaned_message != message: cleaned_message = f"{cleaned_message}\n" @@ -232,6 +234,9 @@ def write(self, message): def flush(self): """Flush stdout.""" + # pylint: disable=import-outside-toplevel + import sys + sys.__stdout__.flush() From 2934cbe9f88a121237a02d4b8a24cdd1b897e2b6 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 18 Nov 2024 23:26:03 -0800 Subject: [PATCH 037/119] more cleanup --- .../extensions/websockets/openbb_websockets/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 671eb7fe545e..24055166c0bb 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -158,7 +158,7 @@ def __init__( # noqa: PLR0913 try: self._setup_database() except DatabaseError as e: - msg = f"Unexpected error setting up the SQLite database and table -> {e.__class___.__name__}: {e.__str__()}" + msg = f"Unexpected error setting up the SQLite database and table -> {e.__class___.__name__}: {e}" self.logger.error(msg) self._exception = OpenBBError(msg) @@ -765,7 +765,9 @@ def send_message( else: client.logger.error("Broadcast process is not running.") except Exception as e: - msg = f"Error sending message to WebSocket process: {e.__class__.__name__} -> {e.__str__()}" + msg = ( + f"Error sending message to WebSocket process: {e.__class__.__name__} -> {e}" + ) client.logger.error(msg) From 7f3b7bc84c876d141ec0a8b197ab5dbf57f0c67c Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 19 Nov 2024 18:31:41 -0800 Subject: [PATCH 038/119] small update --- .../extensions/websockets/README.md | 34 ++++++++++--------- .../websockets/openbb_websockets/models.py | 6 ++-- .../openbb_fmp/models/websocket_connection.py | 8 ++--- .../fmp/openbb_fmp/utils/websocket_client.py | 16 +++++---- .../providers/fmp/tests/test_fmp_fetchers.py | 17 ++++++++++ .../models/websocket_connection.py | 8 ++--- .../openbb_polygon/utils/websocket_client.py | 16 ++++----- .../models/websocket_connection.py | 8 ++--- .../openbb_tiingo/utils/websocket_client.py | 12 +++---- 9 files changed, 70 insertions(+), 55 deletions(-) diff --git a/openbb_platform/extensions/websockets/README.md b/openbb_platform/extensions/websockets/README.md index ee9c120a3cec..128d9010734e 100644 --- a/openbb_platform/extensions/websockets/README.md +++ b/openbb_platform/extensions/websockets/README.md @@ -33,7 +33,7 @@ obb.websockets ``` > Except for, `get_results`, functions do not return the data or stream. Outputs will be a WebSocketConnectionStatus instance, or a string message. -> All functions, except `create_connection`, assume that a connection has already been establiehd and are referenced by parameters: +> All functions, except `create_connection`, assume that a connection has already been established and are referenced by parameters: > > |Parameter|Type | Required| Description | > |:-------|:-----|:--------:|------------:| @@ -52,7 +52,7 @@ All other endpoints require this to be used first. It is the only function mappi |:-------|:-----|:--------:|------------:| |provider |String |Yes |Name of the provider - i.e, `"polygon"`, `"fmp"`, `"tiingo"` | |name |String |Yes |Name to assign the connection. This is the 'name' parameter in the other endpoints.| -|auth_token |String |No |When supplied, the same token must be passed for all future interactions with the connection, or to read from the broadcast server. | +|auth_token |String |No |When supplied, the same token must be passed as a URL parameter to the broadcast server, and to interact with the client from the API. | |results_file |String |No |Absolute path to the file for continuous writing. Temp file is created by default. Unless 'save_results' is True, discarded on exit. | |save_results |Boolean |No |Whether to persist the file after the session ends, default is `False` | |table_name |String |No |Name of the SQL table to write the results to, consisting of an auto-increment ID and a serialized JSON string of the data. Default is `"records"`| @@ -107,7 +107,7 @@ conn.results.model_dump() ``` ```sh -{'name': 'crypto_tiingo', +{'status': {'name': 'crypto_tiingo', 'auth_required': False, 'subscribed_symbols': '*', 'is_running': True, @@ -117,9 +117,11 @@ conn.results.model_dump() 'broadcast_pid': 5813, 'results_file': '/var/folders/kc/j2lm7bkd5dsfqqnvz259gm6c0000gn/T/tmpwb4jslbg', 'table_name': 'records', - 'save_results': False} + 'save_results': False}} ``` +> From the Python interface, the client is also included in the results. Access it from `results.client` + All of the currently captured data can be dumped with the `get_results` endpoint. The return will be the typical data response object. ```python @@ -450,8 +452,6 @@ PROVIDER INFO: unsubscribed to: C.XAU/USD 'save_results': False} ``` - - ## Development ### Provider Interface @@ -609,7 +609,7 @@ class FmpWebSocketData(WebSocketData): This model is what we return from the `FmpWebSocketFetcher`. The provider will inherit two fields, only the `client` needs to be included on output. No additional fields should be defined. - client - - Instance of WebSocketClient, not returned to from the Router. + - Instance of WebSocketClient, not returned to the API. - status - Leave empty, the Router fills this and returns it. @@ -701,6 +701,8 @@ from openbb_websockets.client import WebSocketClient This is where things diverge slightly. Instead of returning `FmpWebSocketData`, it gets passed to the client connection for validating records as they are received. What gets returned by the Fetcher is the `WebSocketConnection`. +> + ```python class FmpWebSocketFetcher(Fetcher[FmpWebSocketQueryParams, FmpWebSocketConnection]): """FMP WebSocket model.""" @@ -715,7 +717,7 @@ class FmpWebSocketFetcher(Fetcher[FmpWebSocketQueryParams, FmpWebSocketConnectio query: FmpWebSocketQueryParams, credentials: Optional[dict[str, str]], **kwargs: Any, - ) -> WebSocketClient: + ) -> dict: """Extract data from the WebSocket.""" # pylint: disable=import-outside-toplevel import asyncio @@ -766,19 +768,19 @@ class FmpWebSocketFetcher(Fetcher[FmpWebSocketQueryParams, FmpWebSocketConnectio raise e from e # Check if the process is still running before returning. if client.is_running: - return client + return {"client": client} raise OpenBBError("Failed to connect to the WebSocket.") @staticmethod def transform_data( - data: WebSocketClient, + data: dict, query: FmpWebSocketQueryParams, **kwargs: Any, ) -> FmpWebSocketConnection: """Return the client as an instance of Data.""" # All we need to do here is return our client wrapped in the WebSocketConnection class. - return FmpWebSocketConnection(client=data) + return FmpWebSocketConnection(client=data["client"]) ``` #### Map To Router @@ -846,7 +848,7 @@ logger = get_logger("openbb.websocket.fmp") # A UUID gets attached to the name s #### `MessageQueue` -This is an async Queue with an input for the message handler. Create a second instance if a separate queue is required for the subscibe events. +This is an async Queue with an input for the message handler. Create a second instance if a separate queue is required for the subscribe events. Define your async message handler function, and create a task to run in the main event loop. @@ -875,7 +877,7 @@ message = await queue.dequeue() Before submitting the record to `write_to_db`, validate and transform the data with the WebSocketData that was created and imported. Use this function right before transmission, a failure will trigger a termination signal from the main application. ```python -# code above confirms that the message being processed is a data message and not an info message or error. +# code above confirms that the message being processed is a data message and not an info or error message. try: result = FmpWebSocketData.model_validate(message).model_dump_json( @@ -920,8 +922,8 @@ if __name__ == "__main__": for sig in (signal.SIGINT, signal.SIGTERM): loop.add_signal_handler(sig, handle_termination_signal, logger) - # asyncio.run_coroutine_threadsafe(some_connect_and_stream_function, loop) - # loop.run_forever() + asyncio.run_coroutine_threadsafe(some_connect_and_stream_function, loop) + loop.run_forever() ... ``` @@ -930,7 +932,7 @@ if __name__ == "__main__": The missing pieces that get created locally include: - Read `stdin` function for receiving subscribe/unsubscribe events while the connection is running. - - Messages to handle will always have the same format: `'{"event": "subscribe", "symbol": ticker}'` + - Messages to handle will always have the same format: `'{"event": "(un)subscribe", "symbol": ticker}'` - Converting for the symbology used by the provider needs to happen here. - Implementation depends on the requirements of the provider - i.e, how to structure send events. - Create the task before the initial `websockets.connect` block. diff --git a/openbb_platform/extensions/websockets/openbb_websockets/models.py b/openbb_platform/extensions/websockets/openbb_websockets/models.py index 8e53e72233ed..ae90ff2352e9 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/models.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/models.py @@ -172,8 +172,8 @@ def _validate_client(cls, v): @model_validator(mode="before") @classmethod - def _validate_inputs(cls, vaules): + def _validate_inputs(cls, values): """Validate the status.""" - if not vaules.get("status") and not vaules.get("client"): + if not values.get("status") and not values.get("client"): raise ValueError("Cannot initialize empty.") - return vaules + return values diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py index b157e572548a..c9fd7a5d7654 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py @@ -140,7 +140,7 @@ async def aextract_data( query: FmpWebSocketQueryParams, credentials: Optional[dict[str, str]], **kwargs: Any, - ) -> WebSocketClient: + ) -> dict: """Extract data from the WebSocket.""" # pylint: disable=import-outside-toplevel import asyncio @@ -183,15 +183,15 @@ async def aextract_data( raise e from e if client.is_running: - return client + return {"client": client} raise OpenBBError("Failed to connect to the WebSocket.") @staticmethod def transform_data( - data: WebSocketClient, + data: dict, query: FmpWebSocketQueryParams, **kwargs: Any, ) -> FmpWebSocketConnection: """Return the client as an instance of Data.""" - return FmpWebSocketConnection(client=data) + return FmpWebSocketConnection(client=data["client"]) diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py index b54890874bc8..75bfb8915da6 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -48,7 +48,7 @@ async def login(websocket, api_key): msg = message.get("message") logger.info("PROVIDER INFO: %s", msg) except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e.__str__()}" + msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" logger.error(msg) sys.exit(1) @@ -66,7 +66,7 @@ async def subscribe(websocket, symbol, event): await websocket.send(json.dumps(subscribe_event)) await asyncio.sleep(1) except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e.__str__()}" + msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" logger.error(msg) @@ -105,6 +105,10 @@ async def process_message(message, results_path, table_name, limit): if "you are not authorized" in message.get("message", "").lower(): msg = f"UnauthorizedError -> FMP Message: {message['message']}" logger.error(msg) + elif "Connected from another location" in message.get("message", ""): + msg = f"UnauthorizedError -> FMP Message: {message.get('message')}" + logger.info(msg) + sys.exit(0) else: msg = f"PROVIDER INFO: {message.get('message')}" logger.info(msg) @@ -156,9 +160,7 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim await asyncio.shield(queue.enqueue(json.loads(message))) except websockets.ConnectionClosed as e: - msg = ( - f"PROVIDER INFO: The WebSocket connection was closed -> {e.__str__()}" - ) + msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" logger.info(msg) # Attempt to reopen the connection logger.info("PROVIDER INFO: Attempting to reconnect after five seconds.") @@ -170,7 +172,7 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim sys.exit(1) except Exception as e: - msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e.__str__()}" + msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e}" logger.error(msg) sys.exit(1) @@ -207,7 +209,7 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim logger.error("PROVIDER ERROR: WebSocket connection closed") except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e.__str__()}" + msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" logger.error(msg) finally: diff --git a/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py b/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py index 60074329bc6f..42d48f6e8f3d 100644 --- a/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py +++ b/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py @@ -70,6 +70,7 @@ from openbb_fmp.models.risk_premium import FMPRiskPremiumFetcher from openbb_fmp.models.share_statistics import FMPShareStatisticsFetcher from openbb_fmp.models.treasury_rates import FMPTreasuryRatesFetcher +from openbb_fmp.models.websocket_connection import FmpWebSocketFetcher from openbb_fmp.models.world_news import FMPWorldNewsFetcher from openbb_fmp.models.yield_curve import FMPYieldCurveFetcher @@ -777,3 +778,19 @@ def test_fmp_historical_market_cap_fetcher(credentials=test_credentials): fetcher = FmpHistoricalMarketCapFetcher() result = fetcher.test(params, credentials) assert result is None + + +@pytest.mark.record_screen +def test_fmp_websocket_fetcher(credentials=test_credentials): + """Test FMP Websocket fetcher.""" + import asyncio + + params = {"symbol": "btcusd", "name": "test", "limit": 10, "asset_type": "crypto"} + + try: + fetcher = FmpWebSocketFetcher() + result = fetcher.test(params, credentials) + response = asyncio.run(fetcher.fetch_data(params, credentials)) + assert result is None + finally: + response.client.disconnect() diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 0960bb4850db..48d3f708ffdd 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -1219,7 +1219,7 @@ def extract_data( query: PolygonWebSocketQueryParams, credentials: Optional[dict[str, str]], **kwargs: Any, - ) -> WebSocketClient: + ) -> dict: """Extract data from the WebSocket.""" api_key = credentials.get("polygon_api_key") if credentials else "" url = URL_MAP[query.asset_type] @@ -1261,15 +1261,15 @@ def extract_data( raise client._exception from client._exception if client.is_running: - return client + return {"client": client} raise OpenBBError("Failed to connect to the WebSocket.") @staticmethod def transform_data( - data: WebSocketClient, + data: dict, query: PolygonWebSocketQueryParams, **kwargs: Any, ) -> PolygonWebSocketConnection: """Return the client as an instance of Data.""" - return PolygonWebSocketConnection(client=data) + return PolygonWebSocketConnection(client=data["client"]) diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index e22421abe2bd..cdb706542ee8 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -233,27 +233,23 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim except websockets.InvalidStatusCode as e: if e.status_code == 404: - msg = f"PROVIDER ERROR: {e.__str__()}" + msg = f"PROVIDER ERROR: {e}" logger.error(msg) sys.exit(1) else: raise except websockets.InvalidURI as e: - msg = f"PROVIDER ERROR: {e.__str__()}" + msg = f"PROVIDER ERROR: {e}" logger.error(msg) sys.exit(1) except websockets.ConnectionClosedOK as e: - msg = ( - f"PROVIDER INFO: The WebSocket connection was closed -> {e.__str__()}" - ) + msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" logger.info(msg) sys.exit(0) except websockets.ConnectionClosed as e: - msg = ( - f"PROVIDER INFO: The WebSocket connection was closed -> {e.__str__()}" - ) + msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" logger.info(msg) # Attempt to reopen the connection logger.info("PROVIDER INFO: Attempting to reconnect after five seconds.") @@ -261,12 +257,12 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim await connect_and_stream(url, symbol, api_key, results_path, table_name, limit) except websockets.WebSocketException as e: - msg = f"PROVIDER ERROR: WebSocketException -> {e.__str__()}" + msg = f"PROVIDER ERROR: WebSocketException -> {e}" logger.error(msg) sys.exit(1) except Exception as e: - msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e.__str__()}" + msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e}" logger.error(msg) sys.exit(1) diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py index 3b083645952d..ffeb3a3781d6 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py @@ -233,7 +233,7 @@ async def aextract_data( query: TiingoWebSocketQueryParams, credentials: Optional[dict[str, str]], **kwargs: Any, - ) -> WebSocketClient: + ) -> dict: """Initiailze the WebSocketClient and connect.""" # pylint: disable=import-outside-toplevel from asyncio import sleep @@ -291,16 +291,16 @@ async def aextract_data( raise OpenBBError(exc) if client.is_running: - return client + return {"client": client} client.disconnect() raise OpenBBError("Failed to connect to the WebSocket.") @staticmethod def transform_data( - data: WebSocketClient, + data: dict, query: TiingoWebSocketQueryParams, **kwargs: Any, ) -> TiingoWebSocketConnection: """Return the client as an instance of Data.""" - return TiingoWebSocketConnection(client=data) + return TiingoWebSocketConnection(client=data["client"]) diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index 50ca68c09c19..25243c9dc55a 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -236,9 +236,7 @@ async def connect_and_stream( sys.exit(1) except websockets.ConnectionClosed as e: - msg = ( - f"PROVIDER INFO: The WebSocket connection was closed -> {e.__str__()}" - ) + msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" logger.info(msg) # Attempt to reopen the connection logger.info("PROVIDER INFO: Attempting to reconnect after five seconds...") @@ -248,11 +246,11 @@ async def connect_and_stream( ) except websockets.WebSocketException as e: - logger.error(str(e)) - sys.exit(1) + logger.info(str(e)) + sys.exit(0) except Exception as e: - msg = f"Unexpected error -> {e.__class__.__name__}: {e.__str__()}" + msg = f"Unexpected error -> {e.__class__.__name__}: {e}" logger.error(msg) sys.exit(1) @@ -286,7 +284,7 @@ async def connect_and_stream( loop.run_forever() except Exception as e: # pylint: disable=broad-except - msg = f"Unexpected error -> {e.__class__.__name__}: {e.__str__()}" + msg = f"Unexpected error -> {e.__class__.__name__}: {e}" logger.error(msg) finally: From 5d8d9b47d1c64ec59454d12ae7047954b70316c2 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:04:08 -0800 Subject: [PATCH 039/119] don't decrypt auth_token when value is None --- .../websockets/openbb_websockets/client.py | 2 +- .../openbb_websockets/websockets_router.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 24055166c0bb..a243fd779a81 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -643,7 +643,7 @@ def start_broadcasting( f"port={open_port}", f"results_file={self.results_file}", f"table_name={self.table_name}", - f"auth_token={self._decrypt_value(self._auth_token)}", + f"auth_token={self._decrypt_value(self._auth_token) if self._auth_token else None}", ] if kwargs: for kwarg in kwargs: diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index cd2a9c280344..f75af9b37f3e 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -67,8 +67,16 @@ async def create_connection( raise OpenBBError("Client failed to connect.") if hasattr(extra_params, "start_broadcast") and extra_params.start_broadcast: - client.start_broadcasting() - await asyncio.sleep(1) + try: + client.start_broadcasting() + await asyncio.sleep(1) + if client._exception is not None: + exc = getattr(client, "_exception", None) + client._exception = None + raise OpenBBError(exc) + except Exception as e: # pylint: disable=broad-except + client._atexit() + raise e from e client_name = client.name connected_clients[client_name] = client From a4336725c653769d3a0a5551458a72d6b0ec0db8 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 21 Nov 2024 16:09:07 -0800 Subject: [PATCH 040/119] handle url in parse_kwargs --- .../extensions/websockets/openbb_websockets/helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 78a5b2b8b396..ae7f30836f1a 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -134,8 +134,12 @@ def parse_kwargs(): import sys args = sys.argv[1:].copy() + sys.stdout.write(f"ARGS: {args}\n") _kwargs: dict = {} for i, arg in enumerate(args): + if arg.startswith("url"): + _kwargs["url"] = arg[4:] + continue if "=" in arg: key, value = arg.split("=") From e6367521d7b35331796849ef89632666967787ac Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 30 Nov 2024 18:55:54 -0800 Subject: [PATCH 041/119] small touchups --- .../websockets/openbb_websockets/client.py | 36 ++++++++++++------- .../websockets/openbb_websockets/helpers.py | 16 +++++++++ 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index a243fd779a81..e2ffbb0184cb 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -197,6 +197,7 @@ def _setup_database(self) -> None: # pylint: disable=import-outside-toplevel import asyncio # noqa import threading + import time from openbb_websockets.helpers import setup_database def run_in_new_loop(): @@ -214,16 +215,14 @@ def run_in_thread(): """Run setup in separate thread.""" thread = threading.Thread(target=run_in_new_loop) thread.start() + time.sleep(0.01) thread.join() try: - try: - loop = asyncio.get_running_loop() # noqa - run_in_thread() - except RuntimeError: - run_in_new_loop() - finally: - return + loop = asyncio.get_running_loop() # noqa + run_in_thread() + except RuntimeError: + run_in_new_loop() def _log_provider_output(self, output_queue) -> None: """Log output from the provider logger, handling exceptions, errors, and messages that are not data.""" @@ -294,6 +293,12 @@ def _log_provider_output(self, output_queue) -> None: break output = clean_message(output) + + if output.startswith("ERROR:"): + output = output.replace("ERROR:", "PROVIDER ERROR:") + elif output.startswith("INFO:"): + output = output.replace("INFO:", "PROVIDER INFO:") + output = output + "\n" sys.stdout.write(output + "\n") sys.stdout.flush() @@ -735,8 +740,10 @@ def non_blocking_websocket(client, output_queue, provider_message_queue) -> None output_queue.put(output.strip()) except Exception as e: + client.logger.error( + f"Unexpected error in non_blocking_websocket: {e.__class__.__name__} -> {e}" + ) raise e from e - client.logger.error(f"Error in non_blocking_websocket: {e}") finally: client._process.stdout.close() client._process.wait() @@ -765,9 +772,7 @@ def send_message( else: client.logger.error("Broadcast process is not running.") except Exception as e: - msg = ( - f"Error sending message to WebSocket process: {e.__class__.__name__} -> {e}" - ) + msg = f"Error sending message to the {target} process: {e.__class__.__name__} -> {e}" client.logger.error(msg) @@ -788,7 +793,10 @@ def read_message_queue( if message: send_message(client, message, target="broadcast") except Exception as e: - err = f"Error reading message queue: {e.args[0]} -> {message}" + err = ( + f"Error while attempting to transmit from the outgoing message queue: {e.__class__.__name__} " + f"-> {e} -> {message}" + ) client.logger.error(err) finally: break @@ -807,7 +815,9 @@ def non_blocking_broadcast(client, output_queue, broadcast_message_queue) -> Non if output: output_queue.put(output.strip()) except Exception as e: - client.logger.error(f"Error in non_blocking_broadcast: {e}") + client.logger.error( + f"Unexpected error in non_blocking_broadcast: {e.__class__.__name__} -> {e}" + ) finally: client._broadcast_process.stdout.close() client._broadcast_process.wait() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index ae7f30836f1a..3c4ebed3a191 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -195,11 +195,24 @@ async def write_to_db(message, results_path, table_name, limit): import aiosqlite conn = await aiosqlite.connect(results_path) + + # Check if the table exists and create it if it doesn't + await conn.execute( + f""" + CREATE TABLE IF NOT EXISTS {table_name} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message TEXT + ) + """ + ) + await conn.commit() + await conn.execute( f"INSERT INTO {table_name} (message) VALUES (?)", # noqa (json.dumps(message),), ) await conn.commit() + records = await conn.execute(f"SELECT COUNT(*) FROM {table_name}") # noqa count = (await records.fetchone())[0] count = await conn.execute(f"SELECT COUNT(*) FROM {table_name}") # noqa @@ -265,6 +278,7 @@ def __init__(self, max_size: int = 1000, max_retries=5, backoff_factor=0.5): self.logger = get_logger("openbb.websocket.queue") async def dequeue(self): + """Dequeue a message.""" return await self.queue.get() async def enqueue(self, message): @@ -286,10 +300,12 @@ async def enqueue(self, message): self.logger.error("Failed to enqueue message after maximum retries.") async def process_queue(self, handler): + """Process the message queue.""" while True: message = await self.queue.get() await self._process_message(message, handler) self.queue.task_done() async def _process_message(self, message, handler): + """Process the message with the handler coroutine.""" await handler(message) From 020becf2c0e51365c77ee13f1ec8958fec27bcd7 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sun, 1 Dec 2024 13:20:40 -0800 Subject: [PATCH 042/119] raise from client when attempting to subscribe or unsubscribe while connection is not running --- .../websockets/openbb_websockets/broadcast.py | 11 ++--- .../websockets/openbb_websockets/client.py | 42 ++++++++++++------- .../websockets/openbb_websockets/helpers.py | 6 +-- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py index 3612a1f7dafb..707b5400347b 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -40,12 +40,7 @@ async def read_stdin(broadcast_server): if line.strip().startswith("{") or line.strip().startswith("[") else line.strip() ) - msg = ( - "BROADCAST INFO: Message received from parent process and relayed to active listeners ->" - + f" {json.dumps(command)}" - ) await broadcast_server.broadcast(json.dumps(command)) - broadcast_server.logger.info(msg) except json.JSONDecodeError: broadcast_server.logger.error("Invalid JSON received from stdin") @@ -270,10 +265,12 @@ def main(): **kwargs, ) except TypeError as e: - msg = f"Invalid keyword argument passed to unvicorn. -> {e.args[0]}\n" + msg = ( + f"ERROR: Invalid keyword argument passed to unvicorn. -> {e.args[0]}\n" + ) broadcast_server.logger.error(msg) except KeyboardInterrupt: - broadcast_server.logger.info("Broadcast server terminated.") + broadcast_server.logger.info("INFO: Broadcast server terminated.") finally: sys.exit(0) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index e2ffbb0184cb..a30de3bf6bfe 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -477,6 +477,9 @@ def subscribe(self, symbol) -> None: import time from openbb_core.app.model.abstract.error import OpenBBError + if not self.is_running: + raise OpenBBError("Provider connection is not running.") + ticker = symbol if isinstance(symbol, list) else symbol.split(",") msg = {"event": "subscribe", "symbol": ticker} self.send_message(json.dumps(msg)) @@ -492,15 +495,21 @@ def subscribe(self, symbol) -> None: def unsubscribe(self, symbol) -> None: """Unsubscribe from a symbol or list of symbols.""" # pylint: disable=import-outside-toplevel - import json + import json # noqa + import time + from openbb_core.app.model.abstract.error import OpenBBError if not self.symbol: self.logger.info("No subscribed symbols.") return + if not self.is_running: + raise OpenBBError("Provider connection is not running.") + ticker = symbol if isinstance(symbol, list) else symbol.split(",") msg = {"event": "unsubscribe", "symbol": ticker} self.send_message(json.dumps(msg)) + time.sleep(0.1) old_symbols = self.symbol.split(",") new_symbols = list(set(old_symbols) - set(ticker)) self._symbol = ",".join(new_symbols) @@ -782,22 +791,25 @@ def read_message_queue( """Read messages from the queue and send them to the WebSocketConnection process.""" while not message_queue.empty(): try: - if target == "provider": - while not client._stop_log_thread_event.is_set(): - message = message_queue.get(timeout=1) - if message: + message = message_queue.get(timeout=1) + if message: + try: + if ( + target == "provider" + and not client._stop_log_thread_event.is_set() + ): send_message(client, message, target="provider") - elif target == "broadcast": - while not client._stop_broadcasting_event.is_set(): - message = message_queue.get(timeout=1) - if message: + elif ( + target == "broadcast" + and not client._stop_broadcasting_event.is_set() + ): send_message(client, message, target="broadcast") - except Exception as e: - err = ( - f"Error while attempting to transmit from the outgoing message queue: {e.__class__.__name__} " - f"-> {e} -> {message}" - ) - client.logger.error(err) + except Exception as e: + err = ( + f"Error while attempting to transmit from the outgoing message queue: {e.__class__.__name__} " + f"-> {e} -> {message}" + ) + client.logger.error(err) finally: break diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 3c4ebed3a191..11a52cab71bd 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -169,14 +169,14 @@ async def setup_database(results_path, table_name): import os # noqa import aiosqlite - async with aiosqlite.connect(results_path) as conn: + async with aiosqlite.connect(results_path, check_same_thread=False) as conn: if os.path.exists(results_path): try: await conn.execute("SELECT name FROM sqlite_master WHERE type='table';") except aiosqlite.DatabaseError: os.remove(results_path) - async with aiosqlite.connect(results_path) as conn: + async with aiosqlite.connect(results_path, check_same_thread=False) as conn: await conn.execute( f""" CREATE TABLE IF NOT EXISTS {table_name} ( @@ -194,7 +194,7 @@ async def write_to_db(message, results_path, table_name, limit): import json # noqa import aiosqlite - conn = await aiosqlite.connect(results_path) + conn = await aiosqlite.connect(results_path, check_same_thread=False) # Check if the table exists and create it if it doesn't await conn.execute( From 4fe088449350a71ab6d0662e0aa37356a5080aeb Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sun, 1 Dec 2024 21:52:31 -0800 Subject: [PATCH 043/119] missing docstrings --- .../extensions/websockets/openbb_websockets/helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 11a52cab71bd..2917fcdb1c5b 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -165,6 +165,7 @@ def parse_kwargs(): async def setup_database(results_path, table_name): + """Setup the SQLite database.""" # pylint: disable=import-outside-toplevel import os # noqa import aiosqlite @@ -261,6 +262,7 @@ class AuthTokenFilter(logging.Formatter): """Custom logging formatter to filter auth tokens.""" def format(self, record): + """Format the log record.""" original_message = super().format(record) cleaned_message = AUTH_TOKEN_FILTER.sub(r"\1********", original_message) return cleaned_message From 754697371318117662b305409b6dfbe411ea7209 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:40:59 -0800 Subject: [PATCH 044/119] try adding intrinio stocks --- .../websockets/openbb_websockets/client.py | 6 +- .../websockets/openbb_websockets/helpers.py | 6 +- .../intrinio/openbb_intrinio/__init__.py | 2 + .../models/websocket_connection.py | 174 +++++ .../openbb_intrinio/utils/stocks_client.py | 611 ++++++++++++++++++ .../openbb_intrinio/utils/websocket_client.py | 142 ++++ 6 files changed, 937 insertions(+), 4 deletions(-) create mode 100644 openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py create mode 100644 openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py create mode 100644 openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index a30de3bf6bfe..79e2ae2cde76 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -544,7 +544,11 @@ def results(self) -> Union[list[dict], list["Data"], None]: index, message = row if self.data_model: output.append( - self.data_model.model_validate_json(json.loads(message)) + self.data_model.model_validate_json( + json.loads(message) + if isinstance(message, str) + else message + ) ) else: output.append(json.loads(json.loads(message))) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 2917fcdb1c5b..9854cdadfb4c 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -170,14 +170,14 @@ async def setup_database(results_path, table_name): import os # noqa import aiosqlite - async with aiosqlite.connect(results_path, check_same_thread=False) as conn: + async with aiosqlite.connect(results_path) as conn: if os.path.exists(results_path): try: await conn.execute("SELECT name FROM sqlite_master WHERE type='table';") except aiosqlite.DatabaseError: os.remove(results_path) - async with aiosqlite.connect(results_path, check_same_thread=False) as conn: + async with aiosqlite.connect(results_path) as conn: await conn.execute( f""" CREATE TABLE IF NOT EXISTS {table_name} ( @@ -195,7 +195,7 @@ async def write_to_db(message, results_path, table_name, limit): import json # noqa import aiosqlite - conn = await aiosqlite.connect(results_path, check_same_thread=False) + conn = await aiosqlite.connect(results_path) # Check if the table exists and create it if it doesn't await conn.execute( diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/__init__.py b/openbb_platform/providers/intrinio/openbb_intrinio/__init__.py index 4152bab8de12..fa069937a06c 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/__init__.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/__init__.py @@ -61,6 +61,7 @@ IntrinioSearchAttributesFetcher, ) from openbb_intrinio.models.share_statistics import IntrinioShareStatisticsFetcher +from openbb_intrinio.models.websocket_connection import IntrinioWebSocketFetcher from openbb_intrinio.models.world_news import IntrinioWorldNewsFetcher intrinio_provider = Provider( @@ -108,6 +109,7 @@ "ReportedFinancials": IntrinioReportedFinancialsFetcher, "SearchAttributes": IntrinioSearchAttributesFetcher, "ShareStatistics": IntrinioShareStatisticsFetcher, + "WebSocketConnection": IntrinioWebSocketFetcher, "WorldNews": IntrinioWorldNewsFetcher, }, repr_name="Intrinio", diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py new file mode 100644 index 000000000000..cc926dd0b6ee --- /dev/null +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py @@ -0,0 +1,174 @@ +"""Intrinio WebSocket model.""" + +from datetime import datetime +from typing import Any, Literal, Optional + +from openbb_core.app.model.abstract.error import OpenBBError +from openbb_core.provider.abstract.fetcher import Fetcher +from openbb_websockets.client import WebSocketClient +from openbb_websockets.models import ( + WebSocketConnection, + WebSocketData, + WebSocketQueryParams, +) +from pydantic import Field, field_validator + + +class IntrinioWebSocketQueryParams(WebSocketQueryParams): + """Intrinio WebSocket query parameters.""" + + __json_schema_extra__ = { + "symbol": {"multiple_items_allowed": True}, + "asset_type": { + "multiple_items_allowed": False, + "choices": ["stock"], + }, + "feed": { + "multiple_items_allowed": False, + "choices": ["realtime", "delayed_sip", "nasdaq_basic"], + }, + } + + symbol: str = Field( + description="The Intrinio symbol to get data for.", + ) + asset_type: Literal["stock"] = Field( + default="stock", + description="The asset type associated with the symbol.", + ) + feed: Literal["realtime", "delayed_sip", "nasdaq_basic"] = Field( + default="realtime", + description="The feed to get data from.", + ) + trades_only: bool = Field( + default=False, + description="Whether to only get trade data.", + ) + + +class IntrinioWebSocketData(WebSocketData): + """Intrinio WebSocket data model.""" + + __alias_dict__ = { + "date": "timestamp", + "feed": "subprovider", + "exchange": "market_center", + "volume": "total_volume", + } + + exchange: Optional[str] = Field( + default=None, + description="The exchange of the data.", + ) + type: Literal["quote", "trade"] = Field( + description="The type of data.", + ) + price: Optional[float] = Field( + default=None, + description="The price of the trade or quote.", + json_schema_extra={"x-unit_measurement": "currency"}, + ) + size: Optional[int] = Field( + default=None, + description="The size of the trade or quote.", + ) + volume: Optional[int] = Field( + default=None, + description="The total volume of the trade or quote.", + ) + condition: Optional[str] = Field( + default=None, + description="The condition attached to the trade or quote.", + ) + + @field_validator("date", mode="before", check_fields=False) + def _validate_date(cls, v): + """Validate the date.""" + # pylint: disable=import-outside-toplevel + from pytz import timezone + + if isinstance(v, str): + dt = datetime.fromisoformat(v) + try: + dt = datetime.fromtimestamp(v / 1000) + except Exception: # pylint: disable=broad-except + if isinstance(v, (int, float)): + # Check if the timestamp is in nanoseconds and convert to seconds + if v > 1e12: + v = v / 1e9 # Convert nanoseconds to seconds + dt = datetime.fromtimestamp(v) + + return dt.astimezone(timezone("America/New_York")) + + +class IntrinioWebSocketConnection(WebSocketConnection): + """Intrinio WebSocket connection model.""" + + +class IntrinioWebSocketFetcher( + Fetcher[IntrinioWebSocketQueryParams, IntrinioWebSocketConnection] +): + """Intrinio WebSocket Fetcher.""" + + @staticmethod + def transform_query(params: dict[str, Any]) -> IntrinioWebSocketQueryParams: + """Transform the query parameters.""" + return IntrinioWebSocketQueryParams(**params) + + @staticmethod + async def aextract_data( + query: IntrinioWebSocketQueryParams, + credentials: Optional[dict[str, str]], + **kwargs: Any, + ) -> dict: + """Extract data from the WebSocket.""" + # pylint: disable=import-outside-toplevel + import asyncio + + api_key = credentials.get("intrinio_api_key") if credentials else "" + + kwargs = { + "api_key": api_key, + "feed": query.feed or "realtime", + "connect_kwargs": query.connect_kwargs, + } + + client = WebSocketClient( + name=query.name, + module="openbb_intrinio.utils.websocket_client", + symbol=query.symbol, + limit=query.limit, + results_file=query.results_file, + table_name=query.table_name, + save_results=query.save_results, + data_model=IntrinioWebSocketData, + sleep_time=query.sleep_time, + broadcast_host=query.broadcast_host, + broadcast_port=query.broadcast_port, + auth_token=query.auth_token, + **kwargs, + ) + + try: + client.connect() + await asyncio.sleep(2) + if client._exception: + raise client._exception + except OpenBBError as e: + if client.is_running: + client.disconnect() + raise e from e + + if client.is_running: + return {"client": client} + + raise OpenBBError("Failed to connect to the WebSocket.") + + @staticmethod + def transform_data( + data: dict, + query: IntrinioWebSocketQueryParams, + **kwargs: Any, + ) -> IntrinioWebSocketConnection: + """Return the client as an instance of Data.""" + return IntrinioWebSocketConnection(client=data["client"]) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py new file mode 100644 index 000000000000..bffbfefd7899 --- /dev/null +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py @@ -0,0 +1,611 @@ +"""Intrinio Realtime Stocks Client.""" + +# pylint: disable +# flake8: noqa +# This file is a slightly modified version of the original file from the Intrinio Python SDK. + +import logging +import queue +import struct +import sys +import threading +import time +from typing import Any, Dict, Optional + +import requests +import websocket +from openbb_core.provider.utils.errors import UnauthorizedError + +SELF_HEAL_BACKOFFS = [10, 30, 60, 300, 600] +REALTIME = "REALTIME" +DELAYED_SIP = "DELAYED_SIP" +NASDAQ_BASIC = "NASDAQ_BASIC" +MANUAL = "MANUAL" +PROVIDERS = [REALTIME, MANUAL, DELAYED_SIP, NASDAQ_BASIC] +NO_SUBPROVIDER = "NO_SUBPROVIDER" +CTA_A = "CTA_A" +CTA_B = "CTA_B" +UTP = "UTP" +OTC = "OTC" +NASDAQ_BASIC = "NASDAQ_BASIC" +IEX = "IEX" +SUB_PROVIDERS = [NO_SUBPROVIDER, CTA_A, CTA_B, UTP, OTC, NASDAQ_BASIC, IEX] +MAX_QUEUE_SIZE = 250000 +DEBUGGING = sys.gettrace() is not None +HEADER_MESSAGE_FORMAT_KEY = "UseNewEquitiesFormat" +HEADER_MESSAGE_FORMAT_VALUE = "v2" +HEADER_CLIENT_INFORMATION_KEY = "Client-Information" +HEADER_CLIENT_INFORMATION_VALUE = "IntrinioPythonSDKv5.3.0" + + +class Quote: + def __init__( + self, + symbol, + type, + price, + size, + timestamp, + subprovider, + market_center, + condition, + ): + self.symbol = symbol + self.type = type + self.price = price + self.size = size + self.timestamp = timestamp + self.subprovider = subprovider + self.market_center = market_center + self.condition = condition + + def __str__(self): + return ( + self.symbol + + ", " + + self.type + + ", price: " + + str(self.price) + + ", size: " + + str(self.size) + + ", timestamp: " + + str(self.timestamp) + + ", subprovider: " + + str(self.subprovider) + + ", market_center: " + + str(self.market_center) + + ", condition: " + + str(self.condition) + ) + + +class Trade: + def __init__( + self, + symbol, + price, + size, + total_volume, + timestamp, + subprovider, + market_center, + condition, + ): + self.symbol = symbol + self.price = price + self.size = size + self.total_volume = total_volume + self.timestamp = timestamp + self.subprovider = subprovider + self.market_center = market_center + self.condition = condition + + def __str__(self): + return ( + self.symbol + + ", trade, price: " + + str(self.price) + + ", size: " + + str(self.size) + + ", timestamp: " + + str(self.timestamp) + + ", subprovider: " + + str(self.subprovider) + + ", market_center: " + + str(self.market_center) + + ", condition: " + + str(self.condition) + ) + + def is_darkpool(self): + return ( + not self.market_center + or self.market_center in ("D", "E", "\x00") + or self.market_center.strip() == "" + ) + + +class IntrinioRealtimeClient: + def __init__( + self, + options: Dict[str, Any], + on_trade: Optional[callable], + on_quote: Optional[callable], + ): + if options is None: + raise ValueError("Options parameter is required") + + self.options = options + self.api_key = options.get("api_key") + self.username = options.get("username") + self.password = options.get("password") + self.provider = options.get("provider") + self.ipaddress = options.get("ipaddress") + self.tradesonly = options.get("tradesonly") + self.bypass_parsing = options.get("bypass_parsing", False) + + if "channels" in options: + self.channels = set(options["channels"]) + else: + self.channels = set() + + if "logger" in options: + self.logger = options["logger"] + else: + log_formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + log_handler = logging.StreamHandler() + log_handler.setFormatter(log_formatter) + self.logger = logging.getLogger("intrinio_realtime") + if options.get("debug") is True: + self.logger.setLevel(logging.DEBUG) + else: + self.logger.setLevel(logging.INFO) + self.logger.addHandler(log_handler) + + if "max_queue_size" in options: + self.quotes = queue.Queue(maxsize=options["max_queue_size"]) + else: + self.quotes = queue.Queue(maxsize=MAX_QUEUE_SIZE) + + if self.api_key: + if not self.valid_api_key(self.api_key): + raise ValueError("API Key was formatted invalidly") + else: + if not self.username and not self.password: + raise ValueError("API key or username and password are required") + + if not self.username: + raise ValueError("Parameter 'username' must be specified") + + if not self.password: + raise ValueError("Parameter 'password' must be specified") + + if not callable(on_quote): + self.on_quote = None + raise ValueError("Parameter 'on_quote' must be a function") + else: + self.on_quote = on_quote + + if not callable(on_trade): + self.on_trade = None + raise ValueError("Parameter 'on_trade' must be a function") + else: + self.on_trade = on_trade + + if self.provider not in PROVIDERS: + raise ValueError(f"Parameter 'provider' is invalid, use one of {PROVIDERS}") + + self.ready = False + self.token = None + self.ws = None + self.quote_receiver = None + self.quote_handler = QuoteHandler(self, self.bypass_parsing) + self.joined_channels = set() + self.last_queue_warning_time = 0 + self.last_self_heal_backoff = -1 + self.quote_handler.start() + + def auth_url(self) -> str: + auth_url = "" + + if self.provider == REALTIME: + auth_url = "https://realtime-mx.intrinio.com/auth" + elif self.provider == DELAYED_SIP: + auth_url = "https://realtime-delayed-sip.intrinio.com/auth" + elif self.provider == NASDAQ_BASIC: + auth_url = "https://realtime-nasdaq-basic.intrinio.com/auth" + elif self.provider == MANUAL: + auth_url = "http://" + self.ipaddress + "/auth" + + if self.api_key: + auth_url = self.api_auth_url(auth_url) + + return auth_url + + def api_auth_url(self, auth_url: str) -> str: + auth_url = auth_url + "&" if "?" in auth_url else auth_url + "?" + + return auth_url + "api_key=" + self.api_key + + def websocket_url(self) -> str: + if self.provider == REALTIME: + return ( + "wss://realtime-mx.intrinio.com/socket/websocket?vsn=1.0.0&token=" + + self.token + ) + elif self.provider == DELAYED_SIP: + return ( + "wss://realtime-delayed-sip.intrinio.com/socket/websocket?vsn=1.0.0&token=" + + self.token + ) + elif self.provider == NASDAQ_BASIC: + return ( + "wss://realtime-nasdaq-basic.intrinio.com/socket/websocket?vsn=1.0.0&token=" + + self.token + ) + elif self.provider == MANUAL: + return ( + "ws://" + + self.ipaddress + + "/socket/websocket?vsn=1.0.0&token=" + + self.token + ) + + def do_backoff(self): + self.last_self_heal_backoff += 1 + i = min(self.last_self_heal_backoff, len(SELF_HEAL_BACKOFFS) - 1) + backoff = SELF_HEAL_BACKOFFS[i] + time.sleep(backoff) + + def connect(self): + connected = False + while not connected: + try: + self.logger.info("INFO: Connecting...") + self.ready = False + self.joined_channels = set() + + if self.ws: + self.ws.close() + time.sleep(3) + + self.refresh_token() + self.refresh_websocket() + connected = True + except Exception as e: + self.logger.error(f"Cannot connect: {repr(e)}") + self.do_backoff() + + def disconnect(self): + self.ready = False + self.joined_channels = set() + + if self.ws: + self.ws.close() + time.sleep(1) + + def refresh_token(self): + headers = {HEADER_CLIENT_INFORMATION_KEY: HEADER_CLIENT_INFORMATION_VALUE} + if self.api_key: + response = requests.get(self.auth_url(), headers=headers, timeout=5) + else: + response = requests.get( + self.auth_url(), + auth=(self.username, self.password), + headers=headers, + timeout=5, + ) + + if response.status_code != 200: + raise UnauthorizedError("Auth failed") + + self.token = response.text + self.logger.info("INFO: Authentication successful!") + + def refresh_websocket(self): + self.quote_receiver = QuoteReceiver(self) + self.quote_receiver.start() + + def on_connect(self): + self.ready = True + self.last_self_heal_backoff = -1 + self.refresh_channels() + + def on_queue_full(self): + if time.time() - self.last_queue_warning_time > 1: + self.logger.error("INFO: Quote queue is full! Dropped some new quotes") + self.last_queue_warning_time = time.time() + + def join(self, channels: list[str]): + if isinstance(channels, str): + channels = [channels] + + self.channels = self.channels | set(channels) + self.refresh_channels() + + def leave(self, channels: list[str]): + if isinstance(channels, str): + channels = [channels] + + self.channels = self.channels - set(channels) + self.refresh_channels() + + def leave_all(self): + self.channels = set() + self.refresh_channels() + + def refresh_channels(self): + if self.ready is not True: + return + + # Join new channels + new_channels = self.channels - self.joined_channels + self.logger.debug(f"New channels: {new_channels}") + for channel in new_channels: + msg = self.join_binary_message(channel) + self.ws.send(msg, websocket.ABNF.OPCODE_BINARY) + self.logger.info(f"INFO: Joined channel {channel}") + + # Leave old channels + old_channels = self.joined_channels - self.channels + self.logger.debug(f"Old channels: {old_channels}") + for channel in old_channels: + msg = self.leave_binary_message(channel) + self.ws.send(msg, websocket.ABNF.OPCODE_BINARY) + self.logger.info(f"INFO: Left channel {channel}") + + self.joined_channels = self.channels.copy() + self.logger.debug(f"Current channels: {self.joined_channels}") + + def join_binary_message(self, channel: str): + if channel == "lobby": + message = bytearray([74, 1 if self.tradesonly else 0]) + channel_bytes = bytes("$FIREHOSE", "ascii") + message.extend(channel_bytes) + return message + else: + message = bytearray([74, 1 if self.tradesonly else 0]) + channel_bytes = bytes(channel, "ascii") + message.extend(channel_bytes) + return message + + def leave_binary_message(self, channel: str): + if channel == "lobby": + message = bytearray([76]) + channel_bytes = bytes("$FIREHOSE", "ascii") + message.extend(channel_bytes) + return message + else: + message = bytearray([76]) + channel_bytes = bytes(channel, "ascii") + message.extend(channel_bytes) + return message + + def valid_api_key(self, api_key: str): + return not (not isinstance(api_key, str) or api_key == "") + + +class QuoteReceiver(threading.Thread): + def __init__(self, client): + threading.Thread.__init__(self, args=(), kwargs=None) + self.daemon = True + self.client = client + self.enabled = True + + def run(self): + self.client.ws = websocket.WebSocketApp( + self.client.websocket_url(), + header={ + HEADER_MESSAGE_FORMAT_KEY: HEADER_MESSAGE_FORMAT_VALUE, + HEADER_CLIENT_INFORMATION_KEY: HEADER_CLIENT_INFORMATION_VALUE, + }, + on_open=self.on_open, + on_close=self.on_close, + on_message=self.on_message, + on_error=self.on_error, + ) + + self.client.logger.debug("QuoteReceiver ready") + self.client.ws.run_forever( + skip_utf8_validation=True + ) # skip_utf8_validation for more performance + self.client.logger.debug("QuoteReceiver exiting") + + def on_open(self, ws): + self.client.logger.info("INFO: Websocket opened!") + self.client.on_connect() + + def on_close(self, ws, code, message): + self.client.logger.info("INFO: Websocket closed!") + + def on_error(self, ws, error, *args): + try: + msg = ( + f"Unexpected error -> {error.__class__.__name__}: {repr(error)}" + if "Unauthorized" not in str(error) + else f"UnauthorizedError -> {repr(error)}" + ) + self.client.logger.error(msg) + self.client.connect() + except Exception as e: + msg = f"Unexpected error while handling another error -> {e.__class__.__name__}: {e} -> {repr(error)}" + self.client.logger.error(msg) + raise e + + def on_message(self, ws, message): + try: + if ( + DEBUGGING + ): # This is here for performance reasons so we don't use slow reflection on every message. + if isinstance(message, str): + self.client.logger.debug( + f"Received message (hex): {message.encode('utf-8').hex()}" + ) + elif isinstance(message, bytes): + self.client.logger.debug(f"Received message (hex): {message.hex()}") + self.client.quotes.put_nowait(message) + except queue.Full: + self.client.on_queue_full() + except Exception as e: + hex_message = "" + if isinstance(message, str): + hex_message = message.encode("utf-8").hex() + elif isinstance(message, bytes): + hex_message = message.hex() + self.client.logger.error( + f"Unexpected error -> Message as hex: {hex_message}; error: {repr(e)}" + ) + raise e + + +class QuoteHandler(threading.Thread): + def __init__(self, client, bypass_parsing: bool): + threading.Thread.__init__(self, args=(), kwargs=None) + self.daemon = True + self.client = client + self.bypass_parsing = bypass_parsing + self.subprovider_codes = { + 0: NO_SUBPROVIDER, + 1: CTA_A, + 2: CTA_B, + 3: UTP, + 4: OTC, + 5: NASDAQ_BASIC, + 6: IEX, + } + + def parse_quote(self, quote_bytes: bytes, start_index: int = 0) -> Quote: + buffer = memoryview(quote_bytes) + symbol_length = buffer[start_index + 2] + symbol = ( + buffer[(start_index + 3) : (start_index + 3 + symbol_length)] + .tobytes() + .decode("ascii") + ) + quote_type = "ask" if buffer[start_index] == 1 else "bid" + price, size, timestamp = struct.unpack_from( + " 0: + condition = ( + buffer[ + (start_index + 23 + symbol_length) : ( + start_index + 23 + symbol_length + condition_length + ) + ] + .tobytes() + .decode("ascii") + ) + + subprovider = self.subprovider_codes.get( + buffer[3 + symbol_length + start_index], IEX + ) # default IEX for backward behavior consistency. + market_center = ( + buffer[ + (start_index + 4 + symbol_length) : (start_index + 6 + symbol_length) + ] + .tobytes() + .decode("utf-16") + ) + + return Quote( + symbol, + quote_type, + price, + size, + timestamp, + subprovider, + market_center, + condition, + ) + + def parse_trade(self, trade_bytes: bytes, start_index: int = 0) -> Trade: + buffer = memoryview(trade_bytes) + symbol_length = buffer[start_index + 2] + symbol = ( + buffer[(start_index + 3) : (start_index + 3 + symbol_length)] + .tobytes() + .decode("ascii") + ) + price, size, timestamp, total_volume = struct.unpack_from( + " 0: + condition = ( + buffer[ + (start_index + 27 + symbol_length) : ( + start_index + 27 + symbol_length + condition_length + ) + ] + .tobytes() + .decode("ascii") + ) + + subprovider = self.subprovider_codes.get( + buffer[3 + symbol_length + start_index], IEX + ) # default IEX for backward behavior consistency. + market_center = ( + buffer[ + (start_index + 4 + symbol_length) : (start_index + 6 + symbol_length) + ] + .tobytes() + .decode("utf-16") + ) + + return Trade( + symbol, + price, + size, + total_volume, + timestamp, + subprovider, + market_center, + condition, + ) + + def parse_message( + self, message_bytes: bytes, start_index: int, backlog_len: int + ) -> int: + message_type = message_bytes[start_index] + message_length = message_bytes[start_index + 1] + new_start_index = start_index + message_length + item = None + if message_type == 0: # this is a trade + if callable(self.client.on_trade): + try: + if self.bypass_parsing: + item = message_bytes[start_index : new_start_index - 1] + else: + item = self.parse_trade(message_bytes, start_index) + self.client.on_trade(item, backlog_len) + except Exception as e: + self.client.logger.error(repr(e)) + elif callable(self.client.on_quote): + try: + if self.bypass_parsing: + item = message_bytes[start_index : new_start_index - 1] + else: + item = self.parse_quote(message_bytes, start_index) + self.client.on_quote(item, backlog_len) + except Exception as e: + self.client.logger.error(repr(e)) + + return new_start_index + + def run(self): + self.client.logger.debug("QuoteHandler ready") + while True: + message = self.client.quotes.get() + backlog_len = self.client.quotes.qsize() + items_in_message = message[0] + start_index = 1 + for i in range(0, items_in_message): + start_index = self.parse_message(message, start_index, backlog_len) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py new file mode 100644 index 000000000000..48575f9d6e14 --- /dev/null +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -0,0 +1,142 @@ +"""Intrinio WebSocket server.""" + +import asyncio +import json +import signal +import sys + +from openbb_intrinio.models.websocket_connection import IntrinioWebSocketData +from openbb_intrinio.utils.stocks_client import IntrinioRealtimeClient, Quote, Trade +from openbb_websockets.helpers import ( + MessageQueue, + get_logger, + handle_termination_signal, + handle_validation_error, + parse_kwargs, + write_to_db, +) +from pydantic import ValidationError + +logger = get_logger("openbb.websocket.intrinio") +kwargs = parse_kwargs() +command_queue = MessageQueue() +CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) + + +async def subscribe(client, symbol, event): + """Subscribe or unsubscribe to a symbol.""" + ticker = symbol.split(",") if isinstance(symbol, str) else symbol + try: + if event == "subscribe": + client.join(ticker) + elif event == "unsubscribe": + client.leave(ticker) + except Exception as e: # pylint: disable=broad-except + msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" + logger.error(msg) + + +async def read_stdin_and_queue_commands(): + """Read from stdin and queue commands.""" + while True: + line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline) + sys.stdin.flush() + + if not line: + break + + try: + command = json.loads(line.strip()) + await command_queue.enqueue(command) + except json.JSONDecodeError: + logger.error("Invalid JSON received from stdin -> %s", line.strip()) + + +async def process_stdin_queue(client): + """Process the command queue.""" + while True: + command = await command_queue.dequeue() + symbol = ["lobby" if d == "*" else d.upper() for d in command.get("symbol", [])] + event = command.get("event") + if symbol and event: + await subscribe(client, symbol, event) + + +async def process_message(message): + """Process the message and write to the database.""" + result: dict = {} + message = json.loads(message) if isinstance(message, str) else message + is_trade = isinstance(message, Trade) + is_quote = isinstance(message, Quote) + if hasattr(message, "__dict__"): + message = message.__dict__ + if is_trade or is_quote: + message["type"] = "trade" if is_trade else "quote" + + try: + result = IntrinioWebSocketData.model_validate(message).model_dump_json( + exclude_none=True, exclude_unset=True + ) + result = message + except ValidationError as e: + try: + handle_validation_error(logger, e) + except ValidationError: + raise e from e + if result: + await write_to_db( + message, kwargs["results_file"], kwargs["table_name"], kwargs["limit"] + ) + + +def on_message(message, backlog): + """Process the message and write to the database.""" + asyncio.run(process_message(message)) + + +options = { + "api_key": kwargs.get("api_key", ""), + "provider": kwargs.get("feed", "").upper(), + "logger": logger, +} + +if kwargs.get("trades_only") is True: + options["tradesonly"] = True + +client = IntrinioRealtimeClient( + options=options, on_trade=on_message, on_quote=on_message +) + + +async def connect_and_stream(): + """Connect to the WebSocket and stream data to file.""" + symbol = kwargs.pop("symbol", "lobby") + symbol = ["lobby"] if "*" in symbol else symbol.split(",") + asyncio.create_task(read_stdin_and_queue_commands()) + client.connect() + client.join(symbol) + asyncio.create_task(process_stdin_queue(client)) + + +if __name__ == "__main__": + try: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.set_exception_handler(lambda loop, context: None) + + for sig in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(sig, handle_termination_signal, logger) + + asyncio.run_coroutine_threadsafe(connect_and_stream(), loop) + loop.run_forever() + + except KeyboardInterrupt: + logger.error("PROVIDER INFO: WebSocket connection closed") + + except Exception as e: # pylint: disable=broad-except + msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" + logger.error(msg) + + finally: + client.disconnect() + sys.exit(0) From 0dd018043502b456da8e4d671e6d62863d00086b Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:04:39 -0800 Subject: [PATCH 045/119] codespell --- openbb_platform/extensions/websockets/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/extensions/websockets/README.md b/openbb_platform/extensions/websockets/README.md index 128d9010734e..db21b5a00b78 100644 --- a/openbb_platform/extensions/websockets/README.md +++ b/openbb_platform/extensions/websockets/README.md @@ -336,7 +336,7 @@ Start the broadcast server. |Parameter|Type | Required| Description | |:-------|:-----|:--------:|------------:| |host |String |No |IP address to run the server over, default is `"127.0.0.1"` | -|port |Interger |No |Port to bind the server to, default is `6666` | +|port |Integer |No |Port to bind the server to, default is `6666` | |uvicorn_kwargs| Dictionary |No |Additional keyword arguments to pass directly to `uvicorn.run()`. | #### Example From 726b4c7a721fdc8981b0b942411488e27d188671 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 3 Dec 2024 18:22:44 -0800 Subject: [PATCH 046/119] polygon crypto vwap as optional --- .../polygon/openbb_polygon/models/websocket_connection.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 48d3f708ffdd..1cb593385ef0 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -277,12 +277,13 @@ class PolygonCryptoAggsWebSocketData(WebSocketData): description=DATA_DESCRIPTIONS.get("close", ""), json_schema_extra={"x-unit_measurement": "currency"}, ) - vwap: float = Field( - description=DATA_DESCRIPTIONS.get("vwap", ""), - ) volume: float = Field( description=DATA_DESCRIPTIONS.get("volume", ""), ) + vwap: Optional[float] = Field( + default=None, + description=DATA_DESCRIPTIONS.get("vwap", ""), + ) avg_size: Optional[float] = Field( default=None, description="The average trade size for the aggregate window.", From f57b0d5635b4d4e1b823196e790e9009fabfe3d9 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:48:13 -0800 Subject: [PATCH 047/119] potential unit tests --- .../websockets/openbb_websockets/client.py | 44 +++++++++++-------- .../test_fmp_websocket_fetcher.json | 1 + .../test_fmp_websocket_fetcher.json | 1 + .../providers/fmp/tests/test_fmp_fetchers.py | 16 +++++-- .../test_polygon_websocket_fetcher.json | 1 + .../test_polygon_websocket_fetcher.json | 1 + .../polygon/tests/test_polygon_fetchers.py | 31 +++++++++++++ .../test_tiingo_websocket_fetcher.json | 1 + .../test_tiingo_websocket_fetcher.json | 1 + .../tiingo/tests/test_tiingo_fetchers.py | 30 +++++++++++++ 10 files changed, 104 insertions(+), 23 deletions(-) create mode 100644 openbb_platform/providers/fmp/tests/record/object_hash/test_fmp_fetchers/test_fmp_websocket_fetcher.json create mode 100644 openbb_platform/providers/fmp/tests/record/screen/test_fmp_fetchers/test_fmp_websocket_fetcher.json create mode 100644 openbb_platform/providers/polygon/tests/record/object_hash/test_polygon_fetchers/test_polygon_websocket_fetcher.json create mode 100644 openbb_platform/providers/polygon/tests/record/screen/test_polygon_fetchers/test_polygon_websocket_fetcher.json create mode 100644 openbb_platform/providers/tiingo/tests/record/object_hash/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json create mode 100644 openbb_platform/providers/tiingo/tests/record/screen/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 79e2ae2cde76..51f0e39176df 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -288,8 +288,10 @@ def _log_provider_output(self, output_queue) -> None: self._thread.join() err = ChildProcessError(output) self._exception = err - sys.stdout.write(output + "\n") - sys.stdout.flush() + output = output + "\n" + self.logger.info(output) + # sys.stdout.write(output) + # sys.stdout.flush() break output = clean_message(output) @@ -299,9 +301,10 @@ def _log_provider_output(self, output_queue) -> None: elif output.startswith("INFO:"): output = output.replace("INFO:", "PROVIDER INFO:") - output = output + "\n" - sys.stdout.write(output + "\n") - sys.stdout.flush() + # output = output + "\n" + # sys.stdout.write(output + "\n") + self.logger.info(output) + # sys.stdout.flush() except queue.Empty: continue @@ -534,25 +537,28 @@ def results(self) -> Union[list[dict], list["Data"], None]: # pylint: disable=import-outside-toplevel import json # noqa import sqlite3 + from openbb_core.app.model.abstract.error import OpenBBError output: list = [] file_path = self.results_path if file_path.exists(): with sqlite3.connect(file_path) as conn: - cursor = conn.execute(f"SELECT * FROM {self.table_name}") # noqa - for row in cursor: - index, message = row - if self.data_model: - output.append( - self.data_model.model_validate_json( - json.loads(message) - if isinstance(message, str) - else message - ) - ) - else: - output.append(json.loads(json.loads(message))) - + try: + cursor = conn.execute(f"SELECT * FROM {self.table_name}") # noqa + for row in cursor: + index, message = row + if self.data_model: + message = json.loads(message) + if isinstance(message, (str, bytes)): + output.append( + self.data_model.model_validate_json(message) + ) + elif isinstance(message, dict): + output.append(self.data_model(**message)) + else: + output.append(json.loads(json.loads(message))) + except Exception as e: + raise OpenBBError(f"Error retrieving results: {e}") if output: return output diff --git a/openbb_platform/providers/fmp/tests/record/object_hash/test_fmp_fetchers/test_fmp_websocket_fetcher.json b/openbb_platform/providers/fmp/tests/record/object_hash/test_fmp_fetchers/test_fmp_websocket_fetcher.json new file mode 100644 index 000000000000..c4434157cff3 --- /dev/null +++ b/openbb_platform/providers/fmp/tests/record/object_hash/test_fmp_fetchers/test_fmp_websocket_fetcher.json @@ -0,0 +1 @@ +["b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b", "3dd2e537120ffb2eaa71d091dc92478f3ce8aa403bcd17c12fff1c5194a16118", "fcbcf165908dd18a9e49f7ff27810176db8e9f63b4352213741664245224f8aa"] \ No newline at end of file diff --git a/openbb_platform/providers/fmp/tests/record/screen/test_fmp_fetchers/test_fmp_websocket_fetcher.json b/openbb_platform/providers/fmp/tests/record/screen/test_fmp_fetchers/test_fmp_websocket_fetcher.json new file mode 100644 index 000000000000..43bd419589e7 --- /dev/null +++ b/openbb_platform/providers/fmp/tests/record/screen/test_fmp_fetchers/test_fmp_websocket_fetcher.json @@ -0,0 +1 @@ +{"out": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "err": "PROVIDER INFO: Authenticated\n\nPROVIDER INFO: Subscribed to btcusd\n\nDisconnected from the provider WebSocket.\n\n"} \ No newline at end of file diff --git a/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py b/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py index 563f1b0b4148..6ddaa374a7ae 100644 --- a/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py +++ b/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py @@ -765,17 +765,25 @@ def test_fmp_historical_market_cap_fetcher(credentials=test_credentials): assert result is None -@pytest.mark.record_screen -def test_fmp_websocket_fetcher(credentials=test_credentials): +@pytest.mark.record_verify_screen(hash=True) +@pytest.mark.record_verify_object(hash=False) +def test_fmp_websocket_fetcher(record, credentials=test_credentials): """Test FMP Websocket fetcher.""" import asyncio + import time params = {"symbol": "btcusd", "name": "test", "limit": 10, "asset_type": "crypto"} try: fetcher = FmpWebSocketFetcher() - result = fetcher.test(params, credentials) response = asyncio.run(fetcher.fetch_data(params, credentials)) - assert result is None + time.sleep(1) + record.add_verify(response.client.is_running) + assert response.client.is_running + time.sleep(1) + assert len(response.client.results) > 0 + record.add_verify(list(response.client.results[0].model_dump().keys())) finally: response.client.disconnect() + assert not response.client.is_running + record.add_verify(response.client.is_running) diff --git a/openbb_platform/providers/polygon/tests/record/object_hash/test_polygon_fetchers/test_polygon_websocket_fetcher.json b/openbb_platform/providers/polygon/tests/record/object_hash/test_polygon_fetchers/test_polygon_websocket_fetcher.json new file mode 100644 index 000000000000..bacd124ff3fb --- /dev/null +++ b/openbb_platform/providers/polygon/tests/record/object_hash/test_polygon_fetchers/test_polygon_websocket_fetcher.json @@ -0,0 +1 @@ +["b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b", "1151296bae1bacb7626f9db0e63520e1bc2a193755e1d88ff58ab1f20abcbaed", "fcbcf165908dd18a9e49f7ff27810176db8e9f63b4352213741664245224f8aa"] \ No newline at end of file diff --git a/openbb_platform/providers/polygon/tests/record/screen/test_polygon_fetchers/test_polygon_websocket_fetcher.json b/openbb_platform/providers/polygon/tests/record/screen/test_polygon_fetchers/test_polygon_websocket_fetcher.json new file mode 100644 index 000000000000..0282266fe3e3 --- /dev/null +++ b/openbb_platform/providers/polygon/tests/record/screen/test_polygon_fetchers/test_polygon_websocket_fetcher.json @@ -0,0 +1 @@ +{"out": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "err": "PROVIDER INFO: Connected Successfully\n\nPROVIDER INFO: authenticated\n\nPROVIDER INFO: subscribed to: XAS.BTC-USD\n\nDisconnected from the provider WebSocket.\n\n"} \ No newline at end of file diff --git a/openbb_platform/providers/polygon/tests/test_polygon_fetchers.py b/openbb_platform/providers/polygon/tests/test_polygon_fetchers.py index 4e8676b30792..cfae156c4e0d 100644 --- a/openbb_platform/providers/polygon/tests/test_polygon_fetchers.py +++ b/openbb_platform/providers/polygon/tests/test_polygon_fetchers.py @@ -17,6 +17,7 @@ from openbb_polygon.models.index_historical import ( PolygonIndexHistoricalFetcher, ) +from openbb_polygon.models.websocket_connection import PolygonWebSocketFetcher from openbb_polygon.models.market_snapshots import PolygonMarketSnapshotsFetcher test_credentials = UserService().default_user_settings.credentials.model_dump( @@ -170,3 +171,33 @@ def test_polygon_currency_snapshots_fetcher(credentials=test_credentials): fetcher = PolygonCurrencySnapshotsFetcher() result = fetcher.test(params, credentials) assert result is None + + +@pytest.mark.record_verify_screen(hash=True) +@pytest.mark.record_verify_object(hash=False) +def test_polygon_websocket_fetcher(record, credentials=test_credentials): + """Test Polygon Websocket fetcher.""" + import asyncio + import time + + params = { + "symbol": "btcusd", + "name": "polygon_test", + "limit": 10, + "asset_type": "crypto", + "feed": "aggs_sec", + } + + try: + fetcher = PolygonWebSocketFetcher() + response = asyncio.run(fetcher.fetch_data(params, credentials)) + time.sleep(1) + record.add_verify(response.client.is_running) + assert response.client.is_running + time.sleep(1) + assert len(response.client.results) > 0 + record.add_verify(list(response.client.results[0].model_dump().keys())) + finally: + response.client.disconnect() + assert not response.client.is_running + record.add_verify(response.client.is_running) diff --git a/openbb_platform/providers/tiingo/tests/record/object_hash/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json b/openbb_platform/providers/tiingo/tests/record/object_hash/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json new file mode 100644 index 000000000000..d46ccbdbefbb --- /dev/null +++ b/openbb_platform/providers/tiingo/tests/record/object_hash/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json @@ -0,0 +1 @@ +["b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b", "22eb8665ee9e73f51d599865e64d3d03b16cda20434665fa481080c201dddab9", "fcbcf165908dd18a9e49f7ff27810176db8e9f63b4352213741664245224f8aa"] \ No newline at end of file diff --git a/openbb_platform/providers/tiingo/tests/record/screen/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json b/openbb_platform/providers/tiingo/tests/record/screen/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json new file mode 100644 index 000000000000..6f58aa2e25a2 --- /dev/null +++ b/openbb_platform/providers/tiingo/tests/record/screen/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json @@ -0,0 +1 @@ +{"out": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "err": "PROVIDER INFO: WebSocket connection established.\n\nPROVIDER INFO: Authorization: Success\n\nDisconnected from the provider WebSocket.\n\n"} \ No newline at end of file diff --git a/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py b/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py index 26dcddaf1df6..a08bbf436400 100644 --- a/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py +++ b/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py @@ -9,6 +9,7 @@ from openbb_tiingo.models.currency_historical import TiingoCurrencyHistoricalFetcher from openbb_tiingo.models.equity_historical import TiingoEquityHistoricalFetcher from openbb_tiingo.models.trailing_dividend_yield import TiingoTrailingDivYieldFetcher +from openbb_tiingo.models.websocket_connection import TiingoWebSocketFetcher from openbb_tiingo.models.world_news import TiingoWorldNewsFetcher test_credentials = UserService().default_user_settings.credentials.model_dump( @@ -97,3 +98,32 @@ def test_tiingo_trailing_div_yield_fetcher(credentials=test_credentials): fetcher = TiingoTrailingDivYieldFetcher() result = fetcher.test(params, credentials) assert result is None + + +@pytest.mark.record_verify_screen(hash=True) +@pytest.mark.record_verify_object(hash=False) +def test_tiingo_websocket_fetcher(record, credentials=test_credentials): + """Test Tiingo Websocket fetcher.""" + import asyncio + import time + + params = { + "symbol": "btcusd", + "name": "tiingo_test", + "limit": 10, + "asset_type": "crypto", + } + + try: + fetcher = TiingoWebSocketFetcher() + response = asyncio.run(fetcher.fetch_data(params, credentials)) + time.sleep(1) + record.add_verify(response.client.is_running) + assert response.client.is_running + time.sleep(1) + assert len(response.client.results) > 0 + record.add_verify(list(response.client.results[0].model_dump().keys())) + finally: + response.client.disconnect() + assert not response.client.is_running + record.add_verify(response.client.is_running) From cb8435e2c8bfa41d028f9f67caed964e767fac00 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 7 Dec 2024 18:02:22 -0800 Subject: [PATCH 048/119] missing docstrings --- .../websockets/openbb_websockets/broadcast.py | 11 ++++++++++- .../websockets/openbb_websockets/helpers.py | 1 - 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py index 707b5400347b..3537d8b62fea 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -1,6 +1,7 @@ import asyncio import json import sys +from pathlib import Path from typing import Optional import uvicorn @@ -22,6 +23,7 @@ SLEEP_TIME = kwargs.pop("sleep_time", None) or 0.25 AUTH_TOKEN = kwargs.pop("auth_token", None) + app = FastAPI() @@ -156,7 +158,6 @@ def _decrypt_value(self, value: str) -> str: async def stream_results(self): # noqa: PLR0915 """Continuously read the database and send new messages as JSON via WebSocket.""" import sqlite3 # noqa - from pathlib import Path from openbb_core.app.model.abstract.error import OpenBBError file_path = Path(self.results_file).absolute() @@ -232,6 +233,7 @@ async def broadcast(self, message: str): connected_clients.remove(client) def start_app(self, host: str = "127.0.0.1", port: int = 6666, **kwargs): + """Start the FastAPI app with Uvicorn.""" uvicorn.run( self._app, host=host, @@ -247,10 +249,12 @@ def create_broadcast_server( auth_token: Optional[str] = None, **kwargs, ): + """Create a new BroadcastServer instance.""" return BroadcastServer(results_file, table_name, sleep_time, auth_token) def main(): + """Main entry point.""" broadcast_server = create_broadcast_server( RESULTS_FILE, TABLE_NAME, @@ -276,4 +280,9 @@ def main(): if __name__ == "__main__": + if not RESULTS_FILE: + raise ValueError("Results file path is required for Broadcast server.") + + if not Path(RESULTS_FILE).absolute().exists(): + raise FileNotFoundError(f"Results file not found: {RESULTS_FILE}") main() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 9854cdadfb4c..093834b8aa0b 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -134,7 +134,6 @@ def parse_kwargs(): import sys args = sys.argv[1:].copy() - sys.stdout.write(f"ARGS: {args}\n") _kwargs: dict = {} for i, arg in enumerate(args): if arg.startswith("url"): From bcb8dbc059098e250b098550dfe8575c69301f0a Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 7 Dec 2024 18:10:57 -0800 Subject: [PATCH 049/119] add websocket-client to intrinio pyproject.toml --- openbb_platform/poetry.lock | 26 +- .../providers/intrinio/poetry.lock | 284 +++++++----------- .../providers/intrinio/pyproject.toml | 2 +- 3 files changed, 128 insertions(+), 184 deletions(-) diff --git a/openbb_platform/poetry.lock b/openbb_platform/poetry.lock index 829325ea7eaf..abeaa1bea7a9 100644 --- a/openbb_platform/poetry.lock +++ b/openbb_platform/poetry.lock @@ -2709,6 +2709,19 @@ openbb-core = ">=1.3.6,<2.0.0" openpyxl = ">=3.1.5,<4.0.0" xlrd = ">=2.0.1,<3.0.0" +[[package]] +name = "openbb-websockets" +version = "1.0.0b0" +description = "Websockets extension for OpenBB" +optional = true +python-versions = "*" +files = [] +develop = true + +[package.source] +type = "directory" +url = "extensions/websockets" + [[package]] name = "openbb-wsj" version = "1.3.5" @@ -2991,13 +3004,13 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "poetry" -version = "1.8.4" +version = "1.8.5" description = "Python dependency management and packaging made easy." optional = false python-versions = "<4.0,>=3.8" files = [ - {file = "poetry-1.8.4-py3-none-any.whl", hash = "sha256:1223bb6dfdbdfbebc6790796b9b7a88ea1f1f4679e709594f698499010ffb129"}, - {file = "poetry-1.8.4.tar.gz", hash = "sha256:5490f8da66d17eecd660e091281f8aaa5554381644540291817c249872c99202"}, + {file = "poetry-1.8.5-py3-none-any.whl", hash = "sha256:5505fba69bf2a792b5d7402d21839c853644337392b745109b86a23010cce5f3"}, + {file = "poetry-1.8.5.tar.gz", hash = "sha256:eb2c88d224f58f36df8f7b36d6c380c07d1001bca28bde620f68fc086e881b70"}, ] [package.dependencies] @@ -3012,7 +3025,7 @@ installer = ">=0.7.0,<0.8.0" keyring = ">=24.0.0,<25.0.0" packaging = ">=23.1" pexpect = ">=4.7.0,<5.0.0" -pkginfo = ">=1.10,<2.0" +pkginfo = ">=1.12,<2.0" platformdirs = ">=3.0.0,<5" poetry-core = "1.9.1" poetry-plugin-export = ">=1.6.0,<2.0.0" @@ -4741,7 +4754,7 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", type = ["pytest-mypy"] [extras] -all = ["openbb-alpha-vantage", "openbb-biztoc", "openbb-cboe", "openbb-charting", "openbb-ecb", "openbb-econometrics", "openbb-finra", "openbb-finviz", "openbb-government-us", "openbb-multpl", "openbb-nasdaq", "openbb-quantitative", "openbb-seeking-alpha", "openbb-stockgrid", "openbb-technical", "openbb-tmx", "openbb-tradier", "openbb-wsj"] +all = ["openbb-alpha-vantage", "openbb-biztoc", "openbb-cboe", "openbb-charting", "openbb-ecb", "openbb-econometrics", "openbb-finra", "openbb-finviz", "openbb-government-us", "openbb-multpl", "openbb-nasdaq", "openbb-quantitative", "openbb-seeking-alpha", "openbb-stockgrid", "openbb-technical", "openbb-tmx", "openbb-tradier", "openbb-websockets", "openbb-wsj"] alpha-vantage = ["openbb-alpha-vantage"] biztoc = ["openbb-biztoc"] cboe = ["openbb-cboe"] @@ -4759,9 +4772,10 @@ stockgrid = ["openbb-stockgrid"] technical = ["openbb-technical"] tmx = ["openbb-tmx"] tradier = ["openbb-tradier"] +websockets = ["openbb-websockets"] wsj = ["openbb-wsj"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "a2901737d966bcf33f21094863132898f6ab2b3469e371bdd753f5651ff5a3cb" +content-hash = "f413db7dd49a5054b633e84046b463da37296c1e7f6ff09eeddc4f4271914411" diff --git a/openbb_platform/providers/intrinio/poetry.lock b/openbb_platform/providers/intrinio/poetry.lock index 342b4e8e077f..d4ed52b73def 100644 --- a/openbb_platform/providers/intrinio/poetry.lock +++ b/openbb_platform/providers/intrinio/poetry.lock @@ -13,87 +13,87 @@ files = [ [[package]] name = "aiohttp" -version = "3.11.9" +version = "3.11.10" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.9" files = [ - {file = "aiohttp-3.11.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0411777249f25d11bd2964a230b3ffafcbed6cd65d0f2b132bc2b8f5b8c347c7"}, - {file = "aiohttp-3.11.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:499368eb904566fbdf1a3836a1532000ef1308f34a1bcbf36e6351904cced771"}, - {file = "aiohttp-3.11.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0b5a5009b0159a8f707879dc102b139466d8ec6db05103ec1520394fdd8ea02c"}, - {file = "aiohttp-3.11.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:176f8bb8931da0613bb0ed16326d01330066bb1e172dd97e1e02b1c27383277b"}, - {file = "aiohttp-3.11.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6435a66957cdba1a0b16f368bde03ce9c79c57306b39510da6ae5312a1a5b2c1"}, - {file = "aiohttp-3.11.9-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:202f40fb686e5f93908eee0c75d1e6fbe50a43e9bd4909bf3bf4a56b560ca180"}, - {file = "aiohttp-3.11.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39625703540feb50b6b7f938b3856d1f4886d2e585d88274e62b1bd273fae09b"}, - {file = "aiohttp-3.11.9-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c6beeac698671baa558e82fa160be9761cf0eb25861943f4689ecf9000f8ebd0"}, - {file = "aiohttp-3.11.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:96726839a42429318017e67a42cca75d4f0d5248a809b3cc2e125445edd7d50d"}, - {file = "aiohttp-3.11.9-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3f5461c77649358610fb9694e790956b4238ac5d9e697a17f63619c096469afe"}, - {file = "aiohttp-3.11.9-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4313f3bc901255b22f01663eeeae167468264fdae0d32c25fc631d5d6e15b502"}, - {file = "aiohttp-3.11.9-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:d6e274661c74195708fc4380a4ef64298926c5a50bb10fbae3d01627d7a075b7"}, - {file = "aiohttp-3.11.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:db2914de2559809fdbcf3e48f41b17a493b58cb7988d3e211f6b63126c55fe82"}, - {file = "aiohttp-3.11.9-cp310-cp310-win32.whl", hash = "sha256:27935716f8d62c1c73010428db310fd10136002cfc6d52b0ba7bdfa752d26066"}, - {file = "aiohttp-3.11.9-cp310-cp310-win_amd64.whl", hash = "sha256:afbe85b50ade42ddff5669947afde9e8a610e64d2c80be046d67ec4368e555fa"}, - {file = "aiohttp-3.11.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:afcda759a69c6a8be3aae764ec6733155aa4a5ad9aad4f398b52ba4037942fe3"}, - {file = "aiohttp-3.11.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5bba6b83fde4ca233cfda04cbd4685ab88696b0c8eaf76f7148969eab5e248a"}, - {file = "aiohttp-3.11.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:442356e8924fe1a121f8c87866b0ecdc785757fd28924b17c20493961b3d6697"}, - {file = "aiohttp-3.11.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f737fef6e117856400afee4f17774cdea392b28ecf058833f5eca368a18cf1bf"}, - {file = "aiohttp-3.11.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea142255d4901b03f89cb6a94411ecec117786a76fc9ab043af8f51dd50b5313"}, - {file = "aiohttp-3.11.9-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6e1e9e447856e9b7b3d38e1316ae9a8c92e7536ef48373de758ea055edfd5db5"}, - {file = "aiohttp-3.11.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7f6173302f8a329ca5d1ee592af9e628d3ade87816e9958dcf7cdae2841def7"}, - {file = "aiohttp-3.11.9-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7c6147c6306f537cff59409609508a1d2eff81199f0302dd456bb9e7ea50c39"}, - {file = "aiohttp-3.11.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e9d036a9a41fc78e8a3f10a86c2fc1098fca8fab8715ba9eb999ce4788d35df0"}, - {file = "aiohttp-3.11.9-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:2ac9fd83096df36728da8e2f4488ac3b5602238f602706606f3702f07a13a409"}, - {file = "aiohttp-3.11.9-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:d3108f0ad5c6b6d78eec5273219a5bbd884b4aacec17883ceefaac988850ce6e"}, - {file = "aiohttp-3.11.9-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:96bbec47beb131bbf4bae05d8ef99ad9e5738f12717cfbbf16648b78b0232e87"}, - {file = "aiohttp-3.11.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:fc726c3fa8f606d07bd2b500e5dc4c0fd664c59be7788a16b9e34352c50b6b6b"}, - {file = "aiohttp-3.11.9-cp311-cp311-win32.whl", hash = "sha256:5720ebbc7a1b46c33a42d489d25d36c64c419f52159485e55589fbec648ea49a"}, - {file = "aiohttp-3.11.9-cp311-cp311-win_amd64.whl", hash = "sha256:17af09d963fa1acd7e4c280e9354aeafd9e3d47eaa4a6bfbd2171ad7da49f0c5"}, - {file = "aiohttp-3.11.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:c1f2d7fd583fc79c240094b3e7237d88493814d4b300d013a42726c35a734bc9"}, - {file = "aiohttp-3.11.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4b8a1b6c7a68c73191f2ebd3bf66f7ce02f9c374e309bdb68ba886bbbf1b938"}, - {file = "aiohttp-3.11.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bd3f711f4c99da0091ced41dccdc1bcf8be0281dc314d6d9c6b6cf5df66f37a9"}, - {file = "aiohttp-3.11.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44cb1a1326a0264480a789e6100dc3e07122eb8cd1ad6b784a3d47d13ed1d89c"}, - {file = "aiohttp-3.11.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7a7ddf981a0b953ade1c2379052d47ccda2f58ab678fca0671c7c7ca2f67aac2"}, - {file = "aiohttp-3.11.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6ffa45cc55b18d4ac1396d1ddb029f139b1d3480f1594130e62bceadf2e1a838"}, - {file = "aiohttp-3.11.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cca505829cdab58c2495ff418c96092d225a1bbd486f79017f6de915580d3c44"}, - {file = "aiohttp-3.11.9-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:44d323aa80a867cb6db6bebb4bbec677c6478e38128847f2c6b0f70eae984d72"}, - {file = "aiohttp-3.11.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b2fab23003c4bb2249729a7290a76c1dda38c438300fdf97d4e42bf78b19c810"}, - {file = "aiohttp-3.11.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:be0c7c98e38a1e3ad7a6ff64af8b6d6db34bf5a41b1478e24c3c74d9e7f8ed42"}, - {file = "aiohttp-3.11.9-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5cc5e0d069c56645446c45a4b5010d4b33ac6c5ebfd369a791b5f097e46a3c08"}, - {file = "aiohttp-3.11.9-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9bcf97b971289be69638d8b1b616f7e557e1342debc7fc86cf89d3f08960e411"}, - {file = "aiohttp-3.11.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c7333e7239415076d1418dbfb7fa4df48f3a5b00f8fdf854fca549080455bc14"}, - {file = "aiohttp-3.11.9-cp312-cp312-win32.whl", hash = "sha256:9384b07cfd3045b37b05ed002d1c255db02fb96506ad65f0f9b776b762a7572e"}, - {file = "aiohttp-3.11.9-cp312-cp312-win_amd64.whl", hash = "sha256:f5252ba8b43906f206048fa569debf2cd0da0316e8d5b4d25abe53307f573941"}, - {file = "aiohttp-3.11.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:282e0a7ddd36ebc411f156aeaa0491e8fe7f030e2a95da532cf0c84b0b70bc66"}, - {file = "aiohttp-3.11.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ebd3e6b0c7d4954cca59d241970011f8d3327633d555051c430bd09ff49dc494"}, - {file = "aiohttp-3.11.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30f9f89ae625d412043f12ca3771b2ccec227cc93b93bb1f994db6e1af40a7d3"}, - {file = "aiohttp-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a3b5b2c012d70c63d9d13c57ed1603709a4d9d7d473e4a9dfece0e4ea3d5f51"}, - {file = "aiohttp-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6ef1550bb5f55f71b97a6a395286db07f7f2c01c8890e613556df9a51da91e8d"}, - {file = "aiohttp-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:317251b9c9a2f1a9ff9cd093775b34c6861d1d7df9439ce3d32a88c275c995cd"}, - {file = "aiohttp-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cbe97839b009826a61b143d3ca4964c8590d7aed33d6118125e5b71691ca46"}, - {file = "aiohttp-3.11.9-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:618b18c3a2360ac940a5503da14fa4f880c5b9bc315ec20a830357bcc62e6bae"}, - {file = "aiohttp-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a0cf4d814689e58f57ecd5d8c523e6538417ca2e72ff52c007c64065cef50fb2"}, - {file = "aiohttp-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:15c4e489942d987d5dac0ba39e5772dcbed4cc9ae3710d1025d5ba95e4a5349c"}, - {file = "aiohttp-3.11.9-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:ec8df0ff5a911c6d21957a9182402aad7bf060eaeffd77c9ea1c16aecab5adbf"}, - {file = "aiohttp-3.11.9-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ed95d66745f53e129e935ad726167d3a6cb18c5d33df3165974d54742c373868"}, - {file = "aiohttp-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:647ec5bee7e4ec9f1034ab48173b5fa970d9a991e565549b965e93331f1328fe"}, - {file = "aiohttp-3.11.9-cp313-cp313-win32.whl", hash = "sha256:ef2c9499b7bd1e24e473dc1a85de55d72fd084eea3d8bdeec7ee0720decb54fa"}, - {file = "aiohttp-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:84de955314aa5e8d469b00b14d6d714b008087a0222b0f743e7ffac34ef56aff"}, - {file = "aiohttp-3.11.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e738aabff3586091221044b7a584865ddc4d6120346d12e28e788307cd731043"}, - {file = "aiohttp-3.11.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:28f29bce89c3b401a53d6fd4bee401ee943083bf2bdc12ef297c1d63155070b0"}, - {file = "aiohttp-3.11.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:31de2f10f63f96cc19e04bd2df9549559beadd0b2ee2da24a17e7ed877ca8c60"}, - {file = "aiohttp-3.11.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f31cebd8c27a36af6c7346055ac564946e562080ee1a838da724585c67474f"}, - {file = "aiohttp-3.11.9-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0bcb7f6976dc0b6b56efde13294862adf68dd48854111b422a336fa729a82ea6"}, - {file = "aiohttp-3.11.9-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a8b13b9950d8b2f8f58b6e5842c4b842b5887e2c32e3f4644d6642f1659a530"}, - {file = "aiohttp-3.11.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9c23e62f3545c2216100603614f9e019e41b9403c47dd85b8e7e5015bf1bde0"}, - {file = "aiohttp-3.11.9-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec656680fc53a13f849c71afd0c84a55c536206d524cbc831cde80abbe80489e"}, - {file = "aiohttp-3.11.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:36df00e0541f264ce42d62280281541a47474dfda500bc5b7f24f70a7f87be7a"}, - {file = "aiohttp-3.11.9-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8dcfd14c712aa9dd18049280bfb2f95700ff6a8bde645e09f17c3ed3f05a0130"}, - {file = "aiohttp-3.11.9-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:14624d96f0d69cf451deed3173079a68c322279be6030208b045ab77e1e8d550"}, - {file = "aiohttp-3.11.9-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4b01d9cfcb616eeb6d40f02e66bebfe7b06d9f2ef81641fdd50b8dd981166e0b"}, - {file = "aiohttp-3.11.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:928f92f80e2e8d6567b87d3316c1fd9860ccfe36e87a9a7f5237d4cda8baa1ba"}, - {file = "aiohttp-3.11.9-cp39-cp39-win32.whl", hash = "sha256:c8a02f74ae419e3955af60f570d83187423e42e672a6433c5e292f1d23619269"}, - {file = "aiohttp-3.11.9-cp39-cp39-win_amd64.whl", hash = "sha256:0a97d657f6cf8782a830bb476c13f7d777cfcab8428ac49dde15c22babceb361"}, - {file = "aiohttp-3.11.9.tar.gz", hash = "sha256:a9266644064779840feec0e34f10a89b3ff1d2d6b751fe90017abcad1864fa7c"}, + {file = "aiohttp-3.11.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cbad88a61fa743c5d283ad501b01c153820734118b65aee2bd7dbb735475ce0d"}, + {file = "aiohttp-3.11.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:80886dac673ceaef499de2f393fc80bb4481a129e6cb29e624a12e3296cc088f"}, + {file = "aiohttp-3.11.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:61b9bae80ed1f338c42f57c16918853dc51775fb5cb61da70d590de14d8b5fb4"}, + {file = "aiohttp-3.11.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e2e576caec5c6a6b93f41626c9c02fc87cd91538b81a3670b2e04452a63def6"}, + {file = "aiohttp-3.11.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02c13415b5732fb6ee7ff64583a5e6ed1c57aa68f17d2bda79c04888dfdc2769"}, + {file = "aiohttp-3.11.10-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4cfce37f31f20800a6a6620ce2cdd6737b82e42e06e6e9bd1b36f546feb3c44f"}, + {file = "aiohttp-3.11.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bbbfff4c679c64e6e23cb213f57cc2c9165c9a65d63717108a644eb5a7398df"}, + {file = "aiohttp-3.11.10-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49c7dbbc1a559ae14fc48387a115b7d4bbc84b4a2c3b9299c31696953c2a5219"}, + {file = "aiohttp-3.11.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:68386d78743e6570f054fe7949d6cb37ef2b672b4d3405ce91fafa996f7d9b4d"}, + {file = "aiohttp-3.11.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9ef405356ba989fb57f84cac66f7b0260772836191ccefbb987f414bcd2979d9"}, + {file = "aiohttp-3.11.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:5d6958671b296febe7f5f859bea581a21c1d05430d1bbdcf2b393599b1cdce77"}, + {file = "aiohttp-3.11.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:99b7920e7165be5a9e9a3a7f1b680f06f68ff0d0328ff4079e5163990d046767"}, + {file = "aiohttp-3.11.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0dc49f42422163efb7e6f1df2636fe3db72713f6cd94688e339dbe33fe06d61d"}, + {file = "aiohttp-3.11.10-cp310-cp310-win32.whl", hash = "sha256:40d1c7a7f750b5648642586ba7206999650208dbe5afbcc5284bcec6579c9b91"}, + {file = "aiohttp-3.11.10-cp310-cp310-win_amd64.whl", hash = "sha256:68ff6f48b51bd78ea92b31079817aff539f6c8fc80b6b8d6ca347d7c02384e33"}, + {file = "aiohttp-3.11.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:77c4aa15a89847b9891abf97f3d4048f3c2d667e00f8a623c89ad2dccee6771b"}, + {file = "aiohttp-3.11.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:909af95a72cedbefe5596f0bdf3055740f96c1a4baa0dd11fd74ca4de0b4e3f1"}, + {file = "aiohttp-3.11.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:386fbe79863eb564e9f3615b959e28b222259da0c48fd1be5929ac838bc65683"}, + {file = "aiohttp-3.11.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3de34936eb1a647aa919655ff8d38b618e9f6b7f250cc19a57a4bf7fd2062b6d"}, + {file = "aiohttp-3.11.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c9527819b29cd2b9f52033e7fb9ff08073df49b4799c89cb5754624ecd98299"}, + {file = "aiohttp-3.11.10-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65a96e3e03300b41f261bbfd40dfdbf1c301e87eab7cd61c054b1f2e7c89b9e8"}, + {file = "aiohttp-3.11.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98f5635f7b74bcd4f6f72fcd85bea2154b323a9f05226a80bc7398d0c90763b0"}, + {file = "aiohttp-3.11.10-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:03b6002e20938fc6ee0918c81d9e776bebccc84690e2b03ed132331cca065ee5"}, + {file = "aiohttp-3.11.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6362cc6c23c08d18ddbf0e8c4d5159b5df74fea1a5278ff4f2c79aed3f4e9f46"}, + {file = "aiohttp-3.11.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:3691ed7726fef54e928fe26344d930c0c8575bc968c3e239c2e1a04bd8cf7838"}, + {file = "aiohttp-3.11.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:31d5093d3acd02b31c649d3a69bb072d539d4c7659b87caa4f6d2bcf57c2fa2b"}, + {file = "aiohttp-3.11.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:8b3cf2dc0f0690a33f2d2b2cb15db87a65f1c609f53c37e226f84edb08d10f52"}, + {file = "aiohttp-3.11.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:fbbaea811a2bba171197b08eea288b9402faa2bab2ba0858eecdd0a4105753a3"}, + {file = "aiohttp-3.11.10-cp311-cp311-win32.whl", hash = "sha256:4b2c7ac59c5698a7a8207ba72d9e9c15b0fc484a560be0788b31312c2c5504e4"}, + {file = "aiohttp-3.11.10-cp311-cp311-win_amd64.whl", hash = "sha256:974d3a2cce5fcfa32f06b13ccc8f20c6ad9c51802bb7f829eae8a1845c4019ec"}, + {file = "aiohttp-3.11.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b78f053a7ecfc35f0451d961dacdc671f4bcbc2f58241a7c820e9d82559844cf"}, + {file = "aiohttp-3.11.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ab7485222db0959a87fbe8125e233b5a6f01f4400785b36e8a7878170d8c3138"}, + {file = "aiohttp-3.11.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cf14627232dfa8730453752e9cdc210966490992234d77ff90bc8dc0dce361d5"}, + {file = "aiohttp-3.11.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:076bc454a7e6fd646bc82ea7f98296be0b1219b5e3ef8a488afbdd8e81fbac50"}, + {file = "aiohttp-3.11.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:482cafb7dc886bebeb6c9ba7925e03591a62ab34298ee70d3dd47ba966370d2c"}, + {file = "aiohttp-3.11.10-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bf3d1a519a324af764a46da4115bdbd566b3c73fb793ffb97f9111dbc684fc4d"}, + {file = "aiohttp-3.11.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24213ba85a419103e641e55c27dc7ff03536c4873470c2478cce3311ba1eee7b"}, + {file = "aiohttp-3.11.10-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b99acd4730ad1b196bfb03ee0803e4adac371ae8efa7e1cbc820200fc5ded109"}, + {file = "aiohttp-3.11.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:14cdb5a9570be5a04eec2ace174a48ae85833c2aadc86de68f55541f66ce42ab"}, + {file = "aiohttp-3.11.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:7e97d622cb083e86f18317282084bc9fbf261801b0192c34fe4b1febd9f7ae69"}, + {file = "aiohttp-3.11.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:012f176945af138abc10c4a48743327a92b4ca9adc7a0e078077cdb5dbab7be0"}, + {file = "aiohttp-3.11.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44224d815853962f48fe124748227773acd9686eba6dc102578defd6fc99e8d9"}, + {file = "aiohttp-3.11.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c87bf31b7fdab94ae3adbe4a48e711bfc5f89d21cf4c197e75561def39e223bc"}, + {file = "aiohttp-3.11.10-cp312-cp312-win32.whl", hash = "sha256:06a8e2ee1cbac16fe61e51e0b0c269400e781b13bcfc33f5425912391a542985"}, + {file = "aiohttp-3.11.10-cp312-cp312-win_amd64.whl", hash = "sha256:be2b516f56ea883a3e14dda17059716593526e10fb6303189aaf5503937db408"}, + {file = "aiohttp-3.11.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8cc5203b817b748adccb07f36390feb730b1bc5f56683445bfe924fc270b8816"}, + {file = "aiohttp-3.11.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ef359ebc6949e3a34c65ce20230fae70920714367c63afd80ea0c2702902ccf"}, + {file = "aiohttp-3.11.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9bca390cb247dbfaec3c664326e034ef23882c3f3bfa5fbf0b56cad0320aaca5"}, + {file = "aiohttp-3.11.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:811f23b3351ca532af598405db1093f018edf81368e689d1b508c57dcc6b6a32"}, + {file = "aiohttp-3.11.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddf5f7d877615f6a1e75971bfa5ac88609af3b74796ff3e06879e8422729fd01"}, + {file = "aiohttp-3.11.10-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6ab29b8a0beb6f8eaf1e5049252cfe74adbaafd39ba91e10f18caeb0e99ffb34"}, + {file = "aiohttp-3.11.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c49a76c1038c2dd116fa443eba26bbb8e6c37e924e2513574856de3b6516be99"}, + {file = "aiohttp-3.11.10-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f3dc0e330575f5b134918976a645e79adf333c0a1439dcf6899a80776c9ab39"}, + {file = "aiohttp-3.11.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:efb15a17a12497685304b2d976cb4939e55137df7b09fa53f1b6a023f01fcb4e"}, + {file = "aiohttp-3.11.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:db1d0b28fcb7f1d35600150c3e4b490775251dea70f894bf15c678fdd84eda6a"}, + {file = "aiohttp-3.11.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:15fccaf62a4889527539ecb86834084ecf6e9ea70588efde86e8bc775e0e7542"}, + {file = "aiohttp-3.11.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:593c114a2221444f30749cc5e5f4012488f56bd14de2af44fe23e1e9894a9c60"}, + {file = "aiohttp-3.11.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7852bbcb4d0d2f0c4d583f40c3bc750ee033265d80598d0f9cb6f372baa6b836"}, + {file = "aiohttp-3.11.10-cp313-cp313-win32.whl", hash = "sha256:65e55ca7debae8faaffee0ebb4b47a51b4075f01e9b641c31e554fd376595c6c"}, + {file = "aiohttp-3.11.10-cp313-cp313-win_amd64.whl", hash = "sha256:beb39a6d60a709ae3fb3516a1581777e7e8b76933bb88c8f4420d875bb0267c6"}, + {file = "aiohttp-3.11.10-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0580f2e12de2138f34debcd5d88894786453a76e98febaf3e8fe5db62d01c9bf"}, + {file = "aiohttp-3.11.10-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a55d2ad345684e7c3dd2c20d2f9572e9e1d5446d57200ff630e6ede7612e307f"}, + {file = "aiohttp-3.11.10-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:04814571cb72d65a6899db6099e377ed00710bf2e3eafd2985166f2918beaf59"}, + {file = "aiohttp-3.11.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e44a9a3c053b90c6f09b1bb4edd880959f5328cf63052503f892c41ea786d99f"}, + {file = "aiohttp-3.11.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:502a1464ccbc800b4b1995b302efaf426e8763fadf185e933c2931df7db9a199"}, + {file = "aiohttp-3.11.10-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:613e5169f8ae77b1933e42e418a95931fb4867b2991fc311430b15901ed67079"}, + {file = "aiohttp-3.11.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cca22a61b7fe45da8fc73c3443150c3608750bbe27641fc7558ec5117b27fdf"}, + {file = "aiohttp-3.11.10-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:86a5dfcc39309470bd7b68c591d84056d195428d5d2e0b5ccadfbaf25b026ebc"}, + {file = "aiohttp-3.11.10-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:77ae58586930ee6b2b6f696c82cf8e78c8016ec4795c53e36718365f6959dc82"}, + {file = "aiohttp-3.11.10-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:78153314f26d5abef3239b4a9af20c229c6f3ecb97d4c1c01b22c4f87669820c"}, + {file = "aiohttp-3.11.10-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:98283b94cc0e11c73acaf1c9698dea80c830ca476492c0fe2622bd931f34b487"}, + {file = "aiohttp-3.11.10-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:53bf2097e05c2accc166c142a2090e4c6fd86581bde3fd9b2d3f9e93dda66ac1"}, + {file = "aiohttp-3.11.10-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c5532f0441fc09c119e1dca18fbc0687e64fbeb45aa4d6a87211ceaee50a74c4"}, + {file = "aiohttp-3.11.10-cp39-cp39-win32.whl", hash = "sha256:47ad15a65fb41c570cd0ad9a9ff8012489e68176e7207ec7b82a0940dddfd8be"}, + {file = "aiohttp-3.11.10-cp39-cp39-win_amd64.whl", hash = "sha256:c6b9e6d7e41656d78e37ce754813fa44b455c3d0d0dced2a047def7dc5570b74"}, + {file = "aiohttp-3.11.10.tar.gz", hash = "sha256:b1fc6b45010a8d0ff9e88f9f2418c6fd408c99c211257334aff41597ebece42e"}, ] [package.dependencies] @@ -136,24 +136,24 @@ files = [ [[package]] name = "anyio" -version = "4.6.2.post1" +version = "4.7.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false python-versions = ">=3.9" files = [ - {file = "anyio-4.6.2.post1-py3-none-any.whl", hash = "sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d"}, - {file = "anyio-4.6.2.post1.tar.gz", hash = "sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c"}, + {file = "anyio-4.7.0-py3-none-any.whl", hash = "sha256:ea60c3723ab42ba6fff7e8ccb0488c898ec538ff4df1f1d5e642c3601d07e352"}, + {file = "anyio-4.7.0.tar.gz", hash = "sha256:2f834749c602966b7d456a7567cafcb309f96482b5081d14ac93ccd457f9dd48"}, ] [package.dependencies] exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} idna = ">=2.8" sniffio = ">=1.1" -typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} +typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} [package.extras] -doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] -test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21.0b1)"] +doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"] trio = ["trio (>=0.26.1)"] [[package]] @@ -197,32 +197,6 @@ files = [ {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, ] -[[package]] -name = "cattrs" -version = "24.1.2" -description = "Composable complex class support for attrs and dataclasses." -optional = false -python-versions = ">=3.8" -files = [ - {file = "cattrs-24.1.2-py3-none-any.whl", hash = "sha256:67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0"}, - {file = "cattrs-24.1.2.tar.gz", hash = "sha256:8028cfe1ff5382df59dd36474a86e02d817b06eaf8af84555441bac915d2ef85"}, -] - -[package.dependencies] -attrs = ">=23.1.0" -exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} - -[package.extras] -bson = ["pymongo (>=4.4.0)"] -cbor2 = ["cbor2 (>=5.4.6)"] -msgpack = ["msgpack (>=1.0.5)"] -msgspec = ["msgspec (>=0.18.5)"] -orjson = ["orjson (>=3.9.2)"] -pyyaml = ["pyyaml (>=6.0)"] -tomlkit = ["tomlkit (>=0.11.8)"] -ujson = ["ujson (>=5.7.0)"] - [[package]] name = "certifi" version = "2024.8.30" @@ -389,13 +363,13 @@ test = ["pytest (>=6)"] [[package]] name = "fastapi" -version = "0.115.5" +version = "0.115.6" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = false python-versions = ">=3.8" files = [ - {file = "fastapi-0.115.5-py3-none-any.whl", hash = "sha256:596b95adbe1474da47049e802f9a65ab2ffa9c2b07e7efee70eb8a66c9f2f796"}, - {file = "fastapi-0.115.5.tar.gz", hash = "sha256:0e7a4d0dc0d01c68df21887cce0945e72d3c48b9f4f79dfe7a7d53aa08fbb289"}, + {file = "fastapi-0.115.6-py3-none-any.whl", hash = "sha256:e9240b29e36fa8f4bb7290316988e90c381e5092e0cbe84e7818cc3713bcf305"}, + {file = "fastapi-0.115.6.tar.gz", hash = "sha256:9ec46f7addc14ea472958a96aae5b5de65f39721a46aaf5705c480d9a8b76654"}, ] [package.dependencies] @@ -924,22 +898,6 @@ sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-d test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] xml = ["lxml (>=4.9.2)"] -[[package]] -name = "platformdirs" -version = "4.3.6" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." -optional = false -python-versions = ">=3.8" -files = [ - {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, - {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, -] - -[package.extras] -docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"] -type = ["mypy (>=1.11.2)"] - [[package]] name = "posthog" version = "3.7.4" @@ -1274,36 +1232,6 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] -[[package]] -name = "requests-cache" -version = "1.2.1" -description = "A persistent cache for python requests" -optional = false -python-versions = ">=3.8" -files = [ - {file = "requests_cache-1.2.1-py3-none-any.whl", hash = "sha256:1285151cddf5331067baa82598afe2d47c7495a1334bfe7a7d329b43e9fd3603"}, - {file = "requests_cache-1.2.1.tar.gz", hash = "sha256:68abc986fdc5b8d0911318fbb5f7c80eebcd4d01bfacc6685ecf8876052511d1"}, -] - -[package.dependencies] -attrs = ">=21.2" -cattrs = ">=22.2" -platformdirs = ">=2.5" -requests = ">=2.22" -url-normalize = ">=1.4" -urllib3 = ">=1.25.5" - -[package.extras] -all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] -bson = ["bson (>=0.5)"] -docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] -dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] -json = ["ujson (>=5.4)"] -mongodb = ["pymongo (>=3)"] -redis = ["redis (>=3)"] -security = ["itsdangerous (>=2.0)"] -yaml = ["pyyaml (>=6.0.1)"] - [[package]] name = "ruff" version = "0.7.4" @@ -1333,13 +1261,13 @@ files = [ [[package]] name = "six" -version = "1.16.0" +version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] [[package]] @@ -1393,20 +1321,6 @@ files = [ {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, ] -[[package]] -name = "url-normalize" -version = "1.4.3" -description = "URL normalization for Python" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" -files = [ - {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, - {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, -] - -[package.dependencies] -six = "*" - [[package]] name = "urllib3" version = "2.2.3" @@ -1465,6 +1379,22 @@ files = [ {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, ] +[[package]] +name = "websocket-client" +version = "1.8.0" +description = "WebSocket client for Python with low level API options" +optional = false +python-versions = ">=3.8" +files = [ + {file = "websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526"}, + {file = "websocket_client-1.8.0.tar.gz", hash = "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da"}, +] + +[package.extras] +docs = ["Sphinx (>=6.0)", "myst-parser (>=2.0.0)", "sphinx-rtd-theme (>=1.1.0)"] +optional = ["python-socks", "wsaccel"] +test = ["websockets"] + [[package]] name = "websockets" version = "13.1" @@ -1678,4 +1608,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "1b73ac44ce3dda7c8ee38d5ad98a17e36a79f865193ba478d2e14f7cf362087e" +content-hash = "ef068ab16d6fce8a0531e76d39f681f1d28aca5ac315228460e09d7be9d07e93" diff --git a/openbb_platform/providers/intrinio/pyproject.toml b/openbb_platform/providers/intrinio/pyproject.toml index 1c1959db22cd..cb361c1db3b3 100644 --- a/openbb_platform/providers/intrinio/pyproject.toml +++ b/openbb_platform/providers/intrinio/pyproject.toml @@ -9,7 +9,7 @@ packages = [{ include = "openbb_intrinio" }] [tool.poetry.dependencies] python = "^3.9" -requests-cache = "^1.1.0" +websocket-client = "^1.8.0" openbb-core = "^1.3.6" [build-system] From ff4a6c4472cd437a438a965cb69c80550842f76e Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 7 Dec 2024 18:50:49 -0800 Subject: [PATCH 050/119] some linting --- .../websockets/openbb_websockets/broadcast.py | 2 +- .../websockets/openbb_websockets/client.py | 26 +++++++++---------- .../websockets/openbb_websockets/helpers.py | 8 +++--- .../websockets/openbb_websockets/listen.py | 12 ++++----- .../openbb_intrinio/utils/stocks_client.py | 2 +- .../models/websocket_connection.py | 9 ++++--- 6 files changed, 30 insertions(+), 29 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py index 3537d8b62fea..6f3d37b47aa7 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -10,7 +10,7 @@ from openbb_websockets.helpers import get_logger, parse_kwargs -connected_clients = set() +connected_clients: set = set() kwargs = parse_kwargs() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 51f0e39176df..cd6bee9c82a3 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -149,7 +149,7 @@ def __init__( # noqa: PLR0913 self.results_path = Path(temp_file_path).absolute() self.results_file = temp_file_path - self.results_path = Path(self.results_file).absolute() + self.results_path = Path(self.results_file).absolute() # type: ignore self.save_results = save_results self.logger = logger if logger else get_logger("openbb.websocket.client") @@ -189,8 +189,8 @@ def _atexit(self) -> None: self.stop_broadcasting() if self.save_results: self.logger.info("Websocket results saved to, %s\n", str(self.results_path)) - if os.path.exists(self.results_file) and not self.save_results: - os.remove(self.results_file) + if os.path.exists(self.results_file) and not self.save_results: # type: ignore + os.remove(self.results_file) # type: ignore def _setup_database(self) -> None: """Set up the SQLite database and table.""" @@ -372,7 +372,7 @@ def connect(self) -> None: self.logger.info("No subscribed symbols.") return - command = self.module.copy() + command = self.module command.extend([f"symbol={symbol}"]) command.extend([f"results_file={self.results_file}"]) command.extend([f"table_name={self.table_name}"]), @@ -395,10 +395,10 @@ def connect(self) -> None: if kwargs else None ) - - for kwarg in _kwargs: - if kwarg not in command: - command.extend([kwarg]) + if _kwargs is not None: + for kwarg in _kwargs: + if kwarg not in command: + command.extend([kwarg]) self._process = subprocess.Popen( # noqa command, @@ -411,7 +411,7 @@ def connect(self) -> None: ) self._psutil_process = psutil.Process(self._process.pid) - log_output_queue = queue.Queue() + log_output_queue: queue.Queue = queue.Queue() self._thread = threading.Thread( target=non_blocking_websocket, args=( @@ -564,7 +564,7 @@ def results(self) -> Union[list[dict], list["Data"], None]: self.logger.info("No results found in %s", self.results_file) - return + return None @results.deleter def results(self): @@ -588,7 +588,7 @@ def results(self): self.logger.error("Error clearing results: %s", e) @property - def module(self) -> str: + def module(self) -> list: """Path to the provider connection script.""" return self._module @@ -605,7 +605,7 @@ def module(self, module): ] @property - def symbol(self) -> str: + def symbol(self) -> Union[str, None]: """Symbol(s) requested to subscribe.""" return self._symbol @@ -683,7 +683,7 @@ def start_broadcasting( bufsize=1, ) self._psutil_broadcast_process = psutil.Process(self._broadcast_process.pid) - output_queue = queue.Queue() + output_queue: queue.Queue = queue.Queue() self._broadcast_thread = threading.Thread( target=non_blocking_broadcast, args=( diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 093834b8aa0b..eaeb29381e31 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -153,7 +153,7 @@ def parse_kwargs(): value = args[i + 1] if isinstance(value, str) and value.lower() in ["false", "true"]: - value = value.lower() == "true" + value = value.lower() == "true" # type: ignore elif isinstance(value, str) and value.lower() == "none": value = None _kwargs[key] = value @@ -247,14 +247,14 @@ def write(self, message): cleaned_message = AUTH_TOKEN_FILTER.sub(r"\1********", message) if cleaned_message != message: cleaned_message = f"{cleaned_message}\n" - sys.__stdout__.write(cleaned_message) + sys.__stdout__.write(cleaned_message) # type: ignore def flush(self): """Flush stdout.""" # pylint: disable=import-outside-toplevel import sys - sys.__stdout__.flush() + sys.__stdout__.flush() # type: ignore class AuthTokenFilter(logging.Formatter): @@ -273,7 +273,7 @@ def __init__(self, max_size: int = 1000, max_retries=5, backoff_factor=0.5): # pylint: disable=import-outside-toplevel from asyncio import Queue - self.queue = Queue(maxsize=max_size) + self.queue: Queue = Queue(maxsize=max_size) self.max_retries = max_retries self.backoff_factor = backoff_factor self.logger = get_logger("openbb.websocket.queue") diff --git a/openbb_platform/extensions/websockets/openbb_websockets/listen.py b/openbb_platform/extensions/websockets/openbb_websockets/listen.py index e16f96817a64..ae1aad6f5a0b 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/listen.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/listen.py @@ -10,7 +10,7 @@ def __init__(self, **kwargs): self.loop = None self.websocket = None self.current_task = None - self.kwargs = {} + self.kwargs: dict = {} if kwargs: self.kwargs = kwargs @@ -103,18 +103,18 @@ async def listen(self, url, **kwargs): # noqa: PLR0915 def stop(self): if self.current_task: self.current_task.cancel() - self.loop.run_until_complete(self.current_task) + self.loop.run_until_complete(self.current_task) # type: ignore if self.websocket: - self.loop.run_until_complete(self.websocket.close()) - if not self.loop.is_closed(): - self.loop.stop() + self.loop.run_until_complete(self.websocket.close()) # type: ignore + if not self.loop.is_closed(): # type: ignore + self.loop.stop() # type: ignore async def start_listening(self, url, **kwargs): # pylint: disable=import-outside-toplevel import asyncio import contextlib - self.current_task = self.loop.create_task(self.listen(url, **kwargs)) + self.current_task = self.loop.create_task(self.listen(url, **kwargs)) # type: ignore with contextlib.suppress(asyncio.CancelledError): await self.current_task diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py index bffbfefd7899..21954b897871 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py @@ -1,6 +1,6 @@ """Intrinio Realtime Stocks Client.""" -# pylint: disable +# pylint: skip-file # flake8: noqa # This file is a slightly modified version of the original file from the Intrinio Python SDK. diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 1cb593385ef0..250f8a213319 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -1186,19 +1186,20 @@ def __new__(cls, **data): ).startswith("O:") if options_symbol: - model = OPTIONS_MODEL_MAP.get(data.get("ev")) or OPTIONS_MODEL_MAP.get( - data.get("type") + model = OPTIONS_MODEL_MAP.get(data.get("ev", "")) or OPTIONS_MODEL_MAP.get( + data.get("type", "") ) else: model = ( MODEL_MAP["A"] if index_symbol - else MODEL_MAP.get(data.get("ev")) or MODEL_MAP.get(data.get("type")) + else MODEL_MAP.get(data.get("ev", "")) + or MODEL_MAP.get(data.get("type", "")) ) if not model: return super().__new__(cls) - return model.model_validate(data) + return model.model_validate(data) # type: ignore class PolygonWebSocketConnection(WebSocketConnection): From d3c65f8ee16327b8fd68dcb9cf813428f35f1395 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 7 Dec 2024 19:03:26 -0800 Subject: [PATCH 051/119] model_config instead of __model_config__ --- .../extensions/websockets/openbb_websockets/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/models.py b/openbb_platform/extensions/websockets/openbb_websockets/models.py index ae90ff2352e9..6912f743dc4e 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/models.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/models.py @@ -146,7 +146,7 @@ class WebSocketData(Data): class WebSocketConnection(Data): """Data model for returning WebSocketClient from the Provider Interface.""" - __model_config__ = ConfigDict( + model_config = ConfigDict( extra="forbid", ) From fe0add91fe893cc58c21b8026326b15d67e2012e Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 7 Dec 2024 19:40:52 -0800 Subject: [PATCH 052/119] bit more cleanup --- .../extensions/websockets/openbb_websockets/client.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index cd6bee9c82a3..9198df3d30fa 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -289,9 +289,8 @@ def _log_provider_output(self, output_queue) -> None: err = ChildProcessError(output) self._exception = err output = output + "\n" - self.logger.info(output) - # sys.stdout.write(output) - # sys.stdout.flush() + sys.stdout.write(output) + sys.stdout.flush() break output = clean_message(output) @@ -301,10 +300,7 @@ def _log_provider_output(self, output_queue) -> None: elif output.startswith("INFO:"): output = output.replace("INFO:", "PROVIDER INFO:") - # output = output + "\n" - # sys.stdout.write(output + "\n") self.logger.info(output) - # sys.stdout.flush() except queue.Empty: continue From 15eeec1d3d08418007bfd2cc6bbc3ac124ef645f Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sun, 8 Dec 2024 19:57:43 -0800 Subject: [PATCH 053/119] linting --- .../extensions/websockets/openbb_websockets/client.py | 6 +++--- .../intrinio/openbb_intrinio/utils/stocks_client.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 9198df3d30fa..183b6fc957b1 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -107,7 +107,7 @@ def __init__( # noqa: PLR0913 from openbb_websockets.helpers import get_logger self.name = name - self.module = module.replace(".py", "") + self.module = module.replace(".py", "") # type: ignore self.results_file = results_file if results_file else None self.table_name = table_name if table_name else "records" self._limit = limit @@ -371,7 +371,7 @@ def connect(self) -> None: command = self.module command.extend([f"symbol={symbol}"]) command.extend([f"results_file={self.results_file}"]) - command.extend([f"table_name={self.table_name}"]), + command.extend([f"table_name={self.table_name}"]) if self.limit: command.extend([f"limit={self.limit}"]) @@ -487,7 +487,7 @@ def subscribe(self, symbol) -> None: exc = getattr(self, "_exception", None) self._exception = None raise OpenBBError(exc) - old_symbols = self.symbol.split(",") + old_symbols = self.symbol.split(",") if self.symbol is not None else [] new_symbols = list(set(old_symbols + ticker)) self._symbol = ",".join(new_symbols) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py index 21954b897871..8c61ad7fee98 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py @@ -2,6 +2,7 @@ # pylint: skip-file # flake8: noqa +# type: ignore # This file is a slightly modified version of the original file from the Intrinio Python SDK. import logging From e553d7a6c4b7f5bff8538a8df8f66e39263cac6d Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sun, 8 Dec 2024 20:04:12 -0800 Subject: [PATCH 054/119] skip type checking on intrinio's code from their python SDK --- .../providers/intrinio/openbb_intrinio/utils/stocks_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py index 8c61ad7fee98..156c58616280 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py @@ -1,8 +1,9 @@ """Intrinio Realtime Stocks Client.""" # pylint: skip-file +# mypy: ignore-errors +# ruff: noqa # flake8: noqa -# type: ignore # This file is a slightly modified version of the original file from the Intrinio Python SDK. import logging From c5d49ae47816229adf464bdfbaac0bbfdae5f9fa Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sun, 8 Dec 2024 20:07:24 -0800 Subject: [PATCH 055/119] skip type checking on intrinio's code from their python SDK --- .../intrinio/openbb_intrinio/utils/stocks_client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py index 156c58616280..49689d77e6d7 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py @@ -4,6 +4,11 @@ # mypy: ignore-errors # ruff: noqa # flake8: noqa +# noqa: E501 +# noqa: F401 +# noqa: F403 +# noqa: F405 + # This file is a slightly modified version of the original file from the Intrinio Python SDK. import logging From 09532eab803ef6e9bc59ab99d3a68947be07d58f Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sun, 8 Dec 2024 20:10:50 -0800 Subject: [PATCH 056/119] skip type checking on intrinio's code from their python SDK --- .../providers/intrinio/openbb_intrinio/utils/stocks_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py index 49689d77e6d7..6600d3823a89 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py @@ -8,6 +8,7 @@ # noqa: F401 # noqa: F403 # noqa: F405 +# pydocstyle: ignore=* # This file is a slightly modified version of the original file from the Intrinio Python SDK. From 31bda8e46b321ae65ef418032175a8c4a683772a Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sun, 8 Dec 2024 20:25:08 -0800 Subject: [PATCH 057/119] more lint --- .../websockets/openbb_websockets/broadcast.py | 6 +++- .../websockets/openbb_websockets/helpers.py | 4 ++- .../websockets/openbb_websockets/listen.py | 3 ++ .../fmp/openbb_fmp/utils/websocket_client.py | 1 + .../openbb_intrinio/utils/stocks_client.py | 36 ++++++++++++++++++- .../openbb_polygon/utils/websocket_client.py | 1 + 6 files changed, 48 insertions(+), 3 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py index 6f3d37b47aa7..f8f4dd71d5b7 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -1,3 +1,5 @@ +"""Broadcast server for streaming results to connected clients via WebSocket.""" + import asyncio import json import sys @@ -51,6 +53,7 @@ async def read_stdin(broadcast_server): async def websocket_endpoint( # noqa: PLR0915 websocket: WebSocket, auth_token: Optional[str] = None ): + """WebSocket endpoint.""" broadcast_server = BroadcastServer( RESULTS_FILE, @@ -128,6 +131,7 @@ def __init__( sleep_time: float = 0.25, auth_token: Optional[str] = None, ): + """Initialize the BroadcastServer instance.""" # pylint: disable=import-outside-toplevel import os @@ -254,7 +258,7 @@ def create_broadcast_server( def main(): - """Main entry point.""" + """The main function.""" broadcast_server = create_broadcast_server( RESULTS_FILE, TABLE_NAME, diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index eaeb29381e31..3f8168acba5b 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -164,7 +164,7 @@ def parse_kwargs(): async def setup_database(results_path, table_name): - """Setup the SQLite database.""" + """Create the SQLite database, if required.""" # pylint: disable=import-outside-toplevel import os # noqa import aiosqlite @@ -268,6 +268,8 @@ def format(self, record): class MessageQueue: + """Async message queue for the WebSocket connection.""" + def __init__(self, max_size: int = 1000, max_retries=5, backoff_factor=0.5): """Initialize the MessageQueue.""" # pylint: disable=import-outside-toplevel diff --git a/openbb_platform/extensions/websockets/openbb_websockets/listen.py b/openbb_platform/extensions/websockets/openbb_websockets/listen.py index ae1aad6f5a0b..61473df05ec8 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/listen.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/listen.py @@ -101,6 +101,7 @@ async def listen(self, url, **kwargs): # noqa: PLR0915 await self.websocket.close() def stop(self): + """Stop the listener.""" if self.current_task: self.current_task.cancel() self.loop.run_until_complete(self.current_task) # type: ignore @@ -110,6 +111,7 @@ def stop(self): self.loop.stop() # type: ignore async def start_listening(self, url, **kwargs): + """Start listening for WebSocket messages.""" # pylint: disable=import-outside-toplevel import asyncio import contextlib @@ -119,6 +121,7 @@ async def start_listening(self, url, **kwargs): await self.current_task def run(self, url, **kwargs): + """Run the listener.""" # pylint: disable=import-outside-toplevel import asyncio diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py index 75bfb8915da6..352c7b3f5e48 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -27,6 +27,7 @@ async def login(websocket, api_key): + """Login to the WebSocket.""" login_event = { "event": "login", "data": { diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py index 6600d3823a89..46c22d81fad0 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py @@ -8,7 +8,6 @@ # noqa: F401 # noqa: F403 # noqa: F405 -# pydocstyle: ignore=* # This file is a slightly modified version of the original file from the Intrinio Python SDK. @@ -67,7 +66,10 @@ def __init__( self.market_center = market_center self.condition = condition + """Intrinio Realtime Stocks Client.""" + def __str__(self): + """Return string representation of the quote.""" return ( self.symbol + ", " @@ -99,6 +101,7 @@ def __init__( market_center, condition, ): + """Initialize the Trade object.""" self.symbol = symbol self.price = price self.size = size @@ -109,6 +112,7 @@ def __init__( self.condition = condition def __str__(self): + """Return string representation of the trade.""" return ( self.symbol + ", trade, price: " @@ -126,6 +130,7 @@ def __str__(self): ) def is_darkpool(self): + """Return True if the trade is a dark pool trade.""" return ( not self.market_center or self.market_center in ("D", "E", "\x00") @@ -140,6 +145,7 @@ def __init__( on_trade: Optional[callable], on_quote: Optional[callable], ): + """Initialize the Intrinio Realtime Client.""" if options is None: raise ValueError("Options parameter is required") @@ -216,6 +222,7 @@ def __init__( self.quote_handler.start() def auth_url(self) -> str: + """Return the authentication URL.""" auth_url = "" if self.provider == REALTIME: @@ -233,11 +240,13 @@ def auth_url(self) -> str: return auth_url def api_auth_url(self, auth_url: str) -> str: + """Return the API authentication URL.""" auth_url = auth_url + "&" if "?" in auth_url else auth_url + "?" return auth_url + "api_key=" + self.api_key def websocket_url(self) -> str: + """Return the websocket URL.""" if self.provider == REALTIME: return ( "wss://realtime-mx.intrinio.com/socket/websocket?vsn=1.0.0&token=" @@ -262,12 +271,14 @@ def websocket_url(self) -> str: ) def do_backoff(self): + """Perform a backoff.""" self.last_self_heal_backoff += 1 i = min(self.last_self_heal_backoff, len(SELF_HEAL_BACKOFFS) - 1) backoff = SELF_HEAL_BACKOFFS[i] time.sleep(backoff) def connect(self): + """Connect to the websocket.""" connected = False while not connected: try: @@ -287,6 +298,7 @@ def connect(self): self.do_backoff() def disconnect(self): + """Disconnect from the websocket.""" self.ready = False self.joined_channels = set() @@ -295,6 +307,7 @@ def disconnect(self): time.sleep(1) def refresh_token(self): + """Refresh the authentication token.""" headers = {HEADER_CLIENT_INFORMATION_KEY: HEADER_CLIENT_INFORMATION_VALUE} if self.api_key: response = requests.get(self.auth_url(), headers=headers, timeout=5) @@ -313,20 +326,24 @@ def refresh_token(self): self.logger.info("INFO: Authentication successful!") def refresh_websocket(self): + """Refresh the websocket connection.""" self.quote_receiver = QuoteReceiver(self) self.quote_receiver.start() def on_connect(self): + """Handle the connection event.""" self.ready = True self.last_self_heal_backoff = -1 self.refresh_channels() def on_queue_full(self): + """Handle the queue full event.""" if time.time() - self.last_queue_warning_time > 1: self.logger.error("INFO: Quote queue is full! Dropped some new quotes") self.last_queue_warning_time = time.time() def join(self, channels: list[str]): + """Join the specified channels.""" if isinstance(channels, str): channels = [channels] @@ -334,6 +351,7 @@ def join(self, channels: list[str]): self.refresh_channels() def leave(self, channels: list[str]): + """Leave the specified channels.""" if isinstance(channels, str): channels = [channels] @@ -341,10 +359,12 @@ def leave(self, channels: list[str]): self.refresh_channels() def leave_all(self): + """Leave all channels.""" self.channels = set() self.refresh_channels() def refresh_channels(self): + """Refresh the channels.""" if self.ready is not True: return @@ -368,6 +388,7 @@ def refresh_channels(self): self.logger.debug(f"Current channels: {self.joined_channels}") def join_binary_message(self, channel: str): + """Return the binary message to join the specified channel.""" if channel == "lobby": message = bytearray([74, 1 if self.tradesonly else 0]) channel_bytes = bytes("$FIREHOSE", "ascii") @@ -380,6 +401,7 @@ def join_binary_message(self, channel: str): return message def leave_binary_message(self, channel: str): + """Return the binary message to leave the specified channel.""" if channel == "lobby": message = bytearray([76]) channel_bytes = bytes("$FIREHOSE", "ascii") @@ -392,17 +414,20 @@ def leave_binary_message(self, channel: str): return message def valid_api_key(self, api_key: str): + """Return True if the API key is valid.""" return not (not isinstance(api_key, str) or api_key == "") class QuoteReceiver(threading.Thread): def __init__(self, client): + """Initialize the QuoteReceiver.""" threading.Thread.__init__(self, args=(), kwargs=None) self.daemon = True self.client = client self.enabled = True def run(self): + """Run the QuoteReceiver.""" self.client.ws = websocket.WebSocketApp( self.client.websocket_url(), header={ @@ -422,13 +447,16 @@ def run(self): self.client.logger.debug("QuoteReceiver exiting") def on_open(self, ws): + """Handle the open event.""" self.client.logger.info("INFO: Websocket opened!") self.client.on_connect() def on_close(self, ws, code, message): + """Handle the close event.""" self.client.logger.info("INFO: Websocket closed!") def on_error(self, ws, error, *args): + """Handle the error event.""" try: msg = ( f"Unexpected error -> {error.__class__.__name__}: {repr(error)}" @@ -443,6 +471,7 @@ def on_error(self, ws, error, *args): raise e def on_message(self, ws, message): + """Handle the message event.""" try: if ( DEBUGGING @@ -470,6 +499,7 @@ def on_message(self, ws, message): class QuoteHandler(threading.Thread): def __init__(self, client, bypass_parsing: bool): + """Initialize the QuoteHandler.""" threading.Thread.__init__(self, args=(), kwargs=None) self.daemon = True self.client = client @@ -485,6 +515,7 @@ def __init__(self, client, bypass_parsing: bool): } def parse_quote(self, quote_bytes: bytes, start_index: int = 0) -> Quote: + """Parse the quote.""" buffer = memoryview(quote_bytes) symbol_length = buffer[start_index + 2] symbol = ( @@ -533,6 +564,7 @@ def parse_quote(self, quote_bytes: bytes, start_index: int = 0) -> Quote: ) def parse_trade(self, trade_bytes: bytes, start_index: int = 0) -> Trade: + """Parse the trade.""" buffer = memoryview(trade_bytes) symbol_length = buffer[start_index + 2] symbol = ( @@ -582,6 +614,7 @@ def parse_trade(self, trade_bytes: bytes, start_index: int = 0) -> Trade: def parse_message( self, message_bytes: bytes, start_index: int, backlog_len: int ) -> int: + """Parse the message.""" message_type = message_bytes[start_index] message_length = message_bytes[start_index + 1] new_start_index = start_index + message_length @@ -609,6 +642,7 @@ def parse_message( return new_start_index def run(self): + """Run the QuoteHandler.""" self.client.logger.debug("QuoteHandler ready") while True: message = self.client.quotes.get() diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index cdb706542ee8..891201c20589 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -85,6 +85,7 @@ async def handle_symbol(symbol): async def login(websocket, api_key): + """Login to the WebSocket.""" login_event = f'{{"action":"auth","params":"{api_key}"}}' try: await websocket.send(login_event) From 9fd137c1d24ef2ed4a44a55068d00b3e07061d37 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sun, 8 Dec 2024 20:35:22 -0800 Subject: [PATCH 058/119] more lint --- .../websockets/openbb_websockets/broadcast.py | 2 +- .../intrinio/openbb_intrinio/utils/stocks_client.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py index f8f4dd71d5b7..a3cf9a911f39 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -53,7 +53,7 @@ async def read_stdin(broadcast_server): async def websocket_endpoint( # noqa: PLR0915 websocket: WebSocket, auth_token: Optional[str] = None ): - """WebSocket endpoint.""" + """Connect to the broadcast server.""" broadcast_server = BroadcastServer( RESULTS_FILE, diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py index 46c22d81fad0..d3d78e8b9d12 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py @@ -46,6 +46,8 @@ class Quote: + """Intrinio Realtime Stocks Quote.""" + def __init__( self, symbol, @@ -57,6 +59,7 @@ def __init__( market_center, condition, ): + """Initialize the Quote object.""" self.symbol = symbol self.type = type self.price = price @@ -66,8 +69,6 @@ def __init__( self.market_center = market_center self.condition = condition - """Intrinio Realtime Stocks Client.""" - def __str__(self): """Return string representation of the quote.""" return ( @@ -90,6 +91,8 @@ def __str__(self): class Trade: + """Intrinio Realtime Stocks Trade.""" + def __init__( self, symbol, @@ -139,6 +142,8 @@ def is_darkpool(self): class IntrinioRealtimeClient: + """Intrinio Realtime Stocks Client.""" + def __init__( self, options: Dict[str, Any], @@ -419,6 +424,8 @@ def valid_api_key(self, api_key: str): class QuoteReceiver(threading.Thread): + """Intrinio Realtime Stocks Quote Receiver.""" + def __init__(self, client): """Initialize the QuoteReceiver.""" threading.Thread.__init__(self, args=(), kwargs=None) @@ -498,6 +505,8 @@ def on_message(self, ws, message): class QuoteHandler(threading.Thread): + """Intrinio Realtime Stocks Quote Handler.""" + def __init__(self, client, bypass_parsing: bool): """Initialize the QuoteHandler.""" threading.Thread.__init__(self, args=(), kwargs=None) From 9a1d4c52ef45282ba9949f21f9e9aee55bc2719e Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sun, 8 Dec 2024 20:39:27 -0800 Subject: [PATCH 059/119] more lint --- .../extensions/websockets/openbb_websockets/broadcast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py index a3cf9a911f39..576b602a559f 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -258,7 +258,7 @@ def create_broadcast_server( def main(): - """The main function.""" + """Run the main function.""" broadcast_server = create_broadcast_server( RESULTS_FILE, TABLE_NAME, From 9ab9c6e769ea862fa45f6d05e10eae2db302a16c Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:45:23 -0800 Subject: [PATCH 060/119] lots of linting --- .../websockets/openbb_websockets/broadcast.py | 58 ++++--- .../websockets/openbb_websockets/client.py | 163 +++++++----------- .../websockets/openbb_websockets/helpers.py | 3 +- .../websockets/openbb_websockets/listen.py | 40 ++--- .../openbb_websockets/websockets_router.py | 13 +- .../openbb_fmp/models/websocket_connection.py | 5 + .../fmp/openbb_fmp/utils/websocket_client.py | 41 ++--- .../models/websocket_connection.py | 13 +- .../openbb_intrinio/utils/websocket_client.py | 85 ++++----- .../models/websocket_connection.py | 7 +- .../models/websocket_connection.py | 12 +- .../openbb_tiingo/utils/websocket_client.py | 53 +++--- 12 files changed, 241 insertions(+), 252 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py index 576b602a559f..dee00bd9f4fe 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -10,7 +10,7 @@ from fastapi import FastAPI, WebSocket, WebSocketDisconnect from starlette.websockets import WebSocketState -from openbb_websockets.helpers import get_logger, parse_kwargs +from openbb_websockets.helpers import get_logger, parse_kwargs, setup_database connected_clients: set = set() @@ -65,7 +65,10 @@ async def websocket_endpoint( # noqa: PLR0915 if ( broadcast_server.auth_token is not None - and auth_token != broadcast_server._decrypt_value(broadcast_server.auth_token) + and auth_token + != broadcast_server._decrypt_value( # pylint: disable=protected-access + broadcast_server.auth_token + ) ): await websocket.accept() await websocket.send_text( @@ -92,10 +95,9 @@ async def websocket_endpoint( # noqa: PLR0915 except WebSocketDisconnect: pass - except Exception as e: + except Exception as e: # pylint: disable=broad-except msg = f"Unexpected error: {e.__class__.__name__} -> {e}" broadcast_server.logger.error(msg) - pass finally: if broadcast_server in connected_clients: connected_clients.remove(broadcast_server) @@ -106,7 +108,7 @@ async def websocket_endpoint( # noqa: PLR0915 await stdin_task except asyncio.CancelledError: broadcast_server.logger.info("INFO: A listener task was cancelled.") - except Exception as e: + except Exception as e: # pylint: disable=broad-except msg = f"Unexpected error while cancelling stream task: {e.__class__.__name__} -> {e}" broadcast_server.logger.error(msg) if websocket.client_state != WebSocketState.DISCONNECTED: @@ -117,7 +119,7 @@ async def websocket_endpoint( # noqa: PLR0915 broadcast_server.logger.error(msg) -class BroadcastServer: +class BroadcastServer: # pylint: disable=too-many-instance-attributes """Stream new results from a continuously written SQLite database. Not intended to be used directly, it is initialized by the server app when it accepts a new connection. @@ -144,6 +146,7 @@ def __init__( self._iv = os.urandom(16) self.auth_token = self._encrypt_value(auth_token) if auth_token else None self.websocket = None + self.kwargs = kwargs def _encrypt_value(self, value: str) -> str: """Encrypt the value for storage.""" @@ -159,8 +162,11 @@ def _decrypt_value(self, value: str) -> str: return decrypt_value(self._key, self._iv, value) - async def stream_results(self): # noqa: PLR0915 + async def stream_results( # noqa: PLR0915 # pylint: disable=too-many-branches + self, + ): """Continuously read the database and send new messages as JSON via WebSocket.""" + # pylint: disable=import-outside-toplevel import sqlite3 # noqa from openbb_core.app.model.abstract.error import OpenBBError @@ -168,16 +174,15 @@ async def stream_results(self): # noqa: PLR0915 last_id = 0 if not file_path.exists(): - self.logger.error(f"Results file not found: {file_path}") + self.logger.error("Results file not found: %s", str(file_path)) return - else: - conn = sqlite3.connect(self.results_file) - cursor = conn.cursor() - cursor.execute(f"SELECT MAX(id) FROM {self.table_name}") # noqa:S608 - last_id = cursor.fetchone()[0] or 0 - conn.close() + conn = sqlite3.connect(self.results_file) + cursor = conn.cursor() + cursor.execute(f"SELECT MAX(id) FROM {self.table_name}") # noqa:S608 + last_id = cursor.fetchone()[0] or 0 + conn.close() - try: + try: # pylint: disable=too-many-nested-blocks while True: try: if file_path.exists(): @@ -192,7 +197,7 @@ async def stream_results(self): # noqa: PLR0915 if rows: for row in rows: - index, message = row + _, message = row await self.broadcast(json.dumps(json.loads(message))) last_id = max(row[0] for row in rows) else: @@ -209,17 +214,15 @@ async def stream_results(self): # noqa: PLR0915 "Results file was removed by the parent process." ) break - else: - raise OpenBBError(e) from e + raise OpenBBError(e) from e except asyncio.CancelledError: break except WebSocketDisconnect: pass - except Exception as e: + except Exception as e: # pylint: disable=broad-except msg = f"Unexpected error: {e.__class__.__name__} -> {e}" self.logger.error(msg) - finally: - return + return async def broadcast(self, message: str): """Broadcast a message to all connected connected clients.""" @@ -229,14 +232,15 @@ async def broadcast(self, message: str): await client.websocket.send_text(message) except WebSocketDisconnect: disconnected_clients.add(client) - except Exception as e: - self.logger.error(f"Unexpected error: {e}") + except Exception as e: # pylint: disable=broad-except + msg = f"Unexpected error: {e.__class__.__name__} -> {e}" + self.logger.error(msg) disconnected_clients.add(client) # Remove disconnected connected clients for client in disconnected_clients: connected_clients.remove(client) - def start_app(self, host: str = "127.0.0.1", port: int = 6666, **kwargs): + def start_app(self, host: str = "127.0.0.1", port: int = 6666): """Start the FastAPI app with Uvicorn.""" uvicorn.run( self._app, @@ -251,7 +255,6 @@ def create_broadcast_server( table_name: str, sleep_time: float = 0.25, auth_token: Optional[str] = None, - **kwargs, ): """Create a new BroadcastServer instance.""" return BroadcastServer(results_file, table_name, sleep_time, auth_token) @@ -288,5 +291,8 @@ def main(): raise ValueError("Results file path is required for Broadcast server.") if not Path(RESULTS_FILE).absolute().exists(): - raise FileNotFoundError(f"Results file not found: {RESULTS_FILE}") + # pylint: disable=import-outside-toplevel + from openbb_core.provider.utils.helpers import run_async + + run_async(setup_database, RESULTS_FILE, TABLE_NAME) main() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 183b6fc957b1..ac9b258e34f4 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -1,6 +1,6 @@ """WebSocket Client module for interacting with a provider websocket in a non-blocking pattern.""" -# pylint: disable=too-many-statements +# pylint: disable=too-many-statements,protected-access # flake8: noqa: PLR0915 import logging @@ -10,7 +10,7 @@ from openbb_core.provider.abstract.data import Data -class WebSocketClient: +class WebSocketClient: # pylint: disable=too-many-instance-attributes """Client for interacting with a websocket server in a non-blocking pattern. Parameters @@ -79,7 +79,7 @@ class WebSocketClient: Send a message to the WebSocket process. Messages can be sent to "provider" or "broadcast" targets. """ - def __init__( # noqa: PLR0913 + def __init__( # noqa: PLR0913 # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals self, name: str, module: str, @@ -95,8 +95,7 @@ def __init__( # noqa: PLR0913 ) -> None: """Initialize the WebSocketClient class.""" # pylint: disable=import-outside-toplevel - import asyncio # noqa - import atexit + import atexit # noqa import os import tempfile import threading @@ -105,6 +104,7 @@ def __init__( # noqa: PLR0913 from pathlib import Path from openbb_core.app.model.abstract.error import OpenBBError from openbb_websockets.helpers import get_logger + from openbb_websockets.helpers import encrypt_value self.name = name self.module = module.replace(".py", "") # type: ignore @@ -115,13 +115,14 @@ def __init__( # noqa: PLR0913 self._symbol = symbol self._key = os.urandom(32) self._iv = os.urandom(16) - self._auth_token = self._encrypt_value(auth_token) if auth_token else None + self._auth_token = ( + encrypt_value(self._key, self._iv, auth_token) if auth_token else None + ) # strings in kwargs are encrypted before storing in the class but unencrypted when passed to the provider module. if kwargs: for k, v in kwargs.items(): if isinstance(v, str): - encrypted_value = self._encrypt_value(v) - kwargs[k] = encrypted_value + kwargs[k] = encrypt_value(self._key, self._iv, v) else: kwargs[k] = v @@ -144,7 +145,6 @@ def __init__( # noqa: PLR0913 if not results_file: with tempfile.NamedTemporaryFile(delete=False) as temp_file: - pass temp_file_path = temp_file.name self.results_path = Path(temp_file_path).absolute() self.results_file = temp_file_path @@ -162,20 +162,6 @@ def __init__( # noqa: PLR0913 self.logger.error(msg) self._exception = OpenBBError(msg) - def _encrypt_value(self, value): - """Encrypt a value before storing.""" - # pylint: disable=import-outside-toplevel - from openbb_websockets.helpers import encrypt_value - - return encrypt_value(self._key, self._iv, value) - - def _decrypt_value(self, encrypted_value): - """Decrypt the value for use.""" - # pylint: disable=import-outside-toplevel - from openbb_websockets.helpers import decrypt_value - - return decrypt_value(self._key, self._iv, encrypted_value) - def _atexit(self) -> None: """Clean up the WebSocket client processes at exit.""" # pylint: disable=import-outside-toplevel @@ -195,34 +181,10 @@ def _atexit(self) -> None: def _setup_database(self) -> None: """Set up the SQLite database and table.""" # pylint: disable=import-outside-toplevel - import asyncio # noqa - import threading - import time + from openbb_core.provider.utils.helpers import run_async # noqa from openbb_websockets.helpers import setup_database - def run_in_new_loop(): - """Run setup in new event loop.""" - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - loop.run_until_complete( - setup_database(self.results_path, self.table_name) - ) - finally: - loop.close() - - def run_in_thread(): - """Run setup in separate thread.""" - thread = threading.Thread(target=run_in_new_loop) - thread.start() - time.sleep(0.01) - thread.join() - - try: - loop = asyncio.get_running_loop() # noqa - run_in_thread() - except RuntimeError: - run_in_new_loop() + run_async(setup_database, self.results_path, self.table_name) def _log_provider_output(self, output_queue) -> None: """Log output from the provider logger, handling exceptions, errors, and messages that are not data.""" @@ -346,17 +308,17 @@ def _log_broadcast_output(self, output_queue) -> None: except queue.Empty: continue - def connect(self) -> None: + def connect(self) -> None: # pylint: disable=too-many-locals """Connect to the provider WebSocket.""" # pylint: disable=import-outside-toplevel - import json # noqa - import os + import os # noqa import psutil import queue import subprocess import threading import time from openbb_core.app.model.abstract.error import OpenBBError + from openbb_websockets.helpers import decrypt_value if self.is_running: self.logger.info("Provider connection already running.") @@ -381,7 +343,9 @@ def connect(self) -> None: if kwargs: for k, v in kwargs.items(): if isinstance(v, str): - unencrypted_value = self._decrypt_value(v) + unencrypted_value = decrypt_value( + self._key, self._iv, v # pylint: disable=protected-access + ) kwargs[k] = unencrypted_value else: kwargs[k] = v @@ -396,7 +360,7 @@ def connect(self) -> None: if kwarg not in command: command.extend([kwarg]) - self._process = subprocess.Popen( # noqa + self._process = subprocess.Popen( # noqa # pylint: disable=consider-using-with command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -430,9 +394,9 @@ def connect(self) -> None: time.sleep(2) if self._exception is not None: - exc = getattr(self, "_exception", None) - self._exception = None - raise OpenBBError(exc) + with self._exception as exc: + self._exception = None + raise OpenBBError(exc) if not self.is_running: self.logger.error("The provider server failed to start.") @@ -542,7 +506,7 @@ def results(self) -> Union[list[dict], list["Data"], None]: try: cursor = conn.execute(f"SELECT * FROM {self.table_name}") # noqa for row in cursor: - index, message = row + _, message = row if self.data_model: message = json.loads(message) if isinstance(message, (str, bytes)): @@ -554,7 +518,7 @@ def results(self) -> Union[list[dict], list["Data"], None]: else: output.append(json.loads(json.loads(message))) except Exception as e: - raise OpenBBError(f"Error retrieving results: {e}") + raise OpenBBError(f"Error retrieving results: {e}") from e if output: return output @@ -580,8 +544,9 @@ def results(self): self.table_name, self.results_file, ) - except Exception as e: - self.logger.error("Error clearing results: %s", e) + except Exception as e: # pylint: disable=broad-except + msg = f"Error clearing results: {e.__class__.__name__}: {e}" + self.logger.error(msg) @property def module(self) -> list: @@ -624,7 +589,7 @@ def broadcast_address(self) -> Union[str, None]: else None ) - def start_broadcasting( + def start_broadcasting( # pylint: disable=too-many-locals self, host: str = "127.0.0.1", port: int = 6666, @@ -639,14 +604,14 @@ def start_broadcasting( import psutil import queue from openbb_platform_api.utils.api import check_port + from openbb_websockets.helpers import decrypt_value if ( self._broadcast_process is not None and self._broadcast_process.poll() is None ): - self.logger.info( - f"WebSocket broadcast already running on: {self._broadcast_address}" - ) + msg = f"WebSocket broadcast already running on: {self._broadcast_address}" + self.logger.info(msg) return open_port = check_port(host, port) @@ -663,20 +628,22 @@ def start_broadcasting( f"port={open_port}", f"results_file={self.results_file}", f"table_name={self.table_name}", - f"auth_token={self._decrypt_value(self._auth_token) if self._auth_token else None}", + f"auth_token={decrypt_value(self._key, self._iv, self._auth_token) if self._auth_token else None}", ] if kwargs: - for kwarg in kwargs: - command.extend([f"{kwarg}={kwargs[kwarg]}"]) - - self._broadcast_process = subprocess.Popen( # noqa - command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - stdin=subprocess.PIPE, - env=os.environ, - text=True, - bufsize=1, + for k, v in kwargs.items(): + command.extend([f"{k}={v}"]) + + self._broadcast_process = ( + subprocess.Popen( # noqa # pylint: disable=consider-using-with + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdin=subprocess.PIPE, + env=os.environ, + text=True, + bufsize=1, + ) ) self._psutil_broadcast_process = psutil.Process(self._broadcast_process.pid) output_queue: queue.Queue = queue.Queue() @@ -786,7 +753,7 @@ def send_message( client._broadcast_process.stdin.flush() else: client.logger.error("Broadcast process is not running.") - except Exception as e: + except Exception as e: # pylint: disable=broad-except msg = f"Error sending message to the {target} process: {e.__class__.__name__} -> {e}" client.logger.error(msg) @@ -796,28 +763,22 @@ def read_message_queue( ): """Read messages from the queue and send them to the WebSocketConnection process.""" while not message_queue.empty(): - try: - message = message_queue.get(timeout=1) - if message: - try: - if ( - target == "provider" - and not client._stop_log_thread_event.is_set() - ): - send_message(client, message, target="provider") - elif ( - target == "broadcast" - and not client._stop_broadcasting_event.is_set() - ): - send_message(client, message, target="broadcast") - except Exception as e: - err = ( - f"Error while attempting to transmit from the outgoing message queue: {e.__class__.__name__} " - f"-> {e} -> {message}" - ) - client.logger.error(err) - finally: - break + message = message_queue.get(timeout=1) + if message: + try: + if target == "provider" and not client._stop_log_thread_event.is_set(): + send_message(client, message, target="provider") + elif ( + target == "broadcast" + and not client._stop_broadcasting_event.is_set() + ): + send_message(client, message, target="broadcast") + except Exception as e: # pylint: disable=broad-except + err = ( + f"Error while attempting to transmit from the outgoing message queue: {e.__class__.__name__} " + f"-> {e} -> {message}" + ) + client.logger.error(err) def non_blocking_broadcast(client, output_queue, broadcast_message_queue) -> None: @@ -832,7 +793,7 @@ def non_blocking_broadcast(client, output_queue, broadcast_message_queue) -> Non break if output: output_queue.put(output.strip()) - except Exception as e: + except Exception as e: # pylint: disable=broad-except client.logger.error( f"Unexpected error in non_blocking_broadcast: {e.__class__.__name__} -> {e}" ) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 3f8168acba5b..c2888b7585f5 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -1,5 +1,7 @@ """WebSockets helpers.""" +# pylint: disable=protected-access + import logging import re from typing import Any, Optional @@ -24,7 +26,6 @@ def clean_message(message: str) -> str: def get_logger(name, level=logging.INFO): """Get a logger instance.""" # pylint: disable=import-outside-toplevel - import logging import uuid logger = logging.getLogger(f"{name}-{uuid.uuid4()}") diff --git a/openbb_platform/extensions/websockets/openbb_websockets/listen.py b/openbb_platform/extensions/websockets/openbb_websockets/listen.py index 61473df05ec8..db28a80766bd 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/listen.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/listen.py @@ -10,11 +10,14 @@ def __init__(self, **kwargs): self.loop = None self.websocket = None self.current_task = None + self.logger = None self.kwargs: dict = {} if kwargs: self.kwargs = kwargs - async def listen(self, url, **kwargs): # noqa: PLR0915 + async def listen( # noqa: PLR0915 # pylint: disable=too-many-branches,too-many-statements,too-many-locals + self, url, **kwargs + ): """Listen for WebSocket messages.""" # pylint: disable=import-outside-toplevel import asyncio # noqa @@ -47,9 +50,8 @@ async def listen(self, url, **kwargs): # noqa: PLR0915 async with websockets.connect(url, **kwargs) as websocket: self.websocket = websocket url = clean_message(url) - self.logger.info( - f"\nListening for messages from {clean_message(url)}" - ) + msg = f"\nListening for messages from {clean_message(url)}" + self.logger.info(msg) for handler in self.logger.handlers: handler.flush() async for message in websocket: @@ -71,30 +73,30 @@ async def listen(self, url, **kwargs): # noqa: PLR0915 websockets.ConnectionClosedError, asyncio.IncompleteReadError, ): - self.logger.error( - f"The process hosting {clean_message(url)} was terminated." - ) + msg = f"The process hosting {clean_message(url)} was terminated." + self.logger.error(msg) break except websockets.exceptions.InvalidURI as error: - self.logger.error(f"Invalid URI -> {error}") + msg = f"Invalid URI -> {error}" + self.logger.error(msg) break except InvalidStatusCode as error: - self.logger.error(f"Invalid status code -> {error}") + msg = f"Invalid status code -> {error}" + self.logger.error(msg) break except OSError as error: if "Multiple exceptions" in str(error): err = str(error).split("Multiple exceptions:")[1].strip() err = err.split("[")[-1].strip().replace("]", ":") - self.logger.error( - f"An error occurred while attempting to connect to: {clean_message(url)} -> {err}" - ) + msg = f"An error occurred while attempting to connect to: {clean_message(url)} -> {err}" + self.logger.error(msg) else: - self.logger.error( - f"An error occurred while attempting to connect to: {clean_message(url)} -> {error}" - ) + msg = f"An error occurred while attempting to connect to: {clean_message(url)} -> {error}" + self.logger.error(msg) break - except Exception as error: - self.logger.error(f"An unexpected error occurred: {error}") + except Exception as error: # pylint: disable=broad-except + msg = f"Unexpected error -> {error.__class__.__name__}: {error}" + self.logger.error(msg) raise OpenBBError(error) from error finally: if self.websocket: @@ -154,7 +156,5 @@ def listen(url, **kwargs): try: listener = Listener(**kwargs) listener.run(url, **kwargs) - except Exception as e: + except Exception as e: # pylint: disable=broad-except raise OpenBBError(e) from e - finally: - return diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index f75af9b37f3e..720061a702f2 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -1,6 +1,6 @@ """Websockets Router.""" -# pylint: disable=unused-argument +# pylint: disable=unused-argument,protected-access,unused-import import asyncio import sys @@ -34,6 +34,17 @@ @router.command( model="WebSocketConnection", + examples=[ + APIEx( + parameters={ + "name": "client1", + "provider": "fmp", + "asset_type": "crypto", + "symbol": "btcusd,ethusd,solusd", + "start_broadcast": True, + } + ) + ], ) async def create_connection( cc: CommandContext, diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py index c9fd7a5d7654..b9779e5f0d22 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py @@ -1,5 +1,7 @@ """FMP WebSocket model.""" +# pylint: disable=unused-argument,protected-access + from datetime import datetime from typing import Any, Literal, Optional @@ -92,11 +94,13 @@ class FmpWebSocketData(WebSocketData): ) @field_validator("symbol", mode="before") + @classmethod def _validate_symbol(cls, v): """Validate the symbol.""" return v.upper() @field_validator("type", mode="before", check_fields=False) + @classmethod def _valiidate_data_type(cls, v): """Validate the data type.""" return ( @@ -104,6 +108,7 @@ def _valiidate_data_type(cls, v): ) @field_validator("date", mode="before", check_fields=False) + @classmethod def _validate_date(cls, v): """Validate the date.""" # pylint: disable=import-outside-toplevel diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py index 352c7b3f5e48..5090088e0df4 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -1,4 +1,4 @@ -"""FMP WebSocket server.""" +"""FMP WebSocket client.""" import asyncio import json @@ -24,14 +24,15 @@ queue = MessageQueue() command_queue = MessageQueue() CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) +kwargs["results_file"] = os.path.abspath(kwargs["results_file"]) -async def login(websocket, api_key): +async def login(websocket): """Login to the WebSocket.""" login_event = { "event": "login", "data": { - "apiKey": api_key, + "apiKey": kwargs["api_key"], }, } try: @@ -97,7 +98,7 @@ async def process_stdin_queue(websocket): await subscribe(websocket, symbol, event) -async def process_message(message, results_path, table_name, limit): +async def process_message(message): """Process the message and write to the database.""" result: dict = {} message = json.loads(message) if isinstance(message, str) else message @@ -124,24 +125,27 @@ async def process_message(message, results_path, table_name, limit): except ValidationError: raise e from e if result: - await write_to_db(result, results_path, table_name, limit) + await write_to_db( + result, + kwargs["results_file"], + kwargs["table_name"], + kwargs.get("limit"), + ) -async def connect_and_stream(url, symbol, api_key, results_path, table_name, limit): +async def connect_and_stream(): """Connect to the WebSocket and stream data to file.""" handler_task = asyncio.create_task( - queue.process_queue( - lambda message: process_message(message, results_path, table_name, limit) - ) + queue.process_queue(lambda message: process_message(message)) ) stdin_task = asyncio.create_task(read_stdin_and_queue_commands()) try: - websocket = await websockets.connect(url, **CONNECT_KWARGS) - await login(websocket, api_key) - await subscribe(websocket, symbol, "subscribe") + websocket = await websockets.connect(kwargs["url"], **CONNECT_KWARGS) + await login(websocket) + await subscribe(websocket, kwargs["symbol"], "subscribe") while True: ws_task = asyncio.create_task(websocket.recv()) @@ -166,13 +170,13 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim # Attempt to reopen the connection logger.info("PROVIDER INFO: Attempting to reconnect after five seconds.") await asyncio.sleep(5) - await connect_and_stream(url, symbol, api_key, results_path, table_name, limit) + await connect_and_stream() except websockets.WebSocketException as e: logger.error(e) sys.exit(1) - except Exception as e: + except Exception as e: # pylint: disable=broad-except msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e}" logger.error(msg) sys.exit(1) @@ -194,14 +198,7 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim loop.add_signal_handler(sig, handle_termination_signal, logger) asyncio.run_coroutine_threadsafe( - connect_and_stream( - kwargs["url"], - kwargs["symbol"], - kwargs["api_key"], - os.path.abspath(kwargs["results_file"]), - kwargs["table_name"], - kwargs.get("limit", None), - ), + connect_and_stream(), loop, ) loop.run_forever() diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py index cc926dd0b6ee..4f5f89b98e15 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py @@ -1,5 +1,7 @@ """Intrinio WebSocket model.""" +# pylint: disable=unused-argument + from datetime import datetime from typing import Any, Literal, Optional @@ -82,6 +84,7 @@ class IntrinioWebSocketData(WebSocketData): ) @field_validator("date", mode="before", check_fields=False) + @classmethod def _validate_date(cls, v): """Validate the date.""" # pylint: disable=import-outside-toplevel @@ -100,6 +103,12 @@ def _validate_date(cls, v): return dt.astimezone(timezone("America/New_York")) + @field_validator("condition", mode="before", check_fields=False) + @classmethod + def _validate_condition(cls, v): + """Strip the empty spaces from the condition.""" + return v.strip().replace(" ", "") if v and isinstance(v, str) else None + class IntrinioWebSocketConnection(WebSocketConnection): """Intrinio WebSocket connection model.""" @@ -152,8 +161,8 @@ async def aextract_data( try: client.connect() await asyncio.sleep(2) - if client._exception: - raise client._exception + if client._exception: # pylint: disable=protected-access + raise client._exception # pylint: disable=protected-access except OpenBBError as e: if client.is_running: client.disconnect() diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py index 48575f9d6e14..49764c6dcd54 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -1,10 +1,13 @@ """Intrinio WebSocket server.""" +# pylint: disable=unused-argument + import asyncio import json import signal import sys +from openbb_core.provider.utils.helpers import run_async from openbb_intrinio.models.websocket_connection import IntrinioWebSocketData from openbb_intrinio.utils.stocks_client import IntrinioRealtimeClient, Quote, Trade from openbb_websockets.helpers import ( @@ -23,45 +26,6 @@ CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) -async def subscribe(client, symbol, event): - """Subscribe or unsubscribe to a symbol.""" - ticker = symbol.split(",") if isinstance(symbol, str) else symbol - try: - if event == "subscribe": - client.join(ticker) - elif event == "unsubscribe": - client.leave(ticker) - except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" - logger.error(msg) - - -async def read_stdin_and_queue_commands(): - """Read from stdin and queue commands.""" - while True: - line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline) - sys.stdin.flush() - - if not line: - break - - try: - command = json.loads(line.strip()) - await command_queue.enqueue(command) - except json.JSONDecodeError: - logger.error("Invalid JSON received from stdin -> %s", line.strip()) - - -async def process_stdin_queue(client): - """Process the command queue.""" - while True: - command = await command_queue.dequeue() - symbol = ["lobby" if d == "*" else d.upper() for d in command.get("symbol", [])] - event = command.get("event") - if symbol and event: - await subscribe(client, symbol, event) - - async def process_message(message): """Process the message and write to the database.""" result: dict = {} @@ -91,7 +55,7 @@ async def process_message(message): def on_message(message, backlog): """Process the message and write to the database.""" - asyncio.run(process_message(message)) + run_async(process_message, message) options = { @@ -108,6 +72,45 @@ def on_message(message, backlog): ) +async def subscribe(symbol, event): + """Subscribe or unsubscribe to a symbol.""" + ticker = symbol.split(",") if isinstance(symbol, str) else symbol + try: + if event == "subscribe": + client.join(ticker) + elif event == "unsubscribe": + client.leave(ticker) + except Exception as e: # pylint: disable=broad-except + msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" + logger.error(msg) + + +async def read_stdin_and_queue_commands(): + """Read from stdin and queue commands.""" + while True: + line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline) + sys.stdin.flush() + + if not line: + break + + try: + command = json.loads(line.strip()) + await command_queue.enqueue(command) + except json.JSONDecodeError: + logger.error("Invalid JSON received from stdin -> %s", line.strip()) + + +async def process_stdin_queue(): + """Process the command queue.""" + while True: + command = await command_queue.dequeue() + symbol = ["lobby" if d == "*" else d.upper() for d in command.get("symbol", [])] + event = command.get("event") + if symbol and event: + await subscribe(symbol, event) + + async def connect_and_stream(): """Connect to the WebSocket and stream data to file.""" symbol = kwargs.pop("symbol", "lobby") @@ -115,7 +118,7 @@ async def connect_and_stream(): asyncio.create_task(read_stdin_and_queue_commands()) client.connect() client.join(symbol) - asyncio.create_task(process_stdin_queue(client)) + asyncio.create_task(process_stdin_queue()) if __name__ == "__main__": diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 250f8a213319..99db223ac12a 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -371,6 +371,7 @@ def _validate_exchange(cls, v): return CRYPTO_EXCHANGE_MAP.get(v, str(v)) @model_validator(mode="before") + @classmethod def _validate_model(cls, values): """Validate the model.""" _ = values.pop("i", None) @@ -1184,7 +1185,7 @@ def __new__(cls, **data): options_symbol = data.get("sym", "").startswith("O:") or data.get( "symbol", "" ).startswith("O:") - + model = "" if options_symbol: model = OPTIONS_MODEL_MAP.get(data.get("ev", "")) or OPTIONS_MODEL_MAP.get( data.get("type", "") @@ -1259,8 +1260,8 @@ def extract_data( client.disconnect() raise e from e - if client._exception: - raise client._exception from client._exception + if client._exception: # pylint: disable=protected-access + raise client._exception # pylint: disable=protected-access if client.is_running: return {"client": client} diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py index ffeb3a3781d6..7afe43199189 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py @@ -1,5 +1,7 @@ """Tiingo WebSocket model.""" +# pylint: disable=unused-argument + from datetime import datetime from typing import Any, Literal, Optional @@ -166,11 +168,13 @@ class TiingoWebSocketData(WebSocketData): ) @field_validator("symbol", mode="before", check_fields=False) + @classmethod def _validate_symbol(cls, v): """Validate the symbol.""" return v.upper() @field_validator("type", mode="before", check_fields=False) + @classmethod def _valiidate_data_type(cls, v): """Validate the data type.""" return ( @@ -178,6 +182,7 @@ def _valiidate_data_type(cls, v): ) @field_validator("date", "timestamp", mode="before", check_fields=False) + @classmethod def _validate_date(cls, v): """Validate the date.""" # pylint: disable=import-outside-toplevel @@ -284,10 +289,9 @@ async def aextract_data( await sleep(1) - if client._exception: - exc = getattr(client, "_exception", None) - client._exception = None - client._atexit() + if getattr(client, "_exception", None): + exc = client._exception # pylint: disable=protected-access + client._exception = None # pylint: disable=protected-access raise OpenBBError(exc) if client.is_running: diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index 25243c9dc55a..a384962dfe97 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -73,6 +73,7 @@ logger = get_logger("openbb.websocket.tiingo") kwargs = parse_kwargs() CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) +kwargs["results_file"] = os.path.abspath(kwargs["results_file"]) # Subscribe and unsubscribe events are handled in a separate connection using the subscription_id set by the login event. @@ -123,7 +124,7 @@ async def read_stdin_and_update_symbols(): await update_symbols(symbol, event) -async def process_message(message, results_path, table_name, limit): +async def process_message(message): # pylint: disable=too-many-branches """Process the message and write to the database.""" result: dict = {} data_message: dict = {} @@ -148,7 +149,7 @@ async def process_message(message, results_path, table_name, limit): msg = f"PROVIDER INFO: Authorization: {response.get('message')}" logger.info(msg) if message.get("data", {}).get("subscriptionId"): - global SUBSCRIPTION_ID # noqa: PLW0603 + global SUBSCRIPTION_ID # noqa: PLW0603 # pylint: disable=global-statement SUBSCRIPTION_ID = message["data"]["subscriptionId"] if "tickers" in response.get("data", {}): @@ -188,31 +189,32 @@ async def process_message(message, results_path, table_name, limit): raise e from e if result: - await write_to_db(result, results_path, table_name, limit) + await write_to_db( + result, + kwargs["results_file"], + kwargs["table_name"], + kwargs.get("limit"), + ) return -async def connect_and_stream( - url, symbol, threshold_level, api_key, results_path, table_name, limit -): +async def connect_and_stream(): """Connect to the WebSocket and stream data to file.""" handler_task = asyncio.create_task( - queue.process_queue( - lambda message: process_message(message, results_path, table_name, limit) - ) + queue.process_queue(lambda message: process_message(message)) ) - stdin_task = asyncio.create_task(read_stdin_and_update_symbols()) + ticker: list = [] - if isinstance(symbol, str): - ticker = symbol.lower().split(",") + if isinstance(kwargs["symbol"], str): + ticker = kwargs["symbol"].lower().split(",") subscribe_event = { "eventName": "subscribe", - "authorization": api_key, + "authorization": kwargs["api_key"], "eventData": { - "thresholdLevel": threshold_level, + "thresholdLevel": kwargs["threshold_level"], "tickers": ticker, }, } @@ -224,7 +226,7 @@ async def connect_and_stream( try: try: - async with websockets.connect(url, **connect_kwargs) as websocket: + async with websockets.connect(kwargs["url"], **connect_kwargs) as websocket: logger.info("PROVIDER INFO: WebSocket connection established.") await websocket.send(json.dumps(subscribe_event)) while True: @@ -241,15 +243,13 @@ async def connect_and_stream( # Attempt to reopen the connection logger.info("PROVIDER INFO: Attempting to reconnect after five seconds...") await asyncio.sleep(5) - await connect_and_stream( - url, symbol, threshold_level, api_key, results_path, table_name, limit - ) + await connect_and_stream() except websockets.WebSocketException as e: logger.info(str(e)) sys.exit(0) - except Exception as e: + except Exception as e: # pylint: disable=broad-except msg = f"Unexpected error -> {e.__class__.__name__}: {e}" logger.error(msg) sys.exit(1) @@ -270,24 +270,15 @@ async def connect_and_stream( loop.add_signal_handler(signal.SIGTERM, handle_termination_signal, logger) asyncio.run_coroutine_threadsafe( - connect_and_stream( - kwargs["url"], - kwargs["symbol"], - kwargs["threshold_level"], - kwargs["api_key"], - os.path.abspath(kwargs["results_file"]), - kwargs["table_name"], - kwargs.get("limit", None), - ), + connect_and_stream(), loop, ) loop.run_forever() except Exception as e: # pylint: disable=broad-except - msg = f"Unexpected error -> {e.__class__.__name__}: {e}" - logger.error(msg) + ERR = f"Unexpected error -> {e.__class__.__name__}: {e}" + logger.error(ERR) finally: loop.stop() - loop.close sys.exit(0) From 662e5da3daaecd04336296426ed1ad739a57235b Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 9 Dec 2024 14:51:07 -0800 Subject: [PATCH 061/119] polygon quote fields as optional --- .../intrinio/openbb_intrinio/models/websocket_connection.py | 2 +- .../intrinio/openbb_intrinio/utils/stocks_client.py | 6 +++--- .../polygon/openbb_polygon/models/websocket_connection.py | 6 ++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py index 4f5f89b98e15..d10d5b1c0c87 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py @@ -107,7 +107,7 @@ def _validate_date(cls, v): @classmethod def _validate_condition(cls, v): """Strip the empty spaces from the condition.""" - return v.strip().replace(" ", "") if v and isinstance(v, str) else None + return str(v).strip().replace(" ", "") if v else None class IntrinioWebSocketConnection(WebSocketConnection): diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py index d3d78e8b9d12..318f0bf68622 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py @@ -67,7 +67,7 @@ def __init__( self.timestamp = timestamp self.subprovider = subprovider self.market_center = market_center - self.condition = condition + self.condition = str(condition.strip().replace(" ", "")) def __str__(self): """Return string representation of the quote.""" @@ -112,7 +112,7 @@ def __init__( self.timestamp = timestamp self.subprovider = subprovider self.market_center = market_center - self.condition = condition + self.condition = str(condition).strip().replace(" ", "") def __str__(self): """Return string representation of the trade.""" @@ -160,7 +160,7 @@ def __init__( self.password = options.get("password") self.provider = options.get("provider") self.ipaddress = options.get("ipaddress") - self.tradesonly = options.get("tradesonly") + self.tradesonly = None self.bypass_parsing = options.get("bypass_parsing", False) if "channels" in options: diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 99db223ac12a..c79ed4b1fc9c 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -771,7 +771,8 @@ class PolygonStockQuoteWebSocketData(WebSocketData): symbol: str = Field( description=DATA_DESCRIPTIONS.get("symbol", ""), ) - bid_exchange: str = Field( + bid_exchange: Optional[str] = Field( + default=None, description="The exchange where the bid originated.", ) bid_size: Optional[float] = Field( @@ -792,7 +793,8 @@ class PolygonStockQuoteWebSocketData(WebSocketData): default=None, description="The size of the ask.", ) - ask_exchange: str = Field( + ask_exchange: Optional[str] = Field( + default=None, description="The exchange where the ask originated.", ) tape: str = Field( From a4e7678b2178c02c3e4062fef97b132ccd11f28e Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:13:04 -0800 Subject: [PATCH 062/119] some final stragglers...? --- .../polygon/openbb_polygon/models/websocket_connection.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index c79ed4b1fc9c..b2d8cc0f5edf 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -1187,15 +1187,14 @@ def __new__(cls, **data): options_symbol = data.get("sym", "").startswith("O:") or data.get( "symbol", "" ).startswith("O:") - model = "" if options_symbol: - model = OPTIONS_MODEL_MAP.get(data.get("ev", "")) or OPTIONS_MODEL_MAP.get( + model = OPTIONS_MODEL_MAP.get(data.get("ev", "")) or OPTIONS_MODEL_MAP.get( # type: ignore data.get("type", "") ) else: model = ( MODEL_MAP["A"] - if index_symbol + if index_symbol # type: ignore else MODEL_MAP.get(data.get("ev", "")) or MODEL_MAP.get(data.get("type", "")) ) From 461a041426470d488729f985c34250859e0387f0 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 9 Dec 2024 16:23:15 -0800 Subject: [PATCH 063/119] some final stragglers...? --- .../extensions/websockets/openbb_websockets/listen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/listen.py b/openbb_platform/extensions/websockets/openbb_websockets/listen.py index db28a80766bd..75e20bed126f 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/listen.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/listen.py @@ -135,7 +135,7 @@ def run(self, url, **kwargs): try: self.loop.run_until_complete(self.start_listening(url, **kwargs)) except KeyboardInterrupt: - self.logger.info("\nWebSocket listener terminated.") + self.logger.info("\nWebSocket listener terminated.") # type: ignore finally: self.stop() From 53cd9987b79c2a3285bf4f8ba77b881b46a19de3 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:22:16 -0800 Subject: [PATCH 064/119] some more intrinio.. --- .../openbb_intrinio/models/websocket_connection.py | 2 +- .../intrinio/openbb_intrinio/utils/stocks_client.py | 2 +- .../openbb_intrinio/utils/websocket_client.py | 12 ++++-------- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py index d10d5b1c0c87..b3b2c853337f 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py @@ -62,7 +62,7 @@ class IntrinioWebSocketData(WebSocketData): default=None, description="The exchange of the data.", ) - type: Literal["quote", "trade"] = Field( + type: Literal["bid", "ask", "trade"] = Field( description="The type of data.", ) price: Optional[float] = Field( diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py index 318f0bf68622..aeb2f31bf60a 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py @@ -112,7 +112,7 @@ def __init__( self.timestamp = timestamp self.subprovider = subprovider self.market_center = market_center - self.condition = str(condition).strip().replace(" ", "") + self.condition = condition def __str__(self): """Return string representation of the trade.""" diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py index 49764c6dcd54..2d63f20a9e07 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -9,7 +9,7 @@ from openbb_core.provider.utils.helpers import run_async from openbb_intrinio.models.websocket_connection import IntrinioWebSocketData -from openbb_intrinio.utils.stocks_client import IntrinioRealtimeClient, Quote, Trade +from openbb_intrinio.utils.stocks_client import IntrinioRealtimeClient, Trade from openbb_websockets.helpers import ( MessageQueue, get_logger, @@ -31,17 +31,13 @@ async def process_message(message): result: dict = {} message = json.loads(message) if isinstance(message, str) else message is_trade = isinstance(message, Trade) - is_quote = isinstance(message, Quote) if hasattr(message, "__dict__"): message = message.__dict__ - if is_trade or is_quote: + if is_trade: message["type"] = "trade" if is_trade else "quote" try: - result = IntrinioWebSocketData.model_validate(message).model_dump_json( - exclude_none=True, exclude_unset=True - ) - result = message + result = IntrinioWebSocketData.model_validate(message).model_dump_json() except ValidationError as e: try: handle_validation_error(logger, e) @@ -49,7 +45,7 @@ async def process_message(message): raise e from e if result: await write_to_db( - message, kwargs["results_file"], kwargs["table_name"], kwargs["limit"] + result, kwargs["results_file"], kwargs["table_name"], kwargs["limit"] ) From 078f5be3fe626fe530948df3c4a02b43eebb3d56 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:55:05 -0800 Subject: [PATCH 065/119] are we done yet? --- .../fmp/openbb_fmp/utils/websocket_client.py | 4 +- .../openbb_intrinio/utils/websocket_client.py | 8 +-- .../models/websocket_connection.py | 9 +-- .../openbb_polygon/utils/websocket_client.py | 55 ++++++++----------- 4 files changed, 35 insertions(+), 41 deletions(-) diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py index 5090088e0df4..90a4f53487e2 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -207,8 +207,8 @@ async def connect_and_stream(): logger.error("PROVIDER ERROR: WebSocket connection closed") except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" - logger.error(msg) + ERR = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" + logger.error(ERR) finally: sys.exit(0) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py index 2d63f20a9e07..62b942331295 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -77,8 +77,8 @@ async def subscribe(symbol, event): elif event == "unsubscribe": client.leave(ticker) except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" - logger.error(msg) + exc = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" + logger.error(exc) async def read_stdin_and_queue_commands(): @@ -133,8 +133,8 @@ async def connect_and_stream(): logger.error("PROVIDER INFO: WebSocket connection closed") except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" - logger.error(msg) + EXC = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" + logger.error(EXC) finally: client.disconnect() diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index b2d8cc0f5edf..305b779da146 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -122,7 +122,8 @@ def validate_date(cls, v): v = v / 1e9 # Convert nanoseconds to seconds dt = datetime.fromtimestamp(v, tz=timezone("UTC")) dt = dt.astimezone(timezone("America/New_York")) - return dt + + return dt class PolygonWebSocketQueryParams(WebSocketQueryParams): @@ -352,9 +353,9 @@ def _validate_conditions(cls, v): """Validate the conditions.""" if v is None or isinstance(v, list) and v[0] == 0: return None - elif isinstance(v, list) and v[0] == 1: + if isinstance(v, list) and v[0] == 1: return "sellside" - elif isinstance(v, list) and v[0] == 2: + if isinstance(v, list) and v[0] == 2: return "buyside" return str(v) @@ -1199,7 +1200,7 @@ def __new__(cls, **data): or MODEL_MAP.get(data.get("type", "")) ) if not model: - return super().__new__(cls) + return super().__new__(cls) # type: ignore return model.model_validate(data) # type: ignore diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index 891201c20589..c1749b9bb8a6 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -30,6 +30,7 @@ CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) FEED = kwargs.pop("feed", None) ASSET_TYPE = kwargs.pop("asset_type", None) +kwargs["results_file"] = os.path.abspath(kwargs["results_file"]) async def handle_symbol(symbol): @@ -100,16 +101,14 @@ async def login(websocket, api_key): err = f"UnauthorizedError -> {msg.get('message')}" logger.error(err) sys.exit(1) - break if msg.get("status") != "auth_success": err = ( f"UnauthorizedError -> {msg.get('status')} -> {msg.get('message')}" ) logger.error(err) sys.exit(1) - break logger.info("PROVIDER INFO: %s", msg.get("message")) - except Exception as e: + except Exception as e: # pylint: disable=broad-except logger.error("PROVIDER ERROR: %s -> %s", e.__class__.__name__, e.args[0]) sys.exit(1) @@ -124,12 +123,12 @@ async def subscribe(websocket, symbol, event): subscribe_event = f'{{"action":"{event}","params":"{ticker}"}}' try: await websocket.send(subscribe_event) - except Exception as e: + except Exception as e: # pylint: disable=broad-except msg = f"PROVIDER ERROR: {e.__class__.__name__} -> {e}" logger.error(msg) -async def read_stdin(command_queue): +async def read_stdin(): """Read from stdin and queue commands.""" while True: line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline) @@ -155,7 +154,7 @@ async def process_stdin_queue(websocket): await subscribe(websocket, symbol, event) -async def process_message(message, results_path, table_name, limit): +async def process_message(message): """Process the WebSocket message.""" messages = message if isinstance(message, list) else [message] for msg in messages: @@ -168,7 +167,6 @@ async def process_message(message, results_path, table_name, limit): err = f"UnauthorizedError -> {msg.get('message')}" logger.error(err) sys.exit(1) - break logger.info("PROVIDER INFO: %s", msg.get("message")) elif msg and "ev" in msg and "status" not in msg: @@ -183,20 +181,23 @@ async def process_message(message, results_path, table_name, limit): raise e from e if result: - await write_to_db(result, results_path, table_name, limit) + await write_to_db( + result, + kwargs["results_path"], + kwargs["table_name"], + kwargs.get("limit"), + ) else: logger.info("PROVIDER INFO: %s", msg) -async def connect_and_stream(url, symbol, api_key, results_path, table_name, limit): +async def connect_and_stream(): # pylint: disable=too-many-branches, too-many-statements """Connect to the WebSocket and stream data to file.""" handler_task = asyncio.create_task( - queue.process_queue( - lambda message: process_message(message, results_path, table_name, limit) - ) + queue.process_queue(lambda message: process_message(message)) ) - stdin_task = asyncio.create_task(read_stdin(command_queue)) + stdin_task = asyncio.create_task(read_stdin()) try: connect_kwargs = CONNECT_KWARGS.copy() if "ping_timeout" not in connect_kwargs: @@ -205,16 +206,15 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim connect_kwargs["close_timeout"] = None try: - async with websockets.connect(url, **connect_kwargs) as websocket: - - await login(websocket, api_key) + async with websockets.connect(kwargs["url"], **connect_kwargs) as websocket: + await login(websocket, kwargs["api_key"]) response = await websocket.recv() messages = json.loads(response) - await process_message(messages, results_path, table_name, limit) - await subscribe(websocket, symbol, "subscribe") + await process_message(messages) + await subscribe(websocket, kwargs["symbol"], "subscribe") response = await websocket.recv() messages = json.loads(response) - await process_message(messages, results_path, table_name, limit) + await process_message(messages) while True: cmd_task = asyncio.create_task(process_stdin_queue(websocket)) msg_task = asyncio.create_task(websocket.recv()) @@ -255,14 +255,14 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim # Attempt to reopen the connection logger.info("PROVIDER INFO: Attempting to reconnect after five seconds.") await asyncio.sleep(5) - await connect_and_stream(url, symbol, api_key, results_path, table_name, limit) + await connect_and_stream() except websockets.WebSocketException as e: msg = f"PROVIDER ERROR: WebSocketException -> {e}" logger.error(msg) sys.exit(1) - except Exception as e: + except Exception as e: # pylint: disable=broad-except msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e}" logger.error(msg) sys.exit(1) @@ -284,14 +284,7 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim loop.add_signal_handler(sig, handle_termination_signal, logger) asyncio.run_coroutine_threadsafe( - connect_and_stream( - kwargs["url"], - kwargs["symbol"], - kwargs["api_key"], - os.path.abspath(kwargs["results_file"]), - kwargs["table_name"], - kwargs.get("limit", None), - ), + connect_and_stream(), loop, ) loop.run_forever() @@ -300,8 +293,8 @@ async def connect_and_stream(url, symbol, api_key, results_path, table_name, lim logger.error("PROVIDER ERROR: WebSocket connection closed") except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: {e.__class__.__name__} -> {e}" - logger.error(msg) + ERR = f"PROVIDER ERROR: {e.__class__.__name__} -> {e}" + logger.error(ERR) finally: sys.exit(0) From 02d92cddf56e723ec15c5b6ff8f41c35d2c77053 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:57:56 -0800 Subject: [PATCH 066/119] are we sure now --- .../polygon/openbb_polygon/models/websocket_connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 305b779da146..f5a76aa73eaa 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -1200,7 +1200,7 @@ def __new__(cls, **data): or MODEL_MAP.get(data.get("type", "")) ) if not model: - return super().__new__(cls) # type: ignore + return super().__new__(cls) # pylint: disable=E1120 return model.model_validate(data) # type: ignore From 7935d6802b59cd1da2fca5ffdf301c984b9fe7eb Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 9 Dec 2024 19:31:18 -0800 Subject: [PATCH 067/119] and some more.. --- .../websockets/openbb_websockets/client.py | 17 +++++++----- .../models/websocket_connection.py | 10 +++---- .../openbb_polygon/utils/websocket_client.py | 26 +++++++++---------- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index ac9b258e34f4..d73a4a32bbda 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -237,6 +237,8 @@ def _log_provider_output(self, output_queue) -> None: if "SymbolError" in output: err = ValueError(output) self._exception = err + sys.stdout.write(output + "\n") + sys.stdout.flush() continue # Other errors are logged to stdout and the process is killed. # If the exception is raised by the parent thread, it will be treated as an unexpected error. @@ -262,7 +264,8 @@ def _log_provider_output(self, output_queue) -> None: elif output.startswith("INFO:"): output = output.replace("INFO:", "PROVIDER INFO:") - self.logger.info(output) + sys.stdout.write(output + "\n") + sys.stdout.flush() except queue.Empty: continue @@ -394,9 +397,9 @@ def connect(self) -> None: # pylint: disable=too-many-locals time.sleep(2) if self._exception is not None: - with self._exception as exc: - self._exception = None - raise OpenBBError(exc) + exc = getattr(self, "_exception", None) + self._exception = None + raise OpenBBError(exc) if not self.is_running: self.logger.error("The provider server failed to start.") @@ -722,9 +725,10 @@ def non_blocking_websocket(client, output_queue, provider_message_queue) -> None output_queue.put(output.strip()) except Exception as e: - client.logger.error( + msg = ( f"Unexpected error in non_blocking_websocket: {e.__class__.__name__} -> {e}" ) + client.logger.error(msg) raise e from e finally: client._process.stdout.close() @@ -794,9 +798,10 @@ def non_blocking_broadcast(client, output_queue, broadcast_message_queue) -> Non if output: output_queue.put(output.strip()) except Exception as e: # pylint: disable=broad-except - client.logger.error( + err = ( f"Unexpected error in non_blocking_broadcast: {e.__class__.__name__} -> {e}" ) + client.logger.error(err) finally: client._broadcast_process.stdout.close() client._broadcast_process.wait() diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index f5a76aa73eaa..59e726eb0a32 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -122,8 +122,8 @@ def validate_date(cls, v): v = v / 1e9 # Convert nanoseconds to seconds dt = datetime.fromtimestamp(v, tz=timezone("UTC")) dt = dt.astimezone(timezone("America/New_York")) - - return dt + return dt + return v class PolygonWebSocketQueryParams(WebSocketQueryParams): @@ -353,9 +353,9 @@ def _validate_conditions(cls, v): """Validate the conditions.""" if v is None or isinstance(v, list) and v[0] == 0: return None - if isinstance(v, list) and v[0] == 1: + elif isinstance(v, list) and v[0] == 1: return "sellside" - if isinstance(v, list) and v[0] == 2: + elif isinstance(v, list) and v[0] == 2: return "buyside" return str(v) @@ -1200,7 +1200,7 @@ def __new__(cls, **data): or MODEL_MAP.get(data.get("type", "")) ) if not model: - return super().__new__(cls) # pylint: disable=E1120 + return super().__new__(cls) return model.model_validate(data) # type: ignore diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index c1749b9bb8a6..2a18dc2c5171 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -1,4 +1,4 @@ -"""Polygon WebSocket server.""" +"""Polygon WebSocket client.""" import asyncio import json @@ -30,7 +30,7 @@ CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) FEED = kwargs.pop("feed", None) ASSET_TYPE = kwargs.pop("asset_type", None) -kwargs["results_file"] = os.path.abspath(kwargs["results_file"]) +kwargs["results_file"] = os.path.abspath(kwargs.get("results_file")) async def handle_symbol(symbol): @@ -85,9 +85,8 @@ async def handle_symbol(symbol): return ",".join(new_symbols) -async def login(websocket, api_key): - """Login to the WebSocket.""" - login_event = f'{{"action":"auth","params":"{api_key}"}}' +async def login(websocket): + login_event = f'{{"action":"auth","params":"{kwargs["api_key"]}"}}' try: await websocket.send(login_event) res = await websocket.recv() @@ -108,7 +107,7 @@ async def login(websocket, api_key): logger.error(err) sys.exit(1) logger.info("PROVIDER INFO: %s", msg.get("message")) - except Exception as e: # pylint: disable=broad-except + except Exception as e: logger.error("PROVIDER ERROR: %s -> %s", e.__class__.__name__, e.args[0]) sys.exit(1) @@ -123,7 +122,7 @@ async def subscribe(websocket, symbol, event): subscribe_event = f'{{"action":"{event}","params":"{ticker}"}}' try: await websocket.send(subscribe_event) - except Exception as e: # pylint: disable=broad-except + except Exception as e: msg = f"PROVIDER ERROR: {e.__class__.__name__} -> {e}" logger.error(msg) @@ -183,7 +182,7 @@ async def process_message(message): if result: await write_to_db( result, - kwargs["results_path"], + kwargs["results_file"], kwargs["table_name"], kwargs.get("limit"), ) @@ -191,7 +190,7 @@ async def process_message(message): logger.info("PROVIDER INFO: %s", msg) -async def connect_and_stream(): # pylint: disable=too-many-branches, too-many-statements +async def connect_and_stream(): """Connect to the WebSocket and stream data to file.""" handler_task = asyncio.create_task( @@ -207,7 +206,8 @@ async def connect_and_stream(): # pylint: disable=too-many-branches, too-many-s try: async with websockets.connect(kwargs["url"], **connect_kwargs) as websocket: - await login(websocket, kwargs["api_key"]) + + await login(websocket) response = await websocket.recv() messages = json.loads(response) await process_message(messages) @@ -262,7 +262,7 @@ async def connect_and_stream(): # pylint: disable=too-many-branches, too-many-s logger.error(msg) sys.exit(1) - except Exception as e: # pylint: disable=broad-except + except Exception as e: msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e}" logger.error(msg) sys.exit(1) @@ -293,8 +293,8 @@ async def connect_and_stream(): # pylint: disable=too-many-branches, too-many-s logger.error("PROVIDER ERROR: WebSocket connection closed") except Exception as e: # pylint: disable=broad-except - ERR = f"PROVIDER ERROR: {e.__class__.__name__} -> {e}" - logger.error(ERR) + msg = f"PROVIDER ERROR: {e.__class__.__name__} -> {e}" + logger.error(msg) finally: sys.exit(0) From a40c84530462c3d95dd0b57e287870a4ea51b81a Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:21:23 -0800 Subject: [PATCH 068/119] define message field as JSON in SQL --- .../websockets/openbb_websockets/broadcast.py | 2 +- .../websockets/openbb_websockets/client.py | 2 +- .../websockets/openbb_websockets/helpers.py | 89 +++++++++++-------- .../models/websocket_connection.py | 37 ++++++-- .../openbb_intrinio/utils/references.py | 38 +++++++- .../openbb_intrinio/utils/websocket_client.py | 16 +++- .../openbb_polygon/utils/websocket_client.py | 4 +- 7 files changed, 134 insertions(+), 54 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py index dee00bd9f4fe..5c23de4a18d8 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -229,7 +229,7 @@ async def broadcast(self, message: str): disconnected_clients = set() for client in connected_clients.copy(): try: - await client.websocket.send_text(message) + await client.websocket.send_json(message) except WebSocketDisconnect: disconnected_clients.add(client) except Exception as e: # pylint: disable=broad-except diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index d73a4a32bbda..bfd9c6f0cc19 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -519,7 +519,7 @@ def results(self) -> Union[list[dict], list["Data"], None]: elif isinstance(message, dict): output.append(self.data_model(**message)) else: - output.append(json.loads(json.loads(message))) + output.append(json.loads(message)) except Exception as e: raise OpenBBError(f"Error retrieving results: {e}") from e if output: diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index c2888b7585f5..914e8b70cfeb 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -174,15 +174,18 @@ async def setup_database(results_path, table_name): if os.path.exists(results_path): try: await conn.execute("SELECT name FROM sqlite_master WHERE type='table';") - except aiosqlite.DatabaseError: - os.remove(results_path) - + except aiosqlite.DatabaseError as e: + raise OpenBBError( + "Unexpected error caused by an invalid SQLite database file." + "Please check the path, and inspect the file if it exists." + + f" -> {e}" + ) async with aiosqlite.connect(results_path) as conn: await conn.execute( f""" CREATE TABLE IF NOT EXISTS {table_name} ( id INTEGER PRIMARY KEY AUTOINCREMENT, - message TEXT + message JSON ) """ ) @@ -197,44 +200,46 @@ async def write_to_db(message, results_path, table_name, limit): conn = await aiosqlite.connect(results_path) - # Check if the table exists and create it if it doesn't - await conn.execute( - f""" - CREATE TABLE IF NOT EXISTS {table_name} ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - message TEXT - ) - """ - ) - await conn.commit() - - await conn.execute( - f"INSERT INTO {table_name} (message) VALUES (?)", # noqa - (json.dumps(message),), - ) - await conn.commit() - - records = await conn.execute(f"SELECT COUNT(*) FROM {table_name}") # noqa - count = (await records.fetchone())[0] - count = await conn.execute(f"SELECT COUNT(*) FROM {table_name}") # noqa - current_count = int((await count.fetchone())[0]) - limit = 0 if limit is None else int(limit) - - if current_count > limit and limit != 0: + try: + # Check if the table exists and create it if it doesn't await conn.execute( f""" - DELETE FROM {table_name} - WHERE id IN ( - SELECT id FROM {table_name} - ORDER BY id DESC - LIMIT -1 OFFSET ? + CREATE TABLE IF NOT EXISTS {table_name} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message JSON ) - """, # noqa: S608 - (limit,), + """ ) + await conn.commit() - await conn.commit() - await conn.close() + await conn.execute( + f"INSERT INTO {table_name} (message) VALUES (?)", # noqa + (message,), + ) + await conn.commit() + + records = await conn.execute(f"SELECT COUNT(*) FROM {table_name}") # noqa + count = (await records.fetchone())[0] + count = await conn.execute(f"SELECT COUNT(*) FROM {table_name}") # noqa + current_count = int((await count.fetchone())[0]) + limit = 0 if limit is None else int(limit) + + if current_count > limit and limit != 0: + await conn.execute( + f""" + DELETE FROM {table_name} + WHERE id IN ( + SELECT id FROM {table_name} + ORDER BY id DESC + LIMIT -1 OFFSET ? + ) + """, # noqa: S608 + (limit,), + ) + + await conn.commit() + finally: + await conn.close() class StdOutSink: @@ -271,7 +276,13 @@ def format(self, record): class MessageQueue: """Async message queue for the WebSocket connection.""" - def __init__(self, max_size: int = 1000, max_retries=5, backoff_factor=0.5): + def __init__( + self, + max_size: int = 5000, + max_retries=5, + backoff_factor=0.5, + logger: Optional[logging.Logger] = None, + ): """Initialize the MessageQueue.""" # pylint: disable=import-outside-toplevel from asyncio import Queue @@ -279,7 +290,7 @@ def __init__(self, max_size: int = 1000, max_retries=5, backoff_factor=0.5): self.queue: Queue = Queue(maxsize=max_size) self.max_retries = max_retries self.backoff_factor = backoff_factor - self.logger = get_logger("openbb.websocket.queue") + self.logger = logger if logger else get_logger("openbb.websocket.queue") async def dequeue(self): """Dequeue a message.""" diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py index b3b2c853337f..f4b8b05f1a53 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py @@ -7,13 +7,14 @@ from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.abstract.fetcher import Fetcher +from openbb_intrinio.utils.references import TRADE_CONDITIONS, VENUES from openbb_websockets.client import WebSocketClient from openbb_websockets.models import ( WebSocketConnection, WebSocketData, WebSocketQueryParams, ) -from pydantic import Field, field_validator +from pydantic import Field, field_validator, model_validator class IntrinioWebSocketQueryParams(WebSocketQueryParams): @@ -78,7 +79,7 @@ class IntrinioWebSocketData(WebSocketData): default=None, description="The total volume of the trade or quote.", ) - condition: Optional[str] = Field( + conditions: Optional[str] = Field( default=None, description="The condition attached to the trade or quote.", ) @@ -103,11 +104,35 @@ def _validate_date(cls, v): return dt.astimezone(timezone("America/New_York")) - @field_validator("condition", mode="before", check_fields=False) + @field_validator("exchange", mode="before", check_fields=False) @classmethod - def _validate_condition(cls, v): - """Strip the empty spaces from the condition.""" - return str(v).strip().replace(" ", "") if v else None + def _validate_exchange(cls, v): + """Validate the exchange.""" + return VENUES.get(v, v) + + @model_validator(mode="before") + @classmethod + def _validate_conditions(cls, values): + """Validate the exchange.""" + new_values = values.copy() + conditions = new_values.pop("condition", None) + trade_type = new_values.get("type") + if trade_type == "trade": + + if not conditions: + return new_values + + new_conditions = [] + conditions = conditions.replace(" ", "") + for char in range(len(conditions)): + if trade_type == "trade": + new_conditions.append( + TRADE_CONDITIONS.get(conditions[char], conditions[char]) + ) + new_values["conditions"] = "; ".join(new_conditions) + else: + new_values["conditions"] = conditions if conditions else None + return new_values class IntrinioWebSocketConnection(WebSocketConnection): diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/references.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/references.py index 23c15c14c6c4..79131bf8f76c 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/references.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/references.py @@ -36,10 +36,46 @@ "X": "NASDAQ OMX PSX, Inc. LLC", "Y": "Bats BYX Exchange, Inc.", "Z": "Bats BZX Exchange, Inc.", - "u": "Other OTC Markets", + "U": "Other OTC Markets", } +TRADE_CONDITIONS = { + "@": "Regular Sale", + "A": "Acquisition", + "B": "Bunched Trade", + "C": "Cash Sale", + "D": "Distribution", + "E": "Placeholder", + "F": "Intermarket Sweep", + "G": "Bunched Sold Trade", + "H": "Priced Variation Trade", + "I": "Odd Lot Trade", + "K": "Rule 155 Trade (AMEX)", + "L": "Sold Last", + "M": "Market Center Official Close", + "N": "Next Day", + "O": "Opening Prints", + "P": "Prior Reference Price", + "Q": "Market Center Official Open", + "R": "Seller", + "S": "Split Trade", + "T": "Form T", + "U": "Extended Trading Hours (Sold Out of Sequence)", + "V": "Contingent Trade", + "W": "Average Price Trade", + "X": "Cross/Periodic Auction Trade", + "Y": "Yellow Flag Regular Trade", + "Z": "Sold (Out of Sequence)", + "1": "Stopped Stock (Regular Trade)", + "4": "Derivatively Priced", + "5": "Re-Opening Prints", + "6": "Closing Prints", + "7": "Qualified Contingent Trade (QCT)", + "8": "Placeholder for 611 Exempt", + "9": "Corrected Consolidated Close", +} + ETF_EXCHANGES = Literal[ "xnas", "arcx", diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py index 62b942331295..9020afb8f87d 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -22,7 +22,7 @@ logger = get_logger("openbb.websocket.intrinio") kwargs = parse_kwargs() -command_queue = MessageQueue() +command_queue = MessageQueue(logger=logger) CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) @@ -34,10 +34,15 @@ async def process_message(message): if hasattr(message, "__dict__"): message = message.__dict__ if is_trade: - message["type"] = "trade" if is_trade else "quote" + message["type"] = "trade" try: - result = IntrinioWebSocketData.model_validate(message).model_dump_json() + result = IntrinioWebSocketData.model_validate(message) + result = ( + {} + if result.exchange == "!" or result.price == 0 + else result.model_dump_json() + ) except ValidationError as e: try: handle_validation_error(logger, e) @@ -45,7 +50,10 @@ async def process_message(message): raise e from e if result: await write_to_db( - result, kwargs["results_file"], kwargs["table_name"], kwargs["limit"] + result, + kwargs["results_file"], + kwargs["table_name"], + kwargs["limit"], ) diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index 2a18dc2c5171..97f6ea998f2d 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -23,8 +23,8 @@ from pydantic import ValidationError logger = get_logger("openbb.websocket.polygon") -queue = MessageQueue() -command_queue = MessageQueue() +queue = MessageQueue(logger=logger) +command_queue = MessageQueue(logger=logger) kwargs = parse_kwargs() CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) From 8394ddfffab9d95674f4bd75f6bfe6946834661e Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:24:36 -0800 Subject: [PATCH 069/119] one more file for that commit.. --- .../providers/tiingo/openbb_tiingo/utils/websocket_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index a384962dfe97..463f4ebcf407 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -69,8 +69,8 @@ "ask_price", ] SUBSCRIPTION_ID = "" -queue = MessageQueue() logger = get_logger("openbb.websocket.tiingo") +queue = MessageQueue(logger=logger) kwargs = parse_kwargs() CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) kwargs["results_file"] = os.path.abspath(kwargs["results_file"]) From 8cea1fbac65781625bbdb649bcbfcd729de72baa Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:57:27 -0800 Subject: [PATCH 070/119] lint --- .../intrinio/openbb_intrinio/utils/websocket_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py index 9020afb8f87d..788c7001d57c 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -38,10 +38,10 @@ async def process_message(message): try: result = IntrinioWebSocketData.model_validate(message) - result = ( + result = ( # type: ignore {} - if result.exchange == "!" or result.price == 0 - else result.model_dump_json() + if result.exchange == "!" or result.price == 0 # type: ignore + else result.model_dump_json() # type: ignore ) except ValidationError as e: try: From 489c7fd63fdd91540e2d182d63289ad96f55c584 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 10 Dec 2024 18:00:23 -0800 Subject: [PATCH 071/119] weird that wasn't already caught.. --- .../providers/polygon/openbb_polygon/utils/websocket_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index 97f6ea998f2d..ecb51e9265df 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -86,6 +86,7 @@ async def handle_symbol(symbol): async def login(websocket): + """Login to the WebSocket.""" login_event = f'{{"action":"auth","params":"{kwargs["api_key"]}"}}' try: await websocket.send(login_event) From 0834fc458c6d67d1b511ff6ff46e26c3b9e98f55 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:39:39 -0800 Subject: [PATCH 072/119] some intrinio updates --- .../models/websocket_connection.py | 11 ++++- .../openbb_intrinio/utils/references.py | 29 ++++++------ .../openbb_intrinio/utils/stocks_client.py | 46 ++++++++++++++++++- .../openbb_intrinio/utils/websocket_client.py | 14 ++---- 4 files changed, 72 insertions(+), 28 deletions(-) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py index f4b8b05f1a53..c970c264eb2f 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py @@ -83,6 +83,10 @@ class IntrinioWebSocketData(WebSocketData): default=None, description="The condition attached to the trade or quote.", ) + is_darkpool: Optional[bool] = Field( + default=None, + description="Flag if the trade is reported from an unlit venue.", + ) @field_validator("date", mode="before", check_fields=False) @classmethod @@ -115,23 +119,26 @@ def _validate_exchange(cls, v): def _validate_conditions(cls, values): """Validate the exchange.""" new_values = values.copy() - conditions = new_values.pop("condition", None) trade_type = new_values.get("type") - if trade_type == "trade": + conditions = new_values.pop("condition", None) + if trade_type == "trade": if not conditions: return new_values new_conditions = [] conditions = conditions.replace(" ", "") + for char in range(len(conditions)): if trade_type == "trade": new_conditions.append( TRADE_CONDITIONS.get(conditions[char], conditions[char]) ) + new_values["conditions"] = "; ".join(new_conditions) else: new_values["conditions"] = conditions if conditions else None + return new_values diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/references.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/references.py index 79131bf8f76c..eb3e957096da 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/references.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/references.py @@ -18,24 +18,25 @@ ] VENUES = { - "A": "NYSE MKT LLC", - "B": "NASDAQ OMX BX, Inc.", - "C": "National Stock Exchange Inc. (NSX)", + "A": "NYSE MKT", + "B": "NASDAQ OMX BX", + "C": "National Stock Exchange", "D": "FINRA ADF", - "I": "International Securities Exchange, LLC", - "J": "Bats EDGA Exchange, INC", - "K": "Bats EDGX Exchange, Inc.", - "M": "Chicago Stock Exchange, Inc. (CHX)", - "N": "New York Stock Exchange LLC", - "P": "NYSE Arca, Inc.", + "I": "International Securities Exchange", + "J": "Bats EDGA Exchange", + "K": "Bats EDGX Exchange", + "L": "Long-term Stock Exchange", + "M": "Chicago Stock Exchange", + "N": "New York Stock Exchange", + "P": "NYSE Arca", "S": "Consolidated Tape System", "T": "NASDAQ (Tape A, B securities)", "Q": "NASDAQ (Tape C securities)", - "V": "The Investors' Exchange, LLC (IEX)", - "W": "Chicago Broad Options Exchange, Inc. (CBOE)", - "X": "NASDAQ OMX PSX, Inc. LLC", - "Y": "Bats BYX Exchange, Inc.", - "Z": "Bats BZX Exchange, Inc.", + "V": "The Investors Exchange", + "W": "Cboe", + "X": "NASDAQ OMX PSX", + "Y": "Bats BYX Exchange", + "Z": "Bats BZX Exchange", "U": "Other OTC Markets", } diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py index aeb2f31bf60a..76e3dc2a7efd 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py @@ -11,11 +11,13 @@ # This file is a slightly modified version of the original file from the Intrinio Python SDK. +import json import logging import queue import struct import sys import threading + import time from typing import Any, Dict, Optional @@ -67,7 +69,7 @@ def __init__( self.timestamp = timestamp self.subprovider = subprovider self.market_center = market_center - self.condition = str(condition.strip().replace(" ", "")) + self.condition = condition def __str__(self): """Return string representation of the quote.""" @@ -89,6 +91,30 @@ def __str__(self): + str(self.condition) ) + def is_darkpool(self): + """Return True if the trade is a dark pool trade.""" + return ( + not self.market_center + or self.market_center in ("D", "E", "\x00") + or self.market_center.strip() == "" + ) + + def to_json(self): + return json.dumps( + dict( + symbol=self.symbol, + type=self.type, + price=self.price, + size=self.size, + total_volume=None, + timestamp=self.timestamp, + subprovider=self.subprovider, + market_center=self.market_center, + condition=self.condition, + is_darkpool=self.is_darkpool() if self.is_darkpool() else None, + ) + ) + class Trade: """Intrinio Realtime Stocks Trade.""" @@ -140,6 +166,22 @@ def is_darkpool(self): or self.market_center.strip() == "" ) + def to_json(self): + return json.dumps( + dict( + symbol=self.symbol, + type="trade", + price=self.price, + size=self.size, + total_volume=self.total_volume, + timestamp=self.timestamp, + subprovider=self.subprovider, + market_center=self.market_center, + condition=self.condition, + is_darkpool=self.is_darkpool() if self.is_darkpool() else None, + ) + ) + class IntrinioRealtimeClient: """Intrinio Realtime Stocks Client.""" @@ -160,7 +202,7 @@ def __init__( self.password = options.get("password") self.provider = options.get("provider") self.ipaddress = options.get("ipaddress") - self.tradesonly = None + self.tradesonly = options.get("tradesonly") self.bypass_parsing = options.get("bypass_parsing", False) if "channels" in options: diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py index 788c7001d57c..9596f4248bd8 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -6,10 +6,11 @@ import json import signal import sys +from typing import Any from openbb_core.provider.utils.helpers import run_async from openbb_intrinio.models.websocket_connection import IntrinioWebSocketData -from openbb_intrinio.utils.stocks_client import IntrinioRealtimeClient, Trade +from openbb_intrinio.utils.stocks_client import IntrinioRealtimeClient from openbb_websockets.helpers import ( MessageQueue, get_logger, @@ -28,16 +29,9 @@ async def process_message(message): """Process the message and write to the database.""" - result: dict = {} - message = json.loads(message) if isinstance(message, str) else message - is_trade = isinstance(message, Trade) - if hasattr(message, "__dict__"): - message = message.__dict__ - if is_trade: - message["type"] = "trade" - + result: Any = None try: - result = IntrinioWebSocketData.model_validate(message) + result = IntrinioWebSocketData.model_validate_json(message.to_json()) result = ( # type: ignore {} if result.exchange == "!" or result.price == 0 # type: ignore From c465374e68e1a78812a1054352caffcc4e746116 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 11 Dec 2024 13:04:03 -0800 Subject: [PATCH 073/119] linting --- .../extensions/websockets/openbb_websockets/helpers.py | 8 ++++++-- .../openbb_intrinio/models/websocket_connection.py | 4 +++- .../polygon/openbb_polygon/models/websocket_connection.py | 6 +++--- .../polygon/openbb_polygon/utils/websocket_client.py | 5 +++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 914e8b70cfeb..9c19fb8ae33c 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -179,7 +179,7 @@ async def setup_database(results_path, table_name): "Unexpected error caused by an invalid SQLite database file." "Please check the path, and inspect the file if it exists." + f" -> {e}" - ) + ) from e async with aiosqlite.connect(results_path) as conn: await conn.execute( f""" @@ -195,7 +195,6 @@ async def setup_database(results_path, table_name): async def write_to_db(message, results_path, table_name, limit): """Write the WebSocket message to the SQLite database.""" # pylint: disable=import-outside-toplevel - import json # noqa import aiosqlite conn = await aiosqlite.connect(results_path) @@ -238,6 +237,11 @@ async def write_to_db(message, results_path, table_name, limit): ) await conn.commit() + except Exception as e: # pylint: disable=broad-except + raise OpenBBError( + f"Unexpected error encountered while inserting message into the database. -> {e.__class__.__name__}: {e}" + ) from e + finally: await conn.close() diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py index c970c264eb2f..4f1d385fdf5a 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py @@ -129,7 +129,9 @@ def _validate_conditions(cls, values): new_conditions = [] conditions = conditions.replace(" ", "") - for char in range(len(conditions)): + for char in range( + len(conditions) + ): # pylint: disable=consider-using-enumerate if trade_type == "trade": new_conditions.append( TRADE_CONDITIONS.get(conditions[char], conditions[char]) diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 59e726eb0a32..2594dd33c1f4 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -353,9 +353,9 @@ def _validate_conditions(cls, v): """Validate the conditions.""" if v is None or isinstance(v, list) and v[0] == 0: return None - elif isinstance(v, list) and v[0] == 1: + if isinstance(v, list) and v[0] == 1: return "sellside" - elif isinstance(v, list) and v[0] == 2: + if isinstance(v, list) and v[0] == 2: return "buyside" return str(v) @@ -1200,7 +1200,7 @@ def __new__(cls, **data): or MODEL_MAP.get(data.get("type", "")) ) if not model: - return super().__new__(cls) + return super().__new__(cls) # pylint: disable=E1120 return model.model_validate(data) # type: ignore diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index ecb51e9265df..4daab5892a6a 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -294,8 +294,9 @@ async def connect_and_stream(): logger.error("PROVIDER ERROR: WebSocket connection closed") except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: {e.__class__.__name__} -> {e}" - logger.error(msg) + ERR = f"PROVIDER ERROR: {e.__class__.__name__} -> {e}" + logger.error(ERR) finally: + loop.stop() sys.exit(0) From 01e08aae67673dd6110c8524f5082ea2f6b3e7c7 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 11 Dec 2024 14:36:22 -0800 Subject: [PATCH 074/119] missing doctstring --- .../intrinio/openbb_intrinio/utils/stocks_client.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py index 76e3dc2a7efd..ed2e6803c2a4 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/stocks_client.py @@ -100,6 +100,7 @@ def is_darkpool(self): ) def to_json(self): + """Return the quote as a JSON string.""" return json.dumps( dict( symbol=self.symbol, @@ -167,6 +168,7 @@ def is_darkpool(self): ) def to_json(self): + """Return the trade as a JSON string.""" return json.dumps( dict( symbol=self.symbol, @@ -341,7 +343,7 @@ def connect(self): self.refresh_websocket() connected = True except Exception as e: - self.logger.error(f"Cannot connect: {repr(e)}") + self.logger.error(f"Unexpected error while connecting -> {repr(e)}") self.do_backoff() def disconnect(self): @@ -367,7 +369,9 @@ def refresh_token(self): ) if response.status_code != 200: - raise UnauthorizedError("Auth failed") + raise UnauthorizedError( + f"""Connection failed with status code {response.status_code} and message "{response.reason}".""" + ) self.token = response.text self.logger.info("INFO: Authentication successful!") From 8cffe6b755f7405b0c453edc789e6ea2c5603f26 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 11 Dec 2024 14:56:25 -0800 Subject: [PATCH 075/119] pylint --- .../intrinio/openbb_intrinio/models/websocket_connection.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py index 4f1d385fdf5a..c845fa0172da 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py @@ -129,9 +129,7 @@ def _validate_conditions(cls, values): new_conditions = [] conditions = conditions.replace(" ", "") - for char in range( - len(conditions) - ): # pylint: disable=consider-using-enumerate + for char in range(len(conditions)): # pylint: C0200 if trade_type == "trade": new_conditions.append( TRADE_CONDITIONS.get(conditions[char], conditions[char]) From 58ece30f8ad5516b3e74feefacc9891d086e1ffc Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 11 Dec 2024 15:04:08 -0800 Subject: [PATCH 076/119] pylint.. --- .../intrinio/openbb_intrinio/models/websocket_connection.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py index c845fa0172da..1672ccdbd35a 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py @@ -129,7 +129,9 @@ def _validate_conditions(cls, values): new_conditions = [] conditions = conditions.replace(" ", "") - for char in range(len(conditions)): # pylint: C0200 + for char in range( # pylint: disable=consider-using-enumerate + len(conditions) + ): if trade_type == "trade": new_conditions.append( TRADE_CONDITIONS.get(conditions[char], conditions[char]) From 052899210cb7bf4fbb00b57f282ca4378e37efdd Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:20:17 -0800 Subject: [PATCH 077/119] better and more practical unit fetcher tests --- .../websockets/openbb_websockets/client.py | 1 + .../websockets/openbb_websockets/helpers.py | 7 +- .../test_fmp_websocket_fetcher.json | 1 - .../test_fmp_websocket_fetcher.json | 1 - .../providers/fmp/tests/test_fmp_fetchers.py | 147 +++++++++++- .../intrinio/tests/test_intrinio_fetchers.py | 194 ++++++++++++++++ .../test_polygon_websocket_fetcher.json | 1 - .../test_polygon_websocket_fetcher.json | 1 - .../polygon/tests/test_polygon_fetchers.py | 215 +++++++++++++++--- .../test_tiingo_websocket_fetcher.json | 1 - .../test_tiingo_websocket_fetcher.json | 1 - .../tiingo/tests/test_tiingo_fetchers.py | 167 +++++++++++--- 12 files changed, 665 insertions(+), 72 deletions(-) delete mode 100644 openbb_platform/providers/fmp/tests/record/object_hash/test_fmp_fetchers/test_fmp_websocket_fetcher.json delete mode 100644 openbb_platform/providers/fmp/tests/record/screen/test_fmp_fetchers/test_fmp_websocket_fetcher.json delete mode 100644 openbb_platform/providers/polygon/tests/record/object_hash/test_polygon_fetchers/test_polygon_websocket_fetcher.json delete mode 100644 openbb_platform/providers/polygon/tests/record/screen/test_polygon_fetchers/test_polygon_websocket_fetcher.json delete mode 100644 openbb_platform/providers/tiingo/tests/record/object_hash/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json delete mode 100644 openbb_platform/providers/tiingo/tests/record/screen/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index bfd9c6f0cc19..2c669035d565 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -537,6 +537,7 @@ def results(self): try: with sqlite3.connect(self.results_path) as conn: + conn.execute("PRAGMA journal_mode=WAL;") conn.execute(f"DELETE FROM {self.table_name}") # noqa conn.commit() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 9c19fb8ae33c..62d7c36b9fda 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -170,8 +170,8 @@ async def setup_database(results_path, table_name): import os # noqa import aiosqlite - async with aiosqlite.connect(results_path) as conn: - if os.path.exists(results_path): + if os.path.exists(results_path): + async with aiosqlite.connect(results_path) as conn: try: await conn.execute("SELECT name FROM sqlite_master WHERE type='table';") except aiosqlite.DatabaseError as e: @@ -180,6 +180,7 @@ async def setup_database(results_path, table_name): "Please check the path, and inspect the file if it exists." + f" -> {e}" ) from e + async with aiosqlite.connect(results_path) as conn: await conn.execute( f""" @@ -198,7 +199,7 @@ async def write_to_db(message, results_path, table_name, limit): import aiosqlite conn = await aiosqlite.connect(results_path) - + await conn.execute("PRAGMA journal_mode=WAL;") try: # Check if the table exists and create it if it doesn't await conn.execute( diff --git a/openbb_platform/providers/fmp/tests/record/object_hash/test_fmp_fetchers/test_fmp_websocket_fetcher.json b/openbb_platform/providers/fmp/tests/record/object_hash/test_fmp_fetchers/test_fmp_websocket_fetcher.json deleted file mode 100644 index c4434157cff3..000000000000 --- a/openbb_platform/providers/fmp/tests/record/object_hash/test_fmp_fetchers/test_fmp_websocket_fetcher.json +++ /dev/null @@ -1 +0,0 @@ -["b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b", "3dd2e537120ffb2eaa71d091dc92478f3ce8aa403bcd17c12fff1c5194a16118", "fcbcf165908dd18a9e49f7ff27810176db8e9f63b4352213741664245224f8aa"] \ No newline at end of file diff --git a/openbb_platform/providers/fmp/tests/record/screen/test_fmp_fetchers/test_fmp_websocket_fetcher.json b/openbb_platform/providers/fmp/tests/record/screen/test_fmp_fetchers/test_fmp_websocket_fetcher.json deleted file mode 100644 index 43bd419589e7..000000000000 --- a/openbb_platform/providers/fmp/tests/record/screen/test_fmp_fetchers/test_fmp_websocket_fetcher.json +++ /dev/null @@ -1 +0,0 @@ -{"out": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "err": "PROVIDER INFO: Authenticated\n\nPROVIDER INFO: Subscribed to btcusd\n\nDisconnected from the provider WebSocket.\n\n"} \ No newline at end of file diff --git a/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py b/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py index 35e6c952db8d..6386a4c7b3c2 100644 --- a/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py +++ b/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py @@ -1,7 +1,9 @@ """Unit tests for FMP provider modules.""" import re +import time from datetime import date +from unittest.mock import MagicMock, patch import pytest from openbb_core.app.service.user_service import UserService @@ -45,6 +47,7 @@ from openbb_fmp.models.financial_ratios import FMPFinancialRatiosFetcher from openbb_fmp.models.forward_ebitda_estimates import FMPForwardEbitdaEstimatesFetcher from openbb_fmp.models.forward_eps_estimates import FMPForwardEpsEstimatesFetcher +from openbb_fmp.models.government_trades import FMPGovernmentTradesFetcher from openbb_fmp.models.historical_dividends import FMPHistoricalDividendsFetcher from openbb_fmp.models.historical_employees import FMPHistoricalEmployeesFetcher from openbb_fmp.models.historical_eps import FMPHistoricalEpsFetcher @@ -69,16 +72,96 @@ from openbb_fmp.models.risk_premium import FMPRiskPremiumFetcher from openbb_fmp.models.share_statistics import FMPShareStatisticsFetcher from openbb_fmp.models.treasury_rates import FMPTreasuryRatesFetcher -from openbb_fmp.models.websocket_connection import FmpWebSocketFetcher +from openbb_fmp.models.websocket_connection import ( + FmpWebSocketConnection, + FmpWebSocketData, + FmpWebSocketFetcher, +) from openbb_fmp.models.world_news import FMPWorldNewsFetcher from openbb_fmp.models.yield_curve import FMPYieldCurveFetcher -from openbb_fmp.models.government_trades import FMPGovernmentTradesFetcher +from openbb_websockets.client import WebSocketClient test_credentials = UserService().default_user_settings.credentials.model_dump( mode="json" ) +MOCK_WEBSOCKET_DATA = [ + { + "date": "2024-12-12T21:13:17.925000-05:00", + "symbol": "BTCUSD", + "exchange": "gdax", + "type": "trade", + "last_price": 99344.20658975664, + "last_size": 1.1701095899999996, + }, + { + "date": "2024-12-12T21:13:19.134000-05:00", + "symbol": "BTCUSD", + "exchange": "gemini", + "type": "trade", + "last_price": 99356.52, + "last_size": 0.01106662, + }, + { + "date": "2024-12-12T21:13:23.017000-05:00", + "symbol": "BTCUSD", + "exchange": "gdax", + "type": "trade", + "last_price": 99359.9798569503, + "last_size": 0.16290354, + }, + { + "date": "2024-12-12T21:13:27.253000-05:00", + "symbol": "BTCUSD", + "exchange": "gemini", + "type": "trade", + "last_price": 99350.54645901076, + "last_size": 0.10452787999999999, + }, + { + "date": "2024-12-12T21:13:28.332000-05:00", + "symbol": "BTCUSD", + "exchange": "gdax", + "type": "trade", + "last_price": 99351.54026716301, + "last_size": 0.11509677, + }, + { + "date": "2024-12-12T21:13:33.842000-05:00", + "symbol": "BTCUSD", + "exchange": "gdax", + "type": "trade", + "last_price": 99353.72826832562, + "last_size": 0.00476533, + }, + { + "date": "2024-12-12T21:13:37.500000-05:00", + "symbol": "BTCUSD", + "exchange": "kraken", + "type": "trade", + "last_price": 99325.1, + "last_size": 0.00302361, + }, + { + "date": "2024-12-12T21:13:38.764000-05:00", + "symbol": "BTCUSD", + "exchange": "gemini", + "type": "trade", + "last_price": 99376.83, + "last_size": 0.00739754, + }, + { + "date": "2024-12-12T21:13:39.310000-05:00", + "symbol": "BTCUSD", + "exchange": "gdax", + "type": "trade", + "last_price": 99363.03061442816, + "last_size": 0.04905618, + }, +] + + def response_filter(response): """Filter the response.""" if "Location" in response["headers"]: @@ -101,6 +184,66 @@ def vcr_config(): } +@pytest.fixture +def mock_websocket_connection(): + """Mock websocket client.""" + + mock_connection = FmpWebSocketConnection( + client=MagicMock( + spec=WebSocketClient( + name="fmp_test", + module="openbb_fmp.utils.websocket_client", + symbol="btcusd", + limit=10, + data_model=FmpWebSocketData, + url="wss://mock.fmp.com/crypto", + api_key="MOCK_API_KEY", + ) + ) + ) + mock_connection.client.is_running = False + mock_results = [] + + def mock_connect(): + mock_connection.client.is_running = True + for data in MOCK_WEBSOCKET_DATA: + mock_results.append(FmpWebSocketData(**data)) + time.sleep(0.1) + + def mock_get_results(): + return mock_results + + mock_connection.client.connect = mock_connect + mock_connection.client.results = mock_get_results + + return mock_connection + + +@pytest.mark.asyncio +async def test_websocket_fetcher( + mock_websocket_connection, credentials=test_credentials +): + """Test websocket fetcher.""" + fetcher = FmpWebSocketFetcher() + params = { + "symbol": "btcusd", + "name": "fmp_test", + "limit": 10, + "asset_type": "crypto", + } + + with patch.object(fetcher, "fetch_data", return_value=mock_websocket_connection): + result = await fetcher.fetch_data(params, credentials) + + # Ensure the client is not running initially + assert not result.client.is_running + assert result.client.results() == [] + result.client.connect() + assert result.client.is_running + assert len(result.client.results()) == len(MOCK_WEBSOCKET_DATA) + assert result.client.results()[0] == FmpWebSocketData(**MOCK_WEBSOCKET_DATA[0]) + + @pytest.mark.record_http def test_fmp_company_filings_fetcher(credentials=test_credentials): """Test FMP company filings fetcher.""" diff --git a/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py b/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py index a10769baafa2..a49177d2c896 100644 --- a/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py +++ b/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py @@ -1,7 +1,9 @@ """Test Intrinio fetchers.""" +import time from datetime import date from unittest import mock +from unittest.mock import MagicMock, patch import pytest from openbb_core.app.service.user_service import UserService @@ -65,13 +67,178 @@ IntrinioSearchAttributesFetcher, ) from openbb_intrinio.models.share_statistics import IntrinioShareStatisticsFetcher +from openbb_intrinio.models.websocket_connection import ( + IntrinioWebSocketConnection, + IntrinioWebSocketData, + IntrinioWebSocketFetcher, +) from openbb_intrinio.models.world_news import IntrinioWorldNewsFetcher +from openbb_websockets.client import WebSocketClient test_credentials = UserService().default_user_settings.credentials.model_dump( mode="json" ) +MOCK_WEBSOCKET_DATA = [ + { + "date": "2024-12-12T18:27:22.943000-05:00", + "symbol": "DJT", + "exchange": "Bats EDGX Exchange", + "type": "trade", + "price": 36.2400016784668, + "size": 155, + "volume": 15968471, + "conditions": "Regular Sale; Form T", + "is_darkpool": None, + "feed": "UTP", + }, + { + "date": "2024-12-12T18:27:22.873000-05:00", + "symbol": "CLF", + "exchange": "NYSE Arca", + "type": "ask", + "price": 11.0, + "size": 4, + "volume": None, + "conditions": None, + "is_darkpool": None, + "feed": "CTA_A", + }, + { + "date": "2024-12-12T18:27:22.873000-05:00", + "symbol": "CLF", + "exchange": "NYSE Arca", + "type": "bid", + "price": 10.960000038146973, + "size": 3, + "volume": None, + "conditions": None, + "is_darkpool": None, + "feed": "CTA_A", + }, + { + "date": "2024-12-12T18:27:22.752000-05:00", + "symbol": "TSLZ", + "exchange": "NYSE Arca", + "type": "ask", + "price": 2.609999895095825, + "size": 223, + "volume": None, + "conditions": None, + "is_darkpool": None, + "feed": "UTP", + }, + { + "date": "2024-12-12T18:27:22.796000-05:00", + "symbol": "QQQM", + "exchange": "NASDAQ (Tape C securities)", + "type": "ask", + "price": 217.67999267578125, + "size": 56, + "volume": None, + "conditions": None, + "is_darkpool": None, + "feed": "UTP", + }, + { + "date": "2024-12-12T18:27:22.796000-05:00", + "symbol": "QQQM", + "exchange": "NASDAQ (Tape C securities)", + "type": "bid", + "price": 217.38999938964844, + "size": 100, + "volume": None, + "conditions": None, + "is_darkpool": None, + "feed": "UTP", + }, + { + "date": "2024-12-12T18:27:22.796000-05:00", + "symbol": "QQQM", + "exchange": "NYSE Arca", + "type": "trade", + "price": 217.5500030517578, + "size": 300, + "volume": 2234058, + "conditions": "Regular Sale; Form T", + "is_darkpool": None, + "feed": "UTP", + }, + { + "date": "2024-12-12T18:27:22.935000-05:00", + "symbol": "LCTX", + "exchange": "Bats BZX Exchange", + "type": "ask", + "price": 0.5356000065803528, + "size": 12, + "volume": None, + "conditions": None, + "is_darkpool": None, + "feed": "CTA_B", + }, + { + "date": "2024-12-12T18:27:22.870000-05:00", + "symbol": "BITU", + "exchange": "Bats EDGX Exchange", + "type": "ask", + "price": 59.5, + "size": 1, + "volume": None, + "conditions": None, + "is_darkpool": None, + "feed": "CTA_B", + }, + { + "date": "2024-12-12T18:27:22.870000-05:00", + "symbol": "BITU", + "exchange": "Bats EDGX Exchange", + "type": "bid", + "price": 58.33000183105469, + "size": 1, + "volume": None, + "conditions": None, + "is_darkpool": None, + "feed": "CTA_B", + }, +] + + +@pytest.fixture +def mock_websocket_connection(): + """Mock websocket client.""" + + mock_connection = IntrinioWebSocketConnection( + client=MagicMock( + spec=WebSocketClient( + name="intrinio_test", + module="openbb_intrinio.utils.websocket_client", + symbol="btcusd", + limit=10, + data_model=IntrinioWebSocketData, + url="wss://mock.intrinio.com/", + api_key="MOCK_API_KEY", + ) + ) + ) + mock_connection.client.is_running = False + mock_results = [] + + def mock_connect(): + mock_connection.client.is_running = True + for data in MOCK_WEBSOCKET_DATA: + mock_results.append(IntrinioWebSocketData(**data)) + time.sleep(0.1) + + def mock_get_results(): + return mock_results + + mock_connection.client.connect = mock_connect + mock_connection.client.results = mock_get_results + + return mock_connection + + @pytest.fixture(scope="module") def vcr_config(): """VCR configuration.""" @@ -99,6 +266,33 @@ def mock_cpu_count(): yield +@pytest.mark.asyncio +async def test_websocket_fetcher( + mock_websocket_connection, credentials=test_credentials +): + """Test websocket fetcher.""" + fetcher = IntrinioWebSocketFetcher() + params = { + "symbol": "*", + "name": "intrinio_test", + "limit": 10, + "asset_type": "stock", + } + + with patch.object(fetcher, "fetch_data", return_value=mock_websocket_connection): + result = await fetcher.fetch_data(params, credentials) + + # Ensure the client is not running initially + assert not result.client.is_running + assert result.client.results() == [] + result.client.connect() + assert result.client.is_running + assert len(result.client.results()) == len(MOCK_WEBSOCKET_DATA) + assert result.client.results()[0] == IntrinioWebSocketData( + **MOCK_WEBSOCKET_DATA[0] + ) + + @pytest.mark.record_http def test_intrinio_equity_historical_fetcher(credentials=test_credentials): """Test equity historical fetcher.""" diff --git a/openbb_platform/providers/polygon/tests/record/object_hash/test_polygon_fetchers/test_polygon_websocket_fetcher.json b/openbb_platform/providers/polygon/tests/record/object_hash/test_polygon_fetchers/test_polygon_websocket_fetcher.json deleted file mode 100644 index bacd124ff3fb..000000000000 --- a/openbb_platform/providers/polygon/tests/record/object_hash/test_polygon_fetchers/test_polygon_websocket_fetcher.json +++ /dev/null @@ -1 +0,0 @@ -["b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b", "1151296bae1bacb7626f9db0e63520e1bc2a193755e1d88ff58ab1f20abcbaed", "fcbcf165908dd18a9e49f7ff27810176db8e9f63b4352213741664245224f8aa"] \ No newline at end of file diff --git a/openbb_platform/providers/polygon/tests/record/screen/test_polygon_fetchers/test_polygon_websocket_fetcher.json b/openbb_platform/providers/polygon/tests/record/screen/test_polygon_fetchers/test_polygon_websocket_fetcher.json deleted file mode 100644 index 0282266fe3e3..000000000000 --- a/openbb_platform/providers/polygon/tests/record/screen/test_polygon_fetchers/test_polygon_websocket_fetcher.json +++ /dev/null @@ -1 +0,0 @@ -{"out": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "err": "PROVIDER INFO: Connected Successfully\n\nPROVIDER INFO: authenticated\n\nPROVIDER INFO: subscribed to: XAS.BTC-USD\n\nDisconnected from the provider WebSocket.\n\n"} \ No newline at end of file diff --git a/openbb_platform/providers/polygon/tests/test_polygon_fetchers.py b/openbb_platform/providers/polygon/tests/test_polygon_fetchers.py index cfae156c4e0d..e62c458c331a 100644 --- a/openbb_platform/providers/polygon/tests/test_polygon_fetchers.py +++ b/openbb_platform/providers/polygon/tests/test_polygon_fetchers.py @@ -1,6 +1,8 @@ """Test the Polygon fetchers.""" +import time from datetime import date +from unittest.mock import MagicMock, patch import pytest from openbb_core.app.service.user_service import UserService @@ -17,13 +19,131 @@ from openbb_polygon.models.index_historical import ( PolygonIndexHistoricalFetcher, ) -from openbb_polygon.models.websocket_connection import PolygonWebSocketFetcher from openbb_polygon.models.market_snapshots import PolygonMarketSnapshotsFetcher +from openbb_polygon.models.websocket_connection import ( + PolygonWebSocketConnection, + PolygonWebSocketData, + PolygonWebSocketFetcher, +) +from openbb_websockets.client import WebSocketClient test_credentials = UserService().default_user_settings.credentials.model_dump( mode="json" ) +MOCK_WEBSOCKET_DATA = [ + { + "date": "2024-12-12T21:04:02-05:00", + "symbol": "BTC-USD", + "type": "XAS", + "open": 99445.07, + "high": 99445.08, + "low": 99445.07, + "close": 99445.08, + "volume": 0.00188791, + "vwap": 99445.0778, + }, + { + "date": "2024-12-12T21:04:03-05:00", + "symbol": "BTC-USD", + "type": "XAS", + "open": 99445.07, + "high": 99445.08, + "low": 99445.07, + "close": 99445.07, + "volume": 0.00670653, + "vwap": 99445.0702, + }, + { + "date": "2024-12-12T21:04:04-05:00", + "symbol": "BTC-USD", + "type": "XAS", + "open": 99445.07, + "high": 99445.07, + "low": 99445.07, + "close": 99445.07, + "volume": 0.00007158, + "vwap": 99445.07, + }, + { + "date": "2024-12-12T21:04:05-05:00", + "symbol": "BTC-USD", + "type": "XAS", + "open": 99445.07, + "high": 99445.07, + "low": 99428.34, + "close": 99428.34, + "volume": 0.6058888, + "vwap": 99441.6093, + }, + { + "date": "2024-12-12T21:04:06-05:00", + "symbol": "BTC-USD", + "type": "XAS", + "open": 99428.33, + "high": 99428.34, + "low": 99428.33, + "close": 99428.34, + "volume": 0.01953452, + "vwap": 99428.3301, + }, + { + "date": "2024-12-12T21:04:07-05:00", + "symbol": "BTC-USD", + "type": "XAS", + "open": 99428.33, + "high": 99428.34, + "low": 99428.33, + "close": 99428.33, + "volume": 0.1214753, + "vwap": 99428.33, + }, + { + "date": "2024-12-12T21:04:08-05:00", + "symbol": "BTC-USD", + "type": "XAS", + "open": 99428.33, + "high": 99435.48, + "low": 99428.33, + "close": 99435.48, + "volume": 0.20089819, + "vwap": 99429.29, + }, + { + "date": "2024-12-12T21:04:09-05:00", + "symbol": "BTC-USD", + "type": "XAS", + "open": 99435.47, + "high": 99435.47, + "low": 99435.02, + "close": 99435.02, + "volume": 0.03098464, + "vwap": 99435.3318, + }, + { + "date": "2024-12-12T21:04:10-05:00", + "symbol": "BTC-USD", + "type": "XAS", + "open": 99445.08, + "high": 99445.08, + "low": 99445.05, + "close": 99445.05, + "volume": 0.00245657, + "vwap": 99445.0502, + }, + { + "date": "2024-12-12T21:04:11-05:00", + "symbol": "BTC-USD", + "type": "XAS", + "open": 99445.08, + "high": 99445.08, + "low": 99440.55, + "close": 99440.55, + "volume": 0.06000562, + "vwap": 99443.6374, + }, +] + @pytest.fixture(scope="module") def vcr_config(): @@ -36,6 +156,69 @@ def vcr_config(): } +@pytest.fixture +def mock_websocket_connection(): + """Mock websocket client.""" + + mock_connection = PolygonWebSocketConnection( + client=MagicMock( + spec=WebSocketClient( + name="polygon_test", + module="openbb_polygon.utils.websocket_client", + symbol="btcusd", + limit=10, + data_model=PolygonWebSocketData, + url="wss://mock.polygon.com/crypto", + api_key="MOCK_API_KEY", + ) + ) + ) + mock_connection.client.is_running = False + mock_results = [] + + def mock_connect(): + mock_connection.client.is_running = True + for data in MOCK_WEBSOCKET_DATA: + mock_results.append(PolygonWebSocketData(**data)) + time.sleep(0.1) + + def mock_get_results(): + return mock_results + + mock_connection.client.connect = mock_connect + mock_connection.client.results = mock_get_results + + return mock_connection + + +@pytest.mark.asyncio +async def test_websocket_fetcher( + mock_websocket_connection, credentials=test_credentials +): + """Test websocket fetcher.""" + fetcher = PolygonWebSocketFetcher() + params = { + "symbol": "btcusd", + "name": "polygon_test", + "limit": 10, + "asset_type": "crypto", + "feed": "aggs_sec", + } + + with patch.object(fetcher, "fetch_data", return_value=mock_websocket_connection): + result = await fetcher.fetch_data(params, credentials) + + # Ensure the client is not running initially + assert not result.client.is_running + assert result.client.results() == [] + result.client.connect() + assert result.client.is_running + assert len(result.client.results()) == len(MOCK_WEBSOCKET_DATA) + assert result.client.results()[0] == PolygonWebSocketData( + **MOCK_WEBSOCKET_DATA[0] + ) + + @pytest.mark.record_http def test_polygon_equity_historical_fetcher(credentials=test_credentials): """Test the Polygon Equity Historical fetcher.""" @@ -171,33 +354,3 @@ def test_polygon_currency_snapshots_fetcher(credentials=test_credentials): fetcher = PolygonCurrencySnapshotsFetcher() result = fetcher.test(params, credentials) assert result is None - - -@pytest.mark.record_verify_screen(hash=True) -@pytest.mark.record_verify_object(hash=False) -def test_polygon_websocket_fetcher(record, credentials=test_credentials): - """Test Polygon Websocket fetcher.""" - import asyncio - import time - - params = { - "symbol": "btcusd", - "name": "polygon_test", - "limit": 10, - "asset_type": "crypto", - "feed": "aggs_sec", - } - - try: - fetcher = PolygonWebSocketFetcher() - response = asyncio.run(fetcher.fetch_data(params, credentials)) - time.sleep(1) - record.add_verify(response.client.is_running) - assert response.client.is_running - time.sleep(1) - assert len(response.client.results) > 0 - record.add_verify(list(response.client.results[0].model_dump().keys())) - finally: - response.client.disconnect() - assert not response.client.is_running - record.add_verify(response.client.is_running) diff --git a/openbb_platform/providers/tiingo/tests/record/object_hash/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json b/openbb_platform/providers/tiingo/tests/record/object_hash/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json deleted file mode 100644 index d46ccbdbefbb..000000000000 --- a/openbb_platform/providers/tiingo/tests/record/object_hash/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json +++ /dev/null @@ -1 +0,0 @@ -["b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b", "22eb8665ee9e73f51d599865e64d3d03b16cda20434665fa481080c201dddab9", "fcbcf165908dd18a9e49f7ff27810176db8e9f63b4352213741664245224f8aa"] \ No newline at end of file diff --git a/openbb_platform/providers/tiingo/tests/record/screen/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json b/openbb_platform/providers/tiingo/tests/record/screen/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json deleted file mode 100644 index 6f58aa2e25a2..000000000000 --- a/openbb_platform/providers/tiingo/tests/record/screen/test_tiingo_fetchers/test_tiingo_websocket_fetcher.json +++ /dev/null @@ -1 +0,0 @@ -{"out": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "err": "PROVIDER INFO: WebSocket connection established.\n\nPROVIDER INFO: Authorization: Success\n\nDisconnected from the provider WebSocket.\n\n"} \ No newline at end of file diff --git a/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py b/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py index a08bbf436400..8fbf39aff7ba 100644 --- a/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py +++ b/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py @@ -1,6 +1,8 @@ """Test Tiingo fetchers.""" +import time from datetime import date +from unittest.mock import MagicMock, patch import pytest from openbb_core.app.service.user_service import UserService @@ -9,13 +11,85 @@ from openbb_tiingo.models.currency_historical import TiingoCurrencyHistoricalFetcher from openbb_tiingo.models.equity_historical import TiingoEquityHistoricalFetcher from openbb_tiingo.models.trailing_dividend_yield import TiingoTrailingDivYieldFetcher -from openbb_tiingo.models.websocket_connection import TiingoWebSocketFetcher +from openbb_tiingo.models.websocket_connection import ( + TiingoWebSocketConnection, + TiingoWebSocketData, + TiingoWebSocketFetcher, +) from openbb_tiingo.models.world_news import TiingoWorldNewsFetcher +from openbb_websockets.client import WebSocketClient test_credentials = UserService().default_user_settings.credentials.model_dump( mode="json" ) +MOCK_WEBSOCKET_DATA = [ + { + "date": "2024-12-12 16:57:50.164993-0500", + "symbol": "BTCUSD", + "type": "trade", + "exchange": "gdax", + "last_price": 99818.08814355676, + "last_size": 0.01445296, + }, + { + "date": "2024-12-12 16:57:50.697317-0500", + "symbol": "BTCUSD", + "type": "trade", + "exchange": "kraken", + "last_price": 99810.1, + "last_size": 6.856e-05, + }, + { + "date": "2024-12-12 16:57:51.119000-0500", + "symbol": "BTCUSD", + "type": "trade", + "exchange": "gemini", + "last_price": 99827.02771283902, + "last_size": 0.19860106, + }, + { + "date": "2024-12-12 16:57:52.573000-0500", + "symbol": "BTCUSD", + "type": "trade", + "exchange": "bitfinex", + "last_price": 99780.0, + "last_size": 8.6e-05, + }, + { + "date": "2024-12-12 16:57:55.187865-0500", + "symbol": "BTCUSD", + "type": "trade", + "exchange": "gdax", + "last_price": 99837.23198173672, + "last_size": 1.50565886, + }, + { + "date": "2024-12-12 16:57:55-0500", + "symbol": "BTCUSD", + "type": "trade", + "exchange": "bitstamp", + "last_price": 99862.0, + "last_size": 0.00212959, + }, + { + "date": "2024-12-12 16:57:57.647609-0500", + "symbol": "BTCUSD", + "type": "trade", + "exchange": "kraken", + "last_price": 99819.24927536234, + "last_size": 0.00207, + }, + { + "date": "2024-12-12 16:58:00.009694-0500", + "symbol": "BTCUSD", + "type": "trade", + "exchange": "gdax", + "last_price": 99838.34191368945, + "last_size": 0.037644090000000005, + }, +] + @pytest.fixture(scope="module") def vcr_config(): @@ -28,6 +102,68 @@ def vcr_config(): } +@pytest.fixture +def mock_websocket_connection(): + """Mock websocket client.""" + + mock_connection = TiingoWebSocketConnection( + client=MagicMock( + spec=WebSocketClient( + name="tiingo_test", + module="openbb_tiingo.utils.websocket_client", + symbol="btcusd", + limit=10, + data_model=TiingoWebSocketData, + url="wss://mock.tiingo.com/iex", + api_key="MOCK_TOKEN", + ) + ) + ) + mock_connection.client.is_running = False + mock_results = [] + + def mock_connect(): + mock_connection.client.is_running = True + for data in MOCK_WEBSOCKET_DATA: + mock_results.append(TiingoWebSocketData(**data)) + time.sleep(0.1) + + def mock_get_results(): + return mock_results + + mock_connection.client.connect = mock_connect + mock_connection.client.results = mock_get_results + + return mock_connection + + +@pytest.mark.asyncio +async def test_websocket_fetcher( + mock_websocket_connection, credentials=test_credentials +): + """Test websocket fetcher.""" + fetcher = TiingoWebSocketFetcher() + params = { + "symbol": "btcusd", + "name": "tiingo_test", + "limit": 10, + "asset_type": "crypto", + } + + with patch.object(fetcher, "fetch_data", return_value=mock_websocket_connection): + result = await fetcher.fetch_data(params, credentials) + + # Ensure the client is not running initially + assert not result.client.is_running + assert result.client.results() == [] + result.client.connect() + assert result.client.is_running + assert len(result.client.results()) == len(MOCK_WEBSOCKET_DATA) + assert result.client.results()[0] == TiingoWebSocketData( + **MOCK_WEBSOCKET_DATA[0] + ) + + @pytest.mark.record_http def test_tiingo_equity_historical_fetcher(credentials=test_credentials): """Test Tiingo equity historical fetcher.""" @@ -98,32 +234,3 @@ def test_tiingo_trailing_div_yield_fetcher(credentials=test_credentials): fetcher = TiingoTrailingDivYieldFetcher() result = fetcher.test(params, credentials) assert result is None - - -@pytest.mark.record_verify_screen(hash=True) -@pytest.mark.record_verify_object(hash=False) -def test_tiingo_websocket_fetcher(record, credentials=test_credentials): - """Test Tiingo Websocket fetcher.""" - import asyncio - import time - - params = { - "symbol": "btcusd", - "name": "tiingo_test", - "limit": 10, - "asset_type": "crypto", - } - - try: - fetcher = TiingoWebSocketFetcher() - response = asyncio.run(fetcher.fetch_data(params, credentials)) - time.sleep(1) - record.add_verify(response.client.is_running) - assert response.client.is_running - time.sleep(1) - assert len(response.client.results) > 0 - record.add_verify(list(response.client.results[0].model_dump().keys())) - finally: - response.client.disconnect() - assert not response.client.is_running - record.add_verify(response.client.is_running) From 7c92fc1c672877768108e0061058c115730d4cf3 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 12 Dec 2024 20:31:50 -0800 Subject: [PATCH 078/119] don't need that version of the fmp unit test --- .../providers/fmp/tests/test_fmp_fetchers.py | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py b/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py index 6386a4c7b3c2..ec5853ca9ca4 100644 --- a/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py +++ b/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py @@ -909,30 +909,6 @@ def test_fmp_historical_market_cap_fetcher(credentials=test_credentials): assert result is None -@pytest.mark.record_verify_screen(hash=True) -@pytest.mark.record_verify_object(hash=False) -def test_fmp_websocket_fetcher(record, credentials=test_credentials): - """Test FMP Websocket fetcher.""" - import asyncio - import time - - params = {"symbol": "btcusd", "name": "test", "limit": 10, "asset_type": "crypto"} - - try: - fetcher = FmpWebSocketFetcher() - response = asyncio.run(fetcher.fetch_data(params, credentials)) - time.sleep(1) - record.add_verify(response.client.is_running) - assert response.client.is_running - time.sleep(1) - assert len(response.client.results) > 0 - record.add_verify(list(response.client.results[0].model_dump().keys())) - finally: - response.client.disconnect() - assert not response.client.is_running - record.add_verify(response.client.is_running) - - @pytest.mark.record_http def test_fmp_government_trades_fetcher(credentials=test_credentials): """Test FMP government trades fetcher. From 4a04f32c2e2016f447bd8f3f99e3875942e1a4cd Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 12 Dec 2024 20:50:30 -0800 Subject: [PATCH 079/119] add intrinio to integration tests --- .../integration/test_websockets_api.py | 60 +++++++++++++++++++ .../integration/test_websockets_python.py | 60 +++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/openbb_platform/extensions/websockets/integration/test_websockets_api.py b/openbb_platform/extensions/websockets/integration/test_websockets_api.py index a01657cecb69..bc8849c7a99e 100644 --- a/openbb_platform/extensions/websockets/integration/test_websockets_api.py +++ b/openbb_platform/extensions/websockets/integration/test_websockets_api.py @@ -81,6 +81,25 @@ def headers(): "connect_kwargs": None, } ), + ( + { + "name": "test_intrinio", + "provider": "intrinio", + "symbol": "spy,qqq,iwm,tsla,nvda", + "asset_type": "stock", + "feed": "realtime", + "auth_token": None, + "results_file": None, + "save_results": False, + "table_name": "records", + "limit": 10, + "sleep_time": 0.25, + "broadcast_host": "0.0.0.0", # noqa: S104 + "broadcast_port": 6666, + "start_broadcast": False, + "connect_kwargs": None, + } + ), ], ) @pytest.mark.integration @@ -115,6 +134,10 @@ def test_websockets_create_connection(params, headers): "name": "test_polygon", "auth_token": None, }, + { + "name": "test_intrinio", + "auth_token": None, + }, ], ) @pytest.mark.integration @@ -144,6 +167,10 @@ def test_websockets_get_results(params, headers): "name": "test_polygon", "auth_token": None, }, + { + "name": "test_intrinio", + "auth_token": None, + }, ], ) @pytest.mark.integration @@ -176,6 +203,11 @@ def test_websockets_clear_results(params, headers): "symbol": "ethusd", "auth_token": None, }, + { + "name": "test_intrinio", + "symbol": "amzn", + "auth_token": None, + }, ], ) @pytest.mark.integration @@ -214,6 +246,13 @@ def test_websockets_subscribe(params, headers): "port": 6668, "uvicorn_kwargs": None, }, + { + "name": "test_intrinio", + "auth_token": None, + "host": "0.0.0.0", # noqa: S104 + "port": 6669, + "uvicorn_kwargs": None, + }, ], ) @pytest.mark.integration @@ -246,6 +285,11 @@ def test_websockets_start_broadcasting(params, headers): "auth_token": None, "symbol": "ethusd", }, + { + "name": "test_intrinio", + "auth_token": None, + "symbol": "amzn", + }, ], ) @pytest.mark.integration @@ -275,6 +319,10 @@ def test_websockets_unsubscribe(params, headers): "name": "test_polygon", "auth_token": None, }, + { + "name": "test_intrinio", + "auth_token": None, + }, ], ) @pytest.mark.integration @@ -304,6 +352,10 @@ def test_websockets_stop_connection(params, headers): "name": "test_polygon", "auth_token": None, }, + { + "name": "test_intrinio", + "auth_token": None, + }, ], ) @pytest.mark.integration @@ -333,6 +385,10 @@ def test_websockets_restart_connection(params, headers): "name": "test_polygon", "auth_token": None, }, + { + "name": "test_intrinio", + "auth_token": None, + }, ], ) @pytest.mark.integration @@ -362,6 +418,10 @@ def test_websockets_stop_broadcasting(params, headers): "name": "test_polygon", "auth_token": None, }, + { + "name": "test_intrinio", + "auth_token": None, + }, ], ) @pytest.mark.integration diff --git a/openbb_platform/extensions/websockets/integration/test_websockets_python.py b/openbb_platform/extensions/websockets/integration/test_websockets_python.py index abd6f27740f6..063857ebf574 100644 --- a/openbb_platform/extensions/websockets/integration/test_websockets_python.py +++ b/openbb_platform/extensions/websockets/integration/test_websockets_python.py @@ -77,6 +77,25 @@ def obb(pytestconfig): "connect_kwargs": None, } ), + ( + { + "name": "test_intrinio", + "provider": "intrinio", + "symbol": "spy,qqq,iwm,tsla,nvda", + "asset_type": "stock", + "feed": "realtime", + "auth_token": None, + "results_file": None, + "save_results": False, + "table_name": "records", + "limit": 10, + "sleep_time": 0.25, + "broadcast_host": "0.0.0.0", # noqa: S104 + "broadcast_port": 6666, + "start_broadcast": False, + "connect_kwargs": None, + } + ), ], ) @pytest.mark.integration @@ -108,6 +127,10 @@ def test_websockets_create_connection(params, obb): "name": "test_polygon", "auth_token": None, }, + { + "name": "test_intrinio", + "auth_token": None, + }, ], ) @pytest.mark.integration @@ -167,6 +190,11 @@ def test_websockets_clear_results(params, obb): "symbol": "ethusd", "auth_token": None, }, + { + "name": "test_intrinio", + "symbol": "amzn", + "auth_token": None, + }, ], ) @pytest.mark.integration @@ -204,6 +232,13 @@ def test_websockets_subscribe(params, obb): "port": None, "uvicorn_kwargs": None, }, + { + "name": "test_intrinio", + "auth_token": None, + "host": "0.0.0.0", # noqa: S104 + "port": None, + "uvicorn_kwargs": None, + }, ], ) @pytest.mark.integration @@ -235,6 +270,11 @@ def test_websockets_start_broadcasting(params, obb): "symbol": "ethusd", "auth_token": None, }, + { + "name": "test_intrinio", + "symbol": "amzn", + "auth_token": None, + }, ], ) @pytest.mark.integration @@ -263,6 +303,10 @@ def test_websockets_unsubscribe(params, obb): "name": "test_polygon", "auth_token": None, }, + { + "name": "test_intrinio", + "auth_token": None, + }, ], ) @pytest.mark.integration @@ -291,6 +335,10 @@ def test_websockets_get_client(params, obb): "name": "test_polygon", "auth_token": None, }, + { + "name": "test_intrinio", + "auth_token": None, + }, ], ) @pytest.mark.integration @@ -319,6 +367,10 @@ def test_websockets_stop_connection(params, obb): "name": "test_polygon", "auth_token": None, }, + { + "name": "test_intrinio", + "auth_token": None, + }, ], ) @pytest.mark.integration @@ -347,6 +399,10 @@ def test_websockets_restart_connection(params, obb): "name": "test_polygon", "auth_token": None, }, + { + "name": "test_intrinio", + "auth_token": None, + }, ], ) @pytest.mark.integration @@ -375,6 +431,10 @@ def test_websockets_stop_broadcasting(params, obb): "name": "test_polygon", "auth_token": None, }, + { + "name": "test_intrinio", + "auth_token": None, + }, ], ) @pytest.mark.integration From aa715cdcf37048052d0f04a8d724cfdacce8602d Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 12 Dec 2024 22:46:17 -0800 Subject: [PATCH 080/119] missing docstring examples --- .../openbb_websockets/websockets_router.py | 90 ++++++++++++++++++- .../models/websocket_connection.py | 4 +- .../models/websocket_connection.py | 4 +- 3 files changed, 92 insertions(+), 6 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index 720061a702f2..a70c1fa0df91 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -8,7 +8,7 @@ from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.app.model.command_context import CommandContext -from openbb_core.app.model.example import APIEx +from openbb_core.app.model.example import APIEx, PythonEx from openbb_core.app.model.obbject import OBBject from openbb_core.app.provider_interface import ( ExtraParams, @@ -43,7 +43,20 @@ "symbol": "btcusd,ethusd,solusd", "start_broadcast": True, } - ) + ), + APIEx( + parameters={ + "name": "client2", + "provider": "polygon", + "asset_type": "stock_delayed", + "feed": "aggs_sec", + "symbol": "*", + "limit": "None", + "results_file": "/path/to/results.db", + "save_results": "True", + "auth_token": "someAuthToken123$", + } + ), ], ) async def create_connection( @@ -98,6 +111,12 @@ async def create_connection( @router.command( methods=["GET"], + examples=[ + PythonEx( + description="Get all written results from a client connection.", + code=["res = obb.websockets.get_results(name='client1')", "res.to_df()"], + ) + ], ) async def get_results(name: str, auth_token: Optional[str] = None) -> OBBject: """Get all recorded results from a client connection. @@ -128,6 +147,12 @@ async def get_results(name: str, auth_token: Optional[str] = None) -> OBBject: @router.command( methods=["GET"], + examples=[ + PythonEx( + description="Clear all results from a client connection database.", + code=["obb.websockets.clear_results(name='client1')"], + ) + ], ) async def clear_results(name: str, auth_token: Optional[str] = None) -> OBBject[str]: """Clear all stored results from a client connection. Does not stop the client or broadcast. @@ -154,6 +179,12 @@ async def clear_results(name: str, auth_token: Optional[str] = None) -> OBBject[ @router.command( methods=["GET"], + examples=[ + PythonEx( + description="Subscribe to a new symbol in an active client connection.", + code=["obb.websockets.subscribe(name='client1', subscribe='ethusd')"], + ) + ], ) async def subscribe( name: str, symbol: str, auth_token: Optional[str] = None @@ -200,6 +231,12 @@ async def subscribe( @router.command( methods=["GET"], + examples=[ + PythonEx( + description="Unsubscribe from a symbol in an active client connection.", + code=["obb.websockets.unsubscribe(name='client1', symbol='btcusd')"], + ) + ], ) async def unsubscribe( name: str, symbol: str, auth_token: Optional[str] = None @@ -238,6 +275,16 @@ async def unsubscribe( @router.command( methods=["GET"], + examples=[ + PythonEx( + description="Get the status of all created clients which have not been killed.", + code=["obb.websockets.get_client_status()"], + ), + PythonEx( + description="Get the status of a specific client.", + code=["obb.websockets.get_client_status(name='client1')"], + ), + ], ) async def get_client_status( name: str = "all", @@ -268,6 +315,13 @@ async def get_client_status( @router.command( methods=["GET"], include_in_schema=False, + examples=[ + PythonEx( + description="Get the Python client object by 'name'." + + " Useful if the local was collected by the Garbage Collector.", + code=["obb.websockets.get_client(name='client1')"], + ), + ], ) async def get_client(name: str, auth_token: Optional[str] = None) -> OBBject: """Get an open client connection object. This endpoint is only available from the Python interface. @@ -292,6 +346,12 @@ async def get_client(name: str, auth_token: Optional[str] = None) -> OBBject: @router.command( methods=["GET"], + examples=[ + PythonEx( + description="Stop the connection to a provider websocket.", + code=["obb.websockets.stop_connection(name='client2')"], + ), + ], ) async def stop_connection( name: str, auth_token: Optional[str] = None @@ -321,6 +381,12 @@ async def stop_connection( @router.command( methods=["GET"], + examples=[ + PythonEx( + description="Restart a stopped connection to a provider websocket.", + code=["obb.websockets.restart_connection(name='client2')"], + ), + ], ) async def restart_connection( name: str, auth_token: Optional[str] = None @@ -357,6 +423,12 @@ async def restart_connection( @router.command( methods=["GET"], + examples=[ + PythonEx( + description="Stop broadcasting the results file.", + code=["obb.websockets.stop_broadcasting(name='client1')"], + ), + ], ) async def stop_broadcasting( name: str, auth_token: Optional[str] = None @@ -402,6 +474,14 @@ async def stop_broadcasting( @router.command( methods=["GET"], + examples=[ + PythonEx( + description="Start broadcasting the results file.", + code=[ + "obb.websockets.start_broadcasting(name='client1', host='0.0.0.0', port=6942)" + ], + ), + ], ) async def start_broadcasting( name: str, @@ -452,6 +532,12 @@ async def start_broadcasting( @router.command( methods=["GET"], + examples=[ + PythonEx( + description="Kill all associated processes with a websocket connection.", + code=["obb.websockets.kill(name='client2')"], + ), + ], ) async def kill(name: str, auth_token: Optional[str] = None) -> OBBject[str]: """Kills a client. diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 2594dd33c1f4..a94daca58c42 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -202,8 +202,8 @@ class PolygonWebSocketQueryParams(WebSocketQueryParams): feed: Literal["aggs_min", "aggs_sec", "trade", "quote", "l2", "fmv", "value"] = ( Field( default="aggs_sec", - description="The feed type to subscribe to. Choose from: aggs_min, aggs_sec, trade, quote, l2, fmv, value" - + "l2 is only available for crypto. value is only available for index.", + description="The asset type associated with the symbol." + + "l2 is only available for crypto, and value is only available for index.", ) ) diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py index 7afe43199189..33c3f1828e2e 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py @@ -99,8 +99,8 @@ class TiingoWebSocketQueryParams(WebSocketQueryParams): ) feed: Literal["trade", "trade_and_quote"] = Field( default="trade_and_quote", - description="The type of data feed to subscribe to. FX only supports quote." - + " Choices are 'trade' or 'trade_and_quote'.", + description="The asset type associated with the symbol. Choices are 'trade' or 'trade_and_quote'." + + " FX only supports quote.", ) From b4c528d3aa4e205164d93e6436475c10bd5d9c2c Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 12 Dec 2024 23:15:07 -0800 Subject: [PATCH 081/119] some fails --- .../integration/test_websockets_api.py | 34 +++++++++++++++++++ .../integration/test_websockets_python.py | 33 ++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/openbb_platform/extensions/websockets/integration/test_websockets_api.py b/openbb_platform/extensions/websockets/integration/test_websockets_api.py index bc8849c7a99e..6d2c15c4b9d6 100644 --- a/openbb_platform/extensions/websockets/integration/test_websockets_api.py +++ b/openbb_platform/extensions/websockets/integration/test_websockets_api.py @@ -88,6 +88,7 @@ def headers(): "symbol": "spy,qqq,iwm,tsla,nvda", "asset_type": "stock", "feed": "realtime", + "trades_only": True, "auth_token": None, "results_file": None, "save_results": False, @@ -304,6 +305,39 @@ def test_websockets_unsubscribe(params, headers): assert result.status_code == 200 +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + { + "name": "test_intrinio", + "auth_token": None, + }, + ], +) +@pytest.mark.skip(reason="Python interface only.") +def test_websockets_get_client(params, headers): + """Test the websockets_get_client endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/websockets/get_client?{query_str}" + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code == 200 + + @parametrize( "params", [ diff --git a/openbb_platform/extensions/websockets/integration/test_websockets_python.py b/openbb_platform/extensions/websockets/integration/test_websockets_python.py index 063857ebf574..9521070b061e 100644 --- a/openbb_platform/extensions/websockets/integration/test_websockets_python.py +++ b/openbb_platform/extensions/websockets/integration/test_websockets_python.py @@ -84,6 +84,7 @@ def obb(pytestconfig): "symbol": "spy,qqq,iwm,tsla,nvda", "asset_type": "stock", "feed": "realtime", + "trades_only": True, "auth_token": None, "results_file": None, "save_results": False, @@ -320,6 +321,38 @@ def test_websockets_get_client(params, obb): assert result.results is not None +@parametrize( + "params", + [ + { + "name": "test_fmp", + "auth_token": None, + }, + { + "name": "test_tiingo", + "auth_token": None, + }, + { + "name": "test_polygon", + "auth_token": None, + }, + { + "name": "test_intrinio", + "auth_token": None, + }, + ], +) +@pytest.mark.integration +def test_websockets_get_client_status(params, obb): + """Test the websockets_get_client endpoint.""" + params = {p: v for p, v in params.items() if v} + + result = obb.websockets.get_client_status(**params) + assert result + assert isinstance(result, OBBject) + assert result.results is not None + + @parametrize( "params", [ From dcfa890608139a7fd64fa422801a23ef0046efd6 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:57:57 -0800 Subject: [PATCH 082/119] move some files around and create unit tests for MessageQueue --- .../provider/utils/websockets/__init__.py | 1 + .../provider/utils/websockets/helpers.py | 278 ++++++++++++++++++ .../utils/websockets/message_queue.py | 62 ++++ .../utils/websockets/test_message_queue.py | 69 +++++ .../fmp/openbb_fmp/utils/websocket_client.py | 6 +- .../openbb_intrinio/utils/websocket_client.py | 8 +- .../openbb_polygon/utils/websocket_client.py | 12 +- .../openbb_tiingo/utils/websocket_client.py | 6 +- 8 files changed, 426 insertions(+), 16 deletions(-) create mode 100644 openbb_platform/core/openbb_core/provider/utils/websockets/__init__.py create mode 100644 openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py create mode 100644 openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py create mode 100644 openbb_platform/core/tests/provider/utils/websockets/test_message_queue.py diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/__init__.py b/openbb_platform/core/openbb_core/provider/utils/websockets/__init__.py new file mode 100644 index 000000000000..41e2a62502a7 --- /dev/null +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/__init__.py @@ -0,0 +1 @@ +"""WebSocket Utilities.""" diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py b/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py new file mode 100644 index 000000000000..66a74e3f0b19 --- /dev/null +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py @@ -0,0 +1,278 @@ +"""WebSocket Helper Functions.""" + +# pylint: disable=protected-access + +import logging +import re +from typing import Any, Optional + +from openbb_core.app.model.abstract.error import OpenBBError +from openbb_core.provider.utils.errors import UnauthorizedError +from pydantic import ValidationError + +AUTH_TOKEN_FILTER = re.compile( + r"(auth_token=)([^&]*)", + re.IGNORECASE | re.MULTILINE, +) + +connected_clients: dict = {} + + +def clean_message(message: str) -> str: + """Clean the message.""" + return AUTH_TOKEN_FILTER.sub(r"\1********", message) + + +def get_logger(name, level=logging.INFO): + """Get a logger instance.""" + # pylint: disable=import-outside-toplevel + import uuid + + logger = logging.getLogger(f"{name}-{uuid.uuid4()}") + handler = logging.StreamHandler() + handler.setLevel(level) + formatter = logging.Formatter("%(message)s\n") + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(level) + + return logger + + +def handle_validation_error(logger: logging.Logger, error: ValidationError): + """Log and raise a Pydantic ValidationError from a provider connection.""" + err = f"{error.__class__.__name__} -> {error.title}: {error.json()}" + logger.error(err) + raise error from error + + +async def get_status(name: Optional[str] = None, client: Optional[Any] = None) -> dict: + """Get the status of a client.""" + if name and name not in connected_clients: + raise OpenBBError(f"Client {name} not connected.") + if not name and not client: + raise OpenBBError("Either name or client must be provided.") + client = client if client else connected_clients[name] + provider_pid = client._psutil_process.pid if client.is_running else None + broadcast_pid = ( + client._psutil_broadcast_process.pid if client.is_broadcasting else None + ) + status = { + "name": client.name, + "auth_required": client._auth_token is not None, + "subscribed_symbols": client.symbol, + "is_running": client.is_running, + "provider_pid": provider_pid, + "is_broadcasting": client.is_broadcasting, + "broadcast_address": client.broadcast_address, + "broadcast_pid": broadcast_pid, + "results_file": client.results_file, + "table_name": client.table_name, + "save_results": client.save_results, + } + return status + + +def encrypt_value(key, iv, value): + """Encrypt a value before storing.""" + # pylint: disable=import-outside-toplevel + import base64 # noqa + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + + backend = default_backend() + cipher = Cipher(algorithms.AES(key), modes.CFB(iv), backend=backend) + encryptor = cipher.encryptor() + encrypted_value = encryptor.update(value.encode()) + encryptor.finalize() + return base64.b64encode(encrypted_value).decode() + + +def decrypt_value(key, iv, encrypted_value): + """Decrypt the value for use.""" + # pylint: disable=import-outside-toplevel + import base64 # noqa + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + + backend = default_backend() + cipher = Cipher(algorithms.AES(key), modes.CFB(iv), backend=backend) + decryptor = cipher.decryptor() + decrypted_value = ( + decryptor.update(base64.b64decode(encrypted_value)) + decryptor.finalize() + ) + return decrypted_value.decode() + + +async def check_auth(name: str, auth_token: Optional[str] = None) -> bool: + """Check the auth token.""" + if name not in connected_clients: + raise OpenBBError(f"Client {name} not connected.") + client = connected_clients[name] + if client._auth_token is None: + return True + if auth_token is None: + raise UnauthorizedError(f"Client authorization token is required for {name}.") + if auth_token != client._decrypt_value(client._auth_token): + raise UnauthorizedError(f"Invalid client authorization token for {name}.") + return True + + +def handle_termination_signal(logger): + """Handle termination signals to ensure graceful shutdown.""" + # pylint: disable=import-outside-toplevel + import sys + + logger.info( + "PROVIDER INFO: Termination signal received. WebSocket connection closed." + ) + sys.exit(0) + + +def parse_kwargs(): + """Parse command line keyword arguments.""" + # pylint: disable=import-outside-toplevel + import json + import sys + + args = sys.argv[1:].copy() + _kwargs: dict = {} + for i, arg in enumerate(args): + if arg.startswith("url"): + _kwargs["url"] = arg[4:] + continue + if "=" in arg: + key, value = arg.split("=") + + if key == "connect_kwargs": + value = {} if value == "None" else json.loads(value) + + _kwargs[key] = value + elif arg.startswith("--"): + key = arg[2:] + + if i + 1 < len(args) and not args[i + 1].startswith("--"): + value = args[i + 1] + + if isinstance(value, str) and value.lower() in ["false", "true"]: + value = value.lower() == "true" # type: ignore + elif isinstance(value, str) and value.lower() == "none": + value = None + _kwargs[key] = value + else: + _kwargs[key] = True + + return _kwargs + + +async def setup_database(results_path, table_name): + """Create the SQLite database, if required.""" + # pylint: disable=import-outside-toplevel + import os # noqa + import aiosqlite + + if os.path.exists(results_path): + async with aiosqlite.connect(results_path) as conn: + try: + await conn.execute("SELECT name FROM sqlite_master WHERE type='table';") + except aiosqlite.DatabaseError as e: + raise OpenBBError( + "Unexpected error caused by an invalid SQLite database file." + "Please check the path, and inspect the file if it exists." + + f" -> {e}" + ) from e + + async with aiosqlite.connect(results_path) as conn: + await conn.execute( + f""" + CREATE TABLE IF NOT EXISTS {table_name} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message JSON + ) + """ + ) + await conn.commit() + + +async def write_to_db(message, results_path, table_name, limit): + """Write the WebSocket message to the SQLite database.""" + # pylint: disable=import-outside-toplevel + import aiosqlite + + conn = await aiosqlite.connect(results_path) + await conn.execute("PRAGMA journal_mode=WAL;") + try: + # Check if the table exists and create it if it doesn't + await conn.execute( + f""" + CREATE TABLE IF NOT EXISTS {table_name} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message JSON + ) + """ + ) + await conn.commit() + + await conn.execute( + f"INSERT INTO {table_name} (message) VALUES (?)", # noqa + (message,), + ) + await conn.commit() + + records = await conn.execute(f"SELECT COUNT(*) FROM {table_name}") # noqa + count = (await records.fetchone())[0] + count = await conn.execute(f"SELECT COUNT(*) FROM {table_name}") # noqa + current_count = int((await count.fetchone())[0]) + limit = 0 if limit is None else int(limit) + + if current_count > limit and limit != 0: + await conn.execute( + f""" + DELETE FROM {table_name} + WHERE id IN ( + SELECT id FROM {table_name} + ORDER BY id DESC + LIMIT -1 OFFSET ? + ) + """, # noqa: S608 + (limit,), + ) + + await conn.commit() + except Exception as e: # pylint: disable=broad-except + raise OpenBBError( + f"Unexpected error encountered while inserting message into the database. -> {e.__class__.__name__}: {e}" + ) from e + + finally: + await conn.close() + + +class StdOutSink: + """Filter stdout for PII.""" + + def write(self, message): + """Write to stdout.""" + # pylint: disable=import-outside-toplevel + import sys + + cleaned_message = AUTH_TOKEN_FILTER.sub(r"\1********", message) + if cleaned_message != message: + cleaned_message = f"{cleaned_message}\n" + sys.__stdout__.write(cleaned_message) # type: ignore + + def flush(self): + """Flush stdout.""" + # pylint: disable=import-outside-toplevel + import sys + + sys.__stdout__.flush() # type: ignore + + +class AuthTokenFilter(logging.Formatter): + """Custom logging formatter to filter auth tokens.""" + + def format(self, record): + """Format the log record.""" + original_message = super().format(record) + cleaned_message = AUTH_TOKEN_FILTER.sub(r"\1********", original_message) + return cleaned_message diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py b/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py new file mode 100644 index 000000000000..f626af8b51bf --- /dev/null +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py @@ -0,0 +1,62 @@ +"""Async WebSocket Message Queue.""" + +import logging +from typing import Optional + + +class MessageQueue: + """Async message queue for the WebSocket connection.""" + + def __init__( + self, + max_size: int = 5000, + max_retries=5, + backoff_factor=0.5, + logger: Optional[logging.Logger] = None, + ): + """Initialize the MessageQueue.""" + # pylint: disable=import-outside-toplevel + from asyncio import Queue # noqa + from openbb_core.provider.utils.websockets.helpers import get_logger + + self.queue: Queue = Queue(maxsize=max_size) + self.max_retries = max_retries + self.backoff_factor = backoff_factor + self.logger = ( + logger + if logger + else get_logger("openbb.websocket.queue", level=logging.WARN) + ) + + async def dequeue(self): + """Dequeue a message.""" + return await self.queue.get() + + async def enqueue(self, message): + """Enqueue a message.""" + # pylint: disable=import-outside-toplevel + from asyncio import sleep + + retries = 0 + while retries < self.max_retries: + if self.queue.full(): + retries += 1 + msg = f"Queue is full. Retrying {retries}/{self.max_retries}..." + self.logger.warning(msg) + await sleep(self.backoff_factor * retries) + else: + await self.queue.put(message) + return + + self.logger.warn("Failed to enqueue message after maximum retries.") + + async def process_queue(self, handler): + """Process the message queue.""" + while not self.queue.empty(): + message = await self.queue.get() + await self._process_message(message, handler) + self.queue.task_done() + + async def _process_message(self, message, handler): + """Process the message with the handler coroutine.""" + await handler(message) diff --git a/openbb_platform/core/tests/provider/utils/websockets/test_message_queue.py b/openbb_platform/core/tests/provider/utils/websockets/test_message_queue.py new file mode 100644 index 000000000000..ba0c6102cbfc --- /dev/null +++ b/openbb_platform/core/tests/provider/utils/websockets/test_message_queue.py @@ -0,0 +1,69 @@ +"""Unit Tests For MessageQueue Class.""" + +import asyncio + +import pytest +from openbb_core.provider.utils.websockets.message_queue import MessageQueue + +MOCK_MESSAGES = [ + {"message": "test1"}, + {"message": "test2"}, + {"message": "test3"}, +] + + +@pytest.fixture +def message_queue(): + """Return a MessageQueue instance.""" + return MessageQueue(max_size=2, max_retries=2, backoff_factor=0.1) + + +@pytest.mark.asyncio +async def test_enqueue_dequeue(message_queue): + """Test the enqueue and dequeue methods.""" + await message_queue.enqueue(MOCK_MESSAGES[0]) + assert not message_queue.queue.empty() + await message_queue.enqueue(MOCK_MESSAGES[1]) + assert message_queue.queue.qsize() == 2 + assert await message_queue.dequeue() == MOCK_MESSAGES[0] + assert message_queue.queue.qsize() == 1 + assert await message_queue.dequeue() == MOCK_MESSAGES[1] + assert message_queue.queue.empty() + + +@pytest.mark.asyncio +async def test_enqueue_full(message_queue): + """Test the enqueue method when the queue is full.""" + await message_queue.enqueue(MOCK_MESSAGES[0]) + await message_queue.enqueue(MOCK_MESSAGES[1]) + + assert message_queue.queue.full() + + with pytest.warns(Warning): + await message_queue.enqueue(MOCK_MESSAGES[2]) + assert message_queue.queue.qsize() == 2 + + assert await message_queue.dequeue() == MOCK_MESSAGES[0] + assert message_queue.queue.qsize() == 1 + await message_queue.enqueue(MOCK_MESSAGES[2]) + assert await message_queue.dequeue() == MOCK_MESSAGES[1] + assert await message_queue.dequeue() == MOCK_MESSAGES[2] + assert message_queue.queue.empty() + + +@pytest.mark.asyncio +async def test_process_queue(message_queue): + """Test the process_queue method.""" + + NUM_MESSAGES = 0 + + async def handler(message): + """Test handler.""" + nonlocal NUM_MESSAGES + NUM_MESSAGES += 1 + + await message_queue.enqueue(MOCK_MESSAGES[0]) + await message_queue.enqueue(MOCK_MESSAGES[1]) + asyncio.create_task(message_queue.process_queue(handler)) + await asyncio.sleep(0.1) + assert NUM_MESSAGES == 2 diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py index 90a4f53487e2..c28715185a9b 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -8,15 +8,15 @@ import websockets import websockets.exceptions -from openbb_fmp.models.websocket_connection import FmpWebSocketData -from openbb_websockets.helpers import ( - MessageQueue, +from openbb_core.provider.utils.websockets.helpers import ( get_logger, handle_termination_signal, handle_validation_error, parse_kwargs, write_to_db, ) +from openbb_core.provider.utils.websockets.message_queue import MessageQueue +from openbb_fmp.models.websocket_connection import FmpWebSocketData from pydantic import ValidationError logger = get_logger("openbb.websocket.fmp") diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py index 9596f4248bd8..470cc375b4bd 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -9,16 +9,16 @@ from typing import Any from openbb_core.provider.utils.helpers import run_async -from openbb_intrinio.models.websocket_connection import IntrinioWebSocketData -from openbb_intrinio.utils.stocks_client import IntrinioRealtimeClient -from openbb_websockets.helpers import ( - MessageQueue, +from openbb_core.provider.utils.websockets.helpers import ( get_logger, handle_termination_signal, handle_validation_error, parse_kwargs, write_to_db, ) +from openbb_core.provider.utils.websockets.message_queue import MessageQueue +from openbb_intrinio.models.websocket_connection import IntrinioWebSocketData +from openbb_intrinio.utils.stocks_client import IntrinioRealtimeClient from pydantic import ValidationError logger = get_logger("openbb.websocket.intrinio") diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index 4daab5892a6a..a5fe307d682a 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -8,18 +8,18 @@ import websockets import websockets.exceptions -from openbb_polygon.models.websocket_connection import ( - FEED_MAP, - PolygonWebSocketData, -) -from openbb_websockets.helpers import ( - MessageQueue, +from openbb_core.provider.utils.websockets.helpers import ( get_logger, handle_termination_signal, handle_validation_error, parse_kwargs, write_to_db, ) +from openbb_core.provider.utils.websockets.message_queue import MessageQueue +from openbb_polygon.models.websocket_connection import ( + FEED_MAP, + PolygonWebSocketData, +) from pydantic import ValidationError logger = get_logger("openbb.websocket.polygon") diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index 463f4ebcf407..e977d2d31100 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -8,15 +8,15 @@ import websockets from openbb_core.provider.utils.errors import UnauthorizedError -from openbb_tiingo.models.websocket_connection import TiingoWebSocketData -from openbb_websockets.helpers import ( - MessageQueue, +from openbb_core.provider.utils.websockets.helpers import ( get_logger, handle_termination_signal, handle_validation_error, parse_kwargs, write_to_db, ) +from openbb_core.provider.utils.websockets.message_queue import MessageQueue +from openbb_tiingo.models.websocket_connection import TiingoWebSocketData from pydantic import ValidationError # These are the data array definitions. From 9dd226a7163135fb2f1b5d8a49eb7cb122765bae Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:33:42 -0800 Subject: [PATCH 083/119] missing integration test --- .../integration/test_websockets_api.py | 29 +++++++++++++++++++ .../integration/test_websockets_python.py | 4 --- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/openbb_platform/extensions/websockets/integration/test_websockets_api.py b/openbb_platform/extensions/websockets/integration/test_websockets_api.py index 6d2c15c4b9d6..965a7ef6141f 100644 --- a/openbb_platform/extensions/websockets/integration/test_websockets_api.py +++ b/openbb_platform/extensions/websockets/integration/test_websockets_api.py @@ -305,6 +305,35 @@ def test_websockets_unsubscribe(params, headers): assert result.status_code == 200 +@parametrize( + "params", + [ + { + "name": "test_fmp", + }, + { + "name": "test_tiingo", + }, + { + "name": "test_polygon", + }, + { + "name": "test_intrinio", + }, + ], +) +@pytest.mark.integration +def test_websockets_get_client_status(params, headers): + """Test the websockets_get_client_status endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/websockets/get_client?{query_str}" + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code == 200 + + @parametrize( "params", [ diff --git a/openbb_platform/extensions/websockets/integration/test_websockets_python.py b/openbb_platform/extensions/websockets/integration/test_websockets_python.py index 9521070b061e..211f5520229c 100644 --- a/openbb_platform/extensions/websockets/integration/test_websockets_python.py +++ b/openbb_platform/extensions/websockets/integration/test_websockets_python.py @@ -326,19 +326,15 @@ def test_websockets_get_client(params, obb): [ { "name": "test_fmp", - "auth_token": None, }, { "name": "test_tiingo", - "auth_token": None, }, { "name": "test_polygon", - "auth_token": None, }, { "name": "test_intrinio", - "auth_token": None, }, ], ) From ba8c83963e9bb5c1b307ab2120991078e543e7b0 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:57:15 -0800 Subject: [PATCH 084/119] create Database class for handling SQL --- .../provider/utils/websockets/database.py | 424 ++++++++++++++++++ .../provider/utils/websockets/helpers.py | 93 +--- .../utils/websockets/test_database.py | 116 +++++ .../websockets/openbb_websockets/helpers.py | 270 +---------- 4 files changed, 569 insertions(+), 334 deletions(-) create mode 100644 openbb_platform/core/openbb_core/provider/utils/websockets/database.py create mode 100644 openbb_platform/core/tests/provider/utils/websockets/test_database.py diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py new file mode 100644 index 000000000000..2a7feff32e25 --- /dev/null +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -0,0 +1,424 @@ +"""Database module for serialized websockets results.""" + +from typing import TYPE_CHECKING, Any, Optional, Union + +from openbb_core.app.model.abstract.error import OpenBBError +from openbb_core.provider.utils.helpers import run_async + +if TYPE_CHECKING: + import asyncio + import logging + + from pydantic import BaseModel + + +class Database: + """ + Class to read from, and write to, the SQL file using aiosqlite. + + Each write or delete operation uses a new connection context in WAL mode. + + The table always contains two columns: + "id" - an auto-incrementing ID + "message" - a JSON serialized row of data + + If a path is not specified, a temporary file will be created and used. + + The limit parameter can be used to set a maximum number of records to keep in the database. + + If the number of records exceeds the limit, the oldest records will be deleted. + + The "id" column will not be reset, i.e., the numbering will continue despite deletions. + + Parameters + ---------- + results_file : Optional[str] + The full path to the SQLite database file. If not specified, a temporary file will be created. + Each websocket client should have its own database file. + table_name : Optional[str] + The name of the table to write to. Default is "records". + data_model : Optional[BaseModel] + A Pydantic model to validate the JSON data. Default is None. + limit: Optional[int] + The maximum number of records to keep in the database. Default is None. + keep_results: Optional[bool] + Whether to persist the file on disk after the Python session is over. Default is False. + logger : Optional[logging.Logger] + A custom logger to use. If not provided, a new logger will be created. + loop: Optional[asyncio.AbstractEventLoop] + An asyncio event loop. + **kwargs + Additional keyword arguments to pass to the SQLite connection at creation. + + Methods + ------- + write_to_db(message) -> None + Write the WebSocket message to the SQLite database. + fetch_all(limit: Optional[int] = None) -> list + Read the WebSocket message from the SQLite database. + query(sql: str) -> list + Run a SELECT query to the database. + clear_results() -> None + Clear all results from the SQLite database. + """ + + def __init__( + self, + results_file: Optional[str] = None, + table_name: Optional[str] = None, + data_model: Optional["BaseModel"] = None, + limit: Optional[int] = None, + keep_results: Optional[bool] = False, + logger: Optional["logging.Logger"] = None, + loop: Optional["asyncio.AbstractEventLoop"] = None, + **kwargs, + ): + """Initialize the ResultsDB class.""" + # pylint: disable=import-outside-toplevel + import atexit # noqa + import tempfile + from pathlib import Path + from openbb_core.provider.utils.websockets.helpers import get_logger + + self.results_file = None + self.table_exists = False + self._exception = None + self.logger = logger if logger else get_logger("openbb.websocket.database") + + if not results_file: + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + temp_file_path = temp_file.name + self.results_path = Path(temp_file_path).absolute() + self.results_file = temp_file_path + else: + self.results_path = Path(results_file).absolute() + self.results_file = results_file + + self.keep_results = keep_results + self.table_name = table_name if table_name else "records" + self.limit = limit + self.loop = loop + atexit.register(self._atexit) + self.kwargs = kwargs if kwargs else {} + run_async(self._setup_database) + self.data_model = data_model + + async def _setup_database(self): + """Create the SQLite database, if required.""" + # pylint: disable=import-outside-toplevel + import os # noqa + import aiosqlite + + try: + if os.path.exists(self.results_file): + async with aiosqlite.connect( + self.results_file, loop=self.loop, **self.kwargs + ) as conn: + try: + cursor = await conn.execute( + "SELECT name FROM sqlite_master WHERE type='table';" + ) + await cursor.close() + except aiosqlite.DatabaseError as e: + msg = ( + "Unexpected error caused by an invalid SQLite database file." + "Please check the path, and inspect the file if it exists." + + f" -> {e}" + ) + self.logger.error(msg) + self._exception = e + raise OpenBBError(msg) from e + + async with aiosqlite.connect( + self.results_file, loop=self.loop, **self.kwargs + ) as conn: + cursor = await conn.execute( + f""" + CREATE TABLE IF NOT EXISTS {self.table_name} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + symbol TEXT, + message TEXT NOT NULL + ) + """ + ) + await cursor.close() + await conn.commit() + self.table_exists = True + except Exception as e: + msg = f"Unexpected error while creating SQLite database -> {e.__class__.__name__}: {e}" + self.logger.error(msg) + self._exception = e + raise OpenBBError(msg) from e + + def _atexit(self) -> None: + """Clean up the WebSocket client processes at exit.""" + # pylint: disable=import-outside-toplevel + import os + + self._exception = None + if self.keep_results: + self.logger.info("Results database saved to, %s\n", str(self.results_file)) + if os.path.exists(self.results_file) and not self.keep_results: # type: ignore + os.remove(self.results_file) # type: ignore + + async def _write_to_db(self, message) -> None: + """Write the WebSocket message to the SQLite database.""" + # pylint: disable=import-outside-toplevel + import aiosqlite # noqa + import json + + try: + if isinstance(message, bytes): + message = message.decode("utf-8") + + if not isinstance(message, str): + message = json.dumps(message) + + async with aiosqlite.connect( + self.results_file, loop=self.loop, **self.kwargs + ) as conn: + cursor = await conn.execute("PRAGMA journal_mode=WAL;") + await cursor.close() + if not self.table_exists: + cursor = await conn.execute( + f""" + CREATE TABLE IF NOT EXISTS {self.table_name} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message TEXT NOT NULL + ) + """ + ) + await conn.commit() + await cursor.close() + self.table_exists = True + cursor = await conn.execute( + f""" + INSERT INTO {self.table_name} (message) + VALUES (?) + """, # noqa + (message,), + ) + await conn.commit() + await cursor.close() + + if self.limit is not None: + records = await conn.execute( + f"SELECT COUNT(*) FROM {self.table_name}" # noqa + ) + count = (await records.fetchone())[0] + await records.close() + count = await conn.execute( + f"SELECT COUNT(*) FROM {self.table_name}" # noqa + ) + current_count = int((await count.fetchone())[0]) + await count.close() + limit = 0 if int(self.limit) < 0 else int(self.limit) + + if current_count > limit and limit != 0: + del_cursor = await conn.execute( + f""" + DELETE FROM {self.table_name} + WHERE id IN ( + SELECT id FROM {self.table_name} + ORDER BY id DESC + LIMIT -1 OFFSET ? + ) + """, # noqa: S608 + (limit,), + ) + await del_cursor.close() + + await conn.commit() + + except Exception as e: # pylint: disable=broad-except + raise e from e + + def write_to_db(self, message) -> None: + """Write the WebSocket message to the SQLite database.""" + try: + run_async(self._write_to_db, message) + except Exception as e: # pylint: disable=broad-except + msg = f"Unexpected error while writing to SQLite database -> {e.__class__.__name__}: {e}" + self.logger.error(msg) + self._exception = e + raise OpenBBError(msg) from e + + async def _fetch_all(self, limit: Optional[int] = None) -> list: + """Read the WebSocket message from the SQLite database.""" + # pylint: disable=import-outside-toplevel + import aiosqlite # noqa + + try: + rows: list = [] + async with aiosqlite.connect( + self.results_file, loop=self.loop, **self.kwargs + ) as conn: + query = ( + f"SELECT message FROM {self.table_name} ORDER BY id DESC" # noqa + ) + if limit: + query += f" LIMIT {limit}" + async with conn.execute(query) as cursor: + async for row in cursor: + rows.append(self.deserialize_row(row)) + + return rows + + except Exception as e: + raise e from e + + def deserialize_row(self, row: str) -> Union["BaseModel", Any]: + """Deserialize a row from the SQLite database.""" + # pylint: disable=import-outside-toplevel + import json + + try: + return ( + self.data_model.model_validate_json(row[0]) + if self.data_model is not None + else ( + json.loads(row[0]) + if ( + ( + isinstance(row[0], str) + and (row[0].startswith("{") or row[0].startswith("[")) + ) + or isinstance(row[0], bytes) + ) + else row[0] + ) + ) + except Exception as e: + msg = f"Unexpected error while deserializing row -> {e.__class__.__name__}: {e}" + self.logger.error(msg) + self._exception = e + raise OpenBBError(msg) from e + + def fetch_all(self, limit: Optional[int] = None) -> list: + """Fetch all the results from the SQLite database.""" + try: + return run_async(self._fetch_all, limit) + except Exception as e: + msg = f"Unexpected error while reading from SQLite database -> {e.__class__.__name__}: {e}" + self.logger.error(msg) + self._exception = e + raise OpenBBError(msg) from e + + async def _query_db(self, sql) -> list: + """Query the SQLite database.""" + # pylint: disable=import-outside-toplevel + import aiosqlite # noqa + import json + + query = ( + sql + if sql.startswith("SELECT") + else f"SELECT message FROM {self.table_name} WHERE {sql}" # noqa + ) + if not query.endswith(";"): + query += ";" + + if not query.startswith("SELECT"): + raise OpenBBError( + "Operation not allowed. Only 'SELECT' operations allowed." + ) + rows: list = [] + try: + async with aiosqlite.connect( + self.results_file, loop=self.loop, **self.kwargs + ) as conn, conn.execute(query) as cursor: + async for row in cursor: + rows.append( + self.data_model.model_validate_json(row[0]) + if self.data_model is not None + and query.startswith( + f"SELECT message FROM {self.table_name}" # noqa + ) + else ( + json.loads(row[0]) + if ( + ( + isinstance(row[0], str) + and ( + row[0].startswith("{") or row[0].startswith("[") + ) + ) + or isinstance(row[0], bytes) + ) + else row[0] + ) + ) + return rows + except Exception as e: # pylint: disable=broad-except + raise e from e + + def query(self, sql: str) -> list: + """ + Run a SELECT query to the database. + + Begin after WHERE, using the built-in JSON functions, or provide a full query string. + + Parameters + ---------- + sql : str + The SQL query string to run. + + Examples + -------- + # Start the query string after WHERE. + >>> database.query("json_extract (message, '$.price') > 100") + # Or provide a full query string by starting with SELECT. + >>> query = ( + "SELECT json_extract (message, '$.symbol')" + "FROM test_table WHERE json_extract (message, '$.type') = 'trade';" + ) + >>> database.query(query) + >>> + """ + try: + return run_async(self._query_db, sql) + except Exception as e: # pylint: disable=broad-except + msg = f"Unexpected error while querying SQLite database -> {e.__class__.__name__}: {e}" + self.logger.error(msg) + raise OpenBBError(e) from e + + async def _clear_results(self): + """Clear the results from the SQLite database.""" + # pylint: disable=import-outside-toplevel + import aiosqlite + + try: + async with aiosqlite.connect( + self.results_file, loop=self.loop, **self.kwargs + ) as conn: + cursor = await conn.execute("PRAGMA journal_mode=WAL;") + await cursor.close() + cursor = await conn.execute(f"DELETE FROM {self.table_name}") # noqa + await cursor.close() + cursor = await conn.execute( + f""" + CREATE TABLE IF NOT EXISTS {self.table_name} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message TEXT NOT NULL + ) + """ + ) # noqa + await conn.commit() + await cursor.close() + + self.logger.info( + "Results cleared from table, '%s', in %s", + self.table_name, + self.results_file, + ) + except Exception as e: # pylint: disable=broad-except + raise e from e + + def clear_results(self) -> None: + """Clear all results from the SQLite database.""" + try: + run_async(self._clear_results) + except Exception as e: # pylint: disable=broad-except + msg = f"Error clearing results: {e.__class__.__name__}: {e}" + self.logger.error(msg) + raise OpenBBError(msg) from e diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py b/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py index 66a74e3f0b19..2b50875f5cde 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py @@ -4,10 +4,8 @@ import logging import re -from typing import Any, Optional from openbb_core.app.model.abstract.error import OpenBBError -from openbb_core.provider.utils.errors import UnauthorizedError from pydantic import ValidationError AUTH_TOKEN_FILTER = re.compile( @@ -15,8 +13,6 @@ re.IGNORECASE | re.MULTILINE, ) -connected_clients: dict = {} - def clean_message(message: str) -> str: """Clean the message.""" @@ -46,33 +42,6 @@ def handle_validation_error(logger: logging.Logger, error: ValidationError): raise error from error -async def get_status(name: Optional[str] = None, client: Optional[Any] = None) -> dict: - """Get the status of a client.""" - if name and name not in connected_clients: - raise OpenBBError(f"Client {name} not connected.") - if not name and not client: - raise OpenBBError("Either name or client must be provided.") - client = client if client else connected_clients[name] - provider_pid = client._psutil_process.pid if client.is_running else None - broadcast_pid = ( - client._psutil_broadcast_process.pid if client.is_broadcasting else None - ) - status = { - "name": client.name, - "auth_required": client._auth_token is not None, - "subscribed_symbols": client.symbol, - "is_running": client.is_running, - "provider_pid": provider_pid, - "is_broadcasting": client.is_broadcasting, - "broadcast_address": client.broadcast_address, - "broadcast_pid": broadcast_pid, - "results_file": client.results_file, - "table_name": client.table_name, - "save_results": client.save_results, - } - return status - - def encrypt_value(key, iv, value): """Encrypt a value before storing.""" # pylint: disable=import-outside-toplevel @@ -103,20 +72,6 @@ def decrypt_value(key, iv, encrypted_value): return decrypted_value.decode() -async def check_auth(name: str, auth_token: Optional[str] = None) -> bool: - """Check the auth token.""" - if name not in connected_clients: - raise OpenBBError(f"Client {name} not connected.") - client = connected_clients[name] - if client._auth_token is None: - return True - if auth_token is None: - raise UnauthorizedError(f"Client authorization token is required for {name}.") - if auth_token != client._decrypt_value(client._auth_token): - raise UnauthorizedError(f"Invalid client authorization token for {name}.") - return True - - def handle_termination_signal(logger): """Handle termination signals to ensure graceful shutdown.""" # pylint: disable=import-outside-toplevel @@ -128,8 +83,19 @@ def handle_termination_signal(logger): sys.exit(0) -def parse_kwargs(): - """Parse command line keyword arguments.""" +def parse_kwargs() -> dict: + """ + Parse command line keyword arguments supplied to a script file. + + Accepts arguments in the form of `key=value` or `--key value`. + + Keys and values should not contain spaces. + + Returns + ------- + dict + A Python dictionary with the parsed kwargs. + """ # pylint: disable=import-outside-toplevel import json import sys @@ -137,7 +103,7 @@ def parse_kwargs(): args = sys.argv[1:].copy() _kwargs: dict = {} for i, arg in enumerate(args): - if arg.startswith("url"): + if arg.startswith("url") or arg.startswith("uri"): _kwargs["url"] = arg[4:] continue if "=" in arg: @@ -245,34 +211,3 @@ async def write_to_db(message, results_path, table_name, limit): finally: await conn.close() - - -class StdOutSink: - """Filter stdout for PII.""" - - def write(self, message): - """Write to stdout.""" - # pylint: disable=import-outside-toplevel - import sys - - cleaned_message = AUTH_TOKEN_FILTER.sub(r"\1********", message) - if cleaned_message != message: - cleaned_message = f"{cleaned_message}\n" - sys.__stdout__.write(cleaned_message) # type: ignore - - def flush(self): - """Flush stdout.""" - # pylint: disable=import-outside-toplevel - import sys - - sys.__stdout__.flush() # type: ignore - - -class AuthTokenFilter(logging.Formatter): - """Custom logging formatter to filter auth tokens.""" - - def format(self, record): - """Format the log record.""" - original_message = super().format(record) - cleaned_message = AUTH_TOKEN_FILTER.sub(r"\1********", original_message) - return cleaned_message diff --git a/openbb_platform/core/tests/provider/utils/websockets/test_database.py b/openbb_platform/core/tests/provider/utils/websockets/test_database.py new file mode 100644 index 000000000000..ff50d674cc78 --- /dev/null +++ b/openbb_platform/core/tests/provider/utils/websockets/test_database.py @@ -0,0 +1,116 @@ +"""Unit Tests For Database Operations.""" + +import pytest +from openbb_core.provider.utils.websockets.database import Database + +MOCK_MESSAGES = [ + {"type": "trade", "symbol": "test1", "price": 100}, + {"type": "quote", "symbol": "test2", "price": 200}, + {"type": "trade", "symbol": "test3", "price": 300}, +] + + +@pytest.fixture(scope="module") +def database(): + """Return a MessageQueue instance.""" + return Database(table_name="test_table") + + +def test_setup_database(database): + """Test if the database was setup.""" + assert database + assert database.fetch_all() == [] + + +def test_write_to_db(database): + """Test if the database was setup.""" + assert database + database.write_to_db(MOCK_MESSAGES[0]) + assert database.fetch_all()[0] == MOCK_MESSAGES[0] + database.write_to_db(MOCK_MESSAGES[1]) + assert database.fetch_all()[0] == MOCK_MESSAGES[1] + database.write_to_db(MOCK_MESSAGES[2]) + assert database.fetch_all(limit=1)[0] == MOCK_MESSAGES[2] + database.write_to_db(MOCK_MESSAGES[0]) + assert len(database.fetch_all()) == 4 + + +def test_fetch_all(database): + """Test if the database was setup.""" + assert database + assert len(database.fetch_all()) == len(MOCK_MESSAGES) + 1 + + +def test_clear_results(database): + """Test if the database was setup.""" + assert database + assert len(database.fetch_all()) == 4 + database.clear_results() + assert database.fetch_all() == [] + + +def test_multiple_connections(database): + """Test interacting with the database from multiple connections.""" + assert database + assert len(database.fetch_all()) == 0 + database.write_to_db(MOCK_MESSAGES[0]) + assert database.fetch_all()[0] == MOCK_MESSAGES[0] + new_db = Database( + results_file=database.results_file, + table_name=database.table_name, + ) + another_db = Database( + results_file=database.results_file, + table_name="other_table", + ) + assert new_db.fetch_all()[0] == MOCK_MESSAGES[0] + database.write_to_db(MOCK_MESSAGES[1]) + another_db.write_to_db(MOCK_MESSAGES[2]) + assert new_db.fetch_all(limit=1)[0] == MOCK_MESSAGES[1] + assert another_db.fetch_all(limit=1)[0] != new_db.fetch_all(limit=1)[0] + new_db.write_to_db(MOCK_MESSAGES[2]) + assert len(new_db.fetch_all()) == 3 + new_db.clear_results() + assert new_db.fetch_all() == [] + assert database.fetch_all() == [] + assert another_db.fetch_all() == [MOCK_MESSAGES[2]] + + +def test_query_db(database): + """Test querying the database.""" + assert database + assert len(database.fetch_all()) == 0 + for message in MOCK_MESSAGES: + database.write_to_db(message) + assert len(database.fetch_all()) == 3 + query = "json_extract (message, '$.price') > 100" + assert len(database.query(query)) == 2 + query = "json_extract (message, '$.type') == 'quote'" + assert len(database.query(query)) == 1 + query = "SELECT message FROM test_table WHERE json_extract (message, '$.type') = 'trade'" + assert len(database.query(query)) == 2 + query = "SELECT json_extract (message, '$.symbol') FROM test_table WHERE json_extract (message, '$.type') = 'trade'" + assert database.query(query) == ["test1", "test3"] + + +def test_limit(): + """Test if the limit parameter is working and that the auto increment index doesn't reset when cleared.""" + database = Database( + table_name="test_limit_table", + limit=2, + ) + assert database + assert len(database.fetch_all()) == 0 + database.write_to_db(MOCK_MESSAGES[0]) + assert len(database.fetch_all()) == 1 + database.write_to_db(MOCK_MESSAGES[1]) + assert len(database.fetch_all()) == 2 + database.write_to_db(MOCK_MESSAGES[2]) + assert len(database.fetch_all()) == 2 + assert database.fetch_all()[1] == MOCK_MESSAGES[1] + assert database.fetch_all()[0] == MOCK_MESSAGES[2] + database.clear_results() + assert database.fetch_all() == [] + database.write_to_db(MOCK_MESSAGES[0]) + query = "SELECT id FROM test_limit_table" + assert database.query(query)[0] > 3 diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 62d7c36b9fda..889d1eb83e1a 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -1,51 +1,15 @@ """WebSockets helpers.""" -# pylint: disable=protected-access - import logging -import re from typing import Any, Optional from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.utils.errors import UnauthorizedError -from pydantic import ValidationError - -AUTH_TOKEN_FILTER = re.compile( - r"(auth_token=)([^&]*)", - re.IGNORECASE | re.MULTILINE, -) +from openbb_core.provider.utils.websockets.helpers import AUTH_TOKEN_FILTER connected_clients: dict = {} -def clean_message(message: str) -> str: - """Clean the message.""" - return AUTH_TOKEN_FILTER.sub(r"\1********", message) - - -def get_logger(name, level=logging.INFO): - """Get a logger instance.""" - # pylint: disable=import-outside-toplevel - import uuid - - logger = logging.getLogger(f"{name}-{uuid.uuid4()}") - handler = logging.StreamHandler() - handler.setLevel(level) - formatter = logging.Formatter("%(message)s\n") - handler.setFormatter(formatter) - logger.addHandler(handler) - logger.setLevel(level) - - return logger - - -def handle_validation_error(logger: logging.Logger, error: ValidationError): - """Log and raise a Pydantic ValidationError from a provider connection.""" - err = f"{error.__class__.__name__} -> {error.title}: {error.json()}" - logger.error(err) - raise error from error - - async def get_status(name: Optional[str] = None, client: Optional[Any] = None) -> dict: """Get the status of a client.""" if name and name not in connected_clients: @@ -53,13 +17,20 @@ async def get_status(name: Optional[str] = None, client: Optional[Any] = None) - if not name and not client: raise OpenBBError("Either name or client must be provided.") client = client if client else connected_clients[name] - provider_pid = client._psutil_process.pid if client.is_running else None + provider_pid = ( + client._psutil_process.pid # pylint: disable=protected-access + if client.is_running + else None + ) broadcast_pid = ( - client._psutil_broadcast_process.pid if client.is_broadcasting else None + client._psutil_broadcast_process.pid # pylint: disable=protected-access + if client.is_broadcasting + else None ) status = { "name": client.name, - "auth_required": client._auth_token is not None, + "auth_required": client._auth_token # pylint: disable=protected-access + is not None, "subscribed_symbols": client.symbol, "is_running": client.is_running, "provider_pid": provider_pid, @@ -73,180 +44,22 @@ async def get_status(name: Optional[str] = None, client: Optional[Any] = None) - return status -def encrypt_value(key, iv, value): - """Encrypt a value before storing.""" - # pylint: disable=import-outside-toplevel - import base64 # noqa - from cryptography.hazmat.backends import default_backend - from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes - - backend = default_backend() - cipher = Cipher(algorithms.AES(key), modes.CFB(iv), backend=backend) - encryptor = cipher.encryptor() - encrypted_value = encryptor.update(value.encode()) + encryptor.finalize() - return base64.b64encode(encrypted_value).decode() - - -def decrypt_value(key, iv, encrypted_value): - """Decrypt the value for use.""" - # pylint: disable=import-outside-toplevel - import base64 # noqa - from cryptography.hazmat.backends import default_backend - from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes - - backend = default_backend() - cipher = Cipher(algorithms.AES(key), modes.CFB(iv), backend=backend) - decryptor = cipher.decryptor() - decrypted_value = ( - decryptor.update(base64.b64decode(encrypted_value)) + decryptor.finalize() - ) - return decrypted_value.decode() - - async def check_auth(name: str, auth_token: Optional[str] = None) -> bool: """Check the auth token.""" if name not in connected_clients: raise OpenBBError(f"Client {name} not connected.") client = connected_clients[name] - if client._auth_token is None: + if client._auth_token is None: # pylint: disable=protected-access return True if auth_token is None: raise UnauthorizedError(f"Client authorization token is required for {name}.") - if auth_token != client._decrypt_value(client._auth_token): + if auth_token != client._decrypt_value( + client._auth_token # pylint: disable=protected-access + ): raise UnauthorizedError(f"Invalid client authorization token for {name}.") return True -def handle_termination_signal(logger): - """Handle termination signals to ensure graceful shutdown.""" - # pylint: disable=import-outside-toplevel - import sys - - logger.info( - "PROVIDER INFO: Termination signal received. WebSocket connection closed." - ) - sys.exit(0) - - -def parse_kwargs(): - """Parse command line keyword arguments.""" - # pylint: disable=import-outside-toplevel - import json - import sys - - args = sys.argv[1:].copy() - _kwargs: dict = {} - for i, arg in enumerate(args): - if arg.startswith("url"): - _kwargs["url"] = arg[4:] - continue - if "=" in arg: - key, value = arg.split("=") - - if key == "connect_kwargs": - value = {} if value == "None" else json.loads(value) - - _kwargs[key] = value - elif arg.startswith("--"): - key = arg[2:] - - if i + 1 < len(args) and not args[i + 1].startswith("--"): - value = args[i + 1] - - if isinstance(value, str) and value.lower() in ["false", "true"]: - value = value.lower() == "true" # type: ignore - elif isinstance(value, str) and value.lower() == "none": - value = None - _kwargs[key] = value - else: - _kwargs[key] = True - - return _kwargs - - -async def setup_database(results_path, table_name): - """Create the SQLite database, if required.""" - # pylint: disable=import-outside-toplevel - import os # noqa - import aiosqlite - - if os.path.exists(results_path): - async with aiosqlite.connect(results_path) as conn: - try: - await conn.execute("SELECT name FROM sqlite_master WHERE type='table';") - except aiosqlite.DatabaseError as e: - raise OpenBBError( - "Unexpected error caused by an invalid SQLite database file." - "Please check the path, and inspect the file if it exists." - + f" -> {e}" - ) from e - - async with aiosqlite.connect(results_path) as conn: - await conn.execute( - f""" - CREATE TABLE IF NOT EXISTS {table_name} ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - message JSON - ) - """ - ) - await conn.commit() - - -async def write_to_db(message, results_path, table_name, limit): - """Write the WebSocket message to the SQLite database.""" - # pylint: disable=import-outside-toplevel - import aiosqlite - - conn = await aiosqlite.connect(results_path) - await conn.execute("PRAGMA journal_mode=WAL;") - try: - # Check if the table exists and create it if it doesn't - await conn.execute( - f""" - CREATE TABLE IF NOT EXISTS {table_name} ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - message JSON - ) - """ - ) - await conn.commit() - - await conn.execute( - f"INSERT INTO {table_name} (message) VALUES (?)", # noqa - (message,), - ) - await conn.commit() - - records = await conn.execute(f"SELECT COUNT(*) FROM {table_name}") # noqa - count = (await records.fetchone())[0] - count = await conn.execute(f"SELECT COUNT(*) FROM {table_name}") # noqa - current_count = int((await count.fetchone())[0]) - limit = 0 if limit is None else int(limit) - - if current_count > limit and limit != 0: - await conn.execute( - f""" - DELETE FROM {table_name} - WHERE id IN ( - SELECT id FROM {table_name} - ORDER BY id DESC - LIMIT -1 OFFSET ? - ) - """, # noqa: S608 - (limit,), - ) - - await conn.commit() - except Exception as e: # pylint: disable=broad-except - raise OpenBBError( - f"Unexpected error encountered while inserting message into the database. -> {e.__class__.__name__}: {e}" - ) from e - - finally: - await conn.close() - - class StdOutSink: """Filter stdout for PII.""" @@ -276,56 +89,3 @@ def format(self, record): original_message = super().format(record) cleaned_message = AUTH_TOKEN_FILTER.sub(r"\1********", original_message) return cleaned_message - - -class MessageQueue: - """Async message queue for the WebSocket connection.""" - - def __init__( - self, - max_size: int = 5000, - max_retries=5, - backoff_factor=0.5, - logger: Optional[logging.Logger] = None, - ): - """Initialize the MessageQueue.""" - # pylint: disable=import-outside-toplevel - from asyncio import Queue - - self.queue: Queue = Queue(maxsize=max_size) - self.max_retries = max_retries - self.backoff_factor = backoff_factor - self.logger = logger if logger else get_logger("openbb.websocket.queue") - - async def dequeue(self): - """Dequeue a message.""" - return await self.queue.get() - - async def enqueue(self, message): - """Enqueue a message.""" - # pylint: disable=import-outside-toplevel - from asyncio import sleep - from queue import Full - - retries = 0 - while retries < self.max_retries: - try: - await self.queue.put(message) - return - except Full: - retries += 1 - msg = f"Queue is full. Retrying {retries}/{self.max_retries}..." - self.logger.warning(msg) - await sleep(self.backoff_factor * retries) - self.logger.error("Failed to enqueue message after maximum retries.") - - async def process_queue(self, handler): - """Process the message queue.""" - while True: - message = await self.queue.get() - await self._process_message(message, handler) - self.queue.task_done() - - async def _process_message(self, message, handler): - """Process the message with the handler coroutine.""" - await handler(message) From f7faf07ea8032397d588ded1fbc46e42483bc379 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:36:08 -0800 Subject: [PATCH 085/119] fix some imports and make default message queue 10000 --- .../provider/utils/websockets/database.py | 19 +---- .../utils/websockets/message_queue.py | 4 +- .../websockets/openbb_websockets/broadcast.py | 11 +-- .../websockets/openbb_websockets/client.py | 84 ++++++------------- .../websockets/openbb_websockets/listen.py | 5 +- .../fmp/openbb_fmp/utils/websocket_client.py | 18 ++-- .../openbb_intrinio/utils/websocket_client.py | 23 ++--- .../openbb_polygon/utils/websocket_client.py | 16 ++-- .../openbb_tiingo/utils/websocket_client.py | 16 ++-- 9 files changed, 77 insertions(+), 119 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 2a7feff32e25..cefb2545a1e5 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -41,8 +41,6 @@ class Database: A Pydantic model to validate the JSON data. Default is None. limit: Optional[int] The maximum number of records to keep in the database. Default is None. - keep_results: Optional[bool] - Whether to persist the file on disk after the Python session is over. Default is False. logger : Optional[logging.Logger] A custom logger to use. If not provided, a new logger will be created. loop: Optional[asyncio.AbstractEventLoop] @@ -75,8 +73,7 @@ def __init__( ): """Initialize the ResultsDB class.""" # pylint: disable=import-outside-toplevel - import atexit # noqa - import tempfile + import tempfile # noqa from pathlib import Path from openbb_core.provider.utils.websockets.helpers import get_logger @@ -94,11 +91,9 @@ def __init__( self.results_path = Path(results_file).absolute() self.results_file = results_file - self.keep_results = keep_results self.table_name = table_name if table_name else "records" self.limit = limit self.loop = loop - atexit.register(self._atexit) self.kwargs = kwargs if kwargs else {} run_async(self._setup_database) self.data_model = data_model @@ -136,7 +131,6 @@ async def _setup_database(self): f""" CREATE TABLE IF NOT EXISTS {self.table_name} ( id INTEGER PRIMARY KEY AUTOINCREMENT, - symbol TEXT, message TEXT NOT NULL ) """ @@ -150,17 +144,6 @@ async def _setup_database(self): self._exception = e raise OpenBBError(msg) from e - def _atexit(self) -> None: - """Clean up the WebSocket client processes at exit.""" - # pylint: disable=import-outside-toplevel - import os - - self._exception = None - if self.keep_results: - self.logger.info("Results database saved to, %s\n", str(self.results_file)) - if os.path.exists(self.results_file) and not self.keep_results: # type: ignore - os.remove(self.results_file) # type: ignore - async def _write_to_db(self, message) -> None: """Write the WebSocket message to the SQLite database.""" # pylint: disable=import-outside-toplevel diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py b/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py index f626af8b51bf..d54d590e2786 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py @@ -9,7 +9,7 @@ class MessageQueue: def __init__( self, - max_size: int = 5000, + max_size: int = 10000, max_retries=5, backoff_factor=0.5, logger: Optional[logging.Logger] = None, @@ -52,7 +52,7 @@ async def enqueue(self, message): async def process_queue(self, handler): """Process the message queue.""" - while not self.queue.empty(): + while True: message = await self.queue.get() await self._process_message(message, handler) self.queue.task_done() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py index 5c23de4a18d8..a39c75dfe9e4 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -8,10 +8,10 @@ import uvicorn from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from openbb_core.provider.utils.websockets.database import Database +from openbb_core.provider.utils.websockets.helpers import get_logger, parse_kwargs from starlette.websockets import WebSocketState -from openbb_websockets.helpers import get_logger, parse_kwargs, setup_database - connected_clients: set = set() kwargs = parse_kwargs() @@ -25,6 +25,7 @@ SLEEP_TIME = kwargs.pop("sleep_time", None) or 0.25 AUTH_TOKEN = kwargs.pop("auth_token", None) +DATABASE = Database(results_file=RESULTS_FILE, table_name=TABLE_NAME) app = FastAPI() @@ -151,14 +152,14 @@ def __init__( def _encrypt_value(self, value: str) -> str: """Encrypt the value for storage.""" # pylint: disable=import-outside-toplevel - from openbb_websockets.helpers import encrypt_value + from openbb_core.provider.utils.websockets.helpers import encrypt_value return encrypt_value(self._key, self._iv, value) def _decrypt_value(self, value: str) -> str: """Decrypt the value for use.""" # pylint: disable=import-outside-toplevel - from openbb_websockets.helpers import decrypt_value + from openbb_core.provider.utils.websockets.helpers import decrypt_value return decrypt_value(self._key, self._iv, value) @@ -294,5 +295,5 @@ def main(): # pylint: disable=import-outside-toplevel from openbb_core.provider.utils.helpers import run_async - run_async(setup_database, RESULTS_FILE, TABLE_NAME) + run_async(DATABASE._setup_database) main() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 2c669035d565..1b14d3dce886 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -99,12 +99,14 @@ def __init__( # noqa: PLR0913 # pylint: disable=too-many-arguments,too-many-po import os import tempfile import threading - from aiosqlite import DatabaseError from queue import Queue from pathlib import Path from openbb_core.app.model.abstract.error import OpenBBError - from openbb_websockets.helpers import get_logger - from openbb_websockets.helpers import encrypt_value + from openbb_core.provider.utils.websockets.database import Database + from openbb_core.provider.utils.websockets.helpers import ( + encrypt_value, + get_logger, + ) self.name = name self.module = module.replace(".py", "") # type: ignore @@ -156,8 +158,15 @@ def __init__( # noqa: PLR0913 # pylint: disable=too-many-arguments,too-many-po atexit.register(self._atexit) try: - self._setup_database() - except DatabaseError as e: + self.database = Database( + results_file=self.results_file, + table_name=self.table_name, + limit=self._limit, + keep_results=self.save_results, + logger=self.logger, + data_model=self.data_model, + ) + except Exception as e: # pylint: disable=broad-except msg = f"Unexpected error setting up the SQLite database and table -> {e.__class___.__name__}: {e}" self.logger.error(msg) self._exception = OpenBBError(msg) @@ -178,14 +187,6 @@ def _atexit(self) -> None: if os.path.exists(self.results_file) and not self.save_results: # type: ignore os.remove(self.results_file) # type: ignore - def _setup_database(self) -> None: - """Set up the SQLite database and table.""" - # pylint: disable=import-outside-toplevel - from openbb_core.provider.utils.helpers import run_async # noqa - from openbb_websockets.helpers import setup_database - - run_async(setup_database, self.results_path, self.table_name) - def _log_provider_output(self, output_queue) -> None: """Log output from the provider logger, handling exceptions, errors, and messages that are not data.""" # pylint: disable=import-outside-toplevel @@ -193,7 +194,7 @@ def _log_provider_output(self, output_queue) -> None: import queue import sys from openbb_core.provider.utils.errors import UnauthorizedError - from openbb_websockets.helpers import clean_message + from openbb_core.provider.utils.websockets.helpers import clean_message from pydantic import ValidationError while not self._stop_log_thread_event.is_set(): @@ -274,7 +275,7 @@ def _log_broadcast_output(self, output_queue) -> None: # pylint: disable=import-outside-toplevel import queue # noqa import sys - from openbb_websockets.helpers import clean_message + from openbb_core.provider.utils.websockets.helpers import clean_message while not self._stop_broadcasting_event.is_set(): try: @@ -321,7 +322,7 @@ def connect(self) -> None: # pylint: disable=too-many-locals import threading import time from openbb_core.app.model.abstract.error import OpenBBError - from openbb_websockets.helpers import decrypt_value + from openbb_core.provider.utils.websockets.helpers import decrypt_value if self.is_running: self.logger.info("Provider connection already running.") @@ -495,59 +496,22 @@ def is_broadcasting(self) -> bool: return False @property - def results(self) -> Union[list[dict], list["Data"], None]: + def results(self) -> list: """Retrieve the deserialized results from the results file.""" # pylint: disable=import-outside-toplevel - import json # noqa - import sqlite3 from openbb_core.app.model.abstract.error import OpenBBError - output: list = [] - file_path = self.results_path - if file_path.exists(): - with sqlite3.connect(file_path) as conn: - try: - cursor = conn.execute(f"SELECT * FROM {self.table_name}") # noqa - for row in cursor: - _, message = row - if self.data_model: - message = json.loads(message) - if isinstance(message, (str, bytes)): - output.append( - self.data_model.model_validate_json(message) - ) - elif isinstance(message, dict): - output.append(self.data_model(**message)) - else: - output.append(json.loads(message)) - except Exception as e: - raise OpenBBError(f"Error retrieving results: {e}") from e - if output: - return output - - self.logger.info("No results found in %s", self.results_file) - - return None + try: + return self.database.fetch_all() + except Exception as e: # pylint: disable=broad-except + raise OpenBBError(f"Error retrieving results: {e}") from e @results.deleter def results(self): """Clear results stored from the WebSocket stream.""" # pylint: disable=import-outside-toplevel - import sqlite3 # noqa - try: - with sqlite3.connect(self.results_path) as conn: - conn.execute("PRAGMA journal_mode=WAL;") - conn.execute(f"DELETE FROM {self.table_name}") # noqa - conn.commit() - - self._setup_database() - - self.logger.info( - "Results cleared from table %s in %s", - self.table_name, - self.results_file, - ) + self.database.clear_results() except Exception as e: # pylint: disable=broad-except msg = f"Error clearing results: {e.__class__.__name__}: {e}" self.logger.error(msg) @@ -608,7 +572,7 @@ def start_broadcasting( # pylint: disable=too-many-locals import psutil import queue from openbb_platform_api.utils.api import check_port - from openbb_websockets.helpers import decrypt_value + from openbb_core.provider.utils.websockets.helpers import decrypt_value if ( self._broadcast_process is not None diff --git a/openbb_platform/extensions/websockets/openbb_websockets/listen.py b/openbb_platform/extensions/websockets/openbb_websockets/listen.py index 75e20bed126f..329dc745828e 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/listen.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/listen.py @@ -25,7 +25,10 @@ async def listen( # noqa: PLR0915 # pylint: disable=too-many-branches,too-many import websockets from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.utils.errors import UnauthorizedError - from openbb_websockets.helpers import clean_message, get_logger + from openbb_core.provider.utils.websockets.helpers import ( + clean_message, + get_logger, + ) from websockets.exceptions import InvalidStatusCode kwargs = kwargs or {} diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py index c28715185a9b..67b06be549f3 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -2,18 +2,17 @@ import asyncio import json -import os import signal import sys import websockets import websockets.exceptions +from openbb_core.provider.utils.websockets.database import Database from openbb_core.provider.utils.websockets.helpers import ( get_logger, handle_termination_signal, handle_validation_error, parse_kwargs, - write_to_db, ) from openbb_core.provider.utils.websockets.message_queue import MessageQueue from openbb_fmp.models.websocket_connection import FmpWebSocketData @@ -24,7 +23,13 @@ queue = MessageQueue() command_queue = MessageQueue() CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) -kwargs["results_file"] = os.path.abspath(kwargs["results_file"]) + +DATABASE = Database( + results_file=kwargs["results_file"], + table_name=kwargs["table_name"], + limit=kwargs.get("limit"), + logger=logger, +) async def login(websocket): @@ -125,12 +130,7 @@ async def process_message(message): except ValidationError: raise e from e if result: - await write_to_db( - result, - kwargs["results_file"], - kwargs["table_name"], - kwargs.get("limit"), - ) + await DATABASE._write_to_db(result) # pylint: disable=protected-access async def connect_and_stream(): diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py index 470cc375b4bd..803787aaeaac 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -9,12 +9,12 @@ from typing import Any from openbb_core.provider.utils.helpers import run_async +from openbb_core.provider.utils.websockets.database import Database from openbb_core.provider.utils.websockets.helpers import ( get_logger, handle_termination_signal, handle_validation_error, parse_kwargs, - write_to_db, ) from openbb_core.provider.utils.websockets.message_queue import MessageQueue from openbb_intrinio.models.websocket_connection import IntrinioWebSocketData @@ -26,8 +26,15 @@ command_queue = MessageQueue(logger=logger) CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) +DATABASE = Database( + results_file=kwargs["results_file"], + table_name=kwargs["table_name"], + limit=kwargs.get("limit"), + logger=logger, +) + -async def process_message(message): +def process_message(message): """Process the message and write to the database.""" result: Any = None try: @@ -43,17 +50,12 @@ async def process_message(message): except ValidationError: raise e from e if result: - await write_to_db( - result, - kwargs["results_file"], - kwargs["table_name"], - kwargs["limit"], - ) + DATABASE.write_to_db(result) def on_message(message, backlog): """Process the message and write to the database.""" - run_async(process_message, message) + process_message(message) options = { @@ -73,6 +75,7 @@ def on_message(message, backlog): async def subscribe(symbol, event): """Subscribe or unsubscribe to a symbol.""" ticker = symbol.split(",") if isinstance(symbol, str) else symbol + ticker = ["lobby"] if "*" in ticker or "LOBBY" in ticker else ticker try: if event == "subscribe": client.join(ticker) @@ -123,7 +126,7 @@ async def connect_and_stream(): try: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - loop.set_exception_handler(lambda loop, context: None) + # loop.set_exception_handler(lambda loop, context: None) for sig in (signal.SIGINT, signal.SIGTERM): loop.add_signal_handler(sig, handle_termination_signal, logger) diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index a5fe307d682a..b6345d32ea8e 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -8,12 +8,12 @@ import websockets import websockets.exceptions +from openbb_core.provider.utils.websockets.database import Database from openbb_core.provider.utils.websockets.helpers import ( get_logger, handle_termination_signal, handle_validation_error, parse_kwargs, - write_to_db, ) from openbb_core.provider.utils.websockets.message_queue import MessageQueue from openbb_polygon.models.websocket_connection import ( @@ -32,6 +32,13 @@ ASSET_TYPE = kwargs.pop("asset_type", None) kwargs["results_file"] = os.path.abspath(kwargs.get("results_file")) +DATABASE = Database( + results_file=kwargs["results_file"], + table_name=kwargs["table_name"], + limit=kwargs.get("limit"), + logger=logger, +) + async def handle_symbol(symbol): """Handle the symbol and map it to the correct format.""" @@ -181,12 +188,7 @@ async def process_message(message): raise e from e if result: - await write_to_db( - result, - kwargs["results_file"], - kwargs["table_name"], - kwargs.get("limit"), - ) + await DATABASE._write_to_db(result) else: logger.info("PROVIDER INFO: %s", msg) diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index e977d2d31100..4a6a17bf5114 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -8,12 +8,12 @@ import websockets from openbb_core.provider.utils.errors import UnauthorizedError +from openbb_core.provider.utils.websockets.database import Database from openbb_core.provider.utils.websockets.helpers import ( get_logger, handle_termination_signal, handle_validation_error, parse_kwargs, - write_to_db, ) from openbb_core.provider.utils.websockets.message_queue import MessageQueue from openbb_tiingo.models.websocket_connection import TiingoWebSocketData @@ -75,6 +75,13 @@ CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) kwargs["results_file"] = os.path.abspath(kwargs["results_file"]) +DATABASE = Database( + results_file=kwargs["results_file"], + table_name=kwargs["table_name"], + limit=kwargs.get("limit"), + logger=logger, +) + # Subscribe and unsubscribe events are handled in a separate connection using the subscription_id set by the login event. async def update_symbols(symbol, event): @@ -189,12 +196,7 @@ async def process_message(message): # pylint: disable=too-many-branches raise e from e if result: - await write_to_db( - result, - kwargs["results_file"], - kwargs["table_name"], - kwargs.get("limit"), - ) + await DATABASE._write_to_db(result) # pylint: disable=protected-access return From 7efdd2e371e7e17fb7208ed4ad63673f0c01666c Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:47:23 -0800 Subject: [PATCH 086/119] don't need those helpers anymore --- .../provider/utils/websockets/helpers.py | 84 ------------------- 1 file changed, 84 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py b/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py index 2b50875f5cde..693faa1fae1a 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py @@ -5,7 +5,6 @@ import logging import re -from openbb_core.app.model.abstract.error import OpenBBError from pydantic import ValidationError AUTH_TOKEN_FILTER = re.compile( @@ -128,86 +127,3 @@ def parse_kwargs() -> dict: _kwargs[key] = True return _kwargs - - -async def setup_database(results_path, table_name): - """Create the SQLite database, if required.""" - # pylint: disable=import-outside-toplevel - import os # noqa - import aiosqlite - - if os.path.exists(results_path): - async with aiosqlite.connect(results_path) as conn: - try: - await conn.execute("SELECT name FROM sqlite_master WHERE type='table';") - except aiosqlite.DatabaseError as e: - raise OpenBBError( - "Unexpected error caused by an invalid SQLite database file." - "Please check the path, and inspect the file if it exists." - + f" -> {e}" - ) from e - - async with aiosqlite.connect(results_path) as conn: - await conn.execute( - f""" - CREATE TABLE IF NOT EXISTS {table_name} ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - message JSON - ) - """ - ) - await conn.commit() - - -async def write_to_db(message, results_path, table_name, limit): - """Write the WebSocket message to the SQLite database.""" - # pylint: disable=import-outside-toplevel - import aiosqlite - - conn = await aiosqlite.connect(results_path) - await conn.execute("PRAGMA journal_mode=WAL;") - try: - # Check if the table exists and create it if it doesn't - await conn.execute( - f""" - CREATE TABLE IF NOT EXISTS {table_name} ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - message JSON - ) - """ - ) - await conn.commit() - - await conn.execute( - f"INSERT INTO {table_name} (message) VALUES (?)", # noqa - (message,), - ) - await conn.commit() - - records = await conn.execute(f"SELECT COUNT(*) FROM {table_name}") # noqa - count = (await records.fetchone())[0] - count = await conn.execute(f"SELECT COUNT(*) FROM {table_name}") # noqa - current_count = int((await count.fetchone())[0]) - limit = 0 if limit is None else int(limit) - - if current_count > limit and limit != 0: - await conn.execute( - f""" - DELETE FROM {table_name} - WHERE id IN ( - SELECT id FROM {table_name} - ORDER BY id DESC - LIMIT -1 OFFSET ? - ) - """, # noqa: S608 - (limit,), - ) - - await conn.commit() - except Exception as e: # pylint: disable=broad-except - raise OpenBBError( - f"Unexpected error encountered while inserting message into the database. -> {e.__class__.__name__}: {e}" - ) from e - - finally: - await conn.close() From a4fda300582b1da19773e893e0b2962daf35ef87 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:56:34 -0800 Subject: [PATCH 087/119] lint --- .../provider/utils/websockets/database.py | 27 ++++++++++++++----- .../websockets/openbb_websockets/client.py | 5 +++- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index cefb2545a1e5..e1aa70bb99a7 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -105,7 +105,7 @@ async def _setup_database(self): import aiosqlite try: - if os.path.exists(self.results_file): + if self.results_file is not None and os.path.exists(self.results_file): # type: ignore async with aiosqlite.connect( self.results_file, loop=self.loop, **self.kwargs ) as conn: @@ -139,7 +139,10 @@ async def _setup_database(self): await conn.commit() self.table_exists = True except Exception as e: - msg = f"Unexpected error while creating SQLite database -> {e.__class__.__name__}: {e}" + msg = ( + "Unexpected error while creating SQLite database ->" + f" {e.__class__.__name__ if hasattr(e, "__class__") else e.__name__}: {e}" + ) self.logger.error(msg) self._exception = e raise OpenBBError(msg) from e @@ -221,7 +224,10 @@ def write_to_db(self, message) -> None: try: run_async(self._write_to_db, message) except Exception as e: # pylint: disable=broad-except - msg = f"Unexpected error while writing to SQLite database -> {e.__class__.__name__}: {e}" + msg = ( + "Unexpected error while writing to SQLite database ->" + f" {e.__class__.__name__ if hasattr(e, "__class__") else e.__name__}: {e}" + ) self.logger.error(msg) self._exception = e raise OpenBBError(msg) from e @@ -272,7 +278,10 @@ def deserialize_row(self, row: str) -> Union["BaseModel", Any]: ) ) except Exception as e: - msg = f"Unexpected error while deserializing row -> {e.__class__.__name__}: {e}" + msg = ( + "Unexpected error while deserializing row -> " + f" {e.__class__.__name__ if hasattr(e, "__class__") else e.__name__}: {e}" + ) self.logger.error(msg) self._exception = e raise OpenBBError(msg) from e @@ -282,7 +291,10 @@ def fetch_all(self, limit: Optional[int] = None) -> list: try: return run_async(self._fetch_all, limit) except Exception as e: - msg = f"Unexpected error while reading from SQLite database -> {e.__class__.__name__}: {e}" + msg = ( + "Unexpected error while reading from SQLite database ->" + f" {e.__class__.__name__ if hasattr(e, "__class__") else e.__name__}: {e}" + ) self.logger.error(msg) self._exception = e raise OpenBBError(msg) from e @@ -402,6 +414,9 @@ def clear_results(self) -> None: try: run_async(self._clear_results) except Exception as e: # pylint: disable=broad-except - msg = f"Error clearing results: {e.__class__.__name__}: {e}" + msg = ( + "Error clearing results: " + f" {e.__class__.__name__ if hasattr(e, "__class__") else e.__name__}: {e}" + ) self.logger.error(msg) raise OpenBBError(msg) from e diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index 1b14d3dce886..f32bb7ce1ec3 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -167,7 +167,10 @@ def __init__( # noqa: PLR0913 # pylint: disable=too-many-arguments,too-many-po data_model=self.data_model, ) except Exception as e: # pylint: disable=broad-except - msg = f"Unexpected error setting up the SQLite database and table -> {e.__class___.__name__}: {e}" + msg = ( + "Unexpected error setting up the SQLite database and table ->" + f" {e.__class__.__name__ if hasattr(e, "__class__") else e.__name__}: {e}" + ) self.logger.error(msg) self._exception = OpenBBError(msg) From fb38ad2d307777c561fc3e3256f1598b0a896792 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 17 Dec 2024 17:05:56 -0800 Subject: [PATCH 088/119] inner quotes --- .../openbb_core/provider/utils/websockets/database.py | 10 +++++----- .../extensions/websockets/openbb_websockets/client.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index e1aa70bb99a7..f711776766cc 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -141,7 +141,7 @@ async def _setup_database(self): except Exception as e: msg = ( "Unexpected error while creating SQLite database ->" - f" {e.__class__.__name__ if hasattr(e, "__class__") else e.__name__}: {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__}: {e}" ) self.logger.error(msg) self._exception = e @@ -226,7 +226,7 @@ def write_to_db(self, message) -> None: except Exception as e: # pylint: disable=broad-except msg = ( "Unexpected error while writing to SQLite database ->" - f" {e.__class__.__name__ if hasattr(e, "__class__") else e.__name__}: {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__}: {e}" ) self.logger.error(msg) self._exception = e @@ -280,7 +280,7 @@ def deserialize_row(self, row: str) -> Union["BaseModel", Any]: except Exception as e: msg = ( "Unexpected error while deserializing row -> " - f" {e.__class__.__name__ if hasattr(e, "__class__") else e.__name__}: {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__}: {e}" ) self.logger.error(msg) self._exception = e @@ -293,7 +293,7 @@ def fetch_all(self, limit: Optional[int] = None) -> list: except Exception as e: msg = ( "Unexpected error while reading from SQLite database ->" - f" {e.__class__.__name__ if hasattr(e, "__class__") else e.__name__}: {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__}: {e}" ) self.logger.error(msg) self._exception = e @@ -416,7 +416,7 @@ def clear_results(self) -> None: except Exception as e: # pylint: disable=broad-except msg = ( "Error clearing results: " - f" {e.__class__.__name__ if hasattr(e, "__class__") else e.__name__}: {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__}: {e}" ) self.logger.error(msg) raise OpenBBError(msg) from e diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/extensions/websockets/openbb_websockets/client.py index f32bb7ce1ec3..82b04254c1cf 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/client.py @@ -169,7 +169,7 @@ def __init__( # noqa: PLR0913 # pylint: disable=too-many-arguments,too-many-po except Exception as e: # pylint: disable=broad-except msg = ( "Unexpected error setting up the SQLite database and table ->" - f" {e.__class__.__name__ if hasattr(e, "__class__") else e.__name__}: {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__}: {e}" ) self.logger.error(msg) self._exception = OpenBBError(msg) From 70caac38e9383d6d2558d83b1ca331978ac32bd6 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 17 Dec 2024 17:11:42 -0800 Subject: [PATCH 089/119] duplicate query --- .../core/openbb_core/provider/utils/websockets/database.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index f711776766cc..4765cc068062 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -188,11 +188,6 @@ async def _write_to_db(self, message) -> None: await cursor.close() if self.limit is not None: - records = await conn.execute( - f"SELECT COUNT(*) FROM {self.table_name}" # noqa - ) - count = (await records.fetchone())[0] - await records.close() count = await conn.execute( f"SELECT COUNT(*) FROM {self.table_name}" # noqa ) From 88978b35bbe8f018a3df2d3b7f15fe5b960fd044 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 17 Dec 2024 23:05:48 -0800 Subject: [PATCH 090/119] move client to core.provider --- .../provider/utils/websockets}/client.py | 258 +++++++++++++----- .../provider/utils/websockets/database.py | 5 +- .../websockets/openbb_websockets/models.py | 3 +- .../openbb_fmp/models/websocket_connection.py | 12 +- .../fmp/openbb_fmp/utils/websocket_client.py | 11 +- .../providers/fmp/tests/test_fmp_fetchers.py | 2 +- .../models/websocket_connection.py | 2 +- .../intrinio/tests/test_intrinio_fetchers.py | 2 +- .../models/websocket_connection.py | 16 +- .../openbb_polygon/utils/websocket_client.py | 19 +- .../polygon/tests/test_polygon_fetchers.py | 2 +- .../models/websocket_connection.py | 12 +- .../openbb_tiingo/utils/websocket_client.py | 16 +- .../tiingo/tests/test_tiingo_fetchers.py | 2 +- 14 files changed, 238 insertions(+), 124 deletions(-) rename openbb_platform/{extensions/websockets/openbb_websockets => core/openbb_core/provider/utils/websockets}/client.py (76%) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/client.py b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py similarity index 76% rename from openbb_platform/extensions/websockets/openbb_websockets/client.py rename to openbb_platform/core/openbb_core/provider/utils/websockets/client.py index 82b04254c1cf..35accd184b97 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/client.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py @@ -1,48 +1,60 @@ -"""WebSocket Client module for interacting with a provider websocket in a non-blocking pattern.""" +"""Module for running OpenBB Provider websocket connection scripts.""" # pylint: disable=too-many-statements,protected-access # flake8: noqa: PLR0915 -import logging from typing import TYPE_CHECKING, Any, Literal, Optional, Union if TYPE_CHECKING: - from openbb_core.provider.abstract.data import Data + import logging + + from pydantic import BaseModel class WebSocketClient: # pylint: disable=too-many-instance-attributes - """Client for interacting with a websocket server in a non-blocking pattern. + """ + Client for interacting with a websocket server in a non-blocking pattern, and handling the subprocesses. Parameters ---------- name : str - Name to assign the WebSocket connection. Used to identify and manage multiple instances. + Name to assign the WebSocket connection. Used to identify and manage multiple instances from the API. module : str - The Python module for the provider websocket_client module. Runs in a separate thread. - Example: 'openbb_fmp.utils.websocket_client'. Pass additional keyword arguments by including kwargs. + The Python module for the provider websocket_client module. + Runs in a separate thread, and is an equivalent to 'python -m module'. + Example: 'openbb_fmp.utils.websocket_client'. + Pass additional keyword arguments to the script by including **kwargs. symbol : Optional[str] - The symbol(s) requested to subscribe. Enter multiple symbols separated by commas without spaces. + The symbol(s) requested to subscribe on start. Enter multiple symbols separated by commas, without spaces. + Where supported by the provider, * represents all symbols within the feed. limit : Optional[int] The limit of records to hold in memory. Once the limit is reached, the oldest records are removed. - Default is 1000. Set to None to keep all records. + Default is 5000. Set to None to keep all records. results_file : Optional[str] Absolute path to the file for continuous writing. By default, a temporary file is created. + File is discarded when the Python session ends unless 'save_results' is set to True. + The connection can be re-established with the same results file to continue writing. + EACH NEW CONNECTION SHOULD HAVE A UNIQUE RESULTS FILE. + If the intention is to permanently store the results for historical records, + save the current session to a new file and copy new records at periodic intervals into the master. table_name : Optional[str] SQL table name to store serialized data messages. By default, 'records'. save_results : bool Whether to persist the results after the main Python session ends. Default is False. - data_model : Optional[Data] + data_model : Optional[BaseModel] Pydantic data model to validate the results before storing them in the database. Also used to deserialize the results from the database. auth_token : Optional[str] - The authentication token to use for the WebSocket connection. Default is None. - Only used for API and Python application endpoints. + Used to limit access to the broadcast stream. When provided, listeners should supply as a URL parameter. + Example: 'ws://127.0.0.1:6666/?auth_token=SOME_TOKEN'. + When provided, the auth_token is required to interact with the instance of this class + from the REST API and Python application endpoints. logger : Optional[logging.Logger] - The pre-configured logger instance to use for this connection. By default, a new logger is created. + A pre-configured logger to use for this instance. By default, a new logger is created. kwargs : Optional[dict] Additional keyword arguments to pass to the target provider module. Keywords and values must not contain spaces. To pass items to 'websocket.connect()', include them in the 'kwargs' dictionary as, - {'connect_kwargs': {'key': 'value'}}. + {'api_key': 'MY_API_KEY', 'connect_kwargs': {'key': 'value'}}. Properties ---------- @@ -55,10 +67,9 @@ class WebSocketClient: # pylint: disable=too-many-instance-attributes is_broadcasting : bool Check if the broadcast server process is running. broadcast_address : str - URI address for the results broadcast server. + URI address for connecting to the broadcast stream. results : list[Data] - All stored results from the provider's WebSocket stream. - Results are stored in a SQLite database as a serialized JSON string, this property deserializes the results. + All stored results from the provider connection. Clear the results by deleting the property. e.g., del client.results Methods @@ -68,15 +79,17 @@ class WebSocketClient: # pylint: disable=too-many-instance-attributes disconnect Disconnect from the provider WebSocket. subscribe - Subscribe to a new symbol or list of symbols. + Send a subscribe message to the provider connection. unsubscribe - Unsubscribe from a symbol or list of symbols. + Send an unsubscribe message to the provider connection. start_broadcasting - Start the broadcast server to stream results over a network connection. + Start the broadcast server to stream results over a network. stop_broadcasting Stop the broadcast server and disconnect all listening clients. send_message Send a message to the WebSocket process. Messages can be sent to "provider" or "broadcast" targets. + query_database + Run a SELECT query to the database. Returns a list of deserialized results, or validated models. """ def __init__( # noqa: PLR0913 # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals @@ -84,13 +97,13 @@ def __init__( # noqa: PLR0913 # pylint: disable=too-many-arguments,too-many-po name: str, module: str, symbol: Optional[str] = None, - limit: Optional[int] = 1000, + limit: Optional[int] = 5000, results_file: Optional[str] = None, table_name: Optional[str] = None, save_results: bool = False, - data_model: Optional["Data"] = None, + data_model: Optional["BaseModel"] = None, auth_token: Optional[str] = None, - logger: Optional[logging.Logger] = None, + logger: Optional["logging.Logger"] = None, **kwargs, ) -> None: """Initialize the WebSocketClient class.""" @@ -162,20 +175,21 @@ def __init__( # noqa: PLR0913 # pylint: disable=too-many-arguments,too-many-po results_file=self.results_file, table_name=self.table_name, limit=self._limit, - keep_results=self.save_results, logger=self.logger, data_model=self.data_model, ) except Exception as e: # pylint: disable=broad-except msg = ( "Unexpected error setting up the SQLite database and table ->" - f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__}: {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__} -> {e}" ) self.logger.error(msg) self._exception = OpenBBError(msg) + self._atexit() + raise OpenBBError(msg) from e def _atexit(self) -> None: - """Clean up the WebSocket client processes at exit.""" + """Clean up the running processes at exit.""" # pylint: disable=import-outside-toplevel import os @@ -310,13 +324,23 @@ def _log_broadcast_output(self, output_queue) -> None: output = output.replace("INFO:", "BROADCAST INFO:") + "\n" output = output[0] if isinstance(output, tuple) else output output = clean_message(output) + if ( + output.startswith("BROADCAST ERROR:") + or "unexpected error" in output.lower() + ): + self._psutil_broadcast_process.kill() + self._broadcast_process.wait() + self._broadcast_thread.join() + sys.stdout.write(output + "\n") + sys.stdout.flush() + break sys.stdout.write(output + "\n") sys.stdout.flush() except queue.Empty: continue def connect(self) -> None: # pylint: disable=too-many-locals - """Connect to the provider WebSocket.""" + """Connect to the provider client connection.""" # pylint: disable=import-outside-toplevel import os # noqa import psutil @@ -367,35 +391,44 @@ def connect(self) -> None: # pylint: disable=too-many-locals if kwarg not in command: command.extend([kwarg]) - self._process = subprocess.Popen( # noqa # pylint: disable=consider-using-with - command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - stdin=subprocess.PIPE, - env=os.environ, - text=True, - bufsize=1, - ) - self._psutil_process = psutil.Process(self._process.pid) + try: + self._process = ( + subprocess.Popen( # noqa # pylint: disable=consider-using-with + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdin=subprocess.PIPE, + env=os.environ, + text=True, + bufsize=1, + ) + ) + self._psutil_process = psutil.Process(self._process.pid) + + log_output_queue: queue.Queue = queue.Queue() + self._thread = threading.Thread( + target=non_blocking_websocket, + args=( + self, + log_output_queue, + self._provider_message_queue, + ), + ) + self._thread.daemon = True + self._thread.start() - log_output_queue: queue.Queue = queue.Queue() - self._thread = threading.Thread( - target=non_blocking_websocket, - args=( - self, - log_output_queue, - self._provider_message_queue, - ), - ) - self._thread.daemon = True - self._thread.start() + self._log_thread = threading.Thread( + target=self._log_provider_output, + args=(log_output_queue,), + ) + self._log_thread.daemon = True + self._log_thread.start() - self._log_thread = threading.Thread( - target=self._log_provider_output, - args=(log_output_queue,), - ) - self._log_thread.daemon = True - self._log_thread.start() + except Exception as e: # pylint: disable=broad-except + msg = f"Unexpected error -> {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__} -> {e}" + self.logger.error(msg) + self._atexit() + raise OpenBBError(msg) from e # Give it some startup time to allow the connection to be established and for exceptions to populate. time.sleep(2) @@ -406,12 +439,14 @@ def connect(self) -> None: # pylint: disable=too-many-locals raise OpenBBError(exc) if not self.is_running: - self.logger.error("The provider server failed to start.") + self.logger.error( + "Unexpected error -> Provider connection process failed to start." + ) def send_message( self, message, target: Literal["provider", "broadcast"] = "provider" ) -> None: - """Send a message to the WebSocket process.""" + """Write to the provider, or broadcast, process stdin.""" if target == "provider": self._provider_message_queue.put(message) read_message_queue(self, self._provider_message_queue) @@ -420,10 +455,10 @@ def send_message( read_message_queue(self, self._broadcast_message_queue, target="broadcast") def disconnect(self) -> None: - """Disconnect from the provider WebSocket.""" + """Disconnect from the provider connection.""" self._stop_log_thread_event.set() if self._process is None or self.is_running is False: - self.logger.info("Not connected to the provider WebSocket.") + self.logger.info("Provider client connection is not running.") return if ( self._psutil_process is not None @@ -435,13 +470,18 @@ def disconnect(self) -> None: self._thread.join() self._log_thread.join() self._stop_log_thread_event.clear() - self.logger.info("Disconnected from the provider WebSocket.") + self.logger.info("Disconnected from the provider server.") if hasattr(self, "_exception") and self._exception: raise self._exception return def subscribe(self, symbol) -> None: - """Subscribe to a new symbol or list of symbols.""" + """ + Send a subscribe message to the active provider connection. + + Messages are sent as JSON strings formatted as: + {"event": "subscribe", "symbol": "AAPL,MSFT"} + """ # pylint: disable=import-outside-toplevel import json # noqa import time @@ -463,7 +503,12 @@ def subscribe(self, symbol) -> None: self._symbol = ",".join(new_symbols) def unsubscribe(self, symbol) -> None: - """Unsubscribe from a symbol or list of symbols.""" + """ + Unsubscribe from a symbol or list of symbols. + + Messages are sent as JSON strings formatted as: + {"event": "unsubscribe", "symbol": "AAPL,MSFT"} + """ # pylint: disable=import-outside-toplevel import json # noqa import time @@ -500,23 +545,33 @@ def is_broadcasting(self) -> bool: @property def results(self) -> list: - """Retrieve the deserialized results from the results file.""" + """ + Retrieve the deserialized results from the active Database. + + Clear the results by deleting the property. e.g., del client.results + """ # pylint: disable=import-outside-toplevel from openbb_core.app.model.abstract.error import OpenBBError try: return self.database.fetch_all() except Exception as e: # pylint: disable=broad-except - raise OpenBBError(f"Error retrieving results: {e}") from e + msg = ( + "Error retrieving results:" + f" {e.__class__.__name__ if hasattr(e,'__class__') else e.__name__} -> {e}" + ) + raise OpenBBError(msg) from e @results.deleter def results(self): - """Clear results stored from the WebSocket stream.""" - # pylint: disable=import-outside-toplevel + """Clear results stored by the active WebSocket stream.""" try: self.database.clear_results() except Exception as e: # pylint: disable=broad-except - msg = f"Error clearing results: {e.__class__.__name__}: {e}" + msg = ( + "Error clearing results:" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__} -> {e}" + ) self.logger.error(msg) @property @@ -560,13 +615,63 @@ def broadcast_address(self) -> Union[str, None]: else None ) + def query_database( + self, + sql: Optional[str] = None, + limit: Optional[int] = 25, + ) -> list: + """ + Make a SELECT query to the database for results. + + The database always contains two columns: + "id" - an auto-incrementing ID + "message" - a JSON serialized row of data + + Parameters + ---------- + sql : Optional[str] + SQL query to execute. Default is None. + limit : Optional[int] + Limit the number of records returned, by most recent. Default is 25, set to None to return all records. + + Returns + ------- + list + A list of deserialized results from the database. + If a 'data_model' was supplied at initialization, it will be a list of validated models. + """ + if not sql: + query = f"SELECT message FROM {self.table_name} ORDER BY id DESC" # noqa + if limit is not None: + query += f" LIMIT {limit};" + else: + query = ( + sql.replace(";", "") + f" LIMIT {limit}" + if limit is not None and "LIMIT" not in sql.upper() + else sql + ) + + return self.database.query(query) + def start_broadcasting( # pylint: disable=too-many-locals self, host: str = "127.0.0.1", port: int = 6666, **kwargs, ) -> None: - """Broadcast results over a network connection.""" + """ + Broadcast results over a network connection. + + Parameters + ---------- + host : str + The host address to broadcast results to. Default is 127.0.0.1 + port : int + The port to broadcast results to. Default is 6666 + If the port is already in use, the next available port is used. + **kwargs: dict + Additional keyword arguments to pass to the `uvicorn.run`. + """ # pylint: disable=import-outside-toplevel import os # noqa import subprocess @@ -692,9 +797,10 @@ def non_blocking_websocket(client, output_queue, provider_message_queue) -> None if output: output_queue.put(output.strip()) - except Exception as e: + except Exception as e: # pylint: disable=broad-except msg = ( - f"Unexpected error in non_blocking_websocket: {e.__class__.__name__} -> {e}" + "Unexpected error in non_blocking_websocket:" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__} -> {e}" ) client.logger.error(msg) raise e from e @@ -710,7 +816,7 @@ def send_message( # pylint: disable=import-outside-toplevel import json - if isinstance(message, (dict, list)): + if not isinstance(message, str): message = json.dumps(message) try: if target == "provider": @@ -726,7 +832,10 @@ def send_message( else: client.logger.error("Broadcast process is not running.") except Exception as e: # pylint: disable=broad-except - msg = f"Error sending message to the {target} process: {e.__class__.__name__} -> {e}" + msg = ( + f"Error sending message to the {target} process:" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__} -> {e}" + ) client.logger.error(msg) @@ -747,8 +856,8 @@ def read_message_queue( send_message(client, message, target="broadcast") except Exception as e: # pylint: disable=broad-except err = ( - f"Error while attempting to transmit from the outgoing message queue: {e.__class__.__name__} " - f"-> {e} -> {message}" + "Error while attempting to transmit from the outgoing message queue:" + f"{e.__class__.__name__ if hasattr(e, '__class__') else e.__name__} -> {e} -> {message}" ) client.logger.error(err) @@ -767,7 +876,8 @@ def non_blocking_broadcast(client, output_queue, broadcast_message_queue) -> Non output_queue.put(output.strip()) except Exception as e: # pylint: disable=broad-except err = ( - f"Unexpected error in non_blocking_broadcast: {e.__class__.__name__} -> {e}" + f"Unexpected error in non_blocking_broadcast:" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__} -> {e}" ) client.logger.error(err) finally: diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 4765cc068062..19968f88e68b 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -66,7 +66,6 @@ def __init__( table_name: Optional[str] = None, data_model: Optional["BaseModel"] = None, limit: Optional[int] = None, - keep_results: Optional[bool] = False, logger: Optional["logging.Logger"] = None, loop: Optional["asyncio.AbstractEventLoop"] = None, **kwargs, @@ -87,6 +86,10 @@ def __init__( temp_file_path = temp_file.name self.results_path = Path(temp_file_path).absolute() self.results_file = temp_file_path + elif results_file and "://" in results_file: + self.results_file = results_file + self.results_path = results_file + kwargs["uri"] = True else: self.results_path = Path(results_file).absolute() self.results_file = results_file diff --git a/openbb_platform/extensions/websockets/openbb_websockets/models.py b/openbb_platform/extensions/websockets/openbb_websockets/models.py index 6912f743dc4e..e2bfb02314e2 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/models.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/models.py @@ -9,10 +9,9 @@ from openbb_core.provider.utils.descriptions import ( DATA_DESCRIPTIONS, ) +from openbb_core.provider.utils.websockets.client import WebSocketClient from pydantic import ConfigDict, Field, field_validator, model_validator -from openbb_websockets.client import WebSocketClient - class WebSocketQueryParams(QueryParams): """Query parameters for WebSocket connection creation.""" diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py index b9779e5f0d22..fa305da597eb 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py @@ -7,7 +7,7 @@ from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.abstract.fetcher import Fetcher -from openbb_websockets.client import WebSocketClient +from openbb_core.provider.utils.websockets.client import WebSocketClient from openbb_websockets.models import ( WebSocketConnection, WebSocketData, @@ -15,12 +15,6 @@ ) from pydantic import Field, field_validator -URL_MAP = { - "stock": "wss://websockets.financialmodelingprep.com", - "fx": "wss://forex.financialmodelingprep.com", - "crypto": "wss://crypto.financialmodelingprep.com", -} - class FmpWebSocketQueryParams(WebSocketQueryParams): """FMP WebSocket query parameters.""" @@ -151,12 +145,10 @@ async def aextract_data( import asyncio api_key = credentials.get("fmp_api_key") if credentials else "" - url = URL_MAP[query.asset_type] symbol = query.symbol.lower() - kwargs = { - "url": url, + "asset_type": query.asset_type, "api_key": api_key, "connect_kwargs": query.connect_kwargs, } diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py index 67b06be549f3..44a0240f70e7 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -18,11 +18,20 @@ from openbb_fmp.models.websocket_connection import FmpWebSocketData from pydantic import ValidationError +URL_MAP = { + "stock": "wss://websockets.financialmodelingprep.com", + "fx": "wss://forex.financialmodelingprep.com", + "crypto": "wss://crypto.financialmodelingprep.com", +} + logger = get_logger("openbb.websocket.fmp") kwargs = parse_kwargs() queue = MessageQueue() command_queue = MessageQueue() CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) +URL = URL_MAP.get(kwargs.pop("asset_type"), None) +if not URL: + raise ValueError("Invalid asset type provided.") DATABASE = Database( results_file=kwargs["results_file"], @@ -143,7 +152,7 @@ async def connect_and_stream(): stdin_task = asyncio.create_task(read_stdin_and_queue_commands()) try: - websocket = await websockets.connect(kwargs["url"], **CONNECT_KWARGS) + websocket = await websockets.connect(URL, **CONNECT_KWARGS) await login(websocket) await subscribe(websocket, kwargs["symbol"], "subscribe") diff --git a/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py b/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py index ec5853ca9ca4..ab0b59402a95 100644 --- a/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py +++ b/openbb_platform/providers/fmp/tests/test_fmp_fetchers.py @@ -7,6 +7,7 @@ import pytest from openbb_core.app.service.user_service import UserService +from openbb_core.provider.utils.websockets.client import WebSocketClient from openbb_fmp.models.analyst_estimates import FMPAnalystEstimatesFetcher from openbb_fmp.models.available_indices import FMPAvailableIndicesFetcher from openbb_fmp.models.balance_sheet import FMPBalanceSheetFetcher @@ -79,7 +80,6 @@ ) from openbb_fmp.models.world_news import FMPWorldNewsFetcher from openbb_fmp.models.yield_curve import FMPYieldCurveFetcher -from openbb_websockets.client import WebSocketClient test_credentials = UserService().default_user_settings.credentials.model_dump( mode="json" diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py index 1672ccdbd35a..850a7c792dd1 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py @@ -7,8 +7,8 @@ from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.abstract.fetcher import Fetcher +from openbb_core.provider.utils.websockets.client import WebSocketClient from openbb_intrinio.utils.references import TRADE_CONDITIONS, VENUES -from openbb_websockets.client import WebSocketClient from openbb_websockets.models import ( WebSocketConnection, WebSocketData, diff --git a/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py b/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py index a49177d2c896..dd0e3d60f12a 100644 --- a/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py +++ b/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py @@ -7,6 +7,7 @@ import pytest from openbb_core.app.service.user_service import UserService +from openbb_core.provider.utils.websockets.client import WebSocketClient from openbb_intrinio.models.balance_sheet import IntrinioBalanceSheetFetcher from openbb_intrinio.models.calendar_ipo import IntrinioCalendarIpoFetcher from openbb_intrinio.models.cash_flow import IntrinioCashFlowStatementFetcher @@ -73,7 +74,6 @@ IntrinioWebSocketFetcher, ) from openbb_intrinio.models.world_news import IntrinioWorldNewsFetcher -from openbb_websockets.client import WebSocketClient test_credentials = UserService().default_user_settings.credentials.model_dump( mode="json" diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index a94daca58c42..3277d4985464 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -9,6 +9,7 @@ from openbb_core.provider.abstract.data import Data from openbb_core.provider.abstract.fetcher import Fetcher from openbb_core.provider.utils.descriptions import DATA_DESCRIPTIONS +from openbb_core.provider.utils.websockets.client import WebSocketClient from openbb_polygon.utils.constants import ( CRYPTO_EXCHANGE_MAP, FX_EXCHANGE_MAP, @@ -20,7 +21,6 @@ STOCK_TRADE_CONDITIONS, ) from openbb_polygon.utils.helpers import map_tape -from openbb_websockets.client import WebSocketClient from openbb_websockets.models import ( WebSocketConnection, WebSocketData, @@ -28,16 +28,6 @@ ) from pydantic import Field, field_validator, model_validator -URL_MAP = { - "stock": "wss://socket.polygon.io/stocks", - "stock_delayed": "wss://delayed.polygon.io/stocks", - "options": "wss://socket.polygon.io/options", - "options_delayed": "wss://delayed.polygon.io/options", - "fx": "wss://socket.polygon.io/forex", - "crypto": "wss://socket.polygon.io/crypto", - "index": "wss://socket.polygon.io/indices", - "index_delayed": "wss://delayed.polygon.io/indices", -} ASSET_CHOICES = [ "stock", @@ -1227,12 +1217,8 @@ def extract_data( ) -> dict: """Extract data from the WebSocket.""" api_key = credentials.get("polygon_api_key") if credentials else "" - url = URL_MAP[query.asset_type] - symbol = query.symbol.upper() - kwargs = { - "url": url, "asset_type": query.asset_type, "feed": query.feed, "api_key": api_key, diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index b6345d32ea8e..1c6fd004ec8c 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -22,6 +22,17 @@ ) from pydantic import ValidationError +URL_MAP = { + "stock": "wss://socket.polygon.io/stocks", + "stock_delayed": "wss://delayed.polygon.io/stocks", + "options": "wss://socket.polygon.io/options", + "options_delayed": "wss://delayed.polygon.io/options", + "fx": "wss://socket.polygon.io/forex", + "crypto": "wss://socket.polygon.io/crypto", + "index": "wss://socket.polygon.io/indices", + "index_delayed": "wss://delayed.polygon.io/indices", +} + logger = get_logger("openbb.websocket.polygon") queue = MessageQueue(logger=logger) command_queue = MessageQueue(logger=logger) @@ -29,8 +40,12 @@ kwargs = parse_kwargs() CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) FEED = kwargs.pop("feed", None) -ASSET_TYPE = kwargs.pop("asset_type", None) +ASSET_TYPE = kwargs.pop("asset_type", "crypto") kwargs["results_file"] = os.path.abspath(kwargs.get("results_file")) +URL = URL_MAP.get(ASSET_TYPE) + +if not URL: + raise ValueError("Invalid asset type provided.") DATABASE = Database( results_file=kwargs["results_file"], @@ -208,7 +223,7 @@ async def connect_and_stream(): connect_kwargs["close_timeout"] = None try: - async with websockets.connect(kwargs["url"], **connect_kwargs) as websocket: + async with websockets.connect(URL, **connect_kwargs) as websocket: await login(websocket) response = await websocket.recv() diff --git a/openbb_platform/providers/polygon/tests/test_polygon_fetchers.py b/openbb_platform/providers/polygon/tests/test_polygon_fetchers.py index e62c458c331a..76e0db916cf0 100644 --- a/openbb_platform/providers/polygon/tests/test_polygon_fetchers.py +++ b/openbb_platform/providers/polygon/tests/test_polygon_fetchers.py @@ -6,6 +6,7 @@ import pytest from openbb_core.app.service.user_service import UserService +from openbb_core.provider.utils.websockets.client import WebSocketClient from openbb_polygon.models.balance_sheet import PolygonBalanceSheetFetcher from openbb_polygon.models.cash_flow import PolygonCashFlowStatementFetcher from openbb_polygon.models.company_news import PolygonCompanyNewsFetcher @@ -25,7 +26,6 @@ PolygonWebSocketData, PolygonWebSocketFetcher, ) -from openbb_websockets.client import WebSocketClient test_credentials = UserService().default_user_settings.credentials.model_dump( mode="json" diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py index 33c3f1828e2e..b8d7073e4b96 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py @@ -10,7 +10,7 @@ from openbb_core.provider.utils.descriptions import ( QUERY_DESCRIPTIONS, ) -from openbb_websockets.client import WebSocketClient +from openbb_core.provider.utils.websockets.client import WebSocketClient from openbb_websockets.models import ( WebSocketConnection, WebSocketData, @@ -18,12 +18,6 @@ ) from pydantic import Field, field_validator, model_validator -URL_MAP = { - "stock": "wss://api.tiingo.com/iex", - "fx": "wss://api.tiingo.com/fx", - "crypto": "wss://api.tiingo.com/crypto", -} - # These are the data array order of definitions. IEX_FIELDS = [ "type", @@ -244,7 +238,6 @@ async def aextract_data( from asyncio import sleep api_key = credentials.get("tiingo_token") if credentials else "" - url = URL_MAP[query.asset_type] threshold_level = ( 5 if query.asset_type == "fx" or query.feed == "trade" @@ -254,12 +247,11 @@ async def aextract_data( else 0 ) ) - symbol = query.symbol.lower() kwargs = { - "url": url, "api_key": api_key, + "asset_type": query.asset_type, "threshold_level": threshold_level, "connect_kwargs": query.connect_kwargs, } diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index 4a6a17bf5114..ef1664d8e207 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -19,6 +19,12 @@ from openbb_tiingo.models.websocket_connection import TiingoWebSocketData from pydantic import ValidationError +URL_MAP = { + "stock": "wss://api.tiingo.com/iex", + "fx": "wss://api.tiingo.com/fx", + "crypto": "wss://api.tiingo.com/crypto", +} + # These are the data array definitions. IEX_FIELDS = [ "type", @@ -74,6 +80,10 @@ kwargs = parse_kwargs() CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) kwargs["results_file"] = os.path.abspath(kwargs["results_file"]) +URL = URL_MAP.get(kwargs.pop("asset_type", "crypto")) + +if not URL: + raise ValueError("Invalid asset type provided.") DATABASE = Database( results_file=kwargs["results_file"], @@ -86,8 +96,6 @@ # Subscribe and unsubscribe events are handled in a separate connection using the subscription_id set by the login event. async def update_symbols(symbol, event): """Update the symbols to subscribe to.""" - url = kwargs["url"] - if not SUBSCRIPTION_ID: logger.error( "PROVIDER ERROR: Must be assigned a subscription ID to update symbols. Try logging in." @@ -103,7 +111,7 @@ async def update_symbols(symbol, event): }, } - async with websockets.connect(url) as websocket: + async with websockets.connect(URL) as websocket: await websocket.send(json.dumps(update_event)) response = await websocket.recv() message = json.loads(response) @@ -228,7 +236,7 @@ async def connect_and_stream(): try: try: - async with websockets.connect(kwargs["url"], **connect_kwargs) as websocket: + async with websockets.connect(URL, **connect_kwargs) as websocket: logger.info("PROVIDER INFO: WebSocket connection established.") await websocket.send(json.dumps(subscribe_event)) while True: diff --git a/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py b/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py index 8fbf39aff7ba..26fb140de193 100644 --- a/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py +++ b/openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py @@ -6,6 +6,7 @@ import pytest from openbb_core.app.service.user_service import UserService +from openbb_provider.utils.websockets.client import WebSocketClient from openbb_tiingo.models.company_news import TiingoCompanyNewsFetcher from openbb_tiingo.models.crypto_historical import TiingoCryptoHistoricalFetcher from openbb_tiingo.models.currency_historical import TiingoCurrencyHistoricalFetcher @@ -17,7 +18,6 @@ TiingoWebSocketFetcher, ) from openbb_tiingo.models.world_news import TiingoWorldNewsFetcher -from openbb_websockets.client import WebSocketClient test_credentials = UserService().default_user_settings.credentials.model_dump( mode="json" From 78a4969ae1bace6a89fde0855cd04843d043a209 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 17 Dec 2024 23:25:56 -0800 Subject: [PATCH 091/119] lint --- .../provider/utils/websockets/client.py | 10 +++++----- .../provider/utils/websockets/database.py | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py index 35accd184b97..1fecc2096ebe 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py @@ -181,7 +181,7 @@ def __init__( # noqa: PLR0913 # pylint: disable=too-many-arguments,too-many-po except Exception as e: # pylint: disable=broad-except msg = ( "Unexpected error setting up the SQLite database and table ->" - f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__} -> {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) self.logger.error(msg) self._exception = OpenBBError(msg) @@ -570,7 +570,7 @@ def results(self): except Exception as e: # pylint: disable=broad-except msg = ( "Error clearing results:" - f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__} -> {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) self.logger.error(msg) @@ -800,7 +800,7 @@ def non_blocking_websocket(client, output_queue, provider_message_queue) -> None except Exception as e: # pylint: disable=broad-except msg = ( "Unexpected error in non_blocking_websocket:" - f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__} -> {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) client.logger.error(msg) raise e from e @@ -834,7 +834,7 @@ def send_message( except Exception as e: # pylint: disable=broad-except msg = ( f"Error sending message to the {target} process:" - f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__} -> {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) client.logger.error(msg) @@ -877,7 +877,7 @@ def non_blocking_broadcast(client, output_queue, broadcast_message_queue) -> Non except Exception as e: # pylint: disable=broad-except err = ( f"Unexpected error in non_blocking_broadcast:" - f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__} -> {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) client.logger.error(err) finally: diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 19968f88e68b..4f3552218c3f 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -88,7 +88,7 @@ def __init__( self.results_file = temp_file_path elif results_file and "://" in results_file: self.results_file = results_file - self.results_path = results_file + self.results_path = results_file # type: ignore kwargs["uri"] = True else: self.results_path = Path(results_file).absolute() @@ -144,10 +144,10 @@ async def _setup_database(self): except Exception as e: msg = ( "Unexpected error while creating SQLite database ->" - f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__}: {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) self.logger.error(msg) - self._exception = e + self._exception = e # type: ignore raise OpenBBError(msg) from e async def _write_to_db(self, message) -> None: @@ -224,7 +224,7 @@ def write_to_db(self, message) -> None: except Exception as e: # pylint: disable=broad-except msg = ( "Unexpected error while writing to SQLite database ->" - f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__}: {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) self.logger.error(msg) self._exception = e @@ -278,10 +278,10 @@ def deserialize_row(self, row: str) -> Union["BaseModel", Any]: except Exception as e: msg = ( "Unexpected error while deserializing row -> " - f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__}: {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) self.logger.error(msg) - self._exception = e + self._exception = e # type: ignore raise OpenBBError(msg) from e def fetch_all(self, limit: Optional[int] = None) -> list: @@ -291,7 +291,7 @@ def fetch_all(self, limit: Optional[int] = None) -> list: except Exception as e: msg = ( "Unexpected error while reading from SQLite database ->" - f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__}: {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) self.logger.error(msg) self._exception = e @@ -414,7 +414,7 @@ def clear_results(self) -> None: except Exception as e: # pylint: disable=broad-except msg = ( "Error clearing results: " - f" {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__}: {e}" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) self.logger.error(msg) raise OpenBBError(msg) from e From 1fdf45e03a3716cdf8f650edabd9ece15600f091 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 17 Dec 2024 23:30:43 -0800 Subject: [PATCH 092/119] lint --- .../core/openbb_core/provider/utils/websockets/client.py | 6 +++--- .../core/openbb_core/provider/utils/websockets/database.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py index 1fecc2096ebe..bcb6d634450e 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py @@ -425,7 +425,7 @@ def connect(self) -> None: # pylint: disable=too-many-locals self._log_thread.start() except Exception as e: # pylint: disable=broad-except - msg = f"Unexpected error -> {e.__class__.__name__ if hasattr(e, '__class__') else e.__name__} -> {e}" + msg = f"Unexpected error -> {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" self.logger.error(msg) self._atexit() raise OpenBBError(msg) from e @@ -558,7 +558,7 @@ def results(self) -> list: except Exception as e: # pylint: disable=broad-except msg = ( "Error retrieving results:" - f" {e.__class__.__name__ if hasattr(e,'__class__') else e.__name__} -> {e}" + f" {e.__class__.__name__ if hasattr(e,'__class__') else e} -> {e.args}" ) raise OpenBBError(msg) from e @@ -857,7 +857,7 @@ def read_message_queue( except Exception as e: # pylint: disable=broad-except err = ( "Error while attempting to transmit from the outgoing message queue:" - f"{e.__class__.__name__ if hasattr(e, '__class__') else e.__name__} -> {e} -> {message}" + f"{e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args} -> {message}" ) client.logger.error(err) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 4f3552218c3f..0c2724012992 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -227,7 +227,7 @@ def write_to_db(self, message) -> None: f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) self.logger.error(msg) - self._exception = e + self._exception = e # type: ignore raise OpenBBError(msg) from e async def _fetch_all(self, limit: Optional[int] = None) -> list: @@ -294,7 +294,7 @@ def fetch_all(self, limit: Optional[int] = None) -> list: f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) self.logger.error(msg) - self._exception = e + self._exception = e # type: ignore raise OpenBBError(msg) from e async def _query_db(self, sql) -> list: From cf934d96fa68003f3d3ae07ac98351b3e58a3245 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 18 Dec 2024 00:11:22 -0800 Subject: [PATCH 093/119] adjust process_queue test --- .../provider/utils/websockets/test_message_queue.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openbb_platform/core/tests/provider/utils/websockets/test_message_queue.py b/openbb_platform/core/tests/provider/utils/websockets/test_message_queue.py index ba0c6102cbfc..dc182212afd8 100644 --- a/openbb_platform/core/tests/provider/utils/websockets/test_message_queue.py +++ b/openbb_platform/core/tests/provider/utils/websockets/test_message_queue.py @@ -61,9 +61,14 @@ async def handler(message): """Test handler.""" nonlocal NUM_MESSAGES NUM_MESSAGES += 1 + assert message in MOCK_MESSAGES await message_queue.enqueue(MOCK_MESSAGES[0]) await message_queue.enqueue(MOCK_MESSAGES[1]) - asyncio.create_task(message_queue.process_queue(handler)) - await asyncio.sleep(0.1) + while not message_queue.queue.empty(): + await message_queue._process_message(await message_queue.dequeue(), handler) + await asyncio.sleep(0.1) + message_queue.queue.task_done() + assert NUM_MESSAGES == 2 + assert message_queue.queue.empty() From 19813cacf153773af27b94cfa0262b6d3742bf7d Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:12:06 -0800 Subject: [PATCH 094/119] more lint --- .../core/openbb_core/provider/utils/websockets/client.py | 1 + .../core/openbb_core/provider/utils/websockets/database.py | 2 +- .../extensions/websockets/openbb_websockets/broadcast.py | 7 ++++--- .../extensions/websockets/openbb_websockets/helpers.py | 2 +- .../intrinio/openbb_intrinio/utils/websocket_client.py | 1 - .../polygon/openbb_polygon/utils/websocket_client.py | 2 +- 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py index bcb6d634450e..7db087ac055a 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py @@ -264,6 +264,7 @@ def _log_provider_output(self, output_queue) -> None: "server rejected" in output.lower() or "PROVIDER ERROR" in output or "unexpected error" in output.lower() + or "Error:" in output ): self._psutil_process.kill() self._process.wait() diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 0c2724012992..7e241046f289 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -60,7 +60,7 @@ class Database: Clear all results from the SQLite database. """ - def __init__( + def __init__( # pylint: disable=too-many-positional-arguments self, results_file: Optional[str] = None, table_name: Optional[str] = None, diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py index a39c75dfe9e4..9089e192228c 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -288,12 +288,13 @@ def main(): if __name__ == "__main__": + # pylint: disable=import-outside-toplevel + from openbb_core.provider.utils.helpers import run_async + if not RESULTS_FILE: raise ValueError("Results file path is required for Broadcast server.") if not Path(RESULTS_FILE).absolute().exists(): - # pylint: disable=import-outside-toplevel - from openbb_core.provider.utils.helpers import run_async + run_async(DATABASE._setup_database) # pylint: disable=protected-access - run_async(DATABASE._setup_database) main() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 889d1eb83e1a..82556aa8d32b 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -53,7 +53,7 @@ async def check_auth(name: str, auth_token: Optional[str] = None) -> bool: return True if auth_token is None: raise UnauthorizedError(f"Client authorization token is required for {name}.") - if auth_token != client._decrypt_value( + if auth_token != client._decrypt_value( # pylint: disable=protected-access client._auth_token # pylint: disable=protected-access ): raise UnauthorizedError(f"Invalid client authorization token for {name}.") diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py index 803787aaeaac..74abf43bc3d4 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -8,7 +8,6 @@ import sys from typing import Any -from openbb_core.provider.utils.helpers import run_async from openbb_core.provider.utils.websockets.database import Database from openbb_core.provider.utils.websockets.helpers import ( get_logger, diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index 1c6fd004ec8c..9ecc4807c50a 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -203,7 +203,7 @@ async def process_message(message): raise e from e if result: - await DATABASE._write_to_db(result) + await DATABASE._write_to_db(result) # pylint: disable=protected-access else: logger.info("PROVIDER INFO: %s", msg) From 8647d09b4039415692cd527e9ffd3b6b260664c8 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:18:33 -0800 Subject: [PATCH 095/119] okk.. --- .../extensions/websockets/openbb_websockets/broadcast.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py index 9089e192228c..950a7a41ac0e 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py @@ -8,6 +8,7 @@ import uvicorn from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from openbb_core.provider.utils.helpers import run_async from openbb_core.provider.utils.websockets.database import Database from openbb_core.provider.utils.websockets.helpers import get_logger, parse_kwargs from starlette.websockets import WebSocketState @@ -288,9 +289,6 @@ def main(): if __name__ == "__main__": - # pylint: disable=import-outside-toplevel - from openbb_core.provider.utils.helpers import run_async - if not RESULTS_FILE: raise ValueError("Results file path is required for Broadcast server.") From 38331f561848ad6dfab818cb1ab6c3222d109e87 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:45:21 -0800 Subject: [PATCH 096/119] database read-only connections don't need to check same thread --- .../provider/utils/websockets/database.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 7e241046f289..f4df4b1344c4 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -237,13 +237,18 @@ async def _fetch_all(self, limit: Optional[int] = None) -> list: try: rows: list = [] + conn_kwargs = self.kwargs.copy() + if not conn_kwargs.get("check_same_thread"): + conn_kwargs["check_same_thread"] = False async with aiosqlite.connect( - self.results_file, loop=self.loop, **self.kwargs + self.results_file, + loop=self.loop, + **conn_kwargs, ) as conn: query = ( f"SELECT message FROM {self.table_name} ORDER BY id DESC" # noqa ) - if limit: + if limit is not None: query += f" LIMIT {limit}" async with conn.execute(query) as cursor: async for row in cursor: @@ -317,8 +322,13 @@ async def _query_db(self, sql) -> list: ) rows: list = [] try: + conn_kwargs = self.kwargs.copy() + if not conn_kwargs.get("check_same_thread"): + conn_kwargs["check_same_thread"] = False async with aiosqlite.connect( - self.results_file, loop=self.loop, **self.kwargs + self.results_file, + loop=self.loop, + **conn_kwargs, ) as conn, conn.execute(query) as cursor: async for row in cursor: rows.append( From 46a45f1bf4c052476c9747e1b66a4fc41d70668a Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 21 Dec 2024 09:14:23 -0800 Subject: [PATCH 097/119] some optimization updates --- .../provider/utils/websockets/broadcast.py | 378 ++++++++++++++++++ .../provider/utils/websockets/client.py | 57 ++- .../provider/utils/websockets/database.py | 317 ++++++++++----- .../utils/websockets/message_queue.py | 2 +- .../websockets/openbb_websockets/listen.py | 14 +- .../openbb_tiingo/utils/websocket_client.py | 32 +- 6 files changed, 668 insertions(+), 132 deletions(-) create mode 100644 openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py new file mode 100644 index 000000000000..4f9c5d717969 --- /dev/null +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py @@ -0,0 +1,378 @@ +"""Broadcast server for streaming results to connected clients via WebSocket.""" + +import asyncio +import json +import logging +import os +import sys +from pathlib import Path +from typing import Optional + +import uvicorn +from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from openbb_core.provider.utils.websockets.database import CHECK_FOR, Database +from openbb_core.provider.utils.websockets.helpers import parse_kwargs +from starlette.websockets import WebSocketState + +kwargs = parse_kwargs() + +HOST = kwargs.pop("host", None) or "localhost" +PORT = kwargs.pop("port", None) or 6666 +PORT = int(PORT) + +RESULTS_FILE = kwargs.pop("results_file", None) +TABLE_NAME = kwargs.pop("table_name", None) or "records" +SLEEP_TIME = kwargs.pop("sleep_time", None) or 0.25 +AUTH_TOKEN = kwargs.pop("auth_token", None) + +SQL = kwargs.pop("sql", None) +SQL_CONNECT_KWARGS = kwargs.pop("sql_connect_kwargs", None) or {} + +app = FastAPI() + + +async def read_stdin(broadcast_server): + """Read from stdin.""" + while True: + line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline) + sys.stdin.flush() + sys.stdout.flush() + + if not line: + break + + try: + command = ( + json.loads(line.strip()) + if line.strip().startswith("{") or line.strip().startswith("[") + else line.strip() + ) + await broadcast_server.websocket.send_json(json.dumps(command)) + except json.JSONDecodeError: + broadcast_server.logger.error("Invalid JSON received from stdin") + + +@app.websocket("/") +async def websocket_endpoint( # noqa: PLR0915 + websocket: WebSocket, + auth_token: Optional[str] = None, + replay: bool = False, +): + """Connect to the broadcast server.""" + headers = dict(websocket.headers) + sql = None + + if headers.get("sql"): + sql = headers.pop("sql", None) + + broadcast_server = BroadcastServer( + RESULTS_FILE, + TABLE_NAME, + SLEEP_TIME, + str(AUTH_TOKEN), + sql=sql, + ) + broadcast_server.replay = replay + auth_token = str(auth_token) + + if sql and ( + any(x in sql for x in CHECK_FOR) + or (broadcast_server.table_name not in sql and "message" not in sql) + ): + await websocket.accept() + await websocket.send_text("Connection refused because of invalid SQL.") + broadcast_server.logger.info("Invalid SQL query passed. -> %s", sql) + await websocket.close(code=1008, reason="Invalid parameter values.") + return + + if ( + broadcast_server.auth_token is not None + and auth_token + != broadcast_server._decrypt_value( # pylint: disable=protected-access + broadcast_server.auth_token + ) + ): + await websocket.accept() + await websocket.send_text( + "UnauthorizedError: Invalid authentication token. Could not connect to the broadcast." + ) + broadcast_server.logger.error( + "Invalid authentication token passed by a client connecting." + ) + await websocket.close(code=1008, reason="Invalid authentication token") + return + + await websocket.accept() + + if RESULTS_FILE is None: + raise ValueError("Results file path is required for WebSocket server.") + + broadcast_server.websocket = websocket + + stream_task = asyncio.create_task(broadcast_server.stream_results()) + stdin_task = asyncio.create_task(read_stdin(broadcast_server)) + try: + await websocket.receive_text() + + except WebSocketDisconnect: + pass + except Exception as e: # pylint: disable=broad-except + msg = f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" + broadcast_server.logger.error(msg) + finally: + stream_task.cancel() + stdin_task.cancel() + try: + await stream_task + await stdin_task + except asyncio.CancelledError: + broadcast_server.logger.info("A listener task was cancelled.") + for handler in broadcast_server.logger.handlers: + handler.flush() + except Exception as e: # pylint: disable=broad-except + msg = ( + f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" + ) + broadcast_server.logger.error(msg) + for handler in broadcast_server.logger.handlers: + handler.flush() + if websocket.client_state != WebSocketState.DISCONNECTED: + try: + await websocket.close() + except RuntimeError as e: + msg = f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" + broadcast_server.logger.error(msg) + for handler in broadcast_server.logger.handlers: + handler.flush() + + +class BroadcastServer: # pylint: disable=too-many-instance-attributes + """Stream new results from a continuously written SQLite database. + + Not intended to be used directly, it is initialized by the server app when it accepts a new connection. + It is responsible for reading the results database and sending new messages to the connected client(s). + """ + + def __init__( + self, + results_file, + table_name, + sleep_time: float = 0.25, + auth_token: Optional[str] = None, + sql_connect_kwargs: Optional[dict] = None, + sql: Optional[str] = None, + logger: Optional[logging.Logger] = None, + **kwargs, + ): + """Initialize the BroadcastServer instance.""" + self.results_file = results_file + self.table_name = table_name + self.logger = logger if logger else logging.getLogger("uvicorn.error") + self.sleep_time = sleep_time + self._app = app + self._key = os.urandom(32) + self._iv = os.urandom(16) + self.auth_token = self._encrypt_value(auth_token) if auth_token else None + self.websocket = None + self.kwargs = kwargs + self.sql_connect_kwargs = ( + sql_connect_kwargs if sql_connect_kwargs is not None else {} + ) + self.database = Database( + results_file=results_file, + table_name=table_name, + logger=self.logger, + **sql_connect_kwargs or {}, + ) + self.sql = sql + + def _encrypt_value(self, value: str) -> str: + """Encrypt the value for storage.""" + # pylint: disable=import-outside-toplevel + from openbb_core.provider.utils.websockets.helpers import encrypt_value + + return encrypt_value(self._key, self._iv, value) + + def _decrypt_value(self, value: str) -> str: + """Decrypt the value for use.""" + # pylint: disable=import-outside-toplevel + from openbb_core.provider.utils.websockets.helpers import decrypt_value + + return decrypt_value(self._key, self._iv, value) + + async def stream_results( # noqa: PLR0915 # pylint: disable=too-many-branches + self, + sql: str = None, + replay: bool = False, + ): + """Continuously read the database and send new messages as JSON via WebSocket.""" + # pylint: disable=import-outside-toplevel + import aiosqlite + from openbb_core.app.model.abstract.error import OpenBBError + + file_path = Path(self.results_file).absolute() + last_id = 0 + + if not file_path.exists(): + self.logger.error("Results file not found: %s", str(file_path)) + return + + query = f"SELECT MAX(id) FROM {self.table_name}" # noqa:S608 + last_id = ( + 0 + if hasattr(self, "replay") and self.replay is True or replay is True + else self.database.query(query)[0] + ) + if sql and self.sql is None: + self.sql = sql + elif self.sql is not None and sql is None: + sql = self.sql + + if sql and sql.lower().startswith("json_extract"): + sql = f"SELECT * FROM {self.table_name} WHERE {sql}" # noqa:S608 + + if sql and ( + any(x.lower() in sql.lower() for x in CHECK_FOR) + or (self.table_name not in sql and "message" not in sql) + ): + await self.websocket.accept() + await self.websocket.send_text("Invalid SQL query passed.") + await self.websocket.close(code=1008, reason="Invalid query") + self.logger.error( + "Invalid query passed to the stream_results method: %s", sql + ) + return + + try: # pylint: disable=too-many-nested-blocks + while True: + try: + if file_path.exists(): + query = ( + sql.replace(";", "") + + f" {'AND' if 'WHERE' in sql else 'WHERE'} id > {last_id}" + if sql is not None + else f"SELECT * FROM {self.table_name} WHERE id > {last_id}" # noqa:S608 + ) + + if ":" not in self.results_file: + results_file = ( + "file:" + + ( + self.results_file + if self.results_file.startswith("/") + else "/" + self.results_file + ) + + "?mode=ro" + ) + else: + results_file = ( + self.results_file + + f"{'&mode=ro' if '?' in self.results_file else '?mode=ro'}" + ) + conn_kwargs = self.sql_connect_kwargs.copy() + conn_kwargs["uri"] = True + conn_kwargs["check_same_thread"] = False + + async with aiosqlite.connect( + results_file, **conn_kwargs + ) as conn, conn.execute(query) as cursor: + async for row in cursor: + last_id = row[0] if row[0] > last_id else last_id + await self.websocket.send_json( + json.dumps(json.loads(row[1])) + ) + if self.replay is True: + await asyncio.sleep(self.sleep_time / 10) + else: + self.logger.error("Results file not found: %s", str(file_path)) + break + + await asyncio.sleep(self.sleep_time) + except KeyboardInterrupt: + self.logger.info("\nResults stream cancelled.") + break + except aiosqlite.OperationalError as e: + if "no such table" in str(e): + self.logger.error( + "Results file was removed by the parent process." + ) + break + raise OpenBBError(e) from e + except asyncio.CancelledError: + break + except WebSocketDisconnect: + pass + except Exception as e: # pylint: disable=broad-except + msg = f"{e.__class__.__name__ if hasattr(e, "__class__") else e} -> {e}" + self.logger.error(msg) + return + + def start_app(self, host: str = "127.0.0.1", port: int = 6666): + """Start the FastAPI app with Uvicorn.""" + uvicorn.run( + self._app, + host=host, + port=port, + **kwargs, + ) + + +def create_broadcast_server( + results_file: str, + table_name: str, + sleep_time: float = 0.25, + auth_token: Optional[str] = None, + sql_connect_kwargs: Optional[dict] = None, + sql: Optional[str] = None, + **kwargs, +): + """Create a new BroadcastServer instance.""" + return BroadcastServer( + results_file, + table_name, + sleep_time, + auth_token, + sql_connect_kwargs, + sql, + **kwargs, + ) + + +def main(): + """Run the main function.""" + broadcast_server = create_broadcast_server( + RESULTS_FILE, + TABLE_NAME, + SLEEP_TIME, + str(AUTH_TOKEN), + SQL_CONNECT_KWARGS, + SQL, + ) + + try: + broadcast_server.start_app( + host=HOST, + port=PORT, + **kwargs, + ) + except TypeError as e: + msg = f"Invalid keyword argument passed to unvicorn. -> {e.args[0]}" + broadcast_server.logger.error(msg) + for handler in broadcast_server.logger.handlers: + handler.flush() + except KeyboardInterrupt: + broadcast_server.logger.info("Broadcast server terminated.") + for handler in broadcast_server.logger.handlers: + handler.flush() + finally: + sys.exit(0) + + +if __name__ == "__main__": + if not RESULTS_FILE: + raise ValueError("Results file path is required for Broadcast server.") + + try: + main() + except KeyboardInterrupt: + sys.exit(0) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py index 7db087ac055a..e08af867fe59 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py @@ -28,7 +28,8 @@ class WebSocketClient: # pylint: disable=too-many-instance-attributes The symbol(s) requested to subscribe on start. Enter multiple symbols separated by commas, without spaces. Where supported by the provider, * represents all symbols within the feed. limit : Optional[int] - The limit of records to hold in memory. Once the limit is reached, the oldest records are removed. + The limit of records to store. Once the limit is reached, a one-in-one-out policy is used. + A limit of None is the most efficient setting, but requires adequate disk storage to handle high volume. Default is 5000. Set to None to keep all records. results_file : Optional[str] Absolute path to the file for continuous writing. By default, a temporary file is created. @@ -40,10 +41,9 @@ class WebSocketClient: # pylint: disable=too-many-instance-attributes table_name : Optional[str] SQL table name to store serialized data messages. By default, 'records'. save_results : bool - Whether to persist the results after the main Python session ends. Default is False. + Whether to persist the results after exiting. Default is False. data_model : Optional[BaseModel] Pydantic data model to validate the results before storing them in the database. - Also used to deserialize the results from the database. auth_token : Optional[str] Used to limit access to the broadcast stream. When provided, listeners should supply as a URL parameter. Example: 'ws://127.0.0.1:6666/?auth_token=SOME_TOKEN'. @@ -53,8 +53,9 @@ class WebSocketClient: # pylint: disable=too-many-instance-attributes A pre-configured logger to use for this instance. By default, a new logger is created. kwargs : Optional[dict] Additional keyword arguments to pass to the target provider module. Keywords and values must not contain spaces. - To pass items to 'websocket.connect()', include them in the 'kwargs' dictionary as, - {'api_key': 'MY_API_KEY', 'connect_kwargs': {'key': 'value'}}. + To pass items to 'websocket.connect()', include them in the 'kwargs' dictionary as a nested dictionary, + with key, 'connect_kwargs'. + {'api_key': 'MY_API_KEY', 'connect_kwargs': {'key': 'value'}}. Properties ---------- @@ -68,6 +69,8 @@ class WebSocketClient: # pylint: disable=too-many-instance-attributes Check if the broadcast server process is running. broadcast_address : str URI address for connecting to the broadcast stream. + num_results : int + Number of results stored in the database. results : list[Data] All stored results from the provider connection. Clear the results by deleting the property. e.g., del client.results @@ -88,8 +91,10 @@ class WebSocketClient: # pylint: disable=too-many-instance-attributes Stop the broadcast server and disconnect all listening clients. send_message Send a message to the WebSocket process. Messages can be sent to "provider" or "broadcast" targets. + get_latest_results + Get the latest results from the database, optionally filter by symbol. query_database - Run a SELECT query to the database. Returns a list of deserialized results, or validated models. + Run a SELECT query to the database. Returns a list of deserialized results. """ def __init__( # noqa: PLR0913 # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals @@ -309,9 +314,6 @@ def _log_broadcast_output(self, output_queue) -> None: output = "INFO: " + f"Stream results from {address}" self._broadcast_address = address - if output and "Started server process" in output: - output = None - if output and "Waiting for application startup." in output: output = None @@ -325,16 +327,16 @@ def _log_broadcast_output(self, output_queue) -> None: output = output.replace("INFO:", "BROADCAST INFO:") + "\n" output = output[0] if isinstance(output, tuple) else output output = clean_message(output) - if ( - output.startswith("BROADCAST ERROR:") - or "unexpected error" in output.lower() - ): - self._psutil_broadcast_process.kill() - self._broadcast_process.wait() - self._broadcast_thread.join() - sys.stdout.write(output + "\n") - sys.stdout.flush() - break + # if ( + # output.startswith("BROADCAST ERROR:") + # or "unexpected error" in output.lower() + # ): + # self._psutil_broadcast_process.kill() + # self._broadcast_process.wait() + # self._broadcast_thread.join() + # sys.stdout.write(output + "\n") + # sys.stdout.flush() + # continue sys.stdout.write(output + "\n") sys.stdout.flush() except queue.Empty: @@ -544,6 +546,13 @@ def is_broadcasting(self) -> bool: return self._psutil_broadcast_process.is_running() return False + @property + def num_results(self) -> int: + """Get the number of results stored in the database.""" + return self.query_database(f"SELECT COUNT(*) FROM {self.table_name};")[ # noqa + 0 + ] + @property def results(self) -> list: """ @@ -616,10 +625,16 @@ def broadcast_address(self) -> Union[str, None]: else None ) + def get_latest_results( + self, symbol: Optional[str] = None, limit: Optional[int] = 100 + ) -> list: + """Get the latest results from the database, optionally filter by symbol.""" + return self.database.get_latest_results(symbol=symbol, limit=limit) + def query_database( self, sql: Optional[str] = None, - limit: Optional[int] = 25, + limit: Optional[int] = 100, ) -> list: """ Make a SELECT query to the database for results. @@ -700,7 +715,7 @@ def start_broadcasting( # pylint: disable=too-many-locals command = [ sys.executable, "-m", - "openbb_websockets.broadcast", + "openbb_core.provider.utils.websockets.broadcast", f"host={host}", f"port={open_port}", f"results_file={self.results_file}", diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index f4df4b1344c4..ef58e03f56a1 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -1,6 +1,6 @@ """Database module for serialized websockets results.""" -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Iterable, Optional from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.utils.helpers import run_async @@ -11,6 +11,50 @@ from pydantic import BaseModel +CHECK_FOR = ( + "DATABASE", + "TABLE", + "BACKUP", + "DELETE", + "UPDATE", + "INSERT", + "CREATE", + "MODIFY", + "PRAGMA", + "ALTER", + "DROP", + "RENAME", + "REPLACE", + "TRUNCATE", + "VACUUM", + "ATTACH", + "DETACH", + "REINDEX", + "MOVE", + "1=1", + "=''", + '=""', + '"=""', + "= ''", + '= ""', + "or ''", + 'or ""', + "OR ''", + 'OR ""', + "AND ''", + 'AND ""', + "and ''", + 'and ""', + "('')", + '("")', + "('',)", + '("",)', + " ''", + ' ""', + "' '", + '" "', +) + class Database: """ @@ -52,12 +96,25 @@ class Database: ------- write_to_db(message) -> None Write the WebSocket message to the SQLite database. + Synchronous wrapper for _write_to_db. fetch_all(limit: Optional[int] = None) -> list Read the WebSocket message from the SQLite database. - query(sql: str) -> list - Run a SELECT query to the database. + Synchronous wrapper for _fetch_all. + get_latest_results(symbol: Optional[str] = None, limit: Optional[int] = None) -> list + Get the latest records from the database. Optionally filter by symbol. + Synchronous wrapper for _get_latest_results. + query(sql: str, parameters: Optional[Iterable[Any]]) -> list + Run a SELECT query to the database. Table name cannot be anything other than the originally assigned name. + For convenience, the query can start after WHERE, or provide a full query string to extract a specific array. + There are only two columns in the table: "id" - auto-increment index - and "message" - serialized JSON data row. + Synchronous wrapper for _query_db. clear_results() -> None Clear all results from the SQLite database. + + Raises + ------ + OpenBBError + All exceptions are raised as OpenBBError. """ def __init__( # pylint: disable=too-many-positional-arguments @@ -74,11 +131,11 @@ def __init__( # pylint: disable=too-many-positional-arguments # pylint: disable=import-outside-toplevel import tempfile # noqa from pathlib import Path + from aiosqlite import ProgrammingError from openbb_core.provider.utils.websockets.helpers import get_logger self.results_file = None self.table_exists = False - self._exception = None self.logger = logger if logger else get_logger("openbb.websocket.database") if not results_file: @@ -86,7 +143,7 @@ def __init__( # pylint: disable=too-many-positional-arguments temp_file_path = temp_file.name self.results_path = Path(temp_file_path).absolute() self.results_file = temp_file_path - elif results_file and "://" in results_file: + if ":" in results_file: self.results_file = results_file self.results_path = results_file # type: ignore kwargs["uri"] = True @@ -94,6 +151,13 @@ def __init__( # pylint: disable=too-many-positional-arguments self.results_path = Path(results_file).absolute() self.results_file = results_file + if ( + " " in table_name + or table_name.isupper() + or any(x.lower() in table_name.lower() for x in CHECK_FOR) + ): + raise OpenBBError(ProgrammingError(f"Invalid table name, {table_name}.")) + self.table_name = table_name if table_name else "records" self.limit = limit self.loop = loop @@ -124,7 +188,6 @@ async def _setup_database(self): + f" -> {e}" ) self.logger.error(msg) - self._exception = e raise OpenBBError(msg) from e async with aiosqlite.connect( @@ -135,19 +198,19 @@ async def _setup_database(self): CREATE TABLE IF NOT EXISTS {self.table_name} ( id INTEGER PRIMARY KEY AUTOINCREMENT, message TEXT NOT NULL - ) + ); """ ) - await cursor.close() await conn.commit() + await cursor.close() self.table_exists = True + except Exception as e: msg = ( "Unexpected error while creating SQLite database ->" f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) self.logger.error(msg) - self._exception = e # type: ignore raise OpenBBError(msg) from e async def _write_to_db(self, message) -> None: @@ -163,23 +226,15 @@ async def _write_to_db(self, message) -> None: if not isinstance(message, str): message = json.dumps(message) + if not self.table_exists: + raise aiosqlite.OperationalError( + "Attempt to write to non-existent table." + ) + async with aiosqlite.connect( self.results_file, loop=self.loop, **self.kwargs ) as conn: - cursor = await conn.execute("PRAGMA journal_mode=WAL;") - await cursor.close() - if not self.table_exists: - cursor = await conn.execute( - f""" - CREATE TABLE IF NOT EXISTS {self.table_name} ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - message TEXT NOT NULL - ) - """ - ) - await conn.commit() - await cursor.close() - self.table_exists = True + await conn.execute("PRAGMA journal_mode=WAL;") cursor = await conn.execute( f""" INSERT INTO {self.table_name} (message) @@ -187,35 +242,39 @@ async def _write_to_db(self, message) -> None: """, # noqa (message,), ) - await conn.commit() - await cursor.close() + self._at_limit = False - if self.limit is not None: - count = await conn.execute( - f"SELECT COUNT(*) FROM {self.table_name}" # noqa - ) - current_count = int((await count.fetchone())[0]) - await count.close() - limit = 0 if int(self.limit) < 0 else int(self.limit) - - if current_count > limit and limit != 0: - del_cursor = await conn.execute( - f""" - DELETE FROM {self.table_name} - WHERE id IN ( - SELECT id FROM {self.table_name} - ORDER BY id DESC - LIMIT -1 OFFSET ? - ) - """, # noqa: S608 - (limit,), + if self.limit is not None and not self._at_limit: + limit = max(0, int(self.limit)) + + if limit > 0: + count_cursor = await conn.execute( + f"SELECT COUNT(*) FROM {self.table_name}" # noqa + ) + count = await count_cursor.fetchone() + + if count[0] > limit: + self._at_limit = True + + await count_cursor.close() + + if self._at_limit: + await conn.execute( + f""" + DELETE FROM {self.table_name} + WHERE id = ( + SELECT id FROM {self.table_name} + ORDER BY id ASC + LIMIT 1 ) - await del_cursor.close() + """, # noqa + ) - await conn.commit() + await cursor.close() + await conn.commit() except Exception as e: # pylint: disable=broad-except - raise e from e + raise OpenBBError(e) from e def write_to_db(self, message) -> None: """Write the WebSocket message to the SQLite database.""" @@ -227,7 +286,6 @@ def write_to_db(self, message) -> None: f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) self.logger.error(msg) - self._exception = e # type: ignore raise OpenBBError(msg) from e async def _fetch_all(self, limit: Optional[int] = None) -> list: @@ -238,10 +296,29 @@ async def _fetch_all(self, limit: Optional[int] = None) -> list: try: rows: list = [] conn_kwargs = self.kwargs.copy() + if not conn_kwargs.get("check_same_thread"): conn_kwargs["check_same_thread"] = False + + if ":" not in self.results_file: + results_file = ( + "file:" + + ( + self.results_file + if self.results_file.startswith("/") + else "/" + self.results_file + ) + + "?mode=ro" + ) + else: + results_file = ( + self.results_file + + f"{'&mode=ro' if '?' in self.results_file else '?mode=ro'}" + ) + conn_kwargs["uri"] = True + async with aiosqlite.connect( - self.results_file, + results_file, loop=self.loop, **conn_kwargs, ) as conn: @@ -249,36 +326,35 @@ async def _fetch_all(self, limit: Optional[int] = None) -> list: f"SELECT message FROM {self.table_name} ORDER BY id DESC" # noqa ) if limit is not None: - query += f" LIMIT {limit}" - async with conn.execute(query) as cursor: + query += " LIMIT ?" + params = (limit,) + else: + params = () + async with conn.execute(query, params) as cursor: async for row in cursor: - rows.append(self.deserialize_row(row)) + rows.append(await self.deserialize_row(row)) return rows except Exception as e: - raise e from e + raise OpenBBError(e) from e - def deserialize_row(self, row: str) -> Union["BaseModel", Any]: + async def deserialize_row(self, row: str) -> dict: """Deserialize a row from the SQLite database.""" # pylint: disable=import-outside-toplevel import json try: return ( - self.data_model.model_validate_json(row[0]) - if self.data_model is not None - else ( - json.loads(row[0]) - if ( - ( - isinstance(row[0], str) - and (row[0].startswith("{") or row[0].startswith("[")) - ) - or isinstance(row[0], bytes) + json.loads(row[0]) + if ( + ( + isinstance(row[0], str) + and (row[0].startswith("{") or row[0].startswith("[")) ) - else row[0] + or isinstance(row[0], bytes) ) + else row[0] ) except Exception as e: msg = ( @@ -286,8 +362,7 @@ def deserialize_row(self, row: str) -> Union["BaseModel", Any]: f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) self.logger.error(msg) - self._exception = e # type: ignore - raise OpenBBError(msg) from e + raise OpenBBError(e) from e def fetch_all(self, limit: Optional[int] = None) -> list: """Fetch all the results from the SQLite database.""" @@ -299,15 +374,50 @@ def fetch_all(self, limit: Optional[int] = None) -> list: f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) self.logger.error(msg) - self._exception = e # type: ignore - raise OpenBBError(msg) from e + raise OpenBBError(e) from e - async def _query_db(self, sql) -> list: + async def _get_latest_results( + self, symbol: Optional[str] = None, limit: Optional[int] = None + ) -> list: + """Get the latest records from the database. Optionally filter by symbol.""" + if symbol: + symbols = symbol.split(",") + sym_str = "(" + for sym in symbols: + sym_str += f"'{sym.upper()}'" + ("," if sym != symbols[-1] else "") + sym_str += ")" + query = f"json_extract (message, '$.symbol') IN {sym_str}" + else: + query = f"SELECT message FROM {self.table_name}" # noqa + + query += " ORDER BY json_extract (message, '$.date') DESC" + + if limit is not None: + query += f" LIMIT {limit};" + + return await self._query_db(query) + + def get_latest_results( + self, symbol: Optional[str] = None, limit: Optional[int] = None + ) -> list: + """Get the latest records from the database. Optionally filter by symbol.""" + try: + return run_async(self._get_latest_results, symbol, limit) + except Exception as e: + msg = ( + "Unexpected error while getting latest records ->" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" + ) + self.logger.error(msg) + + async def _query_db(self, sql, parameters: Optional[Iterable[Any]] = None) -> list: """Query the SQLite database.""" # pylint: disable=import-outside-toplevel import aiosqlite # noqa import json + if not sql or sql in ("", "''"): + raise OpenBBError("Empty query not allowed.") query = ( sql if sql.startswith("SELECT") @@ -316,46 +426,58 @@ async def _query_db(self, sql) -> list: if not query.endswith(";"): query += ";" - if not query.startswith("SELECT"): - raise OpenBBError( - "Operation not allowed. Only 'SELECT' operations allowed." - ) + if ( + not query.startswith("SELECT") + or any(x.lower() in query.lower() for x in CHECK_FOR) + or (self.table_name not in query and "message" not in query) + ): + raise OpenBBError(f"Invalid operation: {sql}.") + rows: list = [] try: conn_kwargs = self.kwargs.copy() if not conn_kwargs.get("check_same_thread"): conn_kwargs["check_same_thread"] = False + + if ":" not in self.results_file: + results_file = ( + "file:" + + ( + self.results_file + if self.results_file.startswith("/") + else "/" + self.results_file + ) + + "?mode=ro" + ) + else: + results_file = ( + self.results_file + + f"{'&mode=ro' if '?' in self.results_file else '?mode=ro'}" + ) + conn_kwargs["uri"] = True + async with aiosqlite.connect( - self.results_file, + results_file, loop=self.loop, **conn_kwargs, - ) as conn, conn.execute(query) as cursor: + ) as conn, conn.execute(query, parameters) as cursor: async for row in cursor: rows.append( - self.data_model.model_validate_json(row[0]) - if self.data_model is not None - and query.startswith( - f"SELECT message FROM {self.table_name}" # noqa - ) - else ( - json.loads(row[0]) - if ( - ( - isinstance(row[0], str) - and ( - row[0].startswith("{") or row[0].startswith("[") - ) - ) - or isinstance(row[0], bytes) + json.loads(row[0]) + if ( + ( + isinstance(row[0], str) + and (row[0].startswith("{") or row[0].startswith("[")) ) - else row[0] + or isinstance(row[0], bytes) ) + else row[0] ) return rows except Exception as e: # pylint: disable=broad-except - raise e from e + raise OpenBBError(e) from e - def query(self, sql: str) -> list: + def query(self, sql: str, parameters: Optional[Iterable[Any]] = None) -> list: """ Run a SELECT query to the database. @@ -379,11 +501,10 @@ def query(self, sql: str) -> list: >>> """ try: - return run_async(self._query_db, sql) + return run_async(self._query_db, sql, parameters) except Exception as e: # pylint: disable=broad-except - msg = f"Unexpected error while querying SQLite database -> {e.__class__.__name__}: {e}" + msg = f"{e.__class__.__name__ if hasattr(e, '__class__') else e}: {e.args}" self.logger.error(msg) - raise OpenBBError(e) from e async def _clear_results(self): """Clear the results from the SQLite database.""" @@ -415,7 +536,7 @@ async def _clear_results(self): self.results_file, ) except Exception as e: # pylint: disable=broad-except - raise e from e + raise OpenBBError(e) from e def clear_results(self) -> None: """Clear all results from the SQLite database.""" diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py b/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py index d54d590e2786..4ecddf979630 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py @@ -11,7 +11,7 @@ def __init__( self, max_size: int = 10000, max_retries=5, - backoff_factor=0.5, + backoff_factor=0.75, logger: Optional[logging.Logger] = None, ): """Initialize the MessageQueue.""" diff --git a/openbb_platform/extensions/websockets/openbb_websockets/listen.py b/openbb_platform/extensions/websockets/openbb_websockets/listen.py index 329dc745828e..da5f5cf21987 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/listen.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/listen.py @@ -53,11 +53,14 @@ async def listen( # noqa: PLR0915 # pylint: disable=too-many-branches,too-many async with websockets.connect(url, **kwargs) as websocket: self.websocket = websocket url = clean_message(url) - msg = f"\nListening for messages from {clean_message(url)}" + msg = f"\nConnecting to {clean_message(url)} ..." self.logger.info(msg) for handler in self.logger.handlers: handler.flush() async for message in websocket: + if "invalid SQL" in message: + raise websockets.exceptions.WebSocketException(message) + if ( isinstance(message, str) and "Invalid authentication token" in message @@ -79,6 +82,9 @@ async def listen( # noqa: PLR0915 # pylint: disable=too-many-branches,too-many msg = f"The process hosting {clean_message(url)} was terminated." self.logger.error(msg) break + except websockets.exceptions.WebSocketException as error: + self.logger.error(error) + break except websockets.exceptions.InvalidURI as error: msg = f"Invalid URI -> {error}" self.logger.error(msg) @@ -97,8 +103,12 @@ async def listen( # noqa: PLR0915 # pylint: disable=too-many-branches,too-many msg = f"An error occurred while attempting to connect to: {clean_message(url)} -> {error}" self.logger.error(msg) break + except Exception as error: # pylint: disable=broad-except - msg = f"Unexpected error -> {error.__class__.__name__}: {error}" + msg = ( + "Unexpected error -> " + f"{error.__class__.__name__ if hasattr(error, '__class__') else error}: {error.args}" + ) self.logger.error(msg) raise OpenBBError(error) from error finally: diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index ef1664d8e207..139eb41f1a3a 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -131,12 +131,15 @@ async def read_stdin_and_update_symbols(): if not line: break - line = json.loads(line.strip()) + if "qsize" in line: + logger.info(f"PROVIDER INFO: Queue size: {queue.queue.qsize()}") + else: + line = json.loads(line.strip()) - if line: - symbol = line.get("symbol") - event = line.get("event") - await update_symbols(symbol, event) + if line: + symbol = line.get("symbol") + event = line.get("event") + await update_symbols(symbol, event) async def process_message(message): # pylint: disable=too-many-branches @@ -211,9 +214,17 @@ async def process_message(message): # pylint: disable=too-many-branches async def connect_and_stream(): """Connect to the WebSocket and stream data to file.""" + tasks: set = set() + handler_task = asyncio.create_task( queue.process_queue(lambda message: process_message(message)) ) + tasks.add(handler_task) + for i in range(0, 15): + new_task = asyncio.create_task( + queue.process_queue(lambda message: process_message(message)) + ) + tasks.add(new_task) stdin_task = asyncio.create_task(read_stdin_and_update_symbols()) ticker: list = [] @@ -265,10 +276,11 @@ async def connect_and_stream(): sys.exit(1) finally: - handler_task.cancel() - await handler_task - stdin_task.cancel() - await stdin_task + tasks.add(stdin_task) + for task in tasks: + task.cancel() + await task + asyncio.gather(*tasks, return_exceptions=True) sys.exit(0) @@ -290,5 +302,5 @@ async def connect_and_stream(): logger.error(ERR) finally: - loop.stop() + loop.close() sys.exit(0) From 34a02534574422bc474dee2d40d8ada690a066ce Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 21 Dec 2024 09:18:50 -0800 Subject: [PATCH 098/119] f-string --- .../core/openbb_core/provider/utils/websockets/broadcast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py index 4f9c5d717969..6c33e4bd5ca4 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py @@ -303,7 +303,7 @@ async def stream_results( # noqa: PLR0915 # pylint: disable=too-many-branches except WebSocketDisconnect: pass except Exception as e: # pylint: disable=broad-except - msg = f"{e.__class__.__name__ if hasattr(e, "__class__") else e} -> {e}" + msg = f"{e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e}" self.logger.error(msg) return From 696b1da8c2802ecd07e00ca4545dd4173a75ccd7 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 21 Dec 2024 13:33:42 -0800 Subject: [PATCH 099/119] add some micro scaling to the queue when it is growing --- .../provider/utils/websockets/database.py | 8 +- .../provider/utils/websockets}/listen.py | 0 .../utils/websockets/message_queue.py | 16 + .../utils/websockets/test_database.py | 12 +- .../websockets/openbb_websockets/broadcast.py | 298 ------------------ .../openbb_polygon/utils/websocket_client.py | 41 ++- 6 files changed, 54 insertions(+), 321 deletions(-) rename openbb_platform/{extensions/websockets/openbb_websockets => core/openbb_core/provider/utils/websockets}/listen.py (100%) delete mode 100644 openbb_platform/extensions/websockets/openbb_websockets/broadcast.py diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index ef58e03f56a1..6df601c385bd 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -143,11 +143,11 @@ def __init__( # pylint: disable=too-many-positional-arguments temp_file_path = temp_file.name self.results_path = Path(temp_file_path).absolute() self.results_file = temp_file_path - if ":" in results_file: - self.results_file = results_file - self.results_path = results_file # type: ignore - kwargs["uri"] = True else: + if ":" in results_file: + self.results_file = results_file + self.results_path = results_file # type: ignore + kwargs["uri"] = True self.results_path = Path(results_file).absolute() self.results_file = results_file diff --git a/openbb_platform/extensions/websockets/openbb_websockets/listen.py b/openbb_platform/core/openbb_core/provider/utils/websockets/listen.py similarity index 100% rename from openbb_platform/extensions/websockets/openbb_websockets/listen.py rename to openbb_platform/core/openbb_core/provider/utils/websockets/listen.py diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py b/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py index 4ecddf979630..3eaa396585b4 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py @@ -39,6 +39,22 @@ async def enqueue(self, message): retries = 0 while retries < self.max_retries: + + if self.queue.qsize() / self.queue.maxsize > 0.3: + await sleep(0.000005) + if self.queue.qsize() / self.queue.maxsize > 0.5: + await sleep(0.00005) + if self.queue.qsize() / self.queue.maxsize > 0.55: + await sleep(0.00005) + if self.queue.qsize() / self.queue.maxsize > 0.6: + await sleep(0.00005) + if self.queue.qsize() / self.queue.maxsize > 0.65: + await sleep(0.00005) + if self.queue.qsize() / self.queue.maxsize > 0.7: + await sleep(0.00005) + if self.queue.qsize() / self.queue.maxsize > 0.99: + await sleep(0.00005) + if self.queue.full(): retries += 1 msg = f"Queue is full. Retrying {retries}/{self.max_retries}..." diff --git a/openbb_platform/core/tests/provider/utils/websockets/test_database.py b/openbb_platform/core/tests/provider/utils/websockets/test_database.py index ff50d674cc78..ae4bef7c054d 100644 --- a/openbb_platform/core/tests/provider/utils/websockets/test_database.py +++ b/openbb_platform/core/tests/provider/utils/websockets/test_database.py @@ -13,7 +13,7 @@ @pytest.fixture(scope="module") def database(): """Return a MessageQueue instance.""" - return Database(table_name="test_table") + return Database(table_name="test") def test_setup_database(database): @@ -61,7 +61,7 @@ def test_multiple_connections(database): ) another_db = Database( results_file=database.results_file, - table_name="other_table", + table_name="other_test", ) assert new_db.fetch_all()[0] == MOCK_MESSAGES[0] database.write_to_db(MOCK_MESSAGES[1]) @@ -87,16 +87,16 @@ def test_query_db(database): assert len(database.query(query)) == 2 query = "json_extract (message, '$.type') == 'quote'" assert len(database.query(query)) == 1 - query = "SELECT message FROM test_table WHERE json_extract (message, '$.type') = 'trade'" + query = "SELECT message FROM test WHERE json_extract (message, '$.type') = 'trade'" assert len(database.query(query)) == 2 - query = "SELECT json_extract (message, '$.symbol') FROM test_table WHERE json_extract (message, '$.type') = 'trade'" + query = "SELECT json_extract (message, '$.symbol') FROM test WHERE json_extract (message, '$.type') = 'trade'" assert database.query(query) == ["test1", "test3"] def test_limit(): """Test if the limit parameter is working and that the auto increment index doesn't reset when cleared.""" database = Database( - table_name="test_limit_table", + table_name="test_limit", limit=2, ) assert database @@ -112,5 +112,5 @@ def test_limit(): database.clear_results() assert database.fetch_all() == [] database.write_to_db(MOCK_MESSAGES[0]) - query = "SELECT id FROM test_limit_table" + query = "SELECT id FROM test_limit" assert database.query(query)[0] > 3 diff --git a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py b/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py deleted file mode 100644 index 950a7a41ac0e..000000000000 --- a/openbb_platform/extensions/websockets/openbb_websockets/broadcast.py +++ /dev/null @@ -1,298 +0,0 @@ -"""Broadcast server for streaming results to connected clients via WebSocket.""" - -import asyncio -import json -import sys -from pathlib import Path -from typing import Optional - -import uvicorn -from fastapi import FastAPI, WebSocket, WebSocketDisconnect -from openbb_core.provider.utils.helpers import run_async -from openbb_core.provider.utils.websockets.database import Database -from openbb_core.provider.utils.websockets.helpers import get_logger, parse_kwargs -from starlette.websockets import WebSocketState - -connected_clients: set = set() - -kwargs = parse_kwargs() - -HOST = kwargs.pop("host", None) or "localhost" -PORT = kwargs.pop("port", None) or 6666 -PORT = int(PORT) - -RESULTS_FILE = kwargs.pop("results_file", None) -TABLE_NAME = kwargs.pop("table_name", None) or "records" -SLEEP_TIME = kwargs.pop("sleep_time", None) or 0.25 -AUTH_TOKEN = kwargs.pop("auth_token", None) - -DATABASE = Database(results_file=RESULTS_FILE, table_name=TABLE_NAME) - -app = FastAPI() - - -async def read_stdin(broadcast_server): - """Read from stdin.""" - while True: - line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline) - sys.stdin.flush() - - if not line: - break - - try: - command = ( - json.loads(line.strip()) - if line.strip().startswith("{") or line.strip().startswith("[") - else line.strip() - ) - await broadcast_server.broadcast(json.dumps(command)) - except json.JSONDecodeError: - broadcast_server.logger.error("Invalid JSON received from stdin") - - -@app.websocket("/") -async def websocket_endpoint( # noqa: PLR0915 - websocket: WebSocket, auth_token: Optional[str] = None -): - """Connect to the broadcast server.""" - - broadcast_server = BroadcastServer( - RESULTS_FILE, - TABLE_NAME, - SLEEP_TIME, - str(AUTH_TOKEN), - ) - auth_token = str(auth_token) - - if ( - broadcast_server.auth_token is not None - and auth_token - != broadcast_server._decrypt_value( # pylint: disable=protected-access - broadcast_server.auth_token - ) - ): - await websocket.accept() - await websocket.send_text( - "ERROR: Invalid authentication token. Could not connect to the broadcast." - ) - broadcast_server.logger.error( - "ERROR: Invalid authentication token passed by a client connecting." - ) - await websocket.close(code=1008, reason="Invalid authentication token") - return - - await websocket.accept() - - if RESULTS_FILE is None: - raise ValueError("Results file path is required for WebSocket server.") - - broadcast_server.websocket = websocket - connected_clients.add(broadcast_server) - - stream_task = asyncio.create_task(broadcast_server.stream_results()) - stdin_task = asyncio.create_task(read_stdin(broadcast_server)) - try: - await websocket.receive_text() - - except WebSocketDisconnect: - pass - except Exception as e: # pylint: disable=broad-except - msg = f"Unexpected error: {e.__class__.__name__} -> {e}" - broadcast_server.logger.error(msg) - finally: - if broadcast_server in connected_clients: - connected_clients.remove(broadcast_server) - stream_task.cancel() - stdin_task.cancel() - try: - await stream_task - await stdin_task - except asyncio.CancelledError: - broadcast_server.logger.info("INFO: A listener task was cancelled.") - except Exception as e: # pylint: disable=broad-except - msg = f"Unexpected error while cancelling stream task: {e.__class__.__name__} -> {e}" - broadcast_server.logger.error(msg) - if websocket.client_state != WebSocketState.DISCONNECTED: - try: - await websocket.close() - except RuntimeError as e: - msg = f"Unexpected error while closing websocket: {e.__class__.__name__} -> {e}" - broadcast_server.logger.error(msg) - - -class BroadcastServer: # pylint: disable=too-many-instance-attributes - """Stream new results from a continuously written SQLite database. - - Not intended to be used directly, it is initialized by the server app when it accepts a new connection. - It is responsible for reading the results database and sending new messages to the connected client(s). - """ - - def __init__( - self, - results_file, - table_name, - sleep_time: float = 0.25, - auth_token: Optional[str] = None, - ): - """Initialize the BroadcastServer instance.""" - # pylint: disable=import-outside-toplevel - import os - - self.results_file = results_file - self.table_name = table_name - self.logger = get_logger("openbb.websocket.broadcast_server") - self.sleep_time = sleep_time - self._app = app - self._key = os.urandom(32) - self._iv = os.urandom(16) - self.auth_token = self._encrypt_value(auth_token) if auth_token else None - self.websocket = None - self.kwargs = kwargs - - def _encrypt_value(self, value: str) -> str: - """Encrypt the value for storage.""" - # pylint: disable=import-outside-toplevel - from openbb_core.provider.utils.websockets.helpers import encrypt_value - - return encrypt_value(self._key, self._iv, value) - - def _decrypt_value(self, value: str) -> str: - """Decrypt the value for use.""" - # pylint: disable=import-outside-toplevel - from openbb_core.provider.utils.websockets.helpers import decrypt_value - - return decrypt_value(self._key, self._iv, value) - - async def stream_results( # noqa: PLR0915 # pylint: disable=too-many-branches - self, - ): - """Continuously read the database and send new messages as JSON via WebSocket.""" - # pylint: disable=import-outside-toplevel - import sqlite3 # noqa - from openbb_core.app.model.abstract.error import OpenBBError - - file_path = Path(self.results_file).absolute() - last_id = 0 - - if not file_path.exists(): - self.logger.error("Results file not found: %s", str(file_path)) - return - conn = sqlite3.connect(self.results_file) - cursor = conn.cursor() - cursor.execute(f"SELECT MAX(id) FROM {self.table_name}") # noqa:S608 - last_id = cursor.fetchone()[0] or 0 - conn.close() - - try: # pylint: disable=too-many-nested-blocks - while True: - try: - if file_path.exists(): - conn = sqlite3.connect(self.results_file) - cursor = conn.cursor() - cursor.execute( - f"SELECT * FROM {self.table_name} WHERE id > ?", # noqa:S608 - (last_id,), - ) - rows = cursor.fetchall() - conn.close() - - if rows: - for row in rows: - _, message = row - await self.broadcast(json.dumps(json.loads(message))) - last_id = max(row[0] for row in rows) - else: - self.logger.error("Results file not found: %s", str(file_path)) - break - - await asyncio.sleep(self.sleep_time) - except KeyboardInterrupt: - self.logger.info("\nResults stream cancelled.") - break - except sqlite3.OperationalError as e: - if "no such table" in str(e): - self.logger.error( - "Results file was removed by the parent process." - ) - break - raise OpenBBError(e) from e - except asyncio.CancelledError: - break - except WebSocketDisconnect: - pass - except Exception as e: # pylint: disable=broad-except - msg = f"Unexpected error: {e.__class__.__name__} -> {e}" - self.logger.error(msg) - return - - async def broadcast(self, message: str): - """Broadcast a message to all connected connected clients.""" - disconnected_clients = set() - for client in connected_clients.copy(): - try: - await client.websocket.send_json(message) - except WebSocketDisconnect: - disconnected_clients.add(client) - except Exception as e: # pylint: disable=broad-except - msg = f"Unexpected error: {e.__class__.__name__} -> {e}" - self.logger.error(msg) - disconnected_clients.add(client) - # Remove disconnected connected clients - for client in disconnected_clients: - connected_clients.remove(client) - - def start_app(self, host: str = "127.0.0.1", port: int = 6666): - """Start the FastAPI app with Uvicorn.""" - uvicorn.run( - self._app, - host=host, - port=port, - **kwargs, - ) - - -def create_broadcast_server( - results_file: str, - table_name: str, - sleep_time: float = 0.25, - auth_token: Optional[str] = None, -): - """Create a new BroadcastServer instance.""" - return BroadcastServer(results_file, table_name, sleep_time, auth_token) - - -def main(): - """Run the main function.""" - broadcast_server = create_broadcast_server( - RESULTS_FILE, - TABLE_NAME, - SLEEP_TIME, - str(AUTH_TOKEN), - ) - - try: - broadcast_server.start_app( - host=HOST, - port=PORT, - **kwargs, - ) - except TypeError as e: - msg = ( - f"ERROR: Invalid keyword argument passed to unvicorn. -> {e.args[0]}\n" - ) - broadcast_server.logger.error(msg) - except KeyboardInterrupt: - broadcast_server.logger.info("INFO: Broadcast server terminated.") - finally: - sys.exit(0) - - -if __name__ == "__main__": - if not RESULTS_FILE: - raise ValueError("Results file path is required for Broadcast server.") - - if not Path(RESULTS_FILE).absolute().exists(): - run_async(DATABASE._setup_database) # pylint: disable=protected-access - - main() diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index 9ecc4807c50a..e8045f82558f 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -7,7 +7,6 @@ import sys import websockets -import websockets.exceptions from openbb_core.provider.utils.websockets.database import Database from openbb_core.provider.utils.websockets.helpers import ( get_logger, @@ -34,7 +33,7 @@ } logger = get_logger("openbb.websocket.polygon") -queue = MessageQueue(logger=logger) +queue = MessageQueue(logger=logger, backoff_factor=2) command_queue = MessageQueue(logger=logger) kwargs = parse_kwargs() @@ -159,11 +158,15 @@ async def read_stdin(): if not line: break - try: - command = json.loads(line.strip()) - await command_queue.enqueue(command) - except json.JSONDecodeError: - logger.error("Invalid JSON received from stdin") + if "qsize" in line: + logger.info(f"PROVIDER INFO: Queue size: {queue.queue.qsize()}") + + else: + try: + command = json.loads(line.strip()) + await command_queue.enqueue(command) + except json.JSONDecodeError: + logger.error("Invalid JSON received from stdin") async def process_stdin_queue(websocket): @@ -211,9 +214,17 @@ async def process_message(message): async def connect_and_stream(): """Connect to the WebSocket and stream data to file.""" + tasks: set = set() + handler_task = asyncio.create_task( queue.process_queue(lambda message: process_message(message)) ) + tasks.add(handler_task) + for i in range(0, 48): + new_task = asyncio.create_task( + queue.process_queue(lambda message: process_message(message)) + ) + tasks.add(new_task) stdin_task = asyncio.create_task(read_stdin()) try: connect_kwargs = CONNECT_KWARGS.copy() @@ -281,14 +292,19 @@ async def connect_and_stream(): sys.exit(1) except Exception as e: - msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e}" + msg = ( + f"PROVIDER ERROR: Unexpected error -> " + f"{e.__class__.__name__ if hasattr(e, '__class__') else e}: {e}" + ) logger.error(msg) sys.exit(1) finally: - handler_task.cancel() - stdin_task.cancel() - await asyncio.gather(handler_task, stdin_task, return_exceptions=True) + tasks.add(stdin_task) + for task in tasks: + task.cancel() + await task + asyncio.gather(*tasks, return_exceptions=True) sys.exit(0) @@ -296,7 +312,6 @@ async def connect_and_stream(): try: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - loop.set_exception_handler(lambda loop, context: None) for sig in (signal.SIGINT, signal.SIGTERM): loop.add_signal_handler(sig, handle_termination_signal, logger) @@ -315,5 +330,5 @@ async def connect_and_stream(): logger.error(ERR) finally: - loop.stop() + loop.close() sys.exit(0) From 23013e6c6bc41b8c68f705b4bb1371538e5cc9ec Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 21 Dec 2024 14:28:50 -0800 Subject: [PATCH 100/119] handle huge polgon crypto message volume --- .../openbb_polygon/utils/websocket_client.py | 20 +++++++++---------- .../openbb_tiingo/utils/websocket_client.py | 7 +++++-- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index e8045f82558f..002253409cf4 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -156,11 +156,10 @@ async def read_stdin(): sys.stdin.flush() if not line: - break + continue if "qsize" in line: logger.info(f"PROVIDER INFO: Queue size: {queue.queue.qsize()}") - else: try: command = json.loads(line.strip()) @@ -221,21 +220,20 @@ async def connect_and_stream(): ) tasks.add(handler_task) for i in range(0, 48): - new_task = asyncio.create_task( - queue.process_queue(lambda message: process_message(message)) + new_task = asyncio.shield( + asyncio.create_task( + queue.process_queue(lambda message: process_message(message)) + ) ) tasks.add(new_task) - stdin_task = asyncio.create_task(read_stdin()) + stdin_task = asyncio.shield(asyncio.create_task(read_stdin())) try: connect_kwargs = CONNECT_KWARGS.copy() - if "ping_timeout" not in connect_kwargs: - connect_kwargs["ping_timeout"] = None - if "close_timeout" not in connect_kwargs: - connect_kwargs["close_timeout"] = None + connect_kwargs["max_size"] = None + connect_kwargs["read_limit"] = 2**32 try: async with websockets.connect(URL, **connect_kwargs) as websocket: - await login(websocket) response = await websocket.recv() messages = json.loads(response) @@ -312,7 +310,7 @@ async def connect_and_stream(): try: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - + loop.set_exception_handler(lambda loop, context: None) for sig in (signal.SIGINT, signal.SIGTERM): loop.add_signal_handler(sig, handle_termination_signal, logger) diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index 139eb41f1a3a..0f7cf6a92650 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -288,7 +288,7 @@ async def connect_and_stream(): try: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - + loop.set_exception_handler(lambda loop, context: None) loop.add_signal_handler(signal.SIGTERM, handle_termination_signal, logger) asyncio.run_coroutine_threadsafe( @@ -298,7 +298,10 @@ async def connect_and_stream(): loop.run_forever() except Exception as e: # pylint: disable=broad-except - ERR = f"Unexpected error -> {e.__class__.__name__}: {e}" + ERR = ( + f"PROVIDER ERROR: Unexpected error -> " + f"{e.__class__.__name__ if hasattr(e, '__class__') else e}: {e}" + ) logger.error(ERR) finally: From 9030261acb30c3b3115595a6cc940602523ae41b Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 21 Dec 2024 17:58:32 -0800 Subject: [PATCH 101/119] polygon no longer supports L2 crypto --- .../models/websocket_connection.py | 122 +++++++----------- .../openbb_polygon/utils/websocket_client.py | 67 +++++----- 2 files changed, 83 insertions(+), 106 deletions(-) diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 3277d4985464..312cc25548d0 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -28,7 +28,6 @@ ) from pydantic import Field, field_validator, model_validator - ASSET_CHOICES = [ "stock", "stock_delayed", @@ -46,7 +45,6 @@ "aggs_sec": "XAS", "trade": "XT", "quote": "XQ", - "l2": "XL2", "fmv": "FMV", }, "fx": { @@ -125,6 +123,17 @@ class PolygonWebSocketQueryParams(WebSocketQueryParams): "multiple_items_allowed": False, "choices": ASSET_CHOICES, }, + "feed": { + "multiple_items_allowed": False, + "choices": [ + "aggs_min", + "aggs_sec", + "trade", + "quote", + "fmv", + "value", + ], + }, } symbol: str = Field( @@ -164,7 +173,6 @@ class PolygonWebSocketQueryParams(WebSocketQueryParams): - aggs_sec: XAS. - trade: XT. - quote: XQ. - - l2: XL2. - fmv: FMV. FX @@ -189,12 +197,10 @@ class PolygonWebSocketQueryParams(WebSocketQueryParams): description="The asset type associated with the symbol(s)." + " Choose from: stock, stock_delayed, fx, crypto.", ) - feed: Literal["aggs_min", "aggs_sec", "trade", "quote", "l2", "fmv", "value"] = ( - Field( - default="aggs_sec", - description="The asset type associated with the symbol." - + "l2 is only available for crypto, and value is only available for index.", - ) + feed: Literal["aggs_min", "aggs_sec", "trade", "quote", "fmv", "value"] = Field( + default="aggs_sec", + description="The asset type associated with the symbol." + + "Value is only available for index.", ) @model_validator(mode="before") @@ -203,26 +209,30 @@ def _validate_feed(cls, values): """Validate the feed.""" feed = values.get("feed") asset_type = values.get("asset_type") - if asset_type == "fx" and feed in ["trade", "l2", "value"]: + if asset_type == "fx" and feed in ["trade", "value"]: raise ValueError(f"FX does not support the {feed} feed.") - if asset_type in [ - "stock", - "stock_delayed", - "options", - "options_delayed", - ] and feed in ["l2", "value"]: + if ( + asset_type + in [ + "stock", + "stock_delayed", + "options", + "options_delayed", + ] + and feed == "value" + ): raise ValueError( f"Asset type, {asset_type}, does not support the {feed} feed." ) if asset_type in ["index", "index_delayed"] and feed in [ "trade", "quote", - "l2", "fmv", ]: raise ValueError(f"Index does not support the {feed} feed.") if asset_type == "crypto" and feed == "value": raise ValueError(f"Crypto does not support the {feed} feed.") + return values @@ -291,6 +301,8 @@ def _validate_date(cls, v): def _validate_model(cls, values): """Validate the model.""" _ = values.pop("s", None) + symbol = values.pop("p", "") if "p" in values else values.pop("pair", "") + values["pair"] = symbol.replace("-", "").replace("/", "") return values @@ -366,6 +378,8 @@ def _validate_exchange(cls, v): def _validate_model(cls, values): """Validate the model.""" _ = values.pop("i", None) + symbol = values.pop("pair", "") + values["pair"] = symbol.replace("-", "") return values @@ -433,65 +447,17 @@ def _validate_model(cls, values): """Validate the model.""" lp = values.pop("lp", None) ls = values.pop("ls", None) + if lp: values["last_price"] = lp + if ls: values["last_size"] = ls - return values - - -class PolygonCryptoL2WebSocketData(WebSocketData): - """Polygon Crypto L2 WebSocket data model.""" + symbol = values.pop("pair", "") + values["pair"] = symbol.replace("-", "") - __alias_dict__ = { - "type": "ev", - "symbol": "pair", - "date": "t", - "exchange": "x", - "bid": "b", - "ask": "a", - "received_at": "r", - } - - type: str = Field( - description="The type of data.", - ) - date: datetime = Field( - description=DATA_DESCRIPTIONS.get("date", ""), - ) - received_at: datetime = Field( - description="The time the data was received by Polygon.", - ) - symbol: str = Field( - description=DATA_DESCRIPTIONS.get("symbol", ""), - ) - exchange: str = Field( - default=None, - description="The exchange of the data.", - ) - bid: list[list[float]] = Field( - description="An array of bid prices, where each entry contains two elements:" - + " the first is the bid price, and the second is the size, with a maximum depth of 100.", - json_schema_extra={"x-unit_measurement": "currency"}, - ) - ask: list[list[float]] = Field( - description="An array of ask prices, where each entry contains two elements:" - + " the first is the ask price, and the second is the size, with a maximum depth of 100.", - json_schema_extra={"x-unit_measurement": "currency"}, - ) - - @field_validator("date", "received_at", mode="before", check_fields=False) - @classmethod - def _validate_date(cls, v): - """Validate the date.""" - return validate_date(cls, v) - - @field_validator("exchange", mode="before", check_fields=False) - @classmethod - def _validate_exchange(cls, v): - """Validate the exchange.""" - return CRYPTO_EXCHANGE_MAP.get(v, str(v)) + return values class PolygonFXQuoteWebSocketData(WebSocketData): @@ -545,6 +511,8 @@ def _validate_exchange(cls, v): def _validate_model(cls, values): """Validate the model.""" _ = values.pop("i", None) + symbol = values.pop("p", "") + values["p"] = symbol.replace("/", "") return values @@ -1103,11 +1071,22 @@ class PolygonFairMarketValueData(WebSocketData): json_schema_extra={"x-unit_measurement": "currency"}, ) + @field_validator("date", mode="before", check_fields=False) + @classmethod + def _validate_date(cls, v): + """Validate the date.""" + return validate_date(cls, v) + + @field_validator("symbol", mode="before", check_fields=False) + @classmethod + def _validate_symbol(cls, v): + """Validate the symbol.""" + return v.replace("-", "").replace("/", "") + MODEL_MAP = { "XT": PolygonCryptoTradeWebSocketData, "XQ": PolygonCryptoQuoteWebSocketData, - "XL2": PolygonCryptoL2WebSocketData, "XA": PolygonCryptoAggsWebSocketData, "XAS": PolygonCryptoAggsWebSocketData, "FMV": PolygonFairMarketValueData, @@ -1160,7 +1139,6 @@ class PolygonWebSocketData(Data): - Aggs: XAS, XA - PolygonCryptoAggsWebSocketData - Trade: XT - PolygonCryptoTradeWebSocketData - Quote: XQ - PolygonCryptoQuoteWebSocketData - - L2: XL2 - PolygonCryptoL2WebSocketData - Fair Market Value: FMV - PolygonFairMarketValueData FX diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index 002253409cf4..f0ad1d8b1ef2 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -101,7 +101,11 @@ async def handle_symbol(symbol): _feed, _ticker = ticker.split(".") if "." in ticker else (feed, ticker) ticker = f"{_feed}.O:{_ticker}" - new_symbols.append(ticker) + if ticker == "XL2.*": + symbol_error = f"SymbolError -> {symbol}: L2 Crypto does not support the all-symbols wildcard." + logger.error(symbol_error) + else: + new_symbols.append(ticker) return ",".join(new_symbols) @@ -130,7 +134,11 @@ async def login(websocket): sys.exit(1) logger.info("PROVIDER INFO: %s", msg.get("message")) except Exception as e: - logger.error("PROVIDER ERROR: %s -> %s", e.__class__.__name__, e.args[0]) + logger.error( + "PROVIDER ERROR: %s -> %s", + e.__class__.__name__ if hasattr(e, "__class__") else e, + e.args[0], + ) sys.exit(1) @@ -154,15 +162,9 @@ async def read_stdin(): while True: line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline) sys.stdin.flush() - - if not line: - continue - - if "qsize" in line: - logger.info(f"PROVIDER INFO: Queue size: {queue.queue.qsize()}") - else: + if line: try: - command = json.loads(line.strip()) + command = line.strip() if "qsize" in line else json.loads(line.strip()) await command_queue.enqueue(command) except json.JSONDecodeError: logger.error("Invalid JSON received from stdin") @@ -172,10 +174,13 @@ async def process_stdin_queue(websocket): """Process the command queue.""" while True: command = await command_queue.dequeue() - symbol = command.get("symbol") - event = command.get("event") - if symbol and event: - await subscribe(websocket, symbol, event) + if command == "qsize": + logger.info(f"PROVIDER INFO: Queue size: {queue.queue.qsize()}") + else: + symbol = command.get("symbol") + event = command.get("event") + if symbol and event: + await subscribe(websocket, symbol, event) async def process_message(message): @@ -219,11 +224,9 @@ async def connect_and_stream(): queue.process_queue(lambda message: process_message(message)) ) tasks.add(handler_task) - for i in range(0, 48): - new_task = asyncio.shield( - asyncio.create_task( - queue.process_queue(lambda message: process_message(message)) - ) + for i in range(0, 64): + new_task = asyncio.create_task( + queue.process_queue(lambda message: process_message(message)) ) tasks.add(new_task) stdin_task = asyncio.shield(asyncio.create_task(read_stdin())) @@ -231,6 +234,8 @@ async def connect_and_stream(): connect_kwargs = CONNECT_KWARGS.copy() connect_kwargs["max_size"] = None connect_kwargs["read_limit"] = 2**32 + connect_kwargs["close_timeout"] = 10 + connect_kwargs["ping_timeout"] = None try: async with websockets.connect(URL, **connect_kwargs) as websocket: @@ -242,22 +247,16 @@ async def connect_and_stream(): response = await websocket.recv() messages = json.loads(response) await process_message(messages) + cmd_task = asyncio.get_running_loop().create_task( + process_stdin_queue(websocket) + ) while True: - cmd_task = asyncio.create_task(process_stdin_queue(websocket)) - msg_task = asyncio.create_task(websocket.recv()) - done, pending = await asyncio.wait( - [cmd_task, msg_task], - return_when=asyncio.FIRST_COMPLETED, - ) - for task in pending: - task.cancel() - - for task in done: - if task == cmd_task: - await cmd_task - elif task == msg_task: - messages = task.result() - await asyncio.shield(queue.enqueue(json.loads(messages))) + messages = await websocket.recv() + await queue.enqueue(json.loads(messages)) + + cmd_task.cancel() + await cmd_task + asyncio.gather(*cmd_task, return_exceptions=True) except websockets.InvalidStatusCode as e: if e.status_code == 404: From 8cd63961fc493d7092ad956faffc12b45242458f Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Sat, 21 Dec 2024 18:14:20 -0800 Subject: [PATCH 102/119] pad the queue buffer a little more --- .../provider/utils/websockets/message_queue.py | 8 ++++++-- .../polygon/openbb_polygon/utils/websocket_client.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py b/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py index 3eaa396585b4..f80f0bcb6a58 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py @@ -41,7 +41,7 @@ async def enqueue(self, message): while retries < self.max_retries: if self.queue.qsize() / self.queue.maxsize > 0.3: - await sleep(0.000005) + await sleep(0.00005) if self.queue.qsize() / self.queue.maxsize > 0.5: await sleep(0.00005) if self.queue.qsize() / self.queue.maxsize > 0.55: @@ -52,8 +52,12 @@ async def enqueue(self, message): await sleep(0.00005) if self.queue.qsize() / self.queue.maxsize > 0.7: await sleep(0.00005) - if self.queue.qsize() / self.queue.maxsize > 0.99: + if self.queue.qsize() / self.queue.maxsize > 0.75: + await sleep(0.00005) + if self.queue.qsize() / self.queue.maxsize > 0.8: await sleep(0.00005) + if self.queue.qsize() / self.queue.maxsize > 0.98: + await sleep(0.0005) if self.queue.full(): retries += 1 diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index f0ad1d8b1ef2..d9db715d022c 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -224,7 +224,7 @@ async def connect_and_stream(): queue.process_queue(lambda message: process_message(message)) ) tasks.add(handler_task) - for i in range(0, 64): + for i in range(0, 128): new_task = asyncio.create_task( queue.process_queue(lambda message: process_message(message)) ) From e338efe85a716fd0a8f6584afa37b2c72899d367 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 30 Dec 2024 22:36:08 -0800 Subject: [PATCH 103/119] a few breaking changes.. --- .../provider/utils/websockets/broadcast.py | 195 +++-- .../provider/utils/websockets/client.py | 228 +++-- .../provider/utils/websockets/database.py | 798 +++++++++++++++--- .../provider/utils/websockets/helpers.py | 2 +- .../utils/websockets/message_queue.py | 22 +- .../websockets/openbb_websockets/helpers.py | 2 +- .../websockets/openbb_websockets/models.py | 16 +- .../openbb_websockets/websockets_router.py | 10 +- .../models/websocket_connection.py | 2 +- .../openbb_intrinio/utils/websocket_client.py | 46 +- .../models/websocket_connection.py | 24 +- .../openbb_polygon/utils/websocket_client.py | 275 ++++-- .../models/websocket_connection.py | 25 +- .../openbb_tiingo/utils/websocket_client.py | 140 +-- 14 files changed, 1357 insertions(+), 428 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py index 6c33e4bd5ca4..9e230c538a44 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py @@ -4,14 +4,23 @@ import json import logging import os +import signal import sys from pathlib import Path from typing import Optional import uvicorn from fastapi import FastAPI, WebSocket, WebSocketDisconnect -from openbb_core.provider.utils.websockets.database import CHECK_FOR, Database -from openbb_core.provider.utils.websockets.helpers import parse_kwargs +from openbb_core.provider.utils.websockets.database import ( + CHECK_FOR, + Database, + kill_thread, +) +from openbb_core.provider.utils.websockets.helpers import ( + get_logger, + handle_termination_signal, + parse_kwargs, +) from starlette.websockets import WebSocketState kwargs = parse_kwargs() @@ -30,8 +39,13 @@ app = FastAPI() +CONNECTED_CLIENTS = set() +MAIN_CLIENT = None +STDIN_TASK = None +LOGGER = get_logger("broadcast-server") -async def read_stdin(broadcast_server): + +async def read_stdin(): """Read from stdin.""" while True: line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline) @@ -39,17 +53,30 @@ async def read_stdin(broadcast_server): sys.stdout.flush() if not line: - break + continue - try: - command = ( - json.loads(line.strip()) - if line.strip().startswith("{") or line.strip().startswith("[") - else line.strip() + if line.strip() == "numclients": + MAIN_CLIENT.logger.info( + "Number of connected clients: %i", len(CONNECTED_CLIENTS) ) - await broadcast_server.websocket.send_json(json.dumps(command)) - except json.JSONDecodeError: - broadcast_server.logger.error("Invalid JSON received from stdin") + continue + if len(CONNECTED_CLIENTS) > 0: + try: + command = ( + json.loads(line.strip()) + if line.strip().startswith("{") or line.strip().startswith("[") + else line.strip() + ) + except json.JSONDecodeError: + err_msg = f"Invalid JSON received from stdin -> {line}" + for client in CONNECTED_CLIENTS: + client.logger.error(err_msg) + + for client in CONNECTED_CLIENTS: + if client.websocket.client_state != WebSocketState.DISCONNECTED: + await client.websocket.send_json(command) + else: + CONNECTED_CLIENTS.remove(client) @app.websocket("/") @@ -108,42 +135,31 @@ async def websocket_endpoint( # noqa: PLR0915 raise ValueError("Results file path is required for WebSocket server.") broadcast_server.websocket = websocket - - stream_task = asyncio.create_task(broadcast_server.stream_results()) - stdin_task = asyncio.create_task(read_stdin(broadcast_server)) + CONNECTED_CLIENTS.add(broadcast_server) try: - await websocket.receive_text() + stream_task = asyncio.create_task(broadcast_server.stream_results()) + stdin_task = asyncio.create_task(read_stdin()) + await asyncio.gather(*[stdin_task, stream_task], return_exceptions=True) + + except (asyncio.CancelledError, RuntimeError): + broadcast_server.logger.info("A listener task was cancelled.") except WebSocketDisconnect: - pass + broadcast_server.logger.info("A listener connection was disconnected.") + except Exception as e: # pylint: disable=broad-except msg = f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" broadcast_server.logger.error(msg) + finally: + CONNECTED_CLIENTS.remove(broadcast_server) stream_task.cancel() + await stream_task stdin_task.cancel() try: - await stream_task await stdin_task except asyncio.CancelledError: broadcast_server.logger.info("A listener task was cancelled.") - for handler in broadcast_server.logger.handlers: - handler.flush() - except Exception as e: # pylint: disable=broad-except - msg = ( - f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" - ) - broadcast_server.logger.error(msg) - for handler in broadcast_server.logger.handlers: - handler.flush() - if websocket.client_state != WebSocketState.DISCONNECTED: - try: - await websocket.close() - except RuntimeError as e: - msg = f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" - broadcast_server.logger.error(msg) - for handler in broadcast_server.logger.handlers: - handler.flush() class BroadcastServer: # pylint: disable=too-many-instance-attributes @@ -218,11 +234,18 @@ async def stream_results( # noqa: PLR0915 # pylint: disable=too-many-branches return query = f"SELECT MAX(id) FROM {self.table_name}" # noqa:S608 + + async with self.database.get_connection("read") as conn: + cursor = await conn.execute(query) + last_id = (await cursor.fetchone())[0] + await cursor.close() + last_id = ( 0 if hasattr(self, "replay") and self.replay is True or replay is True - else self.database.query(query)[0] + else last_id ) + if sql and self.sql is None: self.sql = sql elif self.sql is not None and sql is None: @@ -246,46 +269,29 @@ async def stream_results( # noqa: PLR0915 # pylint: disable=too-many-branches try: # pylint: disable=too-many-nested-blocks while True: try: - if file_path.exists(): + async with self.database.get_connection("read") as conn: query = ( sql.replace(";", "") - + f" {'AND' if 'WHERE' in sql else 'WHERE'} id > {last_id}" + + f" {'AND' if 'WHERE' in sql else 'WHERE'} id > ?" if sql is not None - else f"SELECT * FROM {self.table_name} WHERE id > {last_id}" # noqa:S608 + else f"SELECT * FROM {self.table_name} WHERE id > ?" # noqa:S608 + + " ORDER BY json_extract (message, '$.date') ASC" ) - - if ":" not in self.results_file: - results_file = ( - "file:" - + ( - self.results_file - if self.results_file.startswith("/") - else "/" + self.results_file - ) - + "?mode=ro" + params = (last_id,) + cursor = await conn.execute(query, params) + rows = await cursor.fetchall() + if not rows: + await cursor.close() + await asyncio.sleep(1) + continue + for row in rows: + last_id = row[0] if row[0] > last_id else last_id + await self.websocket.send_json( + json.dumps(json.loads(row[1])) ) - else: - results_file = ( - self.results_file - + f"{'&mode=ro' if '?' in self.results_file else '?mode=ro'}" - ) - conn_kwargs = self.sql_connect_kwargs.copy() - conn_kwargs["uri"] = True - conn_kwargs["check_same_thread"] = False - - async with aiosqlite.connect( - results_file, **conn_kwargs - ) as conn, conn.execute(query) as cursor: - async for row in cursor: - last_id = row[0] if row[0] > last_id else last_id - await self.websocket.send_json( - json.dumps(json.loads(row[1])) - ) - if self.replay is True: - await asyncio.sleep(self.sleep_time / 10) - else: - self.logger.error("Results file not found: %s", str(file_path)) - break + if self.replay is True: + await asyncio.sleep(self.sleep_time / 10) + await cursor.close() await asyncio.sleep(self.sleep_time) except KeyboardInterrupt: @@ -305,7 +311,9 @@ async def stream_results( # noqa: PLR0915 # pylint: disable=too-many-branches except Exception as e: # pylint: disable=broad-except msg = f"{e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e}" self.logger.error(msg) - return + finally: + CONNECTED_CLIENTS.remove(self) + self.logger.info("Listener connection was disconnected.") def start_app(self, host: str = "127.0.0.1", port: int = 6666): """Start the FastAPI app with Uvicorn.""" @@ -338,8 +346,19 @@ def create_broadcast_server( ) -def main(): +def run_broadcast_server(broadcast_server, host, port, **kwargs): + """Run the broadcast server.""" + broadcast_server.start_app(host=host, port=port, **kwargs) + + +async def main(): """Run the main function.""" + import threading + + loop = asyncio.get_running_loop() + + STDIN_TASK = loop.create_task(read_stdin()) + broadcast_server = create_broadcast_server( RESULTS_FILE, TABLE_NAME, @@ -348,23 +367,32 @@ def main(): SQL_CONNECT_KWARGS, SQL, ) - + global MAIN_CLIENT # noqa: PLW0603 + MAIN_CLIENT = broadcast_server try: - broadcast_server.start_app( - host=HOST, - port=PORT, - **kwargs, + broadcast_thread = threading.Thread( + target=run_broadcast_server, + args=(broadcast_server, HOST, PORT), + kwargs=kwargs, + daemon=True, ) + broadcast_thread.start() + await asyncio.sleep(0.1) + + await asyncio.gather(STDIN_TASK, return_exceptions=True) + except TypeError as e: msg = f"Invalid keyword argument passed to unvicorn. -> {e.args[0]}" broadcast_server.logger.error(msg) - for handler in broadcast_server.logger.handlers: - handler.flush() except KeyboardInterrupt: broadcast_server.logger.info("Broadcast server terminated.") - for handler in broadcast_server.logger.handlers: - handler.flush() finally: + if STDIN_TASK: + STDIN_TASK.cancel() + broadcast_thread.join() + + loop.stop() + loop.close() sys.exit(0) @@ -373,6 +401,7 @@ def main(): raise ValueError("Results file path is required for Broadcast server.") try: - main() + asyncio.run(main()) + except KeyboardInterrupt: sys.exit(0) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py index e08af867fe59..659874fb81d4 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py @@ -5,6 +5,8 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, Union +from openbb_core.app.model.abstract.error import OpenBBError + if TYPE_CHECKING: import logging @@ -33,14 +35,14 @@ class WebSocketClient: # pylint: disable=too-many-instance-attributes Default is 5000. Set to None to keep all records. results_file : Optional[str] Absolute path to the file for continuous writing. By default, a temporary file is created. - File is discarded when the Python session ends unless 'save_results' is set to True. + File is discarded when the Python session ends unless 'save_database' is set to True. The connection can be re-established with the same results file to continue writing. EACH NEW CONNECTION SHOULD HAVE A UNIQUE RESULTS FILE. If the intention is to permanently store the results for historical records, save the current session to a new file and copy new records at periodic intervals into the master. table_name : Optional[str] SQL table name to store serialized data messages. By default, 'records'. - save_results : bool + save_database : bool Whether to persist the results after exiting. Default is False. data_model : Optional[BaseModel] Pydantic data model to validate the results before storing them in the database. @@ -67,6 +69,8 @@ class WebSocketClient: # pylint: disable=too-many-instance-attributes Check if the provider connection process is running. is_broadcasting : bool Check if the broadcast server process is running. + is_exporting : bool + Check if the export thread is running. broadcast_address : str URI address for connecting to the broadcast stream. num_results : int @@ -105,8 +109,14 @@ def __init__( # noqa: PLR0913 # pylint: disable=too-many-arguments,too-many-po limit: Optional[int] = 5000, results_file: Optional[str] = None, table_name: Optional[str] = None, - save_results: bool = False, + batch_size: int = 5000, + save_database: bool = False, data_model: Optional["BaseModel"] = None, + prune_interval: Optional[int] = None, + export_directory: Optional[str] = None, + export_interval: Optional[int] = None, + compress_export: bool = False, + verbose: bool = True, auth_token: Optional[str] = None, logger: Optional["logging.Logger"] = None, **kwargs, @@ -170,7 +180,7 @@ def __init__( # noqa: PLR0913 # pylint: disable=too-many-arguments,too-many-po self.results_file = temp_file_path self.results_path = Path(self.results_file).absolute() # type: ignore - self.save_results = save_results + self.save_database = save_database self.logger = logger if logger else get_logger("openbb.websocket.client") atexit.register(self._atexit) @@ -183,6 +193,16 @@ def __init__( # noqa: PLR0913 # pylint: disable=too-many-arguments,too-many-po logger=self.logger, data_model=self.data_model, ) + self.database.writer = self.database.create_writer( + queue=None, + prune_interval=prune_interval, + batch_size=batch_size, + export_directory=export_directory, + export_interval=export_interval, + compress_export=compress_export, + verbose=verbose, + ) + except Exception as e: # pylint: disable=broad-except msg = ( "Unexpected error setting up the SQLite database and table ->" @@ -200,14 +220,25 @@ def _atexit(self) -> None: self._exception = None + if self.is_exporting: + self.database.writer.stop_export_task() + if self.is_pruning: + self.database.writer.stop_prune_task() + if self.is_running: self.disconnect() if self.is_broadcasting: self.stop_broadcasting() - if self.save_results: + if self.save_database: self.logger.info("Websocket results saved to, %s\n", str(self.results_path)) - if os.path.exists(self.results_file) and not self.save_results: # type: ignore + if os.path.exists(self.results_file) and not self.save_database: # type: ignore os.remove(self.results_file) # type: ignore + if os.path.exists(self.results_file + "-journal"): + os.remove(self.results_file + "-journal") + if os.path.exists(self.results_file + "-shm"): + os.remove(self.results_file + "-shm") + if os.path.exists(self.results_file + "-wal"): + os.remove(self.results_file + "-wal") def _log_provider_output(self, output_queue) -> None: """Log output from the provider logger, handling exceptions, errors, and messages that are not data.""" @@ -228,7 +259,7 @@ def _log_provider_output(self, output_queue) -> None: if "UnauthorizedError" in output: self._psutil_process.kill() self._process.wait() - self._thread.join() + self._thread.join(timeout=1) err = UnauthorizedError(output) self._exception = err sys.stdout.write(output + "\n") @@ -240,7 +271,7 @@ def _log_provider_output(self, output_queue) -> None: if "ValidationError" in output: self._psutil_process.kill() self._process.wait() - self._thread.join() + self._thread.join(timeout=1) title, errors = output.split(" -> ")[-1].split(": ") line_errors = json.loads(errors.strip()) err = ValidationError.from_exception_data( @@ -273,7 +304,7 @@ def _log_provider_output(self, output_queue) -> None: ): self._psutil_process.kill() self._process.wait() - self._thread.join() + self._thread.join(timeout=1) err = ChildProcessError(output) self._exception = err output = output + "\n" @@ -322,9 +353,9 @@ def _log_broadcast_output(self, output_queue) -> None: if output: if output.startswith("ERROR:"): - output = output.replace("ERROR:", "BROADCAST ERROR:") + "\n" + output = output.replace("ERROR:", "BROADCAST ERROR:") elif output.startswith("INFO:"): - output = output.replace("INFO:", "BROADCAST INFO:") + "\n" + output = output.replace("INFO:", "BROADCAST INFO:") output = output[0] if isinstance(output, tuple) else output output = clean_message(output) # if ( @@ -372,60 +403,65 @@ def connect(self) -> None: # pylint: disable=too-many-locals if self.limit: command.extend([f"limit={self.limit}"]) - kwargs = self._kwargs.copy() + try: + kwargs = self._kwargs.copy() - if kwargs: - for k, v in kwargs.items(): - if isinstance(v, str): - unencrypted_value = decrypt_value( - self._key, self._iv, v # pylint: disable=protected-access + if kwargs: + for k, v in kwargs.items(): + if isinstance(v, str): + unencrypted_value = decrypt_value( + self._key, self._iv, v # pylint: disable=protected-access + ) + kwargs[k] = unencrypted_value + else: + kwargs[k] = v + + _kwargs = ( + [ + f"{k}={str(v).strip().replace(' ', '_')}" + for k, v in kwargs.items() + ] + if kwargs + else None + ) + if _kwargs is not None: + for kwarg in _kwargs: + if kwarg not in command: + command.extend([kwarg]) + + self._process = ( + subprocess.Popen( # noqa # pylint: disable=consider-using-with + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdin=subprocess.PIPE, + env=os.environ, + text=True, + bufsize=1, ) - kwargs[k] = unencrypted_value - else: - kwargs[k] = v - - _kwargs = ( - [f"{k}={str(v).strip().replace(' ', '_')}" for k, v in kwargs.items()] - if kwargs - else None - ) - if _kwargs is not None: - for kwarg in _kwargs: - if kwarg not in command: - command.extend([kwarg]) - - try: - self._process = ( - subprocess.Popen( # noqa # pylint: disable=consider-using-with - command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - stdin=subprocess.PIPE, - env=os.environ, - text=True, - bufsize=1, ) - ) - self._psutil_process = psutil.Process(self._process.pid) - - log_output_queue: queue.Queue = queue.Queue() - self._thread = threading.Thread( - target=non_blocking_websocket, - args=( - self, - log_output_queue, - self._provider_message_queue, - ), - ) - self._thread.daemon = True - self._thread.start() + self._psutil_process = psutil.Process(self._process.pid) + + log_output_queue: queue.Queue = queue.Queue() + self._thread = threading.Thread( + target=non_blocking_websocket, + args=( + self, + log_output_queue, + self._provider_message_queue, + ), + ) + self._thread.name = f"Provider-Connection-{self.name}" + self._thread.daemon = True + self._thread.start() - self._log_thread = threading.Thread( - target=self._log_provider_output, - args=(log_output_queue,), - ) - self._log_thread.daemon = True - self._log_thread.start() + self._log_thread = threading.Thread( + target=self._log_provider_output, + args=(log_output_queue,), + ) + self._log_thread.name = f"Provider-Log-{self.name}" + self._log_thread.daemon = True + self._log_thread.start() except Exception as e: # pylint: disable=broad-except msg = f"Unexpected error -> {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" @@ -433,9 +469,6 @@ def connect(self) -> None: # pylint: disable=too-many-locals self._atexit() raise OpenBBError(msg) from e - # Give it some startup time to allow the connection to be established and for exceptions to populate. - time.sleep(2) - if self._exception is not None: exc = getattr(self, "_exception", None) self._exception = None @@ -446,6 +479,12 @@ def connect(self) -> None: # pylint: disable=too-many-locals "Unexpected error -> Provider connection process failed to start." ) + if self.database.writer.export_interval: + self.database.writer.start_export_task() + + if self.database.writer.prune_interval: + self.database.writer.start_prune_task() + def send_message( self, message, target: Literal["provider", "broadcast"] = "provider" ) -> None: @@ -463,6 +502,7 @@ def disconnect(self) -> None: if self._process is None or self.is_running is False: self.logger.info("Provider client connection is not running.") return + if ( self._psutil_process is not None and hasattr(self._psutil_process, "is_running") @@ -470,8 +510,8 @@ def disconnect(self) -> None: ): self._psutil_process.kill() self._process.wait() - self._thread.join() - self._log_thread.join() + self._thread.join(timeout=1) + self._log_thread.join(timeout=1) self._stop_log_thread_event.clear() self.logger.info("Disconnected from the provider server.") if hasattr(self, "_exception") and self._exception: @@ -546,6 +586,28 @@ def is_broadcasting(self) -> bool: return self._psutil_broadcast_process.is_running() return False + @property + def is_exporting(self) -> bool: + """Check if the database is exporting records.""" + if ( + hasattr(self.database, "writer") + and hasattr(self.database.writer, "export_thread") + and hasattr(self.database.writer.export_thread, "is_alive") + ): + return self.database.writer.export_thread.is_alive() + return False + + @property + def is_pruning(self) -> bool: + """Check if the pruning event is running.""" + if ( + hasattr(self.database, "writer") + and hasattr(self.database.writer, "prune_thread") + and hasattr(self.database.writer.prune_thread, "is_alive") + ): + return self.database.writer.prune_thread.is_alive() + return False + @property def num_results(self) -> int: """Get the number of results stored in the database.""" @@ -560,9 +622,6 @@ def results(self) -> list: Clear the results by deleting the property. e.g., del client.results """ - # pylint: disable=import-outside-toplevel - from openbb_core.app.model.abstract.error import OpenBBError - try: return self.database.fetch_all() except Exception as e: # pylint: disable=broad-except @@ -747,6 +806,7 @@ def start_broadcasting( # pylint: disable=too-many-locals self._broadcast_message_queue, ), ) + self._broadcast_thread.name = f"Broadcast-Connection-{self.name}" self._broadcast_thread.daemon = True self._broadcast_thread.start() @@ -754,6 +814,7 @@ def start_broadcasting( # pylint: disable=too-many-locals target=self._log_broadcast_output, args=(output_queue,), ) + self._broadcast_log_thread.name = f"Broadcast-Log-{self.name}" self._broadcast_log_thread.daemon = True self._broadcast_log_thread.start() @@ -780,8 +841,8 @@ def stop_broadcasting(self): self.logger.info("Stopped broadcasting to: %s", broadcast_address) self._broadcast_process.wait() - self._broadcast_thread.join() - self._broadcast_log_thread.join() + self._broadcast_thread.join(timeout=1) + self._broadcast_log_thread.join(timeout=1) self._broadcast_process = None self._psutil_broadcast_process = None self._broadcast_address = None @@ -791,13 +852,13 @@ def stop_broadcasting(self): def __repr__(self): """Return the WebSocketClient representation.""" return ( - f"WebSocketClient(module={self.module}, symbol={self.symbol}, " + f"WebSocketClient(module={[d for d in self.module if "api_key" not in d]}, symbol={self.symbol}, " f"is_running={self.is_running}, provider_pid: " f"{self._psutil_process.pid if self._psutil_process else ''}, is_broadcasting={self.is_broadcasting}, " f"broadcast_address={self.broadcast_address}, " f"broadcast_pid: {self._psutil_broadcast_process.pid if self._psutil_broadcast_process else ''}, " f"results_file={self.results_file}, table_name={self.table_name}, " - f"save_results={self.save_results})" + f"save_database={self.save_database})" ) @@ -805,11 +866,28 @@ def non_blocking_websocket(client, output_queue, provider_message_queue) -> None """Communicate with the threaded process.""" try: while not client._stop_log_thread_event.is_set(): + + if ( + client.database.writer.prune_interval + and client.database.writer.prune_thread is not None + and not client.database.writer.prune_thread.is_alive() + ): + client.database.writer.start_prune_task() + + if ( + client.database.writer.export_interval is not None + and client.database.writer.export_thread is not None + and not client.database.writer.export_thread.is_alive() + ): + client.database.writer.start_export_task() + while not provider_message_queue.empty(): read_message_queue(client, provider_message_queue) output = client._process.stdout.readline() + if output == "" and client._process.poll() is not None: break + if output: output_queue.put(output.strip()) @@ -823,6 +901,10 @@ def non_blocking_websocket(client, output_queue, provider_message_queue) -> None finally: client._process.stdout.close() client._process.wait() + if client.is_exporting: + client.database.writer.stop_export_task() + if client.is_pruning: + client.database.writer.stop_prune_task() def send_message( diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 6df601c385bd..0be969d2732c 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -1,14 +1,23 @@ """Database module for serialized websockets results.""" -from typing import TYPE_CHECKING, Any, Iterable, Optional +# pylint: disable=too-many-lines,too-many-arguments,too-many-locals,too-many-branches,too-many-statements,protected-access + +import asyncio +import threading +from contextlib import asynccontextmanager +from datetime import timedelta +from typing import TYPE_CHECKING, Any, Iterable, Optional, Union from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.utils.helpers import run_async if TYPE_CHECKING: - import asyncio import logging + from pathlib import Path + from openbb_core.provider.utils.websockets.helpers.message_queue import ( + MessageQueue, + ) from pydantic import BaseModel CHECK_FOR = ( @@ -56,15 +65,39 @@ ) +def kill_thread(thread: threading.Thread) -> None: + """Kill thread by setting a stop flag.""" + # pylint: disable=import-outside-toplevel + import ctypes + + if hasattr(thread, "loop") and thread.loop: + for task in asyncio.all_tasks(thread.loop): + task.cancel() + + if not thread.is_alive(): + return + + thread_id = thread.ident + if thread_id is None: + return + + res = ctypes.pythonapi.PyThreadState_SetAsyncExc( + ctypes.c_long(thread_id), ctypes.py_object(SystemExit) + ) + if res > 1: + ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(thread_id), None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + class Database: """ Class to read from, and write to, the SQL file using aiosqlite. Each write or delete operation uses a new connection context in WAL mode. - The table always contains two columns: - "id" - an auto-incrementing ID - "message" - a JSON serialized row of data + The table always contains only two columns: + "id" - an auto-incrementing ID & primary key + "message" - a JSON serialized row of data (dictionary) If a path is not specified, a temporary file will be created and used. @@ -94,6 +127,9 @@ class Database: Methods ------- + get_connection(name: str = "read") -> aiosqlite.Connection + Get a connection to the SQLite database. Use "read" for read-only connections, and "write" for write connections. + Yielded as an async context manager. write_to_db(message) -> None Write the WebSocket message to the SQLite database. Synchronous wrapper for _write_to_db. @@ -136,7 +172,9 @@ def __init__( # pylint: disable=too-many-positional-arguments self.results_file = None self.table_exists = False - self.logger = logger if logger else get_logger("openbb.websocket.database") + self.logger = ( + logger if logger is not None else get_logger("openbb.websocket.database") + ) if not results_file: with tempfile.NamedTemporaryFile(delete=False) as temp_file: @@ -162,6 +200,7 @@ def __init__( # pylint: disable=too-many-positional-arguments self.limit = limit self.loop = loop self.kwargs = kwargs if kwargs else {} + self._connections = {} run_async(self._setup_database) self.data_model = data_model @@ -169,19 +208,17 @@ async def _setup_database(self): """Create the SQLite database, if required.""" # pylint: disable=import-outside-toplevel import os # noqa - import aiosqlite + from aiosqlite import DatabaseError try: if self.results_file is not None and os.path.exists(self.results_file): # type: ignore - async with aiosqlite.connect( - self.results_file, loop=self.loop, **self.kwargs - ) as conn: + async with self.get_connection("read") as conn: try: cursor = await conn.execute( "SELECT name FROM sqlite_master WHERE type='table';" ) await cursor.close() - except aiosqlite.DatabaseError as e: + except DatabaseError as e: msg = ( "Unexpected error caused by an invalid SQLite database file." "Please check the path, and inspect the file if it exists." @@ -190,9 +227,7 @@ async def _setup_database(self): self.logger.error(msg) raise OpenBBError(msg) from e - async with aiosqlite.connect( - self.results_file, loop=self.loop, **self.kwargs - ) as conn: + async with self.get_connection("write") as conn: cursor = await conn.execute( f""" CREATE TABLE IF NOT EXISTS {self.table_name} ( @@ -201,11 +236,19 @@ async def _setup_database(self): ); """ ) + pragmas = [ + "PRAGMA journal_mode=WAL", + "PRAGMA synchronous=OFF", + ] + + for pragma in pragmas: + await conn.execute(pragma) + await conn.commit() await cursor.close() self.table_exists = True - except Exception as e: + except Exception as e: # pylint: disable=broad-except msg = ( "Unexpected error while creating SQLite database ->" f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" @@ -213,10 +256,45 @@ async def _setup_database(self): self.logger.error(msg) raise OpenBBError(msg) from e + @asynccontextmanager + async def get_connection(self, name: str = "read"): + """Get a connection to the SQLite database.""" + # pylint: disable=import-outside-toplevel + import aiosqlite + + conn_kwargs = self.kwargs.copy() + + if name == "read": + if ":" not in self.results_file: + results_file = ( + "file:" + + ( + self.results_file + if self.results_file.startswith("/") + else "/" + self.results_file + ) + + "?mode=ro" + ) + else: + results_file = ( + self.results_file + + f"{'&mode=ro' if '?' in self.results_file else '?mode=ro'}" + ) + conn_kwargs["uri"] = True + elif name == "write": + results_file = self.results_file + + conn_kwargs["check_same_thread"] = False + + if name not in self._connections: + conn = await aiosqlite.connect(results_file, **conn_kwargs) + self._connections[name] = conn + + yield self._connections[name] + async def _write_to_db(self, message) -> None: """Write the WebSocket message to the SQLite database.""" # pylint: disable=import-outside-toplevel - import aiosqlite # noqa import json try: @@ -226,15 +304,7 @@ async def _write_to_db(self, message) -> None: if not isinstance(message, str): message = json.dumps(message) - if not self.table_exists: - raise aiosqlite.OperationalError( - "Attempt to write to non-existent table." - ) - - async with aiosqlite.connect( - self.results_file, loop=self.loop, **self.kwargs - ) as conn: - await conn.execute("PRAGMA journal_mode=WAL;") + async with self.get_connection("write") as conn: cursor = await conn.execute( f""" INSERT INTO {self.table_name} (message) @@ -277,7 +347,7 @@ async def _write_to_db(self, message) -> None: raise OpenBBError(e) from e def write_to_db(self, message) -> None: - """Write the WebSocket message to the SQLite database.""" + """Write a message to the SQLite database.""" try: run_async(self._write_to_db, message) except Exception as e: # pylint: disable=broad-except @@ -290,38 +360,9 @@ def write_to_db(self, message) -> None: async def _fetch_all(self, limit: Optional[int] = None) -> list: """Read the WebSocket message from the SQLite database.""" - # pylint: disable=import-outside-toplevel - import aiosqlite # noqa - try: rows: list = [] - conn_kwargs = self.kwargs.copy() - - if not conn_kwargs.get("check_same_thread"): - conn_kwargs["check_same_thread"] = False - - if ":" not in self.results_file: - results_file = ( - "file:" - + ( - self.results_file - if self.results_file.startswith("/") - else "/" + self.results_file - ) - + "?mode=ro" - ) - else: - results_file = ( - self.results_file - + f"{'&mode=ro' if '?' in self.results_file else '?mode=ro'}" - ) - conn_kwargs["uri"] = True - - async with aiosqlite.connect( - results_file, - loop=self.loop, - **conn_kwargs, - ) as conn: + async with self.get_connection("read") as conn: query = ( f"SELECT message FROM {self.table_name} ORDER BY id DESC" # noqa ) @@ -332,14 +373,14 @@ async def _fetch_all(self, limit: Optional[int] = None) -> list: params = () async with conn.execute(query, params) as cursor: async for row in cursor: - rows.append(await self.deserialize_row(row)) + rows.append(await self._deserialize_row(row)) return rows - except Exception as e: + except Exception as e: # pylint: disable=broad-except raise OpenBBError(e) from e - async def deserialize_row(self, row: str) -> dict: + async def _deserialize_row(self, row) -> dict: """Deserialize a row from the SQLite database.""" # pylint: disable=import-outside-toplevel import json @@ -356,7 +397,7 @@ async def deserialize_row(self, row: str) -> dict: ) else row[0] ) - except Exception as e: + except Exception as e: # pylint: disable=broad-except msg = ( "Unexpected error while deserializing row -> " f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" @@ -368,7 +409,7 @@ def fetch_all(self, limit: Optional[int] = None) -> list: """Fetch all the results from the SQLite database.""" try: return run_async(self._fetch_all, limit) - except Exception as e: + except Exception as e: # pylint: disable=broad-except msg = ( "Unexpected error while reading from SQLite database ->" f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" @@ -403,7 +444,7 @@ def get_latest_results( """Get the latest records from the database. Optionally filter by symbol.""" try: return run_async(self._get_latest_results, symbol, limit) - except Exception as e: + except Exception as e: # pylint: disable=broad-except msg = ( "Unexpected error while getting latest records ->" f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" @@ -413,7 +454,6 @@ def get_latest_results( async def _query_db(self, sql, parameters: Optional[Iterable[Any]] = None) -> list: """Query the SQLite database.""" # pylint: disable=import-outside-toplevel - import aiosqlite # noqa import json if not sql or sql in ("", "''"): @@ -435,32 +475,9 @@ async def _query_db(self, sql, parameters: Optional[Iterable[Any]] = None) -> li rows: list = [] try: - conn_kwargs = self.kwargs.copy() - if not conn_kwargs.get("check_same_thread"): - conn_kwargs["check_same_thread"] = False - - if ":" not in self.results_file: - results_file = ( - "file:" - + ( - self.results_file - if self.results_file.startswith("/") - else "/" + self.results_file - ) - + "?mode=ro" - ) - else: - results_file = ( - self.results_file - + f"{'&mode=ro' if '?' in self.results_file else '?mode=ro'}" - ) - conn_kwargs["uri"] = True - - async with aiosqlite.connect( - results_file, - loop=self.loop, - **conn_kwargs, - ) as conn, conn.execute(query, parameters) as cursor: + async with self.get_connection("read") as conn, conn.execute( + query, parameters + ) as cursor: async for row in cursor: rows.append( json.loads(row[0]) @@ -495,7 +512,7 @@ def query(self, sql: str, parameters: Optional[Iterable[Any]] = None) -> list: # Or provide a full query string by starting with SELECT. >>> query = ( "SELECT json_extract (message, '$.symbol')" - "FROM test_table WHERE json_extract (message, '$.type') = 'trade';" + "FROM records WHERE json_extract (message, '$.type') = 'trade';" ) >>> database.query(query) >>> @@ -508,15 +525,8 @@ def query(self, sql: str, parameters: Optional[Iterable[Any]] = None) -> list: async def _clear_results(self): """Clear the results from the SQLite database.""" - # pylint: disable=import-outside-toplevel - import aiosqlite - try: - async with aiosqlite.connect( - self.results_file, loop=self.loop, **self.kwargs - ) as conn: - cursor = await conn.execute("PRAGMA journal_mode=WAL;") - await cursor.close() + async with self.get_connection("write") as conn: cursor = await conn.execute(f"DELETE FROM {self.table_name}") # noqa await cursor.close() cursor = await conn.execute( @@ -549,3 +559,595 @@ def clear_results(self) -> None: ) self.logger.error(msg) raise OpenBBError(msg) from e + + def create_writer( + self, + queue: Optional["MessageQueue"] = None, + batch_size: int = 250, + collection_time: int = 0.1, + prune_interval: Optional[int] = None, + export_directory: Optional[Union[str, "Path"]] = None, + export_interval: Optional[int] = None, + compress_export: bool = False, + num_workers: int = 60, + verbose: bool = True, + ): + """ + Create a new DatabaseWriter instance from the initialized Database. + + Returns + ------- + DatabaseWriter + A new DatabaseWriter instance. Use + """ + + return DatabaseWriter( + self, + queue, + batch_size, + collection_time, + prune_interval, + export_directory, + export_interval, + compress_export, + num_workers, + verbose, + ) + + +class DatabaseWriter: + """ + Class responsible for continuously writing messages to the SQLite database, + exporting the database at set intervals, + and pruning the database of older records at a different interval. + + Setting the 'export_interval' will create a new CSV file for each interval, + while leaving as None disables the export feature. + The task is started by the WebsocketClient class when a connection is created from the FastAPI and Python endpoints, + but can be manually started by calling the `start_batch_writer` async method. + + Parameters + ---------- + database : Database + The Database instance to write to. + queue : Optional[MessageQueue] + The MessageQueue instance to use. Default is None, which creates a new instance. + batch_size : int + The target batch size for writing to the database. Default is 250. + collection_time : int + The maximum time in seconds to collect messages before writing to the database. Default is 0.1. + prune_interval : Optional[int] + The interval in minutes to prune the database of older records. Default is None. + export_directory : Optional[Union[str, Path]] + The directory to export the database to, if an interval is set. Default is 'OpenBBUserData/exports/websockets'. + export_interval : Optional[int] + The interval in minutes to export the database to a CSV file. Default is None. + compress_export : bool + Whether to compress the exported CSV files. Default is False. + num_workers : int + The number of parallel writers reading the queue and preparing batches. Default is 60. + verbose : bool + Whether to print verbose output for export and prune tasks. Default is True. + """ + + def __init__( + self, + database: Database, + queue: Optional["MessageQueue"] = None, + batch_size: int = 250, + collection_time: int = 0.1, + prune_interval: Optional[int] = None, + export_directory: Optional[Union[str, "Path"]] = None, + export_interval: Optional[int] = None, + compress_export: bool = False, + num_workers: int = 60, + verbose: bool = True, + ): + """Initialize the DatabaseWriter class.""" + # pylint: disable=import-outside-toplevel + import os # noqa + import time + + from openbb_core.provider.utils.websockets.message_queue import ( + MessageQueue, + ) + from openbb_core.app.service.user_service import UserService + + user_settings = UserService().read_from_file() + obb_export_directory = user_settings.preferences.export_directory + + if not hasattr(database, "loop"): + try: + database.loop = asyncio.new_event_loop() + except (RuntimeError, RuntimeWarning): + database.loop = asyncio.get_event_loop() + + self.database = database + self.batch_size = batch_size + self.queue = queue if queue else MessageQueue(max_size=100000) + export_directory = ( + export_directory + if export_directory + else obb_export_directory + "/websockets" + ) + os.makedirs(export_directory, exist_ok=True) + self.export_directory = export_directory + self.export_interval = export_interval + self.prune_interval = prune_interval + self.compress_export = compress_export + self.verbose = verbose + self.export_thread = None + self.prune_thread = None + self.last_flush = time.time() + self.writer_running = False + self._first_timestamp = None + self._last_processed_timestamp = None + self._conn = None + self.num_workers = num_workers + self.write_tasks = [] + self._export_running = False + self._prune_running = False + self.batch_processor = BatchProcessor(self) + self._shutdown = False + + async def _create_connection(self): + """Create a new connection to the SQLite database.""" + async with self.database.get_connection("write") as conn: + self.writer_running = True + self._conn = conn + + async def start_writer(self): + """Start writing tasks.""" + if not self.writer_running: + asyncio.get_event_loop().run_in_executor(None, self.batch_processor.start()) + await self._create_connection() + for _ in range(self.num_workers): + task = asyncio.create_task(self._process_queue()) + self.write_tasks.append(task) + + async def stop_writer(self): + """Stop all queue processors.""" + self._flush_queue() + self.writer_running = False + await asyncio.gather(*self.write_tasks, return_exceptions=True) + if self._conn: + await self._conn.close() + self.batch_processor.running = False + kill_thread(self.batch_processor) + + async def _process_queue(self): + """Process queue with parallel writers.""" + # pylint: disable=import-outside-toplevel + import time + + batch: list = [] + collection_start = time.time() + while self.writer_running: + try: + while ( + time.time() - collection_start < self.collection_time + and len(batch) < self.batch_size + ): + try: + message = await asyncio.wait_for( + self.queue.dequeue(), timeout=0.1 + ) + batch.append(message) + except asyncio.TimeoutError: + break + if batch: + self._write_batch(batch) + batch = [] + else: + await asyncio.sleep(0.001) + + except Exception as e: # pylint: disable=broad-except + msg = f"\nQueue processing error: {e}" + self.database.logger.error(msg, exc_info=True) + await asyncio.sleep(0.1) + + def _flush_queue(self): + """Flush the queue of messages to the database.""" + batch: list = [] + while not self.queue.queue.empty(): + batch.append(self.queue.queue.get_nowait()) + self._write_batch(batch) + + def _write_batch(self, batch): + """Write the batch of messages to the database.""" + if not batch: + return + self.batch_processor.write_queue.put_nowait(batch) + + async def _export_database(self): + """Export the database to a CSV file at a set interval.""" + # pylint: disable=import-outside-toplevel + import csv # noqa + import gzip + import json + import sys + from anyio import open_file + from collections import OrderedDict + from io import StringIO + from pandas import to_datetime + + chunk_size = 20000 + minutes = self.export_interval or 5 + + if not self._export_running or not self.export_thread: + return + try: + + latest_query = f""" + SELECT json_extract(message, '$.date') + FROM {self.database.table_name} + ORDER BY json_extract(message, '$.date') DESC LIMIT 1 + """ # noqa + + async with self.database.get_connection("read") as conn: + cursor = await conn.execute(latest_query) + latest_date = await cursor.fetchone() + if not latest_date: + return + + latest_timestamp = to_datetime(latest_date[0]) + + # Round down to nearest interval + cutoff_time = latest_timestamp - timedelta( + minutes=latest_timestamp.minute % minutes, + seconds=latest_timestamp.second, + microseconds=latest_timestamp.microsecond, + ) + + # If we have processed data before, use that as reference + if self._last_processed_timestamp: + start_time = self._last_processed_timestamp + else: + start_time = cutoff_time - timedelta(minutes=minutes) + + cutoff_time = self._last_processed_timestamp + timedelta(minutes=minutes) + start_time = self._last_processed_timestamp + results_file = ( + self.export_directory + + "/" + + self.database.results_file.split("/")[-1].split(".")[0] + ) + path = f"{results_file}_{start_time.strftime('%Y%m%dT%H%M')}.csv" + query = f""" + SELECT message + FROM {self.database.table_name} + WHERE json_extract(message, '$.date') >= ? + AND json_extract(message, '$.date') < ? + ORDER BY json_extract(message, '$.date') ASC + """ # noqa + + async with self.database.get_connection( + "read" + ) as conn, conn.cursor() as cursor: + await cursor.execute( + query, (start_time.isoformat(), cutoff_time.isoformat()) + ) + + headers = OrderedDict() + first_rows = await cursor.fetchmany(chunk_size) + if not first_rows: + return + + for row in first_rows: + for key in json.loads(row[0]): + headers[key] = None + + if self.compress_export: + with gzip.open(path, "wt") as gz_file: + writer = csv.DictWriter(gz_file, fieldnames=list(headers)) + await writer.writeheader() + writer.writerows(json.loads(row[0]) for row in first_rows) + + while True: + rows = await cursor.fetchmany(chunk_size) + new_rows: list = [] + if not rows: + break + + for row in rows: + data = json.loads(row[0]) + for key in data: + headers[key] = None + new_rows.append(data) + + writer = csv.DictWriter(gz_file, fieldnames=list(headers)) + writer.writerows(new_rows) + else: + async with await open_file(path, mode="w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=headers) + await writer.writeheader() + buffer = StringIO() + csv_writer = csv.DictWriter(buffer, fieldnames=list(headers)) + csv_writer.writerows(json.loads(row[0]) for row in first_rows) + await f.write(buffer.getvalue()) + + while True: + rows = await cursor.fetchmany(chunk_size) + new_rows: list = [] + if not rows: + break + + for row in rows: + data = json.loads(row[0]) + for key in data: + headers[key] = None + new_rows.append(data) + + buffer = StringIO() + csv_writer = csv.DictWriter( + buffer, fieldnames=list(headers) + ) + csv_writer.writerows(new_rows) + await f.write(buffer.getvalue()) + + self._last_processed_timestamp = cutoff_time + if self.verbose: + msg = f"DATABASE INFO: Interval for period ending {cutoff_time} saved to: {path}" + sys.stdout.write(msg + "\n") + sys.stdout.flush() + + except asyncio.CancelledError: + pass + except Exception as e: + self.database.logger.error( + "Error exporting database: %s", str(e), exc_info=True + ) + sys.exit(1) + + def _run_export_event(self): + """Run the export event loop in a separate process.""" + run_async(self._start_export_task) + + def start_export_task(self): + """Public method to start the background export task.""" + if ( + hasattr(self, "export_thread") + and self.export_thread + and self.export_thread.is_alive() + ): + if not self._export_running: + self._export_running = True + return + try: + self._export_running = True + self.export_thread = threading.Thread( + target=self._run_export_event, name="ExportThread", daemon=True + ) + self.export_thread.start() + finally: + self.export_thread.join(timeout=1) + self._export_running = False + + def stop_export_task(self): + """Public method to stop the background export task.""" + if hasattr(self, "export_thread") and self.export_thread: + self.export_thread.join(timeout=1) + if self.export_thread.is_alive(): + kill_thread(self.export_thread) + self._export_running = False + self.export_thread = None + + def _run_prune_event(self): + """Run the prune event loop in a separate process.""" + run_async(self._start_prune_task) + + def start_prune_task(self): + """Public method to start the background pruning task.""" + if ( + hasattr(self, "prune_thread") + and self.prune_thread + and self.prune_thread.is_alive() + ): + return + + try: + self._prune_running = True + prune_thread = threading.Thread( + target=self._run_prune_event, daemon=True, name="WebSocketPruneThread" + ) + self.prune_thread = prune_thread + self.prune_thread.start() + finally: + self.prune_thread.join(timeout=1) + self._prune_running = False + + def stop_prune_task(self): + """Public method to stop the background pruning task.""" + if hasattr(self, "prune_thread") and self.prune_thread: + self.prune_thread.join(timeout=1) + if self.prune_thread.is_alive(): + kill_thread(self.prune_thread) + self._prune_running = False + self.prune_thread = None + + async def _start_prune_task(self): + """Start the background prune task.""" + # pylint: disable=import-outside-toplevel + import sys # noqa + from pandas import to_datetime + + if not self._prune_running or not self.prune_thread: + return + + try: + minutes = ( + self.prune_interval + if self.prune_interval + else self.export_interval * 2 if self.export_interval else 10 + ) + while self._prune_running is True: + if self.prune_thread is None: + self._prune_running = False + break + + # Stagger the prune task slightly to avoid things happening exactly on the minute. + await asyncio.sleep(minutes * 60) + await asyncio.sleep(7) + + if not self._last_processed_timestamp: + last_date = await self.database._query_db( + f"SELECT json_extract(message, '$.date') FROM {self.database.table_name} ORDER BY json_extract(message, '$.date') DESC LIMIT 1" # noqa + ) + if not last_date: + continue + last_date = to_datetime(last_date[0]) + last_processed_timestamp = last_date.replace( + second=0, microsecond=0 + ) + else: + last_processed_timestamp = self._last_processed_timestamp + + cutoff_time = last_processed_timestamp - timedelta(minutes=minutes) + cutoff_timestamp = cutoff_time.isoformat() + + async with self.database.get_connection("write") as conn: + + if self.verbose: + msg = f"DATABASE INFO: Pruning database of records before: {cutoff_timestamp}" + sys.stdout.write(msg + "\n") + sys.stdout.flush() + + async with conn.execute( + f"DELETE FROM {self.database.table_name} WHERE json_extract(message, '$.date') < ?", # noqa + (cutoff_timestamp,), + ): + await conn.commit() + finally: + if self.prune_thread is not None: + self.prune_thread.join(timeout=1) + + async def _start_export_task(self): + """Start a background task to prune the database periodically.""" + from pandas import to_datetime + + minutes = self.export_interval or 5 + + while self.export_thread is not None: + # Get the initial row to determine the "first time" + try: + query = f"SELECT json_extract(message, '$.date') FROM {self.database.table_name} ORDER BY json_extract(message, '$.date') ASC LIMIT 1" # noqa + initial_row = await self.database._query_db(query) + if not initial_row: + await asyncio.sleep(1) + initial_row = await self.database._query_db(query) + if not initial_row: + continue + + first_time = to_datetime(initial_row[0]) + if not first_time: + await asyncio.sleep(1) + self._first_timestamp = first_time.replace(second=0, microsecond=0) + self._last_processed_timestamp = self._first_timestamp + + while not self._shutdown: + if self.export_thread is None: + break + # Check if the next interval has been reached and export immediately if available. + next_interval = self._last_processed_timestamp + timedelta( + minutes=minutes + ) + cutoff_timestamp = next_interval.isoformat() + query = f"SELECT COUNT(*) FROM {self.database.table_name} WHERE json_extract(message, '$.date') >= ?" # noqa + count = await self.database._query_db(query, (cutoff_timestamp,)) + + if count[0] > 0: + # task = asyncio.create_task(self._export_database()) + task = await asyncio.to_thread(self._export_database) + await asyncio.gather(task) + else: + await asyncio.sleep(minutes * 60) + # Stagger slightly so things don't happen exactly on the minute. + await asyncio.sleep(3) + except asyncio.CancelledError: + break + finally: + if self.export_thread is not None: + self.export_thread.join(timeout=1) + + +class BatchProcessor(threading.Thread): + """ + Batch processor for writing messages to the SQLite database in a new thread. + + This class is a thread intended for use as a subprocess and is called by `DatabaseWriter.start_writer()`. + """ + + def __init__(self, database_writer: DatabaseWriter): + """Initialize the BatchProcessor class.""" + # pylint: disable=import-outside-toplevel + import queue + + super().__init__(daemon=True, name="BatchProcessor") + self.writer = database_writer + self.write_queue: queue.Queue = queue.Queue() + self.running = True + self.loop = None + self.workers: list = [] + + def run(self): + """Run the batch processor as tasks.""" + + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + try: + # Create worker tasks + for _ in range(self.writer.num_workers): + worker = self.loop.create_task(self._worker()) + self.workers.append(worker) + # Run workers + self.loop.run_until_complete(asyncio.gather(*self.workers)) + finally: + self.loop.close() + + async def _worker(self): + # pylint: disable=import-outside-toplevel + import time + + while self.running: + try: + batch = [] + total = 0 + collection_start = time.time() + + while ( + time.time() - collection_start < (self.writer.collection_time * 2.5) + and total < self.batch_size + ): + if not self.write_queue.empty(): + msg = self.write_queue.get_nowait() + batch.append(msg) + total += len(msg) + else: + await asyncio.sleep(0.01) + + if batch: + await self._write_batch(batch) + + except Exception as e: # pylint: disable=broad-except + self.writer.database.logger.error(f"Worker error: {e}") + self.running = False + await asyncio.sleep(0.1) + break + + async def _write_batch(self, batch): + """Write the batch of messages to the database.""" + try: + query = f""" + INSERT INTO {self.writer.database.table_name} (message) + VALUES (?) + """ # noqa + values: list = [] + for b in batch: + values.extend([(msg,) for msg in b]) + async with self.writer.database.get_connection("write") as conn: + async with conn.cursor() as cursor: + await cursor.executemany(query, values) + await conn.commit() + except Exception as e: # pylint: disable=broad-except + self.writer.database.logger.error(f"Error writing batch: {e}") + await asyncio.sleep(0.1) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py b/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py index 693faa1fae1a..23ea8f2f060d 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py @@ -26,7 +26,7 @@ def get_logger(name, level=logging.INFO): logger = logging.getLogger(f"{name}-{uuid.uuid4()}") handler = logging.StreamHandler() handler.setLevel(level) - formatter = logging.Formatter("%(message)s\n") + formatter = logging.Formatter("%(message)s") handler.setFormatter(formatter) logger.addHandler(handler) logger.setLevel(level) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py b/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py index f80f0bcb6a58..f94e640aae64 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/message_queue.py @@ -40,24 +40,28 @@ async def enqueue(self, message): retries = 0 while retries < self.max_retries: + if self.queue.qsize() / self.queue.maxsize > 0.20: + await sleep(self.backoff_factor * 0.00005) + if self.queue.qsize() / self.queue.maxsize > 0.25: + await sleep(self.backoff_factor * 0.00005) if self.queue.qsize() / self.queue.maxsize > 0.3: - await sleep(0.00005) + await sleep(self.backoff_factor * 0.00005) if self.queue.qsize() / self.queue.maxsize > 0.5: - await sleep(0.00005) + await sleep(self.backoff_factor * 0.0001) if self.queue.qsize() / self.queue.maxsize > 0.55: - await sleep(0.00005) + await sleep(self.backoff_factor * 0.0001) if self.queue.qsize() / self.queue.maxsize > 0.6: - await sleep(0.00005) + await sleep(self.backoff_factor * 0.00005) if self.queue.qsize() / self.queue.maxsize > 0.65: - await sleep(0.00005) + await sleep(self.backoff_factor * 0.00005) if self.queue.qsize() / self.queue.maxsize > 0.7: - await sleep(0.00005) + await sleep(self.backoff_factor * 0.00005) if self.queue.qsize() / self.queue.maxsize > 0.75: - await sleep(0.00005) + await sleep(self.backoff_factor * 0.00005) if self.queue.qsize() / self.queue.maxsize > 0.8: - await sleep(0.00005) + await sleep(self.backoff_factor * 0.00005) if self.queue.qsize() / self.queue.maxsize > 0.98: - await sleep(0.0005) + await sleep(self.backoff_factor * 0.0001) if self.queue.full(): retries += 1 diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 82556aa8d32b..0d9a46cac411 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -39,7 +39,7 @@ async def get_status(name: Optional[str] = None, client: Optional[Any] = None) - "broadcast_pid": broadcast_pid, "results_file": client.results_file, "table_name": client.table_name, - "save_results": client.save_results, + "save_database": client.save_database, } return status diff --git a/openbb_platform/extensions/websockets/openbb_websockets/models.py b/openbb_platform/extensions/websockets/openbb_websockets/models.py index e2bfb02314e2..e55fdb136011 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/models.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/models.py @@ -27,7 +27,7 @@ class WebSocketQueryParams(QueryParams): default=None, description="Absolute path to the file for continuous writing. By default, a temporary file is created.", ) - save_results: bool = Field( + save_database: bool = Field( default=False, description="Whether to save the results after the session ends.", ) @@ -40,6 +40,18 @@ class WebSocketQueryParams(QueryParams): description="Maximum number of newest records to keep in the database." + " If None, all records are kept, which can be memory-intensive.", ) + prune_interval: Optional[int] = Field( + default=None, + description="Prune all entries older than the given number of minutes." + + " If export interval is set, the prune interval must be at least twice as long.", + ) + export_interval: Optional[int] = Field( + default=None, description="Export all entries as a CSV file every N minutes." + ) + export_directory: Optional[str] = Field( + default=None, + description="Directory to save the exported CSV files to. Defaults to OpenBBUserData/exports/websockets", + ) sleep_time: float = Field( default=0.25, description="Time to sleep between checking for new records in the database from the broadcast server." @@ -126,7 +138,7 @@ class WebSocketConnectionStatus(Data): default=None, description="Name of the SQL table to write the results to.", ) - save_results: bool = Field( + save_database: bool = Field( description="Whether to save the results after the session ends.", ) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index a70c1fa0df91..3c18e5395f7e 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -53,7 +53,7 @@ "symbol": "*", "limit": "None", "results_file": "/path/to/results.db", - "save_results": "True", + "save_database": "True", "auth_token": "someAuthToken123$", } ), @@ -66,7 +66,6 @@ async def create_connection( extra_params: ExtraParams, ) -> OBBject: """Create a new provider websocket connection.""" - name = extra_params.name if name in connected_clients: broadcast_address = connected_clients[name].broadcast_address @@ -81,6 +80,11 @@ async def create_connection( obbject = await OBBject.from_query(Query(**locals())) client = obbject.results.client + # pylint: disable=import-outside-toplevel + import asyncio + + await asyncio.sleep(1) + if not client.is_running or client._exception is not None: exc = getattr(client, "_exception", None) if exc: @@ -92,8 +96,8 @@ async def create_connection( if hasattr(extra_params, "start_broadcast") and extra_params.start_broadcast: try: - client.start_broadcasting() await asyncio.sleep(1) + client.start_broadcasting() if client._exception is not None: exc = getattr(client, "_exception", None) client._exception = None diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py index 850a7c792dd1..05eac814a9ec 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py @@ -183,7 +183,7 @@ async def aextract_data( limit=query.limit, results_file=query.results_file, table_name=query.table_name, - save_results=query.save_results, + save_database=query.save_database, data_model=IntrinioWebSocketData, sleep_time=query.sleep_time, broadcast_host=query.broadcast_host, diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py index 74abf43bc3d4..4ddfad2fb415 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -8,7 +8,7 @@ import sys from typing import Any -from openbb_core.provider.utils.websockets.database import Database +from openbb_core.provider.utils.websockets.database import Database, DatabaseWriter from openbb_core.provider.utils.websockets.helpers import ( get_logger, handle_termination_signal, @@ -24,12 +24,16 @@ kwargs = parse_kwargs() command_queue = MessageQueue(logger=logger) CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) - -DATABASE = Database( - results_file=kwargs["results_file"], - table_name=kwargs["table_name"], - limit=kwargs.get("limit"), - logger=logger, +db_queue = MessageQueue(logger=logger) + +DATABASE = DatabaseWriter( + database=Database( + results_file=kwargs["results_file"], + table_name=kwargs["table_name"], + limit=kwargs.get("limit"), + logger=logger, + ), + queue=db_queue, ) @@ -49,7 +53,7 @@ def process_message(message): except ValidationError: raise e from e if result: - DATABASE.write_to_db(result) + db_queue.queue.put_nowait(result) def on_message(message, backlog): @@ -95,6 +99,12 @@ async def read_stdin_and_queue_commands(): break try: + if line == "qsize": + logger.info( + "Queue size: %i - Writer Queue: %i", + command_queue.queue.qsize(), + DATABASE.batch_processor.task_queue.qsize(), + ) command = json.loads(line.strip()) await command_queue.enqueue(command) except json.JSONDecodeError: @@ -113,12 +123,20 @@ async def process_stdin_queue(): async def connect_and_stream(): """Connect to the WebSocket and stream data to file.""" - symbol = kwargs.pop("symbol", "lobby") - symbol = ["lobby"] if "*" in symbol else symbol.split(",") - asyncio.create_task(read_stdin_and_queue_commands()) - client.connect() - client.join(symbol) - asyncio.create_task(process_stdin_queue()) + try: + symbol = kwargs.pop("symbol", "lobby") + symbol = ["lobby"] if "*" in symbol else symbol.split(",") + stdin_task = asyncio.create_task(read_stdin_and_queue_commands()) + await DATABASE.start_writer() + client.connect() + client.join(symbol) + process_stdin_task = asyncio.create_task(process_stdin_queue()) + finally: + stdin_task.cancel() + process_stdin_task.cancel() + await stdin_task + await process_stdin_task + await DATABASE.stop_writer() if __name__ == "__main__": diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 312cc25548d0..cb3f078030bb 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -102,7 +102,6 @@ def validate_date(cls, v): try: dt = datetime.utcfromtimestamp(v / 1000).replace(tzinfo=timezone("UTC")) dt = dt.astimezone(timezone("America/New_York")) - return dt except Exception: if isinstance(v, (int, float)): # Check if the timestamp is in nanoseconds and convert to seconds @@ -110,8 +109,10 @@ def validate_date(cls, v): v = v / 1e9 # Convert nanoseconds to seconds dt = datetime.fromtimestamp(v, tz=timezone("UTC")) dt = dt.astimezone(timezone("America/New_York")) - return dt - return v + else: + return v + + return dt.strftime("%Y-%m-%d %H:%M:%S.%f%z") class PolygonWebSocketQueryParams(WebSocketQueryParams): @@ -545,7 +546,8 @@ class PolygonStockAggsWebSocketData(WebSocketData): symbol: str = Field( description=DATA_DESCRIPTIONS.get("symbol", ""), ) - day_open: float = Field( + day_open: Optional[float] = Field( + default=None, description="Today's official opening price.", json_schema_extra={"x-unit_measurement": "currency"}, ) @@ -573,14 +575,16 @@ class PolygonStockAggsWebSocketData(WebSocketData): description=DATA_DESCRIPTIONS.get("vwap", "") + " For the current aggregate window.", ) - day_vwap: float = Field( + day_vwap: Optional[float] = Field( + default=None, description="Today's volume weighted average price.", ) volume: float = Field( description=DATA_DESCRIPTIONS.get("volume", "") + " For the current aggregate window.", ) - day_volume: float = Field( + day_volume: Optional[float] = Field( + default=None, description="Today's accumulated volume.", ) avg_size: Optional[float] = Field( @@ -1194,6 +1198,8 @@ def extract_data( **kwargs: Any, ) -> dict: """Extract data from the WebSocket.""" + import time + api_key = credentials.get("polygon_api_key") if credentials else "" symbol = query.symbol.upper() kwargs = { @@ -1210,8 +1216,11 @@ def extract_data( limit=query.limit, results_file=query.results_file, table_name=query.table_name, - save_results=query.save_results, + save_database=query.save_database, data_model=PolygonWebSocketData, + prune_interval=query.prune_interval, + export_interval=query.export_interval, + export_directory=query.export_directory, sleep_time=query.sleep_time, broadcast_host=query.broadcast_host, broadcast_port=query.broadcast_port, @@ -1221,6 +1230,7 @@ def extract_data( try: client.connect() + time.sleep(0.5) except OpenBBError as e: if client.is_running: client.disconnect() diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index d9db715d022c..cf37ba5687fc 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -1,13 +1,46 @@ -"""Polygon WebSocket client.""" +""" +Polygon WebSocket Client. + +This file should be run as a script, and is intended to be run as a subprocess of PolygonWebSocketFetcher. + +Keyword arguments are passed from the command line as space-delimited, `key=value`, pairs. + +Required Keyword Arguments +-------------------------- + api_key: str + The API key for the Polygon WebSocket. + symbol: str + The symbol to subscribe to. Example: "AAPL" or "AAPL,MSFT". Use "*" to subscribe to all symbols. + feed: str + The feed to subscribe to. Example: "aggs_sec", "aggs_min", "trade", "quote". + results_file: str + The path to the file where the results will be stored. + +Optional Keyword Arguments +-------------------------- + asset_type: str + The asset type to subscribe to. Default is "crypto". + Options: "stock", "stock_delayed", "options", "options_delayed", "fx", "crypto", "index", "index_delayed". + table_name: str + The name of the table to store the data in. Default is "records". + limit: int + The maximum number of rows to store in the database. + connect_kwargs: dict + Additional keyword arguments to pass directly to `websockets.connect()`. + Example: {"ping_timeout": 300} +""" import asyncio import json +import orjson import os import signal import sys +import time import websockets -from openbb_core.provider.utils.websockets.database import Database +from concurrent.futures import ThreadPoolExecutor +from openbb_core.provider.utils.websockets.database import Database, DatabaseWriter from openbb_core.provider.utils.websockets.helpers import ( get_logger, handle_termination_signal, @@ -20,6 +53,7 @@ PolygonWebSocketData, ) from pydantic import ValidationError +from websockets.asyncio.client import connect URL_MAP = { "stock": "wss://socket.polygon.io/stocks", @@ -33,24 +67,34 @@ } logger = get_logger("openbb.websocket.polygon") -queue = MessageQueue(logger=logger, backoff_factor=2) -command_queue = MessageQueue(logger=logger) - +process_queue = MessageQueue(logger=logger, backoff_factor=0) +input_queue = MessageQueue(logger=logger, backoff_factor=0) +command_queue = MessageQueue(logger=logger, backoff_factor=0) +db_queue = MessageQueue(logger=logger, backoff_factor=0, max_size=100000) kwargs = parse_kwargs() CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) FEED = kwargs.pop("feed", None) ASSET_TYPE = kwargs.pop("asset_type", "crypto") kwargs["results_file"] = os.path.abspath(kwargs.get("results_file")) URL = URL_MAP.get(ASSET_TYPE) +SUBSCRIBED_SYMBOLS: set = set() +message_thread_pool = ThreadPoolExecutor(max_workers=8) + +if not kwargs.get("api_key"): + raise ValueError("No API key provided.") if not URL: raise ValueError("Invalid asset type provided.") -DATABASE = Database( - results_file=kwargs["results_file"], - table_name=kwargs["table_name"], - limit=kwargs.get("limit"), - logger=logger, +DATABASE = DatabaseWriter( + database=Database( + results_file=kwargs["results_file"], + table_name=kwargs.get("table_name", "records"), + limit=kwargs.get("limit"), + logger=logger, + ), + batch_size=1, + queue=db_queue, ) @@ -153,9 +197,19 @@ async def subscribe(websocket, symbol, event): try: await websocket.send(subscribe_event) except Exception as e: - msg = f"PROVIDER ERROR: {e.__class__.__name__} -> {e}" + msg = f"PROVIDER ERROR: {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" logger.error(msg) + tickers = ticker.split(",") + if event == "subscribe": + for t in tickers: + SUBSCRIBED_SYMBOLS.add(t) + elif event == "unsubscribe": + for t in tickers: + SUBSCRIBED_SYMBOLS.discard(t) + + kwargs["symbol"] = ",".join(SUBSCRIBED_SYMBOLS) + async def read_stdin(): """Read from stdin and queue commands.""" @@ -175,7 +229,11 @@ async def process_stdin_queue(websocket): while True: command = await command_queue.dequeue() if command == "qsize": - logger.info(f"PROVIDER INFO: Queue size: {queue.queue.qsize()}") + logger.info( + f"PROVIDER INFO: Input Queue: {input_queue.queue.qsize()} -" + f" Processing Queue: {process_queue.queue.qsize()}:{db_queue.queue.qsize()} -" + f" Writing Queue: {DATABASE.batch_processor.write_queue.qsize()}" + ) else: symbol = command.get("symbol") event = command.get("event") @@ -210,7 +268,7 @@ async def process_message(message): raise e from e if result: - await DATABASE._write_to_db(result) # pylint: disable=protected-access + db_queue.queue.put_nowait(result) else: logger.info("PROVIDER INFO: %s", msg) @@ -219,89 +277,140 @@ async def connect_and_stream(): """Connect to the WebSocket and stream data to file.""" tasks: set = set() - - handler_task = asyncio.create_task( - queue.process_queue(lambda message: process_message(message)) + conn_kwargs = CONNECT_KWARGS.copy() + + conn_kwargs.update( + { + "ping_interval": 20, + "ping_timeout": 30, + "close_timeout": 1, + "max_queue": 1000, + } ) - tasks.add(handler_task) - for i in range(0, 128): - new_task = asyncio.create_task( - queue.process_queue(lambda message: process_message(message)) + + async def message_receiver(websocket): + """Message receiver.""" + try: + async for message in websocket: + input_queue.queue.put_nowait(message) + except Exception as e: + raise e from e + + async def process_input_messages(message): + """Process the messages offloaded from the websocket.""" + + def _process_in_thread(): + message_data = orjson.loads(message) + if isinstance(message_data, list): + status_msgs = [ + msg + for msg in message_data + if isinstance(msg, dict) and ("status" in msg or "message" in msg) + ] + data_msgs = [msg for msg in message_data if msg not in status_msgs] + + if status_msgs: + # Convert to sync call + asyncio.run(process_message(status_msgs)) + if data_msgs: + process_queue.queue.put_nowait(data_msgs) + elif isinstance(message_data, dict): + if "status" in message_data or "message" in message_data: + asyncio.run(process_message(message_data)) + else: + process_queue.queue.put_nowait(message_data) + elif isinstance(message_data, str) and "status" in message_data: + asyncio.run(process_message(message_data)) + + # Run processing in thread + await asyncio.get_event_loop().run_in_executor( + message_thread_pool, _process_in_thread ) - tasks.add(new_task) - stdin_task = asyncio.shield(asyncio.create_task(read_stdin())) + try: - connect_kwargs = CONNECT_KWARGS.copy() - connect_kwargs["max_size"] = None - connect_kwargs["read_limit"] = 2**32 - connect_kwargs["close_timeout"] = 10 - connect_kwargs["ping_timeout"] = None + handler_task = asyncio.create_task( + process_queue.process_queue(lambda message: process_message(message)) + ) + tasks.add(handler_task) + stdin_task = asyncio.create_task(read_stdin()) + tasks.add(stdin_task) + + await DATABASE.start_writer() + + for _ in range(30): + processor_task = asyncio.create_task( + input_queue.process_queue( + lambda message: process_input_messages(message) + ) + ) + tasks.add(processor_task) + handler_task = asyncio.create_task( + process_queue.process_queue(lambda message: process_message(message)) + ) + tasks.add(handler_task) + + async for websocket in connect(URL, **conn_kwargs): + try: + if not any( + task.name == "cmd_task" for task in tasks if hasattr(task, "name") + ): + cmd_task = asyncio.create_task( + process_stdin_queue(websocket), name="cmd_task" + ) + tasks.add(cmd_task) - try: - async with websockets.connect(URL, **connect_kwargs) as websocket: await login(websocket) + response = await websocket.recv() messages = json.loads(response) + await process_message(messages) + await subscribe(websocket, kwargs["symbol"], "subscribe") - response = await websocket.recv() - messages = json.loads(response) - await process_message(messages) - cmd_task = asyncio.get_running_loop().create_task( - process_stdin_queue(websocket) - ) - while True: - messages = await websocket.recv() - await queue.enqueue(json.loads(messages)) - cmd_task.cancel() - await cmd_task - asyncio.gather(*cmd_task, return_exceptions=True) + if not any( + task.name == "receiver" for task in tasks if hasattr(task, "name") + ): + receiver_task = asyncio.create_task( + message_receiver(websocket), name="receiver" + ) + await asyncio.gather(receiver_task) + + # Attempt to reopen the connection + except ( + websockets.ConnectionClosed, + websockets.ConnectionClosedError, + ) as e: + msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" + logger.info(msg) + logger.info("PROVIDER INFO: Attempting to reconnect...") + await asyncio.sleep(1) + continue + + except websockets.ConnectionClosedOK as e: + msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" + logger.info(msg) + sys.exit(0) - except websockets.InvalidStatusCode as e: - if e.status_code == 404: - msg = f"PROVIDER ERROR: {e}" + except websockets.WebSocketException as e: + msg = f"PROVIDER ERROR: WebSocketException -> {e}" logger.error(msg) sys.exit(1) - else: - raise - except websockets.InvalidURI as e: - msg = f"PROVIDER ERROR: {e}" - logger.error(msg) - sys.exit(1) - - except websockets.ConnectionClosedOK as e: - msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" - logger.info(msg) - sys.exit(0) - - except websockets.ConnectionClosed as e: - msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" - logger.info(msg) - # Attempt to reopen the connection - logger.info("PROVIDER INFO: Attempting to reconnect after five seconds.") - await asyncio.sleep(5) - await connect_and_stream() - except websockets.WebSocketException as e: - msg = f"PROVIDER ERROR: WebSocketException -> {e}" - logger.error(msg) - sys.exit(1) - - except Exception as e: - msg = ( - f"PROVIDER ERROR: Unexpected error -> " - f"{e.__class__.__name__ if hasattr(e, '__class__') else e}: {e}" - ) - logger.error(msg) - sys.exit(1) + except Exception as e: # pylint: disable=broad-except + msg = ( + f"PROVIDER ERROR: Unexpected error -> " + f"{e.__class__.__name__ if hasattr(e, '__class__') else e}: {e.args}" + ) + logger.error(msg) + sys.exit(1) finally: - tasks.add(stdin_task) for task in tasks: task.cancel() await task asyncio.gather(*tasks, return_exceptions=True) + await DATABASE.stop_writer() sys.exit(0) @@ -310,6 +419,7 @@ async def connect_and_stream(): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) loop.set_exception_handler(lambda loop, context: None) + for sig in (signal.SIGINT, signal.SIGTERM): loop.add_signal_handler(sig, handle_termination_signal, logger) @@ -319,11 +429,22 @@ async def connect_and_stream(): ) loop.run_forever() + except (websockets.ConnectionClosed, websockets.ConnectionClosedError) as e: + msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" + logger.info(msg) + # Attempt to reopen the connection + logger.info("PROVIDER INFO: Attempting to reconnect...") + time.sleep(1) + asyncio.run_coroutine_threadsafe( + connect_and_stream(), + loop, + ) + except (KeyboardInterrupt, websockets.ConnectionClosed): logger.error("PROVIDER ERROR: WebSocket connection closed") except Exception as e: # pylint: disable=broad-except - ERR = f"PROVIDER ERROR: {e.__class__.__name__} -> {e}" + ERR = f"PROVIDER ERROR: {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" logger.error(ERR) finally: diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py index b8d7073e4b96..7deb68c177a4 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py @@ -21,7 +21,7 @@ # These are the data array order of definitions. IEX_FIELDS = [ "type", - "date", + "tiingo_date", "timestamp", "symbol", "bid_size", @@ -40,7 +40,7 @@ FX_FIELDS = [ "type", "symbol", - "date", + "tiingo_date", "bid_size", "bid_price", "mid_price", @@ -51,7 +51,7 @@ CRYPTO_TRADE_FIELDS = [ "type", "symbol", - "date", + "tiingo_date", "exchange", "last_size", "last_price", @@ -59,7 +59,7 @@ CRYPTO_QUOTE_FIELDS = [ "type", "symbol", - "date", + "tiingo_date", "exchange", "bid_size", "bid_price", @@ -101,6 +101,9 @@ class TiingoWebSocketQueryParams(WebSocketQueryParams): class TiingoWebSocketData(WebSocketData): """Tiingo WebSocket data model.""" + timestamp: Optional[datetime] = Field( + description="The timestamp of the data.", + ) type: Literal["quote", "trade", "break"] = Field( description="The type of data.", ) @@ -187,17 +190,18 @@ def _validate_date(cls, v): dt = to_datetime(v, utc=True).tz_convert(timezone("America/New_York")) else: try: - dt = datetime.fromtimestamp(v / 1000) + dt = datetime.fromtimestamp(v / 1000, timezone.utc) + dt = dt.astimezone(timezone("America/New_York")) except Exception: if isinstance(v, (int, float)): # Check if the timestamp is in nanoseconds and convert to seconds if v > 1e12: v = v / 1e9 # Convert nanoseconds to seconds - dt = datetime.fromtimestamp(v) + dt = datetime.fromtimestamp(v, timezone.utc) + dt = dt.astimezone(timezone("America/New_York")) else: dt = v - - return dt + return dt.strftime("%Y-%m-%d %H:%M:%S.%f%z") @model_validator(mode="before") @classmethod @@ -263,8 +267,11 @@ async def aextract_data( limit=query.limit, results_file=query.results_file, table_name=query.table_name, - save_results=query.save_results, + save_database=query.save_database, data_model=TiingoWebSocketData, + prune_interval=query.prune_interval, + export_interval=query.export_interval, + export_directory=query.export_directory, sleep_time=query.sleep_time, broadcast_host=query.broadcast_host, broadcast_port=query.broadcast_port, diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index 0f7cf6a92650..9e04862a93df 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -5,10 +5,11 @@ import os import signal import sys +from datetime import datetime, UTC import websockets from openbb_core.provider.utils.errors import UnauthorizedError -from openbb_core.provider.utils.websockets.database import Database +from openbb_core.provider.utils.websockets.database import Database, DatabaseWriter from openbb_core.provider.utils.websockets.helpers import ( get_logger, handle_termination_signal, @@ -17,6 +18,7 @@ ) from openbb_core.provider.utils.websockets.message_queue import MessageQueue from openbb_tiingo.models.websocket_connection import TiingoWebSocketData +from pandas import to_datetime from pydantic import ValidationError URL_MAP = { @@ -58,7 +60,7 @@ CRYPTO_TRADE_FIELDS = [ "type", "symbol", - "date", + "timestamp", "exchange", "last_size", "last_price", @@ -66,7 +68,7 @@ CRYPTO_QUOTE_FIELDS = [ "type", "symbol", - "date", + "timestamp", "exchange", "bid_size", "bid_price", @@ -76,20 +78,26 @@ ] SUBSCRIPTION_ID = "" logger = get_logger("openbb.websocket.tiingo") -queue = MessageQueue(logger=logger) +input_queue = MessageQueue(logger=logger, backoff_factor=0) +db_queue = MessageQueue(logger=logger, backoff_factor=0) kwargs = parse_kwargs() CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) kwargs["results_file"] = os.path.abspath(kwargs["results_file"]) URL = URL_MAP.get(kwargs.pop("asset_type", "crypto")) +SUBSCRIBED_SYMBOLS: set = set() if not URL: raise ValueError("Invalid asset type provided.") -DATABASE = Database( - results_file=kwargs["results_file"], - table_name=kwargs["table_name"], - limit=kwargs.get("limit"), - logger=logger, +DATABASE = DatabaseWriter( + database=Database( + results_file=kwargs["results_file"], + table_name=kwargs["table_name"], + limit=kwargs.get("limit"), + logger=logger, + ), + queue=db_queue, + batch_size=100, ) @@ -121,6 +129,16 @@ async def update_symbols(symbol, event): msg = f"PROVIDER INFO: Subscribed to {tickers} with threshold level {threshold_level}" logger.info(msg) + symbols = symbol.split(",") + if event == "subscribe": + for sym in symbols: + SUBSCRIBED_SYMBOLS.add(sym) + elif event == "unsubscribe": + for sym in symbols: + SUBSCRIBED_SYMBOLS.discard(sym) + + kwargs["symbol"] = ",".join(SUBSCRIBED_SYMBOLS) + async def read_stdin_and_update_symbols(): """Read from stdin and update symbols.""" @@ -132,7 +150,10 @@ async def read_stdin_and_update_symbols(): break if "qsize" in line: - logger.info(f"PROVIDER INFO: Queue size: {queue.queue.qsize()}") + logger.info( + f"PROVIDER INFO: Input Queue : {input_queue.queue.qsize()}" + f" Database Queue : {db_queue.queue.qsize()}" + ) else: line = json.loads(line.strip()) @@ -175,7 +196,6 @@ async def process_message(message): # pylint: disable=too-many-branches threshold_level = message["data"].get("thresholdLevel") msg = f"PROVIDER INFO: Subscribed to {tickers} with threshold level {threshold_level}" logger.info(msg) - elif message.get("messageType") == "A": data = message.get("data", []) service = message.get("service") @@ -185,17 +205,22 @@ async def process_message(message): # pylint: disable=too-many-branches elif service == "fx": data_message = {FX_FIELDS[i]: data[i] for i in range(len(data))} elif service == "crypto_data": - if data[0] == "T": + if data[0] == "Q": data_message = { - CRYPTO_TRADE_FIELDS[i]: data[i] for i in range(len(data)) + CRYPTO_QUOTE_FIELDS[i]: data[i] for i in range(len(data)) } - elif data[0] == "Q": + elif data[0] == "T": data_message = { - CRYPTO_QUOTE_FIELDS[i]: data[i] for i in range(len(data)) + CRYPTO_TRADE_FIELDS[i]: data[i] for i in range(len(data)) } - else: - return - + data_message["date"] = datetime.now(UTC).isoformat() + tiingo_date = data_message.pop("tiingo_date", None) + if isinstance(tiingo_date, str): + tiingo_date = to_datetime(tiingo_date) + tiingo_date = tiingo_date.tz_convert("America/New_York").to_pydatetime() + data_message["timestamp"] = tiingo_date + + if data_message: try: result = TiingoWebSocketData.model_validate(data_message).model_dump_json( exclude_none=True, exclude_unset=True @@ -207,26 +232,26 @@ async def process_message(message): # pylint: disable=too-many-branches raise e from e if result: - await DATABASE._write_to_db(result) # pylint: disable=protected-access - return + await db_queue.enqueue(result) async def connect_and_stream(): """Connect to the WebSocket and stream data to file.""" tasks: set = set() + ticker: list = [] + + conn_kwargs = CONNECT_KWARGS.copy() - handler_task = asyncio.create_task( - queue.process_queue(lambda message: process_message(message)) + conn_kwargs.update( + { + "ping_interval": 8, + "ping_timeout": 8, + "read_limit": 2**256, + "close_timeout": 1, + "max_queue": 10000, + } ) - tasks.add(handler_task) - for i in range(0, 15): - new_task = asyncio.create_task( - queue.process_queue(lambda message: process_message(message)) - ) - tasks.add(new_task) - stdin_task = asyncio.create_task(read_stdin_and_update_symbols()) - ticker: list = [] if isinstance(kwargs["symbol"], str): ticker = kwargs["symbol"].lower().split(",") @@ -239,31 +264,41 @@ async def connect_and_stream(): "tickers": ticker, }, } - connect_kwargs = CONNECT_KWARGS.copy() - if "ping_timeout" not in connect_kwargs: - connect_kwargs["ping_timeout"] = None - if "close_timeout" not in connect_kwargs: - connect_kwargs["close_timeout"] = None + + async def message_receiver(websocket): + """Receive messages from the WebSocket.""" + while True: + message = await websocket.recv() + await input_queue.enqueue(message) + + stdin_task = asyncio.create_task(read_stdin_and_update_symbols()) + tasks.add(stdin_task) try: - try: - async with websockets.connect(URL, **connect_kwargs) as websocket: - logger.info("PROVIDER INFO: WebSocket connection established.") - await websocket.send(json.dumps(subscribe_event)) - while True: - message = await websocket.recv() - await queue.enqueue(message) + await DATABASE.start_writer() + websocket = await websockets.connect(URL, **conn_kwargs) + receiver_task = asyncio.create_task(message_receiver(websocket)) + tasks.add(receiver_task) + await websocket.send(json.dumps(subscribe_event)) + logger.info("PROVIDER INFO: WebSocket connection established.") + for _ in range(9): + process_task = asyncio.create_task( + input_queue.process_queue(lambda message: process_message(message)) + ) + tasks.add(process_task) - except UnauthorizedError as e: - logger.error(str(e)) - sys.exit(1) + await asyncio.gather(*tasks, return_exceptions=True) + + except UnauthorizedError as e: + logger.error(str(e)) + sys.exit(1) except websockets.ConnectionClosed as e: msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" logger.info(msg) # Attempt to reopen the connection - logger.info("PROVIDER INFO: Attempting to reconnect after five seconds...") - await asyncio.sleep(5) + logger.info("PROVIDER INFO: Attempting to reconnect...") + await asyncio.sleep(1) await connect_and_stream() except websockets.WebSocketException as e: @@ -271,16 +306,17 @@ async def connect_and_stream(): sys.exit(0) except Exception as e: # pylint: disable=broad-except - msg = f"Unexpected error -> {e.__class__.__name__}: {e}" + msg = f"Unexpected error -> {e.__class__.__name__ if hasattr(e, '__class__') else e}: {e.args}" logger.error(msg) sys.exit(1) finally: - tasks.add(stdin_task) + await websocket.close() for task in tasks: task.cancel() await task asyncio.gather(*tasks, return_exceptions=True) + await DATABASE.stop_writer() sys.exit(0) @@ -289,7 +325,8 @@ async def connect_and_stream(): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) loop.set_exception_handler(lambda loop, context: None) - loop.add_signal_handler(signal.SIGTERM, handle_termination_signal, logger) + for sig in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(signal.SIGTERM, handle_termination_signal, logger) asyncio.run_coroutine_threadsafe( connect_and_stream(), @@ -297,6 +334,9 @@ async def connect_and_stream(): ) loop.run_forever() + except (KeyboardInterrupt, websockets.ConnectionClosed): + logger.error("PROVIDER ERROR: WebSocket connection closed") + except Exception as e: # pylint: disable=broad-except ERR = ( f"PROVIDER ERROR: Unexpected error -> " From 4f7d1277bf5830127edc77a73daf5371e2605baa Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 30 Dec 2024 23:37:01 -0800 Subject: [PATCH 104/119] fix some stuff i broke --- .../provider/utils/websockets/database.py | 81 ++++++++----------- .../websockets/openbb_websockets/models.py | 4 + 2 files changed, 37 insertions(+), 48 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 0be969d2732c..6f169f30ec69 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -563,13 +563,11 @@ def clear_results(self) -> None: def create_writer( self, queue: Optional["MessageQueue"] = None, - batch_size: int = 250, - collection_time: int = 0.1, + batch_size: int = 25000, prune_interval: Optional[int] = None, export_directory: Optional[Union[str, "Path"]] = None, export_interval: Optional[int] = None, compress_export: bool = False, - num_workers: int = 60, verbose: bool = True, ): """ @@ -585,12 +583,10 @@ def create_writer( self, queue, batch_size, - collection_time, prune_interval, export_directory, export_interval, compress_export, - num_workers, verbose, ) @@ -613,34 +609,26 @@ class DatabaseWriter: queue : Optional[MessageQueue] The MessageQueue instance to use. Default is None, which creates a new instance. batch_size : int - The target batch size for writing to the database. Default is 250. - collection_time : int - The maximum time in seconds to collect messages before writing to the database. Default is 0.1. + The target batch size for writing to the database. Default is 25000. prune_interval : Optional[int] The interval in minutes to prune the database of older records. Default is None. export_directory : Optional[Union[str, Path]] The directory to export the database to, if an interval is set. Default is 'OpenBBUserData/exports/websockets'. export_interval : Optional[int] The interval in minutes to export the database to a CSV file. Default is None. - compress_export : bool - Whether to compress the exported CSV files. Default is False. num_workers : int - The number of parallel writers reading the queue and preparing batches. Default is 60. - verbose : bool - Whether to print verbose output for export and prune tasks. Default is True. + The number of parallel writers to use. Default is 120. """ def __init__( self, database: Database, queue: Optional["MessageQueue"] = None, - batch_size: int = 250, - collection_time: int = 0.1, + batch_size: int = 25000, prune_interval: Optional[int] = None, export_directory: Optional[Union[str, "Path"]] = None, export_interval: Optional[int] = None, compress_export: bool = False, - num_workers: int = 60, verbose: bool = True, ): """Initialize the DatabaseWriter class.""" @@ -683,7 +671,7 @@ def __init__( self._first_timestamp = None self._last_processed_timestamp = None self._conn = None - self.num_workers = num_workers + self.num_workers = 60 self.write_tasks = [] self._export_running = False self._prune_running = False @@ -707,7 +695,7 @@ async def start_writer(self): async def stop_writer(self): """Stop all queue processors.""" - self._flush_queue() + await self._flush_queue() self.writer_running = False await asyncio.gather(*self.write_tasks, return_exceptions=True) if self._conn: @@ -717,17 +705,11 @@ async def stop_writer(self): async def _process_queue(self): """Process queue with parallel writers.""" - # pylint: disable=import-outside-toplevel - import time + batch = [] - batch: list = [] - collection_start = time.time() while self.writer_running: try: - while ( - time.time() - collection_start < self.collection_time - and len(batch) < self.batch_size - ): + while len(batch) < self.batch_size: try: message = await asyncio.wait_for( self.queue.dequeue(), timeout=0.1 @@ -746,11 +728,12 @@ async def _process_queue(self): self.database.logger.error(msg, exc_info=True) await asyncio.sleep(0.1) - def _flush_queue(self): + async def _flush_queue(self): """Flush the queue of messages to the database.""" batch: list = [] - while not self.queue.queue.empty(): - batch.append(self.queue.queue.get_nowait()) + while not self.queue.queue.empty() and len(batch) < self.batch_size: + batch.append(await self.queue.dequeue()) + self._write_batch(batch) def _write_batch(self, batch): @@ -886,7 +869,8 @@ async def _export_database(self): await f.write(buffer.getvalue()) self._last_processed_timestamp = cutoff_time - if self.verbose: + + if self.verbose is True: msg = f"DATABASE INFO: Interval for period ending {cutoff_time} saved to: {path}" sys.stdout.write(msg + "\n") sys.stdout.flush() @@ -913,15 +897,12 @@ def start_export_task(self): if not self._export_running: self._export_running = True return - try: - self._export_running = True - self.export_thread = threading.Thread( - target=self._run_export_event, name="ExportThread", daemon=True - ) - self.export_thread.start() - finally: - self.export_thread.join(timeout=1) - self._export_running = False + + self._export_running = True + self.export_thread = threading.Thread( + target=self._run_export_event, name="ExportThread", daemon=True + ) + self.export_thread.start() def stop_export_task(self): """Public method to stop the background export task.""" @@ -947,14 +928,13 @@ def start_prune_task(self): try: self._prune_running = True - prune_thread = threading.Thread( - target=self._run_prune_event, daemon=True, name="WebSocketPruneThread" - ) + prune_thread = threading.Thread(target=self._run_prune_event) + prune_thread.daemon = True + prune_thread.name = "WebSocketPruneThread" self.prune_thread = prune_thread self.prune_thread.start() finally: self.prune_thread.join(timeout=1) - self._prune_running = False def stop_prune_task(self): """Public method to stop the background pruning task.""" @@ -1007,7 +987,7 @@ async def _start_prune_task(self): async with self.database.get_connection("write") as conn: - if self.verbose: + if self.verbose is True: msg = f"DATABASE INFO: Pruning database of records before: {cutoff_timestamp}" sys.stdout.write(msg + "\n") sys.stdout.flush() @@ -1077,7 +1057,9 @@ class BatchProcessor(threading.Thread): This class is a thread intended for use as a subprocess and is called by `DatabaseWriter.start_writer()`. """ - def __init__(self, database_writer: DatabaseWriter): + def __init__( + self, database_writer: DatabaseWriter, num_workers=120, collection_time=0.35 + ): """Initialize the BatchProcessor class.""" # pylint: disable=import-outside-toplevel import queue @@ -1087,7 +1069,9 @@ def __init__(self, database_writer: DatabaseWriter): self.write_queue: queue.Queue = queue.Queue() self.running = True self.loop = None + self.num_workers = num_workers self.workers: list = [] + self.collection_time = collection_time def run(self): """Run the batch processor as tasks.""" @@ -1096,7 +1080,7 @@ def run(self): asyncio.set_event_loop(self.loop) try: # Create worker tasks - for _ in range(self.writer.num_workers): + for _ in range(self.num_workers): worker = self.loop.create_task(self._worker()) self.workers.append(worker) # Run workers @@ -1108,6 +1092,7 @@ async def _worker(self): # pylint: disable=import-outside-toplevel import time + batch_size = 200 while self.running: try: batch = [] @@ -1115,8 +1100,8 @@ async def _worker(self): collection_start = time.time() while ( - time.time() - collection_start < (self.writer.collection_time * 2.5) - and total < self.batch_size + time.time() - collection_start < self.collection_time + and total < batch_size ): if not self.write_queue.empty(): msg = self.write_queue.get_nowait() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/models.py b/openbb_platform/extensions/websockets/openbb_websockets/models.py index e55fdb136011..a5e6a20f1318 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/models.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/models.py @@ -76,6 +76,10 @@ class WebSocketQueryParams(QueryParams): description="A formatted dictionary, or serialized JSON string, of keyword arguments to pass" + " directly to websockets.connect().", ) + verbose: bool = Field( + default=True, + description="Whether to print export and prune messages to the console.", + ) @field_validator("connect_kwargs", mode="before", check_fields=False) @classmethod From 8657eaeab363d42f3db24c0f0d710e2fa241c948 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 31 Dec 2024 14:47:51 -0800 Subject: [PATCH 105/119] some attempts to tweak polygon client.. --- .../provider/utils/websockets/database.py | 38 +++++++++------- .../models/websocket_connection.py | 4 ++ .../openbb_intrinio/utils/websocket_client.py | 2 +- .../models/websocket_connection.py | 1 + .../openbb_polygon/utils/websocket_client.py | 43 +++++++++---------- 5 files changed, 49 insertions(+), 39 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 6f169f30ec69..294a02b6879e 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -756,7 +756,7 @@ async def _export_database(self): chunk_size = 20000 minutes = self.export_interval or 5 - + latest_date = None if not self._export_running or not self.export_thread: return try: @@ -768,25 +768,31 @@ async def _export_database(self): """ # noqa async with self.database.get_connection("read") as conn: - cursor = await conn.execute(latest_query) - latest_date = await cursor.fetchone() - if not latest_date: + try: + async with conn.execute(latest_query) as cursor: + latest_date = await cursor.fetchone() + if not latest_date: + return + + except asyncio.InvalidStateError as e: + self.database.logger.error(f"Database connection state error: {e}") + self._export_running = False return - latest_timestamp = to_datetime(latest_date[0]) + latest_timestamp = to_datetime(latest_date[0]) - # Round down to nearest interval - cutoff_time = latest_timestamp - timedelta( - minutes=latest_timestamp.minute % minutes, - seconds=latest_timestamp.second, - microseconds=latest_timestamp.microsecond, - ) + # Round down to nearest interval + cutoff_time = latest_timestamp - timedelta( + minutes=latest_timestamp.minute % minutes, + seconds=latest_timestamp.second, + microseconds=latest_timestamp.microsecond, + ) - # If we have processed data before, use that as reference - if self._last_processed_timestamp: - start_time = self._last_processed_timestamp - else: - start_time = cutoff_time - timedelta(minutes=minutes) + # If we have processed data before, use that as reference + if self._last_processed_timestamp: + start_time = self._last_processed_timestamp + else: + start_time = cutoff_time - timedelta(minutes=minutes) cutoff_time = self._last_processed_timestamp + timedelta(minutes=minutes) start_time = self._last_processed_timestamp diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py index 05eac814a9ec..8133c7699361 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py @@ -189,6 +189,10 @@ async def aextract_data( broadcast_host=query.broadcast_host, broadcast_port=query.broadcast_port, auth_token=query.auth_token, + export_directory=query.export_directory, + export_interval=query.export_interval, + prune_interval=query.prune_interval, + verbose=query.verbose, **kwargs, ) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py index 4ddfad2fb415..917ab79cdf15 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -143,7 +143,7 @@ async def connect_and_stream(): try: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - # loop.set_exception_handler(lambda loop, context: None) + loop.set_exception_handler(lambda loop, context: None) for sig in (signal.SIGINT, signal.SIGTERM): loop.add_signal_handler(sig, handle_termination_signal, logger) diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index cb3f078030bb..ccfae237c355 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -1225,6 +1225,7 @@ def extract_data( broadcast_host=query.broadcast_host, broadcast_port=query.broadcast_port, auth_token=query.auth_token, + verbose=query.verbose, **kwargs, ) diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index cf37ba5687fc..ab6ef1459d16 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -32,14 +32,14 @@ import asyncio import json -import orjson import os import signal import sys import time +from concurrent.futures import ThreadPoolExecutor +import orjson import websockets -from concurrent.futures import ThreadPoolExecutor from openbb_core.provider.utils.websockets.database import Database, DatabaseWriter from openbb_core.provider.utils.websockets.helpers import ( get_logger, @@ -67,10 +67,10 @@ } logger = get_logger("openbb.websocket.polygon") -process_queue = MessageQueue(logger=logger, backoff_factor=0) -input_queue = MessageQueue(logger=logger, backoff_factor=0) -command_queue = MessageQueue(logger=logger, backoff_factor=0) -db_queue = MessageQueue(logger=logger, backoff_factor=0, max_size=100000) +process_queue = MessageQueue(logger=logger, backoff_factor=0, max_size=1000000) +input_queue = MessageQueue(logger=logger, backoff_factor=0, max_size=1000000) +command_queue = MessageQueue(logger=logger, backoff_factor=0, max_size=1000000) +db_queue = MessageQueue(logger=logger, backoff_factor=0, max_size=1000000) kwargs = parse_kwargs() CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) FEED = kwargs.pop("feed", None) @@ -78,7 +78,7 @@ kwargs["results_file"] = os.path.abspath(kwargs.get("results_file")) URL = URL_MAP.get(ASSET_TYPE) SUBSCRIBED_SYMBOLS: set = set() -message_thread_pool = ThreadPoolExecutor(max_workers=8) +message_thread_pool = ThreadPoolExecutor(max_workers=16) if not kwargs.get("api_key"): raise ValueError("No API key provided.") @@ -93,7 +93,6 @@ limit=kwargs.get("limit"), logger=logger, ), - batch_size=1, queue=db_queue, ) @@ -281,10 +280,11 @@ async def connect_and_stream(): conn_kwargs.update( { - "ping_interval": 20, - "ping_timeout": 30, + "ping_interval": None, + "ping_timeout": None, "close_timeout": 1, - "max_queue": 1000, + "max_size": 2**63, + "max_queue": None, } ) @@ -293,6 +293,7 @@ async def message_receiver(websocket): try: async for message in websocket: input_queue.queue.put_nowait(message) + except Exception as e: raise e from e @@ -337,17 +338,14 @@ def _process_in_thread(): await DATABASE.start_writer() - for _ in range(30): - processor_task = asyncio.create_task( - input_queue.process_queue( - lambda message: process_input_messages(message) - ) - ) - tasks.add(processor_task) - handler_task = asyncio.create_task( - process_queue.process_queue(lambda message: process_message(message)) - ) - tasks.add(handler_task) + processor_task = asyncio.create_task( + input_queue.process_queue(lambda message: process_input_messages(message)) + ) + tasks.add(processor_task) + handler_task = asyncio.create_task( + process_queue.process_queue(lambda message: process_message(message)) + ) + tasks.add(handler_task) async for websocket in connect(URL, **conn_kwargs): try: @@ -374,6 +372,7 @@ def _process_in_thread(): receiver_task = asyncio.create_task( message_receiver(websocket), name="receiver" ) + tasks.add(receiver_task) await asyncio.gather(receiver_task) # Attempt to reopen the connection From 9bb13d2813dc696363c22936101bb29fd5f493af Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Tue, 31 Dec 2024 15:53:15 -0800 Subject: [PATCH 106/119] no more changes until next year --- .../openbb_intrinio/utils/websocket_client.py | 13 ++++---- .../openbb_polygon/utils/websocket_client.py | 31 +++++++------------ 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py index 917ab79cdf15..b5fc09de60c6 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -99,12 +99,6 @@ async def read_stdin_and_queue_commands(): break try: - if line == "qsize": - logger.info( - "Queue size: %i - Writer Queue: %i", - command_queue.queue.qsize(), - DATABASE.batch_processor.task_queue.qsize(), - ) command = json.loads(line.strip()) await command_queue.enqueue(command) except json.JSONDecodeError: @@ -115,6 +109,13 @@ async def process_stdin_queue(): """Process the command queue.""" while True: command = await command_queue.dequeue() + if command == "qsize": + logger.info( + "Queue size: %i - Writer Queue: %i", + command_queue.queue.qsize(), + DATABASE.batch_processor.task_queue.qsize(), + ) + symbol = ["lobby" if d == "*" else d.upper() for d in command.get("symbol", [])] event = command.get("event") if symbol and event: diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index ab6ef1459d16..39a23bf165dc 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -324,9 +324,8 @@ def _process_in_thread(): asyncio.run(process_message(message_data)) # Run processing in thread - await asyncio.get_event_loop().run_in_executor( - message_thread_pool, _process_in_thread - ) + process = asyncio.to_thread(_process_in_thread) + asyncio.create_task(process) try: handler_task = asyncio.create_task( @@ -338,14 +337,13 @@ def _process_in_thread(): await DATABASE.start_writer() - processor_task = asyncio.create_task( - input_queue.process_queue(lambda message: process_input_messages(message)) - ) - tasks.add(processor_task) - handler_task = asyncio.create_task( - process_queue.process_queue(lambda message: process_message(message)) - ) - tasks.add(handler_task) + for i in range(16): + processor_task = asyncio.create_task( + input_queue.process_queue( + lambda message: process_input_messages(message) + ) + ) + tasks.add(processor_task) async for websocket in connect(URL, **conn_kwargs): try: @@ -360,20 +358,13 @@ def _process_in_thread(): await login(websocket) response = await websocket.recv() - messages = json.loads(response) + messages = orjson.loads(response) await process_message(messages) await subscribe(websocket, kwargs["symbol"], "subscribe") - if not any( - task.name == "receiver" for task in tasks if hasattr(task, "name") - ): - receiver_task = asyncio.create_task( - message_receiver(websocket), name="receiver" - ) - tasks.add(receiver_task) - await asyncio.gather(receiver_task) + await message_receiver(websocket) # Attempt to reopen the connection except ( From 516a68cf7f6a44d07fa1f18d3a8bc65619f52b36 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 1 Jan 2025 23:47:17 -0800 Subject: [PATCH 107/119] playing with some settings and update database test --- .../provider/utils/websockets/client.py | 2 +- .../provider/utils/websockets/database.py | 175 ++++++++++-------- .../provider/utils/websockets/helpers.py | 34 +++- .../utils/websockets/test_database.py | 52 +++++- .../websockets/openbb_websockets/helpers.py | 2 + .../openbb_polygon/utils/websocket_client.py | 64 ++++--- .../openbb_tiingo/utils/websocket_client.py | 88 +++++---- 7 files changed, 272 insertions(+), 145 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py index 659874fb81d4..704e3aa7be82 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py @@ -852,7 +852,7 @@ def stop_broadcasting(self): def __repr__(self): """Return the WebSocketClient representation.""" return ( - f"WebSocketClient(module={[d for d in self.module if "api_key" not in d]}, symbol={self.symbol}, " + f"WebSocketClient(module={[d for d in self.module if 'api_key' not in d]}, symbol={self.symbol}, " f"is_running={self.is_running}, provider_pid: " f"{self._psutil_process.pid if self._psutil_process else ''}, is_broadcasting={self.is_broadcasting}, " f"broadcast_address={self.broadcast_address}, " diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 294a02b6879e..c307fd946944 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -8,8 +8,12 @@ from datetime import timedelta from typing import TYPE_CHECKING, Any, Iterable, Optional, Union +import uvloop from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.utils.helpers import run_async +from openbb_core.provider.utils.websockets.helpers import kill_thread + +asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) if TYPE_CHECKING: import logging @@ -65,30 +69,6 @@ ) -def kill_thread(thread: threading.Thread) -> None: - """Kill thread by setting a stop flag.""" - # pylint: disable=import-outside-toplevel - import ctypes - - if hasattr(thread, "loop") and thread.loop: - for task in asyncio.all_tasks(thread.loop): - task.cancel() - - if not thread.is_alive(): - return - - thread_id = thread.ident - if thread_id is None: - return - - res = ctypes.pythonapi.PyThreadState_SetAsyncExc( - ctypes.c_long(thread_id), ctypes.py_object(SystemExit) - ) - if res > 1: - ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(thread_id), None) - raise SystemError("PyThreadState_SetAsyncExc failed") - - class Database: """ Class to read from, and write to, the SQL file using aiosqlite. @@ -160,7 +140,7 @@ def __init__( # pylint: disable=too-many-positional-arguments data_model: Optional["BaseModel"] = None, limit: Optional[int] = None, logger: Optional["logging.Logger"] = None, - loop: Optional["asyncio.AbstractEventLoop"] = None, + loop: Optional[asyncio.AbstractEventLoop] = None, **kwargs, ): """Initialize the ResultsDB class.""" @@ -212,7 +192,7 @@ async def _setup_database(self): try: if self.results_file is not None and os.path.exists(self.results_file): # type: ignore - async with self.get_connection("read") as conn: + async with self.get_connection("write") as conn: try: cursor = await conn.execute( "SELECT name FROM sqlite_master WHERE type='table';" @@ -238,7 +218,8 @@ async def _setup_database(self): ) pragmas = [ "PRAGMA journal_mode=WAL", - "PRAGMA synchronous=OFF", + "PRAGMA synchronous=off", + "PRAGMA temp_store=memory", ] for pragma in pragmas: @@ -288,6 +269,15 @@ async def get_connection(self, name: str = "read"): if name not in self._connections: conn = await aiosqlite.connect(results_file, **conn_kwargs) + pragmas = [ + "PRAGMA journal_mode=WAL", + "PRAGMA synchronous=off", + "PRAGMA temp_store=memory", + ] + for pragma in pragmas: + await conn.execute(pragma) + + await conn.commit() self._connections[name] = conn yield self._connections[name] @@ -373,30 +363,39 @@ async def _fetch_all(self, limit: Optional[int] = None) -> list: params = () async with conn.execute(query, params) as cursor: async for row in cursor: - rows.append(await self._deserialize_row(row)) + rows.append(await self._deserialize_row(row, cursor)) return rows except Exception as e: # pylint: disable=broad-except raise OpenBBError(e) from e - async def _deserialize_row(self, row) -> dict: - """Deserialize a row from the SQLite database.""" + async def _deserialize_row(self, row, cursor) -> dict: + """Deserialize a row from the SQLite database. + Handles both single message column and multiple extracted fields.""" # pylint: disable=import-outside-toplevel import json try: - return ( - json.loads(row[0]) - if ( - ( + if len(row) == 1: + # Single column case (full message) + return ( + json.loads(row[0]) + if ( isinstance(row[0], str) and (row[0].startswith("{") or row[0].startswith("[")) ) or isinstance(row[0], bytes) + else row[0] ) - else row[0] - ) + else: + # Multiple column case (extracted fields) + return {cursor.description[i][0]: row[i] for i in range(len(row))} + + except (json.JSONDecodeError, AttributeError) as e: + self.logger.error(f"Failed to deserialize row: {e}") + return row[0] if len(row) == 1 else dict(enumerate(row)) + except Exception as e: # pylint: disable=broad-except msg = ( "Unexpected error while deserializing row -> " @@ -453,14 +452,12 @@ def get_latest_results( async def _query_db(self, sql, parameters: Optional[Iterable[Any]] = None) -> list: """Query the SQLite database.""" - # pylint: disable=import-outside-toplevel - import json if not sql or sql in ("", "''"): raise OpenBBError("Empty query not allowed.") query = ( sql - if sql.startswith("SELECT") + if sql.strip().startswith("SELECT") else f"SELECT message FROM {self.table_name} WHERE {sql}" # noqa ) if not query.endswith(";"): @@ -479,17 +476,7 @@ async def _query_db(self, sql, parameters: Optional[Iterable[Any]] = None) -> li query, parameters ) as cursor: async for row in cursor: - rows.append( - json.loads(row[0]) - if ( - ( - isinstance(row[0], str) - and (row[0].startswith("{") or row[0].startswith("[")) - ) - or isinstance(row[0], bytes) - ) - else row[0] - ) + rows.append(await self._deserialize_row(row, cursor)) return rows except Exception as e: # pylint: disable=broad-except raise OpenBBError(e) from e @@ -563,7 +550,7 @@ def clear_results(self) -> None: def create_writer( self, queue: Optional["MessageQueue"] = None, - batch_size: int = 25000, + batch_size: int = 200, prune_interval: Optional[int] = None, export_directory: Optional[Union[str, "Path"]] = None, export_interval: Optional[int] = None, @@ -624,7 +611,7 @@ def __init__( self, database: Database, queue: Optional["MessageQueue"] = None, - batch_size: int = 25000, + batch_size: int = 200, prune_interval: Optional[int] = None, export_directory: Optional[Union[str, "Path"]] = None, export_interval: Optional[int] = None, @@ -687,7 +674,8 @@ async def _create_connection(self): async def start_writer(self): """Start writing tasks.""" if not self.writer_running: - asyncio.get_event_loop().run_in_executor(None, self.batch_processor.start()) + if not self.batch_processor.is_alive(): + self.batch_processor.start() await self._create_connection() for _ in range(self.num_workers): task = asyncio.create_task(self._process_queue()) @@ -700,7 +688,7 @@ async def stop_writer(self): await asyncio.gather(*self.write_tasks, return_exceptions=True) if self._conn: await self._conn.close() - self.batch_processor.running = False + self.batch_processor.stop() kill_thread(self.batch_processor) async def _process_queue(self): @@ -718,7 +706,7 @@ async def _process_queue(self): except asyncio.TimeoutError: break if batch: - self._write_batch(batch) + await self._write_batch(batch) batch = [] else: await asyncio.sleep(0.001) @@ -734,9 +722,9 @@ async def _flush_queue(self): while not self.queue.queue.empty() and len(batch) < self.batch_size: batch.append(await self.queue.dequeue()) - self._write_batch(batch) + await self._write_batch(batch) - def _write_batch(self, batch): + async def _write_batch(self, batch): """Write the batch of messages to the database.""" if not batch: return @@ -757,6 +745,7 @@ async def _export_database(self): chunk_size = 20000 minutes = self.export_interval or 5 latest_date = None + earliest_date = None if not self._export_running or not self.export_thread: return try: @@ -767,6 +756,12 @@ async def _export_database(self): ORDER BY json_extract(message, '$.date') DESC LIMIT 1 """ # noqa + earliest_query = f""" + SELECT json_extract(message, '$.date') + FROM {self.database.table_name} + ORDER BY json_extract(message, '$.date') ASC LIMIT 1 + """ # noqa + async with self.database.get_connection("read") as conn: try: async with conn.execute(latest_query) as cursor: @@ -774,19 +769,29 @@ async def _export_database(self): if not latest_date: return + async with conn.execute(earliest_query) as cursor: + earliest_date = await cursor.fetchone() + if not latest_date: + return + except asyncio.InvalidStateError as e: self.database.logger.error(f"Database connection state error: {e}") self._export_running = False return latest_timestamp = to_datetime(latest_date[0]) - + earliest_timestamp = to_datetime(earliest_date[0]) # Round down to nearest interval cutoff_time = latest_timestamp - timedelta( minutes=latest_timestamp.minute % minutes, seconds=latest_timestamp.second, microseconds=latest_timestamp.microsecond, ) + earliest_time = earliest_timestamp - timedelta( + minutes=latest_timestamp.minute % minutes, + seconds=latest_timestamp.second, + microseconds=latest_timestamp.microsecond, + ) # If we have processed data before, use that as reference if self._last_processed_timestamp: @@ -794,8 +799,9 @@ async def _export_database(self): else: start_time = cutoff_time - timedelta(minutes=minutes) - cutoff_time = self._last_processed_timestamp + timedelta(minutes=minutes) - start_time = self._last_processed_timestamp + if start_time < earliest_timestamp: + start_time = earliest_time + results_file = ( self.export_directory + "/" @@ -1042,9 +1048,7 @@ async def _start_export_task(self): count = await self.database._query_db(query, (cutoff_timestamp,)) if count[0] > 0: - # task = asyncio.create_task(self._export_database()) - task = await asyncio.to_thread(self._export_database) - await asyncio.gather(task) + await self._export_database() else: await asyncio.sleep(minutes * 60) # Stagger slightly so things don't happen exactly on the minute. @@ -1058,13 +1062,11 @@ async def _start_export_task(self): class BatchProcessor(threading.Thread): """ - Batch processor for writing messages to the SQLite database in a new thread. - This class is a thread intended for use as a subprocess and is called by `DatabaseWriter.start_writer()`. """ def __init__( - self, database_writer: DatabaseWriter, num_workers=120, collection_time=0.35 + self, database_writer: DatabaseWriter, num_workers=120, collection_time=0.25 ): """Initialize the BatchProcessor class.""" # pylint: disable=import-outside-toplevel @@ -1078,20 +1080,42 @@ def __init__( self.num_workers = num_workers self.workers: list = [] self.collection_time = collection_time + self._shutdown = threading.Event() def run(self): """Run the batch processor as tasks.""" - - self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) try: + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) # Create worker tasks - for _ in range(self.num_workers): - worker = self.loop.create_task(self._worker()) - self.workers.append(worker) - # Run workers - self.loop.run_until_complete(asyncio.gather(*self.workers)) + while self.running and not self._shutdown.is_set(): + try: + self.loop.run_until_complete(self._worker()) + except (SystemExit, KeyboardInterrupt): + self.running = False + break + except Exception as e: + self.writer.database.logger.error(f"Batch processing error: {e}") + break finally: + self._cleanup() + + def stop(self): + """Signal thread to stop gracefully.""" + self.running = False + self._shutdown.set() + if self.loop and self.loop.is_running(): + self.loop.call_soon_threadsafe(self.loop.stop) + + def _cleanup(self): + """Clean up resources on shutdown""" + if self.loop: + pending = asyncio.all_tasks(self.loop) + for task in pending: + task.cancel() + self.loop.run_until_complete( + asyncio.gather(*pending, return_exceptions=True) + ) self.loop.close() async def _worker(self): @@ -1101,7 +1125,7 @@ async def _worker(self): batch_size = 200 while self.running: try: - batch = [] + batch: list = [] total = 0 collection_start = time.time() @@ -1136,6 +1160,7 @@ async def _write_batch(self, batch): for b in batch: values.extend([(msg,) for msg in b]) async with self.writer.database.get_connection("write") as conn: + await conn.execute("PRAGMA temp_store=memory") async with conn.cursor() as cursor: await cursor.executemany(query, values) await conn.commit() diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py b/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py index 23ea8f2f060d..df856d64cfe2 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/helpers.py @@ -4,9 +4,13 @@ import logging import re +from typing import TYPE_CHECKING from pydantic import ValidationError +if TYPE_CHECKING: + import threading + AUTH_TOKEN_FILTER = re.compile( r"(auth_token=)([^&]*)", re.IGNORECASE | re.MULTILINE, @@ -73,13 +77,10 @@ def decrypt_value(key, iv, encrypted_value): def handle_termination_signal(logger): """Handle termination signals to ensure graceful shutdown.""" - # pylint: disable=import-outside-toplevel - import sys - logger.info( "PROVIDER INFO: Termination signal received. WebSocket connection closed." ) - sys.exit(0) + raise SystemExit("Termination signal received.") def parse_kwargs() -> dict: @@ -127,3 +128,28 @@ def parse_kwargs() -> dict: _kwargs[key] = True return _kwargs + + +def kill_thread(thread: "threading.Thread") -> None: + """Kill thread by setting a stop flag.""" + # pylint: disable=import-outside-toplevel + import asyncio + import ctypes + + if hasattr(thread, "loop") and thread.loop: + for task in asyncio.all_tasks(thread.loop): + task.cancel() + + if not thread.is_alive(): + return + + thread_id = thread.ident + if thread_id is None: + return + + res = ctypes.pythonapi.PyThreadState_SetAsyncExc( + ctypes.c_long(thread_id), ctypes.py_object(SystemExit) + ) + if res > 1: + ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(thread_id), None) + raise SystemError("PyThreadState_SetAsyncExc failed") diff --git a/openbb_platform/core/tests/provider/utils/websockets/test_database.py b/openbb_platform/core/tests/provider/utils/websockets/test_database.py index ae4bef7c054d..5e2532df7e71 100644 --- a/openbb_platform/core/tests/provider/utils/websockets/test_database.py +++ b/openbb_platform/core/tests/provider/utils/websockets/test_database.py @@ -1,7 +1,11 @@ """Unit Tests For Database Operations.""" +import asyncio +import json + import pytest -from openbb_core.provider.utils.websockets.database import Database +from openbb_core.provider.utils.websockets.database import Database, DatabaseWriter +from openbb_core.provider.utils.websockets.message_queue import MessageQueue MOCK_MESSAGES = [ {"type": "trade", "symbol": "test1", "price": 100}, @@ -16,6 +20,19 @@ def database(): return Database(table_name="test") +@pytest.fixture +def database_writer(): + """Return a MessageQueue instance.""" + writer = DatabaseWriter(database=Database(table_name="test"), queue=MessageQueue()) + yield writer + + +@pytest.fixture +def message_queue(): + """Return a MessageQueue instance.""" + return MessageQueue() + + def test_setup_database(database): """Test if the database was setup.""" assert database @@ -114,3 +131,36 @@ def test_limit(): database.write_to_db(MOCK_MESSAGES[0]) query = "SELECT id FROM test_limit" assert database.query(query)[0] > 3 + + +def test_batch_process_thread(database_writer): + """Test if the batch process thread starts and stops.""" + database_writer.batch_processor.start() + assert database_writer.batch_processor.is_alive() + database_writer.batch_processor.stop() + database_writer.batch_processor.join(timeout=1) + assert not database_writer.batch_processor.is_alive() + + +@pytest.mark.asyncio +async def test_database_writer(database_writer): + """Test if the database writer is working.""" + assert database_writer + writer = database_writer + assert isinstance(writer, DatabaseWriter) + assert len(writer.database.fetch_all()) == 0 + await writer.start_writer() + assert not writer._export_running + assert not writer._prune_running + + for message in MOCK_MESSAGES: + await writer.queue.enqueue(json.dumps(message)) + + await asyncio.sleep(0.5) + + await writer.stop_writer() + + messages = writer.database.fetch_all() + assert len(messages) == 3 + + assert not writer.batch_processor.is_alive() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 0d9a46cac411..5a1ecd3f21b4 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -66,8 +66,10 @@ class StdOutSink: def write(self, message): """Write to stdout.""" # pylint: disable=import-outside-toplevel + import os import sys + os.set_blocking(sys.__stdout__.fileno(), False) # type: ignore cleaned_message = AUTH_TOKEN_FILTER.sub(r"\1********", message) if cleaned_message != message: cleaned_message = f"{cleaned_message}\n" diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index 39a23bf165dc..d12153f3eead 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -36,9 +36,9 @@ import signal import sys import time -from concurrent.futures import ThreadPoolExecutor import orjson +import uvloop import websockets from openbb_core.provider.utils.websockets.database import Database, DatabaseWriter from openbb_core.provider.utils.websockets.helpers import ( @@ -54,6 +54,9 @@ ) from pydantic import ValidationError from websockets.asyncio.client import connect +from websockets.extensions import permessage_deflate + +asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) URL_MAP = { "stock": "wss://socket.polygon.io/stocks", @@ -78,7 +81,6 @@ kwargs["results_file"] = os.path.abspath(kwargs.get("results_file")) URL = URL_MAP.get(ASSET_TYPE) SUBSCRIBED_SYMBOLS: set = set() -message_thread_pool = ThreadPoolExecutor(max_workers=16) if not kwargs.get("api_key"): raise ValueError("No API key provided.") @@ -86,6 +88,7 @@ if not URL: raise ValueError("Invalid asset type provided.") + DATABASE = DatabaseWriter( database=Database( results_file=kwargs["results_file"], @@ -158,8 +161,8 @@ async def login(websocket): login_event = f'{{"action":"auth","params":"{kwargs["api_key"]}"}}' try: await websocket.send(login_event) - res = await websocket.recv() - response = json.loads(res) + res = await websocket.recv(decode=False) + response = orjson.loads(res) messages = response if isinstance(response, list) else [response] for msg in messages: if msg.get("status") == "connected": @@ -213,7 +216,7 @@ async def subscribe(websocket, symbol, event): async def read_stdin(): """Read from stdin and queue commands.""" while True: - line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline) + line = await asyncio.to_thread(sys.stdin.readline) sys.stdin.flush() if line: try: @@ -280,22 +283,21 @@ async def connect_and_stream(): conn_kwargs.update( { - "ping_interval": None, "ping_timeout": None, "close_timeout": 1, - "max_size": 2**63, + "max_size": 32768, "max_queue": None, } ) async def message_receiver(websocket): """Message receiver.""" - try: - async for message in websocket: + while True: + try: + message = await websocket.recv(decode=False) input_queue.queue.put_nowait(message) - - except Exception as e: - raise e from e + except Exception as e: + raise e from e async def process_input_messages(message): """Process the messages offloaded from the websocket.""" @@ -325,7 +327,8 @@ def _process_in_thread(): # Run processing in thread process = asyncio.to_thread(_process_in_thread) - asyncio.create_task(process) + process_task = asyncio.create_task(process) + tasks.add(process_task) try: handler_task = asyncio.create_task( @@ -337,15 +340,25 @@ def _process_in_thread(): await DATABASE.start_writer() - for i in range(16): - processor_task = asyncio.create_task( - input_queue.process_queue( - lambda message: process_input_messages(message) - ) - ) - tasks.add(processor_task) - - async for websocket in connect(URL, **conn_kwargs): + processor_task = asyncio.create_task( + input_queue.process_queue(lambda message: process_input_messages(message)) + ) + tasks.add(processor_task) + + conn_kwargs["extensions"] = [ + permessage_deflate.ClientPerMessageDeflateFactory( + server_max_window_bits=11, + client_max_window_bits=11, + compress_settings={ + "memLevel": 1, + "level": 1, + }, + ), + ] + async for websocket in connect( + URL, + **conn_kwargs, + ): try: if not any( task.name == "cmd_task" for task in tasks if hasattr(task, "name") @@ -357,10 +370,9 @@ def _process_in_thread(): await login(websocket) - response = await websocket.recv() - messages = orjson.loads(response) + response = await websocket.recv(decode=False) - await process_message(messages) + await process_message(orjson.loads(response)) await subscribe(websocket, kwargs["symbol"], "subscribe") @@ -406,7 +418,7 @@ def _process_in_thread(): if __name__ == "__main__": try: - loop = asyncio.new_event_loop() + loop = uvloop.new_event_loop() asyncio.set_event_loop(loop) loop.set_exception_handler(lambda loop, context: None) diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index 9e04862a93df..9f88a6679346 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -1,12 +1,13 @@ """FMP WebSocket server.""" import asyncio -import json import os import signal import sys -from datetime import datetime, UTC +from datetime import UTC, datetime +import orjson +import uvloop import websockets from openbb_core.provider.utils.errors import UnauthorizedError from openbb_core.provider.utils.websockets.database import Database, DatabaseWriter @@ -21,6 +22,8 @@ from pandas import to_datetime from pydantic import ValidationError +asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + URL_MAP = { "stock": "wss://api.tiingo.com/iex", "fx": "wss://api.tiingo.com/fx", @@ -97,7 +100,6 @@ logger=logger, ), queue=db_queue, - batch_size=100, ) @@ -120,9 +122,9 @@ async def update_symbols(symbol, event): } async with websockets.connect(URL) as websocket: - await websocket.send(json.dumps(update_event)) - response = await websocket.recv() - message = json.loads(response) + await websocket.send(orjson.dumps(update_event)) + response = await websocket.recv(decode=False) + message = orjson.loads(response) if "tickers" in message.get("data", {}): tickers = message["data"]["tickers"] threshold_level = message["data"].get("thresholdLevel") @@ -155,7 +157,7 @@ async def read_stdin_and_update_symbols(): f" Database Queue : {db_queue.queue.qsize()}" ) else: - line = json.loads(line.strip()) + line = orjson.loads(line.strip()) if line: symbol = line.get("symbol") @@ -167,7 +169,7 @@ async def process_message(message): # pylint: disable=too-many-branches """Process the message and write to the database.""" result: dict = {} data_message: dict = {} - message = message if isinstance(message, (dict, list)) else json.loads(message) + message = message if isinstance(message, (dict, list)) else orjson.loads(message) msg: str = "" if message.get("messageType") == "E": response = message.get("response", {}) @@ -232,7 +234,7 @@ async def process_message(message): # pylint: disable=too-many-branches raise e from e if result: - await db_queue.enqueue(result) + db_queue.queue.put_nowait(result) async def connect_and_stream(): @@ -247,9 +249,7 @@ async def connect_and_stream(): { "ping_interval": 8, "ping_timeout": 8, - "read_limit": 2**256, "close_timeout": 1, - "max_queue": 10000, } ) @@ -268,38 +268,50 @@ async def connect_and_stream(): async def message_receiver(websocket): """Receive messages from the WebSocket.""" while True: - message = await websocket.recv() - await input_queue.enqueue(message) + message = await websocket.recv(decode=False) + input_queue.queue.put_nowait(orjson.loads(message)) stdin_task = asyncio.create_task(read_stdin_and_update_symbols()) tasks.add(stdin_task) try: await DATABASE.start_writer() - websocket = await websockets.connect(URL, **conn_kwargs) - receiver_task = asyncio.create_task(message_receiver(websocket)) - tasks.add(receiver_task) - await websocket.send(json.dumps(subscribe_event)) - logger.info("PROVIDER INFO: WebSocket connection established.") - for _ in range(9): - process_task = asyncio.create_task( - input_queue.process_queue(lambda message: process_message(message)) - ) - tasks.add(process_task) - await asyncio.gather(*tasks, return_exceptions=True) - - except UnauthorizedError as e: - logger.error(str(e)) - sys.exit(1) - - except websockets.ConnectionClosed as e: - msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" - logger.info(msg) - # Attempt to reopen the connection - logger.info("PROVIDER INFO: Attempting to reconnect...") - await asyncio.sleep(1) - await connect_and_stream() + async for websocket in websockets.connect(URL, **conn_kwargs): + try: + if not any( + task.name == "receiver_task" + for task in tasks + if hasattr(task, "name") + ): + receiver_task = asyncio.create_task( + message_receiver(websocket), name="receiver_task" + ) + tasks.add(receiver_task) + + await websocket.send(orjson.dumps(subscribe_event)) + logger.info("PROVIDER INFO: WebSocket connection established.") + for _ in range(9): + process_task = asyncio.create_task( + input_queue.process_queue( + lambda message: process_message(message) + ) + ) + tasks.add(process_task) + + await asyncio.gather(*tasks, return_exceptions=True) + + except UnauthorizedError as e: + logger.error(str(e)) + sys.exit(1) + + except websockets.ConnectionClosed as e: + msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" + logger.info(msg) + # Attempt to reopen the connection + logger.info("PROVIDER INFO: Attempting to reconnect...") + await asyncio.sleep(1) + continue except websockets.WebSocketException as e: logger.info(str(e)) @@ -311,7 +323,6 @@ async def message_receiver(websocket): sys.exit(1) finally: - await websocket.close() for task in tasks: task.cancel() await task @@ -322,9 +333,10 @@ async def message_receiver(websocket): if __name__ == "__main__": try: - loop = asyncio.new_event_loop() + loop = uvloop.new_event_loop() asyncio.set_event_loop(loop) loop.set_exception_handler(lambda loop, context: None) + for sig in (signal.SIGINT, signal.SIGTERM): loop.add_signal_handler(signal.SIGTERM, handle_termination_signal, logger) From 18bb17f04c80a2d02749dc685099ce294dbed482 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 2 Jan 2025 00:42:27 -0800 Subject: [PATCH 108/119] blockingio error --- .../core/openbb_core/provider/utils/websockets/database.py | 5 ++++- .../extensions/websockets/openbb_websockets/helpers.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index c307fd946944..168eed69c738 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -802,6 +802,9 @@ async def _export_database(self): if start_time < earliest_timestamp: start_time = earliest_time + # Set end_time to be one interval after start_time + end_time = start_time + timedelta(minutes=minutes) + results_file = ( self.export_directory + "/" @@ -820,7 +823,7 @@ async def _export_database(self): "read" ) as conn, conn.cursor() as cursor: await cursor.execute( - query, (start_time.isoformat(), cutoff_time.isoformat()) + query, (start_time.isoformat(), end_time.isoformat()) ) headers = OrderedDict() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 5a1ecd3f21b4..92bba3e38504 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -69,7 +69,7 @@ def write(self, message): import os import sys - os.set_blocking(sys.__stdout__.fileno(), False) # type: ignore + os.set_blocking(sys.__stdout__.fileno(), True) # type: ignore cleaned_message = AUTH_TOKEN_FILTER.sub(r"\1********", message) if cleaned_message != message: cleaned_message = f"{cleaned_message}\n" From 27a67a042d268ee5f2fa0334e8eb5da0f35dc837 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 2 Jan 2025 01:29:03 -0800 Subject: [PATCH 109/119] not ready for uvloop --- .../core/openbb_core/provider/utils/websockets/broadcast.py | 2 ++ .../core/openbb_core/provider/utils/websockets/database.py | 3 --- .../providers/polygon/openbb_polygon/utils/websocket_client.py | 3 --- .../providers/tiingo/openbb_tiingo/utils/websocket_client.py | 3 --- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py index 9e230c538a44..0d53eddba84c 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py @@ -10,6 +10,7 @@ from typing import Optional import uvicorn + from fastapi import FastAPI, WebSocket, WebSocketDisconnect from openbb_core.provider.utils.websockets.database import ( CHECK_FOR, @@ -23,6 +24,7 @@ ) from starlette.websockets import WebSocketState + kwargs = parse_kwargs() HOST = kwargs.pop("host", None) or "localhost" diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 168eed69c738..32441de23822 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -8,13 +8,10 @@ from datetime import timedelta from typing import TYPE_CHECKING, Any, Iterable, Optional, Union -import uvloop from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.utils.helpers import run_async from openbb_core.provider.utils.websockets.helpers import kill_thread -asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) - if TYPE_CHECKING: import logging from pathlib import Path diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index d12153f3eead..af0924f1d5d8 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -38,7 +38,6 @@ import time import orjson -import uvloop import websockets from openbb_core.provider.utils.websockets.database import Database, DatabaseWriter from openbb_core.provider.utils.websockets.helpers import ( @@ -56,8 +55,6 @@ from websockets.asyncio.client import connect from websockets.extensions import permessage_deflate -asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) - URL_MAP = { "stock": "wss://socket.polygon.io/stocks", "stock_delayed": "wss://delayed.polygon.io/stocks", diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index 9f88a6679346..5ad5126c67b9 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -7,7 +7,6 @@ from datetime import UTC, datetime import orjson -import uvloop import websockets from openbb_core.provider.utils.errors import UnauthorizedError from openbb_core.provider.utils.websockets.database import Database, DatabaseWriter @@ -22,8 +21,6 @@ from pandas import to_datetime from pydantic import ValidationError -asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) - URL_MAP = { "stock": "wss://api.tiingo.com/iex", "fx": "wss://api.tiingo.com/fx", From ab1ede4c5a911448da18a34a9c3aabe86044c89a Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Fri, 3 Jan 2025 11:47:21 -0800 Subject: [PATCH 110/119] enough tuning for now --- .../provider/utils/websockets/database.py | 62 ++++---- .../websockets/openbb_websockets/helpers.py | 2 - .../openbb_polygon/utils/websocket_client.py | 142 ++++++++++++------ 3 files changed, 130 insertions(+), 76 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 32441de23822..fe0c9a46640d 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -216,7 +216,6 @@ async def _setup_database(self): pragmas = [ "PRAGMA journal_mode=WAL", "PRAGMA synchronous=off", - "PRAGMA temp_store=memory", ] for pragma in pragmas: @@ -269,7 +268,6 @@ async def get_connection(self, name: str = "read"): pragmas = [ "PRAGMA journal_mode=WAL", "PRAGMA synchronous=off", - "PRAGMA temp_store=memory", ] for pragma in pragmas: await conn.execute(pragma) @@ -768,7 +766,7 @@ async def _export_database(self): async with conn.execute(earliest_query) as cursor: earliest_date = await cursor.fetchone() - if not latest_date: + if not earliest_date: return except asyncio.InvalidStateError as e: @@ -792,13 +790,14 @@ async def _export_database(self): # If we have processed data before, use that as reference if self._last_processed_timestamp: - start_time = self._last_processed_timestamp + start_time = ( + self._last_processed_timestamp + if self._last_processed_timestamp > earliest_time + else earliest_time + ) else: start_time = cutoff_time - timedelta(minutes=minutes) - if start_time < earliest_timestamp: - start_time = earliest_time - # Set end_time to be one interval after start_time end_time = start_time + timedelta(minutes=minutes) @@ -880,10 +879,10 @@ async def _export_database(self): csv_writer.writerows(new_rows) await f.write(buffer.getvalue()) - self._last_processed_timestamp = cutoff_time + self._last_processed_timestamp = end_time if self.verbose is True: - msg = f"DATABASE INFO: Interval for period ending {cutoff_time} saved to: {path}" + msg = f"DATABASE INFO: Interval for period beginning {start_time} and ending {end_time} saved to: {path}" sys.stdout.write(msg + "\n") sys.stdout.flush() @@ -1019,7 +1018,7 @@ async def _start_export_task(self): minutes = self.export_interval or 5 - while self.export_thread is not None: + while self.export_thread is not None and not self._shutdown: # Get the initial row to determine the "first time" try: query = f"SELECT json_extract(message, '$.date') FROM {self.database.table_name} ORDER BY json_extract(message, '$.date') ASC LIMIT 1" # noqa @@ -1034,25 +1033,28 @@ async def _start_export_task(self): if not first_time: await asyncio.sleep(1) self._first_timestamp = first_time.replace(second=0, microsecond=0) - self._last_processed_timestamp = self._first_timestamp + self._last_processed_timestamp = ( + self._first_timestamp + if not self._last_processed_timestamp + else self._last_processed_timestamp + ) - while not self._shutdown: - if self.export_thread is None: - break - # Check if the next interval has been reached and export immediately if available. - next_interval = self._last_processed_timestamp + timedelta( - minutes=minutes - ) - cutoff_timestamp = next_interval.isoformat() - query = f"SELECT COUNT(*) FROM {self.database.table_name} WHERE json_extract(message, '$.date') >= ?" # noqa - count = await self.database._query_db(query, (cutoff_timestamp,)) + # Check if the next interval has been reached and export immediately if available. + next_interval = self._last_processed_timestamp + timedelta( + minutes=minutes + ) + cutoff_timestamp = next_interval.isoformat() + query = f"SELECT COUNT(*) FROM {self.database.table_name} WHERE json_extract(message, '$.date') >= ?" # noqa + count = await self.database._query_db(query, (cutoff_timestamp,)) - if count[0] > 0: - await self._export_database() - else: - await asyncio.sleep(minutes * 60) - # Stagger slightly so things don't happen exactly on the minute. - await asyncio.sleep(3) + if count[0] > 0: + export_task = await asyncio.to_thread(self._export_database) + await export_task + else: + await asyncio.sleep(minutes * 60) + # Stagger slightly so things don't happen exactly on the minute. + await asyncio.sleep(3) + continue except asyncio.CancelledError: break finally: @@ -1138,11 +1140,14 @@ async def _worker(self): batch.append(msg) total += len(msg) else: - await asyncio.sleep(0.01) + break if batch: await self._write_batch(batch) + if not batch: + time.sleep(0.2) + except Exception as e: # pylint: disable=broad-except self.writer.database.logger.error(f"Worker error: {e}") self.running = False @@ -1160,7 +1165,6 @@ async def _write_batch(self, batch): for b in batch: values.extend([(msg,) for msg in b]) async with self.writer.database.get_connection("write") as conn: - await conn.execute("PRAGMA temp_store=memory") async with conn.cursor() as cursor: await cursor.executemany(query, values) await conn.commit() diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 92bba3e38504..0d9a46cac411 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -66,10 +66,8 @@ class StdOutSink: def write(self, message): """Write to stdout.""" # pylint: disable=import-outside-toplevel - import os import sys - os.set_blocking(sys.__stdout__.fileno(), True) # type: ignore cleaned_message = AUTH_TOKEN_FILTER.sub(r"\1********", message) if cleaned_message != message: cleaned_message = f"{cleaned_message}\n" diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index af0924f1d5d8..407174c10230 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -69,7 +69,7 @@ logger = get_logger("openbb.websocket.polygon") process_queue = MessageQueue(logger=logger, backoff_factor=0, max_size=1000000) input_queue = MessageQueue(logger=logger, backoff_factor=0, max_size=1000000) -command_queue = MessageQueue(logger=logger, backoff_factor=0, max_size=1000000) +stdin_queue = MessageQueue(logger=logger) db_queue = MessageQueue(logger=logger, backoff_factor=0, max_size=1000000) kwargs = parse_kwargs() CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) @@ -79,6 +79,11 @@ URL = URL_MAP.get(ASSET_TYPE) SUBSCRIBED_SYMBOLS: set = set() +MESSAGE_COUNT = 0 +AVG_RATE = 0 +LAST_MINUTE_COUNT = 0 +PREVIOUS_MINUTE_COUNT = 0 + if not kwargs.get("api_key"): raise ValueError("No API key provided.") @@ -94,9 +99,30 @@ logger=logger, ), queue=db_queue, + batch_size=1, ) +async def message_rate(): + """Calculate the average message rate.""" + global AVG_RATE # pylint: disable=global-statement # noqa + start = time.time() + while True: + await asyncio.sleep(1) + elapsed = time.time() - start + if elapsed > 0: + AVG_RATE = round(MESSAGE_COUNT / elapsed) + + +async def reset_last_minute_count(): + """Reset the last minute message count every minute.""" + global LAST_MINUTE_COUNT, PREVIOUS_MINUTE_COUNT # pylint: disable=global-statement # noqa + while True: + await asyncio.sleep(60) + PREVIOUS_MINUTE_COUNT = LAST_MINUTE_COUNT + LAST_MINUTE_COUNT = 0 + + async def handle_symbol(symbol): """Handle the symbol and map it to the correct format.""" symbols = symbol.split(",") if isinstance(symbol, str) else symbol @@ -143,10 +169,6 @@ async def handle_symbol(symbol): elif ASSET_TYPE in ["options", "options_delayed"] and ":" not in ticker: _feed, _ticker = ticker.split(".") if "." in ticker else (feed, ticker) ticker = f"{_feed}.O:{_ticker}" - - if ticker == "XL2.*": - symbol_error = f"SymbolError -> {symbol}: L2 Crypto does not support the all-symbols wildcard." - logger.error(symbol_error) else: new_symbols.append(ticker) @@ -217,8 +239,22 @@ async def read_stdin(): sys.stdin.flush() if line: try: - command = line.strip() if "qsize" in line else json.loads(line.strip()) - await command_queue.enqueue(command) + command = line.strip() + if command == "qsize": + logger.info( + f"PROVIDER INFO: Input Queue: {input_queue.queue.qsize()} -" + f" Processing Queue: {process_queue.queue.qsize()}:{db_queue.queue.qsize()} -" + f" Writing Queue: {DATABASE.batch_processor.write_queue.qsize()}" + ) + elif command == "msgrate": + logger.info( + f"PROVIDER INFO: Total Messages: {MESSAGE_COUNT}" + f" - Average Message Rate: {AVG_RATE}" + f" - Messages in Last Minute: {PREVIOUS_MINUTE_COUNT}" + ) + else: + command = json.loads(line.strip()) + await stdin_queue.enqueue(command) except json.JSONDecodeError: logger.error("Invalid JSON received from stdin") @@ -226,18 +262,11 @@ async def read_stdin(): async def process_stdin_queue(websocket): """Process the command queue.""" while True: - command = await command_queue.dequeue() - if command == "qsize": - logger.info( - f"PROVIDER INFO: Input Queue: {input_queue.queue.qsize()} -" - f" Processing Queue: {process_queue.queue.qsize()}:{db_queue.queue.qsize()} -" - f" Writing Queue: {DATABASE.batch_processor.write_queue.qsize()}" - ) - else: - symbol = command.get("symbol") - event = command.get("event") - if symbol and event: - await subscribe(websocket, symbol, event) + command = await stdin_queue.dequeue() + symbol = command.get("symbol") + event = command.get("event") + if symbol and event: + await subscribe(websocket, symbol, event) async def process_message(message): @@ -281,27 +310,39 @@ async def connect_and_stream(): conn_kwargs.update( { "ping_timeout": None, + "ping_interval": 10, "close_timeout": 1, "max_size": 32768, - "max_queue": None, } ) + conn_kwargs["extensions"] = [ + permessage_deflate.ClientPerMessageDeflateFactory( + server_max_window_bits=11, + client_max_window_bits=11, + compress_settings={ + "memLevel": 1, + "level": 1, + }, + ), + ] + async def message_receiver(websocket): """Message receiver.""" while True: - try: - message = await websocket.recv(decode=False) - input_queue.queue.put_nowait(message) - except Exception as e: - raise e from e + message = await websocket.recv(decode=False) + await input_queue.enqueue(message) async def process_input_messages(message): """Process the messages offloaded from the websocket.""" def _process_in_thread(): + global LAST_MINUTE_COUNT, MESSAGE_COUNT # pylint: disable=global-statement # noqa + message_data = orjson.loads(message) if isinstance(message_data, list): + MESSAGE_COUNT += len(message_data) + LAST_MINUTE_COUNT += len(message_data) status_msgs = [ msg for msg in message_data @@ -328,30 +369,24 @@ def _process_in_thread(): tasks.add(process_task) try: - handler_task = asyncio.create_task( - process_queue.process_queue(lambda message: process_message(message)) - ) - tasks.add(handler_task) + for i in range(1, 120): + handler_task = asyncio.create_task( + process_queue.process_queue(lambda message: process_message(message)) + ) + tasks.add(handler_task) stdin_task = asyncio.create_task(read_stdin()) tasks.add(stdin_task) await DATABASE.start_writer() - processor_task = asyncio.create_task( - input_queue.process_queue(lambda message: process_input_messages(message)) - ) - tasks.add(processor_task) - - conn_kwargs["extensions"] = [ - permessage_deflate.ClientPerMessageDeflateFactory( - server_max_window_bits=11, - client_max_window_bits=11, - compress_settings={ - "memLevel": 1, - "level": 1, - }, - ), - ] + for i in range(1, 120): + processor_task = asyncio.create_task( + input_queue.process_queue( + lambda message: process_input_messages(message) + ) + ) + tasks.add(processor_task) + async for websocket in connect( URL, **conn_kwargs, @@ -373,6 +408,11 @@ def _process_in_thread(): await subscribe(websocket, kwargs["symbol"], "subscribe") + rate_task = asyncio.create_task(message_rate()) + tasks.add(rate_task) + minute_task = asyncio.create_task(reset_last_minute_count()) + tasks.add(minute_task) + await message_receiver(websocket) # Attempt to reopen the connection @@ -392,6 +432,18 @@ def _process_in_thread(): sys.exit(0) except websockets.WebSocketException as e: + if "code=1012" in str(e): + await asyncio.sleep(1) + for task in tasks: + task.cancel() + await task + asyncio.gather(*tasks, return_exceptions=True) + await DATABASE.stop_writer() + await asyncio.sleep(1) + asyncio.run_coroutine_threadsafe( + connect_and_stream(), + asyncio.get_event_loop(), + ) msg = f"PROVIDER ERROR: WebSocketException -> {e}" logger.error(msg) sys.exit(1) @@ -415,7 +467,7 @@ def _process_in_thread(): if __name__ == "__main__": try: - loop = uvloop.new_event_loop() + loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) loop.set_exception_handler(lambda loop, context: None) From 593933d0bd4f2675ca873b5ccc7a515525c65f57 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Fri, 3 Jan 2025 21:18:53 -0800 Subject: [PATCH 111/119] move some stuff around --- .../provider/utils/websockets/database.py | 18 +- .../integration/test_websockets_api.py | 28 +- .../integration/test_websockets_python.py | 30 +- .../websockets/openbb_websockets/helpers.py | 5 + .../websockets/openbb_websockets/models.py | 194 --- .../openbb_websockets/websockets_router.py | 2 +- .../extensions/websockets/poetry.lock | 1240 ++++++++--------- .../extensions/websockets/pyproject.toml | 2 +- .../openbb_fmp/models/websocket_connection.py | 8 +- .../fmp/openbb_fmp/utils/websocket_client.py | 130 +- .../models/websocket_connection.py | 5 +- .../openbb_intrinio/utils/websocket_client.py | 6 +- .../models/websocket_connection.py | 11 +- .../openbb_polygon/utils/websocket_client.py | 63 +- .../models/websocket_connection.py | 3 +- .../openbb_tiingo/utils/websocket_client.py | 13 +- 16 files changed, 804 insertions(+), 954 deletions(-) delete mode 100644 openbb_platform/extensions/websockets/openbb_websockets/models.py diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index fe0c9a46640d..36d418f3ab74 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -798,8 +798,10 @@ async def _export_database(self): else: start_time = cutoff_time - timedelta(minutes=minutes) - # Set end_time to be one interval after start_time - end_time = start_time + timedelta(minutes=minutes) + end_time = (start_time + timedelta(minutes=minutes)).replace( + second=0, microsecond=0 + ) + start_time = start_time.replace(second=0, microsecond=0) results_file = ( self.export_directory @@ -882,7 +884,10 @@ async def _export_database(self): self._last_processed_timestamp = end_time if self.verbose is True: - msg = f"DATABASE INFO: Interval for period beginning {start_time} and ending {end_time} saved to: {path}" + msg = ( + "DATABASE INFO: Interval for period beginning" + f" {start_time} and ending {end_time} saved to: {path}" + ) sys.stdout.write(msg + "\n") sys.stdout.flush() @@ -977,8 +982,7 @@ async def _start_prune_task(self): break # Stagger the prune task slightly to avoid things happening exactly on the minute. - await asyncio.sleep(minutes * 60) - await asyncio.sleep(7) + await asyncio.sleep((minutes * 60) + 7) if not self._last_processed_timestamp: last_date = await self.database._query_db( @@ -1097,7 +1101,9 @@ def run(self): self.running = False break except Exception as e: - self.writer.database.logger.error(f"Batch processing error: {e}") + self.writer.database.logger.error( + f"DATABASE ERROR: Batch processing error: {e}" + ) break finally: self._cleanup() diff --git a/openbb_platform/extensions/websockets/integration/test_websockets_api.py b/openbb_platform/extensions/websockets/integration/test_websockets_api.py index 965a7ef6141f..dc0fec403933 100644 --- a/openbb_platform/extensions/websockets/integration/test_websockets_api.py +++ b/openbb_platform/extensions/websockets/integration/test_websockets_api.py @@ -33,7 +33,7 @@ def headers(): "asset_type": "crypto", "auth_token": None, "results_file": None, - "save_results": False, + "save_database": False, "table_name": "records", "limit": 10, "sleep_time": 0.25, @@ -41,6 +41,11 @@ def headers(): "broadcast_port": 6666, "start_broadcast": False, "connect_kwargs": None, + "export_interval": None, + "export_directory": None, + "compress_export": False, + "prune_interval": None, + "verbose": False, } ), ( @@ -52,7 +57,7 @@ def headers(): "feed": "trade_and_quote", "auth_token": None, "results_file": None, - "save_results": False, + "save_database": False, "table_name": "records", "limit": 10, "sleep_time": 0.25, @@ -60,6 +65,11 @@ def headers(): "broadcast_port": 6666, "start_broadcast": False, "connect_kwargs": None, + "export_interval": None, + "export_directory": None, + "compress_export": False, + "prune_interval": None, + "verbose": False, } ), ( @@ -71,7 +81,7 @@ def headers(): "feed": "quote", "auth_token": None, "results_file": None, - "save_results": False, + "save_database": False, "table_name": "records", "limit": 10, "sleep_time": 0.25, @@ -79,6 +89,11 @@ def headers(): "broadcast_port": 6666, "start_broadcast": False, "connect_kwargs": None, + "export_interval": None, + "export_directory": None, + "compress_export": False, + "prune_interval": None, + "verbose": False, } ), ( @@ -91,7 +106,7 @@ def headers(): "trades_only": True, "auth_token": None, "results_file": None, - "save_results": False, + "save_database": False, "table_name": "records", "limit": 10, "sleep_time": 0.25, @@ -99,6 +114,11 @@ def headers(): "broadcast_port": 6666, "start_broadcast": False, "connect_kwargs": None, + "export_interval": None, + "export_directory": None, + "compress_export": False, + "prune_interval": None, + "verbose": False, } ), ], diff --git a/openbb_platform/extensions/websockets/integration/test_websockets_python.py b/openbb_platform/extensions/websockets/integration/test_websockets_python.py index 211f5520229c..e9f9ff84e41c 100644 --- a/openbb_platform/extensions/websockets/integration/test_websockets_python.py +++ b/openbb_platform/extensions/websockets/integration/test_websockets_python.py @@ -5,7 +5,7 @@ import pytest from extensions.tests.conftest import parametrize from openbb_core.app.model.obbject import OBBject -from openbb_websockets.models import WebSocketConnectionStatus +from openbb_core.provider.utils.websockets.models import WebSocketConnectionStatus @pytest.fixture(scope="session") @@ -29,7 +29,7 @@ def obb(pytestconfig): "asset_type": "crypto", "auth_token": None, "results_file": None, - "save_results": False, + "save_database": False, "table_name": "records", "limit": 10, "sleep_time": 0.25, @@ -37,6 +37,11 @@ def obb(pytestconfig): "broadcast_port": 6666, "start_broadcast": False, "connect_kwargs": None, + "export_interval": None, + "export_directory": None, + "compress_export": False, + "prune_interval": None, + "verbose": False, } ), ( @@ -48,7 +53,7 @@ def obb(pytestconfig): "feed": "trade_and_quote", "auth_token": None, "results_file": None, - "save_results": False, + "save_database": False, "table_name": "records", "limit": 10, "sleep_time": 0.25, @@ -56,6 +61,11 @@ def obb(pytestconfig): "broadcast_port": 6666, "start_broadcast": False, "connect_kwargs": None, + "export_interval": None, + "export_directory": None, + "compress_export": False, + "prune_interval": None, + "verbose": False, } ), ( @@ -67,7 +77,7 @@ def obb(pytestconfig): "feed": "quote", "auth_token": None, "results_file": None, - "save_results": False, + "save_database": False, "table_name": "records", "limit": 10, "sleep_time": 0.25, @@ -75,6 +85,11 @@ def obb(pytestconfig): "broadcast_port": 6666, "start_broadcast": False, "connect_kwargs": None, + "export_interval": None, + "export_directory": None, + "compress_export": False, + "prune_interval": None, + "verbose": False, } ), ( @@ -87,7 +102,7 @@ def obb(pytestconfig): "trades_only": True, "auth_token": None, "results_file": None, - "save_results": False, + "save_database": False, "table_name": "records", "limit": 10, "sleep_time": 0.25, @@ -95,6 +110,11 @@ def obb(pytestconfig): "broadcast_port": 6666, "start_broadcast": False, "connect_kwargs": None, + "export_interval": None, + "export_directory": None, + "compress_export": False, + "prune_interval": None, + "verbose": False, } ), ], diff --git a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py index 0d9a46cac411..53bf44c83a2e 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/helpers.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/helpers.py @@ -40,6 +40,11 @@ async def get_status(name: Optional[str] = None, client: Optional[Any] = None) - "results_file": client.results_file, "table_name": client.table_name, "save_database": client.save_database, + "is_pruning": client.is_pruning, + "prune_interval": client.database.writer.prune_interval, + "is_exporting": client.is_exporting, + "export_interval": client.database.writer.export_interval, + "export_directory": client.database.writer.export_directory, } return status diff --git a/openbb_platform/extensions/websockets/openbb_websockets/models.py b/openbb_platform/extensions/websockets/openbb_websockets/models.py deleted file mode 100644 index a5e6a20f1318..000000000000 --- a/openbb_platform/extensions/websockets/openbb_websockets/models.py +++ /dev/null @@ -1,194 +0,0 @@ -"""WebSockets models.""" - -from datetime import datetime -from typing import Any, Optional - -from openbb_core.app.model.abstract.error import OpenBBError -from openbb_core.provider.abstract.data import Data -from openbb_core.provider.abstract.query_params import QueryParams -from openbb_core.provider.utils.descriptions import ( - DATA_DESCRIPTIONS, -) -from openbb_core.provider.utils.websockets.client import WebSocketClient -from pydantic import ConfigDict, Field, field_validator, model_validator - - -class WebSocketQueryParams(QueryParams): - """Query parameters for WebSocket connection creation.""" - - name: str = Field( - description="Name to assign the client connection.", - ) - auth_token: Optional[str] = Field( - default=None, - description="Authentication token for API access control of the client, not related to the provider credentials.", - ) - results_file: Optional[str] = Field( - default=None, - description="Absolute path to the file for continuous writing. By default, a temporary file is created.", - ) - save_database: bool = Field( - default=False, - description="Whether to save the results after the session ends.", - ) - table_name: str = Field( - default="records", - description="Name of the SQL table to write the results to.", - ) - limit: Optional[int] = Field( - default=1000, - description="Maximum number of newest records to keep in the database." - + " If None, all records are kept, which can be memory-intensive.", - ) - prune_interval: Optional[int] = Field( - default=None, - description="Prune all entries older than the given number of minutes." - + " If export interval is set, the prune interval must be at least twice as long.", - ) - export_interval: Optional[int] = Field( - default=None, description="Export all entries as a CSV file every N minutes." - ) - export_directory: Optional[str] = Field( - default=None, - description="Directory to save the exported CSV files to. Defaults to OpenBBUserData/exports/websockets", - ) - sleep_time: float = Field( - default=0.25, - description="Time to sleep between checking for new records in the database from the broadcast server." - + " The default is 0.25 seconds.", - ) - broadcast_host: str = Field( - default="127.0.0.1", - description="IP address to bind the broadcast server to.", - ) - broadcast_port: int = Field( - default=6666, - description="Port to bind the broadcast server to.", - ) - start_broadcast: bool = Field( - default=False, - description="Whether to start the broadcast server." - + " Set to False if system or network conditions do not allow it." - + " Can be started manually with the 'start_broadcasting' method.", - ) - connect_kwargs: Optional[Any] = Field( - default=None, - description="A formatted dictionary, or serialized JSON string, of keyword arguments to pass" - + " directly to websockets.connect().", - ) - verbose: bool = Field( - default=True, - description="Whether to print export and prune messages to the console.", - ) - - @field_validator("connect_kwargs", mode="before", check_fields=False) - @classmethod - def _validate_connect_kwargs(cls, v): - """Validate the connect_kwargs format.""" - # pylint: disable=import-outside-toplevel - import json - - if isinstance(v, str): - try: - v = json.loads(v) - except json.JSONDecodeError as e: - raise OpenBBError( - f"Invalid JSON format for 'connect_kwargs': {e}" - ) from e - if v is not None and not isinstance(v, dict): - raise OpenBBError( - "Invalid 'connect_kwargs' format. Must be a dictionary or serialized JSON string." - ) - - return json.dumps(v, separators=(",", ":")) - - -class WebSocketConnectionStatus(Data): - """Data model for WebSocketConnection status information.""" - - name: str = Field( - description="Name assigned to the client connection.", - ) - auth_required: bool = Field( - description="True when 'auth_token' is supplied at initialization." - " When True, interactions with the client from the Python or API" - + " endpoints requires it to be supplied as a query parameter.", - ) - subscribed_symbols: str = Field( - description="Symbols subscribed to by the client connection.", - ) - is_running: bool = Field( - description="Whether the client connection is running.", - ) - provider_pid: Optional[int] = Field( - default=None, - description="Process ID of the provider connection.", - ) - is_broadcasting: bool = Field( - description="Whether the client connection is broadcasting.", - ) - broadcast_address: Optional[str] = Field( - default=None, - description="URI to the broadcast server.", - ) - broadcast_pid: Optional[int] = Field( - default=None, - description="Process ID of the broadcast server.", - ) - results_file: Optional[str] = Field( - description="Absolute path to the file for continuous writing.", - ) - table_name: Optional[str] = Field( - default=None, - description="Name of the SQL table to write the results to.", - ) - save_database: bool = Field( - description="Whether to save the results after the session ends.", - ) - - -class WebSocketData(Data): - """WebSocket data model.""" - - date: datetime = Field( - description=DATA_DESCRIPTIONS.get("date", ""), - ) - symbol: str = Field( - description=DATA_DESCRIPTIONS.get("symbol", ""), - ) - - -class WebSocketConnection(Data): - """Data model for returning WebSocketClient from the Provider Interface.""" - - model_config = ConfigDict( - extra="forbid", - ) - - client: Optional[Any] = Field( - default=None, - description="Instance of WebSocketClient class initialized by a provider Fetcher." - + " The client is used to communicate with the provider's data stream." - + " It is not returned to the user, but is handled by the router for API access.", - exclude=True, - ) - status: Optional[WebSocketConnectionStatus] = Field( - default=None, - description="Status information for the WebSocket connection.", - ) - - @field_validator("client", mode="before", check_fields=False) - @classmethod - def _validate_client(cls, v): - """Validate the client.""" - if v and not isinstance(v, WebSocketClient): - raise ValueError("Client must be an instance of WebSocketClient.") - return v - - @model_validator(mode="before") - @classmethod - def _validate_inputs(cls, values): - """Validate the status.""" - if not values.get("status") and not values.get("client"): - raise ValueError("Cannot initialize empty.") - return values diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index 3c18e5395f7e..dd7672cdbe7a 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -18,6 +18,7 @@ from openbb_core.app.query import Query from openbb_core.app.router import Router from openbb_core.provider.utils.errors import EmptyDataError, UnauthorizedError +from openbb_core.provider.utils.websockets.models import WebSocketConnectionStatus from pydantic import ValidationError from openbb_websockets.helpers import ( @@ -26,7 +27,6 @@ connected_clients, get_status, ) -from openbb_websockets.models import WebSocketConnectionStatus router = Router("", description="WebSockets Router") sys.stdout = StdOutSink() diff --git a/openbb_platform/extensions/websockets/poetry.lock b/openbb_platform/extensions/websockets/poetry.lock index 88faa6559e70..6809c395f882 100644 --- a/openbb_platform/extensions/websockets/poetry.lock +++ b/openbb_platform/extensions/websockets/poetry.lock @@ -1,142 +1,146 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" -version = "2.4.3" +version = "2.4.4" description = "Happy Eyeballs for asyncio" optional = false python-versions = ">=3.8" files = [ - {file = "aiohappyeyeballs-2.4.3-py3-none-any.whl", hash = "sha256:8a7a83727b2756f394ab2895ea0765a0a8c475e3c71e98d43d76f22b4b435572"}, - {file = "aiohappyeyeballs-2.4.3.tar.gz", hash = "sha256:75cf88a15106a5002a8eb1dab212525c00d1f4c0fa96e551c9fbe6f09a621586"}, + {file = "aiohappyeyeballs-2.4.4-py3-none-any.whl", hash = "sha256:a980909d50efcd44795c4afeca523296716d50cd756ddca6af8c65b996e27de8"}, + {file = "aiohappyeyeballs-2.4.4.tar.gz", hash = "sha256:5fdd7d87889c63183afc18ce9271f9b0a7d32c2303e394468dd45d514a757745"}, ] [[package]] name = "aiohttp" -version = "3.10.10" +version = "3.11.11" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "aiohttp-3.10.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:be7443669ae9c016b71f402e43208e13ddf00912f47f623ee5994e12fc7d4b3f"}, - {file = "aiohttp-3.10.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7b06b7843929e41a94ea09eb1ce3927865387e3e23ebe108e0d0d09b08d25be9"}, - {file = "aiohttp-3.10.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:333cf6cf8e65f6a1e06e9eb3e643a0c515bb850d470902274239fea02033e9a8"}, - {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:274cfa632350225ce3fdeb318c23b4a10ec25c0e2c880eff951a3842cf358ac1"}, - {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9e5e4a85bdb56d224f412d9c98ae4cbd032cc4f3161818f692cd81766eee65a"}, - {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b606353da03edcc71130b52388d25f9a30a126e04caef1fd637e31683033abd"}, - {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab5a5a0c7a7991d90446a198689c0535be89bbd6b410a1f9a66688f0880ec026"}, - {file = "aiohttp-3.10.10-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:578a4b875af3e0daaf1ac6fa983d93e0bbfec3ead753b6d6f33d467100cdc67b"}, - {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8105fd8a890df77b76dd3054cddf01a879fc13e8af576805d667e0fa0224c35d"}, - {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3bcd391d083f636c06a68715e69467963d1f9600f85ef556ea82e9ef25f043f7"}, - {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fbc6264158392bad9df19537e872d476f7c57adf718944cc1e4495cbabf38e2a"}, - {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e48d5021a84d341bcaf95c8460b152cfbad770d28e5fe14a768988c461b821bc"}, - {file = "aiohttp-3.10.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2609e9ab08474702cc67b7702dbb8a80e392c54613ebe80db7e8dbdb79837c68"}, - {file = "aiohttp-3.10.10-cp310-cp310-win32.whl", hash = "sha256:84afcdea18eda514c25bc68b9af2a2b1adea7c08899175a51fe7c4fb6d551257"}, - {file = "aiohttp-3.10.10-cp310-cp310-win_amd64.whl", hash = "sha256:9c72109213eb9d3874f7ac8c0c5fa90e072d678e117d9061c06e30c85b4cf0e6"}, - {file = "aiohttp-3.10.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c30a0eafc89d28e7f959281b58198a9fa5e99405f716c0289b7892ca345fe45f"}, - {file = "aiohttp-3.10.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:258c5dd01afc10015866114e210fb7365f0d02d9d059c3c3415382ab633fcbcb"}, - {file = "aiohttp-3.10.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:15ecd889a709b0080f02721255b3f80bb261c2293d3c748151274dfea93ac871"}, - {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3935f82f6f4a3820270842e90456ebad3af15810cf65932bd24da4463bc0a4c"}, - {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:413251f6fcf552a33c981c4709a6bba37b12710982fec8e558ae944bfb2abd38"}, - {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1720b4f14c78a3089562b8875b53e36b51c97c51adc53325a69b79b4b48ebcb"}, - {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:679abe5d3858b33c2cf74faec299fda60ea9de62916e8b67e625d65bf069a3b7"}, - {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:79019094f87c9fb44f8d769e41dbb664d6e8fcfd62f665ccce36762deaa0e911"}, - {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe2fb38c2ed905a2582948e2de560675e9dfbee94c6d5ccdb1301c6d0a5bf092"}, - {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a3f00003de6eba42d6e94fabb4125600d6e484846dbf90ea8e48a800430cc142"}, - {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1bbb122c557a16fafc10354b9d99ebf2f2808a660d78202f10ba9d50786384b9"}, - {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:30ca7c3b94708a9d7ae76ff281b2f47d8eaf2579cd05971b5dc681db8caac6e1"}, - {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:df9270660711670e68803107d55c2b5949c2e0f2e4896da176e1ecfc068b974a"}, - {file = "aiohttp-3.10.10-cp311-cp311-win32.whl", hash = "sha256:aafc8ee9b742ce75044ae9a4d3e60e3d918d15a4c2e08a6c3c3e38fa59b92d94"}, - {file = "aiohttp-3.10.10-cp311-cp311-win_amd64.whl", hash = "sha256:362f641f9071e5f3ee6f8e7d37d5ed0d95aae656adf4ef578313ee585b585959"}, - {file = "aiohttp-3.10.10-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9294bbb581f92770e6ed5c19559e1e99255e4ca604a22c5c6397b2f9dd3ee42c"}, - {file = "aiohttp-3.10.10-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a8fa23fe62c436ccf23ff930149c047f060c7126eae3ccea005f0483f27b2e28"}, - {file = "aiohttp-3.10.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5c6a5b8c7926ba5d8545c7dd22961a107526562da31a7a32fa2456baf040939f"}, - {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:007ec22fbc573e5eb2fb7dec4198ef8f6bf2fe4ce20020798b2eb5d0abda6138"}, - {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9627cc1a10c8c409b5822a92d57a77f383b554463d1884008e051c32ab1b3742"}, - {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:50edbcad60d8f0e3eccc68da67f37268b5144ecc34d59f27a02f9611c1d4eec7"}, - {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a45d85cf20b5e0d0aa5a8dca27cce8eddef3292bc29d72dcad1641f4ed50aa16"}, - {file = "aiohttp-3.10.10-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b00807e2605f16e1e198f33a53ce3c4523114059b0c09c337209ae55e3823a8"}, - {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f2d4324a98062be0525d16f768a03e0bbb3b9fe301ceee99611dc9a7953124e6"}, - {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:438cd072f75bb6612f2aca29f8bd7cdf6e35e8f160bc312e49fbecab77c99e3a"}, - {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:baa42524a82f75303f714108fea528ccacf0386af429b69fff141ffef1c534f9"}, - {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a7d8d14fe962153fc681f6366bdec33d4356f98a3e3567782aac1b6e0e40109a"}, - {file = "aiohttp-3.10.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c1277cd707c465cd09572a774559a3cc7c7a28802eb3a2a9472588f062097205"}, - {file = "aiohttp-3.10.10-cp312-cp312-win32.whl", hash = "sha256:59bb3c54aa420521dc4ce3cc2c3fe2ad82adf7b09403fa1f48ae45c0cbde6628"}, - {file = "aiohttp-3.10.10-cp312-cp312-win_amd64.whl", hash = "sha256:0e1b370d8007c4ae31ee6db7f9a2fe801a42b146cec80a86766e7ad5c4a259cf"}, - {file = "aiohttp-3.10.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ad7593bb24b2ab09e65e8a1d385606f0f47c65b5a2ae6c551db67d6653e78c28"}, - {file = "aiohttp-3.10.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1eb89d3d29adaf533588f209768a9c02e44e4baf832b08118749c5fad191781d"}, - {file = "aiohttp-3.10.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3fe407bf93533a6fa82dece0e74dbcaaf5d684e5a51862887f9eaebe6372cd79"}, - {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50aed5155f819873d23520919e16703fc8925e509abbb1a1491b0087d1cd969e"}, - {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4f05e9727ce409358baa615dbeb9b969db94324a79b5a5cea45d39bdb01d82e6"}, - {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dffb610a30d643983aeb185ce134f97f290f8935f0abccdd32c77bed9388b42"}, - {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa6658732517ddabe22c9036479eabce6036655ba87a0224c612e1ae6af2087e"}, - {file = "aiohttp-3.10.10-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:741a46d58677d8c733175d7e5aa618d277cd9d880301a380fd296975a9cdd7bc"}, - {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e00e3505cd80440f6c98c6d69269dcc2a119f86ad0a9fd70bccc59504bebd68a"}, - {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ffe595f10566f8276b76dc3a11ae4bb7eba1aac8ddd75811736a15b0d5311414"}, - {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:bdfcf6443637c148c4e1a20c48c566aa694fa5e288d34b20fcdc58507882fed3"}, - {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d183cf9c797a5291e8301790ed6d053480ed94070637bfaad914dd38b0981f67"}, - {file = "aiohttp-3.10.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:77abf6665ae54000b98b3c742bc6ea1d1fb31c394bcabf8b5d2c1ac3ebfe7f3b"}, - {file = "aiohttp-3.10.10-cp313-cp313-win32.whl", hash = "sha256:4470c73c12cd9109db8277287d11f9dd98f77fc54155fc71a7738a83ffcc8ea8"}, - {file = "aiohttp-3.10.10-cp313-cp313-win_amd64.whl", hash = "sha256:486f7aabfa292719a2753c016cc3a8f8172965cabb3ea2e7f7436c7f5a22a151"}, - {file = "aiohttp-3.10.10-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:1b66ccafef7336a1e1f0e389901f60c1d920102315a56df85e49552308fc0486"}, - {file = "aiohttp-3.10.10-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:acd48d5b80ee80f9432a165c0ac8cbf9253eaddb6113269a5e18699b33958dbb"}, - {file = "aiohttp-3.10.10-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3455522392fb15ff549d92fbf4b73b559d5e43dc522588f7eb3e54c3f38beee7"}, - {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45c3b868724137f713a38376fef8120c166d1eadd50da1855c112fe97954aed8"}, - {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:da1dee8948d2137bb51fbb8a53cce6b1bcc86003c6b42565f008438b806cccd8"}, - {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c5ce2ce7c997e1971b7184ee37deb6ea9922ef5163c6ee5aa3c274b05f9e12fa"}, - {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28529e08fde6f12eba8677f5a8608500ed33c086f974de68cc65ab218713a59d"}, - {file = "aiohttp-3.10.10-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f7db54c7914cc99d901d93a34704833568d86c20925b2762f9fa779f9cd2e70f"}, - {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:03a42ac7895406220124c88911ebee31ba8b2d24c98507f4a8bf826b2937c7f2"}, - {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:7e338c0523d024fad378b376a79faff37fafb3c001872a618cde1d322400a572"}, - {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:038f514fe39e235e9fef6717fbf944057bfa24f9b3db9ee551a7ecf584b5b480"}, - {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:64f6c17757251e2b8d885d728b6433d9d970573586a78b78ba8929b0f41d045a"}, - {file = "aiohttp-3.10.10-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:93429602396f3383a797a2a70e5f1de5df8e35535d7806c9f91df06f297e109b"}, - {file = "aiohttp-3.10.10-cp38-cp38-win32.whl", hash = "sha256:c823bc3971c44ab93e611ab1a46b1eafeae474c0c844aff4b7474287b75fe49c"}, - {file = "aiohttp-3.10.10-cp38-cp38-win_amd64.whl", hash = "sha256:54ca74df1be3c7ca1cf7f4c971c79c2daf48d9aa65dea1a662ae18926f5bc8ce"}, - {file = "aiohttp-3.10.10-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:01948b1d570f83ee7bbf5a60ea2375a89dfb09fd419170e7f5af029510033d24"}, - {file = "aiohttp-3.10.10-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9fc1500fd2a952c5c8e3b29aaf7e3cc6e27e9cfc0a8819b3bce48cc1b849e4cc"}, - {file = "aiohttp-3.10.10-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f614ab0c76397661b90b6851a030004dac502e48260ea10f2441abd2207fbcc7"}, - {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00819de9e45d42584bed046314c40ea7e9aea95411b38971082cad449392b08c"}, - {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05646ebe6b94cc93407b3bf34b9eb26c20722384d068eb7339de802154d61bc5"}, - {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:998f3bd3cfc95e9424a6acd7840cbdd39e45bc09ef87533c006f94ac47296090"}, - {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9010c31cd6fa59438da4e58a7f19e4753f7f264300cd152e7f90d4602449762"}, - {file = "aiohttp-3.10.10-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ea7ffc6d6d6f8a11e6f40091a1040995cdff02cfc9ba4c2f30a516cb2633554"}, - {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ef9c33cc5cbca35808f6c74be11eb7f5f6b14d2311be84a15b594bd3e58b5527"}, - {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ce0cdc074d540265bfeb31336e678b4e37316849d13b308607efa527e981f5c2"}, - {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:597a079284b7ee65ee102bc3a6ea226a37d2b96d0418cc9047490f231dc09fe8"}, - {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:7789050d9e5d0c309c706953e5e8876e38662d57d45f936902e176d19f1c58ab"}, - {file = "aiohttp-3.10.10-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:e7f8b04d83483577fd9200461b057c9f14ced334dcb053090cea1da9c8321a91"}, - {file = "aiohttp-3.10.10-cp39-cp39-win32.whl", hash = "sha256:c02a30b904282777d872266b87b20ed8cc0d1501855e27f831320f471d54d983"}, - {file = "aiohttp-3.10.10-cp39-cp39-win_amd64.whl", hash = "sha256:edfe3341033a6b53a5c522c802deb2079eee5cbfbb0af032a55064bd65c73a23"}, - {file = "aiohttp-3.10.10.tar.gz", hash = "sha256:0631dd7c9f0822cc61c88586ca76d5b5ada26538097d0f1df510b082bad3411a"}, + {file = "aiohttp-3.11.11-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a60804bff28662cbcf340a4d61598891f12eea3a66af48ecfdc975ceec21e3c8"}, + {file = "aiohttp-3.11.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4b4fa1cb5f270fb3eab079536b764ad740bb749ce69a94d4ec30ceee1b5940d5"}, + {file = "aiohttp-3.11.11-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:731468f555656767cda219ab42e033355fe48c85fbe3ba83a349631541715ba2"}, + {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb23d8bb86282b342481cad4370ea0853a39e4a32a0042bb52ca6bdde132df43"}, + {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f047569d655f81cb70ea5be942ee5d4421b6219c3f05d131f64088c73bb0917f"}, + {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd7659baae9ccf94ae5fe8bfaa2c7bc2e94d24611528395ce88d009107e00c6d"}, + {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af01e42ad87ae24932138f154105e88da13ce7d202a6de93fafdafb2883a00ef"}, + {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5854be2f3e5a729800bac57a8d76af464e160f19676ab6aea74bde18ad19d438"}, + {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6526e5fb4e14f4bbf30411216780c9967c20c5a55f2f51d3abd6de68320cc2f3"}, + {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:85992ee30a31835fc482468637b3e5bd085fa8fe9392ba0bdcbdc1ef5e9e3c55"}, + {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:88a12ad8ccf325a8a5ed80e6d7c3bdc247d66175afedbe104ee2aaca72960d8e"}, + {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0a6d3fbf2232e3a08c41eca81ae4f1dff3d8f1a30bae415ebe0af2d2458b8a33"}, + {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:84a585799c58b795573c7fa9b84c455adf3e1d72f19a2bf498b54a95ae0d194c"}, + {file = "aiohttp-3.11.11-cp310-cp310-win32.whl", hash = "sha256:bfde76a8f430cf5c5584553adf9926534352251d379dcb266ad2b93c54a29745"}, + {file = "aiohttp-3.11.11-cp310-cp310-win_amd64.whl", hash = "sha256:0fd82b8e9c383af11d2b26f27a478640b6b83d669440c0a71481f7c865a51da9"}, + {file = "aiohttp-3.11.11-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ba74ec819177af1ef7f59063c6d35a214a8fde6f987f7661f4f0eecc468a8f76"}, + {file = "aiohttp-3.11.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4af57160800b7a815f3fe0eba9b46bf28aafc195555f1824555fa2cfab6c1538"}, + {file = "aiohttp-3.11.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ffa336210cf9cd8ed117011085817d00abe4c08f99968deef0013ea283547204"}, + {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81b8fe282183e4a3c7a1b72f5ade1094ed1c6345a8f153506d114af5bf8accd9"}, + {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3af41686ccec6a0f2bdc66686dc0f403c41ac2089f80e2214a0f82d001052c03"}, + {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70d1f9dde0e5dd9e292a6d4d00058737052b01f3532f69c0c65818dac26dc287"}, + {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:249cc6912405917344192b9f9ea5cd5b139d49e0d2f5c7f70bdfaf6b4dbf3a2e"}, + {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0eb98d90b6690827dcc84c246811feeb4e1eea683c0eac6caed7549be9c84665"}, + {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ec82bf1fda6cecce7f7b915f9196601a1bd1a3079796b76d16ae4cce6d0ef89b"}, + {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:9fd46ce0845cfe28f108888b3ab17abff84ff695e01e73657eec3f96d72eef34"}, + {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:bd176afcf8f5d2aed50c3647d4925d0db0579d96f75a31e77cbaf67d8a87742d"}, + {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ec2aa89305006fba9ffb98970db6c8221541be7bee4c1d027421d6f6df7d1ce2"}, + {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:92cde43018a2e17d48bb09c79e4d4cb0e236de5063ce897a5e40ac7cb4878773"}, + {file = "aiohttp-3.11.11-cp311-cp311-win32.whl", hash = "sha256:aba807f9569455cba566882c8938f1a549f205ee43c27b126e5450dc9f83cc62"}, + {file = "aiohttp-3.11.11-cp311-cp311-win_amd64.whl", hash = "sha256:ae545f31489548c87b0cced5755cfe5a5308d00407000e72c4fa30b19c3220ac"}, + {file = "aiohttp-3.11.11-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e595c591a48bbc295ebf47cb91aebf9bd32f3ff76749ecf282ea7f9f6bb73886"}, + {file = "aiohttp-3.11.11-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3ea1b59dc06396b0b424740a10a0a63974c725b1c64736ff788a3689d36c02d2"}, + {file = "aiohttp-3.11.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8811f3f098a78ffa16e0ea36dffd577eb031aea797cbdba81be039a4169e242c"}, + {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7227b87a355ce1f4bf83bfae4399b1f5bb42e0259cb9405824bd03d2f4336a"}, + {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d40f9da8cabbf295d3a9dae1295c69975b86d941bc20f0a087f0477fa0a66231"}, + {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ffb3dc385f6bb1568aa974fe65da84723210e5d9707e360e9ecb51f59406cd2e"}, + {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8f5f7515f3552d899c61202d99dcb17d6e3b0de777900405611cd747cecd1b8"}, + {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3499c7ffbfd9c6a3d8d6a2b01c26639da7e43d47c7b4f788016226b1e711caa8"}, + {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8e2bf8029dbf0810c7bfbc3e594b51c4cc9101fbffb583a3923aea184724203c"}, + {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b6212a60e5c482ef90f2d788835387070a88d52cf6241d3916733c9176d39eab"}, + {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:d119fafe7b634dbfa25a8c597718e69a930e4847f0b88e172744be24515140da"}, + {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:6fba278063559acc730abf49845d0e9a9e1ba74f85f0ee6efd5803f08b285853"}, + {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:92fc484e34b733704ad77210c7957679c5c3877bd1e6b6d74b185e9320cc716e"}, + {file = "aiohttp-3.11.11-cp312-cp312-win32.whl", hash = "sha256:9f5b3c1ed63c8fa937a920b6c1bec78b74ee09593b3f5b979ab2ae5ef60d7600"}, + {file = "aiohttp-3.11.11-cp312-cp312-win_amd64.whl", hash = "sha256:1e69966ea6ef0c14ee53ef7a3d68b564cc408121ea56c0caa2dc918c1b2f553d"}, + {file = "aiohttp-3.11.11-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:541d823548ab69d13d23730a06f97460f4238ad2e5ed966aaf850d7c369782d9"}, + {file = "aiohttp-3.11.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:929f3ed33743a49ab127c58c3e0a827de0664bfcda566108989a14068f820194"}, + {file = "aiohttp-3.11.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0882c2820fd0132240edbb4a51eb8ceb6eef8181db9ad5291ab3332e0d71df5f"}, + {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b63de12e44935d5aca7ed7ed98a255a11e5cb47f83a9fded7a5e41c40277d104"}, + {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa54f8ef31d23c506910c21163f22b124facb573bff73930735cf9fe38bf7dff"}, + {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a344d5dc18074e3872777b62f5f7d584ae4344cd6006c17ba12103759d407af3"}, + {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b7fb429ab1aafa1f48578eb315ca45bd46e9c37de11fe45c7f5f4138091e2f1"}, + {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c341c7d868750e31961d6d8e60ff040fb9d3d3a46d77fd85e1ab8e76c3e9a5c4"}, + {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ed9ee95614a71e87f1a70bc81603f6c6760128b140bc4030abe6abaa988f1c3d"}, + {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:de8d38f1c2810fa2a4f1d995a2e9c70bb8737b18da04ac2afbf3971f65781d87"}, + {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a9b7371665d4f00deb8f32208c7c5e652059b0fda41cf6dbcac6114a041f1cc2"}, + {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:620598717fce1b3bd14dd09947ea53e1ad510317c85dda2c9c65b622edc96b12"}, + {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bf8d9bfee991d8acc72d060d53860f356e07a50f0e0d09a8dfedea1c554dd0d5"}, + {file = "aiohttp-3.11.11-cp313-cp313-win32.whl", hash = "sha256:9d73ee3725b7a737ad86c2eac5c57a4a97793d9f442599bea5ec67ac9f4bdc3d"}, + {file = "aiohttp-3.11.11-cp313-cp313-win_amd64.whl", hash = "sha256:c7a06301c2fb096bdb0bd25fe2011531c1453b9f2c163c8031600ec73af1cc99"}, + {file = "aiohttp-3.11.11-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3e23419d832d969f659c208557de4a123e30a10d26e1e14b73431d3c13444c2e"}, + {file = "aiohttp-3.11.11-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:21fef42317cf02e05d3b09c028712e1d73a9606f02467fd803f7c1f39cc59add"}, + {file = "aiohttp-3.11.11-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1f21bb8d0235fc10c09ce1d11ffbd40fc50d3f08a89e4cf3a0c503dc2562247a"}, + {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1642eceeaa5ab6c9b6dfeaaa626ae314d808188ab23ae196a34c9d97efb68350"}, + {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2170816e34e10f2fd120f603e951630f8a112e1be3b60963a1f159f5699059a6"}, + {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8be8508d110d93061197fd2d6a74f7401f73b6d12f8822bbcd6d74f2b55d71b1"}, + {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4eed954b161e6b9b65f6be446ed448ed3921763cc432053ceb606f89d793927e"}, + {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6c9af134da4bc9b3bd3e6a70072509f295d10ee60c697826225b60b9959acdd"}, + {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:44167fc6a763d534a6908bdb2592269b4bf30a03239bcb1654781adf5e49caf1"}, + {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:479b8c6ebd12aedfe64563b85920525d05d394b85f166b7873c8bde6da612f9c"}, + {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:10b4ff0ad793d98605958089fabfa350e8e62bd5d40aa65cdc69d6785859f94e"}, + {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:b540bd67cfb54e6f0865ceccd9979687210d7ed1a1cc8c01f8e67e2f1e883d28"}, + {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1dac54e8ce2ed83b1f6b1a54005c87dfed139cf3f777fdc8afc76e7841101226"}, + {file = "aiohttp-3.11.11-cp39-cp39-win32.whl", hash = "sha256:568c1236b2fde93b7720f95a890741854c1200fba4a3471ff48b2934d2d93fd3"}, + {file = "aiohttp-3.11.11-cp39-cp39-win_amd64.whl", hash = "sha256:943a8b052e54dfd6439fd7989f67fc6a7f2138d0a2cf0a7de5f18aa4fe7eb3b1"}, + {file = "aiohttp-3.11.11.tar.gz", hash = "sha256:bb49c7f1e6ebf3821a42d81d494f538107610c3a705987f53068546b0e90303e"}, ] [package.dependencies] aiohappyeyeballs = ">=2.3.0" aiosignal = ">=1.1.2" -async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} +async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" -yarl = ">=1.12.0,<2.0" +propcache = ">=0.2.0" +yarl = ">=1.17.0,<2.0" [package.extras] speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"] [[package]] name = "aiosignal" -version = "1.3.1" +version = "1.3.2" description = "aiosignal: a list of registered asynchronous callbacks" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" files = [ - {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, - {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, + {file = "aiosignal-1.3.2-py2.py3-none-any.whl", hash = "sha256:45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5"}, + {file = "aiosignal-1.3.2.tar.gz", hash = "sha256:a8c255c66fafb1e499c9351d0bf32ff2d8a0321595ebac3b93713656d2436f54"}, ] [package.dependencies] frozenlist = ">=1.1.0" +[[package]] +name = "aiosqlite" +version = "0.20.0" +description = "asyncio bridge to the standard sqlite3 module" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiosqlite-0.20.0-py3-none-any.whl", hash = "sha256:36a1deaca0cac40ebe32aac9977a6e2bbc7f5189f23f4a54d5908986729e5bd6"}, + {file = "aiosqlite-0.20.0.tar.gz", hash = "sha256:6d35c8c256637f4672f843c31021464090805bf925385ac39473fb16eaaca3d7"}, +] + +[package.dependencies] +typing_extensions = ">=4.0" + +[package.extras] +dev = ["attribution (==1.7.0)", "black (==24.2.0)", "coverage[toml] (==7.4.1)", "flake8 (==7.0.0)", "flake8-bugbear (==24.2.6)", "flit (==3.9.0)", "mypy (==1.8.0)", "ufmt (==2.3.0)", "usort (==1.0.8.post1)"] +docs = ["sphinx (==7.2.6)", "sphinx-mdinclude (==0.5.3)"] + [[package]] name = "annotated-types" version = "0.7.0" @@ -150,52 +154,52 @@ files = [ [[package]] name = "anyio" -version = "4.6.2.post1" +version = "4.7.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false python-versions = ">=3.9" files = [ - {file = "anyio-4.6.2.post1-py3-none-any.whl", hash = "sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d"}, - {file = "anyio-4.6.2.post1.tar.gz", hash = "sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c"}, + {file = "anyio-4.7.0-py3-none-any.whl", hash = "sha256:ea60c3723ab42ba6fff7e8ccb0488c898ec538ff4df1f1d5e642c3601d07e352"}, + {file = "anyio-4.7.0.tar.gz", hash = "sha256:2f834749c602966b7d456a7567cafcb309f96482b5081d14ac93ccd457f9dd48"}, ] [package.dependencies] exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} idna = ">=2.8" sniffio = ">=1.1" -typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} +typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} [package.extras] -doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] -test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21.0b1)"] +doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"] trio = ["trio (>=0.26.1)"] [[package]] name = "async-timeout" -version = "4.0.3" +version = "5.0.1" description = "Timeout context manager for asyncio programs" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, - {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, + {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, + {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, ] [[package]] name = "attrs" -version = "24.2.0" +version = "24.3.0" description = "Classes Without Boilerplate" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, - {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, + {file = "attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308"}, + {file = "attrs-24.3.0.tar.gz", hash = "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff"}, ] [package.extras] benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] @@ -213,138 +217,125 @@ files = [ [[package]] name = "certifi" -version = "2024.8.30" +version = "2024.12.14" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, - {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, + {file = "certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56"}, + {file = "certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"}, ] [[package]] name = "charset-normalizer" -version = "3.4.0" +version = "3.4.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.7" files = [ - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-win32.whl", hash = "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-win32.whl", hash = "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-win32.whl", hash = "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-win32.whl", hash = "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-win32.whl", hash = "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca"}, - {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, - {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e218488cd232553829be0664c2292d3af2eeeb94b32bea483cf79ac6a694e037"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80ed5e856eb7f30115aaf94e4a08114ccc8813e6ed1b5efa74f9f82e8509858f"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b010a7a4fd316c3c484d482922d13044979e78d1861f0e0650423144c616a46a"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4532bff1b8421fd0a320463030c7520f56a79c9024a4e88f01c537316019005a"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d973f03c0cb71c5ed99037b870f2be986c3c05e63622c017ea9816881d2dd247"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3a3bd0dcd373514dcec91c411ddb9632c0d7d92aed7093b8c3bbb6d69ca74408"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d9c3cdf5390dcd29aa8056d13e8e99526cda0305acc038b96b30352aff5ff2bb"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2bdfe3ac2e1bbe5b59a1a63721eb3b95fc9b6817ae4a46debbb4e11f6232428d"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:eab677309cdb30d047996b36d34caeda1dc91149e4fdca0b1a039b3f79d9a807"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-win32.whl", hash = "sha256:c0429126cf75e16c4f0ad00ee0eae4242dc652290f940152ca8c75c3a4b6ee8f"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:9f0b8b1c6d84c8034a44893aba5e767bf9c7a211e313a9605d9c617d7083829f"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-win32.whl", hash = "sha256:8417cb1f36cc0bc7eaba8ccb0e04d55f0ee52df06df3ad55259b9a323555fc8b"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:d7f50a1f8c450f3925cb367d011448c39239bb3eb4117c36a6d354794de4ce76"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-win32.whl", hash = "sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-win32.whl", hash = "sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f30bf9fd9be89ecb2360c7d94a711f00c09b976258846efe40db3d05828e8089"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:97f68b8d6831127e4787ad15e6757232e14e12060bec17091b85eb1486b91d8d"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7974a0b5ecd505609e3b19742b60cee7aa2aa2fb3151bc917e6e2646d7667dcf"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc54db6c8593ef7d4b2a331b58653356cf04f67c960f584edb7c3d8c97e8f39e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:311f30128d7d333eebd7896965bfcfbd0065f1716ec92bd5638d7748eb6f936a"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:7d053096f67cd1241601111b698f5cad775f97ab25d81567d3f59219b5f1adbd"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:807f52c1f798eef6cf26beb819eeb8819b1622ddfeef9d0977a8502d4db6d534"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:dccbe65bd2f7f7ec22c4ff99ed56faa1e9f785482b9bbd7c717e26fd723a1d1e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:2fb9bd477fdea8684f78791a6de97a953c51831ee2981f8e4f583ff3b9d9687e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:01732659ba9b5b873fc117534143e4feefecf3b2078b0a6a2e925271bb6f4cfa"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-win32.whl", hash = "sha256:7a4f97a081603d2050bfaffdefa5b02a9ec823f8348a572e39032caa8404a487"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7b1bef6280950ee6c177b326508f86cad7ad4dff12454483b51d8b7d673a2c5d"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ecddf25bee22fe4fe3737a399d0d177d72bc22be6913acfab364b40bce1ba83c"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c60ca7339acd497a55b0ea5d506b2a2612afb2826560416f6894e8b5770d4a9"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b7b2d86dd06bfc2ade3312a83a5c364c7ec2e3498f8734282c6c3d4b07b346b8"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd78cfcda14a1ef52584dbb008f7ac81c1328c0f58184bf9a84c49c605002da6"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e27f48bcd0957c6d4cb9d6fa6b61d192d0b13d5ef563e5f2ae35feafc0d179c"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01ad647cdd609225c5350561d084b42ddf732f4eeefe6e678765636791e78b9a"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:619a609aa74ae43d90ed2e89bdd784765de0a25ca761b93e196d938b8fd1dbbd"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:89149166622f4db9b4b6a449256291dc87a99ee53151c74cbd82a53c8c2f6ccd"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:7709f51f5f7c853f0fb938bcd3bc59cdfdc5203635ffd18bf354f6967ea0f824"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:345b0426edd4e18138d6528aed636de7a9ed169b4aaf9d61a8c19e39d26838ca"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0907f11d019260cdc3f94fbdb23ff9125f6b5d1039b76003b5b0ac9d6a6c9d5b"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-win32.whl", hash = "sha256:ea0d8d539afa5eb2728aa1932a988a9a7af94f18582ffae4bc10b3fbdad0626e"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:329ce159e82018d646c7ac45b01a430369d526569ec08516081727a20e9e4af4"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75832c08354f595c760a804588b9357d34ec00ba1c940c15e31e96d902093770"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0af291f4fe114be0280cdd29d533696a77b5b49cfde5467176ecab32353395c4"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2a75d49014d118e4198bcee5ee0a6f25856b29b12dbf7cd012791f8a6cc5c496"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:363e2f92b0f0174b2f8238240a1a30142e3db7b957a5dd5689b0e75fb717cc78"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ab36c8eb7e454e34e60eb55ca5d241a5d18b2c6244f6827a30e451c42410b5f7"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4c0907b1928a36d5a998d72d64d8eaa7244989f7aaaf947500d3a800c83a3fd6"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:04432ad9479fa40ec0f387795ddad4437a2b50417c69fa275e212933519ff294"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-win32.whl", hash = "sha256:3bed14e9c89dcb10e8f3a29f9ccac4955aebe93c71ae803af79265c9ca5644c5"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:49402233c892a461407c512a19435d1ce275543138294f7ef013f0b63d5d3765"}, + {file = "charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85"}, + {file = "charset_normalizer-3.4.1.tar.gz", hash = "sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3"}, ] [[package]] name = "click" -version = "8.1.7" +version = "8.1.8" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, - {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, + {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, + {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, ] [package.dependencies] @@ -377,13 +368,13 @@ test = ["pytest (>=6)"] [[package]] name = "fastapi" -version = "0.115.4" +version = "0.115.6" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = false python-versions = ">=3.8" files = [ - {file = "fastapi-0.115.4-py3-none-any.whl", hash = "sha256:0b504a063ffb3cf96a5e27dc1bc32c80ca743a2528574f9cdc77daa2d31b4742"}, - {file = "fastapi-0.115.4.tar.gz", hash = "sha256:db653475586b091cb8b2fec2ac54a680ac6a158e07406e1abae31679e8826349"}, + {file = "fastapi-0.115.6-py3-none-any.whl", hash = "sha256:e9240b29e36fa8f4bb7290316988e90c381e5092e0cbe84e7818cc3713bcf305"}, + {file = "fastapi-0.115.6.tar.gz", hash = "sha256:9ec46f7addc14ea472958a96aae5b5de65f39721a46aaf5705c480d9a8b76654"}, ] [package.dependencies] @@ -736,88 +727,90 @@ files = [ [[package]] name = "numpy" -version = "2.1.2" +version = "2.2.1" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.10" files = [ - {file = "numpy-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:30d53720b726ec36a7f88dc873f0eec8447fbc93d93a8f079dfac2629598d6ee"}, - {file = "numpy-2.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d3ca0a72dd8846eb6f7dfe8f19088060fcb76931ed592d29128e0219652884"}, - {file = "numpy-2.1.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:fc44e3c68ff00fd991b59092a54350e6e4911152682b4782f68070985aa9e648"}, - {file = "numpy-2.1.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:7c1c60328bd964b53f8b835df69ae8198659e2b9302ff9ebb7de4e5a5994db3d"}, - {file = "numpy-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cdb606a7478f9ad91c6283e238544451e3a95f30fb5467fbf715964341a8a86"}, - {file = "numpy-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d666cb72687559689e9906197e3bec7b736764df6a2e58ee265e360663e9baf7"}, - {file = "numpy-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6eef7a2dbd0abfb0d9eaf78b73017dbfd0b54051102ff4e6a7b2980d5ac1a03"}, - {file = "numpy-2.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:12edb90831ff481f7ef5f6bc6431a9d74dc0e5ff401559a71e5e4611d4f2d466"}, - {file = "numpy-2.1.2-cp310-cp310-win32.whl", hash = "sha256:a65acfdb9c6ebb8368490dbafe83c03c7e277b37e6857f0caeadbbc56e12f4fb"}, - {file = "numpy-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:860ec6e63e2c5c2ee5e9121808145c7bf86c96cca9ad396c0bd3e0f2798ccbe2"}, - {file = "numpy-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b42a1a511c81cc78cbc4539675713bbcf9d9c3913386243ceff0e9429ca892fe"}, - {file = "numpy-2.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:faa88bc527d0f097abdc2c663cddf37c05a1c2f113716601555249805cf573f1"}, - {file = "numpy-2.1.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:c82af4b2ddd2ee72d1fc0c6695048d457e00b3582ccde72d8a1c991b808bb20f"}, - {file = "numpy-2.1.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:13602b3174432a35b16c4cfb5de9a12d229727c3dd47a6ce35111f2ebdf66ff4"}, - {file = "numpy-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ebec5fd716c5a5b3d8dfcc439be82a8407b7b24b230d0ad28a81b61c2f4659a"}, - {file = "numpy-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2b49c3c0804e8ecb05d59af8386ec2f74877f7ca8fd9c1e00be2672e4d399b1"}, - {file = "numpy-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2cbba4b30bf31ddbe97f1c7205ef976909a93a66bb1583e983adbd155ba72ac2"}, - {file = "numpy-2.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8e00ea6fc82e8a804433d3e9cedaa1051a1422cb6e443011590c14d2dea59146"}, - {file = "numpy-2.1.2-cp311-cp311-win32.whl", hash = "sha256:5006b13a06e0b38d561fab5ccc37581f23c9511879be7693bd33c7cd15ca227c"}, - {file = "numpy-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:f1eb068ead09f4994dec71c24b2844f1e4e4e013b9629f812f292f04bd1510d9"}, - {file = "numpy-2.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d7bf0a4f9f15b32b5ba53147369e94296f5fffb783db5aacc1be15b4bf72f43b"}, - {file = "numpy-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b1d0fcae4f0949f215d4632be684a539859b295e2d0cb14f78ec231915d644db"}, - {file = "numpy-2.1.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f751ed0a2f250541e19dfca9f1eafa31a392c71c832b6bb9e113b10d050cb0f1"}, - {file = "numpy-2.1.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:bd33f82e95ba7ad632bc57837ee99dba3d7e006536200c4e9124089e1bf42426"}, - {file = "numpy-2.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b8cde4f11f0a975d1fd59373b32e2f5a562ade7cde4f85b7137f3de8fbb29a0"}, - {file = "numpy-2.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d95f286b8244b3649b477ac066c6906fbb2905f8ac19b170e2175d3d799f4df"}, - {file = "numpy-2.1.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ab4754d432e3ac42d33a269c8567413bdb541689b02d93788af4131018cbf366"}, - {file = "numpy-2.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e585c8ae871fd38ac50598f4763d73ec5497b0de9a0ab4ef5b69f01c6a046142"}, - {file = "numpy-2.1.2-cp312-cp312-win32.whl", hash = "sha256:9c6c754df29ce6a89ed23afb25550d1c2d5fdb9901d9c67a16e0b16eaf7e2550"}, - {file = "numpy-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:456e3b11cb79ac9946c822a56346ec80275eaf2950314b249b512896c0d2505e"}, - {file = "numpy-2.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a84498e0d0a1174f2b3ed769b67b656aa5460c92c9554039e11f20a05650f00d"}, - {file = "numpy-2.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4d6ec0d4222e8ffdab1744da2560f07856421b367928026fb540e1945f2eeeaf"}, - {file = "numpy-2.1.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:259ec80d54999cc34cd1eb8ded513cb053c3bf4829152a2e00de2371bd406f5e"}, - {file = "numpy-2.1.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:675c741d4739af2dc20cd6c6a5c4b7355c728167845e3c6b0e824e4e5d36a6c3"}, - {file = "numpy-2.1.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05b2d4e667895cc55e3ff2b56077e4c8a5604361fc21a042845ea3ad67465aa8"}, - {file = "numpy-2.1.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43cca367bf94a14aca50b89e9bc2061683116cfe864e56740e083392f533ce7a"}, - {file = "numpy-2.1.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:76322dcdb16fccf2ac56f99048af32259dcc488d9b7e25b51e5eca5147a3fb98"}, - {file = "numpy-2.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:32e16a03138cabe0cb28e1007ee82264296ac0983714094380b408097a418cfe"}, - {file = "numpy-2.1.2-cp313-cp313-win32.whl", hash = "sha256:242b39d00e4944431a3cd2db2f5377e15b5785920421993770cddb89992c3f3a"}, - {file = "numpy-2.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:f2ded8d9b6f68cc26f8425eda5d3877b47343e68ca23d0d0846f4d312ecaa445"}, - {file = "numpy-2.1.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2ffef621c14ebb0188a8633348504a35c13680d6da93ab5cb86f4e54b7e922b5"}, - {file = "numpy-2.1.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:ad369ed238b1959dfbade9018a740fb9392c5ac4f9b5173f420bd4f37ba1f7a0"}, - {file = "numpy-2.1.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:d82075752f40c0ddf57e6e02673a17f6cb0f8eb3f587f63ca1eaab5594da5b17"}, - {file = "numpy-2.1.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:1600068c262af1ca9580a527d43dc9d959b0b1d8e56f8a05d830eea39b7c8af6"}, - {file = "numpy-2.1.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a26ae94658d3ba3781d5e103ac07a876b3e9b29db53f68ed7df432fd033358a8"}, - {file = "numpy-2.1.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13311c2db4c5f7609b462bc0f43d3c465424d25c626d95040f073e30f7570e35"}, - {file = "numpy-2.1.2-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:2abbf905a0b568706391ec6fa15161fad0fb5d8b68d73c461b3c1bab6064dd62"}, - {file = "numpy-2.1.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ef444c57d664d35cac4e18c298c47d7b504c66b17c2ea91312e979fcfbdfb08a"}, - {file = "numpy-2.1.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:bdd407c40483463898b84490770199d5714dcc9dd9b792f6c6caccc523c00952"}, - {file = "numpy-2.1.2-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:da65fb46d4cbb75cb417cddf6ba5e7582eb7bb0b47db4b99c9fe5787ce5d91f5"}, - {file = "numpy-2.1.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c193d0b0238638e6fc5f10f1b074a6993cb13b0b431f64079a509d63d3aa8b7"}, - {file = "numpy-2.1.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a7d80b2e904faa63068ead63107189164ca443b42dd1930299e0d1cb041cec2e"}, - {file = "numpy-2.1.2.tar.gz", hash = "sha256:13532a088217fa624c99b843eeb54640de23b3414b14aa66d023805eb731066c"}, + {file = "numpy-2.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5edb4e4caf751c1518e6a26a83501fda79bff41cc59dac48d70e6d65d4ec4440"}, + {file = "numpy-2.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:aa3017c40d513ccac9621a2364f939d39e550c542eb2a894b4c8da92b38896ab"}, + {file = "numpy-2.2.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:61048b4a49b1c93fe13426e04e04fdf5a03f456616f6e98c7576144677598675"}, + {file = "numpy-2.2.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:7671dc19c7019103ca44e8d94917eba8534c76133523ca8406822efdd19c9308"}, + {file = "numpy-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4250888bcb96617e00bfa28ac24850a83c9f3a16db471eca2ee1f1714df0f957"}, + {file = "numpy-2.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7746f235c47abc72b102d3bce9977714c2444bdfaea7888d241b4c4bb6a78bf"}, + {file = "numpy-2.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:059e6a747ae84fce488c3ee397cee7e5f905fd1bda5fb18c66bc41807ff119b2"}, + {file = "numpy-2.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f62aa6ee4eb43b024b0e5a01cf65a0bb078ef8c395e8713c6e8a12a697144528"}, + {file = "numpy-2.2.1-cp310-cp310-win32.whl", hash = "sha256:48fd472630715e1c1c89bf1feab55c29098cb403cc184b4859f9c86d4fcb6a95"}, + {file = "numpy-2.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:b541032178a718c165a49638d28272b771053f628382d5e9d1c93df23ff58dbf"}, + {file = "numpy-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:40f9e544c1c56ba8f1cf7686a8c9b5bb249e665d40d626a23899ba6d5d9e1484"}, + {file = "numpy-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9b57eaa3b0cd8db52049ed0330747b0364e899e8a606a624813452b8203d5f7"}, + {file = "numpy-2.2.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:bc8a37ad5b22c08e2dbd27df2b3ef7e5c0864235805b1e718a235bcb200cf1cb"}, + {file = "numpy-2.2.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:9036d6365d13b6cbe8f27a0eaf73ddcc070cae584e5ff94bb45e3e9d729feab5"}, + {file = "numpy-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51faf345324db860b515d3f364eaa93d0e0551a88d6218a7d61286554d190d73"}, + {file = "numpy-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38efc1e56b73cc9b182fe55e56e63b044dd26a72128fd2fbd502f75555d92591"}, + {file = "numpy-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:31b89fa67a8042e96715c68e071a1200c4e172f93b0fbe01a14c0ff3ff820fc8"}, + {file = "numpy-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4c86e2a209199ead7ee0af65e1d9992d1dce7e1f63c4b9a616500f93820658d0"}, + {file = "numpy-2.2.1-cp311-cp311-win32.whl", hash = "sha256:b34d87e8a3090ea626003f87f9392b3929a7bbf4104a05b6667348b6bd4bf1cd"}, + {file = "numpy-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:360137f8fb1b753c5cde3ac388597ad680eccbbbb3865ab65efea062c4a1fd16"}, + {file = "numpy-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:694f9e921a0c8f252980e85bce61ebbd07ed2b7d4fa72d0e4246f2f8aa6642ab"}, + {file = "numpy-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3683a8d166f2692664262fd4900f207791d005fb088d7fdb973cc8d663626faa"}, + {file = "numpy-2.2.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:780077d95eafc2ccc3ced969db22377b3864e5b9a0ea5eb347cc93b3ea900315"}, + {file = "numpy-2.2.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:55ba24ebe208344aa7a00e4482f65742969a039c2acfcb910bc6fcd776eb4355"}, + {file = "numpy-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b1d07b53b78bf84a96898c1bc139ad7f10fda7423f5fd158fd0f47ec5e01ac7"}, + {file = "numpy-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5062dc1a4e32a10dc2b8b13cedd58988261416e811c1dc4dbdea4f57eea61b0d"}, + {file = "numpy-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fce4f615f8ca31b2e61aa0eb5865a21e14f5629515c9151850aa936c02a1ee51"}, + {file = "numpy-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:67d4cda6fa6ffa073b08c8372aa5fa767ceb10c9a0587c707505a6d426f4e046"}, + {file = "numpy-2.2.1-cp312-cp312-win32.whl", hash = "sha256:32cb94448be47c500d2c7a95f93e2f21a01f1fd05dd2beea1ccd049bb6001cd2"}, + {file = "numpy-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:ba5511d8f31c033a5fcbda22dd5c813630af98c70b2661f2d2c654ae3cdfcfc8"}, + {file = "numpy-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f1d09e520217618e76396377c81fba6f290d5f926f50c35f3a5f72b01a0da780"}, + {file = "numpy-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3ecc47cd7f6ea0336042be87d9e7da378e5c7e9b3c8ad0f7c966f714fc10d821"}, + {file = "numpy-2.2.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f419290bc8968a46c4933158c91a0012b7a99bb2e465d5ef5293879742f8797e"}, + {file = "numpy-2.2.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:5b6c390bfaef8c45a260554888966618328d30e72173697e5cabe6b285fb2348"}, + {file = "numpy-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:526fc406ab991a340744aad7e25251dd47a6720a685fa3331e5c59fef5282a59"}, + {file = "numpy-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f74e6fdeb9a265624ec3a3918430205dff1df7e95a230779746a6af78bc615af"}, + {file = "numpy-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:53c09385ff0b72ba79d8715683c1168c12e0b6e84fb0372e97553d1ea91efe51"}, + {file = "numpy-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f3eac17d9ec51be534685ba877b6ab5edc3ab7ec95c8f163e5d7b39859524716"}, + {file = "numpy-2.2.1-cp313-cp313-win32.whl", hash = "sha256:9ad014faa93dbb52c80d8f4d3dcf855865c876c9660cb9bd7553843dd03a4b1e"}, + {file = "numpy-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:164a829b6aacf79ca47ba4814b130c4020b202522a93d7bff2202bfb33b61c60"}, + {file = "numpy-2.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4dfda918a13cc4f81e9118dea249e192ab167a0bb1966272d5503e39234d694e"}, + {file = "numpy-2.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:733585f9f4b62e9b3528dd1070ec4f52b8acf64215b60a845fa13ebd73cd0712"}, + {file = "numpy-2.2.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:89b16a18e7bba224ce5114db863e7029803c179979e1af6ad6a6b11f70545008"}, + {file = "numpy-2.2.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:676f4eebf6b2d430300f1f4f4c2461685f8269f94c89698d832cdf9277f30b84"}, + {file = "numpy-2.2.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27f5cdf9f493b35f7e41e8368e7d7b4bbafaf9660cba53fb21d2cd174ec09631"}, + {file = "numpy-2.2.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1ad395cf254c4fbb5b2132fee391f361a6e8c1adbd28f2cd8e79308a615fe9d"}, + {file = "numpy-2.2.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:08ef779aed40dbc52729d6ffe7dd51df85796a702afbf68a4f4e41fafdc8bda5"}, + {file = "numpy-2.2.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:26c9c4382b19fcfbbed3238a14abf7ff223890ea1936b8890f058e7ba35e8d71"}, + {file = "numpy-2.2.1-cp313-cp313t-win32.whl", hash = "sha256:93cf4e045bae74c90ca833cba583c14b62cb4ba2cba0abd2b141ab52548247e2"}, + {file = "numpy-2.2.1-cp313-cp313t-win_amd64.whl", hash = "sha256:bff7d8ec20f5f42607599f9994770fa65d76edca264a87b5e4ea5629bce12268"}, + {file = "numpy-2.2.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7ba9cc93a91d86365a5d270dee221fdc04fb68d7478e6bf6af650de78a8339e3"}, + {file = "numpy-2.2.1-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:3d03883435a19794e41f147612a77a8f56d4e52822337844fff3d4040a142964"}, + {file = "numpy-2.2.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4511d9e6071452b944207c8ce46ad2f897307910b402ea5fa975da32e0102800"}, + {file = "numpy-2.2.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5c5cc0cbabe9452038ed984d05ac87910f89370b9242371bd9079cb4af61811e"}, + {file = "numpy-2.2.1.tar.gz", hash = "sha256:45681fd7128c8ad1c379f0ca0776a8b0c6583d2f69889ddac01559dfe4390918"}, ] [[package]] name = "openbb-core" -version = "1.3.5" +version = "1.3.7" description = "OpenBB package with core functionality." optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "openbb_core-1.3.5-py3-none-any.whl", hash = "sha256:bb93b2343eea06faff2644e03fd6ee7a09c37392c5486d96021aa9ae137e1a90"}, - {file = "openbb_core-1.3.5.tar.gz", hash = "sha256:d8abed2a351a0bca1b02fdaba439574b452a7a812d72d5e059f2dbedab55bd19"}, + {file = "openbb_core-1.3.7-py3-none-any.whl", hash = "sha256:2dd620db5d17c8c4802dd0537de92fb88d11f4f7252ab026e54302c6f86467fa"}, + {file = "openbb_core-1.3.7.tar.gz", hash = "sha256:2e6facd812ee4ad43b9444fedcf3211599d7cda9cbb4999647cd8e4d9e4af210"}, ] [package.dependencies] -aiohttp = ">=3.10.4,<4.0.0" +aiohttp = ">=3.10.11,<4.0.0" fastapi = ">=0.115,<0.116" html5lib = ">=1.1,<2.0" importlib-metadata = ">=6.8.0" pandas = ">=1.5.3" posthog = ">=3.3.1,<4.0.0" pydantic = ">=2.5.1,<3.0.0" -pyjwt = ">=2.8.0,<3.0.0" +pyjwt = ">=2.10.1,<3.0.0" python-dotenv = ">=1.0.0,<2.0.0" -python-multipart = ">=0.0.7,<0.0.8" +python-multipart = ">=0.0.18,<0.0.19" requests = ">=2.32.1,<3.0.0" ruff = ">=0.7,<0.8" uuid7 = ">=0.1.0,<0.2.0" @@ -877,9 +870,9 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, {version = ">=1.22.4", markers = "python_version < \"3.11\""}, {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -912,13 +905,13 @@ xml = ["lxml (>=4.9.2)"] [[package]] name = "posthog" -version = "3.7.0" +version = "3.7.5" description = "Integrate PostHog into any python application." optional = false python-versions = "*" files = [ - {file = "posthog-3.7.0-py2.py3-none-any.whl", hash = "sha256:3555161c3a9557b5666f96d8e1f17f410ea0f07db56e399e336a1656d4e5c722"}, - {file = "posthog-3.7.0.tar.gz", hash = "sha256:b095d4354ba23f8b346ab5daed8ecfc5108772f922006982dfe8b2d29ebc6e0e"}, + {file = "posthog-3.7.5-py2.py3-none-any.whl", hash = "sha256:022132c17069dde03c5c5904e2ae1b9bd68d5059cbc5a8dffc5c1537a1b71cb5"}, + {file = "posthog-3.7.5.tar.gz", hash = "sha256:8ba40ab623da35db72715fc87fe7dccb7fc272ced92581fe31db2d4dbe7ad761"}, ] [package.dependencies] @@ -935,129 +928,110 @@ test = ["coverage", "django", "flake8", "freezegun (==0.3.15)", "mock (>=2.0.0)" [[package]] name = "propcache" -version = "0.2.0" +version = "0.2.1" description = "Accelerated property cache" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c5869b8fd70b81835a6f187c5fdbe67917a04d7e52b6e7cc4e5fe39d55c39d58"}, - {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:952e0d9d07609d9c5be361f33b0d6d650cd2bae393aabb11d9b719364521984b"}, - {file = "propcache-0.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:33ac8f098df0585c0b53009f039dfd913b38c1d2edafed0cedcc0c32a05aa110"}, - {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97e48e8875e6c13909c800fa344cd54cc4b2b0db1d5f911f840458a500fde2c2"}, - {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:388f3217649d6d59292b722d940d4d2e1e6a7003259eb835724092a1cca0203a"}, - {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f571aea50ba5623c308aa146eb650eebf7dbe0fd8c5d946e28343cb3b5aad577"}, - {file = "propcache-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3dfafb44f7bb35c0c06eda6b2ab4bfd58f02729e7c4045e179f9a861b07c9850"}, - {file = "propcache-0.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3ebe9a75be7ab0b7da2464a77bb27febcb4fab46a34f9288f39d74833db7f61"}, - {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d2f0d0f976985f85dfb5f3d685697ef769faa6b71993b46b295cdbbd6be8cc37"}, - {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a3dc1a4b165283bd865e8f8cb5f0c64c05001e0718ed06250d8cac9bec115b48"}, - {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9e0f07b42d2a50c7dd2d8675d50f7343d998c64008f1da5fef888396b7f84630"}, - {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e63e3e1e0271f374ed489ff5ee73d4b6e7c60710e1f76af5f0e1a6117cd26394"}, - {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:56bb5c98f058a41bb58eead194b4db8c05b088c93d94d5161728515bd52b052b"}, - {file = "propcache-0.2.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7665f04d0c7f26ff8bb534e1c65068409bf4687aa2534faf7104d7182debb336"}, - {file = "propcache-0.2.0-cp310-cp310-win32.whl", hash = "sha256:7cf18abf9764746b9c8704774d8b06714bcb0a63641518a3a89c7f85cc02c2ad"}, - {file = "propcache-0.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:cfac69017ef97db2438efb854edf24f5a29fd09a536ff3a992b75990720cdc99"}, - {file = "propcache-0.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:63f13bf09cc3336eb04a837490b8f332e0db41da66995c9fd1ba04552e516354"}, - {file = "propcache-0.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608cce1da6f2672a56b24a015b42db4ac612ee709f3d29f27a00c943d9e851de"}, - {file = "propcache-0.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:466c219deee4536fbc83c08d09115249db301550625c7fef1c5563a584c9bc87"}, - {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc2db02409338bf36590aa985a461b2c96fce91f8e7e0f14c50c5fcc4f229016"}, - {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a6ed8db0a556343d566a5c124ee483ae113acc9a557a807d439bcecc44e7dfbb"}, - {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:91997d9cb4a325b60d4e3f20967f8eb08dfcb32b22554d5ef78e6fd1dda743a2"}, - {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c7dde9e533c0a49d802b4f3f218fa9ad0a1ce21f2c2eb80d5216565202acab4"}, - {file = "propcache-0.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffcad6c564fe6b9b8916c1aefbb37a362deebf9394bd2974e9d84232e3e08504"}, - {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:97a58a28bcf63284e8b4d7b460cbee1edaab24634e82059c7b8c09e65284f178"}, - {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:945db8ee295d3af9dbdbb698cce9bbc5c59b5c3fe328bbc4387f59a8a35f998d"}, - {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:39e104da444a34830751715f45ef9fc537475ba21b7f1f5b0f4d71a3b60d7fe2"}, - {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c5ecca8f9bab618340c8e848d340baf68bcd8ad90a8ecd7a4524a81c1764b3db"}, - {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:c436130cc779806bdf5d5fae0d848713105472b8566b75ff70048c47d3961c5b"}, - {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:191db28dc6dcd29d1a3e063c3be0b40688ed76434622c53a284e5427565bbd9b"}, - {file = "propcache-0.2.0-cp311-cp311-win32.whl", hash = "sha256:5f2564ec89058ee7c7989a7b719115bdfe2a2fb8e7a4543b8d1c0cc4cf6478c1"}, - {file = "propcache-0.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e2e54267980349b723cff366d1e29b138b9a60fa376664a157a342689553f71"}, - {file = "propcache-0.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2ee7606193fb267be4b2e3b32714f2d58cad27217638db98a60f9efb5efeccc2"}, - {file = "propcache-0.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:91ee8fc02ca52e24bcb77b234f22afc03288e1dafbb1f88fe24db308910c4ac7"}, - {file = "propcache-0.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2e900bad2a8456d00a113cad8c13343f3b1f327534e3589acc2219729237a2e8"}, - {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f52a68c21363c45297aca15561812d542f8fc683c85201df0bebe209e349f793"}, - {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e41d67757ff4fbc8ef2af99b338bfb955010444b92929e9e55a6d4dcc3c4f09"}, - {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a64e32f8bd94c105cc27f42d3b658902b5bcc947ece3c8fe7bc1b05982f60e89"}, - {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55346705687dbd7ef0d77883ab4f6fabc48232f587925bdaf95219bae072491e"}, - {file = "propcache-0.2.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00181262b17e517df2cd85656fcd6b4e70946fe62cd625b9d74ac9977b64d8d9"}, - {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6994984550eaf25dd7fc7bd1b700ff45c894149341725bb4edc67f0ffa94efa4"}, - {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:56295eb1e5f3aecd516d91b00cfd8bf3a13991de5a479df9e27dd569ea23959c"}, - {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:439e76255daa0f8151d3cb325f6dd4a3e93043e6403e6491813bcaaaa8733887"}, - {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f6475a1b2ecb310c98c28d271a30df74f9dd436ee46d09236a6b750a7599ce57"}, - {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3444cdba6628accf384e349014084b1cacd866fbb88433cd9d279d90a54e0b23"}, - {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a9d9b4d0a9b38d1c391bb4ad24aa65f306c6f01b512e10a8a34a2dc5675d348"}, - {file = "propcache-0.2.0-cp312-cp312-win32.whl", hash = "sha256:69d3a98eebae99a420d4b28756c8ce6ea5a29291baf2dc9ff9414b42676f61d5"}, - {file = "propcache-0.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:ad9c9b99b05f163109466638bd30ada1722abb01bbb85c739c50b6dc11f92dc3"}, - {file = "propcache-0.2.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ecddc221a077a8132cf7c747d5352a15ed763b674c0448d811f408bf803d9ad7"}, - {file = "propcache-0.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0e53cb83fdd61cbd67202735e6a6687a7b491c8742dfc39c9e01e80354956763"}, - {file = "propcache-0.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92fe151145a990c22cbccf9ae15cae8ae9eddabfc949a219c9f667877e40853d"}, - {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6a21ef516d36909931a2967621eecb256018aeb11fc48656e3257e73e2e247a"}, - {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f88a4095e913f98988f5b338c1d4d5d07dbb0b6bad19892fd447484e483ba6b"}, - {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a5b3bb545ead161be780ee85a2b54fdf7092815995661947812dde94a40f6fb"}, - {file = "propcache-0.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67aeb72e0f482709991aa91345a831d0b707d16b0257e8ef88a2ad246a7280bf"}, - {file = "propcache-0.2.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c997f8c44ec9b9b0bcbf2d422cc00a1d9b9c681f56efa6ca149a941e5560da2"}, - {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2a66df3d4992bc1d725b9aa803e8c5a66c010c65c741ad901e260ece77f58d2f"}, - {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3ebbcf2a07621f29638799828b8d8668c421bfb94c6cb04269130d8de4fb7136"}, - {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1235c01ddaa80da8235741e80815ce381c5267f96cc49b1477fdcf8c047ef325"}, - {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3947483a381259c06921612550867b37d22e1df6d6d7e8361264b6d037595f44"}, - {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d5bed7f9805cc29c780f3aee05de3262ee7ce1f47083cfe9f77471e9d6777e83"}, - {file = "propcache-0.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e4a91d44379f45f5e540971d41e4626dacd7f01004826a18cb048e7da7e96544"}, - {file = "propcache-0.2.0-cp313-cp313-win32.whl", hash = "sha256:f902804113e032e2cdf8c71015651c97af6418363bea8d78dc0911d56c335032"}, - {file = "propcache-0.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8f188cfcc64fb1266f4684206c9de0e80f54622c3f22a910cbd200478aeae61e"}, - {file = "propcache-0.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:53d1bd3f979ed529f0805dd35ddaca330f80a9a6d90bc0121d2ff398f8ed8861"}, - {file = "propcache-0.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:83928404adf8fb3d26793665633ea79b7361efa0287dfbd372a7e74311d51ee6"}, - {file = "propcache-0.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:77a86c261679ea5f3896ec060be9dc8e365788248cc1e049632a1be682442063"}, - {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:218db2a3c297a3768c11a34812e63b3ac1c3234c3a086def9c0fee50d35add1f"}, - {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7735e82e3498c27bcb2d17cb65d62c14f1100b71723b68362872bca7d0913d90"}, - {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:20a617c776f520c3875cf4511e0d1db847a076d720714ae35ffe0df3e440be68"}, - {file = "propcache-0.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67b69535c870670c9f9b14a75d28baa32221d06f6b6fa6f77a0a13c5a7b0a5b9"}, - {file = "propcache-0.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4569158070180c3855e9c0791c56be3ceeb192defa2cdf6a3f39e54319e56b89"}, - {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:db47514ffdbd91ccdc7e6f8407aac4ee94cc871b15b577c1c324236b013ddd04"}, - {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:2a60ad3e2553a74168d275a0ef35e8c0a965448ffbc3b300ab3a5bb9956c2162"}, - {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:662dd62358bdeaca0aee5761de8727cfd6861432e3bb828dc2a693aa0471a563"}, - {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:25a1f88b471b3bc911d18b935ecb7115dff3a192b6fef46f0bfaf71ff4f12418"}, - {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:f60f0ac7005b9f5a6091009b09a419ace1610e163fa5deaba5ce3484341840e7"}, - {file = "propcache-0.2.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:74acd6e291f885678631b7ebc85d2d4aec458dd849b8c841b57ef04047833bed"}, - {file = "propcache-0.2.0-cp38-cp38-win32.whl", hash = "sha256:d9b6ddac6408194e934002a69bcaadbc88c10b5f38fb9307779d1c629181815d"}, - {file = "propcache-0.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:676135dcf3262c9c5081cc8f19ad55c8a64e3f7282a21266d05544450bffc3a5"}, - {file = "propcache-0.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:25c8d773a62ce0451b020c7b29a35cfbc05de8b291163a7a0f3b7904f27253e6"}, - {file = "propcache-0.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:375a12d7556d462dc64d70475a9ee5982465fbb3d2b364f16b86ba9135793638"}, - {file = "propcache-0.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1ec43d76b9677637a89d6ab86e1fef70d739217fefa208c65352ecf0282be957"}, - {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f45eec587dafd4b2d41ac189c2156461ebd0c1082d2fe7013571598abb8505d1"}, - {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc092ba439d91df90aea38168e11f75c655880c12782facf5cf9c00f3d42b562"}, - {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa1076244f54bb76e65e22cb6910365779d5c3d71d1f18b275f1dfc7b0d71b4d"}, - {file = "propcache-0.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:682a7c79a2fbf40f5dbb1eb6bfe2cd865376deeac65acf9beb607505dced9e12"}, - {file = "propcache-0.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e40876731f99b6f3c897b66b803c9e1c07a989b366c6b5b475fafd1f7ba3fb8"}, - {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:363ea8cd3c5cb6679f1c2f5f1f9669587361c062e4899fce56758efa928728f8"}, - {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:140fbf08ab3588b3468932974a9331aff43c0ab8a2ec2c608b6d7d1756dbb6cb"}, - {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e70fac33e8b4ac63dfc4c956fd7d85a0b1139adcfc0d964ce288b7c527537fea"}, - {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:b33d7a286c0dc1a15f5fc864cc48ae92a846df287ceac2dd499926c3801054a6"}, - {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f6d5749fdd33d90e34c2efb174c7e236829147a2713334d708746e94c4bde40d"}, - {file = "propcache-0.2.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:22aa8f2272d81d9317ff5756bb108021a056805ce63dd3630e27d042c8092798"}, - {file = "propcache-0.2.0-cp39-cp39-win32.whl", hash = "sha256:73e4b40ea0eda421b115248d7e79b59214411109a5bc47d0d48e4c73e3b8fcf9"}, - {file = "propcache-0.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:9517d5e9e0731957468c29dbfd0f976736a0e55afaea843726e887f36fe017df"}, - {file = "propcache-0.2.0-py3-none-any.whl", hash = "sha256:2ccc28197af5313706511fab3a8b66dcd6da067a1331372c82ea1cb74285e036"}, - {file = "propcache-0.2.0.tar.gz", hash = "sha256:df81779732feb9d01e5d513fad0122efb3d53bbc75f61b2a4f29a020bc985e70"}, + {file = "propcache-0.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6b3f39a85d671436ee3d12c017f8fdea38509e4f25b28eb25877293c98c243f6"}, + {file = "propcache-0.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d51fbe4285d5db5d92a929e3e21536ea3dd43732c5b177c7ef03f918dff9f2"}, + {file = "propcache-0.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6445804cf4ec763dc70de65a3b0d9954e868609e83850a47ca4f0cb64bd79fea"}, + {file = "propcache-0.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9479aa06a793c5aeba49ce5c5692ffb51fcd9a7016e017d555d5e2b0045d212"}, + {file = "propcache-0.2.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9631c5e8b5b3a0fda99cb0d29c18133bca1e18aea9effe55adb3da1adef80d3"}, + {file = "propcache-0.2.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3156628250f46a0895f1f36e1d4fbe062a1af8718ec3ebeb746f1d23f0c5dc4d"}, + {file = "propcache-0.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b6fb63ae352e13748289f04f37868099e69dba4c2b3e271c46061e82c745634"}, + {file = "propcache-0.2.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:887d9b0a65404929641a9fabb6452b07fe4572b269d901d622d8a34a4e9043b2"}, + {file = "propcache-0.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a96dc1fa45bd8c407a0af03b2d5218392729e1822b0c32e62c5bf7eeb5fb3958"}, + {file = "propcache-0.2.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a7e65eb5c003a303b94aa2c3852ef130230ec79e349632d030e9571b87c4698c"}, + {file = "propcache-0.2.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:999779addc413181912e984b942fbcc951be1f5b3663cd80b2687758f434c583"}, + {file = "propcache-0.2.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:19a0f89a7bb9d8048d9c4370c9c543c396e894c76be5525f5e1ad287f1750ddf"}, + {file = "propcache-0.2.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:1ac2f5fe02fa75f56e1ad473f1175e11f475606ec9bd0be2e78e4734ad575034"}, + {file = "propcache-0.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:574faa3b79e8ebac7cb1d7930f51184ba1ccf69adfdec53a12f319a06030a68b"}, + {file = "propcache-0.2.1-cp310-cp310-win32.whl", hash = "sha256:03ff9d3f665769b2a85e6157ac8b439644f2d7fd17615a82fa55739bc97863f4"}, + {file = "propcache-0.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:2d3af2e79991102678f53e0dbf4c35de99b6b8b58f29a27ca0325816364caaba"}, + {file = "propcache-0.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1ffc3cca89bb438fb9c95c13fc874012f7b9466b89328c3c8b1aa93cdcfadd16"}, + {file = "propcache-0.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f174bbd484294ed9fdf09437f889f95807e5f229d5d93588d34e92106fbf6717"}, + {file = "propcache-0.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:70693319e0b8fd35dd863e3e29513875eb15c51945bf32519ef52927ca883bc3"}, + {file = "propcache-0.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b480c6a4e1138e1aa137c0079b9b6305ec6dcc1098a8ca5196283e8a49df95a9"}, + {file = "propcache-0.2.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d27b84d5880f6d8aa9ae3edb253c59d9f6642ffbb2c889b78b60361eed449787"}, + {file = "propcache-0.2.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:857112b22acd417c40fa4595db2fe28ab900c8c5fe4670c7989b1c0230955465"}, + {file = "propcache-0.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf6c4150f8c0e32d241436526f3c3f9cbd34429492abddbada2ffcff506c51af"}, + {file = "propcache-0.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66d4cfda1d8ed687daa4bc0274fcfd5267873db9a5bc0418c2da19273040eeb7"}, + {file = "propcache-0.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2f992c07c0fca81655066705beae35fc95a2fa7366467366db627d9f2ee097f"}, + {file = "propcache-0.2.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:4a571d97dbe66ef38e472703067021b1467025ec85707d57e78711c085984e54"}, + {file = "propcache-0.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:bb6178c241278d5fe853b3de743087be7f5f4c6f7d6d22a3b524d323eecec505"}, + {file = "propcache-0.2.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:ad1af54a62ffe39cf34db1aa6ed1a1873bd548f6401db39d8e7cd060b9211f82"}, + {file = "propcache-0.2.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e7048abd75fe40712005bcfc06bb44b9dfcd8e101dda2ecf2f5aa46115ad07ca"}, + {file = "propcache-0.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:160291c60081f23ee43d44b08a7e5fb76681221a8e10b3139618c5a9a291b84e"}, + {file = "propcache-0.2.1-cp311-cp311-win32.whl", hash = "sha256:819ce3b883b7576ca28da3861c7e1a88afd08cc8c96908e08a3f4dd64a228034"}, + {file = "propcache-0.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:edc9fc7051e3350643ad929df55c451899bb9ae6d24998a949d2e4c87fb596d3"}, + {file = "propcache-0.2.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:081a430aa8d5e8876c6909b67bd2d937bfd531b0382d3fdedb82612c618bc41a"}, + {file = "propcache-0.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d2ccec9ac47cf4e04897619c0e0c1a48c54a71bdf045117d3a26f80d38ab1fb0"}, + {file = "propcache-0.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:14d86fe14b7e04fa306e0c43cdbeebe6b2c2156a0c9ce56b815faacc193e320d"}, + {file = "propcache-0.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:049324ee97bb67285b49632132db351b41e77833678432be52bdd0289c0e05e4"}, + {file = "propcache-0.2.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1cd9a1d071158de1cc1c71a26014dcdfa7dd3d5f4f88c298c7f90ad6f27bb46d"}, + {file = "propcache-0.2.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98110aa363f1bb4c073e8dcfaefd3a5cea0f0834c2aab23dda657e4dab2f53b5"}, + {file = "propcache-0.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:647894f5ae99c4cf6bb82a1bb3a796f6e06af3caa3d32e26d2350d0e3e3faf24"}, + {file = "propcache-0.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfd3223c15bebe26518d58ccf9a39b93948d3dcb3e57a20480dfdd315356baff"}, + {file = "propcache-0.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d71264a80f3fcf512eb4f18f59423fe82d6e346ee97b90625f283df56aee103f"}, + {file = "propcache-0.2.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e73091191e4280403bde6c9a52a6999d69cdfde498f1fdf629105247599b57ec"}, + {file = "propcache-0.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3935bfa5fede35fb202c4b569bb9c042f337ca4ff7bd540a0aa5e37131659348"}, + {file = "propcache-0.2.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f508b0491767bb1f2b87fdfacaba5f7eddc2f867740ec69ece6d1946d29029a6"}, + {file = "propcache-0.2.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1672137af7c46662a1c2be1e8dc78cb6d224319aaa40271c9257d886be4363a6"}, + {file = "propcache-0.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b74c261802d3d2b85c9df2dfb2fa81b6f90deeef63c2db9f0e029a3cac50b518"}, + {file = "propcache-0.2.1-cp312-cp312-win32.whl", hash = "sha256:d09c333d36c1409d56a9d29b3a1b800a42c76a57a5a8907eacdbce3f18768246"}, + {file = "propcache-0.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:c214999039d4f2a5b2073ac506bba279945233da8c786e490d411dfc30f855c1"}, + {file = "propcache-0.2.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:aca405706e0b0a44cc6bfd41fbe89919a6a56999157f6de7e182a990c36e37bc"}, + {file = "propcache-0.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:12d1083f001ace206fe34b6bdc2cb94be66d57a850866f0b908972f90996b3e9"}, + {file = "propcache-0.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d93f3307ad32a27bda2e88ec81134b823c240aa3abb55821a8da553eed8d9439"}, + {file = "propcache-0.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba278acf14471d36316159c94a802933d10b6a1e117b8554fe0d0d9b75c9d536"}, + {file = "propcache-0.2.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4e6281aedfca15301c41f74d7005e6e3f4ca143584ba696ac69df4f02f40d629"}, + {file = "propcache-0.2.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5b750a8e5a1262434fb1517ddf64b5de58327f1adc3524a5e44c2ca43305eb0b"}, + {file = "propcache-0.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf72af5e0fb40e9babf594308911436c8efde3cb5e75b6f206c34ad18be5c052"}, + {file = "propcache-0.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b2d0a12018b04f4cb820781ec0dffb5f7c7c1d2a5cd22bff7fb055a2cb19ebce"}, + {file = "propcache-0.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e800776a79a5aabdb17dcc2346a7d66d0777e942e4cd251defeb084762ecd17d"}, + {file = "propcache-0.2.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:4160d9283bd382fa6c0c2b5e017acc95bc183570cd70968b9202ad6d8fc48dce"}, + {file = "propcache-0.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:30b43e74f1359353341a7adb783c8f1b1c676367b011709f466f42fda2045e95"}, + {file = "propcache-0.2.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:58791550b27d5488b1bb52bc96328456095d96206a250d28d874fafe11b3dfaf"}, + {file = "propcache-0.2.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:0f022d381747f0dfe27e99d928e31bc51a18b65bb9e481ae0af1380a6725dd1f"}, + {file = "propcache-0.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:297878dc9d0a334358f9b608b56d02e72899f3b8499fc6044133f0d319e2ec30"}, + {file = "propcache-0.2.1-cp313-cp313-win32.whl", hash = "sha256:ddfab44e4489bd79bda09d84c430677fc7f0a4939a73d2bba3073036f487a0a6"}, + {file = "propcache-0.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:556fc6c10989f19a179e4321e5d678db8eb2924131e64652a51fe83e4c3db0e1"}, + {file = "propcache-0.2.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:6a9a8c34fb7bb609419a211e59da8887eeca40d300b5ea8e56af98f6fbbb1541"}, + {file = "propcache-0.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ae1aa1cd222c6d205853b3013c69cd04515f9d6ab6de4b0603e2e1c33221303e"}, + {file = "propcache-0.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:accb6150ce61c9c4b7738d45550806aa2b71c7668c6942f17b0ac182b6142fd4"}, + {file = "propcache-0.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5eee736daafa7af6d0a2dc15cc75e05c64f37fc37bafef2e00d77c14171c2097"}, + {file = "propcache-0.2.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7a31fc1e1bd362874863fdeed71aed92d348f5336fd84f2197ba40c59f061bd"}, + {file = "propcache-0.2.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba4cfa1052819d16699e1d55d18c92b6e094d4517c41dd231a8b9f87b6fa681"}, + {file = "propcache-0.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f089118d584e859c62b3da0892b88a83d611c2033ac410e929cb6754eec0ed16"}, + {file = "propcache-0.2.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:781e65134efaf88feb447e8c97a51772aa75e48b794352f94cb7ea717dedda0d"}, + {file = "propcache-0.2.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31f5af773530fd3c658b32b6bdc2d0838543de70eb9a2156c03e410f7b0d3aae"}, + {file = "propcache-0.2.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:a7a078f5d37bee6690959c813977da5291b24286e7b962e62a94cec31aa5188b"}, + {file = "propcache-0.2.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:cea7daf9fc7ae6687cf1e2c049752f19f146fdc37c2cc376e7d0032cf4f25347"}, + {file = "propcache-0.2.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:8b3489ff1ed1e8315674d0775dc7d2195fb13ca17b3808721b54dbe9fd020faf"}, + {file = "propcache-0.2.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:9403db39be1393618dd80c746cb22ccda168efce239c73af13c3763ef56ffc04"}, + {file = "propcache-0.2.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5d97151bc92d2b2578ff7ce779cdb9174337390a535953cbb9452fb65164c587"}, + {file = "propcache-0.2.1-cp39-cp39-win32.whl", hash = "sha256:9caac6b54914bdf41bcc91e7eb9147d331d29235a7c967c150ef5df6464fd1bb"}, + {file = "propcache-0.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:92fc4500fcb33899b05ba73276dfb684a20d31caa567b7cb5252d48f896a91b1"}, + {file = "propcache-0.2.1-py3-none-any.whl", hash = "sha256:52277518d6aae65536e9cea52d4e7fd2f7a66f4aa2d30ed3f2fcea620ace3c54"}, + {file = "propcache-0.2.1.tar.gz", hash = "sha256:3f77ce728b19cb537714499928fe800c3dda29e8d9428778fc7c186da4c09a64"}, ] [[package]] name = "pydantic" -version = "2.9.2" +version = "2.10.4" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, - {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, + {file = "pydantic-2.10.4-py3-none-any.whl", hash = "sha256:597e135ea68be3a37552fb524bc7d0d66dcf93d395acd93a00682f1efcb8ee3d"}, + {file = "pydantic-2.10.4.tar.gz", hash = "sha256:82f12e9723da6de4fe2ba888b5971157b3be7ad914267dea8f05f82b28254f06"}, ] [package.dependencies] annotated-types = ">=0.6.0" -pydantic-core = "2.23.4" -typing-extensions = [ - {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, - {version = ">=4.6.1", markers = "python_version < \"3.13\""}, -] +pydantic-core = "2.27.2" +typing-extensions = ">=4.12.2" [package.extras] email = ["email-validator (>=2.0.0)"] @@ -1065,100 +1039,111 @@ timezone = ["tzdata"] [[package]] name = "pydantic-core" -version = "2.23.4" +version = "2.27.2" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.23.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b"}, - {file = "pydantic_core-2.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63e46b3169866bd62849936de036f901a9356e36376079b05efa83caeaa02ceb"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed1a53de42fbe34853ba90513cea21673481cd81ed1be739f7f2efb931b24916"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cfdd16ab5e59fc31b5e906d1a3f666571abc367598e3e02c83403acabc092e07"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255a8ef062cbf6674450e668482456abac99a5583bbafb73f9ad469540a3a232"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a7cd62e831afe623fbb7aabbb4fe583212115b3ef38a9f6b71869ba644624a2"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f09e2ff1f17c2b51f2bc76d1cc33da96298f0a036a137f5440ab3ec5360b624f"}, - {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e38e63e6f3d1cec5a27e0afe90a085af8b6806ee208b33030e65b6516353f1a3"}, - {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0dbd8dbed2085ed23b5c04afa29d8fd2771674223135dc9bc937f3c09284d071"}, - {file = "pydantic_core-2.23.4-cp310-none-win32.whl", hash = "sha256:6531b7ca5f951d663c339002e91aaebda765ec7d61b7d1e3991051906ddde119"}, - {file = "pydantic_core-2.23.4-cp310-none-win_amd64.whl", hash = "sha256:7c9129eb40958b3d4500fa2467e6a83356b3b61bfff1b414c7361d9220f9ae8f"}, - {file = "pydantic_core-2.23.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8"}, - {file = "pydantic_core-2.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b"}, - {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0"}, - {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64"}, - {file = "pydantic_core-2.23.4-cp311-none-win32.whl", hash = "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f"}, - {file = "pydantic_core-2.23.4-cp311-none-win_amd64.whl", hash = "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3"}, - {file = "pydantic_core-2.23.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231"}, - {file = "pydantic_core-2.23.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126"}, - {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e"}, - {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24"}, - {file = "pydantic_core-2.23.4-cp312-none-win32.whl", hash = "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84"}, - {file = "pydantic_core-2.23.4-cp312-none-win_amd64.whl", hash = "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9"}, - {file = "pydantic_core-2.23.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7530e201d10d7d14abce4fb54cfe5b94a0aefc87da539d0346a484ead376c3cc"}, - {file = "pydantic_core-2.23.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df933278128ea1cd77772673c73954e53a1c95a4fdf41eef97c2b779271bd0bd"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cb3da3fd1b6a5d0279a01877713dbda118a2a4fc6f0d821a57da2e464793f05"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c6dcb030aefb668a2b7009c85b27f90e51e6a3b4d5c9bc4c57631292015b0d"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:696dd8d674d6ce621ab9d45b205df149399e4bb9aa34102c970b721554828510"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2971bb5ffe72cc0f555c13e19b23c85b654dd2a8f7ab493c262071377bfce9f6"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8394d940e5d400d04cad4f75c0598665cbb81aecefaca82ca85bd28264af7f9b"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dff76e0602ca7d4cdaacc1ac4c005e0ce0dcfe095d5b5259163a80d3a10d327"}, - {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7d32706badfe136888bdea71c0def994644e09fff0bfe47441deaed8e96fdbc6"}, - {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ed541d70698978a20eb63d8c5d72f2cc6d7079d9d90f6b50bad07826f1320f5f"}, - {file = "pydantic_core-2.23.4-cp313-none-win32.whl", hash = "sha256:3d5639516376dce1940ea36edf408c554475369f5da2abd45d44621cb616f769"}, - {file = "pydantic_core-2.23.4-cp313-none-win_amd64.whl", hash = "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5"}, - {file = "pydantic_core-2.23.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d4488a93b071c04dc20f5cecc3631fc78b9789dd72483ba15d423b5b3689b555"}, - {file = "pydantic_core-2.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:81965a16b675b35e1d09dd14df53f190f9129c0202356ed44ab2728b1c905658"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffa2ebd4c8530079140dd2d7f794a9d9a73cbb8e9d59ffe24c63436efa8f271"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:61817945f2fe7d166e75fbfb28004034b48e44878177fc54d81688e7b85a3665"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29d2c342c4bc01b88402d60189f3df065fb0dda3654744d5a165a5288a657368"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e11661ce0fd30a6790e8bcdf263b9ec5988e95e63cf901972107efc49218b13"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d18368b137c6295db49ce7218b1a9ba15c5bc254c96d7c9f9e924a9bc7825ad"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec4e55f79b1c4ffb2eecd8a0cfba9955a2588497d96851f4c8f99aa4a1d39b12"}, - {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:374a5e5049eda9e0a44c696c7ade3ff355f06b1fe0bb945ea3cac2bc336478a2"}, - {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5c364564d17da23db1106787675fc7af45f2f7b58b4173bfdd105564e132e6fb"}, - {file = "pydantic_core-2.23.4-cp38-none-win32.whl", hash = "sha256:d7a80d21d613eec45e3d41eb22f8f94ddc758a6c4720842dc74c0581f54993d6"}, - {file = "pydantic_core-2.23.4-cp38-none-win_amd64.whl", hash = "sha256:5f5ff8d839f4566a474a969508fe1c5e59c31c80d9e140566f9a37bba7b8d556"}, - {file = "pydantic_core-2.23.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a4fa4fc04dff799089689f4fd502ce7d59de529fc2f40a2c8836886c03e0175a"}, - {file = "pydantic_core-2.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a7df63886be5e270da67e0966cf4afbae86069501d35c8c1b3b6c168f42cb36"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcedcd19a557e182628afa1d553c3895a9f825b936415d0dbd3cd0bbcfd29b4b"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f54b118ce5de9ac21c363d9b3caa6c800341e8c47a508787e5868c6b79c9323"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d2f57d3e1379a9525c5ab067b27dbb8a0642fb5d454e17a9ac434f9ce523e3"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de6d1d1b9e5101508cb37ab0d972357cac5235f5c6533d1071964c47139257df"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1278e0d324f6908e872730c9102b0112477a7f7cf88b308e4fc36ce1bdb6d58c"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a6b5099eeec78827553827f4c6b8615978bb4b6a88e5d9b93eddf8bb6790f55"}, - {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e55541f756f9b3ee346b840103f32779c695a19826a4c442b7954550a0972040"}, - {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a5c7ba8ffb6d6f8f2ab08743be203654bb1aaa8c9dcb09f82ddd34eadb695605"}, - {file = "pydantic_core-2.23.4-cp39-none-win32.whl", hash = "sha256:37b0fe330e4a58d3c58b24d91d1eb102aeec675a3db4c292ec3928ecd892a9a6"}, - {file = "pydantic_core-2.23.4-cp39-none-win_amd64.whl", hash = "sha256:1498bec4c05c9c787bde9125cfdcc63a41004ff167f495063191b863399b1a29"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f455ee30a9d61d3e1a15abd5068827773d6e4dc513e795f380cdd59932c782d5"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1e90d2e3bd2c3863d48525d297cd143fe541be8bbf6f579504b9712cb6b643ec"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e203fdf807ac7e12ab59ca2bfcabb38c7cf0b33c41efeb00f8e5da1d86af480"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e08277a400de01bc72436a0ccd02bdf596631411f592ad985dcee21445bd0068"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f220b0eea5965dec25480b6333c788fb72ce5f9129e8759ef876a1d805d00801"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d06b0c8da4f16d1d1e352134427cb194a0a6e19ad5db9161bf32b2113409e728"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ba1a0996f6c2773bd83e63f18914c1de3c9dd26d55f4ac302a7efe93fb8e7433"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:9a5bce9d23aac8f0cf0836ecfc033896aa8443b501c58d0602dbfd5bd5b37753"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:78ddaaa81421a29574a682b3179d4cf9e6d405a09b99d93ddcf7e5239c742e21"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:883a91b5dd7d26492ff2f04f40fbb652de40fcc0afe07e8129e8ae779c2110eb"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88ad334a15b32a791ea935af224b9de1bf99bcd62fabf745d5f3442199d86d59"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233710f069d251feb12a56da21e14cca67994eab08362207785cf8c598e74577"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19442362866a753485ba5e4be408964644dd6a09123d9416c54cd49171f50744"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:624e278a7d29b6445e4e813af92af37820fafb6dcc55c012c834f9e26f9aaaef"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f5ef8f42bec47f21d07668a043f077d507e5bf4e668d5c6dfe6aaba89de1a5b8"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:aea443fffa9fbe3af1a9ba721a87f926fe548d32cab71d188a6ede77d0ff244e"}, - {file = "pydantic_core-2.23.4.tar.gz", hash = "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863"}, + {file = "pydantic_core-2.27.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2d367ca20b2f14095a8f4fa1210f5a7b78b8a20009ecced6b12818f455b1e9fa"}, + {file = "pydantic_core-2.27.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:491a2b73db93fab69731eaee494f320faa4e093dbed776be1a829c2eb222c34c"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7969e133a6f183be60e9f6f56bfae753585680f3b7307a8e555a948d443cc05a"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3de9961f2a346257caf0aa508a4da705467f53778e9ef6fe744c038119737ef5"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e2bb4d3e5873c37bb3dd58714d4cd0b0e6238cebc4177ac8fe878f8b3aa8e74c"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:280d219beebb0752699480fe8f1dc61ab6615c2046d76b7ab7ee38858de0a4e7"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47956ae78b6422cbd46f772f1746799cbb862de838fd8d1fbd34a82e05b0983a"}, + {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:14d4a5c49d2f009d62a2a7140d3064f686d17a5d1a268bc641954ba181880236"}, + {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:337b443af21d488716f8d0b6164de833e788aa6bd7e3a39c005febc1284f4962"}, + {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:03d0f86ea3184a12f41a2d23f7ccb79cdb5a18e06993f8a45baa8dfec746f0e9"}, + {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7041c36f5680c6e0f08d922aed302e98b3745d97fe1589db0a3eebf6624523af"}, + {file = "pydantic_core-2.27.2-cp310-cp310-win32.whl", hash = "sha256:50a68f3e3819077be2c98110c1f9dcb3817e93f267ba80a2c05bb4f8799e2ff4"}, + {file = "pydantic_core-2.27.2-cp310-cp310-win_amd64.whl", hash = "sha256:e0fd26b16394ead34a424eecf8a31a1f5137094cabe84a1bcb10fa6ba39d3d31"}, + {file = "pydantic_core-2.27.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:8e10c99ef58cfdf2a66fc15d66b16c4a04f62bca39db589ae8cba08bc55331bc"}, + {file = "pydantic_core-2.27.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:26f32e0adf166a84d0cb63be85c562ca8a6fa8de28e5f0d92250c6b7e9e2aff7"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c19d1ea0673cd13cc2f872f6c9ab42acc4e4f492a7ca9d3795ce2b112dd7e15"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e68c4446fe0810e959cdff46ab0a41ce2f2c86d227d96dc3847af0ba7def306"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9640b0059ff4f14d1f37321b94061c6db164fbe49b334b31643e0528d100d99"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40d02e7d45c9f8af700f3452f329ead92da4c5f4317ca9b896de7ce7199ea459"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c1fd185014191700554795c99b347d64f2bb637966c4cfc16998a0ca700d048"}, + {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d81d2068e1c1228a565af076598f9e7451712700b673de8f502f0334f281387d"}, + {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1a4207639fb02ec2dbb76227d7c751a20b1a6b4bc52850568e52260cae64ca3b"}, + {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:3de3ce3c9ddc8bbd88f6e0e304dea0e66d843ec9de1b0042b0911c1663ffd474"}, + {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:30c5f68ded0c36466acede341551106821043e9afaad516adfb6e8fa80a4e6a6"}, + {file = "pydantic_core-2.27.2-cp311-cp311-win32.whl", hash = "sha256:c70c26d2c99f78b125a3459f8afe1aed4d9687c24fd677c6a4436bc042e50d6c"}, + {file = "pydantic_core-2.27.2-cp311-cp311-win_amd64.whl", hash = "sha256:08e125dbdc505fa69ca7d9c499639ab6407cfa909214d500897d02afb816e7cc"}, + {file = "pydantic_core-2.27.2-cp311-cp311-win_arm64.whl", hash = "sha256:26f0d68d4b235a2bae0c3fc585c585b4ecc51382db0e3ba402a22cbc440915e4"}, + {file = "pydantic_core-2.27.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9e0c8cfefa0ef83b4da9588448b6d8d2a2bf1a53c3f1ae5fca39eb3061e2f0b0"}, + {file = "pydantic_core-2.27.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:83097677b8e3bd7eaa6775720ec8e0405f1575015a463285a92bfdfe254529ef"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:172fce187655fece0c90d90a678424b013f8fbb0ca8b036ac266749c09438cb7"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:519f29f5213271eeeeb3093f662ba2fd512b91c5f188f3bb7b27bc5973816934"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05e3a55d124407fffba0dd6b0c0cd056d10e983ceb4e5dbd10dda135c31071d6"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c3ed807c7b91de05e63930188f19e921d1fe90de6b4f5cd43ee7fcc3525cb8c"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fb4aadc0b9a0c063206846d603b92030eb6f03069151a625667f982887153e2"}, + {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28ccb213807e037460326424ceb8b5245acb88f32f3d2777427476e1b32c48c4"}, + {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:de3cd1899e2c279b140adde9357c4495ed9d47131b4a4eaff9052f23398076b3"}, + {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:220f892729375e2d736b97d0e51466252ad84c51857d4d15f5e9692f9ef12be4"}, + {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a0fcd29cd6b4e74fe8ddd2c90330fd8edf2e30cb52acda47f06dd615ae72da57"}, + {file = "pydantic_core-2.27.2-cp312-cp312-win32.whl", hash = "sha256:1e2cb691ed9834cd6a8be61228471d0a503731abfb42f82458ff27be7b2186fc"}, + {file = "pydantic_core-2.27.2-cp312-cp312-win_amd64.whl", hash = "sha256:cc3f1a99a4f4f9dd1de4fe0312c114e740b5ddead65bb4102884b384c15d8bc9"}, + {file = "pydantic_core-2.27.2-cp312-cp312-win_arm64.whl", hash = "sha256:3911ac9284cd8a1792d3cb26a2da18f3ca26c6908cc434a18f730dc0db7bfa3b"}, + {file = "pydantic_core-2.27.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7d14bd329640e63852364c306f4d23eb744e0f8193148d4044dd3dacdaacbd8b"}, + {file = "pydantic_core-2.27.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:82f91663004eb8ed30ff478d77c4d1179b3563df6cdb15c0817cd1cdaf34d154"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71b24c7d61131bb83df10cc7e687433609963a944ccf45190cfc21e0887b08c9"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa8e459d4954f608fa26116118bb67f56b93b209c39b008277ace29937453dc9"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce8918cbebc8da707ba805b7fd0b382816858728ae7fe19a942080c24e5b7cd1"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eda3f5c2a021bbc5d976107bb302e0131351c2ba54343f8a496dc8783d3d3a6a"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd8086fa684c4775c27f03f062cbb9eaa6e17f064307e86b21b9e0abc9c0f02e"}, + {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8d9b3388db186ba0c099a6d20f0604a44eabdeef1777ddd94786cdae158729e4"}, + {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7a66efda2387de898c8f38c0cf7f14fca0b51a8ef0b24bfea5849f1b3c95af27"}, + {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:18a101c168e4e092ab40dbc2503bdc0f62010e95d292b27827871dc85450d7ee"}, + {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ba5dd002f88b78a4215ed2f8ddbdf85e8513382820ba15ad5ad8955ce0ca19a1"}, + {file = "pydantic_core-2.27.2-cp313-cp313-win32.whl", hash = "sha256:1ebaf1d0481914d004a573394f4be3a7616334be70261007e47c2a6fe7e50130"}, + {file = "pydantic_core-2.27.2-cp313-cp313-win_amd64.whl", hash = "sha256:953101387ecf2f5652883208769a79e48db18c6df442568a0b5ccd8c2723abee"}, + {file = "pydantic_core-2.27.2-cp313-cp313-win_arm64.whl", hash = "sha256:ac4dbfd1691affb8f48c2c13241a2e3b60ff23247cbcf981759c768b6633cf8b"}, + {file = "pydantic_core-2.27.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d3e8d504bdd3f10835468f29008d72fc8359d95c9c415ce6e767203db6127506"}, + {file = "pydantic_core-2.27.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:521eb9b7f036c9b6187f0b47318ab0d7ca14bd87f776240b90b21c1f4f149320"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85210c4d99a0114f5a9481b44560d7d1e35e32cc5634c656bc48e590b669b145"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d716e2e30c6f140d7560ef1538953a5cd1a87264c737643d481f2779fc247fe1"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f66d89ba397d92f840f8654756196d93804278457b5fbede59598a1f9f90b228"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:669e193c1c576a58f132e3158f9dfa9662969edb1a250c54d8fa52590045f046"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdbe7629b996647b99c01b37f11170a57ae675375b14b8c13b8518b8320ced5"}, + {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d262606bf386a5ba0b0af3b97f37c83d7011439e3dc1a9298f21efb292e42f1a"}, + {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cabb9bcb7e0d97f74df8646f34fc76fbf793b7f6dc2438517d7a9e50eee4f14d"}, + {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:d2d63f1215638d28221f664596b1ccb3944f6e25dd18cd3b86b0a4c408d5ebb9"}, + {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bca101c00bff0adb45a833f8451b9105d9df18accb8743b08107d7ada14bd7da"}, + {file = "pydantic_core-2.27.2-cp38-cp38-win32.whl", hash = "sha256:f6f8e111843bbb0dee4cb6594cdc73e79b3329b526037ec242a3e49012495b3b"}, + {file = "pydantic_core-2.27.2-cp38-cp38-win_amd64.whl", hash = "sha256:fd1aea04935a508f62e0d0ef1f5ae968774a32afc306fb8545e06f5ff5cdf3ad"}, + {file = "pydantic_core-2.27.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:c10eb4f1659290b523af58fa7cffb452a61ad6ae5613404519aee4bfbf1df993"}, + {file = "pydantic_core-2.27.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ef592d4bad47296fb11f96cd7dc898b92e795032b4894dfb4076cfccd43a9308"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c61709a844acc6bf0b7dce7daae75195a10aac96a596ea1b776996414791ede4"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c5f762659e47fdb7b16956c71598292f60a03aa92f8b6351504359dbdba6cf"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c9775e339e42e79ec99c441d9730fccf07414af63eac2f0e48e08fd38a64d76"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57762139821c31847cfb2df63c12f725788bd9f04bc2fb392790959b8f70f118"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d1e85068e818c73e048fe28cfc769040bb1f475524f4745a5dc621f75ac7630"}, + {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:097830ed52fd9e427942ff3b9bc17fab52913b2f50f2880dc4a5611446606a54"}, + {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:044a50963a614ecfae59bb1eaf7ea7efc4bc62f49ed594e18fa1e5d953c40e9f"}, + {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:4e0b4220ba5b40d727c7f879eac379b822eee5d8fff418e9d3381ee45b3b0362"}, + {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e4f4bb20d75e9325cc9696c6802657b58bc1dbbe3022f32cc2b2b632c3fbb96"}, + {file = "pydantic_core-2.27.2-cp39-cp39-win32.whl", hash = "sha256:cca63613e90d001b9f2f9a9ceb276c308bfa2a43fafb75c8031c4f66039e8c6e"}, + {file = "pydantic_core-2.27.2-cp39-cp39-win_amd64.whl", hash = "sha256:77d1bca19b0f7021b3a982e6f903dcd5b2b06076def36a652e3907f596e29f67"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2bf14caea37e91198329b828eae1618c068dfb8ef17bb33287a7ad4b61ac314e"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b0cb791f5b45307caae8810c2023a184c74605ec3bcbb67d13846c28ff731ff8"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:688d3fd9fcb71f41c4c015c023d12a79d1c4c0732ec9eb35d96e3388a120dcf3"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d591580c34f4d731592f0e9fe40f9cc1b430d297eecc70b962e93c5c668f15f"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:82f986faf4e644ffc189a7f1aafc86e46ef70372bb153e7001e8afccc6e54133"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:bec317a27290e2537f922639cafd54990551725fc844249e64c523301d0822fc"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:0296abcb83a797db256b773f45773da397da75a08f5fcaef41f2044adec05f50"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0d75070718e369e452075a6017fbf187f788e17ed67a3abd47fa934d001863d9"}, + {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7e17b560be3c98a8e3aa66ce828bdebb9e9ac6ad5466fba92eb74c4c95cb1151"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c33939a82924da9ed65dab5a65d427205a73181d8098e79b6b426bdf8ad4e656"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:00bad2484fa6bda1e216e7345a798bd37c68fb2d97558edd584942aa41b7d278"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c817e2b40aba42bac6f457498dacabc568c3b7a986fc9ba7c8d9d260b71485fb"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:251136cdad0cb722e93732cb45ca5299fb56e1344a833640bf93b2803f8d1bfd"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d2088237af596f0a524d3afc39ab3b036e8adb054ee57cbb1dcf8e09da5b29cc"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d4041c0b966a84b4ae7a09832eb691a35aec90910cd2dbe7a208de59be77965b"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:8083d4e875ebe0b864ffef72a4304827015cff328a1be6e22cc850753bfb122b"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f141ee28a0ad2123b6611b6ceff018039df17f32ada8b534e6aa039545a3efb2"}, + {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7d0c8399fcc1848491f00e0314bd59fb34a9c008761bcb422a057670c3f65e35"}, + {file = "pydantic_core-2.27.2.tar.gz", hash = "sha256:eb026e5a4c1fee05726072337ff51d1efb6f59090b7da90d30ea58625b1ffb39"}, ] [package.dependencies] @@ -1166,13 +1151,13 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pyjwt" -version = "2.9.0" +version = "2.10.1" description = "JSON Web Token implementation in Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, - {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, + {file = "PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb"}, + {file = "pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953"}, ] [package.extras] @@ -1211,18 +1196,15 @@ cli = ["click (>=5.0)"] [[package]] name = "python-multipart" -version = "0.0.7" +version = "0.0.18" description = "A streaming multipart parser for Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "python_multipart-0.0.7-py3-none-any.whl", hash = "sha256:b1fef9a53b74c795e2347daac8c54b252d9e0df9c619712691c1cc8021bd3c49"}, - {file = "python_multipart-0.0.7.tar.gz", hash = "sha256:288a6c39b06596c1b988bb6794c6fbc80e6c369e35e5062637df256bee0c9af9"}, + {file = "python_multipart-0.0.18-py3-none-any.whl", hash = "sha256:efe91480f485f6a361427a541db4796f9e1591afc0fb8e7a4ba06bfbc6708996"}, + {file = "python_multipart-0.0.18.tar.gz", hash = "sha256:7a68db60c8bfb82e460637fa4750727b45af1d5e2ed215593f917f64694d34fe"}, ] -[package.extras] -dev = ["atomicwrites (==1.2.1)", "attrs (==19.2.0)", "coverage (==6.5.0)", "hatch", "invoke (==2.2.0)", "more-itertools (==4.3.0)", "pbr (==4.3.0)", "pluggy (==1.0.0)", "py (==1.11.0)", "pytest (==7.2.0)", "pytest-cov (==4.0.0)", "pytest-timeout (==2.1.0)", "pyyaml (==5.1)"] - [[package]] name = "pytz" version = "2024.2" @@ -1257,40 +1239,40 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "ruff" -version = "0.7.1" +version = "0.7.4" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.7.1-py3-none-linux_armv6l.whl", hash = "sha256:cb1bc5ed9403daa7da05475d615739cc0212e861b7306f314379d958592aaa89"}, - {file = "ruff-0.7.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:27c1c52a8d199a257ff1e5582d078eab7145129aa02721815ca8fa4f9612dc35"}, - {file = "ruff-0.7.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:588a34e1ef2ea55b4ddfec26bbe76bc866e92523d8c6cdec5e8aceefeff02d99"}, - {file = "ruff-0.7.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94fc32f9cdf72dc75c451e5f072758b118ab8100727168a3df58502b43a599ca"}, - {file = "ruff-0.7.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:985818742b833bffa543a84d1cc11b5e6871de1b4e0ac3060a59a2bae3969250"}, - {file = "ruff-0.7.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32f1e8a192e261366c702c5fb2ece9f68d26625f198a25c408861c16dc2dea9c"}, - {file = "ruff-0.7.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:699085bf05819588551b11751eff33e9ca58b1b86a6843e1b082a7de40da1565"}, - {file = "ruff-0.7.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:344cc2b0814047dc8c3a8ff2cd1f3d808bb23c6658db830d25147339d9bf9ea7"}, - {file = "ruff-0.7.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4316bbf69d5a859cc937890c7ac7a6551252b6a01b1d2c97e8fc96e45a7c8b4a"}, - {file = "ruff-0.7.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79d3af9dca4c56043e738a4d6dd1e9444b6d6c10598ac52d146e331eb155a8ad"}, - {file = "ruff-0.7.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c5c121b46abde94a505175524e51891f829414e093cd8326d6e741ecfc0a9112"}, - {file = "ruff-0.7.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8422104078324ea250886954e48f1373a8fe7de59283d747c3a7eca050b4e378"}, - {file = "ruff-0.7.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:56aad830af8a9db644e80098fe4984a948e2b6fc2e73891538f43bbe478461b8"}, - {file = "ruff-0.7.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:658304f02f68d3a83c998ad8bf91f9b4f53e93e5412b8f2388359d55869727fd"}, - {file = "ruff-0.7.1-py3-none-win32.whl", hash = "sha256:b517a2011333eb7ce2d402652ecaa0ac1a30c114fbbd55c6b8ee466a7f600ee9"}, - {file = "ruff-0.7.1-py3-none-win_amd64.whl", hash = "sha256:f38c41fcde1728736b4eb2b18850f6d1e3eedd9678c914dede554a70d5241307"}, - {file = "ruff-0.7.1-py3-none-win_arm64.whl", hash = "sha256:19aa200ec824c0f36d0c9114c8ec0087082021732979a359d6f3c390a6ff2a37"}, - {file = "ruff-0.7.1.tar.gz", hash = "sha256:9d8a41d4aa2dad1575adb98a82870cf5db5f76b2938cf2206c22c940034a36f4"}, + {file = "ruff-0.7.4-py3-none-linux_armv6l.whl", hash = "sha256:a4919925e7684a3f18e18243cd6bea7cfb8e968a6eaa8437971f681b7ec51478"}, + {file = "ruff-0.7.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:cfb365c135b830778dda8c04fb7d4280ed0b984e1aec27f574445231e20d6c63"}, + {file = "ruff-0.7.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:63a569b36bc66fbadec5beaa539dd81e0527cb258b94e29e0531ce41bacc1f20"}, + {file = "ruff-0.7.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d06218747d361d06fd2fdac734e7fa92df36df93035db3dc2ad7aa9852cb109"}, + {file = "ruff-0.7.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e0cea28d0944f74ebc33e9f934238f15c758841f9f5edd180b5315c203293452"}, + {file = "ruff-0.7.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80094ecd4793c68b2571b128f91754d60f692d64bc0d7272ec9197fdd09bf9ea"}, + {file = "ruff-0.7.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:997512325c6620d1c4c2b15db49ef59543ef9cd0f4aa8065ec2ae5103cedc7e7"}, + {file = "ruff-0.7.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00b4cf3a6b5fad6d1a66e7574d78956bbd09abfd6c8a997798f01f5da3d46a05"}, + {file = "ruff-0.7.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7dbdc7d8274e1422722933d1edddfdc65b4336abf0b16dfcb9dedd6e6a517d06"}, + {file = "ruff-0.7.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e92dfb5f00eaedb1501b2f906ccabfd67b2355bdf117fea9719fc99ac2145bc"}, + {file = "ruff-0.7.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3bd726099f277d735dc38900b6a8d6cf070f80828877941983a57bca1cd92172"}, + {file = "ruff-0.7.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:2e32829c429dd081ee5ba39aef436603e5b22335c3d3fff013cd585806a6486a"}, + {file = "ruff-0.7.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:662a63b4971807623f6f90c1fb664613f67cc182dc4d991471c23c541fee62dd"}, + {file = "ruff-0.7.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:876f5e09eaae3eb76814c1d3b68879891d6fde4824c015d48e7a7da4cf066a3a"}, + {file = "ruff-0.7.4-py3-none-win32.whl", hash = "sha256:75c53f54904be42dd52a548728a5b572344b50d9b2873d13a3f8c5e3b91f5cac"}, + {file = "ruff-0.7.4-py3-none-win_amd64.whl", hash = "sha256:745775c7b39f914238ed1f1b0bebed0b9155a17cd8bc0b08d3c87e4703b990d6"}, + {file = "ruff-0.7.4-py3-none-win_arm64.whl", hash = "sha256:11bff065102c3ae9d3ea4dc9ecdfe5a5171349cdd0787c1fc64761212fc9cf1f"}, + {file = "ruff-0.7.4.tar.gz", hash = "sha256:cd12e35031f5af6b9b93715d8c4f40360070b2041f81273d0527683d5708fce2"}, ] [[package]] name = "six" -version = "1.16.0" +version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] [[package]] @@ -1306,13 +1288,13 @@ files = [ [[package]] name = "starlette" -version = "0.41.2" +version = "0.41.3" description = "The little ASGI library that shines." optional = false python-versions = ">=3.8" files = [ - {file = "starlette-0.41.2-py3-none-any.whl", hash = "sha256:fbc189474b4731cf30fcef52f18a8d070e3f3b46c6a04c97579e85e6ffca942d"}, - {file = "starlette-0.41.2.tar.gz", hash = "sha256:9834fd799d1a87fd346deb76158668cfa0b0d56f85caefe8268e2d97c3468b62"}, + {file = "starlette-0.41.3-py3-none-any.whl", hash = "sha256:44cedb2b7c77a9de33a8b74b2b90e9f50d11fcf25d8270ea525ad71a25374ff7"}, + {file = "starlette-0.41.3.tar.gz", hash = "sha256:0e4ab3d16522a255be6b28260b938eae2482f98ce5cc934cb08dce8dc3ba5835"}, ] [package.dependencies] @@ -1346,13 +1328,13 @@ files = [ [[package]] name = "urllib3" -version = "2.2.3" +version = "2.3.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, - {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, + {file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"}, + {file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"}, ] [package.extras] @@ -1374,13 +1356,13 @@ files = [ [[package]] name = "uvicorn" -version = "0.32.0" +version = "0.32.1" description = "The lightning-fast ASGI server." optional = false python-versions = ">=3.8" files = [ - {file = "uvicorn-0.32.0-py3-none-any.whl", hash = "sha256:60b8f3a5ac027dcd31448f411ced12b5ef452c646f76f02f8cc3f25d8d26fd82"}, - {file = "uvicorn-0.32.0.tar.gz", hash = "sha256:f78b36b143c16f54ccdb8190d0a26b5f1901fe5a3c777e1ab29f26391af8551e"}, + {file = "uvicorn-0.32.1-py3-none-any.whl", hash = "sha256:82ad92fd58da0d12af7482ecdb5f2470a04c9c9a53ced65b9bbb4a205377602e"}, + {file = "uvicorn-0.32.1.tar.gz", hash = "sha256:ee9519c246a72b1c084cea8d3b44ed6026e78a4a309cbedae9c37e4cb9fbb175"}, ] [package.dependencies] @@ -1389,7 +1371,7 @@ h11 = ">=0.8" typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} [package.extras] -standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] +standard = ["colorama (>=0.4)", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] [[package]] name = "webencodings" @@ -1499,93 +1481,93 @@ files = [ [[package]] name = "yarl" -version = "1.17.1" +version = "1.18.3" description = "Yet another URL library" optional = false python-versions = ">=3.9" files = [ - {file = "yarl-1.17.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1794853124e2f663f0ea54efb0340b457f08d40a1cef78edfa086576179c91"}, - {file = "yarl-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fbea1751729afe607d84acfd01efd95e3b31db148a181a441984ce9b3d3469da"}, - {file = "yarl-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8ee427208c675f1b6e344a1f89376a9613fc30b52646a04ac0c1f6587c7e46ec"}, - {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b74ff4767d3ef47ffe0cd1d89379dc4d828d4873e5528976ced3b44fe5b0a21"}, - {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:62a91aefff3d11bf60e5956d340eb507a983a7ec802b19072bb989ce120cd948"}, - {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:846dd2e1243407133d3195d2d7e4ceefcaa5f5bf7278f0a9bda00967e6326b04"}, - {file = "yarl-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e844be8d536afa129366d9af76ed7cb8dfefec99f5f1c9e4f8ae542279a6dc3"}, - {file = "yarl-1.17.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc7c92c1baa629cb03ecb0c3d12564f172218fb1739f54bf5f3881844daadc6d"}, - {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ae3476e934b9d714aa8000d2e4c01eb2590eee10b9d8cd03e7983ad65dfbfcba"}, - {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c7e177c619342e407415d4f35dec63d2d134d951e24b5166afcdfd1362828e17"}, - {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:64cc6e97f14cf8a275d79c5002281f3040c12e2e4220623b5759ea7f9868d6a5"}, - {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:84c063af19ef5130084db70ada40ce63a84f6c1ef4d3dbc34e5e8c4febb20822"}, - {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:482c122b72e3c5ec98f11457aeb436ae4aecca75de19b3d1de7cf88bc40db82f"}, - {file = "yarl-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:380e6c38ef692b8fd5a0f6d1fa8774d81ebc08cfbd624b1bca62a4d4af2f9931"}, - {file = "yarl-1.17.1-cp310-cp310-win32.whl", hash = "sha256:16bca6678a83657dd48df84b51bd56a6c6bd401853aef6d09dc2506a78484c7b"}, - {file = "yarl-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:561c87fea99545ef7d692403c110b2f99dced6dff93056d6e04384ad3bc46243"}, - {file = "yarl-1.17.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cbad927ea8ed814622305d842c93412cb47bd39a496ed0f96bfd42b922b4a217"}, - {file = "yarl-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fca4b4307ebe9c3ec77a084da3a9d1999d164693d16492ca2b64594340999988"}, - {file = "yarl-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff5c6771c7e3511a06555afa317879b7db8d640137ba55d6ab0d0c50425cab75"}, - {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b29beab10211a746f9846baa39275e80034e065460d99eb51e45c9a9495bcca"}, - {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a52a1ffdd824fb1835272e125385c32fd8b17fbdefeedcb4d543cc23b332d74"}, - {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:58c8e9620eb82a189c6c40cb6b59b4e35b2ee68b1f2afa6597732a2b467d7e8f"}, - {file = "yarl-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d216e5d9b8749563c7f2c6f7a0831057ec844c68b4c11cb10fc62d4fd373c26d"}, - {file = "yarl-1.17.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:881764d610e3269964fc4bb3c19bb6fce55422828e152b885609ec176b41cf11"}, - {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8c79e9d7e3d8a32d4824250a9c6401194fb4c2ad9a0cec8f6a96e09a582c2cc0"}, - {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:299f11b44d8d3a588234adbe01112126010bd96d9139c3ba7b3badd9829261c3"}, - {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:cc7d768260f4ba4ea01741c1b5fe3d3a6c70eb91c87f4c8761bbcce5181beafe"}, - {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:de599af166970d6a61accde358ec9ded821234cbbc8c6413acfec06056b8e860"}, - {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2b24ec55fad43e476905eceaf14f41f6478780b870eda5d08b4d6de9a60b65b4"}, - {file = "yarl-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9fb815155aac6bfa8d86184079652c9715c812d506b22cfa369196ef4e99d1b4"}, - {file = "yarl-1.17.1-cp311-cp311-win32.whl", hash = "sha256:7615058aabad54416ddac99ade09a5510cf77039a3b903e94e8922f25ed203d7"}, - {file = "yarl-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:14bc88baa44e1f84164a392827b5defb4fa8e56b93fecac3d15315e7c8e5d8b3"}, - {file = "yarl-1.17.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:327828786da2006085a4d1feb2594de6f6d26f8af48b81eb1ae950c788d97f61"}, - {file = "yarl-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cc353841428d56b683a123a813e6a686e07026d6b1c5757970a877195f880c2d"}, - {file = "yarl-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c73df5b6e8fabe2ddb74876fb82d9dd44cbace0ca12e8861ce9155ad3c886139"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdff5e0995522706c53078f531fb586f56de9c4c81c243865dd5c66c132c3b5"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:06157fb3c58f2736a5e47c8fcbe1afc8b5de6fb28b14d25574af9e62150fcaac"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1654ec814b18be1af2c857aa9000de7a601400bd4c9ca24629b18486c2e35463"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f6595c852ca544aaeeb32d357e62c9c780eac69dcd34e40cae7b55bc4fb1147"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:459e81c2fb920b5f5df744262d1498ec2c8081acdcfe18181da44c50f51312f7"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e48cdb8226644e2fbd0bdb0a0f87906a3db07087f4de77a1b1b1ccfd9e93685"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d9b6b28a57feb51605d6ae5e61a9044a31742db557a3b851a74c13bc61de5172"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e594b22688d5747b06e957f1ef822060cb5cb35b493066e33ceac0cf882188b7"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5f236cb5999ccd23a0ab1bd219cfe0ee3e1c1b65aaf6dd3320e972f7ec3a39da"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a2a64e62c7a0edd07c1c917b0586655f3362d2c2d37d474db1a509efb96fea1c"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d0eea830b591dbc68e030c86a9569826145df485b2b4554874b07fea1275a199"}, - {file = "yarl-1.17.1-cp312-cp312-win32.whl", hash = "sha256:46ddf6e0b975cd680eb83318aa1d321cb2bf8d288d50f1754526230fcf59ba96"}, - {file = "yarl-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:117ed8b3732528a1e41af3aa6d4e08483c2f0f2e3d3d7dca7cf538b3516d93df"}, - {file = "yarl-1.17.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5d1d42556b063d579cae59e37a38c61f4402b47d70c29f0ef15cee1acaa64488"}, - {file = "yarl-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c0167540094838ee9093ef6cc2c69d0074bbf84a432b4995835e8e5a0d984374"}, - {file = "yarl-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2f0a6423295a0d282d00e8701fe763eeefba8037e984ad5de44aa349002562ac"}, - {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5b078134f48552c4d9527db2f7da0b5359abd49393cdf9794017baec7506170"}, - {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d401f07261dc5aa36c2e4efc308548f6ae943bfff20fcadb0a07517a26b196d8"}, - {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5f1ac7359e17efe0b6e5fec21de34145caef22b260e978336f325d5c84e6938"}, - {file = "yarl-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f63d176a81555984e91f2c84c2a574a61cab7111cc907e176f0f01538e9ff6e"}, - {file = "yarl-1.17.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e275792097c9f7e80741c36de3b61917aebecc08a67ae62899b074566ff8556"}, - {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:81713b70bea5c1386dc2f32a8f0dab4148a2928c7495c808c541ee0aae614d67"}, - {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:aa46dce75078fceaf7cecac5817422febb4355fbdda440db55206e3bd288cfb8"}, - {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1ce36ded585f45b1e9bb36d0ae94765c6608b43bd2e7f5f88079f7a85c61a4d3"}, - {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2d374d70fdc36f5863b84e54775452f68639bc862918602d028f89310a034ab0"}, - {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2d9f0606baaec5dd54cb99667fcf85183a7477f3766fbddbe3f385e7fc253299"}, - {file = "yarl-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b0341e6d9a0c0e3cdc65857ef518bb05b410dbd70d749a0d33ac0f39e81a4258"}, - {file = "yarl-1.17.1-cp313-cp313-win32.whl", hash = "sha256:2e7ba4c9377e48fb7b20dedbd473cbcbc13e72e1826917c185157a137dac9df2"}, - {file = "yarl-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:949681f68e0e3c25377462be4b658500e85ca24323d9619fdc41f68d46a1ffda"}, - {file = "yarl-1.17.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8994b29c462de9a8fce2d591028b986dbbe1b32f3ad600b2d3e1c482c93abad6"}, - {file = "yarl-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f9cbfbc5faca235fbdf531b93aa0f9f005ec7d267d9d738761a4d42b744ea159"}, - {file = "yarl-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b40d1bf6e6f74f7c0a567a9e5e778bbd4699d1d3d2c0fe46f4b717eef9e96b95"}, - {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5efe0661b9fcd6246f27957f6ae1c0eb29bc60552820f01e970b4996e016004"}, - {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5c4804e4039f487e942c13381e6c27b4b4e66066d94ef1fae3f6ba8b953f383"}, - {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5d6a6c9602fd4598fa07e0389e19fe199ae96449008d8304bf5d47cb745462e"}, - {file = "yarl-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f4c9156c4d1eb490fe374fb294deeb7bc7eaccda50e23775b2354b6a6739934"}, - {file = "yarl-1.17.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6324274b4e0e2fa1b3eccb25997b1c9ed134ff61d296448ab8269f5ac068c4c"}, - {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d8a8b74d843c2638f3864a17d97a4acda58e40d3e44b6303b8cc3d3c44ae2d29"}, - {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:7fac95714b09da9278a0b52e492466f773cfe37651cf467a83a1b659be24bf71"}, - {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:c180ac742a083e109c1a18151f4dd8675f32679985a1c750d2ff806796165b55"}, - {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:578d00c9b7fccfa1745a44f4eddfdc99d723d157dad26764538fbdda37209857"}, - {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:1a3b91c44efa29e6c8ef8a9a2b583347998e2ba52c5d8280dbd5919c02dfc3b5"}, - {file = "yarl-1.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a7ac5b4984c468ce4f4a553df281450df0a34aefae02e58d77a0847be8d1e11f"}, - {file = "yarl-1.17.1-cp39-cp39-win32.whl", hash = "sha256:7294e38f9aa2e9f05f765b28ffdc5d81378508ce6dadbe93f6d464a8c9594473"}, - {file = "yarl-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:eb6dce402734575e1a8cc0bb1509afca508a400a57ce13d306ea2c663bad1138"}, - {file = "yarl-1.17.1-py3-none-any.whl", hash = "sha256:f1790a4b1e8e8e028c391175433b9c8122c39b46e1663228158e61e6f915bf06"}, - {file = "yarl-1.17.1.tar.gz", hash = "sha256:067a63fcfda82da6b198fa73079b1ca40b7c9b7994995b6ee38acda728b64d47"}, + {file = "yarl-1.18.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7df647e8edd71f000a5208fe6ff8c382a1de8edfbccdbbfe649d263de07d8c34"}, + {file = "yarl-1.18.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c69697d3adff5aa4f874b19c0e4ed65180ceed6318ec856ebc423aa5850d84f7"}, + {file = "yarl-1.18.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:602d98f2c2d929f8e697ed274fbadc09902c4025c5a9963bf4e9edfc3ab6f7ed"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c654d5207c78e0bd6d749f6dae1dcbbfde3403ad3a4b11f3c5544d9906969dde"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5094d9206c64181d0f6e76ebd8fb2f8fe274950a63890ee9e0ebfd58bf9d787b"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35098b24e0327fc4ebdc8ffe336cee0a87a700c24ffed13161af80124b7dc8e5"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3236da9272872443f81fedc389bace88408f64f89f75d1bdb2256069a8730ccc"}, + {file = "yarl-1.18.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2c08cc9b16f4f4bc522771d96734c7901e7ebef70c6c5c35dd0f10845270bcd"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:80316a8bd5109320d38eef8833ccf5f89608c9107d02d2a7f985f98ed6876990"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c1e1cc06da1491e6734f0ea1e6294ce00792193c463350626571c287c9a704db"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e3b9fd71836999aad54084906f8663dffcd2a7fb5cdafd6c37713b2e72be1760"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:757e81cae69244257d125ff31663249b3013b5dc0a8520d73694aed497fb195b"}, + {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b1771de9944d875f1b98a745bc547e684b863abf8f8287da8466cf470ef52690"}, + {file = "yarl-1.18.3-cp310-cp310-win32.whl", hash = "sha256:8874027a53e3aea659a6d62751800cf6e63314c160fd607489ba5c2edd753cf6"}, + {file = "yarl-1.18.3-cp310-cp310-win_amd64.whl", hash = "sha256:93b2e109287f93db79210f86deb6b9bbb81ac32fc97236b16f7433db7fc437d8"}, + {file = "yarl-1.18.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8503ad47387b8ebd39cbbbdf0bf113e17330ffd339ba1144074da24c545f0069"}, + {file = "yarl-1.18.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:02ddb6756f8f4517a2d5e99d8b2f272488e18dd0bfbc802f31c16c6c20f22193"}, + {file = "yarl-1.18.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:67a283dd2882ac98cc6318384f565bffc751ab564605959df4752d42483ad889"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d980e0325b6eddc81331d3f4551e2a333999fb176fd153e075c6d1c2530aa8a8"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b643562c12680b01e17239be267bc306bbc6aac1f34f6444d1bded0c5ce438ca"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c017a3b6df3a1bd45b9fa49a0f54005e53fbcad16633870104b66fa1a30a29d8"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75674776d96d7b851b6498f17824ba17849d790a44d282929c42dbb77d4f17ae"}, + {file = "yarl-1.18.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ccaa3a4b521b780a7e771cc336a2dba389a0861592bbce09a476190bb0c8b4b3"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2d06d3005e668744e11ed80812e61efd77d70bb7f03e33c1598c301eea20efbb"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:9d41beda9dc97ca9ab0b9888cb71f7539124bc05df02c0cff6e5acc5a19dcc6e"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ba23302c0c61a9999784e73809427c9dbedd79f66a13d84ad1b1943802eaaf59"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6748dbf9bfa5ba1afcc7556b71cda0d7ce5f24768043a02a58846e4a443d808d"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0b0cad37311123211dc91eadcb322ef4d4a66008d3e1bdc404808992260e1a0e"}, + {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0fb2171a4486bb075316ee754c6d8382ea6eb8b399d4ec62fde2b591f879778a"}, + {file = "yarl-1.18.3-cp311-cp311-win32.whl", hash = "sha256:61b1a825a13bef4a5f10b1885245377d3cd0bf87cba068e1d9a88c2ae36880e1"}, + {file = "yarl-1.18.3-cp311-cp311-win_amd64.whl", hash = "sha256:b9d60031cf568c627d028239693fd718025719c02c9f55df0a53e587aab951b5"}, + {file = "yarl-1.18.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1dd4bdd05407ced96fed3d7f25dbbf88d2ffb045a0db60dbc247f5b3c5c25d50"}, + {file = "yarl-1.18.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7c33dd1931a95e5d9a772d0ac5e44cac8957eaf58e3c8da8c1414de7dd27c576"}, + {file = "yarl-1.18.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:25b411eddcfd56a2f0cd6a384e9f4f7aa3efee14b188de13048c25b5e91f1640"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:436c4fc0a4d66b2badc6c5fc5ef4e47bb10e4fd9bf0c79524ac719a01f3607c2"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e35ef8683211db69ffe129a25d5634319a677570ab6b2eba4afa860f54eeaf75"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:84b2deecba4a3f1a398df819151eb72d29bfeb3b69abb145a00ddc8d30094512"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba"}, + {file = "yarl-1.18.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0e883008013c0e4aef84dcfe2a0b172c4d23c2669412cf5b3371003941f72bb"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5a3f356548e34a70b0172d8890006c37be92995f62d95a07b4a42e90fba54272"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:ccd17349166b1bee6e529b4add61727d3f55edb7babbe4069b5764c9587a8cc6"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b958ddd075ddba5b09bb0be8a6d9906d2ce933aee81100db289badbeb966f54e"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c7d79f7d9aabd6011004e33b22bc13056a3e3fb54794d138af57f5ee9d9032cb"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4891ed92157e5430874dad17b15eb1fda57627710756c27422200c52d8a4e393"}, + {file = "yarl-1.18.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ce1af883b94304f493698b00d0f006d56aea98aeb49d75ec7d98cd4a777e9285"}, + {file = "yarl-1.18.3-cp312-cp312-win32.whl", hash = "sha256:f91c4803173928a25e1a55b943c81f55b8872f0018be83e3ad4938adffb77dd2"}, + {file = "yarl-1.18.3-cp312-cp312-win_amd64.whl", hash = "sha256:7e2ee16578af3b52ac2f334c3b1f92262f47e02cc6193c598502bd46f5cd1477"}, + {file = "yarl-1.18.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:90adb47ad432332d4f0bc28f83a5963f426ce9a1a8809f5e584e704b82685dcb"}, + {file = "yarl-1.18.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:913829534200eb0f789d45349e55203a091f45c37a2674678744ae52fae23efa"}, + {file = "yarl-1.18.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ef9f7768395923c3039055c14334ba4d926f3baf7b776c923c93d80195624782"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88a19f62ff30117e706ebc9090b8ecc79aeb77d0b1f5ec10d2d27a12bc9f66d0"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e17c9361d46a4d5addf777c6dd5eab0715a7684c2f11b88c67ac37edfba6c482"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a74a13a4c857a84a845505fd2d68e54826a2cd01935a96efb1e9d86c728e186"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41f7ce59d6ee7741af71d82020346af364949314ed3d87553763a2df1829cc58"}, + {file = "yarl-1.18.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f52a265001d830bc425f82ca9eabda94a64a4d753b07d623a9f2863fde532b53"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:82123d0c954dc58db301f5021a01854a85bf1f3bb7d12ae0c01afc414a882ca2"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2ec9bbba33b2d00999af4631a3397d1fd78290c48e2a3e52d8dd72db3a067ac8"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:fbd6748e8ab9b41171bb95c6142faf068f5ef1511935a0aa07025438dd9a9bc1"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:877d209b6aebeb5b16c42cbb377f5f94d9e556626b1bfff66d7b0d115be88d0a"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b464c4ab4bfcb41e3bfd3f1c26600d038376c2de3297760dfe064d2cb7ea8e10"}, + {file = "yarl-1.18.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8d39d351e7faf01483cc7ff7c0213c412e38e5a340238826be7e0e4da450fdc8"}, + {file = "yarl-1.18.3-cp313-cp313-win32.whl", hash = "sha256:61ee62ead9b68b9123ec24bc866cbef297dd266175d53296e2db5e7f797f902d"}, + {file = "yarl-1.18.3-cp313-cp313-win_amd64.whl", hash = "sha256:578e281c393af575879990861823ef19d66e2b1d0098414855dd367e234f5b3c"}, + {file = "yarl-1.18.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:61e5e68cb65ac8f547f6b5ef933f510134a6bf31bb178be428994b0cb46c2a04"}, + {file = "yarl-1.18.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719"}, + {file = "yarl-1.18.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a440a2a624683108a1b454705ecd7afc1c3438a08e890a1513d468671d90a04e"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09c7907c8548bcd6ab860e5f513e727c53b4a714f459b084f6580b49fa1b9cee"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4f6450109834af88cb4cc5ecddfc5380ebb9c228695afc11915a0bf82116789"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9ca04806f3be0ac6d558fffc2fdf8fcef767e0489d2684a21912cc4ed0cd1b8"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77a6e85b90a7641d2e07184df5557132a337f136250caafc9ccaa4a2a998ca2c"}, + {file = "yarl-1.18.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6333c5a377c8e2f5fae35e7b8f145c617b02c939d04110c76f29ee3676b5f9a5"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0b3c92fa08759dbf12b3a59579a4096ba9af8dd344d9a813fc7f5070d86bbab1"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:4ac515b860c36becb81bb84b667466885096b5fc85596948548b667da3bf9f24"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:045b8482ce9483ada4f3f23b3774f4e1bf4f23a2d5c912ed5170f68efb053318"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:a4bb030cf46a434ec0225bddbebd4b89e6471814ca851abb8696170adb163985"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:54d6921f07555713b9300bee9c50fb46e57e2e639027089b1d795ecd9f7fa910"}, + {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1d407181cfa6e70077df3377938c08012d18893f9f20e92f7d2f314a437c30b1"}, + {file = "yarl-1.18.3-cp39-cp39-win32.whl", hash = "sha256:ac36703a585e0929b032fbaab0707b75dc12703766d0b53486eabd5139ebadd5"}, + {file = "yarl-1.18.3-cp39-cp39-win_amd64.whl", hash = "sha256:ba87babd629f8af77f557b61e49e7c7cac36f22f871156b91e10a6e9d4f829e9"}, + {file = "yarl-1.18.3-py3-none-any.whl", hash = "sha256:b57f4f58099328dfb26c6a771d09fb20dbbae81d20cfb66141251ea063bd101b"}, + {file = "yarl-1.18.3.tar.gz", hash = "sha256:ac1801c45cbf77b6c99242eeff4fffb5e4e73a800b5c4ad4fc0be5def634d2e1"}, ] [package.dependencies] @@ -1595,13 +1577,13 @@ propcache = ">=0.2.0" [[package]] name = "zipp" -version = "3.20.2" +version = "3.21.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, - {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, + {file = "zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931"}, + {file = "zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4"}, ] [package.extras] @@ -1615,4 +1597,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "891d55f4c0fd00c315fb2a7f25dfcbfbe0991a9861b1aa66f5378eb82f614063" +content-hash = "c0fe280144a714cfc920fb25cc2db1ece3c888cd973a7ef76c124e6ab90a8c23" diff --git a/openbb_platform/extensions/websockets/pyproject.toml b/openbb_platform/extensions/websockets/pyproject.toml index c53ca99fd173..8187d0f28abf 100644 --- a/openbb_platform/extensions/websockets/pyproject.toml +++ b/openbb_platform/extensions/websockets/pyproject.toml @@ -9,7 +9,7 @@ packages = [{ include = "openbb_websockets" }] [tool.poetry.dependencies] python = "^3.9" -openbb-core = "^1.3.5" +openbb-core = "^1.3.7" aiosqlite = "^0.20.0" [build-system] diff --git a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py index fa305da597eb..19e65b7fa983 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py +++ b/openbb_platform/providers/fmp/openbb_fmp/models/websocket_connection.py @@ -8,7 +8,7 @@ from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.abstract.fetcher import Fetcher from openbb_core.provider.utils.websockets.client import WebSocketClient -from openbb_websockets.models import ( +from openbb_core.provider.utils.websockets.models import ( WebSocketConnection, WebSocketData, WebSocketQueryParams, @@ -160,12 +160,16 @@ async def aextract_data( limit=query.limit, results_file=query.results_file, table_name=query.table_name, - save_results=query.save_results, + save_database=query.save_database, data_model=FmpWebSocketData, sleep_time=query.sleep_time, broadcast_host=query.broadcast_host, broadcast_port=query.broadcast_port, auth_token=query.auth_token, + prune_interval=query.prune_interval, + export_directory=query.export_directory, + export_interval=query.export_interval, + compress_export=query.compress_export, **kwargs, ) diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py index 44a0240f70e7..98a3ff2febf9 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -1,13 +1,12 @@ """FMP WebSocket client.""" import asyncio -import json import signal import sys +import orjson as json import websockets -import websockets.exceptions -from openbb_core.provider.utils.websockets.database import Database +from openbb_core.provider.utils.websockets.database import Database, DatabaseWriter from openbb_core.provider.utils.websockets.helpers import ( get_logger, handle_termination_signal, @@ -26,18 +25,26 @@ logger = get_logger("openbb.websocket.fmp") kwargs = parse_kwargs() -queue = MessageQueue() +input_queue = MessageQueue() command_queue = MessageQueue() +database_queue = MessageQueue() CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) URL = URL_MAP.get(kwargs.pop("asset_type"), None) + if not URL: raise ValueError("Invalid asset type provided.") -DATABASE = Database( - results_file=kwargs["results_file"], - table_name=kwargs["table_name"], - limit=kwargs.get("limit"), - logger=logger, +if not kwargs.get("api_key"): + raise ValueError("API key is required.") + +DATABASE = DatabaseWriter( + database=Database( + results_file=kwargs.get("results_file"), + table_name=kwargs.get("table_name"), + limit=kwargs.get("limit"), + logger=logger, + ), + queue=database_queue, ) @@ -64,7 +71,7 @@ async def login(websocket): msg = message.get("message") logger.info("PROVIDER INFO: %s", msg) except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" + msg = f"PROVIDER ERROR: {e.__class__.__name__ if hasattr(e, '__class__') else e}: {e.args}" logger.error(msg) sys.exit(1) @@ -80,9 +87,8 @@ async def subscribe(websocket, symbol, event): } try: await websocket.send(json.dumps(subscribe_event)) - await asyncio.sleep(1) except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" + msg = f"PROVIDER ERROR: {e.__class__.__name__ if hasattr(e, '__class__') else e}: {e}" logger.error(msg) @@ -139,63 +145,73 @@ async def process_message(message): except ValidationError: raise e from e if result: - await DATABASE._write_to_db(result) # pylint: disable=protected-access + await database_queue.enqueue(result) async def connect_and_stream(): """Connect to the WebSocket and stream data to file.""" handler_task = asyncio.create_task( - queue.process_queue(lambda message: process_message(message)) + input_queue.process_queue(lambda message: process_message(message)) ) stdin_task = asyncio.create_task(read_stdin_and_queue_commands()) - try: - websocket = await websockets.connect(URL, **CONNECT_KWARGS) - await login(websocket) - await subscribe(websocket, kwargs["symbol"], "subscribe") + await DATABASE.start_writer() - while True: - ws_task = asyncio.create_task(websocket.recv()) - cmd_task = asyncio.create_task(process_stdin_queue(websocket)) + disconnects = 0 - done, pending = await asyncio.wait( - [ws_task, cmd_task], return_when=asyncio.FIRST_COMPLETED - ) - for task in pending: - task.cancel() - - for task in done: - if task == cmd_task: - await cmd_task - elif task == ws_task: - message = task.result() - await asyncio.shield(queue.enqueue(json.loads(message))) - - except websockets.ConnectionClosed as e: - msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" - logger.info(msg) - # Attempt to reopen the connection - logger.info("PROVIDER INFO: Attempting to reconnect after five seconds.") - await asyncio.sleep(5) - await connect_and_stream() - - except websockets.WebSocketException as e: - logger.error(e) - sys.exit(1) + async for websocket in websockets.connect(URL, **CONNECT_KWARGS): + try: + await login(websocket) - except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e}" - logger.error(msg) - sys.exit(1) + await subscribe(websocket, kwargs["symbol"], "subscribe") - finally: - await websocket.close() - handler_task.cancel() - stdin_task.cancel() - await asyncio.gather(handler_task, stdin_task, return_exceptions=True) - sys.exit(0) + while True: + ws_task = asyncio.create_task(websocket.recv()) + cmd_task = asyncio.create_task(process_stdin_queue(websocket)) + + done, pending = await asyncio.wait( + [ws_task, cmd_task], return_when=asyncio.FIRST_COMPLETED + ) + for task in pending: + task.cancel() + + for task in done: + if task == cmd_task: + await cmd_task + elif task == ws_task: + message = task.result() + await input_queue.enqueue(json.loads(message)) + + except websockets.ConnectionClosed as e: + msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" + logger.info(msg) + # Attempt to reopen the connection + logger.info("PROVIDER INFO: Attempting to reconnect...") + await asyncio.sleep(2) + disconnects += 1 + if disconnects > 5: + logger.error("PROVIDER ERROR: Too many disconnects. Exiting...") + sys.exit(1) + continue + + except websockets.WebSocketException as e: + logger.error(e) + sys.exit(1) + + except Exception as e: # pylint: disable=broad-except + msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e}" + logger.error(msg) + sys.exit(1) + + finally: + await websocket.close() + handler_task.cancel() + stdin_task.cancel() + await asyncio.gather(handler_task, stdin_task, return_exceptions=True) + await DATABASE.stop_writer() + sys.exit(0) if __name__ == "__main__": @@ -216,8 +232,10 @@ async def connect_and_stream(): logger.error("PROVIDER ERROR: WebSocket connection closed") except Exception as e: # pylint: disable=broad-except - ERR = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" + ERR = f"PROVIDER ERROR: {e.__class__.__name__ if hasattr(e, '__class__') else e}" logger.error(ERR) finally: + loop.call_soon_threadsafe(loop.stop) + loop.close() sys.exit(0) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py index 8133c7699361..3d82ce13c692 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/websocket_connection.py @@ -8,12 +8,12 @@ from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.abstract.fetcher import Fetcher from openbb_core.provider.utils.websockets.client import WebSocketClient -from openbb_intrinio.utils.references import TRADE_CONDITIONS, VENUES -from openbb_websockets.models import ( +from openbb_core.provider.utils.websockets.models import ( WebSocketConnection, WebSocketData, WebSocketQueryParams, ) +from openbb_intrinio.utils.references import TRADE_CONDITIONS, VENUES from pydantic import Field, field_validator, model_validator @@ -192,6 +192,7 @@ async def aextract_data( export_directory=query.export_directory, export_interval=query.export_interval, prune_interval=query.prune_interval, + compress_export=query.compress_export, verbose=query.verbose, **kwargs, ) diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py index b5fc09de60c6..2743acaeaf2a 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -85,7 +85,7 @@ async def subscribe(symbol, event): elif event == "unsubscribe": client.leave(ticker) except Exception as e: # pylint: disable=broad-except - exc = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" + exc = f"PROVIDER ERROR: {e.__class__.__name__ if hasattr(e, '__class__') else e}: {e.args}" logger.error(exc) @@ -156,9 +156,11 @@ async def connect_and_stream(): logger.error("PROVIDER INFO: WebSocket connection closed") except Exception as e: # pylint: disable=broad-except - EXC = f"PROVIDER ERROR: {e.__class__.__name__}: {e}" + EXC = f"PROVIDER ERROR: {e.__class__.__name__ if hasattr(e, '__class__') else e}: {e.args}" logger.error(EXC) finally: client.disconnect() + loop.call_soon_threadsafe(loop.stop) + loop.close() sys.exit(0) diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index ccfae237c355..0a8b0a085fb2 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -10,6 +10,11 @@ from openbb_core.provider.abstract.fetcher import Fetcher from openbb_core.provider.utils.descriptions import DATA_DESCRIPTIONS from openbb_core.provider.utils.websockets.client import WebSocketClient +from openbb_core.provider.utils.websockets.models import ( + WebSocketConnection, + WebSocketData, + WebSocketQueryParams, +) from openbb_polygon.utils.constants import ( CRYPTO_EXCHANGE_MAP, FX_EXCHANGE_MAP, @@ -21,11 +26,6 @@ STOCK_TRADE_CONDITIONS, ) from openbb_polygon.utils.helpers import map_tape -from openbb_websockets.models import ( - WebSocketConnection, - WebSocketData, - WebSocketQueryParams, -) from pydantic import Field, field_validator, model_validator ASSET_CHOICES = [ @@ -1221,6 +1221,7 @@ def extract_data( prune_interval=query.prune_interval, export_interval=query.export_interval, export_directory=query.export_directory, + compress_export=query.compress_export, sleep_time=query.sleep_time, broadcast_host=query.broadcast_host, broadcast_port=query.broadcast_port, diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index 407174c10230..0878d95bc114 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -53,7 +53,6 @@ ) from pydantic import ValidationError from websockets.asyncio.client import connect -from websockets.extensions import permessage_deflate URL_MAP = { "stock": "wss://socket.polygon.io/stocks", @@ -75,7 +74,6 @@ CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) FEED = kwargs.pop("feed", None) ASSET_TYPE = kwargs.pop("asset_type", "crypto") -kwargs["results_file"] = os.path.abspath(kwargs.get("results_file")) URL = URL_MAP.get(ASSET_TYPE) SUBSCRIBED_SYMBOLS: set = set() @@ -99,7 +97,6 @@ logger=logger, ), queue=db_queue, - batch_size=1, ) @@ -305,27 +302,14 @@ async def connect_and_stream(): """Connect to the WebSocket and stream data to file.""" tasks: set = set() - conn_kwargs = CONNECT_KWARGS.copy() - - conn_kwargs.update( - { - "ping_timeout": None, - "ping_interval": 10, - "close_timeout": 1, - "max_size": 32768, - } - ) - - conn_kwargs["extensions"] = [ - permessage_deflate.ClientPerMessageDeflateFactory( - server_max_window_bits=11, - client_max_window_bits=11, - compress_settings={ - "memLevel": 1, - "level": 1, - }, - ), - ] + conn_kwargs = { + "ping_timeout": None, + "ping_interval": 10, + "close_timeout": 1, + "max_size": 32768, + "compression": None, + } + conn_kwargs.update(CONNECT_KWARGS) async def message_receiver(websocket): """Message receiver.""" @@ -351,7 +335,6 @@ def _process_in_thread(): data_msgs = [msg for msg in message_data if msg not in status_msgs] if status_msgs: - # Convert to sync call asyncio.run(process_message(status_msgs)) if data_msgs: process_queue.queue.put_nowait(data_msgs) @@ -369,23 +352,23 @@ def _process_in_thread(): tasks.add(process_task) try: - for i in range(1, 120): - handler_task = asyncio.create_task( - process_queue.process_queue(lambda message: process_message(message)) - ) - tasks.add(handler_task) + handler_task = asyncio.create_task( + process_queue.process_queue(lambda message: process_message(message)) + ) + tasks.add(handler_task) stdin_task = asyncio.create_task(read_stdin()) tasks.add(stdin_task) await DATABASE.start_writer() - for i in range(1, 120): - processor_task = asyncio.create_task( - input_queue.process_queue( - lambda message: process_input_messages(message) - ) - ) - tasks.add(processor_task) + processor_task = asyncio.create_task( + input_queue.process_queue(lambda message: process_input_messages(message)) + ) + tasks.add(processor_task) + rate_task = asyncio.create_task(message_rate()) + tasks.add(rate_task) + minute_task = asyncio.create_task(reset_last_minute_count()) + tasks.add(minute_task) async for websocket in connect( URL, @@ -408,11 +391,6 @@ def _process_in_thread(): await subscribe(websocket, kwargs["symbol"], "subscribe") - rate_task = asyncio.create_task(message_rate()) - tasks.add(rate_task) - minute_task = asyncio.create_task(reset_last_minute_count()) - tasks.add(minute_task) - await message_receiver(websocket) # Attempt to reopen the connection @@ -499,5 +477,6 @@ def _process_in_thread(): logger.error(ERR) finally: + loop.call_soon_threadsafe(loop.stop) loop.close() sys.exit(0) diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py index 7deb68c177a4..8cba93413693 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py @@ -11,7 +11,7 @@ QUERY_DESCRIPTIONS, ) from openbb_core.provider.utils.websockets.client import WebSocketClient -from openbb_websockets.models import ( +from openbb_core.provider.utils.websockets.models import ( WebSocketConnection, WebSocketData, WebSocketQueryParams, @@ -272,6 +272,7 @@ async def aextract_data( prune_interval=query.prune_interval, export_interval=query.export_interval, export_directory=query.export_directory, + compress_export=query.compress_export, sleep_time=query.sleep_time, broadcast_host=query.broadcast_host, broadcast_port=query.broadcast_port, diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index 5ad5126c67b9..ae213457219f 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -20,6 +20,7 @@ from openbb_tiingo.models.websocket_connection import TiingoWebSocketData from pandas import to_datetime from pydantic import ValidationError +from websockets.asyncio.client import connect URL_MAP = { "stock": "wss://api.tiingo.com/iex", @@ -118,7 +119,7 @@ async def update_symbols(symbol, event): }, } - async with websockets.connect(URL) as websocket: + async with connect(URL) as websocket: await websocket.send(orjson.dumps(update_event)) response = await websocket.recv(decode=False) message = orjson.loads(response) @@ -198,12 +199,14 @@ async def process_message(message): # pylint: disable=too-many-branches elif message.get("messageType") == "A": data = message.get("data", []) service = message.get("service") + if service == "iex": data_message = {IEX_FIELDS[i]: data[i] for i in range(len(data))} _ = data_message.pop("timestamp", None) elif service == "fx": data_message = {FX_FIELDS[i]: data[i] for i in range(len(data))} elif service == "crypto_data": + if data[0] == "Q": data_message = { CRYPTO_QUOTE_FIELDS[i]: data[i] for i in range(len(data)) @@ -214,6 +217,7 @@ async def process_message(message): # pylint: disable=too-many-branches } data_message["date"] = datetime.now(UTC).isoformat() tiingo_date = data_message.pop("tiingo_date", None) + if isinstance(tiingo_date, str): tiingo_date = to_datetime(tiingo_date) tiingo_date = tiingo_date.tz_convert("America/New_York").to_pydatetime() @@ -231,7 +235,7 @@ async def process_message(message): # pylint: disable=too-many-branches raise e from e if result: - db_queue.queue.put_nowait(result) + await db_queue.enqueue(result) async def connect_and_stream(): @@ -274,7 +278,7 @@ async def message_receiver(websocket): try: await DATABASE.start_writer() - async for websocket in websockets.connect(URL, **conn_kwargs): + async for websocket in connect(URL, **conn_kwargs): try: if not any( task.name == "receiver_task" @@ -330,7 +334,7 @@ async def message_receiver(websocket): if __name__ == "__main__": try: - loop = uvloop.new_event_loop() + loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) loop.set_exception_handler(lambda loop, context: None) @@ -354,5 +358,6 @@ async def message_receiver(websocket): logger.error(ERR) finally: + loop.call_soon_threadsafe(loop.stop) loop.close() sys.exit(0) From df72101bccedffd21823c4321444b21d05c7e1d6 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Fri, 3 Jan 2025 21:20:07 -0800 Subject: [PATCH 112/119] add that file to the commit --- .../provider/utils/websockets/models.py | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 openbb_platform/core/openbb_core/provider/utils/websockets/models.py diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/models.py b/openbb_platform/core/openbb_core/provider/utils/websockets/models.py new file mode 100644 index 000000000000..4cc018574629 --- /dev/null +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/models.py @@ -0,0 +1,221 @@ +"""WebSockets models.""" + +from datetime import datetime +from typing import Any, Optional + +from openbb_core.app.model.abstract.error import OpenBBError +from openbb_core.provider.abstract.data import Data +from openbb_core.provider.abstract.query_params import QueryParams +from openbb_core.provider.utils.descriptions import ( + DATA_DESCRIPTIONS, +) +from openbb_core.provider.utils.websockets.client import WebSocketClient +from pydantic import ConfigDict, Field, field_validator, model_validator + +# In the Provider Interface, we map to: WebSocketConnection + + +class WebSocketQueryParams(QueryParams): + """Query parameters for WebSocket connection creation.""" + + name: str = Field( + description="Name to assign the client connection.", + ) + auth_token: Optional[str] = Field( + default=None, + description="Authentication token for API access control of the client, not related to the provider credentials.", + ) + results_file: Optional[str] = Field( + default=None, + description="Absolute path to the file for continuous writing. By default, a temporary file is created.", + ) + save_database: bool = Field( + default=False, + description="Whether to save the results after the session ends.", + ) + table_name: str = Field( + default="records", + description="Name of the SQL table to write the results to.", + ) + limit: Optional[int] = Field( + default=1000, + description="Maximum number of newest records to keep in the database." + + " If None, all records are kept, which can be memory-intensive.", + ) + prune_interval: Optional[int] = Field( + default=None, + description="Prune all entries older than the given number of minutes." + + " If 'export_interval' is set, 'prune_interval' must be at least twice as long.", + ) + export_interval: Optional[int] = Field( + default=None, + description="Export all entries as a CSV file every N minutes. Off unless a value is supplied.", + ) + export_directory: Optional[str] = Field( + default=None, + description="Directory to save the exported CSV files to. Defaults to OpenBBUserData/exports/websockets", + ) + compress_export: bool = Field( + default=False, + description="Whether to apply gzip compression to the exported CSV files. Default is False.", + ) + sleep_time: float = Field( + default=0.25, + description="Time to sleep, for the broadcast server, between checking for new records in the database." + + " The default is 0.25 seconds.", + ) + broadcast_host: str = Field( + default="127.0.0.1", + description="IP address to bind the broadcast server to.", + ) + broadcast_port: int = Field( + default=6666, + description="Port to bind the broadcast server to.", + ) + start_broadcast: bool = Field( + default=False, + description="Whether to start the broadcast server." + + " Set to False if system or network conditions do not allow it." + + " Can be started manually with the 'start_broadcasting' method," + + " where additional keyword arguments can be passed to `uvicorn.run`.", + ) + connect_kwargs: Optional[Any] = Field( + default=None, + description="A formatted dictionary, or serialized JSON string, of keyword arguments to pass" + + " directly to websockets.connect().", + ) + verbose: bool = Field( + default=True, + description="Whether to print export and prune messages to the console.", + ) + + @field_validator("connect_kwargs", mode="before", check_fields=False) + @classmethod + def _validate_connect_kwargs(cls, v): + """Validate the connect_kwargs format.""" + # pylint: disable=import-outside-toplevel + import json + + if isinstance(v, str): + try: + v = json.loads(v) + except json.JSONDecodeError as e: + raise OpenBBError( + f"Invalid JSON format for 'connect_kwargs': {e}" + ) from e + if v is not None and not isinstance(v, dict): + raise OpenBBError( + "Invalid 'connect_kwargs' format. Must be a dictionary or serialized JSON string." + ) + + return json.dumps(v, separators=(",", ":")) + + +class WebSocketConnectionStatus(Data): + """Data model for WebSocketConnection status information.""" + + name: str = Field( + description="Name assigned to the client connection.", + ) + auth_required: bool = Field( + description="True when 'auth_token' is supplied at initialization." + " When True, interactions with the client from the Python or API" + + " endpoints requires it to be supplied as a query parameter.", + ) + subscribed_symbols: str = Field( + description="Symbols subscribed to by the client connection.", + ) + is_running: bool = Field( + description="Whether the client connection is running.", + ) + provider_pid: Optional[int] = Field( + default=None, + description="Process ID of the provider connection.", + ) + is_broadcasting: bool = Field( + description="Whether the client connection is broadcasting.", + ) + broadcast_address: Optional[str] = Field( + default=None, + description="URI to the broadcast server.", + ) + broadcast_pid: Optional[int] = Field( + default=None, + description="Process ID of the broadcast server.", + ) + results_file: Optional[str] = Field( + default=None, + description="Absolute path to the file for continuous writing.", + ) + table_name: Optional[str] = Field( + default=None, + description="Name of the SQL table to write the results to.", + ) + save_database: bool = Field( + description="Whether to save the results after the session ends.", + ) + is_exporting: bool = Field( + description="Whether the client connection is actively exporting.", + ) + export_interval: Optional[int] = Field( + default=None, + description="The interval in minutes for exporting records to a CSV file.", + ) + export_directory: Optional[str] = Field( + default=None, + description="Directory to save the exported CSV files to.", + ) + is_pruning: bool = Field( + description="Whether the client connection is actively pruning records.", + ) + prune_interval: Optional[int] = Field( + default=None, + description="The interval in minutes for pruning records from the database, starting at the most recent entry.", + ) + + +class WebSocketData(Data): + """WebSocket data model.""" + + date: datetime = Field( + description=DATA_DESCRIPTIONS.get("date", ""), + ) + symbol: str = Field( + description=DATA_DESCRIPTIONS.get("symbol", ""), + ) + + +class WebSocketConnection(Data): + """Data model for returning WebSocketClient from the Provider Interface.""" + + model_config = ConfigDict( + extra="forbid", + ) + + client: Optional[Any] = Field( + default=None, + description="Instance of WebSocketClient class initialized by a provider Fetcher." + + " The client is used to communicate with the provider's data stream." + + " It is not returned to the user, but is handled by the router for API access.", + exclude=True, + ) + status: Optional[WebSocketConnectionStatus] = Field( + default=None, + description="Status information for the WebSocket connection.", + ) + + @field_validator("client", mode="before", check_fields=False) + @classmethod + def _validate_client(cls, v): + """Validate the client.""" + if v and not isinstance(v, WebSocketClient): + raise ValueError("Client must be an instance of WebSocketClient.") + return v + + @model_validator(mode="before") + @classmethod + def _validate_inputs(cls, values): + """Validate the status.""" + if not values.get("status") and not values.get("client"): + raise ValueError("Cannot initialize empty.") + return values From 58dadc5b1eb8adddb33ae7ae0273bfbc687fb619 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 6 Jan 2025 19:31:46 -0800 Subject: [PATCH 113/119] some cleanup --- .../provider/utils/websockets/broadcast.py | 23 ++--- .../provider/utils/websockets/client.py | 12 +-- .../provider/utils/websockets/database.py | 74 +++++++-------- .../fmp/openbb_fmp/utils/websocket_client.py | 47 +++++++++- .../openbb_polygon/utils/websocket_client.py | 8 +- .../models/websocket_connection.py | 17 ++-- .../openbb_tiingo/utils/websocket_client.py | 90 +++++++++++++------ 7 files changed, 171 insertions(+), 100 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py index 0d53eddba84c..fdd6e9bf7214 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py @@ -4,27 +4,22 @@ import json import logging import os -import signal import sys from pathlib import Path from typing import Optional import uvicorn - from fastapi import FastAPI, WebSocket, WebSocketDisconnect from openbb_core.provider.utils.websockets.database import ( CHECK_FOR, Database, - kill_thread, ) from openbb_core.provider.utils.websockets.helpers import ( get_logger, - handle_termination_signal, parse_kwargs, ) from starlette.websockets import WebSocketState - kwargs = parse_kwargs() HOST = kwargs.pop("host", None) or "localhost" @@ -41,7 +36,7 @@ app = FastAPI() -CONNECTED_CLIENTS = set() +CONNECTED_CLIENTS: set = set() MAIN_CLIENT = None STDIN_TASK = None LOGGER = get_logger("broadcast-server") @@ -58,7 +53,7 @@ async def read_stdin(): continue if line.strip() == "numclients": - MAIN_CLIENT.logger.info( + MAIN_CLIENT.logger.info( # type: ignore "Number of connected clients: %i", len(CONNECTED_CLIENTS) ) continue @@ -101,7 +96,7 @@ async def websocket_endpoint( # noqa: PLR0915 str(AUTH_TOKEN), sql=sql, ) - broadcast_server.replay = replay + broadcast_server.replay = replay # type: ignore auth_token = str(auth_token) if sql and ( @@ -244,7 +239,7 @@ async def stream_results( # noqa: PLR0915 # pylint: disable=too-many-branches last_id = ( 0 - if hasattr(self, "replay") and self.replay is True or replay is True + if hasattr(self, "replay") and self.replay is True or replay is True # type: ignore else last_id ) @@ -260,10 +255,10 @@ async def stream_results( # noqa: PLR0915 # pylint: disable=too-many-branches any(x.lower() in sql.lower() for x in CHECK_FOR) or (self.table_name not in sql and "message" not in sql) ): - await self.websocket.accept() - await self.websocket.send_text("Invalid SQL query passed.") - await self.websocket.close(code=1008, reason="Invalid query") - self.logger.error( + await self.websocket.accept() # type: ignore + await self.websocket.send_text("Invalid SQL query passed.") # type: ignore + await self.websocket.close(code=1008, reason="Invalid query") # type: ignore + self.logger.error( # type: ignore "Invalid query passed to the stream_results method: %s", sql ) return @@ -291,7 +286,7 @@ async def stream_results( # noqa: PLR0915 # pylint: disable=too-many-branches await self.websocket.send_json( json.dumps(json.loads(row[1])) ) - if self.replay is True: + if self.replay is True: # type: ignore await asyncio.sleep(self.sleep_time / 10) await cursor.close() diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py index 704e3aa7be82..c7dae8e53b29 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py @@ -233,12 +233,12 @@ def _atexit(self) -> None: self.logger.info("Websocket results saved to, %s\n", str(self.results_path)) if os.path.exists(self.results_file) and not self.save_database: # type: ignore os.remove(self.results_file) # type: ignore - if os.path.exists(self.results_file + "-journal"): - os.remove(self.results_file + "-journal") - if os.path.exists(self.results_file + "-shm"): - os.remove(self.results_file + "-shm") - if os.path.exists(self.results_file + "-wal"): - os.remove(self.results_file + "-wal") + if os.path.exists(self.results_file + "-journal"): # type: ignore + os.remove(self.results_file + "-journal") # type: ignore + if os.path.exists(self.results_file + "-shm"): # type: ignore + os.remove(self.results_file + "-shm") # type: ignore + if os.path.exists(self.results_file + "-wal"): # type: ignore + os.remove(self.results_file + "-wal") # type: ignore def _log_provider_output(self, output_queue) -> None: """Log output from the provider logger, handling exceptions, errors, and messages that are not data.""" diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 36d418f3ab74..4e380c3b5531 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -166,10 +166,10 @@ def __init__( # pylint: disable=too-many-positional-arguments self.results_path = Path(results_file).absolute() self.results_file = results_file - if ( - " " in table_name - or table_name.isupper() - or any(x.lower() in table_name.lower() for x in CHECK_FOR) + if table_name and ( + " " in table_name # type: ignore + or table_name.isupper() # type: ignore + or any(x.lower() in table_name.lower() for x in CHECK_FOR) # type: ignore ): raise OpenBBError(ProgrammingError(f"Invalid table name, {table_name}.")) @@ -177,7 +177,7 @@ def __init__( # pylint: disable=too-many-positional-arguments self.limit = limit self.loop = loop self.kwargs = kwargs if kwargs else {} - self._connections = {} + self._connections: dict = {} run_async(self._setup_database) self.data_model = data_model @@ -242,29 +242,29 @@ async def get_connection(self, name: str = "read"): conn_kwargs = self.kwargs.copy() if name == "read": - if ":" not in self.results_file: - results_file = ( + if ":" not in self.results_file: # type: ignore + results_file = ( # type: ignore "file:" + ( - self.results_file - if self.results_file.startswith("/") - else "/" + self.results_file + self.results_file # type: ignore + if self.results_file.startswith("/") # type: ignore + else "/" + self.results_file # type: ignore ) + "?mode=ro" ) else: - results_file = ( - self.results_file - + f"{'&mode=ro' if '?' in self.results_file else '?mode=ro'}" + results_file = ( # type: ignore + self.results_file # type: ignore + + f"{'&mode=ro' if '?' in self.results_file else '?mode=ro'}" # type: ignore ) conn_kwargs["uri"] = True elif name == "write": - results_file = self.results_file + results_file = self.results_file # type: ignore conn_kwargs["check_same_thread"] = False if name not in self._connections: - conn = await aiosqlite.connect(results_file, **conn_kwargs) + conn = await aiosqlite.connect(results_file, **conn_kwargs) # type: ignore pragmas = [ "PRAGMA journal_mode=WAL", "PRAGMA synchronous=off", @@ -355,8 +355,8 @@ async def _fetch_all(self, limit: Optional[int] = None) -> list: query += " LIMIT ?" params = (limit,) else: - params = () - async with conn.execute(query, params) as cursor: + params = None + async with conn.execute(query, params) as cursor: # type: ignore async for row in cursor: rows.append(await self._deserialize_row(row, cursor)) @@ -444,6 +444,7 @@ def get_latest_results( f" {e.__class__.__name__ if hasattr(e, '__class__') else e} -> {e.args}" ) self.logger.error(msg) + return [] async def _query_db(self, sql, parameters: Optional[Iterable[Any]] = None) -> list: """Query the SQLite database.""" @@ -472,10 +473,11 @@ async def _query_db(self, sql, parameters: Optional[Iterable[Any]] = None) -> li ) as cursor: async for row in cursor: rows.append(await self._deserialize_row(row, cursor)) - return rows except Exception as e: # pylint: disable=broad-except raise OpenBBError(e) from e + return rows + def query(self, sql: str, parameters: Optional[Iterable[Any]] = None) -> list: """ Run a SELECT query to the database. @@ -654,7 +656,7 @@ def __init__( self._last_processed_timestamp = None self._conn = None self.num_workers = 60 - self.write_tasks = [] + self.write_tasks: list = [] self._export_running = False self._prune_running = False self.batch_processor = BatchProcessor(self) @@ -688,7 +690,7 @@ async def stop_writer(self): async def _process_queue(self): """Process queue with parallel writers.""" - batch = [] + batch: list = [] while self.writer_running: try: @@ -833,6 +835,8 @@ async def _export_database(self): for key in json.loads(row[0]): headers[key] = None + new_rows: list = [] + if self.compress_export: with gzip.open(path, "wt") as gz_file: writer = csv.DictWriter(gz_file, fieldnames=list(headers)) @@ -841,7 +845,7 @@ async def _export_database(self): while True: rows = await cursor.fetchmany(chunk_size) - new_rows: list = [] + if not rows: break @@ -864,7 +868,7 @@ async def _export_database(self): while True: rows = await cursor.fetchmany(chunk_size) - new_rows: list = [] + if not rows: break @@ -915,17 +919,17 @@ def start_export_task(self): return self._export_running = True - self.export_thread = threading.Thread( + self.export_thread = threading.Thread( # type: ignore target=self._run_export_event, name="ExportThread", daemon=True ) - self.export_thread.start() + self.export_thread.start() # type: ignore def stop_export_task(self): """Public method to stop the background export task.""" if hasattr(self, "export_thread") and self.export_thread: - self.export_thread.join(timeout=1) - if self.export_thread.is_alive(): - kill_thread(self.export_thread) + self.export_thread.join(timeout=1) # type: ignore + if self.export_thread.is_alive(): # type: ignore + kill_thread(self.export_thread) # type: ignore self._export_running = False self.export_thread = None @@ -1091,12 +1095,12 @@ def __init__( def run(self): """Run the batch processor as tasks.""" try: - self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) + self.loop = asyncio.new_event_loop() # type: ignore + asyncio.set_event_loop(self.loop) # type: ignore # Create worker tasks while self.running and not self._shutdown.is_set(): try: - self.loop.run_until_complete(self._worker()) + self.loop.run_until_complete(self._worker()) # type: ignore except (SystemExit, KeyboardInterrupt): self.running = False break @@ -1112,19 +1116,19 @@ def stop(self): """Signal thread to stop gracefully.""" self.running = False self._shutdown.set() - if self.loop and self.loop.is_running(): - self.loop.call_soon_threadsafe(self.loop.stop) + if self.loop and self.loop.is_running(): # type: ignore + self.loop.call_soon_threadsafe(self.loop.stop) # type: ignore def _cleanup(self): """Clean up resources on shutdown""" if self.loop: - pending = asyncio.all_tasks(self.loop) + pending = asyncio.all_tasks(self.loop) # type: ignore for task in pending: task.cancel() - self.loop.run_until_complete( + self.loop.run_until_complete( # type: ignore asyncio.gather(*pending, return_exceptions=True) ) - self.loop.close() + self.loop.close() # type: ignore async def _worker(self): # pylint: disable=import-outside-toplevel diff --git a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py index 98a3ff2febf9..b608ff7154e5 100644 --- a/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py +++ b/openbb_platform/providers/fmp/openbb_fmp/utils/websocket_client.py @@ -1,10 +1,38 @@ -"""FMP WebSocket client.""" +""" +FMP WebSocket Client. + +This file should be run as a script, and is intended to be run as a subprocess of FmpWebSocketFetcher. + +Keyword arguments are passed from the command line as space-delimited, `key=value`, pairs. + +Required Keyword Arguments +-------------------------- + api_key: str + The API key for the Polygon WebSocket. + asset_type: str + The asset type to subscribe to. Default is "crypto". + Options: "stock", "crypto", "fx" + symbol: str + The symbol to subscribe to. Example: "AAPL" or "AAPL,MSFT". + results_file: str + The path to the file where the results will be stored. + +Optional Keyword Arguments +-------------------------- + table_name: str + The name of the table to store the data in. Default is "records". + limit: int + The maximum number of rows to store in the database. + connect_kwargs: dict + Additional keyword arguments to pass directly to `websockets.connect()`. + Example: {"ping_timeout": 300} +""" import asyncio +import json import signal import sys -import orjson as json import websockets from openbb_core.provider.utils.websockets.database import Database, DatabaseWriter from openbb_core.provider.utils.websockets.helpers import ( @@ -31,6 +59,8 @@ CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) URL = URL_MAP.get(kwargs.pop("asset_type"), None) +SUBSCRIBED_SYMBOLS: set = set() + if not URL: raise ValueError("Invalid asset type provided.") @@ -87,6 +117,14 @@ async def subscribe(websocket, symbol, event): } try: await websocket.send(json.dumps(subscribe_event)) + + for t in ticker: + if event == "subscribe": + SUBSCRIBED_SYMBOLS.add(t) + else: + SUBSCRIBED_SYMBOLS.discard(t) + + kwargs["symbol"] = ",".join(SUBSCRIBED_SYMBOLS) except Exception as e: # pylint: disable=broad-except msg = f"PROVIDER ERROR: {e.__class__.__name__ if hasattr(e, '__class__') else e}: {e}" logger.error(msg) @@ -201,7 +239,10 @@ async def connect_and_stream(): sys.exit(1) except Exception as e: # pylint: disable=broad-except - msg = f"PROVIDER ERROR: Unexpected error -> {e.__class__.__name__}: {e}" + msg = ( + "PROVIDER ERROR: Unexpected error ->" + f" {e.__class__.__name__ if hasattr(e, '__class__') else e}: {e.args}" + ) logger.error(msg) sys.exit(1) diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index 0878d95bc114..dea05d4df59e 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -32,12 +32,10 @@ import asyncio import json -import os import signal import sys import time -import orjson import websockets from openbb_core.provider.utils.websockets.database import Database, DatabaseWriter from openbb_core.provider.utils.websockets.helpers import ( @@ -178,7 +176,7 @@ async def login(websocket): try: await websocket.send(login_event) res = await websocket.recv(decode=False) - response = orjson.loads(res) + response = json.loads(res) messages = response if isinstance(response, list) else [response] for msg in messages: if msg.get("status") == "connected": @@ -323,7 +321,7 @@ async def process_input_messages(message): def _process_in_thread(): global LAST_MINUTE_COUNT, MESSAGE_COUNT # pylint: disable=global-statement # noqa - message_data = orjson.loads(message) + message_data = json.loads(message) if isinstance(message_data, list): MESSAGE_COUNT += len(message_data) LAST_MINUTE_COUNT += len(message_data) @@ -387,7 +385,7 @@ def _process_in_thread(): response = await websocket.recv(decode=False) - await process_message(orjson.loads(response)) + await process_message(json.loads(response)) await subscribe(websocket, kwargs["symbol"], "subscribe") diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py index 8cba93413693..989999e77747 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py @@ -92,7 +92,7 @@ class TiingoWebSocketQueryParams(WebSocketQueryParams): description="The asset type for the feed. Choices are 'stock', 'fx', or 'crypto'.", ) feed: Literal["trade", "trade_and_quote"] = Field( - default="trade_and_quote", + default="trade", description="The asset type associated with the symbol. Choices are 'trade' or 'trade_and_quote'." + " FX only supports quote.", ) @@ -102,6 +102,7 @@ class TiingoWebSocketData(WebSocketData): """Tiingo WebSocket data model.""" timestamp: Optional[datetime] = Field( + default=None, description="The timestamp of the data.", ) type: Literal["quote", "trade", "break"] = Field( @@ -242,28 +243,20 @@ async def aextract_data( from asyncio import sleep api_key = credentials.get("tiingo_token") if credentials else "" - threshold_level = ( - 5 - if query.asset_type == "fx" or query.feed == "trade" - else ( - 2 - if query.asset_type == "crypto" and query.feed == "trade_and_quote" - else 0 - ) - ) + symbol = query.symbol.lower() kwargs = { "api_key": api_key, "asset_type": query.asset_type, - "threshold_level": threshold_level, + "feed": query.feed, "connect_kwargs": query.connect_kwargs, } client = WebSocketClient( name=query.name, module="openbb_tiingo.utils.websocket_client", - symbol=symbol.lower(), + symbol=symbol, limit=query.limit, results_file=query.results_file, table_name=query.table_name, diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py index ae213457219f..0049847d9a96 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/utils/websocket_client.py @@ -1,12 +1,41 @@ -"""FMP WebSocket server.""" +""" +Tiingo WebSocket Client. + +This file should be run as a script, and is intended to be run as a subprocess of TiingoWebSocketFetcher. + +Keyword arguments are passed from the command line as space-delimited, `key=value`, pairs. + +Required Keyword Arguments +-------------------------- + api_key: str + The API key for the Polygon WebSocket. + asset_type: str + The asset type to subscribe to. Default is "crypto". + Options: "stock", "crypto", "fx" + symbol: str + The symbol to subscribe to. Example: "AAPL" or "AAPL,MSFT". Use "*" to subscribe to all symbols. + feed: str + The feed to subscribe to. One of: "trade" or "trade_and_quote". + results_file: str + The path to the file where the results will be stored. + +Optional Keyword Arguments +-------------------------- + table_name: str + The name of the table to store the data in. Default is "records". + limit: int + The maximum number of rows to store in the database. + connect_kwargs: dict + Additional keyword arguments to pass directly to `websockets.connect()`. + Example: {"ping_timeout": 300} +""" import asyncio -import os +import json import signal import sys -from datetime import UTC, datetime +from datetime import datetime -import orjson import websockets from openbb_core.provider.utils.errors import UnauthorizedError from openbb_core.provider.utils.websockets.database import Database, DatabaseWriter @@ -20,6 +49,7 @@ from openbb_tiingo.models.websocket_connection import TiingoWebSocketData from pandas import to_datetime from pydantic import ValidationError +from pytz import UTC from websockets.asyncio.client import connect URL_MAP = { @@ -79,14 +109,28 @@ ] SUBSCRIPTION_ID = "" logger = get_logger("openbb.websocket.tiingo") -input_queue = MessageQueue(logger=logger, backoff_factor=0) -db_queue = MessageQueue(logger=logger, backoff_factor=0) +input_queue = MessageQueue(logger=logger) +db_queue = MessageQueue(logger=logger) kwargs = parse_kwargs() CONNECT_KWARGS = kwargs.pop("connect_kwargs", {}) -kwargs["results_file"] = os.path.abspath(kwargs["results_file"]) -URL = URL_MAP.get(kwargs.pop("asset_type", "crypto")) +ASSET_TYPE = kwargs.pop("asset_type", "crypto") +FEED = kwargs.pop("feed", "trade") + + SUBSCRIBED_SYMBOLS: set = set() +THRESHOLD_LEVEL = ( + 5 + if ASSET_TYPE == "fx" or FEED == "trade" + else (2 if ASSET_TYPE == "crypto" and FEED == "trade_and_quote" else 0) +) + +URL = URL_MAP.get(ASSET_TYPE) + +if not kwargs.get("api_key"): + raise ValueError("No API key provided.") + + if not URL: raise ValueError("Invalid asset type provided.") @@ -120,9 +164,9 @@ async def update_symbols(symbol, event): } async with connect(URL) as websocket: - await websocket.send(orjson.dumps(update_event)) + await websocket.send(json.dumps(update_event)) response = await websocket.recv(decode=False) - message = orjson.loads(response) + message = json.loads(response) if "tickers" in message.get("data", {}): tickers = message["data"]["tickers"] threshold_level = message["data"].get("thresholdLevel") @@ -155,7 +199,7 @@ async def read_stdin_and_update_symbols(): f" Database Queue : {db_queue.queue.qsize()}" ) else: - line = orjson.loads(line.strip()) + line = json.loads(line.strip()) if line: symbol = line.get("symbol") @@ -167,7 +211,7 @@ async def process_message(message): # pylint: disable=too-many-branches """Process the message and write to the database.""" result: dict = {} data_message: dict = {} - message = message if isinstance(message, (dict, list)) else orjson.loads(message) + message = message if isinstance(message, (dict, list)) else json.loads(message) msg: str = "" if message.get("messageType") == "E": response = message.get("response", {}) @@ -243,16 +287,12 @@ async def connect_and_stream(): tasks: set = set() ticker: list = [] - - conn_kwargs = CONNECT_KWARGS.copy() - - conn_kwargs.update( - { - "ping_interval": 8, - "ping_timeout": 8, - "close_timeout": 1, - } - ) + conn_kwargs = { + "ping_interval": 8, + "ping_timeout": 8, + "close_timeout": 1, + } + conn_kwargs.update(CONNECT_KWARGS) if isinstance(kwargs["symbol"], str): ticker = kwargs["symbol"].lower().split(",") @@ -261,7 +301,7 @@ async def connect_and_stream(): "eventName": "subscribe", "authorization": kwargs["api_key"], "eventData": { - "thresholdLevel": kwargs["threshold_level"], + "thresholdLevel": THRESHOLD_LEVEL, "tickers": ticker, }, } @@ -270,7 +310,7 @@ async def message_receiver(websocket): """Receive messages from the WebSocket.""" while True: message = await websocket.recv(decode=False) - input_queue.queue.put_nowait(orjson.loads(message)) + input_queue.queue.put_nowait(json.loads(message)) stdin_task = asyncio.create_task(read_stdin_and_update_symbols()) tasks.add(stdin_task) @@ -290,7 +330,7 @@ async def message_receiver(websocket): ) tasks.add(receiver_task) - await websocket.send(orjson.dumps(subscribe_event)) + await websocket.send(json.dumps(subscribe_event)) logger.info("PROVIDER INFO: WebSocket connection established.") for _ in range(9): process_task = asyncio.create_task( From 854052d43a20ad050e2210a55c7dc0af540ae07a Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 6 Jan 2025 19:37:32 -0800 Subject: [PATCH 114/119] more linting --- .../provider/utils/websockets/broadcast.py | 4 ++-- .../provider/utils/websockets/database.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py index fdd6e9bf7214..62ef54fcaadd 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py @@ -215,7 +215,7 @@ def _decrypt_value(self, value: str) -> str: async def stream_results( # noqa: PLR0915 # pylint: disable=too-many-branches self, - sql: str = None, + sql: Optional[str] = None, replay: bool = False, ): """Continuously read the database and send new messages as JSON via WebSocket.""" @@ -283,7 +283,7 @@ async def stream_results( # noqa: PLR0915 # pylint: disable=too-many-branches continue for row in rows: last_id = row[0] if row[0] > last_id else last_id - await self.websocket.send_json( + await self.websocket.send_json( # type: ignore json.dumps(json.loads(row[1])) ) if self.replay is True: # type: ignore diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 4e380c3b5531..19f41a7bcc85 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -244,7 +244,7 @@ async def get_connection(self, name: str = "read"): if name == "read": if ":" not in self.results_file: # type: ignore results_file = ( # type: ignore - "file:" + "file:" # type: ignore + ( self.results_file # type: ignore if self.results_file.startswith("/") # type: ignore @@ -506,6 +506,7 @@ def query(self, sql: str, parameters: Optional[Iterable[Any]] = None) -> list: except Exception as e: # pylint: disable=broad-except msg = f"{e.__class__.__name__ if hasattr(e, '__class__') else e}: {e.args}" self.logger.error(msg) + return [] async def _clear_results(self): """Clear the results from the SQLite database.""" @@ -951,17 +952,17 @@ def start_prune_task(self): prune_thread = threading.Thread(target=self._run_prune_event) prune_thread.daemon = True prune_thread.name = "WebSocketPruneThread" - self.prune_thread = prune_thread - self.prune_thread.start() + self.prune_thread = prune_thread # type: ignore + self.prune_thread.start() # type: ignore finally: - self.prune_thread.join(timeout=1) + self.prune_thread.join(timeout=1) # type: ignore def stop_prune_task(self): """Public method to stop the background pruning task.""" if hasattr(self, "prune_thread") and self.prune_thread: - self.prune_thread.join(timeout=1) - if self.prune_thread.is_alive(): - kill_thread(self.prune_thread) + self.prune_thread.join(timeout=1) # type: ignore + if self.prune_thread.is_alive(): # type: ignore + kill_thread(self.prune_thread) # type: ignore self._prune_running = False self.prune_thread = None From 5d35199148aaf1137e258c684df210cb4ef11455 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 6 Jan 2025 19:40:53 -0800 Subject: [PATCH 115/119] more linting --- .../tiingo/openbb_tiingo/models/websocket_connection.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py index 989999e77747..c70035e45e34 100644 --- a/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py +++ b/openbb_platform/providers/tiingo/openbb_tiingo/models/websocket_connection.py @@ -184,21 +184,21 @@ def _valiidate_data_type(cls, v): def _validate_date(cls, v): """Validate the date.""" # pylint: disable=import-outside-toplevel - from pandas import to_datetime - from pytz import timezone + from pandas import to_datetime # noqa + from pytz import timezone, UTC if isinstance(v, str): dt = to_datetime(v, utc=True).tz_convert(timezone("America/New_York")) else: try: - dt = datetime.fromtimestamp(v / 1000, timezone.utc) + dt = datetime.fromtimestamp(v / 1000, UTC) dt = dt.astimezone(timezone("America/New_York")) except Exception: if isinstance(v, (int, float)): # Check if the timestamp is in nanoseconds and convert to seconds if v > 1e12: v = v / 1e9 # Convert nanoseconds to seconds - dt = datetime.fromtimestamp(v, timezone.utc) + dt = datetime.fromtimestamp(v, UTC) dt = dt.astimezone(timezone("America/New_York")) else: dt = v From da7ccbc5f29efceea93da1e3a749a5c6e7015324 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 6 Jan 2025 19:45:20 -0800 Subject: [PATCH 116/119] more linting --- .../openbb_core/provider/utils/websockets/database.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 19f41a7bcc85..31ed2ef5e451 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -366,8 +366,9 @@ async def _fetch_all(self, limit: Optional[int] = None) -> list: raise OpenBBError(e) from e async def _deserialize_row(self, row, cursor) -> dict: - """Deserialize a row from the SQLite database. - Handles both single message column and multiple extracted fields.""" + """ + Deserialize a row from the SQLite database. Handles both single message column and multiple extracted fields. + """ # pylint: disable=import-outside-toplevel import json @@ -1072,9 +1073,7 @@ async def _start_export_task(self): class BatchProcessor(threading.Thread): - """ - This class is a thread intended for use as a subprocess and is called by `DatabaseWriter.start_writer()`. - """ + """This class is a thread intended for use as a subprocess and is called by `DatabaseWriter.start_writer()`.""" def __init__( self, database_writer: DatabaseWriter, num_workers=120, collection_time=0.25 From b5770d6ab1ae15901a3d7d96274842f23f85f4f0 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 6 Jan 2025 19:50:45 -0800 Subject: [PATCH 117/119] more linting --- .../core/openbb_core/provider/utils/websockets/database.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 31ed2ef5e451..5afee8365e29 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -366,9 +366,7 @@ async def _fetch_all(self, limit: Optional[int] = None) -> list: raise OpenBBError(e) from e async def _deserialize_row(self, row, cursor) -> dict: - """ - Deserialize a row from the SQLite database. Handles both single message column and multiple extracted fields. - """ + """Deserialize a row from the SQLite database.""" # pylint: disable=import-outside-toplevel import json From 062b8cdc0b8ec1ae2e308efbf6317c56386cd3c3 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 6 Jan 2025 20:16:47 -0800 Subject: [PATCH 118/119] more linting --- .../provider/utils/websockets/broadcast.py | 31 ++++++++++--------- .../provider/utils/websockets/client.py | 5 --- .../provider/utils/websockets/database.py | 21 ++++++++----- .../openbb_websockets/websockets_router.py | 3 -- .../openbb_intrinio/utils/websocket_client.py | 4 +-- .../models/websocket_connection.py | 1 + .../openbb_polygon/utils/websocket_client.py | 4 +-- 7 files changed, 35 insertions(+), 34 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py index 62ef54fcaadd..10517d4cf299 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py @@ -1,5 +1,7 @@ """Broadcast server for streaming results to connected clients via WebSocket.""" +# pylint: disable=too-many-positional-arguments + import asyncio import json import logging @@ -20,19 +22,19 @@ ) from starlette.websockets import WebSocketState -kwargs = parse_kwargs() +KWARGS = parse_kwargs() -HOST = kwargs.pop("host", None) or "localhost" -PORT = kwargs.pop("port", None) or 6666 +HOST = KWARGS.pop("host", None) or "localhost" +PORT = KWARGS.pop("port", None) or 6666 PORT = int(PORT) -RESULTS_FILE = kwargs.pop("results_file", None) -TABLE_NAME = kwargs.pop("table_name", None) or "records" -SLEEP_TIME = kwargs.pop("sleep_time", None) or 0.25 -AUTH_TOKEN = kwargs.pop("auth_token", None) +RESULTS_FILE = KWARGS.pop("results_file", None) +TABLE_NAME = KWARGS.pop("table_name", None) or "records" +SLEEP_TIME = KWARGS.pop("sleep_time", None) or 0.25 +AUTH_TOKEN = KWARGS.pop("auth_token", None) -SQL = kwargs.pop("sql", None) -SQL_CONNECT_KWARGS = kwargs.pop("sql_connect_kwargs", None) or {} +SQL = KWARGS.pop("sql", None) +SQL_CONNECT_KWARGS = KWARGS.pop("sql_connect_kwargs", None) or {} app = FastAPI() @@ -69,7 +71,7 @@ async def read_stdin(): for client in CONNECTED_CLIENTS: client.logger.error(err_msg) - for client in CONNECTED_CLIENTS: + for client in CONNECTED_CLIENTS.copy(): if client.websocket.client_state != WebSocketState.DISCONNECTED: await client.websocket.send_json(command) else: @@ -96,7 +98,7 @@ async def websocket_endpoint( # noqa: PLR0915 str(AUTH_TOKEN), sql=sql, ) - broadcast_server.replay = replay # type: ignore + broadcast_server.replay = replay # type: ignore # pylint: disable=attribute-defined-outside-init auth_token = str(auth_token) if sql and ( @@ -318,7 +320,7 @@ def start_app(self, host: str = "127.0.0.1", port: int = 6666): self._app, host=host, port=port, - **kwargs, + **KWARGS, ) @@ -350,6 +352,7 @@ def run_broadcast_server(broadcast_server, host, port, **kwargs): async def main(): """Run the main function.""" + # pylint: disable=import-outside-toplevel import threading loop = asyncio.get_running_loop() @@ -364,13 +367,13 @@ async def main(): SQL_CONNECT_KWARGS, SQL, ) - global MAIN_CLIENT # noqa: PLW0603 + global MAIN_CLIENT # noqa: PLW0603 pylint: disable=global-statement MAIN_CLIENT = broadcast_server try: broadcast_thread = threading.Thread( target=run_broadcast_server, args=(broadcast_server, HOST, PORT), - kwargs=kwargs, + kwargs=KWARGS, daemon=True, ) broadcast_thread.start() diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py index c7dae8e53b29..3f92170db1b4 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/client.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/client.py @@ -129,7 +129,6 @@ def __init__( # noqa: PLR0913 # pylint: disable=too-many-arguments,too-many-po import threading from queue import Queue from pathlib import Path - from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.utils.websockets.database import Database from openbb_core.provider.utils.websockets.helpers import ( encrypt_value, @@ -381,8 +380,6 @@ def connect(self) -> None: # pylint: disable=too-many-locals import queue import subprocess import threading - import time - from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.provider.utils.websockets.helpers import decrypt_value if self.is_running: @@ -528,7 +525,6 @@ def subscribe(self, symbol) -> None: # pylint: disable=import-outside-toplevel import json # noqa import time - from openbb_core.app.model.abstract.error import OpenBBError if not self.is_running: raise OpenBBError("Provider connection is not running.") @@ -555,7 +551,6 @@ def unsubscribe(self, symbol) -> None: # pylint: disable=import-outside-toplevel import json # noqa import time - from openbb_core.app.model.abstract.error import OpenBBError if not self.symbol: self.logger.info("No subscribed symbols.") diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 5afee8365e29..7afd9bf02093 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -1,6 +1,6 @@ """Database module for serialized websockets results.""" -# pylint: disable=too-many-lines,too-many-arguments,too-many-locals,too-many-branches,too-many-statements,protected-access +# pylint: disable=too-many-lines,too-many-arguments,too-many-locals,too-many-branches,too-many-statements,protected-access,too-many-instance-attributes,too-many-positional-arguments import asyncio import threading @@ -297,7 +297,7 @@ async def _write_to_db(self, message) -> None: """, # noqa (message,), ) - self._at_limit = False + self._at_limit = False # pylint: disable=attribute-defined-outside-init if self.limit is not None and not self._at_limit: limit = max(0, int(self.limit)) @@ -309,7 +309,9 @@ async def _write_to_db(self, message) -> None: count = await count_cursor.fetchone() if count[0] > limit: - self._at_limit = True + self._at_limit = ( + True # pylint: disable=attribute-defined-outside-init + ) await count_cursor.close() @@ -382,9 +384,7 @@ async def _deserialize_row(self, row, cursor) -> dict: or isinstance(row[0], bytes) else row[0] ) - else: - # Multiple column case (extracted fields) - return {cursor.description[i][0]: row[i] for i in range(len(row))} + return {cursor.description[i][0]: row[i] for i in range(len(row))} except (json.JSONDecodeError, AttributeError) as e: self.logger.error(f"Failed to deserialize row: {e}") @@ -990,7 +990,8 @@ async def _start_prune_task(self): if not self._last_processed_timestamp: last_date = await self.database._query_db( - f"SELECT json_extract(message, '$.date') FROM {self.database.table_name} ORDER BY json_extract(message, '$.date') DESC LIMIT 1" # noqa + "SELECT json_extract(message, '$.date') FROM" # noqa + f" {self.database.table_name} ORDER BY json_extract(message, '$.date') DESC LIMIT 1" ) if not last_date: continue @@ -1022,6 +1023,7 @@ async def _start_prune_task(self): async def _start_export_task(self): """Start a background task to prune the database periodically.""" + # pylint: disable=import-outside-toplevel from pandas import to_datetime minutes = self.export_interval or 5 @@ -1029,7 +1031,10 @@ async def _start_export_task(self): while self.export_thread is not None and not self._shutdown: # Get the initial row to determine the "first time" try: - query = f"SELECT json_extract(message, '$.date') FROM {self.database.table_name} ORDER BY json_extract(message, '$.date') ASC LIMIT 1" # noqa + query = ( + "SELECT json_extract(message, '$.date') FROM" # noqa + f" {self.database.table_name} ORDER BY json_extract(message, '$.date') ASC LIMIT 1" + ) initial_row = await self.database._query_db(query) if not initial_row: await asyncio.sleep(1) diff --git a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py index dd7672cdbe7a..1642f8917aef 100644 --- a/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py +++ b/openbb_platform/extensions/websockets/openbb_websockets/websockets_router.py @@ -80,9 +80,6 @@ async def create_connection( obbject = await OBBject.from_query(Query(**locals())) client = obbject.results.client - # pylint: disable=import-outside-toplevel - import asyncio - await asyncio.sleep(1) if not client.is_running or client._exception is not None: diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py index 2743acaeaf2a..0afd5dadcf94 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/utils/websocket_client.py @@ -124,14 +124,14 @@ async def process_stdin_queue(): async def connect_and_stream(): """Connect to the WebSocket and stream data to file.""" + stdin_task = asyncio.create_task(read_stdin_and_queue_commands()) + process_stdin_task = asyncio.create_task(process_stdin_queue()) try: symbol = kwargs.pop("symbol", "lobby") symbol = ["lobby"] if "*" in symbol else symbol.split(",") - stdin_task = asyncio.create_task(read_stdin_and_queue_commands()) await DATABASE.start_writer() client.connect() client.join(symbol) - process_stdin_task = asyncio.create_task(process_stdin_queue()) finally: stdin_task.cancel() process_stdin_task.cancel() diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py index 0a8b0a085fb2..d716ad481fbb 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py +++ b/openbb_platform/providers/polygon/openbb_polygon/models/websocket_connection.py @@ -1198,6 +1198,7 @@ def extract_data( **kwargs: Any, ) -> dict: """Extract data from the WebSocket.""" + # pylint: disable=import-outside-toplevel import time api_key = credentials.get("polygon_api_key") if credentials else "" diff --git a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py index dea05d4df59e..d73ab6352b4d 100644 --- a/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py +++ b/openbb_platform/providers/polygon/openbb_polygon/utils/websocket_client.py @@ -457,8 +457,8 @@ def _process_in_thread(): loop.run_forever() except (websockets.ConnectionClosed, websockets.ConnectionClosedError) as e: - msg = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" - logger.info(msg) + MSG = f"PROVIDER INFO: The WebSocket connection was closed -> {e}" + logger.info(MSG) # Attempt to reopen the connection logger.info("PROVIDER INFO: Attempting to reconnect...") time.sleep(1) From 6f8c5438ef20fd8af4f808aa006e687506e0bff8 Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 6 Jan 2025 20:25:09 -0800 Subject: [PATCH 119/119] more linting --- .../core/openbb_core/provider/utils/websockets/broadcast.py | 1 - .../core/openbb_core/provider/utils/websockets/database.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py index 10517d4cf299..bb7bda190d9e 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/broadcast.py @@ -40,7 +40,6 @@ CONNECTED_CLIENTS: set = set() MAIN_CLIENT = None -STDIN_TASK = None LOGGER = get_logger("broadcast-server") diff --git a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py index 7afd9bf02093..f5732ec58e13 100644 --- a/openbb_platform/core/openbb_core/provider/utils/websockets/database.py +++ b/openbb_platform/core/openbb_core/provider/utils/websockets/database.py @@ -309,8 +309,8 @@ async def _write_to_db(self, message) -> None: count = await count_cursor.fetchone() if count[0] > limit: - self._at_limit = ( - True # pylint: disable=attribute-defined-outside-init + self._at_limit = ( # pylint: disable=attribute-defined-outside-init + True ) await count_cursor.close()