From 6ec8d734b0f73539c7beac20dbefc8b0f7ff3132 Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Thu, 19 Oct 2023 00:01:29 -0400 Subject: [PATCH] feat: :sparkles: programmatically track provider errors in optional cache This change is meant to lay the skeleton for dynamically determining which order to rank providers for a given chain. This first PR is aimed at adding a key in the cache when errors occur. An exponential decay is used to store a count of the errors incurred by a given provider --- package.json | 1 + src/utils/ProviderUtils.ts | 19 ++++++- src/utils/RedisUtils.ts | 106 ++++++++++++++++++++++++++++++++++++- src/utils/TypeGuards.ts | 13 +++++ 4 files changed, 136 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index e5fe83694..ef7859bab 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "lodash.get": "^4.4.2", "minimist": "^1.2.8", "redis4": "npm:redis@^4.1.0", + "superstruct": "^1.0.3", "ts-node": "^10.9.1", "winston": "^3.10.0", "zksync-web3": "^0.14.3" diff --git a/src/utils/ProviderUtils.ts b/src/utils/ProviderUtils.ts index 5d48500aa..37f6eae9b 100644 --- a/src/utils/ProviderUtils.ts +++ b/src/utils/ProviderUtils.ts @@ -4,7 +4,7 @@ import lodash from "lodash"; import winston from "winston"; import { isPromiseFulfilled, isPromiseRejected } from "./TypeGuards"; import createQueue, { QueueObject } from "async/queue"; -import { getRedis, RedisClient, setRedisKey } from "./RedisUtils"; +import { getRedis, incrementProviderErrorCount, RedisClient, setRedisKey } from "./RedisUtils"; import { CHAIN_CACHE_FOLLOW_DISTANCE, PROVIDER_CACHE_TTL, @@ -242,7 +242,7 @@ export class RetryProvider extends ethers.providers.StaticJsonRpcProvider { readonly delay: number, readonly maxConcurrency: number, providerCacheNamespace: string, - redisClient?: RedisClient + readonly redisClient?: RedisClient ) { // Initialize the super just with the chainId, which stops it from trying to immediately send out a .send before // this derived class is initialized. @@ -303,6 +303,21 @@ export class RetryProvider extends ethers.providers.StaticJsonRpcProvider { const results = await Promise.allSettled(requiredProviders.map(tryWithFallback)); + // Iterate over all the errors and increment the error count for each provider. + await Promise.all( + errors.map(async ([provider]) => { + // Increment the error count for this provider. + // Note: redis must be enabled for this to work. + if (this.redisClient) { + await incrementProviderErrorCount( + this.redisClient, + getOriginFromURL(provider.connection.url), + this.network.chainId + ); + } + }) + ); + if (!results.every(isPromiseFulfilled)) { // Format the error so that it's very clear which providers failed and succeeded. const errorTexts = errors.map(([provider, errorText]) => formatProviderError(provider, errorText)); diff --git a/src/utils/RedisUtils.ts b/src/utils/RedisUtils.ts index 07a815fee..c684ced81 100644 --- a/src/utils/RedisUtils.ts +++ b/src/utils/RedisUtils.ts @@ -1,4 +1,4 @@ -import { assert, toBN, BigNumberish, isDefined } from "./"; +import { assert, toBN, BigNumberish, isDefined, getCurrentTime, isProviderErrorCount } from "./"; import { REDIS_URL_DEFAULT } from "../common/Constants"; import { createClient } from "redis4"; import winston from "winston"; @@ -153,3 +153,107 @@ export function objectWithBigNumberReviver(_: string, value: { type: string; hex } return toBN(value.hex); } + +export type ProviderErrorCount = { + chainId: number; + provider: string; + errorCount: number; + lastTime: number; +}; + +function providerErrorCountKey(provider: string, chainId: number): string { + return `provider_error_count_${provider}_${chainId}`; +} + +/** + * This function returns the ProviderErrorCount object for a given provider and chainId. + * @param client The redis client + * @param provider The provider name + * @param chainId The chainId + * @returns The ProviderErrorCount object. If the key doesn't exist, it will be created and returned with an errorCount of 0. + * @note This function will always return an object, even if the key doesn't exist. + * @note This function will mutate the redis database if the key doesn't exist. + */ +export async function getProviderErrorCount( + client: RedisClient, + provider: string, + chainId: number +): Promise { + // Resolve the key for convenience + const key = providerErrorCountKey(provider, chainId); + // Get the value from redis + const errorCount = await client.get(key); + // Attempt to parse the object OR an empty object + // Note: we want this to always return an object, even if it's empty + const parsedValue = JSON.parse(errorCount ?? "{}"); + // Check if the parsed value is a ProviderErrorCount + if (isProviderErrorCount(parsedValue)) { + // If it is, return it + return parsedValue; + } + // If not, create a new ProviderErrorCount object, set it + // in redis, and return it + else { + return setProviderErrorCount(client, provider, chainId, { + chainId, + provider, + errorCount: 0, + lastTime: getCurrentTime(), + }); + } +} + +/** + * This function sets the ProviderErrorCount object for a given provider and chainId. + * @param client The redis client + * @param provider The name of the provider + * @param chainId The chainId + * @param providerErrorCount The ProviderErrorCount object + * @returns The ProviderErrorCount object that was just set. + */ +export async function setProviderErrorCount( + client: RedisClient, + provider: string, + chainId: number, + providerErrorCount: ProviderErrorCount +): Promise { + // Resolve the key for convenience + const key = providerErrorCountKey(provider, chainId); + // Set the value in redis + await client.set(key, JSON.stringify(providerErrorCount), 86400); // Set the expiry to 1 day + // Return the value that was just set + return providerErrorCount; +} + +/** + * This function increments the errorCount for a given provider and chainId. It does so by using an exponential decay function. + * @param client The redis client + * @param provider The provider name + * @param chainId The chainId + * @returns The ProviderErrorCount object that was just updated. + */ +export async function incrementProviderErrorCount( + client: RedisClient, + provider: string, + chainId: number +): Promise { + // Grab the ProviderErrorCount object from redis + const providerErrorCount = await getProviderErrorCount(client, provider, chainId); + // Find the time since the last error and normalize by the seconds in a day + const timeSinceLastError = (getCurrentTime() - providerErrorCount.lastTime) / 86400; // 86400 seconds in a day + // Calculate the decay factor with a modified exponential decay function + // Note: the decay factor is calculated using the formula: e^(-t) where t is the time since the last error + // Note: as the time increases, we tend towards 0 + // Note: we can enforce that anything over 1 day old will have a decay factor of 0 + const decayFactor = timeSinceLastError >= 1.0 ? 0 : Math.exp(-timeSinceLastError); + // Calculate the new error count + const newErrorCount = providerErrorCount.errorCount * decayFactor + 1; + // Update the ProviderErrorCount object + const updatedProviderErrorCount = { + ...providerErrorCount, + errorCount: newErrorCount, + lastTime: getCurrentTime(), + }; + // Set the ProviderErrorCount object in redis and return it + return setProviderErrorCount(client, provider, chainId, updatedProviderErrorCount); +} diff --git a/src/utils/TypeGuards.ts b/src/utils/TypeGuards.ts index c8bdfa2b1..801d20f8b 100644 --- a/src/utils/TypeGuards.ts +++ b/src/utils/TypeGuards.ts @@ -1,4 +1,6 @@ import { utils } from "@across-protocol/sdk-v2"; +import * as superstruct from "superstruct"; +import { ProviderErrorCount } from "./RedisUtils"; export const { isDefined, isPromiseFulfilled, isPromiseRejected } = utils; @@ -13,3 +15,14 @@ export function isKeyOf( ): input is T { return input in obj; } + +const providerErrorCountSchema = superstruct.object({ + chainId: superstruct.integer(), + provider: superstruct.string(), + errorCount: superstruct.min(superstruct.number(), 0), + lastTime: superstruct.min(superstruct.integer(), 0), +}); + +export function isProviderErrorCount(input: unknown): input is ProviderErrorCount { + return providerErrorCountSchema.is(input); +}