feat: ✨ programmatically track provider errors in cache

package.json

@@ -37,6 +37,7 @@
     "lodash.get": "^4.4.2",
     "minimist": "^1.2.8",
     "redis4": "npm:redis@^4.1.0",
+    "superstruct": "^1.0.3",
     "ts-node": "^10.9.1",
     "winston": "^3.10.0",
     "zksync-web3": "^0.14.3"

src/utils/ProviderUtils.ts

@@ -4,7 +4,7 @@ import lodash from "lodash";
 import winston from "winston";
 import { isPromiseFulfilled, isPromiseRejected } from "./TypeGuards";
 import createQueue, { QueueObject } from "async/queue";
-import { getRedis, RedisClient, setRedisKey } from "./RedisUtils";
+import { getRedis, incrementProviderErrorCount, RedisClient, setRedisKey } from "./RedisUtils";
 import {
   CHAIN_CACHE_FOLLOW_DISTANCE,
   PROVIDER_CACHE_TTL,
@@ -242,7 +242,7 @@ export class RetryProvider extends ethers.providers.StaticJsonRpcProvider {
     readonly delay: number,
     readonly maxConcurrency: number,
     providerCacheNamespace: string,
-    redisClient?: RedisClient
+    readonly redisClient?: RedisClient
   ) {
     // Initialize the super just with the chainId, which stops it from trying to immediately send out a .send before
     // this derived class is initialized.
@@ -303,6 +303,21 @@ export class RetryProvider extends ethers.providers.StaticJsonRpcProvider {
 
     const results = await Promise.allSettled(requiredProviders.map(tryWithFallback));
 
+    // Iterate over all the errors and increment the error count for each provider.
+    await Promise.all(
+      errors.map(async ([provider]) => {
+        // Increment the error count for this provider.
+        // Note: redis must be enabled for this to work.
+        if (this.redisClient) {
+          await incrementProviderErrorCount(
+            this.redisClient,
+            getOriginFromURL(provider.connection.url),
+            this.network.chainId
+          );
+        }
+      })
+    );
+
     if (!results.every(isPromiseFulfilled)) {
       // Format the error so that it's very clear which providers failed and succeeded.
       const errorTexts = errors.map(([provider, errorText]) => formatProviderError(provider, errorText));

src/utils/RedisUtils.ts

@@ -1,4 +1,4 @@
-import { assert, toBN, BigNumberish, isDefined } from "./";
+import { assert, toBN, BigNumberish, isDefined, getCurrentTime, isProviderErrorCount } from "./";
 import { REDIS_URL_DEFAULT } from "../common/Constants";
 import { createClient } from "redis4";
 import winston from "winston";
@@ -153,3 +153,107 @@ export function objectWithBigNumberReviver(_: string, value: { type: string; hex
   }
   return toBN(value.hex);
 }
+
+export type ProviderErrorCount = {
+  chainId: number;
+  provider: string;
+  errorCount: number;
+  lastTime: number;
+};
+
+function providerErrorCountKey(provider: string, chainId: number): string {
+  return `provider_error_count_${provider}_${chainId}`;
+}
+
+/**
+ * This function returns the ProviderErrorCount object for a given provider and chainId.
+ * @param client The redis client
+ * @param provider The provider name
+ * @param chainId The chainId
+ * @returns The ProviderErrorCount object. If the key doesn't exist, it will be created and returned with an errorCount of 0.
+ * @note This function will always return an object, even if the key doesn't exist.
+ * @note This function will mutate the redis database if the key doesn't exist.
+ */
+export async function getProviderErrorCount(
+  client: RedisClient,
+  provider: string,
+  chainId: number
+): Promise<ProviderErrorCount> {
+  // Resolve the key for convenience
+  const key = providerErrorCountKey(provider, chainId);
+  // Get the value from redis
+  const errorCount = await client.get(key);
+  // Attempt to parse the object OR an empty object
+  // Note: we want this to always return an object, even if it's empty
+  const parsedValue = JSON.parse(errorCount ?? "{}");
+  // Check if the parsed value is a ProviderErrorCount
+  if (isProviderErrorCount(parsedValue)) {
+    // If it is, return it
+    return parsedValue;
+  }
+  // If not, create a new ProviderErrorCount object, set it
+  // in redis, and return it
+  else {
+    return setProviderErrorCount(client, provider, chainId, {
+      chainId,
+      provider,
+      errorCount: 0,
+      lastTime: getCurrentTime(),
+    });
+  }
+}
+
+/**
+ * This function sets the ProviderErrorCount object for a given provider and chainId.
+ * @param client The redis client
+ * @param provider The name of the provider
+ * @param chainId The chainId
+ * @param providerErrorCount The ProviderErrorCount object
+ * @returns The ProviderErrorCount object that was just set.
+ */
+export async function setProviderErrorCount(
+  client: RedisClient,
+  provider: string,
+  chainId: number,
+  providerErrorCount: ProviderErrorCount
+): Promise<ProviderErrorCount> {
+  // Resolve the key for convenience
+  const key = providerErrorCountKey(provider, chainId);
+  // Set the value in redis
+  await client.set(key, JSON.stringify(providerErrorCount), 86400); // Set the expiry to 1 day
+  // Return the value that was just set
+  return providerErrorCount;
+}
+
+/**
+ * This function increments the errorCount for a given provider and chainId. It does so by using an exponential decay function.
+ * @param client The redis client
+ * @param provider The provider name
+ * @param chainId The chainId
+ * @returns The ProviderErrorCount object that was just updated.
+ */
+export async function incrementProviderErrorCount(
+  client: RedisClient,
+  provider: string,
+  chainId: number
+): Promise<ProviderErrorCount> {
+  // Grab the ProviderErrorCount object from redis
+  const providerErrorCount = await getProviderErrorCount(client, provider, chainId);
+  // Find the time since the last error and normalize by the seconds in a day
+  const timeSinceLastError = (getCurrentTime() - providerErrorCount.lastTime) / 86400; // 86400 seconds in a day
+  // Calculate the decay factor with a modified exponential decay function
+  // Note: the decay factor is calculated using the formula: e^(-t) where t is the time since the last error
+  // Note: as the time increases, we tend towards 0
+  // Note: we can enforce that anything over 1 day old will have a decay factor of 0
+  const decayFactor = timeSinceLastError >= 1.0 ? 0 : Math.exp(-timeSinceLastError);
+  // Calculate the new error count
+  const newErrorCount = providerErrorCount.errorCount * decayFactor + 1;
+  // Update the ProviderErrorCount object
+  const updatedProviderErrorCount = {
+    ...providerErrorCount,
+    errorCount: newErrorCount,
+    lastTime: getCurrentTime(),
+  };
+  // Set the ProviderErrorCount object in redis and return it
+  return setProviderErrorCount(client, provider, chainId, updatedProviderErrorCount);
+}

src/utils/TypeGuards.ts

@@ -1,4 +1,6 @@
 import { utils } from "@across-protocol/sdk-v2";
+import * as superstruct from "superstruct";
+import { ProviderErrorCount } from "./RedisUtils";
 
 export const { isDefined, isPromiseFulfilled, isPromiseRejected } = utils;
 
@@ -13,3 +15,14 @@ export function isKeyOf<T extends V, V extends number | string | symbol>(
 ): input is T {
   return input in obj;
 }
+
+const providerErrorCountSchema = superstruct.object({
+  chainId: superstruct.integer(),
+  provider: superstruct.string(),
+  errorCount: superstruct.min(superstruct.number(), 0),
+  lastTime: superstruct.min(superstruct.integer(), 0),
+});
+
+export function isProviderErrorCount(input: unknown): input is ProviderErrorCount {
+  return providerErrorCountSchema.is(input);
+}