(incident 2024-05-05): enable redis testing with clustering mode (#804)

fern-api · May 5, 2024 · 3f07e43 · 3f07e43
1 parent 6c2a278
commit 3f07e43
Show file tree

Hide file tree

Showing 9 changed files with 98 additions and 18 deletions.
diff --git a/servers/fdr-deploy/bin/fdr-deploy.ts b/servers/fdr-deploy/bin/fdr-deploy.ts
@@ -30,6 +30,7 @@ async function main() {
                         desiredTaskCount: 2,
                         maxTaskCount: 4,
                         redis: true,
+                        redisClusteringModeEnabled: false,
                         memory: 1024,
                         cpu: 512,
                     },
@@ -50,6 +51,7 @@ async function main() {
                         maxTaskCount: 1,
                         redis: false,
                         memory: 8192,
+                        redisClusteringModeEnabled: false,
                         cpu: 4096,
                     },
                     {

diff --git a/servers/fdr-deploy/scripts/fdr-deploy-stack.ts b/servers/fdr-deploy/scripts/fdr-deploy-stack.ts
@@ -34,6 +34,7 @@ interface ElastiCacheProps {
 
 interface FdrStackOptions {
     redis: boolean;
+    redisClusteringModeEnabled: boolean;
     maxTaskCount: number;
     desiredTaskCount: number;
     cpu: number;
@@ -145,6 +146,7 @@ export class FdrDeployStack extends Stack {
                     DOCS_CACHE_ENDPOINT: fernDocsCacheEndpoint,
                     ENABLE_CUSTOMER_NOTIFICATIONS: (environmentType === "PROD").toString(),
                     REDIS_ENABLED: options.redis.toString(),
+                    REDIS_CLUSTERING_MODE_ENABLED: options.redisClusteringModeEnabled.toString(),
                     APPLICATION_ENVIRONMENT: getEnvironmentVariableOrThrow("APPLICATION_ENVIRONMENT"),
                 },
                 containerName: CONTAINER_NAME,

diff --git a/servers/fdr/docker-compose.ete.yml b/servers/fdr/docker-compose.ete.yml
@@ -16,10 +16,18 @@ services:
       ALGOLIA_SEARCH_INDEX: dummy
       ALGOLIA_SEARCH_API_KEY: dummy
       SLACK_TOKEN: dummy
-      DOCS_CACHE_ENDPOINT: redis:6379
+      DOCS_CACHE_ENDPOINT: redis_0:6379
       ENABLE_CUSTOMER_NOTIFICATIONS: "false"
       REDIS_ENABLED: "true"
+      REDIS_CLUSTERING_ENABLED: "true"
       APPLICATION_ENVIRONMENT: ete
+    depends_on:
+      - redis_0
+      - redis_1
+      - redis_2
+      - redis_3
+      - redis_4
+      - redis_5
   postgres:
     image: postgres:10.3
     restart: always
@@ -35,8 +43,55 @@ services:
       - 9191:9191
     environment:
       initialBuckets: fdr
-  redis:
-    image: redis
+
+  redis_0:
+    image: bitnami/redis-cluster:latest
+    restart: unless-stopped
     ports:
       - 6379:6379
-    command: ["redis-server"]
+    environment:
+      ALLOW_EMPTY_PASSWORD: "yes"
+      REDIS_CLUSTER_CREATOR: "yes"
+      REDIS_CLUSTER_REPLICAS: 1
+      REDIS_NODES: redis_0 redis_1 redis_2 redis_3 redis_4 redis_5
+    depends_on:
+      - redis_1
+      - redis_2
+      - redis_3
+      - redis_4
+      - redis_5
+
+  redis_1:
+    image: bitnami/redis-cluster:latest
+    restart: unless-stopped
+    environment:
+      ALLOW_EMPTY_PASSWORD: "yes"
+      REDIS_NODES: redis_0 redis_1 redis_2 redis_3 redis_4 redis_5
+
+  redis_2:
+    image: bitnami/redis-cluster:latest
+    restart: unless-stopped
+    environment:
+      ALLOW_EMPTY_PASSWORD: "yes"
+      REDIS_NODES: redis_0 redis_1 redis_2 redis_3 redis_4 redis_5
+
+  redis_3:
+    image: bitnami/redis-cluster:latest
+    restart: unless-stopped
+    environment:
+      ALLOW_EMPTY_PASSWORD: "yes"
+      REDIS_NODES: redis_0 redis_1 redis_2 redis_3 redis_4 redis_5
+
+  redis_4:
+    image: bitnami/redis-cluster:latest
+    restart: unless-stopped
+    environment:
+      ALLOW_EMPTY_PASSWORD: "yes"
+      REDIS_NODES: redis_0 redis_1 redis_2 redis_3 redis_4 redis_5
+
+  redis_5:
+    image: bitnami/redis-cluster:latest
+    restart: unless-stopped
+    environment:
+      ALLOW_EMPTY_PASSWORD: "yes"
+      REDIS_NODES: redis_0 redis_1 redis_2 redis_3 redis_4 redis_5
diff --git a/servers/fdr/docker-compose.test.yml b/servers/fdr/docker-compose.test.yml
@@ -16,7 +16,7 @@ services:
     environment:
       initialBuckets: fdr
   redis:
-    image: redis:latest
+    image: redis
     ports:
       - 6379:6379
     command: ["redis-server"]
diff --git a/servers/fdr/redis.conf b/servers/fdr/redis.conf
@@ -0,0 +1,2 @@
+port 6379
+cluster-enabled yes
diff --git a/servers/fdr/src/__test__/mock.ts b/servers/fdr/src/__test__/mock.ts
@@ -123,6 +123,7 @@ export const baseMockFdrConfig: FdrConfig = {
     enableCustomerNotifications: false,
     applicationEnvironment: "mock",
     redisEnabled: false,
+    redisClusteringEnabled: false,
 };
 
 export function getMockFdrConfig(overrides?: Partial<FdrConfig>): FdrConfig {

diff --git a/servers/fdr/src/app/FdrApplication.ts b/servers/fdr/src/app/FdrApplication.ts
@@ -81,7 +81,10 @@ export class FdrApplication {
         this.dao = new FdrDao(prisma);
 
         this.redisDatastore = config.redisEnabled
-            ? new RedisDocsDefinitionStore(`redis://${this.config.docsCacheEndpoint}`)
+            ? new RedisDocsDefinitionStore({
+                  cacheEndpointUrl: `redis://${this.config.docsCacheEndpoint}`,
+                  clusterModeEnabled: config.redisClusteringEnabled,
+              })
             : undefined;
 
         this.docsDefinitionCache = new DocsDefinitionCacheImpl(

diff --git a/servers/fdr/src/app/FdrConfig.ts b/servers/fdr/src/app/FdrConfig.ts
@@ -14,6 +14,7 @@ const LOG_LEVEL_ENV_VAR = "LOG_LEVEL";
 const DOCS_CACHE_ENDPOINT_ENV_VAR = "DOCS_CACHE_ENDPOINT";
 const ENABLE_CUSTOMER_NOTIFICATIONS_ENV_VAR = "ENABLE_CUSTOMER_NOTIFICATIONS";
 const REDIS_ENABLED_ENV_VAR = "REDIS_ENABLED";
+const REDIS_CLUSTERING_ENABLED_ENV_VAR = "REDIS_CLUSTERING_ENABLED";
 const APPLICATION_ENVIRONMENT_ENV_VAR = "APPLICATION_ENVIRONMENT";
 
 export interface FdrConfig {
@@ -33,6 +34,7 @@ export interface FdrConfig {
     docsCacheEndpoint: string;
     enableCustomerNotifications: boolean;
     redisEnabled: boolean;
+    redisClusteringEnabled: boolean;
     applicationEnvironment: string;
 }
 
@@ -54,6 +56,7 @@ export function getConfig(): FdrConfig {
         docsCacheEndpoint: getEnvironmentVariableOrThrow(DOCS_CACHE_ENDPOINT_ENV_VAR),
         enableCustomerNotifications: getEnvironmentVariableOrThrow(ENABLE_CUSTOMER_NOTIFICATIONS_ENV_VAR) === "true",
         redisEnabled: process.env[REDIS_ENABLED_ENV_VAR] === "true",
+        redisClusteringEnabled: process.env[REDIS_CLUSTERING_ENABLED_ENV_VAR] === "true",
         applicationEnvironment: getEnvironmentVariableOrThrow(APPLICATION_ENVIRONMENT_ENV_VAR),
     };
 }

diff --git a/servers/fdr/src/services/docs-cache/RedisDocsDefinitionStore.ts b/servers/fdr/src/services/docs-cache/RedisDocsDefinitionStore.ts
@@ -1,23 +1,35 @@
-import { createClient, RedisClientType } from "redis";
+import { RedisClientType, RedisClusterType, createClient, createCluster } from "redis";
 import { LOGGER } from "../../app/FdrApplication";
 import { CachedDocsResponse } from "./DocsDefinitionCache";
 
+export declare namespace RedisDocsDefinitionStore {
+    interface Args {
+        clusterModeEnabled: boolean;
+        cacheEndpointUrl: string;
+    }
+}
+
 export default class RedisDocsDefinitionStore {
-    private client: RedisClientType;
+    private client: RedisClusterType | RedisClientType;
 
-    public constructor(cacheEndpointUrl: string) {
-        this.client = createClient({ url: cacheEndpointUrl, pingInterval: 10000 });
+    public constructor({ cacheEndpointUrl, clusterModeEnabled }: RedisDocsDefinitionStore.Args) {
+        this.client = clusterModeEnabled
+            ? createCluster({
+                  rootNodes: [
+                      {
+                          url: cacheEndpointUrl,
+                      },
+                  ],
+                  defaults: {
+                      pingInterval: 10000,
+                  },
+              })
+            : createClient({ url: cacheEndpointUrl, pingInterval: 10000 });
+        this.client.on("error", (error) => LOGGER.error("Encountered error from redis", error));
     }
 
     public async initializeCache(): Promise<void> {
-        this.client.on("error", (err) => {
-            LOGGER.error(`Supressed Redis client error: ${err}`);
-        });
-        try {
-            await this.client.connect();
-        } catch (err) {
-            LOGGER.error(`Supressed Redis client error: ${err}`);
-        }
+        await this.client.connect();
     }
 
     public async get({ url }: { url: URL }): Promise<CachedDocsResponse | null> {