From 3f07e434a90019d48e0ffd57d588fbfdd82481ea Mon Sep 17 00:00:00 2001 From: Deep Singhvi Date: Sun, 5 May 2024 08:55:52 -0500 Subject: [PATCH] (incident 2024-05-05): enable redis testing with clustering mode (#804) --- servers/fdr-deploy/bin/fdr-deploy.ts | 2 + .../fdr-deploy/scripts/fdr-deploy-stack.ts | 2 + servers/fdr/docker-compose.ete.yml | 63 +++++++++++++++++-- servers/fdr/docker-compose.test.yml | 2 +- servers/fdr/redis.conf | 2 + servers/fdr/src/__test__/mock.ts | 1 + servers/fdr/src/app/FdrApplication.ts | 5 +- servers/fdr/src/app/FdrConfig.ts | 3 + .../docs-cache/RedisDocsDefinitionStore.ts | 36 +++++++---- 9 files changed, 98 insertions(+), 18 deletions(-) create mode 100644 servers/fdr/redis.conf diff --git a/servers/fdr-deploy/bin/fdr-deploy.ts b/servers/fdr-deploy/bin/fdr-deploy.ts index d2fc70c81e..04c5483dbb 100644 --- a/servers/fdr-deploy/bin/fdr-deploy.ts +++ b/servers/fdr-deploy/bin/fdr-deploy.ts @@ -30,6 +30,7 @@ async function main() { desiredTaskCount: 2, maxTaskCount: 4, redis: true, + redisClusteringModeEnabled: false, memory: 1024, cpu: 512, }, @@ -50,6 +51,7 @@ async function main() { maxTaskCount: 1, redis: false, memory: 8192, + redisClusteringModeEnabled: false, cpu: 4096, }, { diff --git a/servers/fdr-deploy/scripts/fdr-deploy-stack.ts b/servers/fdr-deploy/scripts/fdr-deploy-stack.ts index cdb342e001..6456f68c3e 100644 --- a/servers/fdr-deploy/scripts/fdr-deploy-stack.ts +++ b/servers/fdr-deploy/scripts/fdr-deploy-stack.ts @@ -34,6 +34,7 @@ interface ElastiCacheProps { interface FdrStackOptions { redis: boolean; + redisClusteringModeEnabled: boolean; maxTaskCount: number; desiredTaskCount: number; cpu: number; @@ -145,6 +146,7 @@ export class FdrDeployStack extends Stack { DOCS_CACHE_ENDPOINT: fernDocsCacheEndpoint, ENABLE_CUSTOMER_NOTIFICATIONS: (environmentType === "PROD").toString(), REDIS_ENABLED: options.redis.toString(), + REDIS_CLUSTERING_MODE_ENABLED: options.redisClusteringModeEnabled.toString(), APPLICATION_ENVIRONMENT: getEnvironmentVariableOrThrow("APPLICATION_ENVIRONMENT"), }, containerName: CONTAINER_NAME, diff --git a/servers/fdr/docker-compose.ete.yml b/servers/fdr/docker-compose.ete.yml index ac88e4a02e..1a78e62ce1 100644 --- a/servers/fdr/docker-compose.ete.yml +++ b/servers/fdr/docker-compose.ete.yml @@ -16,10 +16,18 @@ services: ALGOLIA_SEARCH_INDEX: dummy ALGOLIA_SEARCH_API_KEY: dummy SLACK_TOKEN: dummy - DOCS_CACHE_ENDPOINT: redis:6379 + DOCS_CACHE_ENDPOINT: redis_0:6379 ENABLE_CUSTOMER_NOTIFICATIONS: "false" REDIS_ENABLED: "true" + REDIS_CLUSTERING_ENABLED: "true" APPLICATION_ENVIRONMENT: ete + depends_on: + - redis_0 + - redis_1 + - redis_2 + - redis_3 + - redis_4 + - redis_5 postgres: image: postgres:10.3 restart: always @@ -35,8 +43,55 @@ services: - 9191:9191 environment: initialBuckets: fdr - redis: - image: redis + + redis_0: + image: bitnami/redis-cluster:latest + restart: unless-stopped ports: - 6379:6379 - command: ["redis-server"] + environment: + ALLOW_EMPTY_PASSWORD: "yes" + REDIS_CLUSTER_CREATOR: "yes" + REDIS_CLUSTER_REPLICAS: 1 + REDIS_NODES: redis_0 redis_1 redis_2 redis_3 redis_4 redis_5 + depends_on: + - redis_1 + - redis_2 + - redis_3 + - redis_4 + - redis_5 + + redis_1: + image: bitnami/redis-cluster:latest + restart: unless-stopped + environment: + ALLOW_EMPTY_PASSWORD: "yes" + REDIS_NODES: redis_0 redis_1 redis_2 redis_3 redis_4 redis_5 + + redis_2: + image: bitnami/redis-cluster:latest + restart: unless-stopped + environment: + ALLOW_EMPTY_PASSWORD: "yes" + REDIS_NODES: redis_0 redis_1 redis_2 redis_3 redis_4 redis_5 + + redis_3: + image: bitnami/redis-cluster:latest + restart: unless-stopped + environment: + ALLOW_EMPTY_PASSWORD: "yes" + REDIS_NODES: redis_0 redis_1 redis_2 redis_3 redis_4 redis_5 + + redis_4: + image: bitnami/redis-cluster:latest + restart: unless-stopped + environment: + ALLOW_EMPTY_PASSWORD: "yes" + REDIS_NODES: redis_0 redis_1 redis_2 redis_3 redis_4 redis_5 + + redis_5: + image: bitnami/redis-cluster:latest + restart: unless-stopped + environment: + ALLOW_EMPTY_PASSWORD: "yes" + REDIS_NODES: redis_0 redis_1 redis_2 redis_3 redis_4 redis_5 diff --git a/servers/fdr/docker-compose.test.yml b/servers/fdr/docker-compose.test.yml index 7dbddad4b5..f682402d1a 100644 --- a/servers/fdr/docker-compose.test.yml +++ b/servers/fdr/docker-compose.test.yml @@ -16,7 +16,7 @@ services: environment: initialBuckets: fdr redis: - image: redis:latest + image: redis ports: - 6379:6379 command: ["redis-server"] diff --git a/servers/fdr/redis.conf b/servers/fdr/redis.conf new file mode 100644 index 0000000000..91b1a58462 --- /dev/null +++ b/servers/fdr/redis.conf @@ -0,0 +1,2 @@ +port 6379 +cluster-enabled yes \ No newline at end of file diff --git a/servers/fdr/src/__test__/mock.ts b/servers/fdr/src/__test__/mock.ts index 84a4945a59..6bbcc87720 100644 --- a/servers/fdr/src/__test__/mock.ts +++ b/servers/fdr/src/__test__/mock.ts @@ -123,6 +123,7 @@ export const baseMockFdrConfig: FdrConfig = { enableCustomerNotifications: false, applicationEnvironment: "mock", redisEnabled: false, + redisClusteringEnabled: false, }; export function getMockFdrConfig(overrides?: Partial): FdrConfig { diff --git a/servers/fdr/src/app/FdrApplication.ts b/servers/fdr/src/app/FdrApplication.ts index d04c34aaf1..8b52e32216 100644 --- a/servers/fdr/src/app/FdrApplication.ts +++ b/servers/fdr/src/app/FdrApplication.ts @@ -81,7 +81,10 @@ export class FdrApplication { this.dao = new FdrDao(prisma); this.redisDatastore = config.redisEnabled - ? new RedisDocsDefinitionStore(`redis://${this.config.docsCacheEndpoint}`) + ? new RedisDocsDefinitionStore({ + cacheEndpointUrl: `redis://${this.config.docsCacheEndpoint}`, + clusterModeEnabled: config.redisClusteringEnabled, + }) : undefined; this.docsDefinitionCache = new DocsDefinitionCacheImpl( diff --git a/servers/fdr/src/app/FdrConfig.ts b/servers/fdr/src/app/FdrConfig.ts index c1438683de..40ee0bbdda 100644 --- a/servers/fdr/src/app/FdrConfig.ts +++ b/servers/fdr/src/app/FdrConfig.ts @@ -14,6 +14,7 @@ const LOG_LEVEL_ENV_VAR = "LOG_LEVEL"; const DOCS_CACHE_ENDPOINT_ENV_VAR = "DOCS_CACHE_ENDPOINT"; const ENABLE_CUSTOMER_NOTIFICATIONS_ENV_VAR = "ENABLE_CUSTOMER_NOTIFICATIONS"; const REDIS_ENABLED_ENV_VAR = "REDIS_ENABLED"; +const REDIS_CLUSTERING_ENABLED_ENV_VAR = "REDIS_CLUSTERING_ENABLED"; const APPLICATION_ENVIRONMENT_ENV_VAR = "APPLICATION_ENVIRONMENT"; export interface FdrConfig { @@ -33,6 +34,7 @@ export interface FdrConfig { docsCacheEndpoint: string; enableCustomerNotifications: boolean; redisEnabled: boolean; + redisClusteringEnabled: boolean; applicationEnvironment: string; } @@ -54,6 +56,7 @@ export function getConfig(): FdrConfig { docsCacheEndpoint: getEnvironmentVariableOrThrow(DOCS_CACHE_ENDPOINT_ENV_VAR), enableCustomerNotifications: getEnvironmentVariableOrThrow(ENABLE_CUSTOMER_NOTIFICATIONS_ENV_VAR) === "true", redisEnabled: process.env[REDIS_ENABLED_ENV_VAR] === "true", + redisClusteringEnabled: process.env[REDIS_CLUSTERING_ENABLED_ENV_VAR] === "true", applicationEnvironment: getEnvironmentVariableOrThrow(APPLICATION_ENVIRONMENT_ENV_VAR), }; } diff --git a/servers/fdr/src/services/docs-cache/RedisDocsDefinitionStore.ts b/servers/fdr/src/services/docs-cache/RedisDocsDefinitionStore.ts index 5ed4d1d2b3..334e1ee49f 100644 --- a/servers/fdr/src/services/docs-cache/RedisDocsDefinitionStore.ts +++ b/servers/fdr/src/services/docs-cache/RedisDocsDefinitionStore.ts @@ -1,23 +1,35 @@ -import { createClient, RedisClientType } from "redis"; +import { RedisClientType, RedisClusterType, createClient, createCluster } from "redis"; import { LOGGER } from "../../app/FdrApplication"; import { CachedDocsResponse } from "./DocsDefinitionCache"; +export declare namespace RedisDocsDefinitionStore { + interface Args { + clusterModeEnabled: boolean; + cacheEndpointUrl: string; + } +} + export default class RedisDocsDefinitionStore { - private client: RedisClientType; + private client: RedisClusterType | RedisClientType; - public constructor(cacheEndpointUrl: string) { - this.client = createClient({ url: cacheEndpointUrl, pingInterval: 10000 }); + public constructor({ cacheEndpointUrl, clusterModeEnabled }: RedisDocsDefinitionStore.Args) { + this.client = clusterModeEnabled + ? createCluster({ + rootNodes: [ + { + url: cacheEndpointUrl, + }, + ], + defaults: { + pingInterval: 10000, + }, + }) + : createClient({ url: cacheEndpointUrl, pingInterval: 10000 }); + this.client.on("error", (error) => LOGGER.error("Encountered error from redis", error)); } public async initializeCache(): Promise { - this.client.on("error", (err) => { - LOGGER.error(`Supressed Redis client error: ${err}`); - }); - try { - await this.client.connect(); - } catch (err) { - LOGGER.error(`Supressed Redis client error: ${err}`); - } + await this.client.connect(); } public async get({ url }: { url: URL }): Promise {