(incident-2024-05-05): add a healthcheck that gets and sets a redis r…

…ecord (#803)

servers/fdr/docker-compose.ete.yml

@@ -16,9 +16,9 @@ services:
       ALGOLIA_SEARCH_INDEX: dummy
       ALGOLIA_SEARCH_API_KEY: dummy
       SLACK_TOKEN: dummy
-      DOCS_CACHE_ENDPOINT: dummy
+      DOCS_CACHE_ENDPOINT: redis:6379
       ENABLE_CUSTOMER_NOTIFICATIONS: "false"
-      REDIS_ENABLED: "false"
+      REDIS_ENABLED: "true"
       APPLICATION_ENVIRONMENT: ete
   postgres:
     image: postgres:10.3

servers/fdr/src/__test__/ete/ete.test.ts

@@ -1,10 +1,13 @@
-import axios from "axios";
-
 const PORT = 8080;
 
-// We don't spin up the server in this test.
 it("check health", async () => {
+    // wait ten seconds
+    await sleep(10000);
     // register empty definition
-    const healthResponse = await axios.get(`http://localhost:${PORT}/health`);
-    expect(healthResponse.status).toEqual(200);
-});
+    const response = await fetch(`http://localhost:${PORT}/health`);
+    expect(response.status).toEqual(200);
+}, 100_000);
+
+function sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

servers/fdr/src/app/FdrApplication.ts

@@ -46,6 +46,7 @@ export class FdrApplication {
     public readonly dao: FdrDao;
     public readonly docsDefinitionCache: DocsDefinitionCache;
     public readonly logger = LOGGER;
+    public readonly redisDatastore;
 
     public constructor(
         public readonly config: FdrConfig,
@@ -79,15 +80,15 @@ export class FdrApplication {
 
         this.dao = new FdrDao(prisma);
 
-        const redisDatastore = config.redisEnabled
+        this.redisDatastore = config.redisEnabled
             ? new RedisDocsDefinitionStore(`redis://${this.config.docsCacheEndpoint}`)
             : undefined;
 
         this.docsDefinitionCache = new DocsDefinitionCacheImpl(
             this,
             this.dao,
             new LocalDocsDefinitionStore(),
-            redisDatastore,
+            this.redisDatastore,
         );
 
         if ("prepareStackTrace" in Error) {

servers/fdr/src/healthchecks/checkRedis.ts

@@ -0,0 +1,55 @@
+import { LOGGER } from "../app/FdrApplication";
+import { CachedDocsResponse } from "../services/docs-cache/DocsDefinitionCache";
+import RedisDocsDefinitionStore from "../services/docs-cache/RedisDocsDefinitionStore";
+
+const HEALTHCHECK_KEY = "https://healthcheck.buildwithfern.com";
+const HEALTHCHECK_DOCS_RESPONSE: CachedDocsResponse = {
+    dbFiles: {},
+    isPrivate: true,
+    response: {
+        baseUrl: {
+            domain: "healthcheck.buildwithfern.com",
+        },
+        definition: {
+            pages: {},
+            apis: {},
+            config: {
+                navigation: {
+                    items: [],
+                },
+            },
+            files: {},
+            filesV2: {},
+            search: {
+                type: "singleAlgoliaIndex",
+                value: {
+                    type: "unversioned",
+                    indexSegment: {
+                        id: "healthcheck",
+                        searchApiKey: "dummy",
+                    },
+                },
+            },
+        },
+        lightModeEnabled: true,
+    },
+    updatedTime: new Date(),
+    version: "v1",
+};
+
+export async function checkRedis({ redis }: { redis: RedisDocsDefinitionStore }): Promise<boolean> {
+    try {
+        const healthcheckURL = new URL(HEALTHCHECK_KEY);
+        await redis.set({ url: healthcheckURL, value: HEALTHCHECK_DOCS_RESPONSE });
+        const record = await redis.get({ url: healthcheckURL });
+
+        if (record?.response.baseUrl.domain !== healthcheckURL.hostname) {
+            return false;
+        }
+
+        return true;
+    } catch (err) {
+        LOGGER.error("Encountered error while retrieving and storing redis entries", err);
+        return false;
+    }
+}

servers/fdr/src/server.ts

@@ -16,6 +16,7 @@ import { getDocsWriteV2Service } from "./controllers/docs/v2/getDocsWriteV2Servi
 import { getSnippetsFactoryService } from "./controllers/snippets/getSnippetsFactoryService";
 import { getSnippetsService } from "./controllers/snippets/getSnippetsService";
 import { getTemplatesService } from "./controllers/snippets/getTemplatesService";
+import { checkRedis } from "./healthchecks/checkRedis";
 
 const PORT = 8080;
 
@@ -53,14 +54,20 @@ expressApp.use(compression());
 
 const app = new FdrApplication(config);
 
-expressApp.get("/health", (_req, res) => {
+expressApp.get("/health", async (_req, res) => {
     const cacheInitialized = app.docsDefinitionCache.isInitialized();
     if (!cacheInitialized) {
         app.logger.error("The docs definition cache is not initilialized. Erroring the health check.");
         res.sendStatus(500);
-    } else {
-        res.sendStatus(200);
     }
+    if (app.redisDatastore != null) {
+        const redisHealthCheckSuccessful = await checkRedis({ redis: app.redisDatastore });
+        if (!redisHealthCheckSuccessful) {
+            app.logger.error("Records cannot be successfully written and read from redis");
+            res.sendStatus(500);
+        }
+    }
+    res.sendStatus(200);
 });
 
 void startServer();

servers/fdr/src/services/docs-cache/DocsDefinitionCache.ts

@@ -115,7 +115,7 @@ export class DocsDefinitionCacheImpl implements DocsDefinitionCache {
 
         // we don't want to cache from READ if we are currently updating the cache via WRITE
         if (!this.getDocsWriteMonitor(url.hostname).isLocked()) {
-            this.cacheResponse({ url, value: dbResponse });
+            await this.cacheResponse({ url, value: dbResponse });
         }
 
         if (dbResponse.isPrivate) {
@@ -169,15 +169,15 @@ export class DocsDefinitionCacheImpl implements DocsDefinitionCache {
                 return await this.getDocsWriteMonitor(docsUrl.hostname).use(async () => {
                     const url = docsUrl.toURL();
                     const dbResponse = await this.getDocsForUrlFromDatabase({ url });
-                    this.cacheResponse({ url, value: dbResponse });
+                    await this.cacheResponse({ url, value: dbResponse });
                 });
             }),
         );
     }
 
-    private cacheResponse({ url, value }: { url: URL; value: CachedDocsResponse }): void {
+    private async cacheResponse({ url, value }: { url: URL; value: CachedDocsResponse }): Promise<void> {
         if (this.redisDocsCache) {
-            this.redisDocsCache.set({ url, value });
+            await this.redisDocsCache.set({ url, value });
         }
         this.localDocsCache.set({ url, value });
     }

servers/fdr/src/services/docs-cache/RedisDocsDefinitionStore.ts

@@ -28,7 +28,7 @@ export default class RedisDocsDefinitionStore {
         return null;
     }
 
-    public set({ url, value }: { url: URL; value: CachedDocsResponse }): void {
-        this.client.set(url.hostname, JSON.stringify(value));
+    public async set({ url, value }: { url: URL; value: CachedDocsResponse }): Promise<void> {
+        await this.client.set(url.hostname, JSON.stringify(value));
     }
 }