From 8a48c594fb127d27a165021ccba57e95e56f784d Mon Sep 17 00:00:00 2001 From: Dorseuil Nicolas Date: Tue, 14 Jan 2025 13:53:59 +0100 Subject: [PATCH 1/6] implement a multi-tiered cache for aws --- .../incrementalCache/multi-tier-ddb-s3.ts | 172 ++++++++++++++++++ .../src/overrides/incrementalCache/s3-lite.ts | 2 +- packages/open-next/src/types/open-next.ts | 7 +- 3 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts diff --git a/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts b/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts new file mode 100644 index 00000000..07ce5e64 --- /dev/null +++ b/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts @@ -0,0 +1,172 @@ +import type { CacheValue, IncrementalCache } from "types/overrides"; +import S3Cache, { getAwsClient } from "./s3-lite"; +import { customFetchClient } from "utils/fetch"; +import { debug } from "../../adapters/logger"; + +// TTL for the local cache in milliseconds +const localCacheTTL = process.env.OPEN_NEXT_LOCAL_CACHE_TTL + ? Number.parseInt(process.env.OPEN_NEXT_LOCAL_CACHE_TTL) + : 0; +// Maximum size of the local cache in nb of entries +const maxCacheSize = process.env.OPEN_NEXT_LOCAL_CACHE_SIZE + ? Number.parseInt(process.env.OPEN_NEXT_LOCAL_CACHE_SIZE) + : 1000; + +class LRUCache { + private cache: Map< + string, + { + value: CacheValue; + lastModified: number; + } + > = new Map(); + private maxSize: number; + + constructor(maxSize: number) { + this.maxSize = maxSize; + } + + // isFetch is not used here, only used for typing + get(key: string, isFetch?: T) { + return this.cache.get(key) as { + value: CacheValue; + lastModified: number; + }; + } + + set(key: string, value: any) { + if (this.cache.size >= this.maxSize) { + const firstKey = this.cache.keys().next().value; + if (firstKey) { + this.cache.delete(firstKey); + } + } + this.cache.set(key, value); + } + + delete(key: string) { + this.cache.delete(key); + } +} + +const localCache = new LRUCache(maxCacheSize); + +const awsFetch = (body: RequestInit["body"], type: "get" | "set" = "get") => { + const { CACHE_BUCKET_REGION } = process.env; + const client = getAwsClient(); + return customFetchClient(client)( + `https://dynamodb.${CACHE_BUCKET_REGION}.amazonaws.com`, + { + method: "POST", + headers: { + "Content-Type": "application/x-amz-json-1.0", + "X-Amz-Target": `DynamoDB_20120810.${ + type === "get" ? "GetItem" : "PutItem" + }`, + }, + body, + }, + ); +}; + +const buildDynamoKey = (key: string) => { + const { NEXT_BUILD_ID } = process.env; + return `__meta_${NEXT_BUILD_ID}_${key}`; +}; + +/** + * This cache implementation uses a multi-tier cache with a local cache, a DynamoDB metadata cache and an S3 cache. + * It uses the same DynamoDB table as the default tag cache and the same S3 bucket as the default incremental cache. + * It will first check the local cache. + * If the local cache is expired, it will check the DynamoDB metadata cache to see if the local cache is still valid. + * Lastly it will check the S3 cache. + */ +const multiTierCache: IncrementalCache = { + name: "multi-tier-ddb-s3", + async get(key, isFetch) { + // First we check the local cache + const localCacheEntry = localCache.get(key, isFetch); + if (localCacheEntry) { + if (Date.now() - localCacheEntry.lastModified < localCacheTTL) { + debug("Using local cache without checking ddb"); + return localCacheEntry; + } + try { + // Here we'll check ddb metadata to see if the local cache is still valid + const { CACHE_DYNAMO_TABLE } = process.env; + const result = await awsFetch( + JSON.stringify({ + TableName: CACHE_DYNAMO_TABLE, + Key: { + path: { S: buildDynamoKey(key) }, + tag: { S: buildDynamoKey(key) }, + }, + }), + ); + if (result.status === 200) { + const data = await result.json(); + const hasBeenDeleted = data.Item?.deleted?.BOOL; + if (hasBeenDeleted) { + localCache.delete(key); + return { value: undefined, lastModified: 0 }; + } + // If the metadata is older than the local cache, we can use the local cache + // If it's not found we assume that no write has been done yet and we can use the local cache + const lastModified = data.Item?.revalidatedAt?.N + ? Number.parseInt(data.Item.revalidatedAt.N) + : 0; + if (lastModified <= localCacheEntry.lastModified) { + debug("Using local cache after checking ddb"); + return localCacheEntry; + } + } + } catch (e) { + debug("Failed to get metadata from ddb", e); + } + } + const result = await S3Cache.get(key, isFetch); + if (result.value) { + localCache.set(key, { + value: result.value, + lastModified: result.lastModified ?? Date.now(), + }); + } + return result; + }, + async set(key, value, isFetch) { + const revalidatedAt = Date.now(); + await S3Cache.set(key, value, isFetch); + await awsFetch( + JSON.stringify({ + TableName: process.env.CACHE_DYNAMO_TABLE, + Item: { + tag: { S: buildDynamoKey(key) }, + path: { S: buildDynamoKey(key) }, + revalidatedAt: { N: String(revalidatedAt) }, + }, + }), + "set", + ); + localCache.set(key, { + value, + lastModified: revalidatedAt, + }); + }, + async delete(key) { + await S3Cache.delete(key); + await awsFetch( + JSON.stringify({ + TableName: process.env.CACHE_DYNAMO_TABLE, + Item: { + tag: { S: buildDynamoKey(key) }, + path: { S: buildDynamoKey(key) }, + deleted: { BOOL: true }, + }, + }), + "set", + ); + localCache.delete(key); + }, +}; + +export default multiTierCache; diff --git a/packages/open-next/src/overrides/incrementalCache/s3-lite.ts b/packages/open-next/src/overrides/incrementalCache/s3-lite.ts index d6937f69..e24db620 100644 --- a/packages/open-next/src/overrides/incrementalCache/s3-lite.ts +++ b/packages/open-next/src/overrides/incrementalCache/s3-lite.ts @@ -11,7 +11,7 @@ import { parseNumberFromEnv } from "../../adapters/util"; let awsClient: AwsClient | null = null; -const getAwsClient = () => { +export const getAwsClient = () => { const { CACHE_BUCKET_REGION } = process.env; if (awsClient) { return awsClient; diff --git a/packages/open-next/src/types/open-next.ts b/packages/open-next/src/types/open-next.ts index ca69a412..33070809 100644 --- a/packages/open-next/src/types/open-next.ts +++ b/packages/open-next/src/types/open-next.ts @@ -136,7 +136,12 @@ export interface MiddlewareResult export type IncludedQueue = "sqs" | "sqs-lite" | "direct" | "dummy"; -export type IncludedIncrementalCache = "s3" | "s3-lite" | "fs-dev" | "dummy"; +export type IncludedIncrementalCache = + | "s3" + | "s3-lite" + | "multi-tier-ddb-s3" + | "fs-dev" + | "dummy"; export type IncludedTagCache = | "dynamodb" From f6c291401bcfb8d65b80cd869f686d7f60ba8e98 Mon Sep 17 00:00:00 2001 From: Dorseuil Nicolas Date: Tue, 14 Jan 2025 14:15:11 +0100 Subject: [PATCH 2/6] fix linting --- .../src/overrides/incrementalCache/multi-tier-ddb-s3.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts b/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts index 07ce5e64..3da5c6da 100644 --- a/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts +++ b/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts @@ -1,7 +1,7 @@ import type { CacheValue, IncrementalCache } from "types/overrides"; -import S3Cache, { getAwsClient } from "./s3-lite"; import { customFetchClient } from "utils/fetch"; import { debug } from "../../adapters/logger"; +import S3Cache, { getAwsClient } from "./s3-lite"; // TTL for the local cache in milliseconds const localCacheTTL = process.env.OPEN_NEXT_LOCAL_CACHE_TTL From f1c4c839748c2ae7e9f0e339428ae3652504008c Mon Sep 17 00:00:00 2001 From: Dorseuil Nicolas Date: Tue, 14 Jan 2025 14:18:27 +0100 Subject: [PATCH 3/6] changeset --- .changeset/perfect-coats-tell.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/perfect-coats-tell.md diff --git a/.changeset/perfect-coats-tell.md b/.changeset/perfect-coats-tell.md new file mode 100644 index 00000000..65050401 --- /dev/null +++ b/.changeset/perfect-coats-tell.md @@ -0,0 +1,5 @@ +--- +"@opennextjs/aws": minor +--- + +Add a new multi-tiered incremental cache From 6695a2bad64be99c0f39f9b72ace8918c0cafa7c Mon Sep 17 00:00:00 2001 From: Dorseuil Nicolas Date: Tue, 14 Jan 2025 17:13:02 +0100 Subject: [PATCH 4/6] review fix --- .../incrementalCache/multi-tier-ddb-s3.ts | 51 ++++--------------- packages/open-next/src/utils/lru.ts | 28 ++++++++++ 2 files changed, 37 insertions(+), 42 deletions(-) create mode 100644 packages/open-next/src/utils/lru.ts diff --git a/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts b/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts index 3da5c6da..7e713a16 100644 --- a/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts +++ b/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts @@ -1,55 +1,22 @@ import type { CacheValue, IncrementalCache } from "types/overrides"; import { customFetchClient } from "utils/fetch"; +import { LRUCache } from "utils/lru"; import { debug } from "../../adapters/logger"; import S3Cache, { getAwsClient } from "./s3-lite"; // TTL for the local cache in milliseconds -const localCacheTTL = process.env.OPEN_NEXT_LOCAL_CACHE_TTL - ? Number.parseInt(process.env.OPEN_NEXT_LOCAL_CACHE_TTL) +const localCacheTTL = process.env.OPEN_NEXT_LOCAL_CACHE_TTL_MS + ? Number.parseInt(process.env.OPEN_NEXT_LOCAL_CACHE_TTL_MS, 10) : 0; // Maximum size of the local cache in nb of entries const maxCacheSize = process.env.OPEN_NEXT_LOCAL_CACHE_SIZE - ? Number.parseInt(process.env.OPEN_NEXT_LOCAL_CACHE_SIZE) + ? Number.parseInt(process.env.OPEN_NEXT_LOCAL_CACHE_SIZE, 10) : 1000; -class LRUCache { - private cache: Map< - string, - { - value: CacheValue; - lastModified: number; - } - > = new Map(); - private maxSize: number; - - constructor(maxSize: number) { - this.maxSize = maxSize; - } - - // isFetch is not used here, only used for typing - get(key: string, isFetch?: T) { - return this.cache.get(key) as { - value: CacheValue; - lastModified: number; - }; - } - - set(key: string, value: any) { - if (this.cache.size >= this.maxSize) { - const firstKey = this.cache.keys().next().value; - if (firstKey) { - this.cache.delete(firstKey); - } - } - this.cache.set(key, value); - } - - delete(key: string) { - this.cache.delete(key); - } -} - -const localCache = new LRUCache(maxCacheSize); +const localCache = new LRUCache<{ + value: CacheValue; + lastModified: number; +}>(maxCacheSize); const awsFetch = (body: RequestInit["body"], type: "get" | "set" = "get") => { const { CACHE_BUCKET_REGION } = process.env; @@ -85,7 +52,7 @@ const multiTierCache: IncrementalCache = { name: "multi-tier-ddb-s3", async get(key, isFetch) { // First we check the local cache - const localCacheEntry = localCache.get(key, isFetch); + const localCacheEntry = localCache.get(key); if (localCacheEntry) { if (Date.now() - localCacheEntry.lastModified < localCacheTTL) { debug("Using local cache without checking ddb"); diff --git a/packages/open-next/src/utils/lru.ts b/packages/open-next/src/utils/lru.ts new file mode 100644 index 00000000..f0ab508b --- /dev/null +++ b/packages/open-next/src/utils/lru.ts @@ -0,0 +1,28 @@ +export class LRUCache { + private cache: Map = new Map(); + + constructor(private maxSize: number) {} + + get(key: string) { + const result = this.cache.get(key); + if (result) { + this.cache.delete(key); + this.cache.set(key, result); + } + return result; + } + + set(key: string, value: any) { + if (this.cache.size >= this.maxSize) { + const firstKey = this.cache.keys().next().value; + if (firstKey) { + this.cache.delete(firstKey); + } + } + this.cache.set(key, value); + } + + delete(key: string) { + this.cache.delete(key); + } +} From 0b0ccc43648236df3ca69663464cf7358458f69e Mon Sep 17 00:00:00 2001 From: conico974 Date: Tue, 14 Jan 2025 23:27:09 +0100 Subject: [PATCH 5/6] Apply suggestions from code review Co-authored-by: Victor Berchet --- .../src/overrides/incrementalCache/multi-tier-ddb-s3.ts | 2 +- packages/open-next/src/utils/lru.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts b/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts index 7e713a16..2e2566b3 100644 --- a/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts +++ b/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts @@ -80,7 +80,7 @@ const multiTierCache: IncrementalCache = { // If the metadata is older than the local cache, we can use the local cache // If it's not found we assume that no write has been done yet and we can use the local cache const lastModified = data.Item?.revalidatedAt?.N - ? Number.parseInt(data.Item.revalidatedAt.N) + ? Number.parseInt(data.Item.revalidatedAt.N, 10) : 0; if (lastModified <= localCacheEntry.lastModified) { debug("Using local cache after checking ddb"); diff --git a/packages/open-next/src/utils/lru.ts b/packages/open-next/src/utils/lru.ts index f0ab508b..4a1eec03 100644 --- a/packages/open-next/src/utils/lru.ts +++ b/packages/open-next/src/utils/lru.ts @@ -15,7 +15,7 @@ export class LRUCache { set(key: string, value: any) { if (this.cache.size >= this.maxSize) { const firstKey = this.cache.keys().next().value; - if (firstKey) { + if (firstKey !== undefined) { this.cache.delete(firstKey); } } From 2f061864305081bd113936a394299fb9e2a3b8e2 Mon Sep 17 00:00:00 2001 From: Dorseuil Nicolas Date: Wed, 15 Jan 2025 12:14:07 +0100 Subject: [PATCH 6/6] added comment --- .../src/overrides/incrementalCache/multi-tier-ddb-s3.ts | 4 ++++ packages/open-next/src/utils/lru.ts | 2 ++ 2 files changed, 6 insertions(+) diff --git a/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts b/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts index 2e2566b3..bc2e86c4 100644 --- a/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts +++ b/packages/open-next/src/overrides/incrementalCache/multi-tier-ddb-s3.ts @@ -100,6 +100,10 @@ const multiTierCache: IncrementalCache = { } return result; }, + + // Both for set and delete we choose to do the write to S3 first and then to DynamoDB + // Which means that if it fails in DynamoDB, instance that don't have local cache will work as expected. + // But instance that have local cache will have a stale cache until the next working set or delete. async set(key, value, isFetch) { const revalidatedAt = Date.now(); await S3Cache.set(key, value, isFetch); diff --git a/packages/open-next/src/utils/lru.ts b/packages/open-next/src/utils/lru.ts index 4a1eec03..d2c4db6c 100644 --- a/packages/open-next/src/utils/lru.ts +++ b/packages/open-next/src/utils/lru.ts @@ -5,7 +5,9 @@ export class LRUCache { get(key: string) { const result = this.cache.get(key); + // We could have used .has to allow for nullish value to be stored but we don't need that right now if (result) { + // By removing and setting the key again we ensure it's the most recently used this.cache.delete(key); this.cache.set(key, result); }