From b7f4c99a3c4c9c401379b97d28a4c4f3696e2442 Mon Sep 17 00:00:00 2001 From: Clay Tercek <30105080+claytercek@users.noreply.github.com> Date: Wed, 6 Nov 2024 18:15:24 -0500 Subject: [PATCH 1/6] simplify sources syntax by removing neverthrow --- package-lock.json | 14 +- packages/content/package.json | 3 +- packages/content/src/content-config.ts | 4 +- packages/content/src/content-plugin-driver.ts | 1 - .../sources/__tests__/airtable-source.test.ts | 85 +---- .../__tests__/contentful-source.test.ts | 175 ++++------ .../src/sources/__tests__/json-source.test.ts | 83 ++--- .../sources/__tests__/sanity-source.test.ts | 117 +++---- .../sources/__tests__/strapi-source.test.ts | 137 +++----- .../content/src/sources/airtable-source.ts | 300 +++++++----------- .../content/src/sources/contentful-source.ts | 273 +++++++--------- packages/content/src/sources/json-source.ts | 76 ++--- packages/content/src/sources/sanity-source.ts | 213 +++++-------- packages/content/src/sources/source.ts | 61 +--- packages/content/src/sources/strapi-source.ts | 240 ++++++-------- packages/content/src/utils/fetch-paginated.ts | 62 ++-- 16 files changed, 686 insertions(+), 1158 deletions(-) diff --git a/package-lock.json b/package-lock.json index 659626f8..902dd427 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8640,6 +8640,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zod": { + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, "node_modules/zwitch": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", @@ -8664,7 +8673,7 @@ "yargs": "^17.7.2" }, "bin": { - "launchpad": "lib/cli.js" + "launchpad": "dist/cli.js" }, "devDependencies": { "@bluecadet/launchpad-tsconfig": "0.1.0", @@ -8709,7 +8718,8 @@ "p-queue": "^7.1.0", "path-scurry": "^2.0.0", "qs": "^6.11.1", - "sanitize-html": "^2.5.1" + "sanitize-html": "^2.5.1", + "zod": "^3.23.8" }, "devDependencies": { "@bluecadet/launchpad-testing": "0.1.0", diff --git a/packages/content/package.json b/packages/content/package.json index 8bddcdee..4111fa23 100644 --- a/packages/content/package.json +++ b/packages/content/package.json @@ -39,7 +39,8 @@ "p-queue": "^7.1.0", "path-scurry": "^2.0.0", "qs": "^6.11.1", - "sanitize-html": "^2.5.1" + "sanitize-html": "^2.5.1", + "zod": "^3.23.8" }, "optionalDependencies": { "@sanity/client": "^6.4.9", diff --git a/packages/content/src/content-config.ts b/packages/content/src/content-config.ts index ba4f742e..d628cf36 100644 --- a/packages/content/src/content-config.ts +++ b/packages/content/src/content-config.ts @@ -1,10 +1,10 @@ import type { ContentPlugin } from "./content-plugin-driver.js"; -import type { ContentSource, ContentSourceBuilder } from "./sources/source.js"; +import type { ContentSource } from "./sources/source.js"; export const DOWNLOAD_PATH_TOKEN = "%DOWNLOAD_PATH%"; export const TIMESTAMP_TOKEN = "%TIMESTAMP%"; -export type ConfigContentSource = ContentSource | Promise | ReturnType>; +export type ConfigContentSource = ContentSource | Promise; export type ContentConfig = { /** diff --git a/packages/content/src/content-plugin-driver.ts b/packages/content/src/content-plugin-driver.ts index b253906d..587916ea 100644 --- a/packages/content/src/content-plugin-driver.ts +++ b/packages/content/src/content-plugin-driver.ts @@ -1,7 +1,6 @@ import { type BaseHookContext, HookContextProvider, type Plugin, type PluginDriver } from "@bluecadet/launchpad-utils"; import type { ResolvedContentConfig } from "./content-config.js"; import type { DataStore } from "./utils/data-store.js"; -import chalk from "chalk"; export class ContentError extends Error { constructor(...args: ConstructorParameters) { diff --git a/packages/content/src/sources/__tests__/airtable-source.test.ts b/packages/content/src/sources/__tests__/airtable-source.test.ts index 71218c39..6366873e 100644 --- a/packages/content/src/sources/__tests__/airtable-source.test.ts +++ b/packages/content/src/sources/__tests__/airtable-source.test.ts @@ -4,7 +4,6 @@ import { setupServer } from "msw/node"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { DataStore } from "../../utils/data-store.js"; import airtableSource from "../airtable-source.js"; -import { SourceFetchError, SourceParseError } from "../source.js"; const server = setupServer(); @@ -69,29 +68,13 @@ describe("airtableSource", () => { tables: ["table1"], }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); + const result = source.fetch(createFetchContext()); - const result = await sourceValue.fetch(createFetchContext()); + expect(result).toHaveLength(1); - expect(result).toBeOk(); + const data = await result[0]!.data; - const fetchPromises = result._unsafeUnwrap(); - expect(fetchPromises).toHaveLength(1); - - const data = await fetchPromises[0]!.dataPromise; - - expect(data).toBeOk(); - - const tableData = data._unsafeUnwrap(); - expect(tableData).toHaveLength(2); // raw and simplified data - - // Check raw data - expect(tableData[0]!.id).toBe("table1.raw"); - expect(tableData[0]!.data).toHaveLength(2); - // Check simplified data - expect(tableData[1]!.id).toBe("table1"); - expect(tableData[1]!.data).toMatchInlineSnapshot(` + expect(data).toMatchInlineSnapshot(` [ { "id": "1", @@ -161,22 +144,12 @@ describe("airtableSource", () => { keyValueTables: ["settings"], }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); + const result = source.fetch(createFetchContext()); + expect(result).toHaveLength(1); - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeOk(); + const data = await result[0]!.data; - const tableData = data._unsafeUnwrap(); - expect(tableData).toHaveLength(2); // raw and simplified data - - // Check simplified data - expect(tableData[1]!.id).toBe("settings"); - expect(tableData[1]!.data).toEqual({ + expect(data).toEqual({ key1: "value1", key2: 123, // numeric string converted to number key3: true, // 'true' string converted to boolean @@ -198,17 +171,10 @@ describe("airtableSource", () => { tables: ["error-table"], }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); + const result = source.fetch(createFetchContext()); + expect(result).toHaveLength(1); - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeErr(); - expect(data._unsafeUnwrapErr()).toBeInstanceOf(SourceFetchError); - expect(data._unsafeUnwrapErr().message).toContain("Failed to fetch data from Airtable"); + expect(result[0]!.data).rejects.toThrow(); }); it("should handle invalid key-value table data", async () => { @@ -233,20 +199,10 @@ describe("airtableSource", () => { keyValueTables: ["invalid-table"], }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); + const result = source.fetch(createFetchContext()); + expect(result).toHaveLength(1); - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; - - expect(data).toBeErr(); - expect(data._unsafeUnwrapErr()).toBeInstanceOf(SourceParseError); - expect(data._unsafeUnwrapErr().message).toContain("Error processing table invalid-table from Airtable"); - // @ts-expect-error cause is unknown - expect(data._unsafeUnwrapErr().cause.message).toContain("At least 2 columns required"); + expect(result[0]!.data).rejects.toThrow(); }); it("should handle unauthorized access", async () => { @@ -263,16 +219,9 @@ describe("airtableSource", () => { tables: ["table"], }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); + const result = source.fetch(createFetchContext()); + expect(result).toHaveLength(1); - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeErr(); - expect(data._unsafeUnwrapErr()).toBeInstanceOf(SourceFetchError); - expect(data._unsafeUnwrapErr().message).toContain("Failed to fetch data from Airtable"); + expect(result[0]!.data).rejects.toThrow(); }); }); diff --git a/packages/content/src/sources/__tests__/contentful-source.test.ts b/packages/content/src/sources/__tests__/contentful-source.test.ts index 4dc54aaf..119899f2 100644 --- a/packages/content/src/sources/__tests__/contentful-source.test.ts +++ b/packages/content/src/sources/__tests__/contentful-source.test.ts @@ -4,7 +4,6 @@ import { setupServer } from "msw/node"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { DataStore } from "../../utils/data-store.js"; import contentfulSource from "../contentful-source.js"; -import { SourceConfigError, SourceFetchError } from "../source.js"; const server = setupServer(); @@ -29,31 +28,29 @@ function createFetchContext() { describe("contentfulSource", () => { it("should fail with missing delivery token when not using preview", async () => { - // @ts-expect-error - testing invalid options - const result = await contentfulSource({ - id: "test-contentful", - space: "test-space", - // missing deliveryToken - usePreviewApi: false, - }); - - expect(result).toBeErr(); - expect(result._unsafeUnwrapErr()).toBeInstanceOf(SourceConfigError); - expect(result._unsafeUnwrapErr().message).toContain("no deliveryToken is provided"); + expect( + async () => + // @ts-expect-error - testing invalid options + await contentfulSource({ + id: "test-contentful", + space: "test-space", + // missing deliveryToken + usePreviewApi: false, + }), + ).rejects.toThrow(); }); it("should fail with missing preview token when using preview", async () => { - // @ts-expect-error - testing invalid options - const result = await contentfulSource({ - id: "test-contentful", - space: "test-space", - // missing previewToken - usePreviewApi: true, - }); - - expect(result).toBeErr(); - expect(result._unsafeUnwrapErr()).toBeInstanceOf(SourceConfigError); - expect(result._unsafeUnwrapErr().message).toContain("no previewToken is provided"); + expect( + async () => + // @ts-expect-error - testing invalid options + await contentfulSource({ + id: "test-contentful", + space: "test-space", + // missing previewToken + usePreviewApi: true, + }), + ).rejects.toThrow(); }); it("should fetch data with delivery token", async () => { @@ -105,26 +102,15 @@ describe("contentfulSource", () => { usePreviewApi: false, }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); - - const fetchPromises = result._unsafeUnwrap(); - expect(fetchPromises).toHaveLength(1); + const result = await source.fetch(createFetchContext()); - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeOk(); + const data = await result.data; - const content = data._unsafeUnwrap(); - expect(content).toHaveLength(1); - expect(content[0]!.id).toBe("content.json"); - expect(content[0]!.data.entries).toHaveLength(1); - expect(content[0]!.data.assets).toHaveLength(1); + expect(data.entries).toHaveLength(1); + expect(data.assets).toHaveLength(1); // Check first entry - expect(content[0]!.data.entries[0]).toEqual({ + expect(data.entries[0]).toEqual({ sys: { type: "Entry", contentType: { sys: { id: "article" } } }, fields: { title: "Test Entry" }, }); @@ -178,54 +164,41 @@ describe("contentfulSource", () => { usePreviewApi: true, }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); - - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeOk(); - - const content = data._unsafeUnwrap(); - - expect(content).toMatchInlineSnapshot(` - [ - { - "data": { - "assets": [ - { - "fields": { - "file": { - "url": "//test.com/preview.jpg", - }, - "title": "Preview Asset", - }, - "sys": { - "type": "Asset", - }, + const result = source.fetch(createFetchContext()); + + const data = await result.data; + + expect(data).toMatchInlineSnapshot(` + { + "assets": [ + { + "fields": { + "file": { + "url": "//test.com/preview.jpg", }, - ], - "entries": [ - { - "fields": { - "title": "Preview Entry", - }, + "title": "Preview Asset", + }, + "sys": { + "type": "Asset", + }, + }, + ], + "entries": [ + { + "fields": { + "title": "Preview Entry", + }, + "sys": { + "contentType": { "sys": { - "contentType": { - "sys": { - "id": "article", - }, - }, - "type": "Entry", + "id": "article", }, }, - ], + "type": "Entry", + }, }, - "id": "content.json", - }, - ] + ], + } `); }); @@ -243,17 +216,9 @@ describe("contentfulSource", () => { retryOnError: false, }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); + const result = source.fetch(createFetchContext()); - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeErr(); - expect(data._unsafeUnwrapErr()).toBeInstanceOf(SourceFetchError); - expect(data._unsafeUnwrapErr().message).toContain("Error fetching page"); + expect(result.data).rejects.toThrow(); }); it("should handle contentful errors array", async () => { @@ -272,17 +237,9 @@ describe("contentfulSource", () => { deliveryToken: "test-token", }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); + const result = source.fetch(createFetchContext()); - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeErr(); - expect(data._unsafeUnwrapErr()).toBeInstanceOf(SourceFetchError); - expect(data._unsafeUnwrapErr().message).toContain("Invalid content type"); + expect(result.data).rejects.toThrow(); }); it("should respect content type filtering", async () => { @@ -328,18 +285,10 @@ describe("contentfulSource", () => { retryOnError: false, }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); - - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; + const result = source.fetch(createFetchContext()); - expect(data).toBeOk(); + const data = await result.data; - const content = data._unsafeUnwrap(); - expect(content[0]!.data.entries.every((entry) => entry.sys.contentType.sys.id === "article")).toBe(true); + expect(data.entries.every((entry) => entry.sys.contentType.sys.id === "article")).toBe(true); }); }); diff --git a/packages/content/src/sources/__tests__/json-source.test.ts b/packages/content/src/sources/__tests__/json-source.test.ts index ad3f91e0..d14968b9 100644 --- a/packages/content/src/sources/__tests__/json-source.test.ts +++ b/packages/content/src/sources/__tests__/json-source.test.ts @@ -4,7 +4,6 @@ import { setupServer } from "msw/node"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { DataStore } from "../../utils/data-store.js"; import jsonSource from "../json-source.js"; -import { SourceFetchError, SourceParseError } from "../source.js"; const server = setupServer(); @@ -46,26 +45,18 @@ describe("jsonSource", () => { }, }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); + const result = source.fetch(createFetchContext()); + expect(result).toBeInstanceOf(Array); + expect(result).toHaveLength(2); - const result = await sourceValue.fetch(createFetchContext()); + const data1 = await result[0]!.data; + const data2 = await result[1]!.data; - expect(result).toBeOk(); - const fetchPromises = result._unsafeUnwrap(); - expect(fetchPromises).toHaveLength(2); - - const data1 = await fetchPromises[0]!.dataPromise; - const data2 = await fetchPromises[1]!.dataPromise; - - expect(data1).toBeOk(); - expect(data2).toBeOk(); - - expect(data1._unsafeUnwrap()).toEqual([{ id: "data1.json", data: { key: "value1" } }]); - expect(data2._unsafeUnwrap()).toEqual([{ id: "data2.json", data: { key: "value2" } }]); + expect(data1).toEqual({ key: "value1" }); + expect(data2).toEqual({ key: "value2" }); }); - it("should handle fetch errors", async () => { + it("should throw on fetch errors", async () => { server.use( http.get("https://api.example.com/error", () => { return HttpResponse.error(); @@ -79,21 +70,15 @@ describe("jsonSource", () => { }, }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); + const result = source.fetch(createFetchContext()); + expect(result).toHaveLength(1); - expect(result).toBeOk(); - const fetchPromises = result._unsafeUnwrap(); - expect(fetchPromises).toHaveLength(1); - - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeErr(); - expect(data._unsafeUnwrapErr()).toBeInstanceOf(SourceFetchError); + expect(async () => { + await result[0]!.data; + }).rejects.toThrow(); }); - it("should handle parse errors", async () => { + it("should throw on parse errors", async () => { server.use( http.get("https://api.example.com/invalid", () => { return new HttpResponse("Invalid JSON", { @@ -109,18 +94,13 @@ describe("jsonSource", () => { }, }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); + const result = source.fetch(createFetchContext()); - expect(result).toBeOk(); - const fetchPromises = result._unsafeUnwrap(); - expect(fetchPromises).toHaveLength(1); + expect(result).toHaveLength(1); - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeErr(); - expect(data._unsafeUnwrapErr()).toBeInstanceOf(SourceParseError); + expect(async () => { + await result[0]!.data; + }).rejects.toThrow(); }); it("should respect the maxTimeout option", async () => { @@ -138,25 +118,18 @@ describe("jsonSource", () => { }, maxTimeout: 1000, }); + const result = source.fetch(createFetchContext()); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); + const promise = result[0]!.data; - const result = await sourceValue.fetch(createFetchContext()); + expect(promise).rejects.toThrow(); - vi.advanceTimersByTime(1000); - - expect(result).toBeOk(); - const fetchPromises = result._unsafeUnwrap(); - expect(fetchPromises).toHaveLength(1); + // Need to run the timer and wait for the rejection + await vi.runAllTimersAsync(); + }); - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeErr(); - expect(data._unsafeUnwrapErr()).toBeInstanceOf(SourceFetchError); - expect(data._unsafeUnwrapErr().message).toContain("Could not fetch json from https://api.example.com/slow"); - // @ts-expect-error cause is unknown - expect(data._unsafeUnwrapErr().cause.message).toContain("Error during request"); - // @ts-expect-error cause is unknown - expect(data._unsafeUnwrapErr().cause.cause.message).toContain("Request timed out"); + it("should throw on incomplete config", async () => { + // @ts-expect-error - incomplete config + expect(() => jsonSource({})).toThrow(); }); }); diff --git a/packages/content/src/sources/__tests__/sanity-source.test.ts b/packages/content/src/sources/__tests__/sanity-source.test.ts index edd03471..ce402691 100644 --- a/packages/content/src/sources/__tests__/sanity-source.test.ts +++ b/packages/content/src/sources/__tests__/sanity-source.test.ts @@ -4,7 +4,6 @@ import { setupServer } from "msw/node"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { DataStore } from "../../utils/data-store.js"; import sanitySource from "../sanity-source.js"; -import { SourceConfigError, SourceFetchError } from "../source.js"; const server = setupServer(); @@ -30,14 +29,12 @@ function createFetchContext() { describe("sanitySource", () => { it("should fail with missing required options", async () => { // @ts-expect-error - testing invalid options - const result = await sanitySource({ + const result = sanitySource({ id: "test-sanity", // missing projectId and apiToken }); - expect(result).toBeErr(); - expect(result._unsafeUnwrapErr()).toBeInstanceOf(SourceConfigError); - expect(result._unsafeUnwrapErr().message).toContain("Missing projectId and/or apiToken"); + expect(result).rejects.toThrow(); }); it("should fetch data with simple type queries", async () => { @@ -49,28 +46,28 @@ describe("sanitySource", () => { const query = url.searchParams.get("query"); - if (query === '*[_type == "test" ][0..99]') { + if (query === '*[_type == "test"][0..99]') { return HttpResponse.json({ result: [{ _type: "test", title: "Test Document 1" }], ms: 15, }); } - if (query === '*[_type == "test" ][100..199]') { + if (query === '*[_type == "test"][100..199]') { return HttpResponse.json({ result: [{ _type: "test", title: "Test Document 2" }], ms: 15, }); } - if (query === '*[_type == "article" ][0..99]') { + if (query === '*[_type == "article"][0..99]') { return HttpResponse.json({ result: [{ _type: "article", title: "Article 1" }], ms: 15, }); } - if (query === '*[_type == "article" ][100..199]') { + if (query === '*[_type == "article"][100..199]') { return HttpResponse.json({ result: [{ _type: "article", title: "Article 2" }], ms: 15, @@ -89,41 +86,27 @@ describe("sanitySource", () => { projectId: "test-project", apiToken: "test-token", queries: ["test", "article"], + mergePages: true, + useCdn: false, }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); + const result = source.fetch(createFetchContext()); - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); - - const fetchPromises = result._unsafeUnwrap(); - expect(fetchPromises).toHaveLength(2); + expect(result).toHaveLength(2); // Check 'test' type results - const testData = await fetchPromises[0]!.dataPromise; - expect(testData).toBeOk(); - expect(testData._unsafeUnwrap()).toEqual([ - { - id: "test", - data: [ - { _type: "test", title: "Test Document 1" }, - { _type: "test", title: "Test Document 2" }, - ], - }, + const testData = await result[0]!.data; + + expect(testData).toEqual([ + { _type: "test", title: "Test Document 1" }, + { _type: "test", title: "Test Document 2" }, ]); // Check 'article' type results - const articleData = await fetchPromises[1]!.dataPromise; - expect(articleData).toBeOk(); - expect(articleData._unsafeUnwrap()).toEqual([ - { - id: "article", - data: [ - { _type: "article", title: "Article 1" }, - { _type: "article", title: "Article 2" }, - ], - }, + const articleData = await result[1]!.data; + expect(articleData).toEqual([ + { _type: "article", title: "Article 1" }, + { _type: "article", title: "Article 2" }, ]); }); @@ -151,6 +134,8 @@ describe("sanitySource", () => { id: "test-sanity", projectId: "test-project", apiToken: "test-token", + mergePages: true, + useCdn: false, queries: [ { id: "custom", @@ -159,23 +144,13 @@ describe("sanitySource", () => { ], }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); - - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeOk(); - expect(data._unsafeUnwrap()).toEqual([ - { - id: "custom", - data: [ - { _type: "custom", data: "Custom Data" }, - { _type: "custom", data: "Custom Data" }, - ], - }, + const result = source.fetch(createFetchContext()); + expect(result).toHaveLength(1); + + const data = await result[0]!.data; + expect(data).toEqual([ + { _type: "custom", data: "Custom Data" }, + { _type: "custom", data: "Custom Data" }, ]); }); @@ -191,19 +166,13 @@ describe("sanitySource", () => { projectId: "test-project", apiToken: "test-token", queries: ["test"], + mergePages: true, + useCdn: false, }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); + const result = source.fetch(createFetchContext()); - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); - - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeErr(); - expect(data._unsafeUnwrapErr()).toBeInstanceOf(SourceFetchError); - expect(data._unsafeUnwrapErr().message).toContain("Could not fetch page"); + expect(result[0]!.data).rejects.toThrow(); }); it("should respect pagination options", async () => { @@ -213,7 +182,7 @@ describe("sanitySource", () => { const query = url.searchParams.get("query") || ""; const offset = query.match(/\[(\d+)\.\./)?.at(1); - if (offset === "50") { + if (offset === "100") { return HttpResponse.json({ result: [], ms: 5, @@ -234,22 +203,16 @@ describe("sanitySource", () => { queries: ["test"], limit: 50, mergePages: false, - pageNumZeroPad: 2, + useCdn: false, }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); + const result = source.fetch(createFetchContext()); + expect(result).toHaveLength(1); - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeOk(); + const data = (await result[0]!.data) as AsyncGenerator; - // Check that pagination formatting is correct - const pages = data._unsafeUnwrap(); - expect(pages[0]!.id).toBe("test-01"); - expect(pages[0]!.data).toEqual([{ _type: "test", title: "Test Document 0" }]); + expect((await data.next()).value).toEqual([{ _type: "test", title: "Test Document 0" }]); + expect((await data.next()).value).toEqual([{ _type: "test", title: "Test Document 50" }]); + expect((await data.next()).done).toBe(true); }); }); diff --git a/packages/content/src/sources/__tests__/strapi-source.test.ts b/packages/content/src/sources/__tests__/strapi-source.test.ts index 6b594d42..72e3e5f3 100644 --- a/packages/content/src/sources/__tests__/strapi-source.test.ts +++ b/packages/content/src/sources/__tests__/strapi-source.test.ts @@ -3,7 +3,6 @@ import { http, HttpResponse } from "msw"; import { setupServer } from "msw/node"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { DataStore } from "../../utils/data-store.js"; -import { SourceConfigError, SourceFetchError } from "../source.js"; import strapiSource from "../strapi-source.js"; const server = setupServer(); @@ -29,19 +28,17 @@ function createFetchContext() { describe("strapiSource", () => { it("should fail with unsupported version", async () => { - const result = await strapiSource({ - id: "test-strapi", - baseUrl: "http://localhost:1337", - identifier: "test@example.com", - password: "password", - // @ts-expect-error - testing invalid version - version: "5", - queries: ["test-content"], - }); - - expect(result).toBeErr(); - expect(result._unsafeUnwrapErr()).toBeInstanceOf(SourceConfigError); - expect(result._unsafeUnwrapErr().message).toContain("Unsupported strapi version"); + expect(() => + strapiSource({ + id: "test-strapi", + baseUrl: "http://localhost:1337", + identifier: "test@example.com", + password: "password", + // @ts-expect-error - testing invalid version + version: "5", + queries: ["test-content"], + }), + ).rejects.toThrow(); }); describe("Strapi v4", () => { @@ -113,33 +110,24 @@ describe("strapiSource", () => { queries: ["test-content"], }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); + const result = source.fetch(createFetchContext()); + expect(result).toHaveLength(1); - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeOk(); + const data = (await result[0]!.data.next()).value; - const content = data._unsafeUnwrap(); - expect(content).toMatchInlineSnapshot(` + expect(data).toMatchInlineSnapshot(` [ { - "data": [ - { - "attributes": { - "description": "Test Description", - "title": "Test Content", - }, - "id": 1, - }, - ], - "id": "test-content-01.json", + "attributes": { + "description": "Test Description", + "title": "Test Content", + }, + "id": 1, }, ] `); + + expect((await result[0]!.data.next()).done).toBe(true); }); it("should handle custom query objects", async () => { @@ -200,19 +188,11 @@ describe("strapiSource", () => { ], }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); + const result = source.fetch(createFetchContext()); - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; + const data = (await result[0]!.data.next()).value; - expect(data).toBeOk(); - - const content = data._unsafeUnwrap(); - expect(content[0]!.data).toMatchInlineSnapshot(` + expect(data).toMatchInlineSnapshot(` [ { "attributes": { @@ -223,6 +203,8 @@ describe("strapiSource", () => { }, ] `); + + expect((await result[0]!.data.next()).done).toBe(true); }); }); @@ -271,31 +253,21 @@ describe("strapiSource", () => { queries: ["test-content"], }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); + const result = source.fetch(createFetchContext()); + expect(result).toHaveLength(1); - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); - - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; - expect(data).toBeOk(); - - const content = data._unsafeUnwrap(); - expect(content).toMatchInlineSnapshot(` + const data = (await result[0]!.data.next()).value; + expect(data).toMatchInlineSnapshot(` [ { - "data": [ - { - "description": "Test Description V3", - "id": 1, - "title": "Test Content V3", - }, - ], - "id": "test-content-01.json", + "description": "Test Description V3", + "id": 1, + "title": "Test Content V3", }, ] `); + + expect((await result[0]!.data.next()).done).toBe(true); }); }); @@ -306,19 +278,16 @@ describe("strapiSource", () => { }), ); - const source = await strapiSource({ - id: "test-strapi", - version: "4", - baseUrl: "http://localhost:1337", - identifier: "test@example.com", - password: "wrong-password", - queries: ["test-content"], - }); - - expect(source).toBeErr(); - const sourceValue = source._unsafeUnwrapErr(); - expect(sourceValue).toBeInstanceOf(SourceFetchError); - expect(sourceValue.message).toContain("Could not complete request to get JWT for test@example.com"); + expect( + strapiSource({ + id: "test-strapi", + version: "4", + baseUrl: "http://localhost:1337", + identifier: "test@example.com", + password: "wrong-password", + queries: ["test-content"], + }), + ).rejects.toThrow(); }); it("should handle API errors", async () => { @@ -340,17 +309,9 @@ describe("strapiSource", () => { queries: ["test-content"], }); - expect(source).toBeOk(); - const sourceValue = source._unsafeUnwrap(); - - const result = await sourceValue.fetch(createFetchContext()); - expect(result).toBeOk(); - - const fetchPromises = result._unsafeUnwrap(); - const data = await fetchPromises[0]!.dataPromise; + const result = source.fetch(createFetchContext()); + expect(result).toHaveLength(1); - expect(data).toBeErr(); - expect(data._unsafeUnwrapErr()).toBeInstanceOf(SourceFetchError); - expect(data._unsafeUnwrapErr().message).toContain("Could not fetch page"); + expect(async () => (await result[0]!.data.next()).value).rejects.toThrow(); }); }); diff --git a/packages/content/src/sources/airtable-source.ts b/packages/content/src/sources/airtable-source.ts index bf1edfe4..8e4afc5c 100644 --- a/packages/content/src/sources/airtable-source.ts +++ b/packages/content/src/sources/airtable-source.ts @@ -1,88 +1,66 @@ import type { Logger } from "@bluecadet/launchpad-utils"; import type Airtable from "airtable"; -import { type Result, ResultAsync, err, errAsync, ok, okAsync } from "neverthrow"; -import { SourceConfigError, SourceFetchError, SourceMissingDependencyError, SourceParseError, defineSource, type SourceFetchPromise } from "./source.js"; - -type AirtableOptions = { - /** - * Required field to identify this source. Will be used as download path. - */ - id: string; - /** - * Airtable base ID. See https://help.appsheet.com/en/articles/1785063-using-data-from-airtable#:~:text=To%20obtain%20the%20ID%20of,API%20page%20of%20the%20base. - */ - baseId: string; - /** - * The table view which to select for syncing by default. Defaults to 'Grid view'. - */ - defaultView?: string; - /** - * The tables you want to fetch from. Defaults to []. - */ - tables?: string[]; - /** - * As a convenience feature, you can store tables listed here as key/value pairs. Field names should be `"key"` and `"value"`. Defaults to []. - */ - keyValueTables?: string[]; - /** - * The API endpoint to use for Airtable. Defaults to 'https://api.airtable.com'. - */ - endpointUrl?: string; - /** - * Appends the local path of attachments to the saved JSON. Defaults to true. - */ - appendLocalAttachmentPaths?: boolean; - /** - * Airtable API Key - */ - apiKey: string; -}; - -type AirtableOptionsAssembled = Required; - -const AIRTABLE_OPTION_DEFAULTS = { - defaultView: "Grid view", - tables: [], - keyValueTables: [], - endpointUrl: "https://api.airtable.com", - appendLocalAttachmentPaths: true, -} satisfies Partial; +import { defineSource, type SourceFetchResultDocument } from "./source.js"; +import { z } from "zod"; + +const airtableSourceSchema = z.object({ + /** Required field to identify this source. Will be used as download path. */ + id: z.string().describe("Required field to identify this source. Will be used as download path."), + /** Airtable base ID. See https://help.appsheet.com/en/articles/1785063-using-data-from-airtable#:~:text=To%20obtain%20the%20ID%20of,API%20page%20of%20the%20base. */ + baseId: z + .string() + .describe( + "Airtable base ID. See https://help.appsheet.com/en/articles/1785063-using-data-from-airtable#:~:text=To%20obtain%20the%20ID%20of,API%20page%20of%20the%20base.", + ), + /** The table view which to select for syncing by default. Defaults to 'Grid view'. */ + defaultView: z.string().describe("The table view which to select for syncing by default. Defaults to 'Grid view'.").default("Grid view"), + /** The tables you want to fetch from. Defaults to []. */ + tables: z.array(z.string()).describe("The tables you want to fetch from. Defaults to [].").default([]), + /** As a convenience feature, you can store tables listed here as key/value pairs. Field names should be `key` and `value`. Defaults to []. */ + keyValueTables: z + .array(z.string()) + .describe("As a convenience feature, you can store tables listed here as key/value pairs. Field names should be `key` and `value`. Defaults to [].") + .default([]), + /** The API endpoint to use for Airtable. Defaults to 'https://api.airtable.com'. */ + endpointUrl: z.string().describe("The API endpoint to use for Airtable. Defaults to 'https://api.airtable.com'.").default("https://api.airtable.com"), + /** Appends the local path of attachments to the saved JSON. Defaults to true. */ + appendLocalAttachmentPaths: z.boolean().describe("Appends the local path of attachments to the saved JSON. Defaults to true.").default(true), + /** Airtable API Key */ + apiKey: z.string().describe("Airtable API Key"), +}); /** * Fetch data from Airtable. */ -function fetchData(base: Airtable.Base, tableId: string, defaultView: string): ResultAsync[], SourceFetchError> { - return ResultAsync.fromPromise( - new Promise((resolve, reject) => { - const rows: Airtable.Record[] = []; - - base - .table(tableId) - .select({ - view: defaultView, - }) - .eachPage( - (records, fetchNextPage) => { - // This function (`page`) will get called for each page of records. - for (const record of records) { - rows.push(record); - } - - // To fetch the next page of records, call `fetchNextPage`. - // If there are more records, `page` will get called again. - // If there are no more records, `done` will get called. - fetchNextPage(); - }, - (error) => { - if (error) { - reject(error); - } else { - resolve(rows); - } - }, - ); - }), - (error) => new SourceFetchError("Failed to fetch data from Airtable", { cause: error }), +function fetchData(base: Airtable.Base, tableId: string, defaultView: string): Promise[]> { + const rows: Airtable.Record[] = []; + + return new Promise((resolve, reject) => + base + .table(tableId) + .select({ + view: defaultView, + }) + .eachPage( + (records, fetchNextPage) => { + // This function (`page`) will get called for each page of records. + for (const record of records) { + rows.push(record); + } + + // To fetch the next page of records, call `fetchNextPage`. + // If there are more records, `page` will get called again. + // If there are no more records, `done` will get called. + fetchNextPage(); + }, + (error) => { + if (error) { + reject(error); + } else { + resolve(rows); + } + }, + ), ); } @@ -94,7 +72,7 @@ function isBoolStr(value: unknown) { return typeof value === "string" && (value === "true" || value === "false"); } -function processTableToSimplified(tableData: Airtable.Record[], isKeyValueTable: boolean): Result { +function processTableToSimplified(tableData: Airtable.Record[], isKeyValueTable: boolean): unknown { if (isKeyValueTable) { // biome-ignore lint/suspicious/noExplicitAny: TODO const simplifiedData: Record = {}; @@ -103,7 +81,7 @@ function processTableToSimplified(tableData: Airtable.Record[ const fields = row._rawJson.fields; if (Object.keys(fields).length < 2) { - return err(new SourceParseError("At least 2 columns required to map table to a key-value pair")); + throw new Error("At least 2 columns required to map table to a key-value pair"); } const regex = /(.*)\[([0-9]*)\]$/g; @@ -128,119 +106,83 @@ function processTableToSimplified(tableData: Airtable.Record[ } } - return ok(simplifiedData); + return simplifiedData; } - return ok( - tableData.map((row) => ({ - id: row.id, - ...row._rawJson.fields, - })), - ); + return tableData.map((row) => ({ + id: row.id, + ...row._rawJson.fields, + })); } -export default function airtableSource(options: AirtableOptions) { - const assembledOptions = { - ...AIRTABLE_OPTION_DEFAULTS, - ...options, - }; +export default async function airtableSource(options: z.input) { + const assembledOptions = airtableSourceSchema.parse(options); - if (!assembledOptions.apiKey) { - return errAsync(new SourceConfigError("apiKey is required")); - } + const { default: Airtable } = await tryImportAirtable(); - if (!assembledOptions.baseId) { - return errAsync(new SourceConfigError("baseId is required")); - } + Airtable.configure({ + endpointUrl: assembledOptions.endpointUrl, + apiKey: assembledOptions.apiKey, + }); - return ResultAsync.fromPromise( - import("airtable"), - () => new SourceMissingDependencyError('Could not find module "airtable". Make sure you have installed it.'), - ).map(({ default: Airtable }) => { - Airtable.configure({ - endpointUrl: assembledOptions.endpointUrl, - apiKey: assembledOptions.apiKey, - }); + const base = Airtable.base(assembledOptions.baseId); - const base = Airtable.base(assembledOptions.baseId); + const rawAirtableDataCache: Record[]> = {}; - const rawAirtableDataCache: Record[]> = {}; + async function getDataCached(tableId: string, force: boolean, logger: Logger): Promise[]> { + logger.debug(`Fetching ${tableId} from Airtable`); - function getDataCached(tableId: string, force: boolean, logger: Logger): ResultAsync[], SourceFetchError> { - logger.debug(`Fetching ${tableId} from Airtable`); + if (force) { + rawAirtableDataCache[tableId] = []; + } - if (force) { - rawAirtableDataCache[tableId] = []; - } + if (rawAirtableDataCache[tableId] && rawAirtableDataCache[tableId].length > 0) { + logger.debug(`${tableId} found in cache`); + return rawAirtableDataCache[tableId]; + } - if (rawAirtableDataCache[tableId] && rawAirtableDataCache[tableId].length > 0) { - logger.debug(`${tableId} found in cache`); - return okAsync(rawAirtableDataCache[tableId]); - } + const data = await fetchData(base, tableId, assembledOptions.defaultView); + rawAirtableDataCache[tableId] = data; + logger.debug(`${tableId} fetched from Airtable`); + return data; + } - return fetchData(base, tableId, assembledOptions.defaultView).andTee((value) => { - rawAirtableDataCache[tableId] = value as Airtable.Record[]; - logger.debug(`${tableId} fetched from Airtable`); - }); - } + return defineSource({ + id: assembledOptions.id, + fetch: (ctx) => { + const documentFetches: Array<{ id: string; data: Promise }> = []; - return defineSource({ - id: assembledOptions.id, - fetch: (ctx) => { - const tablePromises = [] as SourceFetchPromise[]; - - for (const tableId of assembledOptions.tables) { - tablePromises.push({ - id: tableId, - dataPromise: getDataCached(tableId, false, ctx.logger).andThen((data) => { - const simplifiedTable = processTableToSimplified(data, false); - - if (simplifiedTable.isErr()) { - ctx.logger.error(`Error processing ${tableId} from Airtable`); - return err(new SourceParseError(`Error processing table ${tableId} from Airtable`, { cause: simplifiedTable.error })); - } - - return ok([ - { - id: `${tableId}.raw`, - data, - }, - { - id: tableId, - data: simplifiedTable.value, - }, - ]); - }), - }); - } + for (const tableId of assembledOptions.tables) { + documentFetches.push({ + id: tableId, + data: getDataCached(tableId, false, ctx.logger).then((data) => { + const simplifiedTable = processTableToSimplified(data, false); - for (const tableId of assembledOptions.keyValueTables) { - tablePromises.push({ - id: tableId, - dataPromise: getDataCached(tableId, false, ctx.logger).andThen((data) => { - const simplifiedTable = processTableToSimplified(data, true); - - if (simplifiedTable.isErr()) { - ctx.logger.error(`Error processing ${tableId} from Airtable`); - return err(new SourceParseError(`Error processing table ${tableId} from Airtable`, { cause: simplifiedTable.error })); - } - - return ok([ - { - id: `${tableId}.raw`, - data, - }, - { - id: tableId, - data: simplifiedTable.value, - }, - ]); - }), - }); - } + return simplifiedTable; + }), + }); + } - return ok(tablePromises); - }, - }); + for (const tableId of assembledOptions.keyValueTables) { + documentFetches.push({ + id: tableId, + data: getDataCached(tableId, false, ctx.logger).then((data) => { + const simplifiedTable = processTableToSimplified(data, true); + + return simplifiedTable; + }), + }); + } + + return documentFetches; + }, }); } + +function tryImportAirtable() { + try { + return import("airtable"); + } catch (e) { + throw new Error('Could not find peer dependency "airtable". Make sure you have installed it.', { cause: e }); + } +} diff --git a/packages/content/src/sources/contentful-source.ts b/packages/content/src/sources/contentful-source.ts index 9dfec1cf..21d13d36 100644 --- a/packages/content/src/sources/contentful-source.ts +++ b/packages/content/src/sources/contentful-source.ts @@ -1,179 +1,132 @@ import type { Asset, Entry } from "contentful"; -import { ResultAsync, err, errAsync, ok } from "neverthrow"; import { fetchPaginated } from "../utils/fetch-paginated.js"; -import { SourceConfigError, SourceFetchError, SourceMissingDependencyError, defineSource } from "./source.js"; - -type ContentfulCredentialsDeliveryToken = { - /** - * Content delivery token (all published content). - */ - deliveryToken: string; - /** - * Content preview token (only unpublished/draft content). - */ - previewToken?: string; -}; - -type ContentfulCredentialsPreviewToken = { - /** - * Content preview token (only unpublished/draft content). - */ - previewToken: string; -}; - -type BaseContentfulOptions = { - /** - * Required field to identify this source. Will be used as download path. - */ - id: string; - /** - * Your Contentful space ID. Note that an accessToken is required in addition to this - */ - space: string; - /** - * Optional. Used to pull localized images. - */ - locale?: string; - /** - * Optional. The filename you want to use for where all content (entries and assets metadata) will be stored. Defaults to 'content.json' - */ - filename?: string; - /** - * Optional. Defaults to 'https' - */ - protocol?: string; - /** - * Optional. Defaults to 'cdn.contentful.com', or 'preview.contentful.com' if `usePreviewApi` is true - */ - host?: string; - /** - * Optional. Set to true if you want to use the preview API instead of the production version to view draft content. Defaults to false - */ - usePreviewApi?: boolean; - /** - * Optionally limit queries to these content types. This will also apply to linked assets. Types that link to other types will include up to 10 levels of child content. E.g. filtering by Story, might also include Chapters and Images. Uses `searchParams['sys.contentType.sys.id[in]']` under the hood. - */ - contentTypes?: Array; - /** - * Optional. Supports anything from https://www.contentful.com/developers/docs/references/content-delivery-api/#/reference/search-parameters - */ - searchParams?: Record; -}; - -type ContentfulClientParams = Omit; - -/** - * Configuration options for the Contentful ContentSource. - * - * Also supports all fields of the Contentful SDK's config. - * - * @see Configuration under https://contentful.github.io/contentful.js/contentful/9.1.7/ - */ -type ContentfulOptions = BaseContentfulOptions & ContentfulClientParams & (ContentfulCredentialsDeliveryToken | ContentfulCredentialsPreviewToken); - -const CONTENTFUL_OPTIONS_DEFAULTS = { - locale: "en-US", - filename: "content.json", - protocol: "https", - host: "cdn.contentful.com", - usePreviewApi: false, - contentTypes: [], - searchParams: { - limit: 1000, // This is the max that Contentful supports, - include: 10, // @see https://www.contentful.com/developers/docs/references/content-delivery-api/#/reference/links/retrieval-of-linked-items - } as Record, -} satisfies Partial; - -export default function contentfulSource(options: ContentfulOptions) { - const assembled = { - accessToken: "", - ...CONTENTFUL_OPTIONS_DEFAULTS, - ...options, - }; +import { defineSource } from "./source.js"; +import { z } from "zod"; + +// If deliveryToken is provided, then previewToken is optional. +const contentfulCredentialsSchema = z.union([ + z.object({ + /** Content delivery token (all published content). */ + deliveryToken: z.string().describe("Content delivery token (all published content)."), + /** Content preview token (only unpublished/draft content). */ + previewToken: z.string().optional().describe("Content preview token (only unpublished/draft content)."), + }), + z.object({ + /** Content preview token (only unpublished/draft content). */ + previewToken: z.string().describe("Content preview token (only unpublished/draft content)."), + }), +]); + +const contentfulSourceSchema = z + .object({ + /** Required field to identify this source. Will be used as download path. */ + id: z.string().describe("Required field to identify this source. Will be used as download path."), + /** Required field to identify this source. Will be used as download path. */ + space: z.string().describe("Your Contentful space ID."), + /** Used to pull localized images. */ + locale: z.string().default("en-US").describe("Used to pull localized images."), + /** The filename you want to use for where all content (entries and assets metadata) will be stored. Defaults to 'content.json' */ + filename: z.string().default("content.json").describe("The filename you want to use for where all content (entries and assets metadata) will be stored."), + /** Optional. Defaults to 'https' */ + protocol: z.string().default("https").describe("Optional. Defaults to 'https'"), + /** Optional. Defaults to 'cdn.contentful.com', or 'preview.contentful.com' if `usePreviewApi` is true */ + host: z + .string() + .default("cdn.contentful.com") + .describe("Optional. Defaults to 'cdn.contentful.com', or 'preview.contentful.com' if `usePreviewApi` is true"), + /** Optional. Set to true if you want to use the preview API instead of the production version to view draft content. Defaults to false */ + usePreviewApi: z + .boolean() + .default(false) + .describe("Optional. Set to true if you want to use the preview API instead of the production version to view draft content. Defaults to false"), + /** + * Optionally limit queries to these content types. This will also apply to linked assets. + * Types that link to other types will include up to 10 levels of child content. E.g. filtering by Story, might also include Chapters and Images. + * Uses `searchParams['sys.contentType.sys.id[in]']` under the hood. + */ + contentTypes: z.array(z.string()).default([]).describe( + "Optionally limit queries to these content types. This will also apply to linked assets. \ + Types that link to other types will include up to 10 levels of child content. E.g. filtering by Story, might also include Chapters and Images. \ + Uses `searchParams['sys.contentType.sys.id[in]']` under the hood.", + ), + /** Optional. Supports anything from https://www.contentful.com/developers/docs/references/content-delivery-api/#/reference/search-parameters */ + searchParams: z.record(z.unknown()).default({ + limit: 1000, // This is the max that Contentful supports, + include: 10, // @see https://www.contentful.com/developers/docs/references/content-delivery-api/#/reference/links/retrieval-of-linked-items + }), + }) + .passthrough() + .and(contentfulCredentialsSchema); + +export default async function contentfulSource(options: z.input) { + const assembled = contentfulSourceSchema.parse(options); + + let accessToken: string; if (assembled.usePreviewApi) { if (!assembled.previewToken) { - return errAsync(new SourceConfigError("usePreviewApi is set to true, but no previewToken is provided")); + throw new Error("usePreviewApi is set to true, but no previewToken is provided"); } assembled.host = "preview.contentful.com"; - assembled.accessToken = assembled.previewToken; + accessToken = assembled.previewToken; } else { if (!("deliveryToken" in assembled) || !assembled.deliveryToken) { - return errAsync(new SourceConfigError("usePreviewApi is set to false, but no deliveryToken is provided")); + throw new Error("usePreviewApi is set to false, but no deliveryToken is provided"); } - assembled.accessToken = assembled.deliveryToken; + accessToken = assembled.deliveryToken as string; } if (assembled.contentTypes && assembled.contentTypes.length > 0) { assembled.searchParams["sys.contentType.sys.id[in]"] = assembled.contentTypes.join(","); } - return ResultAsync.fromPromise( - import("contentful"), - () => new SourceMissingDependencyError('Could not find module "contentful". Make sure you have installed it.'), - ).map(({ createClient }) => { - const client = createClient(assembled); - - const source = defineSource<{ entries: Entry[]; assets: Asset[] }>({ - id: options.id, - fetch: (ctx) => { - const fetchResult = fetchPaginated({ - fetchPageFn: (params) => { - return ResultAsync.fromPromise( - client.getEntries({ ...assembled.searchParams, skip: params.offset, limit: params.limit }), - (error) => new SourceFetchError(`Error fetching page: ${error instanceof Error ? error.message : error}`), - ).andThen((rawPage) => { - if (rawPage.errors) { - return err(new SourceFetchError(`Error fetching page: ${rawPage.errors.map((e) => e.message).join(", ")}`)); - } - - const page = rawPage.toPlainObject(); - const entries = parseEntries(page); - const assets = parseAssets(page); - - if (!entries.length) { - return ok(null); // No more pages left - } - - return ok({ - entries, - assets, - }); - }); + const { createClient } = await tryImportContentful(); + + const client = createClient({ + ...assembled, + accessToken, + }); + + return defineSource({ + id: options.id, + fetch: (ctx) => { + return { + id: assembled.filename, + data: fetchPaginated({ + fetchPageFn: async (params) => { + const rawPage = await client.getEntries({ ...assembled.searchParams, skip: params.offset, limit: params.limit }); + + if (rawPage.errors) { + throw new Error(`Error fetching page: ${rawPage.errors.map((e) => e.message).join(", ")}`); + } + + const page = rawPage.toPlainObject(); + const entries = parseEntries(page); + const assets = parseAssets(page); + + if (!entries.length) { + return null; // No more pages left + } + + return { entries, assets }; }, limit: assembled.searchParams.limit as number, logger: ctx.logger, - }); - - return ok([ - { - id: assembled.filename, - dataPromise: fetchResult.map(({ pages }) => { - // combine page results - const combined = pages.reduce( - (acc, page) => { - return { - entries: [...acc.entries, ...page.entries], - assets: [...acc.assets, ...page.assets], - }; - }, - { entries: [], assets: [] }, - ); - - return [ - { - id: assembled.filename, - data: combined, - }, - ]; - }), - }, - ]); - }, - }); - - return source; + mergePages: true, + }).then((fetchResult) => { + return fetchResult.reduce<{ entries: Entry[]; assets: Asset[] }>( + (acc, page) => { + return { + entries: [...acc.entries, ...page.entries], + assets: [...acc.assets, ...page.assets], + }; + }, + { entries: [], assets: [] }, + ); + }), + }; + }, }); } @@ -210,3 +163,11 @@ function parseAssets(responseObj: any): Array { } return assets; } + +async function tryImportContentful() { + try { + return await import("contentful"); + } catch (error) { + throw new Error('Could not find module "contentful". Make sure you have installed it.', { cause: error }); + } +} diff --git a/packages/content/src/sources/json-source.ts b/packages/content/src/sources/json-source.ts index 11e2508e..e409e5ba 100644 --- a/packages/content/src/sources/json-source.ts +++ b/packages/content/src/sources/json-source.ts @@ -1,52 +1,32 @@ import chalk from "chalk"; -import { ok, okAsync } from "neverthrow"; -import { SafeKyParseError, safeKy } from "../utils/safe-ky.js"; -import { SourceFetchError, SourceParseError, defineSource } from "./source.js"; +import { okAsync } from "neverthrow"; +import { defineSource } from "./source.js"; +import { z } from "zod"; +import ky from "ky"; -type JsonSourceOptions = { - /** - * Required field to identify this source. Will be used as download path. - */ - id: string; - /** - * A mapping of json key -> url - */ - files: Record; - /** - * Max request timeout in ms. Defaults to 30 seconds. - */ - maxTimeout?: number; -}; +const jsonSourceSchema = z.object({ + /** required field to identify this source. Will be used as download path. */ + id: z.string().describe("Required field to identify this source. Will be used as download path."), + /** A mapping of json key -> url */ + files: z.record(z.string(), z.string()).describe("A mapping of json key -> url"), + /** Max request timeout in ms. Defaults to 30 seconds. */ + maxTimeout: z.number().describe("Max request timeout in ms.").default(30_000), +}); -export default function jsonSource(options: JsonSourceOptions) { - return okAsync( - defineSource({ - id: options.id, - fetch: (ctx) => { - const jsonFetchPromises = Object.entries(options.files).map(([key, url]) => { - ctx.logger.debug(`Downloading json ${chalk.blue(url)}`); - return { - id: key, - dataPromise: safeKy(url, { timeout: options.maxTimeout }) - .json() - .mapErr((e) => { - if (e instanceof SafeKyParseError) { - return new SourceParseError(`Could not parse json from ${url}`, { cause: e }); - } - return new SourceFetchError(`Could not fetch json from ${url}`, { cause: e }); - }) - .map((data) => { - return [ - { - id: key, - data, - }, - ]; - }), - }; - }); - return ok(jsonFetchPromises); - }, - }), - ); +export default function jsonSource(options: z.input) { + const parsedOptions = jsonSourceSchema.parse(options); + + return defineSource({ + id: parsedOptions.id, + fetch: (ctx) => { + return Object.entries(parsedOptions.files).map(([key, url]) => { + ctx.logger.debug(`Downloading json ${chalk.blue(url)}`); + + return { + id: key, + data: ky.get(url, { timeout: parsedOptions.maxTimeout }).json(), + }; + }); + }, + }); } diff --git a/packages/content/src/sources/sanity-source.ts b/packages/content/src/sources/sanity-source.ts index 5e6e83a7..991752cc 100644 --- a/packages/content/src/sources/sanity-source.ts +++ b/packages/content/src/sources/sanity-source.ts @@ -1,148 +1,87 @@ -import { Result, ResultAsync, err, errAsync, ok, okAsync } from "neverthrow"; import { fetchPaginated } from "../utils/fetch-paginated.js"; -import { SourceConfigError, SourceFetchError, type SourceFetchPromise, SourceMissingDependencyError, defineSource } from "./source.js"; +import { defineSource } from "./source.js"; +import { z } from "zod"; -type BaseSanityOptions = { - /** - * Required field to identify this source. Will be used as download path. - */ - id: string; - /** - * API Version. Defaults to 'v2021-10-21' - */ - apiVersion?: string; - /** - * Sanity Project ID - */ - projectId: string; - /** - * API Token defined in your sanity project. - */ - apiToken: string; - /** - * Dataset. Defaults to 'production' - */ - dataset?: string; - /** - * `false` if you want to ensure fresh data - */ - useCdn?: boolean; - /** - * An array of queries to fetch. Each query can be a string or an object with a query and an id. - */ - queries: Array; - /** - * Max number of entries per page. Defaults to 100. - */ - limit?: number; - /** - * Max number of pages. Use `-1` for all pages. Defaults to -1. - */ - maxNumPages?: number; - /** - * To combine paginated files into a single file. Defaults to false. - */ - mergePages?: boolean; - /** - * How many zeros to pad each json filename index with. Defaults to 0. - */ - pageNumZeroPad?: number; -}; +const sanitySourceSchema = z.object({ + /** Required field to identify this source. Will be used as download path. */ + id: z.string().describe("Required field to identify this source. Will be used as download path."), + /** Sanity API Version. Defaults to 'v2021-10-21' */ + apiVersion: z.string().describe("Sanity API Version").default("v2021-10-21"), + /** Sanity API Token. Required if dataset is private. */ + apiToken: z.string().describe("Sanity API Token. Required if dataset is private.").optional(), + /** Sanity Project ID */ + projectId: z.string().describe("Sanity Project ID"), + /** Sanity Dataset. Defaults to 'production' */ + dataset: z.string().describe("Sanity Dataset").default("production"), + /** `false` if you want to ensure fresh data */ + useCdn: z.boolean().describe("`false` if you want to ensure fresh data").default(true), + /** An array of queries to fetch. Each query can be a string or an object with a query and an id. */ + queries: z + .array(z.union([z.string(), z.object({ query: z.string(), id: z.string() })])) + .describe("An array of queries to fetch. Each query can be a string or an object with a query and an id."), + /** Max number of entries per page. Defaults to 100. */ + limit: z.number().describe("Max number of entries per page").default(100), + /** Max number of pages. Defaults to 1000. */ + maxNumPages: z.number().describe("Max number of pages").default(1000), + /** To combine paginated files into a single file. Defaults to false. */ + mergePages: z.boolean().describe("To combine paginated files into a single file.").default(false), +}); -const SANITY_OPTION_DEFAULTS = { - apiVersion: "v2021-10-21", - dataset: "production", - useCdn: false, - limit: 100, - maxNumPages: -1, - mergePages: true, - pageNumZeroPad: 0, -} satisfies Partial; +export default async function sanitySource(options: z.input) { + const parsedOptions = sanitySourceSchema.parse(options); -export default function sanitySource(options: BaseSanityOptions) { - if (!options.projectId || !options.apiToken) { - return errAsync(new SourceConfigError("Missing projectId and/or apiToken")); - } - - const assembledOptions = { - ...SANITY_OPTION_DEFAULTS, - ...options, - }; - - return ResultAsync.fromPromise( - import("@sanity/client"), - () => new SourceMissingDependencyError('Could not find "@sanity/client". Make sure you have installed it.'), - ).map(({ createClient }) => { - const sanityClient = createClient({ - projectId: assembledOptions.projectId, - dataset: assembledOptions.dataset, - apiVersion: assembledOptions.apiVersion, // use current UTC date - see "specifying API version"! - token: assembledOptions.apiToken, // or leave blank for unauthenticated usage - useCdn: assembledOptions.useCdn, // `false` if you want to ensure fresh data); - }); + const { createClient } = await tryImportSanityClient(); - return defineSource({ - id: options.id, - fetch: (ctx) => { - function combinePages(pages: Array, id: string) { - if (assembledOptions.mergePages) { - const combinedResult = pages.flat(1); - - return [ - { - id, - data: combinedResult, - }, - ]; - } - return pages.map((page, i) => { - const pageNum = i + 1; - const keyWithPageNum = `${id}-${pageNum.toString().padStart(assembledOptions.pageNumZeroPad, "0")}`; + const sanityClient = createClient({ + projectId: parsedOptions.projectId, + dataset: parsedOptions.dataset, + apiVersion: parsedOptions.apiVersion, + token: parsedOptions.apiToken, + useCdn: parsedOptions.useCdn, + }); - return { - id: keyWithPageNum, - data: page, - }; - }); + return defineSource({ + id: parsedOptions.id, + fetch: (ctx) => { + return parsedOptions.queries.map((query) => { + let fullQuery: string; + let id: string; + if (typeof query === "string") { + fullQuery = `*[_type == "${query}"]`; + id = query; + } else { + fullQuery = query.query; + id = query.id; } - const documentFetchPromises: Array = []; - - for (const query of assembledOptions.queries) { - if (typeof query === "string") { - const queryFull = `*[_type == "${query}" ]`; + return { + id, + data: fetchPaginated({ + limit: parsedOptions.limit, + logger: ctx.logger, + maxFetchCount: parsedOptions.maxNumPages, + mergePages: parsedOptions.mergePages, + fetchPageFn: (params) => { + // construct paginated query with groq syntax + const q = `${fullQuery}[${params.offset}..${params.offset + params.limit - 1}]`; - documentFetchPromises.push({ - id: query, - dataPromise: fetchPaginated({ - fetchPageFn: (params) => { - const q = `${queryFull}[${params.offset}..${params.offset + params.limit - 1}]`; - return ResultAsync.fromPromise(sanityClient.fetch(q), (e) => new SourceFetchError(`Could not fetch page with query: '${q}'`, { cause: e })); - }, - limit: assembledOptions.limit, - logger: ctx.logger, - }).map((data) => combinePages(data.pages, query)), - }); - } else if (typeof query === "object" && query.query && query.id) { - documentFetchPromises.push({ - id: query.id, - dataPromise: fetchPaginated({ - fetchPageFn: (params) => { - const q = `${query.query}[${params.offset}..${params.offset + params.limit - 1}]`; - return ResultAsync.fromPromise(sanityClient.fetch(q), (e) => new SourceFetchError(`Could not fetch page with query: '${q}'`, { cause: e })); - }, - limit: assembledOptions.limit, - logger: ctx.logger, - }).map((data) => combinePages(data.pages, query.id)), - }); - } else { - ctx.logger.error(`Invalid query: ${query}`); - return err(new SourceFetchError(`Invalid query: ${query}`)); - } - } - - return ok(documentFetchPromises); - }, - }); + try { + return sanityClient.fetch(q); + } catch (e) { + throw new Error(`Could not fetch page with query: '${q}'`, { cause: e }); + } + }, + }), + }; + }); + }, }); } + +function tryImportSanityClient() { + try { + return import("@sanity/client"); + } catch (e) { + throw new Error('Could not find peer dependency "@sanity/client". Make sure you have installed it.', { cause: e }); + } +} diff --git a/packages/content/src/sources/source.ts b/packages/content/src/sources/source.ts index 03341576..e0f3bf33 100644 --- a/packages/content/src/sources/source.ts +++ b/packages/content/src/sources/source.ts @@ -1,5 +1,4 @@ import type { Logger } from "@bluecadet/launchpad-utils"; -import type { Result, ResultAsync } from "neverthrow"; import type { DataStore } from "../utils/data-store.js"; /** @@ -11,24 +10,12 @@ export type SourceFetchResultDocument = { */ id: string; /** - * serializable data fetched from the source + * Either a promise returning a single document, or an async iterable returning multiple documents. */ - data: T; + data: Promise | AsyncIterable; }; -/** - * Represents a single fetch promise from a source, which can return multiple documents. - */ -export type SourceFetchPromise = { - /** - * Id of the fetch request, used for logging and debugging - */ - id: string; - /** - * Promise that resolves to an array of documents - */ - dataPromise: ResultAsync>, SourceFetchError | SourceParseError>; -}; +export type FetchResult = SourceFetchResultDocument[] | SourceFetchResultDocument; /** * Context object passed to the `fetch` method of a source. @@ -47,50 +34,20 @@ export type FetchContext = { /** * Represents a single content source. */ -export type ContentSource = { +export type ContentSource = FetchResult> = { /** * Id of the source. This will be the 'namespace' for the documents fetched from this source. */ id: string; - fetch: (ctx: FetchContext) => Result>, SourceFetchError | SourceParseError>; + /** + * Fetches the documents from the source. Returns either an array of documents or a single document. + */ + fetch: (ctx: FetchContext) => F; }; -/** - * Represents a function that builds a content source. - */ -export type ContentSourceBuilder = (options: O) => ResultAsync, SourceConfigError | SourceMissingDependencyError>; - /** * This function doesn't do anything, just returns the source parameter. It's just to make it easier to define/type sources. */ -export function defineSource(src: ContentSource) { +export function defineSource = FetchResult>(src: ContentSource) { return src; } - -export class SourceFetchError extends Error { - constructor(message: string, { cause }: { cause?: unknown } = {}) { - super(message, { cause }); - this.name = "SourceFetchError"; - } -} - -export class SourceConfigError extends Error { - constructor(message: string, { cause }: { cause?: unknown } = {}) { - super(message, { cause }); - this.name = "SourceConfigError"; - } -} - -export class SourceParseError extends Error { - constructor(message: string, { cause }: { cause?: unknown } = {}) { - super(message, { cause }); - this.name = "SourceParseError"; - } -} - -export class SourceMissingDependencyError extends Error { - constructor(message: string, { cause }: { cause?: unknown } = {}) { - super(message, { cause }); - this.name = "SourceMissingDependencyError"; - } -} diff --git a/packages/content/src/sources/strapi-source.ts b/packages/content/src/sources/strapi-source.ts index 73b356db..07293c78 100644 --- a/packages/content/src/sources/strapi-source.ts +++ b/packages/content/src/sources/strapi-source.ts @@ -1,9 +1,51 @@ import type { Logger } from "@bluecadet/launchpad-utils"; -import { type ResultAsync, errAsync, ok, okAsync } from "neverthrow"; import qs from "qs"; import { fetchPaginated } from "../utils/fetch-paginated.js"; -import { safeKy } from "../utils/safe-ky.js"; -import { type ContentSource, SourceConfigError, SourceFetchError, defineSource } from "./source.js"; +import { defineSource } from "./source.js"; +import { z } from "zod"; +import ky from "ky"; + +const strapiCredentialsSchema = z.union([ + z.object({ + /** Username or email. Should be configured via `./.env.local` */ + identifier: z.string().describe("Username or email. Should be configured via `./.env.local`"), + /** Password. Should be configured via `./.env.local` */ + password: z.string().describe("Password. Should be configured via `./.env.local`"), + }), + z.object({ + /** A previously generated JWT token. */ + token: z.string().describe("A previously generated JWT token."), + }), +]); + +const strapiSourceSchema = z + .object({ + /** Required field to identify this source. Will be used as download path. */ + id: z.string().describe("Required field to identify this source. Will be used as download path."), + /** Strapi version. Defaults to `3`. */ + version: z.enum(["3", "4"]).describe("Strapi version").default("3"), + /** The base url of your Strapi CMS (with or without trailing slash). */ + baseUrl: z.string().describe("The base url of your Strapi CMS (with or without trailing slash)."), + /** + * Queries for each type of content you want to save. One per content type. Content will be stored as numbered, paginated JSONs. + * You can include all query parameters supported by Strapi. + * You can also pass an object with a `contentType` and `params` property, where `params` is an object of query parameters. + */ + queries: z.array(z.union([z.string(), z.object({ contentType: z.string(), params: z.record(z.any()) })])).describe( + "Queries for each type of content you want to save. One per content type. Content will be stored as numbered, paginated JSONs. \ + You can include all query parameters supported by Strapi. \ + You can also pass an object with a `contentType` and `params` property, where `params` is an object of query parameters.", + ), + /** Max number of entries per page. Defaults to `100`. */ + limit: z.number().describe("Max number of entries per page").default(100), + /** Max number of pages. Defaults to `1000`. */ + maxNumPages: z.number().describe("Max number of pages").default(1000), + /** How many zeros to pad each json filename index with. Defaults to `2`. */ + pageNumZeroPad: z.number().describe("How many zeros to pad each json filename index with").default(2), + }) + .and(strapiCredentialsSchema); + +type StrapiSourceSchemaOutput = z.output; type StrapiObjectQuery = { /** @@ -27,75 +69,11 @@ type StrapiPagination = { limit: number; }; -type StrapiLoginCredentials = { - /** - * Username or email. Should be configured via `./.env.local` - */ - identifier: string; - /** - * Password. Should be configured via `./.env.local` - */ - password: string; -}; - -type StrapiTokenCredentials = { - /** - * A previously generated JWT token. - */ - token: string; -}; - -export type StrapiCredentials = StrapiLoginCredentials | StrapiTokenCredentials; - -export type BaseStrapiOptions = { - /** - * Required field to identify this source. Will be used as download path. - */ - id: string; - /** - * Versions `3` and `4` are supported. Defaults to `3`. - */ - version?: "3" | "4"; - /** - * The base url of your Strapi CMS (with or without trailing slash). - */ - baseUrl: string; - /** - * Queries for each type of content you want to save. One per content type. Content will be stored as numbered, paginated JSONs. - * You can include all query parameters supported by Strapi. - * You can also pass an object with a `contentType` and `params` property, where `params` is an object of query parameters. - */ - queries: Array; - /** - * Max number of entries per page. Defaults to `100`. - */ - limit?: number; - /** - * Max number of pages. Use the default of `-1` for all pages. Defaults to `-1`. - */ - maxNumPages?: number; - /** - * How many zeros to pad each json filename index with. Defaults to `0`. - */ - pageNumZeroPad?: number; -}; - -export type StrapiOptions = BaseStrapiOptions & StrapiCredentials; - -export type StrapiOptionsAssembled = Required & StrapiCredentials; - -const STRAPI_OPTION_DEFAULTS = { - version: "3", - limit: 100, - maxNumPages: -1, - pageNumZeroPad: 2, -} satisfies Partial; - class StrapiVersionUtils { - protected config: StrapiOptionsAssembled; + protected config: StrapiSourceSchemaOutput; protected logger: Logger; - constructor(config: StrapiOptionsAssembled, logger: Logger) { + constructor(config: StrapiSourceSchemaOutput, logger: Logger) { this.config = config; this.logger = logger; } @@ -215,65 +193,53 @@ class StrapiV3 extends StrapiVersionUtils { } } -function getJwt(baseUrl: string, identifier: string, password: string): ResultAsync { - const url = new URL("/auth/local", baseUrl); - - return safeKy(url.toString(), { - method: "POST", - json: { identifier, password }, - }) - .json() - .map((response) => response.jwt) - .mapErr((e) => new SourceFetchError(`Could not complete request to get JWT for ${identifier}`, { cause: e })); -} - -function getToken(assembledOptions: StrapiOptionsAssembled): ResultAsync { +async function getToken(assembledOptions: StrapiSourceSchemaOutput) { if ("token" in assembledOptions) { - return okAsync(assembledOptions.token); + return assembledOptions.token; } - return getJwt(assembledOptions.baseUrl, assembledOptions.identifier, assembledOptions.password); + const url = new URL("/auth/local", assembledOptions.baseUrl); + const response = await ky.post<{ jwt: string }>(url.toString(), { + json: { identifier: assembledOptions.identifier, password: assembledOptions.password }, + }); + + return response.json().then((json) => json.jwt); } -export default function strapiSource(options: StrapiOptions): ResultAsync { - const assembledOptions = { - ...STRAPI_OPTION_DEFAULTS, - ...options, - }; +export default async function strapiSource(options: z.input) { + const assembledOptions = strapiSourceSchema.parse(options); if (assembledOptions.version !== "4" && assembledOptions.version !== "3") { - return errAsync(new SourceConfigError(`Unsupported strapi version '${assembledOptions.version}'`)); + throw new Error(`Unsupported strapi version '${assembledOptions.version}'`); } - return getToken(assembledOptions).map((token) => - defineSource({ - id: options.id, - fetch: (ctx) => { - const versionUtils: StrapiVersionUtils = - assembledOptions.version === "4" ? new StrapiV4(assembledOptions, ctx.logger) : new StrapiV3(assembledOptions, ctx.logger); + const token = await getToken(assembledOptions); - const fetchPromises = assembledOptions.queries.map((query) => { - let parsedQuery: StrapiObjectQuery; + return defineSource({ + id: assembledOptions.id, + fetch: (ctx) => { + const versionUtils: StrapiVersionUtils = + assembledOptions.version === "4" ? new StrapiV4(assembledOptions, ctx.logger) : new StrapiV3(assembledOptions, ctx.logger); - if (typeof query === "string") { - parsedQuery = versionUtils.parseQuery(query); - } else { - parsedQuery = query; - } + return assembledOptions.queries.map((query) => { + let parsedQuery: StrapiObjectQuery; - return { - id: parsedQuery.contentType, - dataPromise: fetchPaginated({ - fetchPageFn: (params) => { - const pageNum = params.offset / params.limit; + if (typeof query === "string") { + parsedQuery = versionUtils.parseQuery(query); + } else { + parsedQuery = query; + } - if (pageNum > assembledOptions.maxNumPages && assembledOptions.maxNumPages !== -1) { - return okAsync(null); - } + return { + id: parsedQuery.contentType, + data: fetchPaginated({ + fetchPageFn: async (params) => { + const pageNum = params.offset / params.limit; - ctx.logger.debug(`Fetching page ${pageNum} of ${parsedQuery.contentType}`); + ctx.logger.debug(`Fetching page ${pageNum} of ${parsedQuery.contentType}`); - return safeKy( + const response = await ky + .get( versionUtils.buildUrl(parsedQuery, { start: params.offset, limit: params.limit, @@ -284,34 +250,22 @@ export default function strapiSource(options: StrapiOptions): ResultAsync { - const transformedContent = versionUtils.transformResult(json); - - if (!transformedContent || !transformedContent.length) { - return null; - } - - return transformedContent; - }) - .mapErr((e) => new SourceFetchError(`Could not fetch page ${pageNum} of ${parsedQuery.contentType}`, { cause: e })); - }, - limit: assembledOptions.limit, - logger: ctx.logger, - }).map((data) => { - return data.pages.map((page, i) => { - const fileName = `${parsedQuery.contentType}-${(i + 1).toString().padStart(assembledOptions.pageNumZeroPad, "0")}.json`; - return { - id: fileName, - data: page, - }; - }); - }), - }; - }); - - return ok(fetchPromises); - }, - }), - ); + .json(); + + const transformedContent = versionUtils.transformResult(response); + + if (!transformedContent || !transformedContent.length) { + return null; // trigger end of pagination + } + + return transformedContent; + }, + maxFetchCount: assembledOptions.maxNumPages, + limit: assembledOptions.limit, + logger: ctx.logger, + }), + }; + }); + }, + }); } diff --git a/packages/content/src/utils/fetch-paginated.ts b/packages/content/src/utils/fetch-paginated.ts index 55734905..2ed9b82a 100644 --- a/packages/content/src/utils/fetch-paginated.ts +++ b/packages/content/src/utils/fetch-paginated.ts @@ -1,8 +1,6 @@ import type { Logger } from "@bluecadet/launchpad-utils"; -import { type ResultAsync, errAsync, okAsync } from "neverthrow"; -import { SourceFetchError } from "../sources/source.js"; -export type FetchPaginatedOptions = { +export type FetchPaginatedOptions = { /** * The number of items to fetch per page */ @@ -14,55 +12,47 @@ export type FetchPaginatedOptions = { /** * A function that takes a params object and returns a ResultAsync of an array of T. To indicate the end of pagination, return an empty array, or null. */ - fetchPageFn: (params: { limit: number; offset: number }) => ResultAsync; + fetchPageFn: (params: { limit: number; offset: number }) => Promise; /** * A logger instance */ logger: Logger; + /** + * Whether to merge pages into a single array. Defaults to false. + */ + mergePages?: Merge; }; -export type FetchPaginatedResult = ResultAsync } : { pages: Array; meta: M }, SourceFetchError>; - /** - * Handles paginated fetching - * @template {unknown} T - * @template {unknown} [M=undefined] - * @param {M extends undefined ? FetchPaginatedOptions : FetchPaginatedOptions & {meta: M}} options - * @returns {FetchPaginatedResult} + * Handles paginated fetching. Returns an async iterable, unless mergePages is true in which case it returns a flattened array. */ -export function fetchPaginated({ +export function fetchPaginated({ fetchPageFn, limit, logger, maxFetchCount = 1000, - ...rest -}: FetchPaginatedOptions & { meta?: M }): FetchPaginatedResult { - const pages: Array = []; - let page = 0; + mergePages, +}: FetchPaginatedOptions): Merge extends true ? Promise : AsyncGenerator { + async function* generator() { + for (let i = 0; i < maxFetchCount; i++) { + logger.debug(`Fetching page ${i}`); + const data = await fetchPageFn({ limit, offset: i * limit }); - const fetchNextPage: () => ResultAsync = () => { - logger.debug(`Fetching page ${page}`); - return fetchPageFn({ limit, offset: page * limit }).andThen((data) => { if (data === null || (Array.isArray(data) && data.length === 0)) { - return okAsync(null); + return; } - pages.push(data); - page++; - if (page >= maxFetchCount) { - return errAsync(new SourceFetchError("Maximum fetch count reached. This is likely a bug. Make sure your fetchPageFn ret")); - } + yield data as T; + } + } - return fetchNextPage(); - }); - }; + return (mergePages ? getFlattened(generator()) : generator()) as Merge extends true ? Promise : AsyncGenerator; +} - return fetchNextPage().andThen(() => { - if ("meta" in rest) { - // Have to cast to FetchPaginatedResult because TS gets confused by the 'M extends undefined ?'... stuff - return okAsync({ pages, meta: rest.meta }) as FetchPaginatedResult; - } - // Have to cast to FetchPaginatedResult because TS gets confused by the 'M extends undefined ?'... stuff - return okAsync({ pages }) as FetchPaginatedResult; - }); +async function getFlattened(generator: AsyncGenerator) { + const pages: T[] = []; + for await (const page of generator) { + pages.push(page); + } + return pages.flat(1); } From b91ae6a82a6768d8d32e9f1b2d8e0f65793ab796 Mon Sep 17 00:00:00 2001 From: Clay Tercek <30105080+claytercek@users.noreply.github.com> Date: Thu, 7 Nov 2024 12:28:34 -0500 Subject: [PATCH 2/6] rewrite datastore to be a fs proxy --- packages/content/src/utils/data-store.ts | 280 +++++++++++++++-------- 1 file changed, 187 insertions(+), 93 deletions(-) diff --git a/packages/content/src/utils/data-store.ts b/packages/content/src/utils/data-store.ts index ec61eb3d..8f3dca7b 100644 --- a/packages/content/src/utils/data-store.ts +++ b/packages/content/src/utils/data-store.ts @@ -1,5 +1,7 @@ import { JSONPath } from "jsonpath-plus"; -import { Result, err, ok } from "neverthrow"; +import { Result, ResultAsync, err, ok } from "neverthrow"; +import * as fs from "node:fs/promises"; +import path from "node:path"; export class DataStoreError extends Error { constructor(...args: ConstructorParameters) { @@ -15,39 +17,123 @@ export type DataKeys = Array; /** * A document represents a single file or resource. + * It is a proxy for the underlying file, and provides a simple api for updating the file. + * When the document is updated, a copy of the original file is created with the `.original` suffix. */ -export class Document { - #id: string; - #originalData: T; +abstract class Document { + protected _id: string; - constructor(id: string, data: T) { - this.#id = id; - this.#originalData = data; + constructor(id: string) { + this._id = id; } get id() { - return this.#id; + return this._id; } /** - * Returns a copy of the document's data. + * Update the document with the given callback. + * @param cb A function that takes the current data and returns the new data. */ - get data() { - return this.#originalData; - } + abstract update(cb: (data: T) => T | Promise): Promise; - update(data: T) { - this.#originalData = data; + /** + * Update the document with the given callback. Same as {@link update}, but returns a neverthrow {@link ResultAsync}. + * @param cb A function that takes the current data and returns the new data. + */ + safeUpdate(cb: (data: T) => T | Promise): ResultAsync { + return ResultAsync.fromPromise(this.update(cb), (e) => new DataStoreError(`Error updating document ${this._id}`, { cause: e })); } /** * Apply a function to each element matching the given jsonpath. */ - apply(pathExpression: string, fn: (x: unknown) => unknown): Result { - // catch errrors thrown from JSONPath OR the fn callback - try { + abstract apply(pathExpression: string, fn: (x: unknown) => unknown): Promise; + + /** + * Apply a function to each element matching the given jsonpath. Same as {@link apply}, but returns a neverthrow {@link ResultAsync}. + */ + safeApply(pathExpression: string, fn: (x: unknown) => unknown): ResultAsync { + return ResultAsync.fromPromise( + this.apply(pathExpression, fn), + (e) => new DataStoreError(`Error applying content transform to document ${this._id}`, { cause: e }), + ); + } + + /** + * Close the file handle. + */ + abstract close(): Promise; + + /** + * Close the file handle. Same as {@link close}, but returns a neverthrow {@link ResultAsync}. + */ + safeClose(): ResultAsync { + return ResultAsync.fromPromise(this.close(), (e) => new DataStoreError(`Error closing document ${this._id}`, { cause: e })); + } +} + +class SingleDocument extends Document { + #hasBeenModified = false; + #handlePromise: Promise | null = null; + #path: string; + + constructor(directory: string, id: string) { + super(id); + const filename = id.includes(".") ? id : `${id}.json`; + this.#path = path.join(directory, filename); + } + + async initialize(data: T | Promise) { + const resolvedData = await data; + + // wx+ opens the file for reading and writing, creating it if it doesn't exist, and erroring if it does + this.#handlePromise = fs.open(this.#path, "wx+").then((handle) => { + handle.write(JSON.stringify(resolvedData)); + return handle; + }); + + return this.#handlePromise.then(() => this); // resolve void + } + + static async create(directory: string, id: string, data: T | Promise) { + const doc = new SingleDocument(directory, id); + await doc.initialize(data); + return doc; + } + + async #getHandle() { + if (!this.#handlePromise) { + throw new DataStoreError(`Document ${this._id} not initialized`); + } + + return await this.#handlePromise; + } + + override async update(cb: (data: T) => T | Promise) { + if (!this.#hasBeenModified) { + // on first modification, copy from the current path to the original path + await fs.copyFile( + this.#path, + this.#path.replace(/(\.[^.]*)$/, ".original$1"), // replace the extension with .original.[extension] + fs.constants.COPYFILE_EXCL, // fail if the original file already exists + ); + + this.#hasBeenModified = true; + } + + const handle = await this.#getHandle(); + const data = await fs.readFile(handle, "utf-8"); + + const updatedData = cb(JSON.parse(data)); + + await fs.writeFile(handle, JSON.stringify(updatedData)); + } + + override async apply(pathExpression: string, fn: (x: unknown) => unknown) { + await this.update((data) => { JSONPath({ - json: this.#originalData as object, + json: data as object, path: pathExpression, resultType: "all", callback: ({ value }, _, { parent, parentProperty }) => { @@ -55,10 +141,60 @@ export class Document { }, }); - return ok(undefined); - } catch (e) { - return err(new DataStoreError("Error applying content transform", { cause: e })); + return data; + }); + } + + override async close() { + const handle = await this.#getHandle(); + await handle.close(); + } +} + +/** + * A batch of documents. This is a wrapper around a list of {@link SingleDocument}s that provides the same api for updating all documents in the batch. + * This is useful for sources that return a list of documents via pagination. + */ +export class BatchDocument extends Document { + #documents: Array> = []; + + /** + * Add a zero-padded index to the document id. If id includes a file extension, it will be preserved after the index. + */ + static getIndexedId(id: string, index: number) { + const paddedIndex = index.toString().padStart(2, "0"); + + const lastDotIndex = id.lastIndexOf("."); + + if (lastDotIndex !== -1) { + return `${id.slice(0, lastDotIndex)}-${paddedIndex}${id.slice(lastDotIndex)}`; } + + return `${id}-${paddedIndex}`; + } + + async initialize(directory: string, data: AsyncIterable) { + for await (const item of data) { + this.#documents.push(await SingleDocument.create(directory, BatchDocument.getIndexedId(this._id, this.#documents.length), item)); + } + } + + static async create(directory: string, id: string, data: AsyncIterable) { + const doc = new BatchDocument(id); + await doc.initialize(directory, data); + return doc; + } + + override async update(cb: (data: T) => T | Promise) { + await Promise.all(this.#documents.map((doc) => doc.update(cb))); + } + + override async apply(pathExpression: string, fn: (x: unknown) => unknown) { + await Promise.all(this.#documents.map((doc) => doc.apply(pathExpression, fn))); + } + + override async close() { + await Promise.all(this.#documents.map((doc) => doc.close())); } } @@ -67,27 +203,33 @@ export class Document { */ class Namespace { #id: string; + #directory: string; #documents = new Map(); - constructor(id: string) { + constructor(parentDirectory: string, id: string) { this.#id = id; + this.#directory = path.join(parentDirectory, id); } get id() { return this.#id; } - insert(id: string, data: unknown): Result { - if (this.#documents.has(id)) { - return err(new DataStoreError(`Document ${id} already exists in namespace ${this.#id}`)); + async insert(data: Promise | AsyncIterable) { + if (data instanceof Promise) { + const doc = await SingleDocument.create(this.#directory, BatchDocument.getIndexedId(this.#id, this.#documents.size), data); + this.#documents.set(doc.id, doc); + } else { + const doc = await BatchDocument.create(this.#directory, this.#id, data); + this.#documents.set(doc.id, doc); } - - this.#documents.set(id, new Document(id, data)); - return ok(undefined); } - get(id: string): Result { + /** + * Get a document from the namespace. + */ + document(id: string): Result { const document = this.#documents.get(id); if (!document) { return err(new DataStoreError(`Document ${id} not found in namespace ${this.#id}`)); @@ -96,19 +238,12 @@ class Namespace { return ok(document); } + /** + * Get all documents in the namespace. + */ documents() { return this.#documents.values(); } - - delete(id: string): Result { - this.#documents.delete(id); - return ok(undefined); - } - - update(id: string, data: unknown): Result { - this.#documents.set(id, new Document(id, data)); - return ok(undefined); - } } /** @@ -117,16 +252,17 @@ class Namespace { */ export class DataStore { #namespaces = new Map(); + #directory: string; - get(namespaceId: string, documentId: string): Result { - const namespace = this.#namespaces.get(namespaceId); - if (!namespace) { - return err(new DataStoreError(`Namespace ${namespaceId} not found in data store`)); - } - - return namespace.get(documentId); + constructor(directory: string) { + this.#directory = directory; + // create the directory if it doesn't exist + fs.mkdir(this.#directory, { recursive: true }); } + /** + * Get a namespace from the data store. + */ namespace(namespaceId: string): Result, DataStoreError> { const namespace = this.#namespaces.get(namespaceId); if (!namespace) { @@ -136,67 +272,25 @@ export class DataStore { return ok(namespace.documents()); } - *allDocuments() { - for (const namespace of this.namespaces()) { - yield* namespace.documents(); - } - } - - namespaces() { - return this.#namespaces.values(); - } - + /** + * Create a new namespace in the data store. + */ createNamespace(namespaceId: string): Result { if (this.#namespaces.has(namespaceId)) { return err(new DataStoreError(`Namespace ${namespaceId} already exists in data store`)); } - this.#namespaces.set(namespaceId, new Namespace(namespaceId)); - return ok(undefined); - } - - createNamespaceFromMap(namespaceId: string, map: Map): Result { - const namespaceResult = this.createNamespace(namespaceId); - - if (namespaceResult.isErr()) { - return namespaceResult; - } - - for (const [documentId, data] of map.entries()) { - const insertResult = this.insert(namespaceId, documentId, data); - if (insertResult.isErr()) { - return insertResult; - } - } - + this.#namespaces.set(namespaceId, new Namespace(this.#directory, namespaceId)); return ok(undefined); } - insert(namespaceId: string, documentId: string, data: unknown): Result { - const namespace = this.#namespaces.get(namespaceId); - if (!namespace) { - return err(new DataStoreError(`Namespace ${namespaceId} not found in data store`)); - } - - return namespace.insert(documentId, data); - } - - delete(namespaceId: string, documentId: string): Result { - const namespace = this.#namespaces.get(namespaceId); - if (!namespace) { - return err(new DataStoreError(`Namespace ${namespaceId} not found in data store`)); - } - - return namespace.delete(documentId); - } - /** * Get lists of documents matching the passed DataKeys grouped by namespace. * @param ids A list containing a combination of namespace ids, and namespace/document id tuples. If not provided, all documents will be matched. */ filter(ids?: DataKeys): Result }>, DataStoreError> { if (!ids) { - return ok(Array.from(this.namespaces()).map((ns) => ({ namespaceId: ns.id, documents: Array.from(ns.documents()) }))); + return ok(Array.from(this.#namespaces.values()).map((ns) => ({ namespaceId: ns.id, documents: Array.from(ns.documents()) }))); } const consolidatedIds = new Map>(); From 5a1b8282ee758ab93e5277997d36970412a9d4e3 Mon Sep 17 00:00:00 2001 From: Clay Tercek <30105080+claytercek@users.noreply.github.com> Date: Thu, 7 Nov 2024 13:14:48 -0500 Subject: [PATCH 3/6] fix datastore related tests --- biome.json | 3 + .../sources/__tests__/airtable-source.test.ts | 2 +- .../__tests__/contentful-source.test.ts | 2 +- .../src/sources/__tests__/json-source.test.ts | 2 +- .../sources/__tests__/sanity-source.test.ts | 2 +- .../sources/__tests__/strapi-source.test.ts | 2 +- .../src/utils/__tests__/data-store.test.ts | 300 +++++++++++------- .../utils/__tests__/fetch-paginated.test.ts | 119 ++++--- packages/content/src/utils/data-store.ts | 44 ++- packages/testing/src/setup.ts | 2 +- 10 files changed, 296 insertions(+), 182 deletions(-) diff --git a/biome.json b/biome.json index 03abb978..4e77d4b4 100644 --- a/biome.json +++ b/biome.json @@ -36,6 +36,9 @@ "rules": { "style": { "noNonNullAssertion": "off" + }, + "suspicious": { + "noExplicitAny": "off" } } } diff --git a/packages/content/src/sources/__tests__/airtable-source.test.ts b/packages/content/src/sources/__tests__/airtable-source.test.ts index 6366873e..215e4136 100644 --- a/packages/content/src/sources/__tests__/airtable-source.test.ts +++ b/packages/content/src/sources/__tests__/airtable-source.test.ts @@ -22,7 +22,7 @@ afterEach(() => server.resetHandlers()); function createFetchContext() { return { logger: createMockLogger(), - dataStore: new DataStore(), + dataStore: new DataStore("/"), }; } diff --git a/packages/content/src/sources/__tests__/contentful-source.test.ts b/packages/content/src/sources/__tests__/contentful-source.test.ts index 119899f2..6c99ecfd 100644 --- a/packages/content/src/sources/__tests__/contentful-source.test.ts +++ b/packages/content/src/sources/__tests__/contentful-source.test.ts @@ -22,7 +22,7 @@ afterEach(() => server.resetHandlers()); function createFetchContext() { return { logger: createMockLogger(), - dataStore: new DataStore(), + dataStore: new DataStore("/"), }; } diff --git a/packages/content/src/sources/__tests__/json-source.test.ts b/packages/content/src/sources/__tests__/json-source.test.ts index d14968b9..fe31bcde 100644 --- a/packages/content/src/sources/__tests__/json-source.test.ts +++ b/packages/content/src/sources/__tests__/json-source.test.ts @@ -22,7 +22,7 @@ afterEach(() => server.resetHandlers()); function createFetchContext() { return { logger: createMockLogger(), - dataStore: new DataStore(), + dataStore: new DataStore("/"), }; } diff --git a/packages/content/src/sources/__tests__/sanity-source.test.ts b/packages/content/src/sources/__tests__/sanity-source.test.ts index ce402691..2580ec53 100644 --- a/packages/content/src/sources/__tests__/sanity-source.test.ts +++ b/packages/content/src/sources/__tests__/sanity-source.test.ts @@ -22,7 +22,7 @@ afterEach(() => server.resetHandlers()); function createFetchContext() { return { logger: createMockLogger(), - dataStore: new DataStore(), + dataStore: new DataStore("/"), }; } diff --git a/packages/content/src/sources/__tests__/strapi-source.test.ts b/packages/content/src/sources/__tests__/strapi-source.test.ts index 72e3e5f3..7b79dae4 100644 --- a/packages/content/src/sources/__tests__/strapi-source.test.ts +++ b/packages/content/src/sources/__tests__/strapi-source.test.ts @@ -22,7 +22,7 @@ afterEach(() => server.resetHandlers()); function createFetchContext() { return { logger: createMockLogger(), - dataStore: new DataStore(), + dataStore: new DataStore("/"), }; } diff --git a/packages/content/src/utils/__tests__/data-store.test.ts b/packages/content/src/utils/__tests__/data-store.test.ts index b78a7d92..4eb999f4 100644 --- a/packages/content/src/utils/__tests__/data-store.test.ts +++ b/packages/content/src/utils/__tests__/data-store.test.ts @@ -1,144 +1,226 @@ -import { describe, expect, it } from "vitest"; -import { DataStore, DataStoreError, Document } from "../data-store.js"; +import { vol } from "memfs"; +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import { DataStore } from "../data-store.js"; +import path from "node:path"; + +describe("SingleDocument", () => { + const TEST_DIR = "/test/store"; + let store: DataStore; + + beforeEach(() => { + vol.reset(); + store = new DataStore(TEST_DIR); + }); -describe("Document", () => { - it("should create a document with id and data", () => { - const doc = new Document("test-id", { content: "test content" }); - expect(doc.id).toBe("test-id"); - expect(doc.data).toEqual({ content: "test content" }); + afterEach(() => { + vol.reset(); }); - it("should update document data", () => { - const doc = new Document("test-id", { content: "test content" }); - doc.update({ content: "updated content" }); - expect(doc.data).toEqual({ content: "updated content" }); + it("should create and read a document", async () => { + const result = await store.createNamespace("test-namespace"); + expect(result).toBeOk(); + + const namespace = result._unsafeUnwrap(); + await namespace.insert("test-doc", Promise.resolve({ content: "test content" })); + + const docResult = namespace.document("test-doc"); + expect(docResult).toBeOk(); + + const fileContent = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", "test-doc.json"), "utf-8"); + expect(JSON.parse(fileContent.toString())).toEqual({ content: "test content" }); }); - it("should apply transformation to document data", () => { - const doc = new Document("test-id", { content: "test content" }); - const result = doc.apply("$.content", (value) => (typeof value === "string" ? value.toUpperCase() : value)); + it("should create backup file on first modification", async () => { + const result = await store.createNamespace("test-namespace"); expect(result).toBeOk(); - expect(doc.data).toEqual({ content: "TEST CONTENT" }); + + const namespace = result._unsafeUnwrap(); + const doc = await namespace.insert("test-doc", Promise.resolve({ content: "original content" })); + + await doc.update((data: any) => ({ + ...data, + content: "modified content", + })); + + const originalContent = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", "test-doc.original.json"), "utf-8"); + expect(JSON.parse(originalContent.toString())).toEqual({ content: "original content" }); + + const modifiedContent = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", "test-doc.json"), "utf-8"); + expect(JSON.parse(modifiedContent.toString())).toEqual({ content: "modified content" }); }); - it("should handle errors in apply transformation", () => { - const doc = new Document("test-id", { content: "test content" }); - const result = doc.apply("$.content", () => { - throw new Error("Test error"); + it("should apply jsonpath transformations", async () => { + const result = await store.createNamespace("test-namespace"); + expect(result).toBeOk(); + + const namespace = result._unsafeUnwrap(); + const doc = await namespace.insert( + "test-doc", + Promise.resolve({ + nested: { content: "test content" }, + }), + ); + + await doc.apply("$.nested.content", (value: unknown) => (typeof value === "string" ? value.toUpperCase() : value)); + + const fileContent = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", "test-doc.json"), "utf-8"); + expect(JSON.parse(fileContent.toString())).toEqual({ + nested: { content: "TEST CONTENT" }, }); - expect(result).toBeErr(); - expect(result._unsafeUnwrapErr()).toBeInstanceOf(DataStoreError); - expect(result._unsafeUnwrapErr().message).toBe("Error applying content transform"); - expect(result._unsafeUnwrapErr().cause).toBeInstanceOf(Error); - // @ts-expect-error cause is unknown - expect(result._unsafeUnwrapErr().cause.message).toBe("Test error"); }); -}); -describe("DataStore", () => { - it("should create a namespace", () => { - const store = new DataStore(); - const result = store.createNamespace("test-namespace"); + it("should keep original file extension", async () => { + const result = await store.createNamespace("test-namespace"); expect(result).toBeOk(); + + const namespace = result._unsafeUnwrap(); + await namespace.insert("test-doc.json", Promise.resolve({ content: "test content A" })); + await namespace.insert("test-doc.extension", Promise.resolve({ content: "test content B" })); + await namespace.insert("test-doc.extension.extension", Promise.resolve({ content: "test content C" })); + + const fileContent = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", "test-doc.json"), "utf-8"); + expect(JSON.parse(fileContent.toString())).toMatchObject({ content: "test content A" }); + + const extensionFileContent = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", "test-doc.extension"), "utf-8"); + expect(JSON.parse(extensionFileContent.toString())).toMatchObject({ content: "test content B" }); + + const extensionExtensionFileContent = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", "test-doc.extension.extension"), "utf-8"); + expect(JSON.parse(extensionExtensionFileContent.toString())).toMatchObject({ content: "test content C" }); }); +}); - it("should not create a duplicate namespace", () => { - const store = new DataStore(); - store.createNamespace("test-namespace"); - const result = store.createNamespace("test-namespace"); - expect(result).toBeErr(); - expect(result._unsafeUnwrapErr()).toBeInstanceOf(DataStoreError); - expect(result._unsafeUnwrapErr().message).toBe("Namespace test-namespace already exists in data store"); +describe("BatchDocument", () => { + const TEST_DIR = "/test/store"; + let store: DataStore; + + beforeEach(() => { + vol.reset(); + store = new DataStore(TEST_DIR); }); - it("should insert a document into a namespace", () => { - const store = new DataStore(); - store.createNamespace("test-namespace"); - const result = store.insert("test-namespace", "test-doc", { content: "test content" }); + afterEach(() => { + vol.reset(); + }); + + it("should handle batch document creation", async () => { + const result = await store.createNamespace("test-namespace"); expect(result).toBeOk(); + + const namespace = result._unsafeUnwrap(); + const items = [ + { id: 1, content: "first" }, + { id: 2, content: "second" }, + { id: 3, content: "third" }, + ]; + + const doc = await namespace.insert( + "test-doc", + (async function* () { + for (const item of items) { + yield item; + } + })(), + ); + + // Check that all files were created + for (let i = 0; i < items.length; i++) { + const filename = `test-doc-${i.toString().padStart(2, "0")}.json`; + const content = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", filename), "utf-8"); + expect(JSON.parse(content.toString())).toEqual(items[i]); + } }); - it("should not insert a duplicate document", () => { - const store = new DataStore(); - store.createNamespace("test-namespace"); - store.insert("test-namespace", "test-doc", { content: "test content" }); - const result = store.insert("test-namespace", "test-doc", { content: "duplicate content" }); - expect(result).toBeErr(); - expect(result._unsafeUnwrapErr()).toBeInstanceOf(DataStoreError); - expect(result._unsafeUnwrapErr().message).toBe("Document test-doc already exists in namespace test-namespace"); + it("should apply updates to all documents in batch", async () => { + const result = await store.createNamespace("test-namespace"); + expect(result).toBeOk(); + + const namespace = result._unsafeUnwrap(); + const items = [{ content: "first" }, { content: "second" }, { content: "third" }]; + + const doc = await namespace.insert( + "test-doc", + (async function* () { + for (const item of items) { + yield item; + } + })(), + ); + + await doc.apply("$.content", (value: unknown) => (typeof value === "string" ? value.toUpperCase() : value)); + + // Verify all documents were updated + for (let i = 0; i < items.length; i++) { + const filename = `test-doc-${i.toString().padStart(2, "0")}.json`; + const content = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", filename), "utf-8"); + expect(JSON.parse(content.toString())).toEqual({ + content: items[i]!.content.toUpperCase(), + }); + } }); +}); - it("should get a document from a namespace", () => { - const store = new DataStore(); - store.createNamespace("test-namespace"); - store.insert("test-namespace", "test-doc", { content: "test content" }); - const result = store.get("test-namespace", "test-doc"); +describe("DataStore", () => { + const TEST_DIR = "/test/store"; + let store: DataStore; + + beforeEach(() => { + vol.reset(); + store = new DataStore(TEST_DIR); + }); + + afterEach(() => { + vol.reset(); + }); + + it("should create namespace directory", async () => { + const result = await store.createNamespace("test-namespace"); expect(result).toBeOk(); - expect(result._unsafeUnwrap().data).toEqual({ content: "test content" }); + + const exists = await vol.existsSync(path.join(TEST_DIR, "test-namespace")); + expect(exists).toBe(true); }); - it("should return error when getting non-existent document", () => { - const store = new DataStore(); + it("should not create duplicate namespace", async () => { store.createNamespace("test-namespace"); - const result = store.get("test-namespace", "non-existent-doc"); + const result = await store.createNamespace("test-namespace"); expect(result).toBeErr(); - expect(result._unsafeUnwrapErr()).toBeInstanceOf(DataStoreError); - expect(result._unsafeUnwrapErr().message).toBe("Document non-existent-doc not found in namespace test-namespace"); + expect(result._unsafeUnwrapErr().message).toBe("Namespace test-namespace already exists in data store"); }); - it("should delete a document from a namespace", () => { - const store = new DataStore(); - store.createNamespace("test-namespace"); - store.insert("test-namespace", "test-doc", { content: "test content" }); - const deleteResult = store.delete("test-namespace", "test-doc"); - expect(deleteResult).toBeOk(); - const getResult = store.get("test-namespace", "test-doc"); - expect(getResult).toBeErr(); - }); - - it("should create a namespace from a map", () => { - const store = new DataStore(); - const map = new Map([ - ["doc1", { content: "content 1" }], - ["doc2", { content: "content 2" }], - ]); - const result = store.createNamespaceFromMap("test-namespace", map); + it("should filter documents by namespace", async () => { + const ns1Result = await store.createNamespace("namespace1"); + const ns2Result = await store.createNamespace("namespace2"); + expect(ns1Result).toBeOk(); + expect(ns2Result).toBeOk(); + + const ns1 = ns1Result._unsafeUnwrap(); + const ns2 = ns2Result._unsafeUnwrap(); + + await ns1.insert("test-doc", Promise.resolve({ content: "content 1" })); + await ns2.insert("test-doc", Promise.resolve({ content: "content 2" })); + + const result = store.filter(["namespace1"]); expect(result).toBeOk(); - const doc1Result = store.get("test-namespace", "doc1"); - expect(doc1Result).toBeOk(); - expect(doc1Result._unsafeUnwrap().data).toEqual({ content: "content 1" }); + + const filtered = result._unsafeUnwrap(); + expect(filtered).toHaveLength(1); + expect(filtered[0]!.namespaceId).toBe("namespace1"); + expect(filtered[0]!.documents).toHaveLength(1); }); - it("should filter documents", () => { - const store = new DataStore(); - store.createNamespace("namespace1"); - store.createNamespace("namespace2"); - store.insert("namespace1", "doc1", { content: "content 1" }); - store.insert("namespace1", "doc2", { content: "content 2" }); - store.insert("namespace2", "doc3", { content: "content 3" }); + it("should filter documents by specific document ids", async () => { + const nsResult = await store.createNamespace("namespace1"); + expect(nsResult).toBeOk(); + const namespace = nsResult._unsafeUnwrap(); - const result = store.filter(["namespace1", ["namespace2", "doc3"]]); - expect(result).toBeOk(); - const filteredDocs = result._unsafeUnwrap(); - expect(filteredDocs).toHaveLength(2); - expect(filteredDocs[0]!.namespaceId).toBe("namespace1"); - expect(filteredDocs[0]!.documents).toHaveLength(2); - expect(filteredDocs[1]!.namespaceId).toBe("namespace2"); - expect(filteredDocs[1]!.documents).toHaveLength(1); - expect(filteredDocs[1]!.documents[0]!.id).toBe("doc3"); - }); - - it("should return all documents when filter is not provided", () => { - const store = new DataStore(); - store.createNamespace("namespace1"); - store.createNamespace("namespace2"); - store.insert("namespace1", "doc1", { content: "content 1" }); - store.insert("namespace2", "doc2", { content: "content 2" }); - - const result = store.filter(); + const doc1 = await namespace.insert("test-doc-1", Promise.resolve({ content: "content 1" })); + await namespace.insert("test-doc-2", Promise.resolve({ content: "content 2" })); + + const result = store.filter([["namespace1", doc1.id]]); expect(result).toBeOk(); - const allDocs = result._unsafeUnwrap(); - expect(allDocs).toHaveLength(2); - expect(allDocs.flatMap((ns) => ns.documents)).toHaveLength(2); + + const filtered = result._unsafeUnwrap(); + expect(filtered[0]!.documents).toHaveLength(1); + expect(filtered[0]!.documents[0]!.id).toBe(doc1.id); }); }); diff --git a/packages/content/src/utils/__tests__/fetch-paginated.test.ts b/packages/content/src/utils/__tests__/fetch-paginated.test.ts index 7e400aa1..a1a5de29 100644 --- a/packages/content/src/utils/__tests__/fetch-paginated.test.ts +++ b/packages/content/src/utils/__tests__/fetch-paginated.test.ts @@ -1,9 +1,7 @@ import { createMockLogger } from "@bluecadet/launchpad-testing/test-utils.ts"; import { http, HttpResponse } from "msw"; import { setupServer } from "msw/node"; -import { ResultAsync } from "neverthrow"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { SourceFetchError } from "../../sources/source.js"; import { fetchPaginated } from "../fetch-paginated.js"; const server = setupServer(); @@ -20,11 +18,40 @@ afterEach(() => { describe("fetchPaginated", () => { const mockLogger = createMockLogger(); - it("empty test", async () => { - expect(true).toBe(true); + it("should fetch all pages as AsyncGenerator", async () => { + let callCount = 0; + server.use( + http.get("http://example.com/api", ({ request }) => { + const url = new URL(request.url); + const limit = Number(url.searchParams.get("limit")); + const offset = Number(url.searchParams.get("offset")); + callCount++; + + if (offset >= 30) { + return HttpResponse.json([]); + } + + return HttpResponse.json(Array.from({ length: limit }, (_, i) => ({ id: offset + i + 1 }))); + }), + ); + + const generator = fetchPaginated({ + limit: 10, + logger: mockLogger, + fetchPageFn: ({ limit, offset }) => fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), + }); + + const pages: unknown[] = []; + for await (const page of generator) { + pages.push(page); + } + + expect(pages).toHaveLength(3); + expect(pages.flat()).toHaveLength(30); + expect(callCount).toBe(4); // 3 successful calls + 1 empty result }); - it("should fetch all pages successfully", async () => { + it("should fetch all pages as merged array when mergePages is true", async () => { let callCount = 0; server.use( http.get("http://example.com/api", ({ request }) => { @@ -44,17 +71,14 @@ describe("fetchPaginated", () => { const result = await fetchPaginated({ limit: 10, logger: mockLogger, - fetchPageFn: ({ limit, offset }) => - ResultAsync.fromPromise( - fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), - (e) => new SourceFetchError("Failed to fetch", { cause: e }), - ), + mergePages: true, + fetchPageFn: ({ limit, offset }) => fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), }); - expect(result).toBeOk(); - const data = result._unsafeUnwrap(); - expect(data.pages).toHaveLength(3); - expect(data.pages.flat()).toHaveLength(30); + expect(Array.isArray(result)).toBe(true); + expect(result).toHaveLength(30); + expect(result[0]).toEqual({ id: 1 }); + expect(result[29]).toEqual({ id: 30 }); expect(callCount).toBe(4); // 3 successful calls + 1 empty result }); @@ -70,19 +94,18 @@ describe("fetchPaginated", () => { }), ); - const result = await fetchPaginated({ + const generator = fetchPaginated({ limit: 1, logger: mockLogger, - fetchPageFn: ({ limit, offset }) => - ResultAsync.fromPromise( - fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), - (e) => new SourceFetchError("Failed to fetch", { cause: e }), - ), + fetchPageFn: ({ limit, offset }) => fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), }); - expect(result).toBeOk(); - const data = result._unsafeUnwrap(); - expect(data.pages).toHaveLength(1); + const pages: unknown[] = []; + for await (const page of generator) { + pages.push(page); + } + + expect(pages).toHaveLength(1); expect(callCount).toBe(2); }); @@ -93,51 +116,43 @@ describe("fetchPaginated", () => { }), ); - const result = await fetchPaginated({ + const generator = fetchPaginated({ limit: 10, logger: mockLogger, fetchPageFn: ({ limit, offset }) => - ResultAsync.fromPromise( - fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => { - if (!res.ok) throw new Error("API error"); - return res.json(); - }), - (e) => new SourceFetchError("Failed to fetch", { cause: e }), - ), + fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => { + if (!res.ok) throw new Error("API error"); + return res.json(); + }), }); - expect(result).toBeErr(); - const error = result._unsafeUnwrapErr(); - expect(error).toBeInstanceOf(SourceFetchError); - expect(error.message).toBe("Failed to fetch"); + await expect(() => generator.next()).rejects.toThrow("API error"); }); - it("should include meta data when provided", async () => { + it("should respect maxFetchCount", async () => { + let callCount = 0; server.use( http.get("http://example.com/api", ({ request }) => { const url = new URL(request.url); - const offset = Number(url.searchParams.get("offset")); - if (offset === 0) { - return HttpResponse.json([{ id: 1 }]); - } - return HttpResponse.json([]); + const limit = Number(url.searchParams.get("limit")); + callCount++; + return HttpResponse.json(Array.from({ length: limit }, (_, i) => ({ id: i + 1 }))); }), ); - const result = await fetchPaginated({ + const generator = fetchPaginated({ limit: 10, + maxFetchCount: 2, logger: mockLogger, - meta: { totalCount: 1 }, - fetchPageFn: ({ limit, offset }) => - ResultAsync.fromPromise( - fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), - (e) => new SourceFetchError("Failed to fetch", { cause: e }), - ), + fetchPageFn: ({ limit, offset }) => fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), }); - expect(result).toBeOk(); - const data = result._unsafeUnwrap(); - expect(data.pages).toHaveLength(1); - expect(data.meta).toEqual({ totalCount: 1 }); + const pages: unknown[] = []; + for await (const page of generator) { + pages.push(page); + } + + expect(pages).toHaveLength(2); + expect(callCount).toBe(2); }); }); diff --git a/packages/content/src/utils/data-store.ts b/packages/content/src/utils/data-store.ts index 8f3dca7b..f6a78b31 100644 --- a/packages/content/src/utils/data-store.ts +++ b/packages/content/src/utils/data-store.ts @@ -1,7 +1,8 @@ import { JSONPath } from "jsonpath-plus"; -import { Result, ResultAsync, err, ok } from "neverthrow"; -import * as fs from "node:fs/promises"; +import { Result, ResultAsync, err, errAsync, ok, okAsync } from "neverthrow"; +import fs from "node:fs/promises"; import path from "node:path"; +import { ensureDir } from "./file-utils.js"; export class DataStoreError extends Error { constructor(...args: ConstructorParameters) { @@ -87,9 +88,12 @@ class SingleDocument extends Document { async initialize(data: T | Promise) { const resolvedData = await data; + // create the directory if it doesn't exist + await fs.mkdir(path.dirname(this.#path), { recursive: true }); + // wx+ opens the file for reading and writing, creating it if it doesn't exist, and erroring if it does - this.#handlePromise = fs.open(this.#path, "wx+").then((handle) => { - handle.write(JSON.stringify(resolvedData)); + this.#handlePromise = fs.open(this.#path, "wx+").then(async (handle) => { + await handle.writeFile(JSON.stringify(resolvedData)); return handle; }); @@ -123,11 +127,13 @@ class SingleDocument extends Document { } const handle = await this.#getHandle(); - const data = await fs.readFile(handle, "utf-8"); + + const data = await handle.readFile("utf-8"); const updatedData = cb(JSON.parse(data)); - await fs.writeFile(handle, JSON.stringify(updatedData)); + await handle.truncate(0); // truncate the file to 0 bytes + await handle.writeFile(JSON.stringify(updatedData)); } override async apply(pathExpression: string, fn: (x: unknown) => unknown) { @@ -216,14 +222,21 @@ class Namespace { return this.#id; } - async insert(data: Promise | AsyncIterable) { + initialize() { + // create the directory if it doesn't exist + return ensureDir(this.#directory); + } + + async insert(id: string, data: Promise | AsyncIterable): Promise> { if (data instanceof Promise) { - const doc = await SingleDocument.create(this.#directory, BatchDocument.getIndexedId(this.#id, this.#documents.size), data); - this.#documents.set(doc.id, doc); - } else { - const doc = await BatchDocument.create(this.#directory, this.#id, data); + const doc = await SingleDocument.create(this.#directory, id, data); this.#documents.set(doc.id, doc); + return doc; } + + const doc = await BatchDocument.create(this.#directory, id, data); + this.#documents.set(doc.id, doc); + return doc; } /** @@ -275,13 +288,14 @@ export class DataStore { /** * Create a new namespace in the data store. */ - createNamespace(namespaceId: string): Result { + createNamespace(namespaceId: string): ResultAsync { if (this.#namespaces.has(namespaceId)) { - return err(new DataStoreError(`Namespace ${namespaceId} already exists in data store`)); + return errAsync(new DataStoreError(`Namespace ${namespaceId} already exists in data store`)); } - this.#namespaces.set(namespaceId, new Namespace(this.#directory, namespaceId)); - return ok(undefined); + const namespace = new Namespace(this.#directory, namespaceId); + this.#namespaces.set(namespaceId, namespace); + return namespace.initialize().andThen(() => okAsync(namespace)); } /** diff --git a/packages/testing/src/setup.ts b/packages/testing/src/setup.ts index 39e78e86..86a345c9 100644 --- a/packages/testing/src/setup.ts +++ b/packages/testing/src/setup.ts @@ -88,7 +88,7 @@ vi.mock("winston-daily-rotate-file", async () => { const { default: Transport } = await import("winston-transport"); class DummyTransport extends Transport { - log(info: LogEntry) { + override log(info: LogEntry) { // do nothing } } From 139a70764ee13124ef666229a385815cfe45ad85 Mon Sep 17 00:00:00 2001 From: Clay Tercek <30105080+claytercek@users.noreply.github.com> Date: Thu, 7 Nov 2024 15:21:18 -0500 Subject: [PATCH 4/6] refactor plugins and content core to work with new datastore --- .../src/__tests__/content-integration.test.ts | 3 +- .../__tests__/content-plugin-driver.test.ts | 2 +- .../src/__tests__/launchpad-content.test.ts | 31 ++---- packages/content/src/launchpad-content.ts | 57 +++-------- .../src/plugins/__tests__/md-to-html.test.ts | 68 +++++++------ .../__tests__/media-downloader.test.ts | 75 +++++++++----- .../plugins/__tests__/plugins.test-utils.ts | 14 +-- .../plugins/__tests__/sanity-to-html.test.ts | 43 ++++---- .../__tests__/sanity-to-markdown.test.ts | 43 ++++---- .../plugins/__tests__/sanity-to-plain.test.ts | 74 +++++++------- packages/content/src/plugins/md-to-html.ts | 2 +- .../content/src/plugins/media-downloader.ts | 98 +++++++++---------- .../content/src/plugins/sanity-to-html.ts | 2 +- .../content/src/plugins/sanity-to-markdown.ts | 2 +- .../content/src/plugins/sanity-to-plain.ts | 2 +- .../__tests__/content-transform-utils.test.ts | 56 ++++++----- .../src/utils/content-transform-utils.ts | 8 +- packages/content/src/utils/data-store.ts | 94 +++++++++++++++--- packages/testing/src/setup.ts | 4 +- 19 files changed, 370 insertions(+), 308 deletions(-) diff --git a/packages/content/src/__tests__/content-integration.test.ts b/packages/content/src/__tests__/content-integration.test.ts index 7506c5b0..4bc6bfbb 100644 --- a/packages/content/src/__tests__/content-integration.test.ts +++ b/packages/content/src/__tests__/content-integration.test.ts @@ -85,7 +85,7 @@ describe("Content Integration", () => { describe("Sanity Source with HTML Conversion", () => { it("should fetch Sanity content and convert blocks to HTML", async () => { server.use( - http.get("https://test-project.api.sanity.io/v2021-10-21/data/query/production", ({ request }) => { + http.get("https://test-project.apicdn.sanity.io/v2021-10-21/data/query/production", ({ request }) => { const url = new URL(request.url); const query = url.searchParams.get("query"); @@ -131,6 +131,7 @@ describe("Content Integration", () => { projectId: "test-project", apiToken: "test-token", queries: ["article"], + mergePages: true, }), ], plugins: [sanityToHtml({ path: "$..content" })], diff --git a/packages/content/src/__tests__/content-plugin-driver.test.ts b/packages/content/src/__tests__/content-plugin-driver.test.ts index e660b7f1..37a7ad1d 100644 --- a/packages/content/src/__tests__/content-plugin-driver.test.ts +++ b/packages/content/src/__tests__/content-plugin-driver.test.ts @@ -7,7 +7,7 @@ import { DataStore } from "../utils/data-store.js"; describe("ContentPluginDriver", () => { const createMockContext = () => { - const dataStore = new DataStore(); + const dataStore = new DataStore("/"); const options = resolveContentConfig({ downloadPath: "/downloads/", tempPath: "/temp/", diff --git a/packages/content/src/__tests__/launchpad-content.test.ts b/packages/content/src/__tests__/launchpad-content.test.ts index 81e5b45d..628bbc22 100644 --- a/packages/content/src/__tests__/launchpad-content.test.ts +++ b/packages/content/src/__tests__/launchpad-content.test.ts @@ -9,11 +9,8 @@ import { LaunchpadContent } from "../launchpad-content.js"; import { defineSource } from "../sources/source.js"; describe("LaunchpadContent", () => { - beforeEach(() => { - vol.reset(); - }); - afterEach(() => { + vol.reset(); vi.clearAllMocks(); }); @@ -26,17 +23,14 @@ describe("LaunchpadContent", () => { defineSource({ id: "test", fetch: () => { - return ok([ + return [ { id: "doc1", - dataPromise: okAsync([ - { - id: "doc1", - data: "doc1", - }, - ]), + data: Promise.resolve({ + hello: "world", + }), }, - ]); + ]; }, }), ], @@ -68,7 +62,7 @@ describe("LaunchpadContent", () => { const filePath = path.join("/downloads", "test", "doc1.json"); expect(vol.existsSync(filePath)).toBe(true); - expect(vol.readFileSync(filePath, "utf8")).toBe("doc1"); + expect(vol.readFileSync(filePath, "utf8")).toBe(JSON.stringify({ hello: "world" })); }); it("should respect keep patterns when clearing directories", async () => { @@ -147,17 +141,6 @@ describe("LaunchpadContent", () => { }); describe("error handling", () => { - it("should wrap filesystem errors", async () => { - // Make downloads directory read-only - vol.mkdirSync("/downloads", { recursive: true, mode: 0o444 }); - const content = new LaunchpadContent(createBasicConfig(), createMockLogger()); - content._dataStore.createNamespaceFromMap("test", new Map([["doc1", "doc1"]])); - const result = await content._writeDataStoreToDisk(content._dataStore); - - expect(result).toBeErr(); - expect(result._unsafeUnwrapErr()).toBeInstanceOf(ContentError); - }); - it("should handle directory clearing errors", async () => { // Make directory read-only vol.mkdirSync("/downloads", { recursive: true, mode: 0o777 }); diff --git a/packages/content/src/launchpad-content.ts b/packages/content/src/launchpad-content.ts index 642b081a..afb626d6 100644 --- a/packages/content/src/launchpad-content.ts +++ b/packages/content/src/launchpad-content.ts @@ -13,7 +13,7 @@ import { } from "./content-config.js"; import { ContentError, ContentPluginDriver } from "./content-plugin-driver.js"; import type { ContentSource } from "./sources/source.js"; -import { DataStore } from "./utils/data-store.js"; +import { DataStore, type DataStoreError } from "./utils/data-store.js"; import * as FileUtils from "./utils/file-utils.js"; export class LaunchpadContent { @@ -29,7 +29,7 @@ export class LaunchpadContent { this._logger = LogManager.getLogger("content", parentLogger); - this._dataStore = new DataStore(); + this._dataStore = new DataStore(this._config.downloadPath); // create all sources this._rawSources = this._config.sources; @@ -69,7 +69,7 @@ export class LaunchpadContent { ) .andThen(() => this._fetchSources(sources)) .andThrough(() => this._pluginDriver.runHookSequential("onContentFetchDone")) - .andThen(() => this._writeDataStoreToDisk(this._dataStore)) + .andThen(() => ResultAsync.fromPromise(this._dataStore.close(), (e) => new ContentError("Failed to close data store", { cause: e }))) .orElse((e) => { this._pluginDriver.runHookSequential("onContentFetchError", e); this._logger.error("Error in content fetch process:", e); @@ -226,12 +226,7 @@ export class LaunchpadContent { // wrap source in promise to ensure it's awaited Promise.resolve(source), (error) => new ContentError("Failed to build source", { cause: error }), - ).andThen((awaited) => { - if ("value" in awaited || "error" in awaited) { - return awaited.mapErr((e) => new ContentError("Failed to build source", { cause: e })); - } - return ok(awaited); - }), + ), ), ).orElse((e) => { this._pluginDriver.runHookSequential("onSetupError", e); @@ -240,48 +235,28 @@ export class LaunchpadContent { } _fetchSources(sources: ContentSource[]): ResultAsync { + // Fetch sources in parallel return ResultAsync.combine( sources.map((source) => { const sourceLogger = LogManager.getLogger(`source:${source.id}`, this._logger); - return source - .fetch({ - logger: sourceLogger, - dataStore: this._dataStore, - }) - .asyncAndThen((calls) => { - return ResultAsync.combine(calls.map((call) => call.dataPromise)); - }) - .mapErr((e) => new ContentError(`Failed to fetch source ${source.id}`, { cause: e })) - .andThrough((fetchResults) => { - const map = new Map(); + const initializedFetch = source.fetch({ + logger: sourceLogger, + dataStore: this._dataStore, + }); - for (const result of fetchResults.flat()) { - map.set(result.id, result.data); - } + const fetchAsArray = Array.isArray(initializedFetch) ? initializedFetch : [initializedFetch]; - return this._dataStore - .createNamespaceFromMap(source.id, map) - .mapErr((e) => new ContentError(`Unable to create namespace for source ${source.id}`, { cause: e })); - }); + return this._dataStore + .createNamespace(source.id) + .andThen((namespace) => { + return ResultAsync.combine(fetchAsArray.map((fetch) => namespace.safeInsert(fetch.id, fetch.data))); + }) + .mapErr((e) => new ContentError(`Failed to fetch source ${source.id}`, { cause: e })); }), ).map(() => undefined); // return void instead of void[] } - _writeDataStoreToDisk(dataStore: DataStore): ResultAsync { - return ResultAsync.combine( - Array.from(dataStore.namespaces()).flatMap((namespace) => - Array.from(namespace.documents()).map((document) => { - const encodeRegex = new RegExp(`[${this._config.encodeChars}]`, "g"); - const filePath = path.join(this._config.downloadPath, namespace.id, document.id).replace(encodeRegex, encodeURIComponent); - return FileUtils.saveJson(document.data, filePath); - }), - ), - ) - .mapErr((e) => new ContentError("Failed to write data store to disk", { cause: e })) - .map(() => undefined); // return void instead of void[] - } - _clearDir(dirPath: string, { removeIfEmpty = true, ignoreKeep = false } = {}): ResultAsync { return FileUtils.pathExists(dirPath) .andThen((exists) => { diff --git a/packages/content/src/plugins/__tests__/md-to-html.test.ts b/packages/content/src/plugins/__tests__/md-to-html.test.ts index 5995d67c..4112c737 100644 --- a/packages/content/src/plugins/__tests__/md-to-html.test.ts +++ b/packages/content/src/plugins/__tests__/md-to-html.test.ts @@ -3,60 +3,66 @@ import mdToHtml from "../md-to-html.js"; import { createTestPluginContext } from "./plugins.test-utils.js"; describe("mdToHtml plugin", () => { - it("should convert markdown to html", () => { - const ctx = createTestPluginContext(); - ctx.data.insert("test", "doc1", { content: "# Hello\n\nThis is **bold** and *italic*." }); + it("should convert markdown to html", async () => { + const ctx = await createTestPluginContext(); + const namespaceResult = await ctx.data.createNamespace("test"); + const namespace = namespaceResult._unsafeUnwrap(); + await namespace.insert("doc1", Promise.resolve({ content: "# Hello\n\nThis is **bold** and *italic*." })); const plugin = mdToHtml({ path: "$.content" }); - plugin.hooks.onContentFetchDone(ctx); + await plugin.hooks.onContentFetchDone(ctx); - const result = ctx.data.get("test", "doc1")._unsafeUnwrap(); - expect((result.data as Record).content).toBe("

Hello

\n

This is bold and italic.

\n"); + const result = await ctx.data.getDocument("test", "doc1")._unsafeUnwrap()._read(); + expect((result as any).content).toBe("

Hello

\n

This is bold and italic.

\n"); }); - it("should convert markdown to simplified html when simplified=true", () => { - const ctx = createTestPluginContext(); - ctx.data.insert("test", "doc1", { content: "This is **bold** and *italic*." }); + it("should convert markdown to simplified html when simplified=true", async () => { + const ctx = await createTestPluginContext(); + const namespace = (await ctx.data.createNamespace("test"))._unsafeUnwrap(); + await namespace.insert("doc1", Promise.resolve({ content: "This is **bold** and *italic*." })); const plugin = mdToHtml({ path: "$.content", simplified: true }); - plugin.hooks.onContentFetchDone(ctx); + await plugin.hooks.onContentFetchDone(ctx); - const result = ctx.data.get("test", "doc1")._unsafeUnwrap(); - expect((result.data as Record).content).toBe("This is bold and italic."); + const result = await ctx.data.getDocument("test", "doc1")._unsafeUnwrap()._read(); + expect((result as any).content).toBe("This is bold and italic."); }); - it("should only transform specified keys", () => { - const ctx = createTestPluginContext(); - ctx.data.createNamespace("skip"); - ctx.data.insert("test", "doc1", { content: "# Hello" }); - ctx.data.insert("skip", "doc2", { content: "# Hello" }); + it("should only transform specified keys", async () => { + const ctx = await createTestPluginContext(); + const namespaceTest = (await ctx.data.createNamespace("test"))._unsafeUnwrap(); + await namespaceTest.insert("doc1", Promise.resolve({ content: "# Hello" })); + const namespaceSkip = (await ctx.data.createNamespace("skip"))._unsafeUnwrap(); + await namespaceSkip.insert("doc2", Promise.resolve({ content: "# Hello" })); const plugin = mdToHtml({ path: "$.content", keys: ["test"] }); - plugin.hooks.onContentFetchDone(ctx); + await plugin.hooks.onContentFetchDone(ctx); - const transformed = ctx.data.get("test", "doc1")._unsafeUnwrap(); - const skipped = ctx.data.get("skip", "doc2")._unsafeUnwrap(); + const transformed = await ctx.data.getDocument("test", "doc1")._unsafeUnwrap()._read(); + const skipped = await ctx.data.getDocument("skip", "doc2")._unsafeUnwrap()._read(); - expect((transformed.data as Record).content).toBe("

Hello

\n"); - expect((skipped.data as Record).content).toBe("# Hello"); + expect((transformed as any).content).toBe("

Hello

\n"); + expect((skipped as any).content).toBe("# Hello"); }); - it("should sanitize html in markdown content", () => { - const ctx = createTestPluginContext(); - ctx.data.insert("test", "doc1", { content: 'Hello ' }); + it("should sanitize html in markdown content", async () => { + const ctx = await createTestPluginContext(); + const namespace = (await ctx.data.createNamespace("test"))._unsafeUnwrap(); + await namespace.insert("doc1", Promise.resolve({ content: 'Hello ' })); const plugin = mdToHtml({ path: "$.content" }); - plugin.hooks.onContentFetchDone(ctx); + await plugin.hooks.onContentFetchDone(ctx); - const result = ctx.data.get("test", "doc1")._unsafeUnwrap(); - expect((result.data as Record).content).not.toContain("' })); + await namespace.insert( + "doc1", + Promise.resolve({ content: 'Hello ' }), + ); const plugin = mdToHtml({ path: "$.content" }); await plugin.hooks.onContentFetchDone(ctx); @@ -63,6 +71,8 @@ describe("mdToHtml plugin", () => { await namespace.insert("doc1", Promise.resolve({ content: { foo: "bar" } })); const plugin = mdToHtml({ path: "$.content" }); - expect(plugin.hooks.onContentFetchDone(ctx)).rejects.toThrow("Error applying content transform"); + expect(plugin.hooks.onContentFetchDone(ctx)).rejects.toThrow( + "Error applying content transform", + ); }); }); diff --git a/packages/content/src/plugins/__tests__/media-downloader.test.ts b/packages/content/src/plugins/__tests__/media-downloader.test.ts index 4986ef9c..e730baf7 100644 --- a/packages/content/src/plugins/__tests__/media-downloader.test.ts +++ b/packages/content/src/plugins/__tests__/media-downloader.test.ts @@ -3,9 +3,15 @@ import { vol } from "memfs"; import { http, HttpResponse } from "msw"; import { setupServer } from "msw/node"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; -import mediaDownloader, { localFilePathFromUrl, checkCacheStatus, downloadFile, findMediaUrls, mediaDownloaderConfigSchema } from "../media-downloader.js"; -import { createTestPluginContext } from "./plugins.test-utils.js"; import type { z } from "zod"; +import mediaDownloader, { + localFilePathFromUrl, + checkCacheStatus, + downloadFile, + findMediaUrls, + mediaDownloaderConfigSchema, +} from "../media-downloader.js"; +import { createTestPluginContext } from "./plugins.test-utils.js"; function getMediaDownloaderConfig(config: z.input) { return mediaDownloaderConfigSchema.parse(config); @@ -21,7 +27,9 @@ describe("mediaDownloader", () => { describe("localFilePathFromUrl", () => { it("should strip protocol and domain", () => { - expect(localFilePathFromUrl("https://example.com/path/to/file.jpg")).toBe(`path${path.sep}to${path.sep}file.jpg`); + expect(localFilePathFromUrl("https://example.com/path/to/file.jpg")).toBe( + `path${path.sep}to${path.sep}file.jpg`, + ); }); it("should handle URLs without leading slash", () => { @@ -29,7 +37,9 @@ describe("mediaDownloader", () => { }); it("should handle URLs with query params", () => { - expect(localFilePathFromUrl("https://example.com/path/file.jpg?size=large")).toBe(`path${path.sep}file.jpg?size=large`); + expect(localFilePathFromUrl("https://example.com/path/file.jpg?size=large")).toBe( + `path${path.sep}file.jpg?size=large`, + ); }); }); @@ -39,7 +49,10 @@ describe("mediaDownloader", () => { "https://example.com/new.jpg", "/downloads/new.jpg", new AbortController().signal, - getMediaDownloaderConfig({ enableIfModifiedSinceCheck: true, enableContentLengthCheck: true }), + getMediaDownloaderConfig({ + enableIfModifiedSinceCheck: true, + enableContentLengthCheck: true, + }), ); expect(result).toBeOk(); @@ -64,7 +77,10 @@ describe("mediaDownloader", () => { "https://example.com/cached.jpg", "/downloads/cached.jpg", new AbortController().signal, - getMediaDownloaderConfig({ enableIfModifiedSinceCheck: true, enableContentLengthCheck: false }), + getMediaDownloaderConfig({ + enableIfModifiedSinceCheck: true, + enableContentLengthCheck: false, + }), ); expect(result).toBeOk(); @@ -90,7 +106,10 @@ describe("mediaDownloader", () => { "https://example.com/size.jpg", "/downloads/size.jpg", new AbortController().signal, - getMediaDownloaderConfig({ enableIfModifiedSinceCheck: false, enableContentLengthCheck: true }), + getMediaDownloaderConfig({ + enableIfModifiedSinceCheck: false, + enableContentLengthCheck: true, + }), ); expect(result).toBeOk(); @@ -148,11 +167,21 @@ describe("mediaDownloader", () => { await namespace.insert( "doc1", Promise.resolve({ - images: ["https://example.com/1.jpg", "https://example.com/1.jpg", "https://example.com/2.png", "not-a-url.txt", "https://example.com/doc.pdf"], + images: [ + "https://example.com/1.jpg", + "https://example.com/1.jpg", + "https://example.com/2.png", + "not-a-url.txt", + "https://example.com/doc.pdf", + ], }), ); - const urls = await findMediaUrls(ctx.data, getMediaDownloaderConfig({ mediaPattern: /\.(jpg|png)$/i }), "$..*[?(@.match(/\\.(jpg|png)$/i))]"); + const urls = await findMediaUrls( + ctx.data, + getMediaDownloaderConfig({ mediaPattern: /\.(jpg|png)$/i }), + "$..*[?(@.match(/\\.(jpg|png)$/i))]", + ); expect(urls).toMatchObject([ { url: "https://example.com/1.jpg", sourceId: "test" }, @@ -173,7 +202,11 @@ describe("mediaDownloader", () => { }), ); - const urls = await findMediaUrls(ctx.data, getMediaDownloaderConfig({ matchPath: "$..*[?(@.url)].url" }), "$..*[?(@.url)].url"); + const urls = await findMediaUrls( + ctx.data, + getMediaDownloaderConfig({ matchPath: "$..*[?(@.url)].url" }), + "$..*[?(@.url)].url", + ); expect(urls).toMatchObject([ { url: "https://example.com/1.jpg", sourceId: "test" }, diff --git a/packages/content/src/plugins/__tests__/plugins.test-utils.ts b/packages/content/src/plugins/__tests__/plugins.test-utils.ts index 78739974..91a098e9 100644 --- a/packages/content/src/plugins/__tests__/plugins.test-utils.ts +++ b/packages/content/src/plugins/__tests__/plugins.test-utils.ts @@ -1,10 +1,10 @@ import { createMockLogger } from "@bluecadet/launchpad-testing/test-utils.ts"; import type { Logger } from "@bluecadet/launchpad-utils"; +import { vol } from "memfs"; import { vi } from "vitest"; +import { afterEach } from "vitest"; import { type ContentConfig, resolveContentConfig } from "../../content-config.js"; import { DataStore } from "../../utils/data-store.js"; -import { afterEach } from "vitest"; -import { vol } from "memfs"; afterEach(() => { vol.reset(); diff --git a/packages/content/src/plugins/__tests__/sanity-to-html.test.ts b/packages/content/src/plugins/__tests__/sanity-to-html.test.ts index de507ffe..f5461d2a 100644 --- a/packages/content/src/plugins/__tests__/sanity-to-html.test.ts +++ b/packages/content/src/plugins/__tests__/sanity-to-html.test.ts @@ -48,6 +48,8 @@ describe("sanityToHtml plugin", () => { await namespace.insert("doc1", Promise.resolve({ content: "not a block" })); const plugin = sanityToHtml({ path: "$.content" }); - await expect(plugin.hooks.onContentFetchDone(ctx)).rejects.toThrow("Error applying content transform"); + await expect(plugin.hooks.onContentFetchDone(ctx)).rejects.toThrow( + "Error applying content transform", + ); }); }); diff --git a/packages/content/src/plugins/__tests__/sanity-to-markdown.test.ts b/packages/content/src/plugins/__tests__/sanity-to-markdown.test.ts index a74aab4a..a4bef2e9 100644 --- a/packages/content/src/plugins/__tests__/sanity-to-markdown.test.ts +++ b/packages/content/src/plugins/__tests__/sanity-to-markdown.test.ts @@ -48,6 +48,8 @@ describe("sanityToMd plugin", () => { await namespace.insert("doc1", Promise.resolve({ content: "not a block" })); const plugin = sanityToMd({ path: "$.content" }); - await expect(plugin.hooks.onContentFetchDone(ctx)).rejects.toThrow("Error applying content transform"); + await expect(plugin.hooks.onContentFetchDone(ctx)).rejects.toThrow( + "Error applying content transform", + ); }); }); diff --git a/packages/content/src/plugins/__tests__/sanity-to-plain.test.ts b/packages/content/src/plugins/__tests__/sanity-to-plain.test.ts index 10c681a8..7e53d591 100644 --- a/packages/content/src/plugins/__tests__/sanity-to-plain.test.ts +++ b/packages/content/src/plugins/__tests__/sanity-to-plain.test.ts @@ -52,7 +52,9 @@ describe("sanityToPlain plugin", () => { await namespace.insert("doc1", Promise.resolve({ content: "not a block" })); const plugin = sanityToPlain({ path: "$.content" }); - await expect(plugin.hooks.onContentFetchDone(ctx)).rejects.toThrow("Error applying content transform"); + await expect(plugin.hooks.onContentFetchDone(ctx)).rejects.toThrow( + "Error applying content transform", + ); }); it("should throw error for block without children", async () => { @@ -64,7 +66,9 @@ describe("sanityToPlain plugin", () => { await namespace.insert("doc1", Promise.resolve({ content: invalidBlock })); const plugin = sanityToPlain({ path: "$.content" }); - await expect(plugin.hooks.onContentFetchDone(ctx)).rejects.toThrow("Error applying content transform"); + await expect(plugin.hooks.onContentFetchDone(ctx)).rejects.toThrow( + "Error applying content transform", + ); }); it("should throw error for block with invalid children", async () => { @@ -82,7 +86,9 @@ describe("sanityToPlain plugin", () => { await namespace.insert("doc1", Promise.resolve({ content: invalidBlock })); const plugin = sanityToPlain({ path: "$.content" }); - await expect(plugin.hooks.onContentFetchDone(ctx)).rejects.toThrow("Error applying content transform"); + await expect(plugin.hooks.onContentFetchDone(ctx)).rejects.toThrow( + "Error applying content transform", + ); }); it("should concatenate multiple text spans", async () => { diff --git a/packages/content/src/plugins/media-downloader.ts b/packages/content/src/plugins/media-downloader.ts index 24446814..258d721d 100644 --- a/packages/content/src/plugins/media-downloader.ts +++ b/packages/content/src/plugins/media-downloader.ts @@ -5,12 +5,12 @@ import { pipeline } from "node:stream/promises"; import chalk from "chalk"; import { JSONPath } from "jsonpath-plus"; import { ResultAsync, errAsync, ok, okAsync } from "neverthrow"; +import { z } from "zod"; import { type ContentHookContext, defineContentPlugin } from "../content-plugin-driver.js"; import type { DataKeys, DataStore } from "../utils/data-store.js"; import * as FileUtils from "../utils/file-utils.js"; import ResultAsyncQueue from "../utils/result-async-queue.js"; import { safeKy } from "../utils/safe-ky.js"; -import { z } from "zod"; const DEFAULT_MEDIA_PATTERN = /\.(jpg|jpeg|png|webp|avi|mov|mp4|mpg|mpeg|webm)$/i; @@ -19,19 +19,35 @@ const DEFAULT_MEDIA_PATTERN = /\.(jpg|jpeg|png|webp|avi|mov|mp4|mpg|mpeg|webm)$/ */ export const mediaDownloaderConfigSchema = z.object({ /** Data keys to search for media urls. If not provided, all keys will be searched. */ - keys: z.array(z.string()).optional().describe("Data keys to search for media urls. If not provided, all keys will be searched."), + keys: z + .array(z.string()) + .optional() + .describe("Data keys to search for media urls. If not provided, all keys will be searched."), /** Regex to match urls to download. */ - mediaPattern: z.instanceof(RegExp).describe("Regex to match urls to download.").default(DEFAULT_MEDIA_PATTERN), + mediaPattern: z + .instanceof(RegExp) + .describe("Regex to match urls to download.") + .default(DEFAULT_MEDIA_PATTERN), /** JSONPath-Plus compatible paths to match urls to download. Overrides `mediaPattern`. */ - matchPath: z.string().optional().describe("JSONPath-Plus compatible paths to match urls to download. Overrides `mediaPattern`."), + matchPath: z + .string() + .optional() + .describe( + "JSONPath-Plus compatible paths to match urls to download. Overrides `mediaPattern`.", + ), /** Number of concurrent downloads */ maxConcurrent: z.number().int().positive().describe("Number of concurrent downloads").default(4), /** Will always download files regardless of whether they've been cached. Defaults to false. */ - ignoreCache: z.boolean().describe("Will always download files regardless of whether they've been cached.").default(false), + ignoreCache: z + .boolean() + .describe("Will always download files regardless of whether they've been cached.") + .default(false), /** Enables the HTTP if-modified-since check. Disabling this will assume that the local file is the same as the remote file if it already exists. Defaults to true. */ enableIfModifiedSinceCheck: z .boolean() - .describe("Enables the HTTP if-modified-since check. Disabling this will assume that the local file is the same as the remote file if it already exists.") + .describe( + "Enables the HTTP if-modified-since check. Disabling this will assume that the local file is the same as the remote file if it already exists.", + ) .default(true), /** Compares the HTTP header content-length with the local file size. Disabling this will assume that the local file is the same as the remote file if it already exists. Defaults to true. */ enableContentLengthCheck: z @@ -41,14 +57,22 @@ export const mediaDownloaderConfigSchema = z.object({ ) .default(true), /** Clear the temp dir before starting downloads. This means the cache will be ignored. Defaults to false. */ - forceClearTempFiles: z.boolean().describe("Clear the temp dir before starting downloads. This means the cache will be ignored.").default(false), + forceClearTempFiles: z + .boolean() + .describe("Clear the temp dir before starting downloads. This means the cache will be ignored.") + .default(false), /** Function to transform the local path of the downloaded file. */ transformLocalPath: z .function(z.tuple([z.string()]), z.string()) .describe("Function to transform the local path of the downloaded file.") .optional(), /** Maximum timeout for the HTTP request. Defaults to 10 seconds. */ - maxTimeout: z.number().int().positive().describe("Maximum timeout for the HTTP request.").default(10000), + maxTimeout: z + .number() + .int() + .positive() + .describe("Maximum timeout for the HTTP request.") + .default(10000), /** If true, the queue will stop and throw an error if any of the media requests fail. If false, the queue will continue to download the remaining files and log all errors, but not throw. Defaults to true. */ abortOnError: z .boolean() @@ -72,13 +96,19 @@ export function checkCacheStatus( url: string, destPath: string, abortSignal: AbortSignal, - config: Pick, + config: Pick< + MediaDownloaderConfigWithDefaults, + "ignoreCache" | "enableIfModifiedSinceCheck" | "enableContentLengthCheck" | "maxTimeout" + >, ) { return FileUtils.pathExists(destPath) .mapErr((err) => new FileSystemError("Failed to check if file exists", err)) .andThen((exists) => exists - ? ResultAsync.fromPromise(fs.promises.lstat(destPath), (err) => new FileSystemError(`Failed to get file stats for ${destPath}`, err)) + ? ResultAsync.fromPromise( + fs.promises.lstat(destPath), + (err) => new FileSystemError(`Failed to get file stats for ${destPath}`, err), + ) : okAsync(null), ) .map((stats) => { @@ -146,7 +176,10 @@ export function downloadFile( } const writer = fs.createWriteStream(filePath); - return ResultAsync.fromPromise(pipeline(res.body, writer), (err) => new FileSystemError(`Failed to write file: ${err}`, err)); + return ResultAsync.fromPromise( + pipeline(res.body, writer), + (err) => new FileSystemError(`Failed to write file: ${err}`, err), + ); }); } @@ -170,14 +203,20 @@ export function downloadMedia( .andThen(() => checkCacheStatus(url, destPath, abortSignal, config)) .andThen((cacheStatus) => { if (!cacheStatus.shouldDownload && cacheStatus.existingFile) { - return FileUtils.copy(cacheStatus.existingFile, tempFilePath).mapErr((e) => new FileSystemError("Failed to copy existing file to temp dir", e)); + return FileUtils.copy(cacheStatus.existingFile, tempFilePath).mapErr( + (e) => new FileSystemError("Failed to copy existing file to temp dir", e), + ); } return downloadFile(url, tempFilePath, abortSignal, config); }) .map(() => tempFilePath); } -export async function findMediaUrls(dataStore: DataStore, options: MediaDownloaderConfigWithDefaults, queryJsonPath: string) { +export async function findMediaUrls( + dataStore: DataStore, + options: MediaDownloaderConfigWithDefaults, + queryJsonPath: string, +) { const returnUrls: { url: string; sourceId: string }[] = []; const filteredResult = dataStore.filter(options.keys); @@ -204,15 +243,23 @@ export async function findMediaUrls(dataStore: DataStore, options: MediaDownload return returnUrls; } -export function setupDownloadDirectories(ctx: ContentHookContext, config: MediaDownloaderConfigWithDefaults): ResultAsync { +export function setupDownloadDirectories( + ctx: ContentHookContext, + config: MediaDownloaderConfigWithDefaults, +): ResultAsync { return ( config.forceClearTempFiles - ? FileUtils.remove(ctx.paths.getTempPath()).andThen(() => FileUtils.ensureDir(ctx.paths.getTempPath())) + ? FileUtils.remove(ctx.paths.getTempPath()).andThen(() => + FileUtils.ensureDir(ctx.paths.getTempPath()), + ) : FileUtils.ensureDir(ctx.paths.getTempPath()) ).mapErr((err) => new FileSystemError("Failed to setup download directories", err)); } -export function cleanupAfterDownload(ctx: ContentHookContext, config: MediaDownloaderConfigWithDefaults): ResultAsync { +export function cleanupAfterDownload( + ctx: ContentHookContext, + config: MediaDownloaderConfigWithDefaults, +): ResultAsync { return FileUtils.copy(ctx.paths.getTempPath(), ctx.paths.getDownloadPath()) .andThen(() => FileUtils.remove(ctx.paths.getTempPath())) .mapErr((err) => new FileSystemError("Failed to cleanup after download", err)); @@ -225,13 +272,22 @@ export default function mediaDownloader(config: z.input @@ -268,7 +324,9 @@ export default function mediaDownloader(config: z.input { - ctx.logger.error(`Encountered ${chalk.red(`${queueErrors.length} error(s)`)} while downloading ${chalk.cyan(tasks.length)} files`); + ctx.logger.error( + `Encountered ${chalk.red(`${queueErrors.length} error(s)`)} while downloading ${chalk.cyan(tasks.length)} files`, + ); for (const error of queueErrors) { ctx.logger.error(chalk.red(error)); diff --git a/packages/content/src/plugins/sanity-to-plain.ts b/packages/content/src/plugins/sanity-to-plain.ts index 8b50bf18..7b2c6921 100644 --- a/packages/content/src/plugins/sanity-to-plain.ts +++ b/packages/content/src/plugins/sanity-to-plain.ts @@ -32,7 +32,9 @@ export default function sanityToPlain({ path, keys }: SanityToPlainOptions) { }); } -function isBlockWithChildren(content: unknown): content is { _type: "block"; children: { text: string }[] } { +function isBlockWithChildren( + content: unknown, +): content is { _type: "block"; children: { text: string }[] } { // check if object if (!isBlockContent(content)) { return false; diff --git a/packages/content/src/sources/__tests__/contentful-source.test.ts b/packages/content/src/sources/__tests__/contentful-source.test.ts index 6c99ecfd..48803b86 100644 --- a/packages/content/src/sources/__tests__/contentful-source.test.ts +++ b/packages/content/src/sources/__tests__/contentful-source.test.ts @@ -55,44 +55,47 @@ describe("contentfulSource", () => { it("should fetch data with delivery token", async () => { server.use( - http.get("https://cdn.contentful.com/spaces/test-space/environments/master/entries", ({ request }) => { - const url = new URL(request.url); - const skip = Number.parseInt(url.searchParams.get("skip") || "0"); + http.get( + "https://cdn.contentful.com/spaces/test-space/environments/master/entries", + ({ request }) => { + const url = new URL(request.url); + const skip = Number.parseInt(url.searchParams.get("skip") || "0"); + + // Return empty results after first page + if (skip > 0) { + return HttpResponse.json({ + items: [], + includes: {}, + total: 1, + skip, + limit: 1000, + }); + } - // Return empty results after first page - if (skip > 0) { return HttpResponse.json({ - items: [], - includes: {}, - total: 1, - skip, - limit: 1000, - }); - } - - return HttpResponse.json({ - items: [ - { - sys: { type: "Entry", contentType: { sys: { id: "article" } } }, - fields: { title: "Test Entry" }, - }, - ], - includes: { - Asset: [ + items: [ { - sys: { type: "Asset", id: "test-asset" }, - fields: { - title: "Test Asset", - file: { url: "//test.com/image.jpg" }, - }, + sys: { type: "Entry", contentType: { sys: { id: "article" } } }, + fields: { title: "Test Entry" }, }, ], - }, - total: 1, - skip: 0, - limit: 1000, - }); - }), + includes: { + Asset: [ + { + sys: { type: "Asset", id: "test-asset" }, + fields: { + title: "Test Asset", + file: { url: "//test.com/image.jpg" }, + }, + }, + ], + }, + total: 1, + skip: 0, + limit: 1000, + }); + }, + ), ); const source = await contentfulSource({ @@ -118,43 +121,46 @@ describe("contentfulSource", () => { it("should fetch data with preview token", async () => { server.use( - http.get("https://preview.contentful.com/spaces/test-space/environments/master/entries", ({ request }) => { - const url = new URL(request.url); - const skip = Number.parseInt(url.searchParams.get("skip") || "0"); + http.get( + "https://preview.contentful.com/spaces/test-space/environments/master/entries", + ({ request }) => { + const url = new URL(request.url); + const skip = Number.parseInt(url.searchParams.get("skip") || "0"); + + if (skip > 0) { + return HttpResponse.json({ + items: [], + includes: {}, + total: 1, + skip, + limit: 1000, + }); + } - if (skip > 0) { return HttpResponse.json({ - items: [], - includes: {}, - total: 1, - skip, - limit: 1000, - }); - } - - return HttpResponse.json({ - items: [ - { - sys: { type: "Entry", contentType: { sys: { id: "article" } } }, - fields: { title: "Preview Entry" }, - }, - ], - includes: { - Asset: [ + items: [ { - sys: { type: "Asset" }, - fields: { - title: "Preview Asset", - file: { url: "//test.com/preview.jpg" }, - }, + sys: { type: "Entry", contentType: { sys: { id: "article" } } }, + fields: { title: "Preview Entry" }, }, ], - }, - total: 1, - skip: 0, - limit: 1000, - }); - }), + includes: { + Asset: [ + { + sys: { type: "Asset" }, + fields: { + title: "Preview Asset", + file: { url: "//test.com/preview.jpg" }, + }, + }, + ], + }, + total: 1, + skip: 0, + limit: 1000, + }); + }, + ), ); const source = await contentfulSource({ @@ -244,37 +250,40 @@ describe("contentfulSource", () => { it("should respect content type filtering", async () => { server.use( - http.get("https://cdn.contentful.com/spaces/test-space/environments/master/entries", ({ request }) => { - const url = new URL(request.url); - const contentType = url.searchParams.get("sys.contentType.sys.id[in]"); + http.get( + "https://cdn.contentful.com/spaces/test-space/environments/master/entries", + ({ request }) => { + const url = new URL(request.url); + const contentType = url.searchParams.get("sys.contentType.sys.id[in]"); + + expect(contentType).toBe("article"); + + const skip = Number.parseInt(url.searchParams.get("skip") || "0"); + + if (skip > 0) { + return HttpResponse.json({ + items: [], + includes: {}, + total: 1, + skip, + limit: 1000, + }); + } - expect(contentType).toBe("article"); - - const skip = Number.parseInt(url.searchParams.get("skip") || "0"); - - if (skip > 0) { return HttpResponse.json({ - items: [], + items: [ + { + sys: { type: "Entry", contentType: { sys: { id: "article" } } }, + fields: { title: "Filtered Entry" }, + }, + ], includes: {}, total: 1, - skip, + skip: 0, limit: 1000, }); - } - - return HttpResponse.json({ - items: [ - { - sys: { type: "Entry", contentType: { sys: { id: "article" } } }, - fields: { title: "Filtered Entry" }, - }, - ], - includes: {}, - total: 1, - skip: 0, - limit: 1000, - }); - }), + }, + ), ); const source = await contentfulSource({ diff --git a/packages/content/src/sources/__tests__/sanity-source.test.ts b/packages/content/src/sources/__tests__/sanity-source.test.ts index 2580ec53..7bae0ff7 100644 --- a/packages/content/src/sources/__tests__/sanity-source.test.ts +++ b/packages/content/src/sources/__tests__/sanity-source.test.ts @@ -41,44 +41,47 @@ describe("sanitySource", () => { // Mock Sanity API responses server.use( // First page of 'test' type - http.get("https://test-project.api.sanity.io/v2021-10-21/data/query/production", ({ request }) => { - const url = new URL(request.url); + http.get( + "https://test-project.api.sanity.io/v2021-10-21/data/query/production", + ({ request }) => { + const url = new URL(request.url); + + const query = url.searchParams.get("query"); + + if (query === '*[_type == "test"][0..99]') { + return HttpResponse.json({ + result: [{ _type: "test", title: "Test Document 1" }], + ms: 15, + }); + } + + if (query === '*[_type == "test"][100..199]') { + return HttpResponse.json({ + result: [{ _type: "test", title: "Test Document 2" }], + ms: 15, + }); + } + + if (query === '*[_type == "article"][0..99]') { + return HttpResponse.json({ + result: [{ _type: "article", title: "Article 1" }], + ms: 15, + }); + } + + if (query === '*[_type == "article"][100..199]') { + return HttpResponse.json({ + result: [{ _type: "article", title: "Article 2" }], + ms: 15, + }); + } - const query = url.searchParams.get("query"); - - if (query === '*[_type == "test"][0..99]') { - return HttpResponse.json({ - result: [{ _type: "test", title: "Test Document 1" }], - ms: 15, - }); - } - - if (query === '*[_type == "test"][100..199]') { - return HttpResponse.json({ - result: [{ _type: "test", title: "Test Document 2" }], - ms: 15, - }); - } - - if (query === '*[_type == "article"][0..99]') { - return HttpResponse.json({ - result: [{ _type: "article", title: "Article 1" }], - ms: 15, - }); - } - - if (query === '*[_type == "article"][100..199]') { return HttpResponse.json({ - result: [{ _type: "article", title: "Article 2" }], - ms: 15, + result: [], + ms: 5, }); - } - - return HttpResponse.json({ - result: [], - ms: 5, - }); - }), + }, + ), ); const source = await sanitySource({ @@ -112,22 +115,28 @@ describe("sanitySource", () => { it("should fetch data with custom query objects", async () => { server.use( - http.get("https://test-project.api.sanity.io/v2021-10-21/data/query/production", ({ request }) => { - const url = new URL(request.url); - const query = url.searchParams.get("query"); + http.get( + "https://test-project.api.sanity.io/v2021-10-21/data/query/production", + ({ request }) => { + const url = new URL(request.url); + const query = url.searchParams.get("query"); + + if ( + query === '*[_type == "custom"][0..99]' || + query === '*[_type == "custom"][100..199]' + ) { + return HttpResponse.json({ + result: [{ _type: "custom", data: "Custom Data" }], + ms: 15, + }); + } - if (query === '*[_type == "custom"][0..99]' || query === '*[_type == "custom"][100..199]') { return HttpResponse.json({ - result: [{ _type: "custom", data: "Custom Data" }], - ms: 15, + result: [], + ms: 5, }); - } - - return HttpResponse.json({ - result: [], - ms: 5, - }); - }), + }, + ), ); const source = await sanitySource({ @@ -177,23 +186,26 @@ describe("sanitySource", () => { it("should respect pagination options", async () => { server.use( - http.get("https://test-project.api.sanity.io/v2021-10-21/data/query/production", ({ request }) => { - const url = new URL(request.url); - const query = url.searchParams.get("query") || ""; - const offset = query.match(/\[(\d+)\.\./)?.at(1); + http.get( + "https://test-project.api.sanity.io/v2021-10-21/data/query/production", + ({ request }) => { + const url = new URL(request.url); + const query = url.searchParams.get("query") || ""; + const offset = query.match(/\[(\d+)\.\./)?.at(1); + + if (offset === "100") { + return HttpResponse.json({ + result: [], + ms: 5, + }); + } - if (offset === "100") { return HttpResponse.json({ - result: [], - ms: 5, + result: [{ _type: "test", title: `Test Document ${offset}` }], + ms: 15, }); - } - - return HttpResponse.json({ - result: [{ _type: "test", title: `Test Document ${offset}` }], - ms: 15, - }); - }), + }, + ), ); const source = await sanitySource({ diff --git a/packages/content/src/sources/airtable-source.ts b/packages/content/src/sources/airtable-source.ts index 8e4afc5c..6c25e325 100644 --- a/packages/content/src/sources/airtable-source.ts +++ b/packages/content/src/sources/airtable-source.ts @@ -1,7 +1,7 @@ import type { Logger } from "@bluecadet/launchpad-utils"; import type Airtable from "airtable"; -import { defineSource, type SourceFetchResultDocument } from "./source.js"; import { z } from "zod"; +import { type SourceFetchResultDocument, defineSource } from "./source.js"; const airtableSourceSchema = z.object({ /** Required field to identify this source. Will be used as download path. */ @@ -13,18 +13,32 @@ const airtableSourceSchema = z.object({ "Airtable base ID. See https://help.appsheet.com/en/articles/1785063-using-data-from-airtable#:~:text=To%20obtain%20the%20ID%20of,API%20page%20of%20the%20base.", ), /** The table view which to select for syncing by default. Defaults to 'Grid view'. */ - defaultView: z.string().describe("The table view which to select for syncing by default. Defaults to 'Grid view'.").default("Grid view"), + defaultView: z + .string() + .describe("The table view which to select for syncing by default. Defaults to 'Grid view'.") + .default("Grid view"), /** The tables you want to fetch from. Defaults to []. */ - tables: z.array(z.string()).describe("The tables you want to fetch from. Defaults to [].").default([]), + tables: z + .array(z.string()) + .describe("The tables you want to fetch from. Defaults to [].") + .default([]), /** As a convenience feature, you can store tables listed here as key/value pairs. Field names should be `key` and `value`. Defaults to []. */ keyValueTables: z .array(z.string()) - .describe("As a convenience feature, you can store tables listed here as key/value pairs. Field names should be `key` and `value`. Defaults to [].") + .describe( + "As a convenience feature, you can store tables listed here as key/value pairs. Field names should be `key` and `value`. Defaults to [].", + ) .default([]), /** The API endpoint to use for Airtable. Defaults to 'https://api.airtable.com'. */ - endpointUrl: z.string().describe("The API endpoint to use for Airtable. Defaults to 'https://api.airtable.com'.").default("https://api.airtable.com"), + endpointUrl: z + .string() + .describe("The API endpoint to use for Airtable. Defaults to 'https://api.airtable.com'.") + .default("https://api.airtable.com"), /** Appends the local path of attachments to the saved JSON. Defaults to true. */ - appendLocalAttachmentPaths: z.boolean().describe("Appends the local path of attachments to the saved JSON. Defaults to true.").default(true), + appendLocalAttachmentPaths: z + .boolean() + .describe("Appends the local path of attachments to the saved JSON. Defaults to true.") + .default(true), /** Airtable API Key */ apiKey: z.string().describe("Airtable API Key"), }); @@ -32,7 +46,11 @@ const airtableSourceSchema = z.object({ /** * Fetch data from Airtable. */ -function fetchData(base: Airtable.Base, tableId: string, defaultView: string): Promise[]> { +function fetchData( + base: Airtable.Base, + tableId: string, + defaultView: string, +): Promise[]> { const rows: Airtable.Record[] = []; return new Promise((resolve, reject) => @@ -72,7 +90,10 @@ function isBoolStr(value: unknown) { return typeof value === "string" && (value === "true" || value === "false"); } -function processTableToSimplified(tableData: Airtable.Record[], isKeyValueTable: boolean): unknown { +function processTableToSimplified( + tableData: Airtable.Record[], + isKeyValueTable: boolean, +): unknown { if (isKeyValueTable) { // biome-ignore lint/suspicious/noExplicitAny: TODO const simplifiedData: Record = {}; @@ -129,7 +150,11 @@ export default async function airtableSource(options: z.input[]> = {}; - async function getDataCached(tableId: string, force: boolean, logger: Logger): Promise[]> { + async function getDataCached( + tableId: string, + force: boolean, + logger: Logger, + ): Promise[]> { logger.debug(`Fetching ${tableId} from Airtable`); if (force) { @@ -183,6 +208,8 @@ function tryImportAirtable() { try { return import("airtable"); } catch (e) { - throw new Error('Could not find peer dependency "airtable". Make sure you have installed it.', { cause: e }); + throw new Error('Could not find peer dependency "airtable". Make sure you have installed it.', { + cause: e, + }); } } diff --git a/packages/content/src/sources/contentful-source.ts b/packages/content/src/sources/contentful-source.ts index 21d13d36..da79f6ec 100644 --- a/packages/content/src/sources/contentful-source.ts +++ b/packages/content/src/sources/contentful-source.ts @@ -1,7 +1,7 @@ import type { Asset, Entry } from "contentful"; +import { z } from "zod"; import { fetchPaginated } from "../utils/fetch-paginated.js"; import { defineSource } from "./source.js"; -import { z } from "zod"; // If deliveryToken is provided, then previewToken is optional. const contentfulCredentialsSchema = z.union([ @@ -9,7 +9,10 @@ const contentfulCredentialsSchema = z.union([ /** Content delivery token (all published content). */ deliveryToken: z.string().describe("Content delivery token (all published content)."), /** Content preview token (only unpublished/draft content). */ - previewToken: z.string().optional().describe("Content preview token (only unpublished/draft content)."), + previewToken: z + .string() + .optional() + .describe("Content preview token (only unpublished/draft content)."), }), z.object({ /** Content preview token (only unpublished/draft content). */ @@ -20,25 +23,36 @@ const contentfulCredentialsSchema = z.union([ const contentfulSourceSchema = z .object({ /** Required field to identify this source. Will be used as download path. */ - id: z.string().describe("Required field to identify this source. Will be used as download path."), + id: z + .string() + .describe("Required field to identify this source. Will be used as download path."), /** Required field to identify this source. Will be used as download path. */ space: z.string().describe("Your Contentful space ID."), /** Used to pull localized images. */ locale: z.string().default("en-US").describe("Used to pull localized images."), /** The filename you want to use for where all content (entries and assets metadata) will be stored. Defaults to 'content.json' */ - filename: z.string().default("content.json").describe("The filename you want to use for where all content (entries and assets metadata) will be stored."), + filename: z + .string() + .default("content.json") + .describe( + "The filename you want to use for where all content (entries and assets metadata) will be stored.", + ), /** Optional. Defaults to 'https' */ protocol: z.string().default("https").describe("Optional. Defaults to 'https'"), /** Optional. Defaults to 'cdn.contentful.com', or 'preview.contentful.com' if `usePreviewApi` is true */ host: z .string() .default("cdn.contentful.com") - .describe("Optional. Defaults to 'cdn.contentful.com', or 'preview.contentful.com' if `usePreviewApi` is true"), + .describe( + "Optional. Defaults to 'cdn.contentful.com', or 'preview.contentful.com' if `usePreviewApi` is true", + ), /** Optional. Set to true if you want to use the preview API instead of the production version to view draft content. Defaults to false */ usePreviewApi: z .boolean() .default(false) - .describe("Optional. Set to true if you want to use the preview API instead of the production version to view draft content. Defaults to false"), + .describe( + "Optional. Set to true if you want to use the preview API instead of the production version to view draft content. Defaults to false", + ), /** * Optionally limit queries to these content types. This will also apply to linked assets. * Types that link to other types will include up to 10 levels of child content. E.g. filtering by Story, might also include Chapters and Images. @@ -95,10 +109,16 @@ export default async function contentfulSource(options: z.input { - const rawPage = await client.getEntries({ ...assembled.searchParams, skip: params.offset, limit: params.limit }); + const rawPage = await client.getEntries({ + ...assembled.searchParams, + skip: params.offset, + limit: params.limit, + }); if (rawPage.errors) { - throw new Error(`Error fetching page: ${rawPage.errors.map((e) => e.message).join(", ")}`); + throw new Error( + `Error fetching page: ${rawPage.errors.map((e) => e.message).join(", ")}`, + ); } const page = rawPage.toPlainObject(); @@ -168,6 +188,8 @@ async function tryImportContentful() { try { return await import("contentful"); } catch (error) { - throw new Error('Could not find module "contentful". Make sure you have installed it.', { cause: error }); + throw new Error('Could not find module "contentful". Make sure you have installed it.', { + cause: error, + }); } } diff --git a/packages/content/src/sources/json-source.ts b/packages/content/src/sources/json-source.ts index e409e5ba..4c3aece5 100644 --- a/packages/content/src/sources/json-source.ts +++ b/packages/content/src/sources/json-source.ts @@ -1,8 +1,8 @@ import chalk from "chalk"; +import ky from "ky"; import { okAsync } from "neverthrow"; -import { defineSource } from "./source.js"; import { z } from "zod"; -import ky from "ky"; +import { defineSource } from "./source.js"; const jsonSourceSchema = z.object({ /** required field to identify this source. Will be used as download path. */ diff --git a/packages/content/src/sources/sanity-source.ts b/packages/content/src/sources/sanity-source.ts index 991752cc..92bf7954 100644 --- a/packages/content/src/sources/sanity-source.ts +++ b/packages/content/src/sources/sanity-source.ts @@ -1,6 +1,6 @@ +import { z } from "zod"; import { fetchPaginated } from "../utils/fetch-paginated.js"; import { defineSource } from "./source.js"; -import { z } from "zod"; const sanitySourceSchema = z.object({ /** Required field to identify this source. Will be used as download path. */ @@ -18,7 +18,9 @@ const sanitySourceSchema = z.object({ /** An array of queries to fetch. Each query can be a string or an object with a query and an id. */ queries: z .array(z.union([z.string(), z.object({ query: z.string(), id: z.string() })])) - .describe("An array of queries to fetch. Each query can be a string or an object with a query and an id."), + .describe( + "An array of queries to fetch. Each query can be a string or an object with a query and an id.", + ), /** Max number of entries per page. Defaults to 100. */ limit: z.number().describe("Max number of entries per page").default(100), /** Max number of pages. Defaults to 1000. */ @@ -82,6 +84,9 @@ function tryImportSanityClient() { try { return import("@sanity/client"); } catch (e) { - throw new Error('Could not find peer dependency "@sanity/client". Make sure you have installed it.', { cause: e }); + throw new Error( + 'Could not find peer dependency "@sanity/client". Make sure you have installed it.', + { cause: e }, + ); } } diff --git a/packages/content/src/sources/source.ts b/packages/content/src/sources/source.ts index e0f3bf33..a226ff38 100644 --- a/packages/content/src/sources/source.ts +++ b/packages/content/src/sources/source.ts @@ -48,6 +48,8 @@ export type ContentSource = FetchResult /** * This function doesn't do anything, just returns the source parameter. It's just to make it easier to define/type sources. */ -export function defineSource = FetchResult>(src: ContentSource) { +export function defineSource = FetchResult>( + src: ContentSource, +) { return src; } diff --git a/packages/content/src/sources/strapi-source.ts b/packages/content/src/sources/strapi-source.ts index 07293c78..c83263b6 100644 --- a/packages/content/src/sources/strapi-source.ts +++ b/packages/content/src/sources/strapi-source.ts @@ -1,9 +1,9 @@ import type { Logger } from "@bluecadet/launchpad-utils"; +import ky from "ky"; import qs from "qs"; +import { z } from "zod"; import { fetchPaginated } from "../utils/fetch-paginated.js"; import { defineSource } from "./source.js"; -import { z } from "zod"; -import ky from "ky"; const strapiCredentialsSchema = z.union([ z.object({ @@ -21,27 +21,38 @@ const strapiCredentialsSchema = z.union([ const strapiSourceSchema = z .object({ /** Required field to identify this source. Will be used as download path. */ - id: z.string().describe("Required field to identify this source. Will be used as download path."), + id: z + .string() + .describe("Required field to identify this source. Will be used as download path."), /** Strapi version. Defaults to `3`. */ version: z.enum(["3", "4"]).describe("Strapi version").default("3"), /** The base url of your Strapi CMS (with or without trailing slash). */ - baseUrl: z.string().describe("The base url of your Strapi CMS (with or without trailing slash)."), + baseUrl: z + .string() + .describe("The base url of your Strapi CMS (with or without trailing slash)."), /** * Queries for each type of content you want to save. One per content type. Content will be stored as numbered, paginated JSONs. * You can include all query parameters supported by Strapi. * You can also pass an object with a `contentType` and `params` property, where `params` is an object of query parameters. */ - queries: z.array(z.union([z.string(), z.object({ contentType: z.string(), params: z.record(z.any()) })])).describe( - "Queries for each type of content you want to save. One per content type. Content will be stored as numbered, paginated JSONs. \ + queries: z + .array( + z.union([z.string(), z.object({ contentType: z.string(), params: z.record(z.any()) })]), + ) + .describe( + "Queries for each type of content you want to save. One per content type. Content will be stored as numbered, paginated JSONs. \ You can include all query parameters supported by Strapi. \ You can also pass an object with a `contentType` and `params` property, where `params` is an object of query parameters.", - ), + ), /** Max number of entries per page. Defaults to `100`. */ limit: z.number().describe("Max number of entries per page").default(100), /** Max number of pages. Defaults to `1000`. */ maxNumPages: z.number().describe("Max number of pages").default(1000), /** How many zeros to pad each json filename index with. Defaults to `2`. */ - pageNumZeroPad: z.number().describe("How many zeros to pad each json filename index with").default(2), + pageNumZeroPad: z + .number() + .describe("How many zeros to pad each json filename index with") + .default(2), }) .and(strapiCredentialsSchema); @@ -140,14 +151,19 @@ class StrapiV4 extends StrapiVersionUtils { } override hasPaginationParams(query: StrapiObjectQuery): boolean { - return query?.params?.pagination?.page !== undefined || query?.params?.pagination?.pageSize !== undefined; + return ( + query?.params?.pagination?.page !== undefined || + query?.params?.pagination?.pageSize !== undefined + ); } override transformResult(result: { data: unknown[] }): unknown[] { return result.data; } - override canFetchMore(result: { meta?: { pagination?: { page: number; pageCount: number } } }): boolean { + override canFetchMore(result: { + meta?: { pagination?: { page: number; pageCount: number } }; + }): boolean { if (result?.meta?.pagination) { const { page, pageCount } = result.meta.pagination; return page < pageCount; @@ -219,7 +235,9 @@ export default async function strapiSource(options: z.input { const versionUtils: StrapiVersionUtils = - assembledOptions.version === "4" ? new StrapiV4(assembledOptions, ctx.logger) : new StrapiV3(assembledOptions, ctx.logger); + assembledOptions.version === "4" + ? new StrapiV4(assembledOptions, ctx.logger) + : new StrapiV3(assembledOptions, ctx.logger); return assembledOptions.queries.map((query) => { let parsedQuery: StrapiObjectQuery; diff --git a/packages/content/src/utils/__tests__/content-transform-utils.test.ts b/packages/content/src/utils/__tests__/content-transform-utils.test.ts index 42948ad5..f1dab175 100644 --- a/packages/content/src/utils/__tests__/content-transform-utils.test.ts +++ b/packages/content/src/utils/__tests__/content-transform-utils.test.ts @@ -1,8 +1,12 @@ import { createMockLogger } from "@bluecadet/launchpad-testing/test-utils.ts"; +import { vol } from "memfs"; import { afterEach, describe, expect, it } from "vitest"; -import { applyTransformToFiles, getMatchingDocuments, isBlockContent } from "../content-transform-utils.js"; +import { + applyTransformToFiles, + getMatchingDocuments, + isBlockContent, +} from "../content-transform-utils.js"; import { DataStore } from "../data-store.js"; -import { vol } from "memfs"; describe("content-transform-utils", () => { afterEach(() => { @@ -44,7 +48,8 @@ describe("content-transform-utils", () => { await namespace._unsafeUnwrap().insert("doc1", Promise.resolve({ content: "test" })); const logger = createMockLogger(); - const transformFn = (content: unknown) => (typeof content === "string" ? content.toUpperCase() : content); + const transformFn = (content: unknown) => + typeof content === "string" ? content.toUpperCase() : content; await applyTransformToFiles({ dataStore, @@ -54,7 +59,9 @@ describe("content-transform-utils", () => { keys: ["test"], }); - expect(((await dataStore.getDocument("test", "doc1")._unsafeUnwrap()._read()) as any).content).toBe("TEST"); + expect( + ((await dataStore.getDocument("test", "doc1")._unsafeUnwrap()._read()) as any).content, + ).toBe("TEST"); expect(logger.debug).toHaveBeenCalled(); }); diff --git a/packages/content/src/utils/__tests__/data-store.test.ts b/packages/content/src/utils/__tests__/data-store.test.ts index 4eb999f4..e5b0204f 100644 --- a/packages/content/src/utils/__tests__/data-store.test.ts +++ b/packages/content/src/utils/__tests__/data-store.test.ts @@ -1,7 +1,7 @@ +import path from "node:path"; import { vol } from "memfs"; -import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { DataStore } from "../data-store.js"; -import path from "node:path"; describe("SingleDocument", () => { const TEST_DIR = "/test/store"; @@ -26,7 +26,10 @@ describe("SingleDocument", () => { const docResult = namespace.document("test-doc"); expect(docResult).toBeOk(); - const fileContent = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", "test-doc.json"), "utf-8"); + const fileContent = await vol.readFileSync( + path.join(TEST_DIR, "test-namespace", "test-doc.json"), + "utf-8", + ); expect(JSON.parse(fileContent.toString())).toEqual({ content: "test content" }); }); @@ -35,17 +38,26 @@ describe("SingleDocument", () => { expect(result).toBeOk(); const namespace = result._unsafeUnwrap(); - const doc = await namespace.insert("test-doc", Promise.resolve({ content: "original content" })); + const doc = await namespace.insert( + "test-doc", + Promise.resolve({ content: "original content" }), + ); await doc.update((data: any) => ({ ...data, content: "modified content", })); - const originalContent = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", "test-doc.original.json"), "utf-8"); + const originalContent = await vol.readFileSync( + path.join(TEST_DIR, "test-namespace", "test-doc.original.json"), + "utf-8", + ); expect(JSON.parse(originalContent.toString())).toEqual({ content: "original content" }); - const modifiedContent = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", "test-doc.json"), "utf-8"); + const modifiedContent = await vol.readFileSync( + path.join(TEST_DIR, "test-namespace", "test-doc.json"), + "utf-8", + ); expect(JSON.parse(modifiedContent.toString())).toEqual({ content: "modified content" }); }); @@ -61,9 +73,14 @@ describe("SingleDocument", () => { }), ); - await doc.apply("$.nested.content", (value: unknown) => (typeof value === "string" ? value.toUpperCase() : value)); + await doc.apply("$.nested.content", (value: unknown) => + typeof value === "string" ? value.toUpperCase() : value, + ); - const fileContent = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", "test-doc.json"), "utf-8"); + const fileContent = await vol.readFileSync( + path.join(TEST_DIR, "test-namespace", "test-doc.json"), + "utf-8", + ); expect(JSON.parse(fileContent.toString())).toEqual({ nested: { content: "TEST CONTENT" }, }); @@ -76,16 +93,32 @@ describe("SingleDocument", () => { const namespace = result._unsafeUnwrap(); await namespace.insert("test-doc.json", Promise.resolve({ content: "test content A" })); await namespace.insert("test-doc.extension", Promise.resolve({ content: "test content B" })); - await namespace.insert("test-doc.extension.extension", Promise.resolve({ content: "test content C" })); + await namespace.insert( + "test-doc.extension.extension", + Promise.resolve({ content: "test content C" }), + ); - const fileContent = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", "test-doc.json"), "utf-8"); + const fileContent = await vol.readFileSync( + path.join(TEST_DIR, "test-namespace", "test-doc.json"), + "utf-8", + ); expect(JSON.parse(fileContent.toString())).toMatchObject({ content: "test content A" }); - const extensionFileContent = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", "test-doc.extension"), "utf-8"); - expect(JSON.parse(extensionFileContent.toString())).toMatchObject({ content: "test content B" }); + const extensionFileContent = await vol.readFileSync( + path.join(TEST_DIR, "test-namespace", "test-doc.extension"), + "utf-8", + ); + expect(JSON.parse(extensionFileContent.toString())).toMatchObject({ + content: "test content B", + }); - const extensionExtensionFileContent = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", "test-doc.extension.extension"), "utf-8"); - expect(JSON.parse(extensionExtensionFileContent.toString())).toMatchObject({ content: "test content C" }); + const extensionExtensionFileContent = await vol.readFileSync( + path.join(TEST_DIR, "test-namespace", "test-doc.extension.extension"), + "utf-8", + ); + expect(JSON.parse(extensionExtensionFileContent.toString())).toMatchObject({ + content: "test content C", + }); }); }); @@ -125,7 +158,10 @@ describe("BatchDocument", () => { // Check that all files were created for (let i = 0; i < items.length; i++) { const filename = `test-doc-${i.toString().padStart(2, "0")}.json`; - const content = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", filename), "utf-8"); + const content = await vol.readFileSync( + path.join(TEST_DIR, "test-namespace", filename), + "utf-8", + ); expect(JSON.parse(content.toString())).toEqual(items[i]); } }); @@ -146,12 +182,17 @@ describe("BatchDocument", () => { })(), ); - await doc.apply("$.content", (value: unknown) => (typeof value === "string" ? value.toUpperCase() : value)); + await doc.apply("$.content", (value: unknown) => + typeof value === "string" ? value.toUpperCase() : value, + ); // Verify all documents were updated for (let i = 0; i < items.length; i++) { const filename = `test-doc-${i.toString().padStart(2, "0")}.json`; - const content = await vol.readFileSync(path.join(TEST_DIR, "test-namespace", filename), "utf-8"); + const content = await vol.readFileSync( + path.join(TEST_DIR, "test-namespace", filename), + "utf-8", + ); expect(JSON.parse(content.toString())).toEqual({ content: items[i]!.content.toUpperCase(), }); @@ -184,7 +225,9 @@ describe("DataStore", () => { store.createNamespace("test-namespace"); const result = await store.createNamespace("test-namespace"); expect(result).toBeErr(); - expect(result._unsafeUnwrapErr().message).toBe("Namespace test-namespace already exists in data store"); + expect(result._unsafeUnwrapErr().message).toBe( + "Namespace test-namespace already exists in data store", + ); }); it("should filter documents by namespace", async () => { diff --git a/packages/content/src/utils/__tests__/fetch-paginated.test.ts b/packages/content/src/utils/__tests__/fetch-paginated.test.ts index a1a5de29..cd2cfb9c 100644 --- a/packages/content/src/utils/__tests__/fetch-paginated.test.ts +++ b/packages/content/src/utils/__tests__/fetch-paginated.test.ts @@ -38,7 +38,8 @@ describe("fetchPaginated", () => { const generator = fetchPaginated({ limit: 10, logger: mockLogger, - fetchPageFn: ({ limit, offset }) => fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), + fetchPageFn: ({ limit, offset }) => + fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), }); const pages: unknown[] = []; @@ -72,7 +73,8 @@ describe("fetchPaginated", () => { limit: 10, logger: mockLogger, mergePages: true, - fetchPageFn: ({ limit, offset }) => fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), + fetchPageFn: ({ limit, offset }) => + fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), }); expect(Array.isArray(result)).toBe(true); @@ -97,7 +99,8 @@ describe("fetchPaginated", () => { const generator = fetchPaginated({ limit: 1, logger: mockLogger, - fetchPageFn: ({ limit, offset }) => fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), + fetchPageFn: ({ limit, offset }) => + fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), }); const pages: unknown[] = []; @@ -144,7 +147,8 @@ describe("fetchPaginated", () => { limit: 10, maxFetchCount: 2, logger: mockLogger, - fetchPageFn: ({ limit, offset }) => fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), + fetchPageFn: ({ limit, offset }) => + fetch(`http://example.com/api?limit=${limit}&offset=${offset}`).then((res) => res.json()), }); const pages: unknown[] = []; diff --git a/packages/content/src/utils/__tests__/file-utils.test.ts b/packages/content/src/utils/__tests__/file-utils.test.ts index 8152f1a1..72a3d800 100644 --- a/packages/content/src/utils/__tests__/file-utils.test.ts +++ b/packages/content/src/utils/__tests__/file-utils.test.ts @@ -71,7 +71,9 @@ describe("FileUtils", () => { it("should exclude specified files", async () => { const result = await FileUtils.removeFilesFromDir("/test-dir", ["*.json", "**/*.csv"]); expect(result).toBeOk(); - expect(vol.readdirSync("/test-dir", { recursive: true })).toEqual(expect.arrayContaining(["file2.json", "subdir/file3.csv", "subdir"])); + expect(vol.readdirSync("/test-dir", { recursive: true })).toEqual( + expect.arrayContaining(["file2.json", "subdir/file3.csv", "subdir"]), + ); }); }); @@ -219,7 +221,9 @@ describe("FileUtils", () => { vi.setSystemTime(date.getTime()); const sourceStats = vol.statSync("/source-file.txt"); vi.setSystemTime(date.getTime() + 1000); - const result = await FileUtils.copy("/source-file.txt", "/dest-file.txt", { preserveTimestamps: true }); + const result = await FileUtils.copy("/source-file.txt", "/dest-file.txt", { + preserveTimestamps: true, + }); expect(result).toBeOk(); const destStats = vol.statSync("/dest-file.txt"); expect(destStats.mtime).toEqual(sourceStats.mtime); diff --git a/packages/content/src/utils/__tests__/result-async-queue.test.ts b/packages/content/src/utils/__tests__/result-async-queue.test.ts index 3308e425..77a1541f 100644 --- a/packages/content/src/utils/__tests__/result-async-queue.test.ts +++ b/packages/content/src/utils/__tests__/result-async-queue.test.ts @@ -16,7 +16,9 @@ describe("ResultAsyncQueue", () => { it("should handle errors in tasks", async () => { const queue = new ResultAsyncQueue(); const task = () => - ResultAsync.fromPromise(Promise.reject(new Error("Task failed")), (error) => (error instanceof Error ? error : new Error(String(error)))); + ResultAsync.fromPromise(Promise.reject(new Error("Task failed")), (error) => + error instanceof Error ? error : new Error(String(error)), + ); const result = await queue.add(task); expect(result).toBeErr(); @@ -40,7 +42,10 @@ describe("ResultAsyncQueue", () => { it("should handle addAll with successful tasks", async () => { const queue = new ResultAsyncQueue(); const logger = createMockLogger(); - const tasks = [() => ResultAsync.fromPromise(Promise.resolve(1), () => new Error()), () => ResultAsync.fromPromise(Promise.resolve(2), () => new Error())]; + const tasks = [ + () => ResultAsync.fromPromise(Promise.resolve(1), () => new Error()), + () => ResultAsync.fromPromise(Promise.resolve(2), () => new Error()), + ]; const result = await queue.addAll(tasks, { logger }); expect(result).toBeOk(); @@ -52,7 +57,10 @@ describe("ResultAsyncQueue", () => { const logger = createMockLogger(); const tasks = [ () => ResultAsync.fromPromise(Promise.resolve(1), () => new Error()), - () => ResultAsync.fromPromise(Promise.reject(new Error("Task 2 failed")), (error) => (error instanceof Error ? error : new Error(String(error)))), + () => + ResultAsync.fromPromise(Promise.reject(new Error("Task 2 failed")), (error) => + error instanceof Error ? error : new Error(String(error)), + ), () => ResultAsync.fromPromise(Promise.resolve(3), () => new Error()), ]; diff --git a/packages/content/src/utils/content-transform-utils.ts b/packages/content/src/utils/content-transform-utils.ts index 2612c138..86f56773 100644 --- a/packages/content/src/utils/content-transform-utils.ts +++ b/packages/content/src/utils/content-transform-utils.ts @@ -6,7 +6,10 @@ import type { DataKeys, DataStore, Document } from "./data-store.js"; /** * @param ids A list containing a combination of namespace ids, and namespace/document id tuples. If not provided, all documents will be matched. */ -export function getMatchingDocuments(dataStore: DataStore, ids?: DataKeys): Result, Error> { +export function getMatchingDocuments( + dataStore: DataStore, + ids?: DataKeys, +): Result, Error> { if (!ids) { return ok(dataStore.allDocuments()); } @@ -25,7 +28,13 @@ type ApplyTransformToFilesParams = { /** * Shared logic for content transforms */ -export async function applyTransformToFiles({ dataStore, path, transformFn, logger, keys }: ApplyTransformToFilesParams) { +export async function applyTransformToFiles({ + dataStore, + path, + transformFn, + logger, + keys, +}: ApplyTransformToFilesParams) { const pathStr = chalk.yellow(path); const matchingDocuments = getMatchingDocuments(dataStore, keys); diff --git a/packages/content/src/utils/data-store.ts b/packages/content/src/utils/data-store.ts index 122f834b..5fabc7fd 100644 --- a/packages/content/src/utils/data-store.ts +++ b/packages/content/src/utils/data-store.ts @@ -1,7 +1,7 @@ -import { JSONPath } from "jsonpath-plus"; -import { Result, ResultAsync, err, errAsync, ok, okAsync } from "neverthrow"; import fs from "node:fs/promises"; import path from "node:path"; +import { JSONPath } from "jsonpath-plus"; +import { Result, ResultAsync, err, errAsync, ok, okAsync } from "neverthrow"; import { ensureDir } from "./file-utils.js"; export class DataStoreError extends Error { @@ -43,7 +43,10 @@ export abstract class Document { * @internal */ _safeRead(): ResultAsync { - return ResultAsync.fromPromise(this._read(), (e) => new DataStoreError(`Error reading document ${this._id}`, { cause: e })); + return ResultAsync.fromPromise( + this._read(), + (e) => new DataStoreError(`Error reading document ${this._id}`, { cause: e }), + ); } /** @@ -57,7 +60,10 @@ export abstract class Document { * @param cb A function that takes the current data and returns the new data. */ safeUpdate(cb: (data: T) => T | Promise): ResultAsync { - return ResultAsync.fromPromise(this.update(cb), (e) => new DataStoreError(`Error updating document ${this._id}`, { cause: e })); + return ResultAsync.fromPromise( + this.update(cb), + (e) => new DataStoreError(`Error updating document ${this._id}`, { cause: e }), + ); } /** @@ -68,17 +74,26 @@ export abstract class Document { /** * Apply a function to each element matching the given jsonpath. Same as {@link apply}, but returns a neverthrow {@link ResultAsync}. */ - safeApply(pathExpression: string, fn: (x: unknown) => unknown): ResultAsync { + safeApply( + pathExpression: string, + fn: (x: unknown) => unknown, + ): ResultAsync { return ResultAsync.fromPromise( this.apply(pathExpression, fn), - (e) => new DataStoreError(`Error applying content transform to document ${this._id}`, { cause: e }), + (e) => + new DataStoreError(`Error applying content transform to document ${this._id}`, { + cause: e, + }), ); } abstract query(pathExpression: string): Promise; safeQuery(pathExpression: string): ResultAsync { - return ResultAsync.fromPromise(this.query(pathExpression), (e) => new DataStoreError(`Error querying document ${this._id}`, { cause: e })); + return ResultAsync.fromPromise( + this.query(pathExpression), + (e) => new DataStoreError(`Error querying document ${this._id}`, { cause: e }), + ); } /** @@ -90,7 +105,10 @@ export abstract class Document { * Close the file handle. Same as {@link close}, but returns a neverthrow {@link ResultAsync}. */ safeClose(): ResultAsync { - return ResultAsync.fromPromise(this.close(), (e) => new DataStoreError(`Error closing document ${this._id}`, { cause: e })); + return ResultAsync.fromPromise( + this.close(), + (e) => new DataStoreError(`Error closing document ${this._id}`, { cause: e }), + ); } } @@ -177,14 +195,21 @@ class SingleDocument extends Document { return data; }); } catch (e) { - throw new DataStoreError(`Error applying content transform to document ${this._id}`, { cause: e }); + throw new DataStoreError(`Error applying content transform to document ${this._id}`, { + cause: e, + }); } } override async query(pathExpression: string): Promise { const handle = await this.#getHandle(); const data = await handle.readFile("utf-8"); - return JSONPath({ json: JSON.parse(data), path: pathExpression, resultType: "value", ignoreEvalErrors: true }); + return JSONPath({ + json: JSON.parse(data), + path: pathExpression, + resultType: "value", + ignoreEvalErrors: true, + }); } override async close() { @@ -221,7 +246,13 @@ export class BatchDocument extends Document { async initialize(directory: string, data: AsyncIterable) { for await (const item of data) { - this.#documents.push(await SingleDocument.create(directory, BatchDocument.getIndexedId(this._id, this.#documents.length), item)); + this.#documents.push( + await SingleDocument.create( + directory, + BatchDocument.getIndexedId(this._id, this.#documents.length), + item, + ), + ); } } @@ -283,8 +314,17 @@ class Namespace { return doc; } - safeInsert(id: string, data: Promise | AsyncIterable): ResultAsync, DataStoreError> { - return ResultAsync.fromPromise(this.insert(id, data), (e) => new DataStoreError(`Error inserting document ${id} into namespace ${this.#id}`, { cause: e })); + safeInsert( + id: string, + data: Promise | AsyncIterable, + ): ResultAsync, DataStoreError> { + return ResultAsync.fromPromise( + this.insert(id, data), + (e) => + new DataStoreError(`Error inserting document ${id} into namespace ${this.#id}`, { + cause: e, + }), + ); } /** @@ -366,9 +406,16 @@ export class DataStore { * Get lists of documents matching the passed DataKeys grouped by namespace. * @param ids A list containing a combination of namespace ids, and namespace/document id tuples. If not provided, all documents will be matched. */ - filter(ids?: DataKeys): Result }>, DataStoreError> { + filter( + ids?: DataKeys, + ): Result }>, DataStoreError> { if (!ids) { - return ok(Array.from(this.#namespaces.values()).map((ns) => ({ namespaceId: ns.id, documents: Array.from(ns.documents()) }))); + return ok( + Array.from(this.#namespaces.values()).map((ns) => ({ + namespaceId: ns.id, + documents: Array.from(ns.documents()), + })), + ); } const consolidatedIds = new Map>(); diff --git a/packages/content/src/utils/fetch-paginated.ts b/packages/content/src/utils/fetch-paginated.ts index 2ed9b82a..49e93cc8 100644 --- a/packages/content/src/utils/fetch-paginated.ts +++ b/packages/content/src/utils/fetch-paginated.ts @@ -46,7 +46,9 @@ export function fetchPaginated({ } } - return (mergePages ? getFlattened(generator()) : generator()) as Merge extends true ? Promise : AsyncGenerator; + return (mergePages ? getFlattened(generator()) : generator()) as Merge extends true + ? Promise + : AsyncGenerator; } async function getFlattened(generator: AsyncGenerator) { diff --git a/packages/content/src/utils/file-utils.ts b/packages/content/src/utils/file-utils.ts index 2045e166..47a33a9e 100644 --- a/packages/content/src/utils/file-utils.ts +++ b/packages/content/src/utils/file-utils.ts @@ -14,7 +14,11 @@ export function isDir(dirPath: string) { return fs.existsSync(dirPath) && fs.lstatSync(dirPath).isDirectory(); } -export function saveJson(json: unknown, filePath: string, appendJsonExtension = true): ResultAsync { +export function saveJson( + json: unknown, + filePath: string, + appendJsonExtension = true, +): ResultAsync { let filePathWithExtension = filePath; if (appendJsonExtension && !filePath.endsWith(".json")) { filePathWithExtension += ".json"; @@ -34,7 +38,10 @@ export function saveJson(json: unknown, filePath: string, appendJsonExtension = * @param dirPath Any absolute directory path * @param exclude Array of glob patterns to exclude (e.g. ['*.json', '** /*.csv', 'my-important-folder/**']). Glob patterns are relative to `dirPath`. */ -export function removeFilesFromDir(dirPath: string, exclude: string[] = []): ResultAsync { +export function removeFilesFromDir( + dirPath: string, + exclude: string[] = [], +): ResultAsync { return ResultAsync.fromPromise( glob(path.join(dirPath, "**/*"), { ignore: exclude.map((pattern) => path.join(dirPath, pattern)), @@ -67,9 +74,10 @@ export function removeFilesFromDir(dirPath: string, exclude: string[] = []): Res const removeDirPromises = dirs.map( (dir) => - ResultAsync.fromPromise(fs.promises.rmdir(dir), (error) => new FileUtilsError(`Failed to remove directory ${dir}`, { cause: error })).orElse(() => - okAsync(undefined), - ), // Ignore errors if directory is not empty + ResultAsync.fromPromise( + fs.promises.rmdir(dir), + (error) => new FileUtilsError(`Failed to remove directory ${dir}`, { cause: error }), + ).orElse(() => okAsync(undefined)), // Ignore errors if directory is not empty ); return ResultAsync.combine(removeDirPromises); }); @@ -101,9 +109,10 @@ export function removeDirIfEmpty(dirPath: string): ResultAsync { // @see https://stackoverflow.com/a/39218759/782899 - return ResultAsync.fromPromise(fs.promises.readdir(dirPath), (e) => new FileUtilsError(`Could not read dir ${dirPath}`, { cause: e })).andThen((files) => - okAsync(files.length === 0), - ); + return ResultAsync.fromPromise( + fs.promises.readdir(dirPath), + (e) => new FileUtilsError(`Could not read dir ${dirPath}`, { cause: e }), + ).andThen((files) => okAsync(files.length === 0)); } /** @@ -120,7 +129,10 @@ export function ensureDir(dirPath: string): ResultAsync { * Removes a file or directory. The directory can have contents. If the path does not exist, silently does nothing. */ export function remove(dir: string): ResultAsync { - return ResultAsync.fromPromise(fs.promises.rm(dir, { recursive: true, force: true }), (e) => new FileUtilsError(`Failed to remove ${dir}`, { cause: e })); + return ResultAsync.fromPromise( + fs.promises.rm(dir, { recursive: true, force: true }), + (e) => new FileUtilsError(`Failed to remove ${dir}`, { cause: e }), + ); } /** @@ -139,8 +151,15 @@ export function pathExists(dir: string): ResultAsync { /** * Copies a file or directory from `src` to `dest`. */ -export function copy(src: string, dest: string, options = { preserveTimestamps: true }): ResultAsync { - return ResultAsync.fromPromise(fs.promises.stat(src), (e) => new FileUtilsError(`Failed to get file stats for ${src}`, { cause: e })) +export function copy( + src: string, + dest: string, + options = { preserveTimestamps: true }, +): ResultAsync { + return ResultAsync.fromPromise( + fs.promises.stat(src), + (e) => new FileUtilsError(`Failed to get file stats for ${src}`, { cause: e }), + ) .andThrough((stats) => { if (stats.isDirectory()) { return copyDir(src, dest, options); @@ -161,10 +180,23 @@ export function copy(src: string, dest: string, options = { preserveTimestamps: /** * Copies a directory from `src` to `dest`. */ -export function copyDir(src: string, dest: string, options = { preserveTimestamps: true }): ResultAsync { +export function copyDir( + src: string, + dest: string, + options = { preserveTimestamps: true }, +): ResultAsync { return ensureDir(dest) - .andThen(() => ResultAsync.fromPromise(fs.promises.readdir(src), (e) => new FileUtilsError(`Failed to read dir ${src}`, { cause: e }))) - .andThen((entries) => ResultAsync.combine(entries.map((entry) => copy(path.join(src, entry), path.join(dest, entry), options)))) + .andThen(() => + ResultAsync.fromPromise( + fs.promises.readdir(src), + (e) => new FileUtilsError(`Failed to read dir ${src}`, { cause: e }), + ), + ) + .andThen((entries) => + ResultAsync.combine( + entries.map((entry) => copy(path.join(src, entry), path.join(dest, entry), options)), + ), + ) .map(() => undefined); } @@ -172,5 +204,8 @@ export function copyDir(src: string, dest: string, options = { preserveTimestamp * Copies a file from `src` to `dest`. */ export function copyFile(src: string, dest: string): ResultAsync { - return ResultAsync.fromPromise(fs.promises.copyFile(src, dest), (e) => new FileUtilsError(`Failed to copy file ${src} to ${dest}`, { cause: e })); + return ResultAsync.fromPromise( + fs.promises.copyFile(src, dest), + (e) => new FileUtilsError(`Failed to copy file ${src} to ${dest}`, { cause: e }), + ); } diff --git a/packages/content/src/utils/result-async-queue.ts b/packages/content/src/utils/result-async-queue.ts index 3f8f2232..7071f88b 100644 --- a/packages/content/src/utils/result-async-queue.ts +++ b/packages/content/src/utils/result-async-queue.ts @@ -7,7 +7,9 @@ export type ResultAsyncTaskOptions = { signal?: AbortSignal; }; -export type ResultAsyncTask = (options: ResultAsyncTaskOptions) => ResultAsync; +export type ResultAsyncTask = ( + options: ResultAsyncTaskOptions, +) => ResultAsync; /** * Wraps a PQueue instance to provide a ResultAsync interface. @@ -22,7 +24,9 @@ export default class ResultAsyncQueue { /** * Add a ResultAsync to the queue, returning a ResultAsync that resolves to the task's result, or void if the task is aborted. */ - add(task: ResultAsyncTask): ResultAsync { + add( + task: ResultAsyncTask, + ): ResultAsync { return ResultAsync.fromSafePromise(this.queue.add(task)).andThen((result) => { if (!result) { return ok(undefined); @@ -42,7 +46,9 @@ export default class ResultAsyncQueue { return (...args) => task(...args).mapErr((e) => { this.queue.clear(); - options.logger.error(`Cancelled ${chalk.red(`${this.queue.size} remaining sync tasks`)} due to ${chalk.red("error")}:`); + options.logger.error( + `Cancelled ${chalk.red(`${this.queue.size} remaining sync tasks`)} due to ${chalk.red("error")}:`, + ); return e; }); }); diff --git a/packages/content/src/utils/safe-ky.ts b/packages/content/src/utils/safe-ky.ts index 0855d628..0a6ad67c 100644 --- a/packages/content/src/utils/safe-ky.ts +++ b/packages/content/src/utils/safe-ky.ts @@ -20,7 +20,10 @@ export function safeKy(input: Input, options?: Options): SafeKyResultAsync { return SafeKyResultAsync.fromRequest(req); } -export type SafeKyResponseResult = Omit, "json" | "text" | "arrayBuffer" | "blob"> & { +export type SafeKyResponseResult = Omit< + KyResponse, + "json" | "text" | "arrayBuffer" | "blob" +> & { // biome-ignore lint/suspicious/noExplicitAny: TODO json: () => ResultAsync; text: () => ResultAsync; @@ -28,7 +31,10 @@ export type SafeKyResponseResult = Omit, "json" | "te blob: () => ResultAsync; }; -class SafeKyResultAsync extends ResultAsync, SafeKyFetchError | SafeKyParseError> { +class SafeKyResultAsync extends ResultAsync< + SafeKyResponseResult, + SafeKyFetchError | SafeKyParseError +> { static fromRequest(promise: ResponsePromise): SafeKyResultAsync { const newPromise = promise .then((res) => { @@ -41,16 +47,35 @@ class SafeKyResultAsync extends ResultAsync url: res.url, redirected: res.redirected, body: res.body, - json: () => ResultAsync.fromPromise(res.json(), (error) => new SafeKyParseError("Error parsing JSON", { cause: error })), - text: () => ResultAsync.fromPromise(res.text(), (error) => new SafeKyParseError("Error parsing text", { cause: error })), - arrayBuffer: () => ResultAsync.fromPromise(res.arrayBuffer(), (error) => new SafeKyParseError("Error parsing array buffer", { cause: error })), - blob: () => ResultAsync.fromPromise(res.blob(), (error) => new SafeKyParseError("Error parsing blob", { cause: error })), + json: () => + ResultAsync.fromPromise( + res.json(), + (error) => new SafeKyParseError("Error parsing JSON", { cause: error }), + ), + text: () => + ResultAsync.fromPromise( + res.text(), + (error) => new SafeKyParseError("Error parsing text", { cause: error }), + ), + arrayBuffer: () => + ResultAsync.fromPromise( + res.arrayBuffer(), + (error) => new SafeKyParseError("Error parsing array buffer", { cause: error }), + ), + blob: () => + ResultAsync.fromPromise( + res.blob(), + (error) => new SafeKyParseError("Error parsing blob", { cause: error }), + ), }; return new Ok(remapped) as Ok, SafeKyFetchError | SafeKyParseError>; }) .catch((error) => { - return new Err(new SafeKyFetchError("Error during request", { cause: error })) as Err, SafeKyFetchError | SafeKyParseError>; + return new Err(new SafeKyFetchError("Error during request", { cause: error })) as Err< + SafeKyResponseResult, + SafeKyFetchError | SafeKyParseError + >; }); return new SafeKyResultAsync(newPromise); diff --git a/packages/monitor/src/__tests__/launchpad-monitor.test.ts b/packages/monitor/src/__tests__/launchpad-monitor.test.ts index e586dc4d..7e7b294f 100644 --- a/packages/monitor/src/__tests__/launchpad-monitor.test.ts +++ b/packages/monitor/src/__tests__/launchpad-monitor.test.ts @@ -162,8 +162,13 @@ describe("LaunchpadMonitor", () => { await monitor.start("test-app"); - expect(plugin.hooks.beforeAppStart).toHaveBeenCalledWith(expect.any(Object), { appName: "test-app" }); - expect(plugin.hooks.afterAppStart).toHaveBeenCalledWith(expect.any(Object), { appName: "test-app", process: expect.any(Object) }); + expect(plugin.hooks.beforeAppStart).toHaveBeenCalledWith(expect.any(Object), { + appName: "test-app", + }); + expect(plugin.hooks.afterAppStart).toHaveBeenCalledWith(expect.any(Object), { + appName: "test-app", + process: expect.any(Object), + }); }); }); @@ -184,8 +189,12 @@ describe("LaunchpadMonitor", () => { await monitor.stop("test-app"); - expect(plugin.hooks.beforeAppStop).toHaveBeenCalledWith(expect.any(Object), { appName: "test-app" }); - expect(plugin.hooks.afterAppStop).toHaveBeenCalledWith(expect.any(Object), { appName: "test-app" }); + expect(plugin.hooks.beforeAppStop).toHaveBeenCalledWith(expect.any(Object), { + appName: "test-app", + }); + expect(plugin.hooks.afterAppStop).toHaveBeenCalledWith(expect.any(Object), { + appName: "test-app", + }); }); }); @@ -226,7 +235,9 @@ describe("LaunchpadMonitor", () => { const exitCode = 123; await monitor.shutdown(exitCode); - expect(plugin.hooks.beforeShutdown).toHaveBeenCalledWith(expect.any(Object), { code: exitCode }); + expect(plugin.hooks.beforeShutdown).toHaveBeenCalledWith(expect.any(Object), { + code: exitCode, + }); }); }); }); diff --git a/packages/monitor/src/core/__tests__/app-manager.test.ts b/packages/monitor/src/core/__tests__/app-manager.test.ts index 3e7c1682..d43ba1af 100644 --- a/packages/monitor/src/core/__tests__/app-manager.test.ts +++ b/packages/monitor/src/core/__tests__/app-manager.test.ts @@ -86,7 +86,9 @@ describe("AppManager", () => { const result = await appManager.startApp("non-existent-app"); expect(result).toBeErr(); - expect(result._unsafeUnwrapErr().message).toContain("No app found with the name 'non-existent-app'"); + expect(result._unsafeUnwrapErr().message).toContain( + "No app found with the name 'non-existent-app'", + ); }); }); @@ -136,7 +138,9 @@ describe("AppManager", () => { // @ts-expect-error Testing invalid input const result = await appManager.validateAppNames(123); expect(result).toBeErr(); - expect(result._unsafeUnwrapErr().message).toContain("appNames must be null, undefined, a string or an iterable array/set of strings"); + expect(result._unsafeUnwrapErr().message).toContain( + "appNames must be null, undefined, a string or an iterable array/set of strings", + ); }); }); @@ -152,7 +156,9 @@ describe("AppManager", () => { const { appManager } = setupTestAppManager(); const result = await appManager.getAppOptions("invalid-app"); expect(result).toBeErr(); - expect(result._unsafeUnwrapErr().message).toContain("No app found with the name 'invalid-app'"); + expect(result._unsafeUnwrapErr().message).toContain( + "No app found with the name 'invalid-app'", + ); }); }); }); diff --git a/packages/monitor/src/core/app-log-router.ts b/packages/monitor/src/core/app-log-router.ts index 43c157c3..61c234b1 100755 --- a/packages/monitor/src/core/app-log-router.ts +++ b/packages/monitor/src/core/app-log-router.ts @@ -23,7 +23,9 @@ class LogRelay { if (appOptions.pm2.output !== "/dev/null" || appOptions.pm2.error !== "/dev/null") { logger.warn("Launchpad is unable to rotate log files generated by pm2"); - logger.warn("Set log mode to 'bus' and unset pm2 output/error properties to hide this warning."); + logger.warn( + "Set log mode to 'bus' and unset pm2 output/error properties to hide this warning.", + ); } this._appOptions = appOptions; @@ -88,10 +90,14 @@ class FileLogRelay extends LogRelay { } if (!outFilepath) { - this._logger.warn(`App process for ${this._appOptions.pm2.name} is missing the 'output' property.`); + this._logger.warn( + `App process for ${this._appOptions.pm2.name} is missing the 'output' property.`, + ); } if (!errFilepath) { - this._logger.warn(`App process for ${this._appOptions.pm2.name} is missing the 'error' property.`); + this._logger.warn( + `App process for ${this._appOptions.pm2.name} is missing the 'error' property.`, + ); } if (outFilepath && this._logOptions.showStdout) { diff --git a/packages/monitor/src/core/app-manager.ts b/packages/monitor/src/core/app-manager.ts index 84b59928..894e7518 100644 --- a/packages/monitor/src/core/app-manager.ts +++ b/packages/monitor/src/core/app-manager.ts @@ -53,7 +53,9 @@ export class AppManager { if (Symbol.iterator in Object(appNames)) { return ok([...appNames]); } - return err(new Error("appNames must be null, undefined, a string or an iterable array/set of strings")); + return err( + new Error("appNames must be null, undefined, a string or an iterable array/set of strings"), + ); } getAllAppNames() { diff --git a/packages/monitor/src/core/bus-manager.ts b/packages/monitor/src/core/bus-manager.ts index ce579dc9..3248ccf2 100644 --- a/packages/monitor/src/core/bus-manager.ts +++ b/packages/monitor/src/core/bus-manager.ts @@ -95,7 +95,8 @@ export class BusManager { } throw new Error("Failed to connect to PM2 bus"); }), - (error) => (error instanceof Error ? error : new Error("Unknown error connecting to PM2 bus")), + (error) => + error instanceof Error ? error : new Error("Unknown error connecting to PM2 bus"), ); } @@ -118,7 +119,9 @@ export class BusManager { } return ok(undefined); } catch (error) { - return err(error instanceof Error ? error : new Error("Unknown error disconnecting from PM2 bus")); + return err( + error instanceof Error ? error : new Error("Unknown error disconnecting from PM2 bus"), + ); } } diff --git a/packages/monitor/src/core/monitor-plugin-driver.ts b/packages/monitor/src/core/monitor-plugin-driver.ts index 27cce2e9..7b9850dc 100644 --- a/packages/monitor/src/core/monitor-plugin-driver.ts +++ b/packages/monitor/src/core/monitor-plugin-driver.ts @@ -1,4 +1,9 @@ -import { type BaseHookContext, HookContextProvider, type Plugin, type PluginDriver } from "@bluecadet/launchpad-utils"; +import { + type BaseHookContext, + HookContextProvider, + type Plugin, + type PluginDriver, +} from "@bluecadet/launchpad-utils"; import type pm2 from "pm2"; import type LaunchpadMonitor from "../launchpad-monitor.js"; @@ -21,19 +26,37 @@ export type MonitorHooks = { /** called after disconnecting from PM2 */ afterDisconnect: (ctx: CombinedMonitorHookContext) => Promise | void; /** called before starting an app */ - beforeAppStart: (ctx: CombinedMonitorHookContext, arg: { appName: string }) => Promise | void; + beforeAppStart: ( + ctx: CombinedMonitorHookContext, + arg: { appName: string }, + ) => Promise | void; /** called after an app is started */ - afterAppStart: (ctx: CombinedMonitorHookContext, arg: { appName: string; process: pm2.ProcessDescription }) => Promise | void; + afterAppStart: ( + ctx: CombinedMonitorHookContext, + arg: { appName: string; process: pm2.ProcessDescription }, + ) => Promise | void; /** called before stopping an app */ - beforeAppStop: (ctx: CombinedMonitorHookContext, arg: { appName: string }) => Promise | void; + beforeAppStop: ( + ctx: CombinedMonitorHookContext, + arg: { appName: string }, + ) => Promise | void; /** called after an app is stopped */ afterAppStop: (ctx: CombinedMonitorHookContext, arg: { appName: string }) => Promise | void; /** called when an app encounters an error */ - onAppError: (ctx: CombinedMonitorHookContext, arg: { appName: string; error: Error }) => Promise | void; + onAppError: ( + ctx: CombinedMonitorHookContext, + arg: { appName: string; error: Error }, + ) => Promise | void; /** called when an app outputs a log message */ - onAppLog: (ctx: CombinedMonitorHookContext, arg: { appName: string; data: string }) => Promise | void; + onAppLog: ( + ctx: CombinedMonitorHookContext, + arg: { appName: string; data: string }, + ) => Promise | void; /** called when an app outputs an error log */ - onAppErrorLog: (ctx: CombinedMonitorHookContext, arg: { appName: string; data: string }) => Promise | void; + onAppErrorLog: ( + ctx: CombinedMonitorHookContext, + arg: { appName: string; data: string }, + ) => Promise | void; /** called before shutting down the monitor */ beforeShutdown: (ctx: CombinedMonitorHookContext, arg: { code?: number }) => Promise | void; }; diff --git a/packages/monitor/src/core/process-manager.ts b/packages/monitor/src/core/process-manager.ts index 625e398a..ae16a37f 100644 --- a/packages/monitor/src/core/process-manager.ts +++ b/packages/monitor/src/core/process-manager.ts @@ -66,7 +66,10 @@ export class ProcessManager { } } -function wrapPm2Function(errorMessage: string, pmFunction: (cb: (err: Error | null, result?: T) => void) => void): ResultAsync { +function wrapPm2Function( + errorMessage: string, + pmFunction: (cb: (err: Error | null, result?: T) => void) => void, +): ResultAsync { return ResultAsync.fromPromise( new Promise((resolve, reject) => { pmFunction((err, result) => { @@ -82,7 +85,10 @@ function wrapPm2Function(errorMessage: string, pmFunction: (cb: (err: Error | ); } -const safeDisconnect = Result.fromThrowable(pm2.disconnect, (error) => new Error("Failed to disconnect from PM2", { cause: error })); +const safeDisconnect = Result.fromThrowable( + pm2.disconnect, + (error) => new Error("Failed to disconnect from PM2", { cause: error }), +); function pingDaemon(): ResultAsync { return ResultAsync.fromPromise( diff --git a/packages/monitor/src/launchpad-monitor.ts b/packages/monitor/src/launchpad-monitor.ts index 440d362c..c3c77fc9 100644 --- a/packages/monitor/src/launchpad-monitor.ts +++ b/packages/monitor/src/launchpad-monitor.ts @@ -8,7 +8,11 @@ import { AppManager } from "./core/app-manager.js"; import { BusManager } from "./core/bus-manager.js"; import { MonitorPluginDriver } from "./core/monitor-plugin-driver.js"; import { ProcessManager } from "./core/process-manager.js"; -import { type MonitorConfig, type ResolvedMonitorConfig, resolveMonitorConfig } from "./monitor-config.js"; +import { + type MonitorConfig, + type ResolvedMonitorConfig, + resolveMonitorConfig, +} from "./monitor-config.js"; export class LaunchpadMonitor { _config: ResolvedMonitorConfig; @@ -132,7 +136,9 @@ export class LaunchpadMonitor { this._pluginDriver .runHookSequential("beforeAppStart", { appName: name }) .andThen(() => this._appManager.startApp(name)) - .andThrough((process) => this._pluginDriver.runHookSequential("afterAppStart", { appName: name, process })), + .andThrough((process) => + this._pluginDriver.runHookSequential("afterAppStart", { appName: name, process }), + ), ), ).andThen(() => this._appManager.applyWindowSettings()); }); @@ -156,7 +162,9 @@ export class LaunchpadMonitor { this._pluginDriver .runHookSequential("beforeAppStop", { appName: name }) .andThen(() => this._appManager.stopApp(name)) - .andThrough(() => this._pluginDriver.runHookSequential("afterAppStop", { appName: name })), + .andThrough(() => + this._pluginDriver.runHookSequential("afterAppStop", { appName: name }), + ), ), ).map(() => undefined); }); @@ -170,7 +178,9 @@ export class LaunchpadMonitor { return this._appManager .validateAppNames(appNames) .asyncAndThen((validatedNames) => { - return ResultAsync.combine(validatedNames.map((name) => this._appManager.isAppRunning(name, true))); + return ResultAsync.combine( + validatedNames.map((name) => this._appManager.isAppRunning(name, true)), + ); }) .map((results) => results.some((isRunning) => isRunning)); } @@ -206,7 +216,9 @@ export class LaunchpadMonitor { this._logger.info("...monitor shut down"); this._logger.close(); - process.exit(eventOrExitCode === undefined || Number.isNaN(+eventOrExitCode) ? 1 : +eventOrExitCode); + process.exit( + eventOrExitCode === undefined || Number.isNaN(+eventOrExitCode) ? 1 : +eventOrExitCode, + ); }) .mapErr((error) => { this._logger.error("Unhandled exit exception:", error); diff --git a/packages/monitor/src/utils/sort-windows.ts b/packages/monitor/src/utils/sort-windows.ts index b2b357aa..ec8c6e79 100644 --- a/packages/monitor/src/utils/sort-windows.ts +++ b/packages/monitor/src/utils/sort-windows.ts @@ -9,7 +9,11 @@ type SortApp = { pid?: number; }; -const sortWindows = async (apps: SortApp[], logger: Logger, minNodeVersion?: string): Promise => { +const sortWindows = async ( + apps: SortApp[], + logger: Logger, + minNodeVersion?: string, +): Promise => { const currNodeVersion = process.version; if (minNodeVersion && !semver.satisfies(currNodeVersion, minNodeVersion)) { return Promise.reject( @@ -31,12 +35,16 @@ const sortWindows = async (apps: SortApp[], logger: Logger, minNodeVersion?: str for (const app of apps) { if (!app.pid) { - logger.warn(`Can't sort windows for ${chalk.blue(app.options.pm2.name)} because it has no pid.`); + logger.warn( + `Can't sort windows for ${chalk.blue(app.options.pm2.name)} because it has no pid.`, + ); continue; } if (!visiblePids.has(app.pid)) { - logger.warn(`No window found for ${chalk.blue(app.options.pm2.name)} with pid ${chalk.blue(app.pid)}.`); + logger.warn( + `No window found for ${chalk.blue(app.options.pm2.name)} with pid ${chalk.blue(app.pid)}.`, + ); continue; } @@ -62,7 +70,9 @@ const sortWindows = async (apps: SortApp[], logger: Logger, minNodeVersion?: str win.minimize(); } if (fgPids.has(win.processId)) { - logger.info(`Foregrounding ${chalk.blue(win.getTitle())} (pid: ${chalk.blue(win.processId)})`); + logger.info( + `Foregrounding ${chalk.blue(win.getTitle())} (pid: ${chalk.blue(win.processId)})`, + ); win.bringToTop(); } } diff --git a/packages/utils/src/__tests__/log-manager.test.ts b/packages/utils/src/__tests__/log-manager.test.ts index e8191c78..42954b26 100644 --- a/packages/utils/src/__tests__/log-manager.test.ts +++ b/packages/utils/src/__tests__/log-manager.test.ts @@ -5,9 +5,11 @@ import winston from "winston"; import { LogManager } from "../log-manager.js"; // we don't want to actually log anything to the console during tests -const consoleLogSpy = vi.spyOn(winston.transports.Console.prototype, "log").mockImplementation((info, cb) => { - if (cb && typeof cb === "function") cb(); -}); +const consoleLogSpy = vi + .spyOn(winston.transports.Console.prototype, "log") + .mockImplementation((info, cb) => { + if (cb && typeof cb === "function") cb(); + }); describe("LogManager", () => { beforeEach(() => { diff --git a/packages/utils/src/__tests__/on-exit.test.ts b/packages/utils/src/__tests__/on-exit.test.ts index ec7053b9..6dc03785 100644 --- a/packages/utils/src/__tests__/on-exit.test.ts +++ b/packages/utils/src/__tests__/on-exit.test.ts @@ -11,7 +11,11 @@ describe("onExit", () => { onExit(callback, false); // Simulate exit events - await Promise.all([process.emit("beforeExit", 0), process.emit("SIGTERM", "SIGTERM"), process.emit("SIGINT", "SIGINT")]); + await Promise.all([ + process.emit("beforeExit", 0), + process.emit("SIGTERM", "SIGTERM"), + process.emit("SIGINT", "SIGINT"), + ]); // Should be called once per event expect(callback).toHaveBeenCalledTimes(3); @@ -22,7 +26,11 @@ describe("onExit", () => { onExit(callback, true); // Simulate multiple exit events - await Promise.all([process.emit("SIGTERM", "SIGTERM"), process.emit("SIGTERM", "SIGTERM"), process.emit("SIGTERM", "SIGTERM")]); + await Promise.all([ + process.emit("SIGTERM", "SIGTERM"), + process.emit("SIGTERM", "SIGTERM"), + process.emit("SIGTERM", "SIGTERM"), + ]); expect(callback).toHaveBeenCalledTimes(1); }); @@ -32,7 +40,11 @@ describe("onExit", () => { onExit(callback, false); // Simulate multiple exit events - await Promise.all([process.emit("SIGTERM", "SIGTERM"), process.emit("SIGTERM", "SIGTERM"), process.emit("SIGTERM", "SIGTERM")]); + await Promise.all([ + process.emit("SIGTERM", "SIGTERM"), + process.emit("SIGTERM", "SIGTERM"), + process.emit("SIGTERM", "SIGTERM"), + ]); expect(callback).toHaveBeenCalledTimes(3); }); diff --git a/packages/utils/src/on-exit.ts b/packages/utils/src/on-exit.ts index 59d04ed9..aa60ba41 100644 --- a/packages/utils/src/on-exit.ts +++ b/packages/utils/src/on-exit.ts @@ -35,7 +35,10 @@ for (const event of events) { continue; } - if (!callback.includeUncaught && (event === "uncaughtException" || event === "unhandledRejection")) { + if ( + !callback.includeUncaught && + (event === "uncaughtException" || event === "unhandledRejection") + ) { continue; } @@ -44,7 +47,11 @@ for (const event of events) { }); } -export const onExit = (callback: () => Promise | void = async () => {}, once = true, includeUncaught = false): void => { +export const onExit = ( + callback: () => Promise | void = async () => {}, + once = true, + includeUncaught = false, +): void => { callbacks.push({ callback, once, includeUncaught }); }; diff --git a/packages/utils/src/plugin-driver.ts b/packages/utils/src/plugin-driver.ts index a36a7c89..0c5f9cf9 100644 --- a/packages/utils/src/plugin-driver.ts +++ b/packages/utils/src/plugin-driver.ts @@ -31,7 +31,10 @@ export type HookSet = Record< (ctx: any, ...args: never[]) => void | PromiseLike >; -export type Plugin = Partial> = { +export type Plugin< + AllowedHooks extends HookSet, + ActualHooks extends Partial = Partial, +> = { name: string; hooks: { [K in keyof ActualHooks]: K extends keyof AllowedHooks ? ActualHooks[K] : never; @@ -107,7 +110,9 @@ export default class PluginDriver { return () => ResultAsync.fromPromise(wrappedHookCall(), (e) => { - this.#getBaseContext(plugin).logger.error(chalk.red(`Error in hook ${String(hookName)}`)); + this.#getBaseContext(plugin).logger.error( + chalk.red(`Error in hook ${String(hookName)}`), + ); this.#getBaseContext(plugin).logger.error(chalk.red(e)); return new PluginError(e, { pluginId: plugin.name }); }); @@ -142,7 +147,10 @@ export default class PluginDriver { return ResultAsync.combineWithAllErrors(hookCalls.map((call) => call())).map(() => undefined); } - async runHookSequential>(hookName: K, ...additionalArgs: Tail>): Promise { + async runHookSequential>( + hookName: K, + ...additionalArgs: Tail> + ): Promise { for (const plugin of this.#plugins) { const hook = plugin.hooks[hookName]; if (hook) { @@ -183,13 +191,21 @@ export class HookContextProvider { hookName: K, ...additionalArgs: Tail> ): ResultAsync { - return this.#innerDriver._runHookSequentialWithCtx(hookName, this._getPluginContext, additionalArgs); + return this.#innerDriver._runHookSequentialWithCtx( + hookName, + this._getPluginContext, + additionalArgs, + ); } runHookParallel>( hookName: K, ...additionalArgs: Tail> ): ResultAsync { - return this.#innerDriver._runHookParallelWithCtx(hookName, this._getPluginContext, additionalArgs); + return this.#innerDriver._runHookParallelWithCtx( + hookName, + this._getPluginContext, + additionalArgs, + ); } }