fix: fetch all references with includes

packages/core/src/fetchers/fetchAllEntities.spec.ts

@@ -0,0 +1,85 @@
+import { entries } from '../test/__fixtures__/entities';
+import { fetchAllEntities } from './fetchAllEntities';
+
+import { describe, afterEach, it, expect, vi, Mock } from 'vitest';
+import { ContentfulClientApi } from 'contentful';
+
+const mockClient = {
+  getAssets: vi.fn(),
+  withoutLinkResolution: {
+    getEntries: vi.fn(),
+  },
+} as unknown as ContentfulClientApi<undefined>;
+
+describe('fetchAllEntities', () => {
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should fetch all entities', async () => {
+    (mockClient.withoutLinkResolution.getEntries as Mock).mockResolvedValue({
+      items: [...entries],
+      total: 100,
+    });
+
+    const params = {
+      ids: entries.map((entry) => entry.sys.id),
+      entityType: 'Entry' as 'Entry' | 'Asset',
+      client: mockClient,
+      locale: 'en-US',
+      limit: 20,
+    };
+
+    await fetchAllEntities(params);
+
+    expect(mockClient.withoutLinkResolution.getEntries).toHaveBeenCalledTimes(5);
+  });
+
+  it('should reduce limit and refetch all entities if response error is gotten', async () => {
+    (mockClient.withoutLinkResolution.getEntries as Mock)
+      .mockRejectedValueOnce(new Error('Response size too big'))
+      .mockResolvedValue({
+        items: [...entries],
+        total: 100,
+      });
+
+    const params = {
+      ids: entries.map((entry) => entry.sys.id),
+      entityType: 'Entry' as 'Entry' | 'Asset',
+      client: mockClient,
+      locale: 'en-US',
+      limit: 20,
+    };
+
+    await fetchAllEntities(params);
+
+    expect(mockClient.withoutLinkResolution.getEntries).toHaveBeenNthCalledWith(11, {
+      'sys.id[in]': params.ids,
+      locale: 'en-US',
+      skip: 90,
+      limit: 10,
+    });
+  });
+
+  it('should never reduce limit to less than MIN_FETCH_LIMIT', async () => {
+    (mockClient.withoutLinkResolution.getEntries as Mock).mockRejectedValue(
+      new Error('Response size too big'),
+    );
+
+    const params = {
+      ids: entries.map((entry) => entry.sys.id),
+      entityType: 'Entry' as 'Entry' | 'Asset',
+      client: mockClient,
+      locale: 'en-US',
+      limit: 20,
+    };
+    await fetchAllEntities(params);
+
+    expect(mockClient.withoutLinkResolution.getEntries).toHaveBeenNthCalledWith(5, {
+      'sys.id[in]': params.ids,
+      locale: 'en-US',
+      skip: 0,
+      limit: 1,
+    });
+  });
+});

packages/core/src/fetchers/fetchAllEntities.ts

@@ -0,0 +1,96 @@
+import { ContentfulClientApi, Entry } from 'contentful';
+import { MinimalEntryCollection } from './gatherAutoFetchedReferentsFromIncludes';
+
+const MIN_FETCH_LIMIT = 1;
+
+const fetchEntities = async ({
+  entityType,
+  client,
+  query,
+}: {
+  entityType: 'Entry' | 'Asset';
+  client: ContentfulClientApi<undefined>;
+  query: { 'sys.id[in]': string[]; locale: string; limit: number; skip: number };
+}) => {
+  if (entityType === 'Asset') {
+    return client.getAssets({ ...query });
+  }
+
+  return client.withoutLinkResolution.getEntries({ ...query });
+};
+
+export const fetchAllEntities = async ({
+  client,
+  entityType,
+  ids,
+  locale,
+  skip = 0,
+  limit = 100,
+  responseItems = [],
+}: {
+  client: ContentfulClientApi<undefined>;
+  entityType: 'Entry' | 'Asset';
+  ids: string[];
+  locale: string;
+  skip?: number;
+  limit?: number;
+  responseItems?: Entry[];
+}) => {
+  try {
+    if (!ids.length || !client) {
+      return {
+        items: [],
+        ...(entityType === 'Entry' && { includes: [] }),
+      };
+    }
+
+    const query = { 'sys.id[in]': ids, locale, limit, skip };
+
+    const response = await fetchEntities({ entityType, client, query });
+
+    if (!response) {
+      return {
+        items: responseItems,
+        ...(entityType === 'Entry' && { includes: [] }),
+      };
+    }
+
+    responseItems.push(...(response.items as Entry[]));
+
+    if (skip + limit < response.total) {
+      await fetchAllEntities({
+        client,
+        entityType,
+        ids,
+        locale,
+        skip: skip + limit,
+        limit,
+        responseItems,
+      });
+    }
+
+    return {
+      items: responseItems,
+      ...(entityType === 'Entry' && { includes: (response as MinimalEntryCollection).includes }),
+    };
+  } catch (error) {
+    if (
+      error instanceof Error &&
+      error.message.includes('size too big') &&
+      limit > MIN_FETCH_LIMIT
+    ) {
+      const newLimit = Math.max(MIN_FETCH_LIMIT, Math.floor(limit / 2));
+      return fetchAllEntities({
+        client,
+        entityType,
+        ids,
+        locale,
+        skip,
+        limit: newLimit,
+        responseItems,
+      });
+    }
+
+    return error;
+  }
+};

packages/core/src/fetchers/fetchReferencedEntities.spec.ts

@@ -87,13 +87,17 @@ describe('fetchReferencedEntities', () => {
     });
 
     expect(mockClient.getAssets).toHaveBeenCalledWith({
+      limit: 100,
+      skip: 0,
       locale: 'en-US',
       'sys.id[in]': assets.map((asset) => asset.sys.id),
     });
 
     expect(mockClient.withoutLinkResolution.getEntries).toHaveBeenCalledWith({
       locale: 'en-US',
       'sys.id[in]': entries.map((entry) => entry.sys.id),
+      limit: 100,
+      skip: 0,
     });
 
     expect(res).toEqual({

packages/core/src/fetchers/fetchReferencedEntities.ts

@@ -6,6 +6,7 @@ import {
   MinimalEntryCollection,
   gatherAutoFetchedReferentsFromIncludes,
 } from './gatherAutoFetchedReferentsFromIncludes';
+import { fetchAllEntities } from './fetchAllEntities';
 
 type FetchReferencedEntitiesArgs = {
   client: ContentfulClientApi<undefined>;
@@ -55,18 +56,16 @@ export const fetchReferencedEntities = async ({
   }
 
   const [entriesResponse, assetsResponse] = (await Promise.all([
-    entryIds.length > 0
-      ? client.withoutLinkResolution.getEntries({ 'sys.id[in]': entryIds, locale })
-      : { items: [], includes: [] },
-    assetIds.length > 0 ? client.getAssets({ 'sys.id[in]': assetIds, locale }) : { items: [] },
+    fetchAllEntities({ client, entityType: 'Entry', ids: entryIds, locale }),
+    fetchAllEntities({ client, entityType: 'Asset', ids: assetIds, locale }),
   ])) as unknown as [MinimalEntryCollection, AssetCollection];
 
   const { autoFetchedReferentAssets, autoFetchedReferentEntries } =
     gatherAutoFetchedReferentsFromIncludes(deepReferences, entriesResponse);
 
   // Using client getEntries resolves all linked entry references, so we do not need to resolve entries in usedComponents
   const allResolvedEntries = [
-    ...((entriesResponse.items ?? []) as Entry[]),
+    ...((entriesResponse?.items ?? []) as Entry[]),
     ...((experienceEntry.fields.usedComponents as ExperienceEntry[]) || []),
     ...autoFetchedReferentEntries,
   ];

packages/experience-builder-sdk/src/hooks/useFetchById.spec.ts

@@ -69,11 +69,15 @@ describe('useFetchById', () => {
       });
 
       expect(clientMock.withoutLinkResolution.getEntries).toHaveBeenNthCalledWith(1, {
+        limit: 100,
+        skip: 0,
         'sys.id[in]': entries.map((entry) => entry.sys.id),
         locale: localeCode,
       });
 
       expect(clientMock.getAssets).toHaveBeenCalledWith({
+        limit: 100,
+        skip: 0,
         'sys.id[in]': assets.map((asset) => asset.sys.id),
         locale: localeCode,
       });

packages/experience-builder-sdk/src/hooks/useFetchBySlug.spec.ts

@@ -77,11 +77,15 @@ describe('useFetchBySlug', () => {
       });
 
       expect(clientMock.withoutLinkResolution.getEntries).toHaveBeenNthCalledWith(1, {
+        limit: 100,
+        skip: 0,
         'sys.id[in]': entries.map((entry) => entry.sys.id),
         locale: localeCode,
       });
 
       expect(clientMock.getAssets).toHaveBeenCalledWith({
+        limit: 100,
+        skip: 0,
         'sys.id[in]': assets.map((asset) => asset.sys.id),
         locale: localeCode,
       });