PRO-6561: export documents which are related via rich text widget "in…

…ternal pages" links and inline images (#98)

* wip

* working

* refactored

* wip

* tests passing

* test cleanup, lint

* cleanup

* a lot of changes around page types to satisfy tests & ensure functionality

* allow both valid aliases for "any page type"

* do not use it.only

* skip inappropriate page type

* fix tests

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## UNRELEASED
+
+### Adds
+
+* Pages linked a page via the "Internal Page" option in the rich text editor are now candidates to be exported as related documents.
+* Images embedded inline in rich text widgets via the `insert: [ 'image' ]` option are now candidates to be exported as related documents.
+
 ## 2.5.0 (2024-11-08)
 
 ### Adds

lib/methods/export.js

@@ -35,7 +35,6 @@ module.exports = self => {
 
       const docs = (await self.getDocs(req, ids, hasRelatedTypes, manager, reporting))
         .map((doc) => self.apos.util.clonePermanent(doc));
-
       if (!hasRelatedTypes) {
         return self.exportFile(
           req,
@@ -210,14 +209,23 @@ module.exports = self => {
       mode = doc.aposMode || req.mode
     }) {
       recursion++;
+      if ((doc.type === '@apostrophecms/rich-text') && (type === 'relationship')) {
+        await self.getRelatedDocsFromRichTextWidget(req, {
+          doc,
+          relatedTypes,
+          storedData,
+          recursion,
+          mode
+        });
+      }
       for (const field of schema) {
         const fieldValue = doc[field.name];
         const shouldRecurse = recursion <= MAX_RECURSION;
 
         if (!field.withType && !fieldValue) {
           continue;
         }
-        if (field.withType && relatedTypes && !relatedTypes.includes(field.withType)) {
+        if (field.withType && relatedTypes && !relatedTypesIncludes(field.withType)) {
           continue;
         }
         if (field.withType && !self.canExport(req, field.withType)) {
@@ -282,6 +290,80 @@ module.exports = self => {
           });
         }
       }
+
+      function relatedTypesIncludes(name) {
+        if ([ '@apostrophecms/any-page-type', '@apostrophecms/page' ].includes(name)) {
+          return relatedTypes.some(type => {
+            const module = self.apos.modules[type];
+            return self.apos.instanceOf(module, '@apostrophecms/page-type');
+          });
+        }
+        return relatedTypes.includes(name);
+      }
+    },
+
+    async getRelatedDocsFromRichTextWidget(req, {
+      doc,
+      relatedTypes,
+      storedData,
+      recursion,
+      mode
+    }) {
+      let linkedDocs = await self.apos.doc.db.find({
+        aposDocId: {
+          $in: doc.permalinkIds
+        }
+      }).project({
+        type: 1,
+        aposDocId: 1,
+        slug: 1
+      }).toArray();
+      // We're likely going to fetch them all with an @apostrophecms/any-page-type query, so we
+      // need to do our real related types check early or we'll allow all page types
+      // whenever we allow even one
+      linkedDocs = linkedDocs.filter(doc => relatedTypes.includes(doc.type));
+      const linkedIdsByType = new Map();
+      for (const linkedDoc of linkedDocs) {
+        // Normalization is a little different here because these
+        // are individual pages or pieces
+        const docType = linkedDoc.slug.startsWith('/') ? '@apostrophecms/any-page-type' : linkedDoc.type;
+        const isTypeStored = linkedIdsByType.has(docType);
+        const linkedIds = isTypeStored ? linkedIdsByType.get(docType) : new Set();
+        linkedIds.add(linkedDoc.aposDocId);
+        if (!isTypeStored) {
+          linkedIdsByType.set(docType, linkedIds);
+        }
+      }
+      if (doc.imageIds?.length > 0) {
+        linkedIdsByType.set('@apostrophecms/image', new Set(doc.imageIds));
+      }
+      const virtualDoc = {
+        type: '@apostrophecms/rich-text_related'
+      };
+      const virtualSchema = [];
+      for (const [ linkedType, linkedIds ] of linkedIdsByType.entries()) {
+        const baseName = self.apos.util.slugify(linkedType);
+        const fieldName = `_${baseName}`;
+        const idsStorage = `${baseName}Ids`;
+        virtualSchema.push({
+          name: fieldName,
+          type: 'relationship',
+          withType: linkedType,
+          idsStorage
+        });
+        const ids = [ ...linkedIds.values() ];
+        virtualDoc[idsStorage] = ids;
+        virtualDoc[fieldName] = ids.map(id => ({ aposDocId: id }));
+      }
+      await self.getRelatedDocsFromSchema(req, {
+        doc: virtualDoc,
+        schema: virtualSchema,
+        relatedTypes,
+        storedData,
+        type: 'relationship',
+        recursion,
+        mode
+      });
     },
 
     async handleRelatedField(req, {
@@ -476,36 +558,79 @@ module.exports = self => {
         timeoutId
       };
     },
-
+    // The entry point. Modifies `related`, also returns `related` because code elsewhere
+    // expects that
     getRelatedTypes(req, schema = [], related = []) {
-      return findSchemaRelatedTypes(schema, related);
-
-      function findSchemaRelatedTypes(schema, related, recursions = 0) {
-        recursions++;
-        if (recursions >= MAX_RECURSION) {
-          return related;
-        }
-        for (const field of schema) {
-          if (
-            field.type === 'relationship' &&
-            self.canExport(req, field.withType) &&
-            !related.includes(field.withType)
-          ) {
-            related.push(field.withType);
-            const relatedManager = self.apos.doc.getManager(field.withType);
-            findSchemaRelatedTypes(relatedManager.schema, related, recursions);
-          } else if ([ 'array', 'object' ].includes(field.type)) {
-            findSchemaRelatedTypes(field.schema, related, recursions);
-          } else if (field.type === 'area') {
-            const widgets = self.apos.area.getWidgets(field.options);
-            for (const widget of Object.keys(widgets)) {
-              const { schema = [] } = self.apos.modules[`${widget}-widget`];
-              findSchemaRelatedTypes(schema, related, recursions);
+      self.findSchemaRelatedTypes(req, schema, related, 0);
+      return related;
+    },
+    // Called recursively for you. Modifies `related`, has no useful return value
+    findSchemaRelatedTypes(req, schema, related, recursions) {
+      recursions++;
+      if (recursions >= MAX_RECURSION) {
+        return;
+      }
+      for (const field of schema) {
+        if (
+          field.type === 'relationship' &&
+          self.canExport(req, field.withType) &&
+          !related.includes(field.withType)
+        ) {
+          self.pushRelatedType(req, related, field.withType, recursions);
+        } else if ([ 'array', 'object' ].includes(field.type)) {
+          self.findSchemaRelatedTypes(req, field.schema, related, recursions);
+        } else if (field.type === 'area') {
+          const widgets = self.apos.area.getWidgets(field.options);
+          for (const [ widget, options ] of Object.entries(widgets)) {
+            const schema = self.apos.area.getWidgetManager(widget).schema || [];
+            if (widget === '@apostrophecms/rich-text') {
+              self.getRelatedTypesFromRichTextWidget(req, options, related, recursions);
             }
+            self.findSchemaRelatedTypes(req, schema, related, recursions);
+          }
+        }
+      }
+    },
+    pushRelatedType(req, related, type, recursions) {
+      if ((type === '@apostrophecms/page') || (type === '@apostrophecms/any-page-type')) {
+        const pageTypes = Object.entries(self.apos.doc.managers).filter(
+          ([ name, module ]) => self.apos.instanceOf(module, '@apostrophecms/page-type'))
+          .map(([ name, module ]) => name);
+        for (const type of pageTypes) {
+          if (type === '@apostrophecms/archive-page') {
+            // It is never appropriate to export the root page of the trash
+            continue;
+          }
+          self.pushRelatedType(req, related, type, recursions);
+        }
+        return;
+      }
+      if (!related.includes(type)) {
+        related.push(type);
+        const relatedManager = self.apos.doc.getManager(type);
+        self.findSchemaRelatedTypes(req, relatedManager.schema, related, recursions);
+      }
+    },
+    // Does not currently utilize req, but it could be relevant in overrides and is
+    // always the first argument by convention, so it is included in the signature
+    getRelatedTypesFromRichTextWidget(req, options, related, recursions) {
+      const manager = self.apos.modules['@apostrophecms/rich-text-widget'];
+      const rteOptions = {
+        ...manager.options.defaultOptions,
+        ...options
+      };
+      if (
+        (rteOptions.toolbar?.includes('image') || rteOptions.insert?.includes('image')) &&
+        !related.includes('@apostrophecms/image')
+      ) {
+        self.pushRelatedType(req, related, '@apostrophecms/image', recursions);
+      }
+      if (rteOptions.toolbar?.includes('link')) {
+        for (const name of manager.linkFields.linkTo.choices.map(choice => choice.value)) {
+          if (self.apos.doc.getManager(name) && !related.includes(name)) {
+            self.pushRelatedType(req, related, name, recursions);
           }
         }
-
-        return related;
       }
     }
   };

lib/methods/import.js

@@ -705,7 +705,6 @@ module.exports = self => {
 
         const type = doc.type;
         const docToInsert = doc;
-
         await manager.convert(_req, doc, docToInsert, {
           presentFieldsOnly: true,
           fetchRelationships: false