From c345fbc3b8cfd1b6505cc9d15cc3c76df05fcc2d Mon Sep 17 00:00:00 2001
From: Ronald Langeveld <hi@ronaldlangeveld.com>
Date: Sun, 19 Jan 2025 19:06:37 +0900
Subject: [PATCH 1/3] Update rendering to allow using Rettiwt as provider

---
 .../lib/nodes/embed/types/twitter.js          |  7 ++-
 .../kg-default-nodes/test/nodes/embed.test.js | 50 +++++++++++++++++++
 2 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/packages/kg-default-nodes/lib/nodes/embed/types/twitter.js b/packages/kg-default-nodes/lib/nodes/embed/types/twitter.js
index 9f8cedfed2..d79ce202d9 100644
--- a/packages/kg-default-nodes/lib/nodes/embed/types/twitter.js
+++ b/packages/kg-default-nodes/lib/nodes/embed/types/twitter.js
@@ -11,6 +11,7 @@ export default function render(node, document, options) {
 
     const tweetData = metadata && metadata.tweet_data;
     const isEmail = options.target === 'email';
+    const source = tweetData && tweetData.source;
 
     if (tweetData && isEmail) {
         const tweetId = tweetData.id;
@@ -39,7 +40,7 @@ export default function render(node, document, options) {
         }
         const hasPoll = tweetData.attachments && tweetData.attachments && tweetData.attachments.poll_ids;
 
-        if (mentions) {
+        if (mentions && source !== 'rettiwt') {
             let last = 0;
             let parts = [];
             let content = toArray(tweetContent);
@@ -91,6 +92,10 @@ export default function render(node, document, options) {
                 return partContent;
             }, '');
         }
+        if (tweetData && source === 'rettiwt') {
+            // Rettiwt doesn't have entity start/end data, so we have to parse it directly
+            tweetContent = tweetContent.replace(/\n/g, '<br>');
+        }
 
         html = `
         <table cellspacing="0" cellpadding="0" border="0" class="kg-twitter-card">
diff --git a/packages/kg-default-nodes/test/nodes/embed.test.js b/packages/kg-default-nodes/test/nodes/embed.test.js
index 29c7490cba..2a31fa18f4 100644
--- a/packages/kg-default-nodes/test/nodes/embed.test.js
+++ b/packages/kg-default-nodes/test/nodes/embed.test.js
@@ -328,6 +328,56 @@ describe('EmbedNode', function () {
             element.outerHTML.should.containEql(`<a href="https://twitter.com/twitter/status/${tweetData.id}"`);
         }));
 
+        it('renders twitter email embed with source as rettiwt', editorTest(function () {
+            const options = {
+                target: 'email'
+            };
+            const tweetData = {
+                id: '1630581157568839683',
+                created_at: '2023-02-28T14:50:17.000Z',
+                author_id: '767545134',
+                edit_history_tweet_ids: ['1630581157568839683'],
+                public_metrics: {
+                    retweet_count: 10,
+                    reply_count: 2,
+                    like_count: 38,
+                    quote_count: 6,
+                    impression_count: 10770
+                },
+                text: 'With the decline of traditional local news outlets, publishers like @MadisonMinutes, @RANGEMedia4all, and @sfsimplified are leading the charge in creating sustainable, community-driven journalism through websites and newsletters.\n' +
+                    '\n' +
+                    'Check out their impact 👇\n' +
+                    'https://t.co/RdNNyY18Iv',
+
+                source: 'rettiwt'
+            };
+
+            const embedNode = $createEmbedNode({
+                url: 'https://twitter.com/ghost/status/1395670367216619520',
+                embedType: 'twitter',
+                html: '<blockquote class="twitter-tweet"><p lang="en" dir="ltr">Ghost 4.0 is out now! 🎉</p>&mdash; Ghost (@ghost) <a href="https://twitter.com/ghost/status/1395670367216619520?ref_src=twsrc%5Etfw">May 21, 2021</a></blockquote> <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>',
+                metadata: {
+                    tweet_data: tweetData,
+                    height: 500,
+                    provider_name: 'Twitter',
+                    provider_url: 'https://twitter.com',
+                    thumbnail_height: 150,
+                    thumbnail_url: 'https://pbs.twimg.com/media/E1Y1q3bXMAU7m4n?format=jpg&name=small',
+                    thumbnail_width: 150,
+                    title: 'Ghost on Twitter: "Ghost 4.0 is out now! 🎉"',
+                    type: 'rich',
+                    version: '1.0',
+                    width: 550
+                },
+                caption: 'caption text'
+            });
+
+            const {element} = embedNode.exportDOM({...exportOptions, ...options});
+
+            element.outerHTML.should.containEql('<table cellspacing="0" cellpadding="0" border="0" class="kg-twitter-card">');
+            element.outerHTML.should.containEql(`<a href="https://twitter.com/twitter/status/${tweetData.id}"`);
+        }));
+
         it('renders video in email', editorTest(function () {
             const options = {
                 target: 'email'

From 31e12824fe08075525fd4304b2d840b7cb87f0ec Mon Sep 17 00:00:00 2001
From: Ronald Langeveld <hi@ronaldlangeveld.com>
Date: Mon, 20 Jan 2025 11:57:34 +0900
Subject: [PATCH 2/3] Added tests and improved logic

---
 .../lib/nodes/embed/types/twitter.js          |  19 +-
 .../kg-default-nodes/test/nodes/embed.test.js | 242 ++++++++++++++++++
 2 files changed, 260 insertions(+), 1 deletion(-)

diff --git a/packages/kg-default-nodes/lib/nodes/embed/types/twitter.js b/packages/kg-default-nodes/lib/nodes/embed/types/twitter.js
index d79ce202d9..2211081436 100644
--- a/packages/kg-default-nodes/lib/nodes/embed/types/twitter.js
+++ b/packages/kg-default-nodes/lib/nodes/embed/types/twitter.js
@@ -92,9 +92,26 @@ export default function render(node, document, options) {
                 return partContent;
             }, '');
         }
+
         if (tweetData && source === 'rettiwt') {
-            // Rettiwt doesn't have entity start/end data, so we have to parse it directly
             tweetContent = tweetContent.replace(/\n/g, '<br>');
+            const tcoLinks = tweetContent.match(/https:\/\/t\.co\/[a-zA-Z0-9]+/g) || [];
+            const displayUrls = urls.map(urlObj => urlObj.display_url);
+            tcoLinks.forEach((tcoLink, index) => {
+                if (index < displayUrls.length) {
+                    tweetContent = tweetContent.replace(tcoLink, `<span style="color: #1DA1F2; word-break: break-all;">${displayUrls[index]}</span>`);
+                }
+            });
+
+            // Replace mentions
+            mentions.forEach((mention) => {
+                tweetContent = tweetContent.replace(`@${mention.username}`, `<span style="color: #1DA1F2;">@${mention.username}</span>`);
+            });
+
+            // Replace hashtags
+            hashtags.forEach((hashtag) => {
+                tweetContent = tweetContent.replace(`#${hashtag.tag}`, `<span style="color: #1DA1F2;">#${hashtag.tag}</span>`);
+            });
         }
 
         html = `
diff --git a/packages/kg-default-nodes/test/nodes/embed.test.js b/packages/kg-default-nodes/test/nodes/embed.test.js
index 2a31fa18f4..6b51943c2e 100644
--- a/packages/kg-default-nodes/test/nodes/embed.test.js
+++ b/packages/kg-default-nodes/test/nodes/embed.test.js
@@ -344,6 +344,15 @@ describe('EmbedNode', function () {
                     quote_count: 6,
                     impression_count: 10770
                 },
+                entities: {
+                    urls: [{
+                        url: 'https://twitter.com',
+                        display_url: 'twitter.com'
+                    }, {
+                        url: 'https://twitter.com/ghost',
+                        display_url: 'twitter.com/ghost'
+                    }]
+                },
                 text: 'With the decline of traditional local news outlets, publishers like @MadisonMinutes, @RANGEMedia4all, and @sfsimplified are leading the charge in creating sustainable, community-driven journalism through websites and newsletters.\n' +
                     '\n' +
                     'Check out their impact 👇\n' +
@@ -378,6 +387,239 @@ describe('EmbedNode', function () {
             element.outerHTML.should.containEql(`<a href="https://twitter.com/twitter/status/${tweetData.id}"`);
         }));
 
+        it('Replaces twitter links with display urls from entities', editorTest(function () {
+            const options = {
+                target: 'email'
+            };
+            const tweetData = {
+                id: '1630581157568839683',
+                created_at: '2023-02-28T14:50:17.000Z',
+                author_id: '767545134',
+                edit_history_tweet_ids: ['1630581157568839683'],
+                public_metrics: {
+                    retweet_count: 10,
+                    reply_count: 2,
+                    like_count: 38,
+                    quote_count: 6,
+                    impression_count: 10770
+                },
+                text: 'With the decline of traditional local news outlets, publishers like @MadisonMinutes, @RANGEMedia4all, and @sfsimplified are leading the charge in creating sustainable, community-driven journalism through websites and newsletters.\n' +
+                    '\n' +
+                    'https://t.co/fakehome' +
+                    'Check out their impact 👇\n' +
+                    'https://t.co/fakeghost',
+                lang: 'en',
+                conversation_id: '1630581157568839683',
+                possibly_sensitive: false,
+                reply_settings: 'everyone',
+                entities: {
+                    urls: [
+                        {
+                            url: 'https://home.com',
+                            display_url: 'home.com'
+                        },
+                        {
+                            url: 'https://ghost.org',
+                            display_url: 'ghost.org'
+                        }
+                    ]
+                },
+                source: 'rettiwt'
+            };
+
+            const embedNode = $createEmbedNode({
+                url: 'https://twitter.com/ghost/status/1395670367216619520',
+                embedType: 'twitter',
+                html: '<blockquote class="twitter-tweet"><p lang="en" dir="ltr">Ghost 4.0 is out now! 🎉</p>&mdash; Ghost (@ghost) <a href="https://twitter.com/ghost/status/1395670367216619520?ref_src=twsrc%5Etfw">May 21, 2021</a></blockquote> <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>',
+                metadata: {
+                    tweet_data: tweetData,
+                    height: 500,
+                    provider_name: 'Twitter',
+                    provider_url: 'https://twitter.com',
+                    thumbnail_height: 150,
+                    thumbnail_url: 'https://pbs.twimg.com/media/E1Y1q3bXMAU7m4n?format=jpg&name=small',
+                    thumbnail_width: 150,
+                    title: 'Ghost on Twitter: "Ghost 4.0 is out now! 🎉"',
+                    type: 'rich',
+                    version: '1.0',
+                    width: 550
+                },
+                caption: 'caption text'
+            });
+
+            const {element} = embedNode.exportDOM({...exportOptions, ...options});
+
+            element.outerHTML.should.containEql('<table cellspacing="0" cellpadding="0" border="0" class="kg-twitter-card">');
+            // the first link should be replaced with the display_url
+            element.outerHTML.should.containEql('<span style="color: #1DA1F2; word-break: break-all;">home.com</span>');
+
+            element.outerHTML.should.containEql('<span style="color: #1DA1F2; word-break: break-all;">ghost.org</span>');
+            // the second link should be replaced with the display_url
+        }));
+
+        it('Wraps mentions in blue span', editorTest(function () {
+            const options = {
+                target: 'email'
+            };
+            const tweetData = {
+                id: '1630581157568839683',
+                created_at: '2023-02-28T14:50:17.000Z',
+                author_id: '767545134',
+                edit_history_tweet_ids: ['1630581157568839683'],
+                public_metrics: {
+                    retweet_count: 10,
+                    reply_count: 2,
+                    like_count: 38,
+                    quote_count: 6,
+                    impression_count: 10770
+                },
+                text: 'With the decline of traditional local news outlets, publishers like @MadisonMinutes, @RANGEMedia4all, and @sfsimplified are leading the charge in creating sustainable, community-driven journalism through websites and newsletters.\n' +
+                    '\n' +
+                    'https://t.co/fakehome' +
+                    'Check out their impact 👇\n' +
+                    'https://t.co/fakeghost',
+                lang: 'en',
+                conversation_id: '1630581157568839683',
+                possibly_sensitive: false,
+                reply_settings: 'everyone',
+                entities: {
+                    urls: [
+                        {
+                            url: 'https://twitter.com',
+                            display_url: 'twitter.com'
+                        },
+                        {
+                            url: 'https://twitter.com/ghost',
+                            display_url: 'twitter.com/ghost'
+                        }
+                    ],
+                    mentions: [
+                        {
+                            username: 'MadisonMinutes'
+                        },
+                        {
+                            username: 'RANGEMedia4all'
+                        },
+                        {
+                            username: 'sfsimplified'
+                        }
+                    ]
+                },
+                source: 'rettiwt'
+            };
+
+            const embedNode = $createEmbedNode({
+                url: 'https://twitter.com/ghost/status/1395670367216619520',
+                embedType: 'twitter',
+                html: '<blockquote class="twitter-tweet"><p lang="en" dir="ltr">Ghost 4.0 is out now! 🎉</p>&mdash; Ghost (@ghost) <a href="https://twitter.com/ghost/status/1395670367216619520?ref_src=twsrc%5Etfw">May 21, 2021</a></blockquote> <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>',
+                metadata: {
+                    tweet_data: tweetData,
+                    height: 500,
+                    provider_name: 'Twitter',
+                    provider_url: 'https://twitter.com',
+                    thumbnail_height: 150,
+                    thumbnail_url: 'https://pbs.twimg.com/media/E1Y1q3bXMAU7m4n?format=jpg&name=small',
+                    thumbnail_width: 150,
+                    title: 'Ghost on Twitter: "Ghost 4.0 is out now! 🎉"',
+                    type: 'rich',
+                    version: '1.0',
+                    width: 550
+                },
+                caption: 'caption text'
+            });
+
+            const {element} = embedNode.exportDOM({...exportOptions, ...options});
+
+            element.outerHTML.should.containEql('<span style="color: #1DA1F2;">@MadisonMinutes</span>');
+            element.outerHTML.should.containEql('<span style="color: #1DA1F2;">@RANGEMedia4all</span>');
+            element.outerHTML.should.containEql('<span style="color: #1DA1F2;">@sfsimplified</span>');
+        }));
+
+        it('Wraps hashtags in blue span', editorTest(function () {
+            const options = {
+                target: 'email'
+            };
+            const tweetData = {
+                id: '1630581157568839683',
+                created_at: '2023-02-28T14:50:17.000Z',
+                author_id: '767545134',
+                edit_history_tweet_ids: ['1630581157568839683'],
+                public_metrics: {
+                    retweet_count: 10,
+                    reply_count: 2,
+                    like_count: 38,
+                    quote_count: 6,
+                    impression_count: 10770
+                },
+                text: 'With the decline of traditional local news outlets, publishers like @MadisonMinutes, @RANGEMedia4all, and @sfsimplified are leading the charge in creating #sustainable, community-driven #journalism through websites and newsletters.\n' +
+                    '\n' +
+                    'https://t.co/fakehome' +
+                    'Check out their impact 👇\n' +
+                    'https://t.co/fakeghost',
+                lang: 'en',
+                conversation_id: '1630581157568839683',
+                possibly_sensitive: false,
+                reply_settings: 'everyone',
+                entities: {
+                    urls: [
+                        {
+                            url: 'https://twitter.com',
+                            display_url: 'twitter.com'
+                        },
+                        {
+                            url: 'https://twitter.com/ghost',
+                            display_url: 'twitter.com/ghost'
+                        }
+                    ],
+                    mentions: [
+                        {
+                            username: 'MadisonMinutes'
+                        },
+                        {
+                            username: 'RANGEMedia4all'
+                        },
+                        {
+                            username: 'sfsimplified'
+                        }
+                    ],
+                    hashtags: [
+                        {
+                            tag: 'sustainable'
+                        },
+                        {
+                            tag: 'journalism'
+                        }
+                    ]
+                },
+                source: 'rettiwt'
+            };
+
+            const embedNode = $createEmbedNode({
+                url: 'https://twitter.com/ghost/status/1395670367216619520',
+                embedType: 'twitter',
+                html: '<blockquote class="twitter-tweet"><p lang="en" dir="ltr">Ghost 4.0 is out now! 🎉</p>&mdash; Ghost (@ghost) <a href="https://twitter.com/ghost/status/1395670367216619520?ref_src=twsrc%5Etfw">May 21, 2021</a></blockquote> <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>',
+                metadata: {
+                    tweet_data: tweetData,
+                    height: 500,
+                    provider_name: 'Twitter',
+                    provider_url: 'https://twitter.com',
+                    thumbnail_height: 150,
+                    thumbnail_url: 'https://pbs.twimg.com/media/E1Y1q3bXMAU7m4n?format=jpg&name=small',
+                    thumbnail_width: 150,
+                    title: 'Ghost on Twitter: "Ghost 4.0 is out now! 🎉"',
+                    type: 'rich',
+                    version: '1.0',
+                    width: 550
+                },
+                caption: 'caption text'
+            });
+
+            const {element} = embedNode.exportDOM({...exportOptions, ...options});
+
+            element.outerHTML.should.containEql('<span style="color: #1DA1F2;">#sustainable</span>');
+            element.outerHTML.should.containEql('<span style="color: #1DA1F2;">#journalism</span>');
+        }));
+
         it('renders video in email', editorTest(function () {
             const options = {
                 target: 'email'

From b6882afff4a567d27ea3abf031599f109ada6b3a Mon Sep 17 00:00:00 2001
From: Ronald Langeveld <hi@ronaldlangeveld.com>
Date: Mon, 20 Jan 2025 12:10:06 +0900
Subject: [PATCH 3/3] Cleaned up function

---
 .../lib/nodes/embed/types/twitter.js          | 32 +++++++++++++------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/packages/kg-default-nodes/lib/nodes/embed/types/twitter.js b/packages/kg-default-nodes/lib/nodes/embed/types/twitter.js
index 2211081436..63c07ecdcb 100644
--- a/packages/kg-default-nodes/lib/nodes/embed/types/twitter.js
+++ b/packages/kg-default-nodes/lib/nodes/embed/types/twitter.js
@@ -40,6 +40,7 @@ export default function render(node, document, options) {
         }
         const hasPoll = tweetData.attachments && tweetData.attachments && tweetData.attachments.poll_ids;
 
+        // for compatibility with previous api provider
         if (mentions && source !== 'rettiwt') {
             let last = 0;
             let parts = [];
@@ -93,25 +94,38 @@ export default function render(node, document, options) {
             }, '');
         }
 
+        // for compatibility with new api provider
         if (tweetData && source === 'rettiwt') {
-            tweetContent = tweetContent.replace(/\n/g, '<br>');
-            const tcoLinks = tweetContent.match(/https:\/\/t\.co\/[a-zA-Z0-9]+/g) || [];
+            const wrapWithStyle = (text, style) => `<span style="${style}">${text}</span>`;
+        
+            let formattedContent = tweetContent.replace(/\n/g, '<br>');
+        
+            const tcoLinks = formattedContent.match(/https:\/\/t\.co\/[a-zA-Z0-9]+/g) || [];
             const displayUrls = urls.map(urlObj => urlObj.display_url);
             tcoLinks.forEach((tcoLink, index) => {
                 if (index < displayUrls.length) {
-                    tweetContent = tweetContent.replace(tcoLink, `<span style="color: #1DA1F2; word-break: break-all;">${displayUrls[index]}</span>`);
+                    formattedContent = formattedContent.replace(
+                        tcoLink,
+                        wrapWithStyle(displayUrls[index], 'color: #1DA1F2; word-break: break-all;')
+                    );
                 }
             });
 
-            // Replace mentions
-            mentions.forEach((mention) => {
-                tweetContent = tweetContent.replace(`@${mention.username}`, `<span style="color: #1DA1F2;">@${mention.username}</span>`);
+            mentions.forEach(({username}) => {
+                formattedContent = formattedContent.replace(
+                    `@${username}`,
+                    wrapWithStyle(`@${username}`, 'color: #1DA1F2;')
+                );
             });
 
-            // Replace hashtags
-            hashtags.forEach((hashtag) => {
-                tweetContent = tweetContent.replace(`#${hashtag.tag}`, `<span style="color: #1DA1F2;">#${hashtag.tag}</span>`);
+            hashtags.forEach(({tag}) => {
+                formattedContent = formattedContent.replace(
+                    `#${tag}`,
+                    wrapWithStyle(`#${tag}`, 'color: #1DA1F2;')
+                );
             });
+
+            tweetContent = formattedContent;
         }
 
         html = `