@@ -393,7 +393,7 @@ func TestMediumImage(t *testing.T) {
`,
}
Rewriter("https://example.org/article", testEntry, "fix_medium_images")
- testEntry.Content = strings.TrimSpace(testEntry.Content)
+ testEntry.WebContent = strings.TrimSpace(testEntry.WebContent)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -402,15 +402,15 @@ func TestMediumImage(t *testing.T) {
func TestRewriteNoScriptImageWithoutNoScriptTag(t *testing.T) {
controlEntry := &model.Entry{
- Title: `A title`,
- Content: `
MDN Logo`,
+ Title: `A title`,
+ WebContent: `
MDN Logo`,
}
testEntry := &model.Entry{
- Title: `A title`,
- Content: `
MDN Logo`,
+ Title: `A title`,
+ WebContent: `
MDN Logo`,
}
Rewriter("https://example.org/article", testEntry, "use_noscript_figure_images")
- testEntry.Content = strings.TrimSpace(testEntry.Content)
+ testEntry.WebContent = strings.TrimSpace(testEntry.WebContent)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -419,15 +419,15 @@ func TestRewriteNoScriptImageWithoutNoScriptTag(t *testing.T) {
func TestRewriteNoScriptImageWithNoScriptTag(t *testing.T) {
controlEntry := &model.Entry{
- Title: `A title`,
- Content: `
MDN Logo`,
+ Title: `A title`,
+ WebContent: `
MDN Logo`,
}
testEntry := &model.Entry{
- Title: `A title`,
- Content: `
MDN Logo`,
+ Title: `A title`,
+ WebContent: `
MDN Logo`,
}
Rewriter("https://example.org/article", testEntry, "use_noscript_figure_images")
- testEntry.Content = strings.TrimSpace(testEntry.Content)
+ testEntry.WebContent = strings.TrimSpace(testEntry.WebContent)
if !reflect.DeepEqual(testEntry, controlEntry) {
t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry)
@@ -436,12 +436,12 @@ func TestRewriteNoScriptImageWithNoScriptTag(t *testing.T) {
func TestRewriteReplaceCustom(t *testing.T) {
controlEntry := &model.Entry{
- Title: `A title`,
- Content: `
`,
+ Title: `A title`,
+ WebContent: `
`,
}
testEntry := &model.Entry{
- Title: `A title`,
- Content: `
`,
+ Title: `A title`,
+ WebContent: `
`,
}
Rewriter("https://example.org/article", testEntry, `replace("article/(.*).svg"|"article/$1.png")`)
@@ -468,12 +468,12 @@ func TestRewriteReplaceTitleCustom(t *testing.T) {
func TestRewriteRemoveCustom(t *testing.T) {
controlEntry := &model.Entry{
- Title: `A title`,
- Content: `
Lorem Ipsum Super important info
`,
+ Title: `A title`,
+ WebContent: `
Lorem Ipsum Super important info
`,
}
testEntry := &model.Entry{
- Title: `A title`,
- Content: `
Lorem Ipsum I dont want to see thisSuper important info
`,
+ Title: `A title`,
+ WebContent: `
Lorem Ipsum I dont want to see thisSuper important info
`,
}
Rewriter("https://example.org/article", testEntry, `remove(".spam, .ads:not(.keep)")`)
@@ -484,12 +484,12 @@ func TestRewriteRemoveCustom(t *testing.T) {
func TestRewriteAddCastopodEpisode(t *testing.T) {
controlEntry := &model.Entry{
- Title: `A title`,
- Content: `
Episode Description`,
+ Title: `A title`,
+ WebContent: `
Episode Description`,
}
testEntry := &model.Entry{
- Title: `A title`,
- Content: `Episode Description`,
+ Title: `A title`,
+ WebContent: `Episode Description`,
}
Rewriter("https://podcast.demo/@demo/episodes/test", testEntry, `add_castopod_episode`)
@@ -500,12 +500,12 @@ func TestRewriteAddCastopodEpisode(t *testing.T) {
func TestRewriteBase64Decode(t *testing.T) {
controlEntry := &model.Entry{
- Title: `A title`,
- Content: `This is some base64 encoded content`,
+ Title: `A title`,
+ WebContent: `This is some base64 encoded content`,
}
testEntry := &model.Entry{
- Title: `A title`,
- Content: `VGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=`,
+ Title: `A title`,
+ WebContent: `VGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=`,
}
Rewriter("https://example.org/article", testEntry, `base64_decode`)
@@ -516,12 +516,12 @@ func TestRewriteBase64Decode(t *testing.T) {
func TestRewriteBase64DecodeInHTML(t *testing.T) {
controlEntry := &model.Entry{
- Title: `A title`,
- Content: `
Lorem Ipsum not valid base64This is some base64 encoded content
`,
+ Title: `A title`,
+ WebContent: `
Lorem Ipsum not valid base64This is some base64 encoded content
`,
}
testEntry := &model.Entry{
- Title: `A title`,
- Content: `
Lorem Ipsum not valid base64VGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=
`,
+ Title: `A title`,
+ WebContent: `
Lorem Ipsum not valid base64VGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=
`,
}
Rewriter("https://example.org/article", testEntry, `base64_decode`)
@@ -532,12 +532,12 @@ func TestRewriteBase64DecodeInHTML(t *testing.T) {
func TestRewriteBase64DecodeArgs(t *testing.T) {
controlEntry := &model.Entry{
- Title: `A title`,
- Content: `
Lorem IpsumThis is some base64 encoded content
`,
+ Title: `A title`,
+ WebContent: `
Lorem IpsumThis is some base64 encoded content
`,
}
testEntry := &model.Entry{
- Title: `A title`,
- Content: `
Lorem IpsumVGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=
`,
+ Title: `A title`,
+ WebContent: `
Lorem IpsumVGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=
`,
}
Rewriter("https://example.org/article", testEntry, `base64_decode(".base64")`)
@@ -548,12 +548,12 @@ func TestRewriteBase64DecodeArgs(t *testing.T) {
func TestRewriteRemoveTables(t *testing.T) {
controlEntry := &model.Entry{
- Title: `A title`,
- Content: `
Test
Hello World!
Test
`,
+ Title: `A title`,
+ WebContent: `
Test
Hello World!
Test
`,
}
testEntry := &model.Entry{
- Title: `A title`,
- Content: `
`,
+ Title: `A title`,
+ WebContent: `
`,
}
Rewriter("https://example.org/article", testEntry, `remove_tables`)
@@ -564,12 +564,12 @@ func TestRewriteRemoveTables(t *testing.T) {
func TestRemoveClickbait(t *testing.T) {
controlEntry := &model.Entry{
- Title: `This Is Amazing`,
- Content: `Some description`,
+ Title: `This Is Amazing`,
+ WebContent: `Some description`,
}
testEntry := &model.Entry{
- Title: `THIS IS AMAZING`,
- Content: `Some description`,
+ Title: `THIS IS AMAZING`,
+ WebContent: `Some description`,
}
Rewriter("https://example.org/article", testEntry, `remove_clickbait`)
diff --git a/internal/storage/entry.go b/internal/storage/entry.go
index fd8a011032a..d290ad8fdab 100644
--- a/internal/storage/entry.go
+++ b/internal/storage/entry.go
@@ -75,11 +75,11 @@ func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
UPDATE
entries
SET
- content=$1, reading_time=$2
+ content=$1, web_content=$2, reading_time=$3
WHERE
- id=$3 AND user_id=$4
+ id=$4 AND user_id=$5
`
- _, err = tx.Exec(query, entry.Content, entry.ReadingTime, entry.ID, entry.UserID)
+ _, err = tx.Exec(query, entry.Content, entry.WebContent, entry.ReadingTime, entry.ID, entry.UserID)
if err != nil {
tx.Rollback()
return fmt.Errorf(`store: unable to update content of entry #%d: %v`, entry.ID, err)
@@ -89,7 +89,7 @@ func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
UPDATE
entries
SET
- document_vectors = setweight(to_tsvector(left(coalesce(title, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce(content, ''), 500000)), 'B')
+ document_vectors = setweight(to_tsvector(left(coalesce(title, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce(content, ''), 500000)), 'B') || setweight(to_tsvector(left(coalesce(web_content, ''), 500000)), 'C')
WHERE
id=$1 AND user_id=$2
`
@@ -98,7 +98,6 @@ func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
tx.Rollback()
return fmt.Errorf(`store: unable to update content of entry #%d: %v`, entry.ID, err)
}
-
return tx.Commit()
}
@@ -113,6 +112,7 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error {
comments_url,
published_at,
content,
+ web_content,
author,
user_id,
feed_id,
@@ -133,9 +133,10 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error {
$8,
$9,
$10,
+ $11,
now(),
- setweight(to_tsvector(left(coalesce($1, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce($6, ''), 500000)), 'B'),
- $11
+ setweight(to_tsvector(left(coalesce($1, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce($6, ''), 500000)), 'B') || setweight(to_tsvector(left(coalesce($7, ''), 500000)), 'C'),
+ $12
)
RETURNING
id, status
@@ -148,13 +149,13 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error {
entry.CommentsURL,
entry.Date,
entry.Content,
+ entry.WebContent,
entry.Author,
entry.UserID,
entry.FeedID,
entry.ReadingTime,
pq.Array(removeDuplicates(entry.Tags)),
).Scan(&entry.ID, &entry.Status)
-
if err != nil {
return fmt.Errorf(`store: unable to create entry %q (feed #%d): %v`, entry.URL, entry.FeedID, err)
}
@@ -183,12 +184,13 @@ func (s *Storage) updateEntry(tx *sql.Tx, entry *model.Entry) error {
url=$2,
comments_url=$3,
content=$4,
- author=$5,
- reading_time=$6,
- document_vectors = setweight(to_tsvector(left(coalesce($1, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce($4, ''), 500000)), 'B'),
- tags=$10
+ web_content=$5,
+ author=$6,
+ reading_time=$7,
+ document_vectors = setweight(to_tsvector(left(coalesce($1, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce($4, ''), 500000)), 'B') || setweight(to_tsvector(left(coalesce($5, ''), 500000)), 'C'),
+ tags=$11
WHERE
- user_id=$7 AND feed_id=$8 AND hash=$9
+ user_id=$8 AND feed_id=$9 AND hash=$10
RETURNING
id
`
@@ -198,6 +200,7 @@ func (s *Storage) updateEntry(tx *sql.Tx, entry *model.Entry) error {
entry.URL,
entry.CommentsURL,
entry.Content,
+ entry.WebContent,
entry.Author,
entry.ReadingTime,
entry.UserID,
@@ -205,7 +208,6 @@ func (s *Storage) updateEntry(tx *sql.Tx, entry *model.Entry) error {
entry.Hash,
pq.Array(removeDuplicates(entry.Tags)),
).Scan(&entry.ID)
-
if err != nil {
return fmt.Errorf(`store: unable to update entry %q: %v`, entry.URL, err)
}
diff --git a/internal/storage/entry_query_builder.go b/internal/storage/entry_query_builder.go
index 2a8defd5dcd..173fe46addb 100644
--- a/internal/storage/entry_query_builder.go
+++ b/internal/storage/entry_query_builder.go
@@ -250,6 +250,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
e.author,
e.share_code,
e.content,
+ e.web_content,
e.status,
e.starred,
e.reading_time,
@@ -315,6 +316,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
&entry.Author,
&entry.ShareCode,
&entry.Content,
+ &entry.WebContent,
&entry.Status,
&entry.Starred,
&entry.ReadingTime,
@@ -337,7 +339,6 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
&iconID,
&tz,
)
-
if err != nil {
return nil, fmt.Errorf("unable to fetch entry row: %v", err)
}
diff --git a/internal/template/templates/views/entry.html b/internal/template/templates/views/entry.html
index 8b55e17ddd6..db0f8560431 100644
--- a/internal/template/templates/views/entry.html
+++ b/internal/template/templates/views/entry.html
@@ -78,11 +78,17 @@
{{ icon "scraper" }}{{ t "entry.scraper.label" }}
+ data-fetch-original-content-url="{{ route "fetchOriginal" "entryID" .entry.ID }}"
+ >{{ icon "scraper" }}{{ if .entry.WebContent }}{{ t "entry.scraper.label.rss" }}{{ else }}{{ t "entry.scraper.label" }}{{ end }}
{{ if .entry.CommentsURL }}
@@ -186,9 +192,17 @@
{{ end }}
{{end}}
{{ if .user }}
- {{ noescape (proxyFilter .entry.Content) }}
+ {{ if .entry.WebContent }}
+ {{ noescape (proxyFilter .entry.WebContent) }}
+ {{ else }}
+ {{ noescape (proxyFilter .entry.Content) }}
+ {{ end }}
{{ else }}
- {{ noescape .entry.Content }}
+ {{ if .entry.WebContent }}
+ {{ noescape .entry.WebContent }}
+ {{ else }}
+ {{ noescape .entry.Content }}
+ {{ end }}
{{ end }}
{{ if .entry.Enclosures }}
@@ -203,11 +217,11 @@
data-last-position="{{ .MediaProgression }}"
data-save-url="{{ route "saveEnclosureProgression" "enclosureID" .ID }}"
>
- {{ if (and $.user (mustBeProxyfied "audio")) }}
-
- {{ else }}
-
- {{ end }}
+ {{ if (and $.user (mustBeProxyfied "audio")) }}
+
+ {{ else }}
+
+ {{ end }}
{{ else if hasPrefix .MimeType "video/" }}
@@ -216,11 +230,11 @@
`,
+ Title: `A title`,
+ WebContent: `
`,
+ Title: `A title`,
+ WebContent: `
`,
+ Title: `A title`,
+ WebContent: `