diff --git a/internal/database/migrations.go b/internal/database/migrations.go index be3bef84c89..2b6b878763e 100644 --- a/internal/database/migrations.go +++ b/internal/database/migrations.go @@ -834,4 +834,11 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE feeds ADD COLUMN icon_url text default ''; + ` + _, err = tx.Exec(sql) + return err + }, } diff --git a/internal/model/feed.go b/internal/model/feed.go index c9562ac9325..46740c012e7 100644 --- a/internal/model/feed.go +++ b/internal/model/feed.go @@ -51,17 +51,17 @@ type Feed struct { FetchViaProxy bool `json:"fetch_via_proxy"` HideGlobally bool `json:"hide_globally"` AppriseServiceURLs string `json:"apprise_service_urls"` + IconURL string `json:"icon_url"` // Non persisted attributes Category *Category `json:"category,omitempty"` Icon *FeedIcon `json:"icon"` Entries Entries `json:"entries,omitempty"` - TTL int `json:"-"` - IconURL string `json:"-"` - UnreadCount int `json:"-"` - ReadCount int `json:"-"` - NumberOfVisibleEntries int `json:"-"` + TTL int `json:"-"` + UnreadCount int `json:"-"` + ReadCount int `json:"-"` + NumberOfVisibleEntries int `json:"-"` } type FeedCounters struct { diff --git a/internal/model/icon.go b/internal/model/icon.go index 7a38b75c90f..51257106c1d 100644 --- a/internal/model/icon.go +++ b/internal/model/icon.go @@ -14,6 +14,7 @@ type Icon struct { Hash string `json:"hash"` MimeType string `json:"mime_type"` Content []byte `json:"-"` + URL string `json:"-"` } // DataURL returns the data URL of the icon. diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go index 36cd2561533..50d1a977069 100644 --- a/internal/reader/handler/handler.go +++ b/internal/reader/handler/handler.go @@ -93,10 +93,8 @@ func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, f checkFeedIcon( store, - requestBuilder, - subscription.ID, - subscription.SiteURL, - subscription.IconURL, + subscription, + false, ) return subscription, nil @@ -185,10 +183,8 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model checkFeedIcon( store, - requestBuilder, - subscription.ID, - subscription.SiteURL, - subscription.IconURL, + subscription, + false, ) return subscription, nil } @@ -321,13 +317,16 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool originalFeed.EtagHeader = responseHandler.ETag() originalFeed.LastModifiedHeader = responseHandler.LastModified() - checkFeedIcon( - store, - requestBuilder, - originalFeed.ID, - originalFeed.SiteURL, - updatedFeed.IconURL, - ) + if originalFeed.IconURL != updatedFeed.IconURL { + originalFeed.IconURL = updatedFeed.IconURL + forceRefresh = true + } else if originalFeed.IconURL != "" { + slog.Debug("Feed icon URL not modified", + slog.Int64("user_id", userID), + slog.Int64("feed_id", feedID), + slog.String("feed_icon_url", originalFeed.IconURL), + ) + } } else { slog.Debug("Feed not modified", slog.Int64("user_id", userID), @@ -335,6 +334,12 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool ) } + checkFeedIcon( + store, + originalFeed, + forceRefresh, + ) + originalFeed.ResetErrorCounter() if storeErr := store.UpdateFeed(originalFeed); storeErr != nil { @@ -347,28 +352,45 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool return nil } -func checkFeedIcon(store *storage.Storage, requestBuilder *fetcher.RequestBuilder, feedID int64, websiteURL, feedIconURL string) { - if !store.HasIcon(feedID) { - iconFinder := icon.NewIconFinder(requestBuilder, websiteURL, feedIconURL) +func checkFeedIcon(store *storage.Storage, feed *model.Feed, forceRefresh bool) { + if !store.HasIcon(feed.ID) || forceRefresh { + requestBuilder := fetcher.NewRequestBuilder() + requestBuilder.WithUsernameAndPassword(feed.Username, feed.Password) + requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent()) + requestBuilder.WithCookie(feed.Cookie) + requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) + requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + requestBuilder.UseProxy(feed.FetchViaProxy) + requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates) + + iconFinder := icon.NewIconFinder(requestBuilder, feed.SiteURL, feed.IconURL) if icon, err := iconFinder.FindIcon(); err != nil { slog.Debug("Unable to find feed icon", - slog.Int64("feed_id", feedID), - slog.String("website_url", websiteURL), - slog.String("feed_icon_url", feedIconURL), + slog.Int64("feed_id", feed.ID), + slog.String("website_url", feed.SiteURL), + slog.String("feed_icon_url", feed.IconURL), slog.Any("error", err), ) } else if icon == nil { slog.Debug("No icon found", - slog.Int64("feed_id", feedID), - slog.String("website_url", websiteURL), - slog.String("feed_icon_url", feedIconURL), + slog.Int64("feed_id", feed.ID), + slog.String("website_url", feed.SiteURL), + slog.String("feed_icon_url", feed.IconURL), ) } else { - if err := store.CreateFeedIcon(feedID, icon); err != nil { + if forceRefresh { + if err := store.RemoveFeedIcon(feed.ID); err != nil { + slog.Error("Unable to remove Feed Icon", + slog.Int64("feed_id", feed.ID), + slog.Any("error", err), + ) + } + } + if err := store.CreateFeedIcon(feed.ID, icon); err != nil { slog.Error("Unable to store feed icon", - slog.Int64("feed_id", feedID), - slog.String("website_url", websiteURL), - slog.String("feed_icon_url", feedIconURL), + slog.Int64("feed_id", feed.ID), + slog.String("website_url", feed.SiteURL), + slog.String("feed_icon_url", feed.IconURL), slog.Any("error", err), ) } diff --git a/internal/reader/icon/finder.go b/internal/reader/icon/finder.go index cb9c7d49cab..dececdca4e1 100644 --- a/internal/reader/icon/finder.go +++ b/internal/reader/icon/finder.go @@ -177,6 +177,7 @@ func (f *IconFinder) DownloadIcon(iconURL string) (*model.Icon, error) { Hash: crypto.HashFromBytes(responseBody), MimeType: responseHandler.ContentType(), Content: responseBody, + URL: iconURL, } return icon, nil diff --git a/internal/storage/feed.go b/internal/storage/feed.go index 1a40102c0f2..11e81e290dc 100644 --- a/internal/storage/feed.go +++ b/internal/storage/feed.go @@ -235,10 +235,11 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { hide_globally, url_rewrite_rules, no_media_player, - apprise_service_urls + apprise_service_urls, + icon_url ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24) + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25) RETURNING id ` @@ -268,6 +269,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { feed.UrlRewriteRules, feed.NoMediaPlayer, feed.AppriseServiceURLs, + feed.IconURL, ).Scan(&feed.ID) if err != nil { return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err) @@ -339,9 +341,10 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) { hide_globally=$24, url_rewrite_rules=$25, no_media_player=$26, - apprise_service_urls=$27 + apprise_service_urls=$27, + icon_url=$28 WHERE - id=$28 AND user_id=$29 + id=$29 AND user_id=$30 ` _, err = s.db.Exec(query, feed.FeedURL, @@ -371,6 +374,7 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) { feed.UrlRewriteRules, feed.NoMediaPlayer, feed.AppriseServiceURLs, + feed.IconURL, feed.ID, feed.UserID, ) diff --git a/internal/storage/feed_query_builder.go b/internal/storage/feed_query_builder.go index 29926b81959..2d2df66974c 100644 --- a/internal/storage/feed_query_builder.go +++ b/internal/storage/feed_query_builder.go @@ -163,7 +163,8 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) { c.hide_globally as category_hidden, fi.icon_id, u.timezone, - f.apprise_service_urls + f.apprise_service_urls, + f.icon_url FROM feeds f LEFT JOIN @@ -230,6 +231,7 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) { &iconID, &tz, &feed.AppriseServiceURLs, + &feed.IconURL, ) if err != nil { diff --git a/internal/storage/icon.go b/internal/storage/icon.go index bd407d92a26..b1c6cc1ee96 100644 --- a/internal/storage/icon.go +++ b/internal/storage/icon.go @@ -115,6 +115,31 @@ func (s *Storage) CreateFeedIcon(feedID int64, icon *model.Icon) error { return nil } +// RemoveFeedIcon removes the feed_icon relation and the icon if no other feed is using it +func (s *Storage) RemoveFeedIcon(feedID int64) error { + var iconID int64 + err := s.db.QueryRow(`SELECT icon_id FROM feed_icons WHERE feed_id=$1`, feedID).Scan(&iconID) + if err == sql.ErrNoRows { + return nil + } else if err != nil { + return fmt.Errorf(`store: unable to fetch iconID: %v`, err) + } + + if _, err := s.db.Exec(`DELETE FROM feed_icons WHERE feed_id=$1`, feedID); err != nil { + return fmt.Errorf(`store: unable to remove feed icon: %v`, err) + } + + query := ` + DELETE FROM icons + WHERE + id=$1 AND NOT EXISTS(SELECT icon_id FROM feed_icons WHERE icon_id=$1); + ` + if _, err := s.db.Exec(query, iconID); err != nil { + return fmt.Errorf(`store: unable to remove feed icon: %v`, err) + } + return nil +} + // Icons returns all icons that belongs to a user. func (s *Storage) Icons(userID int64) (model.Icons, error) { query := `