Add support for refreshing feed icons

internal/database/migrations.go

@@ -834,4 +834,11 @@ var migrations = []func(tx *sql.Tx) error{
 		_, err = tx.Exec(sql)
 		return
 	},
+	func(tx *sql.Tx) (err error) {
+		sql := `
+			ALTER TABLE feeds ADD COLUMN icon_url text default '';
+		`
+		_, err = tx.Exec(sql)
+		return err
+	},
 }

internal/model/feed.go

@@ -51,17 +51,17 @@ type Feed struct {
 	FetchViaProxy               bool      `json:"fetch_via_proxy"`
 	HideGlobally                bool      `json:"hide_globally"`
 	AppriseServiceURLs          string    `json:"apprise_service_urls"`
+	IconURL                     string    `json:"icon_url"`
 
 	// Non persisted attributes
 	Category *Category `json:"category,omitempty"`
 	Icon     *FeedIcon `json:"icon"`
 	Entries  Entries   `json:"entries,omitempty"`
 
-	TTL                    int    `json:"-"`
-	IconURL                string `json:"-"`
-	UnreadCount            int    `json:"-"`
-	ReadCount              int    `json:"-"`
-	NumberOfVisibleEntries int    `json:"-"`
+	TTL                    int `json:"-"`
+	UnreadCount            int `json:"-"`
+	ReadCount              int `json:"-"`
+	NumberOfVisibleEntries int `json:"-"`
 }
 
 type FeedCounters struct {

internal/model/icon.go

@@ -14,6 +14,7 @@ type Icon struct {
 	Hash     string `json:"hash"`
 	MimeType string `json:"mime_type"`
 	Content  []byte `json:"-"`
+	URL      string `json:"-"`
 }
 
 // DataURL returns the data URL of the icon.

internal/reader/handler/handler.go

@@ -93,10 +93,8 @@ func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, f
 
 	checkFeedIcon(
 		store,
-		requestBuilder,
-		subscription.ID,
-		subscription.SiteURL,
-		subscription.IconURL,
+		subscription,
+		false,
 	)
 
 	return subscription, nil
@@ -185,10 +183,8 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model
 
 	checkFeedIcon(
 		store,
-		requestBuilder,
-		subscription.ID,
-		subscription.SiteURL,
-		subscription.IconURL,
+		subscription,
+		false,
 	)
 	return subscription, nil
 }
@@ -321,20 +317,29 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
 		originalFeed.EtagHeader = responseHandler.ETag()
 		originalFeed.LastModifiedHeader = responseHandler.LastModified()
 
-		checkFeedIcon(
-			store,
-			requestBuilder,
-			originalFeed.ID,
-			originalFeed.SiteURL,
-			updatedFeed.IconURL,
-		)
+		if originalFeed.IconURL != updatedFeed.IconURL {
+			originalFeed.IconURL = updatedFeed.IconURL
+			forceRefresh = true
+		} else if originalFeed.IconURL != "" {
+			slog.Debug("Feed icon URL not modified",
+				slog.Int64("user_id", userID),
+				slog.Int64("feed_id", feedID),
+				slog.String("feed_icon_url", originalFeed.IconURL),
+			)
+		}
 	} else {
 		slog.Debug("Feed not modified",
 			slog.Int64("user_id", userID),
 			slog.Int64("feed_id", feedID),
 		)
 	}
 
+	checkFeedIcon(
+		store,
+		originalFeed,
+		forceRefresh,
+	)
+
 	originalFeed.ResetErrorCounter()
 
 	if storeErr := store.UpdateFeed(originalFeed); storeErr != nil {
@@ -347,28 +352,45 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
 	return nil
 }
 
-func checkFeedIcon(store *storage.Storage, requestBuilder *fetcher.RequestBuilder, feedID int64, websiteURL, feedIconURL string) {
-	if !store.HasIcon(feedID) {
-		iconFinder := icon.NewIconFinder(requestBuilder, websiteURL, feedIconURL)
+func checkFeedIcon(store *storage.Storage, feed *model.Feed, forceRefresh bool) {
+	if !store.HasIcon(feed.ID) || forceRefresh {
+		requestBuilder := fetcher.NewRequestBuilder()
+		requestBuilder.WithUsernameAndPassword(feed.Username, feed.Password)
+		requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent())
+		requestBuilder.WithCookie(feed.Cookie)
+		requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
+		requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
+		requestBuilder.UseProxy(feed.FetchViaProxy)
+		requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates)
+
+		iconFinder := icon.NewIconFinder(requestBuilder, feed.SiteURL, feed.IconURL)
 		if icon, err := iconFinder.FindIcon(); err != nil {
 			slog.Debug("Unable to find feed icon",
-				slog.Int64("feed_id", feedID),
-				slog.String("website_url", websiteURL),
-				slog.String("feed_icon_url", feedIconURL),
+				slog.Int64("feed_id", feed.ID),
+				slog.String("website_url", feed.SiteURL),
+				slog.String("feed_icon_url", feed.IconURL),
 				slog.Any("error", err),
 			)
 		} else if icon == nil {
 			slog.Debug("No icon found",
-				slog.Int64("feed_id", feedID),
-				slog.String("website_url", websiteURL),
-				slog.String("feed_icon_url", feedIconURL),
+				slog.Int64("feed_id", feed.ID),
+				slog.String("website_url", feed.SiteURL),
+				slog.String("feed_icon_url", feed.IconURL),
 			)
 		} else {
-			if err := store.CreateFeedIcon(feedID, icon); err != nil {
+			if forceRefresh {
+				if err := store.RemoveFeedIcon(feed.ID); err != nil {
+					slog.Error("Unable to remove Feed Icon",
+						slog.Int64("feed_id", feed.ID),
+						slog.Any("error", err),
+					)
+				}
+			}
+			if err := store.CreateFeedIcon(feed.ID, icon); err != nil {
 				slog.Error("Unable to store feed icon",
-					slog.Int64("feed_id", feedID),
-					slog.String("website_url", websiteURL),
-					slog.String("feed_icon_url", feedIconURL),
+					slog.Int64("feed_id", feed.ID),
+					slog.String("website_url", feed.SiteURL),
+					slog.String("feed_icon_url", feed.IconURL),
 					slog.Any("error", err),
 				)
 			}

internal/reader/icon/finder.go

@@ -177,6 +177,7 @@ func (f *IconFinder) DownloadIcon(iconURL string) (*model.Icon, error) {
 		Hash:     crypto.HashFromBytes(responseBody),
 		MimeType: responseHandler.ContentType(),
 		Content:  responseBody,
+		URL:      iconURL,
 	}
 
 	return icon, nil

internal/storage/feed.go

@@ -235,10 +235,11 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
 			hide_globally,
 			url_rewrite_rules,
 			no_media_player,
-			apprise_service_urls
+			apprise_service_urls,
+			icon_url
 		)
 		VALUES
-			($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24)
+			($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25)
 		RETURNING
 			id
 	`
@@ -268,6 +269,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
 		feed.UrlRewriteRules,
 		feed.NoMediaPlayer,
 		feed.AppriseServiceURLs,
+		feed.IconURL,
 	).Scan(&feed.ID)
 	if err != nil {
 		return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err)
@@ -339,9 +341,10 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
 			hide_globally=$24,
 			url_rewrite_rules=$25,
 			no_media_player=$26,
-			apprise_service_urls=$27
+			apprise_service_urls=$27,
+			icon_url=$28
 		WHERE
-			id=$28 AND user_id=$29
+			id=$29 AND user_id=$30
 	`
 	_, err = s.db.Exec(query,
 		feed.FeedURL,
@@ -371,6 +374,7 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
 		feed.UrlRewriteRules,
 		feed.NoMediaPlayer,
 		feed.AppriseServiceURLs,
+		feed.IconURL,
 		feed.ID,
 		feed.UserID,
 	)

internal/storage/feed_query_builder.go

@@ -163,7 +163,8 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) {
 			c.hide_globally as category_hidden,
 			fi.icon_id,
 			u.timezone,
-			f.apprise_service_urls
+			f.apprise_service_urls,
+			f.icon_url
 		FROM
 			feeds f
 		LEFT JOIN
@@ -230,6 +231,7 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) {
 			&iconID,
 			&tz,
 			&feed.AppriseServiceURLs,
+			&feed.IconURL,
 		)
 
 		if err != nil {

internal/storage/icon.go

@@ -115,6 +115,31 @@ func (s *Storage) CreateFeedIcon(feedID int64, icon *model.Icon) error {
 	return nil
 }
 
+// RemoveFeedIcon removes the feed_icon relation and the icon if no other feed is using it
+func (s *Storage) RemoveFeedIcon(feedID int64) error {
+	var iconID int64
+	err := s.db.QueryRow(`SELECT icon_id FROM feed_icons WHERE feed_id=$1`, feedID).Scan(&iconID)
+	if err == sql.ErrNoRows {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf(`store: unable to fetch iconID: %v`, err)
+	}
+
+	if _, err := s.db.Exec(`DELETE FROM feed_icons WHERE feed_id=$1`, feedID); err != nil {
+		return fmt.Errorf(`store: unable to remove feed icon: %v`, err)
+	}
+
+	query := `
+		DELETE FROM icons
+		WHERE
+			id=$1 AND NOT EXISTS(SELECT icon_id FROM feed_icons WHERE icon_id=$1);
+	`
+	if _, err := s.db.Exec(query, iconID); err != nil {
+		return fmt.Errorf(`store: unable to remove feed icon: %v`, err)
+	}
+	return nil
+}
+
 // Icons returns all icons that belongs to a user.
 func (s *Storage) Icons(userID int64) (model.Icons, error) {
 	query := `