From ef6cfddb433c1a776a42882efec45079d666cfe0 Mon Sep 17 00:00:00 2001 From: telnet23 Date: Tue, 19 Nov 2024 00:57:59 +0000 Subject: [PATCH] feat: optionally fetch watch time from YouTube API instead of website --- internal/config/config_test.go | 18 ++++++++ internal/config/options.go | 9 ++++ internal/config/parser.go | 2 + internal/reader/processor/youtube.go | 65 ++++++++++++++++++++++++++++ miniflux.1 | 5 +++ 5 files changed, 99 insertions(+) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 309d0464368..bfe40450e03 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -2116,6 +2116,24 @@ func TestFetchYouTubeWatchTime(t *testing.T) { } } +func TestYouTubeApiKey(t *testing.T) { + os.Clearenv() + os.Setenv("YOUTUBE_API_KEY", "AAAAAAAAAAAAAaaaaaaaaaaaaa0000000000000") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := "AAAAAAAAAAAAAaaaaaaaaaaaaa0000000000000" + result := opts.YouTubeApiKey() + + if result != expected { + t.Fatalf(`Unexpected YOUTUBE_API_KEY value, got %v instead of %v`, result, expected) + } +} + func TestYouTubeEmbedUrlOverride(t *testing.T) { os.Clearenv() os.Setenv("YOUTUBE_EMBED_URL_OVERRIDE", "https://invidious.custom/embed/") diff --git a/internal/config/options.go b/internal/config/options.go index 303c8ff3d67..015a725e198 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -60,6 +60,7 @@ const ( defaultFetchNebulaWatchTime = false defaultFetchOdyseeWatchTime = false defaultFetchYouTubeWatchTime = false + defaultYouTubeApiKey = "" defaultYouTubeEmbedUrlOverride = "https://www.youtube-nocookie.com/embed/" defaultCreateAdmin = false defaultAdminUsername = "" @@ -149,6 +150,7 @@ type Options struct { fetchOdyseeWatchTime bool fetchYouTubeWatchTime bool filterEntryMaxAgeDays int + youTubeApiKey string youTubeEmbedUrlOverride string oauth2UserCreationAllowed bool oauth2ClientID string @@ -228,6 +230,7 @@ func NewOptions() *Options { fetchNebulaWatchTime: defaultFetchNebulaWatchTime, fetchOdyseeWatchTime: defaultFetchOdyseeWatchTime, fetchYouTubeWatchTime: defaultFetchYouTubeWatchTime, + youTubeApiKey: defaultYouTubeApiKey, youTubeEmbedUrlOverride: defaultYouTubeEmbedUrlOverride, oauth2UserCreationAllowed: defaultOAuth2UserCreation, oauth2ClientID: defaultOAuth2ClientID, @@ -503,6 +506,11 @@ func (o *Options) FetchYouTubeWatchTime() bool { return o.fetchYouTubeWatchTime } +// YouTubeApiKey returns the YouTube API key if defined. +func (o *Options) YouTubeApiKey() string { + return o.youTubeApiKey +} + // YouTubeEmbedUrlOverride returns YouTube URL which will be used for embeds func (o *Options) YouTubeEmbedUrlOverride() string { return o.youTubeEmbedUrlOverride @@ -733,6 +741,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option { "SERVER_TIMING_HEADER": o.serverTimingHeader, "WATCHDOG": o.watchdog, "WORKER_POOL_SIZE": o.workerPoolSize, + "YOUTUBE_API_KEY": redactSecretValue(o.youTubeApiKey, redactSecret), "YOUTUBE_EMBED_URL_OVERRIDE": o.youTubeEmbedUrlOverride, "WEBAUTHN": o.webAuthn, } diff --git a/internal/config/parser.go b/internal/config/parser.go index b443bae0ef0..4d893a3567c 100644 --- a/internal/config/parser.go +++ b/internal/config/parser.go @@ -271,6 +271,8 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.fetchOdyseeWatchTime = parseBool(value, defaultFetchOdyseeWatchTime) case "FETCH_YOUTUBE_WATCH_TIME": p.opts.fetchYouTubeWatchTime = parseBool(value, defaultFetchYouTubeWatchTime) + case "YOUTUBE_API_KEY": + p.opts.youTubeApiKey = parseString(value, defaultYouTubeApiKey) case "YOUTUBE_EMBED_URL_OVERRIDE": p.opts.youTubeEmbedUrlOverride = parseString(value, defaultYouTubeEmbedUrlOverride) case "WATCHDOG": diff --git a/internal/reader/processor/youtube.go b/internal/reader/processor/youtube.go index 52de18c4933..2d41e11f613 100644 --- a/internal/reader/processor/youtube.go +++ b/internal/reader/processor/youtube.go @@ -4,9 +4,11 @@ package processor import ( + "encoding/json" "errors" "fmt" "log/slog" + "net/url" "regexp" "strconv" "time" @@ -33,6 +35,14 @@ func shouldFetchYouTubeWatchTime(entry *model.Entry) bool { } func fetchYouTubeWatchTime(websiteURL string) (int, error) { + if config.Opts.YouTubeApiKey() == "" { + return fetchYouTubeWatchTimeFromWebsite(websiteURL) + } else { + return fetchYouTubeWatchTimeFromApi(websiteURL) + } +} + +func fetchYouTubeWatchTimeFromWebsite(websiteURL string) (int, error) { requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) @@ -63,6 +73,61 @@ func fetchYouTubeWatchTime(websiteURL string) (int, error) { return int(dur.Minutes()), nil } +func fetchYouTubeWatchTimeFromApi(websiteURL string) (int, error) { + requestBuilder := fetcher.NewRequestBuilder() + requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) + requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) + + parsedWebsiteURL, err := url.Parse(websiteURL) + if err != nil { + return 0, fmt.Errorf("unable to parse URL: %v", err) + } + + apiQuery := url.Values{} + apiQuery.Set("id", parsedWebsiteURL.Query().Get("v")) + apiQuery.Set("key", config.Opts.YouTubeApiKey()) + apiQuery.Set("part", "contentDetails") + + apiURL := url.URL{ + Scheme: "https", + Host: "www.googleapis.com", + Path: "youtube/v3/videos", + RawQuery: apiQuery.Encode(), + } + + responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(apiURL.String())) + defer responseHandler.Close() + + if localizedError := responseHandler.LocalizedError(); localizedError != nil { + slog.Warn("Unable to fetch contentDetails from YouTube API", slog.String("website_url", websiteURL), slog.Any("error", localizedError.Error())) + return 0, localizedError.Error() + } + + var videos struct { + Items []struct { + ContentDetails struct { + Duration string `json:"duration"` + } `json:"contentDetails"` + } `json:"items"` + } + + if err := json.NewDecoder(responseHandler.Body(config.Opts.HTTPClientMaxBodySize())).Decode(&videos); err != nil { + return 0, fmt.Errorf("unable to decode JSON: %v", err) + } + + if n := len(videos.Items); n != 1 { + return 0, fmt.Errorf("invalid items length: %d", n) + } + + durs := videos.Items[0].ContentDetails.Duration + dur, err := parseISO8601(durs) + if err != nil { + return 0, fmt.Errorf("unable to parse duration %s: %v", durs, err) + } + + return int(dur.Minutes()), nil +} + func parseISO8601(from string) (time.Duration, error) { var match []string var d time.Duration diff --git a/miniflux.1 b/miniflux.1 index 7c785d3cb1b..528b43fd0df 100644 --- a/miniflux.1 +++ b/miniflux.1 @@ -555,6 +555,11 @@ Number of background workers\&. .br Default is 16 workers\&. .TP +.B YOUTUBE_API_KEY +YouTube API key for use with FETCH_YOUTUBE_WATCH_TIME. If nonempty, the duration will be fetched from the YouTube API. Otherwise, the duration will be fetched from the YouTube website\&. +.br +Default is empty\&. +.TP .B YOUTUBE_EMBED_URL_OVERRIDE YouTube URL which will be used for embeds\&. .br