Add configurable number of retries and timeout for http calls (#40)

README.md

@@ -193,7 +193,9 @@ services:
 | RADARR__ENABLED | No | Enable or disable Radarr cleanup  | false |
 | RADARR__INSTANCES__0__URL | No | First Radarr instance url | http://localhost:8989 |
 | RADARR__INSTANCES__0__APIKEY | No | First Radarr instance API key | empty |
-
+|||||
+| HTTP_MAX_RETRIES | No | The number of times to retry a failed HTTP call (to *arrs, download clients etc.) | 0 |
+| HTTP_TIMEOUT | No | The number of seconds to wait before failing an HTTP call (to *arrs, download clients etc.) | 100 |
 #
 ### To be noted
 

code/Common/Configuration/General/HttpConfig.cs

@@ -0,0 +1,20 @@
+using Microsoft.Extensions.Configuration;
+
+namespace Common.Configuration.General;
+
+public class HttpConfig : IConfig
+{
+    [ConfigurationKeyName("HTTP_MAX_RETRIES")]
+    public ushort MaxRetries { get; init; }
+
+    [ConfigurationKeyName("HTTP_TIMEOUT")]
+    public ushort Timeout { get; init; } = 100;
+
+    public void Validate()
+    {
+        if (Timeout is 0)
+        {
+            throw new ArgumentException("HTTP_TIMEOUT must be greater than 0");
+        }
+    }
+}

code/Common/Configuration/TriggersConfig.cs → code/Common/Configuration/General/TriggersConfig.cs

@@ -1,4 +1,4 @@
-namespace Common.Configuration;
+namespace Common.Configuration.General;
 
 public sealed class TriggersConfig
 {

code/Common/Helpers/Constants.cs

@@ -4,4 +4,6 @@ public static class Constants
 {
     public static readonly TimeSpan TriggerMaxLimit  = TimeSpan.FromHours(6);
     public static readonly TimeSpan CacheLimitBuffer = TimeSpan.FromHours(2);
+
+    public const string HttpClientWithRetryName = "retry";
 }

code/Executable/DependencyInjection/MainDI.cs

@@ -1,14 +1,9 @@
 using System.Net;
-using Common.Configuration;
-using Common.Configuration.ContentBlocker;
-using Executable.Jobs;
-using Infrastructure.Verticals.Arr;
-using Infrastructure.Verticals.ContentBlocker;
-using Infrastructure.Verticals.DownloadClient;
+using Common.Configuration.General;
+using Common.Helpers;
 using Infrastructure.Verticals.DownloadClient.Deluge;
-using Infrastructure.Verticals.DownloadClient.QBittorrent;
-using Infrastructure.Verticals.DownloadClient.Transmission;
-using Infrastructure.Verticals.QueueCleaner;
+using Polly;
+using Polly.Extensions.Http;
 
 namespace Executable.DependencyInjection;
 
@@ -17,16 +12,27 @@ public static class MainDI
     public static IServiceCollection AddInfrastructure(this IServiceCollection services, IConfiguration configuration) =>
         services
             .AddLogging(builder => builder.ClearProviders().AddConsole())
-            .AddHttpClients()
+            .AddHttpClients(configuration)
             .AddConfiguration(configuration)
             .AddMemoryCache()
             .AddServices()
             .AddQuartzServices(configuration);
 
-    private static IServiceCollection AddHttpClients(this IServiceCollection services)
+    private static IServiceCollection AddHttpClients(this IServiceCollection services, IConfiguration configuration)
     {
         // add default HttpClient
         services.AddHttpClient();
+
+        HttpConfig config = configuration.Get<HttpConfig>() ?? new();
+        config.Validate();
+
+        // add retry HttpClient
+        services
+            .AddHttpClient(Constants.HttpClientWithRetryName, x =>
+            {
+                x.Timeout = TimeSpan.FromSeconds(config.Timeout);
+            })
+            .AddRetryPolicyHandler(config);
 
         // add Deluge HttpClient
         services
@@ -44,8 +50,18 @@ private static IServiceCollection AddHttpClients(this IServiceCollection service
                     AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
                     ServerCertificateCustomValidationCallback = (_, _, _, _) => true
                 };
-            });
+            })
+            .AddRetryPolicyHandler(config);
 
         return services;
     }
+
+    private static IHttpClientBuilder AddRetryPolicyHandler(this IHttpClientBuilder builder, HttpConfig config) =>
+        builder.AddPolicyHandler(
+            HttpPolicyExtensions
+                .HandleTransientHttpError()
+                // do not retry on Unauthorized
+                .OrResult(response => !response.IsSuccessStatusCode && response.StatusCode != HttpStatusCode.Unauthorized)
+                .WaitAndRetryAsync(config.MaxRetries, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)))
+        );
 }

code/Executable/DependencyInjection/QuartzDI.cs

@@ -1,5 +1,6 @@
 using Common.Configuration;
 using Common.Configuration.ContentBlocker;
+using Common.Configuration.General;
 using Common.Configuration.QueueCleaner;
 using Common.Helpers;
 using Executable.Jobs;

code/Executable/Executable.csproj

@@ -12,6 +12,7 @@
     <ItemGroup>
         <PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.0" />
         <PackageReference Include="Microsoft.Extensions.Http" Version="9.0.0" />
+        <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="9.0.0" />
         <PackageReference Include="Quartz" Version="3.13.1" />
         <PackageReference Include="Quartz.Extensions.DependencyInjection" Version="3.13.1" />
         <PackageReference Include="Quartz.Extensions.Hosting" Version="3.13.1" />

code/Executable/appsettings.Development.json

@@ -1,4 +1,6 @@
 {
+  "HTTP_MAX_RETRIES": 0,
+  "HTTP_TIMEOUT": 10,
   "Logging": {
     "LogLevel": "Debug",
     "Enhanced": true,

code/Executable/appsettings.json

@@ -1,4 +1,6 @@
 {
+  "HTTP_MAX_RETRIES": 0,
+  "HTTP_TIMEOUT": 100,
   "Logging": {
     "LogLevel": "Information",
     "Enhanced": true,

code/Infrastructure/Infrastructure.csproj

@@ -12,10 +12,10 @@
     </ItemGroup>
 
     <ItemGroup>
-      <PackageReference Include="FLM.Transmission" Version="1.0.0" />
+      <PackageReference Include="FLM.QBittorrent" Version="1.0.0" />
+      <PackageReference Include="FLM.Transmission" Version="1.0.2" />
       <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.0" />
       <PackageReference Include="Microsoft.Extensions.Http" Version="9.0.0" />
-      <PackageReference Include="QBittorrent.Client" Version="1.9.24285.1" />
       <PackageReference Include="Quartz" Version="3.13.1" />
     </ItemGroup>
 

code/Infrastructure/Verticals/Arr/ArrClient.cs

@@ -1,6 +1,7 @@
 using Common.Configuration.Arr;
 using Common.Configuration.Logging;
 using Common.Configuration.QueueCleaner;
+using Common.Helpers;
 using Domain.Enums;
 using Domain.Models.Arr;
 using Domain.Models.Arr.Queue;
@@ -29,7 +30,7 @@ Striker striker
     {
         _logger = logger;
         _striker = striker;
-        _httpClient = httpClientFactory.CreateClient();
+        _httpClient = httpClientFactory.CreateClient(Constants.HttpClientWithRetryName);
         _loggingConfig = loggingConfig.Value;
         _queueCleanerConfig = queueCleanerConfig.Value;
         _striker = striker;

code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs

@@ -2,6 +2,7 @@
 using System.Diagnostics;
 using System.Text.RegularExpressions;
 using Common.Configuration.ContentBlocker;
+using Common.Helpers;
 using Domain.Enums;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@@ -27,7 +28,7 @@ public BlocklistProvider(
     {
         _logger = logger;
         _config = config.Value;
-        _httpClient = httpClientFactory.CreateClient();
+        _httpClient = httpClientFactory.CreateClient(Constants.HttpClientWithRetryName);
 
         _config.Validate();
 

code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs

@@ -1,5 +1,6 @@
 using Common.Configuration.DownloadClient;
 using Common.Configuration.QueueCleaner;
+using Common.Helpers;
 using Infrastructure.Verticals.ContentBlocker;
 using Infrastructure.Verticals.ItemStriker;
 using Microsoft.Extensions.Logging;
@@ -15,6 +16,7 @@ public sealed class QBitService : DownloadServiceBase
 
     public QBitService(
         ILogger<QBitService> logger,
+        IHttpClientFactory httpClientFactory,
         IOptions<QBitConfig> config,
         IOptions<QueueCleanerConfig> queueCleanerConfig,
         FilenameEvaluator filenameEvaluator,
@@ -23,7 +25,7 @@ Striker striker
     {
         _config = config.Value;
         _config.Validate();
-        _client = new(_config.Url);
+        _client = new(httpClientFactory.CreateClient(Constants.HttpClientWithRetryName), _config.Url);
     }
 
     public override async Task LoginAsync()

code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs

@@ -1,5 +1,6 @@
 using Common.Configuration.DownloadClient;
 using Common.Configuration.QueueCleaner;
+using Common.Helpers;
 using Infrastructure.Verticals.ContentBlocker;
 using Infrastructure.Verticals.ItemStriker;
 using Microsoft.Extensions.Logging;
@@ -17,6 +18,7 @@ public sealed class TransmissionService : DownloadServiceBase
     private TorrentInfo[]? _torrentsCache;
 
     public TransmissionService(
+        IHttpClientFactory httpClientFactory,
         ILogger<TransmissionService> logger,
         IOptions<TransmissionConfig> config,
         IOptions<QueueCleanerConfig> queueCleanerConfig,
@@ -27,6 +29,7 @@ Striker striker
         _config = config.Value;
         _config.Validate();
         _client = new(
+            httpClientFactory.CreateClient(Constants.HttpClientWithRetryName),
             new Uri(_config.Url, "/transmission/rpc").ToString(),
             login: _config.Username,
             password: _config.Password

code/test/docker-compose.yml

@@ -176,6 +176,9 @@ services:
       - LOGGING__FILE__PATH=/var/logs
       - LOGGING__ENHANCED=true
 
+      - HTTP_MAX_RETRIES=0
+      - HTTP_TIMEOUT=20
+
       - TRIGGERS__QUEUECLEANER=0/30 * * * * ?
       - TRIGGERS__CONTENTBLOCKER=0/30 * * * * ?