Basic system for sending Discord webhook messages about server problems

To be expanded in the future
space-wizards · Apr 5, 2024 · 58b078c · 58b078c
1 parent 20e7fb3
commit 58b078c
Show file tree

Hide file tree

Showing 6 changed files with 122 additions and 8 deletions.
diff --git a/SS14.Watchdog/Components/Notifications/NotificationManager.cs b/SS14.Watchdog/Components/Notifications/NotificationManager.cs
@@ -0,0 +1,79 @@
+using System;
+using System.Net.Http;
+using System.Net.Http.Json;
+using System.Text.Json.Serialization;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace SS14.Watchdog.Components.Notifications;
+
+/// <summary>
+/// Implements external notifications to various services.
+/// </summary>
+/// <seealso cref="NotificationOptions"/>
+public sealed partial class NotificationManager(
+    IHttpClientFactory http,
+    ILogger<NotificationManager> logger,
+    IOptions<NotificationOptions> options)
+{
+    public const string DiscordHttpClient = "discord_webhook";
+
+    private readonly HttpClient _httpClient = http.CreateClient();
+
+    // I can't wait for this interface to keep expanding in the future.
+    public void SendNotification(string message)
+    {
+        var optionsValue = options.Value;
+        if (optionsValue.DiscordWebhook == null)
+        {
+            logger.LogTrace("Not sending notification: no Discord webhook URL configured");
+            return;
+        }
+
+        SendWebhook(optionsValue, message);
+    }
+
+    private async void SendWebhook(NotificationOptions optionsValue, string message)
+    {
+        logger.LogTrace("Sending notification \"{Message}\" to Discord webhook...", message);
+
+        try
+        {
+            var messageObject = new DiscordWebhookExecute
+            {
+                Content = message,
+                AllowedMentions = DiscordAllowedMentions.None
+            };
+
+            using var response = await _httpClient.PostAsJsonAsync(
+                optionsValue.DiscordWebhook,
+                messageObject,
+                DiscordSourceGenerationContext.Default.DiscordWebhookExecute);
+
+            response.EnsureSuccessStatusCode();
+
+            logger.LogTrace("Succeeded sending notification to Discord webhook");
+        }
+        catch (Exception e)
+        {
+            logger.LogError(e, "Error while sending Discord webhook!");
+        }
+    }
+
+    private sealed class DiscordWebhookExecute
+    {
+        public string? Content { get; set; }
+        public DiscordAllowedMentions? AllowedMentions { get; set; } = null;
+    }
+
+    private sealed class DiscordAllowedMentions
+    {
+        public static readonly DiscordAllowedMentions None = new() { Parse = [] };
+
+        public string[]? Parse { get; set; }
+    }
+
+    [JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.KebabCaseLower)]
+    [JsonSerializable(typeof(DiscordWebhookExecute))]
+    private partial class DiscordSourceGenerationContext : JsonSerializerContext;
+}
diff --git a/SS14.Watchdog/Components/Notifications/NotificationOptions.cs b/SS14.Watchdog/Components/Notifications/NotificationOptions.cs
@@ -0,0 +1,15 @@
+namespace SS14.Watchdog.Components.Notifications;
+
+/// <summary>
+/// Options for notifications the watchdog can send via various channels.
+/// </summary>
+/// <seealso cref="NotificationManager"/>
+public sealed class NotificationOptions
+{
+    public const string Position = "Notification";
+
+    /// <summary>
+    /// A Discord webhook URL to send notifications like server crashes to.
+    /// </summary>
+    public string? DiscordWebhook { get; set; }
+}
diff --git a/SS14.Watchdog/Components/ServerManagement/ServerInstance.Actor.cs b/SS14.Watchdog/Components/ServerManagement/ServerInstance.Actor.cs
@@ -222,6 +222,9 @@ private async Task RunCommandServerExit(CommandServerExit exit, CancellationToke
             return;
         }
 
+        if (exit.ExitCode != 0)
+            _notificationManager.SendNotification($"Server `{Key}` exited with non-success exit code: {exit.ExitCode}. Check server logs for possible causes.");
+
         _runningServer = null;
         if (_lastPing == null)
         {
@@ -235,6 +238,7 @@ private async Task RunCommandServerExit(CommandServerExit exit, CancellationToke
                 _startupFailUpdateWait = true;
                 // Server keeps crashing during init, wait for an update to fix it.
                 _logger.LogWarning("{Key} is failing to start, giving up until update or manual intervention.", Key);
+                _notificationManager.SendNotification($"Server `{Key}` is failing to start, needs manual intervention or update.");
                 return;
             }
         }
@@ -353,7 +357,7 @@ private async void MonitorServer(int startNumber, CancellationToken cancel = def
             _logger.LogInformation("{Key} shut down with exit code {ExitCode}", Key,
                 _runningServer.ExitCode);
 
-            await _commandQueue.Writer.WriteAsync(new CommandServerExit(startNumber), cancel);
+            await _commandQueue.Writer.WriteAsync(new CommandServerExit(startNumber, _runningServer.ExitCode), cancel);
         }
         catch (OperationCanceledException)
         {
@@ -425,7 +429,7 @@ private sealed record CommandTimedOut(int TimeoutCounter) : Command;
     /// <summary>
     /// The server has exited while being monitored.
     /// </summary>
-    private sealed record CommandServerExit(int StartNumber) : Command;
+    private sealed record CommandServerExit(int StartNumber, int ExitCode) : Command;
 
     /// <summary>
     /// The server has sent us a ping, it's still kicking!

diff --git a/SS14.Watchdog/Components/ServerManagement/ServerInstance.cs b/SS14.Watchdog/Components/ServerManagement/ServerInstance.cs
@@ -13,6 +13,7 @@
 using Microsoft.Extensions.Logging;
 using SS14.Watchdog.Components.BackgroundTasks;
 using SS14.Watchdog.Components.DataManagement;
+using SS14.Watchdog.Components.Notifications;
 using SS14.Watchdog.Components.ProcessManagement;
 using SS14.Watchdog.Components.Updates;
 using SS14.Watchdog.Configuration;
@@ -51,6 +52,7 @@ public sealed partial class ServerInstance : IServerInstance
         private readonly IBackgroundTaskQueue _taskQueue;
         private readonly DataManager _dataManager;
         private readonly IProcessManager _processManager;
+        private readonly NotificationManager _notificationManager;
 
         private string? _currentRevision;
         private bool _updateOnRestart = true;
@@ -61,16 +63,16 @@ public sealed partial class ServerInstance : IServerInstance
 
         private IProcessHandle? _runningServer;
 
-        public ServerInstance(
-            string key,
+        public ServerInstance(string key,
             InstanceConfiguration instanceConfig,
             IConfiguration configuration,
             ServersConfiguration serversConfiguration,
             ILogger<ServerInstance> logger,
             IBackgroundTaskQueue taskQueue,
             IServiceProvider serviceProvider,
             DataManager dataManager,
-            IProcessManager processManager)
+            IProcessManager processManager,
+            NotificationManager notificationManager)
         {
             Key = key;
             _instanceConfig = instanceConfig;
@@ -80,6 +82,7 @@ public ServerInstance(
             _taskQueue = taskQueue;
             _dataManager = dataManager;
             _processManager = processManager;
+            _notificationManager = notificationManager;
 
             if (!string.IsNullOrEmpty(_instanceConfig.ApiTokenFile))
             {
@@ -282,6 +285,8 @@ private void TimeoutKill()
             if (_runningServer == null)
                 return;
 
+            _notificationManager.SendNotification($"Server `{Key}` timed and will be killed. Check server logs for possible causes.");
+
             if (_instanceConfig.DumpOnTimeout)
             {
                 if (!OperatingSystem.IsWindows())
@@ -410,4 +415,4 @@ private sealed class ShutdownParameters
             public string Reason { get; set; } = default!;
         }
     }
-}
+}
diff --git a/SS14.Watchdog/Components/ServerManagement/ServerManager.cs b/SS14.Watchdog/Components/ServerManagement/ServerManager.cs
@@ -16,6 +16,7 @@
 using Microsoft.Extensions.Options;
 using SS14.Watchdog.Components.BackgroundTasks;
 using SS14.Watchdog.Components.DataManagement;
+using SS14.Watchdog.Components.Notifications;
 using SS14.Watchdog.Components.ProcessManagement;
 using SS14.Watchdog.Configuration;
 
@@ -31,6 +32,7 @@ public sealed class ServerManager : BackgroundService, IServerManager
         private readonly IProcessManager _processManager;
         private readonly IServer _server;
         private readonly IHostApplicationLifetime _hostApplicationLifetime;
+        private readonly NotificationManager _notificationManager;
         private readonly IConfiguration _configuration;
         private readonly IOptionsMonitor<ServersConfiguration> _serverCfg;
         private readonly Dictionary<string, ServerInstance> _instances = new Dictionary<string, ServerInstance>();
@@ -46,7 +48,8 @@ public ServerManager(
             DataManager dataManager,
             IProcessManager processManager,
             IServer server,
-            IHostApplicationLifetime hostApplicationLifetime)
+            IHostApplicationLifetime hostApplicationLifetime,
+            NotificationManager notificationManager)
         {
             _logger = logger;
             _configuration = configuration;
@@ -56,6 +59,7 @@ public ServerManager(
             _processManager = processManager;
             _server = server;
             _hostApplicationLifetime = hostApplicationLifetime;
+            _notificationManager = notificationManager;
             _serverCfg = instancesOptions;
         }
 
@@ -131,7 +135,8 @@ public override async Task StartAsync(CancellationToken cancellationToken)
                         _taskQueue,
                         _provider,
                         _dataManager,
-                        _processManager);
+                        _processManager,
+                        _notificationManager);
 
                 _instances.Add(key, instance);
             }

diff --git a/SS14.Watchdog/Startup.cs b/SS14.Watchdog/Startup.cs
@@ -11,6 +11,7 @@
 using Serilog;
 using SS14.Watchdog.Components.BackgroundTasks;
 using SS14.Watchdog.Components.DataManagement;
+using SS14.Watchdog.Components.Notifications;
 using SS14.Watchdog.Components.ProcessManagement;
 using SS14.Watchdog.Components.ServerManagement;
 using SS14.Watchdog.Configuration;
@@ -31,6 +32,7 @@ public Startup(IConfiguration configuration)
         public void ConfigureServices(IServiceCollection services)
         {
             services.Configure<DataOptions>(Configuration.GetSection(DataOptions.Position));
+            services.Configure<NotificationOptions>(Configuration.GetSection(NotificationOptions.Position));
             services.Configure<ServersConfiguration>(Configuration.GetSection("Servers"));
 
             services.AddSingleton<DataManager>();
@@ -58,6 +60,10 @@ public void ConfigureServices(IServiceCollection services)
             services.AddSingleton<BackgroundTaskQueue>();
             services.AddSingleton<IBackgroundTaskQueue>(p => p.GetService<BackgroundTaskQueue>()!);
             services.AddHostedService(p => p.GetService<BackgroundTaskQueue>()!);
+
+            services.AddSingleton<NotificationManager>();
+
+            services.AddHttpClient(NotificationManager.DiscordHttpClient);
         }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.