Feature/md7319/cosmosdbusage main (#125)

* [draft]use cosmos db for orcanode monitor data * orcanode data loading in cosmos db * code changes to load data to cosmos db containers . MonitorState Orcanode OrcanodeEvent * [draft]use cosmos db for orcanode monitor data * orcanode data loading in cosmos db * code changes to load data to cosmos db containers . MonitorState Orcanode OrcanodeEvent * Update OrcanodeMonitor/Data/OrcanodeMonitorContext.cs * made all partitionkey as upper-case in code,cosmos db. removed commented code. * Cleanup Signed-off-by: Dave Thaler <[email protected]> --------- Signed-off-by: Dave Thaler <[email protected]> Co-authored-by: Moumita Dutta <[email protected]> Co-authored-by: Dave Thaler <[email protected]>
orcasound · Sep 21, 2024 · 20f99fd · 20f99fd
1 parent ec22b7f
commit 20f99fd
Show file tree

Hide file tree

Showing 8 changed files with 80 additions and 42 deletions.
diff --git a/OrcanodeMonitor/Core/Fetcher.cs b/OrcanodeMonitor/Core/Fetcher.cs
@@ -82,7 +82,12 @@ private static bool NameMatch(string name, string dataplicityName)
         /// <returns></returns>
         private static Orcanode CreateOrcanode(DbSet<Orcanode> nodeList)
         {
-            var newNode = new Orcanode();
+            var newNode = new Orcanode()
+            {
+                ID = Guid.NewGuid().ToString(),
+                PartitionValue = 1
+            };
+
             nodeList.Add(newNode);
             return newNode;
         }
@@ -710,32 +715,34 @@ public static List<OrcanodeEvent> GetEvents(OrcanodeMonitorContext context, int
             return orcanodeEvents;
         }
 
+        private static void AddOrcanodeEvent(OrcanodeMonitorContext context, Orcanode node, string type, string value)
+        {
+            var orcanodeEvent = new OrcanodeEvent(node, type, value, DateTime.UtcNow);
+            context.OrcanodeEvents.Add(orcanodeEvent);
+        }
+
         private static void AddDataplicityConnectionStatusEvent(OrcanodeMonitorContext context, Orcanode node)
         {
             string value = (node.DataplicityConnectionStatus == OrcanodeOnlineStatus.Online) ? "up" : "OFFLINE";
-            var orcanodeEvent = new OrcanodeEvent(node, "dataplicity connection", value, DateTime.UtcNow);
-            context.OrcanodeEvents.Add(orcanodeEvent);
+            AddOrcanodeEvent(context, node, "dataplicity connection", value);
         }
 
         private static void AddDataplicityAgentUpgradeStatusChangeEvent(OrcanodeMonitorContext context, Orcanode node)
         {
             string value = node.DataplicityUpgradeStatus.ToString();
-            var orcanodeEvent = new OrcanodeEvent(node, "agent upgrade status", value, DateTime.UtcNow);
-            context.OrcanodeEvents.Add(orcanodeEvent);
+            AddOrcanodeEvent(context, node, "agent upgrade status", value);
         }
 
         private static void AddDiskCapacityChangeEvent(OrcanodeMonitorContext context, Orcanode node)
         {
             string value = string.Format("{0}G", node.DiskCapacityInGigs);
-            var orcanodeEvent = new OrcanodeEvent(node, "SD card size", value, DateTime.UtcNow);
-            context.OrcanodeEvents.Add(orcanodeEvent);
+            AddOrcanodeEvent(context, node, "SD card size", value);
         }
 
         private static void AddHydrophoneStreamStatusEvent(OrcanodeMonitorContext context, Orcanode node)
         {
             string value = node.OrcasoundOnlineStatusString;
-            var orcanodeEvent = new OrcanodeEvent(node, "hydrophone stream", value, DateTime.UtcNow);
-            context.OrcanodeEvents.Add(orcanodeEvent);
+            AddOrcanodeEvent(context, node, "hydrophone stream", value);
         }
 
         /// <summary>

diff --git a/OrcanodeMonitor/Data/OrcanodeMonitorContext.cs b/OrcanodeMonitor/Data/OrcanodeMonitorContext.cs
@@ -20,16 +20,42 @@ public OrcanodeMonitorContext (DbContextOptions<OrcanodeMonitorContext> options)
 
         protected override void OnModelCreating(ModelBuilder modelBuilder)
         {
-            modelBuilder.Entity<Orcanode>().ToTable("Orcanode");
-            modelBuilder.Entity<OrcanodeEvent>().ToTable("OrcanodeEvent");
+            modelBuilder.Entity<MonitorState>()
+               .ToContainer("MonitorState")
+               .Property(item => item.ID)
+               .HasConversion<string>();
+
+            modelBuilder.Entity<MonitorState>()
+                .ToContainer("MonitorState")
+                .HasPartitionKey(item=>item.ID);
+
+            modelBuilder.Entity<Orcanode>()
+                .ToContainer("Orcanode")
+                .Property(item => item.PartitionValue)
+                .HasConversion<string>();
+
+            modelBuilder.Entity<Orcanode>()
+                .ToContainer("Orcanode")
+                .Property(item => item.ID);
+
+            modelBuilder.Entity<Orcanode>()
+                .ToContainer("Orcanode")
+                .HasPartitionKey(item => item.PartitionValue)
+                .HasKey(item=>item.ID);
 
             modelBuilder.Entity<OrcanodeEvent>()
-                .HasOne(e => e.Orcanode) // Navigation property
-                .WithMany() // Configure the inverse navigation property if needed
-                .HasForeignKey(e => e.OrcanodeId); // Foreign key
+                .ToContainer("OrcanodeEvent")
+                .Property(item => item.Year)
+                .HasConversion<string>();
 
-            modelBuilder.Entity<MonitorState>().ToTable("MonitorState");
+            modelBuilder.Entity<OrcanodeEvent>()
+                .ToContainer("OrcanodeEvent")
+                .HasPartitionKey(item => item.Year)
+                .HasOne(item => item.Orcanode)
+                .WithMany()
+                .HasForeignKey(item=>item.OrcanodeId);
         }
+
         protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
         {
             optionsBuilder.UseLazyLoadingProxies();

diff --git a/OrcanodeMonitor/Models/MonitorState.cs b/OrcanodeMonitor/Models/MonitorState.cs
@@ -26,6 +26,7 @@ public MonitorState()
         /// </summary>
         [DatabaseGenerated(DatabaseGeneratedOption.None)]
         public int ID { get; set; }
+
         public DateTime? LastUpdatedTimestampUtc { get; set; }
         #endregion persisted
 

diff --git a/OrcanodeMonitor/Models/Orcanode.cs b/OrcanodeMonitor/Models/Orcanode.cs
@@ -28,13 +28,13 @@ public enum OrcanodeUpgradeStatus
 
     public class OrcanodeIftttDTO
     {
-        public OrcanodeIftttDTO(int id, string displayName)
+        public OrcanodeIftttDTO(string id, string displayName)
         {
             ID = id;
             DisplayName = displayName;
         }
         [JsonPropertyName("ID")]
-        public int ID { get; private set; }
+        public string ID { get; private set; }
         [JsonPropertyName("display_name")]
         public string DisplayName { get; private set; }
         public override string ToString() => DisplayName;
@@ -58,6 +58,7 @@ public Orcanode()
             DataplicityName = string.Empty;
             DataplicitySerial = string.Empty;
             OrcaHelloId = string.Empty;
+            PartitionValue = 1;            
         }
 
         #region persisted
@@ -80,8 +81,8 @@ public Orcanode()
         /// the Orcasound feed id, since a node is typically detected by dataplicity first when
         /// no Orcasound feed id exists.
         /// </summary>
-        [DatabaseGenerated(DatabaseGeneratedOption.Identity)]
-        public int ID { get; set; }
+
+        public string ID { get; set; }
 
         /// <summary>
         /// Human-readable name at Orcasound.
@@ -177,6 +178,11 @@ public Orcanode()
         /// </summary>
         public string OrcaHelloId { get; set; }
 
+        /// <summary>
+        /// Partition key fixed value.
+        /// </summary>
+        public int PartitionValue { get; set; }
+
         #endregion persisted
 
         #region derived

diff --git a/OrcanodeMonitor/Models/OrcanodeEvent.cs b/OrcanodeMonitor/Models/OrcanodeEvent.cs
@@ -10,7 +10,7 @@ namespace OrcanodeMonitor.Models
 {
     public class OrcanodeEventIftttMeta
     {
-        public OrcanodeEventIftttMeta(int id, DateTime timestamp)
+        public OrcanodeEventIftttMeta(string id, DateTime timestamp)
         {
             Id = id.ToString();
             UnixTimestamp = Fetcher.DateTimeToUnixTimeStamp(timestamp);
@@ -26,7 +26,7 @@ public OrcanodeEventIftttMeta(int id, DateTime timestamp)
     /// </summary>
     public class OrcanodeIftttEventDTO
     {
-        public OrcanodeIftttEventDTO(int id, string nodeName, string slug, string type, string value, DateTime timestamp)
+        public OrcanodeIftttEventDTO(string id, string nodeName, string slug, string type, string value, DateTime timestamp)
         {
             Slug = slug;
             Type = type;
@@ -69,6 +69,8 @@ public OrcanodeEvent(Orcanode node, string type, string value, DateTime timestam
             Value = value;
             DateTimeUtc = timestamp;
             OrcanodeId = node.ID;
+            Year = timestamp.Year;
+            ID = Guid.NewGuid().ToString();
         }
 
         #region persisted
@@ -79,8 +81,8 @@ public OrcanodeEvent(Orcanode node, string type, string value, DateTime timestam
         /// <summary>
         /// Database key for an event.
         /// </summary>
-        [DatabaseGenerated(DatabaseGeneratedOption.Identity)]
-        public int ID { get; set; }
+        [DatabaseGenerated(DatabaseGeneratedOption.None)]
+        public string ID { get; set; }
 
         public string Slug { get; set; }
         public string Type { get; set; }
@@ -89,13 +91,15 @@ public OrcanodeEvent(Orcanode node, string type, string value, DateTime timestam
         /// <summary>
         /// Foreign Key for an Orcanode.
         /// </summary>
-        public int OrcanodeId { get; set; }
+        public string OrcanodeId { get; set; }
 
         // Navigation property that uses OrcanodeId.
         public virtual Orcanode Orcanode { get; set; }
 
         public DateTime DateTimeUtc { get; set; }
 
+        public int Year { get; set; }
+
         #endregion persisted
 
         #region derived

diff --git a/OrcanodeMonitor/OrcanodeMonitor.csproj b/OrcanodeMonitor/OrcanodeMonitor.csproj
@@ -14,6 +14,7 @@
     <PackageReference Include="FFMpegCore" Version="5.1.0" />
     <PackageReference Include="FFMpegInstaller.Windows.x64" Version="0.1.10" />
     <PackageReference Include="Microsoft.AspNetCore.Diagnostics.EntityFrameworkCore" Version="8.0.8" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Cosmos" Version="8.0.8" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Proxies" Version="8.0.8" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.8" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="8.0.8">

diff --git a/OrcanodeMonitor/Program.cs b/OrcanodeMonitor/Program.cs
@@ -6,30 +6,26 @@
 using Microsoft.Extensions.DependencyInjection;
 using OrcanodeMonitor.Data;
 using Microsoft.IdentityModel.Tokens;
+using System;
+using Microsoft.Azure.Cosmos;
 
 var builder = WebApplication.CreateBuilder(args);
 
 // First see if an environment variable specifies a connection string.
-var connection = Environment.GetEnvironmentVariable("AZURE_SQL_CONNECTIONSTRING");
-
-// As discussed in https://learn.microsoft.com/en-us/aspnet/core/data/ef-rp/migrations?view=aspnetcore-8.0&source=recommendations&tabs=visual-studio says:
-// We recommend that production apps not call Database.Migrate at application startup. Migrate shouldn't be called from an app that is deployed to a server farm. If the app is scaled out to multiple server instances, it's hard to ensure database schema updates don't happen from multiple servers or conflict with read/write access.
-// Database migration should be done as part of deployment, and in a controlled way.  Instead,
-// from a developer command prompt, do:
-//    dotnet ef database update --connection "Server=tcp:orcasound-server.database.windows.net,1433;Initial Catalog=OrcasoundFreeDatabase;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;Authentication=\"Active Directory Default\";Pooling=False;"
-bool autoMigrate = false;
-
-// If we have no override, then fall back to using a local SQL database.
+var connection = Environment.GetEnvironmentVariable("AZURE_COSMOS_CONNECTIONSTRING");
 if (connection.IsNullOrEmpty())
 {
     connection = builder.Configuration.GetConnectionString("OrcanodeMonitorContext") ?? throw new InvalidOperationException("Connection string 'OrcanodeMonitorContext' not found.");
-    autoMigrate = true;
 }
 
 // Add services to the container.
 builder.Services.AddRazorPages();
 builder.Services.AddDbContext<OrcanodeMonitorContext>(options =>
-    options.UseSqlServer(connection));
+    options.UseCosmos(
+        connection,
+        databaseName: "OrcaNodeMonitor",
+        options =>
+        { options.ConnectionMode(ConnectionMode.Gateway); }));
 builder.Services.AddHostedService<PeriodicTasks>(); // Register your background service
 builder.Services.AddDatabaseDeveloperPageExceptionFilter();
 
@@ -56,11 +52,6 @@
     var services = scope.ServiceProvider;
 
     var context = services.GetRequiredService<OrcanodeMonitorContext>();
-    if (autoMigrate)
-    {
-        context.Database.Migrate(); // Apply pending migrations
-    }
-    // DbInitializer.Initialize(context); // Optional: Seed data
 }
 
 app.UseHttpsRedirection();

diff --git a/OrcanodeMonitor/appsettings.json b/OrcanodeMonitor/appsettings.json
@@ -7,6 +7,8 @@
   },
   "AllowedHosts": "*",
   "ConnectionStrings": {
-    "OrcanodeMonitorContext": "Server=(localdb)\\mssqllocaldb;Database=OrcanodeMonitorContext-361a3d40-f3a0-4228-92d0-34532be19b05;Trusted_Connection=True;MultipleActiveResultSets=true;Pooling=False"
+    //"OrcanodeMonitorContext": "Server=(localdb)\\mssqllocaldb;Database=OrcanodeMonitorContext-361a3d40-f3a0-4228-92d0-34532be19b05;Trusted_Connection=True;MultipleActiveResultSets=true;Pooling=False"
+    "OrcanodeMonitorContext": "AccountEndpoint=cosmosdburl;AccountKey=accesskeyvalue;"
   }
+
 }