Export Prometheus metrics

loritta-bot-discord/build.gradle.kts

@@ -66,6 +66,8 @@ dependencies {
     implementation("io.ktor:ktor-server-caching-headers:${Versions.KTOR}")
     implementation("io.ktor:ktor-server-sessions:${Versions.KTOR}")
     implementation("io.ktor:ktor-server-compression:${Versions.KTOR}")
+    implementation("io.ktor:ktor-server-metrics-micrometer:${Versions.KTOR}")
+    implementation("io.micrometer:micrometer-registry-prometheus:1.10.3")
 
     implementation("com.google.code.gson:gson:2.10.1")
     implementation("io.pebbletemplates:pebble:3.1.4")

loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/LorittaBot.kt

@@ -68,6 +68,7 @@ import net.perfectdreams.loritta.common.locale.LanguageManager
 import net.perfectdreams.loritta.common.locale.LocaleManager
 import net.perfectdreams.loritta.common.utils.*
 import net.perfectdreams.loritta.common.utils.extensions.getPathFromResources
+import net.perfectdreams.loritta.morenitta.analytics.LorittaMetrics
 import net.perfectdreams.loritta.morenitta.analytics.MagicStats
 import net.perfectdreams.loritta.morenitta.analytics.stats.LorittaStatsCollector
 import net.perfectdreams.loritta.morenitta.bluesky.LorittaBlueskyRelay
@@ -338,6 +339,8 @@ class LorittaBot(
 	val starboardModule = StarboardModule(this)
 	val activityUpdater = ActivityUpdater(this)
 	val loriCoolCardsManager = LoriCoolCardsManager(this.graphicsFonts)
+	val magicStats = MagicStats(this)
+	val metrics = LorittaMetrics(this)
 
 	// Stores if a gateway was successfully resumed during startup
 	val gatewayShardsStartupResumeStatus = ConcurrentHashMap<Int, GatewayShardStartupResumeStatus>()
@@ -529,6 +532,7 @@ class LorittaBot(
 			this,
 			shardManager
 		)
+		metrics.registerMetrics()
 
 		logger.info { "Starting Loritta tasks..." }
 		startTasks()
@@ -1204,7 +1208,7 @@ class LorittaBot(
 		GlobalScope.launch(CoroutineName("Create Twitch Subscriptions Loop")) {
 			twitchSubscriptionsHandler.createSubscriptionsLoop()
 		}
-		scheduleCoroutineAtFixedRate(MagicStats::class.simpleName!!, 15.seconds, action = MagicStats(this))
+		scheduleCoroutineAtFixedRate(MagicStats::class.simpleName!!, 15.seconds, action = magicStats)
 
 		// Update Fan Arts
 		scheduleCoroutineAtFixedRate("GalleryOfDreamsFanArtsUpdater", 1.minutes) {

loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/LorittaLauncher.kt

@@ -10,6 +10,7 @@ import net.perfectdreams.loritta.cinnamon.pudding.Pudding
 import net.perfectdreams.loritta.common.locale.LocaleManager
 import net.perfectdreams.loritta.common.locale.LorittaLanguageManager
 import net.perfectdreams.loritta.common.utils.HostnameUtils
+import net.perfectdreams.loritta.morenitta.analytics.LorittaMetrics
 import net.perfectdreams.loritta.morenitta.utils.config.BaseConfig
 import net.perfectdreams.loritta.morenitta.utils.devious.DeviousConverter
 import net.perfectdreams.loritta.morenitta.utils.devious.GatewayExtrasData
@@ -102,7 +103,9 @@ object LorittaLauncher {
 			config.loritta.pudding.database,
 			config.loritta.pudding.username,
 			config.loritta.pudding.password
-		)
+		) {
+			metricRegistry = LorittaMetrics.appMicrometerRegistry
+		}
 		services.setupShutdownHook()
 
 		logger.info { "Started Pudding client!" }

loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/analytics/LorittaMetrics.kt

@@ -0,0 +1,72 @@
+package net.perfectdreams.loritta.morenitta.analytics
+
+import io.micrometer.core.instrument.Tag
+import io.micrometer.prometheus.PrometheusConfig
+import io.micrometer.prometheus.PrometheusMeterRegistry
+import net.dv8tion.jda.internal.JDAImpl
+import net.perfectdreams.loritta.morenitta.LorittaBot
+import net.perfectdreams.loritta.morenitta.utils.devious.GatewayShardStartupResumeStatus
+import java.lang.management.ManagementFactory
+
+class LorittaMetrics(private val loritta: LorittaBot) {
+    companion object {
+        val appMicrometerRegistry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT)
+    }
+
+    fun registerMetrics() {
+        appMicrometerRegistry.gaugeCollectionSize("loritta.pending_messages", emptyList(), loritta.pendingMessages)
+        appMicrometerRegistry.gauge("jvm.uptime", ManagementFactory.getRuntimeMXBean()) {
+            it.uptime.toDouble()
+        }
+
+        for (shard in loritta.lorittaShards.shardManager.shardCache) {
+            appMicrometerRegistry.gauge("jda.status", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) {
+                shard.status.ordinal.toDouble()
+            }
+
+            appMicrometerRegistry.gauge("loritta.gateway_startup_resume_status", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) {
+                (loritta.gatewayShardsStartupResumeStatus[it.shardInfo.shardId] ?: GatewayShardStartupResumeStatus.UNKNOWN).ordinal.toDouble()
+            }
+
+            appMicrometerRegistry.gauge("jda.gateway_ping", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) {
+                shard.gatewayPing.toDouble()
+            }
+
+            appMicrometerRegistry.gauge("jda.response_total", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) {
+                shard.responseTotal.toDouble()
+            }
+
+            appMicrometerRegistry.gauge(".guilds", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) {
+                shard.guildCache.size().toDouble()
+            }
+
+            appMicrometerRegistry.gauge("jda.unavailable_guilds", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) {
+                (it as JDAImpl).guildSetupController.unavailableGuilds.size().toDouble()
+            }
+
+            appMicrometerRegistry.gauge("jda.cached_users", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) {
+                it.userCache.size().toDouble()
+            }
+
+            appMicrometerRegistry.gauge("jda.cached_channels", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) {
+                it.channelCache.size().toDouble()
+            }
+
+            appMicrometerRegistry.gauge("jda.cached_roles", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) {
+                it.roleCache.size().toDouble()
+            }
+
+            appMicrometerRegistry.gauge("jda.cached_emojis", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) {
+                it.emojiCache.size().toDouble()
+            }
+
+            appMicrometerRegistry.gauge("jda.cached_audiomanagers", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) {
+                it.audioManagerCache.size().toDouble()
+            }
+
+            appMicrometerRegistry.gauge("jda.cached_scheduledevents", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) {
+                it.scheduledEventCache.size().toDouble()
+            }
+        }
+    }
+}

loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/analytics/MagicStats.kt

@@ -1,37 +1,47 @@
 package net.perfectdreams.loritta.morenitta.analytics
 
-import net.dv8tion.jda.internal.JDAImpl
+import io.micrometer.core.instrument.Gauge
+import io.micrometer.core.instrument.MeterRegistry
 import net.perfectdreams.loritta.cinnamon.discord.utils.RunnableCoroutine
 import net.perfectdreams.loritta.cinnamon.pudding.services.UsersService
 import net.perfectdreams.loritta.cinnamon.pudding.tables.BoughtStocks
 import net.perfectdreams.loritta.cinnamon.pudding.tables.Profiles
 import net.perfectdreams.loritta.cinnamon.pudding.tables.TickerPrices
 import net.perfectdreams.loritta.cinnamon.pudding.tables.TotalSonhosStats
-import net.perfectdreams.loritta.cinnamon.pudding.tables.stats.LorittaClusterStats
-import net.perfectdreams.loritta.cinnamon.pudding.tables.stats.LorittaDiscordShardStats
 import net.perfectdreams.loritta.morenitta.LorittaBot
-import org.jetbrains.exposed.sql.*
-import java.lang.management.ManagementFactory
+import org.jetbrains.exposed.sql.and
+import org.jetbrains.exposed.sql.innerJoin
+import org.jetbrains.exposed.sql.insert
+import org.jetbrains.exposed.sql.sum
 import java.time.Instant
+import java.util.concurrent.atomic.AtomicLong
 
 /**
  * A class that periodically stores stats to the database, "poor man's Prometheus"
- *
- * The reason I didn't want to use Prometheus is because I wanted to have more control on how I could store the data and stuffz, and because I already have more knowledge on how
- * PostgreSQL works compared to Prometheus, so I don't want to lose my stats because Prometheus decided to delete it or something like that.
- *
- * I also have more control of when the stats should be deleted.
- *
- * Yeah, one of the bad things is that I can't decide which stats gets deleted (because they are all in the same table)
- *
- * But well you ALSO CAN'T DO THAT IN PROMETHEUS (you can do that with VictoriaMetrics... Enterprise)
  */
 class MagicStats(val loritta: LorittaBot) : RunnableCoroutine {
+    private val totalSonhosGauge = LazyGauge(
+        registry = LorittaMetrics.appMicrometerRegistry,
+        gaugeName = "loritta.total_sonhos"
+    )
+    private val totalSonhosOfBannedUsersGauge = LazyGauge(
+        registry = LorittaMetrics.appMicrometerRegistry,
+        gaugeName = "loritta.total_sonhos_of_banned_users"
+    )
+    private val totalSonhosBrokerGauge = LazyGauge(
+        registry = LorittaMetrics.appMicrometerRegistry,
+        gaugeName = "loritta.total_sonhos_broker"
+    )
+
     override suspend fun run() {
-        loritta.transaction {
-            val now = Instant.now()
+        class Result(
+            val totalSonhos: Long,
+            val totalSonhosOfBannedUsers: Long,
+            val totalSonhosBroker: Long
+        )
 
-            if (loritta.isMainInstance) {
+        if (loritta.isMainInstance) {
+            val result = loritta.transaction {
                 val sumField = Profiles.money.sum()
                 val totalSonhos = Profiles.select(sumField)
                     .where {
@@ -57,48 +67,47 @@ class MagicStats(val loritta: LorittaBot) : RunnableCoroutine {
                     it[TotalSonhosStats.totalSonhosOfBannedUsers] = totalSonhosOfBannedUsers
                     it[TotalSonhosStats.totalSonhosBroker] = totalSonhosBroker
                 }
+
+                return@transaction Result(totalSonhos, totalSonhosOfBannedUsers, totalSonhosBroker)
             }
 
-            val mb = 1024 * 1024
-            val runtime = Runtime.getRuntime()
-            val freeMemory = runtime.freeMemory() / mb
-            val maxMemory = runtime.maxMemory() / mb
-            val totalMemory = runtime.totalMemory() / mb
+            totalSonhosGauge.setValue(result.totalSonhos)
+            totalSonhosOfBannedUsersGauge.setValue(result.totalSonhosOfBannedUsers)
+            totalSonhosBrokerGauge.setValue(result.totalSonhosBroker)
+        }
+    }
+
+    data class LazyGauge(
+        private val registry: MeterRegistry,
+        private val gaugeName: String
+    ) {
+        private val atomicLong = AtomicLong()
 
-            LorittaClusterStats.insert {
-                it[LorittaClusterStats.timestamp] = now
-                it[LorittaClusterStats.clusterId] = loritta.clusterId
-                it[LorittaClusterStats.pendingMessagesCount] = loritta.pendingMessages.size
-                it[LorittaClusterStats.freeMemory] = freeMemory
-                it[LorittaClusterStats.maxMemory] = maxMemory
-                it[LorittaClusterStats.totalMemory] = totalMemory
-                it[LorittaClusterStats.threadCount] = ManagementFactory.getThreadMXBean().threadCount
-                it[LorittaClusterStats.uptime] = ManagementFactory.getRuntimeMXBean().uptime
-                it[LorittaClusterStats.puddingIdleConnections] = loritta.pudding.hikariDataSource.hikariPoolMXBean.idleConnections
-                it[LorittaClusterStats.puddingActiveConnections] = loritta.pudding.hikariDataSource.hikariPoolMXBean.activeConnections
-                it[LorittaClusterStats.puddingTotalConnections] = loritta.pudding.hikariDataSource.hikariPoolMXBean.totalConnections
-            }
+        // Flag to ensure that gauge is registered only once
+        @Volatile
+        private var isGaugeCreated = false
 
-            val shardManager = loritta.lorittaShards.shardManager
-            LorittaDiscordShardStats.batchInsert(shardManager.shardCache, shouldReturnGeneratedValues = false) {
-                this[LorittaDiscordShardStats.timestamp] = now
-                this[LorittaDiscordShardStats.clusterId] = loritta.clusterId
-                this[LorittaDiscordShardStats.shardId] = it.shardInfo.shardId
-                this[LorittaDiscordShardStats.status] = it.status.ordinal
-                this[LorittaDiscordShardStats.gatewayStartupResumeStatus] = loritta.gatewayShardsStartupResumeStatus[it.shardInfo.shardId]?.ordinal
-                this[LorittaDiscordShardStats.gatewayPing] = it.gatewayPing
-                this[LorittaDiscordShardStats.responseTotal] = it.responseTotal
-                this[LorittaDiscordShardStats.guildsCount] = it.guildCache.size()
-                this[LorittaDiscordShardStats.cachedUsersCount] = it.userCache.size()
-                this[LorittaDiscordShardStats.cachedChannelsCount] = it.channelCache.size()
-                this[LorittaDiscordShardStats.cachedRolesCount] = it.roleCache.size()
-                this[LorittaDiscordShardStats.cachedEmojisCount] = it.emojiCache.size()
-                this[LorittaDiscordShardStats.cachedAudioManagerCount] = it.audioManagerCache.size()
-                this[LorittaDiscordShardStats.cachedScheduledEventsCount] = it.scheduledEventCache.size()
+        // Function to set the value to AtomicLong and lazily create a gauge
+        fun setValue(value: Long) {
+            atomicLong.set(value)
 
-                // Stuff that requires casting
-                this[LorittaDiscordShardStats.unavailableGuildsCount] = (it as JDAImpl).guildSetupController.unavailableGuilds.size().toLong()
+            if (!isGaugeCreated) {
+                synchronized(this) {
+                    if (!isGaugeCreated) {
+                        createGauge()
+                        isGaugeCreated = true
+                    }
+                }
             }
         }
+
+        // The actual gauge creation logic
+        private fun createGauge() {
+            Gauge.builder(gaugeName) { atomicLong.get().toDouble() }
+                .register(registry)
+        }
+
+        // Getter for AtomicLong value
+        fun getValue(): Long = atomicLong.get()
     }
 }

loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/bluesky/LorittaBlueskyRelay.kt

@@ -6,11 +6,13 @@ import kotlinx.coroutines.withTimeout
 import mu.KotlinLogging
 import net.perfectdreams.loritta.cinnamon.pudding.tables.servers.moduleconfigs.TrackedBlueskyAccounts
 import net.perfectdreams.loritta.morenitta.LorittaBot
+import net.perfectdreams.loritta.morenitta.analytics.LorittaMetrics
 import net.perfectdreams.loritta.morenitta.utils.DiscordUtils
 import net.perfectdreams.loritta.serializable.internal.requests.LorittaInternalRPCRequest
 import net.perfectdreams.loritta.serializable.internal.responses.LorittaInternalRPCResponse
 import net.perfectdreams.yokye.BlueskyFirehoseClient
 import org.jetbrains.exposed.sql.selectAll
+import java.util.concurrent.atomic.AtomicLong
 import kotlin.time.Duration.Companion.seconds
 
 class LorittaBlueskyRelay(val loritta: LorittaBot) {
@@ -20,6 +22,7 @@ class LorittaBlueskyRelay(val loritta: LorittaBot) {
 
     val firehoseClient = BlueskyFirehoseClient()
     val postStream = firehoseClient.postStream
+    private val postsReceivedGauge = LorittaMetrics.appMicrometerRegistry.gauge("loritta.bluesky.posts_received", AtomicLong(0))
 
     suspend fun startRelay() {
         firehoseClient.connect()
@@ -34,6 +37,7 @@ class LorittaBlueskyRelay(val loritta: LorittaBot) {
                 // We chunk due to limits on IN queries
                 posts.chunked(65_535)
                     .forEach { chunkPosts ->
+                        postsReceivedGauge.incrementAndGet()
                         val repos = chunkPosts.map { it.repo }
 
                         // To avoid spamming all Loritta instances with useless posts, we'll do a "pre-filter" here to know if we need to relay something or not

loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/website/LorittaWebsite.kt

@@ -11,6 +11,7 @@ import io.ktor.server.application.*
 import io.ktor.server.cio.*
 import io.ktor.server.engine.*
 import io.ktor.server.http.content.*
+import io.ktor.server.metrics.micrometer.*
 import io.ktor.server.plugins.cachingheaders.*
 import io.ktor.server.plugins.compression.*
 import io.ktor.server.plugins.statuspages.*
@@ -26,6 +27,7 @@ import kotlinx.html.*
 import kotlinx.html.stream.appendHTML
 import mu.KotlinLogging
 import net.perfectdreams.loritta.morenitta.LorittaBot
+import net.perfectdreams.loritta.morenitta.analytics.LorittaMetrics
 import net.perfectdreams.loritta.morenitta.website.routes.LocalizedRoute
 import net.perfectdreams.loritta.morenitta.website.rpc.processors.Processors
 import net.perfectdreams.loritta.morenitta.website.utils.SVGIconManager
@@ -270,6 +272,11 @@ class LorittaWebsite(
 
 			install(Compression)
 
+			install(MicrometerMetrics) {
+				metricName = "lorittawebserver.ktor.http.server.requests"
+				registry = LorittaMetrics.appMicrometerRegistry
+			}
+
 			routing {
 				static {
 					staticRootFolder = File("${config.websiteFolder}/static/")

loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/websiteinternal/InternalWebServer.kt

@@ -4,6 +4,7 @@ import io.ktor.http.*
 import io.ktor.server.application.*
 import io.ktor.server.cio.*
 import io.ktor.server.engine.*
+import io.ktor.server.metrics.micrometer.*
 import io.ktor.server.plugins.compression.*
 import io.ktor.server.plugins.statuspages.*
 import io.ktor.server.request.*
@@ -24,6 +25,7 @@ import net.perfectdreams.loritta.common.utils.placeholders.BlueskyPostMessagePla
 import net.perfectdreams.loritta.common.utils.placeholders.TwitchStreamOnlineMessagePlaceholders
 import net.perfectdreams.loritta.i18n.I18nKeysData
 import net.perfectdreams.loritta.morenitta.LorittaBot
+import net.perfectdreams.loritta.morenitta.analytics.LorittaMetrics
 import net.perfectdreams.loritta.morenitta.utils.MessageUtils
 import net.perfectdreams.loritta.morenitta.utils.PendingUpdate
 import net.perfectdreams.loritta.morenitta.utils.escapeMentions
@@ -89,6 +91,11 @@ class InternalWebServer(val m: LorittaBot) {
                 }
             }
 
+            install(MicrometerMetrics) {
+                metricName = "internalwebserver.ktor.http.server.requests"
+                registry = LorittaMetrics.appMicrometerRegistry
+            }
+
             routing {
                 post("/rpc") {
                     val body = withContext(Dispatchers.IO) { call.receiveText() }
@@ -108,6 +115,10 @@ class InternalWebServer(val m: LorittaBot) {
                     call.respondText(os.toString(Charsets.UTF_8))
                 }
 
+                get("/metrics") {
+                    call.respond(LorittaMetrics.appMicrometerRegistry.scrape())
+                }
+
                 // Dumps all pending messages on the event queue
                 get("/pending-messages") {
                     val coroutinesInfo = DebugProbes.dumpCoroutinesInfo()