diff --git a/loritta-bot-discord/build.gradle.kts b/loritta-bot-discord/build.gradle.kts index d252eadb21..c493faef60 100644 --- a/loritta-bot-discord/build.gradle.kts +++ b/loritta-bot-discord/build.gradle.kts @@ -66,6 +66,8 @@ dependencies { implementation("io.ktor:ktor-server-caching-headers:${Versions.KTOR}") implementation("io.ktor:ktor-server-sessions:${Versions.KTOR}") implementation("io.ktor:ktor-server-compression:${Versions.KTOR}") + implementation("io.ktor:ktor-server-metrics-micrometer:${Versions.KTOR}") + implementation("io.micrometer:micrometer-registry-prometheus:1.10.3") implementation("com.google.code.gson:gson:2.10.1") implementation("io.pebbletemplates:pebble:3.1.4") diff --git a/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/LorittaBot.kt b/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/LorittaBot.kt index 4b2dc95b1f..165a58f093 100644 --- a/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/LorittaBot.kt +++ b/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/LorittaBot.kt @@ -68,6 +68,7 @@ import net.perfectdreams.loritta.common.locale.LanguageManager import net.perfectdreams.loritta.common.locale.LocaleManager import net.perfectdreams.loritta.common.utils.* import net.perfectdreams.loritta.common.utils.extensions.getPathFromResources +import net.perfectdreams.loritta.morenitta.analytics.LorittaMetrics import net.perfectdreams.loritta.morenitta.analytics.MagicStats import net.perfectdreams.loritta.morenitta.analytics.stats.LorittaStatsCollector import net.perfectdreams.loritta.morenitta.bluesky.LorittaBlueskyRelay @@ -338,6 +339,8 @@ class LorittaBot( val starboardModule = StarboardModule(this) val activityUpdater = ActivityUpdater(this) val loriCoolCardsManager = LoriCoolCardsManager(this.graphicsFonts) + val magicStats = MagicStats(this) + val metrics = LorittaMetrics(this) // Stores if a gateway was successfully resumed during startup val gatewayShardsStartupResumeStatus = ConcurrentHashMap() @@ -529,6 +532,7 @@ class LorittaBot( this, shardManager ) + metrics.registerMetrics() logger.info { "Starting Loritta tasks..." } startTasks() @@ -1204,7 +1208,7 @@ class LorittaBot( GlobalScope.launch(CoroutineName("Create Twitch Subscriptions Loop")) { twitchSubscriptionsHandler.createSubscriptionsLoop() } - scheduleCoroutineAtFixedRate(MagicStats::class.simpleName!!, 15.seconds, action = MagicStats(this)) + scheduleCoroutineAtFixedRate(MagicStats::class.simpleName!!, 15.seconds, action = magicStats) // Update Fan Arts scheduleCoroutineAtFixedRate("GalleryOfDreamsFanArtsUpdater", 1.minutes) { diff --git a/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/LorittaLauncher.kt b/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/LorittaLauncher.kt index cb7478e465..1c515c66e0 100644 --- a/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/LorittaLauncher.kt +++ b/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/LorittaLauncher.kt @@ -10,6 +10,7 @@ import net.perfectdreams.loritta.cinnamon.pudding.Pudding import net.perfectdreams.loritta.common.locale.LocaleManager import net.perfectdreams.loritta.common.locale.LorittaLanguageManager import net.perfectdreams.loritta.common.utils.HostnameUtils +import net.perfectdreams.loritta.morenitta.analytics.LorittaMetrics import net.perfectdreams.loritta.morenitta.utils.config.BaseConfig import net.perfectdreams.loritta.morenitta.utils.devious.DeviousConverter import net.perfectdreams.loritta.morenitta.utils.devious.GatewayExtrasData @@ -102,7 +103,9 @@ object LorittaLauncher { config.loritta.pudding.database, config.loritta.pudding.username, config.loritta.pudding.password - ) + ) { + metricRegistry = LorittaMetrics.appMicrometerRegistry + } services.setupShutdownHook() logger.info { "Started Pudding client!" } diff --git a/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/analytics/LorittaMetrics.kt b/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/analytics/LorittaMetrics.kt new file mode 100644 index 0000000000..cfad861d71 --- /dev/null +++ b/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/analytics/LorittaMetrics.kt @@ -0,0 +1,72 @@ +package net.perfectdreams.loritta.morenitta.analytics + +import io.micrometer.core.instrument.Tag +import io.micrometer.prometheus.PrometheusConfig +import io.micrometer.prometheus.PrometheusMeterRegistry +import net.dv8tion.jda.internal.JDAImpl +import net.perfectdreams.loritta.morenitta.LorittaBot +import net.perfectdreams.loritta.morenitta.utils.devious.GatewayShardStartupResumeStatus +import java.lang.management.ManagementFactory + +class LorittaMetrics(private val loritta: LorittaBot) { + companion object { + val appMicrometerRegistry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT) + } + + fun registerMetrics() { + appMicrometerRegistry.gaugeCollectionSize("loritta.pending_messages", emptyList(), loritta.pendingMessages) + appMicrometerRegistry.gauge("jvm.uptime", ManagementFactory.getRuntimeMXBean()) { + it.uptime.toDouble() + } + + for (shard in loritta.lorittaShards.shardManager.shardCache) { + appMicrometerRegistry.gauge("jda.status", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) { + shard.status.ordinal.toDouble() + } + + appMicrometerRegistry.gauge("loritta.gateway_startup_resume_status", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) { + (loritta.gatewayShardsStartupResumeStatus[it.shardInfo.shardId] ?: GatewayShardStartupResumeStatus.UNKNOWN).ordinal.toDouble() + } + + appMicrometerRegistry.gauge("jda.gateway_ping", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) { + shard.gatewayPing.toDouble() + } + + appMicrometerRegistry.gauge("jda.response_total", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) { + shard.responseTotal.toDouble() + } + + appMicrometerRegistry.gauge(".guilds", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) { + shard.guildCache.size().toDouble() + } + + appMicrometerRegistry.gauge("jda.unavailable_guilds", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) { + (it as JDAImpl).guildSetupController.unavailableGuilds.size().toDouble() + } + + appMicrometerRegistry.gauge("jda.cached_users", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) { + it.userCache.size().toDouble() + } + + appMicrometerRegistry.gauge("jda.cached_channels", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) { + it.channelCache.size().toDouble() + } + + appMicrometerRegistry.gauge("jda.cached_roles", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) { + it.roleCache.size().toDouble() + } + + appMicrometerRegistry.gauge("jda.cached_emojis", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) { + it.emojiCache.size().toDouble() + } + + appMicrometerRegistry.gauge("jda.cached_audiomanagers", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) { + it.audioManagerCache.size().toDouble() + } + + appMicrometerRegistry.gauge("jda.cached_scheduledevents", listOf(Tag.of("shard", shard.shardInfo.shardId.toString())), shard) { + it.scheduledEventCache.size().toDouble() + } + } + } +} \ No newline at end of file diff --git a/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/analytics/MagicStats.kt b/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/analytics/MagicStats.kt index d03cb9a823..1e0426e42b 100644 --- a/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/analytics/MagicStats.kt +++ b/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/analytics/MagicStats.kt @@ -1,37 +1,47 @@ package net.perfectdreams.loritta.morenitta.analytics -import net.dv8tion.jda.internal.JDAImpl +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry import net.perfectdreams.loritta.cinnamon.discord.utils.RunnableCoroutine import net.perfectdreams.loritta.cinnamon.pudding.services.UsersService import net.perfectdreams.loritta.cinnamon.pudding.tables.BoughtStocks import net.perfectdreams.loritta.cinnamon.pudding.tables.Profiles import net.perfectdreams.loritta.cinnamon.pudding.tables.TickerPrices import net.perfectdreams.loritta.cinnamon.pudding.tables.TotalSonhosStats -import net.perfectdreams.loritta.cinnamon.pudding.tables.stats.LorittaClusterStats -import net.perfectdreams.loritta.cinnamon.pudding.tables.stats.LorittaDiscordShardStats import net.perfectdreams.loritta.morenitta.LorittaBot -import org.jetbrains.exposed.sql.* -import java.lang.management.ManagementFactory +import org.jetbrains.exposed.sql.and +import org.jetbrains.exposed.sql.innerJoin +import org.jetbrains.exposed.sql.insert +import org.jetbrains.exposed.sql.sum import java.time.Instant +import java.util.concurrent.atomic.AtomicLong /** * A class that periodically stores stats to the database, "poor man's Prometheus" - * - * The reason I didn't want to use Prometheus is because I wanted to have more control on how I could store the data and stuffz, and because I already have more knowledge on how - * PostgreSQL works compared to Prometheus, so I don't want to lose my stats because Prometheus decided to delete it or something like that. - * - * I also have more control of when the stats should be deleted. - * - * Yeah, one of the bad things is that I can't decide which stats gets deleted (because they are all in the same table) - * - * But well you ALSO CAN'T DO THAT IN PROMETHEUS (you can do that with VictoriaMetrics... Enterprise) */ class MagicStats(val loritta: LorittaBot) : RunnableCoroutine { + private val totalSonhosGauge = LazyGauge( + registry = LorittaMetrics.appMicrometerRegistry, + gaugeName = "loritta.total_sonhos" + ) + private val totalSonhosOfBannedUsersGauge = LazyGauge( + registry = LorittaMetrics.appMicrometerRegistry, + gaugeName = "loritta.total_sonhos_of_banned_users" + ) + private val totalSonhosBrokerGauge = LazyGauge( + registry = LorittaMetrics.appMicrometerRegistry, + gaugeName = "loritta.total_sonhos_broker" + ) + override suspend fun run() { - loritta.transaction { - val now = Instant.now() + class Result( + val totalSonhos: Long, + val totalSonhosOfBannedUsers: Long, + val totalSonhosBroker: Long + ) - if (loritta.isMainInstance) { + if (loritta.isMainInstance) { + val result = loritta.transaction { val sumField = Profiles.money.sum() val totalSonhos = Profiles.select(sumField) .where { @@ -57,48 +67,47 @@ class MagicStats(val loritta: LorittaBot) : RunnableCoroutine { it[TotalSonhosStats.totalSonhosOfBannedUsers] = totalSonhosOfBannedUsers it[TotalSonhosStats.totalSonhosBroker] = totalSonhosBroker } + + return@transaction Result(totalSonhos, totalSonhosOfBannedUsers, totalSonhosBroker) } - val mb = 1024 * 1024 - val runtime = Runtime.getRuntime() - val freeMemory = runtime.freeMemory() / mb - val maxMemory = runtime.maxMemory() / mb - val totalMemory = runtime.totalMemory() / mb + totalSonhosGauge.setValue(result.totalSonhos) + totalSonhosOfBannedUsersGauge.setValue(result.totalSonhosOfBannedUsers) + totalSonhosBrokerGauge.setValue(result.totalSonhosBroker) + } + } + + data class LazyGauge( + private val registry: MeterRegistry, + private val gaugeName: String + ) { + private val atomicLong = AtomicLong() - LorittaClusterStats.insert { - it[LorittaClusterStats.timestamp] = now - it[LorittaClusterStats.clusterId] = loritta.clusterId - it[LorittaClusterStats.pendingMessagesCount] = loritta.pendingMessages.size - it[LorittaClusterStats.freeMemory] = freeMemory - it[LorittaClusterStats.maxMemory] = maxMemory - it[LorittaClusterStats.totalMemory] = totalMemory - it[LorittaClusterStats.threadCount] = ManagementFactory.getThreadMXBean().threadCount - it[LorittaClusterStats.uptime] = ManagementFactory.getRuntimeMXBean().uptime - it[LorittaClusterStats.puddingIdleConnections] = loritta.pudding.hikariDataSource.hikariPoolMXBean.idleConnections - it[LorittaClusterStats.puddingActiveConnections] = loritta.pudding.hikariDataSource.hikariPoolMXBean.activeConnections - it[LorittaClusterStats.puddingTotalConnections] = loritta.pudding.hikariDataSource.hikariPoolMXBean.totalConnections - } + // Flag to ensure that gauge is registered only once + @Volatile + private var isGaugeCreated = false - val shardManager = loritta.lorittaShards.shardManager - LorittaDiscordShardStats.batchInsert(shardManager.shardCache, shouldReturnGeneratedValues = false) { - this[LorittaDiscordShardStats.timestamp] = now - this[LorittaDiscordShardStats.clusterId] = loritta.clusterId - this[LorittaDiscordShardStats.shardId] = it.shardInfo.shardId - this[LorittaDiscordShardStats.status] = it.status.ordinal - this[LorittaDiscordShardStats.gatewayStartupResumeStatus] = loritta.gatewayShardsStartupResumeStatus[it.shardInfo.shardId]?.ordinal - this[LorittaDiscordShardStats.gatewayPing] = it.gatewayPing - this[LorittaDiscordShardStats.responseTotal] = it.responseTotal - this[LorittaDiscordShardStats.guildsCount] = it.guildCache.size() - this[LorittaDiscordShardStats.cachedUsersCount] = it.userCache.size() - this[LorittaDiscordShardStats.cachedChannelsCount] = it.channelCache.size() - this[LorittaDiscordShardStats.cachedRolesCount] = it.roleCache.size() - this[LorittaDiscordShardStats.cachedEmojisCount] = it.emojiCache.size() - this[LorittaDiscordShardStats.cachedAudioManagerCount] = it.audioManagerCache.size() - this[LorittaDiscordShardStats.cachedScheduledEventsCount] = it.scheduledEventCache.size() + // Function to set the value to AtomicLong and lazily create a gauge + fun setValue(value: Long) { + atomicLong.set(value) - // Stuff that requires casting - this[LorittaDiscordShardStats.unavailableGuildsCount] = (it as JDAImpl).guildSetupController.unavailableGuilds.size().toLong() + if (!isGaugeCreated) { + synchronized(this) { + if (!isGaugeCreated) { + createGauge() + isGaugeCreated = true + } + } } } + + // The actual gauge creation logic + private fun createGauge() { + Gauge.builder(gaugeName) { atomicLong.get().toDouble() } + .register(registry) + } + + // Getter for AtomicLong value + fun getValue(): Long = atomicLong.get() } } \ No newline at end of file diff --git a/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/bluesky/LorittaBlueskyRelay.kt b/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/bluesky/LorittaBlueskyRelay.kt index 35bf595501..6a415aca0b 100644 --- a/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/bluesky/LorittaBlueskyRelay.kt +++ b/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/bluesky/LorittaBlueskyRelay.kt @@ -6,11 +6,13 @@ import kotlinx.coroutines.withTimeout import mu.KotlinLogging import net.perfectdreams.loritta.cinnamon.pudding.tables.servers.moduleconfigs.TrackedBlueskyAccounts import net.perfectdreams.loritta.morenitta.LorittaBot +import net.perfectdreams.loritta.morenitta.analytics.LorittaMetrics import net.perfectdreams.loritta.morenitta.utils.DiscordUtils import net.perfectdreams.loritta.serializable.internal.requests.LorittaInternalRPCRequest import net.perfectdreams.loritta.serializable.internal.responses.LorittaInternalRPCResponse import net.perfectdreams.yokye.BlueskyFirehoseClient import org.jetbrains.exposed.sql.selectAll +import java.util.concurrent.atomic.AtomicLong import kotlin.time.Duration.Companion.seconds class LorittaBlueskyRelay(val loritta: LorittaBot) { @@ -20,6 +22,7 @@ class LorittaBlueskyRelay(val loritta: LorittaBot) { val firehoseClient = BlueskyFirehoseClient() val postStream = firehoseClient.postStream + private val postsReceivedGauge = LorittaMetrics.appMicrometerRegistry.gauge("loritta.bluesky.posts_received", AtomicLong(0)) suspend fun startRelay() { firehoseClient.connect() @@ -34,6 +37,7 @@ class LorittaBlueskyRelay(val loritta: LorittaBot) { // We chunk due to limits on IN queries posts.chunked(65_535) .forEach { chunkPosts -> + postsReceivedGauge.incrementAndGet() val repos = chunkPosts.map { it.repo } // To avoid spamming all Loritta instances with useless posts, we'll do a "pre-filter" here to know if we need to relay something or not diff --git a/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/website/LorittaWebsite.kt b/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/website/LorittaWebsite.kt index 0810b4e68e..26aee4bcbb 100644 --- a/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/website/LorittaWebsite.kt +++ b/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/website/LorittaWebsite.kt @@ -11,6 +11,7 @@ import io.ktor.server.application.* import io.ktor.server.cio.* import io.ktor.server.engine.* import io.ktor.server.http.content.* +import io.ktor.server.metrics.micrometer.* import io.ktor.server.plugins.cachingheaders.* import io.ktor.server.plugins.compression.* import io.ktor.server.plugins.statuspages.* @@ -26,6 +27,7 @@ import kotlinx.html.* import kotlinx.html.stream.appendHTML import mu.KotlinLogging import net.perfectdreams.loritta.morenitta.LorittaBot +import net.perfectdreams.loritta.morenitta.analytics.LorittaMetrics import net.perfectdreams.loritta.morenitta.website.routes.LocalizedRoute import net.perfectdreams.loritta.morenitta.website.rpc.processors.Processors import net.perfectdreams.loritta.morenitta.website.utils.SVGIconManager @@ -270,6 +272,11 @@ class LorittaWebsite( install(Compression) + install(MicrometerMetrics) { + metricName = "lorittawebserver.ktor.http.server.requests" + registry = LorittaMetrics.appMicrometerRegistry + } + routing { static { staticRootFolder = File("${config.websiteFolder}/static/") diff --git a/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/websiteinternal/InternalWebServer.kt b/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/websiteinternal/InternalWebServer.kt index 6c36a0a625..963b7a1e32 100644 --- a/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/websiteinternal/InternalWebServer.kt +++ b/loritta-bot-discord/src/main/kotlin/net/perfectdreams/loritta/morenitta/websiteinternal/InternalWebServer.kt @@ -4,6 +4,7 @@ import io.ktor.http.* import io.ktor.server.application.* import io.ktor.server.cio.* import io.ktor.server.engine.* +import io.ktor.server.metrics.micrometer.* import io.ktor.server.plugins.compression.* import io.ktor.server.plugins.statuspages.* import io.ktor.server.request.* @@ -24,6 +25,7 @@ import net.perfectdreams.loritta.common.utils.placeholders.BlueskyPostMessagePla import net.perfectdreams.loritta.common.utils.placeholders.TwitchStreamOnlineMessagePlaceholders import net.perfectdreams.loritta.i18n.I18nKeysData import net.perfectdreams.loritta.morenitta.LorittaBot +import net.perfectdreams.loritta.morenitta.analytics.LorittaMetrics import net.perfectdreams.loritta.morenitta.utils.MessageUtils import net.perfectdreams.loritta.morenitta.utils.PendingUpdate import net.perfectdreams.loritta.morenitta.utils.escapeMentions @@ -89,6 +91,11 @@ class InternalWebServer(val m: LorittaBot) { } } + install(MicrometerMetrics) { + metricName = "internalwebserver.ktor.http.server.requests" + registry = LorittaMetrics.appMicrometerRegistry + } + routing { post("/rpc") { val body = withContext(Dispatchers.IO) { call.receiveText() } @@ -108,6 +115,10 @@ class InternalWebServer(val m: LorittaBot) { call.respondText(os.toString(Charsets.UTF_8)) } + get("/metrics") { + call.respond(LorittaMetrics.appMicrometerRegistry.scrape()) + } + // Dumps all pending messages on the event queue get("/pending-messages") { val coroutinesInfo = DebugProbes.dumpCoroutinesInfo()