diff --git a/loricoolcards-generator/src/main/kotlin/net/perfectdreams/loritta/loricoolcards/generator/DailyCheckTest.kt b/loricoolcards-generator/src/main/kotlin/net/perfectdreams/loritta/loricoolcards/generator/DailyCheckTest.kt new file mode 100644 index 0000000000..5b030c0286 --- /dev/null +++ b/loricoolcards-generator/src/main/kotlin/net/perfectdreams/loritta/loricoolcards/generator/DailyCheckTest.kt @@ -0,0 +1,82 @@ +package net.perfectdreams.loritta.loricoolcards.generator + +import io.ktor.client.* +import net.perfectdreams.loritta.cinnamon.pudding.Pudding +import net.perfectdreams.loritta.cinnamon.pudding.services.UsersService.Companion.validBannedUsersList +import net.perfectdreams.loritta.cinnamon.pudding.tables.BannedUsers +import net.perfectdreams.loritta.cinnamon.pudding.tables.BrowserFingerprints +import net.perfectdreams.loritta.cinnamon.pudding.tables.Dailies +import net.perfectdreams.loritta.cinnamon.pudding.tables.Profiles +import net.perfectdreams.loritta.loricoolcards.generator.utils.config.LoriCoolCardsGeneratorProductionStickersConfig +import net.perfectdreams.loritta.morenitta.utils.readConfigurationFromFile +import org.jetbrains.exposed.sql.and +import org.jetbrains.exposed.sql.innerJoin +import org.jetbrains.exposed.sql.selectAll +import java.io.File +import java.sql.Connection +import java.time.Instant + +suspend fun main() { + val http = HttpClient {} + + val configurationFile = File(System.getProperty("conf") ?: "./loricoolcards-production-stickers-generator.conf") + + if (!configurationFile.exists()) { + println("Missing configuration file!") + System.exit(1) + return + } + + val config = readConfigurationFromFile(configurationFile) + + val pudding = Pudding.createPostgreSQLPudding( + config.pudding.address, + config.pudding.database, + config.pudding.username, + config.pudding.password + ) + + + + pudding.transaction(transactionIsolation = Connection.TRANSACTION_READ_UNCOMMITTED) { + val now = Instant.now() + .minusSeconds(604_800) // 7 days + + val dailiesRecentlyRetrievedHours = Dailies + .innerJoin(BrowserFingerprints) + .innerJoin(Profiles, { Profiles.id }, { Dailies.receivedById }) + .selectAll() + .where { + Dailies.receivedAt greaterEq now.toEpochMilli() and (Dailies.receivedById notInSubQuery validBannedUsersList(now.toEpochMilli())) + } + .toList() + + val allClientIds = dailiesRecentlyRetrievedHours.map { it[BrowserFingerprints.clientId] } + val alreadyChecked = mutableSetOf() + + allClientIds.chunked(10_000).forEach { clientIds -> + val clientIdsThatAreBanned = Dailies + .innerJoin(BrowserFingerprints) + .innerJoin(BannedUsers, { Dailies.receivedById }, { BannedUsers.userId }) + .selectAll() + .where { + BrowserFingerprints.clientId inList clientIds and (BannedUsers.userId inSubQuery validBannedUsersList(now.toEpochMilli())) + } + .toList() + + for (user in dailiesRecentlyRetrievedHours) { + if (user[Dailies.receivedById] in alreadyChecked) + continue + + val bannedUsersAssociatedWithThisUser = clientIdsThatAreBanned.filter { it[BrowserFingerprints.clientId] == user[BrowserFingerprints.clientId] } + + if (bannedUsersAssociatedWithThisUser.isNotEmpty()) { + println("ban ${user[Dailies.receivedById]}") + alreadyChecked.add(user[Dailies.receivedById]) + } + } + } + + println("Finished! $alreadyChecked") + } +} \ No newline at end of file diff --git a/loricoolcards-generator/src/main/kotlin/net/perfectdreams/loritta/loricoolcards/generator/GrepSorter.kt b/loricoolcards-generator/src/main/kotlin/net/perfectdreams/loritta/loricoolcards/generator/GrepSorter.kt new file mode 100644 index 0000000000..5d8aca43fd --- /dev/null +++ b/loricoolcards-generator/src/main/kotlin/net/perfectdreams/loritta/loricoolcards/generator/GrepSorter.kt @@ -0,0 +1,89 @@ +package net.perfectdreams.loritta.loricoolcards.generator + +import java.io.File +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +data class LogEntry(val timestamp: LocalDateTime, val content: String) + +fun parseTimestamp(logEntry: String): LocalDateTime { + // Extract the timestamp part from the log entry using regex or string manipulation + val timestampRegex = Regex("""\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}""") + val timestampString = timestampRegex.find(logEntry)?.value + ?: throw IllegalArgumentException("No valid timestamp found in the log entry.") + + // Define the formatter to match the log's timestamp format + val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss,SSS") + + // Parse the timestamp string to LocalDateTime + return LocalDateTime.parse(timestampString, formatter) +} + +fun processLogFile(file: File): List { + // Read the entire file content + val content = file.readText() + + // Split log entries by "--" + val logEntries = content.split("--").filter { it.isNotBlank() } + + // Map log entries to LogEntry objects, parsing the timestamp + return logEntries.map { logEntry -> + LogEntry(timestamp = parseTimestamp(logEntry), content = logEntry.trim()) + }.sortedBy { it.timestamp } +} + +fun main() { + // List of files to process + val logFiles = listOf( + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-1.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-2.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-3.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-4.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-5.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-6.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-7.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-8.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-9.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-10.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-11.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-12.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-13.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-14.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-15.log"), + File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\webhooks-16.log"), + ) + + // Process each file and sort the entries by timestamp + val outputLogFile = File("C:\\Users\\leona\\Documents\\LorittaInteractionIssues\\output.log") + val sortedEntries = mutableListOf() + for (logFile in logFiles) { + println("Processing file: ${logFile.name}") + + // Get sorted log entries + try { + sortedEntries.addAll(processLogFile(logFile)) + } catch (e: Exception) { + e.printStackTrace() + } + } + + sortedEntries.groupBy { it.timestamp.hour.toString().padStart(2, '0') + ":" + it.timestamp.minute.toString().padStart(2, '0') } + .toList() + .sortedBy { it.first } + .forEach { (t, u) -> + println("$t: ${u.size}") + } + + + /* sortedEntries.filter { it.content.contains("COMBO") }.groupBy { it.timestamp.hour.toString().padStart(2, '0') + ":" + it.timestamp.minute.toString().padStart(2, '0') } + .toList() + .sortedBy { it.first } + .forEach { (t, u) -> + println("$t: ${u.size}") + } */ + + // Print sorted log entries + for (entry in sortedEntries.sortedBy { it.timestamp }) { + // outputLogFile.appendText("--\n${entry.content}\n--") + } +} \ No newline at end of file diff --git a/loricoolcards-generator/src/main/kotlin/net/perfectdreams/loritta/loricoolcards/generator/QueryTest.kt b/loricoolcards-generator/src/main/kotlin/net/perfectdreams/loritta/loricoolcards/generator/QueryTest.kt index 353709227e..a772893890 100644 --- a/loricoolcards-generator/src/main/kotlin/net/perfectdreams/loritta/loricoolcards/generator/QueryTest.kt +++ b/loricoolcards-generator/src/main/kotlin/net/perfectdreams/loritta/loricoolcards/generator/QueryTest.kt @@ -304,6 +304,7 @@ suspend fun main() { is StoredGarticosTransferTransaction -> TODO() is StoredLorittaItemShopBoughtBackgroundTransaction -> TODO() is StoredLorittaItemShopBoughtProfileDesignTransaction -> TODO() + is StoredReactionEventSonhosTransaction -> TODO() } } } diff --git a/loricoolcards-generator/src/main/kotlin/net/perfectdreams/loritta/loricoolcards/generator/VMTest.kt b/loricoolcards-generator/src/main/kotlin/net/perfectdreams/loritta/loricoolcards/generator/VMTest.kt new file mode 100644 index 0000000000..cb4430a5f7 --- /dev/null +++ b/loricoolcards-generator/src/main/kotlin/net/perfectdreams/loritta/loricoolcards/generator/VMTest.kt @@ -0,0 +1,358 @@ +package net.perfectdreams.loritta.loricoolcards.generator + +import io.ktor.client.* +import io.ktor.client.request.* +import kotlinx.coroutines.* +import kotlinx.serialization.json.* +import net.perfectdreams.loritta.cinnamon.pudding.Pudding +import net.perfectdreams.loritta.cinnamon.pudding.tables.stats.LorittaDiscordShardStats +import net.perfectdreams.loritta.loricoolcards.generator.utils.config.LoriCoolCardsGeneratorProductionStickersConfig +import net.perfectdreams.loritta.morenitta.utils.readConfigurationFromFile +import org.jetbrains.exposed.sql.SortOrder +import org.jetbrains.exposed.sql.selectAll +import java.io.File +import java.sql.Connection + +suspend fun main() { + val http = HttpClient {} + + val configurationFile = File(System.getProperty("conf") ?: "./loricoolcards-production-stickers-generator.conf") + + if (!configurationFile.exists()) { + println("Missing configuration file!") + System.exit(1) + return + } + + val config = readConfigurationFromFile(configurationFile) + + val pudding = Pudding.createPostgreSQLPudding( + config.pudding.address, + config.pudding.database, + config.pudding.username, + config.pudding.password + ) + + + + pudding.transaction(transactionIsolation = Connection.TRANSACTION_READ_UNCOMMITTED) { + val list = LorittaDiscordShardStats.selectAll() + .limit(1_000_000) + .orderBy(LorittaDiscordShardStats.timestamp, SortOrder.DESC) + .toList() + + println(list.first()[LorittaDiscordShardStats.timestamp]) + println(list.last()[LorittaDiscordShardStats.timestamp]) + + var i = 0 + list.chunked(256) + .forEach { + val jobs = mutableListOf() + + it.forEach { + val timestamp = it[LorittaDiscordShardStats.timestamp].toEpochMilli() + val clusterId = it[LorittaDiscordShardStats.clusterId].toString() + val shardId = it[LorittaDiscordShardStats.shardId].toString() + + jobs.add( + GlobalScope.launch(Dispatchers.IO) { + http.post("http://127.0.0.1:8428/api/v1/import") { + setBody( + buildJsonObject { + putJsonObject("metric") { + put("__name__", "jda_gateway_ping") + put("loritta_cluster_id", clusterId) + put("jda_shard_id", shardId) + } + + putJsonArray("values") { + add(it[LorittaDiscordShardStats.gatewayPing]) + } + + putJsonArray("timestamps") { + add(timestamp) + } + }.toString() + ) + } + } + ) + + jobs.add( + GlobalScope.launch(Dispatchers.IO) { + http.post("http://127.0.0.1:8428/api/v1/import") { + setBody( + buildJsonObject { + putJsonObject("metric") { + put("__name__", "jda_response_total") + put("loritta_cluster_id", clusterId) + put("jda_shard_id", shardId) + } + + putJsonArray("values") { + add(it[LorittaDiscordShardStats.responseTotal]) + } + + putJsonArray("timestamps") { + add(timestamp) + } + }.toString() + ) + } + } + ) + + jobs.add( + GlobalScope.launch(Dispatchers.IO) { + http.post("http://127.0.0.1:8428/api/v1/import") { + setBody( + buildJsonObject { + putJsonObject("metric") { + put("__name__", "jda_status") + put("loritta_cluster_id", clusterId) + put("jda_shard_id", shardId) + } + + putJsonArray("values") { + add(it[LorittaDiscordShardStats.status]) + } + + putJsonArray("timestamps") { + add(timestamp) + } + }.toString() + ) + } + } + ) + + if (it[LorittaDiscordShardStats.gatewayStartupResumeStatus] != null) { + jobs.add( + GlobalScope.launch(Dispatchers.IO) { + http.post("http://127.0.0.1:8428/api/v1/import") { + setBody( + buildJsonObject { + putJsonObject("metric") { + put("__name__", "jda_gateway_startup_resume_status") + put("loritta_cluster_id", clusterId) + put("jda_shard_id", shardId) + } + + putJsonArray("values") { + add(it[LorittaDiscordShardStats.gatewayStartupResumeStatus]) + } + + putJsonArray("timestamps") { + add(timestamp) + } + }.toString() + ) + } + } + ) + } + + jobs.add( + GlobalScope.launch(Dispatchers.IO) { + http.post("http://127.0.0.1:8428/api/v1/import") { + setBody( + buildJsonObject { + putJsonObject("metric") { + put("__name__", "jda_cached_users") + put("loritta_cluster_id", clusterId) + put("jda_shard_id", shardId) + } + + putJsonArray("values") { + add(it[LorittaDiscordShardStats.cachedUsersCount]) + } + + putJsonArray("timestamps") { + add(timestamp) + } + }.toString() + ) + } + } + ) + + jobs.add( + GlobalScope.launch(Dispatchers.IO) { + http.post("http://127.0.0.1:8428/api/v1/import") { + setBody( + buildJsonObject { + putJsonObject("metric") { + put("__name__", "jda_guilds") + put("loritta_cluster_id", clusterId) + put("jda_shard_id", shardId) + } + + putJsonArray("values") { + add(it[LorittaDiscordShardStats.guildsCount]) + } + + putJsonArray("timestamps") { + add(timestamp) + } + }.toString() + ) + } + } + ) + + jobs.add( + GlobalScope.launch(Dispatchers.IO) { + http.post("http://127.0.0.1:8428/api/v1/import") { + setBody( + buildJsonObject { + putJsonObject("metric") { + put("__name__", "jda_unavailable_guilds") + put("loritta_cluster_id", clusterId) + put("jda_shard_id", shardId) + } + + putJsonArray("values") { + add(it[LorittaDiscordShardStats.unavailableGuildsCount]) + } + + putJsonArray("timestamps") { + add(timestamp) + } + }.toString() + ) + } + } + ) + + jobs.add( + GlobalScope.launch(Dispatchers.IO) { + http.post("http://127.0.0.1:8428/api/v1/import") { + setBody( + buildJsonObject { + putJsonObject("metric") { + put("__name__", "jda_cached_roles") + put("loritta_cluster_id", clusterId) + put("jda_shard_id", shardId) + } + + putJsonArray("values") { + add(it[LorittaDiscordShardStats.cachedRolesCount]) + } + + putJsonArray("timestamps") { + add(timestamp) + } + }.toString() + ) + } + } + ) + + jobs.add( + GlobalScope.launch(Dispatchers.IO) { + http.post("http://127.0.0.1:8428/api/v1/import") { + setBody( + buildJsonObject { + putJsonObject("metric") { + put("__name__", "jda_cached_channels") + put("loritta_cluster_id", clusterId) + put("jda_shard_id", shardId) + } + + putJsonArray("values") { + add(it[LorittaDiscordShardStats.cachedChannelsCount]) + } + + putJsonArray("timestamps") { + add(timestamp) + } + }.toString() + ) + } + } + ) + + jobs.add( + GlobalScope.launch(Dispatchers.IO) { + http.post("http://127.0.0.1:8428/api/v1/import") { + setBody( + buildJsonObject { + putJsonObject("metric") { + put("__name__", "jda_cached_scheduledevents") + put("loritta_cluster_id", clusterId) + put("jda_shard_id", shardId) + } + + putJsonArray("values") { + add(it[LorittaDiscordShardStats.cachedScheduledEventsCount]) + } + + putJsonArray("timestamps") { + add(timestamp) + } + }.toString() + ) + } + } + ) + + jobs.add( + GlobalScope.launch(Dispatchers.IO) { + http.post("http://127.0.0.1:8428/api/v1/import") { + setBody( + buildJsonObject { + putJsonObject("metric") { + put("__name__", "jda_cached_emojis") + put("loritta_cluster_id", clusterId) + put("jda_shard_id", shardId) + } + + putJsonArray("values") { + add(it[LorittaDiscordShardStats.cachedEmojisCount]) + } + + putJsonArray("timestamps") { + add(timestamp) + } + }.toString() + ) + } + } + ) + + jobs.add( + GlobalScope.launch(Dispatchers.IO) { + http.post("http://127.0.0.1:8428/api/v1/import") { + setBody( + buildJsonObject { + putJsonObject("metric") { + put("__name__", "jda_cached_audiomanagers") + put("loritta_cluster_id", clusterId) + put("jda_shard_id", shardId) + } + + putJsonArray("values") { + add(it[LorittaDiscordShardStats.cachedAudioManagerCount]) + } + + putJsonArray("timestamps") { + add(timestamp) + } + }.toString() + ) + } + } + ) + } + + jobs.joinAll() + + println("Progress: ${i}/${list.size}") + i += it.size + } + + println(list.first()[LorittaDiscordShardStats.timestamp]) + println(list.last()[LorittaDiscordShardStats.timestamp]) + + } +} \ No newline at end of file