Optimizations for Governance Transactions Queries

.editorconfig

@@ -0,0 +1,32 @@
+[*.{kt,kts}]
+
+ktlint_standard_filename=disabled
+ktlint_standard_chain-wrapping=disabled
+ktlint_standard_enum-entry-name-case=disabled
+ktlint_standard_multiline-if-else=disabled
+ktlint_standard_property-naming=disabled
+ktlint_standard_max-line-length=disabled
+ktlint_standard_value-parameter-comment=disabled
+ktlint_standard_no-empty-file=disabled
+ktlint_standard_function-naming=disabled
+
+ktlint_standard_function-signature=disabled
+ktlint_standard_chain-method-continuation=disabled
+ktlint_standard_multiline-expression-wrapping=disabled
+ktlint_standard_trailing-comma-on-call-site=disabled
+ktlint_standard_trailing-comma-on-declaration-site=disabled
+ktlint_standard_class-signature=disabled
+ktlint_standard_blank-line-before-declaration=disabled
+ktlint_standard_no-empty-first-line-in-class-body=disabled
+ktlint_standard_parameter-list-wrapping=disabled
+ktlint_standard_argument-list-wrapping=disabled
+ktlint_standard_function-expression-body=disabled
+ktlint_standard_string-template-indent=disabled
+ktlint_standard_indent=disabled
+ktlint_standard_if-else-wrapping=disabled
+ktlint_standard_condition-wrapping=disabled
+ktlint_standard_statement-wrapping=disabled
+ktlint_standard_try-catch-finally-spacing=disabled
+ktlint_standard_if-else-bracing=disabled
+ktlint_standard_no-blank-line-in-list=disabled
+ktlint_standard_no-single-line-block-comment=disabled

CHANGELOG.md

@@ -33,6 +33,14 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ## Unreleased
 
+## Improvements
+* Removed materialized view that is unnecessary and causes misdirection when debugging performance issues
+* Add a caching interface for block and transaction queries that tend to be used often
+* Optimize governance transaction fetching to return results in a timely fashion
+
+### Bug Fixes
+* Resolve explorer service restarts due to database connection starvation caused by long running governance transaction queries
+
 ## [v6.0.0](https://github.com/provenance-io/explorer-service/releases/tag/v6.0.0) - 2024-11-15
 
 ### Features

buildSrc/src/main/kotlin/Dependencies.kt

@@ -1,3 +1,4 @@
+
 object PluginIds { // please keep this sorted in sections
     // Kotlin
     const val Kotlin = "kotlin"
@@ -55,6 +56,7 @@ object Versions {
     const val Postgres = "42.2.23"
     const val Protobuf = "3.21.9"
     const val Reflections = "0.9.12"
+    const val Caffeine = "2.9.2"
 
     // Testing
     const val Jupiter = "5.9.1"
@@ -92,6 +94,7 @@ object Libraries {
     const val KaseChange = "net.pearx.kasechange:kasechange:${Versions.KaseChange}"
     const val Json = "org.json:json:${Versions.Json}"
     const val Reflections = "org.reflections:reflections:${Versions.Reflections}"
+    const val Caffeine = "com.github.ben-manes.caffeine:caffeine:${Versions.Caffeine}"
 
     // Protobuf
     const val GrpcNetty = "io.grpc:grpc-netty:${Versions.Grpc}"
@@ -105,6 +108,7 @@ object Libraries {
     const val SpringBootConfigProcessor = "org.springframework.boot:spring-boot-configuration-processor:${Versions.SpringBoot}"
     const val SpringBootStarterValidation = "org.springframework.boot:spring-boot-starter-validation:${Versions.SpringBoot}"
     const val SpringBootStarterWeb = "org.springframework.boot:spring-boot-starter-web:${Versions.SpringBoot}"
+    const val SpringBootStarterCache = "org.springframework.boot:spring-boot-starter-cache:${Versions.SpringBoot}"
     const val SpringBootStarterTest = "org.springframework.boot:spring-boot-starter-test:${Versions.SpringBoot}"
 
     const val Swagger = "io.springfox:springfox-boot-starter:${Versions.Swagger}"

database/src/main/resources/db/migration/V1_97__Update_validator_state_view.sql

@@ -0,0 +1,22 @@
+SELECT 'Modify `current_validator_state` view' AS comment;
+DROP MATERIALIZED VIEW IF EXISTS current_validator_state;
+
+CREATE VIEW current_validator_state AS
+SELECT DISTINCT ON (vs.operator_addr_id) vs.operator_addr_id,
+    vs.operator_address,
+    vs.block_height,
+    vs.moniker,
+    vs.status,
+    vs.jailed,
+    vs.token_count,
+    vs.json,
+    svc.account_address,
+    svc.consensus_address,
+    svc.consensus_pubkey,
+    vs.commission_rate,
+    vs.removed,
+    ai.image_url
+FROM validator_state vs
+    JOIN staking_validator_cache svc on vs.operator_addr_id = svc.id
+    LEFT JOIN address_image ai ON svc.operator_address = ai.address
+ORDER BY vs.operator_addr_id, vs.block_height desc;

service/build.gradle.kts

@@ -28,11 +28,13 @@ dependencies {
     implementation(Libraries.ProtobufKotlin)
     implementation(Libraries.ProvenanceProto)
     implementation(Libraries.Reflections)
+    implementation(Libraries.Caffeine)
 
     implementation(Libraries.SpringBootStarterWeb)
     implementation(Libraries.SpringBootStarterJdbc)
     implementation(Libraries.SpringBootStarterActuator)
     implementation(Libraries.SpringBootStarterValidation)
+    implementation(Libraries.SpringBootStarterCache)
     kapt(Libraries.SpringBootConfigProcessor)
 
     implementation(Libraries.BouncyCastle)

service/src/main/kotlin/io/provenance/explorer/config/CacheConfig.kt

@@ -0,0 +1,21 @@
+import org.springframework.cache.CacheManager
+import org.springframework.cache.annotation.EnableCaching
+import org.springframework.cache.caffeine.CaffeineCacheManager
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+import java.util.concurrent.TimeUnit
+
+@Configuration
+@EnableCaching
+class CacheConfig {
+    @Bean
+    fun cacheManager(): CacheManager {
+        val cacheManager = CaffeineCacheManager("responses")
+        cacheManager.setCaffeine(
+            com.github.benmanes.caffeine.cache.Caffeine.newBuilder()
+                .expireAfterWrite(30, TimeUnit.SECONDS) // Cache expires after 10 seconds
+                .maximumSize(100)
+        ) // Optional, limits the number of cached items
+        return cacheManager
+    }
+}

service/src/main/kotlin/io/provenance/explorer/domain/entities/Transactions.kt

@@ -6,7 +6,7 @@ import cosmos.tx.v1beta1.ServiceOuterClass
 import io.provenance.explorer.OBJECT_MAPPER
 import io.provenance.explorer.VANILLA_MAPPER
 import io.provenance.explorer.config.ExplorerProperties
-import io.provenance.explorer.domain.core.sql.Distinct
+import io.provenance.explorer.domain.core.logger
 import io.provenance.explorer.domain.core.sql.jsonb
 import io.provenance.explorer.domain.core.sql.toProcedureObject
 import io.provenance.explorer.domain.entities.FeeType.BASE_FEE_OVERAGE
@@ -58,7 +58,6 @@ import org.jetbrains.exposed.sql.SizedIterable
 import org.jetbrains.exposed.sql.SortOrder
 import org.jetbrains.exposed.sql.TextColumnType
 import org.jetbrains.exposed.sql.VarCharColumnType
-import org.jetbrains.exposed.sql.alias
 import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.andWhere
 import org.jetbrains.exposed.sql.countDistinct
@@ -347,8 +346,6 @@ object TxMessageTable : IntIdTable(name = "tx_message") {
 
 class TxMessageRecord(id: EntityID<Int>) : IntEntity(id) {
     companion object : IntEntityClass<TxMessageRecord>(TxMessageTable) {
-
-        val distId = Distinct(TxMessageTable.id, IntegerColumnType()).alias("dist")
         val tableColSet = TxMessageTable.columns.toMutableList()
 
         fun findByHash(hash: String) = transaction {
@@ -369,7 +366,7 @@ class TxMessageRecord(id: EntityID<Int>) : IntEntity(id) {
         fun findByHashIdPaginated(hashId: Int, msgTypes: List<Int>, limit: Int, offset: Int) = transaction {
             val query = TxMessageTable
                 .innerJoin(TxMsgTypeSubtypeTable, { TxMessageTable.id }, { TxMsgTypeSubtypeTable.txMsgId })
-                .slice(listOf(distId) + tableColSet)
+                .slice(tableColSet)
                 .select { TxMessageTable.txHashId eq hashId }
             if (msgTypes.isNotEmpty()) {
                 query.andWhere { TxMsgTypeQueryTable.typeId inList msgTypes }
@@ -393,10 +390,19 @@ class TxMessageRecord(id: EntityID<Int>) : IntEntity(id) {
         }
 
         fun findByQueryForResults(txQueryParams: TxQueryParams) = transaction {
-            val query = findByQueryParams(txQueryParams, listOf(distId) + tableColSet)
-                .orderBy(Pair(TxMessageTable.blockHeight, SortOrder.DESC))
-                .limit(txQueryParams.count, txQueryParams.offset.toLong())
-            TxMessageRecord.wrapRows(query).toSet()
+            findByQueryParams(txQueryParams, tableColSet).let {
+                // Because of the way the db takes hints on when to materialize data for sorting, it is in our best interest
+                // to sort by the deepest level of the inner joins that have been executed. In this case this will yield a
+                // query that takes less than a second whereas ordering by the same field in the tx message table takes 18 minutes.
+                if ((txQueryParams.addressId != null && txQueryParams.addressType != null) || txQueryParams.address != null)
+                    it.orderBy(Pair(TxAddressJoinTable.blockHeight, SortOrder.DESC))
+                else
+                    it.orderBy(Pair(TxMessageTable.blockHeight, SortOrder.DESC))
+            }.let {
+                it.limit(txQueryParams.count, txQueryParams.offset.toLong())
+            }.let {
+                TxMessageRecord.wrapRows(it).toSet()
+            }
         }
 
         fun findByQueryParamsForCount(txQueryParams: TxQueryParams) = transaction {
@@ -405,6 +411,8 @@ class TxMessageRecord(id: EntityID<Int>) : IntEntity(id) {
         }
 
         private fun findByQueryParams(tqp: TxQueryParams, distinctQuery: List<Expression<*>>?) = transaction {
+            logger().info("Query Params: $tqp")
+
             var join: ColumnSet = TxMessageTable
 
             if (tqp.msgTypes.isNotEmpty())

service/src/main/kotlin/io/provenance/explorer/domain/entities/Validators.kt

@@ -150,11 +150,6 @@ class ValidatorStateRecord(id: EntityID<Int>) : IntEntity(id) {
                 .toList()
         }
 
-        fun refreshCurrentStateView() = transaction {
-            val query = "REFRESH MATERIALIZED VIEW current_validator_state"
-            this.exec(query)
-        }
-
         fun findAll(activeSet: Int) = transaction {
             val query = "SELECT * FROM get_all_validator_state(?, ?, NULL)".trimIndent()
             val arguments = mutableListOf<Pair<ColumnType, *>>(

service/src/main/kotlin/io/provenance/explorer/service/TransactionService.kt

@@ -111,6 +111,7 @@ class TransactionService(
         ibcSrcPort: String? = null,
         ibcSrcChannel: String? = null
     ): PagedResults<TxSummary> {
+        logger.info("Fetching transactions with address: $address, denom: $denom, module: $module, msgType: $msgType, txHeight: $txHeight, txStatus: $txStatus, count: $count, page: $page, fromDate: $fromDate, toDate: $toDate, nftAddr: $nftAddr, ibcChain: $ibcChain, ibcSrcPort: $ibcSrcPort, ibcSrcChannel: $ibcSrcChannel")
         val msgTypes = if (msgType != null) listOf(msgType) else (module?.getValuesPlusAddtnl() ?: listOf())
         val msgTypeIds = transaction { TxMessageTypeRecord.findByType(msgTypes).map { it.id.value } }.toList()
         val addr = transaction { address?.getAddressType(valService.getActiveSet()) }

service/src/main/kotlin/io/provenance/explorer/service/ValidatorService.kt

@@ -132,8 +132,7 @@ class ValidatorService(
                     getImgUrl(it.description.identity)
                         ?.let { img -> AddressImageRecord.upsert(it.operatorAddress, img) }
                 }
-            }.also { ValidatorStateRecord.refreshCurrentStateView() }
-            .let { ValidatorStateRecord.findByOperator(getActiveSet(), address)!! }
+            }.let { ValidatorStateRecord.findByOperator(getActiveSet(), address)!! }
     }
 
     fun validateValidator(validator: String) =
@@ -149,7 +148,6 @@ class ValidatorService(
             val votingPowerTotal = validatorSet.sumOf { it.votingPower.toBigInteger() }
             val slashingParams = getSlashingParams()
             validateStatus(addr, latestValidator, addr.operatorAddrId)
-                .also { if (it) ValidatorStateRecord.refreshCurrentStateView() }
             val stakingValidator = getStakingValidator(addr.operatorAddress)
             ValidatorDetails(
                 if (latestValidator != null) {
@@ -245,7 +243,7 @@ class ValidatorService(
                 } else {
                     false
                 }
-            }.also { map -> if (map.contains(true)) ValidatorStateRecord.refreshCurrentStateView() }
+            }
     }
 
     // Updates the staking validator cache
@@ -294,7 +292,7 @@ class ValidatorService(
         ).validatorsList
         getStakingValidators(status).map { v ->
             validateStatus(v, validatorSet.firstOrNull { it.address == v.consensusAddr }, v.operatorAddrId)
-        }.also { map -> if (map.contains(true)) ValidatorStateRecord.refreshCurrentStateView() }
+        }
         val stakingValidators = getStakingValidators(status, null, page.toOffset(count), count)
         val results = hydrateValidators(validatorSet, hr24ChangeSet, stakingValidators, height)
         val totalCount = getStakingValidatorsCount(status, null)

service/src/main/kotlin/io/provenance/explorer/service/async/BlockAndTxProcessor.kt

@@ -224,7 +224,7 @@ class BlockAndTxProcessor(
                         TxAddressJoinType.OPERATOR.name -> validatorService.updateStakingValidators(
                             ent.value,
                             blockRes.block.height()
-                        ).also { updated -> if (updated) ValidatorStateRecord.refreshCurrentStateView() }
+                        )
                     }
                 }
             }

service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt

@@ -211,7 +211,7 @@ class ScheduledTaskService(
     @Scheduled(initialDelay = 0L, fixedDelay = 5000L)
     fun updateSpotlight() = explorerService.createSpotlight()
 
-    @Scheduled(initialDelay = 0L, fixedDelay = 30000L)
+    @Scheduled(initialDelay = 0L, fixedDelay = 5000L)
     fun retryBlockTxs() {
         logger.info("Retrying block/tx records")
         BlockTxRetryRecord.getRecordsToRetry().map { height ->
@@ -263,7 +263,7 @@ class ScheduledTaskService(
         tokenService.updateAndSaveTokenHistoricalData(startDate, today)
     }
 
-    @Scheduled(initialDelay = 0L, fixedDelay = 300000L) // Every 5 minutes
+    @Scheduled(initialDelay = 0L, fixedDelay = 5000L)
     fun updateTokenLatest() {
         val today = DateTime.now().withZone(DateTimeZone.UTC)
         val startDate = today.minusDays(1)

service/src/main/kotlin/io/provenance/explorer/service/utility/UtilityService.kt

@@ -11,7 +11,6 @@ import io.provenance.explorer.domain.entities.TxFeeRecord
 import io.provenance.explorer.domain.entities.TxMessageRecord
 import io.provenance.explorer.domain.entities.TxMessageTypeRecord
 import io.provenance.explorer.domain.entities.UnknownTxType
-import io.provenance.explorer.domain.entities.ValidatorStateRecord
 import io.provenance.explorer.domain.extensions.fromBase64
 import io.provenance.explorer.domain.extensions.toObjectNode
 import io.provenance.explorer.domain.models.explorer.BlockProposer
@@ -127,8 +126,6 @@ class UtilityService(
             "( '$proto', '$module', '$type', '$category' )"
         }
     }
-
-    fun refreshCurrentValidatorState() = ValidatorStateRecord.refreshCurrentStateView()
 }
 
 data class ProtoBreakout(

service/src/main/kotlin/io/provenance/explorer/web/v2/BlockController.kt

@@ -4,6 +4,7 @@ import io.provenance.explorer.service.ExplorerService
 import io.swagger.annotations.Api
 import io.swagger.annotations.ApiOperation
 import io.swagger.annotations.ApiParam
+import org.springframework.cache.annotation.Cacheable
 import org.springframework.http.MediaType
 import org.springframework.validation.annotation.Validated
 import org.springframework.web.bind.annotation.GetMapping
@@ -34,6 +35,7 @@ class BlockController(private val explorerService: ExplorerService) {
     fun blockHeight(@PathVariable height: Int) = explorerService.getBlockAtHeight(height)
 
     @ApiOperation("Returns X most recent blocks")
+    @Cacheable(value = ["responses"], key = "{#root.methodName, #count, #page}")
     @GetMapping("/recent")
     fun recentBlocks(
         @ApiParam(value = "Record count between 1 and 200", defaultValue = "10", required = false)

service/src/main/kotlin/io/provenance/explorer/web/v2/TransactionControllerV2.kt

@@ -7,6 +7,7 @@ import io.swagger.annotations.Api
 import io.swagger.annotations.ApiOperation
 import io.swagger.annotations.ApiParam
 import org.joda.time.DateTime
+import org.springframework.cache.annotation.Cacheable
 import org.springframework.format.annotation.DateTimeFormat
 import org.springframework.http.MediaType
 import org.springframework.validation.annotation.Validated
@@ -31,6 +32,7 @@ class TransactionControllerV2(private val transactionService: TransactionService
 
     @ApiOperation("Return the latest transactions with query params")
     @GetMapping("/recent")
+    @Cacheable(value = ["responses"], key = "{#root.methodName, #count, #page, #msgType, #txStatus, #fromDate, #toDate}")
     fun txsRecent(
         @ApiParam(defaultValue = "1", required = false)
         @RequestParam(defaultValue = "1")

service/src/main/kotlin/io/provenance/explorer/web/v2/utility/UtilityController.kt

@@ -75,8 +75,4 @@ class UtilityController(private val us: UtilityService) {
     @ApiOperation("Parses and tries to save raw tx json, formatted as a string. Used for debugging a tx response")
     @PostMapping("/parse/tx_json/save")
     fun saveTxResponseObject(@RequestBody rawJson: String) = ResponseEntity.ok(us.saveRawTxJson(rawJson))
-
-    @ApiOperation("Refresh the current validator state view")
-    @GetMapping("/validator_refresh")
-    fun refreshValidatorStateView() = ResponseEntity.ok(us.refreshCurrentValidatorState())
 }