Merge branch 'master' of https://github.com/TencentBlueKing/bk-repo

src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/controller/user/UserController.kt

@@ -240,6 +240,7 @@ class UserController @Autowired constructor(
 
     @ApiOperation("校验用户token")
     @PostMapping("/token")
+    @Deprecated("no need work")
     fun checkToken(@RequestParam uid: String, @RequestParam token: String): Response<Boolean> {
         preCheckContextUser(uid)
         userService.findUserByUserToken(uid, token) ?: return ResponseBuilder.success(false)

src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/util/query/UserQueryHelper.kt

@@ -13,7 +13,7 @@ object UserQueryHelper {
             Criteria.where(TUser::pwd.name).`is`(hashPwd),
             Criteria.where("tokens.id").`is`(pwd),
             Criteria.where("tokens.id").`is`(sm3HashPwd)
-        ).and(TUser::userId.name).`is`(userId)
+        ).and(TUser::userId.name).`is`(userId).and(TUser::locked.name).`is`(false)
         return query(criteria)
     }
 

src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/interceptor/devx/DevXAccessInterceptor.kt

@@ -27,14 +27,16 @@
 
 package com.tencent.bkrepo.common.security.interceptor.devx
 
+import com.fasterxml.jackson.core.type.TypeReference
+import com.fasterxml.jackson.module.kotlin.jacksonTypeRef
 import com.google.common.cache.CacheBuilder
 import com.google.common.cache.CacheLoader
 import com.google.common.cache.LoadingCache
 import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER
 import com.tencent.bkrepo.common.api.exception.SystemErrorException
 import com.tencent.bkrepo.common.api.message.CommonMessageCode
 import com.tencent.bkrepo.common.api.util.IpUtils
-import com.tencent.bkrepo.common.api.util.readJsonString
+import com.tencent.bkrepo.common.api.util.JsonUtils
 import com.tencent.bkrepo.common.api.util.toJsonString
 import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID
 import com.tencent.bkrepo.common.security.exception.AuthenticationException
@@ -57,7 +59,7 @@ open class DevXAccessInterceptor(private val devXProperties: DevXProperties) : H
     private val projectIpsCache: LoadingCache<String, Set<String>> = CacheBuilder.newBuilder()
         .maximumSize(devXProperties.cacheSize)
         .expireAfterWrite(devXProperties.cacheExpireTime)
-        .build(CacheLoader.from { key -> listIpFromProject(key) })
+        .build(CacheLoader.from { key -> listIpFromProject(key) + listCvmIpFromProject(key) + listIpFromProps(key) })
 
     override fun preHandle(request: HttpServletRequest, response: HttpServletResponse, handler: Any): Boolean {
         val user = SecurityUtils.getUserId()
@@ -117,29 +119,49 @@ open class DevXAccessInterceptor(private val devXProperties: DevXProperties) : H
     }
 
     private fun listIpFromProject(projectId: String): Set<String> {
-        val apiAuth = ApiAuth(devXProperties.appCode, devXProperties.appSecret)
-        val token = apiAuth.toJsonString().replace(System.lineSeparator(), "")
-        val workspaceUrl = devXProperties.workspaceUrl
-        val request = Request.Builder()
-            .url("$workspaceUrl?project_id=$projectId")
-            .header("X-Bkapi-Authorization", token)
-            .build()
+        val reqBuilder = Request.Builder().url("${devXProperties.workspaceUrl}?project_id=$projectId")
         logger.info("Update project[$projectId] ips.")
-        val response = httpClient.newCall(request).execute()
-        if (!response.isSuccessful || response.body == null) {
-            val errorMsg = response.body?.bytes()?.let { String(it) }
-            logger.error("${response.code} $errorMsg")
-            return devXProperties.projectCvmWhiteList[projectId] ?: emptySet()
-        }
         val ips = HashSet<String>()
-        devXProperties.projectCvmWhiteList[projectId]?.let { ips.addAll(it) }
-        response.body!!.byteStream().readJsonString<QueryResponse>().data.forEach { workspace ->
+        doRequest<List<DevXWorkSpace>>(reqBuilder, jacksonTypeRef())?.forEach { workspace ->
             workspace.innerIp?.substringAfter('.')?.let { ips.add(it) }
         }
         return ips
     }
 
+    private fun listIpFromProps(projectId: String) = devXProperties.projectCvmWhiteList[projectId] ?: emptySet()
+
+    private fun listCvmIpFromProject(projectId: String): Set<String> {
+        val workspaceUrl = devXProperties.cvmWorkspaceUrl.replace("{projectId}", projectId)
+        val reqBuilder = Request
+            .Builder()
+            .url("$workspaceUrl?pageSize=${devXProperties.cvmWorkspacePageSize}")
+            .header("X-DEVOPS-UID", devXProperties.cvmWorkspaceUid)
+        logger.info("Update project[$projectId] cvm ips.")
+        val res = doRequest<PageResponse<DevXCvmWorkspace>>(reqBuilder, jacksonTypeRef())
+        if ((res?.totalPages ?: 0) > 1) {
+            logger.error("[$projectId] has [${res?.totalPages}] page cvm workspace")
+        }
+        return res?.records?.mapTo(HashSet()) { it.ip } ?: emptySet()
+    }
 
+    private fun <T> doRequest(requestBuilder: Request.Builder, jacksonTypeRef: TypeReference<QueryResponse<T>>): T? {
+        val apiAuth = ApiAuth(devXProperties.appCode, devXProperties.appSecret)
+        val token = apiAuth.toJsonString().replace(System.lineSeparator(), "")
+        val req = requestBuilder.header("X-Bkapi-Authorization", token).build()
+        val response = httpClient.newCall(req).execute()
+        if (!response.isSuccessful || response.body == null) {
+            val errorMsg = response.body?.bytes()?.let { String(it) }
+            logger.error("${response.code} $errorMsg")
+            return null
+        }
+
+        val queryResponse = JsonUtils.objectMapper.readValue(response.body!!.byteStream(), jacksonTypeRef)
+        if (queryResponse.status != 0) {
+            logger.error("request bkapi failed ${response.code} status:${queryResponse.status}")
+            return null
+        }
+        return queryResponse.data!!
+    }
 
     companion object {
         private val logger = LoggerFactory.getLogger(DevXAccessInterceptor::class.java)

src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/interceptor/devx/DevXCvmWorkspace.kt

@@ -0,0 +1,37 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2024 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.security.interceptor.devx
+
+data class DevXCvmWorkspace(
+    val itemId: String,
+    val itemName: String,
+    val ip: String,
+    val resourceType: String,
+    val regionId: String,
+    val zoneId: String,
+)

src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/interceptor/devx/DevXProperties.kt

@@ -52,6 +52,18 @@ data class DevXProperties(
      * 查询云研发工作空间的URL
      * */
     var workspaceUrl: String = "",
+    /**
+     * 查询cvm workspace的url
+     */
+    var cvmWorkspaceUrl: String = "",
+    /**
+     * 查询cvm workspace时使用的用户名
+     */
+    var cvmWorkspaceUid: String = "bkrepo",
+    /**
+     * 查询cvm workspace 页大小
+     */
+    var cvmWorkspacePageSize: Int = 500,
     /**
      * 缓存的项目ip过期时间
      */

src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/interceptor/devx/PageResponse.kt

@@ -0,0 +1,43 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2024 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.security.interceptor.devx
+
+import io.swagger.annotations.ApiModelProperty
+
+data class PageResponse<out T>(
+    @ApiModelProperty("记录条数")
+    val count: Long,
+    @ApiModelProperty("页码(从1页开始)")
+    val page: Int,
+    @ApiModelProperty("每页多少条")
+    val pageSize: Int,
+    @ApiModelProperty("总页数")
+    val totalPages: Long,
+    @ApiModelProperty("数据列表")
+    val records: List<T>
+)

src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/interceptor/devx/QueryResponse.kt

@@ -27,7 +27,7 @@
 
 package com.tencent.bkrepo.common.security.interceptor.devx
 
-data class QueryResponse(
+data class QueryResponse<out T>(
     val status: Int,
-    val data: List<DevXWorkSpace>,
+    val data: T?,
 )

src/backend/common/common-storage/storage-service/build.gradle.kts

@@ -47,6 +47,7 @@ dependencies {
     implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml")
     implementation("com.tencent.polaris:polaris-discovery-factory")
     implementation("com.tencent.bk.sdk:crypto-java-sdk")
+    compileOnly("io.micrometer:micrometer-core")
     compileOnly("org.springframework.boot:spring-boot-starter-data-redis")
     testImplementation(project(":common:common-redis"))
     testImplementation("it.ozimov:embedded-redis:${Versions.EmbeddedRedis}") {

src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/StorageAutoConfiguration.kt

@@ -43,7 +43,7 @@ import com.tencent.bkrepo.common.storage.core.locator.HashFileLocator
 import com.tencent.bkrepo.common.storage.core.simple.SimpleStorageService
 import com.tencent.bkrepo.common.storage.credentials.StorageType
 import com.tencent.bkrepo.common.storage.filesystem.FileSystemStorage
-import com.tencent.bkrepo.common.storage.filesystem.cleanup.FileExpireResolver
+import com.tencent.bkrepo.common.storage.filesystem.cleanup.FileRetainResolver
 import com.tencent.bkrepo.common.storage.innercos.InnerCosFileStorage
 import com.tencent.bkrepo.common.storage.monitor.StorageHealthMonitor
 import com.tencent.bkrepo.common.storage.monitor.StorageHealthMonitorHelper
@@ -93,17 +93,12 @@ class StorageAutoConfiguration {
     fun storageService(
         properties: StorageProperties,
         threadPoolTaskExecutor: ThreadPoolTaskExecutor,
-        fileExpireResolver: FileExpireResolver?,
+        fileRetainResolver: FileRetainResolver?,
     ): StorageService {
-        fileExpireResolver?.let {
-            logger.info("Use FileExpireResolver[${fileExpireResolver::class.simpleName}].")
-        }
+        fileRetainResolver?.let { logger.info("Use FileRetainResolver[${fileRetainResolver::class.simpleName}].") }
         val cacheEnabled = properties.defaultStorageCredentials().cache.enabled
         val storageService = if (cacheEnabled) {
-            CacheStorageService(
-                threadPoolTaskExecutor,
-                fileExpireResolver,
-            )
+            CacheStorageService(threadPoolTaskExecutor, fileRetainResolver)
         } else {
             SimpleStorageService()
         }

src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageService.kt

@@ -43,8 +43,7 @@ import com.tencent.bkrepo.common.storage.filesystem.check.SynchronizeResult
 import com.tencent.bkrepo.common.storage.filesystem.cleanup.BasedAtimeAndMTimeFileExpireResolver
 import com.tencent.bkrepo.common.storage.filesystem.cleanup.CleanupFileVisitor
 import com.tencent.bkrepo.common.storage.filesystem.cleanup.CleanupResult
-import com.tencent.bkrepo.common.storage.filesystem.cleanup.CompositeFileExpireResolver
-import com.tencent.bkrepo.common.storage.filesystem.cleanup.FileExpireResolver
+import com.tencent.bkrepo.common.storage.filesystem.cleanup.FileRetainResolver
 import com.tencent.bkrepo.common.storage.monitor.StorageHealthMonitor
 import org.slf4j.LoggerFactory
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor
@@ -58,7 +57,7 @@ import java.nio.file.Paths
  */
 class CacheStorageService(
     private val threadPoolTaskExecutor: ThreadPoolTaskExecutor,
-    private val fileExpireResolver: FileExpireResolver? = null,
+    private val fileRetainResolver: FileRetainResolver? = null,
 ) : AbstractStorageService() {
 
     private val cacheFileEventPublisher by lazy { CacheFileEventPublisher(publisher) }
@@ -170,12 +169,7 @@ class CacheStorageService(
         val rootPath = Paths.get(credentials.cache.path)
         val tempPath = getTempPath(credentials)
         val stagingPath = getStagingPath(credentials)
-        val resolver = if (fileExpireResolver != null) {
-            val baseFileExpireResolver = BasedAtimeAndMTimeFileExpireResolver(credentials.cache.expireDuration)
-            CompositeFileExpireResolver(listOf(baseFileExpireResolver, fileExpireResolver))
-        } else {
-            BasedAtimeAndMTimeFileExpireResolver(credentials.cache.expireDuration)
-        }
+        val resolver = BasedAtimeAndMTimeFileExpireResolver(credentials.cache.expireDuration)
         val visitor = CleanupFileVisitor(
             rootPath,
             tempPath,
@@ -185,6 +179,7 @@ class CacheStorageService(
             credentials,
             resolver,
             publisher,
+            fileRetainResolver
         )
         getCacheClient(credentials).walk(visitor)
         val result = mutableMapOf<Path, CleanupResult>()
@@ -233,15 +228,17 @@ class CacheStorageService(
         path: String,
         filename: String,
         credentials: StorageCredentials,
-    ) {
-        if (doExist(path, filename, credentials)) {
+    ): Boolean {
+        return if (doExist(path, filename, credentials)) {
             val cacheFilePath = "${credentials.cache.path}$path$filename"
             val size = File(cacheFilePath).length()
             getCacheClient(credentials).delete(path, filename)
             cacheFileEventPublisher.publishCacheFileDeletedEvent(path, filename, size, credentials)
             logger.info("Cache [${credentials.cache.path}/$path/$filename] was deleted")
+            true
         } else {
             logger.info("Cache file[${credentials.cache.path}/$path/$filename] was not in storage")
+            false
         }
     }
 

src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/indexer/StorageCacheIndexConfiguration.kt

@@ -32,11 +32,14 @@ import com.tencent.bkrepo.common.storage.core.cache.CacheStorageService
 import com.tencent.bkrepo.common.storage.core.cache.indexer.StorageCacheIndexProperties.Companion.CACHE_TYPE_REDIS_LRU
 import com.tencent.bkrepo.common.storage.core.cache.indexer.StorageCacheIndexProperties.Companion.CACHE_TYPE_REDIS_SLRU
 import com.tencent.bkrepo.common.storage.core.cache.indexer.listener.StorageEldestRemovedListener
+import com.tencent.bkrepo.common.storage.core.cache.indexer.metrics.StorageCacheIndexerMetrics
 import com.tencent.bkrepo.common.storage.core.cache.indexer.redis.RedisLRUCacheIndexer
 import com.tencent.bkrepo.common.storage.core.cache.indexer.redis.RedisSLRUCacheIndexer
 import com.tencent.bkrepo.common.storage.core.locator.FileLocator
 import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
 import com.tencent.bkrepo.common.storage.util.toPath
+import io.micrometer.core.instrument.MeterRegistry
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBean
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
@@ -50,6 +53,12 @@ import org.springframework.data.redis.core.RedisTemplate
 @ConditionalOnProperty(prefix = "storage.cache.index", name = ["enabled"])
 @EnableConfigurationProperties(StorageCacheIndexProperties::class)
 class StorageCacheIndexConfiguration {
+    @Bean
+    @ConditionalOnBean(MeterRegistry::class)
+    fun storageCacheIndexerMetrics(registry: MeterRegistry): StorageCacheIndexerMetrics {
+        return StorageCacheIndexerMetrics(registry)
+    }
+
     @Bean
     fun storageCacheIndexerManager(
         cacheFactory: StorageCacheIndexerFactory<String, Long>,
@@ -114,10 +123,13 @@ class StorageCacheIndexConfiguration {
     fun defaultIndexerCustomizer(
         storageService: CacheStorageService,
         fileLocator: FileLocator,
+        storageCacheIndexerMetrics: StorageCacheIndexerMetrics? = null,
     ): IndexerCustomizer<String, Long> {
         return object : IndexerCustomizer<String, Long> {
             override fun customize(indexer: StorageCacheIndexer<String, Long>, credentials: StorageCredentials) {
-                val listener = StorageEldestRemovedListener(credentials, fileLocator, storageService)
+                val listener = StorageEldestRemovedListener(
+                    credentials, fileLocator, storageService, storageCacheIndexerMetrics
+                )
                 indexer.addEldestRemovedListener(listener)
             }
         }

src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/indexer/listener/StorageEldestRemovedListener.kt

@@ -27,7 +27,9 @@
 
 package com.tencent.bkrepo.common.storage.core.cache.indexer.listener
 
+import com.tencent.bkrepo.common.artifact.constant.DEFAULT_STORAGE_KEY
 import com.tencent.bkrepo.common.storage.core.cache.CacheStorageService
+import com.tencent.bkrepo.common.storage.core.cache.indexer.metrics.StorageCacheIndexerMetrics
 import com.tencent.bkrepo.common.storage.core.locator.FileLocator
 import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
 
@@ -39,10 +41,12 @@ open class StorageEldestRemovedListener(
     protected var storageCredentials: StorageCredentials,
     protected val fileLocator: FileLocator,
     protected val storageService: CacheStorageService,
+    protected val storageCacheIndexerMetrics: StorageCacheIndexerMetrics? = null,
 ) : UpdatableEldestRemovedListener<String, Long> {
     override fun onEldestRemoved(key: String, value: Long) {
         val path = fileLocator.locate(key)
-        storageService.deleteCacheFile(path, key, storageCredentials)
+        val deleted = storageService.deleteCacheFile(path, key, storageCredentials)
+        storageCacheIndexerMetrics?.evicted(storageCredentials.key ?: DEFAULT_STORAGE_KEY, value, deleted)
     }
 
     override fun updateCredentials(credentials: StorageCredentials) {