Merge branch 'master' of https://github.com/TencentBlueKing/bk-repo

src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/Constants.kt

@@ -115,3 +115,18 @@ const val MS_REQUEST_SRC_CLUSTER = "X-BKREPO-MS-CLUSTER"
  */
 const val CLIENT_ADDRESS = "clientAddress"
 const val DOWNLOAD_SOURCE = "downloadSource"
+
+/**
+ * 用于标记访问来源，web或api
+ */
+const val HEADER_ACCESS_FROM = "X-BKREPO-ACCESS-FROM"
+
+/**
+ * 来源于API调用
+ */
+const val ACCESS_FROM_API = "api"
+
+/**
+ * 来源于浏览器访问
+ */
+const val ACCESS_FROM_WEB = "web"

src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/constant/ArtifactConstants.kt

@@ -162,3 +162,16 @@ const val SCAN_STATUS = "scanStatus"
 
 const val METADATA_KEY_PACKAGE_NAME = "packageName"
 const val METADATA_KEY_PACKAGE_VERSION = "packageVersion"
+
+/**
+ * 节点链接的目标项目
+ */
+const val METADATA_KEY_LINK_PROJECT = "targetProjectId"
+/**
+ * 节点链接的目标仓库
+ */
+const val METADATA_KEY_LINK_REPO = "targetRepoName"
+/**
+ * 节点链接的目标路径
+ */
+const val METADATA_KEY_LINK_FULL_PATH = "targetFullPath"

src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/message/ArtifactMessageCode.kt

@@ -67,7 +67,8 @@ enum class ArtifactMessageCode(private val key: String) : MessageCode {
     ARTIFACT_TYPE_UNSUPPORTED("artifact.type.unsupported"),
     ARTIFACT_FORBIDDEN("artifact.forbidden"),
     NODE_CREATE_TIMEOUT("artifact.node.create.timeout"),
-    SIZE_CHECK_FAILED("artifact.size.check-failed")
+    SIZE_CHECK_FAILED("artifact.size.check-failed"),
+    NODE_LINK_FOLDER_UNSUPPORTED("artifact.node.link-folder-unsupported"),
     ;
 
     override fun getBusinessCode() = ordinal + 1

src/backend/common/common-artifact/artifact-api/src/main/resources/i18n/messages_en.properties

@@ -45,6 +45,7 @@ artifact.node.existed=Node [{0}] existed
 artifact.node.conflict=Node [{0}] conflict
 artifact.node.create.timeout=Node [{0}] create timeout
 artifact.node.list.too-large=Node list count too large
+artifact.node.link-folder-unsupported=Link folder[{0}] was unsupported
 artifact.stage.upgrade.error=Upgrade artifact stage error: [{0}]
 artifact.stage.downgrade.error=Downgrade artifact stage error: [{0}]
 artifact.package.not-found=Package [{0}] not found

src/backend/common/common-artifact/artifact-api/src/main/resources/i18n/messages_zh_CN.properties

@@ -42,6 +42,7 @@ artifact.node.existed=节点[{0}]已存在
 artifact.node.conflict=已存在同名文件，且不允许覆盖
 artifact.node.create.timeout=节点[{0}]创建超时
 artifact.node.list.too-large=节点列表数量过大
+artifact.node.link-folder-unsupported=无法链接到目录[{0}]
 artifact.stage.upgrade.error=制品晋级失败: {0}
 artifact.stage.downgrade.error=制品降级失败: {0}
 artifact.package.not-found=制品包[{0}]不存在

src/backend/common/common-artifact/artifact-api/src/main/resources/i18n/messages_zh_TW.properties

@@ -42,6 +42,7 @@ artifact.node.existed=節點[{0}]已存在
 artifact.node.conflict=已存在同名文件，且不允許覆蓋
 artifact.node.create.timeout=節點[{0}]創建超時
 artifact.node.list.too-large=節點列表數量過大
+artifact.node.link-folder-unsupported=無法連結到目錄[{0}]
 artifact.stage.upgrade.error=制品晉級失敗: {0}
 artifact.stage.downgrade.error=制品降級失敗: {0}
 artifact.package.not-found=制品包[{0}]不存在

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/ArtifactContextConfiguration.kt

@@ -38,6 +38,7 @@ import com.tencent.bkrepo.common.artifact.repository.proxy.ProxyRepository
 import com.tencent.bkrepo.common.artifact.repository.redirect.CosRedirectService
 import com.tencent.bkrepo.common.artifact.repository.redirect.DownloadRedirectManager
 import com.tencent.bkrepo.common.artifact.repository.redirect.EdgeNodeRedirectService
+import com.tencent.bkrepo.common.artifact.repository.redirect.LinkNodeRedirectService
 import org.springframework.context.annotation.Configuration
 import org.springframework.context.annotation.Import
 
@@ -49,6 +50,7 @@ import org.springframework.context.annotation.Import
     ProxyRepository::class,
     EdgeNodeRedirectService::class,
     CosRedirectService::class,
+    LinkNodeRedirectService::class,
     DownloadRedirectManager::class,
 )
 class ArtifactContextConfiguration

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/redirect/LinkNodeRedirectService.kt

@@ -0,0 +1,92 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.artifact.repository.redirect
+
+import com.tencent.bkrepo.common.api.constant.ACCESS_FROM_API
+import com.tencent.bkrepo.common.api.constant.HEADER_ACCESS_FROM
+import com.tencent.bkrepo.common.api.constant.HttpHeaders
+import com.tencent.bkrepo.common.api.constant.HttpStatus
+import com.tencent.bkrepo.common.artifact.constant.METADATA_KEY_LINK_FULL_PATH
+import com.tencent.bkrepo.common.artifact.constant.METADATA_KEY_LINK_PROJECT
+import com.tencent.bkrepo.common.artifact.constant.METADATA_KEY_LINK_REPO
+import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
+import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder.RepositoryId
+import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext
+import com.tencent.bkrepo.common.storage.core.StorageProperties
+import com.tencent.bkrepo.repository.pojo.node.NodeDetail
+import org.springframework.core.annotation.Order
+import org.springframework.stereotype.Service
+
+/**
+ * 软链接节点下载重定向
+ */
+@Service
+@Order(2)
+class LinkNodeRedirectService(
+    private val storageProperties: StorageProperties
+) : DownloadRedirectService {
+
+    override fun shouldRedirect(context: ArtifactDownloadContext): Boolean {
+        return storageProperties.redirect.enabled
+                && isLinkNode(ArtifactContextHolder.getNodeDetail(context.artifactInfo))
+    }
+
+    override fun redirect(context: ArtifactDownloadContext) {
+        val node = ArtifactContextHolder.getNodeDetail(context.artifactInfo)!!
+        val targetProjectId = node.metadata[METADATA_KEY_LINK_PROJECT] as String
+        val targetRepoName = node.metadata[METADATA_KEY_LINK_REPO] as String
+        val targetFullPath = node.metadata[METADATA_KEY_LINK_FULL_PATH] as String
+
+        // 获取存储所在区域制品库集群url
+        val targetRepo = ArtifactContextHolder.getRepoDetail(RepositoryId(targetProjectId, targetRepoName))
+        val storageKey = targetRepo.storageCredentials?.key ?: "default"
+        val storageHost = storageProperties.redirect.storageHosts[storageKey]
+        val redirectUrlBuilder = StringBuilder()
+        if (!storageHost.isNullOrEmpty()) {
+            context.response.setHeader(HttpHeaders.CONNECTION, "close")
+            redirectUrlBuilder.append(storageHost)
+        }
+
+        // 判断请求来源是浏览器还是API调用
+        if (context.request.getHeader(HEADER_ACCESS_FROM) != ACCESS_FROM_API) {
+            redirectUrlBuilder.append("/web")
+        }
+
+        // 设置重定向地址
+        redirectUrlBuilder.append("/generic/$targetProjectId/$targetRepoName/$targetFullPath")
+        context.response.setHeader(HttpHeaders.LOCATION, redirectUrlBuilder.toString())
+        context.response.status = HttpStatus.MOVED_PERMANENTLY.value
+    }
+
+    private fun isLinkNode(node: NodeDetail?): Boolean {
+        val projectId = node?.metadata?.get(METADATA_KEY_LINK_PROJECT) as? String
+        val repo = node?.metadata?.get(METADATA_KEY_LINK_REPO) as? String
+        val fullPath = node?.metadata?.get(METADATA_KEY_LINK_FULL_PATH) as? String
+        return node != null && !projectId.isNullOrEmpty() && !repo.isNullOrEmpty() && !fullPath.isNullOrEmpty()
+    }
+}

src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/AbstractMongoDao.kt

@@ -44,6 +44,7 @@ import org.springframework.data.mongodb.core.aggregation.AggregationResults
 import org.springframework.data.mongodb.core.mapping.Document
 import org.springframework.data.mongodb.core.query.Query
 import org.springframework.data.mongodb.core.query.Update
+import org.springframework.data.util.CloseableIterator
 import java.lang.reflect.ParameterizedType
 
 /**
@@ -74,6 +75,10 @@ abstract class AbstractMongoDao<E> : MongoDao<E> {
         return findAll(classType)
     }
 
+    fun stream(query: Query): CloseableIterator<E> {
+        return stream(query, classType)
+    }
+
     override fun <T> findOne(query: Query, clazz: Class<T>): T? {
         if (logger.isDebugEnabled) {
             logger.debug("Mongo Dao findOne: [$query] [$clazz]")
@@ -180,6 +185,13 @@ abstract class AbstractMongoDao<E> : MongoDao<E> {
         return determineMongoTemplate().findAndModify(query, update, options, clazz, determineCollectionName(query))
     }
 
+    override fun <T> stream(query: Query, clazz: Class<T>): CloseableIterator<T> {
+        if (logger.isDebugEnabled) {
+            logger.debug("Mongo Dao stream query: [$query]")
+        }
+        return determineMongoTemplate().stream(query, clazz, determineCollectionName(query))
+    }
+
     protected open fun determineCollectionName(): String {
         var collectionName: String? = null
         if (classType.isAnnotationPresent(Document::class.java)) {

src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/MongoDao.kt

@@ -38,6 +38,7 @@ import org.springframework.data.mongodb.core.aggregation.Aggregation
 import org.springframework.data.mongodb.core.aggregation.AggregationResults
 import org.springframework.data.mongodb.core.query.Query
 import org.springframework.data.mongodb.core.query.Update
+import org.springframework.data.util.CloseableIterator
 
 /**
  * mongo db 数据访问层接口
@@ -113,4 +114,9 @@ interface MongoDao<E> {
      * 查询并更新操作
      */
     fun <T> findAndModify(query: Query, update: Update, options: FindAndModifyOptions, clazz: Class<T>): T?
+
+    /**
+     * 流式查询
+     */
+    fun <T> stream(query: Query, clazz: Class<T>): CloseableIterator<T>
 }

src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/config/RedirectProperties.kt

@@ -57,5 +57,12 @@ data class RedirectProperties(
     /**
      * 重定向URL的过期时间
      */
-    var redirectUrlExpireTime: Duration = Duration.ofMinutes(3L)
+    var redirectUrlExpireTime: Duration = Duration.ofMinutes(3L),
+
+    /**
+     * 指定存储在重定向时使用的域名
+     * key为存储Key
+     * value为域名，例如http://bkrepo.example.com
+     */
+    var storageHosts: Map<String, String> = emptyMap()
 )

src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt

@@ -94,6 +94,7 @@ class RouteConfiguration(
             POST("/create$DEFAULT_MAPPING_URI", nodeOperationsHandler::createNode)
             DELETE("/delete$DEFAULT_MAPPING_URI", nodeOperationsHandler::deleteNode)
             POST("/mkdir$DEFAULT_MAPPING_URI", nodeOperationsHandler::mkdir)
+            POST("/mknod$DEFAULT_MAPPING_URI", nodeOperationsHandler::mknod)
             PUT("/set-length$DEFAULT_MAPPING_URI", nodeOperationsHandler::setLength)
             GET("/stat$DEFAULT_MAPPING_URI", nodeOperationsHandler::getStat)
             GET("/info$DEFAULT_MAPPING_URI", nodeOperationsHandler::info)

src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt

@@ -136,13 +136,17 @@ class NodeOperationsHandler(
                 gid = attrMap[NodeAttribute::gid.name] as? String ?: NOBODY,
                 mode = attrMap[NodeAttribute::mode.name] as? Int ?: DEFAULT_MODE,
                 flags = attrMap[NodeAttribute::flags.name] as? Int,
+                rdev = attrMap[NodeAttribute::rdev.name] as? Int,
+                type = attrMap[NodeAttribute::type.name] as? Int
             )
 
             val attributes = NodeAttribute(
                 uid = uid ?: preFsAttribute.uid,
                 gid = gid ?: preFsAttribute.gid,
                 mode = mode ?: preFsAttribute.mode,
                 flags = flags ?: preFsAttribute.flags,
+                rdev = rdev ?: preFsAttribute.rdev,
+                type = type ?: preFsAttribute.type
             )
             val fsAttr = MetadataModel(
                 key = FS_ATTR_KEY,
@@ -243,6 +247,11 @@ class NodeOperationsHandler(
         return ReactiveResponseBuilder.success(node.nodeInfo.toNode())
     }
 
+    suspend fun mknod(request: ServerRequest): ServerResponse {
+        val node = createNode(request, false)
+        return ReactiveResponseBuilder.success(node.nodeInfo.toNode())
+    }
+
     private suspend fun createNode(request: ServerRequest, folder: Boolean): NodeDetail {
         with(NodeRequest(request)) {
             val user = ReactiveSecurityUtils.getUser()
@@ -251,7 +260,9 @@ class NodeOperationsHandler(
                 uid = NOBODY,
                 gid = NOBODY,
                 mode = mode ?: DEFAULT_MODE,
-                flags = flags
+                flags = flags,
+                rdev = rdev,
+                type = type
             )
             val fsAttr = MetadataModel(
                 key = FS_ATTR_KEY,

src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/model/NodeAttribute.kt

@@ -1,10 +1,18 @@
 package com.tencent.bkrepo.fs.server.model
 
 data class NodeAttribute(
+    // 用户id
     val uid: String,
+    // 组id
     val gid: String,
+    // 文件权限，八进制
     val mode: Int? = DEFAULT_MODE,
+    // windows文件flag，十六进制
     val flags: Int? = null,
+    // 设备文件设备号
+    val rdev: Int? = null,
+    // 文件类型
+    val type: Int? = null
 ) {
     companion object {
         const val DEFAULT_MODE = 644

src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/request/NodeRequest.kt

@@ -49,6 +49,8 @@ open class NodeRequest(
     open val category: String = RepositoryCategory.LOCAL.name,
     open val mode: Int? = null,
     open val flags: Int? = null,
+    open val rdev: Int? = null,
+    open val type: Int? = null,
 ) {
     constructor(request: ServerRequest) : this(
         projectId = request.pathVariable(PROJECT_ID),
@@ -57,6 +59,8 @@ open class NodeRequest(
         category = request.queryParamOrNull("category") ?: RepositoryCategory.LOCAL.name,
         mode = request.queryParamOrNull("mode")?.toIntOrNull(),
         flags = request.queryParamOrNull("flags")?.toIntOrNull(),
+        rdev = request.queryParamOrNull("rdev")?.toIntOrNull(),
+        type = request.queryParamOrNull("type")?.toIntOrNull()
     )
 
     override fun toString(): String {

src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/api/OperateResource.kt

@@ -58,6 +58,8 @@ interface OperateResource {
         @ApiParam("是否包含目录", required = false, defaultValue = "false")
         @RequestParam includeFolder: Boolean = true,
         @ApiParam("是否深度查询文件", required = false, defaultValue = "false")
-        @RequestParam deep: Boolean = false
+        @RequestParam deep: Boolean = false,
+        @ApiParam("是否包含元数据", required = false, defaultValue = "false")
+        @RequestParam includeMetadata: Boolean = false,
     ): Response<List<FileInfo>>
 }

src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/pojo/FileInfo.kt

@@ -65,5 +65,7 @@ data class FileInfo(
     @ApiModelProperty("项目ID")
     val projectId: String,
     @ApiModelProperty("仓库名称")
-    val repoName: String
+    val repoName: String,
+    @ApiModelProperty("元数据")
+    val metadata: Map<String, Any>? = null,
 )

src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/OperateResourceImpl.kt

@@ -56,11 +56,14 @@ class OperateResourceImpl @Autowired constructor(
         userId: String,
         artifactInfo: ArtifactInfo,
         includeFolder: Boolean,
-        deep: Boolean
+        deep: Boolean,
+        includeMetadata: Boolean,
     ): Response<List<FileInfo>> {
         return artifactInfo.run {
             val path = getArtifactFullPath()
-            val fileInfoList = operateService.listFile(userId, projectId, repoName, path, includeFolder, deep)
+            val fileInfoList = operateService.listFile(
+                userId, projectId, repoName, path, includeFolder, deep, includeMetadata
+            )
             ResponseBuilder.success(fileInfoList)
         }
     }

src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/OperateService.kt

@@ -50,9 +50,14 @@ class OperateService(
         repoName: String,
         path: String,
         includeFolder: Boolean,
-        deep: Boolean
+        deep: Boolean,
+        includeMetadata: Boolean,
     ): List<FileInfo> {
-        return nodeClient.listNode(projectId, repoName, path, includeFolder, deep).data.orEmpty().map { toFileInfo(it) }
+        return nodeClient
+            .listNode(projectId, repoName, path, includeFolder, deep, includeMetadata)
+            .data
+            .orEmpty()
+            .map { toFileInfo(it) }
     }
 
     companion object {
@@ -71,7 +76,8 @@ class OperateService(
                     sha256 = it.sha256,
                     md5 = it.md5,
                     projectId = it.projectId,
-                    repoName = it.repoName
+                    repoName = it.repoName,
+                    metadata = it.metadata,
                 )
             }
         }