diff --git a/src/backend/ci/buildSrc/src/main/kotlin/constants/Versions.kt b/src/backend/ci/buildSrc/src/main/kotlin/constants/Versions.kt index 30f3648ece5..fca1108c6c4 100644 --- a/src/backend/ci/buildSrc/src/main/kotlin/constants/Versions.kt +++ b/src/backend/ci/buildSrc/src/main/kotlin/constants/Versions.kt @@ -12,7 +12,7 @@ object Versions { const val JsonSchema = "2.2.6" const val Jasypt = "3.0.3" const val Swagger = "1.6.2" - const val orgJson = "20210307" + const val orgJson = "20230618" const val JsonLib = "2.4" const val CronUtils = "9.1.6" const val Thumbnailator = "0.4.8" diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt index cb74eea41cd..9a5ce748d76 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt @@ -174,6 +174,13 @@ object CommonMessageCode { const val ERROR_INTERFACE_RETRY_NUM_EXCEEDED = "2100123" // 接口连续重试次数超过{0}次，请稍后再试 const val ERROR_PIPELINE_API_ACCESS_NO_PERMISSION = "2100124" // 流水线[{0}]没有接口[{1}]的访问权限 const val TEMPLATE_PLUGIN_NOT_ALLOWED_USE = "2100125" // 模板中插件【{0}】的【{1}】版本的状态是【{2}】，不允许使用 + const val ADD_MR_FAIL = "2100126" // 添加MR失败 + + // 互转使用 + const val ELEMENT_UPDATE_WRONG_PATH = "2100127" // 更新插件的标注位置有误 + const val ELEMENT_NOT_SUPPORT_TRANSFER = "2100128" // 如下插件在 Code 方式下已不支持，请修改后再切换: \n[{0}] + const val DISPATCH_NOT_SUPPORT_TRANSFER = "2100129" // 如下构建环境在 Code 方式下不支持转换，请修改后再切换: \n[{0}] + const val YAML_NOT_VALID = "2100130" // yaml不合法 {0} const val GIT_INVALID_PRIVATE_KEY = "2100131" // 不支持的SSH私钥格式，仅支持rsa格式私钥 const val GIT_INVALID_PRIVATE_KEY_OR_PASSWORD = "2100132" // 第三方服务[{0}]操作失败，失败详情：{1} @@ -235,6 +242,8 @@ object CommonMessageCode { const val OPERATION_LIST_WEBHOOK = "bkOperationListWebhook" // 查询WEBHOOK const val OPERATION_ADD_COMMIT_CHECK = "bkOperationAddCommitCheck" // 添加COMMIT CHECK const val OPERATION_ADD_MR_COMMENT = "bkOperationAddMrComment" // 添加MR COMMENT + const val OPERATION_LIST_MR = "bkOperationListMr" // 添加MR + const val OPERATION_ADD_MR = "bkOperationAddMr" // 添加MR const val OPERATION_COMMIT = "bkOperationCommit" // 拉提交记录 const val OPERATION_COMMIT_DIFF = "bkOperationCommitDiff" // 查询commit变化 const val OPERATION_UNLOCK_HOOK_LOCK = "bkOperationUnlockHookLock" // 解锁hook锁 diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/CheckoutRepositoryType.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/CheckoutRepositoryType.kt new file mode 100644 index 00000000000..6e8a1b26336 --- /dev/null +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/CheckoutRepositoryType.kt @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.api.enums + +/** + * 代码库拉取代码库类型 + */ +enum class CheckoutRepositoryType { + ID, + NAME, + URL, + SELF + ; + + companion object { + fun parseType(type: String?): CheckoutRepositoryType { + if (type.isNullOrBlank()) return ID + return valueOf(type) + } + + /** + * 拉取类型跳过定时触发源代码变更检查 + */ + fun skipTimerTriggerChange(type: String?): Boolean { + return when (type) { + URL.name, SELF.name -> true + else -> false + } + } + } +} diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/RepositoryConfig.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/RepositoryConfig.kt index b12a463fa88..19fe35e3484 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/RepositoryConfig.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/RepositoryConfig.kt @@ -30,6 +30,7 @@ package com.tencent.devops.common.api.enums import com.fasterxml.jackson.annotation.JsonIgnore import com.tencent.devops.common.api.exception.ParamBlankException import io.swagger.v3.oas.annotations.media.Schema + import java.net.URLEncoder /** @@ -43,6 +44,25 @@ class RepositoryConfig( @get:Schema(title = "新版的git插件的类型") val repositoryType: RepositoryType ) { + + constructor( + repositoryHashId: String?, + repositoryName: String?, + triggerRepositoryType: TriggerRepositoryType?, + selfRepoHashId: String? + ) : this( + repositoryHashId = + if (triggerRepositoryType == TriggerRepositoryType.SELF) { + selfRepoHashId + } else { + repositoryHashId + }, + repositoryName = repositoryName, + repositoryType = TriggerRepositoryType.toRepositoryType( + triggerRepositoryType + ) ?: RepositoryType.ID + ) + @JsonIgnore fun getRepositoryId(): String { return when (repositoryType) { diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/ScmType.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/ScmType.kt index 6ed2ff1b2af..32d0e905770 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/ScmType.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/ScmType.kt @@ -27,13 +27,13 @@ package com.tencent.devops.common.api.enums -enum class ScmType { - CODE_SVN, - CODE_GIT, - CODE_GITLAB, - GITHUB, - CODE_TGIT, - CODE_P4 +enum class ScmType(val alis: String) { + CODE_SVN("svn"), + CODE_GIT("git"), + CODE_GITLAB("gitlab"), + GITHUB("github"), + CODE_TGIT("tgit"), + CODE_P4("p4") ; companion object { @@ -47,5 +47,13 @@ enum class ScmType { CODE_P4 -> 6.toShort() } } + + fun parse(alis: String?): ScmType? { + if (alis.isNullOrBlank()) return null + values().forEach { + if (alis == it.alis) return it + } + return null + } } } diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/RepositoryTypeNew.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/TriggerRepositoryType.kt similarity index 84% rename from src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/RepositoryTypeNew.kt rename to src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/TriggerRepositoryType.kt index 2fe10833f49..6fd5e4093c5 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/RepositoryTypeNew.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/TriggerRepositoryType.kt @@ -27,16 +27,18 @@ package com.tencent.devops.common.api.enums -enum class RepositoryTypeNew { +enum class TriggerRepositoryType { ID, NAME, - URL - ; + SELF; companion object { - fun parseType(type: String?): RepositoryTypeNew { - if (type.isNullOrBlank()) return ID - return valueOf(type) + fun toRepositoryType(type: TriggerRepositoryType?): RepositoryType? { + return when (type) { + ID, SELF -> RepositoryType.ID + NAME -> RepositoryType.NAME + else -> null + } } } } diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/I18Variable.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/I18Variable.kt index eda192c0f10..ddf54083ac4 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/I18Variable.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/I18Variable.kt @@ -36,7 +36,7 @@ data class I18Variable( @get:Schema(title = "国际化变量名") val code: String, @get:Schema(title = "国际化参数") - val params: List, + val params: List? = emptyList(), @get:Schema(title = "默认信息") val defaultMessage: String? = null ) { diff --git a/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt b/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt index a54b456f535..a698cc85ee0 100644 --- a/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt +++ b/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt @@ -320,4 +320,16 @@ object MQ { // 数据库分片 const val EXCHANGE_SHARDING_ROUTING_RULE_FANOUT = "e.sharding.routing.rule.exchange.fanout" + + // pac每条流水线触发事件 + const val EXCHANGE_PIPELINE_YAML_LISTENER = "e.pipeline.yaml.listener" + // pac开启流水线事件 + const val ROUTE_PIPELINE_YAML_ENABLE_EVENT = "r.pipeline.yaml.enable.event" + const val QUEUE_PIPELINE_YAML_ENABLE_EVENT = "q.pipeline.yaml.enable.event" + // pac触发事件 + const val ROUTE_PIPELINE_YAML_TRIGGER_EVENT = "r.pipeline.yaml.trigger.event" + const val QUEUE_PIPELINE_YAML_TRIGGER_EVENT = "q.pipeline.yaml.trigger.event" + // pac关闭流水线事件 + const val ROUTE_PIPELINE_YAML_DISABLE_EVENT = "r.pipeline.yaml.disable.event" + const val QUEUE_PIPELINE_YAML_DISABLE_EVENT = "q.pipeline.yaml.disable.event" } diff --git a/src/backend/ci/core/common/common-notify/src/main/kotlin/com/tencent/devops/common/notify/enums/NotifyType.kt b/src/backend/ci/core/common/common-notify/src/main/kotlin/com/tencent/devops/common/notify/enums/NotifyType.kt index 834b68f9ed5..8baa32dfafb 100644 --- a/src/backend/ci/core/common/common-notify/src/main/kotlin/com/tencent/devops/common/notify/enums/NotifyType.kt +++ b/src/backend/ci/core/common/common-notify/src/main/kotlin/com/tencent/devops/common/notify/enums/NotifyType.kt @@ -42,7 +42,7 @@ enum class NotifyType { */ companion object { fun opEditable(): List { - return listOf(EMAIL, RTX, WECHAT) + return listOf(EMAIL, RTX, WECHAT, WEWORK_GROUP) } } } diff --git a/src/backend/ci/core/common/common-notify/src/main/kotlin/com/tencent/devops/common/notify/utils/NotifyUtils.kt b/src/backend/ci/core/common/common-notify/src/main/kotlin/com/tencent/devops/common/notify/utils/NotifyUtils.kt index 8883b99e11d..0bcb5a65a69 100644 --- a/src/backend/ci/core/common/common-notify/src/main/kotlin/com/tencent/devops/common/notify/utils/NotifyUtils.kt +++ b/src/backend/ci/core/common/common-notify/src/main/kotlin/com/tencent/devops/common/notify/utils/NotifyUtils.kt @@ -4,12 +4,12 @@ import com.tencent.devops.common.notify.enums.NotifyType object NotifyUtils { const val WEWORK_GROUP_KEY = "__WEWORK_GROUP__" - fun checkNotifyType(notifyType: MutableList?): MutableSet? { + fun checkNotifyType(notifyType: MutableList?): MutableSet { if (notifyType != null) { val allTypeSet = NotifyType.values().map { it.name }.toMutableSet() allTypeSet.remove(NotifyType.SMS.name) return (notifyType.toSet() intersect allTypeSet).toMutableSet() } - return notifyType + return mutableSetOf() } } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/build.gradle.kts b/src/backend/ci/core/common/common-pipeline-yaml/build.gradle.kts index 6a8409de08e..6ff51e0a718 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/build.gradle.kts +++ b/src/backend/ci/core/common/common-pipeline-yaml/build.gradle.kts @@ -29,12 +29,20 @@ dependencies { api(project(":core:common:common-webhook:biz-common-webhook")) api(project(":core:common:common-expression")) api(project(":core:store:api-store")) + api(project(":core:auth:api-auth")) api(project(":core:quality:api-quality")) + api(project(":core:dispatch:api-dispatch-docker")) api("org.apache.ant:ant") api("org.apache.commons:commons-text") - api("org.yaml:snakeyaml") api("com.github.fge:json-schema-validator") + // https://mvnrepository.com/artifact/io.github.java-diff-utils/java-diff-utils + implementation("io.github.java-diff-utils:java-diff-utils:4.12") + // https://mvnrepository.com/artifact/org.json/json + implementation("org.json:json:20230618") + // https://mvnrepository.com/artifact/org.yaml/snakeyaml +// implementation("org.yaml:snakeyaml:2.1") + testImplementation(project(":core:common:common-test")) } plugins { diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelCommon.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelCommon.kt similarity index 98% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelCommon.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelCommon.kt index 84eeaa5a3f7..7750d9df125 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelCommon.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelCommon.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.modelCreate +package com.tencent.devops.process.yaml.creator import com.tencent.devops.common.client.Client import com.tencent.devops.common.pipeline.enums.ChannelCode @@ -42,8 +42,10 @@ object ModelCommon { return when { ref.startsWith("refs/heads/") -> ref.removePrefix("refs/heads/") + ref.startsWith("refs/tags/") -> ref.removePrefix("refs/tags/") + else -> ref } } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelContainer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelContainer.kt similarity index 97% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelContainer.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelContainer.kt index d0ac40162bf..85d039c9165 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelContainer.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelContainer.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.modelCreate +package com.tencent.devops.process.yaml.creator import com.fasterxml.jackson.databind.ObjectMapper import com.tencent.devops.common.api.constant.CommonMessageCode.BK_ENV_NOT_YET_SUPPORTED @@ -47,14 +47,15 @@ import com.tencent.devops.common.pipeline.option.MatrixControlOption import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.process.pojo.BuildTemplateAcrossInfo -import com.tencent.devops.process.yaml.modelCreate.inner.InnerModelCreator +import com.tencent.devops.process.yaml.creator.inner.InnerModelCreator +import com.tencent.devops.process.yaml.transfer.TransferCacheService import com.tencent.devops.process.yaml.pojo.StreamDispatchInfo import com.tencent.devops.process.yaml.utils.ModelCreateUtil -import com.tencent.devops.process.yaml.utils.StreamDispatchUtils import com.tencent.devops.process.yaml.v2.models.IfType import com.tencent.devops.process.yaml.v2.models.Resources import com.tencent.devops.process.yaml.v2.models.job.Job import com.tencent.devops.process.yaml.v2.models.job.Mutex +import com.tencent.devops.process.yaml.v2.utils.StreamDispatchUtils import com.tencent.devops.store.api.container.ServiceContainerAppResource import javax.ws.rs.core.Response import org.springframework.beans.factory.annotation.Autowired @@ -65,7 +66,8 @@ class ModelContainer @Autowired(required = false) constructor( val client: Client, val objectMapper: ObjectMapper, @Autowired(required = false) - val inner: InnerModelCreator? + val inner: InnerModelCreator?, + val transferCache: TransferCacheService ) { fun addVmBuildContainer( diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelCreate.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelCreate.kt similarity index 96% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelCreate.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelCreate.kt index bc6471362ad..0f19dcd012d 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelCreate.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelCreate.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.modelCreate +package com.tencent.devops.process.yaml.creator import com.tencent.devops.common.api.constant.CommonMessageCode.BK_BUILD_TRIGGER import com.tencent.devops.common.api.constant.CommonMessageCode.BK_MANUAL_TRIGGER @@ -43,10 +43,10 @@ import com.tencent.devops.process.engine.common.VMUtils import com.tencent.devops.process.pojo.classify.PipelineGroup import com.tencent.devops.process.pojo.classify.PipelineGroupCreate import com.tencent.devops.process.pojo.classify.PipelineLabelCreate -import com.tencent.devops.process.pojo.setting.PipelineModelAndSetting -import com.tencent.devops.process.pojo.setting.PipelineRunLockType -import com.tencent.devops.process.pojo.setting.PipelineSetting -import com.tencent.devops.process.yaml.modelCreate.inner.ModelCreateEvent +import com.tencent.devops.common.pipeline.pojo.PipelineModelAndSetting +import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType +import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting +import com.tencent.devops.process.yaml.creator.inner.ModelCreateEvent import com.tencent.devops.process.yaml.pojo.QualityElementInfo import com.tencent.devops.process.yaml.v2.models.ScriptBuildYaml import java.util.concurrent.TimeUnit diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelCreateException.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelCreateException.kt similarity index 97% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelCreateException.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelCreateException.kt index 11fbbc379a1..7c09e531746 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelCreateException.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelCreateException.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.modelCreate +package com.tencent.devops.process.yaml.creator /** * ModelCreate产生的异常都应该收归到这里 diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelElement.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelElement.kt similarity index 96% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelElement.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelElement.kt index d23c678b888..03c29e11d28 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelElement.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelElement.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.modelCreate +package com.tencent.devops.process.yaml.creator import com.tencent.devops.common.client.Client import com.tencent.devops.common.pipeline.NameAndValue @@ -36,8 +36,8 @@ import com.tencent.devops.common.pipeline.pojo.element.RunCondition import com.tencent.devops.common.pipeline.pojo.element.agent.LinuxScriptElement import com.tencent.devops.common.pipeline.pojo.element.agent.WindowsScriptElement import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildAtomElement -import com.tencent.devops.process.yaml.modelCreate.inner.InnerModelCreator -import com.tencent.devops.process.yaml.modelCreate.inner.ModelCreateEvent +import com.tencent.devops.process.yaml.creator.inner.InnerModelCreator +import com.tencent.devops.process.yaml.creator.inner.ModelCreateEvent import com.tencent.devops.process.yaml.utils.ModelCreateUtil import com.tencent.devops.process.yaml.utils.PathMatchUtils import com.tencent.devops.process.yaml.v2.models.IfType @@ -170,7 +170,7 @@ class ModelElement @Autowired(required = false) constructor( if (job.runsOn.agentSelector.isNullOrEmpty()) { linux } else { - when (job.runsOn.agentSelector.first()) { + when (job.runsOn.agentSelector!!.first()) { "linux" -> linux "macos" -> linux "windows" -> WindowsScriptElement( diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelStage.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelStage.kt similarity index 98% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelStage.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelStage.kt index 6fcdbfc39b6..19df065848f 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/ModelStage.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelStage.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.modelCreate +package com.tencent.devops.process.yaml.creator import com.fasterxml.jackson.databind.ObjectMapper import com.tencent.devops.common.api.exception.RemoteServiceException @@ -43,15 +43,15 @@ import com.tencent.devops.common.pipeline.pojo.element.atom.ManualReviewParamTyp import com.tencent.devops.common.quality.pojo.enums.QualityOperation import com.tencent.devops.process.engine.common.VMUtils import com.tencent.devops.process.pojo.BuildTemplateAcrossInfo -import com.tencent.devops.process.yaml.modelCreate.inner.InnerModelCreator -import com.tencent.devops.process.yaml.modelCreate.inner.ModelCreateEvent +import com.tencent.devops.process.yaml.creator.inner.InnerModelCreator +import com.tencent.devops.process.yaml.creator.inner.ModelCreateEvent import com.tencent.devops.process.yaml.pojo.QualityElementInfo import com.tencent.devops.process.yaml.utils.ModelCreateUtil import com.tencent.devops.process.yaml.utils.PathMatchUtils import com.tencent.devops.process.yaml.v2.models.Resources import com.tencent.devops.process.yaml.v2.models.job.JobRunsOnType -import com.tencent.devops.process.yaml.v2.stageCheck.ReviewVariable -import com.tencent.devops.process.yaml.v2.stageCheck.StageCheck +import com.tencent.devops.process.yaml.v2.check.ReviewVariable +import com.tencent.devops.process.yaml.v2.check.StageCheck import com.tencent.devops.quality.api.v2.pojo.ControlPointPosition import com.tencent.devops.quality.api.v3.ServiceQualityRuleResource import com.tencent.devops.quality.api.v3.pojo.request.RuleCreateRequestV3 @@ -63,6 +63,7 @@ import org.springframework.util.AntPathMatcher import com.tencent.devops.process.yaml.v2.models.stage.Stage as StreamV2Stage @Component +@Suppress("ComplexMethod") class ModelStage @Autowired(required = false) constructor( val client: Client, val objectMapper: ObjectMapper, diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/inner/InnerModelCreator.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/inner/InnerModelCreator.kt similarity index 98% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/inner/InnerModelCreator.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/inner/InnerModelCreator.kt index 55461c796d1..9aa91f43d78 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/inner/InnerModelCreator.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/inner/InnerModelCreator.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.modelCreate.inner +package com.tencent.devops.process.yaml.creator.inner import com.tencent.devops.common.client.Client import com.tencent.devops.common.pipeline.pojo.element.ElementAdditionalOptions diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/inner/ModelCreateEvent.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/inner/ModelCreateEvent.kt similarity index 98% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/inner/ModelCreateEvent.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/inner/ModelCreateEvent.kt index de0b553202c..49bdf31cbaf 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/modelCreate/inner/ModelCreateEvent.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/inner/ModelCreateEvent.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.modelCreate.inner +package com.tencent.devops.process.yaml.creator.inner import com.tencent.devops.process.pojo.BuildTemplateAcrossInfo import com.tencent.devops.process.yaml.v2.enums.StreamObjectKind diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/pojo/TemplatePath.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/pojo/TemplatePath.kt new file mode 100644 index 00000000000..fc85bfac630 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/pojo/TemplatePath.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.pojo + +data class TemplatePath( + val path: String, + val ref: String? = null +) { + override fun toString(): String { + return path + if (ref != null) "[$ref]" else "" + } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/pojo/ThirdPartyContainerInfo.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/pojo/ThirdPartyContainerInfo.kt index bf6b3a6fadc..232677e13fd 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/pojo/ThirdPartyContainerInfo.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/pojo/ThirdPartyContainerInfo.kt @@ -28,6 +28,7 @@ package com.tencent.devops.process.yaml.pojo import com.tencent.devops.common.pipeline.type.agent.DockerOptions +import com.tencent.devops.common.pipeline.type.docker.ImageType data class ThirdPartyContainerInfo( val image: String, @@ -36,5 +37,6 @@ data class ThirdPartyContainerInfo( val credId: String?, val acrossTemplateId: String?, val options: DockerOptions?, - val imagePullPolicy: String? + val imagePullPolicy: String?, + val imageType: ImageType? = ImageType.THIRD ) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/pojo/ThirdPartyContainerInfoV3.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/pojo/ThirdPartyContainerInfoV3.kt new file mode 100644 index 00000000000..a5973c74e15 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/pojo/ThirdPartyContainerInfoV3.kt @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.pojo + +import com.tencent.devops.common.pipeline.type.agent.DockerOptions +import com.tencent.devops.common.pipeline.type.docker.ImageType + +data class ThirdPartyContainerInfoV3( + val image: String?, + val imageCode: String?, + val imageVersion: String?, + val userName: String?, + val password: String?, + val credId: String?, + val acrossTemplateId: String?, + val options: DockerOptions?, + val imagePullPolicy: String?, + val imageType: ImageType? = ImageType.THIRD +) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/pojo/YamlVersionParser.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/pojo/YamlVersionParser.kt new file mode 100644 index 00000000000..c2dbc28777a --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/pojo/YamlVersionParser.kt @@ -0,0 +1,24 @@ +package com.tencent.devops.process.yaml.pojo + +interface YamlVersionParser { + + fun yamlVersion(): YamlVersion +} + +enum class YamlVersion(val tag: String) { + V2_0("v2.0"), + V3_0("v3.0"); + + companion object { + const val V2 = "v2.0" + const val V3 = "v3.0" + + fun parse(yamlTag: String): YamlVersion? { + return when (yamlTag) { + V2 -> V2_0 + V3 -> V3_0 + else -> null + } + } + } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ContainerTransfer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ContainerTransfer.kt new file mode 100644 index 00000000000..33812d46fb8 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ContainerTransfer.kt @@ -0,0 +1,441 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.transfer + +import com.fasterxml.jackson.databind.ObjectMapper +import com.tencent.devops.common.api.constant.CommonMessageCode +import com.tencent.devops.common.api.exception.OperationException +import com.tencent.devops.common.api.util.YamlUtil +import com.tencent.devops.common.client.Client +import com.tencent.devops.common.pipeline.container.Container +import com.tencent.devops.common.pipeline.container.MutexGroup +import com.tencent.devops.common.pipeline.container.NormalContainer +import com.tencent.devops.common.pipeline.container.VMBuildContainer +import com.tencent.devops.common.pipeline.enums.DependOnType +import com.tencent.devops.common.pipeline.enums.JobRunCondition +import com.tencent.devops.common.pipeline.option.JobControlOption +import com.tencent.devops.common.pipeline.option.MatrixControlOption +import com.tencent.devops.common.pipeline.pojo.element.Element +import com.tencent.devops.common.pipeline.pojo.transfer.IfType +import com.tencent.devops.common.pipeline.pojo.transfer.PreStep +import com.tencent.devops.common.pipeline.pojo.transfer.Resources +import com.tencent.devops.common.pipeline.type.BuildType +import com.tencent.devops.common.pipeline.type.StoreDispatchType +import com.tencent.devops.common.pipeline.type.docker.ImageType +import com.tencent.devops.common.pipeline.utils.TransferUtil +import com.tencent.devops.process.pojo.BuildTemplateAcrossInfo +import com.tencent.devops.process.yaml.transfer.VariableDefault.DEFAULT_CONTINUE_WHEN_FAILED +import com.tencent.devops.process.yaml.transfer.VariableDefault.DEFAULT_JOB_MAX_QUEUE_MINUTES +import com.tencent.devops.process.yaml.transfer.VariableDefault.DEFAULT_MUTEX_QUEUE_LENGTH +import com.tencent.devops.process.yaml.transfer.VariableDefault.DEFAULT_MUTEX_TIMEOUT_MINUTES +import com.tencent.devops.process.yaml.transfer.VariableDefault.nullIfDefault +import com.tencent.devops.process.yaml.transfer.inner.TransferCreator +import com.tencent.devops.process.yaml.v3.models.job.Container3 +import com.tencent.devops.process.yaml.v3.models.job.Job +import com.tencent.devops.process.yaml.v3.models.job.JobRunsOnPoolType +import com.tencent.devops.process.yaml.v3.models.job.JobRunsOnType +import com.tencent.devops.process.yaml.v3.models.job.Mutex +import com.tencent.devops.process.yaml.v3.models.job.PreJob +import com.tencent.devops.process.yaml.v3.models.job.RunsOn +import com.tencent.devops.process.yaml.v3.models.job.Strategy +import org.json.JSONObject +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +@Component +@Suppress("NestedBlockDepth", "ComplexMethod") +class ContainerTransfer @Autowired(required = false) constructor( + val client: Client, + val objectMapper: ObjectMapper, + val transferCache: TransferCacheService, + val dispatchTransfer: DispatchTransfer, + val inner: TransferCreator +) { + + private val defaultRunsOn = JSONObject( + RunsOn( + selfHosted = null, + poolType = null, + poolName = JobRunsOnType.DOCKER.type, + container = Container3( + imageCode = inner.defaultImageCode, + imageVersion = inner.defaultImageVersion + ) + ) + ) + + fun addVmBuildContainer( + job: Job, + elementList: List, + containerList: MutableList, + jobIndex: Int, + projectCode: String, + userId: String, + finalStage: Boolean = false, + jobEnable: Boolean = true, + resources: Resources? = null, + buildTemplateAcrossInfo: BuildTemplateAcrossInfo? + ) { + val buildEnv = if (job.runsOn.selfHosted == false) job.runsOn.needs?.ifEmpty { null } else null + val (dispatchType, baseOS) = kotlin.runCatching { + dispatchTransfer.makeDispatchType( + job = job, + buildTemplateAcrossInfo = buildTemplateAcrossInfo + ) + }.onFailure { + if (it is OperationException) { + throw PipelineTransferException( + CommonMessageCode.YAML_NOT_VALID, + arrayOf("${it.message}") + ) + } + }.getOrThrow() + if (dispatchType is StoreDispatchType && dispatchType.imageType == ImageType.BKSTORE) { + val imageName = transferCache.getStoreImageDetail( + userId, dispatchType.imageCode!!, dispatchType.imageVersion + )?.name + dispatchType.imageName = imageName + } + val vmContainer = VMBuildContainer( + jobId = job.id, + name = job.name ?: "Job-${jobIndex + 1}", + elements = elementList, + mutexGroup = getMutexModel(job.mutex), + baseOS = baseOS, + vmNames = setOf(), + maxQueueMinutes = DEFAULT_JOB_MAX_QUEUE_MINUTES, + maxRunningMinutes = job.timeoutMinutes?.toIntOrNull() ?: VariableDefault.DEFAULT_JOB_MAX_RUNNING_MINUTES, + buildEnv = buildEnv, + customBuildEnv = job.env, + jobControlOption = getJobControlOption( + job = job, jobEnable = jobEnable, finalStage = finalStage + ), + dispatchType = dispatchType, + matrixGroupFlag = job.strategy != null, + matrixControlOption = getMatrixControlOption(job) + ).apply { + nfsSwitch = buildEnv != null + } + containerList.add(vmContainer) + } + + fun addNormalContainer( + job: Job, + elementList: List, + containerList: MutableList, + jobIndex: Int, + jobEnable: Boolean = true, + finalStage: Boolean = false + ) { + + containerList.add( + NormalContainer( + jobId = job.id, + name = job.name ?: "Job-${jobIndex + 1}", + elements = elementList, + jobControlOption = getJobControlOption( + job = job, jobEnable = jobEnable, finalStage = finalStage + ), + mutexGroup = getMutexModel(job.mutex), + matrixGroupFlag = job.strategy != null, + matrixControlOption = getMatrixControlOption(job) + ) + ) + } + + fun addYamlNormalContainer( + job: NormalContainer, + steps: List? + ): PreJob { + return PreJob( + name = job.name, + runsOn = RunsOn( + selfHosted = null, + poolName = JobRunsOnType.AGENT_LESS.type, + poolType = null + ), + mutex = getMutexYaml(job.mutexGroup), + container = null, + ifField = when (job.jobControlOption?.runCondition) { + JobRunCondition.CUSTOM_CONDITION_MATCH -> job.jobControlOption?.customCondition + JobRunCondition.CUSTOM_VARIABLE_MATCH -> TransferUtil.customVariableMatch( + job.jobControlOption?.customVariables + ) + + JobRunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN -> TransferUtil.customVariableMatchNotRun( + job.jobControlOption?.customVariables + ) + + else -> null + }, + steps = steps, + timeoutMinutes = makeJobTimeout(job.jobControlOption), + env = null, + continueOnError = job.jobControlOption?.continueWhenFailed.nullIfDefault(DEFAULT_CONTINUE_WHEN_FAILED), + strategy = if (job.matrixGroupFlag == true) { + getMatrixFromJob(job.matrixControlOption) + } else null, + // 蓝盾这边是自定义Job ID + dependOn = when (job.jobControlOption?.dependOnType) { + DependOnType.ID -> job.jobControlOption?.dependOnId?.ifEmpty { null } + DependOnType.NAME -> job.jobControlOption?.dependOnName?.ifBlank { null }?.split(",") + else -> null + } + ) + } + + fun addYamlVMBuildContainer( + userId: String, + projectId: String, + job: VMBuildContainer, + steps: List? + ): PreJob { + return PreJob( + name = job.name, + runsOn = dispatchTransfer.makeRunsOn(job)?.fix(userId, projectId, job.dispatchType?.buildType()), + container = null, + services = null, + mutex = getMutexYaml(job.mutexGroup), + ifField = when (job.jobControlOption?.runCondition) { + JobRunCondition.CUSTOM_CONDITION_MATCH -> job.jobControlOption?.customCondition + JobRunCondition.CUSTOM_VARIABLE_MATCH -> TransferUtil.customVariableMatch( + job.jobControlOption?.customVariables + ) + + JobRunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN -> TransferUtil.customVariableMatchNotRun( + job.jobControlOption?.customVariables + ) + + else -> null + }, + steps = steps, + timeoutMinutes = makeJobTimeout(job.jobControlOption), + env = null, + continueOnError = job.jobControlOption?.continueWhenFailed.nullIfDefault(DEFAULT_CONTINUE_WHEN_FAILED), + strategy = if (job.matrixGroupFlag == true) { + getMatrixFromJob(job.matrixControlOption) + } else null, + dependOn = when (job.jobControlOption?.dependOnType) { + DependOnType.ID -> job.jobControlOption?.dependOnId?.ifEmpty { null } + DependOnType.NAME -> job.jobControlOption?.dependOnName?.ifBlank { null }?.split(",") + else -> null + } + ) + } + + private fun RunsOn.fix(userId: String, projectId: String, buildType: BuildType?): RunsOn? { + /*修正私有构建机数据*/ + when (poolType) { + JobRunsOnPoolType.AGENT_ID.name -> { + nodeName = transferCache.getThirdPartyAgent( + poolType = JobRunsOnPoolType.AGENT_ID, + userId = userId, + projectId = projectId, + value = nodeName + ) ?: throw PipelineTransferException( + CommonMessageCode.DISPATCH_NOT_SUPPORT_TRANSFER, + arrayOf("agentId: $nodeName") + ) + } + + JobRunsOnPoolType.ENV_ID.name -> { + poolName = transferCache.getThirdPartyAgent( + poolType = JobRunsOnPoolType.ENV_ID, + userId = userId, + projectId = envProjectId ?: projectId, + value = poolName + ) ?: throw PipelineTransferException( + CommonMessageCode.DISPATCH_NOT_SUPPORT_TRANSFER, + arrayOf("envId: $poolName") + ) + } + } + + /*修正docker配额数据*/ + if (hwSpec != null && buildType != null) { + kotlin.run { + val res = transferCache.getDockerResource(userId, projectId, buildType) + // hwSpec为0时为特殊值，表示默认配置 + if (res?.default == hwSpec || hwSpec == "0") { + hwSpec = null + return@run + } + val hw = res?.dockerResourceOptionsMaps?.find { + it.id == hwSpec + } + hwSpec = hw?.dockerResourceOptionsShow?.description ?: throw PipelineTransferException( + CommonMessageCode.DISPATCH_NOT_SUPPORT_TRANSFER, + arrayOf("poolName:$poolName,hwSpec:$hwSpec") + ) + } + } + if (JSONObject(this).similar(defaultRunsOn)) { + return null + } + return this + } + + private fun makeJobTimeout(controlOption: JobControlOption?): String? { + return controlOption?.timeoutVar.nullIfDefault( + VariableDefault.DEFAULT_JOB_MAX_RUNNING_MINUTES.toString() + ) ?: controlOption?.timeout.nullIfDefault(VariableDefault.DEFAULT_JOB_MAX_RUNNING_MINUTES)?.toString() + } + + @Suppress("UNCHECKED_CAST") + private fun getMatrixControlOption(job: Job): MatrixControlOption? { + + val strategy = job.strategy ?: return null + + with(strategy) { + if (matrix is Map<*, *>) { + val yaml = matrix as MutableMap + val include = if ("include" in yaml.keys && yaml["include"] != null) { + YamlUtil.toYaml(yaml["include"]!!) + } else { + null + } + val exclude = if ("exclude" in yaml.keys && yaml["exclude"] != null) { + YamlUtil.toYaml(yaml["exclude"]!!) + } else { + null + } + val json = matrix + json.remove("include") + json.remove("exclude") + + return MatrixControlOption( + strategyStr = YamlUtil.toYaml(json), + includeCaseStr = include, + excludeCaseStr = exclude, + fastKill = fastKill, + maxConcurrency = maxParallel + ) + } else { + return MatrixControlOption( + strategyStr = matrix.toString(), + fastKill = fastKill, + maxConcurrency = maxParallel + ) + } + } + } + + private fun getJobControlOption( + job: Job, + jobEnable: Boolean = true, + finalStage: Boolean = false + ): JobControlOption { + val timeout = job.timeoutMinutes?.toIntOrNull() ?: VariableDefault.DEFAULT_JOB_MAX_RUNNING_MINUTES + val timeoutVar = job.timeoutMinutes ?: VariableDefault.DEFAULT_JOB_MAX_RUNNING_MINUTES.toString() + + val dependOnName = job.dependOn?.joinToString(",") + + return if (finalStage) { + JobControlOption( + timeout = timeout, + timeoutVar = timeoutVar, + runCondition = when (job.ifField) { + IfType.SUCCESS.name -> JobRunCondition.PREVIOUS_STAGE_SUCCESS + IfType.FAILURE.name -> JobRunCondition.PREVIOUS_STAGE_FAILED + IfType.CANCELLED.name, IfType.CANCELED.name -> JobRunCondition.PREVIOUS_STAGE_CANCEL + else -> JobRunCondition.STAGE_RUNNING + }, + dependOnType = DependOnType.NAME, + dependOnName = dependOnName, + prepareTimeout = job.runsOn.queueTimeoutMinutes ?: VariableDefault.DEFAULT_JOB_PREPARE_TIMEOUT, + continueWhenFailed = job.continueOnError ?: DEFAULT_CONTINUE_WHEN_FAILED + ) + } else { + val runCondition = kotlin.run { + if (!job.ifField.isNullOrBlank()) JobRunCondition.CUSTOM_CONDITION_MATCH else null + } ?: JobRunCondition.STAGE_RUNNING + JobControlOption( + enable = jobEnable, + timeout = timeout, + timeoutVar = timeoutVar, + runCondition = runCondition, + customCondition = if (runCondition == JobRunCondition.CUSTOM_CONDITION_MATCH) { + job.ifField + } else { + null + }, + dependOnType = DependOnType.NAME, + dependOnName = dependOnName, + prepareTimeout = job.runsOn.queueTimeoutMinutes ?: VariableDefault.DEFAULT_JOB_PREPARE_TIMEOUT, + continueWhenFailed = job.continueOnError ?: DEFAULT_CONTINUE_WHEN_FAILED + ) + } + } + + private fun getMutexModel(resource: Mutex?): MutexGroup? { + if (resource == null) { + return null + } + return MutexGroup( + enable = true, + mutexGroupName = resource.label, + queueEnable = resource.queueLength != null, + queue = resource.queueLength ?: DEFAULT_MUTEX_QUEUE_LENGTH, + timeout = resource.timeoutMinutes?.toIntOrNull() ?: DEFAULT_MUTEX_TIMEOUT_MINUTES, + timeoutVar = resource.timeoutMinutes ?: DEFAULT_MUTEX_TIMEOUT_MINUTES.toString() + ) + } + + private fun getMutexYaml(resource: MutexGroup?): Mutex? { + if (resource?.mutexGroupName.isNullOrBlank()) { + return null + } + return Mutex( + label = resource?.mutexGroupName!!, + queueLength = if (resource.queueEnable) { + resource.queue + } else { + null + }, + timeoutMinutes = if (resource.queueEnable) { + resource.timeoutVar.nullIfDefault(DEFAULT_MUTEX_TIMEOUT_MINUTES.toString()) + ?: resource.timeout.nullIfDefault(DEFAULT_MUTEX_TIMEOUT_MINUTES)?.toString() + } else { + null + } + ) + } + + private fun getMatrixFromJob( + matrixControlOption: MatrixControlOption? + ): Strategy? { + if (matrixControlOption == null) { + return null + } + return Strategy( + matrix = matrixControlOption.convertMatrixToYamlConfig() ?: return null, + fastKill = matrixControlOption.fastKill, + maxParallel = matrixControlOption.maxConcurrency + ) + } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/DispatchTransfer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/DispatchTransfer.kt new file mode 100644 index 00000000000..ed62b451422 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/DispatchTransfer.kt @@ -0,0 +1,217 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.transfer + +import com.fasterxml.jackson.databind.ObjectMapper +import com.tencent.devops.common.api.constant.CommonMessageCode +import com.tencent.devops.common.client.Client +import com.tencent.devops.common.pipeline.container.VMBuildContainer +import com.tencent.devops.common.pipeline.enums.VMBaseOS +import com.tencent.devops.common.pipeline.type.DispatchType +import com.tencent.devops.common.pipeline.type.agent.Credential +import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentDockerInfo +import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentEnvDispatchType +import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentIDDispatchType +import com.tencent.devops.process.pojo.BuildTemplateAcrossInfo +import com.tencent.devops.process.yaml.transfer.VariableDefault.DEFAULT_JOB_PREPARE_TIMEOUT +import com.tencent.devops.process.yaml.transfer.VariableDefault.nullIfDefault +import com.tencent.devops.process.yaml.transfer.inner.TransferCreator +import com.tencent.devops.process.yaml.v3.models.image.Pool +import com.tencent.devops.process.yaml.v3.models.image.PoolImage +import com.tencent.devops.process.yaml.v3.models.image.PoolType +import com.tencent.devops.process.yaml.v3.models.job.Job +import com.tencent.devops.process.yaml.v3.models.job.RunsOn +import com.tencent.devops.process.yaml.v3.utils.StreamDispatchUtils +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +@Component +class DispatchTransfer @Autowired(required = false) constructor( + val client: Client, + val objectMapper: ObjectMapper, + val inner: TransferCreator +) { + + companion object { + private val logger = LoggerFactory.getLogger(DispatchTransfer::class.java) + val LINUX_TYPE = setOf("docker", "linux") + val MACOS_TYPE = setOf("macos-11.4", "macos-12.4", "macos-latest", "macos") + val WINDOWS_TYPE = setOf("windows-2016", "windows") + } + + fun makeDispatchType( + job: Job, + buildTemplateAcrossInfo: BuildTemplateAcrossInfo? + ): Pair { + // linux构建机 + dispatcherLinux(job, buildTemplateAcrossInfo)?.let { return Pair(it, VMBaseOS.LINUX) } + // 第三方构建机 + dispatcherThirdPartyAgent(job, buildTemplateAcrossInfo)?.let { return Pair(it, getBaseOs(job)) } + // windows构建机 + dispatcherWindows(job)?.let { return Pair(it, VMBaseOS.WINDOWS) } + // macos构建机 + dispatcherMacos(job)?.let { return Pair(it, VMBaseOS.MACOS) } + // 转换失败 + throw PipelineTransferException( + CommonMessageCode.DISPATCH_NOT_SUPPORT_TRANSFER, + arrayOf("job: ${job.name}") + ) + } + + fun makeRunsOn( + job: VMBuildContainer + ): RunsOn? { + val dispatchType = job.dispatchType + if (dispatchType == null) { + logger.warn("job.dispatchType can not be null") + return null + } + val runsOn = dispatch2RunsOn(dispatchType) ?: throw PipelineTransferException( + CommonMessageCode.DISPATCH_NOT_SUPPORT_TRANSFER, + arrayOf(dispatchType.buildType().name) + ) + if (dispatchType is ThirdPartyAgentEnvDispatchType || dispatchType is ThirdPartyAgentIDDispatchType) { + runsOn.agentSelector = when (job.baseOS) { + VMBaseOS.WINDOWS -> listOf("windows") + VMBaseOS.LINUX -> listOf("linux") + VMBaseOS.MACOS -> listOf("macos") + else -> null + } + } + runsOn.needs = job.buildEnv?.ifEmpty { null } + runsOn.queueTimeoutMinutes = job.jobControlOption?.prepareTimeout?.nullIfDefault(DEFAULT_JOB_PREPARE_TIMEOUT) + return runsOn + } + + fun dispatcherLinux( + job: Job, + buildTemplateAcrossInfo: BuildTemplateAcrossInfo? + ): DispatchType? { + // 公共docker构建机 + if (job.runsOn.checkLinux()) { + val info = if (job.runsOn.container != null) { + StreamDispatchUtils.parseRunsOnContainer( + job = job, + buildTemplateAcrossInfo = buildTemplateAcrossInfo + ) + } else null + return PoolType.DockerOnVm.toDispatchType( + Pool( + credentialId = getDockerInfo(job, buildTemplateAcrossInfo)?.credential?.credentialId, + image = PoolImage( + imageCode = info?.imageCode ?: inner.defaultImageCode, + imageVersion = info?.imageVersion ?: inner.defaultImageVersion, + imageType = info?.imageType + ), + performanceConfigId = job.runsOn.hwSpec + ) + ) + } + return null + } + + fun dispatcherMacos( + job: Job + ): DispatchType? { + return null + } + + fun dispatcherWindows( + job: Job + ): DispatchType? { + return null + } + + fun dispatcherThirdPartyAgent( + job: Job, + buildTemplateAcrossInfo: BuildTemplateAcrossInfo? + ): DispatchType? { + // 第三方构建机 + if (job.runsOn.selfHosted == true) { + return PoolType.SelfHosted.toDispatchType( + with(job.runsOn) { + Pool( + envName = poolName, + workspace = workspace, + agentName = nodeName, + dockerInfo = getDockerInfo(job, buildTemplateAcrossInfo), + envProjectId = envProjectId + ) + } + ) + } + return null + } + + private fun getDockerInfo( + job: Job, + buildTemplateAcrossInfo: BuildTemplateAcrossInfo? + ): ThirdPartyAgentDockerInfo? { + return if (job.runsOn.container != null) { + val info = StreamDispatchUtils.parseRunsOnContainer( + job = job, + buildTemplateAcrossInfo = buildTemplateAcrossInfo + ) + ThirdPartyAgentDockerInfo( + image = info.image ?: "", + credential = Credential( + user = info.userName, + password = info.password, + credentialId = info.credId, + acrossTemplateId = info.acrossTemplateId, + jobId = job.id + ), + options = info.options, + imagePullPolicy = info.imagePullPolicy + ) + } else null + } + + fun dispatch2RunsOn(dispatcher: DispatchType) = + PoolType.SelfHosted.toRunsOn(dispatcher) + ?: PoolType.DockerOnVm.toRunsOn(dispatcher) + + private fun getBaseOs(job: Job): VMBaseOS { + val poolName = job.runsOn.poolName + when { + LINUX_TYPE.contains(poolName) -> return VMBaseOS.LINUX + MACOS_TYPE.contains(poolName) -> return VMBaseOS.MACOS + WINDOWS_TYPE.contains(poolName) -> return VMBaseOS.WINDOWS + } + + val selector = job.runsOn.agentSelector?.get(0) + return when { + selector == null -> VMBaseOS.ALL + LINUX_TYPE.contains(selector) -> VMBaseOS.LINUX + MACOS_TYPE.contains(selector) -> VMBaseOS.MACOS + WINDOWS_TYPE.contains(selector) -> VMBaseOS.WINDOWS + else -> VMBaseOS.LINUX + } + } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ElementTransfer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ElementTransfer.kt new file mode 100644 index 00000000000..9b95d3d04eb --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ElementTransfer.kt @@ -0,0 +1,598 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.transfer + +import com.fasterxml.jackson.core.type.TypeReference +import com.tencent.devops.common.api.enums.ScmType +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.client.Client +import com.tencent.devops.common.pipeline.NameAndValue +import com.tencent.devops.common.pipeline.container.Container +import com.tencent.devops.common.pipeline.enums.BuildScriptType +import com.tencent.devops.common.pipeline.enums.CharsetType +import com.tencent.devops.common.pipeline.pojo.element.Element +import com.tencent.devops.common.pipeline.pojo.element.ElementAdditionalOptions +import com.tencent.devops.common.pipeline.pojo.element.RunCondition +import com.tencent.devops.common.pipeline.pojo.element.agent.LinuxScriptElement +import com.tencent.devops.common.pipeline.pojo.element.agent.ManualReviewUserTaskElement +import com.tencent.devops.common.pipeline.pojo.element.agent.WindowsScriptElement +import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildAtomElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeGitWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeGithubWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeGitlabWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeP4WebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeSVNWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeTGitWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.ManualTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.RemoteTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.TimerTriggerElement +import com.tencent.devops.common.pipeline.pojo.transfer.IfType +import com.tencent.devops.common.pipeline.pojo.transfer.PreStep +import com.tencent.devops.common.pipeline.pojo.transfer.RunAtomParam +import com.tencent.devops.common.pipeline.utils.TransferUtil +import com.tencent.devops.process.yaml.creator.ModelCreateException +import com.tencent.devops.process.yaml.transfer.VariableDefault.nullIfDefault +import com.tencent.devops.process.yaml.transfer.aspect.PipelineTransferAspectWrapper +import com.tencent.devops.process.yaml.transfer.inner.TransferCreator +import com.tencent.devops.process.yaml.transfer.pojo.CheckoutAtomParam +import com.tencent.devops.process.yaml.transfer.pojo.WebHookTriggerElementChanger +import com.tencent.devops.process.yaml.transfer.pojo.YamlTransferInput +import com.tencent.devops.process.yaml.v3.models.TriggerType +import com.tencent.devops.process.yaml.v3.models.job.Job +import com.tencent.devops.process.yaml.v3.models.job.JobRunsOnType +import com.tencent.devops.process.yaml.v3.models.on.EnableType +import com.tencent.devops.process.yaml.v3.models.on.ManualRule +import com.tencent.devops.process.yaml.v3.models.on.RemoteRule +import com.tencent.devops.process.yaml.v3.models.on.SchedulesRule +import com.tencent.devops.process.yaml.v3.models.on.TriggerOn +import com.tencent.devops.process.yaml.v3.models.step.PreCheckoutStep +import com.tencent.devops.process.yaml.v3.models.step.PreManualReviewUserTaskElement +import com.tencent.devops.process.yaml.v3.models.step.Step +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +@Component +@Suppress("ComplexMethod") +class ElementTransfer @Autowired(required = false) constructor( + val client: Client, + @Autowired(required = false) + val creator: TransferCreator, + val transferCache: TransferCacheService, + val triggerTransfer: TriggerTransfer +) { + companion object { + private val logger = LoggerFactory.getLogger(ElementTransfer::class.java) + } + + fun yaml2Triggers(yamlInput: YamlTransferInput, elements: MutableList) { + yamlInput.yaml.formatTriggerOn(yamlInput.defaultScmType).forEach { + yamlInput.aspectWrapper.setYamlTriggerOn(it.second, PipelineTransferAspectWrapper.AspectType.BEFORE) + when (it.first) { + TriggerType.BASE -> triggerTransfer.yaml2TriggerBase(yamlInput, it.second, elements) + TriggerType.CODE_GIT -> triggerTransfer.yaml2TriggerGit(it.second, elements) + TriggerType.CODE_TGIT -> triggerTransfer.yaml2TriggerTGit(it.second, elements) + TriggerType.GITHUB -> triggerTransfer.yaml2TriggerGithub(it.second, elements) + TriggerType.CODE_SVN -> triggerTransfer.yaml2TriggerSvn(it.second, elements) + TriggerType.CODE_P4 -> triggerTransfer.yaml2TriggerP4(it.second, elements) + TriggerType.CODE_GITLAB -> triggerTransfer.yaml2TriggerGitlab(it.second, elements) + } + yamlInput.aspectWrapper.setModelElement4Model( + elements.last(), + PipelineTransferAspectWrapper.AspectType.AFTER + ) + } + } + + fun baseTriggers2yaml(elements: List, aspectWrapper: PipelineTransferAspectWrapper): TriggerOn? { + val triggerOn = lazy { TriggerOn() } + val schedules = mutableListOf() + triggerOn.value.manual = ManualRule( + enable = false + ) + elements.forEach { element -> + aspectWrapper.setModelElement4Model( + element, + PipelineTransferAspectWrapper.AspectType.BEFORE + ) + if (element is ManualTriggerElement) { + triggerOn.value.manual = ManualRule( + name = element.name, + enable = element.isElementEnable().nullIfDefault(true), + canElementSkip = element.canElementSkip.nullIfDefault(false), + useLatestParameters = element.useLatestParameters.nullIfDefault(false) + ) + return@forEach + } + if (element is TimerTriggerElement) { + val timePoints = element.newExpression?.map { + val (_, m, h) = it.split(" ") + "${h.padStart(2, '0')}:${m.padStart(2, '0')}" + } + val week: List? = element.newExpression + ?.firstOrNull() + ?.split(" ") + ?.get(5) + ?.split(",") + ?.map m@{ w -> + when (w) { + "1" -> "Sun" + "2" -> "Mon" + "3" -> "Tue" + "4" -> "Wed" + "5" -> "Thu" + "6" -> "Fri" + "7" -> "Sat" + else -> return@m "" + } + } + schedules.add( + SchedulesRule( + name = element.name, + interval = week?.let { SchedulesRule.Interval(week, timePoints) }, + cron = if (element.advanceExpression?.size == 1) { + element.advanceExpression?.first() + } else { + element.advanceExpression + }, + repoId = element.repoHashId, + repoName = element.repoName, + branches = element.branches, + always = (element.noScm != true).nullIfDefault(false), + enable = element.isElementEnable().nullIfDefault(true) + ) + ) + return@forEach + } + if (element is RemoteTriggerElement) { + triggerOn.value.remote = if (element.isElementEnable()) { + RemoteRule(element.name, EnableType.TRUE.value) + } else { + RemoteRule(element.name, EnableType.FALSE.value) + } + } + } + if (schedules.isNotEmpty()) { + triggerOn.value.schedules = schedules + } + if (triggerOn.isInitialized()) { + aspectWrapper.setYamlTriggerOn( + triggerOn.value, + PipelineTransferAspectWrapper.AspectType.AFTER + ) + return triggerOn.value + } + return null + } + + fun scmTriggers2Yaml( + elements: List, + projectId: String, + aspectWrapper: PipelineTransferAspectWrapper + ): Map> { + val res = mutableMapOf>() + val fix = elements.groupBy { it.getClassType() } + + val gitElement = fix[CodeGitWebHookTriggerElement.classType]?.map { + aspectWrapper.setModelElement4Model(it, PipelineTransferAspectWrapper.AspectType.BEFORE) + WebHookTriggerElementChanger(it as CodeGitWebHookTriggerElement) + } + if (!gitElement.isNullOrEmpty()) { + val gitTrigger = triggerTransfer.git2YamlTriggerOn( + elements = gitElement, + projectId = projectId, + aspectWrapper = aspectWrapper, + defaultName = "Git事件触发" + ) + res.putAll(gitTrigger.groupBy { ScmType.CODE_GIT }) + } + + val tGitElement = fix[CodeTGitWebHookTriggerElement.classType]?.map { + aspectWrapper.setModelElement4Model(it, PipelineTransferAspectWrapper.AspectType.BEFORE) + WebHookTriggerElementChanger(it as CodeTGitWebHookTriggerElement) + } + if (!tGitElement.isNullOrEmpty()) { + val gitTrigger = triggerTransfer.git2YamlTriggerOn( + elements = tGitElement, + projectId = projectId, + aspectWrapper = aspectWrapper, + defaultName = "TGit事件触发" + ) + res.putAll(gitTrigger.groupBy { ScmType.CODE_TGIT }) + } + + val githubElement = fix[CodeGithubWebHookTriggerElement.classType]?.map { + aspectWrapper.setModelElement4Model(it, PipelineTransferAspectWrapper.AspectType.BEFORE) + WebHookTriggerElementChanger(it as CodeGithubWebHookTriggerElement) + } + if (!githubElement.isNullOrEmpty()) { + val gitTrigger = triggerTransfer.git2YamlTriggerOn( + elements = githubElement, + projectId = projectId, + aspectWrapper = aspectWrapper, + defaultName = "GitHub事件触发" + ) + res.putAll(gitTrigger.groupBy { ScmType.GITHUB }) + } + + val svnElement = fix[CodeSVNWebHookTriggerElement.classType]?.map { + aspectWrapper.setModelElement4Model(it, PipelineTransferAspectWrapper.AspectType.BEFORE) + WebHookTriggerElementChanger(it as CodeSVNWebHookTriggerElement) + } + if (!svnElement.isNullOrEmpty()) { + val gitTrigger = triggerTransfer.git2YamlTriggerOn( + elements = svnElement, + projectId = projectId, + aspectWrapper = aspectWrapper, + defaultName = "SVN事件触发" + ) + res.putAll(gitTrigger.groupBy { ScmType.CODE_SVN }) + } + + val p4Element = fix[CodeP4WebHookTriggerElement.classType]?.map { + aspectWrapper.setModelElement4Model(it, PipelineTransferAspectWrapper.AspectType.BEFORE) + WebHookTriggerElementChanger(it as CodeP4WebHookTriggerElement) + } + if (!p4Element.isNullOrEmpty()) { + val gitTrigger = triggerTransfer.git2YamlTriggerOn( + elements = p4Element, + projectId = projectId, + aspectWrapper = aspectWrapper, + defaultName = "P4事件触发" + ) + res.putAll(gitTrigger.groupBy { ScmType.CODE_P4 }) + } + + val gitlabElement = fix[CodeGitlabWebHookTriggerElement.classType]?.map { + aspectWrapper.setModelElement4Model(it, PipelineTransferAspectWrapper.AspectType.BEFORE) + WebHookTriggerElementChanger(it as CodeGitlabWebHookTriggerElement) + } + if (!gitlabElement.isNullOrEmpty()) { + val gitTrigger = triggerTransfer.git2YamlTriggerOn( + elements = gitlabElement, + projectId = projectId, + aspectWrapper = aspectWrapper, + defaultName = "Gitlab变更触发" + ) + res.putAll(gitTrigger.groupBy { ScmType.CODE_GITLAB }) + } + return res + } + + @Suppress("ComplexMethod", "NestedBlockDepth") + fun yaml2Elements( + job: Job, + yamlInput: YamlTransferInput + ): MutableList { + // 解析service + val elementList = makeServiceElementList(job) + // 解析job steps + job.steps!!.forEach { step -> + yamlInput.aspectWrapper.setYamlStep4Yaml( + yamlStep = step, + aspectType = PipelineTransferAspectWrapper.AspectType.BEFORE + ) + val element: Element = yaml2element( + userId = yamlInput.userId, + step = step, + agentSelector = job.runsOn.agentSelector?.first(), + jobRunsOnType = JobRunsOnType.parse(job.runsOn.poolName) + ) + yamlInput.aspectWrapper.setModelElement4Model(element, PipelineTransferAspectWrapper.AspectType.AFTER) + elementList.add(element) + } + + return elementList + } + + fun yaml2element( + userId: String, + step: Step, + agentSelector: String?, + jobRunsOnType: JobRunsOnType? = null + ): Element { + val runCondition = when { + step.ifFiled.isNullOrBlank() -> RunCondition.PRE_TASK_SUCCESS + IfType.ALWAYS_UNLESS_CANCELLED.name == (step.ifFiled) -> + RunCondition.PRE_TASK_FAILED_BUT_CANCEL + + IfType.ALWAYS.name == (step.ifFiled) -> + RunCondition.PRE_TASK_FAILED_EVEN_CANCEL + + IfType.FAILURE.name == (step.ifFiled) -> + RunCondition.PRE_TASK_FAILED_ONLY + + else -> { + RunCondition.CUSTOM_CONDITION_MATCH + } + } + val continueOnError = Step.ContinueOnErrorType.parse(step.continueOnError) + val additionalOptions = ElementAdditionalOptions( + enable = step.enable ?: true, + continueWhenFailed = continueOnError != null, + manualSkip = continueOnError == Step.ContinueOnErrorType.MANUAL_SKIP, + timeout = step.timeoutMinutes?.toLongOrNull() ?: VariableDefault.DEFAULT_TASK_TIME_OUT, + timeoutVar = step.timeoutMinutes ?: VariableDefault.DEFAULT_TASK_TIME_OUT.toString(), + retryWhenFailed = step.retryTimes != null, + retryCount = step.retryTimes ?: VariableDefault.DEFAULT_RETRY_COUNT, + enableCustomEnv = false, + customEnv = getElementEnv(step.env), + runCondition = runCondition, + customCondition = if (runCondition == RunCondition.CUSTOM_CONDITION_MATCH) step.ifFiled else null, + manualRetry = step.manualRetry ?: false, + subscriptionPauseUser = userId + ) + + // bash + val element: Element = when { + step.run != null -> { + makeRunElement(step, agentSelector) + } + + step.uses?.contains("${LinuxScriptElement.classType}@") == true -> { + LinuxScriptElement( + id = step.taskId, + name = step.name ?: "linuxScript", + stepId = step.id, + scriptType = BuildScriptType.SHELL, + script = step.with?.get(LinuxScriptElement::script.name) as String, + continueNoneZero = (step.with?.get(LinuxScriptElement::continueNoneZero.name) as Boolean?) ?: false, + enableArchiveFile = (step.with?.get(LinuxScriptElement::enableArchiveFile.name) as Boolean?) + ?: false, + archiveFile = step.with?.get(LinuxScriptElement::archiveFile.name) as String? + ) + } + + step.uses?.contains("${WindowsScriptElement.classType}@") == true -> { + WindowsScriptElement( + id = step.taskId, + name = step.name ?: "windowsScript", + stepId = step.id, + scriptType = BuildScriptType.BAT, + script = step.with?.get(WindowsScriptElement::script.name) as String, + charsetType = (step.with?.get(WindowsScriptElement::charsetType.name) as String?)?.let { + CharsetType.valueOf(it) + } + ) + } + + step.uses?.contains("${ManualReviewUserTaskElement.classType}@") == true -> { + val pre = step.with?.let { + JsonUtil.anyTo(it, object : TypeReference() {}) + } + ManualReviewUserTaskElement( + id = step.taskId, + name = step.name ?: "manualReviewUserTask", + reviewUsers = pre?.reviewUsers ?: mutableListOf(), + desc = pre?.desc, + suggest = pre?.suggest, + params = pre?.params ?: mutableListOf(), + namespace = pre?.namespace, + notifyType = pre?.notifyType, + notifyTitle = pre?.notifyTitle, + markdownContent = pre?.markdownContent, + notifyGroup = pre?.notifyGroup, + reminderTime = pre?.reminderTime + ) + } + + step.checkout != null -> { + creator.transferCheckoutElement(step) + } + + jobRunsOnType == JobRunsOnType.AGENT_LESS -> { + creator.transferMarketBuildLessAtomElement(step) + } + + else -> { + creator.transferMarketBuildAtomElement(step) + } + }.apply { + this.additionalOptions = additionalOptions + } + return element + } + + private fun makeRunElement( + step: Step, + agentSelector: String? + ): Element { + val type = step.runAdditionalOptions?.get(RunAtomParam::shell.name) + ?: when (agentSelector) { + "windows" -> + RunAtomParam.ShellType.CMD.shellName + + null -> null + else -> RunAtomParam.ShellType.BASH.shellName + } + return when (type) { +// RunAtomParam.ShellType.BASH.shellName -> LinuxScriptElement( +// id = step.taskId, +// name = step.name ?: "run", +// stepId = step.id, +// scriptType = BuildScriptType.SHELL, +// script = step.run ?: "", +// continueNoneZero = false +// ) +// +// RunAtomParam.ShellType.CMD.shellName -> WindowsScriptElement( +// id = step.taskId, +// name = step.name ?: "run", +// stepId = step.id, +// scriptType = BuildScriptType.BAT, +// script = step.run ?: "", +// charsetType = step.with?.get(RunAtomParam::charsetType.name)?.toString() +// ?.let { CharsetType.valueOf(it) } +// ) + + else -> { + val data = mutableMapOf() + data["input"] = mapOf( + RunAtomParam::script.name to step.run, + RunAtomParam::shell.name to type, + RunAtomParam::charsetType.name to RunAtomParam.CharsetType.parse( + step.with?.get(RunAtomParam::charsetType.name)?.toString() + ) + ) + MarketBuildAtomElement( + id = step.taskId, + name = step.name ?: "run", + stepId = step.id, + atomCode = creator.runPlugInAtomCode ?: throw ModelCreateException("runPlugInAtomCode must exist"), + version = creator.runPlugInVersion ?: throw ModelCreateException("runPlugInVersion must exist"), + data = data + ) + } + } + } + + fun model2YamlSteps( + job: Container, + projectId: String, + aspectWrapper: PipelineTransferAspectWrapper + ): List { + val stepList = mutableListOf() + job.elements.forEach { element -> + aspectWrapper.setModelElement4Model(element, PipelineTransferAspectWrapper.AspectType.BEFORE) + val step = element2YamlStep(element, projectId) + aspectWrapper.setYamlStep4Yaml( + yamlPreStep = step, + aspectType = PipelineTransferAspectWrapper.AspectType.AFTER + ) + if (step != null) { + stepList.add(step) + } + } + return stepList + } + + @Suppress("ComplexMethod") + fun element2YamlStep(element: Element, projectId: String): PreStep? { + val uses = "${element.getAtomCode()}@${element.version}" + return when { + element.getAtomCode() == "checkout" && element is MarketBuildAtomElement -> { + val input = element.data["input"] as Map? ?: emptyMap() + val repositoryType = input[CheckoutAtomParam::repositoryType.name].toString().ifBlank { null }?.let { + CheckoutAtomParam.CheckoutRepositoryType.valueOf(it) + } + val repositoryHashId = input[CheckoutAtomParam::repositoryHashId.name].toString().ifBlank { null } + val repositoryName = input[CheckoutAtomParam::repositoryName.name].toString().ifBlank { null } + val repositoryUrl = input[CheckoutAtomParam::repositoryUrl.name].toString().ifBlank { null } + val checkout = when (repositoryType) { + CheckoutAtomParam.CheckoutRepositoryType.ID -> PreCheckoutStep(repoId = repositoryHashId) + CheckoutAtomParam.CheckoutRepositoryType.NAME -> PreCheckoutStep(repoName = repositoryName) + CheckoutAtomParam.CheckoutRepositoryType.URL -> repositoryUrl + CheckoutAtomParam.CheckoutRepositoryType.SELF -> "self" + else -> null + } ?: "self" + // todo 等待checkout插件新增self参数 + PreStep( + name = element.name, + id = element.stepId, + // 插件上的 + ifFiled = TransferUtil.parseStepIfFiled(element), + uses = null, + with = TransferUtil.simplifyParams(transferCache.getAtomDefaultValue(uses), input).apply { + this.remove(CheckoutAtomParam::repositoryType.name) + this.remove(CheckoutAtomParam::repositoryHashId.name) + this.remove(CheckoutAtomParam::repositoryName.name) + this.remove(CheckoutAtomParam::repositoryUrl.name) + }.ifEmpty { null }, + checkout = checkout + ) + } + + element.getAtomCode() == creator.runPlugInAtomCode && element is MarketBuildAtomElement -> { + val input = element.data["input"] as Map? ?: emptyMap() + PreStep( + name = element.name, + id = element.stepId, + // 插件上的 + ifFiled = TransferUtil.parseStepIfFiled(element), + uses = null, + with = TransferUtil.simplifyParams( + transferCache.getAtomDefaultValue(uses), + input.filterNot { + it.key == RunAtomParam::shell.name || it.key == RunAtomParam::script.name + } + ).ifEmpty { null }, + run = input[RunAtomParam::script.name]?.toString(), + shell = input[RunAtomParam::shell.name]?.toString() + ) + } + + else -> element.transferYaml(transferCache.getAtomDefaultValue(uses)) + }?.apply { + this.enable = element.isElementEnable().nullIfDefault(true) + this.timeoutMinutes = element.additionalOptions?.timeoutVar.nullIfDefault( + VariableDefault.DEFAULT_TASK_TIME_OUT.toString() + ) ?: element.additionalOptions?.timeout.nullIfDefault(VariableDefault.DEFAULT_TASK_TIME_OUT)?.toString() + + this.continueOnError = when { + element.additionalOptions?.manualSkip == true -> Step.ContinueOnErrorType.MANUAL_SKIP.alis + element.additionalOptions?.continueWhenFailed == true -> true + else -> null + } + this.retryTimes = if (element.additionalOptions?.retryWhenFailed == true) { + element.additionalOptions?.retryCount + } else null + this.manualRetry = element.additionalOptions?.manualRetry?.nullIfDefault(false) + this.env = if (element.additionalOptions?.enableCustomEnv == true) { + element.additionalOptions?.customEnv?.associateBy({ it.key ?: "" }) { it.value } + ?.ifEmpty { null } + } else { + null + } + } + } + + protected fun makeServiceElementList(job: Job): MutableList { + return mutableListOf() + } + + private fun getElementEnv(env: Map?): List? { + return emptyList() + // 互转暂不支持 element env +// if (env == null) { +// return null +// } +// +// val nameAndValueList = mutableListOf() +// env.forEach { +// // todo 001 +// nameAndValueList.add( +// NameAndValue( +// key = it.key, +// value = it.value.toString() +// ) +// ) +// } +// +// return nameAndValueList + } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ModelTransfer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ModelTransfer.kt new file mode 100644 index 00000000000..76f9f63900b --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ModelTransfer.kt @@ -0,0 +1,427 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.transfer + +import com.tencent.devops.common.api.constant.CommonMessageCode.YAML_NOT_VALID +import com.tencent.devops.common.client.Client +import com.tencent.devops.common.pipeline.Model +import com.tencent.devops.common.pipeline.container.Stage +import com.tencent.devops.common.pipeline.container.TriggerContainer +import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType +import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting +import com.tencent.devops.common.pipeline.pojo.setting.Subscription +import com.tencent.devops.common.pipeline.pojo.transfer.IfType +import com.tencent.devops.common.pipeline.utils.PIPELINE_SETTING_CONCURRENCY_GROUP_DEFAULT +import com.tencent.devops.process.yaml.pojo.YamlVersion +import com.tencent.devops.process.yaml.transfer.VariableDefault.nullIfDefault +import com.tencent.devops.process.yaml.transfer.aspect.PipelineTransferAspectWrapper +import com.tencent.devops.process.yaml.transfer.pojo.ModelTransferInput +import com.tencent.devops.process.yaml.transfer.pojo.YamlTransferInput +import com.tencent.devops.process.yaml.v3.models.Concurrency +import com.tencent.devops.process.yaml.v3.models.Extends +import com.tencent.devops.process.yaml.v3.models.GitNotices +import com.tencent.devops.process.yaml.v3.models.IPreTemplateScriptBuildYamlParser +import com.tencent.devops.process.yaml.v3.models.Notices +import com.tencent.devops.process.yaml.v3.models.PacNotices +import com.tencent.devops.process.yaml.v3.models.PreTemplateScriptBuildYamlParser +import com.tencent.devops.process.yaml.v3.models.PreTemplateScriptBuildYamlV3Parser +import com.tencent.devops.process.yaml.v3.models.on.IPreTriggerOn +import com.tencent.devops.process.yaml.v3.models.on.PreTriggerOn +import com.tencent.devops.process.yaml.v3.models.on.PreTriggerOnV3 +import com.tencent.devops.process.yaml.v3.models.stage.PreStage +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +@Component +@Suppress("ComplexMethod") +class ModelTransfer @Autowired constructor( + val client: Client, + val modelStage: StageTransfer, + val elementTransfer: ElementTransfer, + val variableTransfer: VariableTransfer, + val transferCache: TransferCacheService +) { + + companion object { + private val logger = LoggerFactory.getLogger(ModelTransfer::class.java) + } + + fun yaml2Labels(yamlInput: YamlTransferInput): List { + return preparePipelineLabels(yamlInput.userId, yamlInput.projectCode, yamlInput.yaml) + } + + fun yaml2Setting(yamlInput: YamlTransferInput): PipelineSetting { + val yaml = yamlInput.yaml + return PipelineSetting( + projectId = yamlInput.pipelineInfo?.projectId ?: "", + pipelineId = yamlInput.pipelineInfo?.pipelineId ?: "", + buildNumRule = yaml.customBuildNum, + pipelineName = yaml.name ?: yamlInput.pipelineInfo?.pipelineName ?: "", + desc = yaml.desc ?: yamlInput.pipelineInfo?.pipelineDesc ?: "", + concurrencyGroup = yaml.concurrency?.group ?: PIPELINE_SETTING_CONCURRENCY_GROUP_DEFAULT, + // Cancel-In-Progress 配置group后默认为true + concurrencyCancelInProgress = yaml.concurrency?.cancelInProgress ?: false, + runLockType = when { + yaml.disablePipeline == true -> PipelineRunLockType.LOCK + yaml.concurrency?.group != null -> PipelineRunLockType.GROUP_LOCK + else -> PipelineRunLockType.MULTIPLE + }, + waitQueueTimeMinute = yaml.concurrency?.queueTimeoutMinutes + ?: VariableDefault.DEFAULT_WAIT_QUEUE_TIME_MINUTE, + maxQueueSize = yaml.concurrency?.queueLength ?: VariableDefault.DEFAULT_PIPELINE_SETTING_MAX_QUEUE_SIZE, + labels = yaml2Labels(yamlInput), + pipelineAsCodeSettings = yamlInput.asCodeSettings, + successSubscriptionList = yamlNotice2Setting( + projectId = yamlInput.projectCode, + notices = yaml.notices?.filter { it.checkNotifyForSuccess() } + ), + failSubscriptionList = yamlNotice2Setting( + projectId = yamlInput.projectCode, + notices = yaml.notices?.filter { it.checkNotifyForFail() } + ) + ) + } + + private fun yamlNotice2Setting(projectId: String, notices: List?): List { + if (notices.isNullOrEmpty()) return listOf() + return notices.map { + val res = it.toSubscription() + prepareModelGroups(projectId, res) + } + } + + private fun prepareModelGroups(projectId: String, notice: Subscription): Subscription { + if (notice.groups.isEmpty()) return notice + val info = transferCache.getProjectGroupAndUsers(projectId)?.associateBy { it.displayName } ?: return notice + val groups = notice.groups.map { info[it]?.roleName ?: "" }.toSet() + return notice.copy(groups = groups) + } + + private fun prepareYamlGroups(projectId: String, notice: PacNotices): PacNotices { + if (notice.groups.isNullOrEmpty()) return notice + val info = transferCache.getProjectGroupAndUsers(projectId)?.associateBy { it.roleName } ?: return notice + val groups = notice.groups.mapNotNull { info[it]?.displayName }.ifEmpty { null } + return notice.copy(groups = groups) + } + + fun yaml2Model( + yamlInput: YamlTransferInput + ): Model { + yamlInput.aspectWrapper.setYaml4Yaml(yamlInput.yaml, PipelineTransferAspectWrapper.AspectType.BEFORE) + val stageList = mutableListOf() + val model = Model( + name = yamlInput.yaml.name ?: yamlInput.pipelineInfo?.pipelineName ?: "", + desc = yamlInput.yaml.desc ?: yamlInput.pipelineInfo?.pipelineDesc ?: "", + stages = stageList, + labels = emptyList(), + instanceFromTemplate = false, + pipelineCreator = yamlInput.pipelineInfo?.creator ?: yamlInput.userId + ) + model.latestVersion = yamlInput.pipelineInfo?.version ?: 0 + + // 蓝盾引擎会将stageId从1开始顺序强制重写，因此在生成model时保持一致 + var stageIndex = 1 + stageList.add(modelStage.yaml2TriggerStage(yamlInput, stageIndex++)) + + // 其他的stage + yamlInput.yaml.formatStages().forEach { stage -> + yamlInput.aspectWrapper.setYamlStage4Yaml( + yamlStage = stage, + aspectType = PipelineTransferAspectWrapper.AspectType.BEFORE + ) + stageList.add( + modelStage.yaml2Stage( + stage = stage, + // stream的stage标号从1开始，后续都加1 + stageIndex = stageIndex++, + yamlInput = yamlInput + ).also { + yamlInput.aspectWrapper.setModelStage4Model(it, PipelineTransferAspectWrapper.AspectType.AFTER) + } + ) + } + // 添加finally + val finallyJobs = yamlInput.yaml.formatFinallyStage() + if (finallyJobs.isNotEmpty()) { + stageList.add( + modelStage.yaml2FinallyStage( + stageIndex = stageIndex, + finallyJobs = finallyJobs, + yamlInput = yamlInput + ) + ) + } + checkExtends(yamlInput.yaml.templateFilter().extends, model) + yamlInput.aspectWrapper.setModel4Model(model, PipelineTransferAspectWrapper.AspectType.AFTER) + return model + } + + fun model2yaml(modelInput: ModelTransferInput): IPreTemplateScriptBuildYamlParser { + modelInput.aspectWrapper.setModel4Model(modelInput.model, PipelineTransferAspectWrapper.AspectType.BEFORE) + val label = prepareYamlLabels(modelInput.userId, modelInput.setting).ifEmpty { null } + val yaml = when (modelInput.version) { +// YamlVersion.V2_0 -> PreTemplateScriptBuildYamlParser( +// version = "v2.0", +// name = modelInput.setting.pipelineName, +// desc = modelInput.setting.desc.ifEmpty { null }, +// label = label, +// resources = modelInput.model.resources, +// notices = makeNoticesV2(modelInput.setting) +// ) + YamlVersion.V3_0 -> PreTemplateScriptBuildYamlV3Parser( + version = "v3.0", + name = modelInput.setting.pipelineName, + desc = modelInput.setting.desc.ifEmpty { null }, + label = label, + resources = modelInput.model.resources, + notices = makeNoticesV3(modelInput.setting) + ) + else -> { + throw PipelineTransferException( + YAML_NOT_VALID, + arrayOf("only support v3") + ) + } + } + if (modelInput.model.template != null) { + yaml.extends = Extends( + modelInput.model.template!!, + modelInput.model.ref, + modelInput.model.variables + ) + return yaml + } + + val triggerOn = makeTriggerOn(modelInput) + when (modelInput.version) { + YamlVersion.V2_0 -> { + (yaml as PreTemplateScriptBuildYamlParser).triggerOn = triggerOn.firstOrNull() as PreTriggerOn? + } + YamlVersion.V3_0 -> { + (yaml as PreTemplateScriptBuildYamlV3Parser).triggerOn = + triggerOn.ifEmpty { null }?.let { if (it.size == 1) it.first() else it } + } + } + val stages = mutableListOf() + modelInput.model.stages.forEachIndexed { index, stage -> + if (index == 0 || stage.finally) return@forEachIndexed + modelInput.aspectWrapper.setModelStage4Model(stage, PipelineTransferAspectWrapper.AspectType.BEFORE) + val ymlStage = modelStage.model2YamlStage( + stage = stage, + userId = modelInput.userId, + projectId = modelInput.setting.projectId, + aspectWrapper = modelInput.aspectWrapper + ) + modelInput.aspectWrapper.setYamlStage4Yaml( + yamlPreStage = ymlStage, + aspectType = PipelineTransferAspectWrapper.AspectType.AFTER + ) + stages.add(ymlStage) + } + yaml.stages = TransferMapper.anyTo(stages) + yaml.variables = variableTransfer.makeVariableFromModel(modelInput.model) + val lastStage = modelInput.model.stages.last() + val finally = if (lastStage.finally) { + modelInput.aspectWrapper.setModelStage4Model(lastStage, PipelineTransferAspectWrapper.AspectType.BEFORE) + modelStage.model2YamlStage( + stage = lastStage, + userId = modelInput.userId, + projectId = modelInput.setting.projectId, + aspectWrapper = modelInput.aspectWrapper + ).jobs + } else null + yaml.finally = finally as LinkedHashMap? + yaml.concurrency = makeConcurrency(modelInput.setting) + yaml.customBuildNum = modelInput.setting.buildNumRule + yaml.recommendedVersion = variableTransfer.makeRecommendedVersion(modelInput.model) + yaml.disablePipeline = (modelInput.setting.runLockType == PipelineRunLockType.LOCK).nullIfDefault(false) + modelInput.aspectWrapper.setYaml4Yaml(yaml, PipelineTransferAspectWrapper.AspectType.AFTER) + return yaml + } + + private fun makeNoticesV2(setting: PipelineSetting): List? { + val res = mutableListOf() + setting.successSubscriptionList?.forEach { + if (it.types.isNotEmpty()) { + res.add(GitNotices(it, IfType.SUCCESS.name)) + } + } + setting.failSubscriptionList?.forEach { + if (it.types.isNotEmpty()) { + res.add(GitNotices(it, IfType.FAILURE.name)) + } + } + return res.ifEmpty { null } + } + + private fun makeNoticesV3(setting: PipelineSetting): List? { + val res = mutableListOf() + setting.successSubscriptionList?.ifEmpty { listOf(setting.successSubscription) }?.forEach { + if (it.types.isNotEmpty()) { + val notice = PacNotices(it, IfType.SUCCESS.name) + res.add(prepareYamlGroups(setting.projectId, notice)) + } + } + setting.failSubscriptionList?.ifEmpty { listOf(setting.failSubscription) }?.forEach { + if (it.types.isNotEmpty()) { + val notice = PacNotices(it, IfType.FAILURE.name) + res.add(prepareYamlGroups(setting.projectId, notice)) + } + } + return res.ifEmpty { null } + } + + private fun makeConcurrency(setting: PipelineSetting): Concurrency? { + if (setting.runLockType == PipelineRunLockType.GROUP_LOCK || + setting.runLockType == PipelineRunLockType.LOCK + ) { + return Concurrency( + group = setting.concurrencyGroup, + cancelInProgress = setting.concurrencyCancelInProgress.nullIfDefault(false), + queueLength = setting.maxQueueSize + .nullIfDefault(VariableDefault.DEFAULT_PIPELINE_SETTING_MAX_QUEUE_SIZE), + queueTimeoutMinutes = setting.waitQueueTimeMinute + .nullIfDefault(VariableDefault.DEFAULT_WAIT_QUEUE_TIME_MINUTE) + ) + } + return null + } + + private fun makeTriggerOn(modelInput: ModelTransferInput): List { + modelInput.aspectWrapper.setModelStage4Model( + modelInput.model.stages[0], + PipelineTransferAspectWrapper.AspectType.BEFORE + ) + modelInput.aspectWrapper.setModelJob4Model( + modelInput.model.stages[0].containers[0], + PipelineTransferAspectWrapper.AspectType.BEFORE + ) + val triggers = (modelInput.model.stages[0].containers[0] as TriggerContainer).elements + val baseTrigger = elementTransfer.baseTriggers2yaml(triggers, modelInput.aspectWrapper) + ?.toPre(modelInput.version) + val scmTrigger = elementTransfer.scmTriggers2Yaml( + triggers, modelInput.setting.projectId, modelInput.aspectWrapper + ) + when (modelInput.version) { + YamlVersion.V2_0 -> { + // 融合默认git触发器 + 基础触发器 + if (scmTrigger[modelInput.defaultScmType] != null && + scmTrigger[modelInput.defaultScmType]!!.size == 1 + ) { + val res = scmTrigger[modelInput.defaultScmType]!!.first().toPre(modelInput.version) as PreTriggerOn + return listOf( + res.copy( + manual = baseTrigger?.manual, + schedules = baseTrigger?.schedules, + remote = baseTrigger?.remote + ) + ) + } + // 只带基础触发器 + if (baseTrigger != null) { + return listOf(baseTrigger) + } + // 不带触发器 + return emptyList() + } + + YamlVersion.V3_0 -> { + val trigger = mutableListOf() + val triggerV3 = mutableListOf() + scmTrigger.map { on -> + on.value.forEach { pre -> + triggerV3.add(pre.toPre(modelInput.version).also { + it as PreTriggerOnV3 + if (!it.repoName.isNullOrBlank()) { + it.type = on.key.alis + } + }) + } + } + if (baseTrigger != null) { + when (triggerV3.size) { + // 只带基础触发器 + 0 -> return listOf(baseTrigger) + // 融合一个git触发器 + 基础触发器 + 1 -> return listOf( + (triggerV3.first() as PreTriggerOnV3).copy( + manual = baseTrigger.manual, + schedules = baseTrigger.schedules, + remote = baseTrigger.remote + ) + ) + // 队列首插入基础触发器 + else -> trigger.add(0, baseTrigger) + } + } + trigger.addAll(triggerV3) + return trigger + } + } + } + + @Suppress("NestedBlockDepth") + private fun preparePipelineLabels( + userId: String, + projectCode: String, + yaml: IPreTemplateScriptBuildYamlParser + ): List { + val ymlLabel = yaml.label ?: return emptyList() + val labels = mutableListOf() + + transferCache.getPipelineLabel(userId, projectCode)?.forEach { group -> + group.labels.forEach { + if (ymlLabel.contains(it.name)) labels.add(it.id) + } + } + return labels + } + + private fun prepareYamlLabels( + userId: String, + pipelineSetting: PipelineSetting + ): List { + val labels = mutableListOf() + + transferCache.getPipelineLabel(userId, pipelineSetting.projectId)?.forEach { group -> + group.labels.forEach { + if (pipelineSetting.labels.contains(it.id)) labels.add(it.name) + } + } + return labels + } + + private fun checkExtends(extends: Extends?, model: Model) { + if (extends != null) { + model.template = extends.template + model.ref = extends.ref + model.variables = extends.variables + } + } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/PacYamlNotValidException.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/PacYamlNotValidException.kt new file mode 100644 index 00000000000..d9f73df4841 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/PacYamlNotValidException.kt @@ -0,0 +1,5 @@ +package com.tencent.devops.process.yaml.transfer + +class PacYamlNotValidException( + errorMessage: String +) : Exception(errorMessage) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/PipelineTransferException.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/PipelineTransferException.kt new file mode 100644 index 00000000000..6085eb79f24 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/PipelineTransferException.kt @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.transfer + +import com.tencent.devops.common.api.exception.ErrorCodeException + +class PipelineTransferException( + errorCode: String, + params: Array? = null +) : ErrorCodeException(errorCode = errorCode, params = params) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/StageTransfer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/StageTransfer.kt new file mode 100644 index 00000000000..02709b6d50e --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/StageTransfer.kt @@ -0,0 +1,428 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.transfer + +import com.fasterxml.jackson.databind.ObjectMapper +import com.tencent.devops.common.api.constant.CommonMessageCode +import com.tencent.devops.common.client.Client +import com.tencent.devops.common.pipeline.container.Container +import com.tencent.devops.common.pipeline.container.NormalContainer +import com.tencent.devops.common.pipeline.container.Stage +import com.tencent.devops.common.pipeline.container.TriggerContainer +import com.tencent.devops.common.pipeline.container.VMBuildContainer +import com.tencent.devops.common.pipeline.enums.StageRunCondition +import com.tencent.devops.common.pipeline.option.StageControlOption +import com.tencent.devops.common.pipeline.pojo.BuildNo +import com.tencent.devops.common.pipeline.pojo.StagePauseCheck +import com.tencent.devops.common.pipeline.pojo.StageReviewGroup +import com.tencent.devops.common.pipeline.pojo.element.Element +import com.tencent.devops.common.pipeline.pojo.element.atom.ManualReviewParam +import com.tencent.devops.common.pipeline.pojo.element.atom.ManualReviewParamPair +import com.tencent.devops.common.pipeline.pojo.element.atom.ManualReviewParamType +import com.tencent.devops.common.pipeline.utils.TransferUtil +import com.tencent.devops.common.web.utils.I18nUtil +import com.tencent.devops.process.engine.common.VMUtils +import com.tencent.devops.process.utils.FIXVERSION +import com.tencent.devops.process.utils.MAJORVERSION +import com.tencent.devops.process.utils.MINORVERSION +import com.tencent.devops.process.yaml.creator.ModelCommon +import com.tencent.devops.process.yaml.creator.ModelCreateException +import com.tencent.devops.process.yaml.transfer.VariableDefault.DEFAULT_CHECKIN_TIMEOUT_HOURS +import com.tencent.devops.process.yaml.transfer.VariableDefault.nullIfDefault +import com.tencent.devops.process.yaml.transfer.aspect.PipelineTransferAspectWrapper +import com.tencent.devops.process.yaml.transfer.inner.TransferCreator +import com.tencent.devops.process.yaml.transfer.pojo.YamlTransferInput +import com.tencent.devops.process.yaml.v3.check.PreFlow +import com.tencent.devops.process.yaml.v3.check.PreStageCheck +import com.tencent.devops.process.yaml.v3.check.PreStageReviews +import com.tencent.devops.process.yaml.v3.check.ReviewVariable +import com.tencent.devops.process.yaml.v3.check.StageCheck +import com.tencent.devops.process.yaml.v3.enums.ContentFormat +import com.tencent.devops.process.yaml.v3.models.IPreTemplateScriptBuildYamlParser +import com.tencent.devops.process.yaml.v3.models.RecommendedVersion +import com.tencent.devops.process.yaml.v3.models.Variable +import com.tencent.devops.process.yaml.v3.models.VariablePropType +import com.tencent.devops.process.yaml.v3.models.VariableProps +import com.tencent.devops.process.yaml.v3.models.job.Job +import com.tencent.devops.process.yaml.v3.models.job.JobRunsOnType +import com.tencent.devops.process.yaml.v3.models.stage.PreStage +import com.tencent.devops.process.yaml.v3.models.stage.StageLabel +import com.tencent.devops.process.yaml.v3.utils.ScriptYmlUtils +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component +import com.tencent.devops.process.yaml.v3.models.stage.Stage as StreamV3Stage + +@Component +@Suppress("ComplexMethod") +class StageTransfer @Autowired(required = false) constructor( + val client: Client, + val objectMapper: ObjectMapper, + val containerTransfer: ContainerTransfer, + val elementTransfer: ElementTransfer, + val variableTransfer: VariableTransfer, + val transferCacheService: TransferCacheService, + val transferCreator: TransferCreator +) { + companion object { + private val logger = LoggerFactory.getLogger(StageTransfer::class.java) + } + + fun yaml2TriggerStage(yamlInput: YamlTransferInput, stageIndex: Int): Stage { + // 第一个stage，触发类 + val triggerElementList = mutableListOf() + elementTransfer.yaml2Triggers(yamlInput, triggerElementList) + + val triggerContainer = TriggerContainer( + id = "0", + name = I18nUtil.getCodeLanMessage(CommonMessageCode.BK_BUILD_TRIGGER), + elements = triggerElementList, + status = null, + startEpoch = null, + systemElapsed = null, + elementElapsed = null, + params = variableTransfer.makeVariableFromYaml(makeVariables(yamlInput.yaml)) + ) + with(yamlInput.yaml.recommendedVersion) { + if (this != null && this.enabled) { + triggerContainer.buildNo = BuildNo( + this.buildNo.initialValue, + RecommendedVersion.Strategy.parse(this.buildNo.strategy).toBuildNoType(), + this.allowModifyAtStartup + ) + } + } + yamlInput.aspectWrapper.setModelJob4Model(triggerContainer, PipelineTransferAspectWrapper.AspectType.AFTER) + + val stageId = VMUtils.genStageId(stageIndex) + return Stage(listOf(triggerContainer), id = stageId, name = stageId) + } + + private fun makeVariables(yaml: IPreTemplateScriptBuildYamlParser): Map { + val variable = yaml.formatVariables() + if (yaml.recommendedVersion == null || yaml.recommendedVersion?.enabled == false) return variable + return with(yaml.recommendedVersion) { + variable.plus( + mapOf( + MAJORVERSION to Variable( + value = this!!.major.toString(), + allowModifyAtStartup = allowModifyAtStartup, + props = VariableProps( + type = VariablePropType.VUEX_INPUT.name, + description = I18nUtil.getCodeLanMessage(MAJORVERSION) + ) + ), + MINORVERSION to Variable( + value = this!!.minor.toString(), + allowModifyAtStartup = allowModifyAtStartup, + props = VariableProps( + type = VariablePropType.VUEX_INPUT.name, + description = I18nUtil.getCodeLanMessage(MINORVERSION) + ) + ), + FIXVERSION to Variable( + value = this!!.fix.toString(), + allowModifyAtStartup = allowModifyAtStartup, + props = VariableProps( + type = VariablePropType.VUEX_INPUT.name, + description = I18nUtil.getCodeLanMessage(FIXVERSION) + ) + ) + ) + ) + } + } + + fun yaml2FinallyStage( + stageIndex: Int, + finallyJobs: List, + yamlInput: YamlTransferInput + ): Stage { + return yaml2Stage( + stage = StreamV3Stage( + name = "Finally", + label = emptyList(), + ifField = null, + fastKill = false, + jobs = finallyJobs, + checkIn = null, + checkOut = null + ), + stageIndex = stageIndex, + yamlInput = yamlInput, + finalStage = true + ) + } + + fun yaml2Stage( + stage: StreamV3Stage, + stageIndex: Int, + yamlInput: YamlTransferInput, + finalStage: Boolean = false + ): Stage { + val containerList = mutableListOf() + + stage.jobs.forEachIndexed { jobIndex, job -> + yamlInput.aspectWrapper.setYamlJob4Yaml( + yamlJob = job, + aspectType = PipelineTransferAspectWrapper.AspectType.BEFORE + ) + preCheckJob(job, yamlInput) + val elementList = elementTransfer.yaml2Elements( + job = job, + yamlInput = yamlInput + ) + + val jobEnable = if (job.enable != null) job.enable!! else true + if (job.runsOn.poolName == JobRunsOnType.AGENT_LESS.type) { + containerTransfer.addNormalContainer( + job = job, + elementList = elementList, + containerList = containerList, + jobIndex = jobIndex, + jobEnable = jobEnable, + finalStage = finalStage + ) + } else { + containerTransfer.addVmBuildContainer( + job = job, + elementList = elementList, + containerList = containerList, + jobIndex = jobIndex, + projectCode = yamlInput.projectCode, + userId = yamlInput.userId, + finalStage = finalStage, + jobEnable = jobEnable, + resources = yamlInput.yaml.formatResources(), + buildTemplateAcrossInfo = yamlInput.jobTemplateAcrossInfo?.get(job.id) + ) + } + yamlInput.aspectWrapper.setModelJob4Model( + containerList.last(), + PipelineTransferAspectWrapper.AspectType.AFTER + ) + } + + val stageEnable = if (stage.enable != null) stage.enable!! else true + + // 根据if设置stageController + val stageControlOption = if (!finalStage) { + val runCondition = kotlin.run { + if (!stage.ifField.isNullOrBlank()) StageRunCondition.CUSTOM_CONDITION_MATCH else null + } ?: StageRunCondition.AFTER_LAST_FINISHED + StageControlOption( + enable = stageEnable, + runCondition = runCondition, + customCondition = if (runCondition == StageRunCondition.CUSTOM_CONDITION_MATCH) { + stage.ifField + } else { + null + } + ) + } else StageControlOption( + enable = stageEnable + ) + + val stageId = VMUtils.genStageId(stageIndex) + return Stage( + id = stageId, + name = stage.name ?: if (finalStage) { + "Final" + } else { + VMUtils.genStageId(stageIndex - 1) + }, + tag = stage.label, + fastKill = stage.fastKill, + stageControlOption = stageControlOption, + containers = containerList, + finally = finalStage, + checkIn = createStagePauseCheck( + stageCheck = stage.checkIn + ), + checkOut = createStagePauseCheck( + stageCheck = stage.checkOut + ) + ) + } + + fun model2YamlStage( + stage: Stage, + userId: String, + projectId: String, + aspectWrapper: PipelineTransferAspectWrapper + ): PreStage { + val jobs = stage.containers.associateTo(LinkedHashMap()) { job -> + aspectWrapper.setModelJob4Model(job, PipelineTransferAspectWrapper.AspectType.BEFORE) + val steps = elementTransfer.model2YamlSteps(job, projectId, aspectWrapper) + + (job.jobId?.ifBlank { null } ?: ScriptYmlUtils.randomString("job_")) to when (job.getClassType()) { + NormalContainer.classType -> containerTransfer.addYamlNormalContainer(job as NormalContainer, steps) + VMBuildContainer.classType -> containerTransfer.addYamlVMBuildContainer( + userId = userId, + projectId = projectId, + job = job as VMBuildContainer, + steps = steps + ) + + else -> throw ModelCreateException("unknown classType:(${job.getClassType()})") + }.also { preJob -> + aspectWrapper.setYamlJob4Yaml( + yamlPreJob = preJob, + aspectType = PipelineTransferAspectWrapper.AspectType.AFTER + ) + } + } + return PreStage( + name = stage.name, + label = maskYamlStageLabel(stage.tag).ifEmpty { null }, + ifField = when (stage.stageControlOption?.runCondition) { + StageRunCondition.CUSTOM_CONDITION_MATCH -> stage.stageControlOption?.customCondition + StageRunCondition.CUSTOM_VARIABLE_MATCH -> TransferUtil.customVariableMatch( + stage.stageControlOption?.customVariables + ) + + StageRunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN -> TransferUtil.customVariableMatchNotRun( + stage.stageControlOption?.customVariables + ) + + else -> null + }, + fastKill = if (stage.fastKill == true) true else null, + jobs = jobs, + checkIn = getCheckInForStage(stage), + // TODO 暂时不支持准出和gates的导出 + checkOut = null + ) + } + + private fun maskYamlStageLabel(tags: List?): List { + if (tags.isNullOrEmpty()) return emptyList() + return tags.map { StageLabel.parseById(it).value } + } + + private fun getCheckInForStage(stage: Stage): PreStageCheck? { + val reviews = PreStageReviews( + flows = stage.checkIn?.reviewGroups?.map { PreFlow(it.name, it.reviewers) }, + variables = stage.checkIn?.reviewParams?.associate { + it.key to ReviewVariable( + label = it.chineseName ?: it.key, + type = when (it.valueType) { + ManualReviewParamType.TEXTAREA -> "TEXTAREA" + ManualReviewParamType.ENUM -> "SELECTOR" + ManualReviewParamType.MULTIPLE -> "SELECTOR-MULTIPLE" + ManualReviewParamType.BOOLEAN -> "BOOL" + else -> "INPUT" + }, + default = it.value, + values = it.options?.map { mit -> mit.key }, + description = it.desc + ) + }, + description = stage.checkIn?.reviewDesc, + contentFormat = ContentFormat.parse(stage.checkIn?.markdownContent).nullIfDefault(ContentFormat.TEXT)?.text, + notifyType = stage.checkIn?.notifyType?.ifEmpty { null }, + notifyGroups = stage.checkIn?.notifyGroup?.ifEmpty { null } + ) + if (reviews.flows.isNullOrEmpty()) { + return null + } + return PreStageCheck( + reviews = reviews, + gates = null, + timeoutHours = stage.checkIn?.timeout?.nullIfDefault(DEFAULT_CHECKIN_TIMEOUT_HOURS) + ) + } + + private fun createStagePauseCheck( + stageCheck: StageCheck? + ): StagePauseCheck? { + if (stageCheck == null) return null + val check = StagePauseCheck() + check.timeout = stageCheck.timeoutHours ?: DEFAULT_CHECKIN_TIMEOUT_HOURS + if (stageCheck.reviews?.flows?.isNotEmpty() == true) { + check.manualTrigger = true + check.reviewDesc = stageCheck.reviews.description + check.reviewParams = createReviewParams(stageCheck.reviews.variables) + check.reviewGroups = stageCheck.reviews.flows.map { + StageReviewGroup( + name = it.name, + reviewers = ModelCommon.parseReceivers(it.reviewers).toList() + ) + }.toMutableList() + check.markdownContent = stageCheck.reviews.contentFormat == ContentFormat.MARKDOWN + check.notifyType = stageCheck.reviews.notifyType?.toMutableList() ?: mutableListOf() + check.notifyGroup = stageCheck.reviews.notifyGroups?.toMutableList() ?: mutableListOf() + } + return check + } + + private fun createReviewParams(variables: Map?): List? { + if (variables.isNullOrEmpty()) return null + val params = mutableListOf() + variables.forEach { (key, variable) -> + params.add( + ManualReviewParam( + key = key, + value = variable.default, + required = true, + valueType = when (variable.type) { + "TEXTAREA" -> ManualReviewParamType.TEXTAREA + "SELECTOR" -> ManualReviewParamType.ENUM + "SELECTOR-MULTIPLE" -> ManualReviewParamType.MULTIPLE + "BOOL" -> ManualReviewParamType.BOOLEAN + else -> ManualReviewParamType.STRING + }, + chineseName = variable.label, + desc = variable.description, + options = (variable.values.takeIf { it is List<*>? } as List<*>?)?.map { + ManualReviewParamPair( + it.toString(), + it.toString() + ) + }, + variableOption = variable.values.takeIf { it is String? } as String? + ) + ) + } + return params + } + + private fun preCheckJob( + job: Job, + yamlInput: YamlTransferInput + ) { + if (job.runsOn.hwSpec != null) { + val hw = transferCacheService.getDockerResource( + userId = yamlInput.userId, + projectId = yamlInput.projectCode, + buildType = transferCreator.defaultLinuxDispatchType() + )?.dockerResourceOptionsMaps?.find { it.dockerResourceOptionsShow.description == job.runsOn.hwSpec } + job.runsOn.hwSpec = hw?.id + } + } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TransferCacheService.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TransferCacheService.kt new file mode 100644 index 00000000000..3ebe69e9ac8 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TransferCacheService.kt @@ -0,0 +1,172 @@ +package com.tencent.devops.process.yaml.transfer + +import com.github.benmanes.caffeine.cache.Caffeine +import com.tencent.devops.auth.api.service.ServiceProjectAuthResource +import com.tencent.devops.common.api.enums.RepositoryConfig +import com.tencent.devops.common.api.enums.RepositoryType +import com.tencent.devops.common.auth.api.pojo.BkAuthGroupAndUserList +import com.tencent.devops.common.client.Client +import com.tencent.devops.common.client.ClientTokenService +import com.tencent.devops.common.pipeline.type.BuildType +import com.tencent.devops.dispatch.docker.api.service.ServiceDockerResourceConfigResource +import com.tencent.devops.dispatch.docker.pojo.resource.UserDockerResourceOptionsVO +import com.tencent.devops.environment.api.ServiceEnvironmentResource +import com.tencent.devops.environment.api.thirdpartyagent.ServiceThirdPartyAgentResource +import com.tencent.devops.process.api.service.ServicePipelineGroupResource +import com.tencent.devops.process.api.service.ServicePipelineResource +import com.tencent.devops.process.pojo.classify.PipelineGroup +import com.tencent.devops.process.yaml.v3.models.job.JobRunsOnPoolType +import com.tencent.devops.repository.api.ServiceRepositoryResource +import com.tencent.devops.repository.pojo.Repository +import com.tencent.devops.store.api.atom.ServiceMarketAtomResource +import com.tencent.devops.store.api.image.ServiceStoreImageResource +import com.tencent.devops.store.pojo.atom.ElementThirdPartySearchParam +import com.tencent.devops.store.pojo.image.response.ImageDetail +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import java.util.concurrent.TimeUnit + +@Service +class TransferCacheService @Autowired constructor( + private val client: Client, + private val tokenService: ClientTokenService +) { + companion object { + private val logger = LoggerFactory.getLogger(TransferCacheService::class.java) + } + + private val atomDefaultValueCache = Caffeine.newBuilder() + .maximumSize(1000) + .expireAfterWrite(10, TimeUnit.MINUTES) + .build> { key -> + kotlin.runCatching { + val (atomCode, version) = key.split("@") + client.get(ServiceMarketAtomResource::class) + .getAtomsDefaultValue(ElementThirdPartySearchParam(atomCode, version)).data + }.onFailure { logger.warn("get $key default value error.", it) }.getOrNull() ?: emptyMap() + } + private val storeImageInfoCache = Caffeine.newBuilder() + .maximumSize(1000) + .expireAfterWrite(10, TimeUnit.MINUTES) + .build { key -> + kotlin.runCatching { + val (userId, imageCode, imageVersion) = key.split("@@") + client.get(ServiceStoreImageResource::class) + .getImagesByCodeAndVersion( + userId = userId, + imageCode = imageCode, + version = imageVersion + ).data + }.onFailure { logger.warn("get $key ImageInfoByCodeAndVersion value error.", it) }.getOrNull() + } + private val projectGroupAndUsersCache = Caffeine.newBuilder() + .maximumSize(1000) + .expireAfterWrite(10, TimeUnit.MINUTES) + .build?> { key -> + kotlin.runCatching { + client.get(ServiceProjectAuthResource::class) + .getProjectGroupAndUserList( + token = tokenService.getSystemToken(), + projectCode = key + ).data + }.onFailure { logger.warn("get $key ProjectGroupAndUserList error.", it) }.getOrNull() + } + private val pipelineLabel = Caffeine.newBuilder() + .maximumSize(1000) + .expireAfterWrite(10, TimeUnit.MINUTES) + .build?> { key -> + kotlin.runCatching { + val (userId, projectId) = key.split("@@") + client.get(ServicePipelineGroupResource::class) + .getGroups(userId, projectId) + .data + }.onFailure { logger.warn("get $key pipeline label value error.", it) }.getOrNull() + } + private val gitRepository = Caffeine.newBuilder() + .maximumSize(1000) + .expireAfterWrite(10, TimeUnit.MINUTES) + .build { key -> + kotlin.runCatching { + val (projectId, type, value) = key.split("@@") + val repositoryType = RepositoryType.valueOf(type) + val config = RepositoryConfig( + if (repositoryType == RepositoryType.ID) value else null, + if (repositoryType == RepositoryType.NAME) value else null, + RepositoryType.valueOf(type) + ) + client.get(ServiceRepositoryResource::class) + .get(projectId, config.getURLEncodeRepositoryId(), config.repositoryType) + .data + }.onFailure { logger.warn("get $key git repository error.") }.getOrNull() + } + private val pipelineRemoteToken = Caffeine.newBuilder() + .maximumSize(1000) + .expireAfterWrite(10, TimeUnit.MINUTES) + .build { key -> + kotlin.runCatching { + val (userId, projectId, pipelineId) = key.split("@@") + client.get(ServicePipelineResource::class) + .generateRemoteToken(userId, projectId, pipelineId) + .data?.token + }.onFailure { logger.warn("get $key remote token value error.", it) }.getOrNull() + } + private val thirdPartyAgent = Caffeine.newBuilder() + .maximumSize(1000) + .expireAfterWrite(1, TimeUnit.MINUTES) + .build { key -> + kotlin.runCatching { + val (poolType, userId, projectId, value) = key.split("@@") + when (poolType) { + JobRunsOnPoolType.ENV_ID.name -> { + client.get(ServiceEnvironmentResource::class) + .get(userId, projectId, value) + .data?.name + } + + JobRunsOnPoolType.AGENT_ID.name -> { + client.get(ServiceThirdPartyAgentResource::class) + .getAgentDetail(userId, projectId, value) + .data?.displayName + } + + else -> null + } + }.onFailure { logger.warn("get $key thirdPartyAgent value error.", it) }.getOrNull() + } + + private val dockerResource = Caffeine.newBuilder() + .maximumSize(1000) + .expireAfterWrite(1, TimeUnit.MINUTES) + .build { key -> + kotlin.runCatching { + val (userId, projectId, buildType) = key.split("@@") + client.get(ServiceDockerResourceConfigResource::class) + .getDockerResourceConfigList(userId, projectId, buildType) + .data + }.onFailure { logger.warn("get $key dockerResource value error.", it) }.getOrNull() + } + + fun getAtomDefaultValue(key: String) = atomDefaultValueCache.get(key) ?: emptyMap() + + fun getStoreImageDetail(userId: String, imageCode: String, imageVersion: String?) = + storeImageInfoCache.get("$userId@@$imageCode@@${imageVersion ?: ""}") + + fun getProjectGroupAndUsers(projectId: String) = projectGroupAndUsersCache.get(projectId) + + fun getPipelineLabel(userId: String, projectId: String) = pipelineLabel.get("$userId@@$projectId") + + fun getGitRepository(projectId: String, repositoryType: RepositoryType, value: String) = + gitRepository.get("$projectId@@${repositoryType.name}@@$value") + + fun getPipelineRemoteToken(userId: String, projectId: String, pipelineId: String) = + pipelineRemoteToken.get("$userId@@$projectId@@$pipelineId") + + fun getThirdPartyAgent(poolType: JobRunsOnPoolType, userId: String, projectId: String, value: String?): String? { + if (value == null) return null + return thirdPartyAgent.get("${poolType.name}@@$userId@@$projectId@@$value") + } + + fun getDockerResource(userId: String, projectId: String, buildType: BuildType) = + dockerResource.get("$userId@@$projectId@@${buildType.name}") +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TransferMapper.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TransferMapper.kt new file mode 100644 index 00000000000..e2c26bbdd8b --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TransferMapper.kt @@ -0,0 +1,933 @@ +package com.tencent.devops.process.yaml.transfer + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.core.io.IOContext +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter +import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.dataformat.yaml.YAMLFactoryBuilder +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator +import com.fasterxml.jackson.dataformat.yaml.util.StringQuotingChecker +import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import com.github.difflib.DiffUtils +import com.github.difflib.algorithm.myers.MeyersDiffWithLinearSpace +import com.github.difflib.patch.DeltaType +import com.tencent.devops.common.api.constant.CommonMessageCode.ELEMENT_UPDATE_WRONG_PATH +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.api.util.ReflectUtil +import com.tencent.devops.common.pipeline.pojo.transfer.ElementInsertBody +import com.tencent.devops.common.pipeline.pojo.transfer.PositionResponse +import com.tencent.devops.common.pipeline.pojo.transfer.PreStep +import com.tencent.devops.common.pipeline.pojo.transfer.TransferMark +import com.tencent.devops.common.pipeline.pojo.transfer.YAME_META_DATA_JSON_FILTER +import com.tencent.devops.process.yaml.v3.models.ITemplateFilter +import com.tencent.devops.process.yaml.v3.models.job.PreJob +import com.tencent.devops.process.yaml.v3.models.stage.PreStage +import java.io.StringWriter +import java.io.Writer +import java.util.function.Supplier +import java.util.regex.Pattern +import org.json.JSONArray +import org.json.JSONObject +import org.slf4j.LoggerFactory +import org.yaml.snakeyaml.DumperOptions +import org.yaml.snakeyaml.LoaderOptions +import org.yaml.snakeyaml.Yaml +import org.yaml.snakeyaml.composer.Composer +import org.yaml.snakeyaml.constructor.SafeConstructor +import org.yaml.snakeyaml.error.Mark +import org.yaml.snakeyaml.events.Event +import org.yaml.snakeyaml.events.NodeEvent +import org.yaml.snakeyaml.nodes.AnchorNode +import org.yaml.snakeyaml.nodes.MappingNode +import org.yaml.snakeyaml.nodes.Node +import org.yaml.snakeyaml.nodes.NodeId +import org.yaml.snakeyaml.nodes.NodeTuple +import org.yaml.snakeyaml.nodes.ScalarNode +import org.yaml.snakeyaml.nodes.SequenceNode +import org.yaml.snakeyaml.nodes.Tag +import org.yaml.snakeyaml.parser.Parser +import org.yaml.snakeyaml.representer.Representer +import org.yaml.snakeyaml.resolver.Resolver +import org.yaml.snakeyaml.serializer.AnchorGenerator + +@Suppress("ComplexCondition", "ComplexMethod", "NestedBlockDepth") +object TransferMapper { + private val logger = LoggerFactory.getLogger(TransferMapper::class.java) + + /** + * 重写StringQuotingChecker 以支持on关键字特性 + */ + class CustomStringQuotingChecker : StringQuotingChecker() { + override fun needToQuoteName(name: String): Boolean { + // 自定义字符串引号检查逻辑 + return reservedKeyword(name) || looksLikeYAMLNumber(name) + } + + override fun needToQuoteValue(value: String): Boolean { + // Only consider reserved keywords but not numbers? + return isReservedKeyword(value) || valueHasQuotableChar(value) + } + + /* + *重写此处逻辑，以兼容对on关键字的特殊用法 + */ + private fun reservedKeyword(value: String): Boolean { + if (value == "on") return false + return if (value.isEmpty()) { + true + } else _isReservedKeyword(value[0].code, value) + } + } + + /* + * 实现Parser接口，以支持由events生成yaml node + * */ + class CustomParser(private val events: List) : Parser { + var idx = 0 + private var currentEvent: Event? = null + override fun checkEvent(choice: Event.ID): Boolean { + peekEvent() + return currentEvent != null && currentEvent?.`is`(choice) == true + } + + override fun peekEvent(): Event? { + if (currentEvent == null) { + currentEvent = events[idx++] + } + return currentEvent + } + + override fun getEvent(): Event? { + peekEvent() + val value = currentEvent + currentEvent = null + return value + } + } + + /* + * Parser 的空实现 + * */ + class EmptyParser : Parser { + override fun checkEvent(choice: Event.ID): Boolean { + return true + } + + override fun peekEvent(): Event? { + return null + } + + override fun getEvent(): Event? { + return null + } + } + + private val BOOL_PATTERN = Pattern + .compile("^(?:yes|Yes|YES|no|No|NO|true|True|TRUE|false|False|FALSE|On|ON|off|Off|OFF)$") + + private val resolver = object : Resolver() { + + override fun addImplicitResolver(tag: Tag, regexp: Pattern, first: String?) { + if (tag == Tag.BOOL) { + super.addImplicitResolver(tag, BOOL_PATTERN, first) + } else { + super.addImplicitResolver(tag, regexp, first) + } + } + } + + private val dumper = DumperOptions().apply { + this.isPrettyFlow = false + this.splitLines = false + this.defaultScalarStyle = DumperOptions.ScalarStyle.LITERAL + this.defaultFlowStyle = DumperOptions.FlowStyle.FLOW + this.isAllowReadOnlyProperties = true + this.isProcessComments = true + this.anchorGenerator = CustomAnchorGenerator() + } + + private val loader = LoaderOptions().apply { + this.isProcessComments = true + } + + private fun eventsComposer(events: List) = Composer( + CustomParser(events), resolver, loader + ) + + private val constructor = SafeConstructor(loader) + private fun node2JsonString(node: Node): String { + constructor.setComposer( + object : Composer( + EmptyParser(), Resolver(), loader + ) { + override fun getSingleNode(): Node { + return node + } + } + ) + val res = constructor.getSingleData(Any::class.java) + return JsonUtil.toJson(res, false) + } + + private fun checkCommentEvent(comments: List): List { + var index = comments.size + for (i in (comments.size - 1) downTo 0) { + if (comments[i].eventId != Event.ID.Comment) { + break + } + index = i + } + return comments.subList(index, comments.size) + } + + private fun anchorNode(node: Node, anchors: MutableMap) { + var realNode = node + if (node.nodeId == NodeId.anchor) { + realNode = (node as AnchorNode).realNode + } + if (realNode.anchor != null) { + anchors[realNode.anchor] = realNode + } + when (realNode.nodeId) { + NodeId.sequence -> { + val seqNode = realNode as SequenceNode + val list = seqNode.value + for (item in list) { + anchorNode(item, anchors) + } + } + + NodeId.mapping -> { + val mNode = realNode as MappingNode + val map = mNode.value + for (obj in map) { + val key = obj.keyNode + val value = obj.valueNode + anchorNode(key, anchors) + anchorNode(value, anchors) + } + } + } + } + + /** + * @param node 当前需要做替换的节点 + * @param anchors 锚点信息 + */ + private fun replaceAnchor(node: Node, anchors: Map) { + when (node.nodeId) { + NodeId.scalar -> {} + NodeId.anchor -> {} + NodeId.sequence -> { + val seqNode = node as SequenceNode + val list = seqNode.value + for (item in list) { + replaceAnchor(item, anchors) + } + if (node.anchor != null) return + anchors.forEach { (key, n) -> + if (n !is SequenceNode) return@forEach + if (exactlyTheSameNode(node, n)) { + node.anchor = key + return@forEach + } + } + } + + NodeId.mapping -> { + val mNode = node as MappingNode + val map = mNode.value + for (obj in map) { + val key = obj.keyNode + val value = obj.valueNode + replaceAnchor(key, anchors) + replaceAnchor(value, anchors) + } + if (node.anchor != null) return + anchors.forEach anchors@{ key, n -> + if (n !is MappingNode) return@anchors + val needRemove = mutableListOf() + // key 需要全部有 + n.value.forEach value@{ v -> + val index = map.find { + it.keyNode is ScalarNode && + v.keyNode is ScalarNode && + (it.keyNode as ScalarNode).value == (v.keyNode as ScalarNode).value + } ?: return@anchors + + if (exactlyTheSameNode(index.valueNode, v.valueNode)) { + needRemove.add(index) + } + } + // value相同的去掉 + map.removeAll(needRemove) + map.add( + NodeTuple( + ScalarNode( + /* tag = */ Tag.MERGE, + /* value = */ "<<", + /* startMark = */ n.startMark, + /* endMark = */ n.endMark, + /* style = */ DumperOptions.ScalarStyle.PLAIN + ), + n + ) + ) + } + } + } + } + + data class NodeIndex( + val key: String? = null, + val index: Int? = null, + val next: NodeIndex? = null + ) { + override fun toString(): String { + return key ?: "array($index)" + (next?.toString() ?: "") + } + } + + private fun indexNode(node: Node, marker: TransferMark.Mark): NodeIndex? { + var realNode = node + if (node.nodeId == NodeId.anchor) { + realNode = (node as AnchorNode).realNode + } + if (realNode is ScalarNode && checkMarker(realNode.startMark, realNode.endMark, marker)) { + return NodeIndex(key = realNode.value, index = null, next = null) + } + when (realNode.nodeId) { + NodeId.sequence -> { + val seqNode = realNode as SequenceNode + val list = seqNode.value + list.forEachIndexed { index, node -> + indexNode(node, marker)?.run { + return NodeIndex(key = null, index = index, next = this) + } + } + } + + NodeId.mapping -> { + val mNode = realNode as MappingNode + val map = mNode.value + for (obj in map) { + val key = obj.keyNode + val value = obj.valueNode + indexNode(key, marker)?.run { + return this + } + indexNode(value, marker)?.run { + val k = if (key.nodeId == NodeId.scalar) key as ScalarNode else null + return NodeIndex(key = k?.value ?: key.toString(), index = null, next = this) + } + } + } + } + return null + } + + private fun markNode(node: Node, nodeIndex: NodeIndex): TransferMark? { + var realNode = node + if (node.nodeId == NodeId.anchor) { + realNode = (node as AnchorNode).realNode + } + if (nodeIndex.key != null && nodeIndex.next == null && + realNode is ScalarNode && nodeIndex.key == realNode.value + ) { + return TransferMark( + startMark = TransferMark.Mark( + realNode.startMark.line, realNode.startMark.column + ), + endMark = TransferMark.Mark( + realNode.endMark.line, realNode.endMark.column + ) + ) + } + when (realNode.nodeId) { + NodeId.sequence -> { + val seqNode = realNode as SequenceNode + val list = seqNode.value + list.forEachIndexed { index, node -> + if (nodeIndex.index == null) return null + if (index != nodeIndex.index) return@forEachIndexed + if (nodeIndex.next == null) return TransferMark( + startMark = TransferMark.Mark( + node.startMark.line, node.startMark.column + ), + endMark = TransferMark.Mark( + node.endMark.line, node.endMark.column + ) + ) + return markNode(node, nodeIndex.next) + } + } + + NodeId.mapping -> { + val mNode = realNode as MappingNode + val map = mNode.value + for (obj in map) { + val key = obj.keyNode + val value = obj.valueNode + if (nodeIndex.key == null) return null + val k = if (key.nodeId == NodeId.scalar) key as ScalarNode else null + if (k?.value != nodeIndex.key) continue + if (nodeIndex.next == null) return TransferMark( + startMark = TransferMark.Mark( + value.startMark.line, value.startMark.column + ), + endMark = TransferMark.Mark( + value.endMark.line, value.endMark.column + ) + ) + return markNode(value, nodeIndex.next) + } + } + } + return null + } + + private fun checkMarker(start: Mark, end: Mark, marker: TransferMark.Mark): Boolean { + return marker.bigger(start) != false && marker.bigger(end) != true + } + + private fun TransferMark.Mark.bigger(start: Mark) = when { + line > start.line -> true + line == start.line && column > start.column -> true + line == start.line && column == start.column -> null + else -> false + } + + private fun exactlyTheSameNode(l: Node, r: Node): Boolean { + if (l.nodeId != r.nodeId) return false + + when (l.nodeId) { + NodeId.scalar -> { + val ln = l as ScalarNode + val rn = r as ScalarNode + if (ln.value != rn.value) return false + } + + NodeId.sequence -> { + val ls = node2JsonString(l) + val rs = node2JsonString(r) + return JSONArray(ls).similar(JSONArray(rs)) + } + + NodeId.mapping -> { + val ls = node2JsonString(l) + val rs = node2JsonString(r) + return JSONObject(ls).similar(JSONObject(rs)) + } + + NodeId.anchor -> { + val ln = l as AnchorNode + val rn = r as AnchorNode + return exactlyTheSameNode(ln.realNode, rn.realNode) + } + } + return true + } + + /** + * 重写YAMLFactory 以支持一些yaml规范外的特性 + */ + private val CustomYAMLFactoryBuilder = object : YAMLFactoryBuilder() { + override fun build(): YAMLFactory { + return object : YAMLFactory(this) { + override fun _createGenerator(out: Writer, ctxt: IOContext): YAMLGenerator { + val feats = _yamlGeneratorFeatures + return object : YAMLGenerator( + ctxt, _generatorFeatures, feats, + _quotingChecker, _objectCodec, out, _version + ) { + override fun writeString(text: String) { + super.writeString(removeTrailingSpaces(text)) + } + + /* + * 去掉换行符前的空格，以支持yaml block输出 + * */ + private fun removeTrailingSpaces(text: String): String { + val result = StringBuilder(text.length) + var start = 0 + var end = 0 + val chars = text.toCharArray() + + while (end < chars.size) { + if (chars[end] == '\n') { + val line = chars.sliceArray(start until end) + var endIdx = end - start - 1 + while (endIdx >= 0 && line[endIdx] == ' ') { + endIdx-- + } + result.append(line.sliceArray(0 until endIdx + 1)) + result.append('\n') + start = end + 1 + } + end++ + } + + if (start < chars.size) { + val line = chars.sliceArray(start until chars.size) + var endIdx = end - start - 1 + while (endIdx >= 0 && line[endIdx] == ' ') { + endIdx-- + } + result.append(line.sliceArray(0 until endIdx + 1)) + } + return result.toString() + } + } + } + } + } + } + + private val yamlObjectMapper = ObjectMapper( + CustomYAMLFactoryBuilder.enable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE) + .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) + .enable(YAMLGenerator.Feature.ALWAYS_QUOTE_NUMBERS_AS_STRINGS) + .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) + .disable(YAMLGenerator.Feature.SPLIT_LINES) + .disable(YAMLGenerator.Feature.USE_NATIVE_TYPE_ID) + .stringQuotingChecker(CustomStringQuotingChecker()).build() + ).setSerializationInclusion(JsonInclude.Include.NON_NULL).apply { + registerKotlinModule().setFilterProvider( + SimpleFilterProvider().addFilter( + YAME_META_DATA_JSON_FILTER, + SimpleBeanPropertyFilter.serializeAllExcept(YAME_META_DATA_JSON_FILTER) + ) + ) + } + + /* + * 默认锚点命名生成规则是重命名。而预期的行为是不改变原有锚点命名。 + * 所以此处重写AnchorGenerator方法，对锚点不进行重命名 + * */ + class CustomAnchorGenerator : AnchorGenerator { + override fun nextAnchor(node: Node): String { + return node.anchor + } + } + + private val yamlFactory = ThreadLocal.withInitial( + Supplier { + Yaml(SafeConstructor(loader), Representer(dumper), dumper, loader, resolver) + } + ) + + private fun parseMappingNodeIndex(node: MappingNode): Map { + val res = mutableMapOf() + node.value.forEach { nodeTuple -> + if (nodeTuple.keyNode.nodeId != NodeId.scalar) return@forEach + val kn = nodeTuple.keyNode as ScalarNode + /* 目前页面只关心行坐标，暂对列坐标归零处理 */ + val markFlag = if (nodeTuple.valueNode.endMark.column == 0) 1 else 0 + res[kn.value] = TransferMark( + startMark = TransferMark.Mark( + nodeTuple.valueNode.startMark.line, 0 + ), + endMark = TransferMark.Mark( + nodeTuple.valueNode.endMark.line + markFlag, 0 + ) + ) + } + return res + } + + fun getYamlFactory(): Yaml = yamlFactory.get() + + fun getObjectMapper(): ObjectMapper = yamlObjectMapper + + fun toYaml(bean: Any): String { + if (ReflectUtil.isNativeType(bean) || bean is String) { + return bean.toString() + } + return getObjectMapper().writeValueAsString(bean)!! + } + + fun to(str: String): T = getObjectMapper().readValue(str, object : TypeReference() {}) + + fun anyTo(any: Any?): T = getObjectMapper().readValue( + getObjectMapper().writeValueAsString(any), object : TypeReference() {} + ) + + /** + * 获得 yaml 第一层级的坐标定位信息 + */ + fun getYamlLevelOneIndex(yaml: String): Map { + val node = getYamlFactory().compose(yaml.reader()) + if (node.nodeId != NodeId.mapping) return emptyMap() + return parseMappingNodeIndex(node as MappingNode) + } + + /* + * yaml合并入口 + * 将minor中的内容融合进main中。 + * 融合策略： + * 1.保留注释信息 + * 2.保留锚点信息 + * */ + fun mergeYaml(old: String, new: String): String { + + val oldE = getYamlFactory().parse(old.reader()).toList() + val newL = getYamlFactory().parse(new.reader()).toList() + val newE = newL.toMutableList() + + val patch = DiffUtils.diff(oldE, newE, MeyersDiffWithLinearSpace.factory().create()) + val anchorChecker = mutableMapOf() + for (i in (patch.deltas.size - 1) downTo 0) { + val delta = patch.deltas[i] + when (delta.type) { + DeltaType.INSERT -> { + anchorChecker[delta.source.position]?.let { checker -> + delta.target.lines.forEachIndexed { index, event -> + if (event.eventId == checker.eventId) { + newE[delta.target.position + index] = checker + } + } + } + } + + DeltaType.DELETE -> { + val sourceComment = checkCommentEvent(delta.source.lines) + if (sourceComment.isNotEmpty()) { + newE.addAll(delta.target.position, sourceComment) + } + // 锚点覆写逻辑 + delta.source.lines.forEachIndexed { index, event -> + if (event !is NodeEvent) return@forEachIndexed + if (event.anchor != null) { + anchorChecker[delta.source.position + index] = event + } + } + } + } + } + + val newNode = eventsComposer(newE).singleNode + val anchorNodes = mutableMapOf() + anchorNode(newNode, anchorNodes) + if (anchorNodes.isNotEmpty()) { + replaceAnchor(newNode, anchorNodes) + } + + val stringWriter = StringWriter() + + getYamlFactory().serialize(newNode, stringWriter) + + val out = stringWriter.toString() + +// if (!exactlyTheSameNode(eventsComposer(newL).singleNode, newNode)) { +// throw Exception("not same node") +// } + if (!exactlyTheSameNode(getYamlFactory().compose(new.reader()), getYamlFactory().compose(out.reader()))) { + logger.warn("merge yaml fail|new=\n$new\n|||out=\n$out\n") + return new + } + return out + } + + fun formatYaml(yaml: String): String { + val res = getYamlFactory().load(yaml) as Any + return toYaml(res) + } + + fun indexYaml( + yaml: String, + line: Int, + column: Int + ): NodeIndex? { + return indexNode(getYamlFactory().compose(yaml.reader()), TransferMark.Mark(line, column)) + } + + fun markYaml( + index: NodeIndex, + yaml: String + ): TransferMark? { + return markNode(getYamlFactory().compose(yaml.reader()), index) + } + + fun indexYaml( + position: PositionResponse, + pYml: ITemplateFilter, + yml: PreStep, + type: ElementInsertBody.ElementInsertType + ): NodeIndex { + return when (position.type) { + PositionResponse.PositionType.STEP -> indexInYamlSteps(position, pYml, yml, type) + PositionResponse.PositionType.JOB, PositionResponse.PositionType.STAGE -> indexInYamlJob( + positionResponse = position, + preYaml = pYml, + preStep = yml, + type = type + ) + + else -> addInYamlLastStage(pYml, yml, type) + } + } + + /* + * 光标在一个已有的 step 配置区域，则在该 step 之后 添加一个新的 step + */ + private fun indexInYamlSteps( + positionResponse: PositionResponse, + preYaml: ITemplateFilter, + preStep: PreStep, + type: ElementInsertBody.ElementInsertType = ElementInsertBody.ElementInsertType.INSERT + ): NodeIndex { + if (positionResponse.stageIndex == -1) { + return NodeIndex( + key = ITemplateFilter::finally.name, + next = indexInJob( + positionResponse, + preYaml.finally!! + ) { steps -> + nodeIndexInStep(type, steps, positionResponse, preStep) + } + ) + } + + if (positionResponse.stageIndex != null) { + return NodeIndex( + key = ITemplateFilter::stages.name, + next = indexInStage( + positionResponse, + preYaml.stages!! + ) { steps -> + nodeIndexInStep(type, steps, positionResponse, preStep) + } + ) + } + + if (positionResponse.jobId != null) { + return NodeIndex( + key = ITemplateFilter::jobs.name, + next = indexInJob( + positionResponse, + preYaml.jobs!! + ) { steps -> + nodeIndexInStep(type, steps, positionResponse, preStep) + } + ) + } + + if (positionResponse.stepIndex != null) { + return NodeIndex( + key = PreJob::steps.name, + next = indexInStep( + preYaml.steps!! as ArrayList + ) { steps -> + nodeIndexInStep(type, steps, positionResponse, preStep) + } + ) + } + return NodeIndex() + } + + private fun nodeIndexInStep( + type: ElementInsertBody.ElementInsertType, + steps: ArrayList, + positionResponse: PositionResponse, + preStep: PreStep + ) = when (type) { + ElementInsertBody.ElementInsertType.INSERT -> { + steps.add(positionResponse.stepIndex!! + 1, to(toYaml(preStep))) + NodeIndex( + index = positionResponse.stepIndex!! + 1 + ) + } + + else -> { + steps[positionResponse.stepIndex!!] = to(toYaml(preStep)) + NodeIndex( + index = positionResponse.stepIndex!! + ) + } + } + + /* + * 光标在 job 配置区域，则在 job下的 steps 末尾添加一个新的 step + */ + private fun indexInYamlJob( + positionResponse: PositionResponse, + preYaml: ITemplateFilter, + preStep: PreStep, + type: ElementInsertBody.ElementInsertType + ): NodeIndex { + if (type != ElementInsertBody.ElementInsertType.INSERT) { + throw PipelineTransferException( + ELEMENT_UPDATE_WRONG_PATH + ) + } + if (positionResponse.stageIndex == -1) { + return NodeIndex( + key = ITemplateFilter::finally.name, + next = indexInJob( + positionResponse = positionResponse, + jobs = preYaml.finally!!, + last = true + ) { steps -> + steps.add(preStep) + NodeIndex( + index = steps.size - 1 + ) + } + ) + } + + if (positionResponse.stageIndex != null) { + return NodeIndex( + key = ITemplateFilter::stages.name, + next = indexInStage( + positionResponse, + preYaml.stages!!, + last = true + ) { steps -> + steps.add(preStep) + NodeIndex( + index = steps.size - 1 + ) + } + ) + } + + if (positionResponse.jobId != null) { + return NodeIndex( + key = ITemplateFilter::jobs.name, + next = indexInJob( + positionResponse, + preYaml.jobs!!, + last = true + ) { steps -> + steps.add(preStep) + NodeIndex( + index = steps.size - 1 + ) + } + ) + } + return NodeIndex() + } + + /* + * 光标在 stage 配置/流水线配置区域，则在最后一个 stage 的最后一个 job 末尾添加一个新的 step + */ + private fun addInYamlLastStage( + preYaml: ITemplateFilter, + preStep: PreStep, + type: ElementInsertBody.ElementInsertType + ): NodeIndex { + if (type != ElementInsertBody.ElementInsertType.INSERT) { + throw PipelineTransferException( + ELEMENT_UPDATE_WRONG_PATH + ) + } + if (preYaml.finally != null) { + return NodeIndex( + key = ITemplateFilter::finally.name, + next = indexInJob( + positionResponse = PositionResponse(), + jobs = preYaml.finally!!, + last = true + ) { steps -> + steps.add(preStep) + NodeIndex( + index = steps.size - 1 + ) + } + ) + } + + if (preYaml.stages != null) { + return NodeIndex( + key = ITemplateFilter::stages.name, + next = indexInStage( + positionResponse = PositionResponse(), + stages = preYaml.stages!!, + last = true + ) { steps -> + steps.add(preStep) + NodeIndex( + index = steps.size - 1 + ) + } + ) + } + + if (preYaml.jobs != null) { + return NodeIndex( + key = ITemplateFilter::jobs.name, + next = indexInJob( + positionResponse = PositionResponse(), + jobs = preYaml.jobs!!, + last = true + ) { steps -> + steps.add(preStep) + NodeIndex( + index = steps.size - 1 + ) + } + ) + } + + if (preYaml.steps != null) { + preYaml.steps!!.add(to(toYaml(preStep))) + return NodeIndex( + index = preYaml.steps!!.size - 1 + ) + } + return NodeIndex() + } + + private fun indexInStage( + positionResponse: PositionResponse, + stages: ArrayList>, + last: Boolean = false, + action: (steps: ArrayList) -> NodeIndex? + ): NodeIndex { + if (stages.isEmpty()) { + stages.add(mutableMapOf(PreStage::jobs.name to LinkedHashMap())) + } + val index = if (last && positionResponse.stageIndex == null) stages.lastIndex else positionResponse.stageIndex!! + val jobs = stages[index][PreStage::jobs.name] as LinkedHashMap + return NodeIndex( + index = index, + next = NodeIndex( + key = PreStage::jobs.name, + next = indexInJob(positionResponse, jobs, last, action) + ) + ) + } + + private fun indexInJob( + positionResponse: PositionResponse, + jobs: LinkedHashMap, + last: Boolean = false, + action: (steps: ArrayList) -> NodeIndex? + ): NodeIndex? { + if (jobs.isEmpty()) { + val job = LinkedHashMap() + job[PreJob::steps.name] = ArrayList() + jobs["job_1"] = job + } + val key = if (last && positionResponse.jobId == null) jobs.entries.last().key else positionResponse.jobId + ?: return null + val job = jobs[key] as LinkedHashMap + val steps = job[PreJob::steps.name] as ArrayList + return NodeIndex( + key = key, + next = NodeIndex(key = PreJob::steps.name, next = indexInStep(steps, action)) + ) + } + + private fun indexInStep( + steps: ArrayList, + action: (steps: ArrayList) -> NodeIndex? + ): NodeIndex? { + return action(steps) + } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TriggerTransfer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TriggerTransfer.kt new file mode 100644 index 00000000000..20bc6a8e651 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TriggerTransfer.kt @@ -0,0 +1,748 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.transfer + +import com.tencent.devops.common.api.enums.RepositoryType +import com.tencent.devops.common.api.enums.TriggerRepositoryType +import com.tencent.devops.common.client.Client +import com.tencent.devops.common.pipeline.pojo.element.Element +import com.tencent.devops.common.pipeline.pojo.element.ElementAdditionalOptions +import com.tencent.devops.common.pipeline.pojo.element.RunCondition +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeGitWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeGithubWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeGitlabWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeP4WebHookTriggerData +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeP4WebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeP4WebHookTriggerInput +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeSVNWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeTGitWebHookTriggerData +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeTGitWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeTGitWebHookTriggerInput +import com.tencent.devops.common.pipeline.pojo.element.trigger.ManualTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.RemoteTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.TimerTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.enums.CodeEventType +import com.tencent.devops.common.pipeline.pojo.element.trigger.enums.PathFilterType +import com.tencent.devops.process.yaml.transfer.VariableDefault.nullIfDefault +import com.tencent.devops.process.yaml.transfer.aspect.PipelineTransferAspectWrapper +import com.tencent.devops.process.yaml.transfer.inner.TransferCreator +import com.tencent.devops.process.yaml.transfer.pojo.WebHookTriggerElementChanger +import com.tencent.devops.process.yaml.transfer.pojo.YamlTransferInput +import com.tencent.devops.process.yaml.v3.models.on.EnableType +import com.tencent.devops.process.yaml.v3.models.on.IssueRule +import com.tencent.devops.process.yaml.v3.models.on.MrRule +import com.tencent.devops.process.yaml.v3.models.on.NoteRule +import com.tencent.devops.process.yaml.v3.models.on.PushRule +import com.tencent.devops.process.yaml.v3.models.on.ReviewRule +import com.tencent.devops.process.yaml.v3.models.on.TagRule +import com.tencent.devops.process.yaml.v3.models.on.TriggerOn +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +@Component +class TriggerTransfer @Autowired(required = false) constructor( + val client: Client, + @Autowired(required = false) + val creator: TransferCreator, + val transferCache: TransferCacheService +) { + companion object { + private val logger = LoggerFactory.getLogger(TriggerTransfer::class.java) + } + + @Suppress("ComplexMethod") + fun yaml2TriggerGit(triggerOn: TriggerOn, elementQueue: MutableList) { + val repositoryType = if (triggerOn.repoName.isNullOrBlank()) { + TriggerRepositoryType.SELF + } else { + TriggerRepositoryType.NAME + } + triggerOn.push?.let { push -> + elementQueue.add( + CodeGitWebHookTriggerElement( + name = push.name ?: "Git事件触发", + branchName = push.branches.nonEmptyOrNull()?.join(), + excludeBranchName = push.branchesIgnore.nonEmptyOrNull()?.join(), + includePaths = push.paths.nonEmptyOrNull()?.join(), + excludePaths = push.pathsIgnore.nonEmptyOrNull()?.join(), + includeUsers = push.users, + excludeUsers = push.usersIgnore, + pathFilterType = push.pathFilterType?.let { PathFilterType.valueOf(it) } + ?: PathFilterType.NamePrefixFilter, + eventType = CodeEventType.PUSH, + // todo action + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(push.enable) + ) + } + + triggerOn.tag?.let { tag -> + elementQueue.add( + CodeGitWebHookTriggerElement( + name = tag.name ?: "Git事件触发", + tagName = tag.tags.nonEmptyOrNull()?.join(), + excludeTagName = tag.tagsIgnore.nonEmptyOrNull()?.join(), + fromBranches = tag.fromBranches.nonEmptyOrNull()?.join(), + includeUsers = tag.users, + excludeUsers = tag.usersIgnore, + eventType = CodeEventType.TAG_PUSH, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(tag.enable) + ) + } + + triggerOn.mr?.let { mr -> + elementQueue.add( + CodeGitWebHookTriggerElement( + name = mr.name ?: "Git事件触发", + branchName = mr.targetBranches.nonEmptyOrNull()?.join(), + excludeBranchName = mr.targetBranchesIgnore.nonEmptyOrNull()?.join(), + includeSourceBranchName = mr.sourceBranches.nonEmptyOrNull()?.join(), + excludeSourceBranchName = mr.sourceBranchesIgnore.nonEmptyOrNull()?.join(), + includePaths = mr.paths.nonEmptyOrNull()?.join(), + excludePaths = mr.pathsIgnore.nonEmptyOrNull()?.join(), + includeUsers = mr.users, + excludeUsers = mr.usersIgnore, + block = mr.blockMr, + webhookQueue = mr.webhookQueue, + enableCheck = mr.reportCommitCheck, + pathFilterType = mr.pathFilterType?.let { PathFilterType.valueOf(it) } + ?: PathFilterType.NamePrefixFilter, + // todo action + eventType = CodeEventType.MERGE_REQUEST, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(mr.enable) + ) + } + + triggerOn.review?.let { review -> + elementQueue.add( + CodeGitWebHookTriggerElement( + name = review.name ?: "Git事件触发", + includeCrState = review.states, + includeCrTypes = review.types, + eventType = CodeEventType.REVIEW, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(review.enable) + ) + } + + triggerOn.issue?.let { issue -> + elementQueue.add( + CodeGitWebHookTriggerElement( + name = issue.name ?: "Git事件触发", + includeIssueAction = issue.action, + eventType = CodeEventType.ISSUES, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(issue.enable) + ) + } + + triggerOn.note?.let { note -> + elementQueue.add( + CodeGitWebHookTriggerElement( + name = note.name ?: "Git事件触发", + includeNoteTypes = note.types?.map { + when (it) { + "commit" -> "Commit" + "merge_request" -> "Review" + "issue" -> "Issue" + else -> it + } + }, + includeNoteComment = note.comment.nonEmptyOrNull()?.join(), + eventType = CodeEventType.NOTE, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(note.enable) + ) + } + } + + @Suppress("ComplexMethod") + fun git2YamlTriggerOn( + elements: List, + projectId: String, + aspectWrapper: PipelineTransferAspectWrapper, + defaultName: String + ): List { + val res = mutableMapOf() + elements.forEach { git -> + val name = when (git.repositoryType) { + TriggerRepositoryType.ID -> git.repositoryHashId ?: "" + TriggerRepositoryType.NAME -> git.repositoryName ?: "" + TriggerRepositoryType.SELF -> "self" + else -> "" + } + val nowExist = res.getOrPut(name) { + when (name) { + git.repositoryHashId -> TriggerOn( + repoName = transferCache.getGitRepository(projectId, RepositoryType.ID, name)?.aliasName + ) + + git.repositoryName -> TriggerOn(repoName = name) + else -> TriggerOn() + } + } + when (git.eventType) { + CodeEventType.PUSH -> nowExist.push = PushRule( + name = git.name.nullIfDefault(defaultName), + enable = git.enable.nullIfDefault(true), + branches = git.branchName?.disjoin() ?: emptyList(), + branchesIgnore = git.excludeBranchName?.disjoin(), + paths = git.includePaths?.disjoin(), + pathsIgnore = git.excludePaths?.disjoin(), + users = git.includeUsers, + usersIgnore = git.excludeUsers, + pathFilterType = git.pathFilterType?.name.nullIfDefault(PathFilterType.NamePrefixFilter.name), + // todo action + action = null + ) + + CodeEventType.TAG_PUSH -> nowExist.tag = TagRule( + name = git.name.nullIfDefault(defaultName), + enable = git.enable.nullIfDefault(true), + tags = git.tagName?.disjoin(), + tagsIgnore = git.excludeTagName?.disjoin(), + fromBranches = git.fromBranches?.disjoin(), + users = git.includeUsers, + usersIgnore = git.excludeUsers + ) + + CodeEventType.MERGE_REQUEST -> nowExist.mr = MrRule( + name = git.name.nullIfDefault(defaultName), + enable = git.enable.nullIfDefault(true), + targetBranches = git.branchName?.disjoin(), + targetBranchesIgnore = git.excludeBranchName?.disjoin(), + sourceBranches = git.includeSourceBranchName?.disjoin(), + sourceBranchesIgnore = git.excludeSourceBranchName?.disjoin(), + paths = git.includePaths?.disjoin(), + pathsIgnore = git.excludePaths?.disjoin(), + users = git.includeUsers, + usersIgnore = git.excludeUsers, + blockMr = git.block, + webhookQueue = git.webhookQueue.nullIfDefault(false), + reportCommitCheck = git.enableCheck.nullIfDefault(true), + pathFilterType = git.pathFilterType?.name.nullIfDefault(PathFilterType.NamePrefixFilter.name), + // todo action + action = null + ) + + CodeEventType.REVIEW -> nowExist.review = ReviewRule( + name = git.name.nullIfDefault(defaultName), + enable = git.enable.nullIfDefault(true), + states = git.includeCrState, + types = git.includeCrTypes + ) + + CodeEventType.ISSUES -> nowExist.issue = IssueRule( + name = git.name.nullIfDefault(defaultName), + enable = git.enable.nullIfDefault(true), + action = git.includeIssueAction + ) + + CodeEventType.NOTE -> nowExist.note = NoteRule( + name = git.name.nullIfDefault(defaultName), + enable = git.enable.nullIfDefault(true), + types = git.includeNoteTypes?.map { + when (it) { + "Commit" -> "commit" + "Review" -> "merge_request" + "Issue" -> "issue" + else -> it + } + } + ) + + CodeEventType.POST_COMMIT -> nowExist.push = PushRule( + name = git.name.nullIfDefault(defaultName), + enable = git.enable.nullIfDefault(true), + branches = null, + paths = git.includePaths?.disjoin(), + pathsIgnore = git.excludePaths?.disjoin(), + users = git.includeUsers, + usersIgnore = git.excludeUsers, + pathFilterType = git.pathFilterType?.name.nullIfDefault(PathFilterType.NamePrefixFilter.name) + ) + + CodeEventType.CHANGE_COMMIT -> nowExist.push = PushRule( + name = git.name.nullIfDefault(defaultName), + enable = git.enable.nullIfDefault(true), + branches = null, + branchesIgnore = null, + paths = git.includePaths?.disjoin(), + pathsIgnore = git.excludePaths?.disjoin() + ) + } + aspectWrapper.setYamlTriggerOn(nowExist, PipelineTransferAspectWrapper.AspectType.AFTER) + } + return res.values.toList() + } + + @Suppress("ComplexMethod") + fun yaml2TriggerTGit(triggerOn: TriggerOn, elementQueue: MutableList) { + val repositoryType = if (triggerOn.repoName.isNullOrBlank()) { + TriggerRepositoryType.SELF + } else { + TriggerRepositoryType.NAME + } + triggerOn.push?.let { push -> + elementQueue.add( + CodeTGitWebHookTriggerElement( + name = push.name ?: "TGit事件触发", + data = CodeTGitWebHookTriggerData( + input = CodeTGitWebHookTriggerInput( + branchName = push.branches.nonEmptyOrNull()?.join(), + excludeBranchName = push.branchesIgnore.nonEmptyOrNull()?.join(), + includePaths = push.paths.nonEmptyOrNull()?.join(), + excludePaths = push.pathsIgnore.nonEmptyOrNull()?.join(), + includeUsers = push.users, + excludeUsers = push.usersIgnore, + pathFilterType = push.pathFilterType?.let { PathFilterType.valueOf(it) } + ?: PathFilterType.NamePrefixFilter, + eventType = CodeEventType.PUSH, + // todo action + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ) + ) + ).checkTriggerElementEnable(push.enable) + ) + } + + triggerOn.tag?.let { tag -> + elementQueue.add( + CodeTGitWebHookTriggerElement( + name = tag.name ?: "TGit事件触发", + data = CodeTGitWebHookTriggerData( + input = CodeTGitWebHookTriggerInput( + tagName = tag.tags.nonEmptyOrNull()?.join(), + excludeTagName = tag.tagsIgnore.nonEmptyOrNull()?.join(), + fromBranches = tag.fromBranches.nonEmptyOrNull()?.join(), + includeUsers = tag.users, + excludeUsers = tag.usersIgnore, + eventType = CodeEventType.TAG_PUSH, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ) + ) + ).checkTriggerElementEnable(tag.enable) + ) + } + + triggerOn.mr?.let { mr -> + elementQueue.add( + CodeTGitWebHookTriggerElement( + name = mr.name ?: "TGit事件触发", + data = CodeTGitWebHookTriggerData( + input = CodeTGitWebHookTriggerInput( + branchName = mr.targetBranches.nonEmptyOrNull()?.join(), + excludeBranchName = mr.targetBranchesIgnore.nonEmptyOrNull()?.join(), + includeSourceBranchName = mr.sourceBranches.nonEmptyOrNull()?.join(), + excludeSourceBranchName = mr.sourceBranchesIgnore.nonEmptyOrNull()?.join(), + includePaths = mr.paths.nonEmptyOrNull()?.join(), + excludePaths = mr.pathsIgnore.nonEmptyOrNull()?.join(), + includeUsers = mr.users, + excludeUsers = mr.usersIgnore, + block = mr.blockMr, + webhookQueue = mr.webhookQueue, + enableCheck = mr.reportCommitCheck, + pathFilterType = mr.pathFilterType?.let { PathFilterType.valueOf(it) } + ?: PathFilterType.NamePrefixFilter, + // todo action + eventType = CodeEventType.MERGE_REQUEST, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ) + ) + ).checkTriggerElementEnable(mr.enable) + ) + } + + triggerOn.review?.let { review -> + elementQueue.add( + CodeTGitWebHookTriggerElement( + name = review.name ?: "TGit事件触发", + data = CodeTGitWebHookTriggerData( + input = CodeTGitWebHookTriggerInput( + includeCrState = review.states, + includeCrTypes = review.types, + eventType = CodeEventType.REVIEW, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ) + ) + ).checkTriggerElementEnable(review.enable) + ) + } + + triggerOn.issue?.let { issue -> + elementQueue.add( + CodeTGitWebHookTriggerElement( + name = issue.name ?: "TGit事件触发", + data = CodeTGitWebHookTriggerData( + input = CodeTGitWebHookTriggerInput( + includeIssueAction = issue.action, + eventType = CodeEventType.ISSUES, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ) + ) + ).checkTriggerElementEnable(issue.enable) + ) + } + + triggerOn.note?.let { note -> + elementQueue.add( + CodeTGitWebHookTriggerElement( + name = note.name ?: "TGit事件触发", + data = CodeTGitWebHookTriggerData( + input = CodeTGitWebHookTriggerInput( + includeNoteTypes = note.types?.map { + when (it) { + "commit" -> "Commit" + "merge_request" -> "Review" + "issue" -> "Issue" + else -> it + } + }, + includeNoteComment = note.comment.nonEmptyOrNull()?.join(), + eventType = CodeEventType.NOTE, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ) + ) + ).checkTriggerElementEnable(note.enable) + ) + } + } + + @Suppress("ComplexMethod") + fun yaml2TriggerGithub(triggerOn: TriggerOn, elementQueue: MutableList) { + val repositoryType = if (triggerOn.repoName.isNullOrBlank()) { + TriggerRepositoryType.SELF + } else { + TriggerRepositoryType.NAME + } + triggerOn.push?.let { push -> + elementQueue.add( + CodeGithubWebHookTriggerElement( + name = push.name ?: "GitHub事件触发", + branchName = push.branches.nonEmptyOrNull()?.join(), + excludeBranchName = push.branchesIgnore.nonEmptyOrNull()?.join(), + includePaths = push.paths.nonEmptyOrNull()?.join(), + excludePaths = push.pathsIgnore.nonEmptyOrNull()?.join(), + includeUsers = push.users, + excludeUsers = push.usersIgnore.nonEmptyOrNull()?.join(), + pathFilterType = push.pathFilterType?.let { PathFilterType.valueOf(it) } + ?: PathFilterType.NamePrefixFilter, + eventType = CodeEventType.PUSH, + // todo action + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(push.enable) + ) + } + + triggerOn.tag?.let { tag -> + elementQueue.add( + CodeGithubWebHookTriggerElement( + name = tag.name ?: "GitHub事件触发", + tagName = tag.tags.nonEmptyOrNull()?.join(), + excludeTagName = tag.tagsIgnore.nonEmptyOrNull()?.join(), + fromBranches = tag.fromBranches.nonEmptyOrNull()?.join(), + includeUsers = tag.users, + excludeUsers = tag.usersIgnore.nonEmptyOrNull()?.join(), + eventType = CodeEventType.TAG_PUSH, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(tag.enable) + ) + } + + triggerOn.mr?.let { mr -> + elementQueue.add( + CodeGithubWebHookTriggerElement( + name = mr.name ?: "GitHub事件触发", + branchName = mr.targetBranches.nonEmptyOrNull()?.join(), + excludeBranchName = mr.targetBranchesIgnore.nonEmptyOrNull()?.join(), + includeSourceBranchName = mr.sourceBranches.nonEmptyOrNull()?.join(), + excludeSourceBranchName = mr.sourceBranchesIgnore.nonEmptyOrNull()?.join(), + includePaths = mr.paths.nonEmptyOrNull()?.join(), + excludePaths = mr.pathsIgnore.nonEmptyOrNull()?.join(), + includeUsers = mr.users, + excludeUsers = mr.usersIgnore.nonEmptyOrNull()?.join(), + webhookQueue = mr.webhookQueue, + enableCheck = mr.reportCommitCheck, + pathFilterType = mr.pathFilterType?.let { PathFilterType.valueOf(it) } + ?: PathFilterType.NamePrefixFilter, + // todo action + eventType = CodeEventType.PULL_REQUEST, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(mr.enable) + ) + } + + triggerOn.review?.let { review -> + elementQueue.add( + CodeGithubWebHookTriggerElement( + name = review.name ?: "GitHub事件触发", + includeCrState = review.states, + includeCrTypes = review.types, + eventType = CodeEventType.REVIEW, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(review.enable) + ) + } + + triggerOn.issue?.let { issue -> + elementQueue.add( + CodeGithubWebHookTriggerElement( + name = issue.name ?: "GitHub事件触发", + includeIssueAction = issue.action, + eventType = CodeEventType.ISSUES, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(issue.enable) + ) + } + + triggerOn.note?.let { note -> + elementQueue.add( + CodeGithubWebHookTriggerElement( + name = note.name ?: "GitHub事件触发", + includeNoteTypes = note.types?.map { + when (it) { + "commit" -> "Commit" + "merge_request" -> "Review" + "issue" -> "Issue" + else -> it + } + }, + includeNoteComment = note.comment.nonEmptyOrNull()?.join(), + eventType = CodeEventType.NOTE, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(note.enable) + ) + } + } + + @Suppress("ComplexMethod") + fun yaml2TriggerSvn(triggerOn: TriggerOn, elementQueue: MutableList) { + val repositoryType = if (triggerOn.repoName.isNullOrBlank()) { + TriggerRepositoryType.SELF + } else { + TriggerRepositoryType.NAME + } + triggerOn.push?.let { push -> + elementQueue.add( + CodeSVNWebHookTriggerElement( + name = push.name ?: "SVN事件触发", + relativePath = push.paths.nonEmptyOrNull()?.join(), + excludePaths = push.pathsIgnore.nonEmptyOrNull()?.join(), + includeUsers = push.users, + excludeUsers = push.usersIgnore.nonEmptyOrNull(), + pathFilterType = push.pathFilterType?.let { PathFilterType.valueOf(it) } + ?: PathFilterType.NamePrefixFilter, + // todo action + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(push.enable) + ) + } + } + + @Suppress("ComplexMethod") + fun yaml2TriggerP4(triggerOn: TriggerOn, elementQueue: MutableList) { + val repositoryType = if (triggerOn.repoName.isNullOrBlank()) { + TriggerRepositoryType.SELF + } else { + TriggerRepositoryType.NAME + } + triggerOn.push?.let { push -> + elementQueue.add( + CodeP4WebHookTriggerElement( + name = push.name ?: "P4事件触发", + data = CodeP4WebHookTriggerData( + input = CodeP4WebHookTriggerInput( + includePaths = push.paths.nonEmptyOrNull()?.join(), + excludePaths = push.pathsIgnore.nonEmptyOrNull()?.join(), + eventType = CodeEventType.CHANGE_COMMIT, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ) + ) + ).checkTriggerElementEnable(push.enable).apply { version = "2.*" } + ) + } + } + + @Suppress("ComplexMethod") + fun yaml2TriggerBase(yamlInput: YamlTransferInput, triggerOn: TriggerOn, elementQueue: MutableList) { + triggerOn.manual?.let { manual -> + elementQueue.add( + ManualTriggerElement( + name = manual.name ?: "手动触发", + id = "T-1-1-1", + canElementSkip = manual.canElementSkip, + useLatestParameters = manual.useLatestParameters + ).apply { + this.additionalOptions = ElementAdditionalOptions(enable = manual.enable ?: true) + } + ) + } + + triggerOn.schedules?.let { schedule -> + schedule.forEach { timer -> + val repositoryType = when { + !timer.repoId.isNullOrBlank() -> + RepositoryType.ID + !timer.repoName.isNullOrBlank() -> + RepositoryType.NAME + else -> null + } + elementQueue.add( + TimerTriggerElement( + name = timer.name ?: "定时触发", + repositoryType = repositoryType, + repoHashId = timer.repoId, + repoName = timer.repoName, + branches = timer.branches, + newExpression = timer.newExpression, + advanceExpression = timer.advanceExpression, + noScm = timer.always != true + ).checkTriggerElementEnable(timer.enable) + ) + } + } + + triggerOn.remote?.let { remote -> + elementQueue.add( + RemoteTriggerElement( + name = remote.name ?: "远程触发", + remoteToken = yamlInput.pipelineInfo?.pipelineId?.let { + transferCache.getPipelineRemoteToken( + userId = yamlInput.userId, + projectId = yamlInput.projectCode, + pipelineId = it + ) + } ?: "" + ).checkTriggerElementEnable(remote.enable == EnableType.TRUE.value) + ) + } + } + + @Suppress("ComplexMethod") + fun yaml2TriggerGitlab(triggerOn: TriggerOn, elementQueue: MutableList) { + val repositoryType = if (triggerOn.repoName.isNullOrBlank()) { + TriggerRepositoryType.SELF + } else { + TriggerRepositoryType.NAME + } + triggerOn.push?.let { push -> + elementQueue.add( + CodeGitlabWebHookTriggerElement( + name = push.name ?: "Gitlab变更触发", + branchName = push.branches.nonEmptyOrNull()?.join(), + excludeBranchName = push.branchesIgnore.nonEmptyOrNull()?.join(), + includePaths = push.paths.nonEmptyOrNull()?.join(), + excludePaths = push.pathsIgnore.nonEmptyOrNull()?.join(), + includeUsers = push.users, + excludeUsers = push.usersIgnore, + pathFilterType = push.pathFilterType?.let { PathFilterType.valueOf(it) } + ?: PathFilterType.NamePrefixFilter, + eventType = CodeEventType.PUSH, + // todo action + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(push.enable) + ) + } + + triggerOn.tag?.let { tag -> + elementQueue.add( + CodeGitlabWebHookTriggerElement( + name = tag.name ?: "Gitlab变更触发", + tagName = tag.tags.nonEmptyOrNull()?.join(), + excludeTagName = tag.tagsIgnore.nonEmptyOrNull()?.join(), + includeUsers = tag.users, + excludeUsers = tag.usersIgnore, + eventType = CodeEventType.TAG_PUSH, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(tag.enable) + ) + } + + triggerOn.mr?.let { mr -> + elementQueue.add( + CodeGitlabWebHookTriggerElement( + name = mr.name ?: "Gitlab变更触发", + branchName = mr.targetBranches.nonEmptyOrNull()?.join(), + excludeBranchName = mr.targetBranchesIgnore.nonEmptyOrNull()?.join(), + includeSourceBranchName = mr.sourceBranches.nonEmptyOrNull()?.join(), + excludeSourceBranchName = mr.sourceBranchesIgnore.nonEmptyOrNull()?.join(), + includePaths = mr.paths.nonEmptyOrNull()?.join(), + excludePaths = mr.pathsIgnore.nonEmptyOrNull()?.join(), + includeUsers = mr.users, + excludeUsers = mr.usersIgnore, + block = mr.blockMr, + pathFilterType = mr.pathFilterType?.let { PathFilterType.valueOf(it) } + ?: PathFilterType.NamePrefixFilter, + // todo action + eventType = CodeEventType.MERGE_REQUEST, + repositoryType = repositoryType, + repositoryName = triggerOn.repoName + ).checkTriggerElementEnable(mr.enable) + ) + } + } + + private fun Element.checkTriggerElementEnable(enabled: Boolean?): Element { + if (additionalOptions == null) { + additionalOptions = ElementAdditionalOptions(runCondition = RunCondition.PRE_TASK_SUCCESS) + } + additionalOptions!!.enable = enabled ?: true + return this + } + + private fun List.join() = this.joinToString(separator = ",") + + private fun String.disjoin() = this.split(",") + + private fun List?.nonEmptyOrNull() = this?.ifEmpty { null } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/VariableDefault.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/VariableDefault.kt new file mode 100644 index 00000000000..22c25831447 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/VariableDefault.kt @@ -0,0 +1,25 @@ +package com.tencent.devops.process.yaml.transfer + +import com.tencent.devops.process.yaml.v3.models.on.ManualRule +import com.tencent.devops.process.yaml.v3.models.on.PreTriggerOnV3 +import kotlin.reflect.full.declaredMemberProperties + +object VariableDefault { + const val DEFAULT_TASK_TIME_OUT = 900L + const val DEFAULT_WAIT_QUEUE_TIME_MINUTE = 480 + const val DEFAULT_RETRY_COUNT = 0 + const val DEFAULT_CONTINUE_WHEN_FAILED = false + const val DEFAULT_PIPELINE_SETTING_MAX_QUEUE_SIZE = 10 + const val DEFAULT_JOB_PREPARE_TIMEOUT = 10 + const val DEFAULT_JOB_MAX_QUEUE_MINUTES = 60 + const val DEFAULT_JOB_MAX_RUNNING_MINUTES = 900 + const val DEFAULT_MUTEX_QUEUE_LENGTH = 5 + const val DEFAULT_MUTEX_TIMEOUT_MINUTES = 900 + const val DEFAULT_CHECKIN_TIMEOUT_HOURS = 24 + const val DEFAULT_MUTEX_QUEUE_ENABLE = false + val DEFAULT_MANUAL_RULE = ManualRule(enable = null, canElementSkip = null, useLatestParameters = null) + + fun T.nullIfDefault(value: T) = if (this == value) null else this + + val PreTriggerOnV3Properties = PreTriggerOnV3::class.declaredMemberProperties +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/VariableTransfer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/VariableTransfer.kt new file mode 100644 index 00000000000..bcb2f15f9b6 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/VariableTransfer.kt @@ -0,0 +1,270 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.transfer + +import com.tencent.devops.common.api.constant.CommonMessageCode.YAML_NOT_VALID +import com.tencent.devops.common.api.enums.ScmType +import com.tencent.devops.common.pipeline.Model +import com.tencent.devops.common.pipeline.container.TriggerContainer +import com.tencent.devops.common.pipeline.enums.BuildFormPropertyType +import com.tencent.devops.common.pipeline.pojo.BuildContainerType +import com.tencent.devops.common.pipeline.pojo.BuildFormProperty +import com.tencent.devops.common.pipeline.pojo.BuildFormValue +import com.tencent.devops.process.utils.FIXVERSION +import com.tencent.devops.process.utils.MAJORVERSION +import com.tencent.devops.process.utils.MINORVERSION +import com.tencent.devops.process.yaml.transfer.VariableDefault.nullIfDefault +import com.tencent.devops.process.yaml.v3.models.BuildContainerTypeYaml +import com.tencent.devops.process.yaml.v3.models.RecommendedVersion +import com.tencent.devops.process.yaml.v3.models.Variable +import com.tencent.devops.process.yaml.v3.models.VariablePropOption +import com.tencent.devops.process.yaml.v3.models.VariablePropType +import com.tencent.devops.process.yaml.v3.models.VariableProps +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +@Component +@Suppress("ComplexMethod") +class VariableTransfer @Autowired constructor() { + + companion object { + private val logger = LoggerFactory.getLogger(VariableTransfer::class.java) + private val ignoredVariable = + listOf(MAJORVERSION, "MajorVersion", MINORVERSION, "MinorVersion", FIXVERSION, "FixVersion") + } + + fun makeVariableFromModel(model: Model): Map? { + val result = mutableMapOf() + (model.stages[0].containers[0] as TriggerContainer).params.forEach { + if (it.id in ignoredVariable) return@forEach + val props = when { + // 不带 + it.type == BuildFormPropertyType.STRING && it.desc.isNullOrEmpty() -> null + it.type == BuildFormPropertyType.STRING -> VariableProps( + type = VariablePropType.VUEX_INPUT.value, + description = it.desc.nullIfDefault("") + ) + + it.type == BuildFormPropertyType.TEXTAREA -> VariableProps( + type = VariablePropType.VUEX_TEXTAREA.value, + description = it.desc.nullIfDefault("") + ) + + it.type == BuildFormPropertyType.ENUM -> VariableProps( + type = VariablePropType.SELECTOR.value, + description = it.desc.nullIfDefault(""), + options = it.options?.map { form -> + VariablePropOption(id = form.value, label = form.key) + }, + payload = it.payload + ) + + it.type == BuildFormPropertyType.DATE -> null // not use + it.type == BuildFormPropertyType.LONG -> null // not use + it.type == BuildFormPropertyType.BOOLEAN -> VariableProps( + type = VariablePropType.BOOLEAN.value, + description = it.desc.nullIfDefault("") + ) + + it.type == BuildFormPropertyType.SVN_TAG -> null // not use + it.type == BuildFormPropertyType.GIT_REF -> VariableProps( + type = VariablePropType.GIT_REF.value, + repoHashId = it.repoHashId, + description = it.desc.nullIfDefault("") + ) + + it.type == BuildFormPropertyType.MULTIPLE -> VariableProps( + type = VariablePropType.CHECKBOX.value, + description = it.desc.nullIfDefault(""), + options = it.options?.map { form -> + VariablePropOption(id = form.value, label = form.key) + }, + payload = it.payload + ) + + it.type == BuildFormPropertyType.CODE_LIB -> VariableProps( + type = VariablePropType.CODE_LIB.value, + scmType = it.scmType?.alis, + description = it.desc.nullIfDefault("") + ) + + it.type == BuildFormPropertyType.CONTAINER_TYPE -> VariableProps( + type = VariablePropType.CONTAINER_TYPE.value, + containerType = with(it.containerType) { + this?.let { + BuildContainerTypeYaml( + buildType, os + ) + } + }, + description = it.desc.nullIfDefault("") + ) // 构建机类型(公共构建机，第三方构建机，PCG构建机等) + it.type == BuildFormPropertyType.ARTIFACTORY -> VariableProps( + type = VariablePropType.ARTIFACTORY.value, + glob = it.glob, + properties = it.properties?.ifEmpty { null }, + description = it.desc.nullIfDefault("") + ) // 版本仓库 + it.type == BuildFormPropertyType.SUB_PIPELINE -> VariableProps( + type = VariablePropType.SUB_PIPELINE.value, + description = it.desc.nullIfDefault("") + ) // 子流水线 + it.type == BuildFormPropertyType.CUSTOM_FILE -> VariableProps( + type = VariablePropType.CUSTOM_FILE.value, + description = it.desc.nullIfDefault("") + ) // 自定义仓库文件 + it.type == BuildFormPropertyType.PASSWORD -> null // not use + it.type == BuildFormPropertyType.TEMPORARY -> null // not use + else -> null + } + val const = it.constant.nullIfDefault(false) + result[it.id] = Variable( + value = it.defaultValue.toString(), + name = it.name, + readonly = if (const == true) true else it.readOnly.nullIfDefault(false), + allowModifyAtStartup = if (const != true) it.required.nullIfDefault(true) else null, + valueNotEmpty = it.valueNotEmpty.nullIfDefault(false), + const = const, + props = props + ) + } + return if (result.isEmpty()) { + null + } else { + result + } + } + + fun makeRecommendedVersion(model: Model): RecommendedVersion? { + val triggerContainer = model.stages[0].containers[0] as TriggerContainer + val res = if (triggerContainer.buildNo != null) { + with(triggerContainer.buildNo) { + RecommendedVersion( + enabled = true, allowModifyAtStartup = this!!.required, buildNo = RecommendedVersion.BuildNo( + this.buildNo, + RecommendedVersion.Strategy.parse(this.buildNoType).alis + ) + ) + } + } else return null + + (model.stages[0].containers[0] as TriggerContainer).params.forEach { + if (it.id == MAJORVERSION || it.id == "MajorVersion") { + res.major = it.defaultValue.toString().toIntOrNull() ?: 0 + } + + if (it.id == MINORVERSION || it.id == "MinorVersion") { + res.minor = it.defaultValue.toString().toIntOrNull() ?: 0 + } + + if (it.id == FIXVERSION || it.id == "FixVersion") { + res.fix = it.defaultValue.toString().toIntOrNull() ?: 0 + } + } + return res + } + + fun makeVariableFromYaml( + variables: Map? + ): List { + if (variables.isNullOrEmpty()) { + return emptyList() + } + val buildFormProperties = mutableListOf() + variables.forEach { (key, variable) -> + val type = VariablePropType.findType(variable.props?.type)?.toBuildFormPropertyType() + ?: BuildFormPropertyType.STRING + check(key, variable) + buildFormProperties.add( + BuildFormProperty( + id = key, + name = variable.name, + required = variable.allowModifyAtStartup ?: true, + constant = variable.const ?: false, + type = type, + defaultValue = when (type) { + BuildFormPropertyType.BOOLEAN -> variable.value?.toBoolean() ?: false + else -> variable.value ?: "" + }, + options = variable.props?.options?.map { + BuildFormValue( + key = it.label ?: it.id.toString(), + value = it.id.toString() + ) + }, + desc = variable.props?.description, + repoHashId = variable.props?.repoHashId, + relativePath = null, + scmType = ScmType.parse(variable.props?.scmType), + containerType = with(variable.props?.containerType) { + this?.let { + BuildContainerType( + buildType, os + ) + } + }, + glob = variable.props?.glob, + properties = variable.props?.properties, + readOnly = variable.readonly ?: false, + valueNotEmpty = variable.valueNotEmpty ?: false + ) + ) + } + return buildFormProperties + } + + private fun check(key: String, variable: Variable) { + if (key.length > 64) { + throw PipelineTransferException( + YAML_NOT_VALID, + arrayOf("variable key no more than 64 characters. variable: $key") + ) + } + + if (variable.const == true && variable.readonly == false) { + throw PipelineTransferException( + YAML_NOT_VALID, + arrayOf("When the const attribute is set to true, readonly must be true. variable: $key") + ) + } + if (variable.const == true && variable.allowModifyAtStartup != null) { + throw PipelineTransferException( + YAML_NOT_VALID, + arrayOf( + "The const attribute and the allow-modify-at-startup attribute are mutually exclusive. " + + "If configured at the same time, the verification will fail. variable: $key" + ) + ) + } + + if (variable.const == true && variable.readonly == null) { + variable.readonly = true + } + } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/YamlIndexService.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/YamlIndexService.kt new file mode 100644 index 00000000000..732b1a68657 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/YamlIndexService.kt @@ -0,0 +1,203 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.transfer + +import com.fasterxml.jackson.core.type.TypeReference +import com.tencent.devops.common.api.constant.CommonMessageCode.ELEMENT_NOT_SUPPORT_TRANSFER +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.api.util.YamlUtil +import com.tencent.devops.common.pipeline.pojo.transfer.ElementInsertBody +import com.tencent.devops.common.pipeline.pojo.transfer.ElementInsertResponse +import com.tencent.devops.common.pipeline.pojo.transfer.PositionResponse +import com.tencent.devops.common.pipeline.pojo.transfer.PreStep +import com.tencent.devops.common.pipeline.pojo.transfer.TransferVMBaseOS +import com.tencent.devops.process.yaml.v2.models.job.PreJob +import com.tencent.devops.process.yaml.v2.models.stage.PreStage +import com.tencent.devops.process.yaml.v3.models.ITemplateFilter +import com.tencent.devops.process.yaml.v3.models.job.Job +import com.tencent.devops.process.yaml.v3.models.job.JobRunsOnType +import com.tencent.devops.process.yaml.v3.utils.ScriptYmlUtils +import java.util.concurrent.atomic.AtomicInteger +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +@Component +@Suppress("NestedBlockDepth", "ComplexMethod") +class YamlIndexService @Autowired constructor( + val dispatchTransfer: DispatchTransfer, + val elementTransfer: ElementTransfer +) { + + companion object { + private val logger = LoggerFactory.getLogger(YamlIndexService::class.java) + } + + fun modelTaskInsert( + userId: String, + projectId: String, + pipelineId: String, + line: Int, + column: Int, + data: ElementInsertBody + ): ElementInsertResponse { + val pYml = YamlUtil.getObjectMapper().readValue(data.yaml, object : TypeReference() {}) + val position = position( + userId = userId, line = line, column = column, yaml = data.yaml, preYaml = pYml + ) + val yml = elementTransfer.element2YamlStep(data.data, projectId) ?: throw PipelineTransferException( + ELEMENT_NOT_SUPPORT_TRANSFER, + arrayOf("${data.data.getClassType()}(${data.data.name})") + ) + val index = TransferMapper.indexYaml(position = position, pYml = pYml, yml = yml, type = data.type) + val outYaml = TransferMapper.toYaml(pYml) + return ElementInsertResponse(yaml = outYaml, mark = TransferMapper.markYaml(index, outYaml)) + } + + fun position( + userId: String, + line: Int, + column: Int, + yaml: String, + preYaml: ITemplateFilter + ): PositionResponse { + val index = TransferMapper.indexYaml(yaml, line, column) + ?: return PositionResponse(type = PositionResponse.PositionType.SETTING) + return checkYamlIndex(userId, preYaml, index) + } + + fun checkYamlIndex( + userId: String, + preYaml: ITemplateFilter, + nodeIndex: TransferMapper.NodeIndex + ): PositionResponse { + when (nodeIndex.key) { + ITemplateFilter::stages.name -> { + val next = nodeIndex.next ?: return PositionResponse(type = PositionResponse.PositionType.STAGE) + val index = next.index ?: throw PacYamlNotValidException(nodeIndex.toString()) + return checkStage(userId, preYaml.stages!![index], next.next).apply { + stageIndex = index + } + } + + ITemplateFilter::jobs.name -> { + val jobs = preYaml.jobs!! + val next = nodeIndex.next ?: return PositionResponse(type = PositionResponse.PositionType.STAGE) + val key = next.key ?: throw PacYamlNotValidException(nodeIndex.toString()) + val indexAtomic = AtomicInteger(0) + jobs.forEach { (jobId, job) -> + val index = indexAtomic.getAndIncrement() + if (jobId == key) { + return checkJob(userId, job as Map, next.next).apply { + this.jobId = key + this.containerIndex = index + } + } + } + } + + ITemplateFilter::steps.name -> { + val next = nodeIndex.next ?: return PositionResponse(type = PositionResponse.PositionType.JOB) + val index = next.index ?: throw PacYamlNotValidException(nodeIndex.toString()) + return checkStep(userId, preYaml.steps!![index], next.next).apply { + stepIndex = index + } + } + + ITemplateFilter::finally.name -> { + val next = nodeIndex.next ?: return PositionResponse( + type = PositionResponse.PositionType.STAGE, + stageIndex = -1 + ) + return checkStage(userId, preYaml.finally!!, next.next).apply { + stageIndex = -1 + } + } + } + return PositionResponse(type = PositionResponse.PositionType.SETTING) + } + + fun checkStage( + userId: String, + stage: Map, + nodeIndex: TransferMapper.NodeIndex? + ): PositionResponse { + if (nodeIndex?.key == PreStage::jobs.name) { + val jobs = stage[PreStage::jobs.name] as LinkedHashMap + val next = nodeIndex.next ?: return PositionResponse(type = PositionResponse.PositionType.STAGE) + val key = next.key ?: throw PacYamlNotValidException(nodeIndex.toString()) + val indexAtomic = AtomicInteger(0) + jobs.forEach { (jobId, job) -> + val index = indexAtomic.getAndIncrement() + if (jobId == key) { + return checkJob(userId, job as Map, next.next).apply { + this.jobId = key + this.containerIndex = index + } + } + } + } + return PositionResponse(type = PositionResponse.PositionType.STAGE) + } + + fun checkJob( + userId: String, + job: Map, + nodeIndex: TransferMapper.NodeIndex? + ): PositionResponse { + val runsOn = ScriptYmlUtils.formatRunsOn(job["runs-on"]) + val baseOs = if (runsOn.poolName == JobRunsOnType.AGENT_LESS.type) { + TransferVMBaseOS.BUILD_LESS + } else { + val (_, os) = dispatchTransfer.makeDispatchType(Job(runsOn = runsOn), null) + TransferVMBaseOS.valueOf(os.name) + } + if (nodeIndex?.key == PreJob::steps.name) { + val steps = job[PreJob::steps.name] as List + val next = nodeIndex.next ?: return PositionResponse(type = PositionResponse.PositionType.JOB) + val index = next.index ?: throw PacYamlNotValidException(nodeIndex.toString()) + return checkStep(userId, steps[index] as Map, next.next).apply { + stepIndex = index + jobBaseOs = baseOs + } + } + return PositionResponse(type = PositionResponse.PositionType.JOB, jobBaseOs = baseOs) + } + + fun checkStep( + userId: String, + job: Map, + nodeIndex: TransferMapper.NodeIndex? + ): PositionResponse { + val preStep = JsonUtil.anyTo(job, object : TypeReference() {}) + return PositionResponse( + type = PositionResponse.PositionType.STEP, + element = elementTransfer.yaml2element(userId, ScriptYmlUtils.preStepToStep(preStep), null) + ) + } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/aspect/IPipelineTransferAspect.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/aspect/IPipelineTransferAspect.kt new file mode 100644 index 00000000000..d6acbf09023 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/aspect/IPipelineTransferAspect.kt @@ -0,0 +1,42 @@ +package com.tencent.devops.process.yaml.transfer.aspect + +import com.tencent.devops.common.pipeline.Model +import com.tencent.devops.common.pipeline.container.Container +import com.tencent.devops.common.pipeline.container.Stage +import com.tencent.devops.common.pipeline.pojo.element.Element +import com.tencent.devops.common.pipeline.pojo.transfer.PreStep +import com.tencent.devops.process.yaml.v3.models.IPreTemplateScriptBuildYamlParser +import com.tencent.devops.process.yaml.v3.models.job.PreJob +import com.tencent.devops.process.yaml.v3.models.on.TriggerOn +import com.tencent.devops.process.yaml.v3.models.stage.PreStage +import com.tencent.devops.process.yaml.v3.models.job.Job as YamlV3Job +import com.tencent.devops.process.yaml.v3.models.stage.Stage as YamlV3Stage +import com.tencent.devops.process.yaml.v3.models.step.Step as YamlV3Step + +interface IPipelineTransferAspect { + fun before(jp: PipelineTransferJoinPoint): Any? = null + + fun after(jp: PipelineTransferJoinPoint) = Unit +} + +interface IPipelineTransferAspectTrigger : IPipelineTransferAspect +interface IPipelineTransferAspectElement : IPipelineTransferAspect +interface IPipelineTransferAspectJob : IPipelineTransferAspect +interface IPipelineTransferAspectStage : IPipelineTransferAspect + +interface IPipelineTransferAspectModel : IPipelineTransferAspect + +interface PipelineTransferJoinPoint { + fun modelStage(): Stage? + fun modelJob(): Container? + fun modelElement(): Element? + fun model(): Model? + fun yamlStage(): YamlV3Stage? + fun yamlPreStage(): PreStage? + fun yamlJob(): YamlV3Job? + fun yamlPreJob(): PreJob? + fun yamlStep(): YamlV3Step? + fun yamlPreStep(): PreStep? + fun yamlTriggerOn(): TriggerOn? + fun yaml(): IPreTemplateScriptBuildYamlParser? +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/aspect/PipelineTransferAspectLoader.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/aspect/PipelineTransferAspectLoader.kt new file mode 100644 index 00000000000..72485c57432 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/aspect/PipelineTransferAspectLoader.kt @@ -0,0 +1,144 @@ +package com.tencent.devops.process.yaml.transfer.aspect + +import com.tencent.devops.common.pipeline.pojo.transfer.Resources +import com.tencent.devops.common.pipeline.pojo.transfer.ResourcesPools +import com.tencent.devops.process.yaml.v3.models.PreTemplateScriptBuildYamlV3Parser +import com.tencent.devops.process.yaml.v3.models.job.RunsOn +import java.util.LinkedList +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentMap + +object PipelineTransferAspectLoader { + private val cachedInstances: ConcurrentMap = ConcurrentHashMap() + + fun getOrPutExtension(name: String?, createExtension: () -> IPipelineTransferAspect): IPipelineTransferAspect? { + var instance = cachedInstances[name] + // 从缓存中获取，如果不存在就创建 + if (instance == null) { + synchronized(cachedInstances) { + instance = cachedInstances[name] + if (instance == null) { + instance = createExtension() + cachedInstances[name] = instance + } + } + } + return instance + } + + fun sharedEnvTransfer( + aspects: LinkedList = LinkedList() + ): LinkedList { + val pools = mutableListOf() + aspects.add( + object : IPipelineTransferAspectJob { + override fun before(jp: PipelineTransferJoinPoint): Any? { + if (jp.yamlJob() != null && jp.yaml()?.formatResources()?.pools != null) { + jp.yaml()?.formatResources()?.pools?.find { + it.name == jp.yamlJob()!!.runsOn.poolName + }?.let { pool -> + jp.yamlJob()!!.runsOn.envProjectId = pool.from?.substringBefore("@") + jp.yamlJob()!!.runsOn.poolName = pool.from?.substringAfter("@") + } + } + + return null + } + + override fun after(jp: PipelineTransferJoinPoint) { + if (jp.yamlPreJob()?.runsOn != null && + jp.yamlPreJob()?.runsOn is RunsOn && + (jp.yamlPreJob()?.runsOn as RunsOn).envProjectId != null + ) { + val pool = jp.yamlPreJob()?.runsOn as RunsOn + pools.add( + ResourcesPools( + from = "${pool.envProjectId}@${pool.poolName}", + name = pool.poolName + ) + ) + } + } + } + ) + + aspects.add( + object : IPipelineTransferAspectModel { + override fun after(jp: PipelineTransferJoinPoint) { + if (jp.yaml() != null && + jp.yaml() is PreTemplateScriptBuildYamlV3Parser && + pools.isNotEmpty() + ) { + val v3 = jp.yaml() as PreTemplateScriptBuildYamlV3Parser + v3.resources = Resources( + repositories = v3.resources?.repositories, + pools = v3.resources?.pools?.plus(pools) ?: pools + ) + } + } + } + ) + return aspects + } + + fun initByDefaultTriggerOn( + defaultRepo: () -> String, + aspects: LinkedList = LinkedList() + ): LinkedList { + // val repoName = lazy { defaultRepo() } + /*aspects.add( + object : IPipelineTransferAspectTrigger { + override fun before(jp: PipelineTransferJoinPoint): Any? { + if (jp.yamlTriggerOn() != null && jp.yamlTriggerOn()!!.repoName == null) { + jp.yamlTriggerOn()!!.repoName = repoName.value + } + return null + } + } + )*/ + /*checkout 新增 self类型，此处暂时去掉转换 */ +// aspects.add( +// object : IPipelineTransferAspectElement { +// override fun before(jp: PipelineTransferJoinPoint): Any? { +// if (jp.yamlStep() != null && jp.yamlStep()!!.checkout == "self") { +// jp.yamlStep()!!.checkout = repoName.value +// } +// return null +// } +// } +// ) + /*aspects.add( + // 一个触发器时，如果为默认仓库则忽略repoName和type + object : IPipelineTransferAspectModel { + override fun after(jp: PipelineTransferJoinPoint) { + if (jp.yaml() is PreTemplateScriptBuildYamlV3 && + (jp.yaml() as PreTemplateScriptBuildYamlV3).triggerOn is PreTriggerOnV3 + ) { + val triggerOn = (jp.yaml() as PreTemplateScriptBuildYamlV3).triggerOn as PreTriggerOnV3 + if (triggerOn.repoName == repoName.value) { + triggerOn.repoName = null + triggerOn.type = null + } + } + } + } + )*/ + return aspects + } + + fun checkInvalidElement( + invalidElement: MutableList, + aspects: LinkedList = LinkedList() + ): LinkedList { + aspects.add( + object : IPipelineTransferAspectElement { + override fun after(jp: PipelineTransferJoinPoint) { + if (jp.yamlPreStep() == null) { + invalidElement.add("${jp.modelElement()?.getClassType()}(${jp.modelElement()?.name})") + } + } + } + ) + return aspects + } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/aspect/PipelineTransferAspectWrapper.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/aspect/PipelineTransferAspectWrapper.kt new file mode 100644 index 00000000000..f818d2ae50c --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/aspect/PipelineTransferAspectWrapper.kt @@ -0,0 +1,193 @@ +package com.tencent.devops.process.yaml.transfer.aspect + +import com.tencent.devops.common.pipeline.Model +import com.tencent.devops.common.pipeline.container.Container +import com.tencent.devops.common.pipeline.container.Stage +import com.tencent.devops.common.pipeline.pojo.element.Element +import com.tencent.devops.common.pipeline.pojo.transfer.PreStep +import com.tencent.devops.process.yaml.v3.models.IPreTemplateScriptBuildYamlParser +import com.tencent.devops.process.yaml.v3.models.job.PreJob +import com.tencent.devops.process.yaml.v3.models.on.TriggerOn +import com.tencent.devops.process.yaml.v3.models.stage.PreStage +import java.util.LinkedList +import kotlin.reflect.jvm.jvmName +import com.tencent.devops.process.yaml.v3.models.job.Job as YamlV3Job +import com.tencent.devops.process.yaml.v3.models.stage.Stage as YamlV3Stage +import com.tencent.devops.process.yaml.v3.models.step.Step as YamlV3Step + +class PipelineTransferAspectWrapper { + private val aspectTrigger = LinkedList() + private val aspectElement = LinkedList() + private val aspectJob = LinkedList() + private val aspectStage = LinkedList() + private val aspectModel = LinkedList() + + private val pipelineTransferJoinPoint = PipelineTransferJoinPointImpl() + private var clzName: String? = null + + constructor(list: LinkedList) { + list.forEach { + when (it) { + is IPipelineTransferAspectTrigger -> aspectTrigger.add(it) + is IPipelineTransferAspectElement -> aspectElement.add(it) + is IPipelineTransferAspectJob -> aspectJob.add(it) + is IPipelineTransferAspectStage -> aspectStage.add(it) + is IPipelineTransferAspectModel -> aspectModel.add(it) + } + } + } + + enum class AspectType { + BEFORE, + AFTER; + } + + /* + * AspectType.BEFORE: model -> yaml before stage + * AspectType.AFTER: yaml -> model after stage + */ + fun setModelStage4Model(modelStage: Stage?, aspectType: AspectType) { + pipelineTransferJoinPoint.modelStage = modelStage + clzName = IPipelineTransferAspectStage::class.jvmName + when (aspectType) { + AspectType.BEFORE -> aspectBefore() + AspectType.AFTER -> aspectAfter() + } + } + + /* + * AspectType.BEFORE: model -> yaml before job + * AspectType.AFTER: yaml -> model after job + */ + fun setModelJob4Model(modelJob: Container?, aspectType: AspectType) { + pipelineTransferJoinPoint.modelJob = modelJob + clzName = IPipelineTransferAspectJob::class.jvmName + when (aspectType) { + AspectType.BEFORE -> aspectBefore() + AspectType.AFTER -> aspectAfter() + } + } + + /* + * AspectType.BEFORE: model -> yaml before element + * AspectType.AFTER: yaml -> model after element + */ + fun setModelElement4Model(modelElement: Element?, aspectType: AspectType) { + pipelineTransferJoinPoint.modelElement = modelElement + clzName = IPipelineTransferAspectElement::class.jvmName + when (aspectType) { + AspectType.BEFORE -> aspectBefore() + AspectType.AFTER -> aspectAfter() + } + } + + /* + * AspectType.BEFORE: model -> yaml before Model + * AspectType.AFTER: yaml -> model after Model + */ + fun setModel4Model(model: Model?, aspectType: AspectType) { + pipelineTransferJoinPoint.model = model + clzName = IPipelineTransferAspectModel::class.jvmName + when (aspectType) { + AspectType.BEFORE -> aspectBefore() + AspectType.AFTER -> aspectAfter() + } + } + + /* + * AspectType.BEFORE: yaml -> moel before stage + * AspectType.AFTER: model -> yaml after stage + */ + fun setYamlStage4Yaml(yamlStage: YamlV3Stage? = null, yamlPreStage: PreStage? = null, aspectType: AspectType) { + pipelineTransferJoinPoint.yamlStage = yamlStage + pipelineTransferJoinPoint.yamlPreStage = yamlPreStage + clzName = IPipelineTransferAspectStage::class.jvmName + when (aspectType) { + AspectType.BEFORE -> aspectBefore() + AspectType.AFTER -> aspectAfter() + } + } + + /* + * AspectType.BEFORE: yaml -> moel before job + * AspectType.AFTER: model -> yaml after job + */ + fun setYamlJob4Yaml(yamlJob: YamlV3Job? = null, yamlPreJob: PreJob? = null, aspectType: AspectType) { + pipelineTransferJoinPoint.yamlJob = yamlJob + pipelineTransferJoinPoint.yamlPreJob = yamlPreJob + clzName = IPipelineTransferAspectJob::class.jvmName + when (aspectType) { + AspectType.BEFORE -> aspectBefore() + AspectType.AFTER -> aspectAfter() + } + } + + /* + * AspectType.BEFORE: yaml -> moel before step + * AspectType.AFTER: model -> yaml after step + */ + fun setYamlStep4Yaml(yamlStep: YamlV3Step? = null, yamlPreStep: PreStep? = null, aspectType: AspectType) { + pipelineTransferJoinPoint.yamlStep = yamlStep + pipelineTransferJoinPoint.yamlPreStep = yamlPreStep + clzName = IPipelineTransferAspectElement::class.jvmName + when (aspectType) { + AspectType.BEFORE -> aspectBefore() + AspectType.AFTER -> aspectAfter() + } + } + + /* + * AspectType.BEFORE: yaml -> moel before Yaml + * AspectType.AFTER: model -> yaml after Yaml + */ + fun setYaml4Yaml(yaml: IPreTemplateScriptBuildYamlParser? = null, aspectType: AspectType) { + pipelineTransferJoinPoint.yaml = yaml + clzName = IPipelineTransferAspectModel::class.jvmName + when (aspectType) { + AspectType.BEFORE -> aspectBefore() + AspectType.AFTER -> aspectAfter() + } + } + + /* + * AspectType.BEFORE: yaml -> moel before trigger on + * AspectType.AFTER: model -> yaml after trigger on + */ + fun setYamlTriggerOn(yamlTriggerOn: TriggerOn?, aspectType: AspectType) { + pipelineTransferJoinPoint.yamlTriggerOn = yamlTriggerOn + clzName = IPipelineTransferAspectTrigger::class.jvmName + when (aspectType) { + AspectType.BEFORE -> aspectBefore() + AspectType.AFTER -> aspectAfter() + } + } + + private fun aspectBefore(): Any? { + val wrappers = iPipelineTransferAspects(clzName) + if (wrappers.isNullOrEmpty()) return null + var res: Any? = null + wrappers.forEach { + res = it.before(pipelineTransferJoinPoint) + } + return res + } + + private fun aspectAfter(): Any? { + val wrappers = iPipelineTransferAspects(clzName) + if (wrappers.isNullOrEmpty()) return null + var res: Any? = null + wrappers.forEach { + res = it.after(pipelineTransferJoinPoint) + } + return res + } + + private fun iPipelineTransferAspects(clzName: String?) = when (clzName) { + IPipelineTransferAspectTrigger::class.jvmName -> aspectTrigger + IPipelineTransferAspectElement::class.jvmName -> aspectElement + IPipelineTransferAspectJob::class.jvmName -> aspectJob + IPipelineTransferAspectStage::class.jvmName -> aspectStage + IPipelineTransferAspectModel::class.jvmName -> aspectModel + else -> null + } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/aspect/PipelineTransferJoinPointImpl.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/aspect/PipelineTransferJoinPointImpl.kt new file mode 100644 index 00000000000..e5a008e4523 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/aspect/PipelineTransferJoinPointImpl.kt @@ -0,0 +1,54 @@ +package com.tencent.devops.process.yaml.transfer.aspect + +import com.tencent.devops.common.pipeline.Model +import com.tencent.devops.common.pipeline.container.Container +import com.tencent.devops.common.pipeline.container.Stage +import com.tencent.devops.common.pipeline.pojo.element.Element +import com.tencent.devops.common.pipeline.pojo.transfer.PreStep +import com.tencent.devops.process.yaml.v3.models.IPreTemplateScriptBuildYamlParser +import com.tencent.devops.process.yaml.v3.models.job.PreJob +import com.tencent.devops.process.yaml.v3.models.on.TriggerOn +import com.tencent.devops.process.yaml.v3.models.stage.PreStage +import com.tencent.devops.process.yaml.v3.models.job.Job as YamlV3Job +import com.tencent.devops.process.yaml.v3.models.stage.Stage as YamlV3Stage +import com.tencent.devops.process.yaml.v3.models.step.Step as YamlV3Step + +data class PipelineTransferJoinPointImpl( + var modelStage: Stage? = null, + var modelJob: Container? = null, + var modelElement: Element? = null, + var model: Model? = null, + var yamlStage: YamlV3Stage? = null, + var yamlPreStage: PreStage? = null, + var yamlJob: YamlV3Job? = null, + var yamlPreJob: PreJob? = null, + var yamlStep: YamlV3Step? = null, + var yamlPreStep: PreStep? = null, + var yamlTriggerOn: TriggerOn? = null, + var yaml: IPreTemplateScriptBuildYamlParser? = null +) : PipelineTransferJoinPoint { + + override fun modelStage(): Stage? = modelStage + + override fun modelJob(): Container? = modelJob + + override fun modelElement(): Element? = modelElement + + override fun model(): Model? = model + + override fun yamlStage(): YamlV3Stage? = yamlStage + + override fun yamlPreStage(): PreStage? = yamlPreStage + + override fun yamlJob(): YamlV3Job? = yamlJob + + override fun yamlPreJob(): PreJob? = yamlPreJob + + override fun yamlStep(): YamlV3Step? = yamlStep + + override fun yamlPreStep(): PreStep? = yamlPreStep + + override fun yamlTriggerOn(): TriggerOn? = yamlTriggerOn + + override fun yaml(): IPreTemplateScriptBuildYamlParser? = yaml +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/inner/TransferCreator.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/inner/TransferCreator.kt new file mode 100644 index 00000000000..c81858c5f07 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/inner/TransferCreator.kt @@ -0,0 +1,79 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.transfer.inner + +import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildAtomElement +import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildLessAtomElement +import com.tencent.devops.common.pipeline.type.BuildType +import com.tencent.devops.process.yaml.v3.models.step.Step + +/** + * ModelCreate的内部类，用来放一些不同使用者的不同方法和参数 + */ +interface TransferCreator { + // 控制run插件是否是研发商店插件 + val marketRunTask: Boolean + + // 研发商店的run插件的code + val runPlugInAtomCode: String? + + // 研发商店的run插件的版本 + val runPlugInVersion: String? + + // 默认的公共镜像 + val defaultImageCode: String + + // 默认的公共镜像版本 + val defaultImageVersion: String + + /** + * 构造具有特殊语法的checkout插件 + * @param step 当前step对象 + * @param event model创建的总事件 + * @param additionalOptions 插件的控制参数 + */ + fun transferCheckoutElement( + step: Step + ): MarketBuildAtomElement + + /** + * 构造编译类的插件 + */ + fun transferMarketBuildAtomElement( + step: Step + ): MarketBuildAtomElement + + /** + * 构造编译类的插件 + */ + fun transferMarketBuildLessAtomElement( + step: Step + ): MarketBuildLessAtomElement + + fun defaultLinuxDispatchType(): BuildType +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/inner/TransferCreatorImpl.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/inner/TransferCreatorImpl.kt new file mode 100644 index 00000000000..3468f0736e9 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/inner/TransferCreatorImpl.kt @@ -0,0 +1,147 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.transfer.inner + +import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildAtomElement +import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildLessAtomElement +import com.tencent.devops.common.pipeline.type.BuildType +import com.tencent.devops.process.yaml.transfer.pojo.CheckoutAtomParam +import com.tencent.devops.process.yaml.v3.models.step.Step +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Value +import org.springframework.stereotype.Component + +@Component +class TransferCreatorImpl @Autowired constructor() : TransferCreator { + @Value("\${marketRun.enable:#{false}}") + private val marketRunTaskData: Boolean = false + + @Value("\${marketRun.atomCode:#{null}}") + private val runPlugInAtomCodeData: String? = null + + @Value("\${marketRun.atomVersion:#{null}}") + private val runPlugInVersionData: String? = null + + @Value("\${container.defaultImage:#{null}}") + private val defaultImageData: String = "tlinux3_ci:2.*" + + companion object { + private const val STREAM_CHECK_AUTH_TYPE = "AUTH_USER_TOKEN" + } + + override val marketRunTask: Boolean + get() = marketRunTaskData + + override val runPlugInAtomCode: String? + get() = runPlugInAtomCodeData + + override val runPlugInVersion: String? + get() = runPlugInVersionData + + override val defaultImageCode: String + get() = defaultImageData.substringBefore(":") + + override val defaultImageVersion: String + get() = defaultImageData.substringAfter(":") + + override fun transferCheckoutElement( + step: Step + ): MarketBuildAtomElement { + // checkout插件装配 + val inputMap = mutableMapOf() + if (!step.with.isNullOrEmpty()) { + inputMap.putAll(step.with!!) + } + when { + step.checkout?.self == true -> { + inputMap[CheckoutAtomParam::repositoryType.name] = CheckoutAtomParam.CheckoutRepositoryType.SELF + } + + step.checkout?.repoName != null -> { + inputMap[CheckoutAtomParam::repositoryName.name] = step.checkout?.repoName!! + inputMap[CheckoutAtomParam::repositoryType.name] = CheckoutAtomParam.CheckoutRepositoryType.NAME + } + + step.checkout?.repoId != null -> { + inputMap[CheckoutAtomParam::repositoryHashId.name] = step.checkout?.repoId!! + inputMap[CheckoutAtomParam::repositoryType.name] = CheckoutAtomParam.CheckoutRepositoryType.ID + } + + step.checkout?.url?.startsWith("http") == true -> { + inputMap[CheckoutAtomParam::repositoryUrl.name] = step.checkout?.url!! + inputMap[CheckoutAtomParam::repositoryType.name] = CheckoutAtomParam.CheckoutRepositoryType.URL + } + + else -> { + inputMap[CheckoutAtomParam::repositoryType.name] = CheckoutAtomParam.CheckoutRepositoryType.SELF + } + } + + val data = mutableMapOf() + data["input"] = inputMap + + return MarketBuildAtomElement( + id = step.taskId, + name = step.name ?: "checkout", + stepId = step.id, + atomCode = "checkout", + version = "1.*", + data = data + ) + } + + override fun transferMarketBuildAtomElement( + step: Step + ): MarketBuildAtomElement { + val data = mutableMapOf() + data["input"] = step.with ?: Any() + return MarketBuildAtomElement( + id = step.taskId, + name = step.name ?: step.uses!!.split('@')[0], + stepId = step.id, + atomCode = step.uses!!.split('@')[0], + version = step.uses!!.split('@')[1], + data = data + ) + } + + override fun transferMarketBuildLessAtomElement(step: Step): MarketBuildLessAtomElement { + val data = mutableMapOf() + data["input"] = step.with ?: Any() + return MarketBuildLessAtomElement( + id = step.taskId, + name = step.name ?: step.uses!!.split('@')[0], + stepId = step.id, + atomCode = step.uses!!.split('@')[0], + version = step.uses!!.split('@')[1], + data = data + ) + } + + override fun defaultLinuxDispatchType(): BuildType = BuildType.DOCKER +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/CheckoutAtomParam.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/CheckoutAtomParam.kt new file mode 100644 index 00000000000..3e2b4c83228 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/CheckoutAtomParam.kt @@ -0,0 +1,237 @@ +package com.tencent.devops.process.yaml.transfer.pojo + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonProperty +import com.tencent.devops.common.api.enums.RepositoryConfig +import com.tencent.devops.common.api.enums.RepositoryType + +@JsonInclude(JsonInclude.Include.NON_NULL) +data class CheckoutAtomParam( + /** + * 代码库, 必选, 默认: ID, options: 按代码库选择[ID] | 按代码库别名输入[NAME] | 按仓库URL输入[URL] + */ + var repositoryType: CheckoutRepositoryType? = null, + + /** + * 按代码库选择, 当 [repositoryType] = [ID] 时必选 + */ + var repositoryHashId: String? = null, + + /** + * 按代码库别名输入, 当 [repositoryType] = [NAME] 时必选 + */ + var repositoryName: String? = null, + + /** + * 代码库链接, 当 [repositoryType] = [URL] 时必选 + */ + var repositoryUrl: String? = null, + + /** + * 授权类型, 默认: TICKET, + * 当 [repositoryType] = [URL] 时必选, single, + * + * options: + * + * EMPTY[空] | TICKET[凭证] | ACCESS_TOKEN[access token] | USERNAME_PASSWORD[username/password] | + * START_USER_TOKEN[流水线启动人token] | PERSONAL_ACCESS_TOKEN[工蜂personal_access_token] + */ + var authType: AuthType? = null, + var authUserId: String? = null, + + /** + * 代码库凭证, 当 [repositoryType] = [URL] 和 [authType] = [TICKET] 时必选 + */ + var ticketId: String? = null, + + /** + * access token, 当 [repositoryType] = [URL] 和 [authType] = [ACCESS_TOKEN] 时必选 + */ + var accessToken: String? = null, + + /** + * 工蜂personal_access_token, 当 [repositoryType] = [URL] 和 [authType] = [PERSONAL_ACCESS_TOKEN] 时必选 + */ + var personalAccessToken: String? = null, + + /** + * username, 当 [repositoryType] = [URL] 和 [authType] = [USERNAME_PASSWORD] 时必选 + */ + var username: String? = null, + + /** + * password, 当 [repositoryType] = [URL] 和 [authType] = [USERNAME_PASSWORD] 时必选 + */ + var password: String? = null, + + /** + * 指定拉取方式, 默认: BRANCH, single, options: BRANCH[BRANCH] | TAG[TAG] | COMMIT_ID[COMMIT_ID] + */ + var pullType: String? = null, + + /** + * 分支/TAG/COMMIT, 必选, 默认: master + */ + var refName: String? = null, + + /** + * 代码保存路径 + */ + var localPath: String? = null, + + /** + * 拉取策略, 默认: REVERT_UPDATE, + * + * options: + * + * Revert Update[REVERT_UPDATE] | Fresh Checkout[FRESH_CHECKOUT] | Increment Update[INCREMENT_UPDATE] + */ + var strategy: String? = null, + + /** + * git fetch的depth参数值 + */ + var fetchDepth: Int? = null, + + /** + * 启用拉取指定分支, 默认: false + */ + val enableFetchRefSpec: Boolean? = null, + + /** + * 插件配置的分支不需要设置，默认会设置.配置的分支必须存在，否则会报错, 当 [enableFetchRefSpec] = [true] 时必选 + */ + val fetchRefSpec: String? = null, + + /** + * 是否开启Git Lfs, 默认: true + */ + var enableGitLfs: Boolean? = null, + + /** + * lfs并发上传下载的数量 + */ + val lfsConcurrentTransfers: Int? = null, + + /** + * MR事件触发时执行Pre-Merge, 必选, 默认: true + */ + var enableVirtualMergeBranch: Boolean? = null, + + /** + * 启用子模块, 默认: true + */ + var enableSubmodule: Boolean? = null, + + /** + * 子模块路径当 [enableSubmodule] = [true] 时必选 + */ + var submodulePath: String? = null, + + /** + * 执行git submodule update后面是否带上--remote参数, 默认: false, 当 [enableSubmodule] = [true] 时必选 + */ + var enableSubmoduleRemote: Boolean? = null, + + /** + * 执行git submodule update后面是否带上--recursive参数, 默认: true, 当 [enableSubmodule] = [true] 时必选 + */ + var enableSubmoduleRecursive: Boolean? = null, + + /** + * AutoCrlf配置值, 默认: false, single, options: false[false] | true[true] | input[input] + */ + var autoCrlf: String? = null, + + /** + * 是否开启Git Clean, 必选, 默认: true, 当 [strategy] = [REVERT_UPDATE] 时必选 + */ + var enableGitClean: Boolean? = null, + + /** + * 清理没有版本跟踪的ignored文件, 必选, 默认: true, 当 [strategy] = [REVERT_UPDATE] 和 [enableGitClean] = [true] 时必选 + */ + val enableGitCleanIgnore: Boolean? = null, + + /** + * 清理没有版本跟踪的嵌套仓库, 必选, 默认: false, 当 [strategy] = [REVERT_UPDATE] 和 [enableGitClean] = [true] 时必选 + */ + val enableGitCleanNested: Boolean? = null, + + /** + * 拉取代码库以下路径 + */ + var includePath: String? = null, + + /** + * 排除代码库以下路径 + */ + var excludePath: String? = null, + + // 非前端传递的参数 + @JsonProperty("pipeline.start.type") + val pipelineStartType: String? = null, + val hookEventType: String? = null, + val hookSourceBranch: String? = null, + val hookTargetBranch: String? = null, + val hookSourceUrl: String? = null, + val hookTargetUrl: String? = null, + + @JsonProperty("git_mr_number") + val gitMrNumber: String? = null, + +// 重试时检出的commitId + var retryStartPoint: String? = null, + + /** + * 是否持久化凭证, 默认: true + */ + var persistCredentials: Boolean? = null, + + /** + * 是否开启调试, 必选, 默认: false + */ + var enableTrace: Boolean? = null, + + /** + * 是否开启部分克隆,部分克隆只有git版本大于2.22.0才可以使用 + */ + var enablePartialClone: Boolean? = null, + + /** + * 归档的缓存路径 + */ + val cachePath: String? = null +) { + enum class AuthType { + TICKET, + ACCESS_TOKEN, + USERNAME_PASSWORD, + START_USER_TOKEN, + + // 工蜂专有授权类型 + PERSONAL_ACCESS_TOKEN, + EMPTY, + + // 指定授权用户 + AUTH_USER_TOKEN + } + + enum class CheckoutRepositoryType { + ID, + NAME, + URL, + SELF + } + + @JsonIgnore + fun getRepositoryConfig(): RepositoryConfig { + return RepositoryConfig( + repositoryHashId = repositoryHashId, + repositoryName = repositoryName, + repositoryType = kotlin.runCatching { RepositoryType.valueOf(repositoryType?.name ?: "ID") } + .getOrDefault(RepositoryType.ID) + ) + } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/ModelTransferInput.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/ModelTransferInput.kt new file mode 100644 index 00000000000..7c0223809e3 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/ModelTransferInput.kt @@ -0,0 +1,16 @@ +package com.tencent.devops.process.yaml.transfer.pojo + +import com.tencent.devops.common.api.enums.ScmType +import com.tencent.devops.common.pipeline.Model +import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting +import com.tencent.devops.process.yaml.transfer.aspect.PipelineTransferAspectWrapper +import com.tencent.devops.process.yaml.pojo.YamlVersion + +data class ModelTransferInput( + val userId: String, + var model: Model, + val setting: PipelineSetting, + val version: YamlVersion, + val aspectWrapper: PipelineTransferAspectWrapper, + val defaultScmType: ScmType = ScmType.CODE_GIT +) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/WebHookTriggerElementChanger.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/WebHookTriggerElementChanger.kt new file mode 100644 index 00000000000..aaeef46ac6d --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/WebHookTriggerElementChanger.kt @@ -0,0 +1,247 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.transfer.pojo + +import com.tencent.devops.common.api.enums.TriggerRepositoryType +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeGitWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeGithubWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeGitlabWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeP4WebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeSVNWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeTGitWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.enums.CodeEventType +import com.tencent.devops.common.pipeline.pojo.element.trigger.enums.PathFilterType +import io.swagger.v3.oas.annotations.media.Schema + +data class WebHookTriggerElementChanger( + @get:Schema(title = "任务名称", required = true) + val name: String = "Git变更触发", + @get:Schema(title = "仓库ID", required = true) + val repositoryHashId: String? = null, + @get:Schema(title = "分支名称", required = false) + val branchName: String? = null, + @get:Schema(title = "用于排除的分支名", required = false) + val excludeBranchName: String? = null, + @get:Schema(title = "路径过滤类型", required = true) + val pathFilterType: PathFilterType? = PathFilterType.NamePrefixFilter, + @get:Schema(title = "用于包含的路径", required = false) + val includePaths: String? = null, + @get:Schema(title = "用于排除的路径", required = false) + val excludePaths: String? = null, + @get:Schema(title = "用户白名单", required = false) + val includeUsers: List? = null, + @get:Schema(title = "用于排除的user id", required = false) + val excludeUsers: List? = null, + @get:Schema(title = "事件类型", required = false) + val eventType: CodeEventType?, + @get:Schema(title = "是否为block", required = false) + val block: Boolean? = null, + @get:Schema(title = "新版的git原子的类型") + val repositoryType: TriggerRepositoryType? = null, + @get:Schema(title = "新版的git代码库名") + val repositoryName: String? = null, + @get:Schema(title = "tag名称", required = false) + val tagName: String? = null, + @get:Schema(title = "用于排除的tag名称", required = false) + val excludeTagName: String? = null, + @get:Schema(title = "tag从哪条分支创建", required = false) + val fromBranches: String? = null, + @get:Schema(title = "用于排除的源分支名称", required = false) + val excludeSourceBranchName: String? = null, + @get:Schema(title = "用于包含的源分支名称", required = false) + val includeSourceBranchName: String? = null, + @get:Schema(title = "webhook队列", required = false) + val webhookQueue: Boolean? = false, + @get:Schema(title = "code review 状态", required = false) + val includeCrState: List? = null, + @get:Schema(title = "code review 类型", required = false) + val includeCrTypes: List? = null, + @get:Schema(title = "code note comment", required = false) + val includeNoteComment: String? = null, + @get:Schema(title = "code note 类型", required = false) + val includeNoteTypes: List? = null, + @get:Schema(title = "是否启用回写") + val enableCheck: Boolean? = true, + @get:Schema(title = "issue事件action") + val includeIssueAction: List? = null, + @get:Schema(title = "mr事件action") + val includeMrAction: List? = null, + @get:Schema(title = "push事件action") + val includePushAction: List? = null, + @get:Schema(title = "是否启用第三方过滤") + val enableThirdFilter: Boolean? = false, + @get:Schema(title = "第三方应用地址") + val thirdUrl: String? = null, + @get:Schema(title = "第三方应用鉴权token") + val thirdSecretToken: String? = null, + @get:Schema(title = "是否启用插件") + val enable: Boolean +) { + constructor(data: CodeGitWebHookTriggerElement) : this( + name = data.name, + repositoryHashId = data.repositoryHashId, + branchName = data.branchName, + excludeBranchName = data.excludeBranchName, + pathFilterType = data.pathFilterType, + includePaths = data.includePaths, + excludePaths = data.excludePaths, + includeUsers = data.includeUsers, + excludeUsers = data.excludeUsers, + eventType = data.eventType, + block = data.block, + repositoryType = data.repositoryType, + repositoryName = data.repositoryName, + tagName = data.tagName, + excludeTagName = data.excludeTagName, + fromBranches = data.fromBranches, + excludeSourceBranchName = data.excludeSourceBranchName, + includeSourceBranchName = data.includeSourceBranchName, + webhookQueue = data.webhookQueue, + includeCrState = data.includeCrState, + includeCrTypes = data.includeCrTypes, + includeNoteComment = data.includeNoteComment, + includeNoteTypes = data.includeNoteTypes, + enableCheck = data.enableCheck, + includeIssueAction = data.includeIssueAction, + includeMrAction = data.includeMrAction, + includePushAction = data.includePushAction, + enableThirdFilter = data.enableThirdFilter, + thirdUrl = data.thirdUrl, + thirdSecretToken = data.thirdSecretToken, + enable = data.isElementEnable() + ) + + constructor(data: CodeTGitWebHookTriggerElement) : this( + name = data.name, + repositoryHashId = data.data.input.repositoryHashId, + branchName = data.data.input.branchName, + excludeBranchName = data.data.input.excludeBranchName, + pathFilterType = data.data.input.pathFilterType, + includePaths = data.data.input.includePaths, + excludePaths = data.data.input.excludePaths, + includeUsers = data.data.input.includeUsers, + excludeUsers = data.data.input.excludeUsers, + eventType = data.data.input.eventType, + block = data.data.input.block, + repositoryType = data.data.input.repositoryType, + repositoryName = data.data.input.repositoryName, + tagName = data.data.input.tagName, + excludeTagName = data.data.input.excludeTagName, + fromBranches = data.data.input.fromBranches, + excludeSourceBranchName = data.data.input.excludeSourceBranchName, + includeSourceBranchName = data.data.input.includeSourceBranchName, + webhookQueue = data.data.input.webhookQueue, + includeCrState = data.data.input.includeCrState, + includeCrTypes = data.data.input.includeCrTypes, + includeNoteComment = data.data.input.includeNoteComment, + includeNoteTypes = data.data.input.includeNoteTypes, + enableCheck = data.data.input.enableCheck, + includeIssueAction = data.data.input.includeIssueAction, + includeMrAction = data.data.input.includeMrAction, + includePushAction = data.data.input.includePushAction, + enableThirdFilter = data.data.input.enableThirdFilter, + enable = data.isElementEnable() + ) + + constructor(data: CodeGithubWebHookTriggerElement) : this( + name = data.name, + repositoryHashId = data.repositoryHashId, + branchName = data.branchName, + excludeBranchName = data.excludeBranchName, + pathFilterType = data.pathFilterType, + includePaths = data.includePaths, + excludePaths = data.excludePaths, + includeUsers = data.includeUsers, + excludeUsers = data.excludeUsers?.split(","), + eventType = data.eventType, + repositoryType = data.repositoryType, + repositoryName = data.repositoryName, + tagName = data.tagName, + excludeTagName = data.excludeTagName, + fromBranches = data.fromBranches, + excludeSourceBranchName = data.excludeSourceBranchName, + includeSourceBranchName = data.includeSourceBranchName, + webhookQueue = data.webhookQueue, + includeCrState = data.includeCrState, + includeCrTypes = data.includeCrTypes, + includeNoteComment = data.includeNoteComment, + includeNoteTypes = data.includeNoteTypes, + enableCheck = data.enableCheck, + includeIssueAction = data.includeIssueAction, + includeMrAction = data.includeMrAction, + includePushAction = data.includePushAction, + enableThirdFilter = data.enableThirdFilter, + enable = data.isElementEnable() + ) + + constructor(data: CodeSVNWebHookTriggerElement) : this( + name = data.name, + repositoryHashId = data.repositoryHashId, + pathFilterType = data.pathFilterType, + includePaths = data.relativePath, + excludePaths = data.excludePaths, + includeUsers = data.includeUsers, + excludeUsers = data.excludeUsers, + eventType = CodeEventType.POST_COMMIT, + repositoryType = data.repositoryType, + repositoryName = data.repositoryName, + enable = data.isElementEnable() + ) + + constructor(data: CodeP4WebHookTriggerElement) : this( + name = data.name, + repositoryHashId = data.data.input.repositoryHashId, + includePaths = data.data.input.includePaths, + excludePaths = data.data.input.excludePaths, + eventType = data.data.input.eventType, + repositoryType = data.data.input.repositoryType, + repositoryName = data.data.input.repositoryName, + enable = data.isElementEnable() + ) + + constructor(data: CodeGitlabWebHookTriggerElement) : this( + name = data.name, + repositoryHashId = data.repositoryHashId, + branchName = data.branchName, + excludeBranchName = data.excludeBranchName, + pathFilterType = data.pathFilterType, + includePaths = data.includePaths, + excludePaths = data.excludePaths, + includeUsers = data.includeUsers, + excludeUsers = data.excludeUsers, + eventType = data.eventType, + block = data.block, + repositoryType = data.repositoryType, + repositoryName = data.repositoryName, + tagName = data.tagName, + excludeTagName = data.excludeTagName, + excludeSourceBranchName = data.excludeSourceBranchName, + includeSourceBranchName = data.includeSourceBranchName, + enable = data.isElementEnable() + ) +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/YamlTransferInput.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/YamlTransferInput.kt new file mode 100644 index 00000000000..12cd8f59adf --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/YamlTransferInput.kt @@ -0,0 +1,19 @@ +package com.tencent.devops.process.yaml.transfer.pojo + +import com.tencent.devops.common.api.enums.ScmType +import com.tencent.devops.common.api.pojo.PipelineAsCodeSettings +import com.tencent.devops.process.engine.pojo.PipelineInfo +import com.tencent.devops.process.pojo.BuildTemplateAcrossInfo +import com.tencent.devops.process.yaml.transfer.aspect.PipelineTransferAspectWrapper +import com.tencent.devops.process.yaml.v3.models.IPreTemplateScriptBuildYamlParser + +data class YamlTransferInput( + val userId: String, + val projectCode: String, + val pipelineInfo: PipelineInfo?, + val yaml: IPreTemplateScriptBuildYamlParser, + val aspectWrapper: PipelineTransferAspectWrapper, + val defaultScmType: ScmType = ScmType.CODE_GIT, + val asCodeSettings: PipelineAsCodeSettings? = null, + val jobTemplateAcrossInfo: Map? = null +) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/schema/CodeSchemaCheck.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/schema/CodeSchemaCheck.kt new file mode 100644 index 00000000000..1c7213f50a0 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/schema/CodeSchemaCheck.kt @@ -0,0 +1,184 @@ +package com.tencent.devops.process.yaml.transfer.schema + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator +import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import com.github.benmanes.caffeine.cache.Caffeine +import com.networknt.schema.JsonSchema +import com.networknt.schema.JsonSchemaFactory +import com.networknt.schema.SpecVersion +import com.tencent.devops.common.api.constant.CommonMessageCode.YAML_NOT_VALID +import com.tencent.devops.common.api.util.ReflectUtil +import com.tencent.devops.common.api.util.YamlUtil +import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.process.yaml.pojo.YamlVersion +import com.tencent.devops.process.yaml.transfer.PipelineTransferException +import com.tencent.devops.process.yaml.transfer.TransferMapper +import com.tencent.devops.process.yaml.v2.enums.TemplateType +import java.io.FileNotFoundException +import java.nio.charset.Charset +import java.util.concurrent.TimeUnit +import java.util.function.Supplier +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.core.io.ClassPathResource +import org.springframework.stereotype.Component + +@Component +class CodeSchemaCheck @Autowired constructor( + private val redisOperation: RedisOperation +) { + + companion object { + private const val REDIS_STREAM_YAML_SCHEMA = "pac:yaml.schema:json" + private const val CI_SCHEMA = "ci" + private const val TEMPLATE_EXTEND_SCHEMA = "template-extends" + private const val TEMPLATE_STAGE_SCHEMA = "template-stages" + private const val TEMPLATE_JOB_SCHEMA = "template-jobs" + private const val TEMPLATE_STEP_SCHEMA = "template-steps" + private const val TEMPLATE_VARIABLE_SCHEMA = "template-variables" + private const val TEMPLATE_GATE_SCHEMA = "template-gates" + } + + private val schemaRedisFiles = Caffeine.newBuilder() + .maximumSize(20) + .expireAfterWrite(1, TimeUnit.MINUTES) + .build { key -> + kotlin.runCatching { + redisOperation.get(key) ?: "" + }.onFailure { logger.warn("get $key in schemaRedisFiles error.", it) }.getOrNull() ?: "" + } + + private val logger = + LoggerFactory.getLogger(CodeSchemaCheck::class.java) + + private val schemaFactory = JsonSchemaFactory.builder(JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7)) + .objectMapper(YamlUtil.getObjectMapper()) + .build() + + private val objectMapperFactory = ThreadLocal.withInitial( + Supplier { + ObjectMapper( + YAMLFactory().disable(YAMLGenerator.Feature.SPLIT_LINES) + ).registerKotlinModule() + } + ) + + private val schemaMap = mutableMapOf() + + fun check(originYaml: String) { + check(originYaml, null, true) + } + + // 给来自前端的接口用，直接扔出去就好 + fun check(originYaml: String, templateType: TemplateType?, isCiFile: Boolean) { + try { + checkYamlSchema(originYaml, templateType, isCiFile) + } catch (ignore: FileNotFoundException) { + logger.warn("schema file not find. ${ignore.message}") + } + } + + private fun checkYamlSchema(originYaml: String, templateType: TemplateType? = null, isCiFile: Boolean) { + val loadYaml = try { + toYaml(TransferMapper.getYamlFactory().load(originYaml) as Any) + } catch (ignored: Exception) { + logger.warn("YAML_SCHEMA_CHECK|${ignored.message}|originYaml=$originYaml", ignored) + throw PipelineTransferException( + YAML_NOT_VALID, + arrayOf("There may be a problem with your yaml syntax ${ignored.message}") + ) + } + // 解析锚点 + val yamlJson = TransferMapper.getObjectMapper().readTree(loadYaml) + when (yamlJson.version()) { +// YamlVersion.V2_0.tag -> { +// check(yamlJson, YamlVersion.V2_0, templateType, isCiFile) +// } + + YamlVersion.V3_0.tag -> { + check(yamlJson, YamlVersion.V3_0, templateType, isCiFile) + } + + else -> throw PipelineTransferException( + YAML_NOT_VALID, + arrayOf("yaml version(${yamlJson.version()}) not valid, only support v3.0") + ) + } + } + + private fun check( + yamlJson: JsonNode, + version: YamlVersion, + templateType: TemplateType? = null, + isCiFile: Boolean + ) { + if (isCiFile) { + getSchema(CI_SCHEMA, version).check(yamlJson) + } + if (templateType == null) { + return + } + when (templateType) { + TemplateType.EXTEND -> { + getSchema(TEMPLATE_EXTEND_SCHEMA, version).check(yamlJson) + } + + TemplateType.VARIABLE -> { + getSchema(TEMPLATE_VARIABLE_SCHEMA, version).check(yamlJson) + } + + TemplateType.STAGE -> getSchema(TEMPLATE_STAGE_SCHEMA, version).check(yamlJson) + TemplateType.GATE -> getSchema(TEMPLATE_GATE_SCHEMA, version).check(yamlJson) + TemplateType.JOB -> getSchema(TEMPLATE_JOB_SCHEMA, version).check(yamlJson) + TemplateType.STEP -> getSchema(TEMPLATE_STEP_SCHEMA, version).check(yamlJson) + else -> { + return + } + } + } + + private fun toYaml(bean: Any): String { + if (ReflectUtil.isNativeType(bean) || bean is String) { + return bean.toString() + } + return objectMapperFactory.get().writeValueAsString(bean)!! + } + + private fun getSchema(file: String, version: YamlVersion): JsonSchema { + val str = schemaRedisFiles.get( + "$REDIS_STREAM_YAML_SCHEMA:${version.name}:$file" + )?.ifBlank { null } ?: return getSchemaFromGit(file, version) + return schemaFactory.getSchema(str) + } + + private fun getSchemaFromGit(file: String, version: YamlVersion): JsonSchema { + val path = "schema/${version.name}/$file.json" + if (schemaMap[path] != null) { + return schemaMap[path]!! + } + val schema = schemaFactory.getSchema( + ClassPathResource(path).inputStream.readBytes() + .toString(Charset.defaultCharset()) + ) + schemaMap[path] = schema + return schema + } + + private fun JsonSchema.check(yaml: JsonNode) { + validate(yaml).let { + if (!it.isNullOrEmpty()) { + throw PipelineTransferException( + YAML_NOT_VALID, + arrayOf(it.toString()) + ) + } + } + } + + private fun JsonNode.version(): String? { + return get("version")?.textValue() + } +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/utils/PathMatchUtils.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/utils/PathMatchUtils.kt index 5b5e15aee17..c83b171b847 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/utils/PathMatchUtils.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/utils/PathMatchUtils.kt @@ -27,7 +27,7 @@ package com.tencent.devops.process.yaml.utils -import com.tencent.devops.process.yaml.modelCreate.inner.ModelCreateEvent +import com.tencent.devops.process.yaml.creator.inner.ModelCreateEvent import org.slf4j.LoggerFactory import java.util.regex.Pattern diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/Gate.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/Gate.kt similarity index 97% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/Gate.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/Gate.kt index 3afae9fb419..829ea33f211 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/Gate.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/Gate.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.v2.stageCheck +package com.tencent.devops.process.yaml.v2.check import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/GateTemplate.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/GateTemplate.kt similarity index 96% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/GateTemplate.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/GateTemplate.kt index b55f2509530..34094e3b69f 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/GateTemplate.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/GateTemplate.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.v2.stageCheck +package com.tencent.devops.process.yaml.v2.check import com.tencent.devops.process.yaml.v2.parameter.Parameters diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/PreStageCheck.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/PreStageCheck.kt similarity index 97% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/PreStageCheck.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/PreStageCheck.kt index db840cc593d..f134eee6e04 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/PreStageCheck.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/PreStageCheck.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.v2.stageCheck +package com.tencent.devops.process.yaml.v2.check import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/PreStageReviews.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/PreStageReviews.kt similarity index 97% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/PreStageReviews.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/PreStageReviews.kt index eea10240ee5..6834c121902 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/PreStageReviews.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/PreStageReviews.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.v2.stageCheck +package com.tencent.devops.process.yaml.v2.check import com.fasterxml.jackson.annotation.JsonIgnoreProperties diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/PreTemplateStageCheck.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/PreTemplateStageCheck.kt similarity index 97% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/PreTemplateStageCheck.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/PreTemplateStageCheck.kt index f149be549db..ca30d59e637 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/PreTemplateStageCheck.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/PreTemplateStageCheck.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.v2.stageCheck +package com.tencent.devops.process.yaml.v2.check import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/ReviewVariable.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/ReviewVariable.kt similarity index 96% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/ReviewVariable.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/ReviewVariable.kt index 868b2fb0498..f964acecd64 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/ReviewVariable.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/ReviewVariable.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.v2.stageCheck +package com.tencent.devops.process.yaml.v2.check import com.fasterxml.jackson.annotation.JsonIgnoreProperties diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/StageCheck.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/StageCheck.kt similarity index 96% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/StageCheck.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/StageCheck.kt index 85b1b51fca8..16e77073ebc 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/StageCheck.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/StageCheck.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.v2.stageCheck +package com.tencent.devops.process.yaml.v2.check import com.fasterxml.jackson.annotation.JsonProperty diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/StageReviews.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/StageReviews.kt similarity index 97% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/StageReviews.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/StageReviews.kt index 04498ed1b28..6bfde9e7f9c 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/stageCheck/StageReviews.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/StageReviews.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.v2.stageCheck +package com.tencent.devops.process.yaml.v2.check import com.fasterxml.jackson.annotation.JsonIgnoreProperties diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/enums/StreamTriggerAction.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/enums/StreamTriggerAction.kt index b607729c4fa..86a59eb541d 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/enums/StreamTriggerAction.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/enums/StreamTriggerAction.kt @@ -40,7 +40,7 @@ enum class StreamObjectKind(val value: String) { OPENAPI("openApi"), ISSUE("issue"), REVIEW("review"), - NOTE("note"); + NOTE("note"), } fun StreamObjectKind.needInput() = this == StreamObjectKind.MANUAL || this == StreamObjectKind.OPENAPI diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/PreScriptBuildYaml.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/PreScriptBuildYaml.kt index b99b6775dcf..b8fa8b422c5 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/PreScriptBuildYaml.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/PreScriptBuildYaml.kt @@ -29,6 +29,7 @@ package com.tencent.devops.process.yaml.v2.models import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonProperty import com.tencent.devops.process.yaml.v2.models.job.PreJob import com.tencent.devops.process.yaml.v2.models.on.PreTriggerOn import com.tencent.devops.process.yaml.v2.models.stage.PreStage @@ -65,6 +66,7 @@ data class PreScriptBuildYaml( override var version: String?, override var name: String?, override var label: List? = null, + @JsonProperty("on") override var triggerOn: PreTriggerOn?, override var variables: Map? = null, override var stages: List? = null, diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/PreTemplateScriptBuildYaml.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/PreTemplateScriptBuildYaml.kt index a866f5a1a32..382daa37b07 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/PreTemplateScriptBuildYaml.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/PreTemplateScriptBuildYaml.kt @@ -29,6 +29,7 @@ package com.tencent.devops.process.yaml.v2.models import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonProperty import com.tencent.devops.process.yaml.v2.models.on.PreTriggerOn /** @@ -42,6 +43,7 @@ data class PreTemplateScriptBuildYaml( val version: String?, val name: String?, val label: List? = null, + @JsonProperty("on") val triggerOn: PreTriggerOn?, val variables: Map?, val stages: List>?, diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/ScriptBuildYaml.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/ScriptBuildYaml.kt index 7ad011153da..76fe55f8747 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/ScriptBuildYaml.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/ScriptBuildYaml.kt @@ -29,6 +29,7 @@ package com.tencent.devops.process.yaml.v2.models import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonProperty import com.tencent.devops.process.yaml.v2.models.job.Job import com.tencent.devops.process.yaml.v2.models.on.TriggerOn import com.tencent.devops.process.yaml.v2.models.stage.Stage @@ -44,6 +45,7 @@ data class ScriptBuildYaml( val version: String?, val name: String?, val label: List?, + @JsonProperty("on") val triggerOn: TriggerOn?, val variables: Map?, val stages: List, diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/export/ExportPreScriptBuildYaml.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/export/ExportPreScriptBuildYaml.kt index 0a544af9675..32edba1db6f 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/export/ExportPreScriptBuildYaml.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/export/ExportPreScriptBuildYaml.kt @@ -29,6 +29,7 @@ package com.tencent.devops.process.yaml.v2.models.export import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonProperty import com.tencent.devops.process.yaml.v2.models.Extends import com.tencent.devops.process.yaml.v2.models.GitNotices import com.tencent.devops.process.yaml.v2.models.Resources @@ -48,6 +49,7 @@ data class ExportPreScriptBuildYaml( var version: String?, var name: String?, var label: List? = null, + @JsonProperty("on") var triggerOn: PreTriggerOn?, var variables: Map? = null, var stages: List? = null, diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/stage/PreStage.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/stage/PreStage.kt index 3602e767de4..3f382ba8912 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/stage/PreStage.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/stage/PreStage.kt @@ -29,7 +29,7 @@ package com.tencent.devops.process.yaml.v2.models.stage import com.fasterxml.jackson.annotation.JsonProperty import com.tencent.devops.process.yaml.v2.models.job.PreJob -import com.tencent.devops.process.yaml.v2.stageCheck.PreStageCheck +import com.tencent.devops.process.yaml.v2.check.PreStageCheck import io.swagger.v3.oas.annotations.media.Schema /** diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/stage/Stage.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/stage/Stage.kt index 94ae8566209..409f5dc5472 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/stage/Stage.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/stage/Stage.kt @@ -30,7 +30,7 @@ package com.tencent.devops.process.yaml.v2.models.stage import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty import com.tencent.devops.process.yaml.v2.models.job.Job -import com.tencent.devops.process.yaml.v2.stageCheck.StageCheck +import com.tencent.devops.process.yaml.v2.check.StageCheck import io.swagger.v3.oas.annotations.media.Schema /** diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/ParametersExpressionParse.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/ParametersExpressionParse.kt index fb5f7da908c..bba8eade3cd 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/ParametersExpressionParse.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/ParametersExpressionParse.kt @@ -57,7 +57,9 @@ object ParametersExpressionParse { templateParameters.forEachIndexed { index, param -> if (param.name.contains(".")) { throw error( - Constants.PARAMETER_FORMAT_ERROR.format(path, "parameter name ${param.name} not allow contains '.'") + Constants.PARAMETER_FORMAT_ERROR.format( + path, "parameter name ${param.name} not allow contains '.'" + ) ) } @@ -146,7 +148,9 @@ object ParametersExpressionParse { if (!jsonTree.isArray) { throw error( Constants.PARAMETER_FORMAT_ERROR.format( - path, "array parameter $parameterName value [$value] json type [${jsonTree.nodeType}] not array." + path, + "array parameter $parameterName value [$value] " + + "json type [${jsonTree.nodeType}] not array." ) ) } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/YamlObjects.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/YamlObjects.kt index 84035513ccb..01ba38a0d3b 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/YamlObjects.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/YamlObjects.kt @@ -55,6 +55,7 @@ import com.tencent.devops.process.yaml.v2.parameter.Parameters import com.tencent.devops.process.yaml.v2.parsers.template.models.TemplateDeepTreeNode import com.tencent.devops.process.yaml.v2.utils.StreamEnvUtils +@Suppress("ComplexMethod") object YamlObjects { fun getVariable(fromPath: String, key: String, variable: Map): Variable { diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/YamlTemplate.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/YamlTemplate.kt index ee392b4b9e8..155c441b866 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/YamlTemplate.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/YamlTemplate.kt @@ -50,10 +50,10 @@ import com.tencent.devops.process.yaml.v2.parameter.PreParametersTemplate import com.tencent.devops.process.yaml.v2.parsers.template.models.GetTemplateParam import com.tencent.devops.process.yaml.v2.parsers.template.models.NoReplaceTemplate import com.tencent.devops.process.yaml.v2.parsers.template.models.TemplateDeepTreeNode -import com.tencent.devops.process.yaml.v2.stageCheck.Gate -import com.tencent.devops.process.yaml.v2.stageCheck.GateTemplate -import com.tencent.devops.process.yaml.v2.stageCheck.PreStageCheck -import com.tencent.devops.process.yaml.v2.stageCheck.PreTemplateStageCheck +import com.tencent.devops.process.yaml.v2.check.Gate +import com.tencent.devops.process.yaml.v2.check.GateTemplate +import com.tencent.devops.process.yaml.v2.check.PreStageCheck +import com.tencent.devops.process.yaml.v2.check.PreTemplateStageCheck @Suppress("ALL") class YamlTemplate( diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/models/NoReplaceTemplate.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/models/NoReplaceTemplate.kt index ba5b52b478d..6720d310245 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/models/NoReplaceTemplate.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/models/NoReplaceTemplate.kt @@ -29,6 +29,7 @@ package com.tencent.devops.process.yaml.v2.parsers.template.models import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonProperty import com.tencent.devops.process.yaml.v2.models.Concurrency import com.tencent.devops.process.yaml.v2.models.Extends import com.tencent.devops.process.yaml.v2.models.Resources @@ -41,6 +42,7 @@ data class NoReplaceTemplate( var version: String?, var name: String?, var label: List? = null, + @JsonProperty("on") var triggerOn: PreTriggerOn?, var extends: Extends?, var resources: Resources?, diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/ScriptYmlUtils.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/ScriptYmlUtils.kt index ed74c8e3599..cc91c3ccd8e 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/ScriptYmlUtils.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/ScriptYmlUtils.kt @@ -50,6 +50,7 @@ import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.UUIDUtil import com.tencent.devops.common.api.util.YamlUtil import com.tencent.devops.common.web.utils.I18nUtil +import com.tencent.devops.process.yaml.transfer.TransferMapper import com.tencent.devops.process.yaml.v2.enums.StreamMrEventAction import com.tencent.devops.process.yaml.v2.enums.TemplateType import com.tencent.devops.process.yaml.v2.exception.YamlFormatException @@ -84,10 +85,10 @@ import com.tencent.devops.process.yaml.v2.models.stage.StageLabel import com.tencent.devops.process.yaml.v2.models.step.PreStep import com.tencent.devops.process.yaml.v2.models.step.Step import com.tencent.devops.process.yaml.v2.parameter.ParametersType -import com.tencent.devops.process.yaml.v2.stageCheck.Flow -import com.tencent.devops.process.yaml.v2.stageCheck.PreStageCheck -import com.tencent.devops.process.yaml.v2.stageCheck.StageCheck -import com.tencent.devops.process.yaml.v2.stageCheck.StageReviews +import com.tencent.devops.process.yaml.v2.check.Flow +import com.tencent.devops.process.yaml.v2.check.PreStageCheck +import com.tencent.devops.process.yaml.v2.check.StageCheck +import com.tencent.devops.process.yaml.v2.check.StageReviews import java.io.BufferedReader import java.io.StringReader import java.util.Random @@ -95,7 +96,7 @@ import java.util.regex.Pattern import org.apache.commons.text.StringEscapeUtils import org.slf4j.LoggerFactory -@Suppress("MaximumLineLength", "ComplexCondition") +@Suppress("MaximumLineLength", "ComplexCondition", "ComplexMethod") object ScriptYmlUtils { private val logger = LoggerFactory.getLogger(ScriptYmlUtils::class.java) @@ -116,9 +117,9 @@ object ScriptYmlUtils { @Throws(JsonProcessingException::class) fun formatYaml(yamlStr: String): String { // replace custom tag - val yamlNormal = formatYamlCustom(yamlStr) +// val yamlNormal = formatYamlCustom(yamlStr) // replace anchor tag - return YamlUtil.loadYamlRetryOnAccident(yamlNormal) + return TransferMapper.formatYaml(yamlStr) } fun parseVersion(yamlStr: String?): YmlVersion? { @@ -223,8 +224,8 @@ object ScriptYmlUtils { val startString = line.trim().replace("\\s".toRegex(), "") if (startString.startsWith("if:") || startString.startsWith("-if:")) { val ifPrefix = line.substring(0 until line.indexOfFirst { it == ':' } + 1) - val condition = line.substring(line.indexOfFirst { it == '"' } + 1 until line.length).trimEnd() - .removeSuffix("\"") + val condition = line.removePrefix(ifPrefix).trim() + .removeSurrounding("\"") // 去掉花括号 val baldExpress = condition.replace("\${{", "").replace("}}", "").trim() diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/utils/StreamDispatchUtils.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/StreamDispatchUtils.kt similarity index 99% rename from src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/utils/StreamDispatchUtils.kt rename to src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/StreamDispatchUtils.kt index 291784ebfa3..de274ce1ae0 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/utils/StreamDispatchUtils.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/StreamDispatchUtils.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.yaml.utils +package com.tencent.devops.process.yaml.v2.utils import com.fasterxml.jackson.core.JsonProcessingException import com.tencent.devops.common.api.constant.CommonMessageCode.BUILD_RESOURCE_NOT_EXIST diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/Gate.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/Gate.kt new file mode 100644 index 00000000000..10a2d99d03e --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/Gate.kt @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.v3.check + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty +import com.tencent.devops.process.yaml.v3.models.GateNotices +import com.tencent.devops.process.yaml.v3.models.gate.ContinueOnFail + +@JsonIgnoreProperties(ignoreUnknown = true) +data class Gate( + val name: String, + val rule: List, + @JsonProperty("notify-on-fail") + val notifyOnFail: List, + @JsonProperty("continue-on-fail") + val continueOnFail: ContinueOnFail? +) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/GateTemplate.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/GateTemplate.kt new file mode 100644 index 00000000000..575723ec1d3 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/GateTemplate.kt @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.v3.check + +import com.tencent.devops.process.yaml.v3.parameter.Parameters + +data class GateTemplate( + val gates: List, + val parameters: List? +) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/PreStageCheck.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/PreStageCheck.kt new file mode 100644 index 00000000000..a8403e539ac --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/PreStageCheck.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.v3.check + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +@JsonIgnoreProperties(ignoreUnknown = true) +data class PreStageCheck( + val reviews: PreStageReviews?, + val gates: List?, + @JsonProperty("timeout-hours") + val timeoutHours: Int? +) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/PreStageReviews.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/PreStageReviews.kt new file mode 100644 index 00000000000..76749baebc5 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/PreStageReviews.kt @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.v3.check + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +@JsonIgnoreProperties(ignoreUnknown = true) +data class PreStageReviews( + val flows: List?, + val variables: Map?, + val description: String?, + @JsonProperty("content-format") + val contentFormat: String?, + @JsonProperty("notify-type") + val notifyType: List?, + @JsonProperty("chat-id") + val notifyGroups: List? +) + +@JsonIgnoreProperties(ignoreUnknown = true) +data class PreFlow( + val name: String, + val reviewers: Any +) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/PreTemplateStageCheck.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/PreTemplateStageCheck.kt new file mode 100644 index 00000000000..ae47738ec54 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/check/PreTemplateStageCheck.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.v3.check + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty +import com.tencent.devops.process.yaml.v3.models.Template + +@JsonIgnoreProperties(ignoreUnknown = true) +data class PreTemplateStageCheck( + val reviews: PreStageReviews?, + val gates: List