diff --git a/src/agent/agent/go.mod b/src/agent/agent/go.mod index 55d958217530..937d4212147f 100644 --- a/src/agent/agent/go.mod +++ b/src/agent/agent/go.mod @@ -129,6 +129,7 @@ require ( ) require ( + // 非稳定库，目前只在windows升级中简单使用且主要做对go-ole的封装简化，大规模使用前需要评估 github.com/capnspacehook/taskmaster v0.0.0-20210519235353-1629df7c85e9 github.com/docker/cli v23.0.1+incompatible github.com/docker/go-connections v0.4.0 diff --git a/src/agent/agent/src/cmd/daemon/main_win.go b/src/agent/agent/src/cmd/daemon/main_win.go index 5e0b710a6be4..1ddf8de3e9d3 100644 --- a/src/agent/agent/src/cmd/daemon/main_win.go +++ b/src/agent/agent/src/cmd/daemon/main_win.go @@ -33,7 +33,6 @@ package main import ( "errors" "fmt" - "io" "os" "os/exec" "path/filepath" @@ -129,14 +128,6 @@ func watch() { cmd := exec.Command(agentPath) cmd.Dir = workDir - // 获取 agent 的错误输出，这样有助于打印出崩溃的堆栈方便排查问题 - stdErr, errstd := cmd.StderrPipe() - if errstd != nil { - logs.WithError(errstd).Error("get agent stderr pipe error") - } else { - defer stdErr.Close() - } - logs.Info("start devops agent") if !fileutil.Exists(agentPath) { logs.Errorf("agent file: %s not exists", agentPath) @@ -172,17 +163,6 @@ func watch() { } } logs.WithError(err).Error("agent process error") - - // 读取可能的报错 - if errstd != nil { - return - } - out, err := io.ReadAll(stdErr) - if err != nil { - logs.WithError(err).Error("read agent stderr out error") - } else { - logs.Error("agent process error out", string(out)) - } } logs.Info("agent process exited") diff --git a/src/agent/agent/src/cmd/upgrader/main.go b/src/agent/agent/src/cmd/upgrader/main.go index 6e32c99d89bc..d0e19624922b 100644 --- a/src/agent/agent/src/cmd/upgrader/main.go +++ b/src/agent/agent/src/cmd/upgrader/main.go @@ -59,6 +59,8 @@ func main() { } }() + logs.Infof("version: %s", config.AgentVersion) + if ok := systemutil.CheckProcess(upgraderProcess); !ok { logs.Warn("get process lock failed, exit") return diff --git a/src/agent/agent/src/pkg/agent/agent.go b/src/agent/agent/src/pkg/agent/agent.go index b980f4d4aa79..d54baa8c4178 100644 --- a/src/agent/agent/src/pkg/agent/agent.go +++ b/src/agent/agent/src/pkg/agent/agent.go @@ -28,6 +28,8 @@ package agent import ( + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/systemutil" + "github.com/TencentBlueKing/bk-ci/agent/src/third_components" "time" "github.com/TencentBlueKing/bk-ci/agentcommon/logs" @@ -47,6 +49,10 @@ import ( func Run(isDebug bool) { config.Init(isDebug) + if err := third_components.Init(); err != nil { + logs.WithError(err).Error("init third_components error") + systemutil.ExitProcess(1) + } // 初始化国际化 i18n.InitAgentI18n() diff --git a/src/agent/agent/src/pkg/agent/ask.go b/src/agent/agent/src/pkg/agent/ask.go index b80a0fe4790b..9734aa0a86ef 100644 --- a/src/agent/agent/src/pkg/agent/ask.go +++ b/src/agent/agent/src/pkg/agent/ask.go @@ -1,6 +1,7 @@ package agent import ( + "github.com/TencentBlueKing/bk-ci/agent/src/third_components" "runtime" "github.com/TencentBlueKing/bk-ci/agentcommon/logs" @@ -27,13 +28,13 @@ func genHeartInfoAndUpgrade( }) } - if err := upgrade.SyncJdkVersion(); err != nil { + if err := third_components.Jdk.Jdk17.SyncJdkVersion(); err != nil { logs.Error("ask sync jdkVersion error", err) } if err := upgrade.SyncDockerInitFileMd5(); err != nil { logs.Error("ask sync docker file md5 error", err) } - jdkVersion := upgrade.JdkVersion.GetVersion() + jdkVersion := third_components.Jdk.Jdk17.GetVersion() dockerInitFile := api.DockerInitFileInfo{ FileMd5: upgrade.DockerFileMd5.Md5, NeedUpgrade: upgrade.DockerFileMd5.NeedUpgrade, @@ -42,7 +43,7 @@ func genHeartInfoAndUpgrade( var upg *api.UpgradeInfo = nil if upgradeEnable { upg = &api.UpgradeInfo{ - WorkerVersion: config.GAgentEnv.SlaveVersion, + WorkerVersion: third_components.Worker.GetVersion(), GoAgentVersion: config.AgentVersion, JdkVersion: jdkVersion, DockerInitFileInfo: dockerInitFile, @@ -51,7 +52,7 @@ func genHeartInfoAndUpgrade( return api.AgentHeartbeatInfo{ MasterVersion: config.AgentVersion, - SlaveVersion: config.GAgentEnv.SlaveVersion, + SlaveVersion: third_components.Worker.GetVersion(), HostName: config.GAgentEnv.HostName, AgentIp: config.GAgentEnv.GetAgentIp(), ParallelTaskCount: config.GAgentConfig.ParallelTaskCount, diff --git a/src/agent/agent/src/pkg/api/api.go b/src/agent/agent/src/pkg/api/api.go index 254d515aee5e..caac88cbc7c1 100644 --- a/src/agent/agent/src/pkg/api/api.go +++ b/src/agent/agent/src/pkg/api/api.go @@ -29,6 +29,7 @@ package api import ( "fmt" + "github.com/TencentBlueKing/bk-ci/agent/src/third_components" "reflect" "strconv" @@ -64,7 +65,7 @@ func AgentStartup() (*httputil.DevopsResult, error) { HostIp: config.GAgentEnv.GetAgentIp(), DetectOs: config.GAgentEnv.OsName, MasterVersion: config.AgentVersion, - SlaveVersion: config.GAgentEnv.SlaveVersion, + SlaveVersion: third_components.Worker.GetVersion(), } return httputil.NewHttpClient().Post(url).Body(startInfo, false). diff --git a/src/agent/agent/src/pkg/config/config.go b/src/agent/agent/src/pkg/config/config.go index ee6ba743c94e..f6b5622c2596 100644 --- a/src/agent/agent/src/pkg/config/config.go +++ b/src/agent/agent/src/pkg/config/config.go @@ -36,7 +36,6 @@ import ( "net/http" "os" "path/filepath" - "regexp" "strconv" "strings" "sync" @@ -69,7 +68,9 @@ const ( KeyBatchInstall = "devops.agent.batch.install" KeyLogsKeepHours = "devops.agent.logs.keep.hours" // KeyJdkDirPath 这个key不会预先出现在配置文件中，因为workdir未知，需要第一次动态获取 - KeyJdkDirPath = "devops.agent.jdk.dir.path" + KeyJdkDirPath = "devops.agent.jdk.dir.path" + // KeyJdk17DirPath 最新的 jdk 路径，因为需要一段时间的兼容所以和 KeyJdkDirPath 共存 + KeyJdk17DirPath = "devops.agent.jdk17.dir.path" KeyDockerTaskCount = "devops.docker.parallel.task.count" keyEnableDockerBuild = "devops.docker.enable" KeyLanguage = "devops.language" @@ -95,6 +96,7 @@ type AgentConfig struct { BatchInstallKey string LogsKeepHours int JdkDirPath string + Jdk17DirPath string DockerParallelTaskCount int EnableDockerBuild bool Language string @@ -107,9 +109,10 @@ type AgentEnv struct { OsName string agentIp string HostName string - SlaveVersion string AgentVersion string AgentInstallPath string + // WinTask 启动windows进程的组件如 服务/执行计划 + WinTask string } func (e *AgentEnv) GetAgentIp() string { @@ -162,8 +165,8 @@ func LoadAgentEnv() { GAgentEnv.HostName = systemutil.GetHostName() GAgentEnv.OsName = systemutil.GetOsName() - GAgentEnv.SlaveVersion = DetectWorkerVersion() GAgentEnv.AgentVersion = DetectAgentVersion() + GAgentEnv.WinTask = GetWinTaskType() } // DetectAgentVersion 检测Agent版本 @@ -200,130 +203,6 @@ func DetectAgentVersionByDir(workDir string) string { return strings.TrimSpace(agentVersion) } -// DetectWorkerVersion 检查worker版本 -func DetectWorkerVersion() string { - return DetectWorkerVersionByDir(systemutil.GetWorkDir()) -} - -// DetectWorkerVersionByDir 检测指定目录下的Worker文件版本 -func DetectWorkerVersionByDir(workDir string) string { - jar := fmt.Sprintf("%s/%s", workDir, WorkAgentFile) - tmpDir, _ := systemutil.MkBuildTmpDir() - output, err := command.RunCommand(GetJava(), - []string{"-Djava.io.tmpdir=" + tmpDir, "-Xmx256m", "-cp", jar, "com.tencent.devops.agent.AgentVersionKt"}, - workDir, nil) - - if err != nil { - logs.Errorf("detect worker version failed: %s, output: %s", err.Error(), string(output)) - exitcode.CheckSignalWorkerError(err) - GAgentEnv.SlaveVersion = "" - return "" - } - - detectVersion := parseWorkerVersion(string(output)) - - // 更新下 worker 的版本信息 - if detectVersion == "" { - logs.Warn("parseWorkerVersion null") - } else { - GAgentEnv.SlaveVersion = detectVersion - } - - return detectVersion -} - -// parseWorkerVersion 解析worker版本 -func parseWorkerVersion(output string) string { - // 用函数匹配正确的版本信息, 主要解决tmp空间不足的情况下，jvm会打印出提示信息，导致识别不到worker版本号 - // 兼容旧版本，防止新agent发布后无限升级 - versionRegexp := regexp.MustCompile(`^v(\d+\.)(\d+\.)(\d+)((-RELEASE)|(-SNAPSHOT)?)$`) - lines := strings.Split(output, "\n") - for _, line := range lines { - line = strings.TrimSpace(line) - if !(line == "") && !strings.Contains(line, " ") && !strings.Contains(line, "OPTIONS") { - if len(line) > 64 { - line = line[:64] - } - // 先使用新版本的匹配逻辑匹配，匹配不通则使用旧版本 - if matchWorkerVersion(line) { - logs.Info("match worker version: ", line) - return line - } else { - if versionRegexp != nil { - if versionRegexp.MatchString(line) { - logs.Info("regexp worker version: ", line) - return line - } else { - continue - } - } else { - // 当正则式出错时(versionRegexp = nil)，继续使用原逻辑 - logs.Info("regexp nil worker version: ", line) - return line - } - } - } - } - return "" -} - -// matchWorkerVersion 匹配worker版本信息 -// 版本号为 v数字.数字.数字 || v数字.数字.数字-字符.数字 -// 只匹配以v开头的数字版本即可 -func matchWorkerVersion(line string) bool { - if !strings.HasPrefix(line, "v") { - logs.Warnf("line %s matchWorkerVersion no start 'v'", line) - return false - } - - // 去掉v方便后面计算 - subline := strings.Split(strings.TrimPrefix(line, "v"), ".") - sublen := len(subline) - if sublen < 3 || sublen > 4 { - logs.Warnf("line %s matchWorkerVersion len no match", line) - return false - } - - // v数字.数字.数字 这种去掉v后应该全是数字 - if sublen == 3 { - return checkNumb(subline, line) - } - - // v数字.数字.数字-字符.数字，按照 - 分隔，前面的与len 3一致，后面的两个分别判断，不是数字的是字符，不是字符的是数字 - fSubline := strings.Split(strings.TrimPrefix(line, "v"), "-") - if len(fSubline) != 2 { - logs.Warnf("line %s matchWorkerVersion len no match", line) - return false - } - - if !checkNumb(strings.Split(fSubline[0], "."), line) { - return false - } - - fSubline2 := strings.Split(fSubline[1], ".") - if checkNumb([]string{fSubline2[0]}, line) { - logs.Warnf("line %s matchWorkerVersion not char", line) - return false - } - - if !checkNumb([]string{fSubline2[1]}, line) { - return false - } - - return true -} - -func checkNumb(subs []string, line string) bool { - for _, s := range subs { - _, err := strconv.ParseInt(s, 10, 64) - if err != nil { - logs.Warnf("line %s matchWorkerVersion not numb", line) - return false - } - } - return true -} - // BuildAgentJarPath 生成jar寻址路径 func BuildAgentJarPath() string { return fmt.Sprintf("%s/%s", systemutil.GetWorkDir(), WorkAgentFile) @@ -402,7 +281,15 @@ func LoadAgentConfig() error { jdkDirPath := conf.Section("").Key(KeyJdkDirPath).String() // 如果路径为空，是第一次，需要主动去拿一次 if jdkDirPath == "" { - jdkDirPath = getJavaDir() + workDir := systemutil.GetWorkDir() + if _, err := os.Stat(workDir + "/jdk"); err != nil && !os.IsExist(err) { + jdkDirPath = workDir + "/jre" + } + jdkDirPath = workDir + "/jdk" + } + jdk17DirPath := conf.Section("").Key(KeyJdk17DirPath).String() + if jdk17DirPath == "" { + jdk17DirPath = systemutil.GetWorkDir() + "/jdk17" } // 兼容旧版本 .agent.properties 没有这个键 @@ -481,6 +368,9 @@ func LoadAgentConfig() error { GAgentConfig.JdkDirPath = jdkDirPath logs.Info("jdkDirPath: ", GAgentConfig.JdkDirPath) + GAgentConfig.Jdk17DirPath = jdk17DirPath + logs.Info("jdk17DirPath: ", GAgentConfig.Jdk17DirPath) + GAgentConfig.DockerParallelTaskCount = dockerParallelTaskCount logs.Info("DockerParallelTaskCount: ", GAgentConfig.DockerParallelTaskCount) @@ -525,6 +415,7 @@ func (a *AgentConfig) SaveConfig() error { content.WriteString(KeyIgnoreLocalIps + "=" + GAgentConfig.IgnoreLocalIps + "\n") content.WriteString(KeyLogsKeepHours + "=" + strconv.Itoa(GAgentConfig.LogsKeepHours) + "\n") content.WriteString(KeyJdkDirPath + "=" + GAgentConfig.JdkDirPath + "\n") + content.WriteString(KeyJdk17DirPath + "=" + GAgentConfig.Jdk17DirPath + "\n") content.WriteString(KeyDockerTaskCount + "=" + strconv.Itoa(GAgentConfig.DockerParallelTaskCount) + "\n") content.WriteString(keyEnableDockerBuild + "=" + strconv.FormatBool(GAgentConfig.EnableDockerBuild) + "\n") content.WriteString(KeyLanguage + "=" + GAgentConfig.Language + "\n") @@ -549,36 +440,6 @@ func (a *AgentConfig) GetAuthHeaderMap() map[string]string { return authHeaderMap } -// GetJava 获取本地java命令路径 -func GetJava() string { - if systemutil.IsMacos() { - return GAgentConfig.JdkDirPath + "/Contents/Home/bin/java" - } else { - return GAgentConfig.JdkDirPath + "/bin/java" - } -} - -func SaveJdkDir(dir string) { - if dir == GAgentConfig.JdkDirPath { - return - } - GAgentConfig.JdkDirPath = dir - err := GAgentConfig.SaveConfig() - if err != nil { - logs.Errorf("config.go|SaveJdkDir(dir=%s) failed: %s", dir, err.Error()) - return - } -} - -// getJavaDir 获取本地java文件夹 -func getJavaDir() string { - workDir := systemutil.GetWorkDir() - if _, err := os.Stat(workDir + "/jdk"); err != nil && !os.IsExist(err) { - return workDir + "/jre" - } - return workDir + "/jdk" -} - func GetDockerInitFilePath() string { return systemutil.GetWorkDir() + "/" + DockerInitFile } diff --git a/src/agent/agent/src/pkg/config/config_nowin.go b/src/agent/agent/src/pkg/config/config_nowin.go new file mode 100644 index 000000000000..02d690ffc507 --- /dev/null +++ b/src/agent/agent/src/pkg/config/config_nowin.go @@ -0,0 +1,34 @@ +//go:build linux || darwin + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package config + +func GetWinTaskType() string { + return "" +} diff --git a/src/agent/agent/src/pkg/config/config_win.go b/src/agent/agent/src/pkg/config/config_win.go new file mode 100644 index 000000000000..217cbf45e86f --- /dev/null +++ b/src/agent/agent/src/pkg/config/config_win.go @@ -0,0 +1,53 @@ +//go:build windows + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package config + +import ( + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/wintask" + "github.com/capnspacehook/taskmaster" +) + +func GetWinTaskType() string { + serviceName := "devops_agent_" + GAgentConfig.AgentId + ok := wintask.FindService(serviceName) + if ok { + return string(wintask.ServiceStart) + } + task, taskOk := wintask.FindTask(serviceName) + if taskOk { + // 启用了的task才能进行升级后的启动，否则不能升级Daemon + if task.Enabled && (task.State == taskmaster.TASK_STATE_READY || task.State == taskmaster.TASK_STATE_RUNNING) { + return string(wintask.TaskStart) + } else { + return string(wintask.TaskStart) + "_DISABLE" + } + } + return string(wintask.ManualStart) +} diff --git a/src/agent/agent/src/pkg/config/constant.go b/src/agent/agent/src/pkg/config/constant.go index 5942d6f88a3a..7fea33063e61 100644 --- a/src/agent/agent/src/pkg/config/constant.go +++ b/src/agent/agent/src/pkg/config/constant.go @@ -67,7 +67,8 @@ const ( WorkAgentFile = "worker-agent.jar" - JdkClientFile = "jdk.zip" + JdkClientFile = "jdk.zip" + Jdk17ClientFile = "jdk17.zip" DockerInitFile = "agent_docker_init.sh" ) diff --git a/src/agent/agent/src/pkg/installer/installer.go b/src/agent/agent/src/pkg/installer/installer.go index f3a069c8a354..188b9665f658 100644 --- a/src/agent/agent/src/pkg/installer/installer.go +++ b/src/agent/agent/src/pkg/installer/installer.go @@ -29,6 +29,7 @@ package installer import ( "fmt" + "github.com/TencentBlueKing/bk-ci/agent/src/third_components" "github.com/pkg/errors" "github.com/TencentBlueKing/bk-ci/agentcommon/logs" @@ -45,6 +46,10 @@ import ( func DoInstallAgent() error { logs.Info("start install agent...") config.Init(false) + if err := third_components.Init(); err != nil { + logs.WithError(err).Error("init third_components error") + systemutil.ExitProcess(1) + } if len(config.GAgentConfig.BatchInstallKey) == 0 { return errors.New("file .agent.properties 's devops.agent.batch.install key is null") diff --git a/src/agent/agent/src/pkg/job/build.go b/src/agent/agent/src/pkg/job/build.go index b632ef86daf4..c425c9a33d84 100644 --- a/src/agent/agent/src/pkg/job/build.go +++ b/src/agent/agent/src/pkg/job/build.go @@ -31,6 +31,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "github.com/TencentBlueKing/bk-ci/agent/src/third_components" "github.com/pkg/errors" "io/fs" "os" @@ -171,15 +172,15 @@ func runBuild(buildInfo *api.ThirdPartyBuildInfo) error { if fileutil.Exists(upgradeWorkerFile) { _, err := fileutil.CopyFile(upgradeWorkerFile, agentJarPath, true) - upgradeWorkerFileVersion := config.DetectWorkerVersion() + upgradeWorkerFileVersion := third_components.Worker.DetectWorkerVersion() if err != nil || !strings.HasPrefix(upgradeWorkerFileVersion, "v") { // #5806 宽松判断合法的版本v开头 errorMsg := i18n.Localize("AttemptToRestoreFailed", map[string]interface{}{"filename": agentJarPath, "dir": workDir}) logs.Error(errorMsg) workerBuildFinish(buildInfo.ToFinish(false, errorMsg, api.RecoverRunFileErrorEnum)) } else { // #5806 替换后修正版本号 - if config.GAgentEnv.SlaveVersion != upgradeWorkerFileVersion { - config.GAgentEnv.SlaveVersion = upgradeWorkerFileVersion + if third_components.Worker.GetVersion() != upgradeWorkerFileVersion { + third_components.Worker.SetVersion(upgradeWorkerFileVersion) } } } else { @@ -195,18 +196,20 @@ func runBuild(buildInfo *api.ThirdPartyBuildInfo) error { runUser := config.GAgentConfig.SlaveUser goEnv := map[string]string{ - "DEVOPS_AGENT_VERSION": config.AgentVersion, - "DEVOPS_WORKER_VERSION": config.GAgentEnv.SlaveVersion, - "DEVOPS_PROJECT_ID": buildInfo.ProjectId, - "DEVOPS_BUILD_ID": buildInfo.BuildId, - "DEVOPS_VM_SEQ_ID": buildInfo.VmSeqId, - "DEVOPS_SLAVE_VERSION": config.GAgentEnv.SlaveVersion, //deprecated - "PROJECT_ID": buildInfo.ProjectId, //deprecated - "BUILD_ID": buildInfo.BuildId, //deprecated - "VM_SEQ_ID": buildInfo.VmSeqId, //deprecated - "DEVOPS_FILE_GATEWAY": config.GAgentConfig.FileGateway, - "DEVOPS_GATEWAY": config.GetGateWay(), - "BK_CI_LOCALE_LANGUAGE": config.GAgentConfig.Language, + "DEVOPS_AGENT_VERSION": config.AgentVersion, + "DEVOPS_WORKER_VERSION": third_components.Worker.GetVersion(), + "DEVOPS_PROJECT_ID": buildInfo.ProjectId, + "DEVOPS_BUILD_ID": buildInfo.BuildId, + "DEVOPS_VM_SEQ_ID": buildInfo.VmSeqId, + "DEVOPS_SLAVE_VERSION": third_components.Worker.GetVersion(), //deprecated + "PROJECT_ID": buildInfo.ProjectId, //deprecated + "BUILD_ID": buildInfo.BuildId, //deprecated + "VM_SEQ_ID": buildInfo.VmSeqId, //deprecated + "DEVOPS_FILE_GATEWAY": config.GAgentConfig.FileGateway, + "DEVOPS_GATEWAY": config.GetGateWay(), + "BK_CI_LOCALE_LANGUAGE": config.GAgentConfig.Language, + "DEVOPS_AGENT_JDK_8_PATH": third_components.Jdk.Jdk8.GetJavaOrNull(), + "DEVOPS_AGENT_JDK_17_PATH": third_components.Jdk.Jdk17.GetJavaOrNull(), } if config.GApiEnvVars != nil { config.GApiEnvVars.RangeDo(func(k, v string) bool { diff --git a/src/agent/agent/src/pkg/job/do_build.go b/src/agent/agent/src/pkg/job/do_build.go index d2cb79afc2f3..f2fdaa9b12c2 100644 --- a/src/agent/agent/src/pkg/job/do_build.go +++ b/src/agent/agent/src/pkg/job/do_build.go @@ -32,6 +32,7 @@ package job import ( "fmt" + "github.com/TencentBlueKing/bk-ci/agent/src/third_components" "github.com/pkg/errors" "os" "os/exec" @@ -164,7 +165,7 @@ func writeStartBuildAgentScript(buildInfo *api.ThirdPartyBuildInfo, tmpDir strin fmt.Sprintf("%s -Ddevops.slave.agent.start.file=%s -Ddevops.slave.agent.prepare.start.file=%s "+ "-Ddevops.agent.error.file=%s "+ "-Dbuild.type=AGENT -DAGENT_LOG_PREFIX=%s -Xmx2g -Djava.io.tmpdir=%s -jar %s %s", - config.GetJava(), scriptFile, prepareScriptFile, + third_components.GetJavaLatest(), scriptFile, prepareScriptFile, errorMsgFile, agentLogPrefix, tmpDir, config.BuildAgentJarPath(), getEncodedBuildInfo(buildInfo)), } diff --git a/src/agent/agent/src/pkg/job/do_build_win.go b/src/agent/agent/src/pkg/job/do_build_win.go index 3e49070a7ac2..424c5e131a5c 100644 --- a/src/agent/agent/src/pkg/job/do_build_win.go +++ b/src/agent/agent/src/pkg/job/do_build_win.go @@ -32,6 +32,7 @@ package job import ( "fmt" + "github.com/TencentBlueKing/bk-ci/agent/src/third_components" "github.com/pkg/errors" "os" "os/exec" @@ -55,6 +56,8 @@ func doBuild( goEnv map[string]string, runUser string, ) error { + // windows特有环境变量 + goEnv["DEVOPS_AGENT_WIN_SERVICE"] = config.GAgentEnv.WinTask var err error var exitGroup process.ProcessExitGroup enableExitGroup := config.FetchEnvAndCheck(constant.DevopsAgentEnableExitGroup, "true") @@ -74,7 +77,7 @@ func doBuild( }() } - startCmd := config.GetJava() + startCmd := third_components.GetJavaLatest() agentLogPrefix := fmt.Sprintf("%s_%s_agent", buildInfo.BuildId, buildInfo.VmSeqId) errorMsgFile := getWorkerErrorMsgFile(buildInfo.BuildId, buildInfo.VmSeqId) args := []string{ diff --git a/src/agent/agent/src/pkg/upgrade/download/download_darwin.go b/src/agent/agent/src/pkg/upgrade/download/download_darwin.go index cbf77e93ec66..721348b57cf6 100644 --- a/src/agent/agent/src/pkg/upgrade/download/download_darwin.go +++ b/src/agent/agent/src/pkg/upgrade/download/download_darwin.go @@ -42,7 +42,7 @@ func DownloadAgentFile(saveDir string) (string, error) { func DownloadJdkFile(saveDir string) (string, error) { return api.DownloadUpgradeFile( - "jre/"+strings.TrimPrefix(getServerFileArch(), "_")+"/jre.zip", saveDir+"/"+config.JdkClientFile, + "jre/"+strings.TrimPrefix(getServerFileArch(), "_")+"/jdk17.zip", saveDir+"/"+config.Jdk17ClientFile, ) } diff --git a/src/agent/agent/src/pkg/upgrade/download/download_unix.go b/src/agent/agent/src/pkg/upgrade/download/download_unix.go index 471a32c926d9..bb6d8533cbe5 100644 --- a/src/agent/agent/src/pkg/upgrade/download/download_unix.go +++ b/src/agent/agent/src/pkg/upgrade/download/download_unix.go @@ -43,7 +43,7 @@ func DownloadAgentFile(saveDir string) (string, error) { func DownloadJdkFile(saveDir string) (string, error) { return api.DownloadUpgradeFile( - "jre/"+strings.TrimPrefix(getServerFileArch(), "_")+"/jre.zip", saveDir+"/"+config.JdkClientFile, + "jre/"+strings.TrimPrefix(getServerFileArch(), "_")+"/jdk17.zip", saveDir+"/"+config.Jdk17ClientFile, ) } diff --git a/src/agent/agent/src/pkg/upgrade/download/download_win.go b/src/agent/agent/src/pkg/upgrade/download/download_win.go index 4fafc62025df..3a6e0c8acf27 100644 --- a/src/agent/agent/src/pkg/upgrade/download/download_win.go +++ b/src/agent/agent/src/pkg/upgrade/download/download_win.go @@ -29,7 +29,7 @@ func DownloadAgentFile(saveDir string) (string, error) { func DownloadJdkFile(saveDir string) (string, error) { return api.DownloadUpgradeFile( - "jre/windows/jre.zip", saveDir+"/"+config.JdkClientFile, + "jre/windows/jdk17.zip", saveDir+"/"+config.Jdk17ClientFile, ) } diff --git a/src/agent/agent/src/pkg/upgrade/operation.go b/src/agent/agent/src/pkg/upgrade/operation.go index 27e08a2ffd64..5f4dfbb3c258 100644 --- a/src/agent/agent/src/pkg/upgrade/operation.go +++ b/src/agent/agent/src/pkg/upgrade/operation.go @@ -28,6 +28,7 @@ package upgrade import ( + "github.com/TencentBlueKing/bk-ci/agent/src/third_components" "os" "strconv" "strings" @@ -118,14 +119,14 @@ func runUpgrader(action string) error { } // DoUpgradeOperation 调用升级程序 -func DoUpgradeOperation(changeItems upgradeChangeItem) error { - logs.Info("agentUpgrade|start upgrade, agent changed: ", changeItems.AgentChanged, - ", work agent changed: ", changeItems.WorkAgentChanged, - ", jdk agent changed: ", changeItems.JdkChanged, - ", docker init file changed: ", changeItems.DockerInitFile, +func DoUpgradeOperation(upItems *upgradeItems) error { + logs.Info("agentUpgrade|start upgrade, agent changed: ", upItems.Agent, + ", work agent changed: ", upItems.Worker, + ", jdk agent changed: ", upItems.Jdk, + ", docker init file changed: ", upItems.DockerInitFile, ) - if changeItems.JdkChanged { + if upItems.Jdk { err := DoUpgradeJdk() if err != nil { return err @@ -134,7 +135,7 @@ func DoUpgradeOperation(changeItems upgradeChangeItem) error { logs.Info("agentUpgrade|jdk not changed, skip agent upgrade") } - if changeItems.WorkAgentChanged { + if upItems.Worker { logs.Info("agentUpgrade|work agent changed, replace work agent file") _, err := fileutil.CopyFile( systemutil.GetUpgradeDir()+"/"+config.WorkAgentFile, @@ -146,12 +147,12 @@ func DoUpgradeOperation(changeItems upgradeChangeItem) error { } logs.Info("agentUpgrade|replace agent file done") - config.GAgentEnv.SlaveVersion = config.DetectWorkerVersion() + third_components.Worker.DetectWorkerVersion() } else { logs.Info("agentUpgrade|worker not changed, skip agent upgrade") } - if changeItems.AgentChanged { + if upItems.Agent { logs.Info("agentUpgrade|agent changed, start upgrader") err := runUpgrader(config.ActionUpgrade) if err != nil { @@ -161,7 +162,7 @@ func DoUpgradeOperation(changeItems upgradeChangeItem) error { logs.Info("agentUpgrade|agent not changed, skip agent upgrade") } - if changeItems.DockerInitFile { + if upItems.DockerInitFile { logs.Info("agentUpgrade|docker init file changed, replace docker init file") _, err := fileutil.CopyFile( systemutil.GetUpgradeDir()+"/"+config.DockerInitFile, @@ -189,43 +190,38 @@ func DoUpgradeJdk() error { logs.Info("agentUpgrade|jdk changed, replace jdk file") workDir := systemutil.GetWorkDir() - // 复制出来jdk.zip - _, err := fileutil.CopyFile( - systemutil.GetUpgradeDir()+"/"+config.JdkClientFile, - workDir+"/"+config.JdkClientFile, - true, - ) - if err != nil { - return errors.Wrap(err, "upgrade jdk copy new jdk file error") - } - // 解压缩为一个新文件取代旧文件路径 - jdkTmpName := "jdk" + strconv.FormatInt(time.Now().Unix(), 10) - err = fileutil.Unzip(workDir+"/"+config.JdkClientFile, workDir+"/"+jdkTmpName) + // 解压缩为一个新文件取代旧文件路径，优先使用标准路径 + jdkTmpName := "jdk17" + _, err := os.Stat(workDir + "/" + jdkTmpName) + if !(err != nil && errors.Is(err, os.ErrNotExist)) { + jdkTmpName = "jdk17-" + strconv.FormatInt(time.Now().Unix(), 10) + } + err = fileutil.Unzip(systemutil.GetUpgradeDir()+"/"+config.Jdk17ClientFile, workDir+"/"+jdkTmpName) if err != nil { - return errors.Wrap(err, "upgrade jdk unzip error") + return errors.Wrap(err, "upgrade jdk17 unzip error") } // 删除老的jdk文件，以及之前解压缩或者改名失败残留的，异步删除，删除失败也不影响主进程 go func() { files, err := os.ReadDir(workDir) if err != nil { - logs.WithError(err).Error("agentUpgrade|upgrade jdk remove old jdk file error") + logs.WithError(err).Error("agentUpgrade|upgrade jdk17 remove old jdk file error") return } for _, file := range files { - if (strings.HasPrefix(file.Name(), "jdk") || strings.HasPrefix(file.Name(), "jre")) && - file.Name() != jdkTmpName { + if file.Name() != jdkTmpName && (file.Name() == "jdk17" || file.Name() == "jdk17.zip" || + strings.HasPrefix(file.Name(), "jdk17-")) { err = os.RemoveAll(workDir + "/" + file.Name()) if err != nil { - logs.WithError(err).Error("agentUpgrade|upgrade jdk remove old jdk file error") + logs.WithError(err).Error("agentUpgrade|upgrade jdk17 remove old jdk file error") } } } }() // 修改启动worker的jdk路径 - config.SaveJdkDir(workDir + "/" + jdkTmpName) + third_components.Jdk.Jdk17.SetJavaDir(workDir + "/" + jdkTmpName) logs.Info("agentUpgrade|replace jdk file done") diff --git a/src/agent/agent/src/pkg/upgrade/upgrade.go b/src/agent/agent/src/pkg/upgrade/upgrade.go index 72b9daa54826..b6ad9164f2b8 100644 --- a/src/agent/agent/src/pkg/upgrade/upgrade.go +++ b/src/agent/agent/src/pkg/upgrade/upgrade.go @@ -28,17 +28,13 @@ package upgrade import ( + "github.com/TencentBlueKing/bk-ci/agent/src/third_components" "github.com/pkg/errors" "os" - "strings" - "sync/atomic" "time" - exitcode "github.com/TencentBlueKing/bk-ci/agent/src/pkg/exiterror" "github.com/TencentBlueKing/bk-ci/agent/src/pkg/job" "github.com/TencentBlueKing/bk-ci/agent/src/pkg/upgrade/download" - "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/command" - "github.com/TencentBlueKing/bk-ci/agentcommon/logs" "github.com/TencentBlueKing/bk-ci/agent/src/pkg/api" @@ -47,31 +43,6 @@ import ( "github.com/TencentBlueKing/bk-ci/agentcommon/utils/fileutil" ) -var JdkVersion = &JdkVersionType{} - -// JdkVersionType jdk版本信息缓存 -type JdkVersionType struct { - JdkFileModTime time.Time - // 版本信息，原子级的 []string - version atomic.Value -} - -func (j *JdkVersionType) GetVersion() []string { - data := j.version.Load() - if data == nil { - return make([]string, 0) - } else { - return data.([]string) - } -} - -func (j *JdkVersionType) SetVersion(version []string) { - if version == nil { - version = []string{} - } - j.version.Swap(version) -} - // DockerFileMd5 缓存，用来计算md5 var DockerFileMd5 struct { // 目前非linux机器不支持，以及一些机器不使用docker就不用计算md5 @@ -80,19 +51,16 @@ var DockerFileMd5 struct { Md5 string } -type upgradeChangeItem struct { - AgentChanged bool - WorkAgentChanged bool - JdkChanged bool - DockerInitFile bool +// 升级分为两个阶段，下载和升级，每个阶段都可能存在不升级的情况，这里用一个对象保证生命周期统一 +type upgradeItems struct { + Agent bool + Worker bool + Jdk bool + DockerInitFile bool } -func (u upgradeChangeItem) checkNoChange() bool { - if !u.AgentChanged && !u.WorkAgentChanged && !u.JdkChanged && !u.DockerInitFile { - return true - } - - return false +func (u *upgradeItems) NoChange() bool { + return !u.Agent && !u.Worker && !u.Jdk && !u.DockerInitFile } // AgentUpgrade 升级主逻辑 @@ -107,7 +75,14 @@ func AgentUpgrade(upgradeItem *api.UpgradeItem, hasBuild bool) { } }() - if !upgradeItem.Agent && !upgradeItem.Worker && !upgradeItem.Jdk && !upgradeItem.DockerInitFile { + upItems := &upgradeItems{ + Agent: upgradeItem.Agent, + Worker: upgradeItem.Worker, + Jdk: upgradeItem.Jdk, + DockerInitFile: upgradeItem.DockerInitFile, + } + + if upItems.NoChange() { return } else { // 如果同时还领取了构建任务那么这次的升级取消 @@ -139,14 +114,16 @@ func AgentUpgrade(upgradeItem *api.UpgradeItem, hasBuild bool) { return } - logs.Info("agentUpgrade|download upgrade files start") - changeItems := downloadUpgradeFiles(upgradeItem) - if changeItems.checkNoChange() { + // 下载升级文件 + logs.Infof("agentUpgrade|download upgrade files start %+v", upItems) + downloadUpgradeFiles(upItems) + if upItems.NoChange() { return } - logs.Info("agentUpgrade|download upgrade files done") - err := DoUpgradeOperation(changeItems) + // 升级逻辑 + logs.Infof("agentUpgrade|download upgrade files done %+v", upItems) + err := DoUpgradeOperation(upItems) if err != nil { logs.WithError(err).Error("agentUpgrade|do upgrade operation failed") } else { @@ -154,51 +131,6 @@ func AgentUpgrade(upgradeItem *api.UpgradeItem, hasBuild bool) { } } -// SyncJdkVersion 同步jdk版本信息 -func SyncJdkVersion() error { - // 获取jdk文件状态以及时间 - stat, err := os.Stat(config.GAgentConfig.JdkDirPath) - if err != nil { - if os.IsNotExist(err) { - logs.WithError(err).Error("syncJdkVersion no jdk dir find") - // jdk版本置为空，否则会一直保持有版本的状态 - JdkVersion.SetVersion([]string{}) - return nil - } - return errors.Wrap(err, "agent check jdk dir error") - } - nowModTime := stat.ModTime() - - // 如果为空则必获取 - if len(JdkVersion.GetVersion()) == 0 { - version, err := getJdkVersion() - if err != nil { - // 拿取错误时直接下载新的 - logs.WithError(err).Error("syncJdkVersion getJdkVersion err") - return nil - } - JdkVersion.SetVersion(version) - JdkVersion.JdkFileModTime = nowModTime - return nil - } - - // 判断文件夹最后修改时间，不一致时不用更改 - if nowModTime == JdkVersion.JdkFileModTime { - return nil - } - - version, err := getJdkVersion() - if err != nil { - // 拿取错误时直接下载新的 - logs.WithError(err).Error("syncJdkVersion getJdkVersion err") - JdkVersion.SetVersion([]string{}) - return nil - } - JdkVersion.SetVersion(version) - JdkVersion.JdkFileModTime = nowModTime - return nil -} - func SyncDockerInitFileMd5() error { if !systemutil.IsLinux() || !config.GAgentConfig.EnableDockerBuild { DockerFileMd5.NeedUpgrade = false @@ -243,101 +175,27 @@ func SyncDockerInitFileMd5() error { return nil } -func getJdkVersion() ([]string, error) { - jdkVersion, err := command.RunCommand(config.GetJava(), []string{"-version"}, "", nil) - if err != nil { - logs.WithError(err).Error("agent get jdk version failed") - exitcode.CheckSignalJdkError(err) - return nil, errors.Wrap(err, "agent get jdk version failed") - } - var jdkV []string - if jdkVersion != nil { - versionOutputString := strings.TrimSpace(string(jdkVersion)) - jdkV = trimJdkVersionList(versionOutputString) - } - - return jdkV, nil -} - -// parseJdkVersionList 清洗在解析一些版本信息的干扰信息,避免因tmp空间满等导致识别不准确造成重复不断的升级 -func trimJdkVersionList(versionOutputString string) []string { - /* - OpenJDK 64-Bit Server VM warning: Insufficient space for shared memory file: - 32490 - Try using the -Djava.io.tmpdir= option to select an alternate temp location. - - openjdk version "1.8.0_352" - OpenJDK Runtime Environment (Tencent Kona 8.0.12) (build 1.8.0_352-b1) - OpenJDK 64-Bit Server VM (Tencent Kona 8.0.12) (build 25.352-b1, mixed mode) - Picked up _JAVA_OPTIONS: -Xmx8192m -Xms256m -Xss8m - */ - // 一个JVM版本只需要识别3行。 - var jdkV = make([]string, 3) - - var sep = "\n" - if strings.HasSuffix(versionOutputString, "\r\n") { - sep = "\r\n" - } - - lines := strings.Split(strings.TrimSuffix(versionOutputString, sep), sep) - - var pos = 0 - for i := range lines { - - if pos == 0 { - if strings.Contains(lines[i], " version ") { - jdkV[pos] = lines[i] - pos++ - } - } else if pos == 1 { - if strings.Contains(lines[i], " Runtime Environment ") { - jdkV[pos] = lines[i] - pos++ - } - } else if pos == 2 { - if strings.Contains(lines[i], " Server VM ") { - jdkV[pos] = lines[i] - break - } - } - } - - return jdkV -} - // downloadUpgradeFiles 下载升级文件 -func downloadUpgradeFiles(item *api.UpgradeItem) upgradeChangeItem { +func downloadUpgradeFiles(item *upgradeItems) { workDir := systemutil.GetWorkDir() upgradeDir := systemutil.GetUpgradeDir() _ = os.MkdirAll(upgradeDir, os.ModePerm) - result := upgradeChangeItem{} - - if !item.Agent { - result.AgentChanged = false - } else { - result.AgentChanged = downloadUpgradeAgent(workDir, upgradeDir) + if item.Agent { + item.Agent = downloadUpgradeAgent(workDir, upgradeDir) } - if !item.Worker { - result.WorkAgentChanged = false - } else { - result.WorkAgentChanged = downloadUpgradeWorker(workDir, upgradeDir) + if item.Worker { + item.Worker = downloadUpgradeWorker(workDir, upgradeDir) } - if !item.Jdk { - result.JdkChanged = false - } else { - result.JdkChanged = downloadUpgradeJdk(upgradeDir) + if item.Jdk { + item.Jdk = downloadUpgradeJdk(upgradeDir) } - if !item.DockerInitFile { - result.DockerInitFile = false - } else { - result.DockerInitFile = downloadUpgradeDockerInit(upgradeDir) + if item.DockerInitFile { + item.DockerInitFile = downloadUpgradeDockerInit(upgradeDir) } - - return result } func downloadUpgradeAgent(workDir, upgradeDir string) (agentChanged bool) { @@ -407,7 +265,7 @@ func downloadUpgradeWorker(workDir, upgradeDir string) (workAgentChanged bool) { logs.Info("agentUpgrade|newWorkerMd5=" + newWorkerMd5 + ",workerMd5=" + workerMd5) - workerVersion := config.DetectWorkerVersionByDir(systemutil.GetUpgradeDir()) + workerVersion := third_components.Worker.DetectWorkerVersionByDir(systemutil.GetUpgradeDir()) workAgentChanged = false if len(workerVersion) > 0 { workAgentChanged = workerMd5 != newWorkerMd5 diff --git a/src/agent/agent/src/pkg/upgrader/upgrader.go b/src/agent/agent/src/pkg/upgrader/upgrader.go index 2a0d8e200006..3544a6604aa4 100644 --- a/src/agent/agent/src/pkg/upgrader/upgrader.go +++ b/src/agent/agent/src/pkg/upgrader/upgrader.go @@ -33,6 +33,7 @@ import ( "fmt" "github.com/TencentBlueKing/bk-ci/agent/src/pkg/constant" innerFileUtil "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/fileutil" + "github.com/TencentBlueKing/bk-ci/agent/src/third_components" "github.com/pkg/errors" "os" "strconv" @@ -56,6 +57,10 @@ const ( func DoUpgradeAgent() error { logs.Info("start upgrade agent") config.Init(false) + if err := third_components.Init(); err != nil { + logs.WithError(err).Error("init third_components error") + systemutil.ExitProcess(1) + } totalLock := flock.New(fmt.Sprintf("%s/%s.lock", systemutil.GetRuntimeDir(), systemutil.TotalLock)) err := totalLock.Lock() diff --git a/src/agent/agent/src/pkg/upgrader/upgrader_win.go b/src/agent/agent/src/pkg/upgrader/upgrader_win.go index f39ac1201ba7..8e685f621a07 100644 --- a/src/agent/agent/src/pkg/upgrader/upgrader_win.go +++ b/src/agent/agent/src/pkg/upgrader/upgrader_win.go @@ -31,15 +31,13 @@ package upgrader import ( "fmt" - "github.com/TencentBlueKing/bk-ci/agent/src/pkg/constant" innerFileUtil "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/fileutil" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/wintask" + "github.com/TencentBlueKing/bk-ci/agent/src/third_components" + "github.com/capnspacehook/taskmaster" "github.com/pkg/errors" - "golang.org/x/sys/windows/svc/mgr" "os" - "os/exec" "strconv" - "strings" - "syscall" "time" "github.com/TencentBlueKing/bk-ci/agentcommon/logs" @@ -51,29 +49,25 @@ import ( "github.com/gofrs/flock" - "github.com/capnspacehook/taskmaster" "github.com/shirou/gopsutil/v4/process" ) -type startType string - const ( agentProcess = "agent" daemonProcess = "daemon" - - // 服务，执行计划，手动 - serviceStart startType = "service" - taskStart startType = "task" - manualStart startType = "manual" ) // DoUpgradeAgent 升级agent // 1、通过service启动的daemon因为go本身内部注册了daemon导致权限模型有些未知问题，无法更新daemon后启动，只能更新agent -// 2、通过执行计划启动的daemon因为具有登录态，可以直接执行脚本拉起 +// 2、通过执行计划启动的daemon因为具有登录态，可以直接执行脚本拉起，如果执行计划存在问题，则无法拉起，需要使用 1 中的方式更新 // 3、用户双击启动的daemon和service一样，无法更新daemon，只能更新agent func DoUpgradeAgent() error { logs.Info("start upgrade agent") config.Init(false) + if err := third_components.Init(); err != nil { + logs.WithError(err).Error("init third_components error") + systemutil.ExitProcess(1) + } totalLock := flock.New(fmt.Sprintf("%s/%s.lock", systemutil.GetRuntimeDir(), systemutil.TotalLock)) err := totalLock.Lock() @@ -83,29 +77,35 @@ func DoUpgradeAgent() error { } defer func() { totalLock.Unlock() }() - startT := manualStart + startT := wintask.ManualStart var winTask *taskmaster.RegisteredTask = nil // 先查询服务 serviceName := "devops_agent_" + config.GAgentConfig.AgentId - ok := findService(serviceName) + ok := wintask.FindService(serviceName) if ok { - startT = serviceStart + startT = wintask.ServiceStart } else { - if task, taskOk := findTask(serviceName); taskOk { - winTask = task - startT = taskStart + if task, taskOk := wintask.FindTask(serviceName); taskOk { + // 启用了的task才能进行升级后的启动，否则不能升级Daemon + if task.Enabled && + (task.State == taskmaster.TASK_STATE_READY || task.State == taskmaster.TASK_STATE_RUNNING) { + winTask = task + startT = wintask.TaskStart + } else { + logs.Warnf("win task exist but not enable state: %d", task.State) + } } } // 理论上不可能，但是作为补充可以为后文提供逻辑依据 - if startT == taskStart && winTask == nil { + if startT == wintask.TaskStart && winTask == nil { logs.Warn("win task not exist update agent") - startT = manualStart + startT = wintask.ManualStart } logs.Infof("agent process start by %s", startT) daemonChange := false - if startT == taskStart { + if startT == wintask.TaskStart { daemonChange, err = checkUpgradeFileChange(config.GetClientDaemonFile()) if err != nil { logs.WithError(err).Warn("check daemon upgrade file change failed") @@ -182,12 +182,7 @@ func DoUpgradeAgent() error { // 只有 daemon 被杀才启动，没被杀等待被 daemon 拉起来 if daemonChange { switch startT { - case taskStart: - cmd := exec.Command("cmd.exe", "/c", systemutil.GetWorkDir()+"/devopsctl.vbs") - cmd.SysProcAttr = &syscall.SysProcAttr{ - CreationFlags: constant.WinCommandNewConsole | syscall.CREATE_NEW_PROCESS_GROUP, - NoInheritHandles: true, - } + case wintask.TaskStart: if _, err = winTask.Run(); err != nil { return errors.Wrapf(err, "start win task failed") } @@ -292,37 +287,3 @@ func replaceAgentFile(fileName string) error { return nil } - -func findService(name string) bool { - m, err := mgr.Connect() - if err != nil { - logs.WithError(err).Error("connect manager failed") - return false - } - defer m.Disconnect() - - service, err := m.OpenService(name) - if err != nil { - logs.WithError(err).Error("open manager failed") - return false - } - defer service.Close() - - return true -} - -func findTask(name string) (*taskmaster.RegisteredTask, bool) { - service, err := taskmaster.Connect() - if err != nil { - logs.WithError(err).Error("connect taskmaster failed") - return nil, false - } - - task, err := service.GetRegisteredTask("\\" + name) - if err != nil && !strings.Contains(err.Error(), "error parsing registered task") { - logs.WithError(err).Error("get registered task failed") - return nil, false - } - - return &task, true -} diff --git a/src/agent/agent/src/pkg/util/httputil/devops.go b/src/agent/agent/src/pkg/util/httputil/devops.go index 41b4ee230648..b3d1837fb660 100644 --- a/src/agent/agent/src/pkg/util/httputil/devops.go +++ b/src/agent/agent/src/pkg/util/httputil/devops.go @@ -168,6 +168,7 @@ func DownloadUpgradeFile(url string, headers map[string]string, filepath string) return "", errors.New("file not found") } if resp.StatusCode == http.StatusNotModified { + logs.Infof("download upgrade file %s not modified", filepath) return oldFileMd5, nil } body, _ := io.ReadAll(resp.Body) diff --git a/src/agent/agent/src/pkg/util/wintask/wintask.go b/src/agent/agent/src/pkg/util/wintask/wintask.go new file mode 100644 index 000000000000..1880b28cb25a --- /dev/null +++ b/src/agent/agent/src/pkg/util/wintask/wintask.go @@ -0,0 +1,84 @@ +//go:build windows + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package wintask + +import ( + "github.com/TencentBlueKing/bk-ci/agentcommon/logs" + "github.com/capnspacehook/taskmaster" + "golang.org/x/sys/windows/svc/mgr" + "strings" +) + +type StartType string + +const ( + // ServiceStart 服务 + ServiceStart StartType = "SERVICE" + // TaskStart 执行计划 + TaskStart StartType = "TASK" + // ManualStart 手动 + ManualStart StartType = "MANUAL" +) + +// FindService 查找windows服务 +func FindService(name string) bool { + m, err := mgr.Connect() + if err != nil { + logs.WithError(err).Error("connect manager failed") + return false + } + defer m.Disconnect() + + service, err := m.OpenService(name) + if err != nil { + logs.WithError(err).Error("open manager failed") + return false + } + defer service.Close() + + return true +} + +// FindTask 查找windows执行计划 +func FindTask(name string) (*taskmaster.RegisteredTask, bool) { + service, err := taskmaster.Connect() + if err != nil { + logs.WithError(err).Error("connect taskmaster failed") + return nil, false + } + + task, err := service.GetRegisteredTask("\\" + name) + if err != nil && !strings.Contains(err.Error(), "error parsing registered task") { + logs.WithError(err).Error("get registered task failed") + return nil, false + } + + return &task, true +} diff --git a/src/agent/agent/src/third_components/jdk.go b/src/agent/agent/src/third_components/jdk.go new file mode 100644 index 000000000000..6f2854ca86a6 --- /dev/null +++ b/src/agent/agent/src/third_components/jdk.go @@ -0,0 +1,306 @@ +package third_components + +import ( + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/config" + exitcode "github.com/TencentBlueKing/bk-ci/agent/src/pkg/exiterror" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/command" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/systemutil" + "github.com/TencentBlueKing/bk-ci/agentcommon/logs" + commonutil "github.com/TencentBlueKing/bk-ci/agentcommon/utils" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "os" + "strings" + "sync" + "time" +) + +var Jdk *JdkType + +type jdkVersionNum int + +const ( + jdk8 jdkVersionNum = 8 + jdk17 jdkVersionNum = 17 +) + +// JdkType 用来保存JDK版本信息 +type JdkType struct { + Jdk8 JdkVersionType + Jdk17 JdkVersionType +} + +// JdkVersionType jdk版本信息缓存 +type JdkVersionType struct { + // jdkModTime java应用修改时间 + jdkModTime time.Time + version []string + lock sync.RWMutex + logs *logrus.Entry + vNum jdkVersionNum +} + +func (j *JdkVersionType) IsNull() bool { + return commonutil.IsStringSliceBlank(j.GetVersion()) +} + +func (j *JdkVersionType) GetVersion() []string { + j.lock.RLock() + defer j.lock.RUnlock() + return j.version +} + +func (j *JdkVersionType) GetJdkModTime() time.Time { + j.lock.RLock() + defer j.lock.RUnlock() + return j.jdkModTime +} + +func (j *JdkVersionType) SetVersionAndTime(v []string, t time.Time) { + j.lock.Lock() + defer j.lock.Unlock() + j.version = v + j.jdkModTime = t +} + +// SyncJdkVersion 同步jdk版本信息 +func (j *JdkVersionType) SyncJdkVersion() error { + jdkPath := config.GAgentConfig.Jdk17DirPath + if j.vNum == jdk8 { + jdkPath = config.GAgentConfig.JdkDirPath + } + + // 获取jdk文件状态以及时间 + stat, err := os.Stat(jdkPath) + if err != nil { + if os.IsNotExist(err) { + j.logs.Warnf("syncJdkVersion no %s find", jdkPath) + // jdk版本置为空，否则会一直保持有版本的状态 + j.SetVersionAndTime([]string{}, time.Time{}) + return nil + } + return errors.Wrapf(err, "stat jdk %s error", jdkPath) + } + nowModTime := stat.ModTime() + + // 如果为空则必获取 + if j.IsNull() { + logs.Debugf("is null get jdkversion %+v", j.GetVersion()) + version, err := j.getJdkVersion() + if err != nil { + // 拿取错误时直接下载新的 + j.logs.WithError(err).Error("jVersion is null getJdkVersion err") + return nil + } + j.SetVersionAndTime(version, nowModTime) + logs.Debugf("is null set jdkversion %+v", j.GetVersion()) + return nil + } + + // 判断文件夹最后修改时间，不一致时不用更改 + if nowModTime == j.GetJdkModTime() { + return nil + } + + // 最后修改时间不一致需要重新获取版本信息 + version, err := j.getJdkVersion() + if err != nil { + // 拿取错误时直接下载新的 + j.logs.WithError(err).Error("exist jVersion getJdkVersion err") + j.SetVersionAndTime(nil, time.Time{}) + return nil + } + j.SetVersionAndTime(version, nowModTime) + return nil +} + +func (j *JdkVersionType) getJdkVersion() ([]string, error) { + var jdkVersion = make([]byte, 0) + var err error + switch j.vNum { + case jdk8: + jdkVersion, err = command.RunCommand(j.GetJava(), []string{"-version"}, "", nil) + case jdk17: + jdkVersion, err = command.RunCommand(j.GetJava(), []string{"--version"}, "", nil) + } + + if err != nil { + exitcode.CheckSignalJdkError(err) + return nil, errors.Wrapf(err, "agent get jdk %d version failed", j.vNum) + } + var jdkV []string + if jdkVersion == nil { + return jdkV, nil + } + + versionOutputString := strings.TrimSpace(string(jdkVersion)) + switch j.vNum { + case jdk8: + jdkV = trimJdk8VersionList(versionOutputString) + case jdk17: + jdkV = trimJdk17VersionList(versionOutputString) + } + j.logs.Infof("getJdkVersion %d %+v", j.vNum, string(jdkVersion)) + return jdkV, nil +} + +// trimJdk8VersionList 清洗jdk8在解析一些版本信息的干扰信息,避免因tmp空间满等导致识别不准确造成重复不断的升级 +func trimJdk8VersionList(versionOutputString string) []string { + /* + OpenJDK 64-Bit Server VM warning: Insufficient space for shared memory file: + 32490 + Try using the -Djava.io.tmpdir= option to select an alternate temp location. + + openjdk version "1.8.0_352" + OpenJDK Runtime Environment (Tencent Kona 8.0.12) (build 1.8.0_352-b1) + OpenJDK 64-Bit Server VM (Tencent Kona 8.0.12) (build 25.352-b1, mixed mode) + Picked up _JAVA_OPTIONS: -Xmx8192m -Xms256m -Xss8m + */ + // 一个JVM版本只需要识别3行。 + var jdkV = make([]string, 3) + + var sep = "\n" + if strings.HasSuffix(versionOutputString, "\r\n") { + sep = "\r\n" + } + + lines := strings.Split(strings.TrimSuffix(versionOutputString, sep), sep) + + var pos = 0 + for i := range lines { + if pos == 0 { + if strings.Contains(lines[i], " version ") { + jdkV[pos] = lines[i] + pos++ + } + } else if pos == 1 { + if strings.Contains(lines[i], " Runtime Environment ") { + jdkV[pos] = lines[i] + pos++ + } + } else if pos == 2 { + if strings.Contains(lines[i], " Server VM ") { + jdkV[pos] = lines[i] + break + } + } + } + + return jdkV +} + +// trimJdk17VersionList 清洗jdk17在解析一些版本信息的干扰信息,避免因tmp空间满等导致识别不准确造成重复不断的升级 +func trimJdk17VersionList(versionOutputString string) []string { + /* + OpenJDK 64-Bit Server VM warning: Insufficient space for shared memory file: + 32490 + Try using the -Djava.io.tmpdir= option to select an alternate temp location. + + openjdk 17.0.11 2024-04-23 LTS + OpenJDK Runtime Environment TencentKonaJDK (build 17.0.11+1-LTS) + OpenJDK 64-Bit Server VM TencentKonaJDK (build 17.0.11+1-LTS, mixed mode, sharing) + Picked up _JAVA_OPTIONS: -Xmx8192m -Xms256m -Xss8m + */ + // 一个JVM版本只需要识别3行。 + var jdkV = make([]string, 3) + + var sep = "\n" + if strings.HasSuffix(versionOutputString, "\r\n") { + sep = "\r\n" + } + + lines := strings.Split(strings.TrimSuffix(versionOutputString, sep), sep) + + var pos = 0 + for i := range lines { + if pos == 0 { + if strings.Contains(strings.ToLower(lines[i]), "openjdk 17") { + jdkV[pos] = lines[i] + pos++ + } + } else if pos == 1 { + if strings.Contains(lines[i], " Runtime Environment ") { + jdkV[pos] = lines[i] + pos++ + } + } else if pos == 2 { + if strings.Contains(lines[i], " Server VM ") { + jdkV[pos] = lines[i] + break + } + } + } + + return jdkV +} + +const ( + macosJdkBinPath = "/Contents/Home/bin/java" + linuxJdkBinPath = "/bin/java" + winJdkBinPath = "/bin/java.exe" +) + +// GetJava 获取本地java路径，区分版本 +func (j *JdkVersionType) GetJava() string { + switch j.vNum { + case jdk8: + if systemutil.IsMacos() { + return config.GAgentConfig.JdkDirPath + macosJdkBinPath + } else if systemutil.IsWindows() { + return config.GAgentConfig.JdkDirPath + winJdkBinPath + } else { + return config.GAgentConfig.JdkDirPath + linuxJdkBinPath + } + default: + if systemutil.IsMacos() { + return config.GAgentConfig.Jdk17DirPath + macosJdkBinPath + } else if systemutil.IsWindows() { + return config.GAgentConfig.Jdk17DirPath + winJdkBinPath + } else { + return config.GAgentConfig.Jdk17DirPath + linuxJdkBinPath + } + } +} + +func (j *JdkVersionType) GetJavaOrNull() string { + path := j.GetJava() + if _, err := os.Stat(path); err != nil && !os.IsExist(err) { + logs.WithError(err).Warnf("jdk %d stat %s error", j.vNum, path) + return "" + } + return path +} + +// GetJavaLatest 获取本地java路径，默认使用最新的，没有时使用旧的 +func GetJavaLatest() string { + jdk17path := Jdk.Jdk17.GetJavaOrNull() + if jdk17path == "" { + return Jdk.Jdk8.GetJava() + } + return jdk17path +} + +func (j *JdkVersionType) SetJavaDir(dir string) { + switch j.vNum { + case jdk8: + if dir == config.GAgentConfig.JdkDirPath { + return + } + config.GAgentConfig.JdkDirPath = dir + err := config.GAgentConfig.SaveConfig() + if err != nil { + j.logs.WithError(err).Errorf("saveJdk %d Dir %s failed", j.vNum, dir) + return + } + default: + if dir == config.GAgentConfig.Jdk17DirPath { + return + } + config.GAgentConfig.Jdk17DirPath = dir + err := config.GAgentConfig.SaveConfig() + if err != nil { + j.logs.WithError(err).Errorf("saveJdk %d Dir %s failed", j.vNum, dir) + return + } + } +} diff --git a/src/agent/agent/src/pkg/upgrade/upgrade_test.go b/src/agent/agent/src/third_components/jdk_test.go similarity index 64% rename from src/agent/agent/src/pkg/upgrade/upgrade_test.go rename to src/agent/agent/src/third_components/jdk_test.go index 01e795205e98..3615181ea610 100644 --- a/src/agent/agent/src/pkg/upgrade/upgrade_test.go +++ b/src/agent/agent/src/third_components/jdk_test.go @@ -1,4 +1,4 @@ -package upgrade +package third_components import ( "reflect" @@ -44,7 +44,7 @@ func Test_trimJdkVersionList(t *testing.T) { } func loopTest(t *testing.T, name string, jdkVersionString string, want []string) { - tests := trimJdkVersionList(jdkVersionString) + tests := trimJdk8VersionList(jdkVersionString) t.Run(name+"|length=3", func(t *testing.T) { if len(tests) != len(want) { t.Fatalf("\nFail: len(%d), want(%d)", len(tests), len(want)) @@ -59,3 +59,33 @@ func loopTest(t *testing.T, name string, jdkVersionString string, want []string) }) } } + +func Test_trimJdk17VersionList(t *testing.T) { + type args struct { + versionOutputString string + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "normal", + args: args{ + versionOutputString: "openjdk 17.0.11 2024-04-23 LTS\nOpenJDK Runtime Environment TencentKonaJDK (build 17.0.11+1-LTS)\nOpenJDK 64-Bit Server VM TencentKonaJDK (build 17.0.11+1-LTS, mixed mode, sharing)\n", + }, + want: []string{ + "openjdk 17.0.11 2024-04-23 LTS", + "OpenJDK Runtime Environment TencentKonaJDK (build 17.0.11+1-LTS)", + "OpenJDK 64-Bit Server VM TencentKonaJDK (build 17.0.11+1-LTS, mixed mode, sharing)", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := trimJdk17VersionList(tt.args.versionOutputString); !reflect.DeepEqual(got, tt.want) { + t.Errorf("trimJdk17VersionList() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/src/agent/agent/src/third_components/third_components.go b/src/agent/agent/src/third_components/third_components.go new file mode 100644 index 000000000000..e7b7370aede2 --- /dev/null +++ b/src/agent/agent/src/third_components/third_components.go @@ -0,0 +1,60 @@ +package third_components + +import ( + "github.com/TencentBlueKing/bk-ci/agentcommon/logs" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "sync" + "time" +) + +// 用来处理一些Agent使用的第三方组件信息 + +// Init 初始化第三方组件 +// 需要在日志初始化后 +func Init() error { + if err := initOb(); err != nil { + return err + } + + Worker.DetectWorkerVersion() + + return nil +} + +func initOb() error { + if logs.Logs == nil { + return errors.New("init third components need init log") + } + + Jdk = &JdkType{ + Jdk8: JdkVersionType{ + jdkModTime: time.Time{}, + version: nil, + lock: sync.RWMutex{}, + logs: logs.Logs.WithFields(logrus.Fields{ + "third_component": "jdk8", + }), + vNum: jdk8, + }, + Jdk17: JdkVersionType{ + jdkModTime: time.Time{}, + version: nil, + lock: sync.RWMutex{}, + logs: logs.Logs.WithFields(logrus.Fields{ + "third_component": "jdk17", + }), + vNum: jdk17, + }, + } + + Worker = &WorkerType{ + version: "", + lock: sync.RWMutex{}, + logs: logs.Logs.WithFields(logrus.Fields{ + "third_component": "worker", + }), + } + + return nil +} diff --git a/src/agent/agent/src/third_components/worker.go b/src/agent/agent/src/third_components/worker.go new file mode 100644 index 000000000000..d0d1120a80b0 --- /dev/null +++ b/src/agent/agent/src/third_components/worker.go @@ -0,0 +1,161 @@ +package third_components + +import ( + "fmt" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/config" + exitcode "github.com/TencentBlueKing/bk-ci/agent/src/pkg/exiterror" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/command" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/systemutil" + "github.com/sirupsen/logrus" + "regexp" + "strconv" + "strings" + "sync" +) + +var Worker *WorkerType + +type WorkerType struct { + version string + lock sync.RWMutex + logs *logrus.Entry +} + +func (w *WorkerType) SetVersion(version string) { + w.lock.Lock() + defer w.lock.Unlock() + w.version = version +} + +func (w *WorkerType) GetVersion() string { + w.lock.RLock() + defer w.lock.RUnlock() + return w.version +} + +// DetectWorkerVersion 检查worker版本 +func (w *WorkerType) DetectWorkerVersion() string { + return w.DetectWorkerVersionByDir(systemutil.GetWorkDir()) +} + +// DetectWorkerVersionByDir 检测指定目录下的Worker文件版本 +func (w *WorkerType) DetectWorkerVersionByDir(workDir string) string { + jar := fmt.Sprintf("%s/%s", workDir, config.WorkAgentFile) + tmpDir, _ := systemutil.MkBuildTmpDir() + output, err := command.RunCommand( + GetJavaLatest(), + []string{"-Djava.io.tmpdir=" + tmpDir, "-Xmx256m", "-cp", jar, "com.tencent.devops.agent.AgentVersionKt"}, + workDir, + nil, + ) + + if err != nil { + w.logs.Errorf("detect worker version failed: %s, output: %s", err.Error(), string(output)) + exitcode.CheckSignalWorkerError(err) + w.SetVersion("") + return "" + } + + detectVersion := w.parseWorkerVersion(string(output)) + + // 更新下 worker 的版本信息 + if detectVersion == "" { + w.logs.Warn("parseWorkerVersion null") + } else { + w.SetVersion(detectVersion) + } + + return detectVersion +} + +// parseWorkerVersion 解析worker版本 +func (w *WorkerType) parseWorkerVersion(output string) string { + // 用函数匹配正确的版本信息, 主要解决tmp空间不足的情况下，jvm会打印出提示信息，导致识别不到worker版本号 + // 兼容旧版本，防止新agent发布后无限升级 + versionRegexp := regexp.MustCompile(`^v(\d+\.)(\d+\.)(\d+)((-RELEASE)|(-SNAPSHOT)?)$`) + lines := strings.Split(output, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if !(line == "") && !strings.Contains(line, " ") && !strings.Contains(line, "OPTIONS") { + if len(line) > 64 { + line = line[:64] + } + // 先使用新版本的匹配逻辑匹配，匹配不通则使用旧版本 + if w.matchWorkerVersion(line) { + w.logs.Info("match worker version: ", line) + return line + } else { + if versionRegexp != nil { + if versionRegexp.MatchString(line) { + w.logs.Info("regexp worker version: ", line) + return line + } else { + continue + } + } else { + // 当正则式出错时(versionRegexp = nil)，继续使用原逻辑 + w.logs.Info("regexp nil worker version: ", line) + return line + } + } + } + } + return "" +} + +// matchWorkerVersion 匹配worker版本信息 +// 版本号为 v数字.数字.数字 || v数字.数字.数字-字符.数字 +// 只匹配以v开头的数字版本即可 +func (w *WorkerType) matchWorkerVersion(line string) bool { + if !strings.HasPrefix(line, "v") { + w.logs.Warnf("line %s matchWorkerVersion no start 'v'", line) + return false + } + + // 去掉v方便后面计算 + subline := strings.Split(strings.TrimPrefix(line, "v"), ".") + sublen := len(subline) + if sublen < 3 || sublen > 4 { + w.logs.Warnf("line %s matchWorkerVersion len no match", line) + return false + } + + // v数字.数字.数字 这种去掉v后应该全是数字 + if sublen == 3 { + return w.checkNumb(subline, line) + } + + // v数字.数字.数字-字符.数字，按照 - 分隔，前面的与len 3一致，后面的两个分别判断，不是数字的是字符，不是字符的是数字 + fSubline := strings.Split(strings.TrimPrefix(line, "v"), "-") + if len(fSubline) != 2 { + w.logs.Warnf("line %s matchWorkerVersion len no match", line) + return false + } + + if !w.checkNumb(strings.Split(fSubline[0], "."), line) { + return false + } + + fSubline2 := strings.Split(fSubline[1], ".") + if w.checkNumb([]string{fSubline2[0]}, line) { + w.logs.Warnf("line %s matchWorkerVersion not char", line) + return false + } + + if !w.checkNumb([]string{fSubline2[1]}, line) { + return false + } + + return true +} + +func (w *WorkerType) checkNumb(subs []string, line string) bool { + for _, s := range subs { + _, err := strconv.ParseInt(s, 10, 64) + if err != nil { + w.logs.Warnf("line %s matchWorkerVersion not numb", line) + return false + } + } + return true +} diff --git a/src/agent/agent/src/pkg/config/config_test.go b/src/agent/agent/src/third_components/worker_test.go similarity index 92% rename from src/agent/agent/src/pkg/config/config_test.go rename to src/agent/agent/src/third_components/worker_test.go index 66e841807bee..903c92d2dccf 100644 --- a/src/agent/agent/src/pkg/config/config_test.go +++ b/src/agent/agent/src/third_components/worker_test.go @@ -1,11 +1,10 @@ -package config +package third_components import ( + "github.com/TencentBlueKing/bk-ci/agentcommon/logs" "os" "reflect" "testing" - - "github.com/TencentBlueKing/bk-ci/agentcommon/logs" ) func Test_parseWorkerVersion(t *testing.T) { @@ -88,10 +87,12 @@ func Test_parseWorkerVersion(t *testing.T) { }, } - for _, tt := range tests { + logs.UNTestDebugInit() + _ = initOb() + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - version := parseWorkerVersion(tt.lines) + version := Worker.parseWorkerVersion(tt.lines) if !reflect.DeepEqual(version, tt.want) { t.Fatalf("Fail: %v = %v, want %v", tt.name, version, tt.want) } @@ -117,10 +118,13 @@ func Test_matchWorkerVersion(t *testing.T) { want: true, }, } + logs.UNTestDebugInit() + _ = initOb() + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := matchWorkerVersion(tt.args.line); got != tt.want { + if got := Worker.matchWorkerVersion(tt.args.line); got != tt.want { t.Errorf("matchWorkerVersion() = %v, want %v", got, tt.want) } }) diff --git a/src/agent/common/utils/slice.go b/src/agent/common/utils/slice.go new file mode 100644 index 000000000000..b0930d08cfb9 --- /dev/null +++ b/src/agent/common/utils/slice.go @@ -0,0 +1,15 @@ +package utils + +// IsStringSliceBlank 检查一个字符串切片是否是空字符 +func IsStringSliceBlank(slice []string) bool { + if len(slice) == 0 { + return true + } + f := true + for _, str := range slice { + if str != "" { + f = false + } + } + return f +} diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/functions/Join.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/functions/Join.kt index fde6806eef56..ab225eadde21 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/functions/Join.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/functions/Join.kt @@ -71,7 +71,7 @@ class Join : Function() { } } - (1 until collection.count).forEach { i -> + for (i in 1 until collection.count()) { // Append the separator memory.add(separator) result.append(separator) diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/ExternalThirdPartyAgentResource.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/ExternalThirdPartyAgentResource.kt index 06b27842e4e4..7451528222be 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/ExternalThirdPartyAgentResource.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/ExternalThirdPartyAgentResource.kt @@ -77,6 +77,7 @@ interface ExternalThirdPartyAgentResource { arch: String? ): Response + @Deprecated("没用了") @Operation(summary = "下载JRE") @GET @Path("/{agentId}/jre") diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/OpThirdPartyAgentUpgradeResource.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/OpThirdPartyAgentUpgradeResource.kt index 9692327c6b84..2563aae8f314 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/OpThirdPartyAgentUpgradeResource.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/OpThirdPartyAgentUpgradeResource.kt @@ -28,6 +28,7 @@ package com.tencent.devops.environment.api.thirdpartyagent import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.environment.pojo.AgentUpgradeType import com.tencent.devops.environment.pojo.thirdpartyagent.JDKInfo import io.swagger.v3.oas.annotations.tags.Tag import io.swagger.v3.oas.annotations.Operation @@ -174,6 +175,8 @@ interface OpThirdPartyAgentUpgradeResource { @PUT @Path("/agents/set_priority_upgrade_projects") fun setPriorityUpgradeAgentProjects( + @QueryParam("type") + type: AgentUpgradeType?, @Parameter(description = "projectIds", required = true) projectIds: List ): Result @@ -182,6 +185,8 @@ interface OpThirdPartyAgentUpgradeResource { @DELETE @Path("/agents/unset_priority_upgrade_projects") fun unsetPriorityUpgradeAgentProjects( + @QueryParam("type") + type: AgentUpgradeType?, @Parameter(description = "projectIds", required = true) projectIds: List ): Result @@ -189,17 +194,25 @@ interface OpThirdPartyAgentUpgradeResource { @Operation(summary = "获取优先升级项目列表(所有）") @GET @Path("/agents/get_all_priority_upgrade_projects") - fun getAllPriorityUpgradeAgentProjects(): Result> + fun getAllPriorityUpgradeAgentProjects( + @QueryParam("type") + type: AgentUpgradeType? + ): Result> @Operation(summary = "取消优先升级项目（所有）") @DELETE @Path("/agents/clean_all_priority_upgrade_projects") - fun cleanAllPriorityUpgradeAgentProjects(): Result + fun cleanAllPriorityUpgradeAgentProjects( + @QueryParam("type") + type: AgentUpgradeType? + ): Result @Operation(summary = "设置禁止升级项目（指定项目）") @PUT @Path("/agents/set_deny_upgrade_projects") fun setDenyUpgradeAgentProjects( + @QueryParam("type") + type: AgentUpgradeType?, @Parameter(description = "projectIds", required = true) projectIds: List ): Result @@ -208,6 +221,8 @@ interface OpThirdPartyAgentUpgradeResource { @DELETE @Path("/agents/unset_deny_upgrade_projects") fun unsetDenyUpgradeAgentProjects( + @QueryParam("type") + type: AgentUpgradeType?, @Parameter(description = "agentIds", required = true) projectIds: List ): Result @@ -215,10 +230,16 @@ interface OpThirdPartyAgentUpgradeResource { @Operation(summary = "获取禁止升级项目列表(所有）") @GET @Path("/agents/get_all_deny_upgrade_projects") - fun getAllDenyUpgradeAgentProjects(): Result> + fun getAllDenyUpgradeAgentProjects( + @QueryParam("type") + type: AgentUpgradeType? + ): Result> @Operation(summary = "移除禁止升级项目（所有）") @DELETE @Path("/agents/clean_all_deny_upgrade_projects") - fun cleanAllDenyUpgradeAgentProjects(): Result + fun cleanAllDenyUpgradeAgentProjects( + @QueryParam("type") + type: AgentUpgradeType? + ): Result } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdpartyagent/OpThirdPartyAgentUpgradeResourceImpl.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdpartyagent/OpThirdPartyAgentUpgradeResourceImpl.kt index a71ae1489056..e7c488ed86a7 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdpartyagent/OpThirdPartyAgentUpgradeResourceImpl.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdpartyagent/OpThirdPartyAgentUpgradeResourceImpl.kt @@ -145,47 +145,51 @@ class OpThirdPartyAgentUpgradeResourceImpl @Autowired constructor( ) } - override fun setPriorityUpgradeAgentProjects(projectIds: List): Result { + override fun setPriorityUpgradeAgentProjects(type: AgentUpgradeType?, projectIds: List): Result { return projectScope.setUpgradeProjects( upgradeKey = ProjectScope.UpgradeKey.PRIORITY_PROJECT, - projectIds = projectIds.filter { it.isNotBlank() }.toSet() + projectIds = projectIds.filter { it.isNotBlank() }.toSet(), + type = type ) } - override fun unsetPriorityUpgradeAgentProjects(projectIds: List): Result { + override fun unsetPriorityUpgradeAgentProjects(type: AgentUpgradeType?, projectIds: List): Result { return projectScope.unsetUpgradeProjects( upgradeKey = ProjectScope.UpgradeKey.PRIORITY_PROJECT, - projectIds = projectIds.filter { it.isNotBlank() }.toSet() + projectIds = projectIds.filter { it.isNotBlank() }.toSet(), + type = type ) } - override fun getAllPriorityUpgradeAgentProjects(): Result> { - return Result(projectScope.getAllUpgradeProjects(ProjectScope.UpgradeKey.PRIORITY_PROJECT)) + override fun getAllPriorityUpgradeAgentProjects(type: AgentUpgradeType?): Result> { + return Result(projectScope.getAllUpgradeProjects(ProjectScope.UpgradeKey.PRIORITY_PROJECT, type)) } - override fun cleanAllPriorityUpgradeAgentProjects(): Result { - return Result(projectScope.cleanAllUpgradeProjects(ProjectScope.UpgradeKey.PRIORITY_PROJECT)) + override fun cleanAllPriorityUpgradeAgentProjects(type: AgentUpgradeType?): Result { + return Result(projectScope.cleanAllUpgradeProjects(ProjectScope.UpgradeKey.PRIORITY_PROJECT, type)) } - override fun setDenyUpgradeAgentProjects(projectIds: List): Result { + override fun setDenyUpgradeAgentProjects(type: AgentUpgradeType?, projectIds: List): Result { return projectScope.setUpgradeProjects( upgradeKey = ProjectScope.UpgradeKey.DENY_PROJECT, - projectIds = projectIds.filter { it.isNotBlank() }.toSet() + projectIds = projectIds.filter { it.isNotBlank() }.toSet(), + type = type ) } - override fun unsetDenyUpgradeAgentProjects(projectIds: List): Result { + override fun unsetDenyUpgradeAgentProjects(type: AgentUpgradeType?, projectIds: List): Result { return projectScope.unsetUpgradeProjects( upgradeKey = ProjectScope.UpgradeKey.DENY_PROJECT, - projectIds = projectIds.filter { it.isNotBlank() }.toSet() + projectIds = projectIds.filter { it.isNotBlank() }.toSet(), + type = type ) } - override fun getAllDenyUpgradeAgentProjects(): Result> { - return Result(projectScope.getAllUpgradeProjects(ProjectScope.UpgradeKey.DENY_PROJECT)) + override fun getAllDenyUpgradeAgentProjects(type: AgentUpgradeType?): Result> { + return Result(projectScope.getAllUpgradeProjects(ProjectScope.UpgradeKey.DENY_PROJECT, type)) } - override fun cleanAllDenyUpgradeAgentProjects(): Result { - return Result(projectScope.cleanAllUpgradeProjects(ProjectScope.UpgradeKey.DENY_PROJECT)) + override fun cleanAllDenyUpgradeAgentProjects(type: AgentUpgradeType?): Result { + return Result(projectScope.cleanAllUpgradeProjects(ProjectScope.UpgradeKey.DENY_PROJECT, type)) } } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/AgentUpgradeJob.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/AgentUpgradeJob.kt index 22a61ce5ceb0..ff4436452aba 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/AgentUpgradeJob.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/AgentUpgradeJob.kt @@ -117,7 +117,9 @@ class AgentUpgradeJob @Autowired constructor( ).toMutableSet() // 对于优先升级的项目的 agent 也一并计入并且放到前面 - val upgrades = projectScope.fetchInPriorityUpgradeProject() + val upgrades = mutableSetOf() + upgrades.addAll(projectScope.fetchInPriorityUpgradeProject(AgentUpgradeType.GO_AGENT)) + upgrades.addAll(projectScope.fetchInPriorityUpgradeProject(AgentUpgradeType.WORKER)) if (upgrades.isNotEmpty()) { val upImportOKAgents = thirdPartyAgentDao.listByStatusAndProject( dslContext = dslContext, @@ -131,7 +133,7 @@ class AgentUpgradeJob @Autowired constructor( val needUpgradeAgents = importOKAgents.filter { // #5806 #5045 解决worker过老，或者异常，导致拿不到版本号，而无法自愈或升级的问题 // it.version.isNullOrBlank() || it.masterVersion.isNullOrBlank() -> false - if (checkProjectRouter(it.projectId) && checkProjectUpgrade(it.projectId)) { + if (checkProjectRouter(it.projectId)) { checkCanUpgrade( goAgentCurrentVersion = currentMasterVersion, workCurrentVersion = currentVersion, @@ -161,6 +163,10 @@ class AgentUpgradeJob @Autowired constructor( record: TEnvironmentThirdpartyAgentRecord ): Boolean { AgentUpgradeType.values().forEach { type -> + // 校验这个项目下的这个类型是否可以升级 + if (!checkProjectUpgrade(record.projectId, type)) { + return@forEach + } val res = when (type) { AgentUpgradeType.GO_AGENT -> { goAgentCurrentVersion.trim() != record.masterVersion.trim() @@ -174,7 +180,9 @@ class AgentUpgradeJob @Autowired constructor( val props = agentPropsScope.parseAgentProps(record.agentProps) ?: return@forEach val currentJdkVersion = agentPropsScope.getJdkVersion(record.os, props.arch)?.ifBlank { null } ?: return@forEach - if (props.jdkVersion.size > 2) { + if (props.jdkVersion.isEmpty()) { + true + } else if (props.jdkVersion.size > 2) { currentJdkVersion.trim() != props.jdkVersion.last().trim() } else { false @@ -201,16 +209,15 @@ class AgentUpgradeJob @Autowired constructor( } /** - * 校验这个agent所属的项目是否可以进行升级或者其他属性 + * 校验这个agent所属的项目是否可以进行升级或者其他属性，只支持worker和agent设置，除worker外的类型都和agent设置走 * @return true 可以升级 false 不能进行升级 */ - private fun checkProjectUpgrade(projectId: String): Boolean { + private fun checkProjectUpgrade(projectId: String, type: AgentUpgradeType): Boolean { // 校验不升级项目，这些项目不参与Agent升级 - if (projectScope.checkDenyUpgradeProject(projectId)) { + if (projectScope.checkDenyUpgradeProject(projectId, type)) { return false } - // 校验是否在优先升级的项目列表中，如果不在里面并且优先升级项目的列表为空也允许Agent升级。 - return projectScope.checkInPriorityUpgradeProjectOrEmpty(projectId) + return projectScope.checkInPriorityUpgradeProjectOrEmpty(projectId, type) } } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/DownloadAgentInstallService.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/DownloadAgentInstallService.kt index 1aad8272f746..116b49a1af85 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/DownloadAgentInstallService.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/DownloadAgentInstallService.kt @@ -52,6 +52,7 @@ import java.io.File import java.io.FileInputStream import java.io.FileNotFoundException import java.nio.charset.Charset +import java.util.Locale import javax.ws.rs.NotFoundException import javax.ws.rs.core.MediaType import javax.ws.rs.core.Response @@ -107,7 +108,7 @@ class DownloadAgentInstallService @Autowired constructor( logger.warn("The install script file(${scriptFile.absolutePath}) is not exist") throw FileNotFoundException("The install script file is not exist") } - val map = getAgentReplaceProperties(agentRecord) + val map = getAgentReplaceProperties(agentRecord, true) var result = scriptFile.readText(Charset.forName("UTF-8")) map.forEach { (t, u) -> @@ -207,8 +208,10 @@ class DownloadAgentInstallService @Autowired constructor( private fun getGoAgentJarFiles(os: String, arch: AgentArchType?): List { val agentJar = getAgentJarFile() - val jreFile = getJreZipFile(os, arch) - return listOf(agentJar, jreFile) + // #10586 现阶段保持 8 和 17并行，直到没有 8 的使用 + val jreFile = getJreZipFile(os, arch, JDK8_FILENAME) + val jdk17File = getJreZipFile(os, arch, JDK17_FILENAME) + return listOf(agentJar, jreFile, jdk17File) } private fun getGoFile(os: String, fileName: String, arch: AgentArchType?): File { @@ -229,10 +232,10 @@ class DownloadAgentInstallService @Autowired constructor( return daemonFile } - private fun getGoAgentScriptFiles(agentRecord: TEnvironmentThirdpartyAgentRecord): Map { - val file = File(agentPackage, "script/${agentRecord.os.toLowerCase()}") + private fun getGoAgentScriptFiles(agentRecord: TEnvironmentThirdpartyAgentRecord): Map { + val file = File(agentPackage, "script/${agentRecord.os.lowercase()}") val scripts = file.listFiles() - val map = getAgentReplaceProperties(agentRecord) + val map = getAgentReplaceProperties(agentRecord, false) return scripts?.associate { var content = it.readText(Charsets.UTF_8) map.forEach { (key, value) -> content = content.replace("##$key##", value) } @@ -242,7 +245,7 @@ class DownloadAgentInstallService @Autowired constructor( private fun getPropertyFile(agentRecord: TEnvironmentThirdpartyAgentRecord): Map { val file = File(agentPackage, "config").listFiles() - val map = getAgentReplaceProperties(agentRecord) + val map = getAgentReplaceProperties(agentRecord, false) return file?.filter { it.isFile }?.associate { var content = it.readText(Charsets.UTF_8) map.forEach { (key, value) -> content = content.replace("##$key##", value) } @@ -253,7 +256,7 @@ class DownloadAgentInstallService @Autowired constructor( fun downloadJre(agentId: String, eTag: String?, arch: AgentArchType?): Response { logger.info("downloadJre, agentId: $agentId, eTag: $eTag") val record = getAgentRecord(agentId) - val file = getJreZipFile(record.os, arch) + val file = getJreZipFile(record.os, arch, JDK17_FILENAME) if (!eTag.isNullOrBlank()) { if (eTag == getFileMD5(file)) { @@ -275,7 +278,10 @@ class DownloadAgentInstallService @Autowired constructor( return agentRecord } - private fun getAgentReplaceProperties(agentRecord: TEnvironmentThirdpartyAgentRecord): Map { + private fun getAgentReplaceProperties( + agentRecord: TEnvironmentThirdpartyAgentRecord, + enableCheckFiles: Boolean + ): Map { val agentId = HashUtil.encodeLongId(agentRecord.id) val agentUrl = agentUrlService.genAgentUrl(agentRecord) val gateWay = agentUrlService.genGateway(agentRecord) @@ -289,7 +295,8 @@ class DownloadAgentInstallService @Autowired constructor( "fileGateway" to fileGateway, "landun.env" to profile.getEnv().name, "agentCollectorOn" to agentCollectorOn, - "language" to commonConfig.devopsDefaultLocaleLanguage + "language" to commonConfig.devopsDefaultLocaleLanguage, + "enableCheckFiles" to enableCheckFiles.toString() ) } @@ -302,13 +309,13 @@ class DownloadAgentInstallService @Autowired constructor( return agentJar } - fun getJreZipFile(os: String, arch: AgentArchType?): File { + fun getJreZipFile(os: String, arch: AgentArchType?, fileName: String): File { val archStr = if (arch == null) { "" } else { "_${arch.arch}" } - val file = File(agentPackage, "/jre/${os.toLowerCase()}$archStr/jre.zip") + val file = File(agentPackage, "/jre/${os.lowercase(Locale.getDefault())}$archStr/$fileName") if (!file.exists()) { logger.warn("The jre file(${file.absolutePath}) is not exist") throw FileNotFoundException("The jre file is not exist") @@ -426,5 +433,8 @@ class DownloadAgentInstallService @Autowired constructor( private val logger = LoggerFactory.getLogger(DownloadAgentInstallService::class.java) private const val AGENT_FILE_MODE = 0b111101101 private const val CERT_FILE_NAME = ".cert" + + private const val JDK8_FILENAME = "jre.zip" + private const val JDK17_FILENAME = "jdk17.zip" } } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ThirdPartyAgentMgrService.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ThirdPartyAgentMgrService.kt index 8be997225f02..e1ce2713e176 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ThirdPartyAgentMgrService.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ThirdPartyAgentMgrService.kt @@ -1081,7 +1081,7 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( masterVersion = startInfo.masterVersion ) if (updateCount != 1) { - logger.warn("Fail to update the agent info($updateCount)") + logger.warn("Fail to update the agent($id) info($updateCount)") } if (agentRecord.status == AgentStatus.IMPORT_EXCEPTION.status || @@ -1090,21 +1090,28 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( thirdPartAgentService.addAgentAction(projectId, id, AgentAction.ONLINE) } - dslContext.transactionResult { configuration -> - val context = DSL.using(configuration) - if (agentRecord.nodeId != null) { - val nodeRecord = nodeDao.get(context, projectId, agentRecord.nodeId) - if (nodeRecord != null && (nodeRecord.nodeIp != startInfo.hostIp || - nodeRecord.nodeStatus == NodeStatus.ABNORMAL.name) - ) { - nodeRecord.nodeStatus = NodeStatus.NORMAL.name - nodeRecord.nodeIp = startInfo.hostIp - nodeRecord.agentVersion = startInfo.masterVersion - nodeDao.saveNode(context, nodeRecord) - webSocketDispatcher.dispatch( - websocketService.buildDetailMessage(projectId, "") - ) - } + if (agentRecord.nodeId == null) { + return status + } + + // 更新node表信息 + val nodeRecord = nodeDao.get(dslContext, projectId, agentRecord.nodeId) ?: return status + if (nodeRecord.nodeIp != startInfo.hostIp || nodeRecord.nodeStatus == NodeStatus.ABNORMAL.name) { + nodeRecord.nodeStatus = NodeStatus.NORMAL.name + nodeRecord.nodeIp = startInfo.hostIp + nodeRecord.agentVersion = startInfo.masterVersion + nodeDao.saveNode(dslContext, nodeRecord) + webSocketDispatcher.dispatch( + websocketService.buildDetailMessage(projectId, "") + ) + } else { + // 在IP没变的情况下版本可能变化，这里单独更新下版本 + if (!startInfo.masterVersion.isNullOrBlank() && nodeRecord.agentVersion != startInfo.masterVersion) { + nodeDao.updateDevopsAgentVersionByNodeId( + dslContext = dslContext, + nodeId = agentRecord.nodeId, + agentVersion = startInfo.masterVersion!! + ) } } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/UpgradeService.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/UpgradeService.kt index af6146e2a260..f6f6139a2399 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/UpgradeService.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/UpgradeService.kt @@ -70,7 +70,9 @@ class UpgradeService @Autowired constructor( return AgentResult(status, false) } - if (!checkProjectUpgrade(projectId)) { + if (!checkProjectUpgrade(projectId, AgentUpgradeType.WORKER) && + !checkProjectUpgrade(projectId, AgentUpgradeType.GO_AGENT) + ) { return AgentResult(status = AgentStatus.IMPORT_OK, data = false) } @@ -120,13 +122,6 @@ class UpgradeService @Autowired constructor( ) } - if (!checkProjectUpgrade(projectId)) { - return AgentResult( - AgentStatus.IMPORT_OK, - UpgradeItem(agent = false, worker = false, jdk = false, dockerInitFile = false) - ) - } - val currentWorkerVersion = agentPropsScope.getWorkerVersion() val currentGoAgentVersion = agentPropsScope.getAgentVersion() val currentJdkVersion = agentPropsScope.getJdkVersion(os, props?.arch) @@ -146,6 +141,7 @@ class UpgradeService @Autowired constructor( } val workerVersion = when { + !checkProjectUpgrade(projectId, AgentUpgradeType.WORKER) -> false currentWorkerVersion.isBlank() -> { logger.warn("The current agent version is not exist") false @@ -156,7 +152,10 @@ class UpgradeService @Autowired constructor( else -> canUpgrade && (info.workerVersion.isNullOrBlank() || (currentWorkerVersion != info.workerVersion)) } + val agentProjectCheck = checkProjectUpgrade(projectId, AgentUpgradeType.WORKER) + val goAgentVersion = when { + !agentProjectCheck -> false currentGoAgentVersion.isBlank() -> { logger.warn("The current agent master version is not exist") false @@ -169,6 +168,7 @@ class UpgradeService @Autowired constructor( } val jdkVersion = when { + !agentProjectCheck -> false currentJdkVersion.isNullOrBlank() -> { logger.warn("project: $projectId|agent: $agentId|os: $os|arch: ${props?.arch}|current jdk is null") false @@ -183,6 +183,7 @@ class UpgradeService @Autowired constructor( } val dockerInitFile = when { + !agentProjectCheck -> false info.dockerInitFileInfo == null -> false // 目前存在非linux系统的不支持，旧数据或agent不使用docker构建机，所以不校验升级 info.dockerInitFileInfo?.needUpgrade != true -> false @@ -232,13 +233,13 @@ class UpgradeService @Autowired constructor( * 校验这个agent所属的项目是否可以进行升级或者其他属性 * @return true 可以升级 false 不能进行升级 */ - private fun checkProjectUpgrade(projectId: String): Boolean { + private fun checkProjectUpgrade(projectId: String, type: AgentUpgradeType?): Boolean { // 校验不升级项目，这些项目不参与Agent升级 - if (projectScope.checkDenyUpgradeProject(projectId)) { + if (projectScope.checkDenyUpgradeProject(projectId, type)) { return false } // 校验是否在优先升级的项目列表中，如果不在里面并且优先升级项目的列表为空也允许Agent升级。 - return projectScope.checkInPriorityUpgradeProjectOrEmpty(projectId) + return projectScope.checkInPriorityUpgradeProjectOrEmpty(projectId, type) } } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/upgrade/ProjectScope.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/upgrade/ProjectScope.kt index 7585e2c918f0..967d9ecfa682 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/upgrade/ProjectScope.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/upgrade/ProjectScope.kt @@ -4,6 +4,7 @@ import com.github.benmanes.caffeine.cache.Caffeine import com.github.benmanes.caffeine.cache.LoadingCache import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.environment.pojo.AgentUpgradeType import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component @@ -19,28 +20,40 @@ class ProjectScope @Autowired constructor(private val redisOperation: RedisOpera private val success = Result(true) - private fun getKeyType(upgradeKey: UpgradeKey): String = - when (upgradeKey) { + private fun getKeyType(upgradeKey: UpgradeKey, type: AgentUpgradeType?): String { + if (type == AgentUpgradeType.WORKER) { + when (upgradeKey) { + UpgradeKey.PRIORITY_PROJECT -> WORKER_PRIORITY_UPGRADE_PROJECT_SET // 优先升级项目 + UpgradeKey.DENY_PROJECT -> WORKER_DENY_UPGRADE_PROJECT_SET // 禁止升级项目 + } + } + // 默认全走Agent + return when (upgradeKey) { UpgradeKey.PRIORITY_PROJECT -> AGENT_PRIORITY_UPGRADE_PROJECT_SET // 优先升级项目 UpgradeKey.DENY_PROJECT -> AGENT_DENY_UPGRADE_PROJECT_SET // 禁止升级项目 } + } /** * 如果[projectId]在优先升级列表[AGENT_PRIORITY_UPGRADE_PROJECT_SET]中，返回true * 或者[AGENT_PRIORITY_UPGRADE_PROJECT_SET]为空，表示未设置任何优先升级项目，也返回true */ - fun checkInPriorityUpgradeProjectOrEmpty(projectId: String): Boolean { - val cache = fetchInPriorityUpgradeProject() + fun checkInPriorityUpgradeProjectOrEmpty(projectId: String, type: AgentUpgradeType?): Boolean { + val cache = fetchInPriorityUpgradeProject(type) return cache.isEmpty() || cache.contains(projectId) } // 获取升级列表的项目，方便早点升级 - fun fetchInPriorityUpgradeProject(): Set { - return loadCache(getKeyType(UpgradeKey.PRIORITY_PROJECT)) + fun fetchInPriorityUpgradeProject(type: AgentUpgradeType?): Set { + return loadCache(getKeyType(UpgradeKey.PRIORITY_PROJECT, type)) } - fun setUpgradeProjects(upgradeKey: UpgradeKey, projectIds: Set): Result { - val cacheKey = getKeyType(upgradeKey) + fun setUpgradeProjects( + upgradeKey: UpgradeKey, + projectIds: Set, + type: AgentUpgradeType? + ): Result { + val cacheKey = getKeyType(upgradeKey, type) logger.info("setUpgradeProjects_$upgradeKey| $projectIds") val failList = mutableSetOf() projectIds.forEach { projectId -> @@ -60,8 +73,12 @@ class ProjectScope @Autowired constructor(private val redisOperation: RedisOpera return success } - fun unsetUpgradeProjects(upgradeKey: UpgradeKey, projectIds: Set): Result { - val cacheKey = getKeyType(upgradeKey) + fun unsetUpgradeProjects( + upgradeKey: UpgradeKey, + projectIds: Set, + type: AgentUpgradeType? + ): Result { + val cacheKey = getKeyType(upgradeKey, type) logger.info("unsetUpgradeProjects_$upgradeKey| $projectIds") val failList = mutableSetOf() projectIds.forEach { projectId -> @@ -80,17 +97,20 @@ class ProjectScope @Autowired constructor(private val redisOperation: RedisOpera return success } - fun getAllUpgradeProjects(upgradeKey: UpgradeKey) = loadCache(getKeyType(upgradeKey)) + fun getAllUpgradeProjects(upgradeKey: UpgradeKey, type: AgentUpgradeType?) = loadCache(getKeyType(upgradeKey, type)) - fun cleanAllUpgradeProjects(upgradeKey: UpgradeKey): Boolean { - val key = getKeyType(upgradeKey) + fun cleanAllUpgradeProjects(upgradeKey: UpgradeKey, type: AgentUpgradeType?): Boolean { + val key = getKeyType(upgradeKey, type) val result = redisOperation.delete(key, isDistinguishCluster = true) invalidateCache(key) return result } - fun checkDenyUpgradeProject(projectId: String): Boolean { - return loadCache(AGENT_DENY_UPGRADE_PROJECT_SET).contains(projectId) + fun checkDenyUpgradeProject(projectId: String, type: AgentUpgradeType?): Boolean { + return when (type) { + AgentUpgradeType.WORKER -> loadCache(WORKER_DENY_UPGRADE_PROJECT_SET).contains(projectId) + else -> loadCache(AGENT_DENY_UPGRADE_PROJECT_SET).contains(projectId) + } } private fun loadCache(key: String): Set = projectCache.get(key) ?: setOf() @@ -115,6 +135,8 @@ class ProjectScope @Autowired constructor(private val redisOperation: RedisOpera private const val AGENT_PRIORITY_UPGRADE_PROJECT_SET = "environment:thirdparty:agent.priority.upgrade.project" private const val AGENT_DENY_UPGRADE_PROJECT_SET = "environment:thirdparty:agent.deny.upgrade.project" + private const val WORKER_PRIORITY_UPGRADE_PROJECT_SET = "environment:thirdparty:worker.priority.upgrade.project" + private const val WORKER_DENY_UPGRADE_PROJECT_SET = "environment:thirdparty:worker.deny.upgrade.project" private val logger = LoggerFactory.getLogger(ProjectScope::class.java) } diff --git a/support-files/agent-package/script/linux/install.sh b/support-files/agent-package/script/linux/install.sh index e6edf18abc81..83c2749f05cd 100644 --- a/support-files/agent-package/script/linux/install.sh +++ b/support-files/agent-package/script/linux/install.sh @@ -1,9 +1,19 @@ #!/bin/bash -echo "Start installing the agent..." -t=`date +"%Y-%m-%d_%H-%M-%S"` -workspace=`pwd` -user=${USER} -agent_id='##agentId##' +function checkFiles() +{ + if [[ "$enable_check_files" == "true" ]]; then + # 检查当前目录是否有文件 + if [ "$(find . -maxdepth 1 -type f | wc -l)" -gt 0 ]; then + echo "fatal: current directory is not empty, please install in an empty directory" + exit 1 + fi + fi +} + +function exists() +{ + command -v "$1" >/dev/null 2>&1 +} function initArch() { ARCH=$(uname -m) @@ -22,17 +32,18 @@ function getServiceName() function unzip_jdk() { - echo "start unzipping jdk package" + echo "start unzipping jdk17(jdk17.zip) package" + if [[ -d "jdk17" ]]; then + echo "jdk17 already exists, skip unzip" + else + unzip -q -o jdk17.zip -d jdk17 + fi + echo "start unzipping jdk(jre.zip) package" if [[ -d "jdk" ]]; then echo "jdk already exists, skip unzip" - return + else + unzip -q -o jre.zip -d jdk fi - unzip -q -o jre.zip -d jdk -} - -exists() -{ - command -v "$1" >/dev/null 2>&1 } function download_agent() @@ -43,94 +54,182 @@ function download_agent() return fi if exists curl; then + echo "download using curl" curl -H "X-DEVOPS-PROJECT-ID: ##projectId##" -o agent.zip "##agent_url##" if [[ $? -ne 0 ]]; then echo "Fail to use curl to download the agent, use wget" wget --header="X-DEVOPS-PROJECT-ID: ##projectId##" -O agent.zip "##agent_url##" fi - elif exists wget; then + elif exists wget; then + echo "download using wget" wget --header="X-DEVOPS-PROJECT-ID: ##projectId##" -O agent.zip "##agent_url##" else - echo "Curl & wget command don't exist, download fail" + echo "curl & wget command don't exist, download fail" exit 1 fi } -function installAgentService() +function installRcLocal() { - echo "install agent service with user ${user}" - grep_result=$(grep "${service_name}" /etc/rc.d/rc.local) + echo "install agent service with rc.local" + grep_result=$(grep "$service_name" /etc/rc.d/rc.local) if test -x "/etc/rc.d/rc.local" ; then if [[ -z "$grep_result" ]]; then - echo "cd ${workspace} && ./devopsDaemon & # ${service_name}" >> /etc/rc.d/rc.local - echo "add to rclocal" + echo "add $service_name to rc.local" + echo "cd $workspace && ./devopsDaemon & # $service_name" >> /etc/rc.d/rc.local else - echo "already add to rclocal" + echo "already add $service_name to rc.local, no repeated install" fi fi + cd $workspace + chmod +x *.sh + ${workspace}/start.sh +} +# no using systemd KillMode, will affect Agent self-upgrade +function installSystemd() +{ + echo "install agent service with systemd" + cat < /etc/systemd/system/${service_name}.service +[Unit] +Description=bkdevops agent +After=network.target + +[Service] +Type=forking +ExecStart=$workspace/start.sh +ExecStop=$workspace/stop.sh +WorkingDirectory=$workspace +PrivateTmp=true +KillMode=none + +[Install] +WantedBy=multi-user.target +EOL cd ${workspace} chmod +x *.sh - ${workspace}/start.sh + systemctl daemon-reload + systemctl enable --now $service_name } -function uninstallAgentService() +function installAgentService() { - echo "uninstall agent service" - grep_result=$(grep "${service_name}" /etc/rc.d/rc.local) - if test -x "/etc/rc.d/rc.local" ; then - if [[ -z "$grep_result" ]]; then - echo "already remove from rclocal" - else - sed -i "/${service_name}/d" "/etc/rc.d/rc.local" - echo "removal done" + if exists systemctl; then + installSystemd + else + installRcLocal + fi + echo "service $service_name has been installed" +} + +function doUninstallRcLocal() +{ + if [[ -x "/etc/rc.d/rc.local" ]]; then + if grep -q "$service_name" "/etc/rc.d/rc.local"; then + sed -i "/${service_name}/d" "/etc/rc.d/rc.local" fi fi +} - cd ${workspace} +function uninstallRcLocal() +{ + echo "uninstall agent service $service_name on rc.local" + doUninstallRcLocal + cd $workspace chmod +x *.sh ${workspace}/stop.sh } +function uninstallSystemd() +{ + echo "uninstall agent service $service_name on systemd" + # compatible with old data + doUninstallRcLocal + if systemctl list-unit-files | grep -q "^${service_name}"; then + if systemctl is-active --quiet $service_name; then + systemctl stop $service_name + fi + if systemctl is-enabled --quiet $service_name; then + systemctl disable $service_name + fi + if systemctl is-failed --quiet $service_name; then + systemctl reset-failed $service_name + fi + fi + local SERVICE_FILE="/etc/systemd/system/${service_name}.service" + if [ -f "$SERVICE_FILE" ]; then + rm -f "$SERVICE_FILE" + fi +} + +function uninstallAgentService() +{ + echo "uninstall agent services $service_name to reinstall" + if exists systemctl; then + uninstallSystemd + else + uninstallRcLocal + fi + echo "service $service_name has been uninstalled" +} + function writeSSHConfig() { - echo "writeSSHConfig" + echo "no need write ssh config" } -# if [[ "${workspace}" = ~ ]]; then -# echo 'agent should not install in root of user home directory' -# echo 'please run install script in an empty directory with full permission' -# exit 1 -# fi +# ---------------------------------- -cd ${workspace} +echo "start installing Agent..." +workspace=`pwd` +echo "install Dir: $workspace" +user=$USER +echo "install User: $user" +agent_id='##agentId##' +echo "AgentId: $agent_id" +enable_check_files='##enableCheckFiles##' +echo "EnableCheckFiles: $enable_check_files" +cd $workspace + +checkFiles + +echo "check if the install package(agent.zip) exists" if [[ ! -f "agent.zip" ]]; then + echo "install package does not exist. start download" initArch + echo "download Agent arch: $ARCH" download_agent + echo "unzip install package(agent.zip)" unzip -o agent.zip +else + echo "agent.zip exist. reinstall, do nothing" fi unzip_jdk os=`uname` -arch1=`uname -m` echo "OS: $os" -echo "ARCH: ${arch1}" +arch1=`uname -m` +echo "ARCH: $arch1" +echo "check java17 version" +jdk17/bin/java -version echo "check java version" jdk/bin/java -version -echo "check and write ssh config" +echo "check if write ssh config" writeSSHConfig service_name=`getServiceName` if [[ "$user" = "root" ]]; then - uninstallAgentService - installAgentService + echo "root, instll agent service, $service_name" + uninstallAgentService + installAgentService else - cd ${workspace} - chmod +x *.sh - ${workspace}/start.sh + echo "no root, only start agent" + cd $workspace + chmod +x *.sh + ${workspace}/start.sh fi diff --git a/support-files/agent-package/script/linux/start.sh b/support-files/agent-package/script/linux/start.sh index 7136a4428bb8..533249adcc06 100644 --- a/support-files/agent-package/script/linux/start.sh +++ b/support-files/agent-package/script/linux/start.sh @@ -1,14 +1,8 @@ #!/bin/bash -workspace=`pwd` -user=${USER} - -echo "current user: ${user}" -echo "agent workdir: ${workspace}" - function isPidExists() { for i in `ps aux | grep -v grep | awk '{print $2}'`;do - if [[ $1 == ${i} ]];then + if [[ $1 == $i ]];then return 0 fi done @@ -17,46 +11,58 @@ function isPidExists() function start() { - if [[ ! -d "jdk" ]]; then - echo "unzipping the jdk " + echo "start unzipping jdk17(jdk17.zip) package" + if [[ -d "jdk17" ]]; then + echo "jdk17 already exists, skip unzip" + else + unzip -q -o jdk17.zip -d jdk17 + fi + echo "start unzipping jdk(jre.zip) package" + if [[ -d "jdk" ]]; then + echo "jdk already exists, skip unzip" + else unzip -q -o jre.zip -d jdk fi if [[ ! -d "${workspace}/workspace" ]]; then - echo "create workspace dir: ${workspace}/workspace" + echo "create agent workspace dir: ${workspace}/workspace" mkdir -p ${workspace}/workspace chmod 777 ${workspace}/workspace fi if [[ ! -d "${workspace}/logs" ]]; then - echo "create logs dir: ${workspace}/logs" + echo "create agent logs dir: ${workspace}/logs" mkdir -p ${workspace}/logs chmod 777 ${workspace}/logs fi + echo "start agent daemon" pid=0 if [[ -f "${workspace}/runtime/daemon.pid" ]]; then pid=`cat ${workspace}/runtime/daemon.pid` fi if isPidExists ${pid}; then - echo "agent daemon is running, pid: $pid" + echo "agent daemon already running, pid: $pid" else - echo "start agent" chmod +x devopsDaemon chmod +x devopsAgent chmod +x *.sh - nohup ${workspace}/devopsDaemon $1> /dev/null 2>&1 & - echo "agent starts" - sleep 2s if [[ -f "${workspace}/runtime/daemon.pid" ]]; then pid=`cat ${workspace}/runtime/daemon.pid` - if isPidExists ${pid}; then + if isPidExists $pid; then echo "agent daemon is running, pid: $pid" fi fi fi } +# ---------------------------------- + +workspace=`pwd` +user=$USER +echo "current user: ${user}" +echo "agent workdir: ${workspace}" + start $1 \ No newline at end of file diff --git a/support-files/agent-package/script/linux/stop.sh b/support-files/agent-package/script/linux/stop.sh index cc197a382141..eebe34cae78e 100644 --- a/support-files/agent-package/script/linux/stop.sh +++ b/support-files/agent-package/script/linux/stop.sh @@ -1,11 +1,8 @@ #!/bin/bash -workspace=`pwd` -user=${USER} - function isPidExists() { for i in `ps aux | grep -v grep | awk '{print $2}'`;do - if [[ $1 == ${i} ]];then + if [[ $1 == $i ]];then return 0 fi done @@ -18,25 +15,35 @@ function stop() if [[ -f "${workspace}/runtime/daemon.pid" ]]; then pid=`cat ${workspace}/runtime/daemon.pid` fi - if isPidExists ${pid}; then + if isPidExists $pid; then echo "stop daemon process" - kill -9 "$pid" - echo "daemon is stopped" + if ! kill -9 "$pid"; then + echo "failed to kill daemon process with PID $pid" + else + echo "daemon is stopped" + fi else echo "daemon is not running, skip" fi - pid=0 if [[ -f "${workspace}/runtime/agent.pid" ]]; then pid=`cat ${workspace}/runtime/agent.pid` fi - if isPidExists ${pid}; then + if isPidExists $pid; then echo "stop agent process" - kill -9 "$pid" - echo "agent is stopped" + if ! kill -9 "$pid"; then + echo "failed to kill agent process with PID $pid" + else + echo "agent is stopped" + fi else echo "agent is not running, skip" fi } +# ---------------------------------- + +workspace=`pwd` +user=$USER + stop diff --git a/support-files/agent-package/script/linux/uninstall.sh b/support-files/agent-package/script/linux/uninstall.sh index cfb3a524ee6d..237d624373de 100644 --- a/support-files/agent-package/script/linux/uninstall.sh +++ b/support-files/agent-package/script/linux/uninstall.sh @@ -1,39 +1,81 @@ #!/bin/bash -echo "start uninstalling the agent..." -t=`date +"%Y-%m-%d_%H-%M-%S"` -workspace=`pwd` -user=${USER} -agent_id='##agentId##' +function exists() +{ + command -v "$1" >/dev/null 2>&1 +} function getServiceName() { echo "devops_agent_"${agent_id} } -function uninstallAgentService() +function doUninstallRcLocal() { - echo "uninstall agent service" - grep_result=$(grep "${service_name}" /etc/rc.d/rc.local) - if test -x "/etc/rc.d/rc.local" ; then - if [[ -z "$grep_result" ]]; then - echo "already remove from rclocal" - else - sed -i "/${service_name}/d" "/etc/rc.d/rc.local" - echo "removal done" + if [[ -x "/etc/rc.d/rc.local" ]]; then + if grep -q "$service_name" "/etc/rc.d/rc.local"; then + sed -i "/${service_name}/d" "/etc/rc.d/rc.local" + fi + fi +} + +function uninstallRcLocal() +{ + echo "uninstall agent service $service_name on rc.local" + doUninstallRcLocal +} + +function uninstallSystemd() +{ + echo "uninstall agent service $service_name on systemd" + # 兼容旧数据 + doUninstallRcLocal + if systemctl list-unit-files | grep -q "^${service_name}"; then + if systemctl is-active --quiet $service_name; then + systemctl stop $service_name + fi + if systemctl is-enabled --quiet $service_name; then + systemctl disable $service_name + fi + if systemctl is-failed --quiet $service_name; then + systemctl reset-failed $service_name fi fi + local SERVICE_FILE="/etc/systemd/system/${service_name}.service" + if [ -f "$SERVICE_FILE" ]; then + rm -f "$SERVICE_FILE" + fi +} - cd ${workspace} +function uninstallAgentService() +{ + echo "uninstall agent service" + if exists systemctl; then + uninstallSystemd + else + uninstallRcLocal + fi + cd $workspace chmod +x *.sh ${workspace}/stop.sh + echo "service $service_name has been uninstalled" } +# ---------------------------------- + +echo "start uninstalling Agent..." +workspace=`pwd` +user=$USER +agent_id='##agentId##' +echo "AgentId: $agent_id" + service_name=`getServiceName` if [[ "$user" = "root" ]]; then - uninstallAgentService + echo "root, uninstall agent service name $service_name" + uninstallAgentService else - cd ${workspace} - chmod +x *.sh - ${workspace}/stop.sh + echo "no root, stop agent" + cd $workspace + chmod +x *.sh + ${workspace}/stop.sh fi \ No newline at end of file diff --git a/support-files/agent-package/script/macos/install.sh b/support-files/agent-package/script/macos/install.sh index 5665876c3c11..863d9449d7b0 100644 --- a/support-files/agent-package/script/macos/install.sh +++ b/support-files/agent-package/script/macos/install.sh @@ -1,11 +1,17 @@ #!/bin/bash -echo "Start installing the agent..." -t=`date +"%Y-%m-%d_%H-%M-%S"` -workspace=`pwd` -user=${USER} -agent_id='##agentId##' +function checkFiles() +{ + if [[ "$enable_check_files" == "true" ]]; then + # 检查当前目录是否有文件 + if [ "$(find . -maxdepth 1 -type f | wc -l)" -gt 0 ]; then + echo "fatal: current directory is not empty, please install in an empty directory" + exit 1 + fi + fi +} -function initArch() { +function initArch() +{ ARCH=$(uname -m) case $ARCH in aarch64) ARCH="arm64";; @@ -22,15 +28,21 @@ function getServiceName() function unzip_jdk() { - echo "start unzipping the jdk package" + echo "start unzipping jdk17 package" + if [[ -d "jdk17" ]]; then + echo "jdk17 already exists, skip unzip" + else + unzip -q -o jdk17.zip -d jdk17 + fi + echo "start unzipping jdk package" if [[ -d "jdk" ]]; then echo "jdk already exists, skip unzip" - return + else + unzip -q -o jre.zip -d jdk fi - unzip -q -o jre.zip -d jdk } -exists() +function exists() { command -v "$1" >/dev/null 2>&1 } @@ -85,55 +97,70 @@ EOF function uninstallAgentService() { + echo "uninstall agent services $service_name to reinstall" if [[ "$user" != "root" && -f ~/Library/LaunchAgents/$(getServiceName).plist ]]; then echo "remove run at load" rm -f ~/Library/LaunchAgents/$(getServiceName).plist fi - - cd ${workspace} + cd $workspace chmod +x *.sh ${workspace}/stop.sh + echo "service $service_name has been uninstalled" } function installAgentService() { + echo "install agent services $service_name" if [[ "$user" != "root" ]]; then - echo "add run at load with user ${user}" + echo "add run at load with user $user" addRunAtLoad fi - - cd ${workspace} + cd $workspace chmod +x *.sh ${workspace}/start.sh + echo "service $service_name has been installed" } function writeSSHConfig() { - echo "writeSSHConfig" + echo "no need write ssh config" } -# if [[ "${workspace}" = ~ ]]; then -# echo 'agent should not install in root of user home directory' -# echo 'please run install script in an empty directory with full permission' -# exit 1 -# fi +# ---------------------------------- + +echo "start installing the agent..." +workspace=`pwd` +echo "install Dir: $workspace" +user=$USER +echo "install User: $user" +agent_id='##agentId##' +echo "AgentId: $agent_id" +enable_check_files='##enableCheckFiles##' +echo "EnableCheckFiles: $enable_check_files" + +cd $workspace -cd ${workspace} +checkFiles initArch download_agent + +echo "unzip install package(agent.zip)" unzip -o agent.zip + unzip_jdk os=`uname` -arch1=`uname -m` echo "OS: $os" -echo "ARCH: ${arch1}" +arch1=`uname -m` +echo "ARCH: $arch1" +echo "check java17 version" +jdk17/Contents/Home/bin/java -version echo "check java version" jdk/Contents/Home/bin/java -version -echo "check and write ssh config" +echo "check if write ssh config" writeSSHConfig uninstallAgentService diff --git a/support-files/agent-package/script/macos/start.sh b/support-files/agent-package/script/macos/start.sh index 7136a4428bb8..3f9c305346c1 100644 --- a/support-files/agent-package/script/macos/start.sh +++ b/support-files/agent-package/script/macos/start.sh @@ -1,14 +1,8 @@ #!/bin/bash -workspace=`pwd` -user=${USER} - -echo "current user: ${user}" -echo "agent workdir: ${workspace}" - function isPidExists() { for i in `ps aux | grep -v grep | awk '{print $2}'`;do - if [[ $1 == ${i} ]];then + if [[ $1 == $i ]];then return 0 fi done @@ -17,19 +11,27 @@ function isPidExists() function start() { - if [[ ! -d "jdk" ]]; then - echo "unzipping the jdk " + echo "start unzipping jdk17(jdk17.zip) package" + if [[ -d "jdk17" ]]; then + echo "jdk17 already exists, skip unzip" + else + unzip -q -o jdk17.zip -d jdk17 + fi + echo "start unzipping jdk(jre.zip) package" + if [[ -d "jdk" ]]; then + echo "jdk already exists, skip unzip" + else unzip -q -o jre.zip -d jdk fi if [[ ! -d "${workspace}/workspace" ]]; then - echo "create workspace dir: ${workspace}/workspace" + echo "create agent workspace dir: ${workspace}/workspace" mkdir -p ${workspace}/workspace chmod 777 ${workspace}/workspace fi if [[ ! -d "${workspace}/logs" ]]; then - echo "create logs dir: ${workspace}/logs" + echo "create agent logs dir: ${workspace}/logs" mkdir -p ${workspace}/logs chmod 777 ${workspace}/logs fi @@ -38,25 +40,28 @@ function start() if [[ -f "${workspace}/runtime/daemon.pid" ]]; then pid=`cat ${workspace}/runtime/daemon.pid` fi - if isPidExists ${pid}; then - echo "agent daemon is running, pid: $pid" + if isPidExists $pid; then + echo "agent daemon already running, pid: $pid" else - echo "start agent" chmod +x devopsDaemon chmod +x devopsAgent chmod +x *.sh - nohup ${workspace}/devopsDaemon $1> /dev/null 2>&1 & - echo "agent starts" - sleep 2s if [[ -f "${workspace}/runtime/daemon.pid" ]]; then pid=`cat ${workspace}/runtime/daemon.pid` - if isPidExists ${pid}; then + if isPidExists $pid; then echo "agent daemon is running, pid: $pid" fi fi fi } +# ---------------------------------- + +workspace=`pwd` +user=${USER} +echo "current user: ${user}" +echo "agent workdir: ${workspace}" + start $1 \ No newline at end of file diff --git a/support-files/agent-package/script/macos/stop.sh b/support-files/agent-package/script/macos/stop.sh index b80e19449932..975964ec89f6 100644 --- a/support-files/agent-package/script/macos/stop.sh +++ b/support-files/agent-package/script/macos/stop.sh @@ -1,7 +1,4 @@ #!/bin/bash -workspace=`pwd` -user=${USER} - function isPidExists() { for i in `ps aux | grep -v grep | awk '{print $2}'`;do @@ -18,7 +15,7 @@ function stop() if [[ -f "${workspace}/runtime/daemon.pid" ]]; then pid=`cat "${workspace}/runtime/daemon.pid"` fi - if isPidExists ${pid}; then + if isPidExists $pid; then echo "stop daemon process" kill -9 "$pid" echo "daemon is stopped" @@ -30,7 +27,7 @@ function stop() if [[ -f "${workspace}/runtime/agent.pid" ]]; then pid=`cat "${workspace}/runtime/agent.pid"` fi - if isPidExists ${pid}; then + if isPidExists $pid; then echo "stop agent process" kill -9 "$pid" echo "agent is stopped" @@ -39,4 +36,9 @@ function stop() fi } +# ---------------------------------- + +workspace=`pwd` +user=${USER} + stop \ No newline at end of file diff --git a/support-files/agent-package/script/macos/uninstall.sh b/support-files/agent-package/script/macos/uninstall.sh index e43218c0a21f..d243ccf6c220 100644 --- a/support-files/agent-package/script/macos/uninstall.sh +++ b/support-files/agent-package/script/macos/uninstall.sh @@ -1,10 +1,4 @@ #!/bin/bash -echo "start uninstalling the agent..." -t=`date +"%Y-%m-%d_%H-%M-%S"` -workspace=`pwd` -user=${USER} -agent_id='##agentId##' - function getServiceName() { echo "devops_agent_"${agent_id} @@ -17,9 +11,17 @@ function uninstallAgentService() rm -f ~/Library/LaunchAgents/$(getServiceName).plist fi - cd ${workspace} + cd $workspace chmod +x *.sh ${workspace}/stop.sh } +# ---------------------------------- + +echo "start uninstalling the agent..." +workspace=`pwd` +user=${USER} +agent_id='##agentId##' +echo "AgentId: $agent_id" + uninstallAgentService \ No newline at end of file diff --git a/support-files/agent-package/script/windows/download_install.ps1 b/support-files/agent-package/script/windows/download_install.ps1 index 051159cb3230..a135acf98383 100644 --- a/support-files/agent-package/script/windows/download_install.ps1 +++ b/support-files/agent-package/script/windows/download_install.ps1 @@ -1,6 +1,13 @@ # 不显示进度条用来提速 $ProgressPreference = 'SilentlyContinue' +# 检查当前目录是否有文件 +$files = Get-ChildItem -File -Force +if ($files.Count -gt 0) { + Write-Host "fatal: current directory is not empty, please install in an empty directory" + exit 1 +} + # 下载agent.zip $Uri = '##agent_url##' $InvalidHeaders = @{ diff --git a/support-files/agent-package/script/windows/install.bat b/support-files/agent-package/script/windows/install.bat index e8c7c6124c85..e06b0a45264c 100644 --- a/support-files/agent-package/script/windows/install.bat +++ b/support-files/agent-package/script/windows/install.bat @@ -8,6 +8,13 @@ set service_name=devops_agent_%agent_id% echo work_dir %work_dir% +IF EXIST jdk17 ( + echo "jdk17 already exists, skip unzip" +) else ( + echo "unzip jdk17" + Call :UnZipFile "%~dp0jdk17\" "%~dp0jdk17.zip" +) + IF EXIST jdk ( echo "jdk already exists, skip unzip" ) else ( diff --git a/support-files/agent-package/script/windows/install_schtasks_for_ui.bat b/support-files/agent-package/script/windows/install_schtasks_for_ui.bat index 3bb2afcad6f6..16ca154093ba 100644 --- a/support-files/agent-package/script/windows/install_schtasks_for_ui.bat +++ b/support-files/agent-package/script/windows/install_schtasks_for_ui.bat @@ -8,6 +8,13 @@ set service_name=devops_agent_%agent_id% echo work_dir %work_dir% +IF EXIST jdk17 ( + echo "jdk17 already exists, skip unzip" +) else ( + echo "unzip jdk17" + Call :UnZipFile "%~dp0jdk17\" "%~dp0jdk17.zip" +) + IF EXIST jdk ( echo "jdk already exists, skip unzip" ) else ( @@ -15,6 +22,7 @@ IF EXIST jdk ( Call :UnZipFile "%~dp0jdk\" "%~dp0jre.zip" ) + mkdir logs mkdir workspace diff --git a/support-files/i18n/worker/message_en_US.properties b/support-files/i18n/worker/message_en_US.properties index c3d66efaae4f..ccafbafd07ed 100644 --- a/support-files/i18n/worker/message_en_US.properties +++ b/support-files/i18n/worker/message_en_US.properties @@ -68,7 +68,7 @@ 2130068=From ({0}) to ({1}), Switch pulls code 2130069=Get Svn directory error 2130070=Parameter error -2130071=To run the agent, you need the write permission of the working directory of the build machine, please check the permissions of the agent running account: {0} +2130071=To run the agent, you need the write permission of the working directory of the build machine and disk write space, please check disk is full or the permissions of the agent running account: {0} 2130072=Unknown error: 2130073=Agent DNS Error 2130074=Agent Network Connect Failed diff --git a/support-files/i18n/worker/message_zh_CN.properties b/support-files/i18n/worker/message_zh_CN.properties index a7145d3359db..6551f5ad6e20 100644 --- a/support-files/i18n/worker/message_zh_CN.properties +++ b/support-files/i18n/worker/message_zh_CN.properties @@ -68,7 +68,7 @@ 2130068=从({0})变为({1}), switch拉取代码 2130069=获取Svn目录错误 2130070=参数错误 -2130071=运行Agent需要构建机工作目录的写权限，请检查Agent运行账号相关权限: {0} +2130071=运行Agent需要构建机工作目录的写权限且拥有磁盘写入空间，请检查磁盘是否已满或Agent运行账号相关权限: {0} 2130072=未知错误: 2130073=构建机DNS解析问题 2130074=构建机网络连接问题