From 481aa6703760c5b7da1bd3e0c218bd1babc731bc Mon Sep 17 00:00:00 2001 From: Paolo Chila Date: Fri, 17 Jan 2025 08:59:05 +0100 Subject: [PATCH 1/2] Fixed packaging with manifest when manifest has a different version than desired agent packages --- dev-tools/mage/settings.go | 9 ++++----- dev-tools/packaging/packages.yml | 8 ++++---- magefile.go | 14 +++++++------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index 32bc62a3e9..a6e026d35e 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -471,7 +471,7 @@ func BeatQualifiedVersion() (string, error) { return "", err } // version qualifier can intentionally be set to "" to override build time var - if !versionQualified || versionQualifier == "" { + if PackagingFromManifest || !versionQualified || versionQualifier == "" { return version, nil } return version + "-" + versionQualifier, nil @@ -648,11 +648,10 @@ func (s *BuildVariableSources) GetDocBranch() (string, error) { func parseBeatVersion(data []byte) (string, error) { matches := beatVersionRegex.FindSubmatch(data) - if len(matches) == 2 { - return string(matches[1]), nil + if len(matches) != 2 { + return "", errors.New("failed to parse beat version file") } - - return "", errors.New("failed to parse beat version file") + return string(matches[1]) + SnapshotSuffix(), nil } func parseGoVersion(data []byte) (string, error) { diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 8db52f0ca2..5c7d234706 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -281,16 +281,16 @@ shared: 'data/cloud_downloads/metricbeat.sh': source: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/metricbeat.sh' mode: 0755 - 'data/cloud_downloads/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz': - source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + 'data/cloud_downloads/agentbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz': + source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' mode: 0755 # service build is based on previous cloud variant - &agent_docker_service_spec docker_variant: 'service' files: - 'data/service/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip': - source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip' + 'data/service/connectors-{{ beat_version }}.zip': + source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/connectors-{{ beat_version }}.zip' mode: 0755 'data/{{.BeatName}}-{{ commit_short }}/components/connectors': source: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/connectors.sh' diff --git a/magefile.go b/magefile.go index 4bfa58b704..0303ea2dd5 100644 --- a/magefile.go +++ b/magefile.go @@ -1319,13 +1319,13 @@ func downloadManifestAndSetVersion(ctx context.Context, url string) (*manifest.B return nil, nil, fmt.Errorf("parsing manifest version %s: %w", resp.Version, err) } - // When getting the packageVersion from snapshot we should also update the env of SNAPSHOT=true which is - // something that we use as an implicit parameter to various functions - if parsedVersion.IsSnapshot() { - os.Setenv(snapshotEnv, "true") - mage.Snapshot = true - } - os.Setenv("BEAT_VERSION", parsedVersion.CoreVersion()) + //// When getting the packageVersion from snapshot we should also update the env of SNAPSHOT=true which is + //// something that we use as an implicit parameter to various functions + //if parsedVersion.IsSnapshot() { + // os.Setenv(snapshotEnv, "true") + // mage.Snapshot = true + //} + os.Setenv("BEAT_VERSION", resp.Version) return &resp, parsedVersion, nil } From e04bab55600c3854ebb779454133cbaf74c0dc2a Mon Sep 17 00:00:00 2001 From: Paolo Chila Date: Fri, 17 Jan 2025 15:09:34 +0100 Subject: [PATCH 2/2] Fixed packaging non-snapshot agent version using snapshot manifest --- dev-tools/mage/settings.go | 2 +- magefile.go | 50 +++++++++++++++++++++++--------------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index a6e026d35e..886d159401 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -651,7 +651,7 @@ func parseBeatVersion(data []byte) (string, error) { if len(matches) != 2 { return "", errors.New("failed to parse beat version file") } - return string(matches[1]) + SnapshotSuffix(), nil + return string(matches[1]), nil } func parseGoVersion(data []byte) (string, error) { diff --git a/magefile.go b/magefile.go index 0303ea2dd5..fd5d3a26ca 100644 --- a/magefile.go +++ b/magefile.go @@ -504,22 +504,22 @@ func Package(ctx context.Context) error { var err error var manifestResponse *manifest.Build + var dependenciesVersion string if devtools.PackagingFromManifest { - manifestResponse, _, err = downloadManifestAndSetVersion(ctx, devtools.ManifestURL) + var parsedDependenciesVersion *version.ParsedSemVer + manifestResponse, parsedDependenciesVersion, err = downloadManifestAndSetVersion(ctx, devtools.ManifestURL) if err != nil { return fmt.Errorf("failed downloading manifest: %w", err) } - } - - var dependenciesVersion string - if beatVersion, found := os.LookupEnv("BEAT_VERSION"); !found { - dependenciesVersion = bversion.GetDefaultVersion() + dependenciesVersion = parsedDependenciesVersion.Original() } else { - dependenciesVersion = beatVersion + if beatVersion, found := os.LookupEnv("BEAT_VERSION"); !found { + dependenciesVersion = bversion.GetDefaultVersion() + devtools.SnapshotSuffix() + } else { + dependenciesVersion = beatVersion + } } - - packageAgent(ctx, platforms, dependenciesVersion, manifestResponse, mg.F(devtools.UseElasticAgentPackaging), mg.F(CrossBuild), devtools.SelectedPackageTypes) - return nil + return packageAgent(ctx, platforms, dependenciesVersion, manifestResponse, mg.F(devtools.UseElasticAgentPackaging), mg.F(CrossBuild), devtools.SelectedPackageTypes) } // DownloadManifest downloads the provided manifest file into the predefined folder and downloads all components in the manifest. @@ -984,7 +984,7 @@ func packageAgent(ctx context.Context, platforms []string, dependenciesVersion s } // download/copy all the necessary dependencies for packaging elastic-agent - archivePath, dropPath := collectPackageDependencies(platforms, dependenciesVersion, platformPackageSuffixes, packageTypes) + archivePath, dropPath := collectPackageDependencies(ctx, platforms, dependenciesVersion, platformPackageSuffixes, packageTypes) // cleanup after build defer os.RemoveAll(archivePath) @@ -1018,7 +1018,7 @@ func packageAgent(ctx context.Context, platforms []string, dependenciesVersion s // NOTE: after the build is done the caller must: // - delete archivePath and dropPath contents // - unset AGENT_DROP_PATH environment variable -func collectPackageDependencies(platforms []string, packageVersion string, platformPackageSuffixes []string, packageTypes []mage.PackageType) (archivePath string, dropPath string) { +func collectPackageDependencies(ctx context.Context, platforms []string, packageVersion string, platformPackageSuffixes []string, packageTypes []mage.PackageType) (archivePath string, dropPath string) { dropPath, found := os.LookupEnv(agentDropPath) // try not to shadow too many variables @@ -1038,14 +1038,14 @@ func collectPackageDependencies(platforms []string, packageVersion string, platf } archivePath = movePackagesToArchive(dropPath, platformPackageSuffixes, packageVersion) - if hasSnapshotEnv() { - packageVersion = fmt.Sprintf("%s-SNAPSHOT", packageVersion) - } + //if hasSnapshotEnv() { + // packageVersion = fmt.Sprintf("%s-SNAPSHOT", packageVersion) + //} os.Setenv(agentDropPath, dropPath) - if devtools.ExternalBuild == true { - + if devtools.ExternalBuild { + log.Printf("--- Detected EXTERNAL=%v", devtools.ExternalBuild) // Only log fatal logs for logs produced. This is the global logger // used by github.com/elastic/elastic-agent/dev-tools/mage/downloads which can only be configured globally like this. // @@ -1053,23 +1053,33 @@ func collectPackageDependencies(platforms []string, packageVersion string, platf // download artifacts on unsupported platforms and choose to ignore the errors. // // Change this to InfoLevel to see exactly what the downloader is doing. - downloads.LogLevel.Set(downloads.FatalLevel) + downloads.LogLevel.Set(downloads.TraceLevel) - errGroup, ctx := errgroup.WithContext(context.Background()) + errGroup, ctx := errgroup.WithContext(ctx) completedDownloads := &atomic.Int32{} for _, spec := range manifest.ExpectedBinaries { + if mg.Verbose() { + log.Printf("--- Checking spec %s...", spec.ProjectName) + } for _, platform := range platforms { if !spec.SupportsPlatform(platform) { - fmt.Printf("--- Binary %s does not support %s, download skipped\n", spec.BinaryName, platform) + log.Printf("--- Binary %s does not support %s, download skipped", spec.BinaryName, platform) continue } + if mg.Verbose() { + log.Printf("--- Iterating over pkgTypes %s...", packageTypes) + } for _, pkgType := range packageTypes { if !spec.SupportsPackageType(pkgcommon.PackageType(pkgType)) { + log.Printf("--- Package type %s is not supported for spec %s", pkgType, spec.ProjectName) continue } targetPath := filepath.Join(archivePath, manifest.PlatformPackages[platform]) os.MkdirAll(targetPath, 0o755) packageName := spec.GetPackageName(packageVersion, platform) + if mg.Verbose() { + log.Printf("--- Starting download for package %s into %s...", packageName, targetPath) + } errGroup.Go(downloadBinary(ctx, spec.ProjectName, packageName, spec.BinaryName, platform, packageVersion, targetPath, completedDownloads)) } }