Skip to content

Commit

Permalink
[Backport 7.x] Add core plugins snapshots publishing alongside min sn…
Browse files Browse the repository at this point in the history 
…apshots (#560)

Signed-off-by: Peter Zhu <[email protected]>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
  • Loading branch information
@opensearch-trigger-bot @github-actions
opensearch-trigger-bot[bot] and github-actions[bot] authored Nov 5, 2024
1 parent abea669 commit df62900
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 3 deletions.
5 changes: 4 additions & 1 deletion tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@
uploadMinSnapshotsToS3.library({[email protected], retriever=null})
uploadMinSnapshotsToS3.readYaml({file=tests/data/opensearch-1.3.0.yml})
InputManifest.asBoolean()
uploadMinSnapshotsToS3.readYaml({file=tests/data/tar/builds/opensearch/manifest.yml})
uploadMinSnapshotsToS3.echo(Retreving build manifest from: tests/data/tar/builds/opensearch/manifest.yml)
uploadMinSnapshotsToS3.readYaml({file=tests/data/tar/builds/opensearch/manifest.yml})
BuildManifest.asBoolean()
uploadMinSnapshotsToS3.fileExists(tests/data/tar/builds/opensearch/core-plugins)
uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts)
createSha512Checksums.sh({script=find tests/data/tar/builds/opensearch/dist -type f, returnStdout=true})
createSha512Checksums.echo(Creating sha for opensearch-min-1.3.0-linux-x64.tar.gz)
Expand All @@ -28,7 +30,7 @@ ccc
bbb
ccc
})
uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz)
uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 revision-1.3.0-SNAPSHOT architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz)
uploadMinSnapshotsToS3.sh(
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
Expand All @@ -40,6 +42,7 @@ ccc
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
uploadMinSnapshotsToS3.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
uploadMinSnapshotsToS3.echo(Upload min snapshots)
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz})
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512})
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml})
39 changes: 37 additions & 2 deletions vars/uploadMinSnapshotsToS3.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ void call(Map args = [:]) {

def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: manifest))
String productName = inputManifest.build.getFilename()
String version_plain = inputManifest.build.version
String qualifier = inputManifest.build.qualifier ? '-' + inputManifest.build.qualifier : ''
String revision = version_plain + qualifier + '-SNAPSHOT'
def buildManifestYamlOnly = readYaml(file: "$WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml")
echo("Retreving build manifest from: $WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml")

def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: "$WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml"))
Expand All @@ -41,16 +45,27 @@ void call(Map args = [:]) {
String srcDir = "${WORKSPACE}/${distribution}/builds/${productName}/dist".replace("\\", "/")
String dstDir = "snapshots/core/${productName}/${version}"
String baseName = "${productName}-min-${version}-${platform}-${architecture}"
argsMap = [:]

// Setup core plugins (Tar x64 only)
String corePluginDir = "${WORKSPACE}/${distribution}/builds/${productName}/core-plugins".replace("\\", "/")
boolean corePluginDirExists = fileExists(corePluginDir)
if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {
echo("Create .sha512 for Core Plugins Snapshots")
argsMap['artifactPath'] = corePluginDir
for (Closure action : fileActions) { // running createSha512Checksums()
action(argsMap)
}
}

// Create checksums
echo('Create .sha512 for Min Snapshots Artifacts')
argsMap = [:]
argsMap['artifactPath'] = srcDir
for (Closure action : fileActions) { // running createSha512Checksums()
action(argsMap)
}

echo("Start copying files: version-${version} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}")
echo("Start copying files: version-${version} revision-${revision} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}")

String sedCmd = "sed"
if (platform == "darwin") {
Expand All @@ -68,9 +83,29 @@ void call(Map args = [:]) {
string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'),
string(credentialsId: 'jenkins-artifact-production-bucket-name', variable: 'ARTIFACT_PRODUCTION_BUCKET_NAME')]) {
withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {
// min artifacts
echo("Upload min snapshots")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.sha512")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.build-manifest.yml", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.build-manifest.yml")
// core plugins
if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {
echo("Upload core-plugins snapshots")
List<String> corePluginList = buildManifestYamlOnly.components.artifacts.'core-plugins'[0]
echo("corePluginList: ${corePluginList}")
for (String pluginSubPath : corePluginList) {
String pluginSubFolder = pluginSubPath.split('/')[0]
String pluginNameWithExt = pluginSubPath.split('/')[1]
String pluginName = pluginNameWithExt.replace('-' + revision + '.zip', '')
String pluginFullPath = ['plugins', pluginName, revision].join('/')
s3Upload(
bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}",
path: "snapshots/${pluginFullPath}/",
workingDir: "${corePluginDir}/",
includePathPattern: "**/${pluginName}*"
)
}
}
}
}
}

0 comments on commit df62900

Please sign in to comment.