diff --git a/build.gradle b/build.gradle index 010d9f302..cb5c8545a 100644 --- a/build.gradle +++ b/build.gradle @@ -120,7 +120,7 @@ jacocoTestReport { } } -String version = '5.5.0' +String version = '5.6.0' task updateVersion { doLast { diff --git a/tests/jenkins/TestPatchDockerImage.groovy b/tests/jenkins/TestPatchDockerImage.groovy new file mode 100644 index 000000000..72341624b --- /dev/null +++ b/tests/jenkins/TestPatchDockerImage.groovy @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test + + +class TestPatchDockerImage extends BuildPipelineTest { + + @Before + void setUp() { + this.registerLibTester(new PatchDockerImageLibTester( + 'opensearch', + '1', + 'true' + ) + ) + super.setUp() + } + + @Test + void testPatchDockerImage() { + + super.testPipeline("tests/jenkins/jobs/PatchDockerImage_Jenkinsfile") + } +} diff --git a/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_builds_both.txt b/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_builds_both.txt index b4e3132d7..b5478400f 100644 --- a/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_builds_both.txt +++ b/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_builds_both.txt @@ -8,12 +8,12 @@ buildDockerImage.library({identifier=jenkins@main, retriever=null}) buildDockerImage.readYaml({file=tests/data/opensearch-1.3.0.yml}) InputManifest.asBoolean() - buildDockerImage.echo(Trigger docker-build) + buildDockerImage.echo(Triggering docker-build) buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY, value=https://github.com/opensearch-project/opensearch-build}) buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY_REFERENCE, value=main}) buildDockerImage.string({name=DOCKER_BUILD_SCRIPT_WITH_COMMANDS, value=id && pwd && cd docker/release && curl -sSL opensearch.linux.x64 -o opensearch-x64.tgz && curl -sSL opensearch.linux.arm64 -o opensearch-arm64.tgz && bash build-image-multi-arch.sh -v 1.3.0 -f ./dockerfiles/opensearch.al2.dockerfile -p opensearch -a 'x64,arm64' -r opensearchstaging/opensearch -t 'opensearch-x64.tgz,opensearch-arm64.tgz' -n 33}) - buildDockerImage.build({job=docker-build, parameters=[null, null, null]}) - buildDockerImage.echo(Trigger docker create tag with build number) - buildDockerImage.echo(Trigger docker-scan for opensearch version 1.3.0) + buildDockerImage.build({job=docker-build, propagate=true, wait=true, parameters=[null, null, null]}) + buildDockerImage.echo(Triggering docker create tag with build number) + buildDockerImage.echo(Triggering docker-scan for opensearch version 1.3.0) buildDockerImage.string({name=IMAGE_FULL_NAME, value=opensearchstaging/opensearch:1.3.0}) - buildDockerImage.build({job=docker-scan, parameters=[null]}) + buildDockerImage.build({job=docker-scan, propagate=true, wait=true, parameters=[null]}) diff --git a/tests/jenkins/jobs/BuildDockerImage_Qualifier_Jenkinsfile_builds_both.txt b/tests/jenkins/jobs/BuildDockerImage_Qualifier_Jenkinsfile_builds_both.txt index 640b3b170..90d29366e 100644 --- a/tests/jenkins/jobs/BuildDockerImage_Qualifier_Jenkinsfile_builds_both.txt +++ b/tests/jenkins/jobs/BuildDockerImage_Qualifier_Jenkinsfile_builds_both.txt @@ -8,12 +8,12 @@ buildDockerImage.library({identifier=jenkins@main, retriever=null}) buildDockerImage.readYaml({file=tests/data/opensearch-2.0.0.yml}) InputManifest.asBoolean() - buildDockerImage.echo(Trigger docker-build) + buildDockerImage.echo(Triggering docker-build) buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY, value=https://github.com/opensearch-project/opensearch-build}) buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY_REFERENCE, value=main}) buildDockerImage.string({name=DOCKER_BUILD_SCRIPT_WITH_COMMANDS, value=id && pwd && cd docker/release && curl -sSL opensearch.linux.x64 -o opensearch-x64.tgz && curl -sSL opensearch.linux.arm64 -o opensearch-arm64.tgz && bash build-image-multi-arch.sh -v 2.0.0-alpha1 -f ./dockerfiles/opensearch.al2.dockerfile -p opensearch -a 'x64,arm64' -r opensearchstaging/opensearch -t 'opensearch-x64.tgz,opensearch-arm64.tgz' -n 33}) - buildDockerImage.build({job=docker-build, parameters=[null, null, null]}) - buildDockerImage.echo(Trigger docker create tag with build number) - buildDockerImage.echo(Trigger docker-scan for opensearch version 2.0.0-alpha1) + buildDockerImage.build({job=docker-build, propagate=true, wait=true, parameters=[null, null, null]}) + buildDockerImage.echo(Triggering docker create tag with build number) + buildDockerImage.echo(Triggering docker-scan for opensearch version 2.0.0-alpha1) buildDockerImage.string({name=IMAGE_FULL_NAME, value=opensearchstaging/opensearch:2.0.0-alpha1}) - buildDockerImage.build({job=docker-scan, parameters=[null]}) + buildDockerImage.build({job=docker-scan, propagate=true, wait=true, parameters=[null]}) diff --git a/tests/jenkins/jobs/PatchDockerImage_Jenkinsfile b/tests/jenkins/jobs/PatchDockerImage_Jenkinsfile new file mode 100644 index 000000000..3b04d4fb7 --- /dev/null +++ b/tests/jenkins/jobs/PatchDockerImage_Jenkinsfile @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pipeline { + agent none + stages { + stage('Patch docker image') { + steps { + script { + patchDockerImage( + product: "opensearch", + tag: "1", + re_release: "true" + ) + } + } + } + } +} \ No newline at end of file diff --git a/tests/jenkins/jobs/PatchDockerImage_Jenkinsfile.txt b/tests/jenkins/jobs/PatchDockerImage_Jenkinsfile.txt new file mode 100644 index 000000000..562d14913 --- /dev/null +++ b/tests/jenkins/jobs/PatchDockerImage_Jenkinsfile.txt @@ -0,0 +1,44 @@ + PatchDockerImage_Jenkinsfile.run() + PatchDockerImage_Jenkinsfile.pipeline(groovy.lang.Closure) + PatchDockerImage_Jenkinsfile.echo(Executing on agent [label:none]) + PatchDockerImage_Jenkinsfile.stage(Patch docker image, groovy.lang.Closure) + PatchDockerImage_Jenkinsfile.script(groovy.lang.Closure) + PatchDockerImage_Jenkinsfile.patchDockerImage({product=opensearch, tag=1, re_release=true}) + patchDockerImage.legacySCM(groovy.lang.Closure) + patchDockerImage.library({identifier=jenkins@main, retriever=null}) + patchDockerImage.sh(#!/bin/bash + set -e + set +x + docker pull opensearchproject/opensearch:1 + docker pull opensearchproject/opensearch:latest + ) + patchDockerImage.sh({script=docker inspect --format '{{ index .Config.Labels "org.label-schema.version"}}' opensearchproject/opensearch:1, returnStdout=true}) + patchDockerImage.sh({script=docker inspect --format '{{ index .Config.Labels "org.label-schema.build-date"}}' opensearchproject/opensearch:1, returnStdout=true}) + patchDockerImage.sh({script=docker inspect --format '{{ index .Config.Labels "org.label-schema.description"}}' opensearchproject/opensearch:1, returnStdout=true}) + patchDockerImage.sh({script=docker inspect --format '{{ index .Config.Labels "org.label-schema.version"}}' opensearchproject/opensearch:latest, returnStdout=true}) + patchDockerImage.readYaml({file=manifests/1.3.0/opensearch-1.3.0.yml}) + InputManifest.asBoolean() + patchDockerImage.buildDockerImage({inputManifest=manifests/1.3.0/opensearch-1.3.0.yml, buildNumber=7756, buildDate=20230619, buildOption=re_release_docker_image, artifactUrlX64=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/7756/linux/x64/tar/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz, artifactUrlArm64=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/7756/linux/arm64/tar/dist/opensearch/opensearch-1.3.0-linux-arm64.tar.gz}) + buildDockerImage.legacySCM(groovy.lang.Closure) + buildDockerImage.library({identifier=jenkins@main, retriever=null}) + buildDockerImage.readYaml({file=manifests/1.3.0/opensearch-1.3.0.yml}) + InputManifest.asBoolean() + buildDockerImage.echo(Triggering docker-build) + buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY, value=https://github.com/opensearch-project/opensearch-build}) + buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY_REFERENCE, value=main}) + buildDockerImage.string({name=DOCKER_BUILD_SCRIPT_WITH_COMMANDS, value=id && pwd && cd docker/release && curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/7756/linux/x64/tar/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz -o opensearch-x64.tgz && curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/7756/linux/arm64/tar/dist/opensearch/opensearch-1.3.0-linux-arm64.tar.gz -o opensearch-arm64.tgz && bash build-image-multi-arch.sh -v 1.3.0 -f ./dockerfiles/opensearch.al2.dockerfile -p opensearch -a 'x64,arm64' -r opensearchstaging/opensearch -t 'opensearch-x64.tgz,opensearch-arm64.tgz' -n 7756}) + buildDockerImage.build({job=docker-build, propagate=true, wait=true, parameters=[null, null, null]}) + buildDockerImage.echo(Triggering docker create tag with build number) + buildDockerImage.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + buildDockerImage.string({name=SOURCE_IMAGE, value=opensearch:1.3.0}) + buildDockerImage.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchstaging}) + buildDockerImage.string({name=DESTINATION_IMAGE, value=opensearch:1.3.0.7756.20230619}) + buildDockerImage.build({job=docker-copy, propagate=true, wait=true, parameters=[null, null, null, null]}) + buildDockerImage.echo(Triggering docker-scan for opensearch version 1.3.0) + buildDockerImage.string({name=IMAGE_FULL_NAME, value=opensearchstaging/opensearch:1.3.0}) + buildDockerImage.build({job=docker-scan, propagate=true, wait=true, parameters=[null]}) + patchDockerImage.echo(Triggering docker-promotion) + patchDockerImage.string({name=SOURCE_IMAGES, value=opensearch:1.3.0.7756.20230619}) + patchDockerImage.string({name=RELEASE_VERSION, value=1.3.0}) + patchDockerImage.booleanParam({name=TAG_LATEST, value=false}) + patchDockerImage.build({job=docker-promotion, propagate=true, wait=true, parameters=[null, null, null]}) diff --git a/tests/jenkins/lib-testers/PatchDockerImageLibtester.groovy b/tests/jenkins/lib-testers/PatchDockerImageLibtester.groovy new file mode 100644 index 000000000..a597e4555 --- /dev/null +++ b/tests/jenkins/lib-testers/PatchDockerImageLibtester.groovy @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +import static org.hamcrest.CoreMatchers.notNullValue +import static org.hamcrest.MatcherAssert.assertThat +import org.yaml.snakeyaml.Yaml + + + +class PatchDockerImageLibTester extends LibFunctionTester { + + private String product + private String tag + private String re_release + + public PatchDockerImageLibTester(product, tag, re_release){ + this.product = product + this.tag = tag + this.re_release = re_release + } + + void configure(helper, binding) { + def inputManifest = "tests/data/opensearch-1.3.0.yml" + binding.setVariable('MANIFEST', inputManifest) + + helper.addShMock("""docker inspect --format '{{ index .Config.Labels "org.label-schema.version"}}' opensearchproject/opensearch:1""") { script -> + return [stdout: "1.3.0", exitValue: 0] + } + helper.addShMock("""docker inspect --format '{{ index .Config.Labels "org.label-schema.description"}}' opensearchproject/opensearch:1""") { script -> + return [stdout: "7756", exitValue: 0] + } + helper.addShMock("""docker inspect --format '{{ index .Config.Labels "org.label-schema.build-date"}}' opensearchproject/opensearch:1""") { script -> + return [stdout: "2023-06-19T19:12:59Z", exitValue: 0] + } + helper.addShMock("""docker inspect --format '{{ index .Config.Labels "org.label-schema.version"}}' opensearchproject/opensearch:latest""") { script -> + return [stdout: "2.5.0", exitValue: 0] + } + helper.registerAllowedMethod('readYaml', [Map.class], { args -> + return new Yaml().load((inputManifest as File).text) + }) + helper.registerAllowedMethod("git", [Map]) + } + + void parameterInvariantsAssertions(call) { + assertThat(call.args.product.first(), notNullValue()) + assertThat(call.args.tag.first(), notNullValue()) + } + + boolean expectedParametersMatcher(call) { + return call.args.product.first().toString().equals(this.product) + && call.args.tag.first().toString().equals(this.tag) + } + + String libFunctionName() { + return 'patchDockerImage' + } +} diff --git a/vars/buildDockerImage.groovy b/vars/buildDockerImage.groovy index 68b2571e3..b30590c51 100644 --- a/vars/buildDockerImage.groovy +++ b/vars/buildDockerImage.groovy @@ -6,11 +6,26 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ + +/** +Library to build Docker Image with different Build Options +@param Map[inputManifest] - Path to Input Manifest. +@param Map[buildNumber] - Build number of the corresponding Artifact. +@param Map[buildDate] - Date on which the artifacts were built. +@param Map[artifactUrlX64] - Url Path to X64 Tarball. +@param Map[artifactUrlARM64] - Url Path to ARM64 Tarball. +@param Map[buildOption] - Build Option for building the image with different options. +*/ void call(Map args = [:]) { def lib = library(identifier: 'jenkins@main', retriever: legacySCM(scm)) def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: args.inputManifest)) def build_qualifier = inputManifest.build.qualifier def build_number = args.buildNumber ?: "${BUILD_NUMBER}" + String image_tag = "" + + if (args.buildDate != null && args.buildDate != 'null'){ + image_tag = "." + "${args.buildDate}" + } if (build_qualifier != null && build_qualifier != 'null') { build_qualifier = "-" + build_qualifier @@ -23,9 +38,11 @@ void call(Map args = [:]) { if (args.artifactUrlX64 == null || args.artifactUrlArm64 == null) { echo 'Skipping docker build, one of x64 or arm64 artifacts was not built.' } else { - echo 'Trigger docker-build' + echo 'Triggering docker-build' dockerBuild: { build job: 'docker-build', + propagate: true, + wait: true, parameters: [ string(name: 'DOCKER_BUILD_GIT_REPOSITORY', value: 'https://github.com/opensearch-project/opensearch-build'), string(name: 'DOCKER_BUILD_GIT_REPOSITORY_REFERENCE', value: 'main'), @@ -50,22 +67,26 @@ void call(Map args = [:]) { ] } - echo 'Trigger docker create tag with build number' - if (args.buildOption == "build_docker_with_build_number_tag") { + echo 'Triggering docker create tag with build number' + if (args.buildOption == "build_docker_with_build_number_tag" || args.buildOption == "re_release_docker_image") { dockerCopy: { build job: 'docker-copy', + propagate: true, + wait: true, parameters: [ string(name: 'SOURCE_IMAGE_REGISTRY', value: 'opensearchstaging'), string(name: 'SOURCE_IMAGE', value: "${filename}:${inputManifest.build.version}${build_qualifier}"), string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchstaging'), - string(name: 'DESTINATION_IMAGE', value: "${filename}:${inputManifest.build.version}${build_qualifier}.${build_number}") + string(name: 'DESTINATION_IMAGE', value: "${filename}:${inputManifest.build.version}${build_qualifier}.${build_number}${image_tag}") ] } } - echo "Trigger docker-scan for ${filename} version ${inputManifest.build.version}${build_qualifier}" + echo "Triggering docker-scan for ${filename} version ${inputManifest.build.version}${build_qualifier}" dockerScan: { build job: 'docker-scan', + propagate: true, + wait: true, parameters: [ string(name: 'IMAGE_FULL_NAME', value: "opensearchstaging/${filename}:${inputManifest.build.version}${build_qualifier}") ] diff --git a/vars/patchDockerImage.groovy b/vars/patchDockerImage.groovy new file mode 100644 index 000000000..9bb0053a5 --- /dev/null +++ b/vars/patchDockerImage.groovy @@ -0,0 +1,95 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** +Library to support Docker Image Re-Release Automation +@param Map[product] - Product type refers to opensearch or opensearch-dashboards. +@param Map[tag] - Tag of the product that needs to be re-released. +@param Map[re_release] - This Build-Option can be checked to release the image after Docker-Build. +*/ +void call(Map args = [:]) { + def lib = library(identifier: 'jenkins@main', retriever: legacySCM(scm)) + String docker_image = "opensearchproject/${args.product}:${args.tag}" + String latest_docker_image = "opensearchproject/${args.product}:latest" + boolean tag_latest = false + String build_option = "build_docker_image" + + sh """#!/bin/bash + set -e + set +x + docker pull ${docker_image} + docker pull ${latest_docker_image} + """ + + def version = sh ( + script: """docker inspect --format '{{ index .Config.Labels "org.label-schema.version"}}' ${docker_image}""", + returnStdout: true + ).trim() + def build_time = sh ( + script: """docker inspect --format '{{ index .Config.Labels "org.label-schema.build-date"}}' ${docker_image}""", + returnStdout: true + ).trim() + def build_number = sh ( + script: """docker inspect --format '{{ index .Config.Labels "org.label-schema.description"}}' ${docker_image}""", + returnStdout: true + ).trim() + def latest_version = sh ( + script: """docker inspect --format '{{ index .Config.Labels "org.label-schema.version"}}' ${latest_docker_image}""", + returnStdout: true + ).trim() + + def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: "manifests/${version}/${args.product}-${version}.yml")) + + artifactUrlX64 = "https://ci.opensearch.org/ci/dbc/distribution-build-${args.product}/${version}/${build_number}/linux/x64/tar/dist/${args.product}/${args.product}-${version}-linux-x64.tar.gz" + + artifactUrlARM64 = "https://ci.opensearch.org/ci/dbc/distribution-build-${args.product}/${version}/${build_number}/linux/arm64/tar/dist/${args.product}/${args.product}-${version}-linux-arm64.tar.gz" + + //slice the build-date value (For Example: 2023-08-11T02:17:43Z -> 20230811) + build_date = build_time[0..3] + build_time[5..6] + build_time[8..9] + + def build_qualifier = inputManifest.build.qualifier + + if (build_qualifier != null && build_qualifier != 'null') { + build_qualifier = "-" + build_qualifier + } + else { + build_qualifier = '' + } + + if (latest_version == version){ + tag_latest = true + } + + if (args.re_release){ + build_option = "re_release_docker_image" + } + + buildDockerImage( + inputManifest: "manifests/${version}/${args.product}-${version}.yml", + buildNumber: "${build_number}", + buildDate: "${build_date}", + buildOption: "${build_option}", + artifactUrlX64: "${artifactUrlX64}", + artifactUrlArm64: "${artifactUrlARM64}" + ) + + echo 'Triggering docker-promotion' + if(args.re_release){ + dockerPromote: { + build job: 'docker-promotion', + propagate: true, + wait: true, + parameters: [ + string(name: 'SOURCE_IMAGES', value: "${args.product}:${inputManifest.build.version}${build_qualifier}.${build_number}.${build_date}"), + string(name: 'RELEASE_VERSION', value: "${version}"), + booleanParam(name: 'TAG_LATEST', value: "${tag_latest}") + ] + } + } +}