From f7cad7b0bc6db3ca03cc0069d88e12154834a8b1 Mon Sep 17 00:00:00 2001 From: Nigel Banks Date: Sun, 4 Dec 2022 08:49:38 +0000 Subject: [PATCH] Complete rewrite to better support tests As well as improve speed and fix caching issues. --- build.gradle.kts | 111 +-- gradle.properties | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- src/main/kotlin/IsleDocker.kt | 651 ------------------ src/main/kotlin/plugins/BuildCtlPlugin.kt | 254 +++++++ src/main/kotlin/plugins/BuildKitPlugin.kt | 415 +++++++++++ .../plugins/CertificateGenerationPlugin.kt | 178 +++++ src/main/kotlin/plugins/IslePlugin.kt | 66 ++ src/main/kotlin/plugins/RegistryPlugin.kt | 158 +++++ src/main/kotlin/plugins/ReportPlugin.kt | 166 +++++ src/main/kotlin/plugins/ReportsPlugin.kt | 128 ++++ src/main/kotlin/plugins/TestPlugin.kt | 291 ++++++++ src/main/kotlin/plugins/TestsPlugin.kt | 46 ++ src/main/kotlin/tasks/BuildCtl.kt | 24 + src/main/kotlin/tasks/DockerBuild.kt | 340 --------- src/main/kotlin/tasks/DockerBuilder.kt | 93 --- src/main/kotlin/tasks/DockerClient.kt | 45 -- src/main/kotlin/tasks/DockerCompose.kt | 212 ------ src/main/kotlin/tasks/DockerContainer.kt | 232 +++---- src/main/kotlin/tasks/DockerNetwork.kt | 60 ++ src/main/kotlin/tasks/DockerPull.kt | 95 ++- src/main/kotlin/tasks/DockerVolume.kt | 55 ++ src/main/kotlin/tasks/Download.kt | 9 +- src/main/kotlin/tasks/GenerateCerts.kt | 67 -- src/main/kotlin/tasks/scan/Grype.kt | 86 --- src/main/kotlin/tasks/scan/GrypeUpdateDB.kt | 36 - src/main/kotlin/tasks/scan/Syft.kt | 46 -- .../kotlin/tasks/tests/DockerComposeTest.kt | 19 - .../kotlin/tasks/tests/DockerContainerTest.kt | 35 - .../tests/ServiceStartsWithDefaultsTest.kt | 38 - src/main/kotlin/utils/DockerCommandOptions.kt | 65 -- src/main/kotlin/utils/ProjectExtensions.kt | 34 - 32 files changed, 2041 insertions(+), 2018 deletions(-) delete mode 100644 src/main/kotlin/IsleDocker.kt create mode 100644 src/main/kotlin/plugins/BuildCtlPlugin.kt create mode 100644 src/main/kotlin/plugins/BuildKitPlugin.kt create mode 100644 src/main/kotlin/plugins/CertificateGenerationPlugin.kt create mode 100644 src/main/kotlin/plugins/IslePlugin.kt create mode 100644 src/main/kotlin/plugins/RegistryPlugin.kt create mode 100644 src/main/kotlin/plugins/ReportPlugin.kt create mode 100644 src/main/kotlin/plugins/ReportsPlugin.kt create mode 100644 src/main/kotlin/plugins/TestPlugin.kt create mode 100644 src/main/kotlin/plugins/TestsPlugin.kt create mode 100644 src/main/kotlin/tasks/BuildCtl.kt delete mode 100644 src/main/kotlin/tasks/DockerBuild.kt delete mode 100644 src/main/kotlin/tasks/DockerBuilder.kt delete mode 100644 src/main/kotlin/tasks/DockerClient.kt delete mode 100644 src/main/kotlin/tasks/DockerCompose.kt create mode 100644 src/main/kotlin/tasks/DockerNetwork.kt create mode 100644 src/main/kotlin/tasks/DockerVolume.kt delete mode 100644 src/main/kotlin/tasks/GenerateCerts.kt delete mode 100644 src/main/kotlin/tasks/scan/Grype.kt delete mode 100644 src/main/kotlin/tasks/scan/GrypeUpdateDB.kt delete mode 100644 src/main/kotlin/tasks/scan/Syft.kt delete mode 100644 src/main/kotlin/tasks/tests/DockerComposeTest.kt delete mode 100644 src/main/kotlin/tasks/tests/DockerContainerTest.kt delete mode 100644 src/main/kotlin/tasks/tests/ServiceStartsWithDefaultsTest.kt delete mode 100644 src/main/kotlin/utils/DockerCommandOptions.kt delete mode 100644 src/main/kotlin/utils/ProjectExtensions.kt diff --git a/build.gradle.kts b/build.gradle.kts index 21297cb..c8a0ee0 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,30 +1,21 @@ -version = "0.11" -group = "com.github.nigelgbanks" - -plugins { - id("com.gradle.plugin-publish") version "0.16.0" - `java-gradle-plugin` - `kotlin-dsl` - // Apply the Kotlin JVM plugin to add support for Kotlin. - id("org.jetbrains.kotlin.jvm") version "1.4.31" - `maven-publish` -} +version = "1.0" +group = "io.github.nigelgbanks" repositories { mavenCentral() gradlePluginPortal() } +plugins { + id("com.gradle.plugin-publish") version "1.1.0" + `java-gradle-plugin` + `kotlin-dsl` +} + dependencies { - // Align versions of all Kotlin components - implementation(platform("org.jetbrains.kotlin:kotlin-bom")) - // Use the Kotlin JDK 8 standard library. - implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") - implementation("com.bmuschko:gradle-docker-plugin:7.1.0") - implementation("org.apache.commons:commons-compress:1.21") implementation("org.apache.commons:commons-io:1.3.2") - implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.1") - implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.1") + implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.14.1") + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.14.1") } java { @@ -33,42 +24,58 @@ java { } gradlePlugin { + website.set("https://github.com/Islandora-Devops/isle-gradle-docker-plugin") + vcsUrl.set("https://github.com/Islandora-Devops/isle-gradle-docker-plugin") + plugins { - create("IsleDocker") { - id = "com.github.nigelgbanks.IsleDocker" - implementationClass = "IsleDocker" + create("Isle") { + id = "io.github.nigelgbanks.Isle" + implementationClass = "plugins.IslePlugin" + displayName = "Isle" + description = "Main gradle plugin for the Islandora Isle project" + tags.set(listOf("isle")) } - } -} - -// The configuration example below shows the minimum required properties -// configured to publish your plugin to the plugin portal -pluginBundle { - website = "https://github.com/Islandora-Devops/isle-gradle-docker-plugin" - vcsUrl = "https://github.com/Islandora-Devops/isle-gradle-docker-plugin" - description = "Gradle plugin that supports building interdependent Docker images with Buildkit support for the Isle project." - tags = listOf("isle", "islandora", "docker") - (plugins) { - "IsleDocker" { - displayName = "Docker build plugin for the Islandora Isle project" + create("IsleBuildCtl") { + id = "io.github.nigelgbanks.IsleBuildCtl" + implementationClass = "plugins.BuildCtlPlugin" + displayName = "IsleBuildCtl" + description = "Wrapper around buildctrl for use with buildkit" + tags.set(listOf("isle")) } - } - mavenCoordinates { - groupId = "com.github.nigelgbanks" - artifactId = "isle-docker-plugins" - version = "0.11" - } -} - -publishing { - repositories { - maven { - name = "GitHubPackages" - url = uri("https://maven.pkg.github.com/${System.getenv("GITHUB_REPOSITORY")}") - credentials { - username = System.getenv("GITHUB_ACTOR") - password = System.getenv("GITHUB_TOKEN") - } + create("IsleBuildKit") { + id = "io.github.nigelgbanks.IsleBuildKit" + implementationClass = "plugins.BuildKitPlugin" + displayName = "IsleBuildKit" + description = "Provides buildkit backend via a Docker container" + tags.set(listOf("isle")) + } + create("IsleCertificateGeneration") { + id = "io.github.nigelgbanks.IsleCertificateGeneration" + implementationClass = "plugins.CertificateGenerationPlugin" + displayName = "IsleCertificateGeneration" + description = "Generates development certificates" + tags.set(listOf("isle")) + } + create("IsleReports") { + id = "io.github.nigelgbanks.IsleReports" + implementationClass = "plugins.ReportsPlugin" + displayName = "IsleReports" + description = "Generates security reports for a single project" + tags.set(listOf("isle")) + } + create("IsleRegistry") { + id = "io.github.nigelgbanks.IsleRegistry" + implementationClass = "plugins.RegistryPlugin" + displayName = "IsleRegistry" + description = "Provides local Docker Registry" + tags.set(listOf("isle")) + } + create("IsleTest") { + id = "io.github.nigelgbanks.IsleTest" + implementationClass = "plugins.TestPlugin" + displayName = "IsleTest" + description = "Perform tests with docker-compose files" + tags.set(listOf("isle")) } } } diff --git a/gradle.properties b/gradle.properties index a851d49..a0c2e02 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ # suppress inspection "UnusedProperty" for whole file org.gradle.parallel=true org.gradle.caching=true -version=0.11-SNAPSHOT +version=1.0-SNAPSHOT diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 8049c68..070cb70 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/kotlin/IsleDocker.kt b/src/main/kotlin/IsleDocker.kt deleted file mode 100644 index 7faee0a..0000000 --- a/src/main/kotlin/IsleDocker.kt +++ /dev/null @@ -1,651 +0,0 @@ -import com.github.dockerjava.api.DockerClient -import com.github.dockerjava.core.DefaultDockerClientConfig -import com.github.dockerjava.core.DockerClientImpl -import com.github.dockerjava.httpclient5.ApacheDockerHttpClient -import org.apache.commons.io.FileUtils -import org.gradle.api.GradleException -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.file.RegularFileProperty -import org.gradle.api.logging.LogLevel -import org.gradle.api.provider.Property -import org.gradle.internal.hash.Hashing -import org.gradle.kotlin.dsl.* -import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform -import tasks.* -import tasks.scan.* -import utils.* -import java.io.ByteArrayInputStream -import java.io.ByteArrayOutputStream -import java.io.File -import java.net.HttpURLConnection -import java.net.URI -import java.net.URL -import java.nio.file.Files.setPosixFilePermissions - -@Suppress("unused") -class IsleDocker : Plugin { - override fun apply(pluginProject: Project): Unit = pluginProject.run { - - val os = DefaultNativePlatform.getCurrentOperatingSystem()!! - val arch = DefaultNativePlatform.getCurrentArchitecture()!! - val isleBuildkitGroup = "isle-buildkit" - - extensions.findByName("buildScan")?.withGroovyBuilder { - setProperty("termsOfServiceUrl", "https://gradle.com/terms-of-service") - setProperty("termsOfServiceAgree", "yes") - } - - // If set to true resources are freed as soon as they are no longer needed. - val isCI: Boolean by extra((properties.getOrDefault("isCI", "false") as String).toBoolean()) - - // Only reports issues that have fixes. - val grypeConfig: File? by extra((properties.getOrDefault("grype.config", null) as String?) - ?.let { path -> - val file = project.file(path) - if (file.exists()) - file - else - null - }) - - // Only reports issues that have fixes. - val grypeOnlyFixed: Boolean by extra( - (properties.getOrDefault( - "grype.only-fixed", - "false" - ) as String).toBoolean() - ) - - // Triggers build to fail if security vulnerability is discovered. - // If unspecified the build will continue regardless. - // Possible values: negligible, low, medium, high, critical. - val grypeFailOnSeverity: String? by extra(properties.getOrDefault("grype.fail-on-severity", null) as String?) - - // The format of reports generated by grype. - // Possible values: table, cyclonedx, json, template. - val grypeFormat: String by extra(properties.getOrDefault("grype.format", "table") as String) - - // The build driver to use. - val buildDriver by extra(properties.getOrDefault("docker.driver", "docker") as String) - val isDockerBuild by extra(buildDriver == "docker") - val isContainerBuild by extra(buildDriver == "docker-container") - - // It's important to note that we’re using a domain containing a "." here, i.e. localhost.domain. - // If it were missing Docker would believe that localhost is a username, as in localhost/ubuntu. - // It would then try to push to the default Central Registry rather than our local repository. - val localRepository = "registry.islandora.dev" - val localRepositoryPort = "5000" - - // The repository to place the images into. - val repository by extra(properties.getOrDefault( - "docker.repository", - if (isDockerBuild) "local" else "${localRepository}:${localRepositoryPort}" - ) as String) - - // Conditionally allows pushing when `docker.driver` is set to `docker`. If we - // are building with "docker-container" or "kubernetes" we must push as we need - // to be able to pull from from the registry when building downstream images. - val pushToRemote by extra( - (properties.getOrDefault("docker.push", "false") as String).toBoolean() - .let { push -> - if (!isDockerBuild) - true - else - push - } - ) - - // The mode to use when populating the registry cache. - @Suppress("unused") - val cacheToMode by extra( - properties.getOrDefault( - "docker.cacheToMode", - if (isDockerBuild) "inline" else "max" - ) as String - ) - - // Enable caching from/to repositories. - @Suppress("unused") - val cacheFromEnabled by extra((properties.getOrDefault("docker.cacheFrom", "true") as String).toBoolean()) - - @Suppress("unused") - val cacheToEnabled by extra((properties.getOrDefault("docker.cacheTo", "false") as String).toBoolean()) - - // Sources to search for images to use as caches when building. - @Suppress("unused") - val cacheFromRepositories by extra( - (properties.getOrDefault("docker.cacheFromRepositories", "") as String) - .split(',') - .filter { it.isNotEmpty() } - .map { it.trim() } - .toSet() - .let { repositories -> - repositories.ifEmpty { - if (cacheToEnabled) { - // Can only cache from repositories in which we have cached to. - setOf("islandora", repository) - } else { - // Always cache to/from islandora. - setOf("islandora") - } - } - } - ) - - // Repositories to push cache to (empty by default). - @Suppress("unused") - val cacheToRepositories by extra( - (properties.getOrDefault("docker.cacheToRepositories", "") as String) - .split(',') - .map { it.trim() } - .filter { it.isNotEmpty() } - .toSet() - .let { repositories -> - repositories.ifEmpty { - setOf(repository) - } - } - ) - - // Optionally disable the build cache as well as the remote cache. - @Suppress("unused") - val noBuildCache by extra((properties.getOrDefault("docker.noCache", false) as String).toBoolean()) - - // Platforms to built images to target. - @Suppress("unused") - val buildPlatforms by extra( - (properties.getOrDefault("docker.platforms", "") as String) - .split(',') - .map { it.trim() } - .filter { it.isNotEmpty() } - .toSet() - ) - - // Never empty if user does not specify it will default to 'latest'. - val dockerTags by extra( - (properties.getOrDefault("docker.tags", "") as String) - .split(',') - .filter { it.isNotEmpty() } - .toSet() - .let { tags -> - tags.ifEmpty { - setOf("latest") - } - } - ) - - // Communicate with docker using Java client API. - @Suppress("unused") - val dockerClient: DockerClient by extra { - val config = DefaultDockerClientConfig.createDefaultConfigBuilder().build() - val httpClient = ApacheDockerHttpClient.Builder() - .dockerHost(config.dockerHost) - .sslConfig(config.sslConfig) - .build() - val dockerClient = DockerClientImpl.getInstance(config, httpClient) - project.gradle.buildFinished { - dockerClient.close() - } - dockerClient - } - - val downloadMkCert by tasks.registering(Download::class) { - val version = "v1.4.4" - fun url(name: String) = "https://github.com/FiloSottile/mkcert/releases/download/${version}/$name" - val (url, sha256) = when { - os.isLinux -> { - Pair( - url("mkcert-${version}-linux-amd64"), - "6d31c65b03972c6dc4a14ab429f2928300518b26503f58723e532d1b0a3bbb52" - ) - } - os.isMacOsX -> { - if (arch.isAmd64) { - Pair( - url("mkcert-${version}-darwin-amd64"), - "a32dfab51f1845d51e810db8e47dcf0e6b51ae3422426514bf5a2b8302e97d4e" - ) - } else { - Pair( - url("mkcert-${version}-darwin-arm64"), - "c8af0df44bce04359794dad8ea28d750437411d632748049d08644ffb66a60c6" - ) - } - } - os.isWindows -> { - Pair( - url("mkcert-${version}-windows-amd64.exe"), - "d2660b50a9ed59eada480750561c96abc2ed4c9a38c6a24d93e30e0977631398" - ) - } - else -> { - throw RuntimeException("Unsupported Platform") - } - } - this.url.set(url) - this.sha256.set(sha256) - doLast { - if (!os.isWindows) { - // Make all downloaded files executable. - val perms = setOf( - java.nio.file.attribute.PosixFilePermission.OWNER_READ, - java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE, - java.nio.file.attribute.PosixFilePermission.GROUP_READ, - java.nio.file.attribute.PosixFilePermission.GROUP_EXECUTE, - java.nio.file.attribute.PosixFilePermission.OTHERS_READ, - java.nio.file.attribute.PosixFilePermission.OTHERS_EXECUTE, - ) - setPosixFilePermissions(dest.get().asFile.toPath(), perms) - } - } - } - - val generateCerts by tasks.registering(GenerateCerts::class) { - executable.set(downloadMkCert.flatMap { it.dest }) - } - - val installBinFmt by tasks.registering { - group = isleBuildkitGroup - description = "Install https://github.com/tonistiigi/binfmt to enable multi-arch builds on Linux." - // Cross building with Qemu is already installed with Docker Desktop, so we only need to install on Linux. - // Additionally, it does not work with non x86_64 hosts. - onlyIf { - isContainerBuild && os.isLinux && arch.isAmd64 - } - doLast { - exec { - commandLine = listOf( - "docker", - "run", - "--rm", - "--privileged", - "tonistiigi/binfmt:qemu-v6.2.0-26", - "--install", "all" - ) - } - } - } - - // Local registry for use with the 'docker-container' driver. - val createLocalRegistry by tasks.registering { - group = isleBuildkitGroup - description = "Creates a local docker docker registry ('docker-container' or 'kubernetes' only)" - onlyIf { !isDockerBuild } - - val volume by extra(objects.property()) - volume.convention("isle-buildkit-registry") - - val network by extra(objects.property()) - network.convention("isle-buildkit") - - val configFile by extra(objects.fileProperty()) - configFile.convention(project.rootProject.layout.buildDirectory.file("config.toml")) - - doLast { - // Create network (allows host DNS name resolution between builder and local registry). - network.get().let { - exec { - commandLine = listOf("docker", "network", "create", it) - isIgnoreExitValue = true // If it already exists it will return non-zero. - } - } - - // Create registry volume. - exec { - commandLine = listOf("docker", "volume", "create", volume.get()) - } - - // Check if the container is already running. - val running = ByteArrayOutputStream().use { output -> - exec { - commandLine = - listOf("docker", "container", "inspect", "-f", "{{.State.Running}}", localRepository) - standardOutput = output - isIgnoreExitValue = true // May not be running. - }.exitValue == 0 && output.toString().trim().toBoolean() - } - // Start the local registry if not already started. - if (!running) { - exec { - commandLine = listOf( - "docker", - "run", - "-d", - "--restart=always", - "--network=isle-buildkit", - "--env", "REGISTRY_HTTP_ADDR=0.0.0.0:${localRepositoryPort}", - "--env", "REGISTRY_STORAGE_DELETE_ENABLED=true", - "--env", "REGISTRY_HTTP_TLS_CERTIFICATE=/certs/cert.pem", - "--env", "REGISTRY_HTTP_TLS_KEY=/certs/privkey.pem", - "--name=$localRepository", - "--volume=${generateCerts.get().dest.get().asFile.absolutePath}:/certs", - "--volume=${volume.get()}:/var/lib/registry", - "-p", "${localRepositoryPort}:${localRepositoryPort}", - "registry:2" - ) - } - } - // Allow insecure push / pull. - configFile.get().asFile.run { - val certPath = generateCerts.get().dest.get().asFile.absolutePath - parentFile.mkdirs() - writeText( - """ - [worker.oci] - enabled = true - gc = false - [worker.containerd] - enabled = false - gc = false - [registry."${localRepository}:${localRepositoryPort}"] - ca=["${certPath}/rootCA.pem"] - [[registry."${localRepository}:${localRepositoryPort}".keypair]] - key="${certPath}/privkey.pem" - cert="${certPath}/cert.pem" - """.trimIndent() - ) - } - } - mustRunAfter("destroyLocalRegistry") - dependsOn(generateCerts) - } - - // Destroys resources created by createLocalRegistry. - val destroyLocalRegistry by tasks.registering { - group = isleBuildkitGroup - description = "Destroys the local registry and its backing volume" - doLast { - createLocalRegistry.get().let { task -> - val network: Property by task.extra - val volume: Property by task.extra - exec { - commandLine = listOf("docker", "rm", "-f", localRepository) - isIgnoreExitValue = true - } - exec { - commandLine = listOf("docker", "network", "rm", network.get()) - isIgnoreExitValue = true - } - exec { - commandLine = listOf("docker", "volume", "rm", volume.get()) - isIgnoreExitValue = true - } - } - } - } - - tasks.register("collectGarbageLocalRegistry") { - group = isleBuildkitGroup - description = "Deletes layers not referenced by any manifests in the local repository" - doLast { - exec { - commandLine = listOf( - "docker", - "exec", - localRepository, - "bin/registry", - "garbage-collect", - "/etc/docker/registry/config.yml" - ) - } - } - dependsOn(createLocalRegistry) - } - - val getIpAddressOfLocalRegistry by tasks.registering { - val ipAddress by extra(objects.property()) - doLast { - ipAddress.set( - ByteArrayOutputStream().use { output -> - exec { - commandLine = listOf( - "docker", - "container", - "inspect", - "-f", - "{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}", - localRepository - ) - standardOutput = output - } - output.toString().trim() - } - ) - } - dependsOn(createLocalRegistry) - } - - val clean by tasks.registering { - group = isleBuildkitGroup - description = "Destroy absolutely everything" - doLast { - exec { - commandLine = listOf("docker", "system", "prune", "-f") - isIgnoreExitValue = true - } - } - dependsOn(destroyLocalRegistry) - } - - // Often easier to just use the default builder. - val useDefaultBuilder by tasks.registering { - group = isleBuildkitGroup - description = "Change the builder to use the Docker Daemon" - doLast { - project.exec { - commandLine = listOf("docker", "context", "use", "default") - } - project.exec { - commandLine = listOf("docker", "buildx", "use", "default") - } - } - } - - val pullSyft by tasks.registering(DockerPull::class) { - description = - "Pull anchore/syft for use in generating a Software Bill of Materials for vunerability scanning." - name.set("anchore/syft") - } - - val pullGrype by tasks.registering(DockerPull::class) { - description = - "Pull anchore/grype for use in processing a Software Bill of Materials for vunerability scanning." - name.set("anchore/grype") - } - - val updateGrypeDB by tasks.registering(GrypeUpdateDB::class) { - digest.set(pullGrype.flatMap { it.digestFile }) - imageId.set(pullGrype.map { it.digestFile.get().asFile.readText() }) - } - - tasks.all { - // Common settings for top level tasks. - if (group?.equals(isleBuildkitGroup) == true) { - logging.captureStandardOutput(LogLevel.INFO) - logging.captureStandardError(LogLevel.INFO) - } - } - - subprojects { - // Make all build directories relative to the root, only supports projects up to a depth of one for now. - buildDir = rootProject.buildDir.resolve(projectDir.relativeTo(rootDir)) - layout.buildDirectory.set(buildDir) - - // If there is a docker file in the project add the appropriate tasks. - if (isDockerProject) { - val tag = dockerTags.first() - - val defaultTags = imageTags(repository) - val defaultBuildArgs = mapOf( - "repository" to repository, - "tag" to tag - ) - - // Allows building both x86_64 and arm64 using emulation supported in version 19.03 and up as well Docker Desktop. - val createBuilder by tasks.registering(DockerBuilder::class) { - group = isleBuildkitGroup - description = "Creates and starts the builder ('docker-container' or 'kubernetes' only)" - onlyIf { !isDockerBuild } - // Make sure the builder can find our local repository. - options.run { - append.set(true) - driver.set(buildDriver) - name.set("isle-buildkit-$buildDriver-${project.name}") - node.set("isle-buildkit-$buildDriver-${project.name}-node") - when (buildDriver) { - "docker-container" -> { - driverOpts.set( - createLocalRegistry.map { task -> - val network: Property by task.extra - "network=${network.get()},image=moby/buildkit:v0.10.3" - } - ) - config.set( - createLocalRegistry.map { task -> - val configFile: RegularFileProperty by task.extra - configFile.get() - } - ) - } - } - use.set(false) - } - dependsOn(installBinFmt, createLocalRegistry) - mustRunAfter("destroyBuilder") - } - - val destroyBuilder by tasks.registering { - group = isleBuildkitGroup - description = "Destroy the builder and its cache ('docker-container' or 'kubernetes' only)" - doLast { - exec { - commandLine = listOf("docker", "buildx", "rm", createBuilder.get().options.name.get()) - isIgnoreExitValue = true - } - } - } - - // Clean up builders as well. - clean.configure { - dependsOn(destroyBuilder) - } - - val setupBuilder by tasks.registering { - group = isleBuildkitGroup - description = "Setup the builder according to project properties" - when (buildDriver) { - "docker" -> dependsOn(useDefaultBuilder) - else -> dependsOn(createBuilder) - } - } - - val build by tasks.registering(DockerBuild::class) { - group = isleBuildkitGroup - description = "Build docker image(s)" - options.run { - push.set(pushToRemote) - mustRunAfter("delete") - } - } - - val syft by tasks.registering(Syft::class) { - group = isleBuildkitGroup - description = "Generate a software bill of material with Syft" - syftDigestFile.set(pullSyft.flatMap { it.digestFile }) - imageDigestFile.set(build.flatMap { it.digest }) - image.set(build.map { it.options.tags.get().first() }) - } - - tasks.register("grype") { - group = isleBuildkitGroup - description = "Process the software bill of material with Grype" - config.set(grypeConfig) - failOn.set(grypeFailOnSeverity) - format.set(grypeFormat) - onlyFixed.set(grypeOnlyFixed) - grypeDigestFile.set(pullGrype.flatMap { it.digestFile }) - sbom.set(syft.flatMap { it.sbom }) - dependsOn(updateGrypeDB) - } - - tasks.register("delete") { - group = isleBuildkitGroup - description = "Delete image(s) from local registry" - doLast { - val ipAddress = getIpAddressOfLocalRegistry.get().let { task -> - val ipAddress: Property by task.extra - ipAddress.get() - } - dockerTags.plus("cache").map { tag -> - val baseUrl = "http://$ipAddress/v2/${project.name}/manifests" - val accept = if (tag == "cache") - "application/vnd.oci.image.index.v1+json" - else - "application/vnd.docker.distribution.manifest.v2+json" - (URL("$baseUrl/$tag").openConnection() as HttpURLConnection).run { - requestMethod = "GET" - setRequestProperty("Accept", accept) - headerFields["Docker-Content-Digest"]?.first() - }?.let { digest -> - logger.info("Deleting ${project.name}/$tag:$digest") - (URL("$baseUrl/$digest").openConnection() as HttpURLConnection).run { - requestMethod = "DELETE" - if (responseCode == 200 || responseCode == 202) { - logger.info("Successful ($responseCode): $responseMessage") - } else { - throw RuntimeException("Failed ($responseCode) - $responseMessage") - } - } - } - } - } - dependsOn(getIpAddressOfLocalRegistry) - } - - // Task groups all sub-project tasks.tests into single task. - tasks.register("test") { - group = isleBuildkitGroup - description = "Test docker image(s)" - dependsOn(project.subprojects.mapNotNull { it.tasks.matching { task -> task.name == "test" } }) - } - - // All build tasks have a number of shared defaults that can be overridden. - tasks.withType { - // Default arguments required for building. - options.run { - if (!isDockerBuild) { - dependsOn(getIpAddressOfLocalRegistry) - // Make sure the local repository is accessible. - addHosts.set( - getIpAddressOfLocalRegistry.map { - val ipAddress: Property by it.extra - listOf("$localRepository:${ipAddress.get()}") - } - ) - // Use the chosen builder. - builder.set(createBuilder.map { it.options.name.get() }) - } - tags.convention(defaultTags) - buildArgs.convention(defaultBuildArgs) - } - // Require builder to build. - dependsOn(setupBuilder) - // If destroying resources as well make sure build tasks run after after the destroy tasks. - mustRunAfter(clean, destroyBuilder, destroyLocalRegistry) - // We are either building or pushing neither both in a CI environment. - // This is just to keep us within the ~12 GB of free space that Github Actions gives us. - doLast { - if (!isDockerBuild && isCI) { - exec { - commandLine = - listOf("docker", "buildx", "rm", createBuilder.get().options.name.get()) - isIgnoreExitValue = true - } - } - } - } - } - } - } -} diff --git a/src/main/kotlin/plugins/BuildCtlPlugin.kt b/src/main/kotlin/plugins/BuildCtlPlugin.kt new file mode 100644 index 0000000..d90d6bf --- /dev/null +++ b/src/main/kotlin/plugins/BuildCtlPlugin.kt @@ -0,0 +1,254 @@ +package plugins + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.commons.io.output.ByteArrayOutputStream +import org.apache.commons.io.output.NullOutputStream +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.provider.Provider +import org.gradle.api.tasks.* +import org.gradle.kotlin.dsl.* +import plugins.BuildKitPlugin.Companion.buildKitPlatforms +import plugins.BuildKitPlugin.Companion.buildKitRepository +import plugins.BuildKitPlugin.Companion.buildKitTag +import tasks.BuildCtl + + +// Configures BuildKit. +@Suppress("unused") +class BuildCtlPlugin : Plugin { + + abstract class BuildCtlBuild : BuildCtl() { + // PATH (i.e. Docker build context), trigger re-run if changed. + @InputFiles + @PathSensitive(PathSensitivity.RELATIVE) + val context = project.objects.fileTree().setDir(project.layout.projectDirectory) + + @Input + val repository = project.objects.property().convention(project.buildKitRepository) + + @Input + val tag = project.objects.property().convention(project.buildKitTag) + + @Input + val platforms = project.objects.listProperty().convention(project.buildKitPlatforms) + + @OutputFile + val metadata = project.objects.fileProperty() + + @get:Internal + val image: Provider + get() = project.provider { "${repository.get()}/${project.name}:${tag.get()}" } + + @get:Internal + protected val baseArguments: List + get() = mutableListOf( + executablePath, + "build", + "--frontend=dockerfile.v0", + "--local", "context=.", + "--local", "dockerfile=.", + "--opt", "build-arg:repository=${repository.get()}", + "--opt", "build-arg:tag=${tag.get()}", + "--metadata-file", metadata.get().asFile.absolutePath, + ) + + protected fun parseMetadata(field: String) = metadata.get().asFile.let { + if (it.exists()) { + val node: JsonNode = ObjectMapper().readTree(metadata.get().asFile.readText()) + node.get(field).asText().trim() + } else + "" + } + + init { + val ignore = project.projectDir.resolve(".dockerignore") + if (ignore.exists()) { + context.setExcludes(ignore.readLines()) + } + } + } + + open class BuildCtlBuildImage : BuildCtlBuild() { + + // If we source images change we should rebuild. + @InputFiles + @PathSensitive(PathSensitivity.RELATIVE) + val sourceBuildMetadata = project.objects.fileCollection() + + @Input + val images = project.objects.listProperty() + + // Gets list of images names without repository or tag, required to build this image. + // This assumes all images are built by this project. + @get:Internal + val requiredImages by lazy { + context.dir.resolve("Dockerfile").readText().let { text -> + ("\\" + '$' + """\{repository\}/(?[^:@]+)""") + .toRegex() + .findAll(text) + .map { it.groups["image"]!!.value } + .toSet() + } + } + + private val accept: String + get() = if (platforms.get().isEmpty()) + "application/vnd.docker.distribution.manifest.v2+json" + else + "application/vnd.docker.distribution.manifest.list.v2+json" + + private val url: String + get() = "https://${repository.get()}/v2/${project.name}/manifests/${tag.get()}" + + private val currentDigest: String by lazy { + val arguments = listOf( + "docker", "exec", builder.get(), + "curl", "-s", + "-D", "-", + "--cacert", "/certs/rootCA.pem", + "-H", "'Accept: ${accept}'", + url, + "|", + "grep", "docker-content-digest", + "|", + """awk '{ print $2 }'""" + ) + ByteArrayOutputStream().use { output -> + project.exec { + commandLine("sh", "-c", arguments.joinToString(" ")) + standardOutput = output + // Swallow output. + errorOutput = NullOutputStream() + } + output.toString().trim() + } + } + + @get:Internal + val digest: String + get() = parseMetadata("containerimage.digest") + + init { + // Unique file for build metadata. + metadata.convention(project.layout.buildDirectory.file("build.json")) + + // Always push to the same location we're pulling from to ensure downstream builds get the right image. + images.add(image) + + // Manually check if the current digest matches the last built digest. + outputs.upToDateWhen { + logger.info("Build Digest: $digest") + logger.info("Current Digest: $currentDigest") + digest.isNotEmpty() && digest == currentDigest + } + } + + @TaskAction + fun build() { + val additionalArguments = mutableListOf() + if (platforms.get().isNotEmpty()) { + additionalArguments.addAll( + listOf( + "--opt", "platform=${platforms.get().joinToString(",")}" + ) + ) + } + additionalArguments.addAll( + listOf( + "--import-cache", "type=registry,ref=islandora/${project.name}:cache", + ) + ) + if (System.getenv("GITHUB_ACTIONS") == "true") { + additionalArguments.addAll( + listOf( + "--export-cache", "type=gha", + "--import-cache", "type=gha", + ) + ) + if (System.getenv("GITHUB_REF_NAME") == "main") { + additionalArguments.addAll( + listOf( + "--export-cache", "type=registry,mode=max,compression=estargz,ref=islandora/${project.name}:cache", + ) + ) + } + } + images.get().joinToString(",").let { + additionalArguments.addAll( + listOf("--output", """type=image,"name=$it",push=true""") + ) + } + project.exec { + workingDir(project.projectDir) + environment(hostEnvironmentVariables) + commandLine(baseArguments + additionalArguments) + } + + } + } + + open class BuildCtlLoadImage : BuildCtlBuild() { + + // If we rebuild we should reload. + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + val buildMetadata = project.objects.fileProperty() + + @get:Internal + val digest + get() = parseMetadata("containerimage.config.digest") + + private fun configDigestExists() = project.exec { + commandLine("docker", "inspect", digest) + // Swallow output. + standardOutput = NullOutputStream() + errorOutput = NullOutputStream() + isIgnoreExitValue = true + }.exitValue == 0 + + init { + // Unique file for load metadata. + metadata.convention(project.layout.buildDirectory.file("load.json")) + + // Manually check if the current digest matches the last built digest. + outputs.upToDateWhen { + logger.info("Config Digest: $digest") + configDigestExists() + } + } + + @TaskAction + fun load() { + // Only load the definitive repository/image:tag name, also platform is ignored when loading as we only care + // about the host platform. + val additionalArguments = listOf( + "--output", "type=docker,name=${repository.get()}/${project.name}:${tag.get()}" + ) + project.exec { + workingDir(project.projectDir) + environment(hostEnvironmentVariables) + commandLine( + "sh", "-c", (baseArguments + additionalArguments).joinToString(" ").plus("| docker load") + ) + } + } + } + + override fun apply(pluginProject: Project): Unit = pluginProject.run { + + val build by tasks.registering(BuildCtlBuildImage::class) { + group = "Isle" + description = "Build docker image(s)" + finalizedBy("load") + } + + tasks.register("load") { + group = "Isle" + description = "Load docker image(s) from registry" + buildMetadata.set(build.flatMap { it.metadata }) + } + } +} + diff --git a/src/main/kotlin/plugins/BuildKitPlugin.kt b/src/main/kotlin/plugins/BuildKitPlugin.kt new file mode 100644 index 0000000..d1285bf --- /dev/null +++ b/src/main/kotlin/plugins/BuildKitPlugin.kt @@ -0,0 +1,415 @@ +package plugins + +import org.gradle.api.* +import org.gradle.api.file.RegularFile +import org.gradle.api.model.ObjectFactory +import org.gradle.api.provider.Provider +import org.gradle.api.provider.ProviderFactory +import org.gradle.api.tasks.* +import org.gradle.kotlin.dsl.* +import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform +import plugins.BuildKitPlugin.DockerBuildKitExtension.Companion.buildkit +import plugins.CertificateGenerationPlugin.GenerateCerts +import plugins.IslePlugin.Companion.isDockerProject +import plugins.RegistryPlugin.CreateRegistry +import tasks.BuildCtl +import tasks.DockerContainer.* +import tasks.DockerNetwork.DockerRemoveNetwork +import tasks.DockerVolume.DockerCreateVolume +import tasks.DockerVolume.DockerRemoveVolume +import tasks.Download + + +// Configures BuildKit. +@Suppress("unused") +class BuildKitPlugin : Plugin { + + companion object { + + // The name of the container that is running the buildkit daemon. + val Project.buildKitBuilder: Provider + get() = rootProject.tasks.named("startBuilder").map { it.name.get() } + + val Project.buildKitExecutable: Provider + get() = rootProject.tasks.named("unpackBuildKit").flatMap { it.executable } + + // The registry/repository to use when building/pushing images. + // It will default to the local registry if not given. + val Project.buildKitRepository: Provider + get() = rootProject.tasks.named("createRegistry").map { + (properties["buildkit.build-arg.repository"] as String?) ?: it.registry + } + + // The tag to use when building/pushing images. + val Project.buildKitTag: String + get() = properties.getOrDefault("buildkit.build-arg.tag", "latest") as String + + val Project.buildKitContainer: String + get() = properties.getOrDefault("buildkit.container", "isle-buildkit") as String + + val Project.buildKitVolume: String + get() = properties.getOrDefault("buildkit.volume", "isle-buildkit") as String + + val Project.buildKitImage: String + get() = properties.getOrDefault("buildkit.image", "moby/buildkit:v0.10.6") as String + + val Project.buildKitQemuImage: String + get() = properties.getOrDefault("buildkit.qemu.image", "tonistiigi/binfmt:qemu-v7.0.0-28") as String + + val Project.buildKitPlatforms: Set + get() = (properties.getOrDefault("buildkit.platforms", "") as String) + .split(',') + .map { it.trim() } + .filter { it.isNotEmpty() } + .toSet() + } + + open class DockerBuildKitExtension constructor(objects: ObjectFactory, providers: ProviderFactory) { + open class BuildKitExtension(private val name: String) : Named { + var sha256: String = "" + var platform: Boolean = false + override fun getName(): String = name + } + + val os = DefaultNativePlatform.getCurrentOperatingSystem()!! + val arch = DefaultNativePlatform.getCurrentArchitecture()!! + + var version = "v0.10.6" + var baseUrl = "https://github.com/moby/buildkit/releases/download" + + internal val executables = objects.domainObjectContainer(BuildKitExtension::class.java) + + fun buildkit(name: String, action: Action) { + executables.create(name, action) + } + + val buildkit: BuildKitExtension + get() = executables.find { it.platform }!! + + val url = objects.property().convention(providers.provider { + "${baseUrl}/${version}/${buildkit.name}" + }) + + companion object { + val Project.buildkit: DockerBuildKitExtension + get() = + extensions.findByType() ?: extensions.create("buildkit") + + fun Project.buildkit(action: Action) { + action.execute(buildkit) + } + + } + } + + @CacheableTask + open class BuildkitExecutable : DefaultTask() { + + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + val archive = project.objects.fileProperty() + + @OutputFile + val executable = project.objects.fileProperty().convention(project.layout.buildDirectory.file("buildctl")) + + @TaskAction + fun exec() { + project.copy { + from(project.tarTree(archive.get().asFile)) { + include("bin/buildctl") + eachFile { + path = name + } + } + into(executable.get().asFile.parent) + } + } + } + + // https://github.com/moby/buildkit/blob/v0.10.6/docs/buildkitd.toml.md + @CacheableTask + open class BuildkitConfiguration : DefaultTask() { + @Input + val registry = project.objects.property() + + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + val cert = project.objects.fileProperty() + + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + val key = project.objects.fileProperty() + + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + val rootCA = project.objects.fileProperty() + + @Internal + val dest = project.objects.directoryProperty().convention(project.layout.buildDirectory) + + @OutputFile + val config = project.objects.fileProperty().convention(dest.map { it.file("buildkitd.toml") }) + + @TaskAction + fun exec() { + // GitHub Actions has limited disk space, so we must clean up as we go. + // Additionally, when using CI we do not push to the local registry, but use a remote instead. + // Keep only up to 5GB of storage. + if (System.getenv("GITHUB_ACTIONS") == "true") { + config.get().asFile.writeText( + """ + [worker.containerd] + enabled = false + [worker.oci] + enabled = true + gc = true + gckeepstorage = 5000 + """.trimIndent() + ) + } else { + // Locally developers can run prune when needed, disable GC for speed!!! + // Also, a local registry is required to push / pull form. + config.get().asFile.writeText( + """ + [worker.containerd] + enabled = false + [worker.oci] + enabled = true + gc = false + [registry."${registry.get()}"] + insecure=false + ca=["/certs/${rootCA.get().asFile.name}"] + [[registry."${registry.get()}".keypair]] + key="/certs/${key.get().asFile.name}" + cert="/certs/${cert.get().asFile.name}" + """.trimIndent() + ) + } + } + } + + open class BuildkitDaemon : DockerCreateContainer() { + @Input + val network = project.objects.property() + + @Input + val volume = project.objects.property() + + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + val cert = project.objects.fileProperty() + + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + val key = project.objects.fileProperty() + + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + val rootCA = project.objects.fileProperty() + + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + val config = project.objects.fileProperty() + + init { + options.addAll(project.provider { + listOf( + "--privileged", + "--network", network.get(), + "--volume", "${cert.get().asFile.absolutePath}:/certs/${cert.get().asFile.name}", + "--volume", "${key.get().asFile.absolutePath}:/certs/${key.get().asFile.name}", + "--volume", "${rootCA.get().asFile.absolutePath}:/certs/${rootCA.get().asFile.name}", + "--volume", "${config.get().asFile.absolutePath}:/etc/buildkit/buildkitd.toml", + "--volume", "${volume.get()}:/var/lib/buildkit", + ) + }) + } + } + + open class BuildCtlDiskUsage : BuildCtl() { + @TaskAction + fun diskUsage() { + project.exec { + workingDir(project.projectDir) + environment(hostEnvironmentVariables) + commandLine(executablePath, "du", "-v") + } + } + } + + open class BuildCtlPrune : BuildCtl() { + @TaskAction + fun prune() { + project.exec { + workingDir(project.projectDir) + environment(hostEnvironmentVariables) + commandLine(executablePath, "prune") + } + } + } + + override fun apply(pluginProject: Project): Unit = pluginProject.run { + apply() + apply() + + afterEvaluate { + buildkit { + // Apply defaults if not provided. + if (executables.isEmpty()) { + buildkit("buildkit-${version}.linux-amd64.tar.gz") { + sha256 = "9a21a41298c4a2a7a2b57cb90d37463d3a9057aedfe97a04b0e4fd6f622549d8" + platform = os.isLinux + } + buildkit("buildkit-${version}.darwin-amd64.tar.gz") { + sha256 = "3ffcb3910b337ce74868c36f1fef6ef4c0b32f7f16f54e9acd004ce2f3ae5bd2" + platform = os.isMacOsX && arch.isAmd64 + } + buildkit("buildkit-${version}.darwin-arm64.tar.gz") { + sha256 = "eaad6698b4013e67290fe3515888916b8ea99aa86d296af8d7bcb61da8e95ec5" + platform = os.isMacOsX && arch.isArm + } + buildkit("buildkit-${version}-windows-amd64.tar.gz") { + sha256 = "6095f8f8fab13f3c9cb1df4c63e4160cd092041d70358a9ee2b56db95bd7d1ef" + platform = os.isWindows + } + } + } + } + + val generateCertificates = tasks.named("generateCertificates") + val createRegistry = tasks.named("createRegistry") + val startRegistry = tasks.named("startRegistry") + val destroyRegistryNetwork = tasks.named("destroyRegistryNetwork") + + val downloadBuildKit by tasks.registering(Download::class) { + group = "Isle Buildkit" + description = "Downloads buildctl for interacting with buildkit" + url.set(buildkit.url) + sha256.set(buildkit.buildkit.sha256) + } + + val unpackBuildKit by tasks.registering(BuildkitExecutable::class) { + group = "Isle Buildkit" + description = "Unpacks buildctl from the downloaded archive" + archive.set(downloadBuildKit.flatMap { it.dest }) + } + + val installBinFmt by tasks.registering(Exec::class) { + group = "Isle Buildkit" + description = "Install https://github.com/tonistiigi/binfmt to enable multi-arch builds on Linux." + commandLine = listOf( + "docker", + "container", + "run", + "--rm", + "--privileged", + buildKitQemuImage, + "--install", "all" + ) + // Cross building with Qemu is already installed with Docker Desktop, so we only need to install on Linux. + // Additionally, it does not work with non x86_64 hosts. + onlyIf { + buildkit.os.isLinux && buildkit.arch.isAmd64 + } + } + + val generateBuildkitConfig by tasks.registering(BuildkitConfiguration::class) { + group = "Isle Buildkit" + description = + "Generate https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md to configure buildkit." + registry.set(createRegistry.map { it.registry }) + cert.set(generateCertificates.flatMap { it.cert }) + key.set(generateCertificates.flatMap { it.key }) + rootCA.set(generateCertificates.flatMap { it.rootCA }) + } + + val stopBuilder by tasks.registering(DockerStopContainer::class) { + group = "Isle Registry" + description = "Stops the buildkit container" + name.set(buildKitContainer) + } + + val destroyBuilder by tasks.registering(DockerRemoveContainer::class) { + group = "Isle Buildkit" + description = "Removes the buildkit container" + name.set(buildKitContainer) + dependsOn(stopBuilder) + } + + val destroyBuilderVolume by tasks.registering(DockerRemoveVolume::class) { + group = "Isle Buildkit" + description = "Destroys the buildkit cache volume" + volume.set(buildKitVolume) + dependsOn(destroyBuilder) // Cannot remove volumes of active containers. + } + + val createBuilderVolume by tasks.registering(DockerCreateVolume::class) { + group = "Isle Buildkit" + description = "Creates a volume for the buildkit cache" + volume.set(buildKitVolume) + mustRunAfter(destroyBuilderVolume) + } + + val createBuilder by tasks.registering(BuildkitDaemon::class) { + group = "Isle Buildkit" + description = "Creates a container for the buildkit daemon" + name.set(buildKitContainer) + image.set(buildKitImage) + volume.set(createBuilderVolume.map { it.volume.get() }) + network.set(createRegistry.map { it.network.get() }) + cert.set(generateCertificates.flatMap { it.cert }) + key.set(generateCertificates.flatMap { it.key }) + rootCA.set(generateCertificates.flatMap { it.rootCA }) + config.set(generateBuildkitConfig.flatMap { it.config }) + dependsOn(installBinFmt, unpackBuildKit) + mustRunAfter(destroyBuilder) + } + + tasks.register("startBuilder") { + group = "Isle Buildkit" + description = "Starts the buildkit container" + name.set(buildKitContainer) + dependsOn(createBuilder, startRegistry) // Requires connection with the local registry to push. + doLast { + // We rely on CuRL in the builder to interact with registry to download manifests, etc. + project.exec { + commandLine("docker", "exec", this@register.name.get(), "apk", "add", "curl") + } + } + } + + destroyRegistryNetwork.configure { + dependsOn(destroyBuilder) // Cannot remove networks of active containers as they share the same network. + } + + tasks.register("diskUsage") { + group = "Isle Buildkit" + description = "Display BuildKit disk usage" + } + + tasks.register("prune") { + group = "Isle Buildkit" + description = "Clean BuildKit build cache" + } + + allprojects { + // Auto-apply plugins to relevant projects. + if (isDockerProject) { + apply() + } + } + + subprojects { + // Enforce build order for dependencies. + tasks.withType { + val buildMetadata = project + .rootProject + .allprojects + .filter { it.isDockerProject && requiredImages.contains(it.name) } + .map { it.tasks.named("build").flatMap { task -> task.metadata } } + sourceBuildMetadata.setFrom(buildMetadata) + } + } + + } +} \ No newline at end of file diff --git a/src/main/kotlin/plugins/CertificateGenerationPlugin.kt b/src/main/kotlin/plugins/CertificateGenerationPlugin.kt new file mode 100644 index 0000000..b9f568a --- /dev/null +++ b/src/main/kotlin/plugins/CertificateGenerationPlugin.kt @@ -0,0 +1,178 @@ +package plugins + +import org.gradle.api.* +import org.gradle.api.model.ObjectFactory +import org.gradle.api.provider.ProviderFactory +import org.gradle.api.tasks.* +import org.gradle.internal.jvm.Jvm +import org.gradle.kotlin.dsl.* +import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform +import plugins.CertificateGenerationPlugin.CertificateGenerationExtension.Companion.certificates +import plugins.IslePlugin.Companion.execCaptureOutput +import tasks.Download +import java.nio.file.Files.setPosixFilePermissions + +// Downloads and executes mkcert to generate development certificates. +@Suppress("unused") +class CertificateGenerationPlugin : Plugin { + + open class CertificateGenerationExtension constructor(objects: ObjectFactory, providers: ProviderFactory) { + open class MkCertExtension(private val name: String) : Named { + var sha256: String = "" + var platform: Boolean = false + override fun getName(): String = name + } + + val os = DefaultNativePlatform.getCurrentOperatingSystem()!! + val arch = DefaultNativePlatform.getCurrentArchitecture()!! + + var version = "v1.4.4" + var baseUrl = "https://github.com/FiloSottile/mkcert/releases/download" + var domains = listOf( + "*.islandora.dev", + "islandora.dev", + "localhost", + "127.0.0.1", + "::1", + ) + + internal val executables = objects.domainObjectContainer(MkCertExtension::class.java) + + fun mkcert(name: String, action: Action) { + executables.create(name, action) + } + + val mkcert: MkCertExtension + get() = executables.find { it.platform }!! + + val url = objects.property().convention(providers.provider { + "${baseUrl}/${version}/${mkcert.name}" + }) + + companion object { + val Project.certificates: CertificateGenerationExtension + get() = + extensions.findByType() ?: extensions.create("certificates") + + fun Project.certificates(action: Action) { + action.execute(certificates) + } + + } + } + + open class GenerateCerts : DefaultTask() { + + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + val executable = project.objects.fileProperty() + + @Internal + val dest = project.objects.directoryProperty().convention(project.layout.buildDirectory.dir("certs")) + + @OutputFile + val cert = project.objects.fileProperty().convention(dest.map { it.file("cert.pem") }) + + @OutputFile + val key = project.objects.fileProperty().convention(dest.map { it.file("privkey.pem") }) + + @OutputFile + val rootCA = project.objects.fileProperty().convention(dest.map { it.file("rootCA.pem") }) + + @OutputFile + val rootCAKey = project.objects.fileProperty().convention(dest.map { it.file("rootCA-key.pem") }) + + @Input + val arguments = project.objects.listProperty() + + private val executablePath: String + get() = this@GenerateCerts.executable.get().asFile.absolutePath + + private fun execute(vararg arguments: String) { + project.exec { + commandLine = listOf(executablePath) + arguments + // Exclude JAVA_HOME as we only want to check the local certificates for the system. + environment = Jvm.current().getInheritableEnvironmentVariables(System.getenv()).filterKeys { + !setOf("JAVA_HOME").contains(it) + } + // Note this is allowed to fail on some systems the user may have to manually install the local certificate. + // See the README. + isIgnoreExitValue = true + } + } + + private fun install() { + execute("-install") + val rootStore = + project.file(project.execCaptureOutput(listOf(executablePath, "-CAROOT"), "Failed to find CAROOT")) + listOf(rootCA.get().asFile, rootCAKey.get().asFile).forEach { + rootStore.resolve(it.name).copyTo(it, true) + } + } + + @TaskAction + fun exec() { + install() + execute( + "-cert-file", cert.get().asFile.absolutePath, + "-key-file", key.get().asFile.absolutePath, + *arguments.get().toTypedArray(), + ) + } + + } + + override fun apply(pluginProject: Project): Unit = pluginProject.run { + afterEvaluate { + certificates { + // Apply defaults if not provided. + if (executables.isEmpty()) { + mkcert("mkcert-${version}-linux-amd64") { + sha256 = "6d31c65b03972c6dc4a14ab429f2928300518b26503f58723e532d1b0a3bbb52" + platform = os.isLinux + } + mkcert("mkcert-${version}-darwin-amd64") { + sha256 = "a32dfab51f1845d51e810db8e47dcf0e6b51ae3422426514bf5a2b8302e97d4e" + platform = os.isMacOsX && arch.isAmd64 + } + mkcert("mkcert-${version}-darwin-arm64") { + sha256 = "c8af0df44bce04359794dad8ea28d750437411d632748049d08644ffb66a60c6" + platform = os.isMacOsX && arch.isArm + } + mkcert("mkcert-${version}-windows-amd64.exe") { + sha256 = "d2660b50a9ed59eada480750561c96abc2ed4c9a38c6a24d93e30e0977631398" + platform = os.isWindows + } + } + } + } + + val downloadMkCert by tasks.registering(Download::class) { + group = "Isle Certificates" + description = "Downloads mkcert for generating development certificates" + url.set(certificates.url) + sha256.set(certificates.mkcert.sha256) + doLast { + if (!certificates.os.isWindows) { + // Make all downloaded files executable. + val perms = setOf( + java.nio.file.attribute.PosixFilePermission.OWNER_READ, + java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE, + java.nio.file.attribute.PosixFilePermission.GROUP_READ, + java.nio.file.attribute.PosixFilePermission.GROUP_EXECUTE, + java.nio.file.attribute.PosixFilePermission.OTHERS_READ, + java.nio.file.attribute.PosixFilePermission.OTHERS_EXECUTE, + ) + setPosixFilePermissions(dest.get().asFile.toPath(), perms) + } + } + } + + tasks.register("generateCertificates") { + group = "Isle Certificates" + description = "Generates development certificates" + executable.set(downloadMkCert.flatMap { it.dest }) + arguments.set(certificates.domains) + } + } +} diff --git a/src/main/kotlin/plugins/IslePlugin.kt b/src/main/kotlin/plugins/IslePlugin.kt new file mode 100644 index 0000000..081f62b --- /dev/null +++ b/src/main/kotlin/plugins/IslePlugin.kt @@ -0,0 +1,66 @@ +package plugins + +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.logging.LogLevel +import org.gradle.api.tasks.Delete +import org.gradle.kotlin.dsl.apply +import org.gradle.kotlin.dsl.register +import org.gradle.kotlin.dsl.withGroovyBuilder +import java.io.ByteArrayOutputStream + +@Suppress("unused") +class IslePlugin : Plugin { + + companion object { + // Capture stdout from running a command. + fun Project.execCaptureOutput(command: List, error: String) = ByteArrayOutputStream().use { output -> + val result = this.exec { + standardOutput = output + commandLine = command + } + if (result.exitValue != 0) throw RuntimeException(error) + output.toString() + }.trim() + + // Check if the project should have docker related tasks. + val Project.isDockerProject: Boolean + get() = projectDir.resolve("Dockerfile").exists() + } + + override fun apply(pluginProject: Project): Unit = pluginProject.run { + apply() + apply() + apply() + + extensions.findByName("buildScan")?.withGroovyBuilder { + setProperty("termsOfServiceUrl", "https://gradle.com/terms-of-service") + setProperty("termsOfServiceAgree", "yes") + } + + // Return repository to initial "clean" state. + tasks.register("clean") { + group = "Isle" + description = "Destroy absolutely everything" + delete(layout.buildDirectory) + dependsOn("destroyBuilderVolume", "destroyRegistryVolume") + } + + allprojects { + // Defaults for all tasks created by this plugin. + tasks.configureEach { + val displayOutputExceptions = listOf("diskUsage", "prune") + if (group?.startsWith("Isle") == true && !displayOutputExceptions.contains(name)) { + logging.captureStandardOutput(LogLevel.INFO) + logging.captureStandardError(LogLevel.INFO) + } + } + } + + // Make all build directories relative to the root, only supports projects up to a depth of one for now. + subprojects { + buildDir = rootProject.buildDir.resolve(projectDir.relativeTo(rootDir)) + layout.buildDirectory.set(buildDir) + } + } +} diff --git a/src/main/kotlin/plugins/RegistryPlugin.kt b/src/main/kotlin/plugins/RegistryPlugin.kt new file mode 100644 index 0000000..de51f5b --- /dev/null +++ b/src/main/kotlin/plugins/RegistryPlugin.kt @@ -0,0 +1,158 @@ +package plugins + +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.* +import org.gradle.kotlin.dsl.* +import plugins.CertificateGenerationPlugin.GenerateCerts +import tasks.DockerContainer.* +import tasks.DockerNetwork.DockerCreateNetwork +import tasks.DockerNetwork.DockerRemoveNetwork +import tasks.DockerVolume.DockerCreateVolume +import tasks.DockerVolume.DockerRemoveVolume + +// Creates a docker registry hosted locally with the given parameters that can be used by buildkit. +@Suppress("unused") +class RegistryPlugin : Plugin { + + companion object { + // It's important to note that we’re using a domain containing a "." here, i.e. localhost.domain. + // If it were missing Docker would believe that localhost is a username, as in localhost/ubuntu. + // It would then try to push to the default Central Registry rather than our local repository. + // *.islandora.dev makes for a good default as we can generate certificates for it and avoid many problems. + val Project.registryDomain: String + get() = properties.getOrDefault("registry.domain", "islandora.dev") as String + + val Project.registryPort: Int + get() = (properties.getOrDefault("registry.port", "443") as String).toInt() + + val Project.bindPort: Boolean + get() = (properties.getOrDefault("registry.bind.port", "false") as String).toBoolean() + + // The container should have the same name as the domain so that buildkit builder can find it by name. + val Project.registryContainer: String + get() = properties.getOrDefault("registry.container", "isle-registry") as String + + val Project.registryNetwork: String + get() = properties.getOrDefault("registry.network", "isle-registry") as String + + val Project.registryVolume: String + get() = properties.getOrDefault("registry.volume", "isle-registry") as String + + val Project.registryImage: String + get() = properties.getOrDefault("registry.image", "registry:2") as String + } + + open class CreateRegistry : DockerCreateContainer() { + @Input + val domain = project.objects.property().convention(project.registryDomain) + + @Input + val port = project.objects.property().convention(project.registryPort) + + @Input + val bindPort = project.objects.property().convention(project.bindPort) + + @Input + val network = project.objects.property().convention(project.registryNetwork) + + @Input + val volume = project.objects.property().convention(project.registryVolume) + + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + val cert = project.objects.fileProperty() + + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + val key = project.objects.fileProperty() + + @get:Internal + val registry: String + get() = if (port.get() == 443) domain.get() else "${domain.get()}:${port.get()}" + + init { + options.addAll(project.provider { + listOf( + "--network=${network.get()}", + "--network-alias=${domain.get()}", + "--env", "REGISTRY_HTTP_ADDR=0.0.0.0:${port.get()}", + "--env", "REGISTRY_STORAGE_DELETE_ENABLED=true", + "--env", "REGISTRY_HTTP_TLS_CERTIFICATE=/certs/cert.pem", + "--env", "REGISTRY_HTTP_TLS_KEY=/certs/privkey.pem", + "--volume=${cert.get().asFile.absolutePath}:/certs/cert.pem:ro", + "--volume=${key.get().asFile.absolutePath}:/certs/privkey.pem:ro", + "--volume=${volume.get()}:/var/lib/registry", + ) + if (bindPort.get()) listOf("-p", "${port.get()}:${port.get()}") else emptyList() + }) + } + } + + override fun apply(pluginProject: Project): Unit = pluginProject.run { + apply() + + val stopRegistry by tasks.registering(DockerStopContainer::class) { + group = "Isle Registry" + description = "Stops the local registry" + name.set(registryContainer) + } + + val destroyRegistry by tasks.registering(DockerRemoveContainer::class) { + group = "Isle Registry" + description = "Destroys the local registry" + name.set(registryContainer) + dependsOn(stopRegistry) + finalizedBy("destroyRegistryNetwork") + } + + val destroyRegistryVolume by tasks.registering(DockerRemoveVolume::class) { + group = "Isle Registry" + description = "Destroys the local docker registry volume" + volume.set(registryVolume) + dependsOn(destroyRegistry) // Cannot remove volumes of active containers. + } + + val destroyRegistryNetwork by tasks.registering(DockerRemoveNetwork::class) { + group = "Isle Registry" + description = "Destroys the local docker registry network" + network.set(registryNetwork) + dependsOn(destroyRegistry) // Cannot remove networks of active containers. + } + + val createRegistryVolume by tasks.registering(DockerCreateVolume::class) { + group = "Isle Registry" + description = "Creates a volume for the local docker registry" + volume.set(registryVolume) + mustRunAfter(destroyRegistryVolume) + } + + val createRegistryNetwork by tasks.registering(DockerCreateNetwork::class) { + group = "Isle Registry" + description = "Creates a network for the local docker registry" + network.set(registryNetwork) + mustRunAfter(destroyRegistryNetwork) + } + + val generateCertificates = tasks.named("generateCertificates") + + val createRegistry by tasks.registering(CreateRegistry::class) { + group = "Isle Registry" + description = "Starts a the local docker registry if not already running" + name.set(registryContainer) + image.set(registryImage) + network.set(createRegistryNetwork.map { it.network.get() }) + volume.set(createRegistryVolume.map { it.volume.get() }) + cert.set(generateCertificates.flatMap { it.cert }) + key.set(generateCertificates.flatMap { it.key }) + mustRunAfter(destroyRegistry) + } + + tasks.register("startRegistry") { + group = "Isle Registry" + description = "Starts the local registry" + name.set(registryContainer) + dependsOn(createRegistry) + } + + } +} \ No newline at end of file diff --git a/src/main/kotlin/plugins/ReportPlugin.kt b/src/main/kotlin/plugins/ReportPlugin.kt new file mode 100644 index 0000000..7869704 --- /dev/null +++ b/src/main/kotlin/plugins/ReportPlugin.kt @@ -0,0 +1,166 @@ +package plugins + +import org.gradle.api.DefaultTask +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.* +import org.gradle.kotlin.dsl.* +import plugins.BuildCtlPlugin.BuildCtlLoadImage +import plugins.ReportsPlugin.Companion.grypeConfig +import plugins.ReportsPlugin.Companion.grypeFailOnSeverity +import plugins.ReportsPlugin.Companion.grypeFormat +import plugins.ReportsPlugin.Companion.grypeOnlyFixed +import plugins.ReportsPlugin.UpdateGrypeDB +import tasks.DockerPull + +// Generate reports via Syft and Grype. +@Suppress("unused") +class ReportPlugin : Plugin { + + // Wrapper around a call to `syft`, please refer to the documentation for more information: + // https://github.com/anchore/syft + @CacheableTask + open class Syft : DefaultTask() { + + // anchore/syft image. + @Input + val syft = project.objects.property() + + // The image to process (assumed to exist locally). + @Input + val image = project.objects.property() + + // A json file representing the generated Software Bill of Materials. + @OutputFile + val sbom = project.objects.fileProperty().convention(project.layout.buildDirectory.file("sbom.json")) + + @TaskAction + fun exec() { + sbom.get().asFile.outputStream().use { output -> + project.exec { + standardOutput = output + commandLine = listOf( + "docker", "container", "run", "--rm", + "-v", "/var/run/docker.sock:/var/run/docker.sock", + syft.get(), + "-o", "json", + image.get() + ) + } + } + } + } + + // Wrapper around a call to `syft`, please refer to the documentation for more information: + // https://github.com/anchore/syft + @CacheableTask + open class Grype : DefaultTask() { + // anchore/grype image. + @Input + val grype = project.objects.property() + + // anchore/grype image. + @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) + val database = project.objects.directoryProperty() + + // A json file representing the generated Software Bill of Materials. + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + val sbom = project.objects.fileProperty() + + @Input + val format = project.objects.property().convention("table") + + @Input + @Optional + val failOn = project.objects.property() + + @InputFile + @Optional + @PathSensitive(PathSensitivity.RELATIVE) + val config = project.objects.fileProperty() + + @Input + val onlyFixed = project.objects.property().convention(false) + + @OutputFile + val report = project.objects.fileProperty().convention(format.flatMap { + val dir = project.layout.buildDirectory + val name = "${project.name}-grype" + when (it) { + "json" -> dir.file("${name}.json") + "table" -> dir.file("${name}.md") + "cyclonedx" -> dir.file("${name}.xml") + else -> dir.file("${name}.txt") + } + }) + + @TaskAction + fun exec() { + sbom.get().asFile.inputStream().use { input -> + report.get().asFile.outputStream().use { output -> + // Arguments to docker. + val command = mutableListOf( + "docker", "container", "run", "--rm", "-i", + "-e", "GRYPE_DB_CACHE_DIR=/cache", + "-e", "GRYPE_DB_AUTO_UPDATE=false", + "-v", "${database.get().asFile.absolutePath}:/cache", + ) + if (config.isPresent) { + command.addAll(listOf("-v", "${config.get().asFile.absolutePath}:/grype.yaml")) + } + // Docker image + command.add(grype.get()) + if (config.isPresent) { + command.addAll(listOf("--config", "/grype.yaml")) + } + // Arguments to grype. + if (failOn.isPresent) { + command.addAll(listOf("--fail-on", failOn.get())) + } + if (onlyFixed.get()) { + command.add("--only-fixed") + } + command.addAll(listOf("-o", format.get())) + project.exec { + standardInput = input + standardOutput = output + commandLine = command + } + } + } + } + } + + override fun apply(pluginProject: Project): Unit = pluginProject.run { + // One off tasks live on the root project. + val pullSyft = rootProject.tasks.named("pullSyft") + val pullGrype = rootProject.tasks.named("pullGrype") + val updateGrypeDB = rootProject.tasks.named("updateGrypeDB") + + // Requires the build plugin to be included, or this will fail at configure time. + val load = tasks.named("load") + + val syft by tasks.registering(Syft::class) { + group = "Isle Reports" + description = "Generate a software bill of material with Syft" + syft.set(pullSyft.map { it.digest }) + image.set(load.map { it.digest }) + } + + tasks.register("grype") { + group = "Isle Reports" + description = "Process the software bill of material with Grype" + grype.set(pullGrype.map { it.digest }) + database.set(updateGrypeDB.flatMap { it.database }) + config.set(grypeConfig) + failOn.set(grypeFailOnSeverity) + format.set(grypeFormat) + onlyFixed.set(grypeOnlyFixed) + sbom.set(syft.flatMap { it.sbom }) + dependsOn(":updateGrypeDB") + } + + } +} \ No newline at end of file diff --git a/src/main/kotlin/plugins/ReportsPlugin.kt b/src/main/kotlin/plugins/ReportsPlugin.kt new file mode 100644 index 0000000..c4b76b3 --- /dev/null +++ b/src/main/kotlin/plugins/ReportsPlugin.kt @@ -0,0 +1,128 @@ +package plugins + +import org.apache.commons.io.output.NullOutputStream +import org.gradle.api.DefaultTask +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.file.RegularFile +import org.gradle.api.tasks.CacheableTask +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.TaskAction +import org.gradle.kotlin.dsl.* +import plugins.IslePlugin.Companion.isDockerProject +import tasks.DockerPull + +// Generate reports via Syft and Grype. +@Suppress("unused") +class ReportsPlugin : Plugin { + + companion object { + // Configuration for grype. + val Project.grypeConfig: RegularFile? + get() = (properties["grype.config"] as String?)?.let { path: String -> + project.layout.projectDirectory.file(path).let { file -> + if (file.asFile.exists()) + file + else + null + } + } + + // Only reports issues that have fixes. + val Project.grypeOnlyFixed: Boolean + get() = (properties.getOrDefault("grype.only-fixed", "false") as String).toBoolean() + + // Triggers build to fail if security vulnerability is discovered. + // If unspecified the build will continue regardless. + // Possible values: negligible, low, medium, high, critical. + // Only reports issues that have fixes. + val Project.grypeFailOnSeverity: String? + get() = properties.getOrDefault("grype.fail-on-severity", null) as String? + + // The format of reports generated by grype. + // Possible values: table, cyclonedx, json, template. + val Project.grypeFormat: String + get() = properties.getOrDefault("grype.format", "table") as String + } + + // Updates Grype Database. + @CacheableTask + open class UpdateGrypeDB : DefaultTask() { + @Input + val image = project.objects.property() + + @OutputDirectory + val database = project.objects.directoryProperty().convention(project.layout.buildDirectory.dir("grype")) + + private val baseArguments: List + get() = listOf( + "docker", + "run", + "--rm", + "-e", "GRYPE_DB_CACHE_DIR=/cache", + "-v", "${database.get().asFile.absolutePath}:/cache", + image.get(), + ) + + private fun upToDate() = project.exec { + commandLine( + baseArguments + listOf( + "db", + "status" + ) + ) + standardOutput = NullOutputStream() + errorOutput = NullOutputStream() + isIgnoreExitValue = true + }.exitValue == 0 + + init { + outputs.upToDateWhen { + // If the database is missing the task will re-run or be restored from cache. + // If the database is present check to make sure it is up-to-date, if not run again. + !database.get().asFile.exists() || upToDate() + } + } + + @TaskAction + fun pull() { + project.exec { + commandLine( + baseArguments + listOf( + "db", + "update" + ) + ) + } + } + } + + override fun apply(pluginProject: Project): Unit = pluginProject.run { + + tasks.register("pullSyft") { + group = "Isle Reports" + description = "Pull anchore/syft docker image" + image.set("anchore/syft") + } + + val pullGrype by tasks.registering(DockerPull::class) { + group = "Isle Reports" + description = "Pull anchore/grype docker image" + image.set("anchore/grype") + } + + tasks.register("updateGrypeDB") { + group = "Isle Reports" + description = "Update the Grype Database" + image.set(pullGrype.map { it.digestFile.get().asFile.readText().trim() }) + } + + allprojects { + // Auto-apply plugins to relevant projects. + if (isDockerProject) { + apply() + } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/plugins/TestPlugin.kt b/src/main/kotlin/plugins/TestPlugin.kt new file mode 100644 index 0000000..ab441f0 --- /dev/null +++ b/src/main/kotlin/plugins/TestPlugin.kt @@ -0,0 +1,291 @@ +package plugins + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.module.kotlin.KotlinModule +import com.fasterxml.jackson.module.kotlin.readValue +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.* +import org.gradle.kotlin.dsl.* +import java.io.* +import java.time.Duration +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +// Generate reports via Syft and Grype. +@Suppress("unused") +class TestPlugin : Plugin { + + open class DockerCompose : DefaultTask() { + data class DockerComposeFile(val services: Map) { + companion object { + fun deserialize(file: File): DockerComposeFile = + ObjectMapper(YAMLFactory()) + .registerModule(KotlinModule.Builder().build()) + .configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .readValue(file) + } + } + + data class Service(val image: String) { + companion object { + @Suppress("RegExpRedundantEscape") + val regex = """\$\{(?[^:]+):-(?.+)\}""".toRegex() + } + + private fun variable() = regex + .matchEntire(image) + ?.groups + ?.get("variable") + ?.value + + fun env(image: String) = + variable() + ?.let { variable -> + variable to image + } + } + + @Internal + val baseArguments = listOf( + "docker", + "compose", + "--project-name", project.path + .replace(":", "_") + .toLowerCase() + .removePrefix("_") + ) + + @get:Internal + val dockerCompose by lazy { + DockerComposeFile.deserialize(project.file("docker-compose.yml")) + } + + // Output of load tasks for dependency caching and resolution. + @InputFiles + @PathSensitive(PathSensitivity.RELATIVE) + val metadataFiles = project.objects.fileCollection() + + // Any file might be referenced by the docker-compose.yml file / as a volume, etc. + @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) + val context = project.objects.directoryProperty().convention(project.layout.projectDirectory) + + // Environment for docker-compose not the actual containers. + @Input + val env = project.objects.mapProperty() + } + + @CacheableTask + open class DockerComposeUp : DockerCompose() { + + companion object { + val pool: ExecutorService = Executors.newCachedThreadPool() + } + + @Input + val exitCodeConditions = project.objects.mapProperty() + + @Input + val outputConditions = project.objects.mapProperty() + + // Capture the log output of the command for later inspection. + // Also prevents test from re-running if successful. + @OutputFile + val log = project.objects.fileProperty().convention(project.layout.buildDirectory.file("$name.log")) + + init { + // By default, limit max execution time to 5 minutes. + timeout.convention(Duration.ofMinutes(5)) + + // Expect each container exits 0 by default. + exitCodeConditions.putAll(dockerCompose.services.mapValues { 0 }) + } + + // Gets the identifiers of all the services created by the docker compose file. + private val containers by lazy { + ByteArrayOutputStream().use { output -> + project.exec { + workingDir(project.projectDir) + commandLine(baseArguments + listOf("ps", "-aq")) + standardOutput = output + } + output + .toString() + .lines() + .filter { it.isNotEmpty() } + } + } + + // Performs an `docker inspect` on all the services created by the docker compose file. + // Builds a map of service names paired with their exit codes. + @get:Internal + protected val exitCodes by lazy { + containers.associate { container -> + ByteArrayOutputStream().use { output -> + project.exec { + workingDir(project.projectDir) + commandLine("docker", "inspect", container) + standardOutput = output + } + output.toString() + }.let { + val node: JsonNode = ObjectMapper().readTree(it) + val service = + node.get(0)!!.get("Config")!!.get("Labels")!!.get("com.docker.compose.service")!!.asText()!! + val exitCode = node.get(0)!!.get("State")!!.get("ExitCode")!!.asInt() + Pair(service, exitCode) + } + } + } + + // Helper for writing tests which need to look for specific exit codes. + fun expectOutput(service: String, output: String) { + outputConditions.put(service, output) + } + + // Helper for writing tests which need to look for specific exit codes. + fun expectExitCode(service: String, exitCode: Int) { + exitCodeConditions.put(service, exitCode) + } + + // Monitor output of the given service. + private fun monitorService(service: String, output: String): Triple { + logger.info("""Looking for "$output" in $service logs""") + val process = ProcessBuilder().run { + directory(project.projectDir) + //command("bash", "-c", "while true; do openssl rand -base64 12; sleep 1; done") + command( + baseArguments + listOf( + "logs", + "--follow", + service + ) + ) + redirectErrorStream(true) + start() + } + val reader = CompletableFuture.supplyAsync({ + val reader = BufferedReader(InputStreamReader(process.inputStream)) + var line: String? + while (reader.readLine().also { line = it } != null) { + if (line?.contains(output) == true) { + process.destroy() + logger.info("""Found "$output" in $service logs""") + return@supplyAsync Triple(service, output, true) + } + } + logger.info("""Missing "$output" from $service logs""") + return@supplyAsync Triple(service, output, false) + + }, pool) + reader.whenComplete { _, _ -> + process.destroyForcibly() + } + if (!process.waitFor(timeout.get().toMillis(), TimeUnit.MILLISECONDS)) { + process.destroyForcibly() + } + return reader.get() + } + + @TaskAction + fun up() { + val up = CompletableFuture.supplyAsync( + { + val process = ProcessBuilder().run { + directory(project.projectDir) + command(baseArguments + listOf("up", "--abort-on-container-exit")) + redirectOutput(log.get().asFile) + redirectErrorStream(true) + start() + } + if (!process.waitFor(timeout.get().toMillis(), TimeUnit.MILLISECONDS)) { + process.destroyForcibly() + } + process.exitValue() + }, pool + ) + + // Used to fail the task if any condition was not met. + var failedConditions = false + + if (outputConditions.get().isNotEmpty()) { + val logMonitors = outputConditions.get().map { (service, output) -> + CompletableFuture.supplyAsync({ + monitorService(service, output) + }, pool) + }.toTypedArray() + // Will block until found, or timeout. + CompletableFuture.allOf(*logMonitors).join() + // Exit ignoring the exit code for docker-compose as we look at each container instead. + up.complete(0) + // Check for any monitors that failed to find their expected output. + failedConditions = logMonitors.any { !it.get().third } + } + up.join() // Either ended of its own accord or output conditions have all been satisfied. + // Wait for all containers to come down before we check their exit codes. + project.exec { + workingDir = project.projectDir + commandLine = baseArguments + listOf("stop") + } + exitCodeConditions.get().forEach { (service, expectedExitCode) -> + val exitCode = exitCodes[service] + logger.info("Service ($service) exited with: $exitCode, expected $expectedExitCode") + if (exitCode != expectedExitCode) { + failedConditions = true + } + } + if (failedConditions) { + logger.info("Failed Conditions") + throw GradleException("Failed conditions") + } + } + } + + open class DockerComposeDown : DockerCompose() { + + @TaskAction + fun down() { + project.exec { + workingDir(project.projectDir) + commandLine(baseArguments + listOf("down", "-v")) + } + } + } + + override fun apply(pluginProject: Project): Unit = pluginProject.run { + val cleanUpBefore by tasks.registering(DockerComposeDown::class) { + group = "Isle Tests" + description = "Clean up resources before running test (if interrupted externally, etc)" + } + + // Placeholder which can be overridden in tests. + val setUp by tasks.registering(DockerCompose::class) { + group = "Isle Tests" + description = "Prepare to run test" + dependsOn(cleanUpBefore) + } + + val cleanUpAfter by tasks.registering(DockerComposeDown::class) { + group = "Isle Tests" + description = "Clean up resources after running test" + } + + tasks.register("test") { + group = "Isle Tests" + description = "Perform test" + dependsOn(setUp) + finalizedBy(cleanUpAfter) + } + } + +} \ No newline at end of file diff --git a/src/main/kotlin/plugins/TestsPlugin.kt b/src/main/kotlin/plugins/TestsPlugin.kt new file mode 100644 index 0000000..9d903ea --- /dev/null +++ b/src/main/kotlin/plugins/TestsPlugin.kt @@ -0,0 +1,46 @@ +package plugins + +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.kotlin.dsl.* +import plugins.BuildCtlPlugin.BuildCtlLoadImage +import plugins.IslePlugin.Companion.isDockerProject +import plugins.TestPlugin.DockerComposeUp + + +// Generate reports via Syft and Grype. +@Suppress("unused") +class TestsPlugin : Plugin { + + companion object { + // Check if the project should have docker related tasks. + val Project.isDockerComposeProject: Boolean + get() = projectDir.resolve("docker-compose.yml").exists() + } + + override fun apply(pluginProject: Project): Unit = pluginProject.run { + allprojects { + // Auto-apply plugins to relevant projects in the "tests" folder of docker projects. + if (isDockerProject) { + subprojects { + if (isDockerComposeProject) { + apply() + + // Enforce build order for dependencies. + tasks.withType { + val services = dockerCompose.services.keys + val metadata = project.rootProject.allprojects + .filter { it.isDockerProject && services.contains(it.name) } + .map { it.tasks.named("load").flatMap { task -> task.metadata } } + metadataFiles.setFrom(metadata) + } + } + } + tasks.register("test") { + description = "Test docker image(s)" + dependsOn(project.subprojects.mapNotNull { it.tasks.matching { task -> task.name == "test" } }) + } + } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tasks/BuildCtl.kt b/src/main/kotlin/tasks/BuildCtl.kt new file mode 100644 index 0000000..1886a8c --- /dev/null +++ b/src/main/kotlin/tasks/BuildCtl.kt @@ -0,0 +1,24 @@ +package tasks + +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.* +import org.gradle.kotlin.dsl.property +import plugins.BuildKitPlugin.Companion.buildKitBuilder +import plugins.BuildKitPlugin.Companion.buildKitExecutable + +abstract class BuildCtl : DefaultTask() { + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + val executable = project.objects.fileProperty().convention(project.buildKitExecutable) + + @Input + val builder = project.objects.property().convention(project.buildKitBuilder) + + @get:Internal + val executablePath: String + get() = executable.get().asFile.absolutePath + + @get:Internal + val hostEnvironmentVariables: Map + get() = mapOf("BUILDKIT_HOST" to "docker-container://${builder.get()}") +} \ No newline at end of file diff --git a/src/main/kotlin/tasks/DockerBuild.kt b/src/main/kotlin/tasks/DockerBuild.kt deleted file mode 100644 index 9af9f80..0000000 --- a/src/main/kotlin/tasks/DockerBuild.kt +++ /dev/null @@ -1,340 +0,0 @@ -package tasks - -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import com.github.dockerjava.api.DockerClient -import com.github.dockerjava.api.command.RootFS -import com.github.dockerjava.api.exception.NotFoundException -import com.github.dockerjava.api.model.ContainerConfig -import org.gradle.api.DefaultTask -import org.gradle.api.file.RegularFile -import org.gradle.api.model.ObjectFactory -import org.gradle.api.tasks.* -import org.gradle.kotlin.dsl.* -import utils.DockerCommandOptions -import utils.DockerCommandOptions.Option -import utils.dockerPluginProject -import utils.imageTags - -// Wrapper around a call to `docker buildx build`, please refer to the documentation for more information: -// https://github.com/docker/buildx#documentation -open class DockerBuild : DefaultTask() { - - // Not actually the image digest but rather an approximation that ignores timestamps, etc. - // So we do not build/test unless the image has actually changed, it only checks contents & configuration. - data class ApproximateDigest(val config: ContainerConfig, val rootFS: RootFS) - - @Suppress("unused") - class Options constructor(objects: ObjectFactory) : DockerCommandOptions { - // Add a custom host-to-IP mapping (host:ip) - @Input - @Optional - @Option("--add-host") - val addHosts = objects.listProperty() - - // Allow extra privileged entitlement, e.g. network.host, security.insecure - @Input - @Optional - @Option("--allow") - val allows = objects.listProperty() - - // Set build-time variables - @Input - @Optional - @Option("--build-arg") - val buildArgs = objects.mapProperty() - - // Override the configured builder instance - @Input - @Optional - @Option("--builder") - val builder = objects.property() - - // External cache sources (eg. user/app:cache,type=local,src=path/to/dir) - @Input - @Optional - @Option("--cache-from") - val cacheFrom = objects.setProperty() - - // Cache export destinations (eg. user/app:cache,type=local,dest=path/to/dir) - @Input - @Optional - @Option("--cache-to") - val cacheTo = objects.setProperty() - - // Name of the Dockerfile (Default is 'PATH/Dockerfile') - @InputFile - @PathSensitive(PathSensitivity.RELATIVE) - @Optional - @Option("--file") - val dockerFile = objects.fileProperty() - - // Write the image ID to the file - @OutputFile - @Option("--iidfile") - val imageIdFile = objects.fileProperty() - - // --label stringArray - // Set metadata for an image - @Input - @Optional - @Option("--label") - val labels = objects.mapProperty() - - // Shorthand for --output=type=docker - @Input - @Optional - @Option("--load") - val load = objects.property().convention(false) - - // Set the networking mode for the RUN instructions during build (default "default") - @Input - @Optional - @Option("--network") - val network = objects.property() - - // Do not use cache when building the image - @Input - @Optional - @Option("--no-cache") - val noCache = objects.property().convention(false) - - // Output destination (format: type=local,dest=path) - @Input - @Optional - @Option("--output") - val output = objects.listProperty() - - // Set target platform for build - @Input - @Optional - @Option("--platform") - val platforms = objects.listProperty() - - // Set type of progress output (auto, plain, tty) - // Use plain to show container output - @Input - @Optional - @Option("--progress") - val progress = objects.property().convention("plain") - - // Always attempt to pull a newer version of the image - @Input - @Optional - @Option("--pull") - val pull = objects.property().convention(false) - - // Shorthand for --output=type=registry - @Input - @Optional - @Option("--push") - val push = objects.property().convention(false) - - // Secret file to expose to the build: id=mysecret,src=/local/secret - @Input - @Optional - @Option("--secret") - val secrets = objects.listProperty() - - // SSH agent socket or keys to expose to the build (format: default|[=|[,]]) - @Input - @Optional - @Option("--ssh") - val ssh = objects.listProperty() - - // Name and optionally a tag in the 'name:tag' format - @Input - @Optional - @Option("--tag") - val tags = objects.listProperty() - } - - @Nested - val options = Options(project.objects) - - // PATH (i.e. Docker build context) - @InputFiles - @PathSensitive(PathSensitivity.RELATIVE) - val context = project.objects.fileTree().setDir(project.layout.projectDirectory) - - // A json file whose contents can be used to uniquely identify an image by contents. We do not actually need to - // generate a hash as Gradle will do that when computing dependencies between tasks. This is only used to prevent - // building / testing when the upstream images have not changed. - @OutputFile - val digest = project.objects.fileProperty().convention(project.layout.buildDirectory.file("$name-digest.json")) - - // Gets list of images names without repository or tag, required to build this image. - // This assumes all images are building within this project. - private val requiredImages = options.dockerFile.map { file -> - file.asFile.readText().let { text -> - ("\\" + '$' + """\{repository\}/(?[^:@]+)""") - .toRegex() - .findAll(text) - .map { it.groups["image"]!!.value } - .toSet() - } - } - - // The approximate digests of images required to build this one. This is only used to prevent building / testing - // when the upstream images have not changed. - @InputFiles - @PathSensitive(PathSensitivity.RELATIVE) - @Suppress("unused") - val sourceImageDigests = project.objects.listProperty().convention( - requiredImages.map { images -> - images.mapNotNull { image -> - dockerBuildTasks(name)[image]?.get()?.digest?.get() - } - } - ) - - init { - // Exclude changes to files/directories mentioned in the Docker ignore file. - val ignore = context.dir.resolve(".dockerignore") - if (ignore.exists()) { - ignore.readLines().forEach { line -> - context.exclude(line) - } - } - - options.run { - // Get project properties used to set defaults. - val pluginProject = project.dockerPluginProject() - val buildPlatforms: Set by pluginProject.extra - val cacheFromEnabled: Boolean by pluginProject.extra - val cacheToEnabled: Boolean by pluginProject.extra - val cacheFromRepositories: Set by pluginProject.extra - val cacheToRepositories: Set by pluginProject.extra - val cacheToMode: String by pluginProject.extra - val noBuildCache: Boolean by pluginProject.extra - val isDockerBuild: Boolean by pluginProject.extra - - // Assume docker file is in the project directory. - dockerFile.convention(project.layout.projectDirectory.file("Dockerfile")) - // We always want to generate an imageIdFile if applicable i.e. when --load is specified. - imageIdFile.convention(project.layout.buildDirectory.file("$name-imageId.txt")) - // It is not possible to use --platform with "docker" builder. - if (!isDockerBuild) { - platforms.convention(buildPlatforms) - } - // Load if no platform is given as it will be the host platform so we can safely `load`. - // Also cannot load while pushing: https://github.com/docker/buildx/issues/177 - load.convention(platforms.map { !push.get() && it.isEmpty() }) - if (!noBuildCache) { - // Cache from user provided repositories. - if (cacheFromEnabled) { - cacheFrom.convention( - cacheFromRepositories.map { repository -> - "type=registry,ref=$repository/${project.name}:cache" - } - ) - } - // Cache to registry or cache inline. - if (cacheToEnabled) { - cacheTo.convention( - cacheToRepositories.map { repository -> - when (cacheToMode) { - "min", "max" -> "type=registry,mode=$cacheToMode,ref=$repository/${project.name}:cache" - "inline" -> "type=inline" - else -> throw RuntimeException("Unknown cacheToMode $cacheToMode") - } - } - ) - } - } - // Enable / disable the local build cache. - noCache.convention(noBuildCache) - } - - // Check that another process has not removed the image since it was last built. - outputs.upToDateWhen { task -> (task as DockerBuild).imagesExist() } - - // Enforce build ordering. - this.dependsOn( - requiredImages.map { images -> - images.mapNotNull { image -> - dockerBuildTasks(name)[image] - } - } - ) - } - - // Get list of all DockerBuild tasks with the given name. - private fun dockerBuildTasks(name: String) = project.rootProject.allprojects - .filter { it.projectDir.resolve("Dockerfile").exists() }.associate { project -> - project.name to project.tasks.named(name) - } - - // Checks if all images denoted by the given tag(s) exists locally. - private fun imagesExist(): Boolean { - val dockerClient: DockerClient by project.dockerPluginProject().extra - return options.tags.get().all { tag -> - try { - dockerClient.inspectImageCmd(tag).exec() - true - } catch (e: NotFoundException) { - false - } - } - } - - // --iidfile is only generated when the image is exported to `docker images` - // i.e. with `--load` is specified. - private val shouldCreateImageIdFile - get() = options.load.get() - - // Execute the Docker build command. - private fun build() { - val exclude = if (!shouldCreateImageIdFile) listOf("--iidfile") else emptyList() - val options = options.toList(exclude) - project.exec { - workingDir = context.dir - commandLine = listOf("docker", "buildx", "build") + options + listOf(context.dir.absolutePath) - } - } - - // "push and load may not be set together at the moment", so we must manually pull after building. - // Only applies to when driver is not set to `docker`. - private fun pull() { - val pluginProject = project.dockerPluginProject() - val isDockerBuild: Boolean by pluginProject.extra - if (!isDockerBuild) { - options.tags.get().forEach { tag -> - project.exec { - workingDir = context.dir - commandLine = listOf("docker", "pull", tag) - } - } - } - } - - // Due to https://github.com/docker/buildx/issues/420 we cannot rely on the imageId file to be populated - // correctly so we take matters into our own hands. - private fun updateImageFile() { - val pluginProject = project.dockerPluginProject() - val isDockerBuild: Boolean by pluginProject.extra - val dockerClient: DockerClient by pluginProject.extra - if (!isDockerBuild) { - dockerClient.inspectImageCmd(options.tags.get().first()).exec().run { - options.imageIdFile.get().asFile.writeText(id) - } - } - } - - // We generate an approximate digest to prevent rebuilding downstream images as this will be used as an input to - // those images. - private fun updateDigest() { - val pluginProject = project.dockerPluginProject() - val dockerClient: DockerClient by pluginProject.extra - dockerClient.inspectImageCmd(options.tags.get().first()).exec().run { - digest.get().asFile.writeText(jacksonObjectMapper().writeValueAsString(ApproximateDigest(config, rootFS))) - } - } - - @TaskAction - fun exec() { - build() - pull() - updateDigest() - updateImageFile() - } -} diff --git a/src/main/kotlin/tasks/DockerBuilder.kt b/src/main/kotlin/tasks/DockerBuilder.kt deleted file mode 100644 index a9a8310..0000000 --- a/src/main/kotlin/tasks/DockerBuilder.kt +++ /dev/null @@ -1,93 +0,0 @@ -package tasks - -import org.gradle.api.DefaultTask -import org.gradle.api.model.ObjectFactory -import org.gradle.api.tasks.* -import org.gradle.kotlin.dsl.property -import utils.DockerCommandOptions -import utils.DockerCommandOptions.Option - -// Wrapper around a call to `docker buildx create`, please refer to the documentation for more information: -// https://github.com/docker/buildx#documentation -open class DockerBuilder : DefaultTask() { - - @Suppress("unused") - class Options constructor(objects: ObjectFactory) : DockerCommandOptions { - // Append a node to builder instead of changing it - @Input - @Optional - @Option("--append") - val append = objects.property().convention(false) - - // Override the configured builder instance - @Input - @Optional - @Option("--builder") - val builder = objects.property() - - // Flags for buildkitd daemon - @Input - @Optional - @Option("--buildkitd-flags") - val buildkitdFlags = objects.property() - - // BuildKit config file - @InputFile - @PathSensitive(PathSensitivity.RELATIVE) - @Optional - @Option("--config") - val config = objects.fileProperty() - - // Driver to use (available: [docker docker-container kubernetes]) - @Input - @Optional - @Option("--driver") - val driver = objects.property() - - // Options for the driver - @Input - @Optional - @Option("--driver-opt") - val driverOpts = objects.property() - - // Remove a node from builder instead of changing it - @Input - @Optional - @Option("--leave") - val leave = objects.property().convention(false) - - // Builder instance name - @Input - @Optional - @Option("--name") - val name = objects.property() - - // Create/modify node with given name - @Input - @Optional - @Option("--node") - val node = objects.property() - - // Fixed platforms for current node - @Input - @Optional - @Option("--platform") - val platform = objects.property() - - @Input - @Optional - @Option("--use") - val use = objects.property() - } - - @Nested - val options = Options(project.objects) - - @TaskAction - fun exec() { - project.exec { - workingDir = project.projectDir - commandLine = listOf("docker", "buildx", "create") + options.toList() - } - } -} diff --git a/src/main/kotlin/tasks/DockerClient.kt b/src/main/kotlin/tasks/DockerClient.kt deleted file mode 100644 index a0623dc..0000000 --- a/src/main/kotlin/tasks/DockerClient.kt +++ /dev/null @@ -1,45 +0,0 @@ -package tasks - -import com.github.dockerjava.api.DockerClient -import com.github.dockerjava.core.DefaultDockerClientConfig -import com.github.dockerjava.core.DockerClientImpl -import com.github.dockerjava.httpclient5.ApacheDockerHttpClient -import org.gradle.api.DefaultTask -import org.gradle.api.UnknownTaskException -import org.gradle.api.tasks.Internal -import org.gradle.kotlin.dsl.named -import org.gradle.kotlin.dsl.provideDelegate - -abstract class DockerClient : DefaultTask() { - - @get:Internal - val dockerClient: DockerClient by lazy { - val config = DefaultDockerClientConfig.createDefaultConfigBuilder().build() - val httpClient = ApacheDockerHttpClient.Builder() - .dockerHost(config.dockerHost) - .sslConfig(config.sslConfig) - .build() - val dockerClient = DockerClientImpl.getInstance(config, httpClient) - project.gradle.buildFinished { - dockerClient.close() - } - dockerClient - } - - private val parents by lazy { - project.run { - generateSequence(this) { it.parent } - } - } - - @get:Internal - protected val buildTask by lazy { - parents.forEach { - try { - return@lazy it.tasks.named("build") - } catch (e: UnknownTaskException) { - } - } - return@lazy null - } -} diff --git a/src/main/kotlin/tasks/DockerCompose.kt b/src/main/kotlin/tasks/DockerCompose.kt deleted file mode 100644 index e0c2d8a..0000000 --- a/src/main/kotlin/tasks/DockerCompose.kt +++ /dev/null @@ -1,212 +0,0 @@ -package tasks - -import com.fasterxml.jackson.core.JsonParser -import com.fasterxml.jackson.databind.DeserializationFeature -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import com.fasterxml.jackson.module.kotlin.KotlinModule -import com.fasterxml.jackson.module.kotlin.readValue -import com.github.dockerjava.api.command.InspectContainerResponse -import org.gradle.api.file.RegularFileProperty -import org.gradle.api.tasks.* -import org.gradle.kotlin.dsl.listProperty -import org.gradle.kotlin.dsl.mapProperty -import org.gradle.kotlin.dsl.provideDelegate -import utils.isDockerProject -import java.io.ByteArrayOutputStream -import java.io.File -import java.io.OutputStream -import java.time.Duration - -@Suppress("UnstableApiUsage", "MemberVisibilityCanBePrivate") -@CacheableTask -abstract class DockerCompose : DockerClient() { - - data class DockerComposeFile(val services: Map) { - companion object { - fun deserialize(file: File): DockerComposeFile = - ObjectMapper(YAMLFactory()) - .registerModule(KotlinModule.Builder().build()) - .configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true) - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .readValue(file) - } - } - - data class Service(val image: String) { - companion object { - @Suppress("RegExpRedundantEscape") - val regex = """\$\{(?[^:]+):-(?.+)\}""".toRegex() - } - - private fun variable() = regex - .matchEntire(image) - ?.groups - ?.get("variable") - ?.value - - fun env(image: String) = - variable() - ?.let { variable -> - variable to image - } - } - - // The docker-compose.yml file to run. - @InputFile - @PathSensitive(PathSensitivity.RELATIVE) - val dockerComposeFile = - project.objects.fileProperty().convention(project.layout.projectDirectory.file("docker-compose.yml")) - - @get:Internal - val dockerCompose by lazy { - DockerComposeFile.deserialize(dockerComposeFile.get().asFile) - } - - // Environment variables which allow us to override the image used by the service. - @get:Internal - val imageEnvironmentVariables by lazy { - dockerCompose.services.mapNotNull { (name, service) -> - project.findProject(":$name") - ?.tasks - ?.named("build", DockerBuild::class.java) - ?.get() - ?.options - ?.tags - ?.get() - ?.first() - ?.let { image -> - service.env(image) - } - }.toMap() - } - - // Not actually the image digest but rather an approximation that ignores timestamps, etc. - // So we do not run test unless the image has actually changed. - @InputFiles - @Optional - @PathSensitive(PathSensitivity.RELATIVE) - val digests = project.objects.listProperty().convention( - project.provider { - // If the name of a service matches a known image in this build we will set a dependency on it. - dockerCompose.services.mapNotNull { (name, _) -> - project.findProject(":$name") - ?.tasks - ?.named("build", DockerBuild::class.java) - ?.get() - ?.digest - } - } - ) - - // Capture the log output of the command for later inspection. - @OutputFile - val log = project.objects.fileProperty().convention(project.layout.buildDirectory.file("$name.log")) - - // Environment for docker-compose not the actual containers. - @Input - val environment = project.objects.mapProperty() - - @Internal - val info = project.objects.mapProperty() - - init { - // Rerun test if any of the files in the directory of the docker-compose.yml file changes, as likely they are - // bind mounted or secrets, etc. The could affect the outcome of the test. - inputs.dir(project.projectDir) - // By default limit max execution time to a minute. - timeout.convention(Duration.ofMinutes(5)) - // Ensure we do not leave container running if something goes wrong. - project.gradle.buildFinished { - ByteArrayOutputStream().use { output -> - invoke("down", "-v", output = output, error = output) - logger.info(output.toString()) - } - } - } - - fun invoke( - vararg args: String, - env: Map = imageEnvironmentVariables.plus(environment.get()), - ignoreExitValue: Boolean = false, - output: OutputStream? = null, - error: OutputStream? = null - ) = project.exec { - environment.putAll(env) - workingDir = dockerComposeFile.get().asFile.parentFile - isIgnoreExitValue = ignoreExitValue - if (output != null) standardOutput = output - if (error != null) errorOutput = error - commandLine( - "docker", - "compose", - "--project-name", - project.path - .replace(":", "_") - .toLowerCase() - .removePrefix("_"), - *args - ) - } - - fun up(vararg args: String, ignoreExitValue: Boolean = false) = try { - invoke("up", *args, ignoreExitValue = ignoreExitValue) - } catch (e: Exception) { - log() - throw e - } - - @Suppress("unused") - fun exec(vararg args: String) = invoke("exec", *args) - - fun stop(vararg args: String) = invoke("stop", *args) - - fun down(vararg args: String) = invoke("down", *args) - - fun pull() = dockerCompose.services.keys.mapNotNull { name -> - // Find services that do not match any projects and pull them as they must refer to an external image. - // Other images will be provided by dependency on the image digests. - if (project.rootProject.allprojects.none { it.isDockerProject && it.name == name }) name else null - }.let { services -> - if (services.isNotEmpty()) { - invoke("pull", *services.toTypedArray()) - } - } - - fun log() { - log.get().asFile.outputStream().buffered().use { writer -> - invoke("logs", "--no-color", "--timestamps", output = writer, error = writer) - } - } - - fun inspect() = ByteArrayOutputStream().use { output -> - invoke("ps", "-aq", output = output) - output - .toString() - .lines() - .filter { it.isNotEmpty() } - .map { container -> - dockerClient.inspectContainerCmd(container).exec() - } - } - - fun setUp() { - pull() - } - - fun tearDown() { - stop() - info.set(inspect().associateBy { it.config.labels["com.docker.compose.service"]!! }) - log() - down("-v") - } - - fun checkExitCodes(expected: Long) { - info.get().forEach { (name, info) -> - val state = info.state - if (state.exitCodeLong != expected) { - throw RuntimeException("Service $name exited with ${state.exitCodeLong} and status ${state.status}.") - } - } - } -} diff --git a/src/main/kotlin/tasks/DockerContainer.kt b/src/main/kotlin/tasks/DockerContainer.kt index 3a3751a..977b9e9 100644 --- a/src/main/kotlin/tasks/DockerContainer.kt +++ b/src/main/kotlin/tasks/DockerContainer.kt @@ -1,169 +1,117 @@ package tasks -import com.github.dockerjava.api.async.ResultCallback -import com.github.dockerjava.api.command.CreateContainerCmd -import com.github.dockerjava.api.command.InspectContainerResponse -import com.github.dockerjava.api.exception.NotFoundException -import com.github.dockerjava.api.exception.NotModifiedException -import com.github.dockerjava.api.model.Frame -import org.gradle.api.tasks.* +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.commons.io.output.ByteArrayOutputStream +import org.apache.commons.io.output.NullOutputStream +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.TaskAction +import org.gradle.kotlin.dsl.listProperty import org.gradle.kotlin.dsl.property -import java.time.Instant -import java.time.format.DateTimeFormatter -import kotlin.concurrent.thread - -@Suppress("UnstableApiUsage", "MemberVisibilityCanBePrivate") -@CacheableTask -abstract class DockerContainer : DockerClient() { - - // Not marked as input as the tag can change but the image contents may be the same and we do not need to rerun. - @Internal - val imageId = project.objects.property() - - // Not actually the image digest but rather an approximation that ignores timestamps, etc. - // So we do not run test unless the image has actually changed. - @InputFile - @Optional - @PathSensitive(PathSensitivity.RELATIVE) - val digest = project.objects.fileProperty() - - // Capture the log output of the command for later inspection. - @OutputFile - val log = project.objects.fileProperty().convention(project.layout.buildDirectory.file("$name.log")) - - // Identifier of the container started to run this task. - @Suppress("ANNOTATION_TARGETS_NON_EXISTENT_ACCESSOR") - @get:Internal - val containerId = project.objects.property() - - @Internal - val info = project.objects.property() - - init { - // Ensure we do not leave container running if something goes wrong. - project.gradle.buildFinished { - // May be called before creation of container if build is cancelled etc. - if (containerId.isPresent) { - remove(true) +import org.gradle.kotlin.dsl.provideDelegate + +@Suppress("unused") +class DockerContainer { + + abstract class AbstractNamedDockerContainer : DefaultTask() { + @Internal + protected val baseArguments = listOf("docker", "container") + + @Input + val name = project.objects.property() + + private val inspect by lazy { + ByteArrayOutputStream().use { output -> + project.exec { + commandLine(baseArguments + listOf("inspect", name.get())) + standardOutput = output + errorOutput = NullOutputStream() + isIgnoreExitValue = true + }.let { result -> + if (result.exitValue == 0) output.toString() else null + } + }?.let { json -> + ObjectMapper().readTree(json) } } + + protected fun exists() = inspect !== null + + protected fun running() = inspect?.get(0)?.get("State")?.get("Running")?.asBoolean() ?: false } - // To be able to update the log and view after completion we need it on a separate thread. - @get:Internal - val loggingThread by lazy { - thread(start = false) { - log.get().asFile.bufferedWriter().use { writer -> - dockerClient.logContainerCmd(containerId.get()) - .withFollowStream(true) - .withStdOut(true) - .withStdErr(true) - .exec(object : ResultCallback.Adapter() { - override fun onNext(frame: Frame) { - val timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.now()) - val payload = String(frame.payload).trim { it <= ' ' } - val line = "[$timestamp] ${frame.streamType}: $payload" - logger.info(line) - writer.write("$line\n") - } - }).awaitCompletion() + open class DockerCreateContainer : AbstractNamedDockerContainer() { + + @Input + val options = project.objects.listProperty() + + @Input + val image = project.objects.property() + + @Input + val arguments = project.objects.listProperty() + + init { + @Suppress("LeakingThis") onlyIf { + !exists() } + // Need to have a name, so we can refer to it. + options.add(name.map { "--name=${it}" }) } - } - fun create() { - if (!containerId.isPresent) { - containerId.set(dockerClient.createContainerCmd(imageId.get()).exec().id) + @TaskAction + fun create() { + project.exec { + commandLine = baseArguments + listOf( + "create" + ) + options.get() + listOf(image.get()) + arguments.get() + } } } - fun create(action: CreateContainerCmd.() -> CreateContainerCmd) { - if (!containerId.isPresent) { - dockerClient.createContainerCmd(imageId.get()).let { - containerId.set(action(it).exec().id) + open class DockerStartContainer : AbstractNamedDockerContainer() { + init { + @Suppress("LeakingThis") onlyIf { + !running() } } - } - fun start() { - try { - dockerClient.startContainerCmd(containerId.get()).exec() - } catch (e: NotModifiedException) { - // Ignore if container has already started. + @TaskAction + fun start() { + project.exec { + commandLine = baseArguments + listOf("start", name.get()) + } } - loggingThread.start() } - fun stop() { - try { - dockerClient.stopContainerCmd(containerId.get()).exec() - } catch (e: NotModifiedException) { - // Ignore if not modified, as it has already been stopped. - } catch (e: Exception) { - // Unrecoverable error, user will have to clean up their environment. - throw e + open class DockerStopContainer : AbstractNamedDockerContainer() { + init { + @Suppress("LeakingThis") onlyIf { + exists() + } } - loggingThread.join() // Container has stopped finish logging. - } - fun wait() = - dockerClient.waitContainerCmd(containerId.get()).exec(ResultCallback.Adapter())?.awaitCompletion() - - fun inspect(): InspectContainerResponse = dockerClient.inspectContainerCmd(containerId.get()).exec() - - fun remove(force: Boolean = false) { - try { - dockerClient - .removeContainerCmd(containerId.get()) - .withForce(force) - .exec() - } catch (e: NotFoundException) { - // Ignore if not found, as it has already been removed. - } catch (e: Exception) { - // Unrecoverable error, user will have to clean up their environment. - throw e + @TaskAction + fun stop() { + project.exec { + commandLine = baseArguments + listOf("stop", name.get()) + } } } - // Executes callback for each line of log output until the stream ends or the callback returns false. - fun untilOutput(callback: (String) -> Boolean) { - dockerClient.logContainerCmd(containerId.get()) - .withTailAll() - .withFollowStream(true) - .withStdOut(true) - .withStdErr(true) - .exec(object : ResultCallback.Adapter() { - override fun onNext(frame: Frame) { - val line = String(frame.payload) - if (!callback(line)) { - close() - } - super.onNext(frame) - } - })?.awaitCompletion() - } - - fun setUp() { - create() - start() - } - - fun setUp(action: CreateContainerCmd.() -> CreateContainerCmd) { - create(action) - start() - } - - fun tearDown() { - stop() - info.set(inspect()) - remove(true) - } + open class DockerRemoveContainer : AbstractNamedDockerContainer() { + init { + @Suppress("LeakingThis") onlyIf { + exists() + } + } - fun checkExitCode(expected: Long) { - val name = info.get().name - val state = info.get().state - if (state.exitCodeLong != expected) { - throw RuntimeException("Container Image: '${imageId.get()}' Name: '$name' ID: '${containerId.get()}' exited with ${state.exitCodeLong} and status ${state.status}.") + @TaskAction + fun remove() { + project.exec { + commandLine = baseArguments + listOf("rm", name.get()) + } } } -} +} \ No newline at end of file diff --git a/src/main/kotlin/tasks/DockerNetwork.kt b/src/main/kotlin/tasks/DockerNetwork.kt new file mode 100644 index 0000000..a639897 --- /dev/null +++ b/src/main/kotlin/tasks/DockerNetwork.kt @@ -0,0 +1,60 @@ +package tasks + +import org.apache.commons.io.output.NullOutputStream +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.TaskAction +import org.gradle.kotlin.dsl.property + +@Suppress("unused") +class DockerNetwork { + + abstract class AbstractDockerNetwork : DefaultTask() { + @Input + val network = project.objects.property() + + @Internal + val baseArguments = listOf("docker", "network") + + protected fun exists() = project.exec { + commandLine(baseArguments + listOf("inspect", network.get())) + standardOutput = NullOutputStream() + errorOutput = NullOutputStream() + isIgnoreExitValue = true + }.exitValue == 0 + } + + open class DockerCreateNetwork : AbstractDockerNetwork() { + + init { + @Suppress("LeakingThis") + onlyIf { + !exists() + } + } + + @TaskAction + fun create() { + project.exec { + commandLine = baseArguments + listOf("create", network.get()) + } + } + } + + open class DockerRemoveNetwork : AbstractDockerNetwork() { + init { + @Suppress("LeakingThis") + onlyIf { + exists() + } + } + + @TaskAction + fun remove() { + project.exec { + commandLine = baseArguments + listOf("rm", network.get()) + } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tasks/DockerPull.kt b/src/main/kotlin/tasks/DockerPull.kt index c3bad0e..36ba6ab 100644 --- a/src/main/kotlin/tasks/DockerPull.kt +++ b/src/main/kotlin/tasks/DockerPull.kt @@ -1,73 +1,64 @@ package tasks -import com.github.dockerjava.api.DockerClient -import com.github.dockerjava.api.async.ResultCallback +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.commons.io.output.ByteArrayOutputStream +import org.apache.commons.io.output.NullOutputStream import org.gradle.api.DefaultTask -import org.gradle.api.tasks.* -import org.gradle.kotlin.dsl.* -import utils.dockerPluginProject +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.TaskAction +import org.gradle.kotlin.dsl.property // Pulls down a docker image if not already present. -// https://docs.docker.com/engine/reference/commandline/pull/ +// https://docs.docker.com/engine/reference/commandline/pull open class DockerPull : DefaultTask() { - - // The name of the image. - @Input - val name = project.objects.property() - - // The tag of the image. - @Input - val tag = project.objects.property().convention("latest") - - // The digest of the image. @Input - @Optional - val digest = project.objects.property() + val image = project.objects.property() - // Image digest generated by remote repository, uniquely identifies the image. @OutputFile - val digestFile = project.objects.fileProperty().convention(name.flatMap { project.layout.buildDirectory.file("$it-digest.json") }) + val digestFile = project.objects.fileProperty().convention(image.flatMap { + project.layout.buildDirectory.file(it.replace(Regex("""[:/]"""), ".") + ".digest") + }) @get:Internal - val image: String by lazy { - if (digest.isPresent) { - "${name.get()}:${tag.get()}@${digest.get()}" - } else { - "${name.get()}:${tag.get()}" - } - } - - init { - // Check that another process has not removed the image since it was last built. - outputs.upToDateWhen { task -> (task as DockerPull).imageExist() } - } + val digest: String + get() = digestFile.get().asFile.readText().trim() - private fun imageExist(): Boolean { - val dockerClient: DockerClient by project.dockerPluginProject().extra - return try { - dockerClient.inspectImageCmd(image).exec().run { - digestFile.get().asFile.readText() == repoDigests.first() - } - } catch (e: Exception) { - false + private fun exists() = digestFile.get().asFile.let { file -> + file.exists() && file.readText().trim().let { digest -> + project.exec { + commandLine("docker", "inspect", digest) + standardOutput = NullOutputStream() + errorOutput = NullOutputStream() + isIgnoreExitValue = true + }.exitValue == 0 } } - private fun pull() { - val dockerClient: DockerClient by project.dockerPluginProject().extra - dockerClient.pullImageCmd(image).exec(ResultCallback.Adapter())?.awaitCompletion() - } - - private fun updateDigest() { - val dockerClient: DockerClient by project.dockerPluginProject().extra - dockerClient.inspectImageCmd(image).exec().run { - digestFile.get().asFile.writeText(repoDigests.first()) + init { + outputs.upToDateWhen { + exists() } } @TaskAction - fun exec() { - pull() - updateDigest() + fun pull() { + project.exec { + commandLine("docker", "pull", image.get()) + } + ByteArrayOutputStream().use { output -> + project.exec { + commandLine("docker", "inspect", image.get()) + standardOutput = output + } + output.toString() + }.let { output -> + val node: JsonNode = ObjectMapper().readTree(output) + val content = node.first().get("RepoDigests").first().asText().trim() + digestFile.get().asFile.writeText(content) + } } } + diff --git a/src/main/kotlin/tasks/DockerVolume.kt b/src/main/kotlin/tasks/DockerVolume.kt new file mode 100644 index 0000000..0ada095 --- /dev/null +++ b/src/main/kotlin/tasks/DockerVolume.kt @@ -0,0 +1,55 @@ +package tasks + +import org.apache.commons.io.output.NullOutputStream +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.TaskAction +import org.gradle.kotlin.dsl.property + +@Suppress("unused") +class DockerVolume { + + abstract class AbstractDockerVolume : DefaultTask() { + @Input + val volume = project.objects.property() + + protected fun exists() = project.exec { + commandLine("docker", "volume", "inspect", volume.get()) + standardOutput = NullOutputStream() + errorOutput = NullOutputStream() + isIgnoreExitValue = true + }.exitValue == 0 + } + + open class DockerCreateVolume : AbstractDockerVolume() { + + init { + outputs.upToDateWhen { + exists() + } + } + + @TaskAction + fun create() { + project.exec { + commandLine = listOf("docker", "volume", "create", volume.get()) + } + } + } + + open class DockerRemoveVolume : AbstractDockerVolume() { + init { + @Suppress("LeakingThis") + onlyIf { + exists() + } + } + + @TaskAction + fun remove() { + project.exec { + commandLine = listOf("docker", "volume", "rm", volume.get()) + } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tasks/Download.kt b/src/main/kotlin/tasks/Download.kt index 1cf57e8..80ebfaf 100644 --- a/src/main/kotlin/tasks/Download.kt +++ b/src/main/kotlin/tasks/Download.kt @@ -1,9 +1,12 @@ -package tasks; +package tasks import org.apache.commons.io.FileUtils import org.gradle.api.DefaultTask import org.gradle.api.GradleException -import org.gradle.api.tasks.* +import org.gradle.api.tasks.CacheableTask +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.TaskAction import org.gradle.internal.hash.Hashing import org.gradle.kotlin.dsl.property import java.net.URI @@ -33,4 +36,4 @@ open class Download : DefaultTask() { if (sha256.get() != calculated) throw GradleException("Checksum does not match. Expected: ${sha256.get()}, Calculated: $calculated") } -} +} \ No newline at end of file diff --git a/src/main/kotlin/tasks/GenerateCerts.kt b/src/main/kotlin/tasks/GenerateCerts.kt deleted file mode 100644 index d8b2022..0000000 --- a/src/main/kotlin/tasks/GenerateCerts.kt +++ /dev/null @@ -1,67 +0,0 @@ -package tasks - -import org.gradle.api.DefaultTask -import org.gradle.api.tasks.* -import org.gradle.internal.jvm.Jvm -import utils.execCaptureOutput - -open class GenerateCerts : DefaultTask() { - - @InputFile - @PathSensitive(PathSensitivity.RELATIVE) - val executable = project.objects.fileProperty() - - @Internal - val dest = project.objects.directoryProperty().convention(project.layout.buildDirectory.dir("certs")) - - @OutputFile - val cert = project.objects.fileProperty().convention(dest.map { it.file("cert.pem") }) - - @OutputFile - val key = project.objects.fileProperty().convention(dest.map { it.file("privkey.pem") }) - - @OutputFile - val rootCA = project.objects.fileProperty().convention(dest.map { it.file("rootCA.pem") }) - - @OutputFile - val rootCAKey = project.objects.fileProperty().convention(dest.map { it.file("rootCA-key.pem") }) - - private val executablePath: String - get() = this@GenerateCerts.executable.get().asFile.absolutePath - - private fun execute(vararg arguments: String) { - project.exec { - commandLine = listOf(executablePath) + arguments - // Exclude JAVA_HOME as we only want to check the local certificates for the system. - environment = Jvm.current().getInheritableEnvironmentVariables(System.getenv()).filterKeys { - !setOf("JAVA_HOME").contains(it) - } - // Note this is allowed to fail on some systems the user may have to manually install the local certificate. - // See the README. - isIgnoreExitValue = true - } - } - - private fun install() { - execute("-install") - val rootStore = project.file(project.execCaptureOutput(listOf(executablePath, "-CAROOT"), "Failed to find CAROOT")) - listOf(rootCA.get().asFile, rootCAKey.get().asFile).forEach { - rootStore.resolve(it.name).copyTo(it, true) - } - } - - @TaskAction - fun exec() { - install() - execute( - "-cert-file", cert.get().asFile.absolutePath, - "-key-file", key.get().asFile.absolutePath, - "*.islandora.dev", - "islandora.dev", - "localhost", - "127.0.0.1", - "::1", - ) - } - -} diff --git a/src/main/kotlin/tasks/scan/Grype.kt b/src/main/kotlin/tasks/scan/Grype.kt deleted file mode 100644 index d6a7677..0000000 --- a/src/main/kotlin/tasks/scan/Grype.kt +++ /dev/null @@ -1,86 +0,0 @@ -package tasks.scan - -import org.gradle.api.DefaultTask -import org.gradle.api.tasks.* -import org.gradle.kotlin.dsl.* - -// Wrapper around a call to `syft`, please refer to the documentation for more information: -// https://github.com/anchore/syft -@CacheableTask -open class Grype : DefaultTask() { - - // Digest file for the image anchore/syft. - @InputFile - @PathSensitive(PathSensitivity.RELATIVE) - val grypeDigestFile = project.objects.fileProperty() - - // A json file representing the generated Software Bill of Materials. - @InputFile - @PathSensitive(PathSensitivity.RELATIVE) - val sbom = project.objects.fileProperty() - - @Input - val format = project.objects.property().convention("table") - - @Input - @Optional - val failOn = project.objects.property() - - @InputFile - @Optional - @PathSensitive(PathSensitivity.RELATIVE) - val config = project.objects.fileProperty() - - @Input - val onlyFixed = project.objects.property().convention(false) - - @OutputFile - val report = project.objects.fileProperty().convention(format.flatMap { - val dir = project.layout.buildDirectory - val name = "${project.name}-grype" - when (it) { - "json" -> dir.file("${name}.json") - "table" -> dir.file("${name}.md") - "cyclonedx" -> dir.file("${name}.xml") - else -> dir.file("${name}.txt") - } - }) - - @TaskAction - fun exec() { - sbom.get().asFile.inputStream().use { input -> - report.get().asFile.outputStream().use { output -> - // Arguments to docker. - val command = mutableListOf( - "docker", "run", - "--rm", - "-i", - "-e", "GRYPE_DB_CACHE_DIR=/cache", - "-e", "GRYPE_DB_AUTO_UPDATE=false", - "-v", "grype:/cache:", // Volume created by 'GrypeUpdateDB' task. - ) - if (config.isPresent) { - command.addAll(listOf("-v", "${config.get().asFile.absolutePath}:/grype.yaml")) - } - // Docker image - command.add(grypeDigestFile.get().asFile.readText()) - if (config.isPresent) { - command.addAll(listOf("--config", "/grype.yaml")) - } - // Arguments to grype. - if (failOn.isPresent) { - command.addAll(listOf("--fail-on", failOn.get())) - } - if (onlyFixed.get()) { - command.add("--only-fixed") - } - command.addAll( listOf("-o", format.get())) - project.exec { - standardInput = input - standardOutput = output - commandLine = command - } - } - } - } -} diff --git a/src/main/kotlin/tasks/scan/GrypeUpdateDB.kt b/src/main/kotlin/tasks/scan/GrypeUpdateDB.kt deleted file mode 100644 index 31e9dc9..0000000 --- a/src/main/kotlin/tasks/scan/GrypeUpdateDB.kt +++ /dev/null @@ -1,36 +0,0 @@ -package tasks.scan - -import com.github.dockerjava.api.model.* -import org.gradle.api.tasks.* -import tasks.DockerContainer - -// Downloads and updates the Grype Database. -// https://github.com/anchore/grype -open class GrypeUpdateDB : DockerContainer() { - - init { - // Always fetch the latest database. - outputs.upToDateWhen { false } - } - - @TaskAction - fun exec() { - dockerClient.createVolumeCmd().withName("grype").exec() - setUp { - withEnv("GRYPE_DB_CACHE_DIR=/cache") - withHostConfig( - HostConfig() - .withBinds(Bind("grype", Volume("/cache"))) - ) - withCmd( - listOf( - "db", - "update", - ) - ) - } - wait() // Wait for exit or timeout. - tearDown() - checkExitCode(0L) // Check if any of the containers exited non-zero. - } -} diff --git a/src/main/kotlin/tasks/scan/Syft.kt b/src/main/kotlin/tasks/scan/Syft.kt deleted file mode 100644 index 246ca5f..0000000 --- a/src/main/kotlin/tasks/scan/Syft.kt +++ /dev/null @@ -1,46 +0,0 @@ -package tasks.scan - -import org.gradle.api.DefaultTask -import org.gradle.api.tasks.* -import org.gradle.kotlin.dsl.* - -// Wrapper around a call to `syft`, please refer to the documentation for more information: -// https://github.com/anchore/syft -@CacheableTask -open class Syft : DefaultTask() { - - // Digest file for the image anchore/syft. - @InputFile - @PathSensitive(PathSensitivity.RELATIVE) - val syftDigestFile = project.objects.fileProperty() - - // The image to process (assumed to exits locally). - @Input - val image = project.objects.property() - - // Ensure that we re-run if the digest changes. - @InputFile - @PathSensitive(PathSensitivity.RELATIVE) - val imageDigestFile = project.objects.fileProperty() - - // A json file representing the generated Software Bill of Materials. - @OutputFile - val sbom = project.objects.fileProperty().convention(project.layout.buildDirectory.file("sbom.json")) - - @TaskAction - fun exec() { - sbom.get().asFile.outputStream().use { output -> - project.exec { - standardOutput = output - commandLine = listOf( - "docker", "run", - "--rm", - "-v", "/var/run/docker.sock:/var/run/docker.sock", - syftDigestFile.get().asFile.readText(), - "-o", "json", - image.get() - ) - } - } - } -} diff --git a/src/main/kotlin/tasks/tests/DockerComposeTest.kt b/src/main/kotlin/tasks/tests/DockerComposeTest.kt deleted file mode 100644 index 86546ec..0000000 --- a/src/main/kotlin/tasks/tests/DockerComposeTest.kt +++ /dev/null @@ -1,19 +0,0 @@ -@file:Suppress("unused") - -package tasks.tests - -import org.gradle.api.tasks.CacheableTask -import org.gradle.api.tasks.TaskAction -import tasks.DockerCompose - -@CacheableTask -open class DockerComposeTest : DockerCompose() { - - @TaskAction - fun exec() { - setUp() - up("--abort-on-container-exit") // Wait for exit or timeout. - tearDown() - checkExitCodes(0L) // Check if any of the containers exited non-zero. - } -} diff --git a/src/main/kotlin/tasks/tests/DockerContainerTest.kt b/src/main/kotlin/tasks/tests/DockerContainerTest.kt deleted file mode 100644 index aa3e0d2..0000000 --- a/src/main/kotlin/tasks/tests/DockerContainerTest.kt +++ /dev/null @@ -1,35 +0,0 @@ -package tasks.tests - -import org.gradle.api.tasks.CacheableTask -import org.gradle.api.tasks.TaskAction -import tasks.DockerContainer -import java.time.Duration.ofMinutes - -@CacheableTask -open class DockerContainerTest : DockerContainer() { - - init { - // Rerun test if any of the files in the directory changes, as likely they are - // bind mounted or secrets, etc. The could affect the outcome of the test. - inputs.dir(project.projectDir) - - // By default limit max execution time to a minute. - timeout.convention(ofMinutes(5)) - - // If there is a parent project with a build task assume that docker image is the one we want to use unless - // specified otherwise. - buildTask?.let { - val task = it.get() - imageId.convention(task.options.tags.get().first()) - digest.convention(task.digest) - } - } - - @TaskAction - open fun exec() { - setUp() - wait() // Wait for exit or timeout. - tearDown() - checkExitCode(0L) // Check if any of the containers exited non-zero. - } -} diff --git a/src/main/kotlin/tasks/tests/ServiceStartsWithDefaultsTest.kt b/src/main/kotlin/tasks/tests/ServiceStartsWithDefaultsTest.kt deleted file mode 100644 index 4b30a96..0000000 --- a/src/main/kotlin/tasks/tests/ServiceStartsWithDefaultsTest.kt +++ /dev/null @@ -1,38 +0,0 @@ -package tasks.tests - -import org.gradle.api.tasks.CacheableTask -import org.gradle.api.tasks.Input -import org.gradle.api.tasks.TaskAction -import org.gradle.kotlin.dsl.property - -// Checks that it can bring up the image with default settings. -// Then waits for its services to start, after a set period if no errors occur it will stop the container and check the -// exit code. -@Suppress("UnstableApiUsage") -@CacheableTask -open class ServiceStartsWithDefaultsTest : DockerContainerTest() { - - // Maximum amount of time to wait for an error after services have successfully started (milliseconds). - @Input - val maxWaitForFailure = project.objects.property().convention(10000) - - @Input - val waitForMessage = project.objects.property().convention("[services.d] done.") - - @TaskAction - override fun exec() { - setUp() - untilOutput { line -> - if (line.contains(waitForMessage.get())) { - logger.info("Services have successfully started") - // Services have started, wait for a fixed interval for the container to exited with an error. - Thread.sleep(maxWaitForFailure.get()) - false - } else { - true - } - } - tearDown() - checkExitCode(0L) // Check if any of the containers exited non-zero. - } -} diff --git a/src/main/kotlin/utils/DockerCommandOptions.kt b/src/main/kotlin/utils/DockerCommandOptions.kt deleted file mode 100644 index 9c58cd3..0000000 --- a/src/main/kotlin/utils/DockerCommandOptions.kt +++ /dev/null @@ -1,65 +0,0 @@ -package utils - -import org.gradle.api.file.RegularFile -import org.gradle.api.file.RegularFileProperty -import org.gradle.api.provider.ListProperty -import org.gradle.api.provider.MapProperty -import org.gradle.api.provider.Property -import org.gradle.api.provider.SetProperty -import kotlin.reflect.full.memberProperties - -// Helper functions to clean up argument processing for the various argument types. -@Suppress("UnstableApiUsage") -interface DockerCommandOptions { - // Annotation for serializing command line options to a string. - @Retention(AnnotationRetention.RUNTIME) - @Target(AnnotationTarget.PROPERTY) - annotation class Option(val option: String) - - fun toList(exclude: List = listOf()): List { - fun include(option: Option) = !exclude.contains(option.option) - - fun Property.toOption(option: Option) = - if (get() && include(option)) listOf(option.option) - else emptyList() - - fun Property.toOption(option: Option) = - if (isPresent && include(option)) listOf(option.option, get()) - else emptyList() - - fun RegularFileProperty.toOption(option: Option) = - if (isPresent && include(option)) listOf(option.option, get().asFile.absolutePath) - else emptyList() - - fun ListProperty.toOption(option: Option) = - if (include(option)) get().flatMap { listOf(option.option, it) } - else emptyList() - - fun MapProperty.toOption(option: Option) = - if (include(option)) get().flatMap { listOf(option.option, "${it.key}=${it.value}") } - else emptyList() - - fun SetProperty.toOption(option: Option) = - if (include(option)) get().flatMap { listOf(option.option, it) } - else emptyList() - - @Suppress("UNCHECKED_CAST") - return javaClass.kotlin.memberProperties.flatMap { member -> - member.annotations.filterIsInstance