From 3bb7dfb46044d52599ab1c19dbf609f2a80c2aef Mon Sep 17 00:00:00 2001 From: John Duffell Date: Wed, 3 Jun 2015 17:28:14 +0100 Subject: [PATCH 1/3] upload the original File artifacts to s3 --- .../TestResources/test123/anotherFile | 0 .../TestResources/test123/test234/aFile | 0 .../com/gu/teamcity/ArtifactUploader.scala | 66 +++++++++++-------- .../com/gu/teamcity/ManifestUploader.scala | 10 +-- .../src/main/scala/com/gu/teamcity/S3.scala | 43 ++++++------ .../scala/com/gu/teamcity/TagUploader.scala | 4 +- .../gu/teamcity/ArtifactUploaderSpec.scala | 15 +++++ 7 files changed, 87 insertions(+), 51 deletions(-) create mode 100644 s3-plugin-server/TestResources/test123/anotherFile create mode 100644 s3-plugin-server/TestResources/test123/test234/aFile create mode 100644 s3-plugin-server/src/test/scala/com/gu/teamcity/ArtifactUploaderSpec.scala diff --git a/s3-plugin-server/TestResources/test123/anotherFile b/s3-plugin-server/TestResources/test123/anotherFile new file mode 100644 index 0000000..e69de29 diff --git a/s3-plugin-server/TestResources/test123/test234/aFile b/s3-plugin-server/TestResources/test123/test234/aFile new file mode 100644 index 0000000..e69de29 diff --git a/s3-plugin-server/src/main/scala/com/gu/teamcity/ArtifactUploader.scala b/s3-plugin-server/src/main/scala/com/gu/teamcity/ArtifactUploader.scala index 5f263bc..5be0959 100644 --- a/s3-plugin-server/src/main/scala/com/gu/teamcity/ArtifactUploader.scala +++ b/s3-plugin-server/src/main/scala/com/gu/teamcity/ArtifactUploader.scala @@ -1,11 +1,9 @@ package com.gu.teamcity +import java.io.File import java.util.Date import jetbrains.buildServer.messages.{BuildMessage1, DefaultMessagesInfo, Status} -import jetbrains.buildServer.serverSide.artifacts.BuildArtifacts.BuildArtifactsProcessor -import jetbrains.buildServer.serverSide.artifacts.BuildArtifacts.BuildArtifactsProcessor.Continuation -import jetbrains.buildServer.serverSide.artifacts.{BuildArtifact, BuildArtifactsViewMode} import jetbrains.buildServer.serverSide.{BuildServerAdapter, SRunningBuild} import scala.util.control.NonFatal @@ -19,34 +17,50 @@ class ArtifactUploader(config: S3ConfigManager, s3: S3) extends BuildServerAdapt report("About to upload artifacts to S3") - if (runningBuild.isArtifactsExists) { - runningBuild.getArtifacts(BuildArtifactsViewMode.VIEW_DEFAULT).iterateArtifacts(new BuildArtifactsProcessor { - def processBuildArtifact(buildArtifact: BuildArtifact) = { - if (buildArtifact.isFile || buildArtifact.isArchive) - s3.upload(config.artifactBucket, runningBuild, buildArtifact.getName, buildArtifact.getInputStream, buildArtifact.getSize) map { - uploaded => - if (uploaded) { - Continuation.CONTINUE - } else { - report("Not configured for uploading") - Continuation.BREAK - } - } recover { - case NonFatal(e) => { - runningBuild.addBuildMessage(new BuildMessage1(DefaultMessagesInfo.SOURCE_ID, DefaultMessagesInfo.MSG_BUILD_FAILURE, Status.ERROR, new Date, - s"Error uploading artifacts: ${e.getMessage}")) - Continuation.BREAK - } - } get - else - Continuation.CONTINUE - } - }) + + getAllFiles(runningBuild).foreach { case (name: String, artifact: File) => + config.artifactBucket match { + case None => report("Target artifactBucket was not set") + case Some(bucket) => + s3.upload(bucket, runningBuild, name, artifact).recover { + case NonFatal(e) => + runningBuild.addBuildMessage(new BuildMessage1(DefaultMessagesInfo.SOURCE_ID, DefaultMessagesInfo.MSG_BUILD_FAILURE, Status.ERROR, new Date, + s"Error uploading artifacts: ${e.getMessage}")) + } + } } report("Artifact S3 upload complete") } + def getAllFiles(runningBuild: SRunningBuild): Seq[(String,File)] = { + if (!runningBuild.isArtifactsExists) { + Nil + } else { + + val root = runningBuild.getArtifactsDirectory + + ArtifactUploader.getChildren(root) + + } + } + private def normalMessage(text: String) = new BuildMessage1(DefaultMessagesInfo.SOURCE_ID, DefaultMessagesInfo.MSG_TEXT, Status.NORMAL, new Date, text) } + +object ArtifactUploader { + + def getChildren(file: File, paths: Seq[String] = Nil, current: String = ""): Seq[(String, File)] = { + file.listFiles.toSeq.flatMap { + child => + val newPath = current + child.getName + if (child.isDirectory) { + getChildren(child, paths, newPath + File.separator) + } else { + Seq((newPath, child)) + } + } + } + +} \ No newline at end of file diff --git a/s3-plugin-server/src/main/scala/com/gu/teamcity/ManifestUploader.scala b/s3-plugin-server/src/main/scala/com/gu/teamcity/ManifestUploader.scala index dfdef45..a1de98e 100644 --- a/s3-plugin-server/src/main/scala/com/gu/teamcity/ManifestUploader.scala +++ b/s3-plugin-server/src/main/scala/com/gu/teamcity/ManifestUploader.scala @@ -34,10 +34,12 @@ class ManifestUploader(config: S3ConfigManager, s3: S3) extends BuildServerAdapt val propertiesJSON = pretty(render(properties.foldLeft(JObject())(_ ~ _))) val jsBytes = propertiesJSON.getBytes("UTF-8") - s3.upload(config.buildManifestBucket, runningBuild, "build.json", new ByteArrayInputStream(jsBytes), jsBytes.length) match { - case Failure(e) => runningBuild.addBuildMessage(new BuildMessage1(DefaultMessagesInfo.SOURCE_ID, DefaultMessagesInfo.MSG_BUILD_FAILURE, Status.ERROR, new Date, - s"Error uploading manifest: ${e.getMessage}")) - case Success(status) => if (status) runningBuild.addBuildMessage(normalMessage("Manifest S3 upload complete")) + config.buildManifestBucket.map { bucket => + s3.upload(bucket, runningBuild, "build.json", new ByteArrayInputStream(jsBytes), jsBytes.length) match { + case Failure(e) => runningBuild.addBuildMessage(new BuildMessage1(DefaultMessagesInfo.SOURCE_ID, DefaultMessagesInfo.MSG_BUILD_FAILURE, Status.ERROR, new Date, + s"Error uploading manifest: ${e.getMessage}")) + case Success(_) => runningBuild.addBuildMessage(normalMessage("Manifest S3 upload complete")) + } } } } diff --git a/s3-plugin-server/src/main/scala/com/gu/teamcity/S3.scala b/s3-plugin-server/src/main/scala/com/gu/teamcity/S3.scala index 4bc5521..b7afe39 100644 --- a/s3-plugin-server/src/main/scala/com/gu/teamcity/S3.scala +++ b/s3-plugin-server/src/main/scala/com/gu/teamcity/S3.scala @@ -1,17 +1,14 @@ package com.gu.teamcity -import java.io.InputStream +import java.io.{InputStream, File} import com.amazonaws.ClientConfiguration import com.amazonaws.auth.{AWSCredentialsProviderChain, DefaultAWSCredentialsProviderChain} -import com.amazonaws.event.{ProgressEvent, ProgressListener} import com.amazonaws.services.s3.AmazonS3Client -import com.amazonaws.services.s3.model.{PutObjectRequest, ObjectMetadata} -import com.amazonaws.services.s3.transfer.{PersistableTransfer, TransferManager} -import com.amazonaws.services.s3.transfer.internal.{S3ProgressListenerChain, TransferManagerUtils, S3ProgressListener} +import com.amazonaws.services.s3.model.{ObjectMetadata, PutObjectRequest} +import com.amazonaws.services.s3.transfer.TransferManager import jetbrains.buildServer.serverSide.SBuild -import scala.concurrent.Promise import scala.util.{Success, Try} class S3(config: S3ConfigManager) { @@ -25,19 +22,25 @@ class S3(config: S3ConfigManager) { new AmazonS3Client(credentialsProvider, new ClientConfiguration().withMaxErrorRetry(2)) ) - def upload(targetBucket: Option[String], build: SBuild, fileName: String, contents: InputStream, fileSize: Long): Try[Boolean] = - (for (bucket <- targetBucket) yield - Try { - val uploadDirectory = s"${S3Plugin.cleanFullName(build)}/${build.getBuildNumber}" - val metadata = { - val md = new ObjectMetadata() - md.setContentLength(fileSize) - md - } - val req = new PutObjectRequest(bucket, s"$uploadDirectory/$fileName", contents, metadata) - val upload = transferManager.upload(req) - upload.waitForUploadResult() - true + def upload(bucket: String, build: SBuild, fileName: String, contents: InputStream, fileSize: Long): Try[Unit] = + Try { + val uploadDirectory = s"${S3Plugin.cleanFullName(build)}/${build.getBuildNumber}" + val metadata = { + val md = new ObjectMetadata() + md.setContentLength(fileSize) + md } - ) getOrElse (Success(false)) + val req = new PutObjectRequest(bucket, s"$uploadDirectory/$fileName", contents, metadata) + val upload = transferManager.upload(req) + upload.waitForUploadResult() + } + + def upload(bucket: String, build: SBuild, fileName: String, file: File): Try[Unit] = + Try { + val uploadDirectory = s"${S3Plugin.cleanFullName(build)}/${build.getBuildNumber}" + val req = new PutObjectRequest(bucket, s"$uploadDirectory/$fileName", file) + val upload = transferManager.upload(req) + upload.waitForUploadResult() + } + } diff --git a/s3-plugin-server/src/main/scala/com/gu/teamcity/TagUploader.scala b/s3-plugin-server/src/main/scala/com/gu/teamcity/TagUploader.scala index 2a8f89f..cc56648 100644 --- a/s3-plugin-server/src/main/scala/com/gu/teamcity/TagUploader.scala +++ b/s3-plugin-server/src/main/scala/com/gu/teamcity/TagUploader.scala @@ -14,6 +14,8 @@ class TagUploader(config: S3ConfigManager, s3: S3) extends BuildServerAdapter { val tagJSON =pretty(render(asScalaBuffer(newTags))) val jsBytes = tagJSON.getBytes("UTF-8") - s3.upload(config.tagManifestBucket, build, s"tags.json", new ByteArrayInputStream(jsBytes), jsBytes.length) + config.tagManifestBucket.map { bucket => + s3.upload(bucket, build, s"tags.json", new ByteArrayInputStream(jsBytes), jsBytes.length) + } } } diff --git a/s3-plugin-server/src/test/scala/com/gu/teamcity/ArtifactUploaderSpec.scala b/s3-plugin-server/src/test/scala/com/gu/teamcity/ArtifactUploaderSpec.scala new file mode 100644 index 0000000..8e0d8c4 --- /dev/null +++ b/s3-plugin-server/src/test/scala/com/gu/teamcity/ArtifactUploaderSpec.scala @@ -0,0 +1,15 @@ +package com.gu.teamcity + +import java.io.File + +import org.scalatest._ + +class ArtifactUploaderSpec extends FlatSpec with Matchers { + "getChildren" should "find the right stuff" in { + val data = new File("../s3-plugin-server/TestResources") + val result = ArtifactUploader.getChildren(data) + // not exactly a unit test - good luck making this work in an automated build + result.map(_._1).sorted should be(Seq("test123/test234/aFile", "test123/anotherFile").sorted) + } + +} From 0ceae2d06ca96bca9c0db85eb4d5e93469390683 Mon Sep 17 00:00:00 2001 From: John Duffell Date: Thu, 4 Jun 2015 14:41:32 +0100 Subject: [PATCH 2/3] upload the original File artifacts to s3 --- .../TestResources/.teamcity/evilFile | 0 .../com/gu/teamcity/ArtifactUploader.scala | 18 +++++++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 s3-plugin-server/TestResources/.teamcity/evilFile diff --git a/s3-plugin-server/TestResources/.teamcity/evilFile b/s3-plugin-server/TestResources/.teamcity/evilFile new file mode 100644 index 0000000..e69de29 diff --git a/s3-plugin-server/src/main/scala/com/gu/teamcity/ArtifactUploader.scala b/s3-plugin-server/src/main/scala/com/gu/teamcity/ArtifactUploader.scala index 5be0959..f34b8a8 100644 --- a/s3-plugin-server/src/main/scala/com/gu/teamcity/ArtifactUploader.scala +++ b/s3-plugin-server/src/main/scala/com/gu/teamcity/ArtifactUploader.scala @@ -37,11 +37,7 @@ class ArtifactUploader(config: S3ConfigManager, s3: S3) extends BuildServerAdapt if (!runningBuild.isArtifactsExists) { Nil } else { - - val root = runningBuild.getArtifactsDirectory - - ArtifactUploader.getChildren(root) - + ArtifactUploader.getChildren(runningBuild.getArtifactsDirectory) } } @@ -54,11 +50,15 @@ object ArtifactUploader { def getChildren(file: File, paths: Seq[String] = Nil, current: String = ""): Seq[(String, File)] = { file.listFiles.toSeq.flatMap { child => - val newPath = current + child.getName - if (child.isDirectory) { - getChildren(child, paths, newPath + File.separator) + if (child.isHidden) { + Seq() } else { - Seq((newPath, child)) + val newPath = current + child.getName + if (child.isDirectory) { + getChildren(child, paths, newPath + File.separator) + } else { + Seq((newPath, child)) + } } } } From 1c53c9b9a0e0c188523c65e920f30fdeeafff4fd Mon Sep 17 00:00:00 2001 From: John Duffell Date: Thu, 4 Jun 2015 15:09:18 +0100 Subject: [PATCH 3/3] remove erroneous test --- .../scala/com/gu/teamcity/MakeFileSpec.scala | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 s3-plugin-server/src/test/scala/com/gu/teamcity/MakeFileSpec.scala diff --git a/s3-plugin-server/src/test/scala/com/gu/teamcity/MakeFileSpec.scala b/s3-plugin-server/src/test/scala/com/gu/teamcity/MakeFileSpec.scala deleted file mode 100644 index 710a632..0000000 --- a/s3-plugin-server/src/test/scala/com/gu/teamcity/MakeFileSpec.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.gu.teamcity - -import java.io.ByteArrayInputStream -import java.nio.file.{Files, Paths} - -import org.scalatest._ - -class MakeFileSpec extends FlatSpec with Matchers { - - "file maker" should "turn an input stream into a file" in { - val testData = Array.fill(scala.util.Random.nextInt(4096))((scala.util.Random.nextInt(256) - 128).toByte) - val stream = new ByteArrayInputStream(testData) - val result = MakeFile(stream) - Files.readAllBytes(Paths.get(result.toURI)) should be(testData) - } - -}