Merge pull request #6 from guardian/s3-file-upload

upload the original File artifacts to s3

s3-plugin-server/src/main/scala/com/gu/teamcity/ArtifactUploader.scala

@@ -1,11 +1,9 @@
 package com.gu.teamcity
 
+import java.io.File
 import java.util.Date
 
 import jetbrains.buildServer.messages.{BuildMessage1, DefaultMessagesInfo, Status}
-import jetbrains.buildServer.serverSide.artifacts.BuildArtifacts.BuildArtifactsProcessor
-import jetbrains.buildServer.serverSide.artifacts.BuildArtifacts.BuildArtifactsProcessor.Continuation
-import jetbrains.buildServer.serverSide.artifacts.{BuildArtifact, BuildArtifactsViewMode}
 import jetbrains.buildServer.serverSide.{BuildServerAdapter, SRunningBuild}
 
 import scala.util.control.NonFatal
@@ -19,34 +17,50 @@ class ArtifactUploader(config: S3ConfigManager, s3: S3) extends BuildServerAdapt
 
     report("About to upload artifacts to S3")
 
-    if (runningBuild.isArtifactsExists) {
-      runningBuild.getArtifacts(BuildArtifactsViewMode.VIEW_DEFAULT).iterateArtifacts(new BuildArtifactsProcessor {
-        def processBuildArtifact(buildArtifact: BuildArtifact) = {
-          if (buildArtifact.isFile || buildArtifact.isArchive)
-            s3.upload(config.artifactBucket, runningBuild, buildArtifact.getName, buildArtifact.getInputStream) map {
-              uploaded =>
-                if (uploaded) {
-                  Continuation.CONTINUE
-                } else {
-                  report("Not configured for uploading")
-                  Continuation.BREAK
-                }
-            } recover {
-              case NonFatal(e) => {
-                runningBuild.addBuildMessage(new BuildMessage1(DefaultMessagesInfo.SOURCE_ID, DefaultMessagesInfo.MSG_BUILD_FAILURE, Status.ERROR, new Date,
-                  s"Error uploading artifacts: ${e.getMessage}"))
-                Continuation.BREAK
-              }
-            } get
-          else
-            Continuation.CONTINUE
-        }
-      })
+
+    getAllFiles(runningBuild).foreach { case (name: String, artifact: File) =>
+      config.artifactBucket match {
+        case None => report("Target artifactBucket was not set")
+        case Some(bucket) =>
+          s3.upload(bucket, runningBuild, name, artifact).recover {
+            case NonFatal(e) =>
+              runningBuild.addBuildMessage(new BuildMessage1(DefaultMessagesInfo.SOURCE_ID, DefaultMessagesInfo.MSG_BUILD_FAILURE, Status.ERROR, new Date,
+                s"Error uploading artifacts: ${e.getMessage}"))
+          }
+      }
     }
 
     report("Artifact S3 upload complete")
   }
 
+  def getAllFiles(runningBuild: SRunningBuild): Seq[(String,File)] = {
+    if (!runningBuild.isArtifactsExists) {
+      Nil
+    } else {
+      ArtifactUploader.getChildren(runningBuild.getArtifactsDirectory)
+    }
+  }
+
   private def normalMessage(text: String) =
     new BuildMessage1(DefaultMessagesInfo.SOURCE_ID, DefaultMessagesInfo.MSG_TEXT, Status.NORMAL, new Date, text)
 }
+
+object ArtifactUploader {
+
+  def getChildren(file: File, paths: Seq[String] = Nil, current: String = ""): Seq[(String, File)] = {
+    file.listFiles.toSeq.flatMap {
+      child =>
+        if (child.isHidden) {
+          Seq()
+        } else {
+          val newPath = current + child.getName
+          if (child.isDirectory) {
+            getChildren(child, paths, newPath + File.separator)
+          } else {
+            Seq((newPath, child))
+          }
+        }
+    }
+  }
+
+}

s3-plugin-server/src/main/scala/com/gu/teamcity/ManifestUploader.scala

@@ -34,10 +34,12 @@ class ManifestUploader(config: S3ConfigManager, s3: S3) extends BuildServerAdapt
       val propertiesJSON = pretty(render(properties.foldLeft(JObject())(_ ~ _)))
       val jsBytes = propertiesJSON.getBytes("UTF-8")
 
-      s3.upload(config.buildManifestBucket, runningBuild, "build.json", new ByteArrayInputStream(jsBytes)) match {
-        case Failure(e) => runningBuild.addBuildMessage(new BuildMessage1(DefaultMessagesInfo.SOURCE_ID, DefaultMessagesInfo.MSG_BUILD_FAILURE, Status.ERROR, new Date,
-          s"Error uploading manifest: ${e.getMessage}"))
-        case Success(status) => if (status) runningBuild.addBuildMessage(normalMessage("Manifest S3 upload complete"))
+      config.buildManifestBucket.map { bucket =>
+        s3.upload(bucket, runningBuild, "build.json", new ByteArrayInputStream(jsBytes), jsBytes.length) match {
+          case Failure(e) => runningBuild.addBuildMessage(new BuildMessage1(DefaultMessagesInfo.SOURCE_ID, DefaultMessagesInfo.MSG_BUILD_FAILURE, Status.ERROR, new Date,
+            s"Error uploading manifest: ${e.getMessage}"))
+          case Success(_) => runningBuild.addBuildMessage(normalMessage("Manifest S3 upload complete"))
+        }
       }
     }
   }

s3-plugin-server/src/main/scala/com/gu/teamcity/S3.scala

@@ -1,11 +1,11 @@
 package com.gu.teamcity
 
-import java.io.{File, FileOutputStream, InputStream}
+import java.io.{InputStream, File}
 
 import com.amazonaws.ClientConfiguration
 import com.amazonaws.auth.{AWSCredentialsProviderChain, DefaultAWSCredentialsProviderChain}
 import com.amazonaws.services.s3.AmazonS3Client
-import com.amazonaws.services.s3.model.PutObjectRequest
+import com.amazonaws.services.s3.model.{ObjectMetadata, PutObjectRequest}
 import com.amazonaws.services.s3.transfer.TransferManager
 import jetbrains.buildServer.serverSide.SBuild
 
@@ -22,45 +22,25 @@ class S3(config: S3ConfigManager) {
     new AmazonS3Client(credentialsProvider, new ClientConfiguration().withMaxErrorRetry(2))
   )
 
-  def upload(targetBucket: Option[String], build: SBuild, fileName: String, contents: InputStream): Try[Boolean] =
-    (for (bucket <- targetBucket) yield
-      Try {
-        val uploadDirectory = s"${S3Plugin.cleanFullName(build)}/${build.getBuildNumber}"
-        val req = putRequestAsFile(bucket, s"$uploadDirectory/$fileName", contents)
-        val upload = transferManager.upload(req)
-        upload.waitForUploadResult()
-        true
+  def upload(bucket: String, build: SBuild, fileName: String, contents: InputStream, fileSize: Long): Try[Unit] =
+    Try {
+      val uploadDirectory = s"${S3Plugin.cleanFullName(build)}/${build.getBuildNumber}"
+      val metadata = {
+        val md = new ObjectMetadata()
+        md.setContentLength(fileSize)
+        md
       }
-    ) getOrElse Success(false)
-
-  def putRequestAsFile(bucket: String, targetName: String, contents: InputStream): PutObjectRequest = {
-    // convert to a file, then the S3 client can do parallel upload, and also not fail with reset stream problems
-    val file = MakeFile(contents)
-    val req = new PutObjectRequest(bucket, targetName, file)
-    contents.close()
-    req
-  }
-
-}
-
-/**
- * Return the input stream as a File, consuming everything but not closing it.
- */
-object MakeFile {
+      val req = new PutObjectRequest(bucket, s"$uploadDirectory/$fileName", contents, metadata)
+      val upload = transferManager.upload(req)
+      upload.waitForUploadResult()
+    }
 
-  def apply(input: InputStream): File = {
-    val file = File.createTempFile("s3-plugin", "")
-    val output = new FileOutputStream(file)
-    try {
-      val bytes = new Array[Byte](1024)
-      Iterator
-        .continually(input.read(bytes))
-        .takeWhile(-1 != _)
-        .foreach(read => output.write(bytes, 0, read))
-    } finally {
-      output.close()
+  def upload(bucket: String, build: SBuild, fileName: String, file: File): Try[Unit] =
+    Try {
+      val uploadDirectory = s"${S3Plugin.cleanFullName(build)}/${build.getBuildNumber}"
+      val req = new PutObjectRequest(bucket, s"$uploadDirectory/$fileName", file)
+      val upload = transferManager.upload(req)
+      upload.waitForUploadResult()
     }
-    file
-  }
 
-}
+}

s3-plugin-server/src/main/scala/com/gu/teamcity/TagUploader.scala

@@ -14,6 +14,8 @@ class TagUploader(config: S3ConfigManager, s3: S3) extends BuildServerAdapter {
 
     val tagJSON =pretty(render(asScalaBuffer(newTags)))
     val jsBytes = tagJSON.getBytes("UTF-8")
-    s3.upload(config.tagManifestBucket, build, s"tags.json", new ByteArrayInputStream(jsBytes))
+    config.tagManifestBucket.map { bucket =>
+      s3.upload(bucket, build, s"tags.json", new ByteArrayInputStream(jsBytes), jsBytes.length)
+    }
   }
 }

s3-plugin-server/src/test/scala/com/gu/teamcity/ArtifactUploaderSpec.scala

@@ -0,0 +1,15 @@
+package com.gu.teamcity
+
+import java.io.File
+
+import org.scalatest._
+
+class ArtifactUploaderSpec extends FlatSpec with Matchers {
+  "getChildren" should "find the right stuff" in {
+    val data = new File("../s3-plugin-server/TestResources")
+    val result = ArtifactUploader.getChildren(data)
+    // not exactly a unit test - good luck making this work in an automated build
+    result.map(_._1).sorted should be(Seq("test123/test234/aFile", "test123/anotherFile").sorted)
+  }
+
+}