Improve SBT caching during code generation

modules/codegen-cli/src/smithy4s/codegen/cli/CodegenCommand.scala

@@ -24,6 +24,7 @@ import smithy4s.codegen.CodegenArgs
 import smithy4s.codegen.FileType
 
 import Options._
+import smithy4s.codegen.PathRef
 
 object CodegenCommand {
 
@@ -116,7 +117,7 @@ object CodegenCommand {
             defaultDependencies ++ dependencies.getOrElse(List.empty)
           }
           CodegenArgs(
-            specsArgs,
+            specsArgs.map(PathRef(_)),
             output.getOrElse(os.pwd),
             resourseOutput.getOrElse(os.pwd),
             skip,
@@ -126,7 +127,7 @@ object CodegenCommand {
             repositories.getOrElse(List.empty),
             dependenciesWithDefaults,
             transformers.getOrElse(List.empty),
-            localJars.getOrElse(List.empty)
+            localJars.getOrElse(List.empty).map(PathRef(_))
           )
       }
 

modules/codegen-cli/test/src/smithy4s/codegen/cli/CommandParsingSpec.scala

@@ -21,6 +21,7 @@ import smithy4s.codegen.FileType
 import weaver._
 
 import Defaults.defaultDependencies
+import smithy4s.codegen.PathRef
 
 object CommandParsingSpec extends FunSuite {
 
@@ -81,7 +82,7 @@ object CommandParsingSpec extends FunSuite {
               specs = List(
                 os.pwd / "sampleSpecs" / "pizza.smithy",
                 os.pwd / "sampleSpecs" / "example.smithy"
-              ),
+              ).map(PathRef(_)),
               output = os.pwd / "target",
               resourceOutput = os.pwd / "target" / "openapi",
               skip = Set(FileType.Openapi, FileType.Scala),
@@ -94,7 +95,7 @@ object CommandParsingSpec extends FunSuite {
               localJars = List(
                 os.pwd / "lib1.jar",
                 os.pwd / "lib2.jar"
-              )
+              ).map(PathRef(_))
             )
           )
         )

modules/codegen-plugin/src/smithy4s/codegen/JsonConverters.scala

@@ -30,24 +30,30 @@ private[smithy4s] object JsonConverters {
   // This serialises a path by providing a hash of the content it points to.
   // Because the hash is part of the Json, this allows SBT to detect when a file
   // changes and invalidate its relevant caches, leading to a call to Smithy4s' code generator.
-  implicit val pathFormat: JsonFormat[os.Path] =
-    BasicJsonProtocol.projectFormat[os.Path, HashFileInfo](
+  implicit val pathRefFormat: JsonFormat[PathRef] =
+    BasicJsonProtocol.projectFormat[PathRef, HashFileInfo](
       p => {
-        if (os.isFile(p)) FileInfo.hash(p.toIO)
+        if (os.isFile(p.underlying)) FileInfo.hash(p.underlying.toIO)
         else
           // If the path is a directory, we get the hashes of all files
           // then hash the concatenation of the hash's bytes.
           FileInfo.hash(
-            p.toIO,
+            p.underlying.toIO,
             Hash(
-              os.walk(p)
+              os.walk(p.underlying)
                 .map(_.toIO)
                 .map(Hash(_))
                 .foldLeft(Array.emptyByteArray)(_ ++ _)
             )
           )
       },
-      hash => os.Path(hash.file)
+      hash => PathRef(os.Path(hash.file))
+    )
+
+  implicit val pathFormat: JsonFormat[os.Path] =
+    BasicJsonProtocol.projectFormat[os.Path, String](
+      p => p.toString,
+      str => os.Path(str)
     )
 
   implicit val fileTypeFormat: JsonFormat[FileType] =
@@ -61,8 +67,12 @@ private[smithy4s] object JsonConverters {
     )
 
   // format: off
-  type GenTarget = List[os.Path] :*: os.Path :*: os.Path :*: Set[FileType] :*: Boolean:*: Option[Set[String]] :*: Option[Set[String]] :*: List[String] :*: List[String] :*: List[String] :*: List[os.Path] :*: LNil
+  type GenTarget = List[PathRef] :*: os.Path :*: os.Path :*: Set[FileType] :*: Boolean:*: Option[Set[String]] :*: Option[Set[String]] :*: List[String] :*: List[String] :*: List[String] :*: List[PathRef] :*: LNil
   // format: on
+
+  // `output` and `resourceOutput` are intentionally serialized as strings
+  // instead paths. This is to avoid hashing the directories that are generated by smithy4s anyway
+  // See https://github.com/disneystreaming/smithy4s/issues/1495 for reference on this decision
   implicit val codegenArgsIso = LList.iso[CodegenArgs, GenTarget](
     { ca: CodegenArgs =>
       ("specs", ca.specs) :*:

modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala

@@ -403,28 +403,30 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
       (inputDirs ++ generatedFiles)
         .filter(_.exists())
         .toList
-    val outputPath = (conf / smithy4sOutputDir).value
-    val resourceOutputPath = (conf / smithy4sResourceDir).value
+    val outputPath = (conf / smithy4sOutputDir).value / "smithy4s"
+    val resourceOutputPath = (conf / smithy4sResourceDir).value / "smithy4s"
     val allowedNamespaces =
       (conf / smithy4sAllowedNamespaces).?.value.map(_.toSet)
     val excludedNamespaces =
       (conf / smithy4sExcludedNamespaces).?.value.map(_.toSet)
     val localJars =
-      (conf / smithy4sAllDependenciesAsJars).value.map(os.Path(_)).toList
+      (conf / smithy4sAllDependenciesAsJars).value.toList.sorted
+        .map(p => PathRef(os.Path(p)))
     val res =
       (conf / resolvers).value.toList.collect { case m: MavenRepository =>
         m.root
       }
-    val transforms = (conf / smithy4sModelTransformers).value
+    val transforms = (conf / smithy4sModelTransformers).value.sorted
     val s = (conf / streams).value
     val skipResources: Set[FileType] =
       if ((conf / smithy4sSmithyLibrary).value) Set.empty
       else Set(FileType.Resource)
     val skipSet = skipResources
 
     val filePaths = inputFiles.map(_.getAbsolutePath())
+    val specs = filePaths.sorted.map(p => PathRef(os.Path(p))).toList
     val codegenArgs = CodegenArgs(
-      filePaths.map(os.Path(_)).toList,
+      specs,
       output = os.Path(outputPath),
       resourceOutput = os.Path(resourceOutputPath),
       skip = skipSet,
@@ -443,14 +445,16 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
       ) {
         Function.untupled {
           Tracked.lastOutput[(Boolean, CodegenArgs), Seq[File]](
-            s.cacheStoreFactory.make("output")
+            s.cacheDirectory / "smithy4s-output"
           ) { case ((inputChanged, args), outputs) =>
             if (inputChanged || outputs.isEmpty) {
+              s.log.debug("Regenerating managed sources")
               val resPaths = smithy4s.codegen.Codegen
                 .generateToDisk(args)
                 .toList
               resPaths.map(path => new File(path.toString))
             } else {
+              s.log.debug("Using cached version of outputs")
               outputs.getOrElse(Seq.empty)
             }
           }

modules/codegen/src/smithy4s/codegen/CodegenArgs.scala

@@ -20,7 +20,7 @@ import cats.data.ValidatedNel
 import cats.syntax.all._
 
 final case class CodegenArgs(
-    specs: List[os.Path],
+    specs: List[PathRef],
     output: os.Path,
     resourceOutput: os.Path,
     skip: Set[FileType],
@@ -30,12 +30,13 @@ final case class CodegenArgs(
     repositories: List[String],
     dependencies: List[String],
     transformers: List[String],
-    localJars: List[os.Path]
+    localJars: List[PathRef]
 ) {
   def skipScala: Boolean = skip(FileType.Scala)
   def skipOpenapi: Boolean = skip(FileType.Openapi)
   def skipResources: Boolean = skip(FileType.Resource)
   def skipProto: Boolean = skip(FileType.Proto)
+
 }
 
 sealed abstract class FileType(val name: String)

modules/codegen/src/smithy4s/codegen/PathRef.scala

@@ -0,0 +1,3 @@
+package smithy4s.codegen
+
+final case class PathRef(underlying: os.Path)

modules/codegen/src/smithy4s/codegen/internals/CodegenImpl.scala

@@ -32,12 +32,12 @@ private[codegen] object CodegenImpl { self =>
 
   def generate(args: CodegenArgs): CodegenResult = {
     val (classloader, model): (ClassLoader, Model) = internals.ModelLoader.load(
-      args.specs.map(_.toIO).toSet,
+      args.specs.map(_.underlying.toIO).toSet,
       args.dependencies,
       args.repositories,
       withBuiltinTransformers(args.transformers),
       args.discoverModels,
-      args.localJars
+      args.localJars.map(_.underlying)
     )
 
     val (scalaFiles, smithyResources) = if (!args.skipScala) {
@@ -56,7 +56,7 @@ private[codegen] object CodegenImpl { self =>
       val resources = if (!skipResource) {
         SmithyResources.produce(
           args.resourceOutput,
-          args.specs,
+          args.specs.map(_.underlying),
           generatedNamespaces
         )
       } else List.empty[CodegenEntry]

modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala

@@ -34,6 +34,7 @@ import mill.api.JarManifest
 import mill.scalalib.CrossVersion.Binary
 import mill.scalalib.CrossVersion.Constant
 import mill.scalalib.CrossVersion.Full
+import smithy4s.codegen
 
 trait Smithy4sModule extends ScalaModule {
 
@@ -190,6 +191,8 @@ trait Smithy4sModule extends ScalaModule {
     val specFiles = (smithy4sGeneratedSmithyFiles() ++ smithy4sInputDirs())
       .map(_.path)
       .filter(os.exists(_))
+      .toList
+      .map(codegen.PathRef(_))
 
     val scalaOutput = smithy4sOutputDir().path
     val resourcesOutput = smithy4sResourceOutputDir().path
@@ -205,10 +208,14 @@ trait Smithy4sModule extends ScalaModule {
     val skipSet = skipResources ++ skipOpenApi
 
     val allLocalJars =
-      smithy4sAllDependenciesAsJars().map(_.path).iterator.to(List)
+      smithy4sAllDependenciesAsJars()
+        .map(_.path)
+        .iterator
+        .to(List)
+        .map(codegen.PathRef(_))
 
     val args = CodegenArgs(
-      specs = specFiles.toList,
+      specs = specFiles,
       output = scalaOutput,
       resourceOutput = resourcesOutput,
       skip = skipSet,