Improve SBT caching during code generation

modules/codegen-plugin/src/smithy4s/codegen/JsonConverters.scala

@@ -30,24 +30,30 @@ private[smithy4s] object JsonConverters {
   // This serialises a path by providing a hash of the content it points to.
   // Because the hash is part of the Json, this allows SBT to detect when a file
   // changes and invalidate its relevant caches, leading to a call to Smithy4s' code generator.
-  implicit val pathFormat: JsonFormat[os.Path] =
-    BasicJsonProtocol.projectFormat[os.Path, HashFileInfo](
+  implicit val pathRefFormat: JsonFormat[PathRef] =
+    BasicJsonProtocol.projectFormat[PathRef, HashFileInfo](
       p => {
-        if (os.isFile(p)) FileInfo.hash(p.toIO)
+        if (os.isFile(p.underlying)) FileInfo.hash(p.underlying.toIO)
         else
           // If the path is a directory, we get the hashes of all files
           // then hash the concatenation of the hash's bytes.
           FileInfo.hash(
-            p.toIO,
+            p.underlying.toIO,
             Hash(
-              os.walk(p)
+              os.walk(p.underlying)
                 .map(_.toIO)
                 .map(Hash(_))
                 .foldLeft(Array.emptyByteArray)(_ ++ _)
             )
           )
       },
-      hash => os.Path(hash.file)
+      hash => PathRef(os.Path(hash.file))
+    )
+
+  implicit val pathFormat: JsonFormat[os.Path] =
+    BasicJsonProtocol.projectFormat[os.Path, String](
+      p => p.toString,
+      str => os.Path(str)
     )
 
   implicit val fileTypeFormat: JsonFormat[FileType] =
@@ -61,11 +67,15 @@ private[smithy4s] object JsonConverters {
     )
 
   // format: off
-  type GenTarget = List[os.Path] :*: os.Path :*: os.Path :*: Set[FileType] :*: Boolean:*: Option[Set[String]] :*: Option[Set[String]] :*: List[String] :*: List[String] :*: List[String] :*: List[os.Path] :*: LNil
+  type GenTarget = List[PathRef] :*: os.Path :*: os.Path :*: Set[FileType] :*: Boolean:*: Option[Set[String]] :*: Option[Set[String]] :*: List[String] :*: List[String] :*: List[String] :*: List[PathRef] :*: LNil
   // format: on
+
+  // `output` and `resourceOutput` are intentionally serialized as strings
+  // instead paths. This is to avoid hashing the directories that are generated by smithy4s anyway
+  // See https://github.com/disneystreaming/smithy4s/issues/1495 for reference on this decision
   implicit val codegenArgsIso = LList.iso[CodegenArgs, GenTarget](
     { ca: CodegenArgs =>
-      ("specs", ca.specs) :*:
+      ("specs", ca.specs.map(PathRef(_))) :*:
         ("output", ca.output) :*:
         ("resourceOutput", ca.resourceOutput) :*:
         ("skip", ca.skip) :*:
@@ -75,7 +85,7 @@ private[smithy4s] object JsonConverters {
         ("repositories", ca.repositories) :*:
         ("dependencies", ca.dependencies) :*:
         ("transformers", ca.transformers) :*:
-        ("localJars", ca.localJars) :*:
+        ("localJars", ca.localJars.map(PathRef(_))) :*:
         LNil
     },
     {
@@ -91,7 +101,7 @@ private[smithy4s] object JsonConverters {
           (_, transformers) :*:
           (_, localJars) :*: LNil =>
         CodegenArgs(
-          specs,
+          specs.map(_.underlying),
           output,
           resourceOutput,
           skip,
@@ -101,7 +111,7 @@ private[smithy4s] object JsonConverters {
           repositories,
           dependencies,
           transformers,
-          localJars
+          localJars.map(_.underlying)
         )
     }
   )

modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala

@@ -403,28 +403,30 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
       (inputDirs ++ generatedFiles)
         .filter(_.exists())
         .toList
-    val outputPath = (conf / smithy4sOutputDir).value
-    val resourceOutputPath = (conf / smithy4sResourceDir).value
+    val outputPath = (conf / smithy4sOutputDir).value / "smithy4s"
+    val resourceOutputPath = (conf / smithy4sResourceDir).value / "smithy4s"
     val allowedNamespaces =
       (conf / smithy4sAllowedNamespaces).?.value.map(_.toSet)
     val excludedNamespaces =
       (conf / smithy4sExcludedNamespaces).?.value.map(_.toSet)
     val localJars =
-      (conf / smithy4sAllDependenciesAsJars).value.map(os.Path(_)).toList
+      (conf / smithy4sAllDependenciesAsJars).value.toList.sorted
+        .map(p => os.Path(p))
     val res =
       (conf / resolvers).value.toList.collect { case m: MavenRepository =>
         m.root
       }
-    val transforms = (conf / smithy4sModelTransformers).value
+    val transforms = (conf / smithy4sModelTransformers).value.sorted
     val s = (conf / streams).value
     val skipResources: Set[FileType] =
       if ((conf / smithy4sSmithyLibrary).value) Set.empty
       else Set(FileType.Resource)
     val skipSet = skipResources
 
     val filePaths = inputFiles.map(_.getAbsolutePath())
+    val specs = filePaths.sorted.map(p => os.Path(p)).toList
     val codegenArgs = CodegenArgs(
-      filePaths.map(os.Path(_)).toList,
+      specs,
       output = os.Path(outputPath),
       resourceOutput = os.Path(resourceOutputPath),
       skip = skipSet,
@@ -446,11 +448,13 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
             s.cacheStoreFactory.make("output")
           ) { case ((inputChanged, args), outputs) =>
             if (inputChanged || outputs.isEmpty) {
+              s.log.debug("Regenerating managed sources")
               val resPaths = smithy4s.codegen.Codegen
                 .generateToDisk(args)
                 .toList
               resPaths.map(path => new File(path.toString))
             } else {
+              s.log.debug("Using cached version of outputs")
               outputs.getOrElse(Seq.empty)
             }
           }

modules/codegen/src/smithy4s/codegen/CodegenArgs.scala

@@ -36,6 +36,7 @@ final case class CodegenArgs(
   def skipOpenapi: Boolean = skip(FileType.Openapi)
   def skipResources: Boolean = skip(FileType.Resource)
   def skipProto: Boolean = skip(FileType.Proto)
+
 }
 
 sealed abstract class FileType(val name: String)

modules/codegen/src/smithy4s/codegen/PathRef.scala

@@ -0,0 +1,19 @@
+/*
+ *  Copyright 2021-2024 Disney Streaming
+ *
+ *  Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     https://disneystreaming.github.io/TOST-1.0.txt
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package smithy4s.codegen
+
+final case class PathRef(underlying: os.Path)

modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala

@@ -190,6 +190,7 @@ trait Smithy4sModule extends ScalaModule {
     val specFiles = (smithy4sGeneratedSmithyFiles() ++ smithy4sInputDirs())
       .map(_.path)
       .filter(os.exists(_))
+      .toList
 
     val scalaOutput = smithy4sOutputDir().path
     val resourcesOutput = smithy4sResourceOutputDir().path
@@ -205,10 +206,13 @@ trait Smithy4sModule extends ScalaModule {
     val skipSet = skipResources ++ skipOpenApi
 
     val allLocalJars =
-      smithy4sAllDependenciesAsJars().map(_.path).iterator.to(List)
+      smithy4sAllDependenciesAsJars()
+        .map(_.path)
+        .iterator
+        .to(List)
 
     val args = CodegenArgs(
-      specs = specFiles.toList,
+      specs = specFiles,
       output = scalaOutput,
       resourceOutput = resourcesOutput,
       skip = skipSet,