Compute and cache transitive digests as files in build_resolvers (#3556)

Spawned from discussion [here](#3555 (comment)).

- Adds a builder in build_resolvers which emits a file containing the transitive digest of the library and all its imports.
  - When computing the transitive digest, we always first look for transitive digest files next to all of our immediate library deps, if they exist we merge that file into our digest and don't crawl any deeper.
    - If we can't read any transitive dependency for some reason we emit a warning and just give up in the builder, which means we will never emit a transitive digest file for that library (which is fine, just not optimal).
- The resolver reads these files to establish dependencies - for each library dependency we first check for a transitive digest file, if it exists we just read that and don't crawl that libraries deps. If it doesn't exist then we do the old behavior.
  - There is actually an additional step here that will continue the recursive crawl for libraries that have not yet been seen in order to load them into the resource provider for the analyzer.
- I also exposed a shared resource to avoid duplicate parsing of libraries between the resolver itself and the transitive digest builder. We might eventually want to migrate this to a builder as well, and serialize that information to disk instead, which would avoid some unnecessary re-parsing of libraries on rebuilds.

_test/pubspec.yaml

@@ -9,6 +9,7 @@ dev_dependencies:
   build: any
   build_config: any
   build_modules: any
+  build_resolvers: any
   build_runner: any
   build_runner_core: any
   build_test: any

_test/test/goldens/generated_build_script.dart

@@ -7,9 +7,10 @@ import 'package:build_test/builder.dart' as _i4;
 import 'package:build_config/build_config.dart' as _i5;
 import 'package:build_modules/builders.dart' as _i6;
 import 'package:build/build.dart' as _i7;
-import 'dart:isolate' as _i8;
-import 'package:build_runner/build_runner.dart' as _i9;
-import 'dart:io' as _i10;
+import 'package:build_resolvers/builder.dart' as _i8;
+import 'dart:isolate' as _i9;
+import 'package:build_runner/build_runner.dart' as _i10;
+import 'dart:io' as _i11;
 
 final _builders = <_i1.BuilderApplication>[
   _i1.apply(
@@ -129,6 +130,13 @@ final _builders = <_i1.BuilderApplication>[
         const _i7.BuilderOptions(<String, dynamic>{r'compiler': r'dart2js'}),
     appliesBuilders: const [r'build_web_compilers:dart2js_archive_extractor'],
   ),
+  _i1.apply(
+    r'build_resolvers:transitive_digests',
+    [_i8.transitiveDigestsBuilder],
+    _i1.toAllPackages(),
+    isOptional: true,
+    hideOutput: true,
+  ),
   _i1.applyPostProcess(
     r'build_modules:module_cleanup',
     _i6.moduleCleanup,
@@ -152,12 +160,12 @@ final _builders = <_i1.BuilderApplication>[
 ];
 void main(
   List<String> args, [
-  _i8.SendPort? sendPort,
+  _i9.SendPort? sendPort,
 ]) async {
-  var result = await _i9.run(
+  var result = await _i10.run(
     args,
     _builders,
   );
   sendPort?.send(result);
-  _i10.exitCode = result;
+  _i11.exitCode = result;
 }

build_resolvers/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.3.0
+
+- Improve performance for resolves by adding a builder which serializes
+  transitive digests to disk.
+
 ## 2.2.1
 
 - Allow the latest analyzer (6.x.x).

build_resolvers/build.yaml

@@ -0,0 +1,12 @@
+builders:
+  transitive_digests:
+    import: "package:build_resolvers/builder.dart"
+    builder_factories:
+      - transitiveDigestsBuilder
+    build_extensions:
+      .dart:
+        - .dart.transitive_digest
+    auto_apply: all_packages
+    is_optional: True
+    required_inputs: [".dart"]
+    build_to: cache

build_resolvers/lib/builder.dart

@@ -0,0 +1,68 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:build/build.dart';
+import 'package:convert/convert.dart';
+import 'package:crypto/crypto.dart';
+
+import 'src/build_asset_uri_resolver.dart';
+
+Builder transitiveDigestsBuilder(_) => _TransitiveDigestsBuilder();
+
+/// Computes a digest comprised of the current libraries digest as well as its
+/// transitive dependency digests, and writes it to a file next to the library.
+///
+/// For any dependency that has a transitive digest already written, we just use
+/// that and don't crawl its transitive deps, as the transitive digest includes
+/// all the information we need.
+class _TransitiveDigestsBuilder extends Builder {
+  @override
+  Future<void> build(BuildStep buildStep) async {
+    final seen = <AssetId>{buildStep.inputId};
+    final queue = [...seen];
+    final digestSink = AccumulatorSink<Digest>();
+    final byteSink = md5.startChunkedConversion(digestSink);
+
+    while (queue.isNotEmpty) {
+      final next = queue.removeLast();
+
+      // If we have a transitive digest ID available, just add that digest and
+      // continue.
+      final transitiveDigestId = next.addExtension(transitiveDigestExtension);
+      if (await buildStep.canRead(transitiveDigestId)) {
+        byteSink.add(await buildStep.readAsBytes(transitiveDigestId));
+        continue;
+      }
+
+      // Otherwise, add its digest and queue all its dependencies to crawl.
+      byteSink.add((await buildStep.digest(next)).bytes);
+      final deps = await dependenciesOf(next, buildStep);
+      if (deps == null) {
+        // We warn here but do not fail, the downside is slower builds.
+        log.warning('''
+Unable to read asset, could not compute transitive deps: $next
+
+This may cause less efficient builds, see the following doc for help:
+https://github.com/dart-lang/build/blob/master/docs/faq.md#unable-to-read-asset-could-not-compute-transitive-deps''');
+        return;
+      }
+
+      // Add all previously unseen deps to the queue.
+      for (final dep in deps) {
+        if (seen.add(dep)) queue.add(dep);
+      }
+    }
+    byteSink.close();
+    await buildStep.writeAsBytes(
+        buildStep.inputId.addExtension(transitiveDigestExtension),
+        digestSink.events.single.bytes);
+  }
+
+  @override
+  Map<String, List<String>> get buildExtensions => const {
+        '.dart': ['.dart$transitiveDigestExtension'],
+      };
+}

build_resolvers/lib/src/build_asset_uri_resolver.dart

@@ -12,19 +12,22 @@ import 'package:analyzer/file_system/memory_file_system.dart';
 // ignore: implementation_imports
 import 'package:analyzer/src/clients/build_resolvers/build_resolvers.dart';
 import 'package:build/build.dart' show AssetId, BuildStep;
+import 'package:collection/collection.dart';
 import 'package:crypto/crypto.dart';
 import 'package:graphs/graphs.dart';
 import 'package:path/path.dart' as p;
 import 'package:stream_transform/stream_transform.dart';
 
 const _ignoredSchemes = ['dart', 'dart-ext'];
 
+const transitiveDigestExtension = '.transitive_digest';
+
 class BuildAssetUriResolver extends UriResolver {
   /// A cache of the directives for each Dart library.
   ///
   /// This is stored across builds and is only invalidated if we read a file and
   /// see that it's content is different from what it was last time it was read.
-  final _cachedAssetDependencies = <AssetId, Set<AssetId>>{};
+  final _cachedAssetState = <AssetId, _AssetState>{};
 
   /// A cache of the digest for each Dart asset.
   ///
@@ -51,6 +54,11 @@ class BuildAssetUriResolver extends UriResolver {
   /// input, subsequent calls to a resolver, or a transitive import thereof.
   final _buildStepTransitivelyResolvedAssets = <BuildStep, HashSet<AssetId>>{};
 
+  /// Use the [instance] getter to get an instance.
+  BuildAssetUriResolver._();
+
+  static final BuildAssetUriResolver instance = BuildAssetUriResolver._();
+
   /// Updates [resourceProvider] and the analysis driver given by
   /// `withDriverResource`  with updated versions of [entryPoints].
   ///
@@ -72,9 +80,17 @@ class BuildAssetUriResolver extends UriResolver {
         ? await crawlAsync<AssetId, _AssetState?>(
             uncrawledIds,
             (id) => _updateCachedAssetState(id, buildStep,
-                transitivelyResolved: transitivelyResolved), (id, state) {
+                transitivelyResolved: transitivelyResolved), (id, state) async {
             if (state == null) return const [];
-            return state.dependencies.where(notCrawled);
+            // Establishes a dependency on the transitive deps digest.
+            final hasTransitiveDigestAsset = await buildStep
+                .canRead(id.addExtension(transitiveDigestExtension));
+            return hasTransitiveDigestAsset
+                // Only crawl assets that we haven't yet loaded into the
+                // analyzer if we are using transitive digests for invalidation.
+                ? state.dependencies.whereNot(_cachedAssetDigests.containsKey)
+                // Otherwise fall back on crawling all source deps.
+                : state.dependencies.where(notCrawled);
           }).whereType<_AssetState>().toList()
         : [
             for (final id in uncrawledIds)
@@ -103,32 +119,34 @@ class BuildAssetUriResolver extends UriResolver {
   /// non-null).
   Future<_AssetState?> _updateCachedAssetState(AssetId id, BuildStep buildStep,
       {Set<AssetId>? transitivelyResolved}) async {
-    final path = assetPath(id);
+    late final path = assetPath(id);
     if (!await buildStep.canRead(id)) {
       if (globallySeenAssets.contains(id)) {
         // ignore from this graph, some later build step may still be using it
         // so it shouldn't be removed from [resourceProvider], but we also
         // don't care about it's transitive imports.
         return null;
       }
-      _cachedAssetDependencies.remove(id);
+      _cachedAssetState.remove(id);
       _cachedAssetDigests.remove(id);
       if (resourceProvider.getFile(path).exists) {
         resourceProvider.deleteFile(path);
       }
-      return _AssetState(path, const []);
+      return _AssetState(path, const {});
     }
     globallySeenAssets.add(id);
     transitivelyResolved?.add(id);
     final digest = await buildStep.digest(id);
-    if (_cachedAssetDigests[id] == digest) {
-      return _AssetState(path, _cachedAssetDependencies[id]!);
+
+    final cachedAsset = _cachedAssetDigests[id];
+    if (cachedAsset == digest) {
+      return _cachedAssetState[id];
     } else {
-      final isChange = _cachedAssetDigests.containsKey(id);
+      final isChange = cachedAsset != null;
       final content = await buildStep.readAsString(id);
       if (_cachedAssetDigests[id] == digest) {
         // Cache may have been updated while reading asset content
-        return _AssetState(path, _cachedAssetDependencies[id]!);
+        return _cachedAssetState[id];
       }
       if (isChange) {
         resourceProvider.modifyFile(path, content);
@@ -137,9 +155,8 @@ class BuildAssetUriResolver extends UriResolver {
       }
       _cachedAssetDigests[id] = digest;
       _needsChangeFile.add(path);
-      final dependencies =
-          _cachedAssetDependencies[id] = _parseDirectives(content, id);
-      return _AssetState(path, dependencies);
+      return _cachedAssetState[id] =
+          _AssetState(path, _parseDependencies(content, id));
     }
   }
 
@@ -226,7 +243,7 @@ Future<String> packagePath(String package) async {
 
 /// Returns all the directives from a Dart library that can be resolved to an
 /// [AssetId].
-Set<AssetId> _parseDirectives(String content, AssetId from) => HashSet.of(
+Set<AssetId> _parseDependencies(String content, AssetId from) => HashSet.of(
       parseString(content: content, throwIfDiagnostics: false)
           .unit
           .directives
@@ -240,9 +257,17 @@ Set<AssetId> _parseDirectives(String content, AssetId from) => HashSet.of(
           .map((content) => AssetId.resolve(Uri.parse(content), from: from)),
     );
 
+/// Read the (potentially) cached dependencies of [id] based on parsing the
+/// directives, and cache the results if they weren't already cached.
+Future<Iterable<AssetId>?> dependenciesOf(
+        AssetId id, BuildStep buildStep) async =>
+    (await BuildAssetUriResolver.instance
+            ._updateCachedAssetState(id, buildStep))
+        ?.dependencies;
+
 class _AssetState {
   final String path;
-  final Iterable<AssetId> dependencies;
+  final Set<AssetId> dependencies;
 
   _AssetState(this.path, this.dependencies);
 }