Skip to content

Commit

Permalink
Resolver perf: Implement TreeFS.hierarchicalLookup for getClosestPack…
Browse files Browse the repository at this point in the history
…age (2/n) (facebook#1287)

Summary:
Pull Request resolved: facebook#1287

Implement a new method on `TreeFS` that is able to "natively" perform hierarchical lookup, such as frequently used during node_modules resolution, config path resolution, etc., and use it initially to find the closest package scope of a given candidate path (`context.getClosestPackage()`).

This operation currently dominates resolution time, with an algorithm that uses repeated `path.dirname()` and `fileSystem.lookup()`. The current algorithm is O(n^2) on path segments (~length), because the outer loop is O(n) and each lookup is O(n) - additionally, it's slow in constant time because each `path.dirname()` and `path.join()` involves unnecessarily parsing and normalising their own outputs - `path.join()` calls alone account for >30% of resolution time.

The new implementation:
 - Performs a lookup on the start path, collecting a list of all nodes along the way.
 - Looks back through each node, and checks for the existence of the subpath on each one.

*Benchmarks based on collecting and running all resolutions performed during a build of rn-tester:*
## Before
 - Total resolution time: 1210ms
 - Time in `getClosestPackage`: 910ms

## After
 - Total resolution time: 390ms (~3x faster)
 - Time in `getClosestPackage`: 105ms (~9x faster)

Reviewed By: motiz88

Differential Revision: D58062988
  • Loading branch information
robhogan authored and facebook-github-bot committed Aug 12, 2024
1 parent 021e890 commit 576f6b9
Show file tree
Hide file tree
Showing 7 changed files with 746 additions and 74 deletions.
36 changes: 36 additions & 0 deletions packages/metro-file-map/src/flow-types.js
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,42 @@ export interface FileSystem {
getSerializableSnapshot(): CacheData['fileSystemData'];
getSha1(file: Path): ?string;

/**
* Given a start path (which need not exist), a subpath and type, and
* optionally a 'breakOnSegment', performs the following:
*
* X = mixedStartPath
* do
* if basename(X) === opts.breakOnSegment
* return null
* if X + subpath exists and has type opts.subpathType
* return {
* absolutePath: realpath(X + subpath)
* containerRelativePath: relative(mixedStartPath, X)
* }
* X = dirname(X)
* while X !== dirname(X)
*
* If opts.invalidatedBy is given, collects all absolute, real paths that if
* added or removed may invalidate this result.
*
* Useful for finding the closest package scope (subpath: package.json,
* type f, breakOnSegment: node_modules) or closest potential package root
* (subpath: node_modules/pkg, type: d) in Node.js resolution.
*/
hierarchicalLookup(
mixedStartPath: string,
subpath: string,
opts: {
breakOnSegment: ?string,
invalidatedBy: ?Set<string>,
subpathType: 'f' | 'd',
},
): ?{
absolutePath: string,
containerRelativePath: string,
};

/**
* Analogous to posix lstat. If the file at `file` is a symlink, return
* information about the symlink without following it.
Expand Down
89 changes: 73 additions & 16 deletions packages/metro-file-map/src/lib/RootPathUtils.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
* @flow strict
*/

import invariant from 'invariant';
import * as path from 'path';

/**
Expand Down Expand Up @@ -77,6 +78,10 @@ export class RootPathUtils {
return this.#rootParts[this.#rootParts.length - 1 - n];
}

getParts(): $ReadOnlyArray<string> {
return this.#rootParts;
}

// absolutePath may be any well-formed absolute path.
absoluteToNormal(absolutePath: string): string {
let endOfMatchingPrefix = 0;
Expand Down Expand Up @@ -114,7 +119,7 @@ export class RootPathUtils {
absolutePath,
endOfMatchingPrefix,
upIndirectionsToPrepend,
) ?? this.#slowAbsoluteToNormal(absolutePath)
)?.collapsedPath ?? this.#slowAbsoluteToNormal(absolutePath)
);
}

Expand Down Expand Up @@ -153,25 +158,62 @@ export class RootPathUtils {

relativeToNormal(relativePath: string): string {
return (
this.#tryCollapseIndirectionsInSuffix(relativePath, 0, 0) ??
this.#tryCollapseIndirectionsInSuffix(relativePath, 0, 0)
?.collapsedPath ??
path.relative(this.#rootDir, path.join(this.#rootDir, relativePath))
);
}

// If a path is a direct ancestor of the project root (or the root itself),
// return a number with the degrees of separation, e.g. root=0, parent=1,..
// or null otherwise.
getAncestorOfRootIdx(normalPath: string): ?number {
if (normalPath === '') {
return 0;
}
if (normalPath === '..') {
return 1;
}
// Otherwise a *normal* path is only a root ancestor if it is a sequence of
// '../' segments followed by '..', so the length tells us the number of
// up fragments.
if (normalPath.endsWith(SEP_UP_FRAGMENT)) {
return (normalPath.length + 1) / 3;
}
return null;
}

// Takes a normal and relative path, and joins them efficiently into a normal
// path, including collapsing trailing '..' in the first part with leading
// project root segments in the relative part.
joinNormalToRelative(normalPath: string, relativePath: string): string {
joinNormalToRelative(
normalPath: string,
relativePath: string,
): {normalPath: string, collapsedSegments: number} {
if (normalPath === '') {
return relativePath;
return {collapsedSegments: 0, normalPath: relativePath};
}
if (relativePath === '') {
return normalPath;
return {collapsedSegments: 0, normalPath};
}
const left = normalPath + path.sep;
const rawPath = left + relativePath;
if (normalPath === '..' || normalPath.endsWith(SEP_UP_FRAGMENT)) {
return this.relativeToNormal(normalPath + path.sep + relativePath);
const collapsed = this.#tryCollapseIndirectionsInSuffix(rawPath, 0, 0);
invariant(collapsed != null, 'Failed to collapse');
return {
collapsedSegments: collapsed.collapsedSegments,
normalPath: collapsed.collapsedPath,
};
}
return normalPath + path.sep + relativePath;
return {
collapsedSegments: 0,
normalPath: rawPath,
};
}

relative(from: string, to: string): string {
return path.relative(from, to);
}

// Internal: Tries to collapse sequences like `../root/foo` for root
Expand All @@ -180,8 +222,9 @@ export class RootPathUtils {
fullPath: string, // A string ending with the relative path to process
startOfRelativePart: number, // Index of the start of part to process
implicitUpIndirections: number, // 0=root-relative, 1=dirname(root)-relative...
): ?string {
): ?{collapsedPath: string, collapsedSegments: number} {
let totalUpIndirections = implicitUpIndirections;
let collapsedSegments = 0;
// Allow any sequence of indirection fragments at the start of the
// unmatched suffix e.g /project/[../../foo], but bail out to Node's
// path.relative if we find a possible indirection after any later segment,
Expand All @@ -208,6 +251,7 @@ export class RootPathUtils {
fullPath[segmentToMaybeCollapse.length + pos] === path.sep)
) {
pos += segmentToMaybeCollapse.length + 1;
collapsedSegments++;
totalUpIndirections--;
} else {
break;
Expand All @@ -218,11 +262,15 @@ export class RootPathUtils {
// separator in this case by taking .slice(pos-1). In any other case,
// we know that fullPath[pos] is a separator.
if (pos >= fullPath.length) {
return totalUpIndirections > 0
? UP_FRAGMENT_SEP.repeat(totalUpIndirections - 1) +
'..' +
fullPath.slice(pos - 1)
: '';
return {
collapsedPath:
totalUpIndirections > 0
? UP_FRAGMENT_SEP.repeat(totalUpIndirections - 1) +
'..' +
fullPath.slice(pos - 1)
: '',
collapsedSegments,
};
}
const right = pos > 0 ? fullPath.slice(pos) : fullPath;
if (
Expand All @@ -231,13 +279,22 @@ export class RootPathUtils {
) {
// If we have no right side (or an indirection that would take us
// below the root), just ensure we don't include a trailing separtor.
return UP_FRAGMENT_SEP.repeat(totalUpIndirections).slice(0, -1);
return {
collapsedPath: UP_FRAGMENT_SEP.repeat(totalUpIndirections).slice(
0,
-1,
),
collapsedSegments,
};
}
// Optimisation for the common case, saves a concatenation.
if (totalUpIndirections === 0) {
return right;
return {collapsedPath: right, collapsedSegments};
}
return UP_FRAGMENT_SEP.repeat(totalUpIndirections) + right;
return {
collapsedPath: UP_FRAGMENT_SEP.repeat(totalUpIndirections) + right,
collapsedSegments,
};
}

// Cap the number of indirections at the total number of root segments.
Expand Down
Loading

0 comments on commit 576f6b9

Please sign in to comment.