From 576f6b9dfee6722e58be5b1bc907c35faa1e996e Mon Sep 17 00:00:00 2001 From: Rob Hogan Date: Mon, 12 Aug 2024 04:15:43 -0700 Subject: [PATCH] Resolver perf: Implement TreeFS.hierarchicalLookup for getClosestPackage (2/n) (#1287) Summary: Pull Request resolved: https://github.com/facebook/metro/pull/1287 Implement a new method on `TreeFS` that is able to "natively" perform hierarchical lookup, such as frequently used during node_modules resolution, config path resolution, etc., and use it initially to find the closest package scope of a given candidate path (`context.getClosestPackage()`). This operation currently dominates resolution time, with an algorithm that uses repeated `path.dirname()` and `fileSystem.lookup()`. The current algorithm is O(n^2) on path segments (~length), because the outer loop is O(n) and each lookup is O(n) - additionally, it's slow in constant time because each `path.dirname()` and `path.join()` involves unnecessarily parsing and normalising their own outputs - `path.join()` calls alone account for >30% of resolution time. The new implementation: - Performs a lookup on the start path, collecting a list of all nodes along the way. - Looks back through each node, and checks for the existence of the subpath on each one. *Benchmarks based on collecting and running all resolutions performed during a build of rn-tester:* ## Before - Total resolution time: 1210ms - Time in `getClosestPackage`: 910ms ## After - Total resolution time: 390ms (~3x faster) - Time in `getClosestPackage`: 105ms (~9x faster) Reviewed By: motiz88 Differential Revision: D58062988 --- packages/metro-file-map/src/flow-types.js | 36 ++ .../metro-file-map/src/lib/RootPathUtils.js | 89 +++- packages/metro-file-map/src/lib/TreeFS.js | 404 ++++++++++++++++-- .../src/lib/__tests__/RootPathUtils-test.js | 12 + .../src/lib/__tests__/TreeFS-test.js | 193 +++++++++ packages/metro-file-map/types/flow-types.d.ts | 36 ++ .../metro/src/node-haste/DependencyGraph.js | 50 +-- 7 files changed, 746 insertions(+), 74 deletions(-) diff --git a/packages/metro-file-map/src/flow-types.js b/packages/metro-file-map/src/flow-types.js index 169f6beca..7c60a124a 100644 --- a/packages/metro-file-map/src/flow-types.js +++ b/packages/metro-file-map/src/flow-types.js @@ -179,6 +179,42 @@ export interface FileSystem { getSerializableSnapshot(): CacheData['fileSystemData']; getSha1(file: Path): ?string; + /** + * Given a start path (which need not exist), a subpath and type, and + * optionally a 'breakOnSegment', performs the following: + * + * X = mixedStartPath + * do + * if basename(X) === opts.breakOnSegment + * return null + * if X + subpath exists and has type opts.subpathType + * return { + * absolutePath: realpath(X + subpath) + * containerRelativePath: relative(mixedStartPath, X) + * } + * X = dirname(X) + * while X !== dirname(X) + * + * If opts.invalidatedBy is given, collects all absolute, real paths that if + * added or removed may invalidate this result. + * + * Useful for finding the closest package scope (subpath: package.json, + * type f, breakOnSegment: node_modules) or closest potential package root + * (subpath: node_modules/pkg, type: d) in Node.js resolution. + */ + hierarchicalLookup( + mixedStartPath: string, + subpath: string, + opts: { + breakOnSegment: ?string, + invalidatedBy: ?Set, + subpathType: 'f' | 'd', + }, + ): ?{ + absolutePath: string, + containerRelativePath: string, + }; + /** * Analogous to posix lstat. If the file at `file` is a symlink, return * information about the symlink without following it. diff --git a/packages/metro-file-map/src/lib/RootPathUtils.js b/packages/metro-file-map/src/lib/RootPathUtils.js index c9a04bdab..cfc5c8b4b 100644 --- a/packages/metro-file-map/src/lib/RootPathUtils.js +++ b/packages/metro-file-map/src/lib/RootPathUtils.js @@ -8,6 +8,7 @@ * @flow strict */ +import invariant from 'invariant'; import * as path from 'path'; /** @@ -77,6 +78,10 @@ export class RootPathUtils { return this.#rootParts[this.#rootParts.length - 1 - n]; } + getParts(): $ReadOnlyArray { + return this.#rootParts; + } + // absolutePath may be any well-formed absolute path. absoluteToNormal(absolutePath: string): string { let endOfMatchingPrefix = 0; @@ -114,7 +119,7 @@ export class RootPathUtils { absolutePath, endOfMatchingPrefix, upIndirectionsToPrepend, - ) ?? this.#slowAbsoluteToNormal(absolutePath) + )?.collapsedPath ?? this.#slowAbsoluteToNormal(absolutePath) ); } @@ -153,25 +158,62 @@ export class RootPathUtils { relativeToNormal(relativePath: string): string { return ( - this.#tryCollapseIndirectionsInSuffix(relativePath, 0, 0) ?? + this.#tryCollapseIndirectionsInSuffix(relativePath, 0, 0) + ?.collapsedPath ?? path.relative(this.#rootDir, path.join(this.#rootDir, relativePath)) ); } + // If a path is a direct ancestor of the project root (or the root itself), + // return a number with the degrees of separation, e.g. root=0, parent=1,.. + // or null otherwise. + getAncestorOfRootIdx(normalPath: string): ?number { + if (normalPath === '') { + return 0; + } + if (normalPath === '..') { + return 1; + } + // Otherwise a *normal* path is only a root ancestor if it is a sequence of + // '../' segments followed by '..', so the length tells us the number of + // up fragments. + if (normalPath.endsWith(SEP_UP_FRAGMENT)) { + return (normalPath.length + 1) / 3; + } + return null; + } + // Takes a normal and relative path, and joins them efficiently into a normal // path, including collapsing trailing '..' in the first part with leading // project root segments in the relative part. - joinNormalToRelative(normalPath: string, relativePath: string): string { + joinNormalToRelative( + normalPath: string, + relativePath: string, + ): {normalPath: string, collapsedSegments: number} { if (normalPath === '') { - return relativePath; + return {collapsedSegments: 0, normalPath: relativePath}; } if (relativePath === '') { - return normalPath; + return {collapsedSegments: 0, normalPath}; } + const left = normalPath + path.sep; + const rawPath = left + relativePath; if (normalPath === '..' || normalPath.endsWith(SEP_UP_FRAGMENT)) { - return this.relativeToNormal(normalPath + path.sep + relativePath); + const collapsed = this.#tryCollapseIndirectionsInSuffix(rawPath, 0, 0); + invariant(collapsed != null, 'Failed to collapse'); + return { + collapsedSegments: collapsed.collapsedSegments, + normalPath: collapsed.collapsedPath, + }; } - return normalPath + path.sep + relativePath; + return { + collapsedSegments: 0, + normalPath: rawPath, + }; + } + + relative(from: string, to: string): string { + return path.relative(from, to); } // Internal: Tries to collapse sequences like `../root/foo` for root @@ -180,8 +222,9 @@ export class RootPathUtils { fullPath: string, // A string ending with the relative path to process startOfRelativePart: number, // Index of the start of part to process implicitUpIndirections: number, // 0=root-relative, 1=dirname(root)-relative... - ): ?string { + ): ?{collapsedPath: string, collapsedSegments: number} { let totalUpIndirections = implicitUpIndirections; + let collapsedSegments = 0; // Allow any sequence of indirection fragments at the start of the // unmatched suffix e.g /project/[../../foo], but bail out to Node's // path.relative if we find a possible indirection after any later segment, @@ -208,6 +251,7 @@ export class RootPathUtils { fullPath[segmentToMaybeCollapse.length + pos] === path.sep) ) { pos += segmentToMaybeCollapse.length + 1; + collapsedSegments++; totalUpIndirections--; } else { break; @@ -218,11 +262,15 @@ export class RootPathUtils { // separator in this case by taking .slice(pos-1). In any other case, // we know that fullPath[pos] is a separator. if (pos >= fullPath.length) { - return totalUpIndirections > 0 - ? UP_FRAGMENT_SEP.repeat(totalUpIndirections - 1) + - '..' + - fullPath.slice(pos - 1) - : ''; + return { + collapsedPath: + totalUpIndirections > 0 + ? UP_FRAGMENT_SEP.repeat(totalUpIndirections - 1) + + '..' + + fullPath.slice(pos - 1) + : '', + collapsedSegments, + }; } const right = pos > 0 ? fullPath.slice(pos) : fullPath; if ( @@ -231,13 +279,22 @@ export class RootPathUtils { ) { // If we have no right side (or an indirection that would take us // below the root), just ensure we don't include a trailing separtor. - return UP_FRAGMENT_SEP.repeat(totalUpIndirections).slice(0, -1); + return { + collapsedPath: UP_FRAGMENT_SEP.repeat(totalUpIndirections).slice( + 0, + -1, + ), + collapsedSegments, + }; } // Optimisation for the common case, saves a concatenation. if (totalUpIndirections === 0) { - return right; + return {collapsedPath: right, collapsedSegments}; } - return UP_FRAGMENT_SEP.repeat(totalUpIndirections) + right; + return { + collapsedPath: UP_FRAGMENT_SEP.repeat(totalUpIndirections) + right, + collapsedSegments, + }; } // Cap the number of indirections at the total number of root segments. diff --git a/packages/metro-file-map/src/lib/TreeFS.js b/packages/metro-file-map/src/lib/TreeFS.js index 7b791ebd1..1bd57bd78 100644 --- a/packages/metro-file-map/src/lib/TreeFS.js +++ b/packages/metro-file-map/src/lib/TreeFS.js @@ -27,14 +27,16 @@ type DirectoryNode = Map; type FileNode = FileMetaData; type MixedNode = FileNode | DirectoryNode; +type NormalizedSymlinkTarget = {ancestorOfRootIdx: ?number, normalPath: string}; + /** * OVERVIEW: * * TreeFS is Metro's in-memory representation of the file system. It is * structured as a tree of non-empty maps and leaves (tuples), with the root - * node being representing the given `rootDir`, typically Metro's - * _project root_ (not a filesystem root). Map keys are path segments, and - * branches outside the project root are accessed via `'..'`. + * node representing the given `rootDir`, typically Metro's _project root_ + * (not a filesystem root). Map keys are path segments, and branches outside + * the project root are accessed via `'..'`. * * EXAMPLE: * @@ -77,7 +79,8 @@ type MixedNode = FileNode | DirectoryNode; * a trailing slash */ export default class TreeFS implements MutableFileSystem { - +#cachedNormalSymlinkTargets: WeakMap = new WeakMap(); + +#cachedNormalSymlinkTargets: WeakMap = + new WeakMap(); +#rootDir: Path; #rootNode: DirectoryNode = new Map(); #pathUtils: RootPathUtils; @@ -420,9 +423,16 @@ export default class TreeFS implements MutableFileSystem { _lookupByNormalPath( requestedNormalPath: string, opts: { + collectAncestors?: Array<{ + ancestorOfRootIdx: ?number, + node: DirectoryNode, + normalPath: string, + segmentName: string, + }>, // Mutable Set into which absolute real paths of traversed symlinks will // be added. Omit for performance if not needed. collectLinkPaths?: ?Set, + // Like lstat vs stat, whether to follow a symlink at the basename of // the given path, or return the details of the symlink itself. followLeaf?: boolean, @@ -430,6 +440,13 @@ export default class TreeFS implements MutableFileSystem { // traversal, useful when adding files. Will throw if an expected // directory is already present as a file. makeDirectories?: boolean, + startPathIdx?: number, + startNode?: DirectoryNode, + start?: { + ancestorOfRootIdx: ?number, + node: DirectoryNode, + pathIdx: number, + }, } = {followLeaf: true, makeDirectories: false}, ): | { @@ -448,6 +465,7 @@ export default class TreeFS implements MutableFileSystem { } | { canonicalMissingPath: string, + missingSegmentName: string, exists: false, } { // We'll update the target if we hit a symlink. @@ -456,12 +474,20 @@ export default class TreeFS implements MutableFileSystem { let seen: ?Set; // Pointer to the first character of the current path segment in // targetNormalPath. - let fromIdx = 0; - // The parent of the current segment - let parentNode = this.#rootNode; - // If a returned node is a strict ancestor of the root, this is the number - // of levels below the root, i.e. '..' is 1, '../..' is 2, otherwise null. - let ancestorOfRootIdx: ?number = null; + let fromIdx = opts.start?.pathIdx ?? 0; + // The parent of the current segment. + let parentNode = opts.start?.node ?? this.#rootNode; + // If a returned node is (an ancestor of) the root, this is the number of + // levels below the root, i.e. '' is 0, '..' is 1, '../..' is 2, otherwise + // null. + let ancestorOfRootIdx: ?number = opts.start?.ancestorOfRootIdx ?? 0; + + const collectAncestors = opts.collectAncestors; + // Used only when collecting ancestors, to avoid double-counting nodes and + // paths when traversing a symlink takes us back to rootNode and out again. + // This tracks the first character of the first segment not already + // collected. + let unseenPathFromIdx = 0; while (targetNormalPath.length > fromIdx) { const nextSepIdx = targetNormalPath.indexOf(path.sep, fromIdx); @@ -469,6 +495,7 @@ export default class TreeFS implements MutableFileSystem { const segmentName = isLastSegment ? targetNormalPath.slice(fromIdx) : targetNormalPath.slice(fromIdx, nextSepIdx); + const isUnseen = fromIdx >= unseenPathFromIdx; fromIdx = !isLastSegment ? nextSepIdx + 1 : targetNormalPath.length; if (segmentName === '.') { @@ -479,9 +506,8 @@ export default class TreeFS implements MutableFileSystem { // In normal paths all indirections are at the prefix, so we are at the // nth ancestor of the root iff the path so far is n '..' segments. - if (segmentName === '..') { - ancestorOfRootIdx = - ancestorOfRootIdx == null ? 1 : ancestorOfRootIdx + 1; + if (segmentName === '..' && ancestorOfRootIdx != null) { + ancestorOfRootIdx++; } else if (segmentNode != null) { ancestorOfRootIdx = null; } @@ -493,6 +519,7 @@ export default class TreeFS implements MutableFileSystem { ? targetNormalPath : targetNormalPath.slice(0, fromIdx - 1), exists: false, + missingSegmentName: segmentName, }; } segmentNode = new Map(); @@ -527,6 +554,17 @@ export default class TreeFS implements MutableFileSystem { // If the next node is a directory, go into it if (segmentNode instanceof Map) { parentNode = segmentNode; + if (collectAncestors && isUnseen) { + const currentPath = isLastSegment + ? targetNormalPath + : targetNormalPath.slice(0, fromIdx - 1); + collectAncestors.push({ + ancestorOfRootIdx, + node: segmentNode, + normalPath: currentPath, + segmentName, + }); + } } else { const currentPath = isLastSegment ? targetNormalPath @@ -537,6 +575,7 @@ export default class TreeFS implements MutableFileSystem { return { canonicalMissingPath: currentPath, exists: false, + missingSegmentName: segmentName, }; } @@ -551,14 +590,70 @@ export default class TreeFS implements MutableFileSystem { ); } + const remainingTargetPath = isLastSegment + ? '' + : targetNormalPath.slice(fromIdx); + // Append any subsequent path segments to the symlink target, and reset // with our new target. - targetNormalPath = isLastSegment - ? normalSymlinkTarget - : this.#pathUtils.joinNormalToRelative( - normalSymlinkTarget, - targetNormalPath.slice(fromIdx), - ); + const joinedResult = this.#pathUtils.joinNormalToRelative( + normalSymlinkTarget.normalPath, + remainingTargetPath, + ); + + targetNormalPath = joinedResult.normalPath; + + // Two special cases (covered by unit tests): + // + // If the symlink target is the root, the root should be a counted as + // an ancestor. We'd otherwise miss counting it because we normally + // push new ancestors only when entering a directory. + // + // If the symlink target is an ancestor of the root *and* joining it + // with the remaining path results in collapsing segments, e.g: + // '../..' + 'parentofroot/root/foo.js' = 'foo.js', then we must add + // parentofroot and root as ancestors. + if ( + collectAncestors && + !isLastSegment && + // No-op optimisation to bail out the common case of nothing to do. + (normalSymlinkTarget.ancestorOfRootIdx === 0 || + joinedResult.collapsedSegments > 0) + ) { + let node: MixedNode = this.#rootNode; + let collapsedPath = ''; + const reverseAncestors = []; + for ( + let i = 0; + i <= joinedResult.collapsedSegments && + /* for Flow, always true: */ node instanceof Map; + i++ + ) { + if ( + // Add the root only if the target is the root or we have + // collapsed segments. + i > 0 || + normalSymlinkTarget.ancestorOfRootIdx === 0 || + joinedResult.collapsedSegments > 0 + ) { + reverseAncestors.push({ + ancestorOfRootIdx: i, + node, + normalPath: collapsedPath, + segmentName: this.#pathUtils.getBasenameOfNthAncestor(i), + }); + } + node = node.get('..') ?? new Map(); + collapsedPath = + collapsedPath === '' ? '..' : collapsedPath + path.sep + '..'; + } + collectAncestors.push(...reverseAncestors.reverse()); + } + + // For the purpose of collecting ancestors: Ignore the traversal to + // the symlink target, and start collecting ancestors only when we + // reach the remaining part of the path. + unseenPathFromIdx = normalSymlinkTarget.normalPath.length; if (seen == null) { // Optimisation: set this lazily only when we've encountered a symlink @@ -569,16 +664,18 @@ export default class TreeFS implements MutableFileSystem { return { canonicalMissingPath: targetNormalPath, exists: false, + missingSegmentName: segmentName, }; } seen.add(targetNormalPath); fromIdx = 0; parentNode = this.#rootNode; + ancestorOfRootIdx = 0; } } invariant(parentNode === this.#rootNode, 'Unexpectedly escaped traversal'); return { - ancestorOfRootIdx: null, + ancestorOfRootIdx: 0, canonicalPath: targetNormalPath, exists: true, node: this.#rootNode, @@ -586,6 +683,247 @@ export default class TreeFS implements MutableFileSystem { }; } + /** + * Given a start path (which need not exist), a subpath and type, and + * optionally a 'breakOnSegment', performs the following: + * + * X = mixedStartPath + * do + * if basename(X) === opts.breakOnSegment + * return null + * if X + subpath exists and has type opts.subpathType + * return { + * absolutePath: realpath(X + subpath) + * containerRelativePath: relative(mixedStartPath, X) + * } + * X = dirname(X) + * while X !== dirname(X) + * + * If opts.invalidatedBy is given, collects all absolute, real paths that if + * added or removed may invalidate this result. + * + * Useful for finding the closest package scope (subpath: package.json, + * type f, breakOnSegment: node_modules) or closest potential package root + * (subpath: node_modules/pkg, type: d) in Node.js resolution. + */ + hierarchicalLookup( + mixedStartPath: string, + subpath: string, + opts: { + breakOnSegment: ?string, + invalidatedBy: ?Set, + subpathType: 'f' | 'd', + }, + ): ?{ + absolutePath: string, + containerRelativePath: string, + } { + const ancestorsOfInput: Array<{ + ancestorOfRootIdx: ?number, + node: DirectoryNode, + normalPath: string, + segmentName: string, + }> = []; + const normalPath = this._normalizePath(mixedStartPath); + const invalidatedBy = opts.invalidatedBy; + const closestLookup = this._lookupByNormalPath(normalPath, { + collectAncestors: ancestorsOfInput, + collectLinkPaths: invalidatedBy, + }); + + if (closestLookup.exists && closestLookup.node instanceof Map) { + const maybeAbsolutePathMatch = this.#checkCandidateHasSubpath( + closestLookup.canonicalPath, + subpath, + opts.subpathType, + invalidatedBy, + null, + ); + if (maybeAbsolutePathMatch != null) { + return { + absolutePath: maybeAbsolutePathMatch, + containerRelativePath: '', + }; + } + } else { + if ( + invalidatedBy && + (!closestLookup.exists || !(closestLookup.node instanceof Map)) + ) { + invalidatedBy.add( + this.#pathUtils.normalToAbsolute( + closestLookup.exists + ? closestLookup.canonicalPath + : closestLookup.canonicalMissingPath, + ), + ); + } + if ( + opts.breakOnSegment != null && + !closestLookup.exists && + closestLookup.missingSegmentName === opts.breakOnSegment + ) { + return null; + } + } + + // Let the "common root" be the nearest common ancestor of this.rootDir + // and the input path. We'll look for a match in two stages: + // 1. Every collected ancestor of the input path, from nearest to furthest, + // that is a descendent of the common root + // 2. The common root, and its ancestors. + let commonRoot = this.#rootNode; + let commonRootDepth = 0; + + // Collected ancestors do not include the lookup result itself, so go one + // further if the input path is itself a root ancestor. + if (closestLookup.exists && closestLookup.ancestorOfRootIdx != null) { + commonRootDepth = closestLookup.ancestorOfRootIdx; + invariant( + closestLookup.node instanceof Map, + 'ancestors of the root must be directories', + ); + commonRoot = closestLookup.node; + } else { + // Establish the common root by counting the '..' segments at the start + // of the collected ancestors. + for (const ancestor of ancestorsOfInput) { + if (ancestor.ancestorOfRootIdx == null) { + break; + } + commonRootDepth = ancestor.ancestorOfRootIdx; + commonRoot = ancestor.node; + } + } + + // Phase 1: Consider descendenants of the common root, from deepest to + // shallowest. + for ( + let candidateIdx = ancestorsOfInput.length - 1; + candidateIdx >= commonRootDepth; + --candidateIdx + ) { + const candidate = ancestorsOfInput[candidateIdx]; + if (candidate.segmentName === opts.breakOnSegment) { + return null; + } + const maybeAbsolutePathMatch = this.#checkCandidateHasSubpath( + candidate.normalPath, + subpath, + opts.subpathType, + invalidatedBy, + { + ancestorOfRootIdx: candidate.ancestorOfRootIdx, + node: candidate.node, + pathIdx: + candidate.normalPath.length > 0 + ? candidate.normalPath.length + 1 + : 0, + }, + ); + if (maybeAbsolutePathMatch != null) { + // Determine the input path relative to the current candidate. Note + // that the candidate path will always be canonical (real), whereas the + // input may contain symlinks, so the candidate is not necessarily a + // prefix of the input. Use the fact that each remaining candidate + // corresponds to a leading segment of the input normal path, and + // discard the first candidateIdx + 1 segments of the input path. + // + // The next 5 lines are equivalent to (but faster than) + // normalPath.split('/').slice(candidateIdx + 1).join('/'). + let prefixLength = commonRootDepth * 3; // Leading '../' + for (let i = commonRootDepth; i <= candidateIdx; i++) { + prefixLength = normalPath.indexOf(path.sep, prefixLength + 1); + } + const containerRelativePath = normalPath.slice(prefixLength + 1); + return { + absolutePath: maybeAbsolutePathMatch, + containerRelativePath, + }; + } + } + + // Phase 2: Consider the common root and its ancestors + + // This will be '', '..', '../..', etc. + let candidateNormalPath = + commonRootDepth > 0 ? normalPath.slice(0, 3 * commonRootDepth - 1) : ''; + const remainingNormalPath = normalPath.slice(commonRootDepth * 3); + + let nextNode: ?MixedNode = commonRoot; + let depthBelowCommonRoot = 0; + + while (nextNode instanceof Map) { + const maybeAbsolutePathMatch = this.#checkCandidateHasSubpath( + candidateNormalPath, + subpath, + opts.subpathType, + invalidatedBy, + null, + ); + if (maybeAbsolutePathMatch != null) { + const rootDirParts = this.#pathUtils.getParts(); + const relativeParts = + depthBelowCommonRoot > 0 + ? rootDirParts.slice( + -(depthBelowCommonRoot + commonRootDepth), + commonRootDepth > 0 ? -commonRootDepth : undefined, + ) + : []; + if (remainingNormalPath !== '') { + relativeParts.push(remainingNormalPath); + } + return { + absolutePath: maybeAbsolutePathMatch, + containerRelativePath: relativeParts.join(path.sep), + }; + } + depthBelowCommonRoot++; + candidateNormalPath = + candidateNormalPath === '' + ? '..' + : candidateNormalPath + path.sep + '..'; + nextNode = nextNode.get('..'); + } + return null; + } + + #checkCandidateHasSubpath( + normalCandidatePath: string, + subpath: string, + subpathType: 'f' | 'd', + invalidatedBy: ?Set, + start: ?{ + ancestorOfRootIdx: ?number, + node: DirectoryNode, + pathIdx: number, + }, + ): ?string { + const lookupResult = this._lookupByNormalPath( + this.#pathUtils.joinNormalToRelative(normalCandidatePath, subpath) + .normalPath, + { + collectLinkPaths: invalidatedBy, + }, + ); + if ( + lookupResult.exists && + // Should be a Map iff subpathType is directory + lookupResult.node instanceof Map === (subpathType === 'd') + ) { + return this.#pathUtils.normalToAbsolute(lookupResult.canonicalPath); + } else if (invalidatedBy) { + invalidatedBy.add( + this.#pathUtils.normalToAbsolute( + lookupResult.exists + ? lookupResult.canonicalPath + : lookupResult.canonicalMissingPath, + ), + ); + } + return null; + } + *metadataIterator(opts: { includeSymlinks: boolean, includeNodeModules: boolean, @@ -634,7 +972,7 @@ export default class TreeFS implements MutableFileSystem { parent: ?DirectoryNode, ancestorOfRootIdx: ?number, ): Iterator<[string, MixedNode]> { - if (ancestorOfRootIdx != null && parent) { + if (ancestorOfRootIdx != null && ancestorOfRootIdx > 0 && parent) { yield [ this.#pathUtils.getBasenameOfNthAncestor(ancestorOfRootIdx - 1), parent, @@ -729,7 +1067,7 @@ export default class TreeFS implements MutableFileSystem { yield* this._pathIterator( node, iterationRootParentNode, - ancestorOfRootIdx != null && ancestorOfRootIdx > 1 + ancestorOfRootIdx != null && ancestorOfRootIdx > 0 ? ancestorOfRootIdx - 1 : null, opts, @@ -743,10 +1081,10 @@ export default class TreeFS implements MutableFileSystem { _resolveSymlinkTargetToNormalPath( symlinkNode: FileMetaData, canonicalPathOfSymlink: Path, - ): Path { - let normalSymlinkTarget = this.#cachedNormalSymlinkTargets.get(symlinkNode); - if (normalSymlinkTarget != null) { - return normalSymlinkTarget; + ): NormalizedSymlinkTarget { + const cachedResult = this.#cachedNormalSymlinkTargets.get(symlinkNode); + if (cachedResult != null) { + return cachedResult; } const literalSymlinkTarget = symlinkNode[H.SYMLINK]; @@ -760,9 +1098,17 @@ export default class TreeFS implements MutableFileSystem { '..', // Symlink target is relative to its containing directory. literalSymlinkTarget, // May be absolute, in which case the above are ignored ); - normalSymlinkTarget = path.relative(this.#rootDir, absoluteSymlinkTarget); - this.#cachedNormalSymlinkTargets.set(symlinkNode, normalSymlinkTarget); - return normalSymlinkTarget; + const normalSymlinkTarget = path.relative( + this.#rootDir, + absoluteSymlinkTarget, + ); + const result = { + ancestorOfRootIdx: + this.#pathUtils.getAncestorOfRootIdx(normalSymlinkTarget), + normalPath: normalSymlinkTarget, + }; + this.#cachedNormalSymlinkTargets.set(symlinkNode, result); + return result; } _getFileData( diff --git a/packages/metro-file-map/src/lib/__tests__/RootPathUtils-test.js b/packages/metro-file-map/src/lib/__tests__/RootPathUtils-test.js index e13b1bd19..8fb693c49 100644 --- a/packages/metro-file-map/src/lib/__tests__/RootPathUtils-test.js +++ b/packages/metro-file-map/src/lib/__tests__/RootPathUtils-test.js @@ -141,4 +141,16 @@ describe.each([['win32'], ['posix']])('RootPathUtils on %s', platform => { }, ); }); + + test.each([ + ['foo', null], + ['', 0], + ['..', 1], + [p('../..'), 2], + [p('../../..'), 3], + [p('../../../foo'), null], + [p('../../../..foo'), null], + ])('getAncestorOfRootIdx (%s => %s)', (input, expected) => { + expect(pathUtils.getAncestorOfRootIdx(input)).toEqual(expected); + }); }); diff --git a/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js b/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js index c4077fce5..708855a95 100644 --- a/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js +++ b/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js @@ -213,6 +213,11 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { p('/project/bar.js'), [p('/project/foo/link-up-2')], ], + [ + p('foo/link-up-2/project/foo/link-up-2/project/bar.js'), + p('/project/bar.js'), + [p('/project/foo/link-up-2')], + ], [ p('foo/link-up-2/project/foo/link-up-2/outside/external.js'), p('/outside/external.js'), @@ -284,6 +289,194 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { }); }); + describe('hierarchicalLookup', () => { + let tfs: TreeFSType; + + beforeEach(() => { + tfs = new TreeFS({ + rootDir: p('/A/B/C'), + files: new Map( + [ + [ + p('a/1/package.json'), + ['', 0, 0, 0, '', '', './real-package.json'], + ], + [ + p('a/2/package.json'), + ['', 0, 0, 0, '', '', './notexist-package.json'], + ], + [p('a/b/c/d/link-to-C'), ['', 0, 0, 0, '', '', p('../../../..')]], + [ + p('a/b/c/d/link-to-B'), + ['', 0, 0, 0, '', '', p('../../../../..')], + ], + [ + p('a/b/c/d/link-to-A'), + ['', 0, 0, 0, '', '', p('../../../../../..')], + ], + ].concat( + [ + 'a/package.json', + // A directory named package.json should never match + 'a/b/package.json/index.js', + 'a/b/c/package.json', + 'a/b/c/d/foo.js', + 'a/1/real-package.json', + 'a/b/bar.js', + 'a/n_m/pkg/package.json', + 'a/n_m/pkg/foo.js', + 'a/n_m/pkg/subpath/deep/bar.js', + 'a/n_m/pkg/subpath/package.json', + 'a/n_m/pkg/n_m/pkg2/index.js', + 'a/n_m/pkg/n_m/pkg2/package.json', + '../../package.json', + '../../../a/b/package.json', + ].map(posixPath => [p(posixPath), ['', 0, 0, 0, '', '', 0]]), + ), + ), + }); + }); + + test.each([ + ['/A/B/C/a', '/A/B/C/a/package.json', '', []], + ['/A/B/C/a/b', '/A/B/C/a/package.json', 'b', ['/A/B/C/a/b/package.json']], + [ + '/A/B/C/a/package.json', + '/A/B/C/a/package.json', + 'package.json', + ['/A/B/C/a/package.json'], + ], + [ + '/A/B/C/a/b/notexists', + '/A/B/C/a/package.json', + 'b/notexists', + ['/A/B/C/a/b/notexists', '/A/B/C/a/b/package.json'], + ], + ['/A/B/C/a/b/c', '/A/B/C/a/b/c/package.json', '', []], + [ + '/A/B/C/other', + '/A/package.json', + 'B/C/other', + ['/A/B/C/other', '/A/B/C/package.json', '/A/B/package.json'], + ], + [ + '/A/B/C', + '/A/package.json', + 'B/C', + ['/A/B/C/package.json', '/A/B/package.json'], + ], + ['/A/B', '/A/package.json', 'B', ['/A/B/package.json']], + [ + '/A/B/foo', + '/A/package.json', + 'B/foo', + + ['/A/B/foo', '/A/B/package.json'], + ], + ['/A/foo', '/A/package.json', 'foo', ['/A/foo']], + ['/foo', null, null, ['/foo', '/package.json']], + [ + '/A/B/C/a/b/c/d/link-to-C/foo.js', + '/A/B/C/a/b/c/package.json', + 'd/link-to-C/foo.js', + [ + '/A/B/C/a/b/c/d/link-to-C', + '/A/B/C/a/b/c/d/package.json', + '/A/B/C/foo.js', + '/A/B/C/package.json', + ], + ], + [ + '/A/B/C/a/b/c/d/link-to-B/C/foo.js', + '/A/B/C/a/b/c/package.json', + 'd/link-to-B/C/foo.js', + [ + '/A/B/C/a/b/c/d/link-to-B', + '/A/B/C/a/b/c/d/package.json', + '/A/B/C/foo.js', + '/A/B/C/package.json', + '/A/B/package.json', + ], + ], + [ + '/A/B/C/a/b/c/d/link-to-A/B/C/foo.js', + '/A/package.json', + 'B/C/foo.js', + [ + '/A/B/C/a/b/c/d/link-to-A', + '/A/B/C/foo.js', + '/A/B/C/package.json', + '/A/B/package.json', + ], + ], + [ + '/A/B/C/a/1/foo.js', + '/A/B/C/a/1/real-package.json', + 'foo.js', + ['/A/B/C/a/1/foo.js', '/A/B/C/a/1/package.json'], + ], + [ + '/A/B/C/a/2/foo.js', + '/A/B/C/a/package.json', + '2/foo.js', + [ + '/A/B/C/a/2/foo.js', + '/A/B/C/a/2/notexist-package.json', + '/A/B/C/a/2/package.json', + ], + ], + [ + '/A/B/C/a/n_m/pkg/notexist.js', + '/A/B/C/a/n_m/pkg/package.json', + 'notexist.js', + ['/A/B/C/a/n_m/pkg/notexist.js'], + ], + [ + '/A/B/C/a/n_m/pkg/subpath/notexist.js', + '/A/B/C/a/n_m/pkg/subpath/package.json', + 'notexist.js', + ['/A/B/C/a/n_m/pkg/subpath/notexist.js'], + ], + [ + '/A/B/C/a/n_m/pkg/otherpath/notexist.js', + '/A/B/C/a/n_m/pkg/package.json', + 'otherpath/notexist.js', + ['/A/B/C/a/n_m/pkg/otherpath'], + ], + // pkg3 does not exist, doesn't look beyond the containing n_m + ['/A/B/C/a/n_m/pkg3/foo.js', null, null, ['/A/B/C/a/n_m/pkg3']], + // Does not look beyond n_m, if n_m does not exist + ['/A/B/C/a/b/n_m/pkg/foo', null, null, ['/A/B/C/a/b/n_m']], + ])( + '%s => %s (relative %s, invalidatedBy %s)', + ( + startPath, + expectedPath, + expectedRelativeSubpath, + expectedInvalidatedBy, + ) => { + const pathMap = (normalPosixPath: string) => + mockPathModule.resolve(p('/A/B/C'), p(normalPosixPath)); + const invalidatedBy = new Set(); + expect( + tfs.hierarchicalLookup(p(startPath), 'package.json', { + breakOnSegment: 'n_m', + invalidatedBy, + subpathType: 'f', + }), + ).toEqual( + expectedPath == null + ? null + : { + absolutePath: pathMap(expectedPath), + containerRelativePath: p(expectedRelativeSubpath), + }, + ); + expect(invalidatedBy).toEqual(new Set(expectedInvalidatedBy.map(p))); + }, + ); + }); + describe('matchFiles', () => { test('non-recursive, skipping deep paths', () => { expect( diff --git a/packages/metro-file-map/types/flow-types.d.ts b/packages/metro-file-map/types/flow-types.d.ts index fcfc3b471..c7598ded5 100644 --- a/packages/metro-file-map/types/flow-types.d.ts +++ b/packages/metro-file-map/types/flow-types.d.ts @@ -166,6 +166,42 @@ export interface FileSystem { getSerializableSnapshot(): FileData; getSha1(file: Path): string | null; + /** + * Given a start path (which need not exist), a subpath and type, and + * optionally a 'breakOnSegment', performs the following: + * + * X = mixedStartPath + * do + * if basename(X) === opts.breakOnSegment + * return null + * if X + subpath exists and has type opts.subpathType + * return { + * absolutePath: realpath(X + subpath) + * containerRelativePath: relative(mixedStartPath, X) + * } + * X = dirname(X) + * while X !== dirname(X) + * + * If opts.invalidatedBy is given, collects all absolute, real paths that if + * added or removed may invalidate this result. + * + * Useful for finding the closest package scope (subpath: package.json, + * type f, breakOnSegment: node_modules) or closest potential package root + * (subpath: node_modules/pkg, type: d) in Node.js resolution. + */ + hierarchicalLookup( + mixedStartPath: string, + subpath: string, + opts: { + breakOnSegment: string | null | undefined; + invalidatedBy: Set | null | undefined; + subpathType: 'f' | 'd'; + }, + ): { + absolutePath: string; + containerRelativePath: string; + } | null; + /** * Analogous to posix lstat. If the file at `file` is a symlink, return * information about the symlink without following it. diff --git a/packages/metro/src/node-haste/DependencyGraph.js b/packages/metro/src/node-haste/DependencyGraph.js index eda1e457b..5bad7035a 100644 --- a/packages/metro/src/node-haste/DependencyGraph.js +++ b/packages/metro/src/node-haste/DependencyGraph.js @@ -152,33 +152,6 @@ class DependencyGraph extends EventEmitter { return self; } - _getClosestPackage( - absoluteModulePath: string, - ): ?{packageJsonPath: string, packageRelativePath: string} { - const parsedPath = path.parse(absoluteModulePath); - const root = parsedPath.root; - let dir = path.join(parsedPath.dir, parsedPath.base); - - do { - // If we've hit a node_modules directory, the closest package was not - // found (`filePath` was likely nonexistent). - if (path.basename(dir) === 'node_modules') { - return null; - } - const candidate = path.join(dir, 'package.json'); - if (this._fileSystem.exists(candidate)) { - return { - packageJsonPath: candidate, - // Note that by construction, dir is a prefix of absoluteModulePath, - // so this relative path has no indirections. - packageRelativePath: path.relative(dir, absoluteModulePath), - }; - } - dir = path.dirname(dir); - } while (dir !== '.' && dir !== root); - return null; - } - _onHasteChange({eventsQueue}: ChangeEvent) { this._resolutionCache = new Map(); eventsQueue.forEach(({filePath}) => this._moduleCache.invalidate(filePath)); @@ -247,10 +220,29 @@ class DependencyGraph extends EventEmitter { }); } + _getClosestPackage( + absoluteModulePath: string, + ): ?{packageJsonPath: string, packageRelativePath: string} { + const result = this._fileSystem.hierarchicalLookup( + absoluteModulePath, + 'package.json', + { + breakOnSegment: 'node_modules', + invalidatedBy: null, + subpathType: 'f', + }, + ); + return result + ? { + packageJsonPath: result.absolutePath, + packageRelativePath: result.containerRelativePath, + } + : null; + } + _createModuleCache(): ModuleCache { return new ModuleCache({ - getClosestPackage: absoluteModulePath => - this._getClosestPackage(absoluteModulePath), + getClosestPackage: absolutePath => this._getClosestPackage(absolutePath), }); }