From 2ffdc51a90192d01153cd8b473dc52a566f22da5 Mon Sep 17 00:00:00 2001 From: Hiroo Ono Date: Thu, 11 Jul 2024 12:50:26 +0900 Subject: [PATCH 1/2] Limit concurrent open files during 'npm cache verify' --- lib/entry-index.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/entry-index.js b/lib/entry-index.js index 5bc2189..f73a11c 100644 --- a/lib/entry-index.js +++ b/lib/entry-index.js @@ -19,6 +19,10 @@ const hashToSegments = require('./util/hash-to-segments') const indexV = require('../package.json')['cache-version'].index const { moveFile } = require('@npmcli/fs') +const pMap = require('p-map') +const lsStreamConcurrency = 5 + + module.exports.NotFoundError = class NotFoundError extends Error { constructor (cache, key) { super(`No cache entry for ${key} found in ${cache}`) @@ -182,15 +186,15 @@ function lsStream (cache) { // Set all this up to run on the stream and then just return the stream Promise.resolve().then(async () => { const buckets = await readdirOrEmpty(indexDir) - await Promise.all(buckets.map(async (bucket) => { + await pMap(buckets, async (bucket) => { const bucketPath = path.join(indexDir, bucket) const subbuckets = await readdirOrEmpty(bucketPath) - await Promise.all(subbuckets.map(async (subbucket) => { + await pMap(subbuckets, async (subbucket) => { const subbucketPath = path.join(bucketPath, subbucket) // "/cachename//./*" const subbucketEntries = await readdirOrEmpty(subbucketPath) - await Promise.all(subbucketEntries.map(async (entry) => { + await pMap(subbucketEntries, async (entry) => { const entryPath = path.join(subbucketPath, entry) try { const entries = await bucketEntries(entryPath) @@ -213,9 +217,12 @@ function lsStream (cache) { } throw err } - })) - })) - })) + }, + { concurrency: lsStreamConcurrency }) + }, + { concurrency: lsStreamConcurrency }) + }, + { concurrency: lsStreamConcurrency }) stream.end() return stream }).catch(err => stream.emit('error', err)) From b4088487036a3cac3ffffc7331c0531b0d1eb123 Mon Sep 17 00:00:00 2001 From: Hiroo Ono Date: Fri, 12 Jul 2024 00:45:41 +0900 Subject: [PATCH 2/2] Remove extra blank line. --- lib/entry-index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/entry-index.js b/lib/entry-index.js index f73a11c..89c28f2 100644 --- a/lib/entry-index.js +++ b/lib/entry-index.js @@ -22,7 +22,6 @@ const { moveFile } = require('@npmcli/fs') const pMap = require('p-map') const lsStreamConcurrency = 5 - module.exports.NotFoundError = class NotFoundError extends Error { constructor (cache, key) { super(`No cache entry for ${key} found in ${cache}`)