From 5f2166a37b82fc027c2ed5c5df3b42f2e3d96e2f Mon Sep 17 00:00:00 2001
From: Hiroo Ono <49257691+oikumene@users.noreply.github.com>
Date: Fri, 12 Jul 2024 01:23:22 +0900
Subject: [PATCH] fix: Limit concurrent open files during `lsstream` (#285)

<!-- What / Why -->
<!-- Describe the request in detail. What it does and why it's being
changed. -->
During 'npm cache verify', currently all the cache files are open at the
same time, which will bring EMFILE error in an environment that limit
max open files.
This change limits the concurrent open files in garbageCollect() with
p-map module to avoid this problem.
I first sent this pull request to npm/cli and it was merged, but
realized that the original was in this repository.

## References
https://github.com/npm/cli/pull/7631
https://github.com/npm/cli/issues/4783
---
 lib/entry-index.js | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/lib/entry-index.js b/lib/entry-index.js
index 5bc2189..89c28f2 100644
--- a/lib/entry-index.js
+++ b/lib/entry-index.js
@@ -19,6 +19,9 @@ const hashToSegments = require('./util/hash-to-segments')
 const indexV = require('../package.json')['cache-version'].index
 const { moveFile } = require('@npmcli/fs')
 
+const pMap = require('p-map')
+const lsStreamConcurrency = 5
+
 module.exports.NotFoundError = class NotFoundError extends Error {
   constructor (cache, key) {
     super(`No cache entry for ${key} found in ${cache}`)
@@ -182,15 +185,15 @@ function lsStream (cache) {
   // Set all this up to run on the stream and then just return the stream
   Promise.resolve().then(async () => {
     const buckets = await readdirOrEmpty(indexDir)
-    await Promise.all(buckets.map(async (bucket) => {
+    await pMap(buckets, async (bucket) => {
       const bucketPath = path.join(indexDir, bucket)
       const subbuckets = await readdirOrEmpty(bucketPath)
-      await Promise.all(subbuckets.map(async (subbucket) => {
+      await pMap(subbuckets, async (subbucket) => {
         const subbucketPath = path.join(bucketPath, subbucket)
 
         // "/cachename/<bucket 0xFF>/<bucket 0xFF>./*"
         const subbucketEntries = await readdirOrEmpty(subbucketPath)
-        await Promise.all(subbucketEntries.map(async (entry) => {
+        await pMap(subbucketEntries, async (entry) => {
           const entryPath = path.join(subbucketPath, entry)
           try {
             const entries = await bucketEntries(entryPath)
@@ -213,9 +216,12 @@ function lsStream (cache) {
             }
             throw err
           }
-        }))
-      }))
-    }))
+        },
+        { concurrency: lsStreamConcurrency })
+      },
+      { concurrency: lsStreamConcurrency })
+    },
+    { concurrency: lsStreamConcurrency })
     stream.end()
     return stream
   }).catch(err => stream.emit('error', err))