Option to limit number of rows imported

lib/input/csv-file-reader.js

@@ -15,9 +15,13 @@ const {parse} = require('csv-parse');
  * 
  * @param {string}  csvFilePath - The path to the input CSV file
  */
-async function* getRecordGenerator_Async(csvFilePath) {
+async function* getRecordGenerator_Async(csvFilePath, startIndex = 1, rowsToImport = 1) {
     const parser = fs.createReadStream(csvFilePath)
-        .pipe(parse({columns: true}));
+        .pipe(parse({
+            columns: true,
+            from: startIndex,
+            to: startIndex + rowsToImport - 1
+        }));
 
     for await (const record of parser) {
         yield record;

lib/input/parse-cmdline-args.js

@@ -19,7 +19,7 @@ Implements
  - Migration log file (JSONL)
  - Customizeable migration report file (CSV) -- see the __log-to-report.js module
 
-Usage: $0 --from-csv-file <path> --output-folder <path> --max-concurrent-uploads <number>
+Usage: $0 --from-csv-file <path> --output-folder <path> --max-concurrent-uploads <number> --number-to-import <number>
 
 🤓 To prevent unintentional override of log or report files from prior executions, the script does not proceed if the output folder already contains migration log or report files.`
 
@@ -80,6 +80,32 @@ const cmdline_args = yargs(hideBin(process.argv))
       return intValue;
     }
   })
+  .option('number-to-import', {
+    alias: 'n',
+    description: 'Number of assets to import',
+    type: 'number',
+    demandOption: false, // Required argument
+    coerce: value => {
+      const intValue = parseInt(value, 10);
+      if (isNaN(intValue) || intValue < 1) {
+        throw new Error(`Invalid value for number of assets to import: ${value}`);
+      }
+      return intValue;
+    }
+  })
+  .option('rows-to-skip', {
+    alias: 's',
+    description: 'Number of rows to skip when importing',
+    type: 'number',
+    demandOption: false, // Required argument
+    coerce: value => {
+      const intValue = parseInt(value, 10);
+      if (isNaN(intValue) || intValue < 1) {
+        throw new Error(`Invalid value for number of rows to skip: ${value}`);
+      }
+      return intValue;
+    }
+  })
   .wrap(terminalWidth())
   .argv;
 
@@ -94,6 +120,8 @@ function getReportFilePath(outputFolder) {
 module.exports = {
     fromCsvFile           : cmdline_args['from-csv-file'],
     maxConcurrentUploads  : cmdline_args['max-concurrent-uploads'],
+    numberToImport        : cmdline_args['number-to-import'],
+    rowsToSkip            : cmdline_args['rows-to-skip'],
     logFile               : getLogFilePath(cmdline_args['output-folder']),
     reportFile            : getReportFilePath(cmdline_args['output-folder'])
 }

lib/progress.js

@@ -2,37 +2,6 @@
  * @fileoverview This module encapsulates the logic to display progress bars using the 'cli-progress' library.
  */
 const progress = require('cli-progress');
-const fs = require('fs');
-const readline = require('readline');
-
-/**
- * Counts the number of lines in a file (used to determine ETA).
- * 
- * @param {string} filePath - The path to the input file
- * @returns {Promise<number>} The count of lines in the file
- */
-async function _countLines_Async(filePath) {
-  return new Promise((resolve, reject) => {
-    let lineCount = 0;
-    const readStream = fs.createReadStream(filePath);
-    const lineReader = readline.createInterface({
-      input: readStream,
-      crlfDelay: Infinity
-    });
-
-    lineReader.on('line', () => {
-      lineCount++;
-    });
-
-    lineReader.on('close', () => {
-      resolve(lineCount);
-    });
-
-    lineReader.on('error', (err) => {
-      reject(err);
-    });
-  });
-}
 
 // Create a multi bar container
 const _multiBar = new progress.MultiBar({
@@ -48,19 +17,17 @@ let _progressBar = null;
  * 
  * @param {string} filePath - The path to the migration input CSV file
  */
-async function init_Async(filePath) {
-    const fileCount = await _countLines_Async(filePath);
-    const totalCount = fileCount - 1; // subtracting the header line
+async function init_Async(totalToImport) {
     const init_stats = {
         concurrent: 0,
         succeeded: 0,
         failed: 0
     };
-    _progressBar = _multiBar.create(totalCount, 0, init_stats, {
+    _progressBar = _multiBar.create(totalToImport, 0, init_stats, {
         format: '(🔀{concurrent}) ⏳ [{bar}] ETA: {eta_formatted}',
         fps: 5
     });
-    _statusBar = _multiBar.create(totalCount, 0, init_stats, {
+    _statusBar = _multiBar.create(totalToImport, 0, init_stats, {
         format: 'Attempted: {value} (✅{succeeded} /❌{failed}) out of {total}',
         fps: 5
     });

migrate-remote-assets.js

@@ -23,6 +23,10 @@
  */
 
 require('dotenv').config(); // Load environment variables from .env file
+
+const fs = require('fs');
+const readline = require('readline');
+
 const async = require('async');
 const progress = require('./lib/progress');
 const args = require('./lib/input/parse-cmdline-args');
@@ -50,10 +54,14 @@ const migrationLog = log.migration;
     const migrationOptions = {
         dest_cloud             : cloudinary.config().cloud_name,
         from_csv_file          : args.fromCsvFile,
-        max_concurrent_uploads : args.maxConcurrentUploads
+        max_concurrent_uploads : args.maxConcurrentUploads,
+        number_to_import       : args.numberToImport,
+        rows_to_skip           : args.rowsToSkip
     }
 
-    await confirmMigrationOptionsOrExit_Async(migrationOptions);
+    const totalRows = (await countLines_Async(migrationOptions.from_csv_file)) - 1; //account for the headers
+
+    await confirmMigrationOptionsOrExit_Async(migrationOptions, totalRows);
 
     scriptLog.info(migrationOptions, 'Migration parameters confirmed. Starting migration routine');
     const stats = {
@@ -65,11 +73,14 @@ const migrationLog = log.migration;
 
     console.log('\n\n🚚 Starting migration routine');
 
+    const startIndex = migrationOptions.rows_to_skip !== undefined ? migrationOptions.rows_to_skip+1 : 1;
+    const totalToImport = migrationOptions.number_to_import || totalRows;
+
     // Initializing visual progress bar
-    await progress.init_Async(migrationOptions.from_csv_file);
+    await progress.init_Async(totalToImport);
 
     // Using async generator to avoid loading the entire input file into memory
-    const inputRecordGeneratorAsync = csvReader.getRecordGenerator_Async(migrationOptions.from_csv_file);
+    const inputRecordGeneratorAsync = csvReader.getRecordGenerator_Async(migrationOptions.from_csv_file, startIndex, totalToImport);
 
     // Using async.mapLimit to limit the number of concurrent operations
     await async.mapLimit(inputRecordGeneratorAsync, migrationOptions.max_concurrent_uploads, async (input) => {
@@ -127,12 +138,14 @@ async function ensureCloudinaryConfigOrExit_Async() {
  * @param {string} migrationOptions.dest_cloud    -  Destination Cloudinary environment (sub-account)
  * @param {string} migrationOptions.from_csv_file -  Path to the CSV file with the migration input
  */
-async function confirmMigrationOptionsOrExit_Async(migrationOptions) {
+async function confirmMigrationOptionsOrExit_Async(migrationOptions, total) {
     const migrationPrompt = 
 `❗️WARNING: This script will perform asset migration with the following parameters:
     - source file           :  '${migrationOptions.from_csv_file}'
     - destination cloud     :  '${migrationOptions.dest_cloud}'
     - max concurrent uploads:  ${migrationOptions.max_concurrent_uploads}
+    - number to import      :  ${migrationOptions.number_to_import ? `${migrationOptions.number_to_import} of ${total}` : total}
+    - starting at row       :  ${migrationOptions.rows_to_skip ? migrationOptions.rows_to_skip + 1 : 1}
 Are you sure you want to proceed?`;
     const promptConfirmed = await confirm_Async(migrationPrompt);
     if (!promptConfirmed) {
@@ -156,4 +169,34 @@ async function produceMigrationReport_Async() {
     await new Promise(resolve => setTimeout(resolve, 1500));
     log2Report(args.logFile, args.reportFile);
     console.log(`🏁 Migration report produced.`);
-}
+}
+
+/**
+ * Counts the number of lines in a file.
+ * 
+ * @param {string} filePath - The path to the input file
+ * @returns {Promise<number>} The count of lines in the file
+ */
+async function countLines_Async(filePath) {
+    return new Promise((resolve, reject) => {
+      let lineCount = 0;
+      const readStream = fs.createReadStream(filePath);
+      const lineReader = readline.createInterface({
+        input: readStream,
+        crlfDelay: Infinity
+      });
+
+      lineReader.on('line', () => {
+        lineCount++;
+      });
+
+      lineReader.on('close', () => {
+        resolve(lineCount);
+      });
+
+      lineReader.on('error', (err) => {
+        reject(err);
+      });
+    });
+  }
+