Adds progress bars to the tr-cron-pull-identifiers and `tr-cron-mar…

…k-identifiers-completed` commands (#311)

* Updates

* Updates

package.json

@@ -2,7 +2,7 @@
   "author": "Transcend Inc.",
   "name": "@transcend-io/cli",
   "description": "Small package containing useful typescript utilities.",
-  "version": "4.130.5",
+  "version": "4.131.0",
   "homepage": "https://github.com/transcend-io/cli",
   "repository": {
     "type": "git",

src/cron/markCronIdentifierCompleted.ts

@@ -18,12 +18,12 @@ export type CronIdentifierPush = t.TypeOf<typeof CronIdentifierPush>;
  * @see https://docs.transcend.io/docs/api-reference/PUT/v1/data-silo
  * @param sombra - Sombra instance configured to make requests
  * @param options - Additional options
- * @returns Successfully submitted request
+ * @returns Successfully submitted request, false if not in a state to update
  */
 export async function markCronIdentifierCompleted(
   sombra: Got,
   { nonce, identifier }: CronIdentifierPush,
-): Promise<void> {
+): Promise<boolean> {
   try {
     // Make the GraphQL request
     await sombra.put('v1/data-silo', {
@@ -38,10 +38,11 @@ export async function markCronIdentifierCompleted(
         ],
       },
     });
+    return true;
   } catch (err) {
     // handle gracefully
-    if (err.statusCode === 409) {
-      return;
+    if (err.response?.statusCode === 409) {
+      return false;
     }
     throw new Error(
       `Received an error from server: ${err?.response?.body || err?.message}`,

src/cron/pullCustomSiloOutstandingIdentifiers.ts

@@ -1,5 +1,10 @@
-import { createSombraGotInstance } from '../graphql';
+import {
+  buildTranscendGraphQLClient,
+  createSombraGotInstance,
+  fetchRequestDataSiloActiveCount,
+} from '../graphql';
 import colors from 'colors';
+import cliProgress from 'cli-progress';
 import {
   pullCronPageOfIdentifiers,
   CronIdentifier,
@@ -50,18 +55,36 @@ export async function pullCustomSiloOutstandingIdentifiers({
   // Create sombra instance to communicate with
   const sombra = await createSombraGotInstance(transcendUrl, auth, sombraAuth);
 
+  // Create GraphQL client to connect to Transcend backend
+  const client = buildTranscendGraphQLClient(transcendUrl, auth);
+
+  const totalRequestCount = await fetchRequestDataSiloActiveCount(client, {
+    dataSiloId,
+  });
+
   logger.info(
     colors.magenta(
-      `Pulling outstanding request identifiers for data silo: "${dataSiloId}" for requests of types "${actions.join(
-        '", "',
-      )}"`,
+      `Pulling ${totalRequestCount} outstanding request identifiers ` +
+        `for data silo: "${dataSiloId}" for requests of types "${actions.join(
+          '", "',
+        )}"`,
     ),
   );
 
+  // Time duration
+  const t0 = new Date().getTime();
+  // create a new progress bar instance and use shades_classic theme
+  const progressBar = new cliProgress.SingleBar(
+    {},
+    cliProgress.Presets.shades_classic,
+  );
+  const foundRequestIds = new Set<string>();
+
   // identifiers found in total
   const identifiers: CronIdentifierWithAction[] = [];
 
   // map over each action
+  progressBar.start(totalRequestCount, 0);
   await mapSeries(actions, async (action) => {
     let offset = 0;
     let shouldContinue = true;
@@ -76,19 +99,29 @@ export async function pullCustomSiloOutstandingIdentifiers({
         requestType: action,
       });
       identifiers.push(
-        ...pageIdentifiers.map((identifier) => ({
-          ...identifier,
-          action,
-        })),
+        ...pageIdentifiers.map((identifier) => {
+          foundRequestIds.add(identifier.requestId);
+          return {
+            ...identifier,
+            action,
+          };
+        }),
       );
       shouldContinue = pageIdentifiers.length === pageLimit;
       offset += pageLimit;
+      progressBar.update(foundRequestIds.size);
     }
   });
 
+  progressBar.stop();
+  const t1 = new Date().getTime();
+  const totalTime = t1 - t0;
+
   logger.info(
-    colors.magenta(
-      `Found: ${identifiers.length} outstanding identifiers to parse`,
+    colors.green(
+      `Successfully pulled ${identifiers.length} outstanding identifiers from ${
+        foundRequestIds.size
+      } requests in "${totalTime / 1000}" seconds!`,
     ),
   );
 

src/cron/pushCronIdentifiersFromCsv.ts

@@ -5,6 +5,7 @@ import {
   markCronIdentifierCompleted,
   CronIdentifierPush,
 } from './markCronIdentifierCompleted';
+import cliProgress from 'cli-progress';
 import { logger } from '../logger';
 import { readCsv } from '../requests';
 import { DEFAULT_TRANSCEND_API } from '../constants';
@@ -46,17 +47,53 @@ export async function pushCronIdentifiersFromCsv({
   // Notify Transcend
   logger.info(
     colors.magenta(
-      `Notifying Transcend for data silo "${dataSiloId}" marking "${activeResults.length}" requests as completed.`,
+      `Notifying Transcend for data silo "${dataSiloId}" marking "${activeResults.length}" identifiers as completed.`,
     ),
   );
+
+  // Time duration
+  const t0 = new Date().getTime();
+  // create a new progress bar instance and use shades_classic theme
+  const progressBar = new cliProgress.SingleBar(
+    {},
+    cliProgress.Presets.shades_classic,
+  );
+
+  let successCount = 0;
+  let failureCount = 0;
+  progressBar.start(activeResults.length, 0);
   await map(
     activeResults,
     async (identifier) => {
-      await markCronIdentifierCompleted(sombra, identifier);
+      const success = await markCronIdentifierCompleted(sombra, identifier);
+      if (success) {
+        successCount += 1;
+      } else {
+        failureCount += 1;
+      }
+      progressBar.update(successCount + failureCount);
     },
     { concurrency },
   );
 
-  logger.info(colors.green('Successfully notified Transcend!'));
+  progressBar.stop();
+  const t1 = new Date().getTime();
+  const totalTime = t1 - t0;
+
+  logger.info(
+    colors.green(
+      `Successfully notified Transcend for ${successCount} identifiers in "${
+        totalTime / 1000
+      }" seconds!`,
+    ),
+  );
+  if (failureCount) {
+    logger.info(
+      colors.magenta(
+        `There were ${failureCount} identifiers that were not in a state to be updated.` +
+          'They likely have already been resolved.',
+      ),
+    );
+  }
   return activeResults.length;
 }

src/graphql/fetchRequestDataSiloActiveCount.ts

@@ -0,0 +1,37 @@
+import { GraphQLClient } from 'graphql-request';
+import { REDUCED_REQUESTS_FOR_DATA_SILO_COUNT } from './gqls';
+import { makeGraphQLRequest } from './makeGraphQLRequest';
+
+/**
+ * Get number of open requests for a data silo
+ *
+ * @param client - GraphQL client
+ * @param options - Filter options
+ * @returns List of request identifiers
+ */
+export async function fetchRequestDataSiloActiveCount(
+  client: GraphQLClient,
+  {
+    dataSiloId,
+  }: {
+    /** Data silo ID */
+    dataSiloId: string;
+  },
+): Promise<number> {
+  const {
+    listReducedRequestsForDataSilo: { totalCount },
+  } = await makeGraphQLRequest<{
+    /** Requests */
+    listReducedRequestsForDataSilo: {
+      /** Total count */
+      totalCount: number;
+    };
+  }>(client, REDUCED_REQUESTS_FOR_DATA_SILO_COUNT, {
+    input: {
+      dataSiloId,
+      isResolved: false,
+    },
+  });
+
+  return totalCount;
+}

src/graphql/gqls/RequestDataSilo.ts

@@ -50,3 +50,20 @@ export const RETRY_REQUEST_DATA_SILO = gql`
     }
   }
 `;
+
+// TODO: https://transcend.height.app/T-27909 - enable optimizations
+// isExportCsv: true
+// useMaster: false
+// orderBy: [
+//   { field: createdAt, direction: DESC }
+//   { field: title, direction: ASC, model: dataSilo }
+// ]
+export const REDUCED_REQUESTS_FOR_DATA_SILO_COUNT = gql`
+  query TranscendCliListReducedRequestsForDataSiloCount(
+    $input: BulkCompletionReducedRequestInput!
+  ) {
+    listReducedRequestsForDataSilo(input: $input) {
+      totalCount
+    }
+  }
+`;

src/graphql/index.ts

@@ -18,6 +18,7 @@ export * from './setResourceAttributes';
 export * from './buildTranscendGraphQLClient';
 export * from './gqls';
 export * from './fetchPromptThreads';
+export * from './fetchRequestDataSiloActiveCount';
 export * from './fetchAllAttributeKeys';
 export * from './fetchAllRequests';
 export * from './fetchAllRequestIdentifiers';