Merges

transcend-io · Feb 23, 2024 · cac078f · cac078f
2 parents ebf1b72 + effdfc3
commit cac078f
Show file tree

Hide file tree

Showing 4 changed files with 82 additions and 40 deletions.
diff --git a/package.json b/package.json
@@ -2,7 +2,7 @@
   "author": "Transcend Inc.",
   "name": "@transcend-io/cli",
   "description": "Small package containing useful typescript utilities.",
-  "version": "4.130.4",
+  "version": "4.131.0",
   "homepage": "https://github.com/transcend-io/cli",
   "repository": {
     "type": "git",

diff --git a/src/cron/markCronIdentifierCompleted.ts b/src/cron/markCronIdentifierCompleted.ts
@@ -18,23 +18,34 @@ export type CronIdentifierPush = t.TypeOf<typeof CronIdentifierPush>;
  * @see https://docs.transcend.io/docs/api-reference/PUT/v1/data-silo
  * @param sombra - Sombra instance configured to make requests
  * @param options - Additional options
- * @returns Successfully submitted request
+ * @returns Successfully submitted request, false if not in a state to update
  */
 export async function markCronIdentifierCompleted(
   sombra: Got,
   { nonce, identifier }: CronIdentifierPush,
-): Promise<void> {
-  // Make the GraphQL request
-  await sombra.put('v1/data-silo', {
-    headers: {
-      'x-transcend-nonce': nonce,
-    },
-    json: {
-      profiles: [
-        {
-          profileId: identifier,
-        },
-      ],
-    },
-  });
+): Promise<boolean> {
+  try {
+    // Make the GraphQL request
+    await sombra.put('v1/data-silo', {
+      headers: {
+        'x-transcend-nonce': nonce,
+      },
+      json: {
+        profiles: [
+          {
+            profileId: identifier,
+          },
+        ],
+      },
+    });
+    return true;
+  } catch (err) {
+    // handle gracefully
+    if (err.response?.statusCode === 409) {
+      return false;
+    }
+    throw new Error(
+      `Received an error from server: ${err?.response?.body || err?.message}`,
+    );
+  }
 }
diff --git a/src/cron/pullCustomSiloOutstandingIdentifiers.ts b/src/cron/pullCustomSiloOutstandingIdentifiers.ts
@@ -1,8 +1,7 @@
 import {
+  buildTranscendGraphQLClient,
   createSombraGotInstance,
-  // FIXME
-  // buildTranscendGraphQLClient,
-  // fetchRequestDataSiloActiveCount,
+  fetchRequestDataSiloActiveCount,
 } from '../graphql';
 import colors from 'colors';
 import cliProgress from 'cli-progress';
@@ -70,38 +69,35 @@ export async function pullCustomSiloOutstandingIdentifiers({
   const sombra = await createSombraGotInstance(transcendUrl, auth, sombraAuth);
 
   // Create GraphQL client to connect to Transcend backend
-  // FIXME
-  // const client = buildTranscendGraphQLClient(transcendUrl, auth);
+  const client = buildTranscendGraphQLClient(transcendUrl, auth);
+
+  const totalRequestCount = await fetchRequestDataSiloActiveCount(client, {
+    dataSiloId,
+  });
 
   logger.info(
     colors.magenta(
-      `Pulling outstanding request identifiers for data silo: "${dataSiloId}" for requests of types "${actions.join(
-        '", "',
-      )}"`,
+      `Pulling ${totalRequestCount} outstanding request identifiers ` +
+        `for data silo: "${dataSiloId}" for requests of types "${actions.join(
+          '", "',
+        )}"`,
     ),
   );
 
-  // identifiers found in total
-  const identifiers: CronIdentifierWithAction[] = [];
-
-  // FIXME
-  // const totalRequestCount = await fetchRequestDataSiloActiveCount(client, {
-  //   dataSiloId,
-  // });
-  const totalRequestCount = 69134;
-
   // Time duration
   const t0 = new Date().getTime();
   // create a new progress bar instance and use shades_classic theme
   const progressBar = new cliProgress.SingleBar(
     {},
     cliProgress.Presets.shades_classic,
   );
-
-  progressBar.start(totalRequestCount, 0);
   const foundRequestIds = new Set<string>();
 
+  // identifiers found in total
+  const identifiers: CronIdentifierWithAction[] = [];
+
   // map over each action
+  progressBar.start(totalRequestCount, 0);
   await mapSeries(actions, async (action) => {
     let offset = 0;
     let shouldContinue = true;
@@ -118,7 +114,6 @@ export async function pullCustomSiloOutstandingIdentifiers({
       identifiers.push(
         ...pageIdentifiers.map((identifier) => {
           foundRequestIds.add(identifier.requestId);
-
           return {
             ...identifier,
             action,
@@ -127,7 +122,6 @@ export async function pullCustomSiloOutstandingIdentifiers({
       );
       shouldContinue = pageIdentifiers.length === pageLimit;
       offset += pageLimit;
-
       progressBar.update(foundRequestIds.size);
     }
   });

diff --git a/src/cron/pushCronIdentifiersFromCsv.ts b/src/cron/pushCronIdentifiersFromCsv.ts
@@ -5,6 +5,7 @@ import {
   markCronIdentifierCompleted,
   CronIdentifierPush,
 } from './markCronIdentifierCompleted';
+import cliProgress from 'cli-progress';
 import { logger } from '../logger';
 import { readCsv } from '../requests';
 import { DEFAULT_TRANSCEND_API } from '../constants';
@@ -46,17 +47,53 @@ export async function pushCronIdentifiersFromCsv({
   // Notify Transcend
   logger.info(
     colors.magenta(
-      `Notifying Transcend for data silo "${dataSiloId}" marking "${activeResults.length}" requests as completed.`,
+      `Notifying Transcend for data silo "${dataSiloId}" marking "${activeResults.length}" identifiers as completed.`,
     ),
   );
+
+  // Time duration
+  const t0 = new Date().getTime();
+  // create a new progress bar instance and use shades_classic theme
+  const progressBar = new cliProgress.SingleBar(
+    {},
+    cliProgress.Presets.shades_classic,
+  );
+
+  let successCount = 0;
+  let failureCount = 0;
+  progressBar.start(activeResults.length, 0);
   await map(
     activeResults,
     async (identifier) => {
-      await markCronIdentifierCompleted(sombra, identifier);
+      const success = await markCronIdentifierCompleted(sombra, identifier);
+      if (success) {
+        successCount += 1;
+      } else {
+        failureCount += 1;
+      }
+      progressBar.update(successCount + failureCount);
     },
     { concurrency },
   );
 
-  logger.info(colors.green('Successfully notified Transcend!'));
+  progressBar.stop();
+  const t1 = new Date().getTime();
+  const totalTime = t1 - t0;
+
+  logger.info(
+    colors.green(
+      `Successfully notified Transcend for ${successCount} identifiers in "${
+        totalTime / 1000
+      }" seconds!`,
+    ),
+  );
+  if (failureCount) {
+    logger.info(
+      colors.magenta(
+        `There were ${failureCount} identifiers that were not in a state to be updated.` +
+          'They likely have already been resolved.',
+      ),
+    );
+  }
   return activeResults.length;
 }