deploy: Add rate-limiting aware retries for failed requests

Change-type: patch

lib/utils/compose.ts

@@ -20,6 +20,7 @@ import type * as SDK from 'balena-sdk';
 import type Dockerode = require('dockerode');
 import * as path from 'path';
 import type { Composition, ImageDescriptor } from '@balena/compose/dist/parse';
+import type { RetryParametersObj } from 'pinejs-client-core';
 import type {
 	BuiltImage,
 	ComposeOpts,
@@ -94,22 +95,60 @@ export function createProject(
 	};
 }
 
+const getRequestRetryParameters = (): RetryParametersObj => {
+	const testIntVar = (varName: string) => {
+		if (
+			!process.env.BALENA_CLI_TEST_TYPE ||
+			!varName.startsWith('BALENARCTEST_')
+		) {
+			// We only read the test env vars when in test mode.
+			return;
+		}
+		const { intVar } =
+			require('@balena/env-parsing') as typeof import('@balena/env-parsing');
+		return intVar(varName);
+	};
+	// We use the BALENARCTEST namespace and only parse the env vars while in test mode
+	// since we plan to switch all pinejs clients with the one of the SDK and might not
+	// BALENARCTEST to have to support these env vars.
+	return {
+		minDelayMs: testIntVar('BALENARCTEST_API_RETRY_MIN_DELAY_MS') ?? 1000,
+		maxDelayMs: testIntVar('BALENARCTEST_API_RETRY_MAX_DELAY_MS') ?? 60000,
+		maxAttempts: testIntVar('BALENARCTEST_API_RETRY_MAX_ATTEMPTS') ?? 7,
+	};
+};
+
 export const createRelease = async function (
+	logger: Logger,
 	apiEndpoint: string,
 	auth: string,
 	userId: number,
 	appId: number,
 	composition: Composition,
 	draft: boolean,
-	semver?: string,
-	contract?: string,
+	semver: string | undefined,
+	contract: string | undefined,
 ): Promise<Release> {
 	const _ = require('lodash') as typeof import('lodash');
 	const crypto = require('crypto') as typeof import('crypto');
 	const releaseMod =
 		require('@balena/compose/dist/release') as typeof import('@balena/compose/dist/release');
 
-	const client = releaseMod.createClient({ apiEndpoint, auth });
+	const client = releaseMod.createClient({
+		apiEndpoint,
+		auth,
+		retry: {
+			...getRequestRetryParameters(),
+			onRetry: (err, delayMs, attempt, maxAttempts) => {
+				const code = err?.statusCode ?? 0;
+				logger.logDebug(
+					`API call failed with code ${code}.  Attempting retry ${attempt} of ${maxAttempts} in ${
+						delayMs / 1000
+					} seconds`,
+				);
+			},
+		},
+	});
 
 	const { release, serviceImages } = await releaseMod.create({
 		client,

lib/utils/compose_ts.ts

@@ -1385,6 +1385,7 @@ export async function deployProject(
 		`${prefix}Creating release...`,
 		() =>
 			createRelease(
+				logger,
 				apiEndpoint,
 				auth,
 				userId,

package.json

@@ -196,6 +196,7 @@
   "dependencies": {
     "@balena/compose": "^3.2.0",
     "@balena/dockerignore": "^1.0.2",
+    "@balena/env-parsing": "^1.1.8",
     "@balena/es-version": "^1.0.1",
     "@oclif/core": "^3.14.1",
     "@resin.io/valid-email": "^0.1.0",

tests/commands/deploy.spec.ts

@@ -289,6 +289,8 @@ describe('balena deploy', function () {
 		api.expectPatchImage({
 			replyBody: errMsg,
 			statusCode: 500,
+			// b/c failed requests are retried
+			persist: true,
 			inspectRequest: (_uri, requestBody) => {
 				const imageBody = requestBody as Partial<
 					import('@balena/compose/dist/release/models').ImageModel

tests/config-tests.ts

@@ -26,6 +26,11 @@ process.env.BALENARC_NO_SENTRY = '1';
 // Like the global `--unsupported` flag
 process.env.BALENARC_UNSUPPORTED = '1';
 
+// Reduce the api request retry limits to keep the tests fast.
+process.env.BALENARCTEST_API_RETRY_MIN_DELAY_MS = '100';
+process.env.BALENARCTEST_API_RETRY_MAX_DELAY_MS = '1000';
+process.env.BALENARCTEST_API_RETRY_MAX_ATTEMPTS = '2';
+
 import * as tmp from 'tmp';
 tmp.setGracefulCleanup();
 // Use a temporary dir for tests data