deploy: Add rate-limiting aware retries for failed requests

Change-type: patch

lib/utils/compose.ts

@@ -19,6 +19,7 @@ import type { Renderer } from './compose_ts';
 import type * as SDK from 'balena-sdk';
 import type Dockerode = require('dockerode');
 import * as path from 'path';
+import type { RetryParametersObj } from 'pinejs-client-core';
 import type { Composition, ImageDescriptor } from '@balena/compose/dist/parse';
 import type {
 	BuiltImage,
@@ -94,22 +95,43 @@ export function createProject(
 	};
 }
 
+const retryParameters = {
+	minDelayMs: 1000,
+	maxDelayMs: 60000,
+	maxAttempts: 7,
+} satisfies RetryParametersObj;
+
 export const createRelease = async function (
+	logger: Logger,
 	apiEndpoint: string,
 	auth: string,
 	userId: number,
 	appId: number,
 	composition: Composition,
 	draft: boolean,
-	semver?: string,
-	contract?: string,
+	semver: string | undefined,
+	contract: string | undefined,
 ): Promise<Release> {
 	const _ = require('lodash') as typeof import('lodash');
 	const crypto = require('crypto') as typeof import('crypto');
 	const releaseMod =
 		require('@balena/compose/dist/release') as typeof import('@balena/compose/dist/release');
 
-	const client = releaseMod.createClient({ apiEndpoint, auth });
+	const client = releaseMod.createClient({
+		apiEndpoint,
+		auth,
+		retry: {
+			...retryParameters,
+			onRetry: (err, delayMs, attempt, maxAttempts) => {
+				const code = err?.statusCode ?? 0;
+				logger.logDebug(
+					`API call failed with code ${code}.  Attempting retry ${attempt} of ${maxAttempts} in ${
+						delayMs / 1000
+					} seconds`,
+				);
+			},
+		},
+	});
 
 	const { release, serviceImages } = await releaseMod.create({
 		client,

lib/utils/compose_ts.ts

@@ -1385,6 +1385,7 @@ export async function deployProject(
 		`${prefix}Creating release...`,
 		() =>
 			createRelease(
+				logger,
 				apiEndpoint,
 				auth,
 				userId,