From ebe74edb9315903af8923b124ba3adc42cc6d416 Mon Sep 17 00:00:00 2001 From: philmcmahon Date: Mon, 13 Jan 2025 12:03:50 +0000 Subject: [PATCH] Move uploadFileToGoogleDrive to media-export service --- packages/api/src/services/googleDrive.ts | 82 ----------------------- packages/media-export/src/googleDrive.ts | 84 ++++++++++++++++++++++++ packages/media-export/src/index.ts | 2 +- 3 files changed, 85 insertions(+), 83 deletions(-) create mode 100644 packages/media-export/src/googleDrive.ts diff --git a/packages/api/src/services/googleDrive.ts b/packages/api/src/services/googleDrive.ts index 4e9bf1e..f360365 100644 --- a/packages/api/src/services/googleDrive.ts +++ b/packages/api/src/services/googleDrive.ts @@ -5,7 +5,6 @@ import { TranscriptionConfig, } from '@guardian/transcription-service-backend-common'; import { ZTokenResponse } from '@guardian/transcription-service-common'; -import * as fs from 'node:fs'; import Drive = drive_v3.Drive; const ROOT_FOLDER_NAME = 'Guardian Transcribe Tool'; @@ -130,84 +129,3 @@ export const createExportFolder = async (drive: Drive, name: string) => { } return folderId; }; - -export const uploadFileToGoogleDrive = async ( - fileName: string, - oAuthTokenResponse: ZTokenResponse, - filePath: string, - mimeType: string, - folderId: string, -) => { - const fileSize = fs.statSync(filePath).size; - - const startResumableSessionResponse = await fetch( - 'https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable', - { - method: 'POST', - headers: { - 'X-Upload-Content-Length': `${fileSize}`, - 'X-Upload-Content-Type': mimeType, - 'Content-Type': 'application/json', - Authorization: `Bearer ${oAuthTokenResponse.access_token}`, - }, - body: JSON.stringify({ - name: fileName, - mimeType, - parents: [folderId], - }), - }, - ); - - const uploadUrl = startResumableSessionResponse.headers.get('location'); - - if (!uploadUrl) { - throw new Error('Failed to start resumable upload session'); - } - - //when changing this value consider the amount of memory allocated to the API lambda function - const CHUNK_SIZE = 128 * 1024 * 1024; // 128MB - - const fileStream = fs.createReadStream(filePath, { - highWaterMark: CHUNK_SIZE, - }); - - let offset = 0; - - for await (const chunk of fileStream) { - // pause the stream to prevent node from buffering any more data whilst we upload - fileStream.pause(); - const chunkSize = chunk.length; - const range = `bytes ${offset}-${offset + chunkSize - 1}/${fileSize}`; - - logger.info( - `Uploading chunk: ${range} (Upload ${Math.floor((offset / fileSize) * 100)}% complete)`, - ); - - const response = await fetch(uploadUrl, { - method: 'PUT', - headers: { - 'Content-Range': range, - 'Content-Length': chunkSize, - }, - body: chunk, - }); - - if (response.ok) { - // Response status is 308 until the final chunk. Final response includes file metadata - return ((await response.json()) as { id: string }).id; - } - if (response.status === 308) { - //continue - } else { - const text = await response.text(); - logger.error(`Received ${response.status} from google, error: ${text}`); - throw new Error( - `Failed to upload chunk: ${response.status} ${response.statusText}`, - ); - } - - offset += chunkSize; - fileStream.resume(); - } - - throw new Error('Failed to upload file'); -}; diff --git a/packages/media-export/src/googleDrive.ts b/packages/media-export/src/googleDrive.ts new file mode 100644 index 0000000..141cc70 --- /dev/null +++ b/packages/media-export/src/googleDrive.ts @@ -0,0 +1,84 @@ +import { ZTokenResponse } from '@guardian/transcription-service-common'; +import fs from 'node:fs'; +import { logger } from '@guardian/transcription-service-backend-common'; + +export const uploadFileToGoogleDrive = async ( + fileName: string, + oAuthTokenResponse: ZTokenResponse, + filePath: string, + mimeType: string, + folderId: string, +) => { + const fileSize = fs.statSync(filePath).size; + + const startResumableSessionResponse = await fetch( + 'https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable', + { + method: 'POST', + headers: { + 'X-Upload-Content-Length': `${fileSize}`, + 'X-Upload-Content-Type': mimeType, + 'Content-Type': 'application/json', + Authorization: `Bearer ${oAuthTokenResponse.access_token}`, + }, + body: JSON.stringify({ + name: fileName, + mimeType, + parents: [folderId], + }), + }, + ); + + const uploadUrl = startResumableSessionResponse.headers.get('location'); + + if (!uploadUrl) { + throw new Error('Failed to start resumable upload session'); + } + + //when changing this value consider the amount of memory allocated to the API lambda function + const CHUNK_SIZE = 128 * 1024 * 1024; // 128MB - + const fileStream = fs.createReadStream(filePath, { + highWaterMark: CHUNK_SIZE, + }); + + let offset = 0; + + for await (const chunk of fileStream) { + // pause the stream to prevent node from buffering any more data whilst we upload + fileStream.pause(); + const chunkSize = chunk.length; + const range = `bytes ${offset}-${offset + chunkSize - 1}/${fileSize}`; + + logger.info( + `Uploading chunk: ${range} (Upload ${Math.floor((offset / fileSize) * 100)}% complete)`, + ); + + const response = await fetch(uploadUrl, { + method: 'PUT', + headers: { + 'Content-Range': range, + 'Content-Length': chunkSize, + }, + body: chunk, + }); + + if (response.ok) { + // Response status is 308 until the final chunk. Final response includes file metadata + return ((await response.json()) as { id: string }).id; + } + if (response.status === 308) { + //continue + } else { + const text = await response.text(); + logger.error(`Received ${response.status} from google, error: ${text}`); + throw new Error( + `Failed to upload chunk: ${response.status} ${response.statusText}`, + ); + } + + offset += chunkSize; + fileStream.resume(); + } + + throw new Error('Failed to upload file'); +}; diff --git a/packages/media-export/src/index.ts b/packages/media-export/src/index.ts index 8eb4e12..2d39177 100644 --- a/packages/media-export/src/index.ts +++ b/packages/media-export/src/index.ts @@ -16,9 +16,9 @@ import { TranscriptExportRequest, ZTokenResponse, } from '@guardian/transcription-service-common'; -import { uploadFileToGoogleDrive } from 'api/src/services/googleDrive'; import { updateStatus } from 'api/src/export'; import { LAMBDA_MAX_EPHEMERAL_STORAGE_BYTES } from 'api/src/services/lambda'; +import { uploadFileToGoogleDrive } from './googleDrive'; export const exportMediaToDrive = async ( config: TranscriptionConfig,