diff --git a/serverless-compose.yml b/serverless-compose.yml index 7d1e4400cd..4e8c972e6e 100644 --- a/serverless-compose.yml +++ b/serverless-compose.yml @@ -9,6 +9,8 @@ services: path: src/services/data params: ECSFailureTopicArn: ${alerts.ECSFailureTopicArn} + uploads: + path: src/services/uploads ui-infra: path: src/services/ui-infra api: @@ -17,6 +19,10 @@ services: ECSFailureTopicArn: ${alerts.ECSFailureTopicArn} osDomainArn: ${data.OpenSearchDomainArn} osDomain: ${data.OpenSearchDomainEndpoint} + topicName: ${data.TopicName} + attachmentsBucketName: ${uploads.AttachmentsBucketName} + attachmentsBucketRegion: ${uploads.AttachmentsBucketRegion} + attachmentsBucketArn: ${uploads.AttachmentsBucketArn} auth: path: src/services/auth params: diff --git a/src/services/api/handlers/getAttachmentUrl.ts b/src/services/api/handlers/getAttachmentUrl.ts index b1d3f10465..8747890cc5 100644 --- a/src/services/api/handlers/getAttachmentUrl.ts +++ b/src/services/api/handlers/getAttachmentUrl.ts @@ -69,7 +69,7 @@ export const handler = async (event: APIGatewayEvent) => { } // Now we can generate the presigned url - const url = await generatePresignedS3Url(body.bucket, body.key, 60); + const url = await generateLegacyPresignedS3Url(body.bucket, body.key, 60); return response({ statusCode: 200, @@ -84,32 +84,37 @@ export const handler = async (event: APIGatewayEvent) => { } }; -async function generatePresignedS3Url(bucket, key, expirationInSeconds) { - // Create an S3 client - const roleToAssumeArn = process.env.onemacLegacyS3AccessRoleArn; - - // Create an STS client to make the AssumeRole API call - const stsClient = new STSClient({}); - - // Assume the role - const assumedRoleResponse = await stsClient.send( - new AssumeRoleCommand({ - RoleArn: roleToAssumeArn, - RoleSessionName: "AssumedRoleSession", - }) - ); - - // Extract the assumed role credentials - const assumedCredentials = assumedRoleResponse.Credentials; - - // Create S3 client using the assumed role's credentials - const assumedS3Client = new S3Client({ - credentials: { - accessKeyId: assumedCredentials.AccessKeyId, - secretAccessKey: assumedCredentials.SecretAccessKey, - sessionToken: assumedCredentials.SessionToken, - }, - }); +async function getClient(bucket) { + if (bucket.startsWith("uploads")) { + const stsClient = new STSClient({}); + + // Assume the role + const assumedRoleResponse = await stsClient.send( + new AssumeRoleCommand({ + RoleArn: process.env.onemacLegacyS3AccessRoleArn, + RoleSessionName: "AssumedRoleSession", + }) + ); + + // Extract the assumed role credentials + const assumedCredentials = assumedRoleResponse.Credentials; + + // Create S3 client using the assumed role's credentials + return new S3Client({ + credentials: { + accessKeyId: assumedCredentials.AccessKeyId, + secretAccessKey: assumedCredentials.SecretAccessKey, + sessionToken: assumedCredentials.SessionToken, + }, + }); + } else { + return new S3Client({}); + } +} + +async function generateLegacyPresignedS3Url(bucket, key, expirationInSeconds) { + // Get an S3 client + const client = await getClient(bucket); // Create a command to get the object (you can adjust this according to your use case) const getObjectCommand = new GetObjectCommand({ @@ -118,7 +123,7 @@ async function generatePresignedS3Url(bucket, key, expirationInSeconds) { }); // Generate a presigned URL - const presignedUrl = await getSignedUrl(assumedS3Client, getObjectCommand, { + const presignedUrl = await getSignedUrl(client, getObjectCommand, { expiresIn: expirationInSeconds, }); diff --git a/src/services/api/handlers/getUploadUrl.ts b/src/services/api/handlers/getUploadUrl.ts new file mode 100644 index 0000000000..276ad47e6a --- /dev/null +++ b/src/services/api/handlers/getUploadUrl.ts @@ -0,0 +1,52 @@ +import { response } from "../libs/handler"; +import { APIGatewayEvent } from "aws-lambda"; +import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3"; +import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; +import { v4 as uuidv4 } from "uuid"; + +checkEnvVariables(["attachmentsBucketName", "attachmentsBucketRegion"]); + +const s3 = new S3Client({ + region: process.env.attachmentsBucketRegion, +}); + +export const handler = async (event: APIGatewayEvent) => { + try { + const body = JSON.parse(event.body); + const bucket = process.env.attachmentsBucketName; + const key = uuidv4(); + const url = await getSignedUrl( + s3, + new PutObjectCommand({ + Bucket: bucket, + Key: key, + }), + { + expiresIn: 60, + } + ); + + return response({ + statusCode: 200, + body: { url, bucket, key }, + }); + } catch (error) { + console.error({ error }); + return response({ + statusCode: 500, + body: { message: "Internal server error" }, + }); + } +}; + +function checkEnvVariables(requiredEnvVariables) { + const missingVariables = requiredEnvVariables.filter( + (envVar) => !process.env[envVar] + ); + + if (missingVariables.length > 0) { + throw new Error( + `Missing required environment variables: ${missingVariables.join(", ")}` + ); + } +} diff --git a/src/services/api/serverless.yml b/src/services/api/serverless.yml index a97494897c..e4d902e37f 100644 --- a/src/services/api/serverless.yml +++ b/src/services/api/serverless.yml @@ -39,6 +39,14 @@ provider: - sts:AssumeRole Resource: - ${self:custom.onemacLegacyS3AccessRoleArn} + - Effect: Allow + Action: + - s3:PutObject + - s3:PutObjectTagging + - s3:GetObject + - s3:GetObjectTagging + Resource: + - ${param:attachmentsBucketArn}/* custom: project: ${env:PROJECT} @@ -52,6 +60,7 @@ custom: vpc: ${ssm:/aws/reference/secretsmanager/${self:custom.project}/${sls:stage}/vpc, ssm:/aws/reference/secretsmanager/${self:custom.project}/default/vpc} onemacLegacyS3AccessRoleArn: ${ssm:/aws/reference/secretsmanager/${self:custom.project}/${sls:stage}/onemacLegacyS3AccessRoleArn, ssm:/aws/reference/secretsmanager/${self:custom.project}/default/onemacLegacyS3AccessRoleArn} dbInfo: ${ssm:/aws/reference/secretsmanager/${self:custom.project}/${sls:stage}/seatool/dbInfo, ssm:/aws/reference/secretsmanager/${self:custom.project}/default/seatool/dbInfo} + brokerString: ${ssm:/aws/reference/secretsmanager/${self:custom.project}/${sls:stage}/brokerString, ssm:/aws/reference/secretsmanager/${self:custom.project}/default/brokerString} scriptable: hooks: package:compileEvents: ./handlers/repack.js @@ -115,6 +124,17 @@ functions: subnetIds: >- ${self:custom.vpc.privateSubnets} provisionedConcurrency: ${param:getAttachmentUrlProvisionedConcurrency} + getUploadUrl: + handler: handlers/getUploadUrl.handler + environment: + attachmentsBucketName: ${param:attachmentsBucketName} + attachmentsBucketRegion: ${param:attachmentsBucketRegion} + events: + - http: + path: /getUploadUrl + method: post + cors: true + authorizer: aws_iam item: handler: handlers/item.handler maximumRetryAttempts: 0 @@ -142,6 +162,8 @@ functions: dbPort: ${self:custom.dbInfo.port} dbUser: ${self:custom.dbInfo.user} dbPassword: ${self:custom.dbInfo.password} + topicName: ${param:topicName} + brokerString: ${self:custom.brokerString} events: - http: path: /submit diff --git a/src/services/data/serverless.yml b/src/services/data/serverless.yml index 0e4d748823..5950a795f7 100644 --- a/src/services/data/serverless.yml +++ b/src/services/data/serverless.yml @@ -78,7 +78,6 @@ custom: then aws lambda invoke --region ${self:provider.region} --function-name ${self:service}-${sls:stage}-bootstrapUsers --invocation-type RequestResponse /dev/null fi - stepFunctions: stateMachines: reindex: @@ -681,3 +680,5 @@ resources: Value: !Sub https://${OpenSearch.DomainEndpoint} OpenSearchDashboardEndpoint: Value: !Sub https://${OpenSearch.DomainEndpoint}/_dashboards + TopicName: + Value: ${param:topicNamespace}aws.onemac.migration.cdc diff --git a/src/services/uploads/package.json b/src/services/uploads/package.json new file mode 100644 index 0000000000..c23c026318 --- /dev/null +++ b/src/services/uploads/package.json @@ -0,0 +1,11 @@ +{ + "name": "uploads", + "description": "", + "private": true, + "version": "0.0.0", + "main": "index.js", + "scripts": {}, + "author": "", + "license": "CC0-1.0", + "devDependencies": {} +} diff --git a/src/services/uploads/serverless.yml b/src/services/uploads/serverless.yml new file mode 100644 index 0000000000..280c152962 --- /dev/null +++ b/src/services/uploads/serverless.yml @@ -0,0 +1,195 @@ +service: ${self:custom.project}-uploads +frameworkVersion: "3" + +plugins: + - serverless-esbuild + - serverless-stack-termination-protection + - "@stratiformdigital/serverless-iam-helper" + - "@stratiformdigital/serverless-s3-security-helper" + - serverless-scriptable-plugin +provider: + name: aws + runtime: nodejs18.x + region: ${env:REGION_A} + stackTags: + PROJECT: ${self:custom.project} + SERVICE: ${self:service} + iam: + role: + path: /delegatedadmin/developer/ + permissionsBoundary: arn:aws:iam::${aws:accountId}:policy/cms-cloud-admin/developer-boundary-policy + statements: + - Effect: "Allow" + Action: + - s3:GetObject + - s3:GetObjectTagging + - s3:PutObject + - s3:PutObjectAcl + - s3:PutObjectTagging + - s3:PutObjectVersionTagging + - s3:DeleteObject + - s3:ListBucket + Resource: + - !Sub arn:aws:s3:::${self:service}-${sls:stage}-attachments-${AWS::AccountId}/* + - !Sub arn:aws:s3:::${self:service}-${sls:stage}-avscan-${AWS::AccountId}/* + - Effect: "Allow" + Action: + - s3:ListBucket + Resource: + - !Sub arn:aws:s3:::${self:service}-${sls:stage}-attachments-${AWS::AccountId} + - !Sub arn:aws:s3:::${self:service}-${sls:stage}-avscan-${AWS::AccountId} + - Effect: "Allow" + Action: + - lambda:InvokeFunction + Resource: + - !Sub arn:aws:lambda:${self:provider.region}:${AWS::AccountId}:function:${self:service}-${sls:stage}-avDownloadDefinitions + +custom: + project: ${env:PROJECT} + accountId: !Sub "${AWS::AccountId}" + stage: ${opt:stage, self:provider.stage} + serverlessTerminationProtection: + stages: # Apply CloudFormation termination protection for these stages + - master + - val + - production + scriptable: + hooks: + package:initialize: | + set -e + curl -s -L --output lambda_layer.zip https://github.com/CMSgov/lambda-clamav-layer/releases/download/0.7/lambda_layer.zip + deploy:finalize: | + rm lambda_layer.zip + +layers: + clamDefs: + name: clamDefs-${self:service}-${sls:stage} + package: + artifact: lambda_layer.zip + +functions: + avScan: + handler: src/antivirus.lambdaHandleEvent + name: ${self:service}-${sls:stage}-avScan + timeout: 300 # 300 seconds = 5 minutes. Average scan is 25 seconds. + memorySize: 3008 + layers: + - !Ref ClamDefsLambdaLayer + environment: + CLAMAV_BUCKET_NAME: !Ref ClamDefsBucket + PATH_TO_AV_DEFINITIONS: "lambda/s3-antivirus/av-definitions" + avDownloadDefinitions: + handler: src/download-definitions.lambdaHandleEvent + events: + - schedule: cron(0 10 */1 * ? *) + timeout: 300 # 300 seconds = 5 minutes + memorySize: 1024 + layers: + - !Ref ClamDefsLambdaLayer + environment: + CLAMAV_BUCKET_NAME: !Ref ClamDefsBucket + PATH_TO_AV_DEFINITIONS: "lambda/s3-antivirus/av-definitions" + triggerInitialDownload: + handler: src/triggerInitialDownload.handler + timeout: 300 # 300 seconds = 5 minutes + memorySize: 1024 +resources: + Resources: + AttachmentsBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub ${self:service}-${sls:stage}-attachments-${AWS::AccountId} + # Set the CORS policy + CorsConfiguration: + CorsRules: + - AllowedOrigins: + - "*" + AllowedHeaders: + - "*" + AllowedMethods: + - GET + - PUT + - POST + - DELETE + - HEAD + ExposedHeaders: + - ETag + MaxAge: 3000 + NotificationConfiguration: + LambdaConfigurations: + - Event: s3:ObjectCreated:* + Function: !GetAtt AvScanLambdaFunction.Arn + DependsOn: LambdaInvokePermission + S3CMSReadBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: + Ref: AttachmentsBucket + PolicyDocument: + Statement: + - Effect: Deny + Principal: "*" + Action: "s3:*" + Resource: + - !Sub ${AttachmentsBucket.Arn}/* + - !Sub ${AttachmentsBucket.Arn} + Condition: + Bool: + "aws:SecureTransport": "false" + - Effect: "Deny" + Principal: "*" + Action: + - "s3:GetObject" + Resource: + - !Sub ${AttachmentsBucket.Arn}/* + Condition: + StringNotEquals: + s3:ExistingObjectTag/virusScanStatus: + - "CLEAN" + aws:PrincipalArn: !GetAtt IamRoleLambdaExecution.Arn + LambdaInvokePermission: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !GetAtt AvScanLambdaFunction.Arn + Action: lambda:InvokeFunction + Principal: s3.amazonaws.com + SourceAccount: !Sub ${AWS::AccountId} + SourceArn: !Sub arn:aws:s3:::${self:service}-${sls:stage}-attachments-${AWS::AccountId} + ClamDefsBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub ${self:service}-${sls:stage}-avscan-${AWS::AccountId} + AccessControl: Private + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + ClamDefsBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: + Ref: ClamDefsBucket + PolicyDocument: + Statement: + - Effect: Deny + Principal: "*" + Action: "s3:*" + Resource: + - !Sub ${ClamDefsBucket.Arn}/* + - !Sub ${ClamDefsBucket.Arn} + Condition: + Bool: + "aws:SecureTransport": "false" + TriggerInitialDownload: + Type: Custom::TriggerInitialDownload + Properties: + ServiceToken: !GetAtt TriggerInitialDownloadLambdaFunction.Arn + FunctionName: !GetAtt AvDownloadDefinitionsLambdaFunction.Arn + Outputs: + AttachmentsBucketName: + Value: + Ref: AttachmentsBucket + AttachmentsBucketArn: + Value: !GetAtt AttachmentsBucket.Arn + AttachmentsBucketRegion: + Value: ${self:provider.region} diff --git a/src/services/uploads/src/antivirus.ts b/src/services/uploads/src/antivirus.ts new file mode 100644 index 0000000000..2e0a2d8118 --- /dev/null +++ b/src/services/uploads/src/antivirus.ts @@ -0,0 +1,140 @@ +import { + S3Client, + HeadObjectCommand, + GetObjectCommand, + PutObjectTaggingCommand, +} from "@aws-sdk/client-s3"; +import { randomUUID } from "crypto"; +import fs from "fs"; +import asyncfs from "fs/promises"; + +import { downloadAVDefinitions, scanLocalFile } from "./clamav"; +import * as utils from "./utils"; +import * as constants from "./constants"; + +const s3Client: S3Client = new S3Client(); + +export async function isS3FileTooBig( + key: string, + bucket: string +): Promise { + try { + const res: HeadObjectCommandOutput = await s3Client.send( + new HeadObjectCommand({ Key: key, Bucket: bucket }) + ); + return res.ContentLength > constants.MAX_FILE_SIZE; + } catch (e) { + utils.generateSystemMessage( + `Error finding size of S3 Object: s3://${bucket}/${key}` + ); + return false; + } +} + +async function downloadFileFromS3( + s3ObjectKey: string, + s3ObjectBucket: string +): Promise { + if (!fs.existsSync(constants.TMP_DOWNLOAD_PATH)) { + fs.mkdirSync(constants.TMP_DOWNLOAD_PATH); + } + + const localPath: string = `${constants.TMP_DOWNLOAD_PATH}${randomUUID()}.tmp`; + const writeStream: fs.WriteStream = fs.createWriteStream(localPath); + + utils.generateSystemMessage( + `Downloading file s3://${s3ObjectBucket}/${s3ObjectKey}` + ); + + const options = { + Bucket: s3ObjectBucket, + Key: s3ObjectKey, + }; + + try { + const { Body } = await s3Client.send(new GetObjectCommand(options)); + await asyncfs.writeFile(localPath, Body); + utils.generateSystemMessage( + `Finished downloading new object ${s3ObjectKey}` + ); + return localPath; + } catch (err) { + console.error(err); + throw err; + } +} + +const scanAndTagS3Object = async ( + s3ObjectKey: string, + s3ObjectBucket: string +): Promise => { + utils.generateSystemMessage( + `S3 Bucket and Key\n ${s3ObjectBucket}\n${s3ObjectKey}` + ); + + let virusScanStatus: string; + + if (await isS3FileTooBig(s3ObjectKey, s3ObjectBucket)) { + virusScanStatus = constants.STATUS_SKIPPED_FILE; + utils.generateSystemMessage( + `S3 File is too big. virusScanStatus=${virusScanStatus}` + ); + } else { + utils.generateSystemMessage("Download AV Definitions"); + await downloadAVDefinitions(); + utils.generateSystemMessage("Download File from S3"); + const fileLoc: string = await downloadFileFromS3( + s3ObjectKey, + s3ObjectBucket + ); + utils.generateSystemMessage("Set virusScanStatus"); + virusScanStatus = scanLocalFile(fileLoc); + utils.generateSystemMessage(`virusScanStatus=${virusScanStatus}`); + } + + const taggingParams = { + Bucket: s3ObjectBucket, + Key: s3ObjectKey, + Tagging: utils.generateTagSet(virusScanStatus), + }; + + try { + await s3Client.send(new PutObjectTaggingCommand(taggingParams)); + utils.generateSystemMessage("Tagging successful"); + } catch (err) { + console.error(err); + } + + return virusScanStatus; +}; + +export async function lambdaHandleEvent(event: any): Promise { + utils.generateSystemMessage( + `Start avScan with event ${JSON.stringify(event, null, 2)}` + ); + + let s3ObjectKey: string, s3ObjectBucket: string; + + if (event.s3ObjectKey && event.s3ObjectBucket) { + s3ObjectKey = event.s3ObjectKey; + s3ObjectBucket = event.s3ObjectBucket; + } else if ( + event.Records && + Array.isArray(event.Records) && + event.Records[0]?.eventSource === "aws:s3" + ) { + s3ObjectKey = utils.extractKeyFromS3Event(event); + s3ObjectBucket = utils.extractBucketFromS3Event(event); + } else { + utils.generateSystemMessage( + `Event missing s3ObjectKey or s3ObjectBucket: ${JSON.stringify( + event, + null, + 2 + )}` + ); + return constants.STATUS_ERROR_PROCESSING_FILE; + } + + return await scanAndTagS3Object(s3ObjectKey, s3ObjectBucket); +} diff --git a/src/services/uploads/src/clamav.ts b/src/services/uploads/src/clamav.ts new file mode 100644 index 0000000000..19c6d5d05d --- /dev/null +++ b/src/services/uploads/src/clamav.ts @@ -0,0 +1,244 @@ +import { + S3Client, + ListObjectsV2Command, + GetObjectCommand, + PutObjectCommand, + DeleteObjectsCommand, +} from "@aws-sdk/client-s3"; +import { spawnSync, SpawnSyncReturns } from "child_process"; +import path from "path"; +import fs from "fs"; +import asyncfs from "fs/promises"; +import * as constants from "./constants"; +import * as utils from "./utils"; + +const s3Client: S3Client = new S3Client(); + +export async function listBucketFiles(bucketName: string): Promise { + try { + const listFilesResult = await s3Client.send( + new ListObjectsV2Command({ Bucket: bucketName }) + ); + if (listFilesResult.Contents) { + const keys: string[] = listFilesResult.Contents.map((c) => c.Key); + return keys; + } else { + return []; + } + } catch (err) { + utils.generateSystemMessage("Error listing files"); + console.error(err); + throw err; + } +} + +export const updateAVDefinitonsWithFreshclam = (): boolean => { + try { + const { stdout, stderr, error }: SpawnSyncReturns = spawnSync( + `${constants.PATH_TO_FRESHCLAM}`, + [ + `--config-file=${constants.FRESHCLAM_CONFIG}`, + `--datadir=${constants.FRESHCLAM_WORK_DIR}`, + ] + ); + utils.generateSystemMessage("Update message"); + console.log(stdout.toString()); + + console.log("Downloaded:", fs.readdirSync(constants.FRESHCLAM_WORK_DIR)); + + if (stderr) { + utils.generateSystemMessage("stderr"); + console.log(stderr.toString()); + } + + return true; + } catch (err) { + console.log("in the catch"); + console.log(err); + return false; + } +}; + +/** + * Download the Antivirus definition from S3. + * The definitions are stored on the local disk, ensure there's enough space. + */ +export const downloadAVDefinitions = async (): Promise => { + // list all the files in that bucket + utils.generateSystemMessage("Downloading Definitions"); + const allFileKeys: string[] = await listBucketFiles( + constants.CLAMAV_BUCKET_NAME + ); + + const definitionFileKeys: string[] = allFileKeys + .filter((key) => key.startsWith(constants.PATH_TO_AV_DEFINITIONS)) + .map((fullPath) => path.basename(fullPath)); + + // download each file in the bucket. + const downloadPromises: Promise[] = definitionFileKeys.map( + (filenameToDownload) => { + return new Promise(async (resolve, reject) => { + const destinationFile: string = path.join( + constants.FRESHCLAM_WORK_DIR, + filenameToDownload + ); + + utils.generateSystemMessage( + `Downloading ${filenameToDownload} from S3 to ${destinationFile}` + ); + + const localFileWriteStream = fs.createWriteStream(destinationFile); + + const options = { + Bucket: constants.CLAMAV_BUCKET_NAME, + Key: `${constants.PATH_TO_AV_DEFINITIONS}/${filenameToDownload}`, + }; + + try { + const { Body } = await s3Client.send(new GetObjectCommand(options)); + await asyncfs.writeFile(destinationFile, Body); + utils.generateSystemMessage( + `Finished download ${filenameToDownload}` + ); + resolve(); + } catch (err) { + utils.generateSystemMessage( + `Error downloading definition file ${filenameToDownload}` + ); + console.log(err); + reject(); + } + }); + } + ); + + return await Promise.all(downloadPromises); +}; + +/** + * Uploads the AV definitions to the S3 bucket. + */ +export const uploadAVDefinitions = async (): Promise => { + // delete all the definitions currently in the bucket. + // first list them. + utils.generateSystemMessage("Uploading Definitions"); + const s3AllFullKeys: string[] = await listBucketFiles( + constants.CLAMAV_BUCKET_NAME + ); + const s3DefinitionFileFullKeys: string[] = s3AllFullKeys.filter((key) => + key.startsWith(constants.PATH_TO_AV_DEFINITIONS) + ); + + // If there are any s3 Definition files in the s3 bucket, delete them. + if (s3DefinitionFileFullKeys && s3DefinitionFileFullKeys.length !== 0) { + try { + await s3Client.send( + new DeleteObjectsCommand({ + Bucket: constants.CLAMAV_BUCKET_NAME, + Delete: { + Objects: s3DefinitionFileFullKeys.map((k) => { + return { Key: k }; + }), + }, + }) + ); + + utils.generateSystemMessage( + `Deleted extant definitions: ${s3DefinitionFileFullKeys}` + ); + } catch (err) { + utils.generateSystemMessage( + `Error deleting current definition files: ${s3DefinitionFileFullKeys}` + ); + console.log(err); + throw err; + } + } + + // list all the files in the work dir for upload + const definitionFiles: string[] = fs.readdirSync( + constants.FRESHCLAM_WORK_DIR + ); + + const uploadPromises: Promise[] = definitionFiles.map( + (filenameToUpload) => { + return new Promise(async (resolve, reject) => { + utils.generateSystemMessage( + `Uploading updated definitions for file ${filenameToUpload} ---` + ); + + const options = { + Bucket: constants.CLAMAV_BUCKET_NAME, + Key: `${constants.PATH_TO_AV_DEFINITIONS}/${filenameToUpload}`, + Body: fs.readFileSync( + path.join(constants.FRESHCLAM_WORK_DIR, filenameToUpload) + ), + }; + + try { + await s3Client.send(new PutObjectCommand(options)); + resolve(); + utils.generateSystemMessage( + `--- Finished uploading ${filenameToUpload} ---` + ); + } catch (err) { + utils.generateSystemMessage( + `--- Error uploading ${filenameToUpload} ---` + ); + console.log(err); + reject(); + } + }); + } + ); + + return await Promise.all(uploadPromises); +}; + +/** + * Function to scan the given file. This function requires ClamAV and the definitions to be available. + * This function does not download the file so the file should also be accessible. + * + * Three possible case can happen: + * - The file is clean, the clamAV command returns 0 and the function return "CLEAN" + * - The file is infected, the clamAV command returns 1 and this function will return "INFECTED" + * - Any other error and the function will return null; (falsey) + * + * @param pathToFile Path in the filesystem where the file is stored. + */ +export const scanLocalFile = (pathToFile: string): string | null => { + try { + const avResult: SpawnSyncReturns = spawnSync( + constants.PATH_TO_CLAMAV, + [ + "--stdout", + "-v", + "-a", + "-d", + constants.FRESHCLAM_WORK_DIR, + pathToFile, + ] + ); + + // status 1 means that the file is infected. + if (avResult.status === 1) { + utils.generateSystemMessage("SUCCESSFUL SCAN, FILE INFECTED"); + return constants.STATUS_INFECTED_FILE; + } else if (avResult.status !== 0) { + utils.generateSystemMessage("-- SCAN FAILED WITH ERROR --"); + console.error("stderror", avResult.stderr.toString()); + console.error("stdout", avResult.stdout.toString()); + console.error("err", avResult.error); + return constants.STATUS_ERROR_PROCESSING_FILE; + } + + utils.generateSystemMessage("SUCCESSFUL SCAN, FILE CLEAN"); + console.log(avResult.stdout.toString()); + + return constants.STATUS_CLEAN_FILE; + } catch (err) { + utils.generateSystemMessage("-- SCAN FAILED --"); + console.log(err); + return constants.STATUS_ERROR_PROCESSING_FILE; + } +}; \ No newline at end of file diff --git a/src/services/uploads/src/constants.ts b/src/services/uploads/src/constants.ts new file mode 100644 index 0000000000..61325d3bc6 --- /dev/null +++ b/src/services/uploads/src/constants.ts @@ -0,0 +1,46 @@ +/** + * Exposes the constants used throughout the program. + * + * The following variables have to be set: + * + * CLAMAV_BUCKET_NAME: Name of the bucket where ClamAV and its definitions are stored + * PATH_TO_AV_DEFINITIONS: Path in S3 where the definitions are stored. + * + * The following variables can be overridden: + * + * STATUS_CLEAN_FILE: (default 'CLEAN') Tag that will be added to files that are clean. + * STATUS_INFECTED_FILE: (default 'INFECTED') Tag that will be added to files that are infected. + * STATUS_ERROR_PROCESSING_FILE: (default 'ERROR') Tag that will be added to files where the scan was not successful. + * VIRUS_SCAN_STATUS_KEY: (default 'virusScanStatus') Name of the tag that indicates the status of the virus scan. + * VIRUS_SCAN_TIMESTAMP_KEY: (default 'virusScanTimestamp') Name of the tag that indicates the time of the virus scan. + */ + +import process from "process"; + +// Various paths and application names on S3 +export const ATTACHMENTS_BUCKET: string | undefined = + process.env.ATTACHMENTS_BUCKET; +export const CLAMAV_BUCKET_NAME: string | undefined = + process.env.CLAMAV_BUCKET_NAME; +export const PATH_TO_AV_DEFINITIONS: string | undefined = + process.env.PATH_TO_AV_DEFINITIONS; +export const PATH_TO_FRESHCLAM: string = "/opt/bin/freshclam"; +export const PATH_TO_CLAMAV: string = "/opt/bin/clamscan"; +export const FRESHCLAM_CONFIG: string = "/opt/bin/freshclam.conf"; +export const FRESHCLAM_WORK_DIR: string = "/tmp/"; +export const TMP_DOWNLOAD_PATH: string = "/tmp/download/"; + +// Constants for tagging file after a virus scan. +export const STATUS_CLEAN_FILE: string = + process.env.STATUS_CLEAN_FILE || "CLEAN"; +export const STATUS_INFECTED_FILE: string = + process.env.STATUS_INFECTED_FILE || "INFECTED"; +export const STATUS_ERROR_PROCESSING_FILE: string = + process.env.STATUS_ERROR_PROCESSING_FILE || "ERROR"; +export const STATUS_SKIPPED_FILE: string = + process.env.STATUS_SKIPPED_FILE || "SKIPPED"; +export const VIRUS_SCAN_STATUS_KEY: string = + process.env.VIRUS_SCAN_STATUS_KEY || "virusScanStatus"; +export const VIRUS_SCAN_TIMESTAMP_KEY: string = + process.env.VIRUS_SCAN_TIMESTAMP_KEY || "virusScanTimestamp"; +export const MAX_FILE_SIZE: string = process.env.MAX_FILE_SIZE || "314572800"; diff --git a/src/services/uploads/src/download-definitions.ts b/src/services/uploads/src/download-definitions.ts new file mode 100644 index 0000000000..151a4dd710 --- /dev/null +++ b/src/services/uploads/src/download-definitions.ts @@ -0,0 +1,19 @@ +import * as clamav from "./clamav"; +import { generateSystemMessage, cleanupFolder } from "./utils"; +import { FRESHCLAM_WORK_DIR } from "./constants"; + +export async function lambdaHandleEvent(): Promise { + generateSystemMessage(`AV definition update start time: ${new Date()}`); + + await cleanupFolder(FRESHCLAM_WORK_DIR); + if (await clamav.updateAVDefinitonsWithFreshclam()) { + generateSystemMessage("Folder content after freshclam "); + await clamav.uploadAVDefinitions(); + + generateSystemMessage(`AV definition update end time: ${new Date()}`); + + return "DEFINITION UPDATE SUCCESS"; + } else { + return "DEFINITION UPDATE FAILED"; + } +} \ No newline at end of file diff --git a/src/services/uploads/src/triggerInitialDownload.ts b/src/services/uploads/src/triggerInitialDownload.ts new file mode 100644 index 0000000000..a9ff085c0d --- /dev/null +++ b/src/services/uploads/src/triggerInitialDownload.ts @@ -0,0 +1,27 @@ +import { Handler } from "aws-lambda"; +import { send, SUCCESS, FAILED } from "cfn-response-async"; +type ResponseStatus = typeof SUCCESS | typeof FAILED; +import { LambdaClient, InvokeCommand } from "@aws-sdk/client-lambda"; + +const lambdaClient = new LambdaClient({}); + +export const handler: Handler = async (event, context) => { + console.log("request:", JSON.stringify(event, undefined, 2)); + const responseData = {}; + let responseStatus: ResponseStatus = SUCCESS; + try { + if (event.RequestType == "Create" || event.RequestType == "Update") { + const invokeCommand = new InvokeCommand({ + FunctionName: event.ResourceProperties.FunctionName, + InvocationType: "RequestResponse", + }); + await lambdaClient.send(invokeCommand); + } + } catch (error) { + console.log(error); + responseStatus = FAILED; + } finally { + console.log("finally"); + await send(event, context, responseStatus, responseData); + } +}; diff --git a/src/services/uploads/src/utils.ts b/src/services/uploads/src/utils.ts new file mode 100644 index 0000000000..90ca9c4dbc --- /dev/null +++ b/src/services/uploads/src/utils.ts @@ -0,0 +1,91 @@ +import { execSync } from "child_process"; + +import * as constants from "./constants"; + +interface TagSet { + TagSet: Tag[]; +} + +interface Tag { + Key: string; + Value: string; +} + +/** + * Generates the set of tags that will be used to tag the files of S3. + * @param virusScanStatus String representing the status. + * @return {{TagSet: *[]}} TagSet ready to be attached to an S3 file. + */ +export function generateTagSet(virusScanStatus: string): TagSet { + return { + TagSet: [ + { + Key: constants.VIRUS_SCAN_STATUS_KEY, + Value: virusScanStatus, + }, + { + Key: constants.VIRUS_SCAN_TIMESTAMP_KEY, + Value: new Date().getTime().toString(), + }, + ], + }; +} + +/** + * Cleanup the specific S3 folder by removing all of its content. + * We need that to cleanup the /tmp/ folder after the download of the definitions. + */ +export function cleanupFolder(folderToClean: string): void { + let result: Buffer = execSync(`ls -l ${folderToClean}`); + + console.log("-- Folder before cleanup--"); + console.log(result.toString()); + + execSync(`rm -rf ${folderToClean}*`); + + result = execSync(`ls -l ${folderToClean}`); + + console.log("-- Folder after cleanup --"); + console.log(result.toString()); +} + +/** + * Extract the key from an S3 event. + * @param s3Event Inbound S3 event. + * @return {string} decoded key. + */ +export function extractKeyFromS3Event(s3Event: any): string { + const key: string = s3Event["Records"][0]["s3"]["object"]["key"]; + + if (!key) { + throw new Error("Unable to retrieve key information from the event"); + } + + return decodeURIComponent(key).replace(/\+/g, " "); +} + +/** + * Extract the bucket from an S3 event. + * @param s3Event Inbound S3 event. + * @return {string} Bucket + */ +export function extractBucketFromS3Event(s3Event: any): string { + const bucketName: string = s3Event["Records"][0]["s3"]["bucket"]["name"]; + + if (!bucketName) { + throw new Error("Unable to retrieve bucket information from the event"); + } + + return bucketName; +} + +/** + * Generates & logs a system message (simple --- the message here ---) + * @param systemMessage Inbound message to log and generate. + * @return {string} Formatted message. + */ +export function generateSystemMessage(systemMessage: string): string { + const finalMessage: string = `--- ${systemMessage} ---`; + console.log(finalMessage); + return finalMessage; +} diff --git a/src/services/uploads/tsconfig.json b/src/services/uploads/tsconfig.json new file mode 100644 index 0000000000..8b94d5c8d7 --- /dev/null +++ b/src/services/uploads/tsconfig.json @@ -0,0 +1,15 @@ +{ + "exclude": [ + "./docs/**/*", + "./src/services/**/*", + "./src/libs/**/*", + "./src/tests/**/**" + ], + "compilerOptions": { + "resolveJsonModule": true, + "target": "es5", + "module": "commonjs", + "strict": true, + "allowSyntheticDefaultImports": true + } +}