From 662d7964a8f0c8ffedc64fd38f92cddbe65b27fb Mon Sep 17 00:00:00 2001 From: mj Date: Fri, 4 Oct 2024 16:33:55 +0900 Subject: [PATCH 1/9] feat: integrate MinIO for file storage - Add MinIO support in FilesService - Update docker-compose.yml to include MinIO service - Implement environment variable switching between S3 and MinIO - Add MinIO endpoint configuration - Update documentation for MinIO setup and usage --- backend/.env.development | 11 ++++++--- backend/docker/docker-compose-full.yml | 28 +++++++++++++++++++++- backend/src/files/files.service.ts | 32 +++++++++++++++++++++---- backend/src/utils/types/storage.type.ts | 1 + 4 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 backend/src/utils/types/storage.type.ts diff --git a/backend/.env.development b/backend/.env.development index 236e5076..c32fcf41 100644 --- a/backend/.env.development +++ b/backend/.env.development @@ -64,9 +64,14 @@ LANGCHAIN_PROJECT=your_langsmith_project_name_here # FILE_UPLOAD: Whether to enable file upload to storage -# Set to true if file upload is required. # If set to false, AWS_S3_BUCKET_NAME is not required. +# Set to true if file upload is required. FILE_UPLOAD=false -# AWS_S3_BUCKET_NAME: S3 Bucket name +# S3 or MinIO configuration +BUCKET_TYPE="S3 or MINIO" # This is the name of the S3 Bucket -AWS_S3_BUCKET_NAME="your_s3_bucket_name" \ No newline at end of file +BUCKET_NAME="your_bucket_name" +AWS_REGION="your_aws_region" +MINIO_ENDPOINT="your_minio_endpoint" +MINIO_ACCESS_KEY="your_minio_access_key" +MINIO_SECRET_KEY="your_minio_secret_key" diff --git a/backend/docker/docker-compose-full.yml b/backend/docker/docker-compose-full.yml index 5bec6f56..d42e9ed0 100644 --- a/backend/docker/docker-compose-full.yml +++ b/backend/docker/docker-compose-full.yml @@ -22,15 +22,21 @@ services: LANGCHAIN_API_KEY: "your_langsmith_api_key_here" LANGCHAIN_PROJECT: "your_langsmith_project_name_here" FILE_UPLOAD: false - AWS_S3_BUCKET_NAME: "your_s3_bucket_name" + BUCKET_TYPE: "S3 or MINIO" + BUCKET_NAME: "your_s3_bucket_name_here" + MINIO_ENDPOINT: "your_minio_endpoint_here" + MINIO_ACCESS_KEY: "your_minio_access_key_here" + MINIO_SECRET_KEY: "your_minio_secret_key_here" ports: - "3000:3000" depends_on: - mongo + - minio restart: unless-stopped links: - "mongo:mongo" - "yorkie:yorkie" + - "minio:minio" yorkie: image: "yorkieteam/yorkie:latest" @@ -59,3 +65,23 @@ services: interval: 5s timeout: 2s retries: 20 + + minio: + image: minio/minio + ports: + - "9000:9000" + - "9001:9001" + volumes: + - minio_data:/data + environment: + MINIO_ROOT_USER: ${MINIO_ACCESS_KEY} + MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY} + command: server --console-address ":9001" /data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + +volumes: + minio_data: diff --git a/backend/src/files/files.service.ts b/backend/src/files/files.service.ts index c147bc65..af37c31f 100644 --- a/backend/src/files/files.service.ts +++ b/backend/src/files/files.service.ts @@ -1,4 +1,4 @@ -import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { GetObjectCommand, PutObjectCommand, S3Client, S3ClientConfig } from "@aws-sdk/client-s3"; import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; import { BadRequestException, @@ -13,6 +13,7 @@ import * as htmlPdf from "html-pdf-node"; import * as MarkdownIt from "markdown-it"; import { PrismaService } from "src/db/prisma.service"; import { generateRandomKey } from "src/utils/functions/random-string"; +import { StorageType } from "src/utils/types/storage.type"; import { CreateUploadPresignedUrlResponse } from "./types/create-upload-url-response.type"; import { ExportFileRequestBody, ExportFileResponse } from "./types/export-file.type"; @@ -25,7 +26,7 @@ export class FilesService { private configService: ConfigService, private prismaService: PrismaService ) { - this.s3Client = new S3Client(); + this.s3Client = new S3Client(this.getStorageConfig()); this.markdown = new MarkdownIt({ html: true, breaks: true, @@ -55,7 +56,7 @@ export class FilesService { const fileKey = `${workspace.slug}-${generateRandomKey()}.${contentType.split("/")[1]}`; const command = new PutObjectCommand({ - Bucket: this.configService.get("AWS_S3_BUCKET_NAME"), + Bucket: this.configService.get("BUCKET_NAME"), Key: fileKey, StorageClass: "INTELLIGENT_TIERING", ContentType: contentType, @@ -70,7 +71,7 @@ export class FilesService { async createDownloadPresignedUrl(fileKey: string) { try { const command = new GetObjectCommand({ - Bucket: this.configService.get("AWS_S3_BUCKET_NAME"), + Bucket: this.configService.get("BUCKET_NAME"), Key: fileKey, }); return getSignedUrl(this.s3Client, command, { expiresIn: 3600 }); @@ -125,4 +126,27 @@ export class FilesService { fileName: `${fileName}.pdf`, }; } + + private getStorageConfig = (): S3ClientConfig => { + const bucketType: StorageType = this.configService.get("BUCKET_TYPE") || "S3"; + const region = this.configService.get("AWS_REGION") || "us-east-1"; + if (bucketType === "MINIO") { + const endpoint = this.configService.get("MINIO_ENDPOINT"); + const accessKeyId = this.configService.get("MINIO_ACCESS_KEY"); + const secretAccessKey = this.configService.get("MINIO_SECRET_KEY"); + return { + region, + endpoint, + forcePathStyle: true, + credentials: { + accessKeyId, + secretAccessKey, + }, + }; + } + + return { + region, + }; + }; } diff --git a/backend/src/utils/types/storage.type.ts b/backend/src/utils/types/storage.type.ts new file mode 100644 index 00000000..e1b76854 --- /dev/null +++ b/backend/src/utils/types/storage.type.ts @@ -0,0 +1 @@ +export type StorageType = "S3" | "MINIO"; From 6ccd8d8471b8c29a9fc3562c9b5d0a383ad985c3 Mon Sep 17 00:00:00 2001 From: mj Date: Sat, 12 Oct 2024 11:16:11 +0900 Subject: [PATCH 2/9] feat: Add storage module for file upload - Add storage module to handle file upload functionality - Configure S3Client for storage service - Update files service to use injected S3Client - Update files module to import storage module --- backend/.env.development | 15 ++++----- backend/docker/docker-compose-full.yml | 12 ++++--- backend/src/app.module.ts | 18 ++++++----- backend/src/files/files.module.ts | 4 ++- backend/src/files/files.service.ts | 30 ++---------------- backend/src/storage/storage.module.ts | 43 ++++++++++++++++++++++++++ 6 files changed, 74 insertions(+), 48 deletions(-) create mode 100644 backend/src/storage/storage.module.ts diff --git a/backend/.env.development b/backend/.env.development index c32fcf41..328783b4 100644 --- a/backend/.env.development +++ b/backend/.env.development @@ -62,16 +62,17 @@ LANGCHAIN_API_KEY=your_langsmith_api_key_here # To create a LangSmith project, visit LangSmith: https://www.langchain.com/langsmith LANGCHAIN_PROJECT=your_langsmith_project_name_here - # FILE_UPLOAD: Whether to enable file upload to storage # If set to false, AWS_S3_BUCKET_NAME is not required. # Set to true if file upload is required. FILE_UPLOAD=false -# S3 or MinIO configuration -BUCKET_TYPE="S3 or MINIO" -# This is the name of the S3 Bucket -BUCKET_NAME="your_bucket_name" -AWS_REGION="your_aws_region" -MINIO_ENDPOINT="your_minio_endpoint" +# Storage configuration for either AWS S3 or MinIO +# This is the name of the S3 Bucket or MinIO Bucket +BUCKET_NAME="my-bucket" +# Common access keys for both S3 and MinIO MINIO_ACCESS_KEY="your_minio_access_key" MINIO_SECRET_KEY="your_minio_secret_key" +# AWS Region, only required for AWS S3 +AWS_REGION="your_aws_region" +# MinIO endpoint, only required for MinIO +MINIO_ENDPOINT="your_minio_endpoint" diff --git a/backend/docker/docker-compose-full.yml b/backend/docker/docker-compose-full.yml index d42e9ed0..b74b358e 100644 --- a/backend/docker/docker-compose-full.yml +++ b/backend/docker/docker-compose-full.yml @@ -22,11 +22,12 @@ services: LANGCHAIN_API_KEY: "your_langsmith_api_key_here" LANGCHAIN_PROJECT: "your_langsmith_project_name_here" FILE_UPLOAD: false - BUCKET_TYPE: "S3 or MINIO" BUCKET_NAME: "your_s3_bucket_name_here" + AWS_REGION: "your_aws_region_here" MINIO_ENDPOINT: "your_minio_endpoint_here" - MINIO_ACCESS_KEY: "your_minio_access_key_here" - MINIO_SECRET_KEY: "your_minio_secret_key_here" + # Make sure to also set STORAGE_ACCESS_KEY and STORAGE_SECRET_KEY for the MinIO service below + STORAGE_ACCESS_KEY: "your_storage_access_key_here" + STORAGE_SECRET_KEY: "your_storage_secret_key_here" ports: - "3000:3000" depends_on: @@ -66,6 +67,7 @@ services: timeout: 2s retries: 20 + # If you are not using MinIO, comment out this service and related configurations. minio: image: minio/minio ports: @@ -74,8 +76,8 @@ services: volumes: - minio_data:/data environment: - MINIO_ROOT_USER: ${MINIO_ACCESS_KEY} - MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY} + MINIO_ROOT_USER: "your_storage_access_key_here" + MINIO_ROOT_PASSWORD: "your_storage_secret_key_here" command: server --console-address ":9001" /data healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index 688bf78d..626de627 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -1,19 +1,20 @@ import { Module } from "@nestjs/common"; -import { PrismaService } from "./db/prisma.service"; -import { UsersModule } from "./users/users.module"; -import { AuthModule } from "./auth/auth.module"; import { ConfigModule } from "@nestjs/config"; import { APP_GUARD } from "@nestjs/core/constants"; +import { AuthModule } from "./auth/auth.module"; import { JwtAuthGuard } from "./auth/jwt.guard"; -import { WorkspacesModule } from "./workspaces/workspaces.module"; -import { WorkspaceUsersModule } from "./workspace-users/workspace-users.module"; -import { WorkspaceDocumentsModule } from "./workspace-documents/workspace-documents.module"; -import { DocumentsModule } from "./documents/documents.module"; import { CheckModule } from "./check/check.module"; +import { PrismaService } from "./db/prisma.service"; +import { DocumentsModule } from "./documents/documents.module"; +import { FilesModule } from "./files/files.module"; import { IntelligenceModule } from "./intelligence/intelligence.module"; import { LangchainModule } from "./langchain/langchain.module"; -import { FilesModule } from "./files/files.module"; import { SettingsModule } from "./settings/settings.module"; +import { StorageModule } from "./storage/storage.module"; +import { UsersModule } from "./users/users.module"; +import { WorkspaceDocumentsModule } from "./workspace-documents/workspace-documents.module"; +import { WorkspaceUsersModule } from "./workspace-users/workspace-users.module"; +import { WorkspacesModule } from "./workspaces/workspaces.module"; @Module({ imports: [ @@ -34,6 +35,7 @@ import { SettingsModule } from "./settings/settings.module"; FilesModule, ConfigModule, SettingsModule, + StorageModule, ], controllers: [], providers: [ diff --git a/backend/src/files/files.module.ts b/backend/src/files/files.module.ts index 0f5b0829..8bf7dc91 100644 --- a/backend/src/files/files.module.ts +++ b/backend/src/files/files.module.ts @@ -1,9 +1,11 @@ import { Module } from "@nestjs/common"; +import { PrismaService } from "src/db/prisma.service"; +import { StorageModule } from "src/storage/storage.module"; import { FilesController } from "./files.controller"; import { FilesService } from "./files.service"; -import { PrismaService } from "src/db/prisma.service"; @Module({ + imports: [StorageModule], controllers: [FilesController], providers: [FilesService, PrismaService], }) diff --git a/backend/src/files/files.service.ts b/backend/src/files/files.service.ts index af37c31f..7a79eb44 100644 --- a/backend/src/files/files.service.ts +++ b/backend/src/files/files.service.ts @@ -1,7 +1,8 @@ -import { GetObjectCommand, PutObjectCommand, S3Client, S3ClientConfig } from "@aws-sdk/client-s3"; +import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; import { BadRequestException, + Inject, Injectable, NotFoundException, UnauthorizedException, @@ -13,20 +14,18 @@ import * as htmlPdf from "html-pdf-node"; import * as MarkdownIt from "markdown-it"; import { PrismaService } from "src/db/prisma.service"; import { generateRandomKey } from "src/utils/functions/random-string"; -import { StorageType } from "src/utils/types/storage.type"; import { CreateUploadPresignedUrlResponse } from "./types/create-upload-url-response.type"; import { ExportFileRequestBody, ExportFileResponse } from "./types/export-file.type"; @Injectable() export class FilesService { - private s3Client: S3Client; private readonly markdown: MarkdownIt; constructor( + @Inject("STORAGE_CLIENT") private s3Client: S3Client, private configService: ConfigService, private prismaService: PrismaService ) { - this.s3Client = new S3Client(this.getStorageConfig()); this.markdown = new MarkdownIt({ html: true, breaks: true, @@ -126,27 +125,4 @@ export class FilesService { fileName: `${fileName}.pdf`, }; } - - private getStorageConfig = (): S3ClientConfig => { - const bucketType: StorageType = this.configService.get("BUCKET_TYPE") || "S3"; - const region = this.configService.get("AWS_REGION") || "us-east-1"; - if (bucketType === "MINIO") { - const endpoint = this.configService.get("MINIO_ENDPOINT"); - const accessKeyId = this.configService.get("MINIO_ACCESS_KEY"); - const secretAccessKey = this.configService.get("MINIO_SECRET_KEY"); - return { - region, - endpoint, - forcePathStyle: true, - credentials: { - accessKeyId, - secretAccessKey, - }, - }; - } - - return { - region, - }; - }; } diff --git a/backend/src/storage/storage.module.ts b/backend/src/storage/storage.module.ts new file mode 100644 index 00000000..cb9dbfc7 --- /dev/null +++ b/backend/src/storage/storage.module.ts @@ -0,0 +1,43 @@ +import { S3Client, S3ClientConfig } from "@aws-sdk/client-s3"; +import { Module } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; + +const s3ClientFactory = { + provide: "STORAGE_CLIENT", + useFactory: (configService: ConfigService): S3Client | null => { + if (configService.get("FILE_UPLOAD") === false) { + return null; + } + const region = configService.get("AWS_REGION") || "us-east-1"; + const endpoint = configService.get("MINIO_ENDPOINT"); + const accessKeyId = configService.get("STORAGE_ACCESS_KEY"); + const secretAccessKey = configService.get("STORAGE_SECRET_KEY"); + + const config: S3ClientConfig = endpoint + ? { + region, + endpoint, + forcePathStyle: true, + credentials: { + accessKeyId, + secretAccessKey, + }, + } + : { + region, + credentials: { + accessKeyId, + secretAccessKey, + }, + }; + + return new S3Client(config); + }, + inject: [ConfigService], +}; + +@Module({ + providers: [s3ClientFactory], + exports: [s3ClientFactory], +}) +export class StorageModule {} From 5627a975f8392420c1e5f87e7369e511c32485fe Mon Sep 17 00:00:00 2001 From: mj Date: Sat, 12 Oct 2024 11:20:11 +0900 Subject: [PATCH 3/9] refactor: storage module to remove unnecessary credentials case by aws_s3 --- backend/src/storage/storage.module.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/backend/src/storage/storage.module.ts b/backend/src/storage/storage.module.ts index cb9dbfc7..78b7c48f 100644 --- a/backend/src/storage/storage.module.ts +++ b/backend/src/storage/storage.module.ts @@ -25,10 +25,6 @@ const s3ClientFactory = { } : { region, - credentials: { - accessKeyId, - secretAccessKey, - }, }; return new S3Client(config); From 082502848d5f49cb739ea4632ccf0e8f73725a10 Mon Sep 17 00:00:00 2001 From: mj Date: Tue, 15 Oct 2024 02:11:16 +0900 Subject: [PATCH 4/9] refactor: storage module update condition check logic to use file_upload variable --- backend/src/storage/storage.module.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/storage/storage.module.ts b/backend/src/storage/storage.module.ts index 78b7c48f..038567af 100644 --- a/backend/src/storage/storage.module.ts +++ b/backend/src/storage/storage.module.ts @@ -5,7 +5,8 @@ import { ConfigService } from "@nestjs/config"; const s3ClientFactory = { provide: "STORAGE_CLIENT", useFactory: (configService: ConfigService): S3Client | null => { - if (configService.get("FILE_UPLOAD") === false) { + const fileUpload = configService.get("FILE_UPLOAD"); + if (!fileUpload) { return null; } const region = configService.get("AWS_REGION") || "us-east-1"; @@ -13,7 +14,7 @@ const s3ClientFactory = { const accessKeyId = configService.get("STORAGE_ACCESS_KEY"); const secretAccessKey = configService.get("STORAGE_SECRET_KEY"); - const config: S3ClientConfig = endpoint + const config: S3ClientConfig = fileUpload ? { region, endpoint, From cb403cdaea91d6a26d1dc106f6f8d0d6d90d64b4 Mon Sep 17 00:00:00 2001 From: mj Date: Wed, 16 Oct 2024 12:42:46 +0900 Subject: [PATCH 5/9] feat: set default configuration values for Minio --- backend/docker/docker-compose-full.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/backend/docker/docker-compose-full.yml b/backend/docker/docker-compose-full.yml index b74b358e..29150f6d 100644 --- a/backend/docker/docker-compose-full.yml +++ b/backend/docker/docker-compose-full.yml @@ -21,13 +21,13 @@ services: LANGCHAIN_ENDPOINT: "https://www.langchain.com/langsmith" LANGCHAIN_API_KEY: "your_langsmith_api_key_here" LANGCHAIN_PROJECT: "your_langsmith_project_name_here" - FILE_UPLOAD: false - BUCKET_NAME: "your_s3_bucket_name_here" + FILE_UPLOAD: true AWS_REGION: "your_aws_region_here" - MINIO_ENDPOINT: "your_minio_endpoint_here" - # Make sure to also set STORAGE_ACCESS_KEY and STORAGE_SECRET_KEY for the MinIO service below - STORAGE_ACCESS_KEY: "your_storage_access_key_here" - STORAGE_SECRET_KEY: "your_storage_secret_key_here" + # Default configuration values for using Minio + BUCKET_NAME: "default-storage" + MINIO_ENDPOINT: "http://localhost:9000" + STORAGE_ACCESS_KEY: "minioadmin" + STORAGE_SECRET_KEY: "minioadmin" ports: - "3000:3000" depends_on: @@ -67,7 +67,7 @@ services: timeout: 2s retries: 20 - # If you are not using MinIO, comment out this service and related configurations. + # You can remove the following content if you're using S3 or not using Minio. minio: image: minio/minio ports: @@ -76,9 +76,15 @@ services: volumes: - minio_data:/data environment: - MINIO_ROOT_USER: "your_storage_access_key_here" - MINIO_ROOT_PASSWORD: "your_storage_secret_key_here" - command: server --console-address ":9001" /data + MINIO_ROOT_USER: "minioadmin" + MINIO_ROOT_PASSWORD: "minioadmin" + entrypoint: /bin/sh -c " + /usr/bin/docker-entrypoint.sh server --console-address ':9001' --address ':9000' /data & + sleep 1 && + mc alias set myminio http://localhost:9000 minioadmin minioadmin && + if ! mc ls myminio/default-storage; then mc mb myminio/default-storage; fi && + wait" + command: server --console-address ":9001" --address ":9000" /data healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] interval: 30s From 322201c2ba3016f5ad00678491c245bc66d6036f Mon Sep 17 00:00:00 2001 From: mj Date: Wed, 16 Oct 2024 12:43:32 +0900 Subject: [PATCH 6/9] refactor: update storage module to remove default value for AWS_REGION --- backend/src/storage/storage.module.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/storage/storage.module.ts b/backend/src/storage/storage.module.ts index 038567af..ce18a69a 100644 --- a/backend/src/storage/storage.module.ts +++ b/backend/src/storage/storage.module.ts @@ -9,7 +9,7 @@ const s3ClientFactory = { if (!fileUpload) { return null; } - const region = configService.get("AWS_REGION") || "us-east-1"; + const region = configService.get("AWS_REGION"); const endpoint = configService.get("MINIO_ENDPOINT"); const accessKeyId = configService.get("STORAGE_ACCESS_KEY"); const secretAccessKey = configService.get("STORAGE_SECRET_KEY"); From a1f132bd815b7c94151dc5cecb67c426e86ad950 Mon Sep 17 00:00:00 2001 From: mj Date: Wed, 16 Oct 2024 12:51:56 +0900 Subject: [PATCH 7/9] refactor: remove unused StorageType enum --- backend/src/utils/types/storage.type.ts | 1 - 1 file changed, 1 deletion(-) delete mode 100644 backend/src/utils/types/storage.type.ts diff --git a/backend/src/utils/types/storage.type.ts b/backend/src/utils/types/storage.type.ts deleted file mode 100644 index e1b76854..00000000 --- a/backend/src/utils/types/storage.type.ts +++ /dev/null @@ -1 +0,0 @@ -export type StorageType = "S3" | "MINIO"; From 0d22ee03cf11eb7036ddc7435aac1ad5d32a578a Mon Sep 17 00:00:00 2001 From: mj Date: Thu, 17 Oct 2024 02:19:00 +0900 Subject: [PATCH 8/9] refactor: update .env.development to set default values for Minio --- backend/.env.development | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/.env.development b/backend/.env.development index 328783b4..1995d198 100644 --- a/backend/.env.development +++ b/backend/.env.development @@ -65,14 +65,14 @@ LANGCHAIN_PROJECT=your_langsmith_project_name_here # FILE_UPLOAD: Whether to enable file upload to storage # If set to false, AWS_S3_BUCKET_NAME is not required. # Set to true if file upload is required. -FILE_UPLOAD=false +FILE_UPLOAD=true # Storage configuration for either AWS S3 or MinIO # This is the name of the S3 Bucket or MinIO Bucket -BUCKET_NAME="my-bucket" +BUCKET_NAME="default-storage" +# MinIO endpoint, only required for MinIO +MINIO_ENDPOINT="http://localhost:9000" # Common access keys for both S3 and MinIO -MINIO_ACCESS_KEY="your_minio_access_key" -MINIO_SECRET_KEY="your_minio_secret_key" +MINIO_ACCESS_KEY="minioadmin" +MINIO_SECRET_KEY="minioadmin" # AWS Region, only required for AWS S3 AWS_REGION="your_aws_region" -# MinIO endpoint, only required for MinIO -MINIO_ENDPOINT="your_minio_endpoint" From d38b9d24c053ab4c87ae9282e26c1bb581576ade Mon Sep 17 00:00:00 2001 From: mj Date: Tue, 5 Nov 2024 09:28:46 +0900 Subject: [PATCH 9/9] feat: enhance file upload configuration for MinIO support --- backend/.env.development | 1 + backend/docker/docker-compose-full.yml | 2 +- backend/src/storage/storage.module.ts | 26 ++++++++++++-------------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/backend/.env.development b/backend/.env.development index c6634bc2..84ee895a 100644 --- a/backend/.env.development +++ b/backend/.env.development @@ -68,6 +68,7 @@ LANGCHAIN_API_KEY=your_langsmith_api_key_here LANGCHAIN_PROJECT=your_langsmith_project_name_here # FILE_UPLOAD: Whether to enable file upload to storage +# Available options: false, s3, minio # If set to false, AWS_S3_BUCKET_NAME is not required. # Set to true if file upload is required. FILE_UPLOAD=true diff --git a/backend/docker/docker-compose-full.yml b/backend/docker/docker-compose-full.yml index 9e19bfaa..233ff1f3 100644 --- a/backend/docker/docker-compose-full.yml +++ b/backend/docker/docker-compose-full.yml @@ -26,7 +26,7 @@ services: LANGCHAIN_TRACING_V2: "false" LANGCHAIN_API_KEY: "your_langsmith_api_key_here" LANGCHAIN_PROJECT: "your_langsmith_project_name_here" - FILE_UPLOAD: true + FILE_UPLOAD: false AWS_REGION: "your_aws_region_here" # Default configuration values for using Minio BUCKET_NAME: "default-storage" diff --git a/backend/src/storage/storage.module.ts b/backend/src/storage/storage.module.ts index ce18a69a..b75a261a 100644 --- a/backend/src/storage/storage.module.ts +++ b/backend/src/storage/storage.module.ts @@ -5,7 +5,7 @@ import { ConfigService } from "@nestjs/config"; const s3ClientFactory = { provide: "STORAGE_CLIENT", useFactory: (configService: ConfigService): S3Client | null => { - const fileUpload = configService.get("FILE_UPLOAD"); + const fileUpload = configService.get("FILE_UPLOAD"); if (!fileUpload) { return null; } @@ -14,19 +14,17 @@ const s3ClientFactory = { const accessKeyId = configService.get("STORAGE_ACCESS_KEY"); const secretAccessKey = configService.get("STORAGE_SECRET_KEY"); - const config: S3ClientConfig = fileUpload - ? { - region, - endpoint, - forcePathStyle: true, - credentials: { - accessKeyId, - secretAccessKey, - }, - } - : { - region, - }; + const config: S3ClientConfig = { + region, + ...(fileUpload === "minio" && { + endpoint, + forcePathStyle: true, + credentials: { + accessKeyId, + secretAccessKey, + }, + }), + }; return new S3Client(config); },