Support MinIO for file storage

backend/.env.development

@@ -67,11 +67,18 @@ LANGCHAIN_API_KEY=your_langsmith_api_key_here
 # To create a LangSmith project, visit LangSmith: https://www.langchain.com/langsmith
 LANGCHAIN_PROJECT=your_langsmith_project_name_here
 
-
 # FILE_UPLOAD: Whether to enable file upload to storage
-# Set to true if file upload is required.
+# Available options: false, s3, minio
 # If set to false, AWS_S3_BUCKET_NAME is not required.
-FILE_UPLOAD=false
-# AWS_S3_BUCKET_NAME: S3 Bucket name 
-# This is the name of the S3 Bucket
-AWS_S3_BUCKET_NAME="your_s3_bucket_name"
+# Set to true if file upload is required.
+FILE_UPLOAD=true
+# Storage configuration for either AWS S3 or MinIO
+# This is the name of the S3 Bucket or MinIO Bucket
+BUCKET_NAME="default-storage"
+# MinIO endpoint, only required for MinIO
+MINIO_ENDPOINT="http://localhost:9000"
+# Common access keys for both S3 and MinIO
+MINIO_ACCESS_KEY="minioadmin"
+MINIO_SECRET_KEY="minioadmin"
+# AWS Region, only required for AWS S3
+AWS_REGION="your_aws_region"

backend/docker/docker-compose-full.yml

@@ -27,15 +27,22 @@ services:
             LANGCHAIN_API_KEY: "your_langsmith_api_key_here"
             LANGCHAIN_PROJECT: "your_langsmith_project_name_here"
             FILE_UPLOAD: false
-            AWS_S3_BUCKET_NAME: "your_s3_bucket_name"
+            AWS_REGION: "your_aws_region_here"
+            # Default configuration values for using Minio
+            BUCKET_NAME: "default-storage"
+            MINIO_ENDPOINT: "http://localhost:9000"
+            STORAGE_ACCESS_KEY: "minioadmin"
+            STORAGE_SECRET_KEY: "minioadmin"
         ports:
             - "3000:3000"
         depends_on:
             - mongo
+            - minio
         restart: unless-stopped
         links:
             - "mongo:mongo"
             - "yorkie:yorkie"
+            - "minio:minio"
 
     yorkie:
         image: "yorkieteam/yorkie:0.5.0"
@@ -70,3 +77,30 @@ services:
             interval: 5s
             timeout: 2s
             retries: 20
+
+    # You can remove the following content if you're using S3 or not using Minio.
+    minio:
+        image: minio/minio
+        ports:
+            - "9000:9000"
+            - "9001:9001"
+        volumes:
+            - minio_data:/data
+        environment:
+            MINIO_ROOT_USER: "minioadmin"
+            MINIO_ROOT_PASSWORD: "minioadmin"
+        entrypoint: /bin/sh -c "
+            /usr/bin/docker-entrypoint.sh server --console-address ':9001' --address ':9000' /data &
+            sleep 1 &&
+            mc alias set myminio http://localhost:9000 minioadmin minioadmin &&
+            if ! mc ls myminio/default-storage; then mc mb myminio/default-storage; fi &&
+            wait"
+        command: server --console-address ":9001" --address ":9000" /data
+        healthcheck:
+            test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+            interval: 30s
+            timeout: 20s
+            retries: 3
+
+volumes:
+    minio_data:

backend/src/app.module.ts

@@ -1,19 +1,20 @@
 import { Module } from "@nestjs/common";
-import { PrismaService } from "./db/prisma.service";
-import { UsersModule } from "./users/users.module";
-import { AuthModule } from "./auth/auth.module";
 import { ConfigModule } from "@nestjs/config";
 import { APP_GUARD } from "@nestjs/core/constants";
+import { AuthModule } from "./auth/auth.module";
 import { JwtAuthGuard } from "./auth/jwt.guard";
-import { WorkspacesModule } from "./workspaces/workspaces.module";
-import { WorkspaceUsersModule } from "./workspace-users/workspace-users.module";
-import { WorkspaceDocumentsModule } from "./workspace-documents/workspace-documents.module";
-import { DocumentsModule } from "./documents/documents.module";
 import { CheckModule } from "./check/check.module";
+import { PrismaService } from "./db/prisma.service";
+import { DocumentsModule } from "./documents/documents.module";
+import { FilesModule } from "./files/files.module";
 import { IntelligenceModule } from "./intelligence/intelligence.module";
 import { LangchainModule } from "./langchain/langchain.module";
-import { FilesModule } from "./files/files.module";
 import { SettingsModule } from "./settings/settings.module";
+import { StorageModule } from "./storage/storage.module";
+import { UsersModule } from "./users/users.module";
+import { WorkspaceDocumentsModule } from "./workspace-documents/workspace-documents.module";
+import { WorkspaceUsersModule } from "./workspace-users/workspace-users.module";
+import { WorkspacesModule } from "./workspaces/workspaces.module";
 
 @Module({
 	imports: [
@@ -34,6 +35,7 @@ import { SettingsModule } from "./settings/settings.module";
 		FilesModule,
 		ConfigModule,
 		SettingsModule,
+		StorageModule,
 	],
 	controllers: [],
 	providers: [

backend/src/files/files.module.ts

@@ -1,9 +1,11 @@
 import { Module } from "@nestjs/common";
+import { PrismaService } from "src/db/prisma.service";
+import { StorageModule } from "src/storage/storage.module";
 import { FilesController } from "./files.controller";
 import { FilesService } from "./files.service";
-import { PrismaService } from "src/db/prisma.service";
 
 @Module({
+	imports: [StorageModule],
 	controllers: [FilesController],
 	providers: [FilesService, PrismaService],
 })

backend/src/files/files.service.ts

@@ -2,6 +2,7 @@ import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 import {
 	BadRequestException,
+	Inject,
 	Injectable,
 	NotFoundException,
 	UnauthorizedException,
@@ -18,14 +19,13 @@ import { ExportFileRequestBody, ExportFileResponse } from "./types/export-file.t
 
 @Injectable()
 export class FilesService {
-	private s3Client: S3Client;
 	private readonly markdown: MarkdownIt;
 
 	constructor(
+		@Inject("STORAGE_CLIENT") private s3Client: S3Client,
 		private configService: ConfigService,
 		private prismaService: PrismaService
 	) {
-		this.s3Client = new S3Client();
 		this.markdown = new MarkdownIt({
 			html: true,
 			breaks: true,
@@ -55,7 +55,7 @@ export class FilesService {
 
 		const fileKey = `${workspace.slug}-${generateRandomKey()}.${contentType.split("/")[1]}`;
 		const command = new PutObjectCommand({
-			Bucket: this.configService.get("AWS_S3_BUCKET_NAME"),
+			Bucket: this.configService.get("BUCKET_NAME"),
 			Key: fileKey,
 			StorageClass: "INTELLIGENT_TIERING",
 			ContentType: contentType,
@@ -70,7 +70,7 @@ export class FilesService {
 	async createDownloadPresignedUrl(fileKey: string) {
 		try {
 			const command = new GetObjectCommand({
-				Bucket: this.configService.get("AWS_S3_BUCKET_NAME"),
+				Bucket: this.configService.get("BUCKET_NAME"),
 				Key: fileKey,
 			});
 			return getSignedUrl(this.s3Client, command, { expiresIn: 3600 });

backend/src/storage/storage.module.ts

@@ -0,0 +1,38 @@
+import { S3Client, S3ClientConfig } from "@aws-sdk/client-s3";
+import { Module } from "@nestjs/common";
+import { ConfigService } from "@nestjs/config";
+
+const s3ClientFactory = {
+	provide: "STORAGE_CLIENT",
+	useFactory: (configService: ConfigService): S3Client | null => {
+		const fileUpload = configService.get<boolean | "s3" | "minio">("FILE_UPLOAD");
+		if (!fileUpload) {
+			return null;
+		}
+		const region = configService.get<string>("AWS_REGION");
+		const endpoint = configService.get<string>("MINIO_ENDPOINT");
+		const accessKeyId = configService.get<string>("STORAGE_ACCESS_KEY");
+		const secretAccessKey = configService.get<string>("STORAGE_SECRET_KEY");
+
+		const config: S3ClientConfig = {
+			region,
+			...(fileUpload === "minio" && {
+				endpoint,
+				forcePathStyle: true,
+				credentials: {
+					accessKeyId,
+					secretAccessKey,
+				},
+			}),
+		};
+
+		return new S3Client(config);
+	},
+	inject: [ConfigService],
+};
+
+@Module({
+	providers: [s3ClientFactory],
+	exports: [s3ClientFactory],
+})
+export class StorageModule {}