feat: migrate to aws-sdk-go-v2 (#186)

nhost · Nov 22, 2023 · 51ccc70 · 51ccc70
1 parent 3bdbf46
commit 51ccc70
Show file tree

Hide file tree

Showing 761 changed files with 164,542 additions and 120,166 deletions.
diff --git a/cmd/serve.go b/cmd/serve.go
@@ -1,13 +1,16 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"net/url"
 	"os"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/gin-gonic/gin"
 	"github.com/nhost/hasura-storage/controller"
 	"github.com/nhost/hasura-storage/image"
@@ -35,6 +38,7 @@ const (
 	s3RegionFlag                 = "s3-region"
 	s3BucketFlag                 = "s3-bucket"
 	s3RootFolderFlag             = "s3-root-folder"
+	s3DisableHTTPS               = "s3-disable-http"
 	postgresMigrationsFlag       = "postgres-migrations"
 	postgresMigrationsSourceFlag = "postgres-migrations-source"
 	fastlyServiceFlag            = "fastly-service"
@@ -135,20 +139,33 @@ func getMetadataStorage(endpoint string) *metadata.Hasura {
 }
 
 func getContentStorage(
-	s3Endpoint, region, s3AccessKey, s3SecretKey, bucket, rootFolder string, logger *logrus.Logger,
+	ctx context.Context,
+	s3Endpoint, region, s3AccessKey, s3SecretKey, bucket, rootFolder string, disableHTTPS bool, logger *logrus.Logger,
 ) *storage.S3 {
-	config := &aws.Config{ //nolint: exhaustivestruct
-		Credentials:      credentials.NewStaticCredentials(s3AccessKey, s3SecretKey, ""),
-		Endpoint:         aws.String(s3Endpoint),
-		Region:           aws.String(region),
-		DisableSSL:       aws.Bool(true),
-		S3ForcePathStyle: aws.Bool(true),
+	var cfg aws.Config
+	var err error
+	if s3AccessKey != "" && s3SecretKey != "" {
+		logger.Info("Using static aws credentials")
+		cfg, err = config.LoadDefaultConfig(ctx,
+			config.WithRegion(region),
+			config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s3AccessKey, s3SecretKey, "")),
+		)
+	} else {
+		logger.Info("Using default configuration for aws credentials")
+		cfg, err = config.LoadDefaultConfig(ctx, config.WithRegion(region))
 	}
-
-	st, err := storage.NewS3(config, bucket, rootFolder, s3Endpoint, logger)
 	if err != nil {
 		panic(err)
 	}
+	client := s3.NewFromConfig(
+		cfg,
+		func(o *s3.Options) {
+			o.BaseEndpoint = aws.String(s3Endpoint)
+			o.UsePathStyle = true
+			o.EndpointOptions.DisableHTTPS = disableHTTPS
+		},
+	)
+	st := storage.NewS3(client, bucket, rootFolder, s3Endpoint, disableHTTPS, logger)
 
 	return st
 }
@@ -249,7 +266,7 @@ func init() {
 var serveCmd = &cobra.Command{
 	Use:   "serve",
 	Short: "Starts hasura-storage server",
-	Run: func(_ *cobra.Command, _ []string) {
+	Run: func(cmd *cobra.Command, _ []string) {
 		logger := getLogger()
 
 		logger.Info("storage version ", controller.Version())
@@ -283,12 +300,14 @@ var serveCmd = &cobra.Command{
 		).Debug("parameters")
 
 		contentStorage := getContentStorage(
+			cmd.Context(),
 			viper.GetString(s3EndpointFlag),
 			viper.GetString(s3RegionFlag),
 			viper.GetString(s3AccessKeyFlag),
 			viper.GetString(s3SecretKeyFlag),
 			viper.GetString(s3BucketFlag),
 			viper.GetString(s3RootFolderFlag),
+			viper.GetBool(s3DisableHTTPS),
 			logger,
 		)
 

diff --git a/controller/controller.go b/controller/controller.go
@@ -73,14 +73,14 @@ type MetadataStorage interface {
 
 //go:generate mockgen --build_flags=--mod=mod -destination mock/content_storage.go -package mock . ContentStorage
 type ContentStorage interface {
-	PutFile(content io.ReadSeeker, filepath, contentType string) (string, *APIError)
-	GetFile(filepath string, headers http.Header) (*File, *APIError)
-	CreatePresignedURL(filepath string, expire time.Duration) (string, *APIError)
+	PutFile(ctx context.Context, content io.ReadSeeker, filepath, contentType string) (string, *APIError)
+	GetFile(ctx context.Context, filepath string, headers http.Header) (*File, *APIError)
+	CreatePresignedURL(ctx context.Context, filepath string, expire time.Duration) (string, *APIError)
 	GetFileWithPresignedURL(
 		ctx context.Context, filepath, signature string, headers http.Header,
 	) (*File, *APIError)
-	DeleteFile(filepath string) *APIError
-	ListFiles() ([]string, *APIError)
+	DeleteFile(ctx context.Context, filepath string) *APIError
+	ListFiles(ctx context.Context) ([]string, *APIError)
 }
 
 //go:generate mockgen --build_flags=--mod=mod -destination mock/antivirus.go -package mock . Antivirus

diff --git a/controller/delete_broken_metadata_test.go b/controller/delete_broken_metadata_test.go
@@ -88,7 +88,7 @@ func TestDeleteBrokenMetadata(t *testing.T) {
 				}, nil,
 			)
 
-			contentStorage.EXPECT().ListFiles().Return(
+			contentStorage.EXPECT().ListFiles(gomock.Any()).Return(
 				[]string{
 					"default/7dc0b0d0-b100-4667-89f1-0434942d9c15",
 				}, nil,

diff --git a/controller/delete_file.go b/controller/delete_file.go
@@ -15,7 +15,7 @@ func (ctrl *Controller) deleteFile(ctx *gin.Context) *APIError {
 		return apiErr
 	}
 
-	if apiErr := ctrl.contentStorage.DeleteFile(id); apiErr != nil {
+	if apiErr := ctrl.contentStorage.DeleteFile(ctx, id); apiErr != nil {
 		return apiErr
 	}
 

diff --git a/controller/delete_file_test.go b/controller/delete_file_test.go
@@ -46,6 +46,7 @@ func TestDeleteFile(t *testing.T) {
 			).Return(nil)
 
 			contentStorage.EXPECT().DeleteFile(
+				gomock.Any(),
 				"55af1e60-0f28-454e-885e-ea6aab2bb288",
 			).Return(
 				nil,

diff --git a/controller/delete_orphans.go b/controller/delete_orphans.go
@@ -14,7 +14,7 @@ func (ctrl *Controller) deleteOrphans(ctx *gin.Context) ([]string, *APIError) {
 	}
 
 	for _, f := range toDelete {
-		if apiErr := ctrl.contentStorage.DeleteFile(f); apiErr != nil {
+		if apiErr := ctrl.contentStorage.DeleteFile(ctx, f); apiErr != nil {
 			return nil, apiErr
 		}
 	}

diff --git a/controller/delete_orphans_test.go b/controller/delete_orphans_test.go
@@ -63,15 +63,15 @@ func TestDeleteOrphans(t *testing.T) {
 				}, nil,
 			)
 
-			contentStorage.EXPECT().ListFiles().Return(
+			contentStorage.EXPECT().ListFiles(gomock.Any()).Return(
 				[]string{
 					"default/b3b4e653-ca59-412c-a165-92d251c3fe86",
 					"default/7dc0b0d0-b100-4667-89f1-0434942d9c15",
 					"default/garbage",
 				}, nil,
 			)
 
-			contentStorage.EXPECT().DeleteFile("default/garbage").Return(nil)
+			contentStorage.EXPECT().DeleteFile(gomock.Any(), "default/garbage").Return(nil)
 
 			ctrl := controller.New("http://asd", "/v1", "asdasd", metadataStorage, contentStorage, nil, nil, logger)
 

diff --git a/controller/get_file.go b/controller/get_file.go
@@ -212,7 +212,7 @@ func (ctrl *Controller) getFileProcess(ctx *gin.Context) (*FileResponse, *APIErr
 	}
 
 	downloadFunc := func() (*File, *APIError) {
-		return ctrl.contentStorage.GetFile(fileMetadata.ID, ctx.Request.Header)
+		return ctrl.contentStorage.GetFile(ctx, fileMetadata.ID, ctx.Request.Header)
 	}
 
 	response, apiErr := ctrl.processFileToDownload(

diff --git a/controller/get_file_information.go b/controller/get_file_information.go
@@ -148,7 +148,7 @@ func (ctrl *Controller) getFileInformationProcess(ctx *gin.Context) (*FileRespon
 	}
 
 	if !opts.IsEmpty() {
-		download, apiErr := ctrl.contentStorage.GetFile(fileMetadata.ID, ctx.Request.Header)
+		download, apiErr := ctrl.contentStorage.GetFile(ctx, fileMetadata.ID, ctx.Request.Header)
 		if apiErr != nil {
 			return nil, apiErr
 		}

diff --git a/controller/get_file_presigned_url.go b/controller/get_file_presigned_url.go
@@ -41,6 +41,7 @@ func (ctrl *Controller) getFilePresignedURL(ctx *gin.Context) (GetFilePresignedU
 	}
 
 	signature, apiErr := ctrl.contentStorage.CreatePresignedURL(
+		ctx,
 		fileMetadata.ID,
 		time.Duration(bucketMetadata.DownloadExpiration)*time.Second,
 	)

diff --git a/controller/get_file_presigned_url_test.go b/controller/get_file_presigned_url_test.go
@@ -79,7 +79,7 @@ func TestGetFilePresignedURL(t *testing.T) {
 				}, nil)
 
 				contentStorage.EXPECT().CreatePresignedURL(
-					"55af1e60-0f28-454e-885e-ea6aab2bb288", 30*time.Second,
+					gomock.Any(), "55af1e60-0f28-454e-885e-ea6aab2bb288", 30*time.Second,
 				).Return(
 					"this-is-the-signature", nil,
 				)

diff --git a/controller/get_file_test.go b/controller/get_file_test.go
@@ -63,6 +63,7 @@ func TestGetFile(t *testing.T) {
 			}, nil)
 
 			contentStorage.EXPECT().GetFile(
+				gomock.Any(),
 				"55af1e60-0f28-454e-885e-ea6aab2bb288",
 				gomock.Any(),
 			).Return(

diff --git a/controller/list_broken_metadata.go b/controller/list_broken_metadata.go
@@ -18,7 +18,7 @@ func (ctrl *Controller) listBrokenMetadata(ctx *gin.Context) ([]FileSummary, *AP
 		return nil, apiErr
 	}
 
-	filesInS3, apiErr := ctrl.contentStorage.ListFiles()
+	filesInS3, apiErr := ctrl.contentStorage.ListFiles(ctx)
 	if apiErr != nil {
 		return nil, apiErr
 	}

diff --git a/controller/list_broken_metadata_test.go b/controller/list_broken_metadata_test.go
@@ -88,7 +88,7 @@ func TestListBrokenMetadata(t *testing.T) {
 				}, nil,
 			)
 
-			contentStorage.EXPECT().ListFiles().Return(
+			contentStorage.EXPECT().ListFiles(gomock.Any()).Return(
 				[]string{
 					"app_id/7dc0b0d0-b100-4667-89f1-0434942d9c15",
 				}, nil,

diff --git a/controller/list_orphans.go b/controller/list_orphans.go
@@ -18,7 +18,7 @@ func (ctrl *Controller) listOrphans(ctx *gin.Context) ([]string, *APIError) {
 		return nil, apiErr
 	}
 
-	filesInS3, apiErr := ctrl.contentStorage.ListFiles()
+	filesInS3, apiErr := ctrl.contentStorage.ListFiles(ctx)
 	if apiErr != nil {
 		return nil, apiErr
 	}

diff --git a/controller/list_orphans_test.go b/controller/list_orphans_test.go
@@ -63,7 +63,7 @@ func TestListOrphans(t *testing.T) {
 				}, nil,
 			)
 
-			contentStorage.EXPECT().ListFiles().Return(
+			contentStorage.EXPECT().ListFiles(gomock.Any()).Return(
 				[]string{
 					"app_id/b3b4e653-ca59-412c-a165-92d251c3fe86",
 					"app_id/7dc0b0d0-b100-4667-89f1-0434942d9c15",

diff --git a/controller/mock/content_storage.go b/controller/mock/content_storage.go
diff --git a/controller/update_file.go b/controller/update_file.go
@@ -103,7 +103,7 @@ func (ctrl *Controller) updateFile(ctx *gin.Context) (FileMetadata, *APIError) {
 		return FileMetadata{}, err
 	}
 
-	etag, apiErr := ctrl.contentStorage.PutFile(fileContent, file.ID, contentType)
+	etag, apiErr := ctrl.contentStorage.PutFile(ctx, fileContent, file.ID, contentType)
 	if apiErr != nil {
 		// let's revert the change to isUploaded
 		_ = ctrl.metadataStorage.SetIsUploaded(ctx, file.ID, true, ctx.Request.Header)

diff --git a/controller/update_file_test.go b/controller/update_file_test.go
@@ -123,6 +123,7 @@ func TestUpdateFile(t *testing.T) {
 			).Return(nil)
 
 			contentStorage.EXPECT().PutFile(
+				gomock.Any(),
 				ReaderMatcher(
 					file.contents,
 				),

diff --git a/controller/upload_file.go b/controller/upload_file.go
@@ -116,7 +116,7 @@ func (ctrl *Controller) processFile(
 		return FileMetadata{}, err
 	}
 
-	etag, apiErr := ctrl.contentStorage.PutFile(fileContent, file.ID, contentType)
+	etag, apiErr := ctrl.contentStorage.PutFile(ctx, fileContent, file.ID, contentType)
 	if apiErr != nil {
 		_ = ctrl.metadataStorage.DeleteFileByID(
 			ctx,

diff --git a/controller/upload_file_test.go b/controller/upload_file_test.go
@@ -167,6 +167,7 @@ func TestUploadFile(t *testing.T) {
 				).Return(nil)
 
 				contentStorage.EXPECT().PutFile(
+					gomock.Any(),
 					ReaderMatcher(
 						file.contents,
 					),
@@ -216,6 +217,7 @@ func TestUploadFile(t *testing.T) {
 				).Return(nil)
 
 				contentStorage.EXPECT().PutFile(
+					gomock.Any(),
 					ReaderMatcher(
 						file.contents,
 					),