Skip to content

Commit

Permalink
GCP native support (for the Google Cloud Platform provider)
Browse files Browse the repository at this point in the history 
Signed-off-by: gifi-siby <[email protected]>
  • Loading branch information
@gifi-siby
gifi-siby committed Jan 6, 2025
1 parent 6d40f12 commit dfff96f
Show file tree
Hide file tree
Showing 11 changed files with 449 additions and 38 deletions.
7 changes: 5 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,20 +104,22 @@ Below is a summary of the configurations supported in `backup.yaml`:
| | `tlsMode` | TLS mode (0: none, 1: one-way, 2: two-way). | `0` |
| | `user` | Username for Milvus. | `root` |
| | `password` | Password for Milvus. | `Milvus` |
| `minio` | `storageType` | Storage type for Milvus (e.g., `local`, `minio`, `s3`, `aws`, `gcp`, `ali(aliyun)`, `azure`, `tc(tencent)`). | `minio` |
| `minio` | `storageType` | Storage type for Milvus (e.g., `local`, `minio`, `s3`, `aws`, `gcp`, `ali(aliyun)`, `azure`, `tc(tencent)`, `gcpnative`). Use `gcpnative` for the Google Cloud Platform provider. | `minio` |
| | `address` | MinIO/S3 address. | `localhost` |
| | `port` | MinIO/S3 port. | `9000` |
| | `accessKeyID` | MinIO/S3 access key ID. | `minioadmin` |
| | `secretAccessKey` | MinIO/S3 secret access key. | `minioadmin` |
| | `gcpCredentialJSON` | Path to your GCP credential JSON key file. Used only for the "gcpnative" cloud provider. | `/path/to/file` |
| | `useSSL` | Whether to use SSL for MinIO/S3. | `false` |
| | `bucketName` | Bucket name in MinIO/S3. | `a-bucket` |
| | `rootPath` | Storage root path in MinIO/S3. | `files` |
| `minio (backup)` | `backupStorageType` | Backup storage type (e.g., `local`, `minio`, `s3`, `aws`, `gcp`, `ali(aliyun)`, `azure`, `tc(tencent)`). | `minio` |
| `minio (backup)` | `backupStorageType` | Backup storage type for Milvus (e.g., `local`, `minio`, `s3`, `aws`, `gcp`, `ali(aliyun)`, `azure`, `tc(tencent)`, `gcpnative`). Use `gcpnative` for the Google Cloud Platform provider. | `minio` |
| | `backupAddress` | Address of backup storage. | `localhost` |
| | `backupPort` | Port of backup storage. | `9000` |
| | `backupUseSSL` | Whether to use SSL for backup storage. | `false` |
| | `backupAccessKeyID` | Backup storage access key ID. | `minioadmin` |
| | `backupSecretAccessKey` | Backup storage secret access key. | `minioadmin` |
| | `backupGcpCredentialJSON` | Path to your GCP credential JSON key file. Used only for the "gcpnative" cloud provider. | `/path/to/file` |
| | `backupBucketName` | Bucket name for backups. | `a-bucket` |
| | `backupRootPath` | Root path to store backup data. | `backup` |
| | `crossStorage` | Enable cross-storage backups (e.g., MinIO to AWS S3). | `false` |
Expand Down Expand Up @@ -153,6 +155,7 @@ backupAddress: s3.{your-aws-region}.amazonaws.com # Address of AWS S3 (replace
backupPort: 443 # Default port for AWS S3
backupAccessKeyID: <your-access-key-id> # Access key ID for your AWS S3
backupSecretAccessKey: <your-secret-key> # Secret access key for AWS S3
backupGcpCredentialJSON: "/path/to/file" # Path to your GCP credential JSON key file. Used only for the "gcpnative" cloud provider.
backupBucketName: "your-bucket-name" # Bucket name where the backups will be stored
backupRootPath: "backups" # Root path inside the bucket to store backups
backupUseSSL: true # Use SSL for secure connections (Required)
Expand Down
29 changes: 17 additions & 12 deletions cmd/backup_yaml.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package cmd
import (
"fmt"
"strings"

"github.com/spf13/cobra"
"github.com/zilliztech/milvus-backup/core/paramtable"
"gopkg.in/yaml.v3"
Expand Down Expand Up @@ -41,18 +42,20 @@ type YAMLConFig struct {
Password string `yaml:"password"`
} `yaml:"milvus"`
Minio struct {
Address string `yaml:"address"`
Port int `yaml:"port"`
AccessKeyID string `yaml:"accessKeyID"`
secretAccessKey string `yaml:"secretAccessKey"`
UseSSL bool `yaml:"useSSL"`
UseIAM bool `yaml:"useIAM"`
CloudProvider string `yaml:"cloudProvider"`
IamEndpoint string `yaml:"iamEndpoint"`
BucketName string `yaml:"bucketName"`
RootPath string `yaml:"rootPath"`
BackupBucketName string `yaml:"backupBucketName"`
BackupRootPath string `yaml:"backupRootPath"`
Address string `yaml:"address"`
Port int `yaml:"port"`
AccessKeyID string `yaml:"accessKeyID"`
secretAccessKey string `yaml:"secretAccessKey"`
GcpCredentialJSON string `yaml:"gcpCredentialJSON"`
UseSSL bool `yaml:"useSSL"`
UseIAM bool `yaml:"useIAM"`
CloudProvider string `yaml:"cloudProvider"`
IamEndpoint string `yaml:"iamEndpoint"`
BucketName string `yaml:"bucketName"`
RootPath string `yaml:"rootPath"`
BackupGcpCredentialJSON string `yaml:"backupGcpCredentialJSON"`
BackupBucketName string `yaml:"backupBucketName"`
BackupRootPath string `yaml:"backupRootPath"`
} `yaml:"minio"`
Backup struct {
MaxSegmentGroupSize string `yaml:"maxSegmentGroupSize"`
Expand Down Expand Up @@ -82,12 +85,14 @@ func printParams(base *paramtable.BackupParams) {
yml.Minio.Port = base.ParseIntWithDefault("minio.port", 9000)
yml.Minio.AccessKeyID = base.BaseTable.LoadWithDefault("minio.accessKeyID", "")
yml.Minio.secretAccessKey = base.BaseTable.LoadWithDefault("minio.secretAccessKey", "")
yml.Minio.GcpCredentialJSON = base.BaseTable.LoadWithDefault("minio.gcpCredentialJSON", "")
yml.Minio.UseSSL = base.ParseBool("minio.useSSL", false)
yml.Minio.UseIAM = base.ParseBool("minio.useIAM", false)
yml.Minio.CloudProvider = base.BaseTable.LoadWithDefault("minio.cloudProvider", "aws")
yml.Minio.IamEndpoint = base.BaseTable.LoadWithDefault("minio.iamEndpoint", "")
yml.Minio.BucketName = base.BaseTable.LoadWithDefault("minio.bucketName", "")
yml.Minio.RootPath = base.LoadWithDefault("minio.rootPath", "")
yml.Minio.BackupGcpCredentialJSON = base.BaseTable.LoadWithDefault("minio.backupGcpCredentialJSON", "")
yml.Minio.BackupBucketName = base.LoadWithDefault("minio.backupBucketName", "")
yml.Minio.BackupRootPath = base.LoadWithDefault("minio.backupRootPath", "")

Expand Down
7 changes: 5 additions & 2 deletions configs/backup.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,26 @@ milvus:
# Related configuration of minio, which is responsible for data persistence for Milvus.
minio:
# Milvus storage configs, make them the same with milvus config
storageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent)
storageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent), gcpnative
# You can use "gcpnative" for the Google Cloud Platform provider. Uses service account credentials for authentication.
address: localhost # Address of MinIO/S3
port: 9000 # Port of MinIO/S3
accessKeyID: minioadmin # accessKeyID of MinIO/S3
secretAccessKey: minioadmin # MinIO/S3 encryption string
gcpCredentialJSON: "" # The JSON content contains the gcs service account credentials. Used only for the "gcpnative" cloud provider.
useSSL: false # Access to MinIO/S3 with SSL
useIAM: false
iamEndpoint: ""
bucketName: "a-bucket" # Milvus Bucket name in MinIO/S3, make it the same as your milvus instance
rootPath: "files" # Milvus storage root path in MinIO/S3, make it the same as your milvus instance

# Backup storage configs, the storage you want to put the backup data
backupStorageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent)
backupStorageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent), gcpnative
backupAddress: localhost # Address of MinIO/S3
backupPort: 9000 # Port of MinIO/S3
backupAccessKeyID: minioadmin # accessKeyID of MinIO/S3
backupSecretAccessKey: minioadmin # MinIO/S3 encryption string
backupGcpCredentialJSON: "" # The JSON content contains the gcs service account credentials. Used only for the "gcpnative" cloud provider.
backupBucketName: "a-bucket" # Bucket name to store backup data. Backup data will store to backupBucketName/backupRootPath
backupRootPath: "backup" # Rootpath to store backup data. Backup data will store to backupBucketName/backupRootPath

Expand Down
3 changes: 3 additions & 0 deletions core/backup_context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ func createStorageClient(ctx context.Context, params paramtable.BackupParams) (s
BucketName: params.MinioCfg.BucketName,
AccessKeyID: params.MinioCfg.AccessKeyID,
SecretAccessKeyID: params.MinioCfg.SecretAccessKey,
GcpCredentialJSON: params.MinioCfg.GcpCredentialJSON,
UseSSL: params.MinioCfg.UseSSL,
UseIAM: params.MinioCfg.UseIAM,
IAMEndpoint: params.MinioCfg.IAMEndpoint,
Expand Down Expand Up @@ -169,6 +170,7 @@ func (b *BackupContext) getMilvusStorageClient() storage.ChunkManager {
BucketName: b.params.MinioCfg.BucketName,
AccessKeyID: b.params.MinioCfg.AccessKeyID,
SecretAccessKeyID: b.params.MinioCfg.SecretAccessKey,
GcpCredentialJSON: b.params.MinioCfg.GcpCredentialJSON,
UseSSL: b.params.MinioCfg.UseSSL,
UseIAM: b.params.MinioCfg.UseIAM,
IAMEndpoint: b.params.MinioCfg.IAMEndpoint,
Expand Down Expand Up @@ -200,6 +202,7 @@ func (b *BackupContext) getBackupStorageClient() storage.ChunkManager {
BucketName: b.params.MinioCfg.BackupBucketName,
AccessKeyID: b.params.MinioCfg.BackupAccessKeyID,
SecretAccessKeyID: b.params.MinioCfg.BackupSecretAccessKey,
GcpCredentialJSON: b.params.MinioCfg.BackupGcpCredentialJSON,
UseSSL: b.params.MinioCfg.BackupUseSSL,
UseIAM: b.params.MinioCfg.BackupUseIAM,
IAMEndpoint: b.params.MinioCfg.BackupIAMEndpoint,
Expand Down
11 changes: 11 additions & 0 deletions core/paramtable/base_table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ const (
DefaultMinioPort = "9000"
DefaultMinioAccessKey = "minioadmin"
DefaultMinioSecretAccessKey = "minioadmin"
DefaultGcpCredentialJSON = ""
DefaultMinioUseSSL = "false"
DefaultMinioBucketName = "a-bucket"
DefaultMinioRootPath = "files"
Expand Down Expand Up @@ -428,6 +429,11 @@ func (gp *BaseTable) loadMinioConfig() {
_ = gp.Save("minio.secretAccessKey", minioSecretKey)
}

gcpCredentialJSON := os.Getenv("GCP_KEY_JSON")
if gcpCredentialJSON != "" {
_ = gp.Save("minio.gcpCredentialJSON", gcpCredentialJSON)
}

minioUseSSL := os.Getenv("MINIO_USE_SSL")
if minioUseSSL != "" {
_ = gp.Save("minio.useSSL", minioUseSSL)
Expand Down Expand Up @@ -483,6 +489,11 @@ func (gp *BaseTable) loadMinioConfig() {
_ = gp.Save("minio.backupSecretAccessKey", minioBackupSecretKey)
}

backupGcpCredentialJSON := os.Getenv("BACKUP_GCP_KEY_JSON")
if backupGcpCredentialJSON != "" {
_ = gp.Save("minio.backupGcpCredentialJSON", backupGcpCredentialJSON)
}

minioBackupUseSSL := os.Getenv("MINIO_BACKUP_USE_SSL")
if minioBackupUseSSL != "" {
_ = gp.Save("minio.backupUseSSL", minioBackupUseSSL)
Expand Down
58 changes: 37 additions & 21 deletions core/paramtable/params.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ const (
S3 = "s3"
CloudProviderAWS = "aws"
CloudProviderGCP = "gcp"
CloudProviderGCPNative = "gcpnative"
CloudProviderAli = "ali"
CloudProviderAliyun = "aliyun"
CloudProviderAzure = "azure"
Expand All @@ -184,6 +185,7 @@ var supportedStorageType = map[string]bool{
S3: true,
CloudProviderAWS: true,
CloudProviderGCP: true,
CloudProviderGCPNative: true,
CloudProviderAli: true,
CloudProviderAliyun: true,
CloudProviderAzure: true,
Expand All @@ -196,27 +198,29 @@ type MinioConfig struct {

StorageType string
// Deprecated
CloudProvider string
Address string
Port string
AccessKeyID string
SecretAccessKey string
UseSSL bool
BucketName string
RootPath string
UseIAM bool
IAMEndpoint string

BackupStorageType string
BackupAddress string
BackupPort string
BackupAccessKeyID string
BackupSecretAccessKey string
BackupUseSSL bool
BackupBucketName string
BackupRootPath string
BackupUseIAM bool
BackupIAMEndpoint string
CloudProvider string
Address string
Port string
AccessKeyID string
SecretAccessKey string
GcpCredentialJSON string
UseSSL bool
BucketName string
RootPath string
UseIAM bool
IAMEndpoint string

BackupStorageType string
BackupAddress string
BackupPort string
BackupAccessKeyID string
BackupSecretAccessKey string
BackupGcpCredentialJSON string
BackupUseSSL bool
BackupBucketName string
BackupRootPath string
BackupUseIAM bool
BackupIAMEndpoint string

CrossStorage bool
}
Expand All @@ -229,6 +233,7 @@ func (p *MinioConfig) init(base *BaseTable) {
p.initPort()
p.initAccessKeyID()
p.initSecretAccessKey()
p.initGcpCredentialJSON()
p.initUseSSL()
p.initBucketName()
p.initRootPath()
Expand All @@ -241,6 +246,7 @@ func (p *MinioConfig) init(base *BaseTable) {
p.initBackupPort()
p.initBackupAccessKeyID()
p.initBackupSecretAccessKey()
p.initBackupGcpCredentialJSON()
p.initBackupUseSSL()
p.initBackupBucketName()
p.initBackupRootPath()
Expand Down Expand Up @@ -270,6 +276,11 @@ func (p *MinioConfig) initSecretAccessKey() {
p.SecretAccessKey = key
}

func (p *MinioConfig) initGcpCredentialJSON() {
gcpCredentialJSON := p.Base.LoadWithDefault("minio.gcpCredentialJSON", DefaultGcpCredentialJSON)
p.GcpCredentialJSON = gcpCredentialJSON
}

func (p *MinioConfig) initUseSSL() {
usessl := p.Base.LoadWithDefault("minio.useSSL", DefaultMinioUseSSL)
var err error
Expand Down Expand Up @@ -380,6 +391,11 @@ func (p *MinioConfig) initBackupSecretAccessKey() {
p.BackupSecretAccessKey = key
}

func (p *MinioConfig) initBackupGcpCredentialJSON() {
gcpCredentialJSON := p.Base.LoadWithDefault("minio.backupGcpCredentialJSON", DefaultGcpCredentialJSON)
p.BackupGcpCredentialJSON = gcpCredentialJSON
}

func (p *MinioConfig) initBackupBucketName() {
bucketName := p.Base.LoadWithDefault("minio.backupBucketName",
p.Base.LoadWithDefault("minio.bucketName", DefaultMinioBackupBucketName))
Expand Down
2 changes: 2 additions & 0 deletions core/storage/chunk_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ func NewChunkManager(ctx context.Context, params paramtable.BackupParams, config
case paramtable.CloudProviderAzure:
// todo @wayblink
return newAzureChunkManagerWithParams(ctx, params, config)
case paramtable.CloudProviderGCPNative:
return newGCPNativeChunkManagerWithConfig(ctx, config)
default:
return NewMinioChunkManagerWithConfig(ctx, config)
}
Expand Down
Loading

0 comments on commit dfff96f

Please sign in to comment.