Print configs of milvus-backup

Signed-off-by: wayblink <[email protected]>

azure-identity-token

@@ -0,0 +1 @@
+eyJhbGciOiJSUzI1NiIsImtpZCI6InhEdFRlRF9Ca0l2cVNhLUFqTW5iVVd1d2VidTlzNkJsU3A1ZEw0MXFYbVkifQ.eyJhdWQiOlsiYXBpOi8vQXp1cmVBRFRva2VuRXhjaGFuZ2UiXSwiZXhwIjoxNjk4MDQ5NTA4LCJpYXQiOjE2OTgwNDU5MDgsImlzcyI6Imh0dHBzOi8vd2VzdHVzMy5vaWMucHJvZC1ha3MuYXp1cmUuY29tLzdiMWU0NTBjLTk5ZGQtNGUzNi05Zjg4LTVmYmZkMGQ4Y2M1Zi85YWQ2YWVhOC0zNDZjLTQ1ZmQtYWEyMS1hMGRmYTM1ODRlNDQvIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJtaWx2dXMtdG9vbCIsInBvZCI6eyJuYW1lIjoiYnAtaW4wMS0yZjIzNDFlZTc3ZDg1ZjIteGZyZTdlaCIsInVpZCI6IjU3M2M2ZDEzLTg1ZGYtNDBhYS04Y2U1LTYzN2FjZDJiZjJmOCJ9LCJzZXJ2aWNlYWNjb3VudCI6eyJuYW1lIjoibWlsdnVzLWJ1Y2tldCIsInVpZCI6IjYwYWY2YWUzLTliNmEtNDg0MC05N2RkLWFmNjc4NjNiNzBkZCJ9fSwibmJmIjoxNjk4MDQ1OTA4LCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6bWlsdnVzLXRvb2w6bWlsdnVzLWJ1Y2tldCJ9.a_pkHNHmoncvHlVI-yVEspG6nCyO_PjyUXv7xFxuUoIDk4GRc4qC78aeChrFWAtaW9QMCkJ97_UUSJaaep6NMy6V1sBxp0BOI2i9KTz3InNtoNlo25ya2GadXeNF-MsY9MxPQyJITFmfLi5SdEV6gs5UP2eTr3O_TgEFOx_i0Z9qHwHLJ_2sYfnmyC0RIYx2jZYs4N6n6dI4g0qqBYGRpbrNbgSpHOQFDxMaLT-QRwRIp9f4TntP9QUa7jafgSvYiKnac6cJGnDDhj2OM2qI-EoY6wZRVTTpOwZRLOLRzkfz8BCFOHCY_CMLgVsOMy0yDqNIkHjYRi6D50WvmVjRAw4N40nkj81UpVnnMfidYrz8e_a8yMpm9SgqEE5odKMiu3WvH03hBeB8C5rKQO7Mx9eG2RFka6SNQkV5HjKX7k0cQ21IsbPDNCfJ9XYyEPgAKourxFciyqNuUmD_cijf_a4yiaqFIY1Muqpm_gBgTW7Q6Q6SFY8e3eE7yWVEYR-cMXgdmSKQ8QA28WMg8zwR_ywoqsdHOYjYExn7qYE_itPY8dX-GQ3jl_XeoGYD4Bhd0xuRh75WAv-sckmLjYxX1XpULK6uroqEHFs81WDZ_7M9Q85KGlW4O6EqTlzJ_NTn5FzrHMFgChiuFstmBD2sSDcEaGP-gMl3FP34oqoZUdg

core/backup_context.go

@@ -94,6 +94,8 @@ func (b *BackupContext) Start() error {
 	b.backupNameIdDict = sync.Map{}
 	b.restoreTasks = make(map[string]*backuppb.RestoreBackupTask)
 	b.started = true
+	confString := fmt.Sprintf("%+v", b.params)
+	log.Info(confString)
 	return nil
 }
 

core/paramtable/params.go

@@ -69,7 +69,7 @@ func (p *BackupConfig) initRestoreParallelism() {
 }
 
 func (p *BackupConfig) initBackupCopyDataParallelism() {
-	size := p.Base.ParseIntWithDefault("backup.copydata.parallelism", 10)
+	size := p.Base.ParseIntWithDefault("backup.copydata.parallelism", 128)
 	p.BackupCopyDataParallelism = size
 }
 

patch

@@ -0,0 +1,65 @@
+diff --git a/configs/backup.yaml b/configs/backup.yaml
+index 4daa3ce..396a7ad 100644
+--- a/configs/backup.yaml
++++ b/configs/backup.yaml
+@@ -49,6 +49,7 @@ backup:
+     # which means if you set backup.parallelism = 2 backup.copydata.parallelism = 100, there will be 200 copy executing at the same time.
+     # reduce it if blocks your storage's network bandwidth
+     parallelism: 128
++  keepTempFiles: true
+
+ restore:
+   # Collection level parallelism to restore
+diff --git a/core/backup_impl_restore_backup.go b/core/backup_impl_restore_backup.go
+index 71a0317..0adf2f6 100644
+--- a/core/backup_impl_restore_backup.go
++++ b/core/backup_impl_restore_backup.go
+@@ -453,7 +453,7 @@ func (b *BackupContext) executeRestoreCollectionTask(ctx context.Context, backup
+ 	isSameBucket := b.milvusBucketName == backupBucketName
+ 	// clean the temporary file
+ 	defer func() {
+-		if !isSameBucket {
++		if !isSameBucket && !b.params.BackupCfg.KeepTempFiles {
+ 			log.Info("Delete temporary file", zap.String("dir", tempDir))
+ 			err := b.getStorageClient().RemoveWithPrefix(ctx, b.milvusBucketName, tempDir)
+ 			if err != nil {
+@@ -527,6 +527,7 @@ func (b *BackupContext) restorePartition(ctx context.Context, targetDBName, targ
+ 				if file == "" {
+ 					realFiles[i] = file
+ 				} else {
++					log.Debug("Copy temporary restore file", zap.String("from", file), zap.String("to", tempDir+file))
+ 					err := b.getStorageClient().Copy(ctx, backupBucketName, b.milvusBucketName, file, tempDir+file)
+ 					if err != nil {
+ 						log.Error("fail to copy backup date from backup bucket to restore target milvus bucket", zap.Error(err))
+diff --git a/core/paramtable/params.go b/core/paramtable/params.go
+index fea3c5c..57ff474 100644
+--- a/core/paramtable/params.go
++++ b/core/paramtable/params.go
+@@ -37,6 +37,7 @@ type BackupConfig struct {
+ 	BackupParallelism         int
+ 	RestoreParallelism        int
+ 	BackupCopyDataParallelism int
++	KeepTempFiles             bool
+ }
+
+ func (p *BackupConfig) init(base *BaseTable) {
+@@ -46,6 +47,7 @@ func (p *BackupConfig) init(base *BaseTable) {
+ 	p.initBackupParallelism()
+ 	p.initRestoreParallelism()
+ 	p.initBackupCopyDataParallelism()
++	p.initKeepTempFiles()
+ }
+
+ func (p *BackupConfig) initMaxSegmentGroupSize() {
+@@ -71,6 +73,11 @@ func (p *BackupConfig) initBackupCopyDataParallelism() {
+ 	p.BackupCopyDataParallelism = size
+ }
+
++func (p *BackupConfig) initKeepTempFiles() {
++	keepTempFiles := p.Base.LoadWithDefault("backup.keepTempFiles", "false")
++	p.KeepTempFiles, _ = strconv.ParseBool(keepTempFiles)
++}
++
+ type MilvusConfig struct {
+ 	Base *BaseTable
+