Refine cross storage backup and add doc

Signed-off-by: wayblink <[email protected]>

configs/backup.yaml

@@ -42,6 +42,12 @@ minio:
   backupBucketName: "a-bucket" # Bucket name to store backup data. Backup data will store to backupBucketName/backupRootPath
   backupRootPath: "backup" # Rootpath to store backup data. Backup data will store to backupBucketName/backupRootPath
 
+  # If you need to back up or restore data between two different storage systems, direct client-side copying is not supported. 
+  # Set this option to true to enable data transfer through Milvus Backup.
+  # Note: This option will be automatically set to true if `minio.storageType` and `minio.backupStorageType` differ.
+  # However, if they are the same but belong to different services, you must manually set this option to `true`.
+  crossStorage: "false"
+
 backup:
   maxSegmentGroupSize: 2G
 
@@ -61,8 +67,3 @@ backup:
     enable: true
     seconds: 7200
     address: http://localhost:9091
-
-  # If you need to backup or restore data between two different storage systems, 
-  # direct client-side copying is not supported. 
-  # Set this option to true to enable data transfer through Milvus Backup.
-  copyByServer: "false" 

core/backup_context.go

@@ -217,28 +217,37 @@ func (b *BackupContext) getBackupStorageClient() storage.ChunkManager {
 }
 
 func (b *BackupContext) getBackupCopier() *storage.Copier {
+	crossStorage := b.params.MinioCfg.CrossStorage
+	if b.getBackupStorageClient().Config().StorageType != b.getMilvusStorageClient().Config().StorageType {
+		crossStorage = true
+	}
 	if b.backupCopier == nil {
 		b.backupCopier = storage.NewCopier(
 			b.getMilvusStorageClient(),
 			b.getBackupStorageClient(),
 			storage.CopyOption{
 				WorkerNum:    b.params.BackupCfg.BackupCopyDataParallelism,
 				RPS:          RPS,
-				CopyByServer: b.params.BackupCfg.CopyByServer,
+				CopyByServer: crossStorage,
 			})
 	}
 	return b.backupCopier
 }
 
 func (b *BackupContext) getRestoreCopier() *storage.Copier {
+	crossStorage := b.params.MinioCfg.CrossStorage
+	// force set copyByServer is true if two storage type is different
+	if b.getBackupStorageClient().Config().StorageType != b.getMilvusStorageClient().Config().StorageType {
+		crossStorage = true
+	}
 	if b.restoreCopier == nil {
 		b.restoreCopier = storage.NewCopier(
 			b.getBackupStorageClient(),
 			b.getMilvusStorageClient(),
 			storage.CopyOption{
 				WorkerNum:    b.params.BackupCfg.BackupCopyDataParallelism,
 				RPS:          RPS,
-				CopyByServer: b.params.BackupCfg.CopyByServer,
+				CopyByServer: crossStorage,
 			})
 	}
 	return b.restoreCopier

core/paramtable/params.go

@@ -43,8 +43,6 @@ type BackupConfig struct {
 	GcPauseEnable  bool
 	GcPauseSeconds int
 	GcPauseAddress string
-
-	CopyByServer bool
 }
 
 func (p *BackupConfig) init(base *BaseTable) {
@@ -58,7 +56,6 @@ func (p *BackupConfig) init(base *BaseTable) {
 	p.initGcPauseEnable()
 	p.initGcPauseSeconds()
 	p.initGcPauseAddress()
-	p.initCopyByServer()
 }
 
 func (p *BackupConfig) initMaxSegmentGroupSize() {
@@ -104,15 +101,6 @@ func (p *BackupConfig) initGcPauseAddress() {
 	p.GcPauseAddress = address
 }
 
-func (p *BackupConfig) initCopyByServer() {
-	copyByServer := p.Base.LoadWithDefault("backup.copyByServer", "false")
-	var err error
-	p.CopyByServer, err = strconv.ParseBool(copyByServer)
-	if err != nil {
-		panic("parse bool CopyByServer:" + err.Error())
-	}
-}
-
 type MilvusConfig struct {
 	Base *BaseTable
 
@@ -229,6 +217,8 @@ type MinioConfig struct {
 	BackupRootPath        string
 	BackupUseIAM          bool
 	BackupIAMEndpoint     string
+
+	CrossStorage bool
 }
 
 func (p *MinioConfig) init(base *BaseTable) {
@@ -256,6 +246,8 @@ func (p *MinioConfig) init(base *BaseTable) {
 	p.initBackupRootPath()
 	p.initBackupUseIAM()
 	p.initBackupIAMEndpoint()
+
+	p.initCrossStorage()
 }
 
 func (p *MinioConfig) initAddress() {
@@ -400,6 +392,15 @@ func (p *MinioConfig) initBackupRootPath() {
 	p.BackupRootPath = rootPath
 }
 
+func (p *MinioConfig) initCrossStorage() {
+	crossStorage := p.Base.LoadWithDefault("backup.crossStorage", "false")
+	var err error
+	p.CrossStorage, err = strconv.ParseBool(crossStorage)
+	if err != nil {
+		panic("parse bool CrossStorage:" + err.Error())
+	}
+}
+
 type HTTPConfig struct {
 	Base *BaseTable
 

docs/cross_storage_backup_restore.md

@@ -0,0 +1,90 @@
+# Cross storage backup & restore
+
+Previously, Milvus-backup utilized the Copy API of the storage client to back up data.
+This limited the backup capability to the same storage type as the Milvus cluster.
+However, there's a significant demand for cross-storage backups—for instance,
+backup data from Minio to a local disk or backup from in-house storage to cloud storage.
+
+Starting from version v0.4.21, Milvus-backup now supports cross-storage backups.
+In this process, data is read from the source storage and written to the target storage through the Milvus-backup service.
+
+This feature is currently in Beta.
+
+## Usage
+
+To enable cross-storage backup, you only need to adjust the configurations in backup.yaml.
+
+For example
+
+*Back up data from Minio to a local disk*:
+
+```yaml
+# Related configuration of minio, which is responsible for data persistence for Milvus.
+minio:
+  storageType: "minio"
+  address: localhost
+  port: 9000
+  accessKeyID: minioadmin
+  secretAccessKey: minioadmin
+  bucketName: "a-bucket"
+  rootPath: "files"
+
+  backupStorageType: "local"
+  backupRootPath: "/root/backup/"
+```
+
+*Backup from Minio to S3*
+
+```yaml
+minio:
+  storageType: "minio"
+  address: localhost
+  port: 9000
+  accessKeyID: minioadmin
+  secretAccessKey: minioadmin
+  useSSL: false
+  useIAM: false
+  iamEndpoint: ""
+  bucketName: "a-bucket"
+  rootPath: "files"
+
+  backupStorageType: "s3"
+  backupAddress: s3Address
+  backupPort: 443
+  backupAccessKeyID: s3AccessKey
+  backupSecretAccessKey: s3SecretAccessKey
+  backupBucketName: "s3-bucket"
+  backupRootPath: "s3-backup-path"
+```
+
+*Backup from Minio A to Minio B*
+
+If the two storage locations are of the same type but belong to different services,
+you need to add an additional configuration crossStorage=true to explicitly indicate that it is a cross-storage backup or restore operation.
+```yaml
+minio:
+  storageType: "minio"
+  address: addressA
+  port: 9000
+  accessKeyID: userA
+  secretAccessKey: passwdB
+  useSSL: false
+  useIAM: false
+  iamEndpoint: ""
+  bucketName: "a-bucket"
+  rootPath: "files"
+
+  backupStorageType: "minio"
+  backupAddress: addressB
+  backupPort: 9000
+  backupAccessKeyID: userB
+  backupSecretAccessKey: passwdB
+  backupBucketName: "b-bucket"
+  backupRootPath: "backup"
+
+  # If you need to back up or restore data between two different storage systems, direct client-side copying is not supported. 
+  # Set this option to true to enable data transfer through Milvus Backup.
+  # Note: This option will be automatically set to true if `minio.storageType` and `minio.backupStorageType` differ.
+  # However, if they are the same but belong to different services, you must manually set this option to `true`.
+  crossStorage: "true"
+```