feat: migrate an instance's storage between swift and s3

client/instances.go

@@ -282,6 +282,55 @@ func (ac *AdminClient) ModifyInstance(opts *InstanceOptions) (*Instance, error)
 	return readInstance(res)
 }
 
+// MigrateStorageOptions contains the options for MigrateStorage. It mirrors
+// the fields of storagemigration.Options in the model, without importing
+// that package: the client stays free of any model/... dependency.
+type MigrateStorageOptions struct {
+	To          string
+	DryRun      bool
+	FlagOnly    bool
+	Force       bool
+	PurgeSource bool
+}
+
+// MigrateStorageReport is the report returned by MigrateStorage. It mirrors
+// the fields of storagemigration.Report in the model.
+type MigrateStorageReport struct {
+	Files        int   `json:"Files"`
+	Versions     int   `json:"Versions"`
+	Bytes        int64 `json:"Bytes"`
+	AvatarCopied bool  `json:"AvatarCopied"`
+}
+
+// MigrateStorage moves an instance's object-storage content (files,
+// versions, avatar) from its current backend to the target scheme.
+func (ac *AdminClient) MigrateStorage(domain string, opts MigrateStorageOptions) (*MigrateStorageReport, error) {
+	if !validDomain(domain) {
+		return nil, fmt.Errorf("Invalid domain: %s", domain)
+	}
+	q := url.Values{
+		"to":           {opts.To},
+		"dry_run":      {strconv.FormatBool(opts.DryRun)},
+		"flag_only":    {strconv.FormatBool(opts.FlagOnly)},
+		"force":        {strconv.FormatBool(opts.Force)},
+		"purge_source": {strconv.FormatBool(opts.PurgeSource)},
+	}
+	res, err := ac.Req(&request.Options{
+		Method:  "POST",
+		Path:    "/instances/" + domain + "/migrate-storage",
+		Queries: q,
+	})
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	rep := &MigrateStorageReport{}
+	if err := json.NewDecoder(res.Body).Decode(rep); err != nil {
+		return nil, err
+	}
+	return rep, nil
+}
+
 // DestroyInstance is used to delete an instance and all its data.
 func (ac *AdminClient) DestroyInstance(domain string) error {
 	if !validDomain(domain) {

cmd/instances.go

@@ -45,6 +45,11 @@ var flagPassphrase string
 var flagForce bool
 var flagJSON bool
 var flagSwiftLayout int
+var flagMigrateTo string
+var flagMigrateDryRun bool
+var flagMigrateFlagOnly bool
+var flagMigrateForce bool
+var flagMigratePurgeSource bool
 var flagCouchCluster int
 var flagUUID string
 var flagOIDCID string
@@ -248,6 +253,33 @@ be used as the error message.
 	},
 }
 
+var migrateStorageCmd = &cobra.Command{
+	Use:   "migrate-storage <domain>",
+	Short: "Migrate an instance's file storage to another backend (e.g. s3)",
+	Long: `cozy-stack instances migrate-storage copies an instance's files, file
+versions and avatar to another storage backend and switches the instance to it.
+The source data is kept unless --purge-source is given.`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if len(args) != 1 {
+			return cmd.Usage()
+		}
+		ac := newAdminClient()
+		rep, err := ac.MigrateStorage(args[0], client.MigrateStorageOptions{
+			To:          flagMigrateTo,
+			DryRun:      flagMigrateDryRun,
+			FlagOnly:    flagMigrateFlagOnly,
+			Force:       flagMigrateForce,
+			PurgeSource: flagMigratePurgeSource,
+		})
+		if err != nil {
+			return err
+		}
+		fmt.Fprintf(os.Stdout, "migrated: %d files, %d versions, %d bytes, avatar=%v\n",
+			rep.Files, rep.Versions, rep.Bytes, rep.AvatarCopied)
+		return nil
+	},
+}
+
 var modifyInstanceCmd = &cobra.Command{
 	Use:   "modify <domain>",
 	Short: "Modify the instance properties",
@@ -1064,6 +1096,7 @@ func init() {
 	instanceCmdGroup.AddCommand(showInstanceCmd)
 	instanceCmdGroup.AddCommand(showDBPrefixInstanceCmd)
 	instanceCmdGroup.AddCommand(addInstanceCmd)
+	instanceCmdGroup.AddCommand(migrateStorageCmd)
 	instanceCmdGroup.AddCommand(modifyInstanceCmd)
 	instanceCmdGroup.AddCommand(countInstanceCmd)
 	instanceCmdGroup.AddCommand(lsInstanceCmd)
@@ -1101,6 +1134,11 @@ func init() {
 	addInstanceCmd.Flags().StringVar(&flagPhone, "phone", "", "The phone number of the owner")
 	addInstanceCmd.Flags().StringVar(&flagSettings, "settings", "", "A list of settings (eg context:foo,offer:premium)")
 	addInstanceCmd.Flags().IntVar(&flagSwiftLayout, "swift-layout", -1, "Specify the layout to use for Swift (from 0 for layout V1 to 2 for layout V3, -1 means the default)")
+	migrateStorageCmd.Flags().StringVar(&flagMigrateTo, "to", "s3", "Target storage scheme")
+	migrateStorageCmd.Flags().BoolVar(&flagMigrateDryRun, "dry-run", false, "Report what would be copied without writing or switching")
+	migrateStorageCmd.Flags().BoolVar(&flagMigrateFlagOnly, "flag-only", false, "Switch the backend pointer without copying (rollback to a retained source)")
+	migrateStorageCmd.Flags().BoolVar(&flagMigrateForce, "force", false, "Required with --flag-only; writes since cutover are lost")
+	migrateStorageCmd.Flags().BoolVar(&flagMigratePurgeSource, "purge-source", false, "Delete source objects after a successful switch")
 	addInstanceCmd.Flags().IntVar(&flagCouchCluster, "couch-cluster", -1, "Specify the CouchDB cluster where the instance will be created (-1 means the default)")
 	addInstanceCmd.Flags().StringVar(&flagDiskQuota, "disk-quota", "", "The quota allowed to the instance's VFS")
 	addInstanceCmd.Flags().StringSliceVar(&flagApps, "apps", nil, "Apps to be preinstalled")

docs/cli/cozy-stack_instances.md

@@ -50,6 +50,7 @@ cozy-stack instances <command> [flags]
 * [cozy-stack instances fsck](cozy-stack_instances_fsck.md)	 - Check a vfs
 * [cozy-stack instances import](cozy-stack_instances_import.md)	 - Import data from an export link
 * [cozy-stack instances ls](cozy-stack_instances_ls.md)	 - List instances
+* [cozy-stack instances migrate-storage](cozy-stack_instances_migrate-storage.md)	 - Migrate an instance's file storage to another backend (e.g. s3)
 * [cozy-stack instances modify](cozy-stack_instances_modify.md)	 - Modify the instance properties
 * [cozy-stack instances refresh-token-oauth](cozy-stack_instances_refresh-token-oauth.md)	 - Generate a new OAuth refresh token
 * [cozy-stack instances set-disk-quota](cozy-stack_instances_set-disk-quota.md)	 - Change the disk-quota of the instance

docs/cli/cozy-stack_instances_migrate-storage.md

@@ -0,0 +1,39 @@
+## cozy-stack instances migrate-storage
+
+Migrate an instance's file storage to another backend (e.g. s3)
+
+### Synopsis
+
+cozy-stack instances migrate-storage copies an instance's files, file
+versions and avatar to another storage backend and switches the instance to it.
+The source data is kept unless --purge-source is given.
+
+```
+cozy-stack instances migrate-storage <domain> [flags]
+```
+
+### Options
+
+```
+      --dry-run        Report what would be copied without writing or switching
+      --flag-only      Switch the backend pointer without copying (rollback to a retained source)
+      --force          Required with --flag-only; writes since cutover are lost
+  -h, --help           help for migrate-storage
+      --purge-source   Delete source objects after a successful switch
+      --to string      Target storage scheme (default "s3")
+```
+
+### Options inherited from parent commands
+
+```
+      --admin-host string   administration server host (default "localhost")
+      --admin-port int      administration server port (default 6060)
+  -c, --config string       configuration file (default "$HOME/.cozy.yaml")
+      --host string         server host (default "localhost")
+  -p, --port int            server port (default 8080)
+```
+
+### SEE ALSO
+
+* [cozy-stack instances](cozy-stack_instances.md)	 - Manage instances of a stack
+

docs/config.md

@@ -100,6 +100,34 @@ Magick, konnectors and services for example). And they can take several GB for
 the case of importing a Cozy. If needed, it is possible to configure the
 directory where they will be created via the `TMPDIR` environment variable.
 
+## Storage backend migration
+
+The `fs.url` parameter configures the storage backend (`file://`,
+`swift://` or `s3://`) used by all instances, as shown in
+[cozy.example.yaml](https://github.com/cozy/cozy-stack/blob/master/cozy.example.yaml)
+and detailed in the [S3 storage backend](s3.md) documentation.
+
+To move instances to a different backend one at a time, without changing
+the backend used by the rest of the fleet, an optional `fs.migration_target`
+key can be set to a second storage URL. Its connection is initialized
+alongside the default one at startup, so instances can be migrated to it
+while `fs.url` keeps pointing at the previous backend:
+
+```yaml
+fs:
+  url: swift://openstack/?UserName={{ .Env.OS_USERNAME }}&Password={{ .Env.OS_PASSWORD }}
+  migration_target: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET&region=rbx&bucket_prefix=cozy&use_ssl=true
+```
+
+As with `fs.url`, S3 credentials are passed as `access_key` and `secret_key`
+query parameters of the URL.
+
+Once `fs.migration_target` is set, the
+[`cozy-stack instances migrate-storage`](cli/cozy-stack_instances_migrate-storage.md)
+command can move individual instances to it. See
+[Migrating an instance from Swift to S3](s3.md#migrating-an-instance-from-swift-to-s3)
+for the full procedure, including rollback.
+
 ## Multiple CouchDB clusters
 
 With a large number of instances, a single CouchDB cluster may not be enough.

docs/s3.md

@@ -122,6 +122,97 @@ fs:
 Note: files uploaded to S3 won't be accessible when using the local
 filesystem backend, and vice versa. Each backend has its own storage.
 
+## Migrating an instance from Swift to S3
+
+Instances can be moved from Swift to S3 one at a time, without changing the
+storage backend for the rest of the fleet. This is useful to validate S3 on
+a few instances before committing the whole platform to it.
+
+### 1. Configure the migration target
+
+Set `fs.migration_target` to the S3 URL, keeping `fs.url` on `swift://`.
+Both connections (Swift and S3) are then initialized at startup:
+
+```yaml
+fs:
+  url: swift://openstack/?UserName=...
+  migration_target: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET&region=rbx&bucket_prefix=cozy&use_ssl=true
+```
+
+See [`fs.migration_target`](config.md#storage-backend-migration) in the
+configuration documentation for details on this key.
+
+### 2. Run the migration
+
+```bash
+cozy-stack instances migrate-storage <domain> --to s3
+```
+
+It is recommended to run with `--dry-run` first to see what would be copied
+without writing anything or switching the instance.
+
+The command:
+
+- blocks the instance (read-only, HTTP traffic only) for the duration of
+  the copy;
+- copies the files, file versions, and the user's avatar to the S3 target
+  (thumbnails and installed apps are not copied; they regenerate on the
+  target backend);
+- verifies the copied objects against the source;
+- flips the instance's storage backend to S3 and unblocks the instance.
+
+By default the Swift source is kept as-is after a successful migration, so
+it stays available for a rollback.
+
+**Known limitation:** blocking only gates HTTP traffic. Background workers
+and triggers can still write to the source during the migration window.
+Run migrations during low-activity periods until this is addressed.
+
+### 3. Roll back if needed
+
+If something looks wrong shortly after the switch, before any real write
+has landed on the S3 target, flip the instance back to the retained Swift
+source instantly, without copying anything back:
+
+```bash
+cozy-stack instances migrate-storage <domain> --to swift --flag-only --force
+```
+
+`--force` is required with `--flag-only` because any writes made against S3
+since the cutover are lost.
+
+If real data now lives on S3 and needs to be preserved, run a full
+migration back to Swift instead, which copies the data:
+
+```bash
+cozy-stack instances migrate-storage <domain> --to swift
+```
+
+### 4. Reclaim the source
+
+Once confident the instance is stable on its new backend, delete the
+retained source objects:
+
+```bash
+cozy-stack instances migrate-storage <domain> --to s3 --purge-source
+```
+
+Since the instance already uses `s3`, this runs in purge-only mode: nothing
+is copied, verified, or flipped, and the previously retained Swift data is
+simply deleted. The instance is not blocked for this step. Running the same
+command again is safe and is the way to retry a purge that failed right
+after an earlier switch.
+
+(`--purge-source` can also be supplied on the initial migration if no
+rollback window is needed.)
+
+### 5. Switch the global default
+
+After the whole fleet has been migrated to S3, change the global `fs.url`
+to the S3 URL and remove `fs.migration_target`. From then on, the
+per-instance backend flag set by earlier migrations simply matches the
+global default.
+
 ## Bucket strategy
 
 ### Design rationale

model/instance/instance.go

@@ -88,6 +88,11 @@ type Instance struct {
 	// See model/vfs/vfsswift for more details.
 	SwiftLayout int `json:"swift_cluster,omitempty"`
 
+	// FsScheme, when non-empty, overrides the global fs.url scheme for this
+	// instance. Used to migrate a single instance to another storage backend
+	// (e.g. "s3") without changing the stack-wide default. Empty = global default.
+	FsScheme string `json:"fs_scheme,omitempty"`
+
 	CouchCluster int `json:"couch_cluster,omitempty"`
 
 	// PassphraseHash is a hash of a hash of the user's passphrase: the
@@ -187,6 +192,15 @@ func (i *Instance) DBPrefix() string {
 	return i.Domain
 }
 
+// StorageScheme returns the storage backend scheme effective for this instance:
+// the per-instance FsScheme override when set, otherwise the global fs.url scheme.
+func (i *Instance) StorageScheme() string {
+	if i.FsScheme != "" {
+		return i.FsScheme
+	}
+	return config.FsURL().Scheme
+}
+
 // DomainName returns the main domain name of the instance.
 func (i *Instance) DomainName() string {
 	return i.Domain
@@ -252,14 +266,13 @@ func (i *Instance) MakeVFS() error {
 	if i.vfs != nil {
 		return nil
 	}
-	fsURL := config.FsURL()
 	mutex := config.Lock().ReadWrite(i, "vfs")
 	index := vfs.NewCouchdbIndexer(i)
 	disk := vfs.DiskThresholder(i)
 	var err error
-	switch fsURL.Scheme {
+	switch i.StorageScheme() {
 	case config.SchemeFile, config.SchemeMem:
-		i.vfs, err = vfsafero.New(i, index, disk, mutex, fsURL, i.DirName())
+		i.vfs, err = vfsafero.New(i, index, disk, mutex, config.FsURL(), i.DirName())
 	case config.SchemeSwift, config.SchemeSwiftSecure:
 		switch i.SwiftLayout {
 		case 2:
@@ -270,16 +283,16 @@ func (i *Instance) MakeVFS() error {
 	case config.SchemeS3:
 		i.vfs, err = vfss3.New(i, index, disk, mutex)
 	default:
-		err = fmt.Errorf("instance: unknown storage provider %s", fsURL.Scheme)
+		err = fmt.Errorf("instance: unknown storage provider %s", i.StorageScheme())
 	}
 	return err
 }
 
 // AvatarFS returns the hidden filesystem for storing the avatar.
 func (i *Instance) AvatarFS() vfs.Avatarer {
-	fsURL := config.FsURL()
-	switch fsURL.Scheme {
+	switch i.StorageScheme() {
 	case config.SchemeFile:
+		fsURL := config.FsURL()
 		baseFS := afero.NewBasePathFs(afero.NewOsFs(),
 			path.Join(fsURL.Path, i.DirName(), vfs.ThumbsDirName))
 		return vfsafero.NewAvatarFs(baseFS)
@@ -299,16 +312,16 @@ func (i *Instance) AvatarFS() vfs.Avatarer {
 		keyPrefix := i.DBPrefix() + "/"
 		return vfss3.NewAvatarFs(client, bucket, keyPrefix)
 	default:
-		panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme))
+		panic(fmt.Sprintf("instance: unknown storage provider %s", i.StorageScheme()))
 	}
 }
 
 // ThumbsFS returns the hidden filesystem for storing the thumbnails of the
 // photos/image
 func (i *Instance) ThumbsFS() vfs.Thumbser {
-	fsURL := config.FsURL()
-	switch fsURL.Scheme {
+	switch i.StorageScheme() {
 	case config.SchemeFile:
+		fsURL := config.FsURL()
 		baseFS := afero.NewBasePathFs(afero.NewOsFs(),
 			path.Join(fsURL.Path, i.DirName(), vfs.ThumbsDirName))
 		return vfsafero.NewThumbsFs(baseFS)
@@ -328,7 +341,7 @@ func (i *Instance) ThumbsFS() vfs.Thumbser {
 		keyPrefix := i.DBPrefix() + "/"
 		return vfss3.NewThumbsFs(client, bucket, keyPrefix)
 	default:
-		panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme))
+		panic(fmt.Sprintf("instance: unknown storage provider %s", i.StorageScheme()))
 	}
 }