diff --git a/client/instances.go b/client/instances.go index 07833c035aa..9325cef89c3 100644 --- a/client/instances.go +++ b/client/instances.go @@ -282,6 +282,55 @@ func (ac *AdminClient) ModifyInstance(opts *InstanceOptions) (*Instance, error) return readInstance(res) } +// MigrateStorageOptions contains the options for MigrateStorage. It mirrors +// the fields of storagemigration.Options in the model, without importing +// that package: the client stays free of any model/... dependency. +type MigrateStorageOptions struct { + To string + DryRun bool + FlagOnly bool + Force bool + PurgeSource bool +} + +// MigrateStorageReport is the report returned by MigrateStorage. It mirrors +// the fields of storagemigration.Report in the model. +type MigrateStorageReport struct { + Files int `json:"Files"` + Versions int `json:"Versions"` + Bytes int64 `json:"Bytes"` + AvatarCopied bool `json:"AvatarCopied"` +} + +// MigrateStorage moves an instance's object-storage content (files, +// versions, avatar) from its current backend to the target scheme. +func (ac *AdminClient) MigrateStorage(domain string, opts MigrateStorageOptions) (*MigrateStorageReport, error) { + if !validDomain(domain) { + return nil, fmt.Errorf("Invalid domain: %s", domain) + } + q := url.Values{ + "to": {opts.To}, + "dry_run": {strconv.FormatBool(opts.DryRun)}, + "flag_only": {strconv.FormatBool(opts.FlagOnly)}, + "force": {strconv.FormatBool(opts.Force)}, + "purge_source": {strconv.FormatBool(opts.PurgeSource)}, + } + res, err := ac.Req(&request.Options{ + Method: "POST", + Path: "/instances/" + domain + "/migrate-storage", + Queries: q, + }) + if err != nil { + return nil, err + } + defer res.Body.Close() + rep := &MigrateStorageReport{} + if err := json.NewDecoder(res.Body).Decode(rep); err != nil { + return nil, err + } + return rep, nil +} + // DestroyInstance is used to delete an instance and all its data. func (ac *AdminClient) DestroyInstance(domain string) error { if !validDomain(domain) { diff --git a/cmd/instances.go b/cmd/instances.go index bf60ab23622..2f0b720eb61 100644 --- a/cmd/instances.go +++ b/cmd/instances.go @@ -45,6 +45,11 @@ var flagPassphrase string var flagForce bool var flagJSON bool var flagSwiftLayout int +var flagMigrateTo string +var flagMigrateDryRun bool +var flagMigrateFlagOnly bool +var flagMigrateForce bool +var flagMigratePurgeSource bool var flagCouchCluster int var flagUUID string var flagOIDCID string @@ -248,6 +253,33 @@ be used as the error message. }, } +var migrateStorageCmd = &cobra.Command{ + Use: "migrate-storage ", + Short: "Migrate an instance's file storage to another backend (e.g. s3)", + Long: `cozy-stack instances migrate-storage copies an instance's files, file +versions and avatar to another storage backend and switches the instance to it. +The source data is kept unless --purge-source is given.`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return cmd.Usage() + } + ac := newAdminClient() + rep, err := ac.MigrateStorage(args[0], client.MigrateStorageOptions{ + To: flagMigrateTo, + DryRun: flagMigrateDryRun, + FlagOnly: flagMigrateFlagOnly, + Force: flagMigrateForce, + PurgeSource: flagMigratePurgeSource, + }) + if err != nil { + return err + } + fmt.Fprintf(os.Stdout, "migrated: %d files, %d versions, %d bytes, avatar=%v\n", + rep.Files, rep.Versions, rep.Bytes, rep.AvatarCopied) + return nil + }, +} + var modifyInstanceCmd = &cobra.Command{ Use: "modify ", Short: "Modify the instance properties", @@ -1064,6 +1096,7 @@ func init() { instanceCmdGroup.AddCommand(showInstanceCmd) instanceCmdGroup.AddCommand(showDBPrefixInstanceCmd) instanceCmdGroup.AddCommand(addInstanceCmd) + instanceCmdGroup.AddCommand(migrateStorageCmd) instanceCmdGroup.AddCommand(modifyInstanceCmd) instanceCmdGroup.AddCommand(countInstanceCmd) instanceCmdGroup.AddCommand(lsInstanceCmd) @@ -1101,6 +1134,11 @@ func init() { addInstanceCmd.Flags().StringVar(&flagPhone, "phone", "", "The phone number of the owner") addInstanceCmd.Flags().StringVar(&flagSettings, "settings", "", "A list of settings (eg context:foo,offer:premium)") addInstanceCmd.Flags().IntVar(&flagSwiftLayout, "swift-layout", -1, "Specify the layout to use for Swift (from 0 for layout V1 to 2 for layout V3, -1 means the default)") + migrateStorageCmd.Flags().StringVar(&flagMigrateTo, "to", "s3", "Target storage scheme") + migrateStorageCmd.Flags().BoolVar(&flagMigrateDryRun, "dry-run", false, "Report what would be copied without writing or switching") + migrateStorageCmd.Flags().BoolVar(&flagMigrateFlagOnly, "flag-only", false, "Switch the backend pointer without copying (rollback to a retained source)") + migrateStorageCmd.Flags().BoolVar(&flagMigrateForce, "force", false, "Required with --flag-only; writes since cutover are lost") + migrateStorageCmd.Flags().BoolVar(&flagMigratePurgeSource, "purge-source", false, "Delete source objects after a successful switch") addInstanceCmd.Flags().IntVar(&flagCouchCluster, "couch-cluster", -1, "Specify the CouchDB cluster where the instance will be created (-1 means the default)") addInstanceCmd.Flags().StringVar(&flagDiskQuota, "disk-quota", "", "The quota allowed to the instance's VFS") addInstanceCmd.Flags().StringSliceVar(&flagApps, "apps", nil, "Apps to be preinstalled") diff --git a/docs/cli/cozy-stack_instances.md b/docs/cli/cozy-stack_instances.md index 8a47bfcb0a8..e17f2a7f39d 100644 --- a/docs/cli/cozy-stack_instances.md +++ b/docs/cli/cozy-stack_instances.md @@ -50,6 +50,7 @@ cozy-stack instances [flags] * [cozy-stack instances fsck](cozy-stack_instances_fsck.md) - Check a vfs * [cozy-stack instances import](cozy-stack_instances_import.md) - Import data from an export link * [cozy-stack instances ls](cozy-stack_instances_ls.md) - List instances +* [cozy-stack instances migrate-storage](cozy-stack_instances_migrate-storage.md) - Migrate an instance's file storage to another backend (e.g. s3) * [cozy-stack instances modify](cozy-stack_instances_modify.md) - Modify the instance properties * [cozy-stack instances refresh-token-oauth](cozy-stack_instances_refresh-token-oauth.md) - Generate a new OAuth refresh token * [cozy-stack instances set-disk-quota](cozy-stack_instances_set-disk-quota.md) - Change the disk-quota of the instance diff --git a/docs/cli/cozy-stack_instances_migrate-storage.md b/docs/cli/cozy-stack_instances_migrate-storage.md new file mode 100644 index 00000000000..8bd3876ab81 --- /dev/null +++ b/docs/cli/cozy-stack_instances_migrate-storage.md @@ -0,0 +1,39 @@ +## cozy-stack instances migrate-storage + +Migrate an instance's file storage to another backend (e.g. s3) + +### Synopsis + +cozy-stack instances migrate-storage copies an instance's files, file +versions and avatar to another storage backend and switches the instance to it. +The source data is kept unless --purge-source is given. + +``` +cozy-stack instances migrate-storage [flags] +``` + +### Options + +``` + --dry-run Report what would be copied without writing or switching + --flag-only Switch the backend pointer without copying (rollback to a retained source) + --force Required with --flag-only; writes since cutover are lost + -h, --help help for migrate-storage + --purge-source Delete source objects after a successful switch + --to string Target storage scheme (default "s3") +``` + +### Options inherited from parent commands + +``` + --admin-host string administration server host (default "localhost") + --admin-port int administration server port (default 6060) + -c, --config string configuration file (default "$HOME/.cozy.yaml") + --host string server host (default "localhost") + -p, --port int server port (default 8080) +``` + +### SEE ALSO + +* [cozy-stack instances](cozy-stack_instances.md) - Manage instances of a stack + diff --git a/docs/config.md b/docs/config.md index e10d2459803..1bd70e19731 100644 --- a/docs/config.md +++ b/docs/config.md @@ -100,6 +100,34 @@ Magick, konnectors and services for example). And they can take several GB for the case of importing a Cozy. If needed, it is possible to configure the directory where they will be created via the `TMPDIR` environment variable. +## Storage backend migration + +The `fs.url` parameter configures the storage backend (`file://`, +`swift://` or `s3://`) used by all instances, as shown in +[cozy.example.yaml](https://github.com/cozy/cozy-stack/blob/master/cozy.example.yaml) +and detailed in the [S3 storage backend](s3.md) documentation. + +To move instances to a different backend one at a time, without changing +the backend used by the rest of the fleet, an optional `fs.migration_target` +key can be set to a second storage URL. Its connection is initialized +alongside the default one at startup, so instances can be migrated to it +while `fs.url` keeps pointing at the previous backend: + +```yaml +fs: + url: swift://openstack/?UserName={{ .Env.OS_USERNAME }}&Password={{ .Env.OS_PASSWORD }} + migration_target: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET®ion=rbx&bucket_prefix=cozy&use_ssl=true +``` + +As with `fs.url`, S3 credentials are passed as `access_key` and `secret_key` +query parameters of the URL. + +Once `fs.migration_target` is set, the +[`cozy-stack instances migrate-storage`](cli/cozy-stack_instances_migrate-storage.md) +command can move individual instances to it. See +[Migrating an instance from Swift to S3](s3.md#migrating-an-instance-from-swift-to-s3) +for the full procedure, including rollback. + ## Multiple CouchDB clusters With a large number of instances, a single CouchDB cluster may not be enough. diff --git a/docs/s3.md b/docs/s3.md index c757f3b4455..384c21956a0 100644 --- a/docs/s3.md +++ b/docs/s3.md @@ -122,6 +122,97 @@ fs: Note: files uploaded to S3 won't be accessible when using the local filesystem backend, and vice versa. Each backend has its own storage. +## Migrating an instance from Swift to S3 + +Instances can be moved from Swift to S3 one at a time, without changing the +storage backend for the rest of the fleet. This is useful to validate S3 on +a few instances before committing the whole platform to it. + +### 1. Configure the migration target + +Set `fs.migration_target` to the S3 URL, keeping `fs.url` on `swift://`. +Both connections (Swift and S3) are then initialized at startup: + +```yaml +fs: + url: swift://openstack/?UserName=... + migration_target: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET®ion=rbx&bucket_prefix=cozy&use_ssl=true +``` + +See [`fs.migration_target`](config.md#storage-backend-migration) in the +configuration documentation for details on this key. + +### 2. Run the migration + +```bash +cozy-stack instances migrate-storage --to s3 +``` + +It is recommended to run with `--dry-run` first to see what would be copied +without writing anything or switching the instance. + +The command: + +- blocks the instance (read-only, HTTP traffic only) for the duration of + the copy; +- copies the files, file versions, and the user's avatar to the S3 target + (thumbnails and installed apps are not copied; they regenerate on the + target backend); +- verifies the copied objects against the source; +- flips the instance's storage backend to S3 and unblocks the instance. + +By default the Swift source is kept as-is after a successful migration, so +it stays available for a rollback. + +**Known limitation:** blocking only gates HTTP traffic. Background workers +and triggers can still write to the source during the migration window. +Run migrations during low-activity periods until this is addressed. + +### 3. Roll back if needed + +If something looks wrong shortly after the switch, before any real write +has landed on the S3 target, flip the instance back to the retained Swift +source instantly, without copying anything back: + +```bash +cozy-stack instances migrate-storage --to swift --flag-only --force +``` + +`--force` is required with `--flag-only` because any writes made against S3 +since the cutover are lost. + +If real data now lives on S3 and needs to be preserved, run a full +migration back to Swift instead, which copies the data: + +```bash +cozy-stack instances migrate-storage --to swift +``` + +### 4. Reclaim the source + +Once confident the instance is stable on its new backend, delete the +retained source objects: + +```bash +cozy-stack instances migrate-storage --to s3 --purge-source +``` + +Since the instance already uses `s3`, this runs in purge-only mode: nothing +is copied, verified, or flipped, and the previously retained Swift data is +simply deleted. The instance is not blocked for this step. Running the same +command again is safe and is the way to retry a purge that failed right +after an earlier switch. + +(`--purge-source` can also be supplied on the initial migration if no +rollback window is needed.) + +### 5. Switch the global default + +After the whole fleet has been migrated to S3, change the global `fs.url` +to the S3 URL and remove `fs.migration_target`. From then on, the +per-instance backend flag set by earlier migrations simply matches the +global default. + ## Bucket strategy ### Design rationale diff --git a/model/instance/instance.go b/model/instance/instance.go index 91a7743b56d..ed6e938c150 100644 --- a/model/instance/instance.go +++ b/model/instance/instance.go @@ -88,6 +88,11 @@ type Instance struct { // See model/vfs/vfsswift for more details. SwiftLayout int `json:"swift_cluster,omitempty"` + // FsScheme, when non-empty, overrides the global fs.url scheme for this + // instance. Used to migrate a single instance to another storage backend + // (e.g. "s3") without changing the stack-wide default. Empty = global default. + FsScheme string `json:"fs_scheme,omitempty"` + CouchCluster int `json:"couch_cluster,omitempty"` // PassphraseHash is a hash of a hash of the user's passphrase: the @@ -187,6 +192,15 @@ func (i *Instance) DBPrefix() string { return i.Domain } +// StorageScheme returns the storage backend scheme effective for this instance: +// the per-instance FsScheme override when set, otherwise the global fs.url scheme. +func (i *Instance) StorageScheme() string { + if i.FsScheme != "" { + return i.FsScheme + } + return config.FsURL().Scheme +} + // DomainName returns the main domain name of the instance. func (i *Instance) DomainName() string { return i.Domain @@ -252,14 +266,13 @@ func (i *Instance) MakeVFS() error { if i.vfs != nil { return nil } - fsURL := config.FsURL() mutex := config.Lock().ReadWrite(i, "vfs") index := vfs.NewCouchdbIndexer(i) disk := vfs.DiskThresholder(i) var err error - switch fsURL.Scheme { + switch i.StorageScheme() { case config.SchemeFile, config.SchemeMem: - i.vfs, err = vfsafero.New(i, index, disk, mutex, fsURL, i.DirName()) + i.vfs, err = vfsafero.New(i, index, disk, mutex, config.FsURL(), i.DirName()) case config.SchemeSwift, config.SchemeSwiftSecure: switch i.SwiftLayout { case 2: @@ -270,16 +283,16 @@ func (i *Instance) MakeVFS() error { case config.SchemeS3: i.vfs, err = vfss3.New(i, index, disk, mutex) default: - err = fmt.Errorf("instance: unknown storage provider %s", fsURL.Scheme) + err = fmt.Errorf("instance: unknown storage provider %s", i.StorageScheme()) } return err } // AvatarFS returns the hidden filesystem for storing the avatar. func (i *Instance) AvatarFS() vfs.Avatarer { - fsURL := config.FsURL() - switch fsURL.Scheme { + switch i.StorageScheme() { case config.SchemeFile: + fsURL := config.FsURL() baseFS := afero.NewBasePathFs(afero.NewOsFs(), path.Join(fsURL.Path, i.DirName(), vfs.ThumbsDirName)) return vfsafero.NewAvatarFs(baseFS) @@ -299,16 +312,16 @@ func (i *Instance) AvatarFS() vfs.Avatarer { keyPrefix := i.DBPrefix() + "/" return vfss3.NewAvatarFs(client, bucket, keyPrefix) default: - panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) + panic(fmt.Sprintf("instance: unknown storage provider %s", i.StorageScheme())) } } // ThumbsFS returns the hidden filesystem for storing the thumbnails of the // photos/image func (i *Instance) ThumbsFS() vfs.Thumbser { - fsURL := config.FsURL() - switch fsURL.Scheme { + switch i.StorageScheme() { case config.SchemeFile: + fsURL := config.FsURL() baseFS := afero.NewBasePathFs(afero.NewOsFs(), path.Join(fsURL.Path, i.DirName(), vfs.ThumbsDirName)) return vfsafero.NewThumbsFs(baseFS) @@ -328,7 +341,7 @@ func (i *Instance) ThumbsFS() vfs.Thumbser { keyPrefix := i.DBPrefix() + "/" return vfss3.NewThumbsFs(client, bucket, keyPrefix) default: - panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) + panic(fmt.Sprintf("instance: unknown storage provider %s", i.StorageScheme())) } } diff --git a/model/instance/instance_storage_scheme_test.go b/model/instance/instance_storage_scheme_test.go new file mode 100644 index 00000000000..87df01242d2 --- /dev/null +++ b/model/instance/instance_storage_scheme_test.go @@ -0,0 +1,20 @@ +package instance + +import ( + "testing" + + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/stretchr/testify/assert" +) + +func TestStorageSchemeFallsBackToGlobal(t *testing.T) { + config.UseTestFile(t) + i := &Instance{} + assert.Equal(t, config.FsURL().Scheme, i.StorageScheme()) +} + +func TestStorageSchemeOverridesGlobal(t *testing.T) { + config.UseTestFile(t) + i := &Instance{FsScheme: config.SchemeS3} + assert.Equal(t, config.SchemeS3, i.StorageScheme()) +} diff --git a/model/instance/storagemigration/migration.go b/model/instance/storagemigration/migration.go new file mode 100644 index 00000000000..4f9fcd06e2a --- /dev/null +++ b/model/instance/storagemigration/migration.go @@ -0,0 +1,556 @@ +// Package storagemigration implements the engine that copies an instance's +// object-storage content (files, versions, avatar) from one VFS backend to +// another, without touching the shared CouchDB index. It is used to move an +// instance's files between Swift and S3 — either direction, S3 to Swift as +// well as Swift to S3 — while all other instances sharing the same CouchDB +// cluster keep working against the same io.cozy.files / +// io.cozy.files.versions documents. +package storagemigration + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/cozy/cozy-stack/model/instance" + "github.com/cozy/cozy-stack/model/instance/lifecycle" + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/model/vfs/vfss3" + "github.com/cozy/cozy-stack/model/vfs/vfsswift" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/pkg/prefixer" + "github.com/cozy/cozy-stack/pkg/s3util" +) + +// contentWriter is implemented by target VFS backends that can write object +// bytes for a (docID, internalID) key without creating a CouchDB document. +type contentWriter interface { + WriteContentAt(docID, internalID string, content io.Reader, size int64) error +} + +// contentStater is implemented by target VFS backends that can report the +// byte size of the object backing a (docID, internalID) key without +// touching CouchDB. It returns os.ErrNotExist when the object is absent. +type contentStater interface { + StatContentAt(docID, internalID string) (int64, error) +} + +// Report summarizes a content copy performed by CopyContent. +type Report struct { + Files int + Versions int + Bytes int64 + AvatarCopied bool +} + +func copyContent(db prefixer.Prefixer, src, dst vfs.VFS, srcAv, dstAv vfs.Avatarer) (*Report, error) { + writer, ok := dst.(contentWriter) + if !ok { + return nil, fmt.Errorf("storagemigration: target backend does not support index-free writes") + } + + rep := &Report{} + + if err := copyFiles(db, src, writer, rep); err != nil { + return rep, err + } + if err := copyVersions(db, src, writer, rep); err != nil { + return rep, err + } + if err := copyAvatar(srcAv, dstAv, rep); err != nil { + return rep, err + } + + return rep, nil +} + +func verify(db prefixer.Prefixer, dst vfs.VFS, dstAv vfs.Avatarer, expected *Report) error { + stater, ok := dst.(contentStater) + if !ok { + return fmt.Errorf("storagemigration: target backend does not support index-free stats") + } + + got := &Report{} + + if err := verifyFiles(db, stater, got); err != nil { + return err + } + if err := verifyVersions(db, stater, got); err != nil { + return err + } + + if got.Files != expected.Files { + return fmt.Errorf("storagemigration: verify: expected %d files, found %d on target", expected.Files, got.Files) + } + if got.Versions != expected.Versions { + return fmt.Errorf("storagemigration: verify: expected %d versions, found %d on target", expected.Versions, got.Versions) + } + + if expected.AvatarCopied { + ar, _, err := dstAv.OpenAvatar() + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("storagemigration: verify: avatar missing on target") + } + if err != nil { + return fmt.Errorf("storagemigration: verify: open target avatar: %w", err) + } + _ = ar.Close() + } + + return nil +} + +// sourceReport describes the source content that an already-populated target +// must contain before a flag-only switch. +func sourceReport(db prefixer.Prefixer, srcAv vfs.Avatarer) (*Report, error) { + rep := &Report{} + + err := couchdb.ForeachDocs(db, consts.Files, func(_ string, raw json.RawMessage) error { + var doc vfs.FileDoc + if err := json.Unmarshal(raw, &doc); err != nil { + return fmt.Errorf("storagemigration: decode file doc: %w", err) + } + if doc.Type != consts.DirType { + rep.Files++ + } + return nil + }) + if err != nil { + return nil, err + } + + err = couchdb.ForeachDocs(db, consts.FilesVersions, func(_ string, _ json.RawMessage) error { + rep.Versions++ + return nil + }) + if err != nil { + return nil, err + } + + ar, _, err := srcAv.OpenAvatar() + switch { + case errors.Is(err, os.ErrNotExist): + case err != nil: + return nil, fmt.Errorf("storagemigration: open source avatar: %w", err) + default: + _ = ar.Close() + rep.AvatarCopied = true + } + + return rep, nil +} + +// verifyFiles re-enumerates every io.cozy.files document and confirms the +// target object for each non-directory file exists with a matching size. +func verifyFiles(db prefixer.Prefixer, stater contentStater, got *Report) error { + return couchdb.ForeachDocs(db, consts.Files, func(_ string, raw json.RawMessage) error { + var doc vfs.FileDoc + if err := json.Unmarshal(raw, &doc); err != nil { + return fmt.Errorf("storagemigration: decode file doc: %w", err) + } + if doc.Type == consts.DirType { + return nil + } + + size, err := stater.StatContentAt(doc.DocID, doc.InternalID) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("storagemigration: verify: file %s missing on target", doc.DocID) + } + if err != nil { + return fmt.Errorf("storagemigration: verify: stat target file %s: %w", doc.DocID, err) + } + if size != doc.ByteSize { + return fmt.Errorf("storagemigration: verify: file %s size mismatch: expected %d, got %d", doc.DocID, doc.ByteSize, size) + } + + got.Files++ + return nil + }) +} + +// verifyVersions re-enumerates every io.cozy.files.versions document and +// confirms the target object for each version exists with a matching size. +func verifyVersions(db prefixer.Prefixer, stater contentStater, got *Report) error { + return couchdb.ForeachDocs(db, consts.FilesVersions, func(_ string, raw json.RawMessage) error { + var ver vfs.Version + if err := json.Unmarshal(raw, &ver); err != nil { + return fmt.Errorf("storagemigration: decode version doc: %w", err) + } + + fileID, internalID := splitVersionID(ver.DocID) + + size, err := stater.StatContentAt(fileID, internalID) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("storagemigration: verify: version %s missing on target", ver.DocID) + } + if err != nil { + return fmt.Errorf("storagemigration: verify: stat target version %s: %w", ver.DocID, err) + } + if size != ver.ByteSize { + return fmt.Errorf("storagemigration: verify: version %s size mismatch: expected %d, got %d", ver.DocID, ver.ByteSize, size) + } + + got.Versions++ + return nil + }) +} + +// copyFiles enumerates every io.cozy.files document (ForeachDocs is +// unfiltered, so trashed files are naturally included) and copies the +// content of each file (skipping directories) from src to dst. +func copyFiles(db prefixer.Prefixer, src vfs.VFS, writer contentWriter, rep *Report) error { + return couchdb.ForeachDocs(db, consts.Files, func(_ string, raw json.RawMessage) error { + var doc vfs.FileDoc + if err := json.Unmarshal(raw, &doc); err != nil { + return fmt.Errorf("storagemigration: decode file doc: %w", err) + } + if doc.Type == consts.DirType { + return nil + } + + r, err := src.OpenFile(&doc) + if err != nil { + return fmt.Errorf("storagemigration: open source file %s: %w", doc.DocID, err) + } + defer r.Close() + + if err := writer.WriteContentAt(doc.DocID, doc.InternalID, r, doc.ByteSize); err != nil { + return fmt.Errorf("storagemigration: write target file %s: %w", doc.DocID, err) + } + + rep.Files++ + rep.Bytes += doc.ByteSize + return nil + }) +} + +// copyVersions enumerates every io.cozy.files.versions document and copies +// the content of each version from src to dst. +func copyVersions(db prefixer.Prefixer, src vfs.VFS, writer contentWriter, rep *Report) error { + return couchdb.ForeachDocs(db, consts.FilesVersions, func(_ string, raw json.RawMessage) error { + var ver vfs.Version + if err := json.Unmarshal(raw, &ver); err != nil { + return fmt.Errorf("storagemigration: decode version doc: %w", err) + } + + fileID, internalID := splitVersionID(ver.DocID) + + fileDoc, err := src.FileByID(fileID) + if err != nil { + return fmt.Errorf("storagemigration: file for version %s: %w", ver.DocID, err) + } + + r, err := src.OpenFileVersion(fileDoc, &ver) + if err != nil { + return fmt.Errorf("storagemigration: open source version %s: %w", ver.DocID, err) + } + defer r.Close() + + if err := writer.WriteContentAt(fileID, internalID, r, ver.ByteSize); err != nil { + return fmt.Errorf("storagemigration: write target version %s: %w", ver.DocID, err) + } + + rep.Versions++ + rep.Bytes += ver.ByteSize + return nil + }) +} + +// copyAvatar copies the instance's avatar, if any, from srcAv to dstAv, +// setting rep.AvatarCopied on success. The absence of an avatar +// (os.ErrNotExist) is not an error. +func copyAvatar(srcAv, dstAv vfs.Avatarer, rep *Report) error { + ar, ctype, err := srcAv.OpenAvatar() + switch { + case errors.Is(err, os.ErrNotExist): + return nil + case err != nil: + return fmt.Errorf("storagemigration: open source avatar: %w", err) + } + defer ar.Close() + + w, err := dstAv.CreateAvatar(ctype) + if err != nil { + return fmt.Errorf("storagemigration: create target avatar: %w", err) + } + + if _, err := io.Copy(w, ar); err != nil { + _ = w.Close() + return fmt.Errorf("storagemigration: copy avatar: %w", err) + } + if err := w.Close(); err != nil { + return fmt.Errorf("storagemigration: finalize target avatar: %w", err) + } + + rep.AvatarCopied = true + return nil +} + +// splitVersionID splits a io.cozy.files.versions document id +// ("/") into its fileID and internalID parts. +func splitVersionID(versionDocID string) (fileID, internalID string) { + fileID, internalID, _ = strings.Cut(versionDocID, "/") + return +} + +// Options configures a call to Migrate. +type Options struct { + // To is the target storage scheme: config.SchemeS3 or config.SchemeSwift. + To string + // DryRun copies and verifies the content on the target backend but does + // not flip the instance's FsScheme. Combined with FlagOnly, it verifies + // the retained target without switching. + DryRun bool + // FlagOnly switches to an already-populated target without copying. + FlagOnly bool + // Force acknowledges that FlagOnly may discard writes made since cutover. + Force bool + // PurgeSource deletes the source backend's objects after a successful + // flip. It is a best-effort cleanup performed once the instance is + // already fully served from the target: a failure here does not revert + // the flip. + // + // If To already equals the instance's current storage scheme, + // PurgeSource switches Migrate into purge-only mode: nothing is copied, + // verified, or flipped, and only the OTHER backend's leftover data for + // this instance is deleted. This is what makes a deferred reclaim + // (running --purge-source well after the flip, or retrying an inline + // purge that failed) possible. + PurgeSource bool +} + +// containerNamer is implemented by VFS backends (vfsswift's V3 layout) that +// expose the underlying object-storage container name(s) they use, so +// storagemigration can ensure the target container exists without +// hand-rolling the naming scheme itself. +type containerNamer interface { + ContainerNames() map[string]string +} + +// Migrate moves an instance's object-storage content (files, versions, +// avatar) from its current backend to opts.To, verifies the copy, and only +// then flips the instance's FsScheme to the target. The instance is blocked +// (instance.BlockedMoving) for the duration of the copy/verify and unblocked +// on every return path. +// +// FsScheme is updated ONLY after verification succeeds; a DryRun or failed +// verification always leaves FsScheme unchanged. +// +// If opts.To already equals the instance's current storage scheme AND +// opts.PurgeSource is set, Migrate runs in purge-only mode instead: see +// purgeOnly. Without PurgeSource, opts.To == the current scheme is still an +// error. +func Migrate(inst *instance.Instance, opts Options) (*Report, error) { + switch opts.To { + case config.SchemeS3, config.SchemeSwift: + default: + return nil, fmt.Errorf("storagemigration: unsupported target scheme %q", opts.To) + } + + srcScheme := inst.StorageScheme() + if opts.To == srcScheme { + if opts.PurgeSource { + // Purge-only mode: the instance is already on opts.To (either + // because a previous migration flipped it, or the caller is + // retrying an inline purge that failed after that flip). There + // is nothing to copy, verify, or flip: just reclaim the OTHER + // backend's leftover data for this instance. + return purgeOnly(inst, opts.To) + } + return nil, fmt.Errorf("storagemigration: instance %s already uses %q as its storage scheme", inst.DomainName(), opts.To) + } + + if opts.To == config.SchemeS3 && !config.HasS3Client() { + return nil, errors.New("storagemigration: cannot migrate to s3: no S3 client is configured") + } + if opts.To == config.SchemeSwift && !config.HasSwiftConnection() { + return nil, errors.New("storagemigration: cannot migrate to swift: no swift connection is configured") + } + + if srcScheme == config.SchemeSwift || srcScheme == config.SchemeSwiftSecure { + if inst.SwiftLayout != 2 { + return nil, fmt.Errorf("storagemigration: source swift layout %d is not supported, only layout 2 (v3) can be migrated", inst.SwiftLayout) + } + } + if opts.FlagOnly && !opts.Force { + return nil, errors.New("storagemigration: flag-only migration requires Force: any write performed against the source since the previous cutover would be lost") + } + + // Build the SOURCE from the instance's current backend before touching + // anything (the instance already knows how to build it for its current + // scheme). + src := inst.VFS() + srcAv := inst.AvatarFS() + + dst, dstAv, err := buildTarget(inst, opts.To) + if err != nil { + return nil, err + } + + if err := lifecycle.Block(inst, instance.BlockedMoving.Code); err != nil { + return nil, fmt.Errorf("storagemigration: block instance: %w", err) + } + defer lifecycle.Unblock(inst) + + if opts.FlagOnly { + expected, err := sourceReport(inst, srcAv) + if err != nil { + return nil, err + } + if err := verify(inst, dst, dstAv, expected); err != nil { + return expected, err + } + if opts.DryRun { + return expected, nil + } + return flip(inst, opts, srcScheme, expected) + } + + rep, err := copyContent(inst, src, dst, srcAv, dstAv) + if err != nil { + return rep, err + } + if err := verify(inst, dst, dstAv, rep); err != nil { + return rep, err + } + + if opts.DryRun { + return rep, nil + } + + return flip(inst, opts, srcScheme, rep) +} + +// buildTarget constructs the VFS + Avatarer pair for the target scheme, +// ensuring the underlying bucket/container exists, without touching the +// CouchDB index (no InitFs: the index is shared with the source and must not +// be reinitialized). +func buildTarget(inst *instance.Instance, to string) (vfs.VFS, vfs.Avatarer, error) { + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "vfs-migration-target") + + switch to { + case config.SchemeS3: + dst, err := vfss3.New(inst, index, disk, mutex) + if err != nil { + return nil, nil, fmt.Errorf("storagemigration: build s3 target: %w", err) + } + bucket := vfss3.BucketName(inst.GetOrgID(), config.GetS3BucketPrefix()) + if err := s3util.EnsureBucket(context.Background(), config.GetS3Client(), bucket, config.GetS3Region()); err != nil { + return nil, nil, fmt.Errorf("storagemigration: ensure target bucket: %w", err) + } + dstAv := vfss3.NewAvatarFs(config.GetS3Client(), bucket, inst.DBPrefix()+"/") + return dst, dstAv, nil + + case config.SchemeSwift: + dst, err := vfsswift.NewV3(inst, index, disk, mutex) + if err != nil { + return nil, nil, fmt.Errorf("storagemigration: build swift target: %w", err) + } + if cn, ok := dst.(containerNamer); ok { + if container, ok := cn.ContainerNames()["container"]; ok && container != "" { + if err := config.GetSwiftConnection().ContainerCreate(context.Background(), container, nil); err != nil { + return nil, nil, fmt.Errorf("storagemigration: ensure target container: %w", err) + } + } + } + dstAv := vfsswift.NewAvatarFsV3(config.GetSwiftConnection(), inst) + return dst, dstAv, nil + + default: + return nil, nil, fmt.Errorf("storagemigration: unsupported target scheme %q", to) + } +} + +// flip persists the FsScheme change to the target scheme and, if requested, +// purges the source backend's objects on a best-effort basis. It never +// reverts the flip: once the instance points at the target, the target is +// the source of truth for the instance's content. +func flip(inst *instance.Instance, opts Options, srcScheme string, rep *Report) (*Report, error) { + inst.FsScheme = opts.To + if err := instance.Update(inst); err != nil { + return rep, fmt.Errorf("storagemigration: persist storage scheme flip: %w", err) + } + + if opts.PurgeSource { + if err := purgeSource(inst, srcScheme); err != nil { + return rep, fmt.Errorf("storagemigration: purge source after flip: %w", err) + } + } + + return rep, nil +} + +// purgeSource best-effort deletes the source backend's objects after a +// successful flip. The instance already fully serves reads/writes from the +// target at this point, so a purge failure is reported but never reverts the +// flip. +func purgeSource(inst *instance.Instance, srcScheme string) error { + switch srcScheme { + case config.SchemeS3: + bucket := vfss3.BucketName(inst.GetOrgID(), config.GetS3BucketPrefix()) + prefix := inst.DBPrefix() + "/" + return s3util.DeletePrefixObjects(context.Background(), config.GetS3Client(), bucket, prefix) + case config.SchemeSwift, config.SchemeSwiftSecure: + // The v3 swift layout uses a single, per-instance container (see + // vfsswift.NewV3), so purging the source is just deleting that + // container: build the same source VFS instance destroy/reset use + // (see lifecycle.destroy/reset calling inst.VFS().Delete()) and + // reuse its Delete(), which marks the container to-be-deleted and + // removes all its objects before removing the container itself. + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "vfs-migration-purge-source") + src, err := vfsswift.NewV3(inst, index, disk, mutex) + if err != nil { + return fmt.Errorf("storagemigration: build swift source for purge: %w", err) + } + return src.Delete() + default: + return fmt.Errorf("storagemigration: purging source scheme %q is not implemented", srcScheme) + } +} + +// purgeOnly implements Migrate's purge-only mode: opts.To already equals the +// instance's current storage scheme, so there is nothing to copy, verify, or +// flip. It only deletes the OTHER backend's (still-retained) leftover data +// for this instance, which is what makes the documented deferred reclaim +// step (running --purge-source well after the flip) work, and also gives a +// retry path when an inline purge failed after a previous flip. The active +// backend (to) is never touched and the instance is not blocked, since reads +// and writes against it are unaffected. +func purgeOnly(inst *instance.Instance, to string) (*Report, error) { + var otherScheme string + switch to { + case config.SchemeS3: + otherScheme = config.SchemeSwift + case config.SchemeSwift: + otherScheme = config.SchemeS3 + default: + return nil, fmt.Errorf("storagemigration: unsupported target scheme %q", to) + } + + switch otherScheme { + case config.SchemeS3: + if !config.HasS3Client() { + return nil, errors.New("storagemigration: cannot purge s3: no S3 client is configured") + } + case config.SchemeSwift: + if !config.HasSwiftConnection() { + return nil, errors.New("storagemigration: cannot purge swift: no swift connection is configured") + } + } + + if err := purgeSource(inst, otherScheme); err != nil { + return nil, fmt.Errorf("storagemigration: purge-only: %w", err) + } + + return &Report{}, nil +} diff --git a/model/instance/storagemigration/migration_test.go b/model/instance/storagemigration/migration_test.go new file mode 100644 index 00000000000..d8ae12b9037 --- /dev/null +++ b/model/instance/storagemigration/migration_test.go @@ -0,0 +1,569 @@ +package storagemigration + +import ( + "bytes" + "context" + "crypto/md5" + "errors" + "io" + "net/url" + "testing" + "time" + + "github.com/cozy/cozy-stack/model/instance" + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/model/vfs/vfsafero" + "github.com/cozy/cozy-stack/model/vfs/vfss3" + "github.com/cozy/cozy-stack/model/vfs/vfsswift" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/pkg/utils" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/minio/minio-go/v7" + swiftv2 "github.com/ncw/swift/v2" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +// migrationPrefixer is a minimal vfs.Prefixer (+ GetOrgID, required by +// vfss3.New's bucket-name derivation) implementation local to this external +// test package, mirroring model/vfs/vfs_test.go's contexter fixture. +type migrationPrefixer struct { + cluster int + domain string + prefix string + context string +} + +func (p *migrationPrefixer) DBCluster() int { return p.cluster } +func (p *migrationPrefixer) DomainName() string { return p.domain } +func (p *migrationPrefixer) DBPrefix() string { return p.prefix } +func (p *migrationPrefixer) GetContextName() string { return p.context } +func (p *migrationPrefixer) GetOrgID() string { return "migrationtestorg" } + +// migrationDisk is a minimal vfs.DiskThresholder (unlimited quota). +type migrationDisk struct{} + +func (migrationDisk) DiskQuota() int64 { return 0 } + +// migrationFixture bundles the source (afero) and target (s3) VFS + Avatarer +// pairs, plus the shared prefixer/db handle used to run CopyContent and to +// inspect CouchDB directly for the index-unchanged assertion. +type migrationFixture struct { + db *migrationPrefixer + src vfs.VFS + dst vfs.VFS + + srcAv vfs.Avatarer + dstAv vfs.Avatarer + + // minioClient and bucket give tests raw access to the S3 target, e.g. to + // delete an object behind the target VFS's back for negative-path checks. + minioClient *minio.Client + bucket string +} + +func setupMigrationFixture(t *testing.T) *migrationFixture { + t.Helper() + + config.UseTestFile(t) + + db := &migrationPrefixer{ + cluster: 0, + domain: "io.cozy.storagemigration.test", + prefix: "io.cozy.storagemigration.test", + context: "cozy_beta", + } + index := vfs.NewCouchdbIndexer(db) + + require.NoError(t, couchdb.ResetDB(db, consts.Files)) + require.NoError(t, couchdb.ResetDB(db, consts.FilesVersions)) + t.Cleanup(func() { + _ = couchdb.DeleteDB(db, consts.Files) + _ = couchdb.DeleteDB(db, consts.FilesVersions) + }) + + g, _ := errgroup.WithContext(context.Background()) + couchdb.DefineIndexes(g, db, couchdb.IndexesByDoctype(consts.Files)) + couchdb.DefineViews(g, db, couchdb.ViewsByDoctype(consts.Files)) + require.NoError(t, g.Wait()) + + // Source: afero-backed VFS on a temp dir. + tempdir := t.TempDir() + aferoMutex := config.Lock().ReadWrite(db, "storagemigration-test-afero") + aferoURL := &url.URL{Scheme: "file", Host: "localhost", Path: tempdir} + src, err := vfsafero.New(db, index, &migrationDisk{}, aferoMutex, aferoURL, "io.cozy.vfs.test") + require.NoError(t, err) + require.NoError(t, src.InitFs()) + + baseFS := afero.NewBasePathFs(afero.NewOsFs(), tempdir) + srcAv := vfsafero.NewAvatarFs(baseFS) + + // Target: S3-backed VFS against a MinIO test container. + mf := testutils.StartMinio(t) + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + s3Mutex := config.Lock().ReadWrite(db, "storagemigration-test-s3") + dst, err := vfss3.New(db, index, &migrationDisk{}, s3Mutex) + require.NoError(t, err) + + bucket := vfss3.BucketName(db.GetOrgID(), config.GetS3BucketPrefix()) + client := mf.Client(t) + require.NoError(t, client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{})) + + keyPrefix := db.DBPrefix() + "/" + dstAv := vfss3.NewAvatarFs(client, bucket, keyPrefix) + + return &migrationFixture{ + db: db, + src: src, + dst: dst, + srcAv: srcAv, + dstAv: dstAv, + minioClient: client, + bucket: bucket, + } +} + +// createSourceFile creates a file of the given name/content on the source +// VFS and returns its FileDoc. +func createSourceFile(t *testing.T, fx *migrationFixture, name string, content []byte) *vfs.FileDoc { + t.Helper() + + doc, err := vfs.NewFileDoc(name, "", int64(len(content)), nil, "text/plain", "text", time.Now(), false, false, false, []string{}) + require.NoError(t, err) + + f, err := fx.src.CreateFile(doc, nil) + require.NoError(t, err) + + _, err = io.Copy(f, bytes.NewReader(content)) + require.NoError(t, err) + require.NoError(t, f.Close()) + + got, err := fx.src.FileByPath("/" + name) + require.NoError(t, err) + return got +} + +func TestCopyContentMovesFilesVersionsAndAvatar(t *testing.T) { + fx := setupMigrationFixture(t) + + // 2 live files. + file1 := createSourceFile(t, fx, "file1.txt", []byte("hello from file 1")) + file2 := createSourceFile(t, fx, "file2.txt", []byte("hello from file 2, a bit longer")) + + // 1 file that gets trashed (content copy must still include it, since + // ForeachDocs is unfiltered). + file3 := createSourceFile(t, fx, "file3.txt", []byte("this one goes to the trash")) + file3, err := vfs.TrashFile(fx.src, file3) + require.NoError(t, err) + + // 1 extra version on file1. + versionPayload := []byte("an older revision of file 1") + sum := md5.Sum(versionPayload) + internalID := utils.RandomString(16) + version := &vfs.Version{ + DocID: file1.DocID + "/" + internalID, + ByteSize: int64(len(versionPayload)), + MD5Sum: sum[:], + } + version.Rels.File.Data.ID = file1.DocID + require.NoError(t, fx.src.ImportFileVersion(version, io.NopCloser(bytes.NewReader(versionPayload)))) + + // Avatar. + avatarPayload := []byte("fake png bytes for the avatar") + aw, err := fx.srcAv.CreateAvatar("image/png") + require.NoError(t, err) + _, err = aw.Write(avatarPayload) + require.NoError(t, err) + require.NoError(t, aw.Close()) + + // Capture revs before the copy: CopyContent must not touch the index. + revBefore1 := file1.Rev() + revBefore2 := file2.Rev() + revBefore3 := file3.Rev() + + rep, err := copyContent(fx.db, fx.src, fx.dst, fx.srcAv, fx.dstAv) + require.NoError(t, err) + + assert.Equal(t, 3, rep.Files) // 2 live + 1 trashed + assert.Equal(t, 1, rep.Versions) + assert.True(t, rep.AvatarCopied) + + // Every source file's bytes are now readable from the target VFS. + assertFileContentOn(t, fx.dst, file1, []byte("hello from file 1")) + assertFileContentOn(t, fx.dst, file2, []byte("hello from file 2, a bit longer")) + assertFileContentOn(t, fx.dst, file3, []byte("this one goes to the trash")) + + // The version is readable via the target VFS. + vr, err := fx.dst.OpenFileVersion(file1, version) + require.NoError(t, err) + gotVersion, err := io.ReadAll(vr) + require.NoError(t, err) + require.NoError(t, vr.Close()) + assert.Equal(t, versionPayload, gotVersion) + + // The avatar is readable via the target avatarer. Note: the source + // avatarer is afero-backed, which does not persist a content-type on + // disk and always reports "application/octet-stream" from OpenAvatar + // (see vfsafero's OpenAvatar); CopyContent faithfully forwards whatever + // content-type srcAv.OpenAvatar() reports to dstAv.CreateAvatar(), so + // that is what ends up stored on the target too. + ar, ctype, err := fx.dstAv.OpenAvatar() + require.NoError(t, err) + gotAvatar, err := io.ReadAll(ar) + require.NoError(t, err) + require.NoError(t, ar.Close()) + assert.Equal(t, "application/octet-stream", ctype) + assert.Equal(t, avatarPayload, gotAvatar) + + // The CouchDB index is unchanged: same revs as before the copy. + reread1 := &vfs.FileDoc{} + require.NoError(t, couchdb.GetDoc(fx.db, consts.Files, file1.DocID, reread1)) + reread2 := &vfs.FileDoc{} + require.NoError(t, couchdb.GetDoc(fx.db, consts.Files, file2.DocID, reread2)) + reread3 := &vfs.FileDoc{} + require.NoError(t, couchdb.GetDoc(fx.db, consts.Files, file3.DocID, reread3)) + + assert.Equal(t, revBefore1, reread1.Rev()) + assert.Equal(t, revBefore2, reread2.Rev()) + assert.Equal(t, revBefore3, reread3.Rev()) +} + +func TestVerifySucceedsAfterCopyAndFailsWhenObjectMissing(t *testing.T) { + fx := setupMigrationFixture(t) + + file1 := createSourceFile(t, fx, "file1.txt", []byte("hello from file 1")) + _ = createSourceFile(t, fx, "file2.txt", []byte("hello from file 2, a bit longer")) + + rep, err := copyContent(fx.db, fx.src, fx.dst, fx.srcAv, fx.dstAv) + require.NoError(t, err) + + require.NoError(t, verify(fx.db, fx.dst, fx.dstAv, rep)) + + // Remove one known target object directly via the raw MinIO client, then + // confirm Verify now detects the discrepancy. + keyPrefix := fx.db.DBPrefix() + "/" + objKey := vfss3.MakeObjectKey(keyPrefix, file1.DocID, file1.InternalID) + + require.NoError(t, fx.minioClient.RemoveObject(context.Background(), fx.bucket, objKey, minio.RemoveObjectOptions{})) + + assert.Error(t, verify(fx.db, fx.dst, fx.dstAv, rep)) +} + +func assertFileContentOn(t *testing.T, fs vfs.VFS, doc *vfs.FileDoc, want []byte) { + t.Helper() + r, err := fs.OpenFile(doc) + require.NoError(t, err) + got, err := io.ReadAll(r) + require.NoError(t, err) + require.NoError(t, r.Close()) + assert.Equal(t, want, got) +} + +// setupMigrateInstance creates a real instance (via testutils, on the global +// test backend, "mem") and populates it with a couple of files and an +// avatar, then starts a MinIO test server and wires up the global S3 client +// so config.HasS3Client() is true and Migrate can build an S3 target. +func setupMigrateInstance(t *testing.T) *instance.Instance { + t.Helper() + + if testing.Short() { + t.Skip("an instance is required for this test: test skipped due to the use of --short flag") + } + + config.UseTestFile(t) + setup := testutils.NewSetup(t, t.Name()) + inst := setup.GetTestInstance() + + mf := testutils.StartMinio(t) + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + createInstanceFile(t, inst, "migrate-file1.txt", []byte("hello from migrate file 1")) + createInstanceFile(t, inst, "migrate-file2.txt", []byte("hello from migrate file 2, a bit longer")) + + aw, err := inst.AvatarFS().CreateAvatar("image/png") + require.NoError(t, err) + _, err = aw.Write([]byte("fake png bytes for the migrate avatar")) + require.NoError(t, err) + require.NoError(t, aw.Close()) + + return inst +} + +// createInstanceFile creates a file of the given name/content on the +// instance's current VFS. +func createInstanceFile(t *testing.T, inst *instance.Instance, name string, content []byte) *vfs.FileDoc { + t.Helper() + + doc, err := vfs.NewFileDoc(name, "", int64(len(content)), nil, "text/plain", "text", time.Now(), false, false, false, []string{}) + require.NoError(t, err) + + f, err := inst.VFS().CreateFile(doc, nil) + require.NoError(t, err) + + _, err = io.Copy(f, bytes.NewReader(content)) + require.NoError(t, err) + require.NoError(t, f.Close()) + + got, err := inst.VFS().FileByPath("/" + name) + require.NoError(t, err) + return got +} + +func TestMigrateFlipsSchemeAfterVerify(t *testing.T) { + inst := setupMigrateInstance(t) + + rep, err := Migrate(inst, Options{To: config.SchemeS3}) + require.NoError(t, err) + require.NotNil(t, rep) + + assert.Equal(t, config.SchemeS3, inst.FsScheme) + assert.Greater(t, rep.Files, 0) + assert.False(t, inst.Blocked, "instance must be unblocked after a successful migration") + + // Reads are now served from S3: build a fresh S3 VFS for the instance + // (mirroring what inst.VFS() would now build) and confirm the migrated + // files are readable from it. + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "vfs-migrate-test-read") + s3fs, err := vfss3.New(inst, index, disk, mutex) + require.NoError(t, err) + + doc1, err := s3fs.FileByPath("/migrate-file1.txt") + require.NoError(t, err) + assertFileContentOn(t, s3fs, doc1, []byte("hello from migrate file 1")) + + doc2, err := s3fs.FileByPath("/migrate-file2.txt") + require.NoError(t, err) + assertFileContentOn(t, s3fs, doc2, []byte("hello from migrate file 2, a bit longer")) +} + +func TestMigrateDryRunDoesNotFlip(t *testing.T) { + inst := setupMigrateInstance(t) + + rep, err := Migrate(inst, Options{To: config.SchemeS3, DryRun: true}) + require.NoError(t, err) + require.NotNil(t, rep) + assert.Greater(t, rep.Files, 0) + + assert.Equal(t, "", inst.FsScheme) + assert.False(t, inst.Blocked, "instance must be unblocked after a dry-run migration") +} + +func TestMigrateFlagOnlyRequiresForce(t *testing.T) { + inst := setupMigrateInstance(t) + + _, err := Migrate(inst, Options{To: config.SchemeS3, FlagOnly: true}) + require.Error(t, err) + assert.Equal(t, "", inst.FsScheme) +} + +func TestMigrateFlagOnlyFlipsWhenTargetPopulated(t *testing.T) { + inst := setupMigrateInstance(t) + + _, err := Migrate(inst, Options{To: config.SchemeS3}) + require.NoError(t, err) + require.Equal(t, config.SchemeS3, inst.FsScheme) + + inst.FsScheme = "" + + rep, err := Migrate(inst, Options{To: config.SchemeS3, FlagOnly: true, Force: true}) + require.NoError(t, err) + require.NotNil(t, rep) + assert.Equal(t, config.SchemeS3, inst.FsScheme) + assert.Equal(t, 2, rep.Files) + assert.True(t, rep.AvatarCopied) +} + +func TestMigrateFlagOnlyFailsWhenTargetEmpty(t *testing.T) { + inst := setupMigrateInstance(t) + + _, err := Migrate(inst, Options{To: config.SchemeS3, FlagOnly: true, Force: true}) + require.Error(t, err) + assert.Equal(t, "", inst.FsScheme) +} + +// TestMigratePurgeSourceRemovesSourceObjects covers the IMPORTANT fix: a +// swift source must actually be purged (not return a "not implemented" +// error) after a successful flip. It exercises the real swift-source purge +// path end-to-end: an instance is first migrated from mem to a real +// (in-memory swifttest server) swift backend, populating swift for real; +// it is then migrated from swift to S3 with PurgeSource, and the test +// confirms the swift container backing the instance is gone afterward. +func TestMigratePurgeSourceRemovesSourceObjects(t *testing.T) { + if testing.Short() { + t.Skip("an instance is required for this test: test skipped due to the use of --short flag") + } + + config.UseTestFile(t) + setup := testutils.NewSetup(t, t.Name()) + setup.SetupSwiftTest() + inst := setup.GetTestInstance() + + // GetTestInstance created this instance against the test config's + // default (non-swift) scheme, so it was never assigned a swift layout. + // Migrate requires layout v3 for any swift source (see the SwiftLayout + // guard in Migrate), so set it explicitly here to simulate a real + // swift-scheme instance, as would exist in production. + inst.SwiftLayout = 2 + require.NoError(t, instance.Update(inst)) + + mf := testutils.StartMinio(t) + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + createInstanceFile(t, inst, "purge-file1.txt", []byte("hello from purge file 1")) + createInstanceFile(t, inst, "purge-file2.txt", []byte("hello from purge file 2, a bit longer")) + + // Step 1: migrate mem -> swift for real, so the swift container backing + // this instance is genuinely populated. + _, err := Migrate(inst, Options{To: config.SchemeSwift}) + require.NoError(t, err) + require.Equal(t, config.SchemeSwift, inst.FsScheme) + + containerName := swiftContainerName(t, inst) + + // Sanity check: the container really exists before the purge. + _, _, err = config.GetSwiftConnection().Container(context.Background(), containerName) + require.NoError(t, err, "the swift container must exist after the first migration") + + // Step 2: migrate swift -> S3 with PurgeSource, exercising the swift + // source purge implementation. + _, err = Migrate(inst, Options{To: config.SchemeS3, PurgeSource: true}) + require.NoError(t, err) + assert.Equal(t, config.SchemeS3, inst.FsScheme) + + // The swift container must be gone now: purgeSource must have actually + // deleted it, not returned a "not implemented" error after a + // successful (and now unrevertable) flip. + _, _, err = config.GetSwiftConnection().Container(context.Background(), containerName) + assert.True(t, errors.Is(err, swiftv2.ContainerNotFound), "expected the swift container to be gone after purge, got err=%v", err) +} + +// swiftContainerName builds the same per-instance swift V3 container that +// buildTarget/purgeSource use, so tests can inspect it directly against the +// swift connection. +func swiftContainerName(t *testing.T, inst *instance.Instance) string { + t.Helper() + + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "storagemigration-test-swift-container-name") + + sfs, err := vfsswift.NewV3(inst, index, disk, mutex) + require.NoError(t, err) + + cn, ok := sfs.(interface{ ContainerNames() map[string]string }) + require.True(t, ok, "vfsswift.NewV3 must expose ContainerNames()") + + return cn.ContainerNames()["container"] +} + +// TestMigratePurgeOnlyReclaimsOtherBackend covers the CRITICAL fix: once an +// instance already sits on its target scheme (a previous migration flipped +// it, and the Swift source was deliberately retained for rollback, as +// docs/s3.md step 4 describes), a later call with PurgeSource and the SAME +// To must not hit the "already uses that scheme" guard. Instead it must run +// in purge-only mode: reclaim the other backend's leftover data without +// copying, verifying, or flipping anything. +func TestMigratePurgeOnlyReclaimsOtherBackend(t *testing.T) { + if testing.Short() { + t.Skip("an instance is required for this test: test skipped due to the use of --short flag") + } + + config.UseTestFile(t) + setup := testutils.NewSetup(t, t.Name()) + setup.SetupSwiftTest() + inst := setup.GetTestInstance() + + // See TestMigratePurgeSourceRemovesSourceObjects: a swift source requires + // layout v3 to be migrated. + inst.SwiftLayout = 2 + require.NoError(t, instance.Update(inst)) + + mf := testutils.StartMinio(t) + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + createInstanceFile(t, inst, "purge-only-file1.txt", []byte("hello from purge-only file 1")) + createInstanceFile(t, inst, "purge-only-file2.txt", []byte("hello from purge-only file 2, a bit longer")) + + // Step 1: migrate mem -> swift for real, so the swift container backing + // this instance is genuinely populated. + _, err := Migrate(inst, Options{To: config.SchemeSwift}) + require.NoError(t, err) + require.Equal(t, config.SchemeSwift, inst.FsScheme) + + containerName := swiftContainerName(t, inst) + + // Step 2: migrate swift -> S3 WITHOUT PurgeSource, so the instance ends + // on S3 while the swift source is deliberately retained, exactly as + // docs/s3.md's rollback window describes. + _, err = Migrate(inst, Options{To: config.SchemeS3}) + require.NoError(t, err) + require.Equal(t, config.SchemeS3, inst.FsScheme) + + // Sanity check: the retained swift container still exists after the + // flip, since PurgeSource was not requested. + _, _, err = config.GetSwiftConnection().Container(context.Background(), containerName) + require.NoError(t, err, "the swift container must still exist: PurgeSource was not requested on the flip") + + // Step 3 (the deferred reclaim, run later): call Migrate again with + // To == the instance's CURRENT scheme (s3) and PurgeSource set. This + // must not error out on the "already uses that scheme" guard; it must + // instead purge the other backend (swift) and leave the instance as-is. + rep, err := Migrate(inst, Options{To: config.SchemeS3, PurgeSource: true}) + require.NoError(t, err) + require.NotNil(t, rep) + assert.Equal(t, config.SchemeS3, inst.FsScheme, "purge-only must not change the instance's scheme") + assert.False(t, inst.Blocked, "purge-only must not leave the instance blocked") + + // The swift container must be gone now. + _, _, err = config.GetSwiftConnection().Container(context.Background(), containerName) + assert.True(t, errors.Is(err, swiftv2.ContainerNotFound), "expected the swift container to be gone after purge-only, got err=%v", err) + + // The instance's S3 content (the active backend) must be untouched. + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "vfs-migrate-test-purge-only-read") + s3fs, err := vfss3.New(inst, index, disk, mutex) + require.NoError(t, err) + doc1, err := s3fs.FileByPath("/purge-only-file1.txt") + require.NoError(t, err) + assertFileContentOn(t, s3fs, doc1, []byte("hello from purge-only file 1")) +} + +// TestMigratePurgeOnlyWithoutPurgeFlagStillErrors covers the guard that must +// still hold for a plain re-run against the current scheme without +// PurgeSource: purge-only mode is only entered when PurgeSource is set. +func TestMigratePurgeOnlyWithoutPurgeFlagStillErrors(t *testing.T) { + inst := setupMigrateInstance(t) + + _, err := Migrate(inst, Options{To: config.SchemeS3}) + require.NoError(t, err) + require.Equal(t, config.SchemeS3, inst.FsScheme) + + _, err = Migrate(inst, Options{To: config.SchemeS3}) + require.Error(t, err) + assert.Equal(t, config.SchemeS3, inst.FsScheme) +} + +func TestMigrateFlagOnlyDryRunDoesNotFlip(t *testing.T) { + inst := setupMigrateInstance(t) + + _, err := Migrate(inst, Options{To: config.SchemeS3}) + require.NoError(t, err) + require.Equal(t, config.SchemeS3, inst.FsScheme) + + inst.FsScheme = "" + + rep, err := Migrate(inst, Options{To: config.SchemeS3, FlagOnly: true, Force: true, DryRun: true}) + require.NoError(t, err) + require.NotNil(t, rep) + assert.Equal(t, 2, rep.Files) + assert.Equal(t, "", inst.FsScheme) + assert.False(t, inst.Blocked, "instance must be unblocked after a dry-run flag-only migration") +} diff --git a/model/stack/main.go b/model/stack/main.go index f82bdbe87da..820c36af832 100644 --- a/model/stack/main.go +++ b/model/stack/main.go @@ -90,6 +90,20 @@ security features. Please do not use this binary as your production server. return nil, nil, fmt.Errorf("failed to init the S3 connection: %w", err) } + // When a storage migration target is configured (e.g. migrating instances + // to S3 while the global default is still Swift), init that connection too. + if config.HasS3Target() { + // If fs.url is already an s3 scheme, initializing the migration + // target here would silently overwrite the global S3 client with the + // migration endpoint: refuse to start instead of risking that. + if config.FsURL().Scheme == config.SchemeS3 { + return nil, nil, fmt.Errorf("fs.migration_target must not be set when fs.url is already an s3 scheme") + } + if err := config.InitS3Connection(config.Fs{URL: config.MigrationTargetURL()}); err != nil { + return nil, nil, fmt.Errorf("failed to init the S3 migration target connection: %w", err) + } + } + workersList, err := job.GetWorkersList() if err != nil { return nil, nil, fmt.Errorf("failed to get the workers list: %w", err) diff --git a/model/vfs/vfs.go b/model/vfs/vfs.go index 75a0be44ec7..2a83cd2f1fe 100644 --- a/model/vfs/vfs.go +++ b/model/vfs/vfs.go @@ -270,6 +270,9 @@ type Avatarer interface { // but does if there was a problem deleting it. DeleteAvatar() error ServeAvatarContent(w http.ResponseWriter, req *http.Request) error + // OpenAvatar returns a reader over the stored avatar content and its + // content-type, or os.ErrNotExist if no avatar is stored. + OpenAvatar() (io.ReadCloser, string, error) } // Thumbser defines an interface to define a thumbnail filesystem. diff --git a/model/vfs/vfsafero/avatar.go b/model/vfs/vfsafero/avatar.go index 86b0d1ab398..44d38587339 100644 --- a/model/vfs/vfsafero/avatar.go +++ b/model/vfs/vfsafero/avatar.go @@ -74,6 +74,20 @@ func (a *avatarFS) AvatarExists() (bool, error) { return infos.Size() > 0, nil } +// OpenAvatar returns a reader over the stored avatar content and its +// content-type, or os.ErrNotExist if no avatar is stored. The content-type +// is not persisted on disk by this backend, so a generic value is returned. +func (a *avatarFS) OpenAvatar() (io.ReadCloser, string, error) { + f, err := a.fs.Open(AvatarFilename) + if err != nil { + if os.IsNotExist(err) { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + return f, "application/octet-stream", nil +} + func (a *avatarFS) ServeAvatarContent(w http.ResponseWriter, req *http.Request) error { s, err := a.fs.Stat(AvatarFilename) if err != nil { diff --git a/model/vfs/vfss3/avatar.go b/model/vfs/vfss3/avatar.go index a164ef39c4b..7ff2feea834 100644 --- a/model/vfs/vfss3/avatar.go +++ b/model/vfs/vfss3/avatar.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "os" "time" "github.com/cozy/cozy-stack/model/vfs" @@ -80,6 +81,26 @@ func (a *avatarS3) DeleteAvatar() error { return nil } +// OpenAvatar returns a reader over the stored avatar content and its +// content-type, or os.ErrNotExist if no avatar is stored. +func (a *avatarS3) OpenAvatar() (io.ReadCloser, string, error) { + obj, err := a.client.GetObject(a.ctx, a.bucket, a.avatarKey(), minio.GetObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + info, err := obj.Stat() + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + return obj, info.ContentType, nil +} + func (a *avatarS3) ServeAvatarContent(w http.ResponseWriter, req *http.Request) error { obj, err := a.client.GetObject(a.ctx, a.bucket, a.avatarKey(), minio.GetObjectOptions{}) if err != nil { diff --git a/model/vfs/vfss3/avatar_open_test.go b/model/vfs/vfss3/avatar_open_test.go new file mode 100644 index 00000000000..2da3c0ebfdd --- /dev/null +++ b/model/vfs/vfss3/avatar_open_test.go @@ -0,0 +1,55 @@ +package vfss3_test + +import ( + "context" + "io" + "os" + "testing" + + "github.com/cozy/cozy-stack/model/vfs/vfss3" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestOpenAvatarRoundTrip verifies that OpenAvatar returns the content and +// content-type previously stored via CreateAvatar, and that it reports +// os.ErrNotExist when no avatar has been stored yet. +func TestOpenAvatarRoundTrip(t *testing.T) { + config.UseTestFile(t) + + mf := testutils.StartMinio(t) + + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + bucket := "io-cozy-vfss3-openavatar-test" + keyPrefix := "io.cozy.vfss3.openavatar.test/" + + client := mf.Client(t) + require.NoError(t, client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{})) + + av := vfss3.NewAvatarFs(client, bucket, keyPrefix) + + // No avatar stored yet: OpenAvatar must report os.ErrNotExist. + _, _, err := av.OpenAvatar() + assert.ErrorIs(t, err, os.ErrNotExist) + + // Store an avatar, then read it back. + w, err := av.CreateAvatar("image/png") + require.NoError(t, err) + payload := []byte("fake png bytes") + _, err = w.Write(payload) + require.NoError(t, err) + require.NoError(t, w.Close()) + + r, ctype, err := av.OpenAvatar() + require.NoError(t, err) + defer r.Close() + assert.Equal(t, "image/png", ctype) + + got, err := io.ReadAll(r) + require.NoError(t, err) + assert.Equal(t, payload, got) +} diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go index caa2cbabfa5..891c134870d 100644 --- a/model/vfs/vfss3/impl.go +++ b/model/vfs/vfss3/impl.go @@ -625,6 +625,35 @@ func (sfs *s3VFS) ImportFileVersion(version *vfs.Version, content io.ReadCloser) return sfs.Indexer.CreateVersion(version) } +// WriteContentAt streams content into the object backing the (docID, internalID) +// key, creating NO CouchDB document. It is used by storage migration, which +// preserves the shared index and only moves object bytes. size may be -1 when +// unknown (falls back to multipart). +func (sfs *s3VFS) WriteContentAt(docID, internalID string, content io.Reader, size int64) error { + objKey := MakeObjectKey(sfs.keyPrefix, docID, internalID) + _, err := sfs.client.PutObject(sfs.ctx, sfs.bucket, objKey, content, size, minio.PutObjectOptions{ + ContentType: "application/octet-stream", + SendContentMd5: true, + }) + return err +} + +// StatContentAt returns the byte size of the object backing the (docID, +// internalID) key, without touching CouchDB. It returns os.ErrNotExist when +// the object is absent. Used by storage migration to verify a copy landed on +// the target before flipping the instance's backend flag. +func (sfs *s3VFS) StatContentAt(docID, internalID string) (int64, error) { + objKey := MakeObjectKey(sfs.keyPrefix, docID, internalID) + info, err := sfs.client.StatObject(sfs.ctx, sfs.bucket, objKey, minio.StatObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return 0, os.ErrNotExist + } + return 0, err + } + return info.Size, nil +} + func (sfs *s3VFS) RevertFileVersion(doc *vfs.FileDoc, version *vfs.Version) error { if lockerr := sfs.mu.Lock(); lockerr != nil { return lockerr diff --git a/model/vfs/vfss3/write_content_at_test.go b/model/vfs/vfss3/write_content_at_test.go new file mode 100644 index 00000000000..e513222d0d0 --- /dev/null +++ b/model/vfs/vfss3/write_content_at_test.go @@ -0,0 +1,86 @@ +package vfss3_test + +import ( + "bytes" + "context" + "io" + "testing" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/model/vfs/vfss3" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/require" +) + +// writeContentAtPrefixer is a minimal vfs.Prefixer implementation, local to +// this test, so it can live in an external test package (package +// vfss3_test) without importing anything from the internal vfss3 test +// harness. It also exposes GetOrgID so vfss3.New can build the bucket name. +type writeContentAtPrefixer struct { + cluster int + domain string + prefix string + context string +} + +func (p *writeContentAtPrefixer) DBCluster() int { return p.cluster } +func (p *writeContentAtPrefixer) DomainName() string { return p.domain } +func (p *writeContentAtPrefixer) DBPrefix() string { return p.prefix } +func (p *writeContentAtPrefixer) GetContextName() string { return p.context } +func (p *writeContentAtPrefixer) GetOrgID() string { return "wcatestorg" } + +// writeContentAtDisk is a minimal vfs.DiskThresholder, unused by +// WriteContentAt itself but required by vfss3.New's signature. +type writeContentAtDisk struct{} + +func (writeContentAtDisk) DiskQuota() int64 { return 0 } + +// TestWriteContentAtPutsBytesWithoutIndex verifies that WriteContentAt is a +// pure object-storage primitive: it puts bytes at the object key derived +// from (docID, internalID) and does not touch CouchDB at all (no +// ResetDB/DefineIndexes/InitFs is performed in this test). +func TestWriteContentAtPutsBytesWithoutIndex(t *testing.T) { + config.UseTestFile(t) + + mf := testutils.StartMinio(t) + + db := &writeContentAtPrefixer{ + cluster: 0, + domain: "io.cozy.vfss3.writecontentat.test", + prefix: "io.cozy.vfss3.writecontentat.test", + context: "cozy_beta", + } + index := vfs.NewCouchdbIndexer(db) + + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + mutex := config.Lock().ReadWrite(db, "vfs-s3-writecontentat-test") + sfs, err := vfss3.New(db, index, &writeContentAtDisk{}, mutex) + require.NoError(t, err) + + // WriteContentAt never creates its own bucket (that's InitFs's job, which + // we deliberately skip here since it would also touch CouchDB through + // Indexer.InitIndex). Create the bucket directly against the raw client. + bucket := vfss3.BucketName(db.GetOrgID(), config.GetS3BucketPrefix()) + client := mf.Client(t) + require.NoError(t, client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{})) + + w := sfs.(interface { + WriteContentAt(docID, internalID string, content io.Reader, size int64) error + }) + + docID := "0123456789012345678901234567890a" // 32 chars + internalID := "abcdef0123456789" // 16 chars + payload := []byte("hello s3 migration") + + require.NoError(t, w.WriteContentAt(docID, internalID, bytes.NewReader(payload), int64(len(payload)))) + + objKey := vfss3.MakeObjectKey(db.DBPrefix()+"/", docID, internalID) + obj, err := client.GetObject(context.Background(), bucket, objKey, minio.GetObjectOptions{}) + require.NoError(t, err) + got, err := io.ReadAll(obj) + require.NoError(t, err) + require.Equal(t, payload, got) +} diff --git a/model/vfs/vfsswift/avatar_v3.go b/model/vfs/vfsswift/avatar_v3.go index 917c0f0a3a0..b776cd413ca 100644 --- a/model/vfs/vfsswift/avatar_v3.go +++ b/model/vfs/vfsswift/avatar_v3.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "os" "time" "github.com/cozy/cozy-stack/model/vfs" @@ -45,6 +46,19 @@ func (a *avatarV3) DeleteAvatar() error { return err } +// OpenAvatar returns a reader over the stored avatar content and its +// content-type, or os.ErrNotExist if no avatar is stored. +func (a *avatarV3) OpenAvatar() (io.ReadCloser, string, error) { + f, headers, err := a.c.ObjectOpen(a.ctx, a.container, "avatar", false, nil) + if err != nil { + if err == swift.ObjectNotFound { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + return f, headers["Content-Type"], nil +} + func (a *avatarV3) ServeAvatarContent(w http.ResponseWriter, req *http.Request) error { f, o, err := a.c.ObjectOpen(a.ctx, a.container, "avatar", false, nil) if err != nil { diff --git a/model/vfs/vfsswift/impl_v3.go b/model/vfs/vfsswift/impl_v3.go index e40eab1ef10..e06304c7c2c 100644 --- a/model/vfs/vfsswift/impl_v3.go +++ b/model/vfs/vfsswift/impl_v3.go @@ -579,6 +579,40 @@ func (sfs *swiftVFSV3) ImportFileVersion(version *vfs.Version, content io.ReadCl return sfs.Indexer.CreateVersion(version) } +// WriteContentAt streams content into the object backing the (docID, +// internalID) key in this instance's container, creating NO CouchDB +// document. Used by storage migration, which preserves the shared index and +// only moves bytes. +func (sfs *swiftVFSV3) WriteContentAt(docID, internalID string, content io.Reader, size int64) error { + objName := MakeObjectNameV3(docID, internalID) + f, err := sfs.c.ObjectCreate(sfs.ctx, sfs.container, objName, true, "", "application/octet-stream", nil) + if err != nil { + return err + } + if _, err = io.Copy(f, content); err != nil { + _ = f.Close() + return err + } + return f.Close() +} + +// StatContentAt returns the byte size of the object backing the (docID, +// internalID) key in this instance's container, without touching CouchDB. It +// returns os.ErrNotExist when the object is absent. Used by storage +// migration to verify a copy landed on the target before flipping the +// instance's backend flag. +func (sfs *swiftVFSV3) StatContentAt(docID, internalID string) (int64, error) { + objName := MakeObjectNameV3(docID, internalID) + info, _, err := sfs.c.Object(sfs.ctx, sfs.container, objName) + if errors.Is(err, swift.ObjectNotFound) { + return 0, os.ErrNotExist + } + if err != nil { + return 0, err + } + return info.Bytes, nil +} + func (sfs *swiftVFSV3) RevertFileVersion(doc *vfs.FileDoc, version *vfs.Version) error { if lockerr := sfs.mu.Lock(); lockerr != nil { return lockerr diff --git a/model/vfs/vfsswift/write_content_at_v3_test.go b/model/vfs/vfsswift/write_content_at_v3_test.go new file mode 100644 index 00000000000..42834de0d7a --- /dev/null +++ b/model/vfs/vfsswift/write_content_at_v3_test.go @@ -0,0 +1,107 @@ +package vfsswift_test + +import ( + "bytes" + "context" + "io" + "net/url" + "testing" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/model/vfs/vfsswift" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/ncw/swift/v2/swifttest" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +// writeContentAtPrefixer is a minimal vfs.Prefixer implementation, local to +// this test, so it can live in an external test package (package +// vfsswift_test) without importing anything from the internal vfsswift test +// harness. +type writeContentAtPrefixer struct { + cluster int + domain string + prefix string + context string +} + +func (p *writeContentAtPrefixer) DBCluster() int { return p.cluster } +func (p *writeContentAtPrefixer) DomainName() string { return p.domain } +func (p *writeContentAtPrefixer) DBPrefix() string { return p.prefix } +func (p *writeContentAtPrefixer) GetContextName() string { return p.context } + +// writeContentAtDisk is a minimal vfs.DiskThresholder, unused by +// WriteContentAt itself but required by vfsswift.NewV3's signature. +type writeContentAtDisk struct{} + +func (writeContentAtDisk) DiskQuota() int64 { return 0 } + +// TestWriteContentAtPutsBytesWithoutIndex verifies that WriteContentAt is a +// pure object-storage primitive: it puts bytes at the object key derived +// from (docID, internalID) and does not touch CouchDB (no document is +// created for the write itself). +func TestWriteContentAtPutsBytesWithoutIndex(t *testing.T) { + config.UseTestFile(t) + testutils.NeedCouchdb(t) + + db := &writeContentAtPrefixer{ + cluster: 0, + domain: "io.cozy.vfsswift.writecontentat.test", + prefix: "io.cozy.vfsswift.writecontentat.test", + context: "cozy_beta", + } + index := vfs.NewCouchdbIndexer(db) + + swiftSrv, err := swifttest.NewSwiftServer("localhost") + require.NoError(t, err, "failed to create swift server") + t.Cleanup(func() { swiftSrv.Close() }) + + require.NoError(t, config.InitSwiftConnection(config.Fs{ + URL: &url.URL{ + Scheme: "swift", + Host: "localhost", + RawQuery: "UserName=swifttest&Password=swifttest&AuthURL=" + url.QueryEscape(swiftSrv.AuthURL), + }, + })) + + mutex := config.Lock().ReadWrite(db, "vfs-swiftv3-writecontentat-test") + sfs, err := vfsswift.NewV3(db, index, &writeContentAtDisk{}, mutex) + require.NoError(t, err) + + require.NoError(t, couchdb.ResetDB(db, consts.Files)) + t.Cleanup(func() { _ = couchdb.DeleteDB(db, consts.Files) }) + + g, _ := errgroup.WithContext(context.Background()) + couchdb.DefineIndexes(g, db, couchdb.IndexesByDoctype(consts.Files)) + couchdb.DefineViews(g, db, couchdb.ViewsByDoctype(consts.Files)) + require.NoError(t, g.Wait()) + + require.NoError(t, sfs.InitFs()) + + w := sfs.(interface { + WriteContentAt(docID, internalID string, content io.Reader, size int64) error + }) + + docID := "0123456789012345678901234567890a" // 32 chars + internalID := "abcdef0123456789" // 16 chars + payload := []byte("hello swift migration") + + require.NoError(t, w.WriteContentAt(docID, internalID, bytes.NewReader(payload), int64(len(payload)))) + + cn := sfs.(interface{ ContainerNames() map[string]string }) + container := cn.ContainerNames()["container"] + objName := vfsswift.MakeObjectNameV3(docID, internalID) + + conn := config.GetSwiftConnection() + obj, _, err := conn.ObjectOpen(context.Background(), container, objName, false, nil) + require.NoError(t, err) + defer obj.Close() + + got, err := io.ReadAll(obj) + require.NoError(t, err) + require.Equal(t, payload, got) +} diff --git a/pkg/config/config/config.go b/pkg/config/config/config.go index 5c8d687479f..924099e8550 100644 --- a/pkg/config/config/config.go +++ b/pkg/config/config/config.go @@ -232,6 +232,10 @@ type Fs struct { AutoCleanTrashedAfter map[string]string Versioning FsVersioning Contexts map[string]interface{} + // MigrationTarget, when set, is an alternate storage URL (e.g. s3://...) + // whose connection is initialized alongside the default one, so instances + // can be migrated to it while the global scheme stays unchanged. + MigrationTarget *url.URL } // FsVersioning contains the configuration for the versioning of files @@ -498,6 +502,17 @@ func FsURL() *url.URL { return config.Fs.URL } +// MigrationTargetURL returns the configured storage migration target URL, or nil. +func MigrationTargetURL() *url.URL { + return config.Fs.MigrationTarget +} + +// HasS3Target reports whether an S3 storage migration target is configured. +func HasS3Target() bool { + u := config.Fs.MigrationTarget + return u != nil && u.Scheme == SchemeS3 +} + // ServerAddr returns the address on which the stack is run func ServerAddr() string { return net.JoinHostPort(config.Host, strconv.Itoa(config.Port)) @@ -849,6 +864,14 @@ func UseViper(v *viper.Viper) error { } } + var migrationTarget *url.URL + if raw := v.GetString("fs.migration_target"); raw != "" { + migrationTarget, err = url.Parse(raw) + if err != nil { + return err + } + } + couch, err := makeCouch(v) if err != nil { return err @@ -1163,7 +1186,8 @@ func UseViper(v *viper.Viper) error { MaxNumberToKeep: v.GetInt("fs.versioning.max_number_of_versions_to_keep"), MinDelayBetweenTwoVersions: v.GetDuration("fs.versioning.min_delay_between_two_versions"), }, - Contexts: v.GetStringMap("fs.contexts"), + Contexts: v.GetStringMap("fs.contexts"), + MigrationTarget: migrationTarget, }, CouchDB: couch, Jobs: jobs, diff --git a/pkg/config/config/s3.go b/pkg/config/config/s3.go index 506fd57afe2..0cf7a1e729a 100644 --- a/pkg/config/config/s3.go +++ b/pkg/config/config/s3.go @@ -95,6 +95,14 @@ func GetS3Client() *minio.Client { return s3Client } +// HasS3Client reports whether the global S3 client has been initialized, +// without panicking. Callers that need to guard against a missing S3 +// connection (e.g. before attempting a storage migration to S3) should use +// this instead of recovering from GetS3Client's panic. +func HasS3Client() bool { + return s3Client != nil +} + // GetS3BucketPrefix returns the configured bucket prefix. func GetS3BucketPrefix() string { return s3BucketPrefix diff --git a/pkg/config/config/s3_target_test.go b/pkg/config/config/s3_target_test.go new file mode 100644 index 00000000000..afffacd8b3f --- /dev/null +++ b/pkg/config/config/s3_target_test.go @@ -0,0 +1,42 @@ +package config + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMigrationTargetInitsS3WhenGlobalIsSwift(t *testing.T) { + // A minimal fake S3 endpoint: InitS3Connection only needs a successful + // ListBuckets call (a signed GET on "/") to consider the connection live; + // bucket-creation failures are only logged, so any other response is fine. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/" { + w.Header().Set("Content-Type", "application/xml") + fmt.Fprint(w, ` + + testtest + +`) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + endpoint := srv.Listener.Addr().String() + + swiftURL, _ := url.Parse("swift://openstack/") + s3URL, _ := url.Parse(fmt.Sprintf("s3://%s/?access_key=key&secret_key=secret&bucket_prefix=cozy&use_ssl=false", endpoint)) + config = &Config{Fs: Fs{URL: swiftURL, MigrationTarget: s3URL}} + + require.True(t, HasS3Target()) + // Init the S3 globals from the target even though the global scheme is swift. + require.NoError(t, InitS3Connection(Fs{URL: MigrationTargetURL()})) + assert.NotNil(t, GetS3Client()) + assert.Equal(t, "cozy", GetS3BucketPrefix()) +} diff --git a/pkg/config/config/swift.go b/pkg/config/config/swift.go index 5d7a83a23ca..66c336e2ae8 100644 --- a/pkg/config/config/swift.go +++ b/pkg/config/config/swift.go @@ -102,3 +102,12 @@ func GetSwiftConnection() *swift.Connection { } return swiftConn } + +// HasSwiftConnection reports whether the global swift connection has been +// initialized, without panicking. Callers that need to guard against a +// missing swift connection (e.g. before attempting a storage migration to +// swift) should use this instead of recovering from GetSwiftConnection's +// panic. +func HasSwiftConnection() bool { + return swiftConn != nil +} diff --git a/web/instances/instances.go b/web/instances/instances.go index 4e716c3aea0..1778ad96a95 100644 --- a/web/instances/instances.go +++ b/web/instances/instances.go @@ -15,6 +15,7 @@ import ( "github.com/cozy/cozy-stack/model/app" "github.com/cozy/cozy-stack/model/instance" "github.com/cozy/cozy-stack/model/instance/lifecycle" + "github.com/cozy/cozy-stack/model/instance/storagemigration" "github.com/cozy/cozy-stack/model/notification" "github.com/cozy/cozy-stack/model/notification/center" "github.com/cozy/cozy-stack/model/oauth" @@ -300,6 +301,26 @@ func deleteHandler(c echo.Context) error { return c.NoContent(http.StatusNoContent) } +func migrateStorageHandler(c echo.Context) error { + domain := c.Param("domain") + inst, err := lifecycle.GetInstance(domain) + if err != nil { + return wrapError(err) + } + opts := storagemigration.Options{ + To: c.QueryParam("to"), + DryRun: c.QueryParam("dry_run") == "true", + FlagOnly: c.QueryParam("flag_only") == "true", + Force: c.QueryParam("force") == "true", + PurgeSource: c.QueryParam("purge_source") == "true", + } + rep, err := storagemigration.Migrate(inst, opts) + if err != nil { + return wrapError(err) + } + return c.JSON(http.StatusOK, rep) +} + func setAuthMode(c echo.Context) error { domain := c.Param("domain") inst, err := lifecycle.GetInstance(domain) @@ -785,6 +806,9 @@ func Routes(router *echo.Group) { router.GET("/contexts/:name", showContext) router.GET("/with-app-version/:slug/:version", appVersion) + // Storage migration + router.POST("/:domain/migrate-storage", migrateStorageHandler) + // Checks router.GET("/:domain/fsck", fsckHandler) router.POST("/:domain/checks/triggers", checkTriggers) diff --git a/web/instances/instances_test.go b/web/instances/instances_test.go index e6bc473bb4c..4b1cb6cd523 100644 --- a/web/instances/instances_test.go +++ b/web/instances/instances_test.go @@ -97,4 +97,27 @@ func TestInstances(t *testing.T) { attrs.HasValue("feature_sets", []string{"71df3022-abd9-11ee-b79b-9cb6d0907fa3", "790789f8-abd9-11ee-ae09-9cb6d0907fa3"}) }) }) + + t.Run("MigrateStorage", func(t *testing.T) { + domain := "migrate-storage.cozy.localhost" + t.Cleanup(func() { _ = lifecycle.Destroy(domain) }) + + e := testutils.CreateTestClient(t, ts.URL) + + e.POST("/instances"). + WithQuery("Domain", domain). + WithQuery("Locale", "en"). + WithQuery("SwiftLayout", "-1"). + WithHeader("Authorization", "Bearer "+token). + Expect().Status(201) + + // An unsupported target scheme is rejected by the Migrate guard + // before touching any storage backend, so this exercises the route + // and error propagation without requiring a live S3/Swift target. + e.POST("/instances/"+domain+"/migrate-storage"). + WithQuery("to", "not-a-scheme"). + WithQuery("dry_run", "true"). + WithHeader("Authorization", "Bearer "+token). + Expect().Status(500) + }) }