feat(s3)!: support explicit bucket layouts

cozy.example.yaml

@@ -67,7 +67,20 @@ fs:
 
   # url: file://localhost/var/lib/cozy
   # url: swift://openstack/?UserName={{ .Env.OS_USERNAME }}&Password={{ .Env.OS_PASSWORD }}&ProjectName={{ .Env.OS_PROJECT_NAME }}&UserDomainName={{ .Env.OS_USER_DOMAIN_NAME }}&Timeout={{ .Env.GOSWIFT_TIMEOUT }}
-  # url: s3://{{ .Env.S3_ENDPOINT }}?access_key={{ .Env.S3_ACCESS_KEY }}&secret_key={{ .Env.S3_SECRET_KEY }}&region={{ .Env.S3_REGION }}&bucket_prefix=cozy&use_ssl=true
+  # url: s3://{{ .Env.S3_ENDPOINT }}?access_key={{ .Env.S3_ACCESS_KEY }}&secret_key={{ .Env.S3_SECRET_KEY }}&region={{ .Env.S3_REGION }}&use_ssl=true
+
+  # S3 buckets are created automatically by default. See docs/s3.md.
+  # To use pre-created buckets and optional connections per concrete bucket:
+  # s3:
+  #   auto_create_buckets: false
+  #   buckets:
+  #     default:
+  #       name: company-storage
+  #     files:
+  #       name: company-files
+  #       url: s3://{{ .Env.S3_VFS_ENDPOINT }}?access_key={{ .Env.S3_VFS_ACCESS_KEY }}&secret_key={{ .Env.S3_VFS_SECRET_KEY }}&region={{ .Env.S3_REGION }}
+  #     exports:
+  #       name: company-backups
 
   # Swift FS can be used with advanced parameters to activate TLS properties.
   # For using swift with https, you must use the "swift+https" scheme.

docs/s3.md

@@ -9,24 +9,98 @@ Scaleway, etc.) and does not depend on the AWS SDK.
 
 ## Configuration
 
-The S3 backend is configured via the `fs.url` parameter using the `s3://`
-scheme. All connection parameters are passed as query parameters:
+Set `fs.url` to the default S3 connection and configure destinations in
+`fs.s3.buckets`. Each entry contains a bucket `name` and an optional `url`.
+Bucket names are never generated from prefixes or organization IDs.
 
 ```yaml
 fs:
-  url: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET&region=rbx&bucket_prefix=cozy&use_ssl=true
+  url: s3://s3.example.net?access_key=ACCESS&secret_key=SECRET&region=gra
+  s3:
+    buckets:
+      default:
+        name: company-storage
 ```
 
-| Parameter       | Description                          | Default |
-|-----------------|--------------------------------------|---------|
-| `access_key`    | S3 access key ID                     | —       |
-| `secret_key`    | S3 secret access key                 | —       |
-| `region`        | S3 region                            | —       |
-| `bucket_prefix` | Prefix for all bucket names          | `cozy`  |
-| `use_ssl`       | Use HTTPS for S3 connections         | `true`  |
+This stores everything in one bucket, with separate object paths for each
+storage type. All organizations share the files bucket; each instance has
+its own path beneath `files/`.
 
-The host part of the URL is the S3 endpoint (e.g. `s3.rbx.io.cloud.ovh.net`
-for OVH, `localhost:9000` for MinIO).
+| URL parameter | Description | Default |
+|---------------|-------------|---------|
+| `access_key` | S3 access key ID | — |
+| `secret_key` | S3 secret access key | — |
+| `region` | Region for requests and bucket creation | SDK discovery/default |
+| `use_ssl` | Use HTTPS | `true` |
+
+URL-encode credentials containing reserved query characters. Specify `region`
+in every connection URL to avoid location discovery requests.
+
+### Separate or mixed buckets
+
+Configure a bucket name and connection together for each storage type:
+
+```yaml
+fs:
+  url: s3://s3.example.net?access_key=DEFAULT_KEY&secret_key=DEFAULT_SECRET&region=gra
+  s3:
+    auto_create_buckets: false
+    buckets:
+      default:
+        name: company-storage
+      files:
+        name: company-files
+        url: s3://files.example.net?access_key=FILES_KEY&secret_key=FILES_SECRET&region=rbx
+      exports:
+        name: company-backups
+```
+
+The supported types are `files`, `apps_web`, `apps_konnectors`, `assets`,
+`previews`, and `exports`. Types without an entry use the complete `default`
+entry. To use only separate destinations, omit `default` and specify all six
+types. Several types can use the same bucket name and URL.
+
+Every entry requires `name`. Its optional `url` supplies the endpoint,
+credentials, region and TLS setting; omitting it uses `fs.url`. An explicit
+entry does not inherit fields from `default`. Entries with the same URL reuse
+one connection, and each bucket on that connection is initialized once.
+The same bucket name can also be used on different connections.
+Copies between different connections stream through the stack.
+
+Invalid names, unknown storage types, missing destinations and invalid
+connection URLs cause startup errors. Configuration errors do not print
+connection credentials.
+
+### Bucket provisioning
+
+`auto_create_buckets` defaults to `true`: startup creates each distinct bucket
+once, using its connection's region. Existing buckets are accepted. Bucket
+creation requires `s3:CreateBucket`; bucket enumeration is not used.
+
+With `auto_create_buckets: false`, create the configured buckets before starting
+the stack. Startup checks each distinct bucket with `HeadBucket`, without listing
+or creating buckets. Missing or inaccessible buckets fail startup with a
+bucket-specific error. All startup bucket operations have a 30-second timeout.
+Loading or initializing an instance does not probe S3.
+
+`HeadBucket` requires bucket-scoped `s3:ListBucket` permission on AWS S3; see the
+[HeadBucket API documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadBucket.html).
+Runtime credentials also need the object operations used by the stack, including
+reads, writes, deletes and multipart uploads. Neither `s3:ListAllMyBuckets` nor
+`s3:CreateBucket` is required when creation is disabled. Dynamic asset health
+checks also use `HeadBucket` and honor shorter caller deadlines.
+
+Environment variables override YAML:
+
+- `COZY_FS_S3_BUCKETS_DEFAULT_NAME` sets the default bucket name.
+- `COZY_FS_S3_BUCKETS_FILES_NAME` and `COZY_FS_S3_BUCKETS_FILES_URL` configure
+  the files destination; the other types follow the same pattern, such as
+  `COZY_FS_S3_BUCKETS_APPS_WEB_NAME`.
+- `COZY_FS_S3_AUTO_CREATE_BUCKETS=false` disables creation; `true` enables it.
+
+Changing bucket assignments requires moving the corresponding objects while
+preserving their type and instance paths. The stack does not migrate data or
+read the former generated-bucket layout.
 
 ### Local development with MinIO
 
@@ -57,7 +131,11 @@ Edit your `~/.cozy/cozy.yaml`:
 
 ```yaml
 fs:
-  url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&bucket_prefix=cozy&use_ssl=false
+  url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&use_ssl=false
+  s3:
+    buckets:
+      default:
+        name: company-storage
 ```
 
 **3. Build and start:**
@@ -90,7 +168,7 @@ cozy-stack apps install home --domain your.domain.localhost:8080
 Check that objects appear in MinIO:
 
 ```bash
-docker exec minio mc ls --recursive local/cozy-apps-web/
+docker exec minio mc ls --recursive local/company-storage/apps-web/
 ```
 
 Upload a file via the Drive UI or the API:
@@ -107,7 +185,7 @@ curl -X POST \
 Verify the file is in MinIO:
 
 ```bash
-docker exec minio mc ls --recursive local/cozy-default/
+docker exec minio mc ls --recursive local/company-storage/files/
 ```
 
 **6. Switching back to local filesystem:**
@@ -116,50 +194,25 @@ Comment out the S3 URL in your config and restart cozy-stack:
 
 ```yaml
 fs:
-  # url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&bucket_prefix=cozy&use_ssl=false
+  # url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&use_ssl=false
 ```
 
 Note: files uploaded to S3 won't be accessible when using the local
 filesystem backend, and vice versa. Each backend has its own storage.
 
-## Bucket strategy
-
-### Design rationale
-
-Swift uses one container per instance (`cozy-v3-<DBPrefix>`). This doesn't
-scale well for S3 where bucket creation can be limited (AWS limits to 100
-buckets per account by default, OVH to 100 as well). Instead, the S3 backend
-uses a **shared bucket per organization** with **key prefixes per instance**.
+## Storage namespaces
 
-### Bucket naming
+The same object paths are used whether types share a bucket or use separate
+buckets. Every list and deletion is restricted to its type or instance path.
 
-Each bucket name is derived from the instance's `OrgID` field:
-
-```
-<bucket_prefix>-<sanitized_org_id>
-```
-
-- If `OrgID` is empty, `"default"` is used as fallback
-- The org ID is sanitized: lowercased, underscores/dots replaced by hyphens,
-  non-alphanumeric characters stripped, consecutive hyphens collapsed,
-  truncated to respect the 63-character S3 bucket name limit
-- Examples: `cozy-default`, `cozy-acme-corp`, `cozy-org-12345`
-
-### Dedicated buckets for secondary storage
-
-In addition to the main VFS bucket, the S3 backend uses dedicated buckets for
-other storage needs:
-
-| Bucket                        | Content                                 |
-|-------------------------------|-----------------------------------------|
-| `<prefix>-<orgId>`           | Main VFS data (files, versions)         |
-| `<prefix>-apps-web`          | Web application assets (drive, etc.)    |
-| `<prefix>-apps-konnectors`   | Konnector assets                        |
-| `<prefix>-assets`            | Dynamic assets                          |
-| `<prefix>-previews`          | PDF preview and icon cache              |
-| `<prefix>-exports`           | Instance export archives                |
-
-Buckets are created automatically on first use.
+| Storage type | Object prefix | Content |
+|--------------|---------------|---------|
+| `files` | `files/<DBPrefix>/` | Files, versions, avatars and thumbnails |
+| `apps_web` | `apps-web/` | Installed web applications, markers and tarballs |
+| `apps_konnectors` | `apps-konnectors/` | Installed konnectors, markers and tarballs |
+| `assets` | `assets/` | Dynamic assets, grouped by context |
+| `previews` | `previews/` | PDF previews and icons |
+| `exports` | `exports/` | Instance export archives, grouped by domain |
 
 ## Object key structure
 
@@ -169,32 +222,32 @@ from `DBPrefix()` (typically the instance domain or a CouchDB prefix).
 ### VFS files
 
 ```
-<DBPrefix>/<docID_part1>/<docID_part2>/<docID_part3>/<internalID>
+files/<DBPrefix>/<docID_part1>/<docID_part2>/<docID_part3>/<internalID>
 ```
 
 The document ID (a 32-character UUID v7 hex string) is split into virtual
 subfolders to avoid flat hierarchies:
 
 ```
-cozy218def.../019d35b1-9dc3-78ec-994d-f5/44336/7f1b6/e0AbCdEfGh123456
-              ^^^^^^^^^^^^^^^^^^^^^^       ^^^^^  ^^^^^  ^^^^^^^^^^^^^^^^
-              first 22 chars               5 ch   5 ch   16-char internalID
+files/cozy218def.../019d35b1-9dc3-78ec-994d-f5/44336/7f1b6/e0AbCdEfGh123456
+                    ^^^^^^^^^^^^^^^^^^^^^^       ^^^^^  ^^^^^  ^^^^^^^^^^^^^^^^
+                    first 22 chars               5 ch   5 ch   16-char internalID
 ```
 
 This structure mirrors the Swift V3 layout (`MakeObjectNameV3`).
 
 ### Thumbnails
 
 ```
-<DBPrefix>/thumbs/<docID_split>-<format>
+files/<DBPrefix>/thumbs/<docID_split>-<format>
 ```
 
 Formats: `small`, `medium`, `large`.
 
 ### Avatar
 
 ```
-<DBPrefix>/avatar
+files/<DBPrefix>/avatar
 ```
 
 ## Memory consumption
@@ -244,5 +297,8 @@ COZY_COUCHDB_URL=http://admin:admin@localhost:5984/ \
   go test ./model/vfs/ -run TestVfs -v -count=1 -timeout 300s
 
 # Run naming unit tests (no external deps)
-go test ./model/vfs/vfss3/ -run "TestSanitize|TestBucketName|TestMakeObjectKey|TestMakeDocID" -v
+go test ./model/vfs/vfss3/ -run "TestMakeObjectKey|TestMakeDocID" -v
+
+# Exercise all storage types in one bucket (requires Docker, no CouchDB)
+go test ./model/move/ -run TestS3SharedStorage -v -count=1 -timeout 2m
 ```

model/app/apps.go

@@ -157,9 +157,8 @@ func Copier(appsType consts.AppType, inst *instance.Instance) appfs.Copier {
 	case config.SchemeSwift, config.SchemeSwiftSecure:
 		return appfs.NewSwiftCopier(config.GetSwiftConnection(), appsType)
 	case config.SchemeS3:
-		client := config.GetS3Client()
-		bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), appsType)
-		return appfs.NewS3Copier(client, bucket)
+		storage := config.GetS3Storage(appfs.S3AppsStorageType(appsType))
+		return appfs.NewS3Copier(storage.Client, storage.Bucket, storage.Prefix)
 	default:
 		panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme))
 	}
@@ -180,9 +179,8 @@ func AppsFileServer(i *instance.Instance) appfs.FileServer {
 	case config.SchemeSwift, config.SchemeSwiftSecure:
 		return appfs.NewSwiftFileServer(config.GetSwiftConnection(), consts.WebappType)
 	case config.SchemeS3:
-		client := config.GetS3Client()
-		bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), consts.WebappType)
-		return appfs.NewS3FileServer(client, bucket)
+		storage := config.GetS3Storage(appfs.S3AppsStorageType(consts.WebappType))
+		return appfs.NewS3FileServer(storage.Client, storage.Bucket, storage.Prefix)
 	default:
 		panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme))
 	}
@@ -203,9 +201,8 @@ func KonnectorsFileServer(i *instance.Instance) appfs.FileServer {
 	case config.SchemeSwift, config.SchemeSwiftSecure:
 		return appfs.NewSwiftFileServer(config.GetSwiftConnection(), consts.KonnectorType)
 	case config.SchemeS3:
-		client := config.GetS3Client()
-		bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), consts.KonnectorType)
-		return appfs.NewS3FileServer(client, bucket)
+		storage := config.GetS3Storage(appfs.S3AppsStorageType(consts.KonnectorType))
+		return appfs.NewS3FileServer(storage.Client, storage.Bucket, storage.Prefix)
 	default:
 		panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme))
 	}

model/instance/instance.go

@@ -294,10 +294,9 @@ func (i *Instance) AvatarFS() vfs.Avatarer {
 			panic(ErrInvalidSwiftLayout)
 		}
 	case config.SchemeS3:
-		client := config.GetS3Client()
-		bucket := vfss3.BucketName(i.GetOrgID(), config.GetS3BucketPrefix())
-		keyPrefix := i.DBPrefix() + "/"
-		return vfss3.NewAvatarFs(client, bucket, keyPrefix)
+		storage := config.GetS3Storage("files")
+		keyPrefix := storage.Prefix + i.DBPrefix() + "/"
+		return vfss3.NewAvatarFs(storage.Client, storage.Bucket, keyPrefix)
 	default:
 		panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme))
 	}
@@ -323,10 +322,9 @@ func (i *Instance) ThumbsFS() vfs.Thumbser {
 			panic(ErrInvalidSwiftLayout)
 		}
 	case config.SchemeS3:
-		client := config.GetS3Client()
-		bucket := vfss3.BucketName(i.GetOrgID(), config.GetS3BucketPrefix())
-		keyPrefix := i.DBPrefix() + "/"
-		return vfss3.NewThumbsFs(client, bucket, keyPrefix)
+		storage := config.GetS3Storage("files")
+		keyPrefix := storage.Prefix + i.DBPrefix() + "/"
+		return vfss3.NewThumbsFs(storage.Client, storage.Bucket, keyPrefix)
 	default:
 		panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme))
 	}