scratch: #114 + compile-before-timed-tests (do not merge)

.env.example

@@ -55,6 +55,16 @@ FROM_EMAIL=noreply@example.com
 # SECRETS_REFRESH_INTERVAL=60s
 # SECRETS_MAX_STALENESS=15m
 
+# Optional object storage + image uploads (docs/storage.md): off unless all
+# four STORAGE_S3_* secrets are set (live secrets: from Doppler with
+# SECRETS_PROVIDER=doppler). Region and path style are plain env.
+# STORAGE_S3_ENDPOINT=http://127.0.0.1:9000
+# STORAGE_S3_BUCKET=app
+# STORAGE_S3_ACCESS_KEY_ID=
+# STORAGE_S3_SECRET_ACCESS_KEY=
+# STORAGE_S3_REGION=us-east-1     (empty: auto, for Railway and R2)
+# STORAGE_S3_PATH_STYLE=true      (MinIO)
+
 # make derives a compose project and ports per checkout so parallel worktrees
 # never collide; `make env` prints them. Override only if you need fixed values.
 # COMPOSE_PROJECT_NAME=myapp

.golangci.yml

@@ -78,6 +78,8 @@ linters:
               desc: transports call services (use services.User etc.), never repositories or sqlc
             - pkg: github.com/jayleaton/go-fast-stack/packages/secrets
               desc: secrets reach consumers through config.Secrets accessors wired in cmd/server, never the store itself
+            - pkg: github.com/jayleaton/go-fast-stack/packages/storage
+              desc: transports call services; object storage sits behind the services' ports
         services:
           files:
             - "**/apps/web/internal/services/**"
@@ -102,6 +104,8 @@ linters:
               desc: route access is the transports' coarse gate; services get a services.Actor and decide record-level authorization themselves
             - pkg: github.com/jayleaton/go-fast-stack/packages/secrets
               desc: services declare a port for what a secret enables (a client); cmd/server wires it from config.Secrets
+            - pkg: github.com/jayleaton/go-fast-stack/packages/storage
+              desc: services declare ports (ObjectStore, ImageFetcher, ImageCodec); adapters in apps/web/internal/server wrap packages/storage
         database:
           files:
             - "**/packages/database/**"
@@ -128,6 +132,21 @@ linters:
               desc: packages/secrets is self-contained
             - pkg: go.uber.org/mock
               desc: secrets tests use fake providers over httptest, never mocks
+        # Object storage (packages/storage: S3 client, safefetch, imaging) is
+        # optional and deletable: it imports no app and none of the data,
+        # auth or secrets packages; cmd reveals credentials and passes strings.
+        storage:
+          files:
+            - "**/packages/storage/**"
+          deny:
+            - pkg: github.com/jayleaton/go-fast-stack/apps
+              desc: packages/storage is self-contained; apps/web/internal/server adapts it to the services' ports
+            - pkg: github.com/jayleaton/go-fast-stack/packages/database
+              desc: packages/storage is self-contained
+            - pkg: github.com/jayleaton/go-fast-stack/packages/auth
+              desc: packages/storage is self-contained
+            - pkg: github.com/jayleaton/go-fast-stack/packages/secrets
+              desc: packages/storage takes plain strings revealed in apps/*/cmd
         # Credentials travel as secrets.Secret (prints "[redacted]"), which has
         # no method returning the value; reveal.Value is the only way to read
         # it (package secrets/internal/access keeps it that way). Only the cmd

AGENTS.md

@@ -222,6 +222,7 @@ Exit codes:
 - `apps/web/internal/services`: business rules and the ports they need; `services/mocks` (generated) for service tests. `apps/web/internal/validate`: input types.
 - `apps/web/ui`: templ pages, Datastar handlers (server actions that call `apiclient.Client`), i18n, static assets, and the browser security headers (`security.go`).
 - `packages/uikit`: the templ components and design tokens both UIs share (`theme.css` is the one file to re-theme). It renders translated strings and values only; depguard keeps it from importing `apps/` or the data, auth and secrets packages.
+- `packages/storage`: the optional S3 client, `safefetch` (SSRF-safe fetch of user URLs, rules injected from `services.PublicAddr`/`FetchURLAllowed`) and `imaging` (pure-Go decode/normalise to WebP; build with `-tags nodynamic`). Services reach it only through ports (`ObjectStore`, `ImageFetcher`, `ImageCodec`; depguard). [docs/storage.md](docs/storage.md).
 - `apps/backup`, `packages/backup`: the nightly encrypted Postgres backup and restore ([docs/backups.md](docs/backups.md)).
 - `packages/database`: pool (`Pool`: swaps itself when `DATABASE_URL` rotates), `migrations/` (goose, embedded), `queries/` (sqlc input), `sqlc/` (generated, never edit), repositories, `model/`.
 - `packages/secrets`: live secrets: `Manager` (polling, validation, last-good cache, staleness, change hooks) and providers (`Env`, `Doppler`; Infisical/1Password stubs behind build tags), `Secret`, and the redacting log handler. Self-contained; never reads the env.

Dockerfile

@@ -8,7 +8,11 @@ WORKDIR /src
 COPY go.mod go.sum ./
 RUN go mod download
 COPY . .
-RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/server ./apps/web/cmd/server
+# -tags nodynamic keeps the image codecs from linking purego (a dynamic
+# binary, which distroless/static cannot run); the check fails the build if
+# the tag is ever dropped.
+RUN CGO_ENABLED=0 go build -tags nodynamic -trimpath -ldflags='-s -w' -o /out/server ./apps/web/cmd/server \
+ && go version -m /out/server | grep -q -- '-tags=nodynamic'
 
 FROM gcr.io/distroless/static-debian13:nonroot
 COPY --from=build /out/server /server

Makefile

@@ -38,9 +38,14 @@ help: ## List targets
 dev: ## Run the server (0.0.0.0, PORT or 3000) with .env (USE_DOPPLER=1: secrets from Doppler's dev config)
 	$(if $(filter 1,$(USE_DOPPLER)),doppler run --) $(GO) run ./apps/web/cmd/server serve
 
+# nodynamic: the pure-Go image codecs (packages/storage/imaging) would
+# otherwise link purego to try a system libwebp/libavif, and the binary
+# would come out dynamically linked even with CGO_ENABLED=0.
+STATIC_TAGS := nodynamic
+
 .PHONY: build
 build: ## Build a static binary into bin/server
-	CGO_ENABLED=0 $(GO) build -trimpath -ldflags='-s -w' -o $(BIN) ./apps/web/cmd/server
+	CGO_ENABLED=0 $(GO) build -tags $(STATIC_TAGS) -trimpath -ldflags='-s -w' -o $(BIN) ./apps/web/cmd/server
 
 .PHONY: generate
 generate: ## Regenerate sqlc, go:generate (mocks) and templ (shared uikit + web + admin UIs)

README.md

@@ -32,13 +32,13 @@ Pages are rendered on the server in Go, and [Datastar](https://data-star.dev) ad
 
 | | Measured |
 |---|---|
-| Web server | one static binary, **16.5 MB** (no runtime, no `node_modules`) |
+| Web server | one static binary, **24.5 MB** (no runtime, no `node_modules`) |
 | Client JavaScript | one file, Datastar: **34 KB** (13 KB gzipped) |
 | Memory | **83 MB** right after start, 93 MB after 200 page loads, on a dev machine. On Railway the idle server uses about 75 MB. |
 | Home page | 17 KB of HTML, rendered in about 0.4 ms |
-| Production image | distroless, about 5 MB |
+| Production image | distroless, about 11 MB compressed (0.9 MB base + the binary) |
 
-The binary size, memory and page numbers come from a production build (`-trimpath -ldflags='-s -w'`, Go 1.26.8) at this PR's base. The Railway figure is from the [deploy cost notes](docs/deploy.md#cost).
+7.6 MB of the binary is the pure-Go image codecs used by the optional image uploads ([docs/storage.md](docs/storage.md#pure-go-static-binary)). The binary and image sizes come from a production build (`make build`: `-tags nodynamic -trimpath -ldflags='-s -w'`, Go 1.26.8, linux/amd64); the memory and page numbers come from the same kind of build before the codecs were added. The Railway figure is from the [deploy cost notes](docs/deploy.md#cost).
 
 **Fast CI and fast tests.** CI checks only the packages a change can affect, in one job.
 
@@ -113,6 +113,7 @@ To deploy, run `make railway-setup`. It creates the whole Railway project from [
 - [Security](docs/security.md): what the browser can and can't reach, headers, cookies
 - [Google and Apple sign-in (optional)](docs/sign-in.md): provider credentials, turning it on, deleting it
 - [Admin dashboard (optional)](docs/admin.md): user management and the audit log
+- [Object storage (optional)](docs/storage.md): S3/R2/Railway buckets, safe image uploads and imports by URL
 - [Mobile app](apps/mobile-app/README.md) · [Auth](packages/auth/README.md) · [UI kit](packages/uikit/README.md)
 - [AGENTS.md](AGENTS.md) (rules for contributors and agents) · [CONTEXT.md](CONTEXT.md) (vocabulary)
 

apps/web/cmd/server/main.go

@@ -119,6 +119,10 @@ func serve(ctx context.Context, cfg config.Config, log *slog.Logger) error {
 	gate := access.NewGate(server.Callers)
 	usersAPI := api.NewUsers(users, gate)
 	socialAPI := api.NewSocial(services.NewSocialSignInService(server.NewSocialAuth(authn)), gate)
+	images, err := newImages(ctx, cfg, gate, log)
+	if err != nil {
+		return err
+	}
 	handler := server.New(server.Deps{
 		Config:  cfg,
 		Log:     log,
@@ -127,6 +131,7 @@ func serve(ctx context.Context, cfg config.Config, log *slog.Logger) error {
 		Auth:    authn,
 		Users:   usersAPI,
 		Social:  socialAPI,
+		Images:  images,
 		UI: ui.New(ui.Deps{
 			API: api.NewClient(usersAPI, api.NewSessions(server.NewAuthenticator(authn), gate)).WithSocial(socialAPI), Gate: gate, Features: cfg.Features, AppName: cfg.AppName,
 			SecureCookies: cfg.PublicURL.Scheme == "https", Logger: log,

apps/web/cmd/server/storage.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+
+	"github.com/jayleaton/go-fast-stack/apps/web/internal/access"
+	"github.com/jayleaton/go-fast-stack/apps/web/internal/api"
+	"github.com/jayleaton/go-fast-stack/apps/web/internal/config"
+	"github.com/jayleaton/go-fast-stack/apps/web/internal/server"
+	"github.com/jayleaton/go-fast-stack/apps/web/internal/services"
+	"github.com/jayleaton/go-fast-stack/packages/secrets/reveal"
+	"github.com/jayleaton/go-fast-stack/packages/storage"
+	"github.com/jayleaton/go-fast-stack/packages/storage/imaging"
+)
+
+// imageConcurrency bounds how many images are decoded at once. Measured peak
+// RSS above an idle server: about 30 MB for one 12 MP photo, 70 MB for one
+// 48 MP photo (the decoded pixels and the WebP encoder; resizing uses at most
+// 8 MiB), 120 MB for two 12 MP photos at once (docs/storage.md).
+const imageConcurrency = 2
+
+// newImages builds the image endpoints over object storage, or returns nil
+// (routes not mounted) when STORAGE_S3_BUCKET is unset. A bucket that the
+// keys cannot reach fails boot, like a database that cannot be reached.
+// docs/storage.md.
+func newImages(ctx context.Context, cfg config.Config, gate *access.Gate, log *slog.Logger) (*api.Images, error) {
+	creds, err := cfg.Secrets.Storage()
+	if err != nil {
+		return nil, err
+	}
+	if !creds.Configured() {
+		log.Info("storage: off (STORAGE_S3_BUCKET not set); image routes not mounted")
+		return nil, nil
+	}
+	client, err := storage.New(storageConfig(cfg, creds))
+	if err != nil {
+		return nil, err
+	}
+	probeCtx, cancel := context.WithTimeout(ctx, rotationTimeout)
+	defer cancel()
+	if err := client.Probe(probeCtx); err != nil {
+		return nil, fmt.Errorf("storage: bucket probe failed (%s): %w", creds.Fingerprint(), err)
+	}
+	log.Info("storage: bucket reachable", "storage", creds.Fingerprint())
+	svc := services.NewImageService(
+		server.NewObjectStore(client),
+		server.NewImageFetcher(server.NewSafeFetch(cfg.AppName)),
+		server.NewImageCodec(imaging.New(imageConcurrency)),
+	)
+	return api.NewImages(svc, gate), nil
+}
+
+func storageConfig(cfg config.Config, credentials config.StorageCredentials) storage.Config {
+	return storage.Config{
+		Endpoint:        reveal.Value(credentials.Endpoint),
+		Bucket:          reveal.Value(credentials.Bucket),
+		AccessKeyID:     reveal.Value(credentials.AccessKeyID),
+		SecretAccessKey: reveal.Value(credentials.SecretAccessKey),
+		Region:          cfg.Storage.Region,
+		PathStyle:       cfg.Storage.PathStyle,
+	}
+}

apps/web/internal/api/images.go

@@ -0,0 +1,80 @@
+package api
+
+//go:generate go tool mockgen -source=images.go -destination=mocks/images.go -package=mocks
+
+import (
+	"context"
+	"io"
+	"net/http"
+
+	"github.com/jayleaton/go-fast-stack/apps/web/internal/access"
+	"github.com/jayleaton/go-fast-stack/apps/web/internal/config"
+	"github.com/jayleaton/go-fast-stack/apps/web/internal/services"
+	"github.com/jayleaton/go-fast-stack/apps/web/internal/validate"
+)
+
+// ImageService is the port these endpoints call (*services.ImageService).
+type ImageService interface {
+	Upload(ctx context.Context, actor services.Actor, data []byte) (services.Image, error)
+	Import(ctx context.Context, actor services.Actor, in validate.CreateImageFromURL) (services.Image, error)
+	Open(ctx context.Context, actor services.Actor, id string) (services.ImageFile, error)
+	Delete(ctx context.Context, actor services.Actor, id string) error
+}
+
+var _ ImageService = (*services.ImageService)(nil)
+
+// Images serves the signed-in user's images under /api/core/v1/images. The
+// server mounts it only when object storage is configured. Import and delete
+// are Endpoints (JSON in, envelope out); the upload takes the raw image as
+// its body and the download streams the file, so those two are routes.
+type Images struct {
+	svc  ImageService
+	gate *access.Gate
+
+	importURL Endpoint[validate.CreateImageFromURL, services.Image]
+	delete    Endpoint[NoInput, Success]
+}
+
+// NewImages returns the image endpoints.
+func NewImages(svc ImageService, gate *access.Gate) *Images {
+	images := &Images{svc: svc, gate: gate}
+	routes := config.APIRoutes
+	images.importURL = Endpoint[validate.CreateImageFromURL, services.Image]{"POST", routes.ImageImport, access.SignedIn, images.importFrom}
+	images.delete = Endpoint[NoInput, Success]{"DELETE", routes.ImageByID, access.SignedIn, images.deleteOne}
+	return images
+}
+
+// Routes registers the endpoints on mux.
+func (images *Images) Routes(mux *http.ServeMux) {
+	routes := config.APIRoutes
+	mux.Handle("POST "+routes.Images, RouteLimit(images.gate, access.SignedIn, services.MaxImageBytes, images.upload))
+	mux.Handle(images.importURL.Pattern(), images.importURL.Serve(images.gate))
+	mux.Handle("GET "+routes.ImageByID, RouteFile(images.gate, access.SignedIn, images.get))
+	mux.Handle(images.delete.Pattern(), images.delete.Serve(images.gate))
+}
+
+// upload takes the image as the raw request body (any Content-Type: the
+// service sniffs the bytes).
+func (images *Images) upload(request *http.Request, caller access.Caller) (services.Image, error) {
+	data, err := io.ReadAll(request.Body)
+	if isTooLarge(err) {
+		return services.Image{}, services.ErrImageTooLarge
+	} else if err != nil {
+		return services.Image{}, err
+	}
+	return images.svc.Upload(request.Context(), caller.Actor(), data)
+}
+
+func (images *Images) importFrom(request *http.Request, caller access.Caller, in validate.CreateImageFromURL) (services.Image, error) {
+	return images.svc.Import(request.Context(), caller.Actor(), in)
+}
+
+func (images *Images) get(request *http.Request, caller access.Caller) (File, error) {
+	file, err := images.svc.Open(request.Context(), caller.Actor(), request.PathValue("id"))
+	return File(file), err
+}
+
+func (images *Images) deleteOne(request *http.Request, caller access.Caller, _ NoInput) (Success, error) {
+	err := images.svc.Delete(request.Context(), caller.Actor(), request.PathValue("id"))
+	return Success{err == nil}, err
+}