feat(proxy): scan url-encoded phantom tokens in form bodies

internal/proxy/addon.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"net/url"
 	"runtime/debug"
 	"sort"
 	"strconv"
@@ -641,14 +642,14 @@ func (a *SluiceAddon) Request(f *mitmproxy.Flow) {
 	}
 
 	// Pass 2+3 on URL query.
-	if rawQ := f.Request.URL.RawQuery; bytes.Contains([]byte(rawQ), phantomPrefix) {
+	if rawQ := f.Request.URL.RawQuery; bytesContainsAnyPhantomPrefix([]byte(rawQ)) {
 		f.Request.URL.RawQuery = string(
 			a.swapPhantomBytes([]byte(rawQ), pairs, host, port, "URL query"),
 		)
 	}
 
 	// Pass 2+3 on URL path.
-	if rawP := f.Request.URL.Path; bytes.Contains([]byte(rawP), phantomPrefix) {
+	if rawP := f.Request.URL.Path; bytesContainsAnyPhantomPrefix([]byte(rawP)) {
 		f.Request.URL.Path = string(
 			a.swapPhantomBytes([]byte(rawP), pairs, host, port, "URL path"),
 		)
@@ -1211,41 +1212,75 @@ func releasePhantomPairs(pairs []phantomPair) {
 // hasPhantomPrefix checks whether the request body, headers, or URL
 // contain the phantom prefix bytes.
 func (a *SluiceAddon) hasPhantomPrefix(f *mitmproxy.Flow) bool {
-	if bytes.Contains(f.Request.Body, phantomPrefix) {
+	if bytesContainsAnyPhantomPrefix(f.Request.Body) {
 		return true
 	}
 	for _, vals := range f.Request.Header {
 		for _, v := range vals {
-			if bytes.Contains([]byte(v), phantomPrefix) {
+			if bytesContainsAnyPhantomPrefix([]byte(v)) {
 				return true
 			}
 		}
 	}
-	if bytes.Contains([]byte(f.Request.URL.RawQuery), phantomPrefix) {
+	if bytesContainsAnyPhantomPrefix([]byte(f.Request.URL.RawQuery)) {
 		return true
 	}
-	if bytes.Contains([]byte(f.Request.URL.Path), phantomPrefix) {
+	if bytesContainsAnyPhantomPrefix([]byte(f.Request.URL.Path)) {
 		return true
 	}
 	return false
 }
 
+// bytesContainsAnyPhantomPrefix reports whether the data contains either the
+// literal or URL-encoded phantom prefix. Form-urlencoded request bodies and
+// URL query/path components percent-encode the colon in phantom tokens, so a
+// scan that only checks the literal form would miss phantoms about to leak
+// through OAuth refresh POSTs and other form-encoded paths.
+func bytesContainsAnyPhantomPrefix(data []byte) bool {
+	return bytes.Contains(data, phantomPrefix) || bytes.Contains(data, urlEncodedPhantomPrefix)
+}
+
 // swapPhantomBytes performs Pass 2 (scoped replacement) and Pass 3 (strip
 // unbound) on a byte slice.
+//
+// Each pair is matched in both its literal form (`SLUICE_PHANTOM:<name>`,
+// the shape used in JSON bodies and raw header values) and its URL-encoded
+// form (`SLUICE_PHANTOM%3A<name>`, the shape used in
+// application/x-www-form-urlencoded request bodies and URL components). The
+// encoded path is what makes OAuth refresh round-trips work: refresh POSTs
+// to providers like Anthropic and Google use form-urlencoded bodies, so the
+// colon in the phantom token gets percent-encoded on the wire. Without the
+// encoded scan the upstream receives `SLUICE_PHANTOM%3A...` literally,
+// returns `invalid_grant`, and the agent falls back to a fresh interactive
+// OAuth — every time tokens expire.
+//
+// The literal pass runs first so JSON request bodies (where the colon
+// survives unencoded) take the cheap exact-substring path. The encoded
+// pass then handles the form-urlencoded case. We url-encode the secret
+// before substitution so the wire body remains well-formed
+// application/x-www-form-urlencoded data.
 func (a *SluiceAddon) swapPhantomBytes(data []byte, pairs []phantomPair, host string, port int, location string) []byte {
 	for _, p := range pairs {
 		if bytes.Contains(data, p.phantom) {
 			data = bytes.ReplaceAll(data, p.phantom, p.secret.Bytes())
 		}
+		encoded := []byte(url.QueryEscape(string(p.phantom)))
+		if !bytes.Equal(encoded, p.phantom) && bytes.Contains(data, encoded) {
+			data = bytes.ReplaceAll(data, encoded, []byte(url.QueryEscape(string(p.secret.Bytes()))))
+		}
 	}
-	if bytes.Contains(data, phantomPrefix) {
+	if bytes.Contains(data, phantomPrefix) || bytes.Contains(data, urlEncodedPhantomPrefix) {
 		data = stripUnboundPhantomsFromProvider(data, a.provider)
 		log.Printf("[ADDON-INJECT] stripped unbound phantom token from %s for %s:%d", location, host, port)
 	}
 	return data
 }
 
 // swapPhantomHeaders performs Pass 2+3 on all request headers.
+//
+// Each pair is matched in both its literal and URL-encoded forms so phantom
+// tokens carried in percent-encoded header values (custom cookie schemes,
+// query-style header payloads) cannot bypass the swap.
 func (a *SluiceAddon) swapPhantomHeaders(f *mitmproxy.Flow, pairs []phantomPair, host string, port int) {
 	for key, vals := range f.Request.Header {
 		for i, v := range vals {
@@ -1256,8 +1291,13 @@ func (a *SluiceAddon) swapPhantomHeaders(f *mitmproxy.Flow, pairs []phantomPair,
 					vb = bytes.ReplaceAll(vb, p.phantom, p.secret.Bytes())
 					changed = true
 				}
+				encoded := []byte(url.QueryEscape(string(p.phantom)))
+				if !bytes.Equal(encoded, p.phantom) && bytes.Contains(vb, encoded) {
+					vb = bytes.ReplaceAll(vb, encoded, []byte(url.QueryEscape(string(p.secret.Bytes()))))
+					changed = true
+				}
 			}
-			if bytes.Contains(vb, phantomPrefix) {
+			if bytesContainsAnyPhantomPrefix(vb) {
 				vb = stripUnboundPhantomsFromProvider(vb, a.provider)
 				changed = true
 				log.Printf("[ADDON-INJECT] stripped unbound phantom token from header %q for %s:%d", key, host, port)
@@ -1285,15 +1325,24 @@ type phantomSwapReader struct {
 // maxPhantomLen returns the length of the longest phantom token in the
 // pairs list. Used to determine how much data to hold back from the
 // output buffer to handle tokens that span read boundaries.
+//
+// The result accounts for both literal phantom tokens (SLUICE_PHANTOM:name)
+// and their URL-encoded forms (SLUICE_PHANTOM%3Aname). The encoded form is
+// strictly longer because the colon expands to %3A, so a holdback sized for
+// the literal form alone would lose URL-encoded phantoms that straddle a
+// read boundary.
 func maxPhantomLen(pairs []phantomPair) int {
 	m := 0
 	for _, p := range pairs {
 		if len(p.phantom) > m {
 			m = len(p.phantom)
 		}
+		if encLen := len(url.QueryEscape(string(p.phantom))); encLen > m {
+			m = encLen
+		}
 	}
 	// Also account for the generic phantom prefix pattern.
-	if pLen := len(phantomPrefix) + maxCredNameLen; pLen > m {
+	if pLen := len(urlEncodedPhantomPrefix) + maxCredNameLen; pLen > m {
 		m = pLen
 	}
 	return m
@@ -1340,14 +1389,18 @@ func (r *phantomSwapReader) Read(p []byte) (int, error) {
 		toProcess := r.pending[:safe]
 		r.pending = append([]byte(nil), r.pending[safe:]...)
 
-		// Pass 2: scoped replacement.
+		// Pass 2: scoped replacement, in both literal and URL-encoded forms.
 		for _, pp := range r.pairs {
 			if bytes.Contains(toProcess, pp.phantom) {
 				toProcess = bytes.ReplaceAll(toProcess, pp.phantom, pp.secret.Bytes())
 			}
+			encoded := []byte(url.QueryEscape(string(pp.phantom)))
+			if !bytes.Equal(encoded, pp.phantom) && bytes.Contains(toProcess, encoded) {
+				toProcess = bytes.ReplaceAll(toProcess, encoded, []byte(url.QueryEscape(string(pp.secret.Bytes()))))
+			}
 		}
-		// Pass 3: strip unbound.
-		if bytes.Contains(toProcess, phantomPrefix) {
+		// Pass 3: strip unbound, including URL-encoded phantoms.
+		if bytesContainsAnyPhantomPrefix(toProcess) {
 			toProcess = stripUnboundPhantomsFromProvider(toProcess, r.provider)
 		}
 

internal/proxy/addon_test.go

@@ -726,6 +726,107 @@ func TestRequest_StripUnboundPhantoms(t *testing.T) {
 	}
 }
 
+// TestRequest_PhantomSwapInFormUrlencodedBody covers the OAuth refresh path:
+// the agent posts application/x-www-form-urlencoded data and the phantom's
+// colon is percent-encoded on the wire (SLUICE_PHANTOM%3Aapi_key). The
+// scanner must recognize that form and substitute the URL-encoded secret so
+// the upstream still receives a well-formed form body.
+func TestRequest_PhantomSwapInFormUrlencodedBody(t *testing.T) {
+	addon := newTestAddonWithCreds(
+		t,
+		map[string]string{"api_key": "real value/with+special&chars"},
+		[]vault.Binding{{
+			Destination: "api.example.com",
+			Ports:       []int{443},
+			Credential:  "api_key",
+		}},
+	)
+	client := setupAddonConn(addon, "api.example.com:443")
+
+	f := newTestFlow(client, "POST", "https://api.example.com/oauth/token")
+	f.Request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	f.Request.Body = []byte("grant_type=refresh_token&refresh_token=SLUICE_PHANTOM%3Aapi_key")
+
+	addon.Requestheaders(f)
+	addon.Request(f)
+
+	body := string(f.Request.Body)
+	wantValue := "real+value%2Fwith%2Bspecial%26chars"
+	if !strings.Contains(body, wantValue) {
+		t.Fatalf("body = %q, want url-encoded secret %q", body, wantValue)
+	}
+	if strings.Contains(body, "SLUICE_PHANTOM%3A") {
+		t.Fatalf("url-encoded phantom should have been replaced, got %q", body)
+	}
+	if strings.Contains(body, "SLUICE_PHANTOM:") {
+		t.Fatalf("literal phantom should not appear in body, got %q", body)
+	}
+}
+
+// TestRequest_StripUnboundPhantoms_UrlEncoded asserts that an unbound
+// phantom in url-encoded form does not pass through to the upstream.
+func TestRequest_StripUnboundPhantoms_UrlEncoded(t *testing.T) {
+	addon := newTestAddonWithCreds(
+		t,
+		map[string]string{
+			"api_key":   "real-secret",
+			"other_key": "other-secret",
+		},
+		[]vault.Binding{{
+			Destination: "api.example.com",
+			Ports:       []int{443},
+			Credential:  "api_key",
+		}},
+	)
+	client := setupAddonConn(addon, "api.example.com:443")
+
+	f := newTestFlow(client, "POST", "https://api.example.com/data")
+	f.Request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	f.Request.Body = []byte("key=SLUICE_PHANTOM%3Aapi_key&unbound=SLUICE_PHANTOM%3Aother_key")
+
+	addon.Requestheaders(f)
+	addon.Request(f)
+
+	body := string(f.Request.Body)
+	if !strings.Contains(body, "real-secret") {
+		t.Fatalf("expected bound phantom to be replaced, got %q", body)
+	}
+	if strings.Contains(body, "SLUICE_PHANTOM%3A") {
+		t.Fatalf("expected unbound url-encoded phantom to be stripped, got %q", body)
+	}
+	if strings.Contains(body, "other-secret") {
+		t.Fatalf("unbound phantom should not be replaced with real credential, got %q", body)
+	}
+}
+
+// TestRequest_PhantomSwapInUrlEncodedQuery covers the URL query path:
+// the phantom is percent-encoded in RawQuery and must be swapped with a
+// percent-encoded real value so the resulting query string stays valid.
+func TestRequest_PhantomSwapInUrlEncodedQuery(t *testing.T) {
+	addon := newTestAddonWithCreds(
+		t,
+		map[string]string{"api_key": "real-secret"},
+		[]vault.Binding{{
+			Destination: "api.example.com",
+			Ports:       []int{443},
+			Credential:  "api_key",
+		}},
+	)
+	client := setupAddonConn(addon, "api.example.com:443")
+
+	f := newTestFlow(client, "GET", "https://api.example.com/data?token=SLUICE_PHANTOM%3Aapi_key")
+
+	addon.Requestheaders(f)
+	addon.Request(f)
+
+	if got := f.Request.URL.RawQuery; !strings.Contains(got, "real-secret") {
+		t.Fatalf("RawQuery = %q, want to contain real-secret", got)
+	}
+	if got := f.Request.URL.RawQuery; strings.Contains(got, "SLUICE_PHANTOM%3A") {
+		t.Fatalf("RawQuery still contains url-encoded phantom: %q", got)
+	}
+}
+
 func TestRequest_NoBindingNoChange(t *testing.T) {
 	// No bindings configured. Body without phantom tokens should pass
 	// through unchanged.

internal/proxy/phantom.go

@@ -2,15 +2,30 @@ package proxy
 
 import "regexp"
 
-// phantomPrefix is the byte prefix for all phantom tokens, used for quick
-// detection before applying the more expensive regex strip.
+// phantomPrefix is the byte prefix for all phantom tokens in their literal
+// form, used for quick detection before applying the more expensive regex
+// strip. Literal form appears in JSON bodies, raw header values, and
+// anywhere the colon survives unencoded.
 var phantomPrefix = []byte("SLUICE_PHANTOM:")
 
+// urlEncodedPhantomPrefix is the byte prefix for phantom tokens after URL
+// percent-encoding (the colon becomes %3A). Appears in
+// application/x-www-form-urlencoded request bodies (e.g. OAuth refresh
+// POSTs) and in URL query strings. Without scanning for this form, a
+// phantom embedded in form-urlencoded data would pass through unswapped
+// and the upstream would receive the literal `SLUICE_PHANTOM%3A...`
+// string. The two prefixes are kept side by side rather than computed at
+// runtime so the byte scan stays a single allocation-free contains check.
+var urlEncodedPhantomPrefix = []byte("SLUICE_PHANTOM%3A")
+
 // phantomStripRe is a last-resort regex for stripping phantom tokens when
-// provider.List() cannot enumerate all credential names. It matches word
-// characters, dots, and hyphens.
+// provider.List() cannot enumerate all credential names. It matches both
+// literal (SLUICE_PHANTOM:...) and URL-encoded (SLUICE_PHANTOM%3A...) forms
+// so unbound phantoms cannot leak via either encoding. The character class
+// matches word characters, dots, and hyphens — the same set used by the
+// credential-name and OAuth-suffix grammar.
 // The primary strip path uses exact matching via provider.List().
-var phantomStripRe = regexp.MustCompile(`SLUICE_PHANTOM:[\w.\-]+`)
+var phantomStripRe = regexp.MustCompile(`SLUICE_PHANTOM(?::|%3[Aa])[\w.\-]+`)
 
 // PhantomToken returns the placeholder token for a credential name.
 // Agents use this token in requests. The MITM proxy replaces it with

internal/proxy/phantom_strip.go

@@ -2,6 +2,7 @@ package proxy
 
 import (
 	"bytes"
+	"net/url"
 	"sort"
 
 	"github.com/nemirovsky/sluice/internal/vault"
@@ -47,6 +48,19 @@ func stripUnboundPhantomsFromProvider(data []byte, provider vault.Provider) []by
 			[]byte(PhantomToken(name)),
 		)
 	}
+	// Mirror every phantom with its URL-encoded variant so phantoms carried in
+	// form-urlencoded bodies or URL components are stripped as cleanly as the
+	// literal form. The encoded variant is only added when it differs from the
+	// literal form (i.e. when QueryEscape actually rewrote something) so we
+	// don't waste a duplicate ReplaceAll on the literal scan path.
+	encodedPhantoms := make([][]byte, 0, len(phantoms))
+	for _, p := range phantoms {
+		encoded := []byte(url.QueryEscape(string(p)))
+		if !bytes.Equal(encoded, p) {
+			encodedPhantoms = append(encodedPhantoms, encoded)
+		}
+	}
+	phantoms = append(phantoms, encodedPhantoms...)
 	// Sort by token length descending so longer phantom tokens are stripped
 	// before shorter prefixes that could corrupt them via substring match.
 	sort.Slice(phantoms, func(i, j int) bool {
@@ -58,8 +72,10 @@ func stripUnboundPhantomsFromProvider(data []byte, provider vault.Provider) []by
 		}
 	}
 	// Last-resort regex strip for phantom tokens from providers that
-	// don't support List() (e.g. env provider).
-	if bytes.Contains(data, phantomPrefix) {
+	// don't support List() (e.g. env provider). The regex handles both
+	// literal (SLUICE_PHANTOM:...) and URL-encoded (SLUICE_PHANTOM%3A...)
+	// forms so unbound phantoms in form-urlencoded bodies are caught too.
+	if bytes.Contains(data, phantomPrefix) || bytes.Contains(data, urlEncodedPhantomPrefix) {
 		data = phantomStripRe.ReplaceAll(data, nil)
 	}
 	return data