Skip to content

Commit c92cc34

Browse files
lidellidel
committed
feat(gateway): add AllowCodecConversion config option
Add AllowCodecConversion to gateway.Config to control codec conversion behavior per IPIP-0524. When false (default), the gateway returns 406 Not Acceptable if the requested format doesn't match the block's codec. When true, conversions between codecs are performed for backward compatibility. Codec conversion tests moved here from gateway-conformance since conversions are now an optional implementation feature, not a spec requirement. Gateway-conformance now tests for 406 responses. Ref: ipfs/specs#524 Ref: ipfs/gateway-conformance#254
1 parent 0842ad2 commit c92cc34

File tree

9 files changed

+115
-45
lines changed

9 files changed

+115
-45
lines changed

‎.github/workflows/gateway-conformance.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222
steps:
2323
# 1. Download the gateway-conformance fixtures
2424
- name: Download gateway-conformance fixtures
25-
uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8
25+
uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships
2626
with:
2727
output: fixtures
2828
merged: true
@@ -47,7 +47,7 @@ jobs:
4747

4848
# 4. Run the gateway-conformance tests
4949
- name: Run gateway-conformance tests without IPNS and DNSLink
50-
uses: ipfs/gateway-conformance/.github/actions/test@v0.8
50+
uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships
5151
with:
5252
gateway-url: http://127.0.0.1:8040
5353
subdomain-url: http://example.net:8040
@@ -84,7 +84,7 @@ jobs:
8484
steps:
8585
# 1. Download the gateway-conformance fixtures
8686
- name: Download gateway-conformance fixtures
87-
uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8
87+
uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships
8888
with:
8989
output: fixtures
9090
merged: true
@@ -114,7 +114,7 @@ jobs:
114114

115115
# 4. Run the gateway-conformance tests
116116
- name: Run gateway-conformance tests without IPNS and DNSLink
117-
uses: ipfs/gateway-conformance/.github/actions/test@v0.8
117+
uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships
118118
with:
119119
gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway
120120
subdomain-url: http://example.net:8040
@@ -152,7 +152,7 @@ jobs:
152152
steps:
153153
# 1. Download the gateway-conformance fixtures
154154
- name: Download gateway-conformance fixtures
155-
uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8
155+
uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships
156156
with:
157157
output: fixtures
158158
merged: true
@@ -182,7 +182,7 @@ jobs:
182182

183183
# 4. Run the gateway-conformance tests
184184
- name: Run gateway-conformance tests without IPNS and DNSLink
185-
uses: ipfs/gateway-conformance/.github/actions/test@v0.8
185+
uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships
186186
with:
187187
gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway
188188
subdomain-url: http://example.net:8040

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ The following emojis are used to highlight certain changes:
2222

2323
### Changed
2424

25+
- `gateway`: 🛠 Codec conversions (e.g., dag-pb to dag-json, dag-json to dag-cbor) are no longer performed by default per [IPIP-0524](https://github.com/ipfs/specs/pull/524). Requesting a format that differs from the block's codec now returns HTTP 406 Not Acceptable. Clients should fetch raw blocks (`?format=raw`) and convert in userland. Set `Config.AllowCodecConversion` to `true` to restore the old behavior.
26+
2527
### Removed
2628

2729
### Fixed

‎examples/gateway/car-file/main_test.go‎

Lines changed: 21 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,14 @@
11
package main
22

33
import (
4+
"bytes"
45
"io"
56
"net/http"
67
"net/http/httptest"
78
"testing"
89

910
"github.com/ipfs/boxo/examples/gateway/common"
1011
"github.com/ipfs/boxo/gateway"
11-
"github.com/ipld/go-ipld-prime/codec/dagjson"
12-
"github.com/ipld/go-ipld-prime/node/basicnode"
1312
"github.com/stretchr/testify/assert"
1413
)
1514

@@ -62,48 +61,35 @@ func TestFile(t *testing.T) {
6261
assert.EqualValues(t, string(body), "hello world\n")
6362
}
6463

65-
func TestDirectoryAsDAG(t *testing.T) {
64+
func TestDirectoryAsRawBlock(t *testing.T) {
6665
ts, f, err := newTestServer()
6766
assert.NoError(t, err)
6867
defer f.Close()
6968

70-
res, err := http.Get(ts.URL + "/ipfs/" + BaseCID + "?format=dag-json")
69+
res, err := http.Get(ts.URL + "/ipfs/" + BaseCID + "?format=raw")
7170
assert.NoError(t, err)
7271
defer res.Body.Close()
7372

74-
contentType := res.Header.Get("Content-Type")
75-
assert.EqualValues(t, contentType, "application/vnd.ipld.dag-json")
76-
77-
// Parses the DAG-JSON response.
78-
dag := basicnode.Prototype.Any.NewBuilder()
79-
err = dagjson.Decode(dag, res.Body)
80-
assert.NoError(t, err)
81-
82-
// Checks for the links inside the logical model.
83-
links, err := dag.Build().LookupByString("Links")
84-
assert.NoError(t, err)
85-
86-
// Checks if there are 2 links.
87-
assert.EqualValues(t, links.Length(), 2)
88-
89-
// Check if the first item is correct.
90-
n, err := links.LookupByIndex(0)
91-
assert.NoError(t, err)
92-
assert.NotNil(t, n)
73+
assert.Equal(t, http.StatusOK, res.StatusCode)
9374

94-
nameNode, err := n.LookupByString("Name")
95-
assert.NoError(t, err)
96-
assert.NotNil(t, nameNode)
97-
98-
name, err := nameNode.AsString()
99-
assert.NoError(t, err)
100-
assert.EqualValues(t, name, "eye.png")
75+
contentType := res.Header.Get("Content-Type")
76+
assert.Equal(t, "application/vnd.ipld.raw", contentType)
10177

102-
hashNode, err := n.LookupByString("Hash")
78+
body, err := io.ReadAll(res.Body)
10379
assert.NoError(t, err)
104-
assert.NotNil(t, hashNode)
10580

106-
hash, err := hashNode.AsLink()
107-
assert.NoError(t, err)
108-
assert.EqualValues(t, hash.String(), "bafybeigmlfksb374fdkxih4urny2yiyazyra2375y2e4a72b3jcrnthnau")
81+
// Raw bytes of the dag-pb directory block
82+
expected := []byte{
83+
0x12, 0x33, 0x0a, 0x24, 0x01, 0x70, 0x12, 0x20, 0xcc, 0x59, 0x55, 0x20,
84+
0xef, 0xfc, 0x28, 0xd5, 0x74, 0x1f, 0x94, 0x8b, 0x71, 0xac, 0x23, 0x00,
85+
0xce, 0x22, 0x0d, 0x6f, 0xfd, 0xc6, 0x89, 0xc0, 0x7f, 0x41, 0xda, 0x45,
86+
0x16, 0xcc, 0xed, 0x05, 0x12, 0x07, 0x65, 0x79, 0x65, 0x2e, 0x70, 0x6e,
87+
0x67, 0x18, 0xd0, 0xc8, 0x10, 0x12, 0x33, 0x0a, 0x24, 0x01, 0x55, 0x12,
88+
0x20, 0xa9, 0x48, 0x90, 0x4f, 0x2f, 0x0f, 0x47, 0x9b, 0x8f, 0x81, 0x97,
89+
0x69, 0x4b, 0x30, 0x18, 0x4b, 0x0d, 0x2e, 0xd1, 0xc1, 0xcd, 0x2a, 0x1e,
90+
0xc0, 0xfb, 0x85, 0xd2, 0x99, 0xa1, 0x92, 0xa4, 0x47, 0x12, 0x09, 0x68,
91+
0x65, 0x6c, 0x6c, 0x6f, 0x2e, 0x74, 0x78, 0x74, 0x18, 0x0c, 0x0a, 0x02,
92+
0x08, 0x01,
93+
}
94+
assert.True(t, bytes.Equal(body, expected), "raw block bytes should match")
10995
}

‎examples/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ require (
88
github.com/ipfs/go-cid v0.6.0
99
github.com/ipfs/go-datastore v0.9.0
1010
github.com/ipld/go-car/v2 v2.16.0
11-
github.com/ipld/go-ipld-prime v0.21.0
1211
github.com/libp2p/go-libp2p v0.45.0
1312
github.com/multiformats/go-multiaddr v0.16.1
1413
github.com/multiformats/go-multicodec v0.10.0
@@ -63,6 +62,7 @@ require (
6362
github.com/ipfs/go-peertaskqueue v0.8.2 // indirect
6463
github.com/ipfs/go-unixfsnode v1.10.2 // indirect
6564
github.com/ipld/go-codec-dagpb v1.7.0 // indirect
65+
github.com/ipld/go-ipld-prime v0.21.0 // indirect
6666
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
6767
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
6868
github.com/klauspost/compress v1.18.0 // indirect

‎gateway/gateway.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,19 @@ type Config struct {
5252
// [Trustless Gateway]: https://specs.ipfs.tech/http-gateways/trustless-gateway/
5353
DeserializedResponses bool
5454

55+
// AllowCodecConversion enables automatic conversion between codecs when
56+
// the requested format differs from the block's native codec. For example,
57+
// converting dag-pb (UnixFS) to dag-json.
58+
//
59+
// When false (default), the gateway returns 406 Not Acceptable if the
60+
// requested format doesn't match the block's codec. This follows the
61+
// behavior specified in IPIP-0524.
62+
//
63+
// When true, the gateway attempts to convert between legacy IPLD formats.
64+
// This is provided for backwards compatibility but is not required by
65+
// the gateway specification.
66+
AllowCodecConversion bool
67+
5568
// NoDNSLink configures the gateway to _not_ perform DNS TXT record lookups in
5669
// response to requests with values in `Host` HTTP header. This flag can be
5770
// overridden per FQDN in PublicGateways. To be used with WithHostname.

‎gateway/gateway_test.go‎

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -521,6 +521,7 @@ func TestHeaders(t *testing.T) {
521521
},
522522
},
523523
DeserializedResponses: true,
524+
AllowCodecConversion: true, // Test tests various format conversions
524525
})
525526

526527
runTest := func(name, path, accept, host, expectedContentLocationHdr string) {
@@ -1073,7 +1074,8 @@ func TestDeserializedResponses(t *testing.T) {
10731074
backend, root := newMockBackend(t, "fixtures.car")
10741075

10751076
ts := newTestServerWithConfig(t, backend, Config{
1076-
NoDNSLink: false,
1077+
NoDNSLink: false,
1078+
AllowCodecConversion: true, // Test expects codec conversions to work
10771079
PublicGateways: map[string]*PublicGateway{
10781080
"trustless.com": {
10791081
Paths: []string{"/ipfs", "/ipns"},
@@ -1152,7 +1154,8 @@ func TestDeserializedResponses(t *testing.T) {
11521154
backend.namesys["/ipns/trusted.com"] = newMockNamesysItem(path.FromCid(root), 0)
11531155

11541156
ts := newTestServerWithConfig(t, backend, Config{
1155-
NoDNSLink: false,
1157+
NoDNSLink: false,
1158+
AllowCodecConversion: true, // Test expects codec conversions to work
11561159
PublicGateways: map[string]*PublicGateway{
11571160
"trustless.com": {
11581161
Paths: []string{"/ipfs", "/ipns"},
@@ -1186,6 +1189,58 @@ func TestDeserializedResponses(t *testing.T) {
11861189
})
11871190
}
11881191

1192+
func TestAllowCodecConversion(t *testing.T) {
1193+
t.Parallel()
1194+
1195+
// Use dag-cbor fixture
1196+
backend, dagCborRoot := newMockBackend(t, "path_gateway_dag/dag-cbor-traversal.car")
1197+
1198+
t.Run("AllowCodecConversion=false returns 406 for codec mismatch", func(t *testing.T) {
1199+
t.Parallel()
1200+
1201+
ts := newTestServerWithConfig(t, backend, Config{
1202+
DeserializedResponses: true,
1203+
AllowCodecConversion: false, // IPIP-0524 behavior
1204+
})
1205+
1206+
// Request dag-json for a dag-cbor block - should return 406
1207+
req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil)
1208+
res := mustDoWithoutRedirect(t, req)
1209+
defer res.Body.Close()
1210+
assert.Equal(t, http.StatusNotAcceptable, res.StatusCode)
1211+
})
1212+
1213+
t.Run("AllowCodecConversion=false allows matching codec", func(t *testing.T) {
1214+
t.Parallel()
1215+
1216+
ts := newTestServerWithConfig(t, backend, Config{
1217+
DeserializedResponses: true,
1218+
AllowCodecConversion: false, // IPIP-0524 behavior
1219+
})
1220+
1221+
// Request dag-cbor for a dag-cbor block - should return 200
1222+
req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-cbor", nil)
1223+
res := mustDoWithoutRedirect(t, req)
1224+
defer res.Body.Close()
1225+
assert.Equal(t, http.StatusOK, res.StatusCode)
1226+
})
1227+
1228+
t.Run("AllowCodecConversion=true allows codec conversion", func(t *testing.T) {
1229+
t.Parallel()
1230+
1231+
ts := newTestServerWithConfig(t, backend, Config{
1232+
DeserializedResponses: true,
1233+
AllowCodecConversion: true, // Legacy behavior
1234+
})
1235+
1236+
// Request dag-json for a dag-cbor block - should return 200 with conversion
1237+
req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil)
1238+
res := mustDoWithoutRedirect(t, req)
1239+
defer res.Body.Close()
1240+
assert.Equal(t, http.StatusOK, res.StatusCode)
1241+
})
1242+
}
1243+
11891244
type errorMockBackend struct {
11901245
err error
11911246
}

‎gateway/handler_codec.go‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,20 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt
148148
return false
149149
}
150150

151-
// This handles DAG-* conversions and validations.
151+
// IPIP-0524: Check if codec conversion is allowed
152+
if !i.config.AllowCodecConversion && toCodec != cidCodec {
153+
// Conversion not allowed and codecs don't match - return 406
154+
err := fmt.Errorf("format %q requested but block has codec %q: codec conversion is not supported", rq.responseFormat, cidCodec.String())
155+
i.webError(w, r, err, http.StatusNotAcceptable)
156+
return false
157+
}
158+
159+
// If codecs match, serve raw (no conversion needed)
160+
if toCodec == cidCodec {
161+
return i.serveCodecRaw(ctx, w, r, blockSize, blockData, rq.contentPath, modtime, rq.begin)
162+
}
163+
164+
// AllowCodecConversion is true - perform DAG-* conversion
152165
return i.serveCodecConverted(ctx, w, r, blockCid, blockData, rq.contentPath, toCodec, modtime, rq.begin)
153166
}
154167

‎gateway/testdata/path_gateway_dag/dag-cbor-traversal.car‎

318 Bytes
Binary file not shown.

‎gateway/utilities_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,7 @@ func newTestServerAndNode(t *testing.T, fixturesFile string) (*httptest.Server,
234234
func newTestServer(t *testing.T, backend IPFSBackend) *httptest.Server {
235235
return newTestServerWithConfig(t, backend, Config{
236236
DeserializedResponses: true,
237+
AllowCodecConversion: true, // Enable for backwards compatibility in tests
237238
MetricsRegistry: prometheus.NewRegistry(),
238239
})
239240
}

0 commit comments

Comments
 (0)