From 20c8878c82df6d967b24874a1004e749f8e0110b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 15 May 2024 21:59:50 +0200 Subject: [PATCH 01/43] chore: add todo for bzz --- pkg/api/bzz.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index 65ded851f12..26a289a7b27 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -384,14 +384,14 @@ FETCH: jsonhttp.NotFound(w, "no update found") return } - ref, _, err := parseFeedUpdate(ch) + wc, err := feeds.GetWrappedChunk(ctx, s.storer.ChunkStore(), ch) if err != nil { logger.Debug("bzz download: mapStructure feed update failed", "error", err) logger.Error(nil, "bzz download: mapStructure feed update failed") jsonhttp.InternalServerError(w, "mapStructure feed update") return } - address = ref + address = wc.Address() // FIXME: init manifest with root chunk instead feedDereferenced = true curBytes, err := cur.MarshalBinary() if err != nil { From 6332838249f4628b804ee14e7f35b88992db163a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 15 May 2024 22:04:04 +0200 Subject: [PATCH 02/43] feat: getWrappedChunk with legacy payload handling --- pkg/feeds/getter.go | 50 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/pkg/feeds/getter.go b/pkg/feeds/getter.go index 77b7599dd9e..c2236c77ce1 100644 --- a/pkg/feeds/getter.go +++ b/pkg/feeds/getter.go @@ -16,6 +16,8 @@ import ( "github.com/ethersphere/bee/v2/pkg/swarm" ) +var errNotLegacyPayload = errors.New("feed update is not in the legacy payload structure") + // Lookup is the interface for time based feed lookup type Lookup interface { At(ctx context.Context, at int64, after uint64) (chunk swarm.Chunk, currentIndex, nextIndex Index, err error) @@ -49,19 +51,49 @@ func (f *Getter) Get(ctx context.Context, i Index) (swarm.Chunk, error) { return f.getter.Get(ctx, addr) } -// FromChunk parses out the timestamp and the payload -func FromChunk(ch swarm.Chunk) (uint64, []byte, error) { +func GetWrappedChunk(ctx context.Context, getter storage.Getter, ch swarm.Chunk) (swarm.Chunk, error) { + wc, err := FromChunk(ch) + if err != nil { + return nil, err + } + // try to split the timestamp and reference + // possible values right now: + // unencrypted ref: span+timestamp+ref => 8+8+32=48 + // encrypted ref: span+timestamp+ref+decryptKey => 8+8+64=80 + _, ref, err := LegacyPayload(wc) + if err != nil { + if errors.Is(err, errNotLegacyPayload) { + return wc, nil + } + return nil, err + } + wc, err = getter.Get(ctx, ref) + if err != nil { + return nil, err + } + + return wc, nil +} + +// FromChunk parses out the wrapped chunk +func FromChunk(ch swarm.Chunk) (swarm.Chunk, error) { s, err := soc.FromChunk(ch) if err != nil { - return 0, nil, err + return nil, fmt.Errorf("soc unmarshal: %w", err) } - cac := s.WrappedChunk() - if len(cac.Data()) < 16 { - return 0, nil, errors.New("feed update payload too short") + return s.WrappedChunk(), nil +} + +// LegacyPayload returns back the referenced chunk and datetime from the legacy feed payload +func LegacyPayload(wrappedChunk swarm.Chunk) (uint64, swarm.Address, error) { + cacData := wrappedChunk.Data() + if !(len(cacData) == 16+swarm.HashSize || len(cacData) == 16+swarm.HashSize*2) { + return 0, swarm.ZeroAddress, errNotLegacyPayload } - payload := cac.Data()[16:] - at := binary.BigEndian.Uint64(cac.Data()[8:16]) - return at, payload, nil + address := swarm.NewAddress(cacData[16:]) + at := binary.BigEndian.Uint64(cacData[8:16]) + + return at, address, nil } // UpdatedAt extracts the time of feed other than update From 8c14c90cc620f0fbb3d3b614d93edb20118f50c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 15 May 2024 22:08:15 +0200 Subject: [PATCH 03/43] feat: add content length header for feeds path --- pkg/api/router.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/api/router.go b/pkg/api/router.go index c0c6d02782c..232920e9090 100644 --- a/pkg/api/router.go +++ b/pkg/api/router.go @@ -67,9 +67,11 @@ func (s *Service) MountAPI() { "/bzz", "/bytes", "/chunks", + "/feeds", rootPath + "/bzz", rootPath + "/bytes", rootPath + "/chunks", + rootPath + "/feeds", } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { From 8515af039954f667ff33939c02a1b607c442482c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 15 May 2024 22:14:40 +0200 Subject: [PATCH 04/43] feat: remove payload structure for feeds --- pkg/feeds/putter.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pkg/feeds/putter.go b/pkg/feeds/putter.go index 633276f8f63..abe1972a0a5 100644 --- a/pkg/feeds/putter.go +++ b/pkg/feeds/putter.go @@ -6,7 +6,6 @@ package feeds import ( "context" - "encoding/binary" "github.com/ethersphere/bee/v2/pkg/cac" "github.com/ethersphere/bee/v2/pkg/crypto" @@ -39,12 +38,12 @@ func NewPutter(putter storage.Putter, signer crypto.Signer, topic []byte) (*Putt } // Put pushes an update to the feed through the chunk stores -func (u *Putter) Put(ctx context.Context, i Index, at int64, payload []byte) error { +func (u *Putter) Put(ctx context.Context, i Index, payload []byte) error { id, err := u.Feed.Update(i).Id() if err != nil { return err } - cac, err := toChunk(uint64(at), payload) + cac, err := toChunk(payload) if err != nil { return err } @@ -56,8 +55,6 @@ func (u *Putter) Put(ctx context.Context, i Index, at int64, payload []byte) err return u.putter.Put(ctx, ch) } -func toChunk(at uint64, payload []byte) (swarm.Chunk, error) { - ts := make([]byte, 8) - binary.BigEndian.PutUint64(ts, at) - return cac.New(append(ts, payload...)) +func toChunk(payload []byte) (swarm.Chunk, error) { + return cac.New(payload) } From 33f7b8b595b8992bc8479c1e8bd2fb85290a6d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 15 May 2024 22:18:57 +0200 Subject: [PATCH 05/43] chore: indicating bootstrap fixme --- pkg/node/bootstrap.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/node/bootstrap.go b/pkg/node/bootstrap.go index 3a97168a241..47b6b0f9093 100644 --- a/pkg/node/bootstrap.go +++ b/pkg/node/bootstrap.go @@ -333,12 +333,16 @@ func getLatestSnapshot( return swarm.ZeroAddress, err } - _, ref, err := feeds.FromChunk(u) + _, ref, err := feeds.LegacyPayload(u) if err != nil { - return swarm.ZeroAddress, err + wrappedChunk, err := feeds.FromChunk(u) + if err != nil { + return swarm.ZeroAddress, err + } + ref = wrappedChunk.Address() } - return swarm.NewAddress(ref), nil + return ref, nil // FIXME root chunk return, use only getWrappedChunk } func batchStoreExists(s storage.StateStorer) (bool, error) { From 7edec9e2c2f4b81bc8e2b3fae016cbee4fe3520a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 15 May 2024 22:28:31 +0200 Subject: [PATCH 06/43] chore: update call arguments --- pkg/feeds/epochs/updater.go | 2 +- pkg/feeds/feed.go | 2 +- pkg/feeds/sequence/sequence.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/feeds/epochs/updater.go b/pkg/feeds/epochs/updater.go index b36d77e7d96..f3588b98f1b 100644 --- a/pkg/feeds/epochs/updater.go +++ b/pkg/feeds/epochs/updater.go @@ -34,7 +34,7 @@ func NewUpdater(putter storage.Putter, signer crypto.Signer, topic []byte) (feed // Update pushes an update to the feed through the chunk stores func (u *updater) Update(ctx context.Context, at int64, payload []byte) error { e := next(u.epoch, u.last, uint64(at)) - err := u.Put(ctx, e, at, payload) + err := u.Put(ctx, e, payload) if err != nil { return err } diff --git a/pkg/feeds/feed.go b/pkg/feeds/feed.go index 7c9495cdfac..ac8d232f5ce 100644 --- a/pkg/feeds/feed.go +++ b/pkg/feeds/feed.go @@ -107,7 +107,7 @@ func NewUpdate(f *Feed, idx Index, timestamp int64, payload, sig []byte) (swarm. if err != nil { return nil, fmt.Errorf("update: %w", err) } - cac, err := toChunk(uint64(timestamp), payload) + cac, err := toChunk(payload) if err != nil { return nil, fmt.Errorf("toChunk: %w", err) } diff --git a/pkg/feeds/sequence/sequence.go b/pkg/feeds/sequence/sequence.go index 5361086de4b..c1dc06a720c 100644 --- a/pkg/feeds/sequence/sequence.go +++ b/pkg/feeds/sequence/sequence.go @@ -297,7 +297,7 @@ func NewUpdater(putter storage.Putter, signer crypto.Signer, topic []byte) (feed // Update pushes an update to the feed through the chunk stores func (u *updater) Update(ctx context.Context, at int64, payload []byte) error { - err := u.Put(ctx, &index{u.next}, at, payload) + err := u.Put(ctx, &index{u.next}, payload) if err != nil { return err } From d93cece65b11f8150db71a85706f7d9be41f53ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 15 May 2024 22:29:00 +0200 Subject: [PATCH 07/43] test: adjust feed testing --- pkg/feeds/testing/lookup.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/pkg/feeds/testing/lookup.go b/pkg/feeds/testing/lookup.go index 8c71098f5a6..1b8ff7abc2b 100644 --- a/pkg/feeds/testing/lookup.go +++ b/pkg/feeds/testing/lookup.go @@ -77,24 +77,22 @@ func TestFinderBasic(t *testing.T, finderf func(storage.Getter, *feeds.Feed) fee if err != nil { t.Fatal(err) } - ch, err := feeds.Latest(ctx, finder, 0) + soc, err := feeds.Latest(ctx, finder, 0) if err != nil { t.Fatal(err) } - if ch == nil { + if soc == nil { t.Fatalf("expected to find update, got none") } exp := payload - ts, payload, err := feeds.FromChunk(ch) + cac, err := feeds.FromChunk(soc) if err != nil { t.Fatal(err) } + payload = cac.Data()[swarm.SpanSize:] if !bytes.Equal(payload, exp) { t.Fatalf("result mismatch. want %8x... got %8x...", exp, payload) } - if ts != uint64(at) { - t.Fatalf("timestamp mismatch: expected %v, got %v", at, ts) - } }) } @@ -157,7 +155,8 @@ func TestFinderIntervals(t *testing.T, nextf func() (bool, int64), finderf func( if ch == nil { t.Fatalf("expected to find update, got none") } - ts, payload, err := feeds.FromChunk(ch) + wrappedChunk, err := feeds.FromChunk(ch) + payload := wrappedChunk.Data()[swarm.SpanSize:] if err != nil { t.Fatal(err) } @@ -166,10 +165,6 @@ func TestFinderIntervals(t *testing.T, nextf func() (bool, int64), finderf func( t.Fatalf("payload mismatch: expected %v, got %v", at, content) } - if ts != uint64(at) { - t.Fatalf("timestamp mismatch: expected %v, got %v", at, ts) - } - if current != nil { expectedId := ch.Data()[:32] id, err := feeds.Id(topic, current) From e0923f2a734a16aa189ca63f705c10e73e860fc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 15 May 2024 22:36:02 +0200 Subject: [PATCH 08/43] feat: return chunk payload on api --- pkg/api/feed.go | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/pkg/api/feed.go b/pkg/api/feed.go index 09fdf6515ec..2bc815ce39d 100644 --- a/pkg/api/feed.go +++ b/pkg/api/feed.go @@ -5,11 +5,13 @@ package api import ( - "encoding/binary" + "bytes" "encoding/hex" "errors" "fmt" + "io" "net/http" + "strconv" "time" "github.com/ethereum/go-ethereum/common" @@ -20,7 +22,6 @@ import ( "github.com/ethersphere/bee/v2/pkg/manifest/mantaray" "github.com/ethersphere/bee/v2/pkg/manifest/simple" "github.com/ethersphere/bee/v2/pkg/postage" - "github.com/ethersphere/bee/v2/pkg/soc" storage "github.com/ethersphere/bee/v2/pkg/storage" storer "github.com/ethersphere/bee/v2/pkg/storer" "github.com/ethersphere/bee/v2/pkg/swarm" @@ -93,7 +94,13 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { return } - ref, _, err := parseFeedUpdate(ch) + wc, err := feeds.GetWrappedChunk(r.Context(), s.storer.ChunkStore(), ch) + if err != nil { + logger.Error(nil, "wrapped chunk cannot be retrieved") + jsonhttp.NotFound(w, "wrapped chunk cannot be retrieved") + return + } + wData := wc.Data() if err != nil { logger.Debug("mapStructure feed update failed", "error", err) logger.Error(nil, "mapStructure feed update failed") @@ -117,11 +124,12 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { return } + contenL := len(wData) + w.Header().Set(ContentLengthHeader, strconv.Itoa(contenL)) w.Header().Set(SwarmFeedIndexHeader, hex.EncodeToString(curBytes)) w.Header().Set(SwarmFeedIndexNextHeader, hex.EncodeToString(nextBytes)) w.Header().Set("Access-Control-Expose-Headers", fmt.Sprintf("%s, %s", SwarmFeedIndexHeader, SwarmFeedIndexNextHeader)) - - jsonhttp.OK(w, feedReferenceResponse{Reference: ref}) + _, _ = io.Copy(w, bytes.NewReader(wData)) } func (s *Service) feedPostHandler(w http.ResponseWriter, r *http.Request) { @@ -255,22 +263,3 @@ func (s *Service) feedPostHandler(w http.ResponseWriter, r *http.Request) { jsonhttp.Created(w, feedReferenceResponse{Reference: ref}) } - -func parseFeedUpdate(ch swarm.Chunk) (swarm.Address, int64, error) { - s, err := soc.FromChunk(ch) - if err != nil { - return swarm.ZeroAddress, 0, fmt.Errorf("soc unmarshal: %w", err) - } - - update := s.WrappedChunk().Data() - // split the timestamp and reference - // possible values right now: - // unencrypted ref: span+timestamp+ref => 8+8+32=48 - // encrypted ref: span+timestamp+ref+decryptKey => 8+8+64=80 - if len(update) != 48 && len(update) != 80 { - return swarm.ZeroAddress, 0, errInvalidFeedUpdate - } - ts := binary.BigEndian.Uint64(update[8:16]) - ref := swarm.NewAddress(update[16:]) - return ref, int64(ts), nil -} From b6d382e5a12a967ebf9c172e3213afa2e314a3bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 15 May 2024 22:36:16 +0200 Subject: [PATCH 09/43] test: feed api --- pkg/api/feed_test.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/api/feed_test.go b/pkg/api/feed_test.go index a35b9ce3423..b311c294486 100644 --- a/pkg/api/feed_test.go +++ b/pkg/api/feed_test.go @@ -24,13 +24,14 @@ import ( "github.com/ethersphere/bee/v2/pkg/postage" mockpost "github.com/ethersphere/bee/v2/pkg/postage/mock" testingsoc "github.com/ethersphere/bee/v2/pkg/soc/testing" + testingc "github.com/ethersphere/bee/v2/pkg/storage/testing" mockstorer "github.com/ethersphere/bee/v2/pkg/storer/mock" "github.com/ethersphere/bee/v2/pkg/swarm" ) const ownerString = "8d3766440f0d7b949a5e32995d09619a7f86e632" -var expReference = swarm.MustParseHexAddress("891a1d1c8436c792d02fc2e8883fef7ab387eaeaacd25aa9f518be7be7856d54") +var expReference = swarm.MustParseHexAddress("0033153ac8cfb0c343db1795f578c15ed8ef827f3e68ed3c58329900bf0d7276") func TestFeed_Get(t *testing.T) { t.Parallel() @@ -44,6 +45,15 @@ func TestFeed_Get(t *testing.T) { } mockStorer = mockstorer.New() ) + putter, err := mockStorer.Upload(context.Background(), false, 0) + if err != nil { + t.Fatal(err) + } + mockWrappedCh := testingc.FixtureChunk("0033") + err = putter.Put(context.Background(), mockWrappedCh) + if err != nil { + t.Fatal(err) + } t.Run("with at", func(t *testing.T) { t.Parallel() @@ -61,7 +71,7 @@ func TestFeed_Get(t *testing.T) { ) jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "12"), http.StatusOK, - jsonhttptest.WithExpectedJSONResponse(api.FeedReferenceResponse{Reference: expReference}), + jsonhttptest.WithExpectedResponse(mockWrappedCh.Data()), jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), ) }) @@ -83,7 +93,8 @@ func TestFeed_Get(t *testing.T) { ) jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, - jsonhttptest.WithExpectedJSONResponse(api.FeedReferenceResponse{Reference: expReference}), + jsonhttptest.WithExpectedResponse(mockWrappedCh.Data()), + jsonhttptest.WithExpectedContentLength(len(mockWrappedCh.Data())), jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), ) }) From df017830c108c8e1da84c9f780751f5d67c02d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 22 May 2024 11:31:37 +0200 Subject: [PATCH 10/43] test: correct expected hashes --- pkg/api/feed_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/api/feed_test.go b/pkg/api/feed_test.go index b311c294486..7857339d957 100644 --- a/pkg/api/feed_test.go +++ b/pkg/api/feed_test.go @@ -31,7 +31,7 @@ import ( const ownerString = "8d3766440f0d7b949a5e32995d09619a7f86e632" -var expReference = swarm.MustParseHexAddress("0033153ac8cfb0c343db1795f578c15ed8ef827f3e68ed3c58329900bf0d7276") +var expReference = swarm.MustParseHexAddress("891a1d1c8436c792d02fc2e8883fef7ab387eaeaacd25aa9f518be7be7856d54") func TestFeed_Get(t *testing.T) { t.Parallel() @@ -60,7 +60,7 @@ func TestFeed_Get(t *testing.T) { var ( timestamp = int64(12121212) - ch = toChunk(t, uint64(timestamp), expReference.Bytes()) + ch = toChunk(t, uint64(timestamp), mockWrappedCh.Address().Bytes()) look = newMockLookup(12, 0, ch, nil, &id{}, &id{}) factory = newMockFactory(look) idBytes, _ = (&id{}).MarshalBinary() @@ -81,7 +81,7 @@ func TestFeed_Get(t *testing.T) { var ( timestamp = int64(12121212) - ch = toChunk(t, uint64(timestamp), expReference.Bytes()) + ch = toChunk(t, uint64(timestamp), mockWrappedCh.Address().Bytes()) look = newMockLookup(-1, 2, ch, nil, &id{}, &id{}) factory = newMockFactory(look) idBytes, _ = (&id{}).MarshalBinary() From abe6896c77d9e2b9f4af1ae2b77148f7fbace1cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 22 May 2024 11:36:36 +0200 Subject: [PATCH 11/43] feat: joiner and loadsaver with rootch init --- pkg/api/bzz.go | 5 ++++- pkg/file/joiner/joiner.go | 10 +++++++++ pkg/file/loadsave/loadsave.go | 28 ++++++++++++++++++++---- pkg/node/bootstrap.go | 41 ++++++++++++++--------------------- 4 files changed, 54 insertions(+), 30 deletions(-) diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index 26a289a7b27..45e8c468ac3 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -391,7 +391,10 @@ FETCH: jsonhttp.InternalServerError(w, "mapStructure feed update") return } - address = wc.Address() // FIXME: init manifest with root chunk instead + address = wc.Address() + // modify ls and init with non-existing wrapped chunk + ls = loadsave.NewReadonlyWithRootCh(s.storer.Download(cache), wc) + feedDereferenced = true curBytes, err := cur.MarshalBinary() if err != nil { diff --git a/pkg/file/joiner/joiner.go b/pkg/file/joiner/joiner.go index fcd7e790c10..ca05366e038 100644 --- a/pkg/file/joiner/joiner.go +++ b/pkg/file/joiner/joiner.go @@ -103,6 +103,12 @@ func (g *decoderCache) GetOrCreate(addrs []swarm.Address, shardCnt int) storage. return d } +// NewWithRoot creates a new Joiner with the already fetched root chunk. +// A Joiner provides Read, Seek and Size functionalities. +func NewWithRootCh(ctx context.Context, g storage.Getter, putter storage.Putter, rootAddr swarm.Address, rootChunk swarm.Chunk) (file.Joiner, int64, error) { + return new(ctx, g, putter, rootAddr, rootChunk) +} + // New creates a new Joiner. A Joiner provides Read, Seek and Size functionalities. func New(ctx context.Context, g storage.Getter, putter storage.Putter, address swarm.Address) (file.Joiner, int64, error) { // retrieve the root chunk to read the total data length the be retrieved @@ -116,6 +122,10 @@ func New(ctx context.Context, g storage.Getter, putter storage.Putter, address s return nil, 0, err } + return new(ctx, g, putter, address, rootChunk) +} + +func new(ctx context.Context, g storage.Getter, putter storage.Putter, address swarm.Address, rootChunk swarm.Chunk) (file.Joiner, int64, error) { chunkData := rootChunk.Data() rootData := chunkData[swarm.SpanSize:] refLength := len(address.Bytes()) diff --git a/pkg/file/loadsave/loadsave.go b/pkg/file/loadsave/loadsave.go index 6a2a0bbf782..d329fb870e3 100644 --- a/pkg/file/loadsave/loadsave.go +++ b/pkg/file/loadsave/loadsave.go @@ -29,6 +29,7 @@ type loadSave struct { getter storage.Getter putter storage.Putter pipelineFn func() pipeline.Interface + rootCh swarm.Chunk } // New returns a new read-write load-saver. @@ -48,14 +49,33 @@ func NewReadonly(getter storage.Getter) file.LoadSaver { } } +// NewReadonly returns a new read-only load-saver +// which will error on write. +func NewReadonlyWithRootCh(getter storage.Getter, rootCh swarm.Chunk) file.LoadSaver { + return &loadSave{ + getter: getter, + rootCh: rootCh, + } +} + func (ls *loadSave) Load(ctx context.Context, ref []byte) ([]byte, error) { - j, _, err := joiner.New(ctx, ls.getter, ls.putter, swarm.NewAddress(ref)) - if err != nil { - return nil, err + var j file.Joiner + if ls.rootCh == nil || !bytes.Equal(ls.rootCh.Address().Bytes(), ref[:swarm.HashSize]) { + joiner, _, err := joiner.New(ctx, ls.getter, ls.putter, swarm.NewAddress(ref)) + if err != nil { + return nil, err + } + j = joiner + } else { + joiner, _, err := joiner.NewWithRootCh(ctx, ls.getter, ls.putter, swarm.NewAddress(ref), ls.rootCh) + if err != nil { + return nil, err + } + j = joiner } buf := bytes.NewBuffer(nil) - _, err = file.JoinReadAll(ctx, j, buf) + _, err := file.JoinReadAll(ctx, j, buf) if err != nil { return nil, err } diff --git a/pkg/node/bootstrap.go b/pkg/node/bootstrap.go index 47b6b0f9093..f6d8690466a 100644 --- a/pkg/node/bootstrap.go +++ b/pkg/node/bootstrap.go @@ -196,10 +196,10 @@ func bootstrapNode( logger.Info("bootstrap: trying to fetch stamps snapshot") var ( - snapshotReference swarm.Address - reader file.Joiner - l int64 - eventsJSON []byte + snapshotRootCh swarm.Chunk + reader file.Joiner + l int64 + eventsJSON []byte ) for i := 0; i < getSnapshotRetries; i++ { @@ -210,7 +210,7 @@ func bootstrapNode( ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - snapshotReference, err = getLatestSnapshot(ctx, localStore.Download(true), snapshotFeed) + snapshotRootCh, err = getLatestSnapshot(ctx, localStore.Download(true), snapshotFeed) if err != nil { logger.Warning("bootstrap: fetching snapshot failed", "error", err) continue @@ -229,7 +229,7 @@ func bootstrapNode( ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - reader, l, err = joiner.New(ctx, localStore.Download(true), localStore.Cache(), snapshotReference) + reader, l, err = joiner.NewWithRootCh(ctx, localStore.Download(true), localStore.Cache(), snapshotRootCh.Address(), snapshotRootCh) if err != nil { logger.Warning("bootstrap: file joiner failed", "error", err) continue @@ -278,7 +278,7 @@ func getLatestSnapshot( ctx context.Context, st storage.Getter, address swarm.Address, -) (swarm.Address, error) { +) (swarm.Chunk, error) { ls := loadsave.NewReadonly(st) feedFactory := factory.New(st) @@ -287,12 +287,12 @@ func getLatestSnapshot( ls, ) if err != nil { - return swarm.ZeroAddress, fmt.Errorf("not a manifest: %w", err) + return nil, fmt.Errorf("not a manifest: %w", err) } e, err := m.Lookup(ctx, "/") if err != nil { - return swarm.ZeroAddress, fmt.Errorf("node lookup: %w", err) + return nil, fmt.Errorf("node lookup: %w", err) } var ( @@ -303,46 +303,37 @@ func getLatestSnapshot( if e := meta["swarm-feed-owner"]; e != "" { owner, err = hex.DecodeString(e) if err != nil { - return swarm.ZeroAddress, err + return nil, err } } if e := meta["swarm-feed-topic"]; e != "" { topic, err = hex.DecodeString(e) if err != nil { - return swarm.ZeroAddress, err + return nil, err } } if e := meta["swarm-feed-type"]; e != "" { err := t.FromString(e) if err != nil { - return swarm.ZeroAddress, err + return nil, err } } if len(owner) == 0 || len(topic) == 0 { - return swarm.ZeroAddress, fmt.Errorf("node lookup: %s", "feed metadata absent") + return nil, fmt.Errorf("node lookup: %s", "feed metadata absent") } f := feeds.New(topic, common.BytesToAddress(owner)) l, err := feedFactory.NewLookup(*t, f) if err != nil { - return swarm.ZeroAddress, fmt.Errorf("feed lookup failed: %w", err) + return nil, fmt.Errorf("feed lookup failed: %w", err) } u, _, _, err := l.At(ctx, time.Now().Unix(), 0) if err != nil { - return swarm.ZeroAddress, err - } - - _, ref, err := feeds.LegacyPayload(u) - if err != nil { - wrappedChunk, err := feeds.FromChunk(u) - if err != nil { - return swarm.ZeroAddress, err - } - ref = wrappedChunk.Address() + return nil, err } - return ref, nil // FIXME root chunk return, use only getWrappedChunk + return feeds.GetWrappedChunk(ctx, st, u) } func batchStoreExists(s storage.StateStorer) (bool, error) { From ef05e2eea5a70e66bcae84aef8ddb0378891b445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 22 May 2024 14:04:27 +0200 Subject: [PATCH 12/43] test: chunk wrapping and wrong legacy payload resolution --- pkg/api/feed_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/pkg/api/feed_test.go b/pkg/api/feed_test.go index 7857339d957..9cf836fc18f 100644 --- a/pkg/api/feed_test.go +++ b/pkg/api/feed_test.go @@ -15,6 +15,7 @@ import ( "testing" "github.com/ethersphere/bee/v2/pkg/api" + "github.com/ethersphere/bee/v2/pkg/bmt" "github.com/ethersphere/bee/v2/pkg/feeds" "github.com/ethersphere/bee/v2/pkg/file/loadsave" "github.com/ethersphere/bee/v2/pkg/jsonhttp" @@ -98,6 +99,49 @@ func TestFeed_Get(t *testing.T) { jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), ) }) + + t.Run("chunk wrapping", func(t *testing.T) { + testData := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8} + testDataSpan := bmt.LengthToSpan(int64(len(testData))) + expectedData := append(testDataSpan, testData...) + + var ( + ch = testingsoc.GenerateMockSOC(t, testData).Chunk() + look = newMockLookup(-1, 2, ch, nil, &id{}, &id{}) + factory = newMockFactory(look) + idBytes, _ = (&id{}).MarshalBinary() + + client, _, _, _ = newTestServer(t, testServerOptions{ + Storer: mockStorer, + Feeds: factory, + }) + ) + + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, + jsonhttptest.WithExpectedResponse(expectedData), + jsonhttptest.WithExpectedContentLength(len(expectedData)), + jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), + ) + }) + + t.Run("legacy payload with non existing wrapped chunk", func(t *testing.T) { + t.Parallel() + wrappedRef := mockWrappedCh.Address().Bytes() + wrappedRef[0]++ + + var ( + ch = toChunk(t, uint64(12121212), wrappedRef) + look = newMockLookup(-1, 2, ch, nil, &id{}, &id{}) + factory = newMockFactory(look) + + client, _, _, _ = newTestServer(t, testServerOptions{ + Storer: mockStorer, + Feeds: factory, + }) + ) + + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusNotFound) + }) } // nolint:paralleltest From eccb5e4d10c4281da1b403ebfa0000f59693466b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 22 May 2024 14:22:17 +0200 Subject: [PATCH 13/43] test: changed feed response --- openapi/Swarm.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 7bcd224a7d0..6c5951de009 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -862,9 +862,10 @@ paths: "swarm-feed-index-next": $ref: "SwarmCommon.yaml#/components/headers/SwarmFeedIndexNext" content: - application/json: + application/octet-stream: schema: - $ref: "SwarmCommon.yaml#/components/schemas/ReferenceResponse" + type: string + format: binary "400": $ref: "SwarmCommon.yaml#/components/responses/400" "401": From a7a6e1e2aeb56776e6aa8b5da0ed230f1da0471f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 22 May 2024 17:36:48 +0200 Subject: [PATCH 14/43] chore: remove invalidFeedUpdate error --- pkg/api/feed.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/api/feed.go b/pkg/api/feed.go index 2bc815ce39d..58ab13eb7f0 100644 --- a/pkg/api/feed.go +++ b/pkg/api/feed.go @@ -34,8 +34,6 @@ const ( feedMetadataEntryType = "swarm-feed-type" ) -var errInvalidFeedUpdate = errors.New("invalid feed update") - type feedReferenceResponse struct { Reference swarm.Address `json:"reference"` } From c768d889d6f782c0d879939dee16f04e0a6b44ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 23 May 2024 11:50:46 +0200 Subject: [PATCH 15/43] test: make and copy wrappedRef --- pkg/api/feed_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/api/feed_test.go b/pkg/api/feed_test.go index 9cf836fc18f..e5653ba8829 100644 --- a/pkg/api/feed_test.go +++ b/pkg/api/feed_test.go @@ -126,7 +126,9 @@ func TestFeed_Get(t *testing.T) { t.Run("legacy payload with non existing wrapped chunk", func(t *testing.T) { t.Parallel() - wrappedRef := mockWrappedCh.Address().Bytes() + + wrappedRef := make([]byte, swarm.HashSize) + _ = copy(wrappedRef, mockWrappedCh.Address().Bytes()) wrappedRef[0]++ var ( From 62acd8545d3c2863ca9d1d64f6f8dcd772d1b74f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 23 May 2024 13:48:36 +0200 Subject: [PATCH 16/43] fix: remove payload time related logic --- pkg/feeds/epochs/finder.go | 39 ++++++---------------------------- pkg/feeds/getter.go | 14 ------------ pkg/feeds/sequence/sequence.go | 17 --------------- pkg/feeds/testing/lookup.go | 9 -------- 4 files changed, 6 insertions(+), 73 deletions(-) diff --git a/pkg/feeds/epochs/finder.go b/pkg/feeds/epochs/finder.go index a85ab309e76..2f378249212 100644 --- a/pkg/feeds/epochs/finder.go +++ b/pkg/feeds/epochs/finder.go @@ -7,6 +7,7 @@ package epochs import ( "context" "errors" + "fmt" "github.com/ethersphere/bee/v2/pkg/feeds" storage "github.com/ethersphere/bee/v2/pkg/storage" @@ -51,19 +52,13 @@ func (f *finder) common(ctx context.Context, at int64, after uint64) (*epoch, sw } return e, nil, err } - ts, err := feeds.UpdatedAt(ch) - if err != nil { - return e, nil, err - } - if ts <= uint64(at) { - return e, ch, nil - } + return e, ch, nil } } // at is a non-concurrent recursive Finder function to find the version update chunk at time `at` func (f *finder) at(ctx context.Context, at uint64, e *epoch, ch swarm.Chunk) (swarm.Chunk, error) { - uch, err := f.getter.Get(ctx, e) + _, err := f.getter.Get(ctx, e) if err != nil { // error retrieving if !errors.Is(err, storage.ErrNotFound) { @@ -76,23 +71,8 @@ func (f *finder) at(ctx context.Context, at uint64, e *epoch, ch swarm.Chunk) (s // traverse earlier branch return f.at(ctx, e.start-1, e.left(), ch) } - // epoch found - // check if timestamp is later then target - ts, err := feeds.UpdatedAt(uch) - if err != nil { - return nil, err - } - if ts > at { - if e.isLeft() { - return ch, nil - } - return f.at(ctx, e.start-1, e.left(), ch) - } - if e.level == 0 { // matching update time or finest resolution - return uch, nil - } - // continue traversing based on at - return f.at(ctx, at, e.childAt(at), uch) + + return nil, fmt.Errorf("not implemented") } type result struct { @@ -131,14 +111,7 @@ func (f *asyncFinder) get(ctx context.Context, at int64, e *epoch) (swarm.Chunk, } return nil, nil } - ts, err := feeds.UpdatedAt(u) - if err != nil { - return nil, err - } - diff := at - int64(ts) - if diff < 0 { - return nil, nil - } + return u, nil } diff --git a/pkg/feeds/getter.go b/pkg/feeds/getter.go index c2236c77ce1..9fcb2ccae77 100644 --- a/pkg/feeds/getter.go +++ b/pkg/feeds/getter.go @@ -95,17 +95,3 @@ func LegacyPayload(wrappedChunk swarm.Chunk) (uint64, swarm.Address, error) { return at, address, nil } - -// UpdatedAt extracts the time of feed other than update -func UpdatedAt(ch swarm.Chunk) (uint64, error) { - d := ch.Data() - if len(d) < 113 { - return 0, fmt.Errorf("too short: %d", len(d)) - } - // a soc chunk with time information in the wrapped content addressed chunk - // 0-32 index, - // 65-97 signature, - // 98-105 span of wrapped chunk - // 105-113 timestamp - return binary.BigEndian.Uint64(d[105:113]), nil -} diff --git a/pkg/feeds/sequence/sequence.go b/pkg/feeds/sequence/sequence.go index c1dc06a720c..5184885f1ab 100644 --- a/pkg/feeds/sequence/sequence.go +++ b/pkg/feeds/sequence/sequence.go @@ -79,14 +79,6 @@ func (f *finder) At(ctx context.Context, at int64, _ uint64) (ch swarm.Chunk, cu } return ch, current, &index{i}, nil } - ts, err := feeds.UpdatedAt(u) - if err != nil { - return nil, nil, nil, err - } - // if index is later than the `at` target index, then return previous chunk and index - if ts > uint64(at) { - return ch, &index{i - 1}, &index{i}, nil - } ch = u } } @@ -267,15 +259,6 @@ func (f *asyncFinder) get(ctx context.Context, at int64, idx uint64) (swarm.Chun // if 'not-found' error, then just silence and return nil chunk return nil, nil } - ts, err := feeds.UpdatedAt(u) - if err != nil { - return nil, err - } - // this means the update timestamp is later than the pivot time we are looking for - // handled as if the update was missing but with no uncertainty due to timeout - if at < int64(ts) { - return nil, nil - } return u, nil } diff --git a/pkg/feeds/testing/lookup.go b/pkg/feeds/testing/lookup.go index 1b8ff7abc2b..8fd852888c8 100644 --- a/pkg/feeds/testing/lookup.go +++ b/pkg/feeds/testing/lookup.go @@ -155,15 +155,6 @@ func TestFinderIntervals(t *testing.T, nextf func() (bool, int64), finderf func( if ch == nil { t.Fatalf("expected to find update, got none") } - wrappedChunk, err := feeds.FromChunk(ch) - payload := wrappedChunk.Data()[swarm.SpanSize:] - if err != nil { - t.Fatal(err) - } - content := binary.BigEndian.Uint64(payload) - if content != uint64(at) { - t.Fatalf("payload mismatch: expected %v, got %v", at, content) - } if current != nil { expectedId := ch.Data()[:32] From c7ddac939add6201fb9693be7657057d04b0fdbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Fri, 24 May 2024 10:21:47 +0200 Subject: [PATCH 17/43] feat: give back whole data --- pkg/api/bytes.go | 2 +- pkg/api/bzz.go | 15 ++++++++++++--- pkg/api/feed.go | 19 ++++++++----------- pkg/api/feed_test.go | 13 +++++-------- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/pkg/api/bytes.go b/pkg/api/bytes.go index 9b5d9b902d6..570b9e38b21 100644 --- a/pkg/api/bytes.go +++ b/pkg/api/bytes.go @@ -153,7 +153,7 @@ func (s *Service) bytesGetHandler(w http.ResponseWriter, r *http.Request) { ContentTypeHeader: {"application/octet-stream"}, } - s.downloadHandler(logger, w, r, paths.Address, additionalHeaders, true, false) + s.downloadHandler(logger, w, r, paths.Address, additionalHeaders, true, false, nil) } func (s *Service) bytesHeadHandler(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index 45e8c468ac3..6c24ed5e5fe 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethersphere/bee/v2/pkg/feeds" + "github.com/ethersphere/bee/v2/pkg/file" "github.com/ethersphere/bee/v2/pkg/file/joiner" "github.com/ethersphere/bee/v2/pkg/file/loadsave" "github.com/ethersphere/bee/v2/pkg/file/redundancy" @@ -516,11 +517,11 @@ func (s *Service) serveManifestEntry( additionalHeaders[ContentTypeHeader] = []string{mimeType} } - s.downloadHandler(logger, w, r, manifestEntry.Reference(), additionalHeaders, etag, headersOnly) + s.downloadHandler(logger, w, r, manifestEntry.Reference(), additionalHeaders, etag, headersOnly, nil) } // downloadHandler contains common logic for dowloading Swarm file from API -func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *http.Request, reference swarm.Address, additionalHeaders http.Header, etag, headersOnly bool) { +func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *http.Request, reference swarm.Address, additionalHeaders http.Header, etag, headersOnly bool, rootCh swarm.Chunk) { headers := struct { Strategy *getter.Strategy `map:"Swarm-Redundancy-Strategy"` FallbackMode *bool `map:"Swarm-Redundancy-Fallback-Mode"` @@ -546,7 +547,15 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h return } - reader, l, err := joiner.New(ctx, s.storer.Download(cache), s.storer.Cache(), reference) + var ( + reader file.Joiner + l int64 + ) + if rootCh != nil { + reader, l, err = joiner.NewWithRootCh(ctx, s.storer.Download(cache), s.storer.Cache(), reference, rootCh) + } else { + reader, l, err = joiner.New(ctx, s.storer.Download(cache), s.storer.Cache(), reference) + } if err != nil { if errors.Is(err, storage.ErrNotFound) || errors.Is(err, topology.ErrNotFound) { logger.Debug("api download: not found ", "address", reference, "error", err) diff --git a/pkg/api/feed.go b/pkg/api/feed.go index 58ab13eb7f0..f5a7b482fc7 100644 --- a/pkg/api/feed.go +++ b/pkg/api/feed.go @@ -5,13 +5,9 @@ package api import ( - "bytes" "encoding/hex" "errors" - "fmt" - "io" "net/http" - "strconv" "time" "github.com/ethereum/go-ethereum/common" @@ -98,7 +94,6 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { jsonhttp.NotFound(w, "wrapped chunk cannot be retrieved") return } - wData := wc.Data() if err != nil { logger.Debug("mapStructure feed update failed", "error", err) logger.Error(nil, "mapStructure feed update failed") @@ -122,12 +117,14 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { return } - contenL := len(wData) - w.Header().Set(ContentLengthHeader, strconv.Itoa(contenL)) - w.Header().Set(SwarmFeedIndexHeader, hex.EncodeToString(curBytes)) - w.Header().Set(SwarmFeedIndexNextHeader, hex.EncodeToString(nextBytes)) - w.Header().Set("Access-Control-Expose-Headers", fmt.Sprintf("%s, %s", SwarmFeedIndexHeader, SwarmFeedIndexNextHeader)) - _, _ = io.Copy(w, bytes.NewReader(wData)) + additionalHeaders := http.Header{ + ContentTypeHeader: {"application/octet-stream"}, + SwarmFeedIndexHeader: {hex.EncodeToString(curBytes)}, + SwarmFeedIndexNextHeader: {hex.EncodeToString(nextBytes)}, + "Access-Control-Expose-Headers": {SwarmFeedIndexHeader, SwarmFeedIndexNextHeader}, + } + + s.downloadHandler(logger, w, r, wc.Address(), additionalHeaders, true, false, wc) } func (s *Service) feedPostHandler(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/api/feed_test.go b/pkg/api/feed_test.go index e5653ba8829..501c8ab0bd2 100644 --- a/pkg/api/feed_test.go +++ b/pkg/api/feed_test.go @@ -15,7 +15,6 @@ import ( "testing" "github.com/ethersphere/bee/v2/pkg/api" - "github.com/ethersphere/bee/v2/pkg/bmt" "github.com/ethersphere/bee/v2/pkg/feeds" "github.com/ethersphere/bee/v2/pkg/file/loadsave" "github.com/ethersphere/bee/v2/pkg/jsonhttp" @@ -72,7 +71,7 @@ func TestFeed_Get(t *testing.T) { ) jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "12"), http.StatusOK, - jsonhttptest.WithExpectedResponse(mockWrappedCh.Data()), + jsonhttptest.WithExpectedResponse(mockWrappedCh.Data()[swarm.SpanSize:]), jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), ) }) @@ -94,16 +93,14 @@ func TestFeed_Get(t *testing.T) { ) jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, - jsonhttptest.WithExpectedResponse(mockWrappedCh.Data()), - jsonhttptest.WithExpectedContentLength(len(mockWrappedCh.Data())), + jsonhttptest.WithExpectedResponse(mockWrappedCh.Data()[swarm.SpanSize:]), + jsonhttptest.WithExpectedContentLength(len(mockWrappedCh.Data()[swarm.SpanSize:])), jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), ) }) t.Run("chunk wrapping", func(t *testing.T) { testData := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8} - testDataSpan := bmt.LengthToSpan(int64(len(testData))) - expectedData := append(testDataSpan, testData...) var ( ch = testingsoc.GenerateMockSOC(t, testData).Chunk() @@ -118,8 +115,8 @@ func TestFeed_Get(t *testing.T) { ) jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, - jsonhttptest.WithExpectedResponse(expectedData), - jsonhttptest.WithExpectedContentLength(len(expectedData)), + jsonhttptest.WithExpectedResponse(testData), + jsonhttptest.WithExpectedContentLength(len(testData)), jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), ) }) From dae4346ec4680ccd813ce87f0cc97b627ec4499f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Sat, 25 May 2024 19:25:34 +0200 Subject: [PATCH 18/43] chore: resolve linter issues --- 1 | 53 +++++++++++++++++++++++++++++++++++++++ pkg/api/feed.go | 6 ----- pkg/file/joiner/joiner.go | 6 ++--- 3 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 1 diff --git a/1 b/1 new file mode 100644 index 00000000000..254944e86db --- /dev/null +++ b/1 @@ -0,0 +1,53 @@ +Go is a tool for managing Go source code. + +Usage: + + go [arguments] + +The commands are: + + bug start a bug report + build compile packages and dependencies + clean remove object files and cached files + doc show documentation for package or symbol + env print Go environment information + fix update packages to use new APIs + fmt gofmt (reformat) package sources + generate generate Go files by processing source + get add dependencies to current module and install them + install compile and install packages and dependencies + list list packages or modules + mod module maintenance + work workspace maintenance + run compile and run Go program + test test packages + tool run specified go tool + version print Go version + vet report likely mistakes in packages + +Use "go help " for more information about a command. + +Additional help topics: + + buildconstraint build constraints + buildmode build modes + c calling between Go and C + cache build and test caching + environment environment variables + filetype file types + go.mod the go.mod file + gopath GOPATH environment variable + gopath-get legacy GOPATH go get + goproxy module proxy protocol + importpath import path syntax + modules modules, module versions, and more + module-get module-aware go get + module-auth module authentication using go.sum + packages package lists and patterns + private configuration for downloading non-public code + testflag testing flags + testfunc testing functions + vcs controlling version control with GOVCS + +Use "go help " for more information about that topic. + diff --git a/pkg/api/feed.go b/pkg/api/feed.go index f5a7b482fc7..43256bf59a9 100644 --- a/pkg/api/feed.go +++ b/pkg/api/feed.go @@ -94,12 +94,6 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { jsonhttp.NotFound(w, "wrapped chunk cannot be retrieved") return } - if err != nil { - logger.Debug("mapStructure feed update failed", "error", err) - logger.Error(nil, "mapStructure feed update failed") - jsonhttp.InternalServerError(w, "mapStructure feed update failed") - return - } curBytes, err := cur.MarshalBinary() if err != nil { diff --git a/pkg/file/joiner/joiner.go b/pkg/file/joiner/joiner.go index ca05366e038..f103759dda1 100644 --- a/pkg/file/joiner/joiner.go +++ b/pkg/file/joiner/joiner.go @@ -106,7 +106,7 @@ func (g *decoderCache) GetOrCreate(addrs []swarm.Address, shardCnt int) storage. // NewWithRoot creates a new Joiner with the already fetched root chunk. // A Joiner provides Read, Seek and Size functionalities. func NewWithRootCh(ctx context.Context, g storage.Getter, putter storage.Putter, rootAddr swarm.Address, rootChunk swarm.Chunk) (file.Joiner, int64, error) { - return new(ctx, g, putter, rootAddr, rootChunk) + return newJoiner(ctx, g, putter, rootAddr, rootChunk) } // New creates a new Joiner. A Joiner provides Read, Seek and Size functionalities. @@ -122,10 +122,10 @@ func New(ctx context.Context, g storage.Getter, putter storage.Putter, address s return nil, 0, err } - return new(ctx, g, putter, address, rootChunk) + return newJoiner(ctx, g, putter, address, rootChunk) } -func new(ctx context.Context, g storage.Getter, putter storage.Putter, address swarm.Address, rootChunk swarm.Chunk) (file.Joiner, int64, error) { +func newJoiner(ctx context.Context, g storage.Getter, putter storage.Putter, address swarm.Address, rootChunk swarm.Chunk) (file.Joiner, int64, error) { chunkData := rootChunk.Data() rootData := chunkData[swarm.SpanSize:] refLength := len(address.Bytes()) From 27372fb7a10b1398f2d1cf69f738fe63a36f77a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Mon, 27 May 2024 11:02:03 +0200 Subject: [PATCH 19/43] test: deactivate epoch based finder test --- pkg/feeds/epochs/lookup_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/feeds/epochs/lookup_test.go b/pkg/feeds/epochs/lookup_test.go index e89d8b62b74..a7653de3bc5 100644 --- a/pkg/feeds/epochs/lookup_test.go +++ b/pkg/feeds/epochs/lookup_test.go @@ -16,6 +16,7 @@ import ( func TestFinder_FLAKY(t *testing.T) { t.Parallel() + t.Skip() // TODO fix epoch based feeds testf := func(t *testing.T, finderf func(storage.Getter, *feeds.Feed) feeds.Lookup, updaterf func(putter storage.Putter, signer crypto.Signer, topic []byte) (feeds.Updater, error)) { t.Helper() From 652a89e6aa6629d015b01bb085f56389164e1614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Mon, 27 May 2024 18:04:03 +0200 Subject: [PATCH 20/43] feat: generate mock soc with span --- pkg/soc/testing/soc.go | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/pkg/soc/testing/soc.go b/pkg/soc/testing/soc.go index e5325f464b1..d0f82b590c7 100644 --- a/pkg/soc/testing/soc.go +++ b/pkg/soc/testing/soc.go @@ -36,17 +36,35 @@ func (ms MockSOC) Chunk() swarm.Chunk { func GenerateMockSOC(t *testing.T, data []byte) *MockSOC { t.Helper() - privKey, err := crypto.GenerateSecp256k1Key() + ch, err := cac.New(data) if err != nil { t.Fatal(err) } - signer := crypto.NewDefaultSigner(privKey) - owner, err := signer.EthereumAddress() + + return generateMockSOC(t, ch) +} + +// GenerateMockSOC generates a valid mocked SOC from given chunk data (span + payload). +func GenerateMockSOCWithSpan(t *testing.T, data []byte) *MockSOC { + t.Helper() + + ch, err := cac.NewWithDataSpan(data) if err != nil { t.Fatal(err) } - ch, err := cac.New(data) + return generateMockSOC(t, ch) +} + +func generateMockSOC(t *testing.T, ch swarm.Chunk) *MockSOC { + t.Helper() + + privKey, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + signer := crypto.NewDefaultSigner(privKey) + owner, err := signer.EthereumAddress() if err != nil { t.Fatal(err) } From 7989e4dc030823ee65b0e3ac1d56a3c2ddbfccf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Mon, 27 May 2024 18:05:29 +0200 Subject: [PATCH 21/43] test: resolve content that takes more than one chunk --- pkg/api/feed_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/pkg/api/feed_test.go b/pkg/api/feed_test.go index 501c8ab0bd2..8c2f306d0ba 100644 --- a/pkg/api/feed_test.go +++ b/pkg/api/feed_test.go @@ -5,11 +5,13 @@ package api_test import ( + "bytes" "context" "encoding/binary" "encoding/hex" "errors" "fmt" + "io" "math/big" "net/http" "testing" @@ -17,6 +19,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/api" "github.com/ethersphere/bee/v2/pkg/feeds" "github.com/ethersphere/bee/v2/pkg/file/loadsave" + "github.com/ethersphere/bee/v2/pkg/file/splitter" "github.com/ethersphere/bee/v2/pkg/jsonhttp" "github.com/ethersphere/bee/v2/pkg/jsonhttp/jsonhttptest" "github.com/ethersphere/bee/v2/pkg/log" @@ -27,6 +30,7 @@ import ( testingc "github.com/ethersphere/bee/v2/pkg/storage/testing" mockstorer "github.com/ethersphere/bee/v2/pkg/storer/mock" "github.com/ethersphere/bee/v2/pkg/swarm" + "github.com/ethersphere/bee/v2/pkg/util/testutil" ) const ownerString = "8d3766440f0d7b949a5e32995d09619a7f86e632" @@ -141,6 +145,41 @@ func TestFeed_Get(t *testing.T) { jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusNotFound) }) + + t.Run("bigger payload than one chunk", func(t *testing.T) { + t.Parallel() + + testDataLen := 5000 + testData := testutil.RandBytesWithSeed(t, testDataLen, 1) + s := splitter.NewSimpleSplitter(putter) + addr, err := s.Split(context.Background(), io.NopCloser(bytes.NewReader(testData)), int64(testDataLen), false) + if err != nil { + t.Fatal(err) + } + + // get root ch addr then add wrap it with soc + testRootCh, err := mockStorer.ChunkStore().Get(context.Background(), addr) + if err != nil { + t.Fatal(err) + } + var ( + ch = testingsoc.GenerateMockSOCWithSpan(t, testRootCh.Data()).Chunk() + look = newMockLookup(-1, 2, ch, nil, &id{}, &id{}) + factory = newMockFactory(look) + idBytes, _ = (&id{}).MarshalBinary() + + client, _, _, _ = newTestServer(t, testServerOptions{ + Storer: mockStorer, + Feeds: factory, + }) + ) + + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, + jsonhttptest.WithExpectedResponse(testData), + jsonhttptest.WithExpectedContentLength(testDataLen), + jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), + ) + }) } // nolint:paralleltest From d1141256e9e1f33813f4a023bed131abf2e8b0f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Tue, 28 May 2024 17:10:58 +0200 Subject: [PATCH 22/43] chore: remove generated file --- 1 | 53 ----------------------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 1 diff --git a/1 b/1 deleted file mode 100644 index 254944e86db..00000000000 --- a/1 +++ /dev/null @@ -1,53 +0,0 @@ -Go is a tool for managing Go source code. - -Usage: - - go [arguments] - -The commands are: - - bug start a bug report - build compile packages and dependencies - clean remove object files and cached files - doc show documentation for package or symbol - env print Go environment information - fix update packages to use new APIs - fmt gofmt (reformat) package sources - generate generate Go files by processing source - get add dependencies to current module and install them - install compile and install packages and dependencies - list list packages or modules - mod module maintenance - work workspace maintenance - run compile and run Go program - test test packages - tool run specified go tool - version print Go version - vet report likely mistakes in packages - -Use "go help " for more information about a command. - -Additional help topics: - - buildconstraint build constraints - buildmode build modes - c calling between Go and C - cache build and test caching - environment environment variables - filetype file types - go.mod the go.mod file - gopath GOPATH environment variable - gopath-get legacy GOPATH go get - goproxy module proxy protocol - importpath import path syntax - modules modules, module versions, and more - module-get module-aware go get - module-auth module authentication using go.sum - packages package lists and patterns - private configuration for downloading non-public code - testflag testing flags - testfunc testing functions - vcs controlling version control with GOVCS - -Use "go help " for more information about that topic. - From b8d6ee2e04efacba2ed0b2ad6b90a475ef6737c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Tue, 28 May 2024 17:33:48 +0200 Subject: [PATCH 23/43] fix: epoch finder --- pkg/feeds/epochs/finder.go | 19 +++++++++++++++---- pkg/feeds/epochs/lookup_test.go | 1 - 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/feeds/epochs/finder.go b/pkg/feeds/epochs/finder.go index 2f378249212..e9de47efea5 100644 --- a/pkg/feeds/epochs/finder.go +++ b/pkg/feeds/epochs/finder.go @@ -7,7 +7,6 @@ package epochs import ( "context" "errors" - "fmt" "github.com/ethersphere/bee/v2/pkg/feeds" storage "github.com/ethersphere/bee/v2/pkg/storage" @@ -58,7 +57,7 @@ func (f *finder) common(ctx context.Context, at int64, after uint64) (*epoch, sw // at is a non-concurrent recursive Finder function to find the version update chunk at time `at` func (f *finder) at(ctx context.Context, at uint64, e *epoch, ch swarm.Chunk) (swarm.Chunk, error) { - _, err := f.getter.Get(ctx, e) + uch, err := f.getter.Get(ctx, e) if err != nil { // error retrieving if !errors.Is(err, storage.ErrNotFound) { @@ -71,8 +70,20 @@ func (f *finder) at(ctx context.Context, at uint64, e *epoch, ch swarm.Chunk) (s // traverse earlier branch return f.at(ctx, e.start-1, e.left(), ch) } - - return nil, fmt.Errorf("not implemented") + // epoch found + // check if timestamp is later then target + ts := e.length() * e.start + if ts > at { + if e.isLeft() { + return ch, nil + } + return f.at(ctx, e.start-1, e.left(), ch) + } + if e.level == 0 { // matching update time or finest resolution + return uch, nil + } + // continue traversing based on at + return f.at(ctx, at, e.childAt(at), uch) } type result struct { diff --git a/pkg/feeds/epochs/lookup_test.go b/pkg/feeds/epochs/lookup_test.go index a7653de3bc5..e89d8b62b74 100644 --- a/pkg/feeds/epochs/lookup_test.go +++ b/pkg/feeds/epochs/lookup_test.go @@ -16,7 +16,6 @@ import ( func TestFinder_FLAKY(t *testing.T) { t.Parallel() - t.Skip() // TODO fix epoch based feeds testf := func(t *testing.T, finderf func(storage.Getter, *feeds.Feed) feeds.Lookup, updaterf func(putter storage.Putter, signer crypto.Signer, topic []byte) (feeds.Updater, error)) { t.Helper() From 1a113373aa1b9762c4d00f991068ffd2f9013675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 29 May 2024 14:26:42 +0200 Subject: [PATCH 24/43] feat: only wrapped chunk --- pkg/api/api.go | 1 + pkg/api/feed.go | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/pkg/api/api.go b/pkg/api/api.go index 72100cc0d9b..74c557cf49d 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -77,6 +77,7 @@ const ( SwarmErrorDocumentHeader = "Swarm-Error-Document" SwarmFeedIndexHeader = "Swarm-Feed-Index" SwarmFeedIndexNextHeader = "Swarm-Feed-Index-Next" + SwarmOnlyWrappedChunk = "Swarm-Only-Wrapped-Chunk" SwarmCollectionHeader = "Swarm-Collection" SwarmPostageBatchIdHeader = "Swarm-Postage-Batch-Id" SwarmDeferredUploadHeader = "Swarm-Deferred-Upload" diff --git a/pkg/api/feed.go b/pkg/api/feed.go index 43256bf59a9..86f2eae025e 100644 --- a/pkg/api/feed.go +++ b/pkg/api/feed.go @@ -5,9 +5,13 @@ package api import ( + "bytes" "encoding/hex" "errors" + "io" "net/http" + "strconv" + "strings" "time" "github.com/ethereum/go-ethereum/common" @@ -58,6 +62,14 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { queries.At = time.Now().Unix() } + headers := struct { + OnlyWrappedChunk bool `map:"Swarm-Only-Wrapped-Chunk"` + }{} + if response := s.mapStructure(r.Header, &headers); response != nil { + response("invalid header params", logger, w) + return + } + f := feeds.New(paths.Topic, paths.Owner) lookup, err := s.feedFactory.NewLookup(feeds.Sequence, f) if err != nil { @@ -118,6 +130,16 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { "Access-Control-Expose-Headers": {SwarmFeedIndexHeader, SwarmFeedIndexNextHeader}, } + if headers.OnlyWrappedChunk { + w.Header().Set(ContentLengthHeader, strconv.Itoa(len(wc.Data()))) + // include additional headers + for name, values := range additionalHeaders { + w.Header().Set(name, strings.Join(values, "; ")) + } + _, _ = io.Copy(w, bytes.NewReader(wc.Data())) + return + } + s.downloadHandler(logger, w, r, wc.Address(), additionalHeaders, true, false, wc) } From 381f7bc197ae1178a585c16c268700bdc62c6804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 29 May 2024 14:29:03 +0200 Subject: [PATCH 25/43] test: only wrapped chunk --- pkg/api/feed_test.go | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/pkg/api/feed_test.go b/pkg/api/feed_test.go index 8c2f306d0ba..7f4e5a34946 100644 --- a/pkg/api/feed_test.go +++ b/pkg/api/feed_test.go @@ -174,11 +174,26 @@ func TestFeed_Get(t *testing.T) { }) ) - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, - jsonhttptest.WithExpectedResponse(testData), - jsonhttptest.WithExpectedContentLength(testDataLen), - jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), - ) + t.Run("retrieve chunk tree", func(t *testing.T) { + t.Parallel() + + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, + jsonhttptest.WithExpectedResponse(testData), + jsonhttptest.WithExpectedContentLength(testDataLen), + jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), + ) + }) + + t.Run("retrieve only wrapped chunk", func(t *testing.T) { + t.Parallel() + + jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, + jsonhttptest.WithRequestHeader(api.SwarmOnlyWrappedChunk, "true"), + jsonhttptest.WithExpectedResponse(testRootCh.Data()), + jsonhttptest.WithExpectedContentLength(len(testRootCh.Data())), + jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), + ) + }) }) } From c06cc249b41387e17a0b82be3a12dbc147c404a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 29 May 2024 17:28:26 +0200 Subject: [PATCH 26/43] refactor: only wrapped chunk to only root chunk --- pkg/api/api.go | 2 +- pkg/api/feed.go | 4 ++-- pkg/api/feed_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 74c557cf49d..5709ab0ef76 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -77,7 +77,7 @@ const ( SwarmErrorDocumentHeader = "Swarm-Error-Document" SwarmFeedIndexHeader = "Swarm-Feed-Index" SwarmFeedIndexNextHeader = "Swarm-Feed-Index-Next" - SwarmOnlyWrappedChunk = "Swarm-Only-Wrapped-Chunk" + SwarmOnlyRootChunk = "Swarm-Only-Root-Chunk" SwarmCollectionHeader = "Swarm-Collection" SwarmPostageBatchIdHeader = "Swarm-Postage-Batch-Id" SwarmDeferredUploadHeader = "Swarm-Deferred-Upload" diff --git a/pkg/api/feed.go b/pkg/api/feed.go index 86f2eae025e..1e9bc8038b9 100644 --- a/pkg/api/feed.go +++ b/pkg/api/feed.go @@ -63,7 +63,7 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { } headers := struct { - OnlyWrappedChunk bool `map:"Swarm-Only-Wrapped-Chunk"` + OnlyRootChunk bool `map:"Swarm-Only-Root-Chunk"` }{} if response := s.mapStructure(r.Header, &headers); response != nil { response("invalid header params", logger, w) @@ -130,7 +130,7 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { "Access-Control-Expose-Headers": {SwarmFeedIndexHeader, SwarmFeedIndexNextHeader}, } - if headers.OnlyWrappedChunk { + if headers.OnlyRootChunk { w.Header().Set(ContentLengthHeader, strconv.Itoa(len(wc.Data()))) // include additional headers for name, values := range additionalHeaders { diff --git a/pkg/api/feed_test.go b/pkg/api/feed_test.go index 7f4e5a34946..d4425890ef2 100644 --- a/pkg/api/feed_test.go +++ b/pkg/api/feed_test.go @@ -188,7 +188,7 @@ func TestFeed_Get(t *testing.T) { t.Parallel() jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, - jsonhttptest.WithRequestHeader(api.SwarmOnlyWrappedChunk, "true"), + jsonhttptest.WithRequestHeader(api.SwarmOnlyRootChunk, "true"), jsonhttptest.WithExpectedResponse(testRootCh.Data()), jsonhttptest.WithExpectedContentLength(len(testRootCh.Data())), jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)), From 5215af93b71d324c1645d500c178b92a49d9a4bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 29 May 2024 20:34:32 +0200 Subject: [PATCH 27/43] docs: only root chunk --- openapi/Swarm.yaml | 2 ++ openapi/SwarmCommon.yaml | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 6c5951de009..867f73666cf 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -861,6 +861,8 @@ paths: $ref: "SwarmCommon.yaml#/components/headers/SwarmFeedIndex" "swarm-feed-index-next": $ref: "SwarmCommon.yaml#/components/headers/SwarmFeedIndexNext" + "swarm-only-root-chunk": + $ref: "SwarmCommon.yaml#/components/headers/SwarmOnlyRootChunk" content: application/octet-stream: schema: diff --git a/openapi/SwarmCommon.yaml b/openapi/SwarmCommon.yaml index 84d690fc64f..a754165716d 100644 --- a/openapi/SwarmCommon.yaml +++ b/openapi/SwarmCommon.yaml @@ -977,6 +977,11 @@ components: schema: $ref: "#/components/schemas/HexString" + SwarmOnlyRootChunk: + description: "Returns only the root chunk of the content" + schema: + type: boolean + ETag: description: | The RFC7232 ETag header field in a response provides the current entity- From e8c2ff6309ce68285903f2a73ec71b47cdbc3dbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 29 May 2024 20:43:06 +0200 Subject: [PATCH 28/43] docs: add remaining redundancy related header options --- openapi/Swarm.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 867f73666cf..94930e31930 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -853,6 +853,10 @@ paths: $ref: "SwarmCommon.yaml#/components/schemas/FeedType" required: false description: "Feed indexing scheme (default: sequence)" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmCache" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyStrategyParameter" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyFallbackModeParameter" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmChunkRetrievalTimeoutParameter" responses: "200": description: Latest feed update From 012607600f8f01e127a74e844a7471bd97224b75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 29 May 2024 20:55:30 +0200 Subject: [PATCH 29/43] fix: epoch ts def --- pkg/feeds/epochs/finder.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/feeds/epochs/finder.go b/pkg/feeds/epochs/finder.go index e9de47efea5..d5133fdaa57 100644 --- a/pkg/feeds/epochs/finder.go +++ b/pkg/feeds/epochs/finder.go @@ -51,7 +51,10 @@ func (f *finder) common(ctx context.Context, at int64, after uint64) (*epoch, sw } return e, nil, err } - return e, ch, nil + ts := e.length() * e.start + if ts <= uint64(at) { + return e, ch, nil + } } } @@ -122,7 +125,11 @@ func (f *asyncFinder) get(ctx context.Context, at int64, e *epoch) (swarm.Chunk, } return nil, nil } - + ts := e.length() * e.start + diff := at - int64(ts) + if diff < 0 { + return nil, nil + } return u, nil } From 4e1aea75f8980b49967e62487f7c586ac6429cb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Wed, 29 May 2024 21:28:06 +0200 Subject: [PATCH 30/43] test: parallel --- pkg/api/feed_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/api/feed_test.go b/pkg/api/feed_test.go index d4425890ef2..843756d7237 100644 --- a/pkg/api/feed_test.go +++ b/pkg/api/feed_test.go @@ -104,6 +104,8 @@ func TestFeed_Get(t *testing.T) { }) t.Run("chunk wrapping", func(t *testing.T) { + t.Parallel() + testData := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8} var ( @@ -175,8 +177,6 @@ func TestFeed_Get(t *testing.T) { ) t.Run("retrieve chunk tree", func(t *testing.T) { - t.Parallel() - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, jsonhttptest.WithExpectedResponse(testData), jsonhttptest.WithExpectedContentLength(testDataLen), @@ -185,8 +185,6 @@ func TestFeed_Get(t *testing.T) { }) t.Run("retrieve only wrapped chunk", func(t *testing.T) { - t.Parallel() - jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK, jsonhttptest.WithRequestHeader(api.SwarmOnlyRootChunk, "true"), jsonhttptest.WithExpectedResponse(testRootCh.Data()), From 061bbcacacc37ddc5922469fe7fdedbb64d6119b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 6 Jun 2024 06:52:02 +0200 Subject: [PATCH 31/43] fix: typeo on additional header set --- pkg/api/feed.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/feed.go b/pkg/api/feed.go index 1e9bc8038b9..04103fcdebe 100644 --- a/pkg/api/feed.go +++ b/pkg/api/feed.go @@ -134,7 +134,7 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set(ContentLengthHeader, strconv.Itoa(len(wc.Data()))) // include additional headers for name, values := range additionalHeaders { - w.Header().Set(name, strings.Join(values, "; ")) + w.Header().Set(name, strings.Join(values, ", ")) } _, _ = io.Copy(w, bytes.NewReader(wc.Data())) return From ea0dfb352c1245199458f793fabe42babe65de65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 6 Jun 2024 16:47:08 +0200 Subject: [PATCH 32/43] refactor: remove joiner wrapper --- pkg/file/joiner/joiner.go | 12 ++++-------- pkg/file/loadsave/loadsave.go | 2 +- pkg/node/bootstrap.go | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/pkg/file/joiner/joiner.go b/pkg/file/joiner/joiner.go index f103759dda1..4644f95ca0b 100644 --- a/pkg/file/joiner/joiner.go +++ b/pkg/file/joiner/joiner.go @@ -103,12 +103,6 @@ func (g *decoderCache) GetOrCreate(addrs []swarm.Address, shardCnt int) storage. return d } -// NewWithRoot creates a new Joiner with the already fetched root chunk. -// A Joiner provides Read, Seek and Size functionalities. -func NewWithRootCh(ctx context.Context, g storage.Getter, putter storage.Putter, rootAddr swarm.Address, rootChunk swarm.Chunk) (file.Joiner, int64, error) { - return newJoiner(ctx, g, putter, rootAddr, rootChunk) -} - // New creates a new Joiner. A Joiner provides Read, Seek and Size functionalities. func New(ctx context.Context, g storage.Getter, putter storage.Putter, address swarm.Address) (file.Joiner, int64, error) { // retrieve the root chunk to read the total data length the be retrieved @@ -122,10 +116,12 @@ func New(ctx context.Context, g storage.Getter, putter storage.Putter, address s return nil, 0, err } - return newJoiner(ctx, g, putter, address, rootChunk) + return NewJoiner(ctx, g, putter, address, rootChunk) } -func newJoiner(ctx context.Context, g storage.Getter, putter storage.Putter, address swarm.Address, rootChunk swarm.Chunk) (file.Joiner, int64, error) { +// NewWithRoot creates a new Joiner with the already fetched root chunk. +// A Joiner provides Read, Seek and Size functionalities. +func NewJoiner(ctx context.Context, g storage.Getter, putter storage.Putter, address swarm.Address, rootChunk swarm.Chunk) (file.Joiner, int64, error) { chunkData := rootChunk.Data() rootData := chunkData[swarm.SpanSize:] refLength := len(address.Bytes()) diff --git a/pkg/file/loadsave/loadsave.go b/pkg/file/loadsave/loadsave.go index d329fb870e3..2f94eae8eec 100644 --- a/pkg/file/loadsave/loadsave.go +++ b/pkg/file/loadsave/loadsave.go @@ -67,7 +67,7 @@ func (ls *loadSave) Load(ctx context.Context, ref []byte) ([]byte, error) { } j = joiner } else { - joiner, _, err := joiner.NewWithRootCh(ctx, ls.getter, ls.putter, swarm.NewAddress(ref), ls.rootCh) + joiner, _, err := joiner.NewJoiner(ctx, ls.getter, ls.putter, swarm.NewAddress(ref), ls.rootCh) if err != nil { return nil, err } diff --git a/pkg/node/bootstrap.go b/pkg/node/bootstrap.go index f6d8690466a..cc5748e1888 100644 --- a/pkg/node/bootstrap.go +++ b/pkg/node/bootstrap.go @@ -229,7 +229,7 @@ func bootstrapNode( ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - reader, l, err = joiner.NewWithRootCh(ctx, localStore.Download(true), localStore.Cache(), snapshotRootCh.Address(), snapshotRootCh) + reader, l, err = joiner.NewJoiner(ctx, localStore.Download(true), localStore.Cache(), snapshotRootCh.Address(), snapshotRootCh) if err != nil { logger.Warning("bootstrap: file joiner failed", "error", err) continue From 9d99af1e8cd8bfb353b316d0d69b47a2c09782ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 6 Jun 2024 16:47:23 +0200 Subject: [PATCH 33/43] feat: soc get api --- pkg/api/api.go | 1 + pkg/api/bzz.go | 2 +- pkg/api/router.go | 3 +++ pkg/api/soc.go | 60 +++++++++++++++++++++++++++++++++++++++++++++ pkg/api/soc_test.go | 33 ++++++++++++++++++------- 5 files changed, 89 insertions(+), 10 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 5709ab0ef76..163888c64aa 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -75,6 +75,7 @@ const ( SwarmEncryptHeader = "Swarm-Encrypt" SwarmIndexDocumentHeader = "Swarm-Index-Document" SwarmErrorDocumentHeader = "Swarm-Error-Document" + SwarmSocSignature = "Swarm-Soc-Signature" SwarmFeedIndexHeader = "Swarm-Feed-Index" SwarmFeedIndexNextHeader = "Swarm-Feed-Index-Next" SwarmOnlyRootChunk = "Swarm-Only-Root-Chunk" diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index 6c24ed5e5fe..985ddb97213 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -552,7 +552,7 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h l int64 ) if rootCh != nil { - reader, l, err = joiner.NewWithRootCh(ctx, s.storer.Download(cache), s.storer.Cache(), reference, rootCh) + reader, l, err = joiner.NewJoiner(ctx, s.storer.Download(cache), s.storer.Cache(), reference, rootCh) } else { reader, l, err = joiner.New(ctx, s.storer.Download(cache), s.storer.Cache(), reference) } diff --git a/pkg/api/router.go b/pkg/api/router.go index 232920e9090..3b6101e68e9 100644 --- a/pkg/api/router.go +++ b/pkg/api/router.go @@ -68,10 +68,12 @@ func (s *Service) MountAPI() { "/bytes", "/chunks", "/feeds", + "/soc", rootPath + "/bzz", rootPath + "/bytes", rootPath + "/chunks", rootPath + "/feeds", + rootPath + "/soc", } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -242,6 +244,7 @@ func (s *Service) mountAPI() { }) handle("/soc/{owner}/{id}", jsonhttp.MethodHandler{ + "GET": http.HandlerFunc(s.socGetHandler), "POST": web.ChainHandlers( jsonhttp.NewMaxBodyBytesHandler(swarm.ChunkWithSpanSize), web.FinalHandlerFunc(s.socUploadHandler), diff --git a/pkg/api/soc.go b/pkg/api/soc.go index 0abf338deb9..16d83854966 100644 --- a/pkg/api/soc.go +++ b/pkg/api/soc.go @@ -5,9 +5,13 @@ package api import ( + "bytes" + "encoding/hex" "errors" "io" "net/http" + "strconv" + "strings" "github.com/ethersphere/bee/v2/pkg/cac" "github.com/ethersphere/bee/v2/pkg/jsonhttp" @@ -173,3 +177,59 @@ func (s *Service) socUploadHandler(w http.ResponseWriter, r *http.Request) { jsonhttp.Created(w, chunkAddressResponse{Reference: sch.Address()}) } + +func (s *Service) socGetHandler(w http.ResponseWriter, r *http.Request) { + logger := s.logger.WithName("get_soc").Build() + + paths := struct { + Owner []byte `map:"owner" validate:"required"` + ID []byte `map:"id" validate:"required"` + }{} + if response := s.mapStructure(mux.Vars(r), &paths); response != nil { + response("invalid path params", logger, w) + return + } + + headers := struct { + OnlyRootChunk bool `map:"Swarm-Only-Root-Chunk"` + }{} + if response := s.mapStructure(r.Header, &headers); response != nil { + response("invalid header params", logger, w) + return + } + + address, err := soc.CreateAddress(paths.ID, paths.Owner) + if err != nil { + jsonhttp.BadRequest(w, "soc address cannot be created") + } + + sch, err := s.storer.ChunkStore().Get(r.Context(), address) + if err != nil { + jsonhttp.NotFound(w, "requested chunk cannot be retrievable") + } + socCh, err := soc.FromChunk(sch) + if err != nil { + jsonhttp.InternalServerError(w, "chunk is not a single owner chunk") + } + + sig := socCh.Signature() + wc := socCh.WrappedChunk() + + additionalHeaders := http.Header{ + ContentTypeHeader: {"application/octet-stream"}, + SwarmSocSignature: {hex.EncodeToString(sig)}, + "Access-Control-Expose-Headers": {SwarmSocSignature}, + } + + if headers.OnlyRootChunk { + w.Header().Set(ContentLengthHeader, strconv.Itoa(len(wc.Data()))) + // include additional headers + for name, values := range additionalHeaders { + w.Header().Set(name, strings.Join(values, ", ")) + } + _, _ = io.Copy(w, bytes.NewReader(wc.Data())) + return + } + + s.downloadHandler(logger, w, r, wc.Address(), additionalHeaders, true, false, wc) +} diff --git a/pkg/api/soc_test.go b/pkg/api/soc_test.go index 79e72ffb603..55e1dfe0a17 100644 --- a/pkg/api/soc_test.go +++ b/pkg/api/soc_test.go @@ -85,16 +85,31 @@ func TestSOC(t *testing.T) { ) // try to fetch the same chunk - rsrc := fmt.Sprintf("/chunks/" + s.Address().String()) - resp := request(t, client, http.MethodGet, rsrc, nil, http.StatusOK) - data, err := io.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } + t.Run("chunks fetch", func(t *testing.T) { + rsrc := fmt.Sprintf("/chunks/" + s.Address().String()) + resp := request(t, client, http.MethodGet, rsrc, nil, http.StatusOK) + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(s.Chunk().Data(), data) { + t.Fatal("data retrieved doesn't match uploaded content") + } + }) - if !bytes.Equal(s.Chunk().Data(), data) { - t.Fatal("data retrieved doesn't match uploaded content") - } + t.Run("soc fetch", func(t *testing.T) { + rsrc := fmt.Sprintf("/soc/%s/%s", hex.EncodeToString(s.Owner), hex.EncodeToString(s.ID)) + resp := request(t, client, http.MethodGet, rsrc, nil, http.StatusOK) + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(s.WrappedChunk.Data()[swarm.SpanSize:], data) { + t.Fatal("data retrieved doesn't match uploaded content") + } + }) }) t.Run("postage", func(t *testing.T) { From df16c0cd2b9c6799716bac1a30abc5d72fc5d904 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Fri, 7 Jun 2024 21:33:09 +0200 Subject: [PATCH 34/43] fix: return on error and chunk get --- pkg/api/soc.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/api/soc.go b/pkg/api/soc.go index 16d83854966..e971575413a 100644 --- a/pkg/api/soc.go +++ b/pkg/api/soc.go @@ -200,16 +200,23 @@ func (s *Service) socGetHandler(w http.ResponseWriter, r *http.Request) { address, err := soc.CreateAddress(paths.ID, paths.Owner) if err != nil { + logger.Error(err, "soc address cannot be created") jsonhttp.BadRequest(w, "soc address cannot be created") + return } - sch, err := s.storer.ChunkStore().Get(r.Context(), address) + getter := s.storer.Download(true) + sch, err := getter.Get(r.Context(), address) if err != nil { - jsonhttp.NotFound(w, "requested chunk cannot be retrievable") + logger.Error(err, "soc retrieval has been failed") + jsonhttp.NotFound(w, "requested chunk cannot be retrieved") + return } socCh, err := soc.FromChunk(sch) if err != nil { + logger.Error(err, "chunk is not a signle owner chunk") jsonhttp.InternalServerError(w, "chunk is not a single owner chunk") + return } sig := socCh.Signature() From 7b152196ddd063606915482edd9c1a2235dfda36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 27 Jun 2024 16:51:46 +0200 Subject: [PATCH 35/43] feat: swarm soc signature header in feed endpoint --- pkg/api/feed.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/api/feed.go b/pkg/api/feed.go index 04103fcdebe..40f6d890a84 100644 --- a/pkg/api/feed.go +++ b/pkg/api/feed.go @@ -22,6 +22,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/manifest/mantaray" "github.com/ethersphere/bee/v2/pkg/manifest/simple" "github.com/ethersphere/bee/v2/pkg/postage" + "github.com/ethersphere/bee/v2/pkg/soc" storage "github.com/ethersphere/bee/v2/pkg/storage" storer "github.com/ethersphere/bee/v2/pkg/storer" "github.com/ethersphere/bee/v2/pkg/swarm" @@ -123,11 +124,20 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { return } + socCh, err := soc.FromChunk(ch) + if err != nil { + logger.Error(nil, "wrapped chunk cannot be retrieved") + jsonhttp.NotFound(w, "wrapped chunk cannot be retrieved") + return + } + sig := socCh.Signature() + additionalHeaders := http.Header{ ContentTypeHeader: {"application/octet-stream"}, SwarmFeedIndexHeader: {hex.EncodeToString(curBytes)}, SwarmFeedIndexNextHeader: {hex.EncodeToString(nextBytes)}, - "Access-Control-Expose-Headers": {SwarmFeedIndexHeader, SwarmFeedIndexNextHeader}, + SwarmSocSignature: {hex.EncodeToString(sig)}, + "Access-Control-Expose-Headers": {SwarmFeedIndexHeader, SwarmFeedIndexNextHeader, SwarmSocSignature}, } if headers.OnlyRootChunk { From f2cc8d8660157e10eddb4ae01f759fbe77e9c679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 27 Jun 2024 16:53:57 +0200 Subject: [PATCH 36/43] refactor: swarm signature header --- pkg/api/api.go | 2 +- pkg/api/feed.go | 4 ++-- pkg/api/soc.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 163888c64aa..1f724213adc 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -75,7 +75,7 @@ const ( SwarmEncryptHeader = "Swarm-Encrypt" SwarmIndexDocumentHeader = "Swarm-Index-Document" SwarmErrorDocumentHeader = "Swarm-Error-Document" - SwarmSocSignature = "Swarm-Soc-Signature" + SwarmSocSignatureHeader = "Swarm-Soc-Signature" SwarmFeedIndexHeader = "Swarm-Feed-Index" SwarmFeedIndexNextHeader = "Swarm-Feed-Index-Next" SwarmOnlyRootChunk = "Swarm-Only-Root-Chunk" diff --git a/pkg/api/feed.go b/pkg/api/feed.go index 40f6d890a84..81dd2047949 100644 --- a/pkg/api/feed.go +++ b/pkg/api/feed.go @@ -136,8 +136,8 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) { ContentTypeHeader: {"application/octet-stream"}, SwarmFeedIndexHeader: {hex.EncodeToString(curBytes)}, SwarmFeedIndexNextHeader: {hex.EncodeToString(nextBytes)}, - SwarmSocSignature: {hex.EncodeToString(sig)}, - "Access-Control-Expose-Headers": {SwarmFeedIndexHeader, SwarmFeedIndexNextHeader, SwarmSocSignature}, + SwarmSocSignatureHeader: {hex.EncodeToString(sig)}, + "Access-Control-Expose-Headers": {SwarmFeedIndexHeader, SwarmFeedIndexNextHeader, SwarmSocSignatureHeader}, } if headers.OnlyRootChunk { diff --git a/pkg/api/soc.go b/pkg/api/soc.go index e971575413a..9abdc083c18 100644 --- a/pkg/api/soc.go +++ b/pkg/api/soc.go @@ -224,8 +224,8 @@ func (s *Service) socGetHandler(w http.ResponseWriter, r *http.Request) { additionalHeaders := http.Header{ ContentTypeHeader: {"application/octet-stream"}, - SwarmSocSignature: {hex.EncodeToString(sig)}, - "Access-Control-Expose-Headers": {SwarmSocSignature}, + SwarmSocSignatureHeader: {hex.EncodeToString(sig)}, + "Access-Control-Expose-Headers": {SwarmSocSignatureHeader}, } if headers.OnlyRootChunk { From f4c117f31b9ac047f5e6141ccbb5f5dc97543b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Mon, 1 Jul 2024 10:40:52 +0200 Subject: [PATCH 37/43] docs: update openapi --- openapi/Swarm.yaml | 48 +++++++++++++++++++++++++++++++++++++--- openapi/SwarmCommon.yaml | 14 +++++++++--- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 94930e31930..0f91a724c3d 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -743,12 +743,12 @@ paths: $ref: "SwarmCommon.yaml#/components/schemas/HexString" required: true description: Signature - - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPinParameter" - in: header schema: $ref: "SwarmCommon.yaml#/components/parameters/SwarmPostageBatchId" name: swarm-postage-batch-id required: true + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPinParameter" requestBody: required: true description: The SOC binary data is composed of the span (8 bytes) and the at most 4KB payload. @@ -774,6 +774,47 @@ paths: $ref: "SwarmCommon.yaml#/components/responses/500" default: description: Default response + get: + summary: Resolve Single Owner Chunk data + tags: + - Single owner chunk + parameters: + - in: path + name: owner + schema: + $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" + required: true + description: Ethereum address of the Owner of the SOC + - in: path + name: id + schema: + $ref: "SwarmCommon.yaml#/components/schemas/HexString" + required: true + description: Arbitrary identifier of the related data + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmOnlyRootChunkParameter" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmCache" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyStrategyParameter" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyFallbackModeParameter" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmChunkRetrievalTimeoutParameter" + responses: + "200": + description: Related Single Owner Chunk data + headers: + "swarm-soc-signature": + $ref: "SwarmCommon.yaml#/components/headers/SwarmSocSignature" + content: + application/octet-stream: + schema: + type: string + format: binary + "400": + $ref: "SwarmCommon.yaml#/components/responses/400" + "401": + $ref: "SwarmCommon.yaml#/components/responses/401" + "500": + $ref: "SwarmCommon.yaml#/components/responses/500" + default: + description: Default response "/feeds/{owner}/{topic}": post: @@ -853,6 +894,7 @@ paths: $ref: "SwarmCommon.yaml#/components/schemas/FeedType" required: false description: "Feed indexing scheme (default: sequence)" + - $ref: "SwarmCommon.yaml#/components/headers/SwarmOnlyRootChunkParameter" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmCache" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyStrategyParameter" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyFallbackModeParameter" @@ -861,12 +903,12 @@ paths: "200": description: Latest feed update headers: + "swarm-soc-signature": + $ref: "SwarmCommon.yaml#/components/headers/SwarmSocSignature" "swarm-feed-index": $ref: "SwarmCommon.yaml#/components/headers/SwarmFeedIndex" "swarm-feed-index-next": $ref: "SwarmCommon.yaml#/components/headers/SwarmFeedIndexNext" - "swarm-only-root-chunk": - $ref: "SwarmCommon.yaml#/components/headers/SwarmOnlyRootChunk" content: application/octet-stream: schema: diff --git a/openapi/SwarmCommon.yaml b/openapi/SwarmCommon.yaml index a754165716d..300ec0fae22 100644 --- a/openapi/SwarmCommon.yaml +++ b/openapi/SwarmCommon.yaml @@ -977,10 +977,10 @@ components: schema: $ref: "#/components/schemas/HexString" - SwarmOnlyRootChunk: - description: "Returns only the root chunk of the content" + SwarmSocSignature: + description: "Attached digital signature of the Single Owner Chunk" schema: - type: boolean + $ref: "#/components/schemas/HexString" ETag: description: | @@ -1081,6 +1081,14 @@ components: required: false description: > Specify the timeout for chunk retrieval. The default is 30 seconds. + + SwarmOnlyRootChunkParameter: + in: header + name: swarm-only-root-chunk + schema: + type: boolean + required: false + description: "Returns only the root chunk of the content" ContentTypePreserved: in: header From 3d7de7c7fe9276a4323799908d2bb5cf4d17a34c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Mon, 1 Jul 2024 10:46:54 +0200 Subject: [PATCH 38/43] fix: whitespace check --- openapi/SwarmCommon.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openapi/SwarmCommon.yaml b/openapi/SwarmCommon.yaml index 300ec0fae22..cd8d1814aa4 100644 --- a/openapi/SwarmCommon.yaml +++ b/openapi/SwarmCommon.yaml @@ -1081,7 +1081,7 @@ components: required: false description: > Specify the timeout for chunk retrieval. The default is 30 seconds. - + SwarmOnlyRootChunkParameter: in: header name: swarm-only-root-chunk From a6611dfa62d57a34db57254ff06c1eefb4c435b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 5 Sep 2024 10:35:38 +0200 Subject: [PATCH 39/43] docs: fix method namings --- pkg/file/joiner/joiner.go | 2 +- pkg/file/loadsave/loadsave.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/file/joiner/joiner.go b/pkg/file/joiner/joiner.go index 4644f95ca0b..c3a317854bc 100644 --- a/pkg/file/joiner/joiner.go +++ b/pkg/file/joiner/joiner.go @@ -119,7 +119,7 @@ func New(ctx context.Context, g storage.Getter, putter storage.Putter, address s return NewJoiner(ctx, g, putter, address, rootChunk) } -// NewWithRoot creates a new Joiner with the already fetched root chunk. +// NewJoiner creates a new Joiner with the already fetched root chunk. // A Joiner provides Read, Seek and Size functionalities. func NewJoiner(ctx context.Context, g storage.Getter, putter storage.Putter, address swarm.Address, rootChunk swarm.Chunk) (file.Joiner, int64, error) { chunkData := rootChunk.Data() diff --git a/pkg/file/loadsave/loadsave.go b/pkg/file/loadsave/loadsave.go index 2f94eae8eec..2899d9ed6ab 100644 --- a/pkg/file/loadsave/loadsave.go +++ b/pkg/file/loadsave/loadsave.go @@ -49,7 +49,7 @@ func NewReadonly(getter storage.Getter) file.LoadSaver { } } -// NewReadonly returns a new read-only load-saver +// NewReadonlyWithRootCh returns a new read-only load-saver // which will error on write. func NewReadonlyWithRootCh(getter storage.Getter, rootCh swarm.Chunk) file.LoadSaver { return &loadSave{ From caed9755bbc6656ada0420a84fd55f2ff15af6d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Mon, 16 Sep 2024 11:10:45 +0200 Subject: [PATCH 40/43] fix: params after merge --- pkg/api/bytes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/bytes.go b/pkg/api/bytes.go index 43351b858d7..5eabf41458b 100644 --- a/pkg/api/bytes.go +++ b/pkg/api/bytes.go @@ -182,7 +182,7 @@ func (s *Service) bytesGetHandler(w http.ResponseWriter, r *http.Request) { ContentTypeHeader: {"application/octet-stream"}, } - s.downloadHandler(logger, w, r, paths.Address, additionalHeaders, true, false, nil) + s.downloadHandler(logger, w, r, address, additionalHeaders, true, false, nil) } func (s *Service) bytesHeadHandler(w http.ResponseWriter, r *http.Request) { From c347904b66c85039d6347cccc56ab5fc7d537a98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Mon, 16 Sep 2024 11:29:38 +0200 Subject: [PATCH 41/43] docs: fix duplicate keys --- openapi/Swarm.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 1508ccbc668..e9ef6b840a4 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -844,7 +844,6 @@ paths: name: swarm-postage-batch-id schema: $ref: "SwarmCommon.yaml#/components/parameters/SwarmPostageBatchId" - name: swarm-postage-batch-id required: true - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPinParameter" required: false From 994bc2c60f0b7e8d50f25a822fe2e0d174170ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 19 Sep 2024 11:24:45 +0200 Subject: [PATCH 42/43] feat: add new headers to allowedHeaders --- pkg/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index b69542cd50d..4fc70fbfb17 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -522,7 +522,7 @@ func (s *Service) corsHandler(h http.Handler) http.Handler { allowedHeaders := []string{ "User-Agent", "Accept", "X-Requested-With", "Access-Control-Request-Headers", "Access-Control-Request-Method", "Accept-Ranges", "Content-Encoding", AuthorizationHeader, AcceptEncodingHeader, ContentTypeHeader, ContentDispositionHeader, RangeHeader, OriginHeader, - SwarmTagHeader, SwarmPinHeader, SwarmEncryptHeader, SwarmIndexDocumentHeader, SwarmErrorDocumentHeader, SwarmCollectionHeader, SwarmPostageBatchIdHeader, SwarmPostageStampHeader, SwarmDeferredUploadHeader, SwarmRedundancyLevelHeader, SwarmRedundancyStrategyHeader, SwarmRedundancyFallbackModeHeader, SwarmChunkRetrievalTimeoutHeader, SwarmLookAheadBufferSizeHeader, SwarmFeedIndexHeader, SwarmFeedIndexNextHeader, GasPriceHeader, GasLimitHeader, ImmutableHeader, + SwarmTagHeader, SwarmPinHeader, SwarmEncryptHeader, SwarmIndexDocumentHeader, SwarmErrorDocumentHeader, SwarmCollectionHeader, SwarmPostageBatchIdHeader, SwarmPostageStampHeader, SwarmDeferredUploadHeader, SwarmRedundancyLevelHeader, SwarmRedundancyStrategyHeader, SwarmRedundancyFallbackModeHeader, SwarmChunkRetrievalTimeoutHeader, SwarmLookAheadBufferSizeHeader, SwarmFeedIndexHeader, SwarmFeedIndexNextHeader, SwarmSocSignatureHeader, SwarmOnlyRootChunk, GasPriceHeader, GasLimitHeader, ImmutableHeader, } allowedHeadersStr := strings.Join(allowedHeaders, ", ") From ba593c05b4bba7aea0b3b352e73b175a30309d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 19 Sep 2024 11:42:57 +0200 Subject: [PATCH 43/43] docs: typeo --- pkg/api/bzz.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index fe29ed3b850..559ce652c43 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -557,7 +557,7 @@ func (s *Service) serveManifestEntry( s.downloadHandler(logger, w, r, manifestEntry.Reference(), additionalHeaders, etag, headersOnly, nil) } -// downloadHandler contains common logic for dowloading Swarm file from API +// downloadHandler contains common logic for downloading Swarm file from API func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *http.Request, reference swarm.Address, additionalHeaders http.Header, etag, headersOnly bool, rootCh swarm.Chunk) { headers := struct { Strategy *getter.Strategy `map:"Swarm-Redundancy-Strategy"`