From 1e53b4e4f76fb5fbb0879b8091cc705209710214 Mon Sep 17 00:00:00 2001 From: Adit Sachde <23707194+aditsachde@users.noreply.github.com> Date: Sat, 3 Aug 2024 12:45:36 -0700 Subject: [PATCH] use new path function --- internal/ctmonitor/fetch.go | 6 +++--- internal/ctmonitor/logic.go | 2 +- internal/ctsubmit/bucket.go | 2 +- internal/ctsubmit/config.go | 2 +- internal/sunlight/tile_ol.go | 33 ++++++++++++++++++++++++++++- internal/sunlight/tile_reader_ol.go | 2 +- 6 files changed, 39 insertions(+), 8 deletions(-) diff --git a/internal/ctmonitor/fetch.go b/internal/ctmonitor/fetch.go index 1aee051..f4fd241 100644 --- a/internal/ctmonitor/fetch.go +++ b/internal/ctmonitor/fetch.go @@ -45,12 +45,12 @@ func (f *Fetch) getSth(ctx context.Context) (ct.SignedTreeHead, error) { func (f *Fetch) getTile(ctx context.Context, tile tlog.Tile) ([]byte, error) { fallbackWidth := tile.W tile.W = sunlight.TileWidth - resp, notfound, err := f.s.Get(ctx, tile.Path()) + resp, notfound, err := f.s.Get(ctx, sunlight.Path(tile)) // In case the tile is not found, try to fetch the partial tile - if notfound == true { + if notfound { if fallbackWidth != sunlight.TileWidth { tile.W = fallbackWidth - return f.get(ctx, tile.Path()) + return f.get(ctx, sunlight.Path(tile)) } } return resp, err diff --git a/internal/ctmonitor/logic.go b/internal/ctmonitor/logic.go index 580064b..54c0737 100644 --- a/internal/ctmonitor/logic.go +++ b/internal/ctmonitor/logic.go @@ -89,7 +89,7 @@ func hashreader(ctx context.Context, f Fetch, fallbackTreeSize int64) tlog.HashR // and then fall back to the width actually specified in the tile. data, err := f.getTile(ctx, tile) if err != nil { - return nil, fmt.Errorf("failed to fetch tile %s: %w (fallback %s)", tile.Path(), err, finalTile.Path()) + return nil, fmt.Errorf("failed to fetch tile %s: %w (fallback %s)", sunlight.Path(tile), err, sunlight.Path(finalTile)) } hash, err := tlog.HashFromTile(tile, data, index) if err != nil { diff --git a/internal/ctsubmit/bucket.go b/internal/ctsubmit/bucket.go index 501a680..8d19b97 100644 --- a/internal/ctsubmit/bucket.go +++ b/internal/ctsubmit/bucket.go @@ -23,7 +23,7 @@ type Bucket struct { // -------------------------------------------------------------------------------------------- func (b *Bucket) SetTile(ctx context.Context, tile tlog.Tile, data []byte) error { - return b.S.Set(ctx, tile.Path(), data) + return b.S.Set(ctx, sunlight.Path(tile), data) } func (b *Bucket) SetSth(ctx context.Context, data []byte) error { diff --git a/internal/ctsubmit/config.go b/internal/ctsubmit/config.go index 12d5d6f..5eee408 100644 --- a/internal/ctsubmit/config.go +++ b/internal/ctsubmit/config.go @@ -353,7 +353,7 @@ func LoadLog(ctx context.Context, kvpath, consulAddress string) (*Log, error) { // the data tile is the same as the level zero tile, with L -1 dataTile.Tile.L = -1 - dataTileBytes, err := bucket.S.Get(ctx, dataTile.Path()) + dataTileBytes, err := bucket.S.Get(ctx, sunlight.Path(dataTile.Tile)) if err != nil { return nil, fmt.Errorf("unable to fetch data tile: %v", err) } diff --git a/internal/sunlight/tile_ol.go b/internal/sunlight/tile_ol.go index 9a747cf..aaa01af 100644 --- a/internal/sunlight/tile_ol.go +++ b/internal/sunlight/tile_ol.go @@ -16,10 +16,12 @@ package sunlight import ( "fmt" - "github.com/google/certificate-transparency-go/x509" "math" + "github.com/google/certificate-transparency-go/x509" + "golang.org/x/crypto/cryptobyte" + "golang.org/x/mod/sumdb/tlog" ) const TileHeight = 8 @@ -233,3 +235,32 @@ func addExtensions(b *cryptobyte.Builder, leafIndex uint64) { b.AddBytes(ext) }) } + +// To limit the size of any particular directory listing, +// we encode the (possibly very large) number N +// by encoding three digits at a time. +// For example, 123456789 encodes as x123/x456/789. +// Each directory has at most 1000 each xNNN, NNN, and NNN.p children, +// so there are at most 3000 entries in any one directory. +const pathBase = 1000 + +// Path returns a tile coordinate path describing t. +func Path(t tlog.Tile) string { + n := t.N + nStr := fmt.Sprintf("%03d", n%pathBase) + for n >= pathBase { + n /= pathBase + nStr = fmt.Sprintf("x%03d/%s", n%pathBase, nStr) + } + pStr := "" + if t.W != 1<