diff --git a/storage/go.mod b/storage/go.mod index b0e4dc3a36..a1da6e1504 100644 --- a/storage/go.mod +++ b/storage/go.mod @@ -6,7 +6,7 @@ module go.podman.io/storage require ( github.com/BurntSushi/toml v1.5.0 - github.com/containerd/stargz-snapshotter/estargz v0.18.0 + github.com/containerd/stargz-snapshotter/estargz v0.18.1 github.com/cyphar/filepath-securejoin v0.4.1 github.com/docker/go-units v0.5.0 github.com/google/go-intervals v0.0.2 diff --git a/storage/go.sum b/storage/go.sum index 3391274223..f8547170ce 100644 --- a/storage/go.sum +++ b/storage/go.sum @@ -1,7 +1,7 @@ github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/containerd/stargz-snapshotter/estargz v0.18.0 h1:Ny5yptQgEXSkDFKvlKJGTvf1YJ+4xD8V+hXqoRG0n74= -github.com/containerd/stargz-snapshotter/estargz v0.18.0/go.mod h1:7hfU1BO2KB3axZl0dRQCdnHrIWw7TRDdK6L44Rdeuo0= +github.com/containerd/stargz-snapshotter/estargz v0.18.1 h1:cy2/lpgBXDA3cDKSyEfNOFMA/c10O1axL69EU7iirO8= +github.com/containerd/stargz-snapshotter/estargz v0.18.1/go.mod h1:ALIEqa7B6oVDsrF37GkGN20SuvG/pIMm7FwP7ZmRb0Q= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go index 16bdc90b20..a9e1b72ba7 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go @@ -35,6 +35,7 @@ import ( "runtime" "strings" "sync" + "sync/atomic" "github.com/containerd/stargz-snapshotter/estargz/errorutil" "github.com/klauspost/compress/zstd" @@ -145,8 +146,10 @@ func WithGzipHelperFunc(gzipHelperFunc GzipHelperFunc) Option { // Blob is an eStargz blob. type Blob struct { io.ReadCloser - diffID digest.Digester - tocDigest digest.Digest + diffID digest.Digester + tocDigest digest.Digest + readCompleted *atomic.Bool + uncompressedSize *atomic.Int64 } // DiffID returns the digest of uncompressed blob. @@ -160,6 +163,19 @@ func (b *Blob) TOCDigest() digest.Digest { return b.tocDigest } +// UncompressedSize returns the size of uncompressed blob. +// UncompressedSize should only be called after the blob has been fully read. +func (b *Blob) UncompressedSize() (int64, error) { + switch { + case b.uncompressedSize == nil || b.readCompleted == nil: + return -1, fmt.Errorf("readCompleted or uncompressedSize is not initialized") + case !b.readCompleted.Load(): + return -1, fmt.Errorf("called UncompressedSize before the blob has been fully read") + default: + return b.uncompressedSize.Load(), nil + } +} + // Build builds an eStargz blob which is an extended version of stargz, from a blob (gzip, zstd // or plain tar) passed through the argument. If there are some prioritized files are listed in // the option, these files are grouped as "prioritized" and can be used for runtime optimization @@ -267,17 +283,28 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) { } diffID := digest.Canonical.Digester() pr, pw := io.Pipe() + readCompleted := new(atomic.Bool) + uncompressedSize := new(atomic.Int64) go func() { - r, err := opts.compression.Reader(io.TeeReader(io.MultiReader(append(rs, tocAndFooter)...), pw)) + var size int64 + var decompressFunc func(io.Reader) (io.ReadCloser, error) + if _, ok := opts.compression.(*gzipCompression); ok && opts.gzipHelperFunc != nil { + decompressFunc = opts.gzipHelperFunc + } else { + decompressFunc = opts.compression.Reader + } + decompressR, err := decompressFunc(io.TeeReader(io.MultiReader(append(rs, tocAndFooter)...), pw)) if err != nil { pw.CloseWithError(err) return } - defer r.Close() - if _, err := io.Copy(diffID.Hash(), r); err != nil { + defer decompressR.Close() + if size, err = io.Copy(diffID.Hash(), decompressR); err != nil { pw.CloseWithError(err) return } + uncompressedSize.Store(size) + readCompleted.Store(true) pw.Close() }() return &Blob{ @@ -285,8 +312,10 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) { Reader: pr, closeFunc: layerFiles.CleanupAll, }, - tocDigest: tocDgst, - diffID: diffID, + tocDigest: tocDgst, + diffID: diffID, + readCompleted: readCompleted, + uncompressedSize: uncompressedSize, }, nil } @@ -381,8 +410,9 @@ func sortEntries(in io.ReaderAt, prioritized []string, missedPrioritized *[]stri // Sort the tar file respecting to the prioritized files list. sorted := &tarFile{} + picked := make(map[string]struct{}) for _, l := range prioritized { - if err := moveRec(l, intar, sorted); err != nil { + if err := moveRec(l, intar, sorted, picked); err != nil { if errors.Is(err, errNotFound) && missedPrioritized != nil { *missedPrioritized = append(*missedPrioritized, l) continue // allow not found @@ -410,8 +440,8 @@ func sortEntries(in io.ReaderAt, prioritized []string, missedPrioritized *[]stri }) } - // Dump all entry and concatinate them. - return append(sorted.dump(), intar.dump()...), nil + // Dump prioritized entries followed by the rest entries while skipping picked ones. + return append(sorted.dump(nil), intar.dump(picked)...), nil } // readerFromEntries returns a reader of tar archive that contains entries passed @@ -473,36 +503,42 @@ func importTar(in io.ReaderAt) (*tarFile, error) { return tf, nil } -func moveRec(name string, in *tarFile, out *tarFile) error { +func moveRec(name string, in *tarFile, out *tarFile, picked map[string]struct{}) error { name = cleanEntryName(name) if name == "" { // root directory. stop recursion. if e, ok := in.get(name); ok { // entry of the root directory exists. we should move it as well. // this case will occur if tar entries are prefixed with "./", "/", etc. - out.add(e) - in.remove(name) + if _, done := picked[name]; !done { + out.add(e) + picked[name] = struct{}{} + } } return nil } _, okIn := in.get(name) _, okOut := out.get(name) - if !okIn && !okOut { + _, okPicked := picked[name] + if !okIn && !okOut && !okPicked { return fmt.Errorf("file: %q: %w", name, errNotFound) } parent, _ := path.Split(strings.TrimSuffix(name, "/")) - if err := moveRec(parent, in, out); err != nil { + if err := moveRec(parent, in, out, picked); err != nil { return err } if e, ok := in.get(name); ok && e.header.Typeflag == tar.TypeLink { - if err := moveRec(e.header.Linkname, in, out); err != nil { + if err := moveRec(e.header.Linkname, in, out, picked); err != nil { return err } } + if _, done := picked[name]; done { + return nil + } if e, ok := in.get(name); ok { out.add(e) - in.remove(name) + picked[name] = struct{}{} } return nil } @@ -548,8 +584,18 @@ func (f *tarFile) get(name string) (e *entry, ok bool) { return } -func (f *tarFile) dump() []*entry { - return f.stream +func (f *tarFile) dump(skip map[string]struct{}) []*entry { + if len(skip) == 0 { + return f.stream + } + var out []*entry + for _, e := range f.stream { + if _, ok := skip[cleanEntryName(e.header.Name)]; ok { + continue + } + out = append(out, e) + } + return out } type readCloser struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index de314040b4..e116828419 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -56,6 +56,8 @@ github.com/containerd/platforms ## explicit; go 1.24.0 # github.com/containerd/stargz-snapshotter/estargz v0.18.0 ## explicit; go 1.24.0 +# github.com/containerd/stargz-snapshotter/estargz v0.18.1 +## explicit; go 1.24.0 github.com/containerd/stargz-snapshotter/estargz github.com/containerd/stargz-snapshotter/estargz/errorutil # github.com/containernetworking/cni v1.3.0