Ensure parent dirs exist for partial idb (#1776)

Some runtimes will get mad if the image is missing parent directories,
this ensures we have them.

Signed-off-by: Jon Johnson <[email protected]>

Author: jonjohnsonjr

internal/cli/publish_test.go

@@ -167,7 +167,7 @@ func TestPublishLayering(t *testing.T) {
 
 	// This test will fail if we ever make a change in apko that changes the image.
 	// Sometimes, this is intentional, and we need to change this and bump the version.
-	want := "sha256:88a8065a9ccb964f27ca842a3e297e152997ddae2997fe68ae7535653c1ebc46"
+	want := "sha256:66dc21761826860d1152e8a6defa7919ad3f65af5d56fd75164c4c36ba85c648"
 	require.Equal(t, want, digest.String())
 
 	im, err := idx.IndexManifest()

pkg/build/layers.go

@@ -284,10 +284,6 @@ func splitLayers(ctx context.Context, fsys apkfs.FullFS, groups []*group, pkgToD
 
 	top := newLayerWriter(f)
 
-	// We want to match the file info from the top layer's idb file below,
-	// so when we run into it, just stash it for later.
-	var idb tar.Header
-
 	// In a tar file, it is customary to include directories before files in those directories.
 	// In order to know which directories we need to include, we maintain a directory stack for each layer.
 	// We compare those stacks to the full FS that we're walking whenever we create a new tar entry.
@@ -371,7 +367,6 @@ func splitLayers(ctx context.Context, fsys apkfs.FullFS, groups []*group, pkgToD
 			if err != nil {
 				return nil, fmt.Errorf("opening %s: %w", f.path, err)
 			}
-
 			if _, err := io.CopyBuffer(w.w, data, buf); err != nil {
 				return nil, fmt.Errorf("copying %s: %w", f.path, err)
 			}
@@ -383,35 +378,46 @@ func splitLayers(ctx context.Context, fsys apkfs.FullFS, groups []*group, pkgToD
 		}
 
 		if f.header.Name == "usr/lib/apk/db/installed" {
-			idb = *f.header
-		}
-	}
+			// Add a partial installed db to each layer to satisfy scanners.
+			for _, g := range groups {
+				w := groupToWriter[g]
 
-	// Once we're done walking the FS, we need to finalize each layer...
-	layers := make([]v1.Layer, 0, len(groups)+1)
-	for i, g := range groups {
-		w := groupToWriter[g]
+				// Make sure we have all parent directories to appease AWS Lambda.
+				for _, todo := range w.alignStacks(stack) {
+					todo.header.ModTime = f.header.ModTime
 
-		// Add a partial installed db to satisfy scanners.
-		{
-			var buf bytes.Buffer
-			for _, pkg := range g.pkgs {
-				if _, err := buf.Write(pkgToDiff[pkg]); err != nil {
-					return nil, err
+					if err := w.w.WriteHeader(todo.header); err != nil {
+						return nil, fmt.Errorf("writing header %s: %w", todo.header.Name, err)
+					}
 				}
-			}
 
-			// Only the size should be different across layers.
-			idb.Size = int64(buf.Len())
+				// Accumulate all the idb entries for this layer.
+				var buf bytes.Buffer
+				for _, pkg := range g.pkgs {
+					if _, err := buf.Write(pkgToDiff[pkg]); err != nil {
+						return nil, err
+					}
+				}
 
-			if err := w.w.WriteHeader(&idb); err != nil {
-				return nil, err
-			}
+				// Only the size should be different across layers.
+				idb := *f.header
+				idb.Size = int64(buf.Len())
 
-			if _, err := io.Copy(w.w, &buf); err != nil {
-				return nil, err
+				if err := w.w.WriteHeader(&idb); err != nil {
+					return nil, err
+				}
+
+				if _, err := io.Copy(w.w, &buf); err != nil {
+					return nil, err
+				}
 			}
 		}
+	}
+
+	// Once we're done walking the FS, we need to finalize each layer...
+	layers := make([]v1.Layer, 0, len(groups)+1)
+	for i, g := range groups {
+		w := groupToWriter[g]
 
 		l, err := w.finalize()
 		if err != nil {

pkg/build/layers_test.go

@@ -15,11 +15,16 @@
 package build
 
 import (
+	"archive/tar"
+	"context"
 	"fmt"
+	"io"
+	"os"
 	"slices"
 	"testing"
 
 	"chainguard.dev/apko/pkg/apk/apk"
+	apkfs "chainguard.dev/apko/pkg/apk/fs"
 )
 
 func size(pkgs ...*apk.Package) uint64 {
@@ -224,3 +229,122 @@ func compareStacks(a, b []*file) error {
 
 	return nil
 }
+
+func TestSplitLayersDirectoryCreation(t *testing.T) {
+	// Create a minimal filesystem with an installed DB file
+	fsys := apkfs.NewMemFS()
+
+	// Create the parent directories first
+	if err := fsys.MkdirAll("usr/lib/apk/db", 0755); err != nil {
+		t.Fatalf("failed to create parent directories: %v", err)
+	}
+
+	// Create the installed DB file with some content
+	idbContent := []byte("test db content")
+	if err := fsys.WriteFile("usr/lib/apk/db/installed", idbContent, 0644); err != nil {
+		t.Fatalf("failed to create installed DB file: %v", err)
+	}
+
+	// Create test packages for multiple layers
+	pkg1 := &apk.Package{
+		Name:          "pkg1",
+		Origin:        "pkg1",
+		Version:       "1.0.0",
+		InstalledSize: 1000,
+	}
+	pkg2 := &apk.Package{
+		Name:          "pkg2",
+		Origin:        "pkg2",
+		Version:       "1.0.0",
+		InstalledSize: 2000,
+	}
+
+	// Create package groups (this will result in multiple layers)
+	groups := []*group{
+		{pkgs: []*apk.Package{pkg1}, size: 1000, tiebreaker: "pkg1"},
+		{pkgs: []*apk.Package{pkg2}, size: 2000, tiebreaker: "pkg2"},
+	}
+
+	// Create package diffs (minimal content for each package)
+	pkgToDiff := map[*apk.Package][]byte{
+		pkg1: []byte("pkg1 info\n"),
+		pkg2: []byte("pkg2 info\n"),
+	}
+
+	// Create temp directory for layer files
+	tmpDir, err := os.MkdirTemp("", "layer-test-*")
+	if err != nil {
+		t.Fatalf("failed to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	// Call splitLayers to create the layers
+	ctx := context.Background()
+	layers, err := splitLayers(ctx, fsys, groups, pkgToDiff, tmpDir)
+	if err != nil {
+		t.Fatalf("splitLayers failed: %v", err)
+	}
+
+	// We expect 3 layers: one for each package group + top layer
+	if len(layers) != 3 {
+		t.Fatalf("expected 3 layers, got %d", len(layers))
+	}
+
+	wantDirs := map[string]struct{}{
+		"usr":            {},
+		"usr/lib":        {},
+		"usr/lib/apk":    {},
+		"usr/lib/apk/db": {},
+	}
+
+	foundDirs := map[string]struct{}{}
+
+	// Check each of the first 2 layers (package layers) for directory and file
+	for i := range 2 {
+		layer := layers[i]
+
+		// Get layer content as tar reader
+		rc, err := layer.Uncompressed()
+		if err != nil {
+			t.Fatalf("failed to get layer %d content: %v", i, err)
+		}
+
+		tr := tar.NewReader(rc)
+
+		for {
+			header, err := tr.Next()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				t.Fatalf("failed to read tar entry in layer %d: %v", i, err)
+			}
+
+			// Check for the parent directory
+			if _, ok := wantDirs[header.Name]; ok && header.Typeflag == tar.TypeDir {
+				foundDirs[header.Name] = struct{}{}
+			}
+
+			// Check for the installed DB file
+			if header.Name == "usr/lib/apk/db/installed" && header.Typeflag == tar.TypeReg {
+				// Verify the file has content
+				content, err := io.ReadAll(tr)
+				if err != nil {
+					t.Fatalf("failed to read installed DB content in layer %d: %v", i, err)
+				}
+				if len(content) == 0 {
+					t.Errorf("installed DB file in layer %d is empty", i)
+				}
+			}
+		}
+
+		rc.Close()
+
+		// Verify both directory and file were found
+		for dir := range wantDirs {
+			if _, ok := foundDirs[dir]; !ok {
+				t.Errorf("layer %d missing parent directory %q - this indicates the directory creation fix is not working", i, dir)
+			}
+		}
+	}
+}