Integrate OCI image metadata into index

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ecr v1.27.0
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.58.0
 	github.com/beam-cloud/blobcache-v2 v0.0.0-20250503151236-e2403183f563
-	github.com/beam-cloud/clip v0.0.0-20251101135640-e0fd6437c99e
+	github.com/beam-cloud/clip v0.0.0-20251101191352-e2bbf59d2918
 	github.com/beam-cloud/go-runc v0.0.0-20250911154456-bb45084abfe1
 	github.com/beam-cloud/goproc v0.1.5
 	github.com/beam-cloud/redislock v0.0.0-20250201162619-1b534b3be324

go.sum

@@ -136,8 +136,8 @@ github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxY
 github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/beam-cloud/blobcache-v2 v0.0.0-20250503151236-e2403183f563 h1:kn8/lBy5kRSLpFAX/xV5GDg31R38Kv8Tp6qcuhpJAfQ=
 github.com/beam-cloud/blobcache-v2 v0.0.0-20250503151236-e2403183f563/go.mod h1:RrA2ruMma4/dN9Sa6wwhyAO1uI6di+tlLB5wuM7TuvQ=
-github.com/beam-cloud/clip v0.0.0-20251101135640-e0fd6437c99e h1:Xg3fRf6z0eGeZek0YjfWhYV1guYxJ3N52w1DjvgER+4=
-github.com/beam-cloud/clip v0.0.0-20251101135640-e0fd6437c99e/go.mod h1:Tt5HW/Mp3twQHzal5RE3FYACcxaMaT+QyTBo8aGbsyI=
+github.com/beam-cloud/clip v0.0.0-20251101191352-e2bbf59d2918 h1:NKbCVubDGLtrOOjXXXfgsK+xGnKXVT88nuiG4ZPHUZg=
+github.com/beam-cloud/clip v0.0.0-20251101191352-e2bbf59d2918/go.mod h1:Tt5HW/Mp3twQHzal5RE3FYACcxaMaT+QyTBo8aGbsyI=
 github.com/beam-cloud/geesefs v0.0.0-20250921162405-f983757f032a h1:JQVvP0qKA39xp5ckKGcwkvgMO6BxatnzvgnMz97tt1g=
 github.com/beam-cloud/geesefs v0.0.0-20250921162405-f983757f032a/go.mod h1:utihEuMyzBOeZ6oU2ozzZkJmyzbYBuYrxsLUo1DfZXs=
 github.com/beam-cloud/geesefs/s3ext v0.0.0-20250606164905-2f3593d03f4f h1:XzHOu+erxeBO6D3fKVbd5DAlipl+PYZ3u+Ywb8m7Ovk=

pkg/common/skopeo.go

@@ -54,18 +54,6 @@ type ImageMetadata struct {
 		Annotations any    `json:"Annotations"`
 	} `json:"LayersData"`
 	Env []string `json:"Env"`
-	// Full OCI Config for v2 builds
-	Config *ImageConfig `json:"Config"`
-}
-
-// ImageConfig represents the OCI image configuration
-type ImageConfig struct {
-	User         string            `json:"User"`
-	ExposedPorts map[string]struct{} `json:"ExposedPorts"`
-	Env          []string          `json:"Env"`
-	Entrypoint   []string          `json:"Entrypoint"`
-	Cmd          []string          `json:"Cmd"`
-	WorkingDir   string            `json:"WorkingDir"`
 }
 
 func NewSkopeoClient(config types.AppConfig) SkopeoClient {

pkg/worker/image.go

@@ -298,6 +298,11 @@ func (c *ImageClient) PullLazy(ctx context.Context, request *types.ContainerRequ
 					c.v2ImageRefs.Set(imageId, sourceRef)
 					log.Info().Str("image_id", imageId).Str("source_image", sourceRef).Msg("cached source image reference from clip metadata")
 				}
+
+				// Log that metadata is available in the archive
+				if ociInfo.ImageMetadata != nil {
+					log.Info().Str("image_id", imageId).Msg("image metadata available in clip archive")
+				}
 			}
 		}
 	}
@@ -412,6 +417,44 @@ func (c *ImageClient) GetSourceImageRef(imageId string) (string, bool) {
 	return c.v2ImageRefs.Get(imageId)
 }
 
+// GetCLIPImageMetadata extracts CLIP image metadata from the archive for a v2 image
+// Returns the CLIP metadata directly from the archive (source of truth)
+func (c *ImageClient) GetCLIPImageMetadata(imageId string) (*clipCommon.ImageMetadata, bool) {
+	// Determine the archive path for this image
+	archivePath := fmt.Sprintf("/images/%s.%s", imageId, reg.LocalImageFileExtension)
+
+	// Check if the archive exists
+	if _, err := os.Stat(archivePath); os.IsNotExist(err) {
+		// Try cache path as fallback
+		if c.registry != nil && c.registry.ImageFileExtension != "" {
+			archivePath = fmt.Sprintf("%s/%s.%s", c.imageCachePath, imageId, c.registry.ImageFileExtension)
+		} else {
+			archivePath = fmt.Sprintf("%s/%s.clip", c.imageCachePath, imageId)
+		}
+
+		if _, err := os.Stat(archivePath); os.IsNotExist(err) {
+			return nil, false
+		}
+	}
+
+	// Extract metadata from the CLIP archive
+	archiver := clip.NewClipArchiver()
+	meta, err := archiver.ExtractMetadata(archivePath)
+	if err != nil {
+		log.Warn().Err(err).Str("image_id", imageId).Msg("failed to extract metadata from clip archive")
+		return nil, false
+	}
+
+	// Check if this is an OCI archive with metadata
+	if meta != nil && meta.StorageInfo != nil {
+		if ociInfo, ok := meta.StorageInfo.(clipCommon.OCIStorageInfo); ok && ociInfo.ImageMetadata != nil {
+			return ociInfo.ImageMetadata, true
+		}
+	}
+
+	return nil, false
+}
+
 // createCredentialProvider creates a CLIP credential provider from JSON credentials
 // Credentials are expected in JSON format: {"registry":"...","type":"...","credentials":{...}}
 // This format is used for both build and runtime containers

pkg/worker/lifecycle.go

@@ -17,6 +17,7 @@ import (
 	"github.com/beam-cloud/beta9/pkg/storage"
 	types "github.com/beam-cloud/beta9/pkg/types"
 	pb "github.com/beam-cloud/beta9/proto"
+	clipCommon "github.com/beam-cloud/clip/pkg/common"
 	goproc "github.com/beam-cloud/goproc/pkg"
 	"tags.cncf.io/container-device-interface/pkg/cdi"
 
@@ -360,10 +361,10 @@ func (s *Worker) readBundleConfig(request *types.ContainerRequest) (*specs.Spec,
 	data, err := os.ReadFile(imageConfigPath)
 	if err != nil {
 		// For v2 images, there's no pre-baked config.json in the mounted root.
-		// Derive the spec from the source image using skopeo inspect.
+		// Derive the spec from CLIP metadata embedded in the archive.
 		if os.IsNotExist(err) {
-			log.Info().Str("image_id", request.ImageId).Msg("no bundle config found, deriving from source image")
-			return s.deriveSpecFromSourceImage(request)
+			log.Info().Str("image_id", request.ImageId).Msg("no bundle config found, deriving from v2 image metadata")
+			return s.deriveSpecFromV2Image(request)
 		}
 		log.Error().Str("image_id", request.ImageId).Str("image_config_path", imageConfigPath).Err(err).Msg("failed to read bundle config")
 		return nil, err
@@ -381,85 +382,49 @@ func (s *Worker) readBundleConfig(request *types.ContainerRequest) (*specs.Spec,
 	return &spec, nil
 }
 
-// deriveSpecFromSourceImage creates an OCI spec from the source image metadata.
-// This is used for v2 images where we don't have an unpacked bundle with config.json.
-func (s *Worker) deriveSpecFromSourceImage(request *types.ContainerRequest) (*specs.Spec, error) {
-	// Determine source image reference and credentials
-	sourceImageRef, sourceImageCreds := s.getSourceImageInfo(request)
-	if sourceImageRef == "" {
-		log.Warn().Str("image_id", request.ImageId).Msg("no source image reference, using base spec")
-		return nil, nil
-	}
-
-	// Inspect source image with timeout
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	imgMeta, err := s.imageClient.skopeoClient.Inspect(ctx, sourceImageRef, sourceImageCreds, nil)
-	if err != nil {
+// deriveSpecFromV2Image creates an OCI spec from v2 image metadata.
+// This is ONLY used for v2 images where we don't have an unpacked bundle with config.json.
+// V1 images always have a config.json, so if we're here, it's a v2 image.
+func (s *Worker) deriveSpecFromV2Image(request *types.ContainerRequest) (*specs.Spec, error) {
+	clipMeta, ok := s.imageClient.GetCLIPImageMetadata(request.ImageId)
+	if !ok {
 		log.Warn().
 			Str("image_id", request.ImageId).
-			Str("source_image", sourceImageRef).
-			Err(err).
-			Msg("failed to inspect source image, using base spec")
+			Msg("no metadata found in v2 image archive, using base spec")
 		return nil, nil
 	}
 
 	log.Info().
 		Str("image_id", request.ImageId).
-		Str("source_image", sourceImageRef).
-		Msg("derived spec from source image")
+		Msg("using metadata from v2 clip archive")
 
-	// Build spec from image metadata
-	return s.buildSpecFromImageMetadata(&imgMeta), nil
+	return s.buildSpecFromCLIPMetadata(clipMeta), nil
 }
 
-// getSourceImageInfo retrieves the source image reference and credentials
-func (s *Worker) getSourceImageInfo(request *types.ContainerRequest) (string, string) {
-	// Build containers have source image in BuildOptions
-	if request.BuildOptions.SourceImage != nil {
-		return *request.BuildOptions.SourceImage, request.BuildOptions.SourceImageCreds
-	}
-
-	// Non-build containers: try cache
-	if ref, ok := s.imageClient.GetSourceImageRef(request.ImageId); ok {
-		log.Info().
-			Str("image_id", request.ImageId).
-			Str("source_image", ref).
-			Msg("retrieved source image from cache")
-		return ref, ""
-	}
-
-	return "", ""
-}
-
-// buildSpecFromImageMetadata constructs an OCI spec from image metadata
-func (s *Worker) buildSpecFromImageMetadata(imgMeta *common.ImageMetadata) *specs.Spec {
+// buildSpecFromCLIPMetadata constructs an OCI spec from CLIP image metadata
+// This is the primary path for v2 images with embedded metadata
+func (s *Worker) buildSpecFromCLIPMetadata(clipMeta *clipCommon.ImageMetadata) *specs.Spec {
 	spec := specs.Spec{
 		Process: &specs.Process{
 			Env: []string{},
 		},
 	}
 
-	// Use Config if available, otherwise fallback to legacy Env field
-	if imgMeta.Config != nil {
-		if len(imgMeta.Config.Env) > 0 {
-			spec.Process.Env = imgMeta.Config.Env
-		}
-		if imgMeta.Config.WorkingDir != "" {
-			spec.Process.Cwd = imgMeta.Config.WorkingDir
-		}
-		if imgMeta.Config.User != "" {
-			spec.Process.User.Username = imgMeta.Config.User
-		}
-		// Combine Entrypoint and Cmd, or use Cmd alone
-		if len(imgMeta.Config.Entrypoint) > 0 {
-			spec.Process.Args = append(imgMeta.Config.Entrypoint, imgMeta.Config.Cmd...)
-		} else if len(imgMeta.Config.Cmd) > 0 {
-			spec.Process.Args = imgMeta.Config.Cmd
-		}
-	} else if len(imgMeta.Env) > 0 {
-		spec.Process.Env = imgMeta.Env
+	// CLIP metadata has a flat structure with all fields at the top level
+	if len(clipMeta.Env) > 0 {
+		spec.Process.Env = clipMeta.Env
+	}
+	if clipMeta.WorkingDir != "" {
+		spec.Process.Cwd = clipMeta.WorkingDir
+	}
+	if clipMeta.User != "" {
+		spec.Process.User.Username = clipMeta.User
+	}
+	// Combine Entrypoint and Cmd, or use Cmd alone
+	if len(clipMeta.Entrypoint) > 0 {
+		spec.Process.Args = append(clipMeta.Entrypoint, clipMeta.Cmd...)
+	} else if len(clipMeta.Cmd) > 0 {
+		spec.Process.Args = clipMeta.Cmd
 	}
 
 	return &spec