llbsolver: on-demand CDI devices with automatic setup

Signed-off-by: Tonis Tiigi <[email protected]>

Author: tonistiigi

Dockerfile

@@ -22,6 +22,7 @@ ARG GO_VERSION=1.23
 ARG ALPINE_VERSION=3.21
 ARG XX_VERSION=1.6.1
 ARG BUILDKIT_DEBUG
+ARG EXPORT_BASE=alpine
 
 # minio for s3 integration tests
 FROM minio/minio:${MINIO_VERSION} AS minio
@@ -194,12 +195,28 @@ RUN --mount=from=binaries \
 FROM scratch AS release
 COPY --link --from=releaser /out/ /
 
-FROM alpine:${ALPINE_VERSION} AS buildkit-export
+FROM alpine:${ALPINE_VERSION} AS buildkit-export-alpine
 RUN apk add --no-cache fuse3 git openssh pigz xz iptables ip6tables \
   && ln -s fusermount3 /usr/bin/fusermount
 COPY --link examples/buildctl-daemonless/buildctl-daemonless.sh /usr/bin/
 VOLUME /var/lib/buildkit
 
+FROM ubuntu:24.04 AS buildkit-export-ubuntu
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends \
+    fuse3 \
+    git \
+    openssh-client \
+    pigz \
+    xz-utils \
+    iptables \
+    ca-certificates \
+  && rm -rf /var/lib/apt/lists/*
+COPY --link examples/buildctl-daemonless/buildctl-daemonless.sh /usr/bin/
+VOLUME /var/lib/buildkit
+
+FROM buildkit-export-${EXPORT_BASE} AS buildkit-export
+
 FROM gobuild-base AS containerd-build
 WORKDIR /go/src/github.com/containerd/containerd
 ARG TARGETPLATFORM

api/types/worker.pb.go

@@ -36,4 +36,5 @@ message CDIDevice {
 	string Name = 1;
 	bool AutoAllow = 2;
 	map<string, string> Annotations = 3;
+	bool OnDemand = 4;
 }

api/types/worker.proto

@@ -22,6 +22,7 @@ type CDIDevice struct {
 	Name        string            `json:"name"`
 	AutoAllow   bool              `json:"autoAllow"`
 	Annotations map[string]string `json:"annotations"`
+	OnDemand    bool              `json:"onDemand"`
 }
 
 func (c *Client) Info(ctx context.Context) (*Info, error) {
@@ -52,6 +53,7 @@ func fromAPICDIDevices(in []*apitypes.CDIDevice) []CDIDevice {
 			Name:        d.Name,
 			AutoAllow:   d.AutoAllow,
 			Annotations: d.Annotations,
+			OnDemand:    d.OnDemand,
 		})
 	}
 	return out

api/types/worker_vtproto.pb.go

@@ -86,7 +86,12 @@ func printWorkersVerbose(tw *tabwriter.Writer, winfo []*client.WorkerInfo) {
 			fmt.Fprint(tw, "Devices:\n")
 			for _, d := range wi.CDIDevices {
 				fmt.Fprintf(tw, "\tName:\t%s\n", d.Name)
-				fmt.Fprintf(tw, "\tAutoAllow:\t%v\n", d.AutoAllow)
+				if d.OnDemand {
+					fmt.Fprintf(tw, "\tOnDemand:\t%v\n", d.OnDemand)
+				} else {
+					fmt.Fprintf(tw, "\tAutoAllow:\t%v\n", d.AutoAllow)
+				}
+
 				for _, k := range sortedKeys(d.Annotations) {
 					v := d.Annotations[k]
 					fmt.Fprintf(tw, "\t\t%s:\t%s\n", k, v)

client/info.go

@@ -12,6 +12,7 @@ import (
 	ctd "github.com/containerd/containerd/v2/client"
 	"github.com/containerd/containerd/v2/defaults"
 	"github.com/moby/buildkit/cmd/buildkitd/config"
+	"github.com/moby/buildkit/solver/llbsolver/cdidevices"
 	"github.com/moby/buildkit/util/bklog"
 	"github.com/moby/buildkit/util/disk"
 	"github.com/moby/buildkit/util/network/cniprovider"
@@ -344,7 +345,7 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([
 		ParallelismSem:  parallelismSem,
 		TraceSocket:     common.traceSocket,
 		Runtime:         runtime,
-		CDIManager:      cdiManager,
+		CDIManager:      cdidevices.NewManager(cdiManager),
 	}
 
 	opt, err := containerd.NewWorkerOpt(workerOpts, ctd.WithTimeout(60*time.Second))

cmd/buildctl/debug/workers.go

@@ -33,6 +33,7 @@ import (
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/bboltcachestorage"
 	"github.com/moby/buildkit/solver/llbsolver"
+	"github.com/moby/buildkit/solver/llbsolver/cdidevices"
 	"github.com/moby/buildkit/solver/llbsolver/proc"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/bklog"
@@ -54,7 +55,6 @@ import (
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/timestamppb"
-	"tags.cncf.io/container-device-interface/pkg/cdi"
 )
 
 type Opt struct {
@@ -686,18 +686,18 @@ func toPBBuildkitVersion(in client.BuildkitVersion) *apitypes.BuildkitVersion {
 	}
 }
 
-func toPBCDIDevices(manager *cdi.Cache) []*apitypes.CDIDevice {
+func toPBCDIDevices(manager *cdidevices.Manager) []*apitypes.CDIDevice {
 	if manager == nil {
 		return nil
 	}
 	devs := manager.ListDevices()
 	out := make([]*apitypes.CDIDevice, 0, len(devs))
 	for _, dev := range devs {
-		spec := manager.GetDevice(dev).GetSpec()
 		out = append(out, &apitypes.CDIDevice{
-			Name:        dev,
+			Name:        dev.Name,
 			AutoAllow:   true, // TODO
-			Annotations: spec.Annotations,
+			Annotations: dev.Annotations,
+			OnDemand:    dev.OnDemand,
 		})
 	}
 	return out

cmd/buildkitd/main_containerd_worker.go

@@ -13,7 +13,6 @@ import (
 	"github.com/moby/buildkit/util/bklog"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
-	"tags.cncf.io/container-device-interface/pkg/cdi"
 
 	ctd "github.com/containerd/containerd/v2/client"
 	"github.com/containerd/containerd/v2/core/mount"
@@ -23,6 +22,7 @@ import (
 	resourcestypes "github.com/moby/buildkit/executor/resources/types"
 	gatewayapi "github.com/moby/buildkit/frontend/gateway/pb"
 	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/solver/llbsolver/cdidevices"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/network"
 	"github.com/pkg/errors"
@@ -41,7 +41,7 @@ type containerdExecutor struct {
 	traceSocket      string
 	rootless         bool
 	runtime          *RuntimeInfo
-	cdiManager       *cdi.Cache
+	cdiManager       *cdidevices.Manager
 }
 
 // OnCreateRuntimer provides an alternative to OCI hooks for applying network
@@ -74,7 +74,7 @@ type ExecutorOptions struct {
 	TraceSocket      string
 	Rootless         bool
 	Runtime          *RuntimeInfo
-	CDIManager       *cdi.Cache
+	CDIManager       *cdidevices.Manager
 }
 
 // New creates a new executor backed by connection to containerd API

control/control.go

@@ -17,14 +17,14 @@ import (
 	"github.com/mitchellh/hashstructure/v2"
 	"github.com/moby/buildkit/executor"
 	"github.com/moby/buildkit/snapshot"
+	"github.com/moby/buildkit/solver/llbsolver/cdidevices"
 	"github.com/moby/buildkit/util/network"
 	rootlessmountopts "github.com/moby/buildkit/util/rootless/mountopts"
 	traceexec "github.com/moby/buildkit/util/tracing/exec"
 	"github.com/moby/sys/userns"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/pkg/errors"
-	"tags.cncf.io/container-device-interface/pkg/cdi"
 )
 
 // ProcessMode configures PID namespaces
@@ -60,7 +60,7 @@ func (pm ProcessMode) String() string {
 
 // GenerateSpec generates spec using containerd functionality.
 // opts are ignored for s.Process, s.Hostname, and s.Mounts .
-func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mount, id, resolvConf, hostsFile string, namespace network.Namespace, cgroupParent string, processMode ProcessMode, idmap *idtools.IdentityMapping, apparmorProfile string, selinuxB bool, tracingSocket string, cdiManager *cdi.Cache, opts ...oci.SpecOpts) (*specs.Spec, func(), error) {
+func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mount, id, resolvConf, hostsFile string, namespace network.Namespace, cgroupParent string, processMode ProcessMode, idmap *idtools.IdentityMapping, apparmorProfile string, selinuxB bool, tracingSocket string, cdiManager *cdidevices.Manager, opts ...oci.SpecOpts) (*specs.Spec, func(), error) {
 	c := &containers.Container{
 		ID: id,
 	}

executor/containerdexecutor/executor.go

@@ -5,10 +5,10 @@ import (
 	"github.com/containerd/containerd/v2/pkg/oci"
 	"github.com/containerd/continuity/fs"
 	"github.com/docker/docker/pkg/idtools"
+	"github.com/moby/buildkit/solver/llbsolver/cdidevices"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
-	"tags.cncf.io/container-device-interface/pkg/cdi"
 )
 
 func withProcessArgs(args ...string) oci.SpecOpts {
@@ -64,7 +64,7 @@ func sub(m mount.Mount, subPath string) (mount.Mount, func() error, error) {
 	return m, func() error { return nil }, nil
 }
 
-func generateCDIOpts(_ *cdi.Cache, devices []*pb.CDIDevice) ([]oci.SpecOpts, error) {
+func generateCDIOpts(_ *cdidevices.Manager, devices []*pb.CDIDevice) ([]oci.SpecOpts, error) {
 	if len(devices) == 0 {
 		return nil, nil
 	}

executor/oci/spec.go

@@ -5,10 +5,10 @@ import (
 	"github.com/containerd/containerd/v2/pkg/oci"
 	"github.com/containerd/continuity/fs"
 	"github.com/docker/docker/pkg/idtools"
+	"github.com/moby/buildkit/solver/llbsolver/cdidevices"
 	"github.com/moby/buildkit/solver/pb"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
-	"tags.cncf.io/container-device-interface/pkg/cdi"
 )
 
 func withProcessArgs(args ...string) oci.SpecOpts {
@@ -72,7 +72,7 @@ func sub(m mount.Mount, subPath string) (mount.Mount, func() error, error) {
 	return m, func() error { return nil }, nil
 }
 
-func generateCDIOpts(_ *cdi.Cache, devices []*pb.CDIDevice) ([]oci.SpecOpts, error) {
+func generateCDIOpts(_ *cdidevices.Manager, devices []*pb.CDIDevice) ([]oci.SpecOpts, error) {
 	if len(devices) == 0 {
 		return nil, nil
 	}