WIP: try to get e2e test for extensions

Signed-off-by: Natalie Arellano <narellano@vmware.com>

Author: Natalie Arellano

DEVELOPMENT.md

@@ -91,12 +91,6 @@ IMAGE_REGISTRY=gcr.io/<some-project> \
   make e2e
 ```
 
-* The IMAGE_REGISTRY environment variable must point at a registry with local write access - e.g.
-
-```bash
-export IMAGE_REGISTRY="gcr.io/<some-project>"
-```
-
 * The KPACK_TEST_NAMESPACE_LABELS environment variable allows you to define additional labels for the test namespace, e.g.
 
 ```bash

Makefile

@@ -10,6 +10,6 @@ unit-ci:
 	$(GOCMD) test ./pkg/... -coverprofile=coverage.txt -covermode=atomic
 
 e2e:
-	$(GOCMD) test --timeout=30m -v ./test/...
+	$(GOCMD) test --timeout=30m -failfast -v ./test/...
 
 .PHONY: unit unit-ci e2e

cmd/controller/main.go

@@ -50,8 +50,10 @@ import (
 	"github.com/pivotal/kpack/pkg/reconciler/buildpack"
 	"github.com/pivotal/kpack/pkg/reconciler/clusterbuilder"
 	"github.com/pivotal/kpack/pkg/reconciler/clusterbuildpack"
+	"github.com/pivotal/kpack/pkg/reconciler/clusterextension"
 	"github.com/pivotal/kpack/pkg/reconciler/clusterstack"
 	"github.com/pivotal/kpack/pkg/reconciler/clusterstore"
+	"github.com/pivotal/kpack/pkg/reconciler/extension"
 	"github.com/pivotal/kpack/pkg/reconciler/image"
 	"github.com/pivotal/kpack/pkg/reconciler/lifecycle"
 	"github.com/pivotal/kpack/pkg/reconciler/sourceresolver"
@@ -207,8 +209,10 @@ func main() {
 	sourceResolverController := sourceresolver.NewController(ctx, options, sourceResolverInformer, gitResolver, blobResolver, registryResolver)
 	builderController, builderResync := builder.NewController(ctx, options, builderInformer, builderCreator, keychainFactory, clusterStoreInformer, buildpackInformer, clusterBuildpackInformer, clusterStackInformer, extensionInformer, clusterExtensionInformer)
 	buildpackController := buildpack.NewController(ctx, options, keychainFactory, buildpackInformer, remoteStoreReader)
+	extensionController := extension.NewController(ctx, options, keychainFactory, extensionInformer, remoteStoreReader)
 	clusterBuilderController, clusterBuilderResync := clusterbuilder.NewController(ctx, options, clusterBuilderInformer, builderCreator, keychainFactory, clusterStoreInformer, clusterBuildpackInformer, clusterStackInformer, clusterExtensionInformer)
 	clusterBuildpackController := clusterbuildpack.NewController(ctx, options, keychainFactory, clusterBuildpackInformer, remoteStoreReader)
+	clusterExtensionController := clusterextension.NewController(ctx, options, keychainFactory, clusterExtensionInformer, remoteStoreReader)
 	clusterStoreController := clusterstore.NewController(ctx, options, keychainFactory, clusterStoreInformer, remoteStoreReader)
 	clusterStackController := clusterstack.NewController(ctx, options, keychainFactory, clusterStackInformer, remoteStackReader)
 	lifecycleController := lifecycle.NewController(ctx, options, k8sClient, config.LifecycleConfigName, lifecycleConfigmapInformer, lifecycleProvider)
@@ -243,8 +247,10 @@ func main() {
 		run(buildController, routinesPerController),
 		run(builderController, routinesPerController),
 		run(buildpackController, routinesPerController),
+		run(extensionController, routinesPerController),
 		run(clusterBuilderController, routinesPerController),
 		run(clusterBuildpackController, routinesPerController),
+		run(clusterExtensionController, routinesPerController),
 		run(clusterStoreController, routinesPerController),
 		run(lifecycleController, routinesPerController),
 		run(sourceResolverController, 2*routinesPerController),

config/clusterextension.yaml

@@ -0,0 +1,31 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: clusterextensions.kpack.io
+spec:
+  group: kpack.io
+  versions:
+  - name: v1alpha2
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true
+    subresources:
+      status: {}
+    additionalPrinterColumns:
+    - name: Ready
+      type: string
+      jsonPath: ".status.conditions[?(@.type==\"Ready\")].status"
+  names:
+    kind: ClusterExtension
+    listKind: ClusterExtensionList
+    singular: clusterextension
+    plural: clusterextensions
+    shortNames:
+    - clstext
+    - clstexts
+    categories:
+    - kpack
+  scope: Cluster

config/extension.yaml

@@ -0,0 +1,32 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: extensions.kpack.io
+spec:
+  group: kpack.io
+  versions:
+  - name: v1alpha2
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true
+    subresources:
+      status: {}
+    additionalPrinterColumns:
+    - name: Ready
+      type: string
+      jsonPath: ".status.conditions[?(@.type==\"Ready\")].status"
+  names:
+    kind: Extension
+    listKind: ExtensionList
+    singular: extension
+    plural: extensions
+    shortNames:
+    - ext
+    - exts
+    categories:
+    - kpack
+  scope: Namespaced
+

go.sum

@@ -58,7 +58,7 @@ dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek=
 filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
-github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230618160516-e936619f9f18 h1:rd389Q26LMy03gG4anandGFC2LW/xvjga5GezeeaxQk=
+github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230919221257-8b5d3ce2d11d h1:zjqpY4C7H15HjRPEenkS4SAn3Jy2eRRjkjZbGR30TOg=
 github.com/AliyunContainerService/ack-ram-tool/pkg/credentials/alibabacloudsdkgo/helper v0.2.0 h1:8+4G8JaejP8Xa6W46PzJEwisNgBXMvFcz78N6zG/ARw=
 github.com/AliyunContainerService/ack-ram-tool/pkg/credentials/alibabacloudsdkgo/helper v0.2.0/go.mod h1:GgeIE+1be8Ivm7Sh4RgwI42aTtC9qrcj+Y9Y6CjJhJs=
 github.com/Azure/azure-sdk-for-go v55.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=

hack/common.sh

@@ -38,23 +38,23 @@ function compile() {
   pack_build ${controller_image} "./cmd/controller"
   controller_image=${resolved_image_name}
 
-  pack_build ${build_waiter_image} "./cmd/build-waiter"
-  build_waiter_image=${resolved_image_name}
-
-  pack_build ${webhook_image} "./cmd/webhook"
-  webhook_image=${resolved_image_name}
+#  pack_build ${build_waiter_image} "./cmd/build-waiter"
+#  build_waiter_image=${resolved_image_name}
+#
+#  pack_build ${webhook_image} "./cmd/webhook"
+#  webhook_image=${resolved_image_name}
 
   pack_build ${build_init_image} "./cmd/build-init"
   build_init_image=${resolved_image_name}
 
-  pack_build ${rebase_image} "./cmd/rebase"
-  rebase_image=${resolved_image_name}
-
-  pack_build ${completion_image} "./cmd/completion"
-  completion_image=${resolved_image_name}
-
-  lifecycle_image_build ${lifecycle_image}
-  lifecycle_image=${resolved_image_name}
+#  pack_build ${rebase_image} "./cmd/rebase"
+#  rebase_image=${resolved_image_name}
+#
+#  pack_build ${completion_image} "./cmd/completion"
+#  completion_image=${resolved_image_name}
+#
+#  lifecycle_image_build ${lifecycle_image}
+#  lifecycle_image=${resolved_image_name}
 
   ytt -f config/. \
     -v controller.image=${controller_image} \

pkg/apis/build/v1alpha2/build_pod.go

@@ -126,12 +126,13 @@ func (c BuildContext) os() string {
 }
 
 type BuildPodBuilderConfig struct {
-	StackID      string
-	RunImage     string
-	Uid          int64
-	Gid          int64
-	PlatformAPIs []string
-	OS           string
+	StackID       string
+	RunImage      string
+	Uid           int64
+	Gid           int64
+	PlatformAPIs  []string
+	OS            string
+	HasExtensions bool
 }
 
 var (
@@ -633,6 +634,10 @@ func (b *Build) BuildPod(images BuildPodImages, buildContext BuildContext) (*cor
 		pod = b.useStandardContainers(images.BuildWaiterImage, pod)
 	}
 
+	if buildContext.BuildPodBuilderConfig.HasExtensions && buildContext.os() != "windows" {
+		b.useImageExtensions(pod)
+	}
+
 	return pod, nil
 }
 
@@ -713,7 +718,6 @@ func setUpBuildWaiter(container corev1.Container, waitFile string) corev1.Contai
 }
 
 func (b *Build) useStandardContainers(buildWaiterImage string, pod *corev1.Pod) *corev1.Pod {
-
 	containers := pod.Spec.InitContainers
 	pod.Spec.InitContainers = []corev1.Container{
 		{
@@ -770,6 +774,16 @@ func (b *Build) useStandardContainers(buildWaiterImage string, pod *corev1.Pod)
 	return pod
 }
 
+func (b *Build) useImageExtensions(pod *corev1.Pod) {
+	kanikoVolume := corev1.Volume{
+		Name: "kaniko",
+		VolumeSource: corev1.VolumeSource{
+			EmptyDir: &corev1.EmptyDirVolumeSource{},
+		},
+	}
+	pod.Spec.Volumes = append(pod.Spec.Volumes, kanikoVolume)
+}
+
 func userprofileHomeEnv() stepModifier {
 	return func(container corev1.Container) corev1.Container {
 		for i, env := range container.Env {
@@ -1085,7 +1099,11 @@ func (b *Build) setupCosignVolumes(secrets []corev1.Secret) ([]corev1.Volume, []
 }
 
 var (
-	supportedPlatformAPIVersions = []*semver.Version{semver.MustParse("0.9"), semver.MustParse("0.8"), semver.MustParse("0.7")}
+	supportedPlatformAPIVersions = []*semver.Version{
+		semver.MustParse("0.9"),
+		semver.MustParse("0.8"),
+		semver.MustParse("0.7"),
+	}
 )
 
 func (bc BuildContext) highestSupportedPlatformAPI(b *Build) (*semver.Version, error) {

pkg/apis/build/v1alpha2/build_pod_test.go

@@ -889,6 +889,7 @@ func testBuildPod(t *testing.T, when spec.G, it spec.S) {
 				"someimage/name:tag3",
 			}, pod.Spec.InitContainers[5].Args)
 		})
+
 		it("configures export step with non-web default process", func() {
 			build.Spec.DefaultProcess = "sys-info"
 			pod, err := build.BuildPod(config, buildContext)
@@ -2460,6 +2461,40 @@ func testBuildPod(t *testing.T, when spec.G, it spec.S) {
 			})
 		})
 
+		when.Focus("builder has extensions", func() {
+			it.Before(func() {
+				buildContext.BuildPodBuilderConfig.HasExtensions = true
+			})
+
+			it("runs the extender instead of the builder", func() {
+				pod, err := build.BuildPod(config, buildContext)
+				require.NoError(t, err)
+
+				assert.Equal(t, pod.Spec.)
+			})
+
+			it("adds kaniko volume to pod", func() {
+				pod, err := build.BuildPod(config, buildContext)
+				require.NoError(t, err)
+
+				assert.Contains(t, pod.Spec.Volumes, corev1.Volume{
+					Name:         "kaniko",
+					VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}},
+				})
+			})
+
+			it("mounts kaniko volume during restore and extend", func() {
+				_, err := build.BuildPod(config, buildContext)
+				require.NoError(t, err)
+
+				// assert.Subset(t, pod.Spec.InitContainers[0].VolumeMounts, []corev1.VolumeMount{})
+			})
+
+			it("is possible to use standard containers", func() {
+				// TODO
+			})
+		})
+
 		when("complying with the restricted pod security standard", func() {
 			// enforces https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted
 			var pod *corev1.Pod

pkg/apis/build/v1alpha2/builder_conversion.go

@@ -82,7 +82,7 @@ func (bs *NamespacedBuilderSpec) convertFrom(from *v1alpha1.NamespacedBuilderSpe
 
 func (bst *BuilderStatus) convertFrom(from *v1alpha1.BuilderStatus) {
 	bst.Status = from.Status
-	bst.BuilderMetadata = from.BuilderMetadata
+	bst.BuilderMetadataBuildpacks = from.BuilderMetadata
 	bst.Order = from.Order
 	bst.Stack = from.Stack
 	bst.LatestImage = from.LatestImage
@@ -93,7 +93,7 @@ func (bst *BuilderStatus) convertFrom(from *v1alpha1.BuilderStatus) {
 
 func (bst *BuilderStatus) convertTo(to *v1alpha1.BuilderStatus) {
 	to.Status = bst.Status
-	to.BuilderMetadata = bst.BuilderMetadata
+	to.BuilderMetadata = bst.BuilderMetadataBuildpacks
 	to.Order = bst.Order
 	to.Stack = bst.Stack
 	to.LatestImage = bst.LatestImage