Support putting external images into the local registry of the kubevirtci cluster (#3801)

User can define an environment variable EXTRA_IMAGES that contains a list of
external image urls separated by spaces.

At the end of 'make cluster-up' target the EXTRA_IMAGES are processed
by a script ./cluster-up/extra-images.sh so that for each image url,
it pulls the image and then push the image into the local registry.

Then when 'make cluster-sync' is invoked, it searches all external
image urls in the generated yamls in _out/manifests/ and replaces
them with corresponding urls in the local registry (registry:5000)

Signed-off-by: Howard Gao <[email protected]>

Author: howardgao

Makefile

@@ -126,7 +126,7 @@ docker-registry-cleanup: ## Clean up all cached images from docker registry. Acc
 publish: manifests push ## Generate a cdi-controller and operator manifests and push the built container images to the registry defined in DOCKER_PREFIX
 
 manifests: ## Generate a cdi-controller and operator manifests in '_out/manifests/'.  Accepts [make variables]\(#make-variables\) DOCKER_TAG, DOCKER_PREFIX, VERBOSITY, PULL_POLICY, CSV_VERSION, QUAY_REPOSITORY, QUAY_NAMESPACE
-	${DO_BAZ} "DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} VERBOSITY=${VERBOSITY} PULL_POLICY=${PULL_POLICY} CR_NAME=${CR_NAME} CDI_NAMESPACE=${CDI_NAMESPACE} ./hack/build/build-manifests.sh"
+	${DO_BAZ} "DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} VERBOSITY=${VERBOSITY} PULL_POLICY=${PULL_POLICY} CR_NAME=${CR_NAME} CDI_NAMESPACE=${CDI_NAMESPACE} EXTRA_IMAGES=\"${EXTRA_IMAGES}\" ./hack/build/build-manifests.sh"
 
 release-description: ## Generate a release announcement detailing changes between 2 commits (typically tags).  Expects 'RELREF' and 'PREREF' to be set
 	./hack/build/release-description.sh ${RELREF} ${PREREF}
@@ -140,6 +140,7 @@ openshift-ci-image-push: ## Build and push the OpenShift CI build+test container
 ##@ Local cluster management
 cluster-up: ## Start a default Kubernetes or Open Shift cluster. set KUBEVIRT_PROVIDER environment variable to either 'k8s-1.18' or 'os-3.11.0' to select the type of cluster. set KUBEVIRT_NUM_NODES to something higher than 1 to have more than one node.
 	./cluster-up/up.sh
+	./hack/extra-images.sh
 
 cluster-down: ## Stop the cluster, doing a make cluster-down && make cluster-up will basically restart the cluster into an empty fresh state.
 	./cluster-up/down.sh

hack/build/build-manifests.sh

@@ -32,6 +32,20 @@ source "${script_dir}"/resource-generator.sh
 
 mkdir -p "${MANIFEST_GENERATED_DIR}/"
 
+# Update manifests to replace extra images with internal registry ones
+if [ -n "${EXTRA_IMAGES-}" ]; then
+    for img in ${EXTRA_IMAGES}; do
+        base_image_path="${img#*/}"
+        local_img_url="registry:5000/${base_image_path}"
+        if [ "$img" = "${EXTERNAL_IMAGE_MINIO}" ]; then
+            EXTERNAL_IMAGE_MINIO=$local_img_url
+        fi
+        if [ "$img" = "${EXTERNAL_IMAGE_FAKEOVIRT}" ]; then
+            EXTERNAL_IMAGE_FAKEOVIRT=$local_img_url
+        fi
+    done
+fi
+
 #generate operator related manifests used to deploy cdi with operator-framework
 generateResourceManifest $generator $MANIFEST_GENERATED_DIR "operator" "everything" "operator-everything.yaml.in"
 

hack/build/config.sh

@@ -64,6 +64,9 @@ CDI_LOGO_PATH=${CDI_LOGO_PATH:-"assets/cdi_logo.png"}
 # oVirt populator image, by default it is the same as the importer image
 OVIRT_POPULATOR_IMAGE_NAME=${OVIRT_POPULATOR_IMAGE_NAME:-cdi-importer}
 
+EXTERNAL_IMAGE_MINIO=${EXTERNAL_IMAGE_MINIO:-quay.io/kubevirt/minio:RELEASE.2024-10-02T17-50-41Z}
+EXTERNAL_IMAGE_FAKEOVIRT=${EXTERNAL_IMAGE_FAKEOVIRT:-quay.io/kubevirt/fakeovirt:v1.38.0}
+
 function allPkgs() {
     ret=$(sed "s,kubevirt.io/containerized-data-importer,${CDI_DIR},g" <(go list ./pkg/... ./tools/... ./tests/... ./cmd/... | grep -v "pkg/client" | sort -u))
     echo "$ret"

hack/build/resource-generator.sh

@@ -131,6 +131,8 @@ function populateResourceManifest() {
             -uploadproxy-image="${DOCKER_PREFIX}/${UPLOADPROXY_IMAGE_NAME}:${DOCKER_TAG}" \
             -uploadserver-image="${DOCKER_PREFIX}/${UPLOADSERVER_IMAGE_NAME}:${DOCKER_TAG}" \
             -ovirt-populator-image="${OVIRT_POPULATOR_IMAGE_NAME}" \
+            -external-image-minio="${EXTERNAL_IMAGE_MINIO}" \
+            -external-image-fakeovirt="${EXTERNAL_IMAGE_FAKEOVIRT}" \
             -verbosity="${VERBOSITY}" \
             -pull-policy="${PULL_POLICY}" \
             -cr-name="${CR_NAME}" \
@@ -152,6 +154,8 @@ function populateResourceManifest() {
             -apiserver-image="{{ apiserver_image }}" \
             -uploadproxy-image="{{ uploadproxy_image }}" \
             -uploadserver-image="{{ uploadserver_image }}" \
+            -external-image-minio="{{ external_image_minio }}" \
+            -external-image-fakeovirt="{{ external_image_fakeovirt }}" \
             -verbosity="${VERBOSITY}" \
             -pull-policy="{{ pull_policy }}" \
             -namespace="{{ cdi_namespace }}" \

hack/extra-images.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -e
+
+script_dir="$(cd "$(dirname "$0")" && pwd -P)"
+source "${script_dir}"/build/common.sh
+source ./cluster-up/hack/common.sh
+source ./cluster-up/cluster/${KUBEVIRT_PROVIDER}/provider.sh
+
+cri_bin=$1
+if [ -z $cri_bin ]; then
+    cri_bin=$CDI_CRI
+fi
+
+if [ -n "${EXTRA_IMAGES}" ]; then
+    registry_port=$(_port registry)
+    for img in ${EXTRA_IMAGES}; do
+        base_image_path="${img#*/}"
+        local_img_url="localhost:${registry_port}/${base_image_path}"
+        $cri_bin pull $img
+        $cri_bin tag $img $local_img_url
+        $cri_bin push $local_img_url
+    done
+fi

manifests/templates/file-host.yaml.in

@@ -60,7 +60,7 @@ spec:
           initialDelaySeconds: 20
           periodSeconds: 20
       - name: s3
-        image: quay.io/kubevirt/minio:RELEASE.2024-10-02T17-50-41Z
+        image: {{ .ExternalImageMinio }}
         imagePullPolicy: {{ .PullPolicy }}
         env:
         - name: MINIO_ACCESS_KEY

manifests/templates/imageio.yaml.in

@@ -48,7 +48,7 @@ spec:
           add-ticket myticket.json && tail -n+1 -f /var/log/ovirt-imageio/daemon.log
       - name: fakeovirt
         # Docker file: https://github.com/machacekondra/fakeovirt
-        image: quay.io/kubevirt/fakeovirt:v1.38.0
+        image: {{ .ExternalImageFakeovirt }}
         imagePullPolicy: {{ .PullPolicy }}
         ports:
         - containerPort: 12346

pkg/operator/resources/namespaced/factory.go

@@ -40,6 +40,8 @@ type FactoryArgs struct {
 	APIServerImage         string `required:"true" envconfig:"apiserver_image"`
 	UploadProxyImage       string `required:"true" split_words:"true"`
 	UploadServerImage      string `required:"true" split_words:"true"`
+	ExternalImageMinio     string `split_words:"true"`
+	ExternalImageFakeovirt string `split_words:"true"`
 	Verbosity              string `required:"true"`
 	PullPolicy             string `required:"true" split_words:"true"`
 	ImagePullSecrets       []corev1.LocalObjectReference

tools/manifest-generator/manifest-generator.go

@@ -41,6 +41,8 @@ type templateData struct {
 	APIServerImage         string
 	UploadProxyImage       string
 	UploadServerImage      string
+	ExternalImageMinio     string
+	ExternalImageFakeovirt string
 	Verbosity              string
 	PullPolicy             string
 	CrName                 string
@@ -62,6 +64,8 @@ var (
 	apiServerImage         = flag.String("apiserver-image", "", "")
 	uploadProxyImage       = flag.String("uploadproxy-image", "", "")
 	uploadServerImage      = flag.String("uploadserver-image", "", "")
+	externalImageMinio     = flag.String("external-image-minio", "", "")
+	externalImageFakeovirt = flag.String("external-image-fakeovirt", "", "")
 	verbosity              = flag.String("verbosity", "1", "")
 	pullPolicy             = flag.String("pull-policy", "", "")
 	crName                 = flag.String("cr-name", "", "")
@@ -76,6 +80,9 @@ func main() {
 
 	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(klogFlags)
+
+	klog.Info("========Generator", "templFile", templFile)
+
 	flag.CommandLine.VisitAll(func(f1 *flag.Flag) {
 		f2 := klogFlags.Lookup(f1.Name)
 		if f2 != nil {
@@ -108,6 +115,8 @@ func generateFromFile(templFile string) {
 		APIServerImage:         *apiServerImage,
 		UploadProxyImage:       *uploadProxyImage,
 		UploadServerImage:      *uploadServerImage,
+		ExternalImageMinio:     *externalImageMinio,
+		ExternalImageFakeovirt: *externalImageFakeovirt,
 		PullPolicy:             *pullPolicy,
 		CrName:                 *crName,
 		Namespace:              *namespace,
@@ -205,17 +214,19 @@ func getClusterResources(codeGroup string) ([]client.Object, error) {
 
 func getNamespacedResources(codeGroup string) ([]client.Object, error) {
 	args := &cdinamespaced.FactoryArgs{
-		Verbosity:           *verbosity,
-		OperatorVersion:     *operatorVersion,
-		ControllerImage:     *controllerImage,
-		ImporterImage:       *importerImage,
-		ClonerImage:         *clonerImage,
-		OvirtPopulatorImage: *ovirtPopulatorImage,
-		APIServerImage:      *apiServerImage,
-		UploadProxyImage:    *uploadProxyImage,
-		UploadServerImage:   *uploadServerImage,
-		PullPolicy:          *pullPolicy,
-		Namespace:           *namespace,
+		Verbosity:              *verbosity,
+		OperatorVersion:        *operatorVersion,
+		ControllerImage:        *controllerImage,
+		ImporterImage:          *importerImage,
+		ClonerImage:            *clonerImage,
+		OvirtPopulatorImage:    *ovirtPopulatorImage,
+		APIServerImage:         *apiServerImage,
+		UploadProxyImage:       *uploadProxyImage,
+		UploadServerImage:      *uploadServerImage,
+		ExternalImageMinio:     *externalImageMinio,
+		ExternalImageFakeovirt: *externalImageFakeovirt,
+		PullPolicy:             *pullPolicy,
+		Namespace:              *namespace,
 	}
 
 	return cdinamespaced.CreateResourceGroup(codeGroup, args)