capi: refactor bootstrap building to use shared ConfigBuilder

The intent is that the ConfigBuilder from bare-metal support is reusable.

Author: justinsb

pkg/commands/toolbox_enroll.go

@@ -423,11 +423,11 @@ type BootstrapData struct {
 // ConfigBuilder builds bootstrap configuration for a node.
 type ConfigBuilder struct {
 	// ClusterName is the name of the cluster to build.
-	// Required.
+	// Required (unless Cluster is set).
 	ClusterName string
 
 	// InstanceGroupName is the name of the InstanceGroup we are building configuration for.
-	// Required.
+	// Required (unless InstanceGroup is set).
 	InstanceGroupName string
 
 	// Clientset is the clientset to use to query for clusters / instancegroups etc
@@ -450,9 +450,9 @@ type ConfigBuilder struct {
 	// Use GetInstanceGroup to read and auto-populate.
 	InstanceGroup *kops.InstanceGroup
 
-	// instanceGroups holds the (unexpanded) instance group configurations
+	// InstanceGroups holds the (unexpanded) instance group configurations
 	// Use GetInstanceGroups to read and auto-populate
-	instanceGroups *kops.InstanceGroupList
+	InstanceGroups *kops.InstanceGroupList
 
 	// wellKnownAddresses holds the known IP/host endpoints for the cluster.
 	// Use GetWellKnownAddresses to read and auto-populate.
@@ -603,8 +603,8 @@ func (b *ConfigBuilder) GetCloud(ctx context.Context) (fi.Cloud, error) {
 }
 
 func (b *ConfigBuilder) GetInstanceGroups(ctx context.Context) (*kops.InstanceGroupList, error) {
-	if b.instanceGroups != nil {
-		return b.instanceGroups, nil
+	if b.InstanceGroups != nil {
+		return b.InstanceGroups, nil
 	}
 
 	cluster, err := b.GetCluster(ctx)
@@ -622,7 +622,7 @@ func (b *ConfigBuilder) GetInstanceGroups(ctx context.Context) (*kops.InstanceGr
 		return nil, fmt.Errorf("reading instance groups: %w", err)
 	}
 
-	b.instanceGroups = instanceGroupList
+	b.InstanceGroups = instanceGroupList
 	return instanceGroupList, nil
 }
 

pkg/controllers/clusterapi/kopsconfig_controller.go

@@ -17,10 +17,8 @@ limitations under the License.
 package clusterapi
 
 import (
-	"bytes"
 	"context"
 	"fmt"
-	"sort"
 
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -33,15 +31,11 @@ import (
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/registry"
 	kopsapi "k8s.io/kops/pkg/apis/kops/v1alpha2"
-	"k8s.io/kops/pkg/apis/nodeup"
-	"k8s.io/kops/pkg/assets"
 	"k8s.io/kops/pkg/client/simple/vfsclientset"
+	"k8s.io/kops/pkg/commands"
 	"k8s.io/kops/pkg/kopscodecs"
 	"k8s.io/kops/pkg/model"
-	"k8s.io/kops/pkg/model/resources"
-	"k8s.io/kops/pkg/nodemodel"
 	"k8s.io/kops/pkg/wellknownservices"
-	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup"
 	"k8s.io/kops/util/pkg/vfs"
 	"k8s.io/utils/pointer"
@@ -176,25 +170,14 @@ func (r *KopsConfigReconciler) storeBootstrapData(ctx context.Context, parent *a
 }
 
 func (r *KopsConfigReconciler) buildBootstrapData(ctx context.Context, cluster *kopsapi.Cluster, kopsControlPlane *capikops.KopsControlPlane) ([]byte, error) {
-
 	config, err := BuildNodeupConfig(ctx, cluster, kopsControlPlane)
 	if err != nil {
 		return nil, err
 	}
 	return config.NodeupScript, nil
 }
 
-type NodeupConfig struct {
-	NodeupScript []byte
-	NodeupConfig *nodeup.Config
-}
-
-// TODO: Dedup with b.builder.NodeUpConfigBuilder.BuildConfig
-func BuildNodeupConfig(ctx context.Context, cluster *kopsapi.Cluster, kopsControlPlane *capikops.KopsControlPlane) (*NodeupConfig, error) {
-	// tf := &TemplateFunctions{
-	// 	KopsModelContext: *modelContext,
-	// 	cloud:            cloud,
-	// }
+func BuildNodeupConfig(ctx context.Context, cluster *kopsapi.Cluster, kopsControlPlane *capikops.KopsControlPlane) (*commands.BootstrapData, error) {
 	wellKnownAddresses := model.WellKnownAddresses{}
 	for _, systemEndpoint := range kopsControlPlane.Status.SystemEndpoints {
 		switch systemEndpoint.Type {
@@ -205,210 +188,58 @@ func BuildNodeupConfig(ctx context.Context, cluster *kopsapi.Cluster, kopsContro
 		}
 	}
 
-	// TODO: Sync with other nodeup config builder
 	clusterInternal := &kops.Cluster{}
-	if err := kopscodecs.Scheme.Convert(cluster, clusterInternal, nil); err != nil {
-		return nil, fmt.Errorf("converting cluster object: %w", err)
-	}
-	// TODO: Fix validation
-	clusterInternal.Namespace = ""
 
-	// if clusterInternal.Spec.KubeAPIServer == nil {
-	// 	clusterInternal.Spec.KubeAPIServer = &kops.KubeAPIServerConfig{}
-	// }
+	configBuilder := &commands.ConfigBuilder{}
 
-	// cluster := &kops.Cluster{}
-	// cluster.Spec.KubernetesVersion = "1.28.3"
-	// cluster.Spec.KubeAPIServer = &kops.KubeAPIServerConfig{}
-
-	// if cluster.Spec.KubeAPIServer == nil {
-	// 	cluster.Spec.KubeAPIServer = &kopsapi.KubeAPIServerConfig{}
-	// }
-
-	vfsContext := vfs.NewVFSContext()
-
-	basePath, err := registry.ConfigBase(vfsContext, clusterInternal)
-	if err != nil {
-		return nil, fmt.Errorf("parsing vfs base path: %w", err)
-	}
-
-	clientset := vfsclientset.NewVFSClientset(vfsContext, basePath)
-
-	ig := &kops.InstanceGroup{}
-	// TODO: Name
-	ig.SetName("todo-ig-name")
-	ig.Spec.Role = kops.InstanceGroupRoleNode
-
-	getAssets := false
-	assetBuilder := assets.NewAssetBuilder(vfsContext, clusterInternal.Spec.Assets, getAssets)
-
-	cloud, err := cloudup.BuildCloud(clusterInternal)
-	if err != nil {
-		return nil, fmt.Errorf("building cloud: %w", err)
-	}
-
-	// assetBuilder := assets.NewAssetBuilder(clientset.VFSContext(), cluster.Spec.Assets, cluster.Spec.KubernetesVersion, false)
-	var instanceGroups []*kops.InstanceGroup
-	instanceGroups = append(instanceGroups, ig)
-
-	fullCluster, err := cloudup.PopulateClusterSpec(ctx, clientset, clusterInternal, instanceGroups, cloud, assetBuilder)
-	if err != nil {
-		return nil, fmt.Errorf("building full cluster spec: %w", err)
-	}
-
-	channel, err := cloudup.ChannelForCluster(clientset.VFSContext(), fullCluster)
-	if err != nil {
-		// TODO: Maybe this should be a warning
-		return nil, fmt.Errorf("building channel for cluster: %w", err)
-	}
-
-	var fullInstanceGroups []*kops.InstanceGroup
-	for _, instanceGroup := range instanceGroups {
-		fullGroup, err := cloudup.PopulateInstanceGroupSpec(fullCluster, instanceGroup, cloud, channel)
-		if err != nil {
-			return nil, fmt.Errorf("building full instance group spec: %w", err)
+	{
+		if err := kopscodecs.Scheme.Convert(cluster, clusterInternal, nil); err != nil {
+			return nil, fmt.Errorf("converting cluster object: %w", err)
 		}
-		fullInstanceGroups = append(fullInstanceGroups, fullGroup)
-	}
+		// TODO: Fix validation
+		clusterInternal.Namespace = ""
 
-	encryptionConfigSecretHash := ""
-	// if fi.ValueOf(c.Cluster.Spec.EncryptionConfig) {
-	// 	secret, err := secretStore.FindSecret("encryptionconfig")
-	// 	if err != nil {
-	// 		return fmt.Errorf("could not load encryptionconfig secret: %v", err)
-	// 	}
-	// 	if secret == nil {
-	// 		fmt.Println("")
-	// 		fmt.Println("You have encryptionConfig enabled, but no encryptionconfig secret has been set.")
-	// 		fmt.Println("See `kops create secret encryptionconfig -h` and https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/")
-	// 		return fmt.Errorf("could not find encryptionconfig secret")
-	// 	}
-	// 	hashBytes := sha256.Sum256(secret.Data)
-	// 	encryptionConfigSecretHash = base64.URLEncoding.EncodeToString(hashBytes[:])
-	// }
-
-	nodeUpAssets, err := nodemodel.BuildNodeUpAssets(ctx, assetBuilder)
-	if err != nil {
-		return nil, err
+		configBuilder.Cluster = clusterInternal
+		configBuilder.ClusterName = clusterInternal.Name
 	}
 
-	configBuilder, err := nodemodel.NewNodeUpConfigBuilder(fullCluster, assetBuilder, encryptionConfigSecretHash)
-	if err != nil {
-		return nil, fmt.Errorf("building node config: %w", err)
-	}
-
-	// bootstrapScript := &model.BootstrapScript{
-	// 	// KopsModelContext:    modelContext,
-	// 	Lifecycle: fi.LifecycleSync,
-	// 	// NodeUpConfigBuilder: configBuilder,
-	// 	// NodeUpAssets:        c.NodeUpAssets,
-	// }
-
-	keysets := make(map[string]*fi.Keyset)
-
-	// var keystoreBase vfs.Path
-
-	// if cluster.Spec.ConfigStore.Keypairs == "" {
-	// 	configBase, err := registry.ConfigBase(vfsContext, clusterInternal)
-	// 	if err != nil {
-	// 		return nil, err
-	// 	}
-	// 	keystoreBase = configBase.Join("pki")
-	// } else {
-	// 	storePath, err := vfsContext.BuildVfsPath(cluster.Spec.ConfigStore.Keypairs)
-	// 	if err != nil {
-	// 		return nil, err
-	// 	}
-	// 	keystoreBase = storePath
-	// }
-
-	// keystore := fi.NewVFSCAStore(clusterInternal, keystoreBase)
-	keystore, err := clientset.KeyStore(fullCluster)
-	if err != nil {
-		return nil, err
-	}
+	{
+		vfsContext := vfs.NewVFSContext()
 
-	for _, keyName := range []string{"kubernetes-ca"} {
-		keyset, err := keystore.FindKeyset(ctx, keyName)
+		basePath, err := registry.ConfigBase(vfsContext, clusterInternal)
 		if err != nil {
-			return nil, fmt.Errorf("getting keyset %q: %w", keyName, err)
-		}
-
-		if keyset == nil {
-			return nil, fmt.Errorf("failed to find keyset %q", keyName)
+			return nil, fmt.Errorf("parsing vfs base path: %w", err)
 		}
 
-		keysets[keyName] = keyset
+		clientset := vfsclientset.NewVFSClientset(vfsContext, basePath)
+		configBuilder.Clientset = clientset
 	}
 
-	nodeupConfig, bootConfig, err := configBuilder.BuildConfig(fullInstanceGroups[0], wellKnownAddresses, keysets)
-	if err != nil {
-		return nil, err
-	}
-
-	// configData, err := utils.YamlMarshal(config)
-	// if err != nil {
-	// 	return nil, fmt.Errorf("error converting nodeup config to yaml: %v", err)
-	// }
-	// sum256 := sha256.Sum256(configData)
-	// bootConfig.NodeupConfigHash = base64.StdEncoding.EncodeToString(sum256[:])
-	// b.nodeupConfig.Resource = fi.NewBytesResource(configData)
-
-	var nodeupScript resources.NodeUpScript
-	nodeupScript.NodeUpAssets = nodeUpAssets.NodeUpAssets
-	// nodeupScript.NodeUpAssets = configBuilder.NodeUpAssets()
-	nodeupScript.BootConfig = bootConfig
-
 	{
-		nodeupScript.EnvironmentVariables = func() (string, error) {
-			env := make(map[string]string)
-
-			// env, err := b.buildEnvironmentVariables()
-			// if err != nil {
-			// 	return "", err
-			// }
-
-			// Sort keys to have a stable sequence of "export xx=xxx"" statements
-			var keys []string
-			for k := range env {
-				keys = append(keys, k)
-			}
-			sort.Strings(keys)
-
-			var b bytes.Buffer
-			for _, k := range keys {
-				b.WriteString(fmt.Sprintf("export %s=%s\n", k, env[k]))
-			}
-			return b.String(), nil
-		}
-
-		nodeupScript.ProxyEnv = func() (string, error) {
-			return "", nil
-			// return b.createProxyEnv(cluster.Spec.Networking.EgressProxy)
+		ig := &kops.InstanceGroup{}
+		// TODO: Name
+		ig.SetName("todo-ig-name")
+		ig.Spec.Role = kops.InstanceGroupRoleNode
+
+		configBuilder.InstanceGroup = ig
+		configBuilder.InstanceGroupName = ig.Name
+		configBuilder.InstanceGroups = &kops.InstanceGroupList{
+			Items: []kops.InstanceGroup{*ig},
 		}
 	}
 
-	// TODO: nodeupScript.CompressUserData = fi.ValueOf(b.ig.Spec.CompressUserData)
-
-	// By setting some sysctls early, we avoid broken configurations that prevent nodeup download.
-	// See https://github.com/kubernetes/kops/issues/10206 for details.
-	// TODO: nodeupScript.SetSysctls = setSysctls()
-
-	// nodeupScript.CloudProvider = string(cluster.Spec.GetCloudProvider())
-	nodeupScript.CloudProvider = string(clusterInternal.GetCloudProvider())
-
-	nodeupScriptResource, err := nodeupScript.Build()
-	if err != nil {
-		return nil, err
+	{
+		cloud, err := cloudup.BuildCloud(clusterInternal)
+		if err != nil {
+			return nil, fmt.Errorf("building cloud: %w", err)
+		}
+		configBuilder.Cloud = cloud
 	}
 
-	b, err := fi.ResourceAsBytes(nodeupScriptResource)
+	bootstrapData, err := configBuilder.GetBootstrapData(ctx)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("building bootstrap data: %w", err)
 	}
 
-	return &NodeupConfig{
-		NodeupScript: b,
-		NodeupConfig: nodeupConfig,
-	}, nil
+	return bootstrapData, nil
 }