diff --git a/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml index 993be25cc..9d7ff4091 100644 --- a/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml @@ -33,6 +33,8 @@ configMapGenerator: - DISABLECVM="true" # Set to false to enable confidential VM - INITDATA="" # set default initdata for podvm #- TAGS="" # Uncomment and add key1=value1,key2=value2 etc if you want to use specific tags for podvm + #- IBMCLOUD_DEDICATED_HOST_IDS="" # Uncomment and set if you want to use dedicated hosts, comma separated list + #- IBMCLOUD_DEDICATED_HOST_GROUP_IDS="" # Uncomment and set if you want to use dedicated host groups, comma separated list #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image #- TUNNEL_TYPE="" # Uncomment and set if you want to use a specific tunnel type. Defaults to vxlan #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 diff --git a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_common.go b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_common.go index 27d5d21d4..1036bc00b 100644 --- a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_common.go @@ -980,6 +980,8 @@ func (p *IBMCloudProvisioner) GetProperties(ctx context.Context, cfg *envconf.Co "INITDATA": IBMCloudProps.InitData, "IBMCLOUD_CLUSTER_ID": IBMCloudProps.ClusterID, "TAGS": IBMCloudProps.Tags, + "IBMCLOUD_DEDICATED_HOST_IDS": IBMCloudProps.DedicatedHostIDs, + "IBMCLOUD_DEDICATED_HOST_GROUP_IDS": IBMCloudProps.DedicatedHostGroupIDs, } } diff --git a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_ibmcloud.properties b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_ibmcloud.properties index 94cf105a4..489636146 100644 --- a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_ibmcloud.properties +++ b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_ibmcloud.properties @@ -13,6 +13,10 @@ COS_INSTANCE_ID="crn:v1:bluemix:public:cloud-object-storage:global:a/xxxxxxxxxxx COS_SERVICE_URL="s3.jp-tok.cloud-object-storage.appdomain.cloud" # Resource list -> storage -> a cos service -> instances -> an cos service instance -> service credentials -> apikey COS_APIKEY="${MY_COS_SERVICE_APIKEY}" +# optional +# DEDICATED_HOST_IDS="" +# optional +# DEDICATED_HOST_GROUP_IDS="" IS_SELF_MANAGED_CLUSTER="no" # bz2-2x8 | bx2-2x8 | bz2e-2x8 INSTANCE_PROFILE_NAME="bz2-2x8" diff --git a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_initializer.go b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_initializer.go index 23f98a72a..fc0a83480 100644 --- a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_initializer.go +++ b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_initializer.go @@ -19,42 +19,44 @@ import ( ) type IBMCloudProperties struct { - IBMCloudProvider string - ApiKey string - IamProfileID string - Bucket string - CaaImageTag string - ClusterName string - CosApiKey string - CosInstanceID string - CosServiceURL string - SecurityGroupID string - IamServiceURL string - IksServiceURL string - InitData string - InstanceProfile string - KubeVersion string - PodvmImageID string - PodvmImageArch string - PublicGatewayName string - PublicGatewayID string - Region string - ResourceGroupID string - SshKeyContent string - SshKeyID string - SshKeyName string - SubnetName string - SubnetID string - VpcName string - VpcID string - VpcServiceURL string - WorkerFlavor string - WorkerOS string - Zone string - TunnelType string - VxlanPort string - ClusterID string - Tags string + IBMCloudProvider string + ApiKey string + IamProfileID string + Bucket string + CaaImageTag string + ClusterName string + CosApiKey string + CosInstanceID string + CosServiceURL string + SecurityGroupID string + IamServiceURL string + IksServiceURL string + InitData string + InstanceProfile string + KubeVersion string + PodvmImageID string + PodvmImageArch string + PublicGatewayName string + PublicGatewayID string + Region string + ResourceGroupID string + SshKeyContent string + SshKeyID string + SshKeyName string + SubnetName string + SubnetID string + VpcName string + VpcID string + VpcServiceURL string + WorkerFlavor string + WorkerOS string + Zone string + TunnelType string + VxlanPort string + ClusterID string + Tags string + DedicatedHostIDs string + DedicatedHostGroupIDs string WorkerCount int IsSelfManaged bool @@ -68,41 +70,43 @@ var IBMCloudProps = &IBMCloudProperties{} func InitIBMCloudProperties(properties map[string]string) error { IBMCloudProps = &IBMCloudProperties{ - IBMCloudProvider: properties["IBMCLOUD_PROVIDER"], - ApiKey: properties["APIKEY"], - IamProfileID: properties["IAM_PROFILE_ID"], - Bucket: properties["COS_BUCKET"], - CaaImageTag: properties["CAA_IMAGE_TAG"], - ClusterName: properties["CLUSTER_NAME"], - CosApiKey: properties["COS_APIKEY"], - CosInstanceID: properties["COS_INSTANCE_ID"], - CosServiceURL: properties["COS_SERVICE_URL"], - IamServiceURL: properties["IAM_SERVICE_URL"], - IksServiceURL: properties["IKS_SERVICE_URL"], - InitData: properties["INITDATA"], - InstanceProfile: properties["INSTANCE_PROFILE_NAME"], - KubeVersion: properties["KUBE_VERSION"], - PodvmImageID: properties["PODVM_IMAGE_ID"], - PodvmImageArch: properties["PODVM_IMAGE_ARCH"], - PublicGatewayName: properties["PUBLIC_GATEWAY_NAME"], - Region: properties["REGION"], - ResourceGroupID: properties["RESOURCE_GROUP_ID"], - SshKeyName: properties["SSH_KEY_NAME"], - SshKeyContent: properties["SSH_PUBLIC_KEY_CONTENT"], - SubnetName: properties["VPC_SUBNET_NAME"], - VpcName: properties["VPC_NAME"], - VpcServiceURL: properties["VPC_SERVICE_URL"], - WorkerFlavor: properties["WORKER_FLAVOR"], - WorkerOS: properties["WORKER_OPERATION_SYSTEM"], - Zone: properties["ZONE"], - SshKeyID: properties["SSH_KEY_ID"], - SubnetID: properties["VPC_SUBNET_ID"], - SecurityGroupID: properties["VPC_SECURITY_GROUP_ID"], - VpcID: properties["VPC_ID"], - TunnelType: properties["TUNNEL_TYPE"], - VxlanPort: properties["VXLAN_PORT"], - ClusterID: properties["CLUSTER_ID"], - Tags: properties["TAGS"], + IBMCloudProvider: properties["IBMCLOUD_PROVIDER"], + ApiKey: properties["APIKEY"], + IamProfileID: properties["IAM_PROFILE_ID"], + Bucket: properties["COS_BUCKET"], + CaaImageTag: properties["CAA_IMAGE_TAG"], + ClusterName: properties["CLUSTER_NAME"], + CosApiKey: properties["COS_APIKEY"], + CosInstanceID: properties["COS_INSTANCE_ID"], + CosServiceURL: properties["COS_SERVICE_URL"], + IamServiceURL: properties["IAM_SERVICE_URL"], + IksServiceURL: properties["IKS_SERVICE_URL"], + InitData: properties["INITDATA"], + InstanceProfile: properties["INSTANCE_PROFILE_NAME"], + KubeVersion: properties["KUBE_VERSION"], + PodvmImageID: properties["PODVM_IMAGE_ID"], + PodvmImageArch: properties["PODVM_IMAGE_ARCH"], + PublicGatewayName: properties["PUBLIC_GATEWAY_NAME"], + Region: properties["REGION"], + ResourceGroupID: properties["RESOURCE_GROUP_ID"], + SshKeyName: properties["SSH_KEY_NAME"], + SshKeyContent: properties["SSH_PUBLIC_KEY_CONTENT"], + SubnetName: properties["VPC_SUBNET_NAME"], + VpcName: properties["VPC_NAME"], + VpcServiceURL: properties["VPC_SERVICE_URL"], + WorkerFlavor: properties["WORKER_FLAVOR"], + WorkerOS: properties["WORKER_OPERATION_SYSTEM"], + Zone: properties["ZONE"], + SshKeyID: properties["SSH_KEY_ID"], + SubnetID: properties["VPC_SUBNET_ID"], + SecurityGroupID: properties["VPC_SECURITY_GROUP_ID"], + VpcID: properties["VPC_ID"], + TunnelType: properties["TUNNEL_TYPE"], + VxlanPort: properties["VXLAN_PORT"], + ClusterID: properties["CLUSTER_ID"], + Tags: properties["TAGS"], + DedicatedHostIDs: properties["DEDICATED_HOST_IDS"], + DedicatedHostGroupIDs: properties["DEDICATED_HOST_GROUP_IDS"], } if len(IBMCloudProps.IBMCloudProvider) <= 0 { diff --git a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go index af1807f2e..4622dbc83 100644 --- a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go +++ b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go @@ -67,6 +67,10 @@ func isKustomizeConfigMapKey(key string) bool { return true case "TAGS": return true + case "IBMCLOUD_DEDICATED_HOST_IDS": + return true + case "IBMCLOUD_DEDICATED_HOST_GROUP_IDS": + return true default: return false } diff --git a/src/cloud-providers/ibmcloud/manager.go b/src/cloud-providers/ibmcloud/manager.go index d639cfa66..d9805fd6f 100644 --- a/src/cloud-providers/ibmcloud/manager.go +++ b/src/cloud-providers/ibmcloud/manager.go @@ -45,6 +45,8 @@ func (_ *Manager) ParseCmd(flags *flag.FlagSet) { reg.CustomTypeWithEnv(&ibmcloudVPCConfig.InstanceProfiles, "profile-list", "", "IBMCLOUD_PODVM_INSTANCE_PROFILE_LIST", "List of instance profile names to be used for the Pod VMs, comma separated") reg.CustomTypeWithEnv(&ibmcloudVPCConfig.Images, "image-id", "", "IBMCLOUD_PODVM_IMAGE_ID", "List of Image IDs, comma separated", provider.Required()) reg.CustomTypeWithEnv(&ibmcloudVPCConfig.Tags, "tags", "", "TAGS", "List of tags to attach to the Pod VMs, comma separated") + reg.CustomTypeWithEnv(&ibmcloudVPCConfig.DedicatedHostIDs, "dedicated-host-ids", "", "IBMCLOUD_DEDICATED_HOST_IDS", "List of Dedicated Host IDs, provide one from each Zone") + reg.CustomTypeWithEnv(&ibmcloudVPCConfig.DedicatedHostGroupIDs, "dedicated-host-group-ids", "", "IBMCLOUD_DEDICATED_HOST_GROUP_IDS", "List of Dedicated Host Group IDs, provide one from each Zone") } func (_ *Manager) LoadEnv() { diff --git a/src/cloud-providers/ibmcloud/provider.go b/src/cloud-providers/ibmcloud/provider.go index 599bbe5a0..fb9a1e854 100644 --- a/src/cloud-providers/ibmcloud/provider.go +++ b/src/cloud-providers/ibmcloud/provider.go @@ -136,6 +136,37 @@ func NewProvider(config *Config) (provider.Provider, error) { } } + // Return error early + if config.ZoneName == "" { + return nil, fmt.Errorf("zone was not provided and could not detect automatically") + } + + if len(config.DedicatedHostIDs) > 0 { + selected, err := pickIDInZone( + config.DedicatedHostIDs, + config.ZoneName, + func(id string) (string, error) { return getDedicatedHostZone(vpcV1, id) }, + "Dedicated Host", + ) + if err != nil { + return nil, err + } + config.selectedDedicatedHostID = selected + } + + if len(config.DedicatedHostGroupIDs) > 0 { + selected, err := pickIDInZone( + config.DedicatedHostGroupIDs, + config.ZoneName, + func(id string) (string, error) { return getDedicatedHostGroupZone(vpcV1, id) }, + "Dedicated Host Group", + ) + if err != nil { + return nil, err + } + config.selectedDedicatedHostGroupID = selected + } + gTaggingV1, err := globaltaggingv1.NewGlobalTaggingV1( &globaltaggingv1.GlobalTaggingV1Options{ Authenticator: authenticator, @@ -209,6 +240,59 @@ func fetchVPCDetails(vpcV1 *vpcv1.VpcV1, subnetID string) (vpcID string, resourc return } +func getDedicatedHostZone(vpcV1 *vpcv1.VpcV1, dedicatedHostID string) (string, error) { + dedicatedHostOptions := vpcv1.GetDedicatedHostOptions{ + ID: &dedicatedHostID, + } + dedicatedHost, response, err := vpcV1.GetDedicatedHost(&dedicatedHostOptions) + if err != nil { + return "", fmt.Errorf("VPC error with:\n %w\nfurther details:\n %v", err, response) + } + + return *dedicatedHost.Zone.Name, nil +} + +func getDedicatedHostGroupZone(vpcV1 *vpcv1.VpcV1, dedicatedHostGroupID string) (string, error) { + dedicatedHostGroupOptions := vpcv1.GetDedicatedHostGroupOptions{ + ID: &dedicatedHostGroupID, + } + dedicatedHostGroup, response, err := vpcV1.GetDedicatedHostGroup(&dedicatedHostGroupOptions) + if err != nil { + return "", fmt.Errorf("VPC error with:\n %w\nfurther details:\n %v", err, response) + } + + return *dedicatedHostGroup.Zone.Name, nil +} + +// pickIDInZone finds the first ID whose zone equals zoneName. +// If multiple IDs match the zone, it logs a warning and returns the first. +// If none match, it returns a descriptive error. +func pickIDInZone(ids []string, zoneName string, getZone func(string) (string, error), resourceLabel string) (string, error) { + var selected string + + for _, id := range ids { + zone, err := getZone(id) + if err != nil { + return "", fmt.Errorf("couldn't get %s %s's zone: %w", id, resourceLabel, err) + } + if zone == zoneName { + if selected != "" && logger != nil { + logger.Printf("warning: multiple %ss were provided in zone %s; only one will be used", + resourceLabel, zoneName) + // Continue to keep the first match as the selected one. + continue + } + selected = id + } + } + + if selected == "" { + return "", fmt.Errorf("no %s in zone %s was provided; please provide a %s in the specified zone for High Availability", + resourceLabel, zoneName, resourceLabel) + } + return selected, nil +} + func (p *ibmcloudVPCProvider) getAttachTagOptions(vpcInstanceCRN *string) (*globaltaggingv1.AttachTagOptions, error) { if vpcInstanceCRN == nil { return nil, fmt.Errorf("missing vpc instance crn, can't create attach tag options") @@ -277,6 +361,15 @@ func (p *ibmcloudVPCProvider) getInstancePrototype(instanceName, userData, insta } } + // When both dedicated host id and group id provided the (more specific) dedicated host id will be used as the placement target + if p.serviceConfig.selectedDedicatedHostGroupID != "" { + prototype.PlacementTarget = &vpcv1.InstancePlacementTargetPrototypeDedicatedHostGroupIdentityDedicatedHostGroupIdentityByID{ID: &p.serviceConfig.selectedDedicatedHostGroupID} + } + + if p.serviceConfig.selectedDedicatedHostID != "" { + prototype.PlacementTarget = &vpcv1.InstancePlacementTargetPrototypeDedicatedHostGroupIdentityDedicatedHostGroupIdentityByID{ID: &p.serviceConfig.selectedDedicatedHostID} + } + return prototype } @@ -345,9 +438,8 @@ func (p *ibmcloudVPCProvider) CreateInstance(ctx context.Context, podName, sandb logger.Printf("CreateInstance: name: %q", instanceName) - vpcInstance, resp, err := p.vpc.CreateInstanceWithContext(ctx, &vpcv1.CreateInstanceOptions{InstancePrototype: prototype}) + vpcInstance, err := p.createInstanceWithFallback(ctx, prototype) if err != nil { - logger.Printf("failed to create an instance : %v and the response is %s", err, resp) return nil, err } @@ -390,7 +482,7 @@ func (p *ibmcloudVPCProvider) CreateInstance(ctx context.Context, podName, sandb return instance, fmt.Errorf("failed to get attach tag options: %w", err) } - _, resp, err = p.globalTagging.AttachTagWithContext(ctx, options) + _, resp, err := p.globalTagging.AttachTagWithContext(ctx, options) if err != nil { return instance, fmt.Errorf("failed to attach tags: %w and the response is %s", err, resp) } @@ -399,6 +491,44 @@ func (p *ibmcloudVPCProvider) CreateInstance(ctx context.Context, podName, sandb return instance, nil } +func (p *ibmcloudVPCProvider) createInstanceWithFallback(ctx context.Context, prototype *vpcv1.InstancePrototype) (*vpcv1.Instance, error) { + + dedicatedHostID := p.serviceConfig.selectedDedicatedHostID + dedicatedHostGroupID := p.serviceConfig.selectedDedicatedHostGroupID + + inst, resp, err := p.vpc.CreateInstanceWithContext(ctx, &vpcv1.CreateInstanceOptions{ + InstancePrototype: prototype, + }) + if err == nil { + return inst, nil + } + + // Fallback if both IDs exist + if dedicatedHostID != "" && dedicatedHostGroupID != "" { + logger.Printf("creation failed on dedicated host %q: %v; retrying on dedicated host group %q", dedicatedHostID, err, dedicatedHostGroupID) + + prototype.PlacementTarget = + &vpcv1.InstancePlacementTargetPrototypeDedicatedHostGroupIdentityDedicatedHostGroupIdentityByID{ + ID: &dedicatedHostGroupID, + } + + inst2, resp2, err2 := p.vpc.CreateInstanceWithContext(ctx, &vpcv1.CreateInstanceOptions{ + InstancePrototype: prototype, + }) + if err2 == nil { + return inst2, nil + } + + // Return both errors for context. + return nil, errors.Join( + fmt.Errorf("instance creation on dedicated host %q failed: %w and the response is %s", dedicatedHostID, err, resp), + fmt.Errorf("fallback instance creation on dedicated host group %q failed: %w and the response is %s", dedicatedHostGroupID, err2, resp2), + ) + } + + return nil, fmt.Errorf("failed to create an instance: %w and the response is %s", err, resp) +} + // Select an instance profile based on the memory and vcpu requirements func (p *ibmcloudVPCProvider) selectInstanceProfile(ctx context.Context, spec provider.InstanceTypeSpec) (string, error) { diff --git a/src/cloud-providers/ibmcloud/types.go b/src/cloud-providers/ibmcloud/types.go index 483ec38a5..e81ee821f 100644 --- a/src/cloud-providers/ibmcloud/types.go +++ b/src/cloud-providers/ibmcloud/types.go @@ -70,6 +70,28 @@ func (i *tags) Set(value string) error { return nil } +type dedicatedHostIDs []string + +func (i *dedicatedHostIDs) String() string { + return strings.Join(*i, ", ") +} + +func (i *dedicatedHostIDs) Set(value string) error { + *i = append(*i, toList(value, ",")...) + return nil +} + +type dedicatedHostGroupIDs []string + +func (i *dedicatedHostGroupIDs) String() string { + return strings.Join(*i, ", ") +} + +func (i *dedicatedHostGroupIDs) Set(value string) error { + *i = append(*i, toList(value, ",")...) + return nil +} + type Config struct { ApiKey string IAMProfileID string @@ -91,6 +113,11 @@ type Config struct { DisableCVM bool ClusterID string Tags tags + DedicatedHostIDs dedicatedHostIDs + DedicatedHostGroupIDs dedicatedHostGroupIDs + + selectedDedicatedHostID string + selectedDedicatedHostGroupID string } func (c Config) Redact() Config {