diff --git a/src/cloud-api-adaptor/ibmcloud/README.md b/src/cloud-api-adaptor/ibmcloud/README.md index 08a1ca2f8d..e45dac3d35 100644 --- a/src/cloud-api-adaptor/ibmcloud/README.md +++ b/src/cloud-api-adaptor/ibmcloud/README.md @@ -136,7 +136,6 @@ REGION="$(terraform output --raw region)" RESOURCE_GROUP_ID="$(terraform output --raw resource_group_id)" ZONE="$(terraform output --raw zone)" VPC_SUBNET_ID="$(terraform output --raw subnet_id)" -VPC_SECURITY_GROUP_ID="$(terraform output --raw security_group_id)" VPC_ID="$(terraform output --raw vpc_id)" EOF diff --git a/src/cloud-api-adaptor/ibmcloud/ROKS_SETUP.md b/src/cloud-api-adaptor/ibmcloud/ROKS_SETUP.md index d1ce018954..ac5d59970c 100644 --- a/src/cloud-api-adaptor/ibmcloud/ROKS_SETUP.md +++ b/src/cloud-api-adaptor/ibmcloud/ROKS_SETUP.md @@ -183,7 +183,6 @@ SSH_KEY_ID="$SSH_KEY_ID" PODVM_IMAGE_ID="$PODVM_IMAGE_ID" VPC_ID="$VPC_ID" VPC_SUBNET_ID="$SUBNET_ID" -VPC_SECURITY_GROUP_ID="$(ibmcloud ks security-group ls --cluster $CLUSTER_NAME -json | jq -r '.[] | select(.type == "cluster") | .id')" RESOURCE_GROUP_ID="$(ibmcloud is vpc "$VPC_ID" -json | jq -r .resource_group.id)" ZONE="$(ibmcloud is subnet $SUBNET_ID -json | jq -r .zone.name)" REGION="$(ibmcloud is zone $ZONE -json | jq -r .region.name)" diff --git a/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml index 993be25ccb..c9e8510c02 100644 --- a/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml @@ -27,7 +27,7 @@ configMapGenerator: - IBMCLOUD_PODVM_INSTANCE_PROFILE_LIST="" #optional, comma separated list - IBMCLOUD_ZONE="" #set - IBMCLOUD_VPC_SUBNET_ID="" #set - - IBMCLOUD_VPC_SG_ID="" #set + - IBMCLOUD_SECURITY_GROUP_IDS="" #optional, comma separated list - IBMCLOUD_VPC_ID="" #set - CRI_RUNTIME_ENDPOINT="/run/cri-runtime/containerd.sock" - DISABLECVM="true" # Set to false to enable confidential VM diff --git a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_common.go b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_common.go index 27d5d21d49..1a1c4f6f72 100644 --- a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_common.go @@ -154,16 +154,6 @@ func createVPC() error { } } - sgoptions := &vpcv1.GetVPCDefaultSecurityGroupOptions{} - sgoptions.SetID(IBMCloudProps.VpcID) - defaultSG, _, err := IBMCloudProps.VPC.GetVPCDefaultSecurityGroup(sgoptions) - if err != nil { - return err - } - - IBMCloudProps.SecurityGroupID = *defaultSG.ID - log.Infof("Got VPC default SecurityGroupID %s.", IBMCloudProps.SecurityGroupID) - return nil } @@ -967,7 +957,7 @@ func (p *IBMCloudProvisioner) GetProperties(ctx context.Context, cfg *envconf.Co "IBMCLOUD_PODVM_INSTANCE_PROFILE_NAME": IBMCloudProps.InstanceProfile, "IBMCLOUD_ZONE": IBMCloudProps.Zone, "IBMCLOUD_VPC_SUBNET_ID": IBMCloudProps.SubnetID, - "IBMCLOUD_VPC_SG_ID": IBMCloudProps.SecurityGroupID, + "IBMCLOUD_SECURITY_GROUP_IDS": IBMCloudProps.SecurityGroupIDs, "IBMCLOUD_VPC_ID": IBMCloudProps.VpcID, "CRI_RUNTIME_ENDPOINT": "/run/cri-runtime/containerd.sock", "IBMCLOUD_API_KEY": IBMCloudProps.ApiKey, @@ -982,18 +972,3 @@ func (p *IBMCloudProvisioner) GetProperties(ctx context.Context, cfg *envconf.Co "TAGS": IBMCloudProps.Tags, } } - -func (p *IBMCloudProvisioner) GetVPCDefaultSecurityGroupID(vpcID string) (string, error) { - if len(IBMCloudProps.SecurityGroupID) > 0 { - return IBMCloudProps.SecurityGroupID, nil - } - - options := &vpcv1.GetVPCDefaultSecurityGroupOptions{} - options.SetID(vpcID) - defaultSG, _, err := IBMCloudProps.VPC.GetVPCDefaultSecurityGroup(options) - if err != nil { - return "", err - } - - return *defaultSG.ID, nil -} diff --git a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_ibmcloud.properties b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_ibmcloud.properties index 94cf105a4a..e0503cc58f 100644 --- a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_ibmcloud.properties +++ b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_ibmcloud.properties @@ -44,8 +44,8 @@ PUBLIC_GATEWAY_NAME="" VPC_SUBNET_NAME="" # optional, existing subnet id if using existing VPC and cluster for testing VPC_SUBNET_ID="" -# optional, existing security group id if using existing VPC and cluster for testing -VPC_SECURITY_GROUP_ID="" +# optional, a list of security group IDs if you need additional ones besides the default +IBMCLOUD_SECURITY_GROUP_IDS="" # optional, it'll be set as ${CLUSTER_NAME}-vpc if not provided VPC_NAME="" # optional, existing VPC id if using existing VPC and cluster for testing diff --git a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_initializer.go b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_initializer.go index 23f98a72a3..7e06177f56 100644 --- a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_initializer.go +++ b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_initializer.go @@ -28,7 +28,7 @@ type IBMCloudProperties struct { CosApiKey string CosInstanceID string CosServiceURL string - SecurityGroupID string + SecurityGroupIDs string IamServiceURL string IksServiceURL string InitData string @@ -97,7 +97,7 @@ func InitIBMCloudProperties(properties map[string]string) error { Zone: properties["ZONE"], SshKeyID: properties["SSH_KEY_ID"], SubnetID: properties["VPC_SUBNET_ID"], - SecurityGroupID: properties["VPC_SECURITY_GROUP_ID"], + SecurityGroupIDs: properties["IBMCLOUD_SECURITY_GROUP_IDS"], VpcID: properties["VPC_ID"], TunnelType: properties["TUNNEL_TYPE"], VxlanPort: properties["VXLAN_PORT"], @@ -200,8 +200,8 @@ func InitIBMCloudProperties(properties map[string]string) error { if len(IBMCloudProps.SubnetID) <= 0 { log.Info("[warning] VPC_SUBNET_ID was not set.") } - if len(IBMCloudProps.SecurityGroupID) <= 0 { - log.Info("[warning] VPC_SECURITY_GROUP_ID was not set.") + if len(IBMCloudProps.SecurityGroupIDs) <= 0 { + log.Info("IBMCLOUD_SECURITY_GROUP_IDS was not set.") } if len(IBMCloudProps.VpcID) <= 0 { log.Info("[warning] VPC_ID was not set.") diff --git a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go index af1807f2ea..c0e5f0b871 100644 --- a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go +++ b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go @@ -49,7 +49,7 @@ func isKustomizeConfigMapKey(key string) bool { return true case "IBMCLOUD_VPC_SUBNET_ID": return true - case "IBMCLOUD_VPC_SG_ID": + case "IBMCLOUD_SECURITY_GROUP_IDS": return true case "IBMCLOUD_VPC_ID": return true diff --git a/src/cloud-providers/go.mod b/src/cloud-providers/go.mod index 2e80170d24..0c9de81d94 100644 --- a/src/cloud-providers/go.mod +++ b/src/cloud-providers/go.mod @@ -110,7 +110,7 @@ require ( github.com/google/go-cmp v0.7.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.9 // indirect - github.com/google/uuid v1.6.0 // indirect + github.com/google/uuid v1.6.0 github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/src/cloud-providers/ibmcloud/cluster.go b/src/cloud-providers/ibmcloud/cluster.go new file mode 100644 index 0000000000..7779eed263 --- /dev/null +++ b/src/cloud-providers/ibmcloud/cluster.go @@ -0,0 +1,141 @@ +package ibmcloud + +import ( + "context" + "fmt" + "runtime" + + "github.com/IBM/go-sdk-core/v5/core" + "github.com/google/uuid" +) + +type ClusterV2 struct { + Service *core.BaseService +} + +const DefaultServiceURL = "https://containers.cloud.ibm.com/global" + +type ClusterOptions struct { + Authenticator core.Authenticator +} + +func NewClusterV2Service(options *ClusterOptions) (service *ClusterV2, err error) { + serviceOptions := &core.ServiceOptions{ + URL: DefaultServiceURL, + Authenticator: options.Authenticator, + } + + err = core.ValidateStruct(options, "options") + if err != nil { + err = core.SDKErrorf(err, "", "invalid-global-options", GetComponentInfo()) + return + } + + baseService, err := core.NewBaseService(serviceOptions) + if err != nil { + err = core.SDKErrorf(err, "", "new-base-error", GetComponentInfo()) + return + } + + service = &ClusterV2{ + Service: baseService, + } + + return +} + +func (clusterApi *ClusterV2) GetClusterTypeSecurityGroups(clusterID string) (result []securityGroup, response *core.DetailedResponse, err error) { + builder := core.NewRequestBuilder(core.GET) + builder = builder.WithContext(context.Background()) + builder.EnableGzipCompression = clusterApi.Service.GetEnableGzipCompression() + + // Construct the request URL + _, err = builder.ResolveRequestURL( + clusterApi.Service.Options.URL, + "/network/v2/security-group/getSecurityGroups", + nil, + ) + if err != nil { + err = core.SDKErrorf(err, "", "url-resolve-error", GetComponentInfo()) + return + } + + builder.AddQuery("cluster", clusterID) + builder.AddQuery("type", "cluster") + + // Add headers + sdkHeaders := GetHeaders("kubernetes_service_api", "V2", "GetSecurityGroups") + for headerName, headerValue := range sdkHeaders { + builder.AddHeader(headerName, headerValue) + } + + builder.AddHeader("Accept", "application/json") + + // Build the request + request, err := builder.Build() + if err != nil { + err = core.SDKErrorf(err, "", "build-error", GetComponentInfo()) + return + } + + var rawResponse []securityGroup + response, err = clusterApi.Service.Request(request, &rawResponse) + if err != nil { + err = core.SDKErrorf(err, "", "http-request-err", GetComponentInfo()) + return + } + if rawResponse != nil { + result = rawResponse + response.Result = result + } + + return +} + +type securityGroup struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + UserProvided bool `json:"userProvided"` + Shared bool `json:"shared"` + WorkerPoolID string `json:"workerPoolID"` +} + +const ( + HEADER_NAME_USER_AGENT = "User-Agent" + + NAME = "cloud-api-adaptor-ibm" + + X_REQUEST_ID = "X-Request-Id" + + VERSION = "0.0.1" +) + +func GetHeaders(serviceName string, serviceVersion string, operationId string) map[string]string { + sdkHeaders := make(map[string]string) + + sdkHeaders[HEADER_NAME_USER_AGENT] = GetUserAgentInfo() + sdkHeaders[X_REQUEST_ID] = GetNewXRequestID() + + return sdkHeaders +} + +var UserAgent string = fmt.Sprintf("%s-%s %s", NAME, VERSION, GetSystemInfo()) + +func GetUserAgentInfo() string { + return UserAgent +} + +func GetNewXRequestID() string { + return uuid.New().String() +} + +var systemInfo = fmt.Sprintf("(arch=%s; os=%s; go.version=%s)", runtime.GOARCH, runtime.GOOS, runtime.Version()) + +func GetSystemInfo() string { + return systemInfo +} + +func GetComponentInfo() *core.ProblemComponent { + return core.NewProblemComponent("github.com/confidential-containers/cloud-api-adaptor", VERSION) +} diff --git a/src/cloud-providers/ibmcloud/manager.go b/src/cloud-providers/ibmcloud/manager.go index d639cfa664..847b66689c 100644 --- a/src/cloud-providers/ibmcloud/manager.go +++ b/src/cloud-providers/ibmcloud/manager.go @@ -29,7 +29,6 @@ func (_ *Manager) ParseCmd(flags *flag.FlagSet) { reg.StringWithEnv(&ibmcloudVPCConfig.ProfileName, "profile-name", "", "IBMCLOUD_PODVM_INSTANCE_PROFILE_NAME", "Default instance profile name to be used for the Pod VMs") reg.StringWithEnv(&ibmcloudVPCConfig.ZoneName, "zone-name", "", "IBMCLOUD_ZONE", "Zone name") reg.StringWithEnv(&ibmcloudVPCConfig.PrimarySubnetID, "primary-subnet-id", "", "IBMCLOUD_VPC_SUBNET_ID", "Primary subnet ID") - reg.StringWithEnv(&ibmcloudVPCConfig.PrimarySecurityGroupID, "primary-security-group-id", "", "IBMCLOUD_VPC_SG_ID", "Primary security group ID") reg.StringWithEnv(&ibmcloudVPCConfig.KeyID, "key-id", "", "IBMCLOUD_SSH_KEY_ID", "SSH Key ID") reg.StringWithEnv(&ibmcloudVPCConfig.VpcID, "vpc-id", "", "IBMCLOUD_VPC_ID", "VPC ID") reg.StringWithEnv(&ibmcloudVPCConfig.ClusterID, "cluster-id", "", "IBMCLOUD_CLUSTER_ID", "Cluster ID") @@ -45,6 +44,7 @@ func (_ *Manager) ParseCmd(flags *flag.FlagSet) { reg.CustomTypeWithEnv(&ibmcloudVPCConfig.InstanceProfiles, "profile-list", "", "IBMCLOUD_PODVM_INSTANCE_PROFILE_LIST", "List of instance profile names to be used for the Pod VMs, comma separated") reg.CustomTypeWithEnv(&ibmcloudVPCConfig.Images, "image-id", "", "IBMCLOUD_PODVM_IMAGE_ID", "List of Image IDs, comma separated", provider.Required()) reg.CustomTypeWithEnv(&ibmcloudVPCConfig.Tags, "tags", "", "TAGS", "List of tags to attach to the Pod VMs, comma separated") + reg.CustomTypeWithEnv(&ibmcloudVPCConfig.SecurityGroupIds, "security-group-ids", "", "IBMCLOUD_SECURITY_GROUP_IDS", "List of Security Group IDs to be used for the Pod VM, comma separated") } func (_ *Manager) LoadEnv() { diff --git a/src/cloud-providers/ibmcloud/provider.go b/src/cloud-providers/ibmcloud/provider.go index 599bbe5a05..26d2f76f4f 100644 --- a/src/cloud-providers/ibmcloud/provider.go +++ b/src/cloud-providers/ibmcloud/provider.go @@ -49,9 +49,14 @@ type globalTaggingV1 interface { AttachTagWithContext(ctx context.Context, attachTagOptions *globaltaggingv1.AttachTagOptions) (*globaltaggingv1.TagResults, *core.DetailedResponse, error) } +type clusterV2 interface { + GetClusterTypeSecurityGroups(clusterID string) (result []securityGroup, response *core.DetailedResponse, err error) +} + type ibmcloudVPCProvider struct { vpc vpcV1 globalTagging globalTaggingV1 + cluster clusterV2 serviceConfig *Config } @@ -117,7 +122,7 @@ func NewProvider(config *Config) (provider.Provider, error) { if config.ZoneName == "" { config.ZoneName = nodeLabels["topology.kubernetes.io/zone"] } - vpcID, rgID, sgID, err := fetchVPCDetails(vpcV1, primarySubnetID) + vpcID, rgID, err := fetchVPCDetails(vpcV1, primarySubnetID) if err != nil { logger.Printf("warning, unable to automatically populate VPC details\ndue to: %v\n", err) } else { @@ -130,9 +135,6 @@ func NewProvider(config *Config) (provider.Provider, error) { if config.ResourceGroupID == "" { config.ResourceGroupID = rgID } - if config.PrimarySecurityGroupID == "" { - config.PrimarySecurityGroupID = sgID - } } } @@ -144,9 +146,21 @@ func NewProvider(config *Config) (provider.Provider, error) { return nil, err } + clusterV2, err := NewClusterV2Service(&ClusterOptions{Authenticator: authenticator}) + if err != nil { + return nil, err + } + + sgID, err := fetchClusterSG(clusterV2, config.ClusterID) + if err != nil { + return nil, err + } + config.SecurityGroupIds = append(config.SecurityGroupIds, sgID) + provider := &ibmcloudVPCProvider{ vpc: vpcV1, globalTagging: gTaggingV1, + cluster: clusterV2, serviceConfig: config, } @@ -186,7 +200,7 @@ func getClusterID() (string, error) { return clusterID, nil } -func fetchVPCDetails(vpcV1 *vpcv1.VpcV1, subnetID string) (vpcID string, resourceGroupID string, securityGroupID string, e error) { +func fetchVPCDetails(vpcV1 *vpcv1.VpcV1, subnetID string) (vpcID string, resourceGroupID string, e error) { subnet, response, err := vpcV1.GetSubnet(&vpcv1.GetSubnetOptions{ ID: &subnetID, }) @@ -195,17 +209,27 @@ func fetchVPCDetails(vpcV1 *vpcv1.VpcV1, subnetID string) (vpcID string, resourc return } - sg, response, err := vpcV1.GetVPCDefaultSecurityGroup(&vpcv1.GetVPCDefaultSecurityGroupOptions{ - ID: subnet.VPC.ID, - }) + vpcID = *subnet.VPC.ID + resourceGroupID = *subnet.ResourceGroup.ID + return +} + +func fetchClusterSG(clusterv2 clusterV2, clusterID string) (securityGroupID string, e error) { + securityGroups, response, err := clusterv2.GetClusterTypeSecurityGroups(clusterID) if err != nil { - e = fmt.Errorf("VPC error with:\n %w\nfurther details:\n %v", err, response) + e = fmt.Errorf("cluster error with:\n %w\nfurther details:\n %v", err, response) return } - securityGroupID = *sg.ID - vpcID = *subnet.VPC.ID - resourceGroupID = *subnet.ResourceGroup.ID + expectedSgName := fmt.Sprintf("kube-%s", clusterID) + + for _, sg := range securityGroups { + if sg.Name == expectedSgName { + securityGroupID = sg.ID + return + } + } + e = fmt.Errorf("could not find default cluster security group %s", expectedSgName) return } @@ -227,6 +251,13 @@ func (p *ibmcloudVPCProvider) getAttachTagOptions(vpcInstanceCRN *string) (*glob func (p *ibmcloudVPCProvider) getInstancePrototype(instanceName, userData, instanceProfile, imageId string) *vpcv1.InstancePrototype { + securityGroups := make([]vpcv1.SecurityGroupIdentityIntf, 0, len(p.serviceConfig.SecurityGroupIds)) + for i := range p.serviceConfig.SecurityGroupIds { + securityGroups = append(securityGroups, &vpcv1.SecurityGroupIdentityByID{ + ID: &p.serviceConfig.SecurityGroupIds[i], + }) + } + prototype := &vpcv1.InstancePrototype{ Name: &instanceName, Image: &vpcv1.ImageIdentity{ID: &imageId}, @@ -236,10 +267,8 @@ func (p *ibmcloudVPCProvider) getInstancePrototype(instanceName, userData, insta Keys: []vpcv1.KeyIdentityIntf{}, VPC: &vpcv1.VPCIdentity{ID: &p.serviceConfig.VpcID}, PrimaryNetworkInterface: &vpcv1.NetworkInterfacePrototype{ - Subnet: &vpcv1.SubnetIdentity{ID: &p.serviceConfig.PrimarySubnetID}, - SecurityGroups: []vpcv1.SecurityGroupIdentityIntf{ - &vpcv1.SecurityGroupIdentityByID{ID: &p.serviceConfig.PrimarySecurityGroupID}, - }, + Subnet: &vpcv1.SubnetIdentity{ID: &p.serviceConfig.PrimarySubnetID}, + SecurityGroups: securityGroups, }, MetadataService: &vpcv1.InstanceMetadataServicePrototype{ Enabled: core.BoolPtr(true), diff --git a/src/cloud-providers/ibmcloud/types.go b/src/cloud-providers/ibmcloud/types.go index 483ec38a5b..4be65f722a 100644 --- a/src/cloud-providers/ibmcloud/types.go +++ b/src/cloud-providers/ibmcloud/types.go @@ -70,6 +70,17 @@ func (i *tags) Set(value string) error { return nil } +type securityGroupIds []string + +func (i *securityGroupIds) String() string { + return strings.Join(*i, ", ") +} + +func (i *securityGroupIds) Set(value string) error { + *i = append(*i, toList(value, ",")...) + return nil +} + type Config struct { ApiKey string IAMProfileID string @@ -81,7 +92,7 @@ type Config struct { ZoneName string Images Images PrimarySubnetID string - PrimarySecurityGroupID string + SecurityGroupIds securityGroupIds SecondarySubnetID string SecondarySecurityGroupID string KeyID string