diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index c7627f0468da..aaa241646f84 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -18,6 +18,7 @@ package cmd import ( "fmt" + "net" "runtime" "strings" "time" @@ -61,6 +62,7 @@ const ( kubernetesVersion = "kubernetes-version" noKubernetes = "no-kubernetes" hostOnlyCIDR = "host-only-cidr" + hostOnlyCIDRv6 = "host-only-cidr-v6" containerRuntime = "container-runtime" criSocket = "cri-socket" networkPlugin = "network-plugin" // deprecated, use --cni instead @@ -84,6 +86,8 @@ const ( apiServerPort = "apiserver-port" dnsDomain = "dns-domain" serviceCIDR = "service-cluster-ip-range" + serviceCIDRv6 = "service-cluster-ip-range-v6" + ipFamily = "ip-family" imageRepository = "image-repository" imageMirrorCountry = "image-mirror-country" mountString = "mount-string" @@ -144,8 +148,12 @@ const ( socketVMnetClientPath = "socket-vmnet-client-path" socketVMnetPath = "socket-vmnet-path" staticIP = "static-ip" + staticIPv6 = "static-ipv6" gpus = "gpus" autoPauseInterval = "auto-pause-interval" + subnetv6 = "subnet-v6" + podCIDR = "pod-cidr" + podCIDRv6 = "pod-cidr-v6" ) var ( @@ -209,6 +217,7 @@ func initMinikubeFlags() { startCmd.Flags().Bool(disableMetrics, false, "If set, disables metrics reporting (CPU and memory usage), this can improve CPU usage. Defaults to false.") startCmd.Flags().Bool(disableCoreDNSLog, false, "If set, disable CoreDNS verbose logging. Defaults to false.") startCmd.Flags().String(staticIP, "", "Set a static IP for the minikube cluster, the IP must be: private, IPv4, and the last octet must be between 2 and 254, for example 192.168.200.200 (Docker and Podman drivers only)") + startCmd.Flags().String(staticIPv6, "", "Set a static IPv6 address for the minikube cluster, for example fd00::100 (Docker and Podman drivers only)") startCmd.Flags().StringP(gpus, "g", "", "Allow pods to use your GPUs. Options include: [all,nvidia,amd] (Docker driver with Docker container-runtime only)") startCmd.Flags().Duration(autoPauseInterval, time.Minute*1, "Duration of inactivity before the minikube VM is paused (default 1m0s)") } @@ -260,6 +269,7 @@ func initDriverFlags() { // virtualbox startCmd.Flags().String(hostOnlyCIDR, "192.168.59.1/24", "The CIDR to be used for the minikube VM (virtualbox driver only)") + startCmd.Flags().String(hostOnlyCIDRv6, "fd00::1/64", "The IPv6 CIDR to be used for the minikube VM (virtualbox driver only)") startCmd.Flags().Bool(dnsProxy, false, "Enable proxy for NAT DNS requests (virtualbox driver only)") startCmd.Flags().Bool(hostDNSResolver, true, "Enable host resolver for NAT DNS requests (virtualbox driver only)") startCmd.Flags().Bool(noVTXCheck, false, "Disable checking for the availability of hardware virtualization before the vm is started (virtualbox driver only)") @@ -282,6 +292,9 @@ func initDriverFlags() { startCmd.Flags().String(listenAddress, "", "IP Address to use to expose ports (docker and podman driver only)") startCmd.Flags().StringSlice(ports, []string{}, "List of ports that should be exposed (docker and podman driver only)") startCmd.Flags().String(subnet, "", "Subnet to be used on kic cluster. If left empty, minikube will choose subnet address, beginning from 192.168.49.0. (docker and podman driver only)") + startCmd.Flags().String(subnetv6, "", "IPv6 subnet (CIDR) for the Docker/Podman network. If empty, minikube picks an internal ULA. (docker and podman driver only)") + startCmd.Flags().String(podCIDR, "", "IPv4 CIDR to use for pod IPs (bridge CNI).") + startCmd.Flags().String(podCIDRv6, "", "IPv6 CIDR to use for pod IPs (bridge CNI).") // qemu startCmd.Flags().String(qemuFirmwarePath, "", "Path to the qemu firmware file. Defaults: For Linux, the default firmware location. For macOS, the brew installation location. For Windows, C:\\Program Files\\qemu\\share") @@ -293,7 +306,9 @@ func initNetworkingFlags() { startCmd.Flags().StringSliceVar(®istryMirror, "registry-mirror", nil, "Registry mirrors to pass to the Docker daemon") startCmd.Flags().String(imageRepository, "", "Alternative image repository to pull docker images from. This can be used when you have limited access to gcr.io. Set it to \"auto\" to let minikube decide one for you. For Chinese mainland users, you may use local gcr.io mirrors such as registry.cn-hangzhou.aliyuncs.com/google_containers") startCmd.Flags().String(imageMirrorCountry, "", "Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.") - startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.") + startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The IPv4 CIDR to be used for service cluster IPs.") + startCmd.Flags().String(serviceCIDRv6, constants.DefaultServiceCIDRv6, "The IPv6 CIDR to be used for service cluster IPs.") + startCmd.Flags().String(ipFamily, "ipv4", "Cluster IP family mode: one of 'ipv4' (default), 'ipv6', or 'dual'.") startCmd.Flags().StringArrayVar(&config.DockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)") startCmd.Flags().StringArrayVar(&config.DockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)") @@ -494,6 +509,112 @@ func getNetwork(driverName string, options *run.CommandOptions) string { return n } +// normalizeAndValidateIPFamily sets defaults, validates CIDRs, and adds +// guardrails for IPv6/dual-stack on Docker. +func normalizeAndValidateIPFamily(cc *config.ClusterConfig) { + fam := strings.ToLower(strings.TrimSpace(cc.KubernetesConfig.IPFamily)) + switch fam { + case "", "ipv4", "ipv6", "dual": + // ok + default: + exit.Message(reason.Usage, + "Invalid --ip-family {{.fam}}. Must be one of: ipv4, ipv6, dual.", + out.V{"fam": cc.KubernetesConfig.IPFamily}) + } + + // Default to ipv4 to keep existing behavior for old configs. + if fam == "" { + fam = "ipv4" + cc.KubernetesConfig.IPFamily = fam + } + + // IPv6-capable families: ensure v6 defaults. + if fam != "ipv4" { + if cc.KubernetesConfig.ServiceCIDRv6 == "" { + cc.KubernetesConfig.ServiceCIDRv6 = constants.DefaultServiceCIDRv6 + } + if cc.KubernetesConfig.PodCIDRv6 == "" { + cc.KubernetesConfig.PodCIDRv6 = constants.DefaultPodCIDRv6 + } + } + + // For ipv4 or dual-stack, keep the usual IPv4 pod CIDR default if none was set. + if fam != "ipv6" && cc.KubernetesConfig.PodCIDR == "" { + cc.KubernetesConfig.PodCIDR = cni.DefaultPodCIDR + } + + // --- CIDR / address validation --- + + // Helper: ensure a CIDR is v4 or v6 as expected. + checkCIDR := func(flag string, cidr string, wantV6 bool) { + ip, _, err := net.ParseCIDR(cidr) + if err != nil { + exit.Message(reason.Usage, "{{.flag}} must be a valid CIDR: {{.e}}", + out.V{"flag": flag, "e": err}) + } + isV4 := ip.To4() != nil + if wantV6 && isV4 { + exit.Message(reason.Usage, "{{.flag}} must be a valid IPv6 CIDR: {{.e}}", + out.V{"flag": flag, "e": err}) + } + if !wantV6 && !isV4 { + exit.Message(reason.Usage, "{{.flag}} must be a valid IPv4 CIDR: {{.e}}", + out.V{"flag": flag, "e": err}) + } + } + + // IPv6-only CIDR flags. + if cidr := cc.Subnetv6; cidr != "" { + checkCIDR("--subnet-v6", cidr, true) + } + if cidr := cc.KubernetesConfig.ServiceCIDRv6; cidr != "" { + checkCIDR("--service-cluster-ip-range-v6", cidr, true) + } + if cidr := cc.KubernetesConfig.PodCIDRv6; cidr != "" { + checkCIDR("--pod-cidr-v6", cidr, true) + } + + // IPv4-only pod CIDR. + if cidr := cc.KubernetesConfig.PodCIDR; cidr != "" { + checkCIDR("--pod-cidr", cidr, false) + } + + // validate static IPv6 if provided + if s := cc.StaticIPv6; s != "" { + ip := net.ParseIP(s) + if ip == nil || ip.To4() != nil { + exit.Message(reason.Usage, "--static-ipv6 must be a valid IPv6 address") + } + } + + // Docker driver guardrails: Linux daemon for IPv6, warn on Desktop. + if driver.IsDocker(cc.Driver) && fam != "ipv4" { + // Desktop vs Linux daemon hint (we can't reliably detect IPv6 enabled here). + si, err := oci.CachedDaemonInfo(cc.Driver) + if err != nil { + si, err = oci.DaemonInfo(cc.Driver) + if err != nil { + exit.Message(reason.Usage, "Failed to query Docker daemon info: {{.e}}", out.V{"e": err}) + } + } + // On non-Linux hosts we assume Docker Desktop; on Linux it's a native Engine + // unless DockerOS explicitly says "Docker Desktop". + isLinuxDaemon := runtime.GOOS == "linux" && si.DockerOS != "Docker Desktop" + if !isLinuxDaemon { + if fam == "ipv6" { + exit.Message(reason.Usage, + "IPv6 clusters require a Linux Docker daemon (Desktop is not supported). "+ + "Use a Linux/WSL2 daemon or set --ip-family=ipv4.") + } + out.WarningT("Dual-stack on Docker Desktop may be limited. For full IPv6 support, use a Linux Docker daemon.") + } + // Friendly reminder about enabling daemon IPv6 (actual failure will occur during + // network create if the daemon/network really blocks IPv6 bridge networks). + out.Styled(style.Tip, + "If your Docker daemon/network blocks IPv6 bridge networks, enable IPv6 in /etc/docker/daemon.json and restart:\n {\"ipv6\": true, \"fixed-cidr-v6\": \"fd00:55:66::/64\"}") + } +} + func validateQemuNetwork(n string) string { switch n { case "socket_vmnet": @@ -576,6 +697,7 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str KicBaseImage: viper.GetString(kicBaseImage), Network: getNetwork(drvName, options), Subnet: viper.GetString(subnet), + Subnetv6: viper.GetString(subnetv6), Memory: getMemorySize(cmd, drvName), CPUs: getCPUCount(drvName), DiskSize: getDiskSize(), @@ -590,6 +712,7 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str InsecureRegistry: insecureRegistry, RegistryMirror: registryMirror, HostOnlyCIDR: viper.GetString(hostOnlyCIDR), + HostOnlyCIDRv6: viper.GetString(hostOnlyCIDRv6), HypervVirtualSwitch: viper.GetString(hypervVirtualSwitch), HypervUseExternalSwitch: viper.GetBool(hypervUseExternalSwitch), HypervExternalAdapter: viper.GetString(hypervExternalAdapter), @@ -631,6 +754,7 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str SocketVMnetClientPath: detect.SocketVMNetClientPath(), SocketVMnetPath: detect.SocketVMNetPath(), StaticIP: viper.GetString(staticIP), + StaticIPv6: viper.GetString(staticIPv6), KubernetesConfig: config.KubernetesConfig{ KubernetesVersion: k8sVersion, ClusterName: ClusterFlagValue(), @@ -644,6 +768,10 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str CRISocket: viper.GetString(criSocket), NetworkPlugin: chosenNetworkPlugin, ServiceCIDR: viper.GetString(serviceCIDR), + ServiceCIDRv6: viper.GetString(serviceCIDRv6), + PodCIDR: viper.GetString(podCIDR), + PodCIDRv6: viper.GetString(podCIDRv6), + IPFamily: viper.GetString(ipFamily), ImageRepository: getRepository(cmd, k8sVersion), ExtraOptions: getExtraOptions(), ShouldLoadCachedImages: viper.GetBool(cacheImages), @@ -697,6 +825,8 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str } } + normalizeAndValidateIPFamily(&cc) + return cc } @@ -831,11 +961,15 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC updateStringFromFlag(cmd, &cc.MinikubeISO, isoURL) updateStringFromFlag(cmd, &cc.KicBaseImage, kicBaseImage) updateStringFromFlag(cmd, &cc.Network, network) + updateStringFromFlag(cmd, &cc.Subnetv6, subnetv6) + updateStringFromFlag(cmd, &cc.KubernetesConfig.PodCIDR, podCIDR) + updateStringFromFlag(cmd, &cc.KubernetesConfig.PodCIDRv6, podCIDRv6) updateStringFromFlag(cmd, &cc.HyperkitVpnKitSock, vpnkitSock) updateStringSliceFromFlag(cmd, &cc.HyperkitVSockPorts, vsockPorts) updateStringSliceFromFlag(cmd, &cc.NFSShare, nfsShare) updateStringFromFlag(cmd, &cc.NFSSharesRoot, nfsSharesRoot) updateStringFromFlag(cmd, &cc.HostOnlyCIDR, hostOnlyCIDR) + updateStringFromFlag(cmd, &cc.HostOnlyCIDRv6, hostOnlyCIDRv6) updateStringFromFlag(cmd, &cc.HypervVirtualSwitch, hypervVirtualSwitch) updateBoolFromFlag(cmd, &cc.HypervUseExternalSwitch, hypervUseExternalSwitch) updateStringFromFlag(cmd, &cc.HypervExternalAdapter, hypervExternalAdapter) @@ -853,6 +987,7 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC updateDurationFromFlag(cmd, &cc.StartHostTimeout, waitTimeout) updateStringSliceFromFlag(cmd, &cc.ExposedPorts, ports) updateStringFromFlag(cmd, &cc.SSHIPAddress, sshIPAddress) + updateStringFromFlag(cmd, &cc.StaticIPv6, staticIPv6) updateStringFromFlag(cmd, &cc.SSHUser, sshSSHUser) updateStringFromFlag(cmd, &cc.SSHKey, sshSSHKey) updateIntFromFlag(cmd, &cc.SSHPort, sshSSHPort) @@ -865,6 +1000,8 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC updateStringFromFlag(cmd, &cc.KubernetesConfig.CRISocket, criSocket) updateStringFromFlag(cmd, &cc.KubernetesConfig.NetworkPlugin, networkPlugin) updateStringFromFlag(cmd, &cc.KubernetesConfig.ServiceCIDR, serviceCIDR) + updateStringFromFlag(cmd, &cc.KubernetesConfig.ServiceCIDRv6, serviceCIDRv6) + updateStringFromFlag(cmd, &cc.KubernetesConfig.IPFamily, ipFamily) updateBoolFromFlag(cmd, &cc.KubernetesConfig.ShouldLoadCachedImages, cacheImages) updateDurationFromFlag(cmd, &cc.CertExpiration, certExpiration) updateStringFromFlag(cmd, &cc.MountString, mountString) @@ -921,7 +1058,7 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC if cc.ScheduledStop != nil && time.Until(time.Unix(cc.ScheduledStop.InitiationTime, 0).Add(cc.ScheduledStop.Duration)) <= 0 { cc.ScheduledStop = nil } - + normalizeAndValidateIPFamily(&cc) return cc } diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index f8e35de5f51d..143acb13cadc 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -89,50 +89,115 @@ func (d *Driver) Create() error { OCIBinary: d.NodeConfig.OCIBinary, APIServerPort: d.NodeConfig.APIServerPort, GPUs: d.NodeConfig.GPUs, + IPFamily: strings.ToLower(d.NodeConfig.IPFamily), + IPv6: d.NodeConfig.StaticIPv6, } if params.Memory != "0" { params.Memory += "mb" } - networkName := d.NodeConfig.Network if networkName == "" { networkName = d.NodeConfig.ClusterName } + staticIP := d.NodeConfig.StaticIP - if gateway, err := oci.CreateNetwork(d.OCIBinary, networkName, d.NodeConfig.Subnet, staticIP); err != nil { + // NEW: create network with IPv6/dual awareness + gateway, err := oci.CreateNetworkWithIPFamily( + d.OCIBinary, + networkName, + d.NodeConfig.Subnet, + d.NodeConfig.Subnetv6, + staticIP, + d.NodeConfig.StaticIPv6, + params.IPFamily, + ) + if err != nil { msg := "Unable to create dedicated network, this might result in cluster IP change after restart: {{.error}}" args := out.V{"error": err} if staticIP != "" { + // With a user-requested static IP we can’t safely fall back + // to the default bridge network. exit.Message(reason.IfDedicatedNetwork, msg, args) } out.WarningT(msg, args) - } else if gateway != nil && staticIP != "" { - params.Network = networkName - params.IP = staticIP - } else if gateway != nil { + // NOTE: Do NOT set params.Network here – we’ll fall back to Docker’s + // default bridge network. + } else { + // Only attach to the dedicated network when creation succeeded. + // For IPv6-only clusters the gateway may be nil, but the network + // still exists and can be used. params.Network = networkName - ip := gateway.To4() - // calculate the container IP based on guessing the machine index - index := driver.IndexFromMachineName(d.NodeConfig.MachineName) - if int(ip[3])+index > 253 { // reserve last client ip address for multi-control-plane loadbalancer vip address in ha cluster - return fmt.Errorf("too many machines to calculate an IP") + } + + // Now decide static IPs per family based on the gateway (if any). + switch params.IPFamily { + case "ipv6": + if d.NodeConfig.StaticIPv6 != "" { + params.IPv6 = d.NodeConfig.StaticIPv6 + } + + case "dual": + // IPv4 part (only if Docker reported a v4 gateway) + if g4 := gateway.To4(); g4 != nil { + if staticIP != "" { + params.IP = staticIP + } else { + ip := make(net.IP, len(g4)) + copy(ip, g4) + index := driver.IndexFromMachineName(d.NodeConfig.MachineName) + if int(ip[3])+index > 253 { + return fmt.Errorf("too many machines to calculate an IPv4") + } + ip[3] += byte(index) + klog.Infof("calculated static IPv4 %q for the %q container", ip.String(), d.NodeConfig.MachineName) + params.IP = ip.String() + } + } + if d.NodeConfig.StaticIPv6 != "" { + params.IPv6 = d.NodeConfig.StaticIPv6 + } + + default: // ipv4 + if staticIP != "" { + params.IP = staticIP + } else if gateway != nil { + if g4 := gateway.To4(); g4 != nil { + ip := make(net.IP, len(g4)) + copy(ip, g4) + index := driver.IndexFromMachineName(d.NodeConfig.MachineName) + if int(ip[3])+index > 253 { + return fmt.Errorf("too many machines to calculate an IP") + } + ip[3] += byte(index) + klog.Infof("calculated static IP %q for the %q container", ip.String(), d.NodeConfig.MachineName) + params.IP = ip.String() + } } - ip[3] += byte(index) - klog.Infof("calculated static IP %q for the %q container", ip.String(), d.NodeConfig.MachineName) - params.IP = ip.String() } - drv := d.DriverName() + drv := d.DriverName() + // Default listen address: v4 localhost for ipv4, v6 localhost for ipv6-only listAddr := oci.DefaultBindIPV4 + // IPv6-only clusters must publish on IPv6 loopback so the host can reach them + if params.IPFamily == "ipv6" { + listAddr = "::1" + } + if d.NodeConfig.ListenAddress != "" && d.NodeConfig.ListenAddress != listAddr { out.Step(style.Tip, "minikube is not meant for production use. You are opening non-local traffic") out.WarningT("Listening to {{.listenAddr}}. This is not recommended and can cause a security vulnerability. Use at your own risk", out.V{"listenAddr": d.NodeConfig.ListenAddress}) listAddr = d.NodeConfig.ListenAddress } else if oci.IsExternalDaemonHost(drv) { - out.WarningT("Listening to 0.0.0.0 on external docker host {{.host}}. Please be advised", - out.V{"host": oci.DaemonHost(drv)}) - listAddr = "0.0.0.0" + if params.IPFamily == "ipv6" { + out.WarningT("Listening to :: on external docker host {{.host}}. Please be advised", + out.V{"host": oci.DaemonHost(drv)}) + listAddr = "::" + } else { + out.WarningT("Listening to 0.0.0.0 on external docker host {{.host}}. Please be advised", + out.V{"host": oci.DaemonHost(drv)}) + listAddr = "0.0.0.0" + } } // control plane specific options @@ -293,18 +358,38 @@ func (d *Driver) DriverName() string { // GetIP returns an IP or hostname that this host is available at func (d *Driver) GetIP() (string, error) { - ip, _, err := oci.ContainerIPs(d.OCIBinary, d.MachineName) - return ip, err + ip4, ip6, err := oci.ContainerIPs(d.OCIBinary, d.MachineName) + if err != nil { + return "", err + } + switch strings.ToLower(d.NodeConfig.IPFamily) { + case "ipv6": + if ip6 != "" { + return ip6, nil + } + } + // default / dual prefers IPv4 for backward compat + return ip4, nil } // GetExternalIP returns an IP which is accessible from outside func (d *Driver) GetExternalIP() (string, error) { - return oci.DaemonHost(d.DriverName()), nil + host := oci.DaemonHost(d.DriverName()) + // For local daemons and IPv6-only clusters, ports are published on ::1 + if strings.ToLower(d.NodeConfig.IPFamily) == "ipv6" && !oci.IsExternalDaemonHost(d.DriverName()) { + return "::1", nil + } + return host, nil } // GetSSHHostname returns hostname for use with ssh func (d *Driver) GetSSHHostname() (string, error) { - return oci.DaemonHost(d.DriverName()), nil + host := oci.DaemonHost(d.DriverName()) + // For local daemons and IPv6-only clusters, ports are published on ::1 + if strings.ToLower(d.NodeConfig.IPFamily) == "ipv6" && !oci.IsExternalDaemonHost(d.DriverName()) { + return "::1", nil + } + return host, nil } // GetSSHPort returns port for use with ssh @@ -386,7 +471,11 @@ func (d *Driver) Remove() error { return fmt.Errorf("expected no container ID be found for %q after delete. but got %q", d.MachineName, id) } - if err := oci.RemoveNetwork(d.OCIBinary, d.NodeConfig.ClusterName); err != nil { + networkName := d.NodeConfig.ClusterName + if d.NodeConfig.Network != "" { + networkName = d.NodeConfig.Network + } + if err := oci.RemoveNetwork(d.OCIBinary, networkName); err != nil { klog.Warningf("failed to remove network (which might be okay) %s: %v", d.NodeConfig.ClusterName, err) } return nil diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 8486fa698903..f00e9a712c99 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -88,13 +88,41 @@ func RoutableHostIPFromInside(ociBin string, clusterName string, containerName s // digDNS will get the IP record for a dns func digDNS(ociBin, containerName, dns string) (net.IP, error) { rr, err := runCmd(exec.Command(ociBin, "exec", "-t", containerName, "dig", "+short", dns)) - ip := net.ParseIP(strings.TrimSpace(rr.Stdout.String())) if err != nil { - return ip, errors.Wrapf(err, "resolve dns to ip") + // still try to parse whatever output we got + klog.Infof("dig returned error, attempting to parse output anyway: %v", err) } - - klog.Infof("got host ip for mount in container by digging dns: %s", ip.String()) - return ip, nil + out := strings.TrimSpace(rr.Stdout.String()) + if out == "" { + return nil, errors.Wrapf(err, "resolve dns to ip") + } + // Parse line-by-line. On non-Linux (Docker Desktop), prefer IPv4 for better routability. + var firstIP net.IP + for _, line := range strings.Split(out, "\n") { + s := strings.TrimSpace(line) + if s == "" { + continue + } + ip := net.ParseIP(s) + if ip == nil { + continue + } + if runtime.GOOS != "linux" && ip.To4() == nil { + // Prefer IPv4 on Desktop; keep looking for an A record + if firstIP == nil { + firstIP = ip + } + continue + } + klog.Infof("got host ip for mount in container by digging dns: %s", ip.String()) + return ip, nil + } + // Fallback: return first valid IP if only AAAA answers were present + if firstIP != nil { + klog.Infof("got host ip for mount in container by digging dns (first match): %s", firstIP.String()) + return firstIP, nil + } + return nil, errors.New("no A/AAAA answers returned by dig") } // gatewayIP inspects oci container to find a gateway IP string @@ -107,6 +135,14 @@ func gatewayIP(ociBin, containerName string) (string, error) { return gatewayIP, nil } + // Fallback to IPv6 gateway (needed for IPv6-only / dual-stack) + rr6, err6 := runCmd(exec.Command(ociBin, "container", "inspect", "--format", "{{.NetworkSettings.IPv6Gateway}}", containerName)) + if err6 == nil { + if gatewayIP6 := strings.TrimSpace(rr6.Stdout.String()); gatewayIP6 != "" { + return gatewayIP6, nil + } + } + // https://github.com/kubernetes/minikube/issues/11293 // need to check nested network // check .NetworkSettings.Networks["cluster-name"].Gateway and then @@ -126,16 +162,24 @@ func gatewayIP(ociBin, containerName string) (string, error) { } func networkGateway(ociBin, container, network string) (string, error) { - format := fmt.Sprintf(` -{{ if index .NetworkSettings.Networks %q}} - {{(index .NetworkSettings.Networks %q).Gateway}} -{{ end }} -`, network, network) - rr, err := runCmd(exec.Command(ociBin, "container", "inspect", "--format", format, container)) + // First try IPv4 gateway on the specific network + format4 := fmt.Sprintf(`{{ if index .NetworkSettings.Networks %q}}{{(index .NetworkSettings.Networks %q).Gateway}}{{ end }}`, network, network) + rr, err := runCmd(exec.Command(ociBin, "container", "inspect", "--format", format4, container)) if err != nil { return "", errors.Wrapf(err, "inspect gateway") } - return strings.TrimSpace(rr.Stdout.String()), nil + + gw := strings.TrimSpace(rr.Stdout.String()) + if gw != "" { + return gw, nil + } + // Fallback to IPv6 gateway + format6 := fmt.Sprintf(`{{ if index .NetworkSettings.Networks %q}}{{(index .NetworkSettings.Networks %q).IPv6Gateway}}{{ end }}`, network, network) + rr6, err := runCmd(exec.Command(ociBin, "container", "inspect", "--format", format6, container)) + if err != nil { + return "", errors.Wrapf(err, "inspect ipv6 gateway") + } + return strings.TrimSpace(rr6.Stdout.String()), nil } // containerGatewayIP gets the default gateway ip for the container @@ -188,9 +232,8 @@ func ForwardedPort(ociBin string, ociID string, contPort int) (int, error) { o := strings.TrimSpace(rr.Stdout.String()) o = strings.Trim(o, "'") p, err := strconv.Atoi(o) - if err != nil { - return p, errors.Wrapf(err, "convert host-port %q to number", p) + return 0, errors.Wrapf(err, "convert host-port %q to number", o) } return p, nil diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go index cd38dc785c30..9d78a5ad0046 100644 --- a/pkg/drivers/kic/oci/network_create.go +++ b/pkg/drivers/kic/oci/network_create.go @@ -37,6 +37,12 @@ import ( // it is one octet more than the one used by KVM to avoid possible conflict const defaultFirstSubnetAddr = "192.168.49.0" +// defaultFirstSubnetAddrv6 is the first IPv6 subnet used for kic networks. +// Avoid fd00::/64 because Docker's IPv6 pools (e.g. fixed-cidr-v6) often sit +// under fd00::/xx and will trigger "Pool overlaps with other one on this +// address space". Use a different ULA /64 instead. +const defaultFirstSubnetAddrv6 = "fd01::/64" + // name of the default bridge network, used to lookup the MTU (see #9528) const dockerDefaultBridge = "bridge" @@ -63,60 +69,238 @@ func firstSubnetAddr(subnet string) string { return subnet } +func firstSubnetAddrv6(subnet string) string { + if subnet == "" { + return defaultFirstSubnetAddrv6 + } + return subnet +} + // CreateNetwork creates a network returns gateway and error, minikube creates one network per cluster func CreateNetwork(ociBin, networkName, subnet, staticIP string) (net.IP, error) { - defaultBridgeName := defaultBridgeName(ociBin) - if networkName == defaultBridgeName { + return CreateNetworkWithIPFamily(ociBin, networkName, subnet, "", staticIP, "", "ipv4") +} + +func CreateNetworkWithIPFamily(ociBin, networkName, subnet, subnetv6, staticIP, staticIPv6, ipFamily string) (net.IP, error) { + bridgeName := defaultBridgeName(ociBin) + if networkName == bridgeName { klog.Infof("skipping creating network since default network %s was specified", networkName) return nil, nil } - // check if the network already exists - info, err := containerNetworkInspect(ociBin, networkName) - if err == nil { + // If the network already exists, reuse it. + if info, err := containerNetworkInspect(ociBin, networkName); err == nil { klog.Infof("Found existing network %+v", info) return info.gateway, nil } - // will try to get MTU from the docker network to avoid issue with systems with exotic MTU settings. - // related issue #9528 - info, err = containerNetworkInspect(ociBin, defaultBridgeName) - if err != nil { - klog.Warningf("failed to get mtu information from the %s's default network %q: %v", ociBin, defaultBridgeName, err) + // Learn MTU from the default bridge (best effort). + bridgeInfo, berr := containerNetworkInspect(ociBin, bridgeName) + if berr != nil { + klog.Warningf("failed to get mtu information from the %s's default network %q: %v", ociBin, bridgeName, berr) } - tries := 20 + fam := strings.ToLower(ipFamily) + if fam == "" { + fam = "ipv4" + } + + // ---------- IPv6 / dual-stack flow ---------- + if fam == "ipv6" || fam == "dual" { + // We no longer try to "guess" a /64 from the static IPv6. + // If the user wants a static IPv6, they must also provide --subnet-v6. + if staticIPv6 != "" && subnetv6 == "" { + return nil, fmt.Errorf("invalid IPv6 configuration: --static-ipv6 requires --subnet-v6 to be set") + } + + // If caller provided a v6 subnet, normalize it via firstSubnetAddrv6. + if subnetv6 != "" { + subnetv6 = firstSubnetAddrv6(subnetv6) + } - // we don't want to increment the subnet IP on network creation failure if the user specifies a static IP, so set tries to 1 + baseArgs := []string{"network", "create", "--driver=bridge", "--ipv6"} + + // ----- dual-stack: IPv4 + IPv6 ----- + if fam == "dual" { + tries := 20 + if staticIP != "" { + // Don't walk the IPv4 space when user pinned a specific address. + tries = 1 + subnet = staticIP + } + + var lastErr error + for attempts, subnetAddr := 0, firstSubnetAddr(subnet); attempts < 5; attempts++ { + var p *network.Parameters + p, lastErr = network.FreeSubnet(subnetAddr, 9, tries) + if lastErr != nil { + klog.Errorf("failed to find free IPv4 subnet for %s network %s after %d attempts: %v", ociBin, networkName, 20, lastErr) + return nil, fmt.Errorf("un-retryable: %w", lastErr) + } + + args := append([]string{}, baseArgs...) + // IPv4 part + args = append(args, "--subnet", p.CIDR, "--gateway", p.Gateway) + // IPv6 part (optional, only if caller specified --subnet-v6) + if subnetv6 != "" { + args = append(args, "--subnet", subnetv6) + } + if ociBin == Docker && bridgeInfo.mtu > 0 { + args = append(args, "-o", fmt.Sprintf("com.docker.network.driver.mtu=%d", bridgeInfo.mtu)) + } + args = append(args, + fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), + fmt.Sprintf("--label=%s=%s", ProfileLabelKey, networkName), + networkName, + ) + + rr, err := runCmd(exec.Command(ociBin, args...)) + if err == nil { + ni, _ := containerNetworkInspect(ociBin, networkName) + return ni.gateway, nil + } + + out := rr.Output() + // Respect the same retry conditions as the IPv4-only flow. + if strings.Contains(out, "Pool overlaps") || + (strings.Contains(out, "failed to allocate gateway") && strings.Contains(out, "Address already in use")) || + strings.Contains(out, "is being used by a network interface") || + strings.Contains(out, "is already used on the host or by another config") { + klog.Warningf("failed to create %s network %s %s (dual): %v; retrying with next IPv4 subnet", ociBin, networkName, p.CIDR, err) + subnetAddr = p.IP + continue + } + + // Non-retryable + klog.Errorf("error creating dual-stack network %s: %v", networkName, err) + return nil, fmt.Errorf("un-retryable: %w", err) + } + + return nil, fmt.Errorf("failed to create %s network %s (dual): %w", ociBin, networkName, lastErr) + } + + args := append([]string{}, baseArgs...) + if subnetv6 != "" { + args = append(args, "--subnet", subnetv6) + } + // ipv6-only / “IPv6 + auto IPv4” branch: + // - ipFamily == "ipv6" + // - OR ipFamily == "dual" && staticIP == "" (Docker picks IPv4 range) + args = append(args, "--subnet", subnetv6) + if ociBin == Docker && bridgeInfo.mtu > 0 { + args = append(args, "-o", fmt.Sprintf("com.docker.network.driver.mtu=%d", bridgeInfo.mtu)) + } + args = append(args, + fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), + fmt.Sprintf("--label=%s=%s", ProfileLabelKey, networkName), + networkName, + ) + + if _, err := runCmd(exec.Command(ociBin, args...)); err != nil { + klog.Warningf("failed to create %s network %q (ipv6/dual): %v", ociBin, networkName, err) + return nil, fmt.Errorf("create %s network %q: %w", ociBin, networkName, err) + } + + ni, _ := containerNetworkInspect(ociBin, networkName) + return ni.gateway, nil + } + + // ---------- IPv4-only flow (unchanged behaviour) ---------- + tries := 20 if staticIP != "" { + // we don't want to increment the subnet IP on network creation failure if the user specifies a static IP, so set tries to 1 tries = 1 subnet = staticIP } + var lastErr error // retry up to 5 times to create container network for attempts, subnetAddr := 0, firstSubnetAddr(subnet); attempts < 5; attempts++ { // Rather than iterate through all of the valid subnets, give up at 20 to avoid a lengthy user delay for something that is unlikely to work. // will be like 192.168.49.0/24,..., 192.168.220.0/24 (in increment steps of 9) - var subnet *network.Parameters - subnet, err = network.FreeSubnet(subnetAddr, 9, tries) - if err != nil { - klog.Errorf("failed to find free subnet for %s network %s after %d attempts: %v", ociBin, networkName, 20, err) - return nil, fmt.Errorf("un-retryable: %w", err) + var p *network.Parameters + p, lastErr = network.FreeSubnet(subnetAddr, 9, tries) + if lastErr != nil { + klog.Errorf("failed to find free subnet for %s network %s after %d attempts: %v", ociBin, networkName, 20, lastErr) + return nil, fmt.Errorf("un-retryable: %w", lastErr) } - info.gateway, err = tryCreateDockerNetwork(ociBin, subnet, info.mtu, networkName) + gw, err := tryCreateDockerNetwork(ociBin, p, bridgeInfo.mtu, networkName) if err == nil { - klog.Infof("%s network %s %s created", ociBin, networkName, subnet.CIDR) - return info.gateway, nil + klog.Infof("%s network %s %s created", ociBin, networkName, p.CIDR) + return gw, nil } - // don't retry if error is not address is taken if !errors.Is(err, ErrNetworkSubnetTaken) && !errors.Is(err, ErrNetworkGatewayTaken) { - klog.Errorf("error while trying to create %s network %s %s: %v", ociBin, networkName, subnet.CIDR, err) + klog.Errorf("error while trying to create %s network %s %s: %v", ociBin, networkName, p.CIDR, err) return nil, fmt.Errorf("un-retryable: %w", err) } - klog.Warningf("failed to create %s network %s %s, will retry: %v", ociBin, networkName, subnet.CIDR, err) - subnetAddr = subnet.IP + klog.Warningf("failed to create %s network %s %s, will retry: %v", ociBin, networkName, p.CIDR, err) + subnetAddr = p.IP + } + return nil, fmt.Errorf("failed to create %s network %s: %w", ociBin, networkName, lastErr) +} + +// cidr64ForIP returns a /64 CIDR string covering the provided IPv6 address. +func cidr64ForIP(ipStr string) (string, error) { + ip := net.ParseIP(ipStr) + if ip == nil || ip.To16() == nil || ip.To4() != nil { + return "", fmt.Errorf("not a valid IPv6 address: %q", ipStr) } - return info.gateway, fmt.Errorf("failed to create %s network %s: %w", ociBin, networkName, err) + mask := net.CIDRMask(64, 128) + ipMasked := ip.Mask(mask) + return (&net.IPNet{IP: ipMasked, Mask: mask}).String(), nil +} + +// createV6OrDualNetwork creates a user-defined bridge network with IPv6 enabled, +// and adds both subnets when ipFamily == "dual". Returns the gateway reported by inspect (may be nil for v6-only). +func createV6OrDualNetwork(ociBin, name, subnetV4, subnetV6, ipFamily string) (net.IP, error) { + klog.Infof("creating %s network %q (family=%s, v4=%q, v6=%q)", ociBin, name, ipFamily, subnetV4, subnetV6) + + // If exists, reuse as-is + if info, err := containerNetworkInspect(ociBin, name); err == nil { + klog.Infof("found existing network %q", name) + return info.gateway, nil + } + + // Defaults + if subnetV6 == "" { + subnetV6 = firstSubnetAddrv6("") + } + + // Build args + args := []string{"network", "create", "--driver=bridge"} + if ipFamily == "ipv6" || ipFamily == "dual" { + args = append(args, "--ipv6") + } + if ipFamily == "dual" && subnetV4 != "" { + args = append(args, "--subnet", subnetV4) + } + if subnetV6 != "" { + args = append(args, "--subnet", subnetV6) + } + // Optional bridge knobs (safe to omit) + // if ociBin == Docker { + // args = append(args, "-o", "com.docker.network.bridge.enable_ip_masquerade=true") + // args = append(args, "-o", "com.docker.network.bridge.enable_icc=true") + // } + args = append(args, + fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), + fmt.Sprintf("--label=%s=%s", ProfileLabelKey, name), + name, + ) + + if _, err := runCmd(exec.Command(ociBin, args...)); err != nil { + klog.Warningf("failed to create %s network %q: %v", ociBin, name, err) + return nil, fmt.Errorf("create %s network %q: %w", ociBin, name, err) + } + + // Rely on inspect (gateway may be empty for IPv6) + info, err := containerNetworkInspect(ociBin, name) + if err != nil { + // non-fatal: just return nil gateway + klog.Warningf("post-create inspect failed for %s: %v", name, err) + return nil, nil + } + return info.gateway, nil } func tryCreateDockerNetwork(ociBin string, subnet *network.Parameters, mtu int, name string) (net.IP, error) { @@ -185,21 +369,34 @@ func containerNetworkInspect(ociBin string, name string) (netInfo, error) { // networkInspect is only used to unmarshal the docker network inspect output and translate it to netInfo type networkInspect struct { - Name string - Driver string - Subnet string - Gateway string - MTU int - ContainerIPs []string + Name string `json:"Name"` + Driver string `json:"Driver"` + // Legacy single fields (older template) + Subnet string `json:"Subnet"` + Gateway string `json:"Gateway"` + // Multi-family (new template) + Subnets []string `json:"Subnets"` + Gateways []string `json:"Gateways"` + MTU int `json:"MTU"` + ContainerIPs []string `json:"ContainerIPs"` } var dockerInspectGetter = func(name string) (*RunResult, error) { - // hack -- 'support ancient versions of docker again (template parsing issue) #10362' and resolve 'Template parsing error: template: :1: unexpected "=" in operand' / 'exit status 64' - // note: docker v18.09.7 and older use go v1.10.8 and older, whereas support for '=' operator in go templates came in go v1.11 - cmd := exec.Command(Docker, "network", "inspect", name, "--format", `{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}`) + // keep the old workaround: avoid eq/== in templates; emit trailing commas then strip ",]" + cmd := exec.Command( + Docker, "network", "inspect", name, "--format", + `{"Name":"{{.Name}}","Driver":"{{.Driver}}",`+ + `"Subnet":"{{range .IPAM.Config}}{{.Subnet}}{{end}}",`+ + `"Gateway":"{{range .IPAM.Config}}{{.Gateway}}{{end}}",`+ + `"Subnets":[{{range .IPAM.Config}}{{if .Subnet}}"{{.Subnet}}",{{end}}{{end}}],`+ + `"Gateways":[{{range .IPAM.Config}}{{if .Gateway}}"{{.Gateway}}",{{end}}{{end}}],`+ + `"MTU":{{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}},`+ + `"ContainerIPs":[{{range $k,$v := .Containers}}"{{$v.IPv4Address}}",{{end}}]}`, + ) rr, err := runCmd(cmd) - // remove extra ',' after the last element in the ContainerIPs slice - rr.Stdout = *bytes.NewBuffer(bytes.ReplaceAll(rr.Stdout.Bytes(), []byte(",]"), []byte("]"))) + // remove any trailing commas from arrays we just built + cleaned := bytes.ReplaceAll(rr.Stdout.Bytes(), []byte(",]"), []byte("]")) + rr.Stdout = *bytes.NewBuffer(cleaned) return rr, err } @@ -217,19 +414,51 @@ func dockerNetworkInspect(name string) (netInfo, error) { return info, err } - // results looks like {"Name": "bridge","Driver": "bridge","Subnet": "172.17.0.0/16","Gateway": "172.17.0.1","MTU": 1500, "ContainerIPs": ["172.17.0.3/16", "172.17.0.2/16"]} + // results look like: + // {"Name":"bridge","Driver":"bridge", + // "Subnet":"172.17.0.0/16","Gateway":"172.17.0.1", + // "Subnets":["172.17.0.0/16","fd00::/64"],"Gateways":["172.17.0.1","fd00::1"], + // "MTU":1500,"ContainerIPs":[...]} if err := json.Unmarshal(rr.Stdout.Bytes(), &vals); err != nil { return info, fmt.Errorf("error parsing network inspect output: %q", rr.Stdout.String()) } - info.gateway = net.ParseIP(vals.Gateway) - info.mtu = vals.MTU - - _, info.subnet, err = net.ParseCIDR(vals.Subnet) - if err != nil { - return info, errors.Wrapf(err, "parse subnet for %s", name) + // Choose a subnet/gateway: + // - Prefer an IPv4 entry (back-compat with existing IPv4 flows), + // - else fall back to first entry, + // - else use legacy single fields. + pickSubnet := "" + pickGateway := "" + if len(vals.Subnets) > 0 { + for i, s := range vals.Subnets { + if ip, _, e := net.ParseCIDR(s); e == nil && ip.To4() != nil { + pickSubnet = s + if i < len(vals.Gateways) { + pickGateway = vals.Gateways[i] + } + break + } + } + if pickSubnet == "" { + pickSubnet = vals.Subnets[0] + if len(vals.Gateways) > 0 { + pickGateway = vals.Gateways[0] + } + } } - + if pickSubnet == "" { + pickSubnet = vals.Subnet + pickGateway = vals.Gateway + } + if pickSubnet != "" { + if _, info.subnet, err = net.ParseCIDR(pickSubnet); err != nil { + return info, errors.Wrapf(err, "parse subnet for %s", name) + } + } + if pickGateway != "" { + info.gateway = net.ParseIP(pickGateway) + } + info.mtu = vals.MTU return info, nil } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 6370cb30ad48..a8f4639186ee 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -185,12 +185,40 @@ func CreateContainerNode(p CreateParams) error { //nolint to suppress cyclomatic // label th enode with the node ID "--label", p.NodeLabel, } - // to provide a static IP - if p.Network != "" && p.IP != "" { + + // attach to the user-defined bridge network (once), and set static IPs if provided + if p.Network != "" { runArgs = append(runArgs, "--network", p.Network) - runArgs = append(runArgs, "--ip", p.IP) + if p.IP != "" { + runArgs = append(runArgs, "--ip", p.IP) // IPv4 + } + if p.IPv6 != "" { + runArgs = append(runArgs, "--ip6", p.IPv6) // IPv6 + } + } + + // For IPv6/dual clusters, enable forwarding inside the node container + // (safe sysctl; avoid disable_ipv6 which may be blocked by Docker's safe list) + + // Ensure service rules apply to bridged traffic inside the node container. + // Do both families; harmless if already set. + runArgs = append(runArgs, + "--sysctl", "net.ipv4.ip_forward=1", + "--sysctl", "net.bridge.bridge-nf-call-iptables=1", + // Allow kube-proxy/IPVS or iptables to program and accept IPv4 Service VIPs. + "--sysctl", "net.ipv4.ip_nonlocal_bind=1", + ) + // IPv6/dual clusters need IPv6 forwarding and IPv6 bridge netfilter, too. + if p.IPFamily == "ipv6" || p.IPFamily == "dual" { + runArgs = append(runArgs, + "--sysctl", "net.ipv6.conf.all.forwarding=1", + "--sysctl", "net.bridge.bridge-nf-call-ip6tables=1", + // Allow kube-proxy/IPVS or iptables to program and accept Service VIPs. + "--sysctl", "net.ipv4.ip_nonlocal_bind=1", + // Same for IPv6 VIPs. + "--sysctl", "net.ipv6.ip_nonlocal_bind=1", + ) } - switch p.GPUs { case "all", "nvidia": runArgs = append(runArgs, "--gpus", "all", "--env", "NVIDIA_DRIVER_CAPABILITIES=all") @@ -509,7 +537,12 @@ func generatePortMappings(portMappings ...PortMapping) []string { for _, pm := range portMappings { // let docker pick a host port by leaving it as :: // example --publish=127.0.0.17::8443 will get a random host port for 8443 - publish := fmt.Sprintf("--publish=%s::%d", pm.ListenAddress, pm.ContainerPort) + host := pm.ListenAddress + // If the listen address is an IPv6 literal, bracket it for Docker syntax. + if strings.Contains(host, ":") && !strings.HasPrefix(host, "[") { + host = "[" + host + "]" + } + publish := fmt.Sprintf("--publish=%s::%d", host, pm.ContainerPort) result = append(result, publish) } return result diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go index 894679c4e667..e91a3b98e369 100644 --- a/pkg/drivers/kic/oci/types.go +++ b/pkg/drivers/kic/oci/types.go @@ -61,6 +61,8 @@ type CreateParams struct { OCIBinary string // docker or podman Network string // network name that the container will attach to IP string // static IP to assign the container in the cluster network + IPv6 string // optional static IPv6 to assign to the node container (--ip6) + IPFamily string // "ipv4", "ipv6", or "dual" (from cc.KubernetesConfig.IPFamily) GPUs string // add GPU devices to the container } diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index e88aa6c23fa1..03e68fff97c0 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -66,8 +66,11 @@ type Config struct { ContainerRuntime string // container runtime kic is running Network string // network to run with kic Subnet string // subnet to be used on kic cluster - StaticIP string // static IP for the kic cluster - ExtraArgs []string // a list of any extra option to pass to oci binary during creation time, for example --expose 8080... - ListenAddress string // IP Address to listen to - GPUs string // add GPU devices to the container + Subnetv6 string + StaticIP string // static IP for the kic cluster + StaticIPv6 string + IPFamily string // "ipv4", "ipv6", or "dual" + ExtraArgs []string // a list of any extra option to pass to oci binary during creation time, for example --expose 8080... + ListenAddress string // IP Address to listen to + GPUs string // add GPU devices to the container } diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go index 3e0d021f7245..99936c47ab4d 100644 --- a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go @@ -36,8 +36,10 @@ bootstrapTokens: nodeRegistration: criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}} name: "{{.NodeName}}" +{{- if .NodeIP }} kubeletExtraArgs: node-ip: {{.NodeIP}} +{{- end }} taints: [] --- apiVersion: kubeadm.k8s.io/v1beta1 @@ -93,7 +95,7 @@ staticPodPath: {{.StaticPodPath}} apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration clusterCIDR: "{{.PodSubnet }}" -metricsBindAddress: 0.0.0.0:10249 +metricsBindAddress: {{.KubeProxyMetricsBindAddress}} conntrack: maxPerCore: 0 # Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established" diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta2.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta2.go index fc098623cfed..2f35264676ee 100644 --- a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta2.go +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta2.go @@ -24,7 +24,7 @@ var V1Beta2 = template.Must(template.New("configTmpl-v1beta2").Funcs(template.Fu }).Parse(`apiVersion: kubeadm.k8s.io/v1beta2 kind: InitConfiguration localAPIEndpoint: - advertiseAddress: {{.AdvertiseAddress}} + advertiseAddress: "{{.AdvertiseAddress}}" bindPort: {{.APIServerPort}} bootstrapTokens: - groups: @@ -36,16 +36,28 @@ bootstrapTokens: nodeRegistration: criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}} name: "{{.NodeName}}" +{{- if .NodeIP }} kubeletExtraArgs: - node-ip: {{.NodeIP}} + node-ip: "{{.NodeIP}}" +{{- end }} taints: [] --- apiVersion: kubeadm.k8s.io/v1beta2 kind: ClusterConfiguration {{ if .ImageRepository}}imageRepository: {{.ImageRepository}} {{end}}{{range .ComponentOptions}}{{.Component}}: +{{- if eq .Component "apiServer" }} + {{- if $.APIServerCertSANs }} + certSANs: + {{- range $.APIServerCertSANs }} + - "{{ . }}" + {{- end }} + {{- end }} +{{- end }} {{- range $k, $v := .Pairs }} + {{- if not (and (eq .Component "apiServer") (eq $k "certSANs")) }} {{$k}}: {{$v}} + {{- end }} {{- end}} extraArgs: {{- range $i, $val := printMapInOrder .ExtraArgs ": " }} @@ -72,7 +84,7 @@ kubernetesVersion: {{.KubernetesVersion}} networking: dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}} podSubnet: "{{.PodSubnet }}" - serviceSubnet: {{.ServiceCIDR}} + serviceSubnet: "{{.ServiceCIDR}}" --- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration @@ -96,7 +108,7 @@ staticPodPath: {{.StaticPodPath}} apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration clusterCIDR: "{{.PodSubnet }}" -metricsBindAddress: 0.0.0.0:10249 +metricsBindAddress: "{{.KubeProxyMetricsBindAddress}}" conntrack: maxPerCore: 0 # Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established" diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go index 26de90f42e6f..15261b6370b5 100644 --- a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go @@ -24,7 +24,7 @@ var V1Beta3 = template.Must(template.New("configTmpl-v1beta3").Funcs(template.Fu }).Parse(`apiVersion: kubeadm.k8s.io/v1beta3 kind: InitConfiguration localAPIEndpoint: - advertiseAddress: {{.AdvertiseAddress}} + advertiseAddress: "{{.AdvertiseAddress}}" bindPort: {{.APIServerPort}} bootstrapTokens: - groups: @@ -36,16 +36,28 @@ bootstrapTokens: nodeRegistration: criSocket: {{if .CRISocket}}{{if .PrependCriSocketUnix}}unix://{{end}}{{.CRISocket}}{{else}}{{if .PrependCriSocketUnix}}unix://{{end}}/var/run/dockershim.sock{{end}} name: "{{.NodeName}}" +{{- if .NodeIP }} kubeletExtraArgs: - node-ip: {{.NodeIP}} + node-ip: "{{.NodeIP}}" +{{- end }} taints: [] --- apiVersion: kubeadm.k8s.io/v1beta3 kind: ClusterConfiguration {{ if .ImageRepository}}imageRepository: {{.ImageRepository}} {{end}}{{range .ComponentOptions}}{{.Component}}: +{{- if eq .Component "apiServer" }} + {{- if $.APIServerCertSANs }} + certSANs: + {{- range $.APIServerCertSANs }} + - "{{ . }}" + {{- end }} + {{- end }} +{{- end }} {{- range $k, $v := .Pairs }} + {{- if not (and (eq .Component "apiServer") (eq $k "certSANs")) }} {{$k}}: {{$v}} + {{- end }} {{- end}} extraArgs: {{- range $i, $val := printMapInOrder .ExtraArgs ": " }} @@ -70,7 +82,7 @@ kubernetesVersion: {{.KubernetesVersion}} networking: dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}} podSubnet: "{{.PodSubnet }}" - serviceSubnet: {{.ServiceCIDR}} + serviceSubnet: "{{.ServiceCIDR}}" --- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration @@ -95,7 +107,7 @@ resolvConf: /etc/kubelet-resolv.conf{{end}} apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration clusterCIDR: "{{.PodSubnet }}" -metricsBindAddress: 0.0.0.0:10249 +metricsBindAddress: "{{.KubeProxyMetricsBindAddress}}" conntrack: maxPerCore: 0 # Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established" diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta4.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta4.go index ee28b7d69da1..78ef47dd372f 100644 --- a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta4.go +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta4.go @@ -24,7 +24,7 @@ var V1Beta4 = template.Must(template.New("configTmpl-v1beta4").Funcs(template.Fu }).Parse(`apiVersion: kubeadm.k8s.io/v1beta4 kind: InitConfiguration localAPIEndpoint: - advertiseAddress: {{.AdvertiseAddress}} + advertiseAddress: "{{.AdvertiseAddress}}" bindPort: {{.APIServerPort}} bootstrapTokens: - groups: @@ -36,20 +36,34 @@ bootstrapTokens: nodeRegistration: criSocket: {{if .CRISocket}}{{if .PrependCriSocketUnix}}unix://{{end}}{{.CRISocket}}{{else}}{{if .PrependCriSocketUnix}}unix://{{end}}/var/run/dockershim.sock{{end}} name: "{{.NodeName}}" +{{- if .NodeIP }} kubeletExtraArgs: - name: "node-ip" value: "{{.NodeIP}}" +{{- end }} taints: [] --- apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration {{ if .ImageRepository}}imageRepository: {{.ImageRepository}} -{{end}}{{range .ComponentOptions}}{{.Component}}: -{{- range $k, $v := .Pairs }} +{{end}}{{range .ComponentOptions }} + {{- $co := . }} +{{$co.Component}}: +{{- if eq $co.Component "apiServer" }} + {{- if $.APIServerCertSANs }} + certSANs: + {{- range $.APIServerCertSANs }} + - "{{ . }}" + {{- end }} + {{- end }} +{{- end }} +{{- range $k, $v := $co.Pairs }} + {{- if not (and (eq $co.Component "apiServer") (eq $k "certSANs")) }} {{$k}}: {{$v}} + {{- end }} {{- end}} extraArgs: -{{- range $key, $val := .ExtraArgs }} +{{- range $key, $val := $co.ExtraArgs }} - name: "{{$key}}" value: "{{$val}}" {{- end}} @@ -59,7 +73,7 @@ kind: ClusterConfiguration {{end -}}{{end -}} certificatesDir: {{.CertDir}} clusterName: mk -controlPlaneEndpoint: {{.ControlPlaneAddress}}:{{.APIServerPort}} +controlPlaneEndpoint: "{{.ControlPlaneEndpoint}}" etcd: local: dataDir: {{.EtcdDataDir}} @@ -73,7 +87,7 @@ kubernetesVersion: {{.KubernetesVersion}} networking: dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}} podSubnet: "{{.PodSubnet }}" - serviceSubnet: {{.ServiceCIDR}} + serviceSubnet: "{{.ServiceCIDR}}" --- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration @@ -98,7 +112,7 @@ resolvConf: /etc/kubelet-resolv.conf{{end}} apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration clusterCIDR: "{{.PodSubnet }}" -metricsBindAddress: 0.0.0.0:10249 +metricsBindAddress: "{{.KubeProxyMetricsBindAddress}}" conntrack: maxPerCore: 0 # Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established" diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go index 5a06e5a9c566..84b7a9f6a08b 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go +++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go @@ -22,6 +22,7 @@ import ( "fmt" "path" "slices" + "strings" "github.com/blang/semver/v4" "github.com/pkg/errors" @@ -77,13 +78,25 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana return nil, errors.Wrap(err, "cni") } - podCIDR := cnm.CIDR() - overrideCIDR := k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm) - if overrideCIDR != "" { - podCIDR = overrideCIDR + // Build podSubnet(s) based on IP family + family := strings.ToLower(k8s.IPFamily) + v4Pod := cnm.CIDR() + if o := k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm); o != "" { + v4Pod = o + } + var podSubnets []string + if family != "ipv6" && v4Pod != "" { + podSubnets = append(podSubnets, v4Pod) + } + if family != "ipv4" && k8s.PodCIDRv6 != "" { + podSubnets = append(podSubnets, k8s.PodCIDRv6) + } + podCIDR := strings.Join(podSubnets, ",") + if podCIDR != "" { + klog.Infof("Using pod subnet(s): %s", podCIDR) + } else { + klog.Infof("No pod subnet set via kubeadm (CNI will configure)") } - klog.Infof("Using pod CIDR: %s", podCIDR) - // ref: https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/#kubelet-config-k8s-io-v1beta1-KubeletConfiguration kubeletConfigOpts := kubeletConfigOpts(k8s.ExtraOptions) // container-runtime-endpoint kubelet flag was deprecated but corresponding containerRuntimeEndpoint kubelet config field is "required" but supported only from k8s v1.27 @@ -108,36 +121,107 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana kubeletConfigOpts["runtimeRequestTimeout"] = "15m" } + // Build serviceSubnet(s) per IP family + // v4 default comes from constants; v6 must be provided via ServiceCIDRv6 + v4Svc := constants.DefaultServiceCIDR + if k8s.ServiceCIDR != "" { + v4Svc = k8s.ServiceCIDR + } + var svcSubnets []string + if family != "ipv6" && v4Svc != "" { + svcSubnets = append(svcSubnets, v4Svc) + } + if family != "ipv4" && k8s.ServiceCIDRv6 != "" { + svcSubnets = append(svcSubnets, k8s.ServiceCIDRv6) + } + serviceCIDR := strings.Join(svcSubnets, ",") + + // Choose advertise address & nodeIP according to family + advertiseAddress := n.IP + nodeIP := n.IP + if family == "ipv6" && n.IPv6 != "" { + advertiseAddress = n.IPv6 + nodeIP = n.IPv6 + } else if family == "dual" { + // let kubelet auto-detect both; don’t force a single family + nodeIP = "" + } + + cpEndpoint := fmt.Sprintf("%s:%d", constants.ControlPlaneAlias, nodePort) + if family == "ipv6" && advertiseAddress != "" { + cpEndpoint = fmt.Sprintf("[%s]:%d", advertiseAddress, nodePort) + } + + if family == "ipv6" || family == "dual" { + ensured := false + for i := range componentOpts { + // match "apiServer" regardless of accidental casing + if strings.EqualFold(componentOpts[i].Component, "apiServer") { + if componentOpts[i].ExtraArgs == nil { + componentOpts[i].ExtraArgs = map[string]string{} + } + if _, ok := componentOpts[i].ExtraArgs["bind-address"]; !ok { + componentOpts[i].ExtraArgs["bind-address"] = "::" + } + // normalize the component name so the template emits 'apiServer' + componentOpts[i].Component = "apiServer" + ensured = true + break + } + } + if !ensured { + componentOpts = append(componentOpts, componentOptions{ + Component: "apiServer", + ExtraArgs: map[string]string{ + "bind-address": "::", + }, + }) + } + } + + apiServerCertSANs := []string{constants.ControlPlaneAlias} + switch strings.ToLower(k8s.IPFamily) { + case "ipv6": + apiServerCertSANs = append(apiServerCertSANs, "::1") + case "dual": + apiServerCertSANs = append(apiServerCertSANs, "127.0.0.1", "::1") + default: // ipv4 + apiServerCertSANs = append(apiServerCertSANs, "127.0.0.1") + } opts := struct { - CertDir string - ServiceCIDR string - PodSubnet string - AdvertiseAddress string - APIServerPort int - KubernetesVersion string - EtcdDataDir string - EtcdExtraArgs map[string]string - ClusterName string - NodeName string - DNSDomain string - CRISocket string - ImageRepository string - ComponentOptions []componentOptions - FeatureArgs map[string]bool - NodeIP string - CgroupDriver string - ClientCAFile string - StaticPodPath string - ControlPlaneAddress string - KubeProxyOptions map[string]string - ResolvConfSearchRegression bool - KubeletConfigOpts map[string]string - PrependCriSocketUnix bool + CertDir string + ServiceCIDR string + PodSubnet string + AdvertiseAddress string + APIServerCertSANs []string + APIServerPort int + KubernetesVersion string + EtcdDataDir string + EtcdExtraArgs map[string]string + ClusterName string + NodeName string + DNSDomain string + CRISocket string + ImageRepository string + ComponentOptions []componentOptions + FeatureArgs map[string]bool + NodeIP string + CgroupDriver string + ClientCAFile string + StaticPodPath string + ControlPlaneAddress string + KubeProxyOptions map[string]string + ResolvConfSearchRegression bool + KubeletConfigOpts map[string]string + PrependCriSocketUnix bool + ControlPlaneEndpoint string + KubeProxyMetricsBindAddress string }{ CertDir: vmpath.GuestKubernetesCertsDir, - ServiceCIDR: constants.DefaultServiceCIDR, + ServiceCIDR: serviceCIDR, PodSubnet: podCIDR, - AdvertiseAddress: n.IP, + AdvertiseAddress: advertiseAddress, + APIServerCertSANs: apiServerCertSANs, APIServerPort: nodePort, KubernetesVersion: k8s.KubernetesVersion, EtcdDataDir: EtcdDataDir(), @@ -150,7 +234,7 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana ComponentOptions: componentOpts, FeatureArgs: kubeadmFeatureArgs, DNSDomain: k8s.DNSDomain, - NodeIP: n.IP, + NodeIP: nodeIP, CgroupDriver: cgroupDriver, ClientCAFile: path.Join(vmpath.GuestKubernetesCertsDir, "ca.crt"), StaticPodPath: vmpath.GuestManifestsDir, @@ -158,10 +242,15 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana KubeProxyOptions: createKubeProxyOptions(k8s.ExtraOptions), ResolvConfSearchRegression: HasResolvConfSearchRegression(k8s.KubernetesVersion), KubeletConfigOpts: kubeletConfigOpts, - } - - if k8s.ServiceCIDR != "" { - opts.ServiceCIDR = k8s.ServiceCIDR + ControlPlaneEndpoint: cpEndpoint, + KubeProxyMetricsBindAddress: func() string { + switch strings.ToLower(k8s.IPFamily) { + case "ipv6": + return "[::]:10249" + default: // ipv4 or dual + return "0.0.0.0:10249" + } + }(), } configTmpl := ktmpl.V1Beta1 diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go index e0811107eb42..84aaf6f62941 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubelet.go +++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "path" + "strings" "github.com/blang/semver/v4" "github.com/pkg/errors" @@ -82,8 +83,23 @@ func extraKubeletOpts(mc config.ClusterConfig, nc config.Node, r cruntime.Manage } } + // Pick node-ip based on requested IP family if _, ok := extraOpts["node-ip"]; !ok { - extraOpts["node-ip"] = nc.IP + family := strings.ToLower(k8s.IPFamily) + switch family { + case "ipv6": + if nc.IPv6 != "" { + extraOpts["node-ip"] = nc.IPv6 + } else { + // fallback if IPv6 wasn’t wired yet + extraOpts["node-ip"] = nc.IP + } + case "dual": + // Don’t set node-ip at all; kubelet will advertise both families. + // (If a user explicitly set node-ip, we honor it above.) + default: // "ipv4" or empty + extraOpts["node-ip"] = nc.IP + } } if _, ok := extraOpts["hostname-override"]; !ok { diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index 2dc7b987072e..23c53209d257 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -257,14 +257,31 @@ func generateProfileCerts(cfg config.ClusterConfig, n config.Node, shared shared klog.Info("generating profile certs ...") k8s := cfg.KubernetesConfig - serviceIP, err := util.ServiceClusterIP(k8s.ServiceCIDR) if err != nil { return nil, errors.Wrap(err, "get service cluster ip") } + // Collect both service VIPs if present + var serviceIPv6 net.IP + if k8s.ServiceCIDRv6 != "" { + if sip6, err := util.ServiceClusterIP(k8s.ServiceCIDRv6); err == nil { + serviceIPv6 = sip6 + } else { + klog.Warningf("failed to compute service IPv6 from %q: %v", k8s.ServiceCIDRv6, err) + } + } + apiServerIPs := append([]net.IP{}, k8s.APIServerIPs...) - apiServerIPs = append(apiServerIPs, serviceIP, net.ParseIP(oci.DefaultBindIPV4), net.ParseIP("10.0.0.1")) + apiServerIPs = append(apiServerIPs, serviceIP) + if serviceIPv6 != nil { + apiServerIPs = append(apiServerIPs, serviceIPv6) + } + // Always include loopbacks for both families; the docker driver publishes ports on ::1. + apiServerIPs = append(apiServerIPs, net.ParseIP("127.0.0.1"), net.ParseIP("::1")) + // Common local addresses used by the node runtime/bridge + apiServerIPs = append(apiServerIPs, net.ParseIP(oci.DefaultBindIPV4), net.ParseIP("10.0.0.1")) + // append ip addresses of all control-plane nodes for _, n := range config.ControlPlanes(cfg) { apiServerIPs = append(apiServerIPs, net.ParseIP(n.IP)) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index b4b7f4699dc6..147cf387787f 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -39,6 +39,7 @@ import ( "github.com/pkg/errors" core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -296,6 +297,28 @@ func (k *Bootstrapper) init(cfg config.ClusterConfig, options *run.CommandOption klog.Warningf("apiserver tunnel failed: %v", err) } + // Ensure system Services (e.g., kube-dns) have correct IP families immediately after init. + cp, err := config.ControlPlane(cfg) + if err != nil { + klog.Warningf("get control-plane node failed: %v", err) + } else { + hostname, _, port, derr := driver.ControlPlaneEndpoint(&cfg, &cp, cfg.Driver) + if derr != nil { + klog.Warningf("resolve control-plane endpoint failed: %v", derr) + } else { + client, cerr := k.client(hostname, port) + if cerr != nil { + klog.Warningf("build k8s client failed: %v", cerr) + } else { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + if err := k.ensureSystemServicesIPFamilies(ctx, client, desiredFamilyFromCfg(cfg)); err != nil { + klog.Warningf("ensureSystemServicesIPFamilies failed: %v", err) + } + } + } + } + return nil } @@ -395,6 +418,43 @@ func (k *Bootstrapper) unpause(cfg config.ClusterConfig) error { return nil } +// ensureControlPlaneAlias adds control-plane.minikube.internal -> IP mapping in /etc/hosts +func (k *Bootstrapper) ensureControlPlaneAlias(cfg config.ClusterConfig) error { + family := strings.ToLower(cfg.KubernetesConfig.IPFamily) + + // HA: use the VIP that kube-vip sets + if config.IsHA(cfg) { + if ip := net.ParseIP(cfg.KubernetesConfig.APIServerHAVIP); ip != nil { + return machine.AddHostAlias(k.c, constants.ControlPlaneAlias, ip) + } + return nil + } + + // Single-CP: pick the right address based on IP family + cp, err := config.ControlPlane(cfg) + if err != nil { + return errors.Wrap(err, "get control-plane node") + } + + // For ipv6-only or dual, add AAAA + if family == "ipv6" || family == "dual" { + if ip6 := net.ParseIP(cp.IPv6); ip6 != nil { + if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, ip6); err != nil { + return errors.Wrap(err, "add control-plane alias (ipv6)") + } + } + } + // For ipv4-only or dual, add A + if family != "ipv6" { + if ip4 := net.ParseIP(cp.IP); ip4 != nil { + if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, ip4); err != nil { + return errors.Wrap(err, "add control-plane alias (ipv4)") + } + } + } + return nil +} + // StartCluster starts the cluster func (k *Bootstrapper) StartCluster(cfg config.ClusterConfig, options *run.CommandOptions) error { start := time.Now() @@ -432,6 +492,10 @@ func (k *Bootstrapper) StartCluster(cfg config.ClusterConfig, options *run.Comma return errors.Wrap(err, "cp") } + if err := k.ensureControlPlaneAlias(cfg); err != nil { + klog.Warningf("could not ensure control-plane alias: %v", err) + } + err := k.init(cfg, options) if err == nil { return nil @@ -590,6 +654,10 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time adviseNodePressure(err, cfg.Name, cfg.Driver) return errors.Wrap(err, "node pressure") } + + ctx2, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + _ = k.ensureSystemServicesIPFamilies(ctx2, client, desiredFamilyFromCfg(cfg)) return nil } @@ -702,6 +770,13 @@ func (k *Bootstrapper) restartPrimaryControlPlane(cfg config.ClusterConfig) erro return errors.Wrap(err, "system pods") } + // Re-ensure IP families after restart/reconfigure as well. + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + if err := k.ensureSystemServicesIPFamilies(ctx, client, desiredFamilyFromCfg(cfg)); err != nil { + klog.Warningf("ensureSystemServicesIPFamilies (restart) failed: %v", err) + } + if err := kverify.NodePressure(client); err != nil { adviseNodePressure(err, cfg.Name, cfg.Driver) } @@ -752,6 +827,17 @@ func (k *Bootstrapper) restartPrimaryControlPlane(cfg config.ClusterConfig) erro return nil } +func advertiseIP(cc config.ClusterConfig, n config.Node) string { + switch strings.ToLower(cc.KubernetesConfig.IPFamily) { + case "ipv6": + if n.IPv6 != "" { + return n.IPv6 + } + } + // default / ipv4 / dual: keep IPv4 + return n.IP +} + // JoinCluster adds new node to an existing cluster. func (k *Bootstrapper) JoinCluster(cc config.ClusterConfig, n config.Node, joinCmd string) error { // Join the control plane by specifying its token @@ -764,7 +850,9 @@ func (k *Bootstrapper) JoinCluster(cc config.ClusterConfig, n config.Node, joinC // ref: https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-join/#options // "If the node should host a new control plane instance, the IP address the API Server will advertise it's listening on. If not set the default network interface will be used." // "If the node should host a new control plane instance, the port for the API Server to bind to." - joinCmd += " --apiserver-advertise-address=" + n.IP + + // pick IPv6 for ipv6 clusters, otherwise IPv4 + addr := advertiseIP(cc, n) + joinCmd += " --apiserver-advertise-address=" + addr + " --apiserver-bind-port=" + strconv.Itoa(n.Port) } @@ -1005,18 +1093,45 @@ func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cru // add "control-plane.minikube.internal" dns alias // note: needs to be called after APIServerHAVIP is set (in startPrimaryControlPlane()) and before kubeadm kicks off - cpIP := cfg.KubernetesConfig.APIServerHAVIP - if !config.IsHA(cfg) { + + family := strings.ToLower(cfg.KubernetesConfig.IPFamily) + + if config.IsHA(cfg) { + // For HA we already have APIServerHAVIP set appropriately by kube-vip generation + if ip := net.ParseIP(cfg.KubernetesConfig.APIServerHAVIP); ip != nil { + if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, ip); err != nil { + return errors.Wrap(err, "add HA control-plane alias") + } + } + } else { cp, err := config.ControlPlane(cfg) if err != nil { return errors.Wrap(err, "get control-plane node") } - cpIP = cp.IP - } - if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, net.ParseIP(cpIP)); err != nil { - return errors.Wrap(err, "add control-plane alias") - } + // ipv6-only → write AAAA; ipv4/dual → write A; dual → write both + if family == "ipv6" { + if ip6 := net.ParseIP(cp.IPv6); ip6 != nil { + if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, ip6); err != nil { + return errors.Wrap(err, "add control-plane alias (ipv6)") + } + } + } else { + if ip4 := net.ParseIP(cp.IP); ip4 != nil { + if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, ip4); err != nil { + return errors.Wrap(err, "add control-plane alias (ipv4)") + } + } + if family == "dual" { + if ip6 := net.ParseIP(cp.IPv6); ip6 != nil { + // add AAAA alongside A + if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, ip6); err != nil { + return errors.Wrap(err, "add control-plane alias (dual, ipv6)") + } + } + } + } + } // "ensure" kubelet is started, intentionally non-fatal in case of an error if err := sysinit.New(k.c).Start("kubelet"); err != nil { klog.Errorf("Couldn't ensure kubelet is started this might cause issues (will continue): %v", err) @@ -1236,3 +1351,97 @@ func adviseNodePressure(err error, name string, drv string) { return } } + +// detectClusterFamily returns "dual" | "ipv6" | "ipv4" based on Node.Spec.PodCIDRs. +func (k *Bootstrapper) detectClusterFamily(ctx context.Context, cs kubernetes.Interface) (string, error) { + nodes, err := cs.CoreV1().Nodes().List(ctx, meta.ListOptions{}) + if err != nil { + return "", err + } + if len(nodes.Items) == 0 { + return "", fmt.Errorf("no nodes found") + } + podCIDRs := nodes.Items[0].Spec.PodCIDRs + switch { + case len(podCIDRs) >= 2: + return "dual", nil + case len(podCIDRs) == 1 && strings.Contains(podCIDRs[0], ":"): + return "ipv6", nil + default: + return "ipv4", nil + } +} + +func (k *Bootstrapper) patchServiceForFamily(ctx context.Context, cs kubernetes.Interface, ns, name, family string) error { + var patch []byte + switch family { + case "dual": + patch = []byte(`{"spec":{"ipFamilyPolicy":"PreferDualStack","ipFamilies":["IPv4","IPv6"]}}`) + case "ipv6": + patch = []byte(`{"spec":{"ipFamilyPolicy":"SingleStack","ipFamilies":["IPv6"]}}`) + default: + patch = []byte(`{"spec":{"ipFamilyPolicy":"SingleStack","ipFamilies":["IPv4"]}}`) + } + _, err := cs.CoreV1().Services(ns).Patch(ctx, name, types.MergePatchType, patch, meta.PatchOptions{}) + return err +} + +// desiredFamilyFromCfg returns "dual"|"ipv6"|"ipv4" from cfg, or "" if unknown. +func desiredFamilyFromCfg(cfg config.ClusterConfig) string { + f := strings.ToLower(cfg.KubernetesConfig.IPFamily) + switch f { + case "dual", "ipv6", "ipv4": + return f + default: + return "" + } +} + +// ensureSystemServicesIPFamilies waits for kube-dns Service and patches it to the right families. +// Add more Services here if you want them defaulted too. +func (k *Bootstrapper) ensureSystemServicesIPFamilies(ctx context.Context, cs kubernetes.Interface, family string) error { + // wait until kube-dns Service exists (kubeadm creates it) + if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + _, err := cs.CoreV1().Services("kube-system").Get(ctx, "kube-dns", meta.GetOptions{}) + return err == nil, nil + }); err != nil { + return fmt.Errorf("waiting for kube-dns service: %w", err) + } + + // Prefer the configured family; if empty, fall back to kube-proxy config, then node CIDRs. + if family == "" { + if cm, err := cs.CoreV1().ConfigMaps("kube-system").Get(ctx, "kube-proxy", meta.GetOptions{}); err == nil { + // config key is usually "config.conf"; older variants used "kubeproxy.config" + if cfgTxt, ok := cm.Data["config.conf"]; ok { + if strings.Contains(cfgTxt, ",") && strings.Contains(cfgTxt, ":") { + family = "dual" + } else if strings.Contains(cfgTxt, ":") { + family = "ipv6" + } else { + family = "ipv4" + } + } else if cfgTxt, ok := cm.Data["kubeproxy.config"]; ok { + if strings.Contains(cfgTxt, ",") && strings.Contains(cfgTxt, ":") { + family = "dual" + } else if strings.Contains(cfgTxt, ":") { + family = "ipv6" + } else { + family = "ipv4" + } + } + } + } + if family == "" { + if f, _ := k.detectClusterFamily(ctx, cs); f != "" { + family = f + } else { + family = "ipv4" + } + } + + if err := k.patchServiceForFamily(ctx, cs, "kube-system", "kube-dns", family); err != nil { + klog.Warningf("failed to patch kube-dns for %s: %v", family, err) + return err + } + return nil +} diff --git a/pkg/minikube/cni/bridge.go b/pkg/minikube/cni/bridge.go index d4264920ddb0..d3f2d0dd8bcb 100644 --- a/pkg/minikube/cni/bridge.go +++ b/pkg/minikube/cni/bridge.go @@ -17,10 +17,10 @@ limitations under the License. package cni import ( - "bytes" + "encoding/json" "fmt" "os/exec" - "text/template" + "strings" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/assets" @@ -32,38 +32,73 @@ import ( // ref: https://www.cni.dev/plugins/current/meta/portmap/ // ref: https://www.cni.dev/plugins/current/meta/firewall/ -// note: "cannot set hairpin mode and promiscuous mode at the same time" -// ref: https://github.com/containernetworking/plugins/blob/7e9ada51e751740541969e1ea5a803cbf45adcf3/plugins/main/bridge/bridge.go#L424 -var bridgeConf = template.Must(template.New("bridge").Parse(` -{ - "cniVersion": "0.4.0", - "name": "bridge", - "plugins": [ - { - "type": "bridge", - "bridge": "bridge", - "addIf": "true", - "isDefaultGateway": true, - "forceAddress": false, - "ipMasq": true, - "hairpinMode": true, - "ipam": { - "type": "host-local", - "subnet": "{{.PodCIDR}}" - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - }, - { - "type": "firewall" - } - ] +// renderBridgeConflist builds a bridge CNI config that supports IPv4-only, IPv6-only, or dual-stack. +func renderBridgeConflist(k8s config.KubernetesConfig) ([]byte, error) { + // minimal structs for JSON marshal + type rng struct { + Subnet string `json:"subnet"` + } + type ipam struct { + Type string `json:"type"` + Subnet string `json:"subnet,omitempty"` // single-stack (v4 or v6) + Ranges [][]rng `json:"ranges,omitempty"` // dual-stack + } + type bridge struct { + Type string `json:"type"` + Bridge string `json:"bridge"` + IsDefaultGateway bool `json:"isDefaultGateway"` + HairpinMode bool `json:"hairpinMode"` + IPMasq bool `json:"ipMasq"` + IPAM ipam `json:"ipam"` + } + type plugin struct { + Type string `json:"type"` + Capabilities map[string]bool `json:"capabilities,omitempty"` + } + type conflist struct { + CNIVersion string `json:"cniVersion"` + Name string `json:"name"` + Plugins []interface{} `json:"plugins"` + } + + v4 := k8s.PodCIDR != "" + v6 := k8s.PodCIDRv6 != "" + + cfgIPAM := ipam{Type: "host-local"} + switch { + case v4 && v6: + cfgIPAM.Ranges = [][]rng{{{Subnet: k8s.PodCIDR}}, {{Subnet: k8s.PodCIDRv6}}} + case v6: + cfgIPAM.Subnet = k8s.PodCIDRv6 + default: + // fall back to previous default if unset upstream + cidr := k8s.PodCIDR + if cidr == "" { + cidr = DefaultPodCIDR + } + cfgIPAM.Subnet = cidr + } + + // NAT generally not desired for IPv6; keep masquerade only for v4 + ipMasq := v4 && !v6 + + br := bridge{ + Type: "bridge", Bridge: "cni0", + IsDefaultGateway: true, + HairpinMode: true, + IPMasq: ipMasq, + IPAM: cfgIPAM, + } + portmap := plugin{Type: "portmap", Capabilities: map[string]bool{"portMappings": true}} + firewall := plugin{Type: "firewall"} + + out := conflist{ + CNIVersion: "1.0.0", + Name: "k8s-pod-network", + Plugins: []interface{}{br, portmap, firewall}, + } + return json.MarshalIndent(out, "", " ") } -`)) // Bridge is a simple CNI manager for single-node usage type Bridge struct { @@ -76,14 +111,11 @@ func (c Bridge) String() string { } func (c Bridge) netconf() (assets.CopyableFile, error) { - input := &tmplInput{PodCIDR: DefaultPodCIDR} - - b := bytes.Buffer{} - if err := bridgeConf.Execute(&b, input); err != nil { + cfgBytes, err := renderBridgeConflist(c.cc.KubernetesConfig) + if err != nil { return nil, err } - - return assets.NewMemoryAssetTarget(b.Bytes(), "/etc/cni/net.d/1-k8s.conflist", "0644"), nil + return assets.NewMemoryAssetTarget(cfgBytes, "/etc/cni/net.d/1-k8s.conflist", "0644"), nil } // Apply enables the CNI @@ -110,5 +142,15 @@ func (c Bridge) Apply(r Runner) error { // CIDR returns the default CIDR used by this CNI func (c Bridge) CIDR() string { + + // Prefer explicitly-set CIDRs from the cluster config. + k := c.cc.KubernetesConfig + if k.PodCIDRv6 != "" && (strings.ToLower(k.IPFamily) == "ipv6" || k.PodCIDR == "") { + return k.PodCIDRv6 + } + + if k.PodCIDR != "" { + return k.PodCIDR + } return DefaultPodCIDR } diff --git a/pkg/minikube/cni/calico.go b/pkg/minikube/cni/calico.go index 6f7b55c4b7c1..8334e038aa92 100644 --- a/pkg/minikube/cni/calico.go +++ b/pkg/minikube/cni/calico.go @@ -19,6 +19,9 @@ package cni import ( "bytes" "fmt" + "os/exec" + "strings" + "time" // goembed needs this _ "embed" @@ -29,6 +32,7 @@ import ( "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/util" ) @@ -46,11 +50,20 @@ type Calico struct { } type calicoTmplStruct struct { + // IPv4/IPv6/dual inputs for the template + IPFamily string PodCIDR string + PodCIDRv6 string + ServiceCIDR string + ServiceCIDRv6 string + ControlPlaneAlias string + APIServerPort int DeploymentImageName string DaemonSetImageName string BinaryImageName string LegacyPodDisruptionBudget bool + EnableIPv4 bool + EnableIPv6 bool } // String returns a string representation of this CNI @@ -65,12 +78,48 @@ func (c Calico) manifest() (assets.CopyableFile, error) { return nil, fmt.Errorf("failed to parse Kubernetes version: %v", err) } + k := c.cc.KubernetesConfig + ipFamily := strings.ToLower(k.IPFamily) + // Defaults/fallbacks to stay safe if flags weren’t provided + v4Pod := k.PodCIDR + if v4Pod == "" { + v4Pod = DefaultPodCIDR + } + v6Pod := k.PodCIDRv6 + // Ensure an IPv6 pod CIDR exists when the ily is v6 or dual + if v6Pod == "" && (ipFamily == "ipv6" || ipFamily == "dual") { + v6Pod = constants.DefaultPodCIDRv6 + } + svcV4 := k.ServiceCIDR + if svcV4 == "" { + svcV4 = constants.DefaultServiceCIDR + } + svcV6 := k.ServiceCIDRv6 + if svcV6 == "" && ipFamily != "ipv4" { + svcV6 = constants.DefaultServiceCIDRv6 + } + apiPort := c.cc.APIServerPort + if apiPort == 0 { + apiPort = 8443 + } + + enablev4 := (ipFamily == "ipv4" || ipFamily == "dual") + enablev6 := (ipFamily == "ipv6" || ipFamily == "dual") + input := &calicoTmplStruct{ - PodCIDR: DefaultPodCIDR, - DeploymentImageName: images.CalicoDeployment(c.cc.KubernetesConfig.ImageRepository), - DaemonSetImageName: images.CalicoDaemonSet(c.cc.KubernetesConfig.ImageRepository), - BinaryImageName: images.CalicoBin(c.cc.KubernetesConfig.ImageRepository), + IPFamily: ipFamily, + PodCIDR: v4Pod, + PodCIDRv6: v6Pod, + ServiceCIDR: svcV4, + ServiceCIDRv6: svcV6, + ControlPlaneAlias: constants.ControlPlaneAlias, + APIServerPort: apiPort, + DeploymentImageName: images.CalicoDeployment(k.ImageRepository), + DaemonSetImageName: images.CalicoDaemonSet(k.ImageRepository), + BinaryImageName: images.CalicoBin(k.ImageRepository), LegacyPodDisruptionBudget: k8sVersion.LT(semver.Version{Major: 1, Minor: 25}), + EnableIPv4: enablev4, + EnableIPv6: enablev6, } b := bytes.Buffer{} @@ -86,11 +135,97 @@ func (c Calico) Apply(r Runner) error { if err != nil { return errors.Wrap(err, "manifest") } - return applyManifest(c.cc, r, m) + // Phase 1: apply core Calico (includes CRDs) + if err := applyManifest(c.cc, r, m); err != nil { + return err + } + + // Phase 2: wait for CRD to be Established, then apply IPPools + if err := waitForCRDEstablished(r, c.cc.KubernetesConfig.KubernetesVersion, "ippools.crd.projectcalico.org", 90*time.Second); err != nil { + // Non-fatal: log and try to continue; but usually this must succeed. + return errors.Wrap(err, "waiting for Calico IPPool CRD") + } + + ipPoolsYAML := renderCalicoIPPools(c.cc.KubernetesConfig) + if ipPoolsYAML == "" { + return nil + } + poolsAsset := assets.NewMemoryAssetTarget([]byte(ipPoolsYAML), "/var/tmp/minikube/calico-ippools.yaml", "0644") + return applyManifest(c.cc, r, poolsAsset) +} + +// waitForCRDEstablished waits until the given CRD reports Established=True. +func waitForCRDEstablished(r Runner, k8sVersion string, crd string, to time.Duration) error { + kubectlPath := fmt.Sprintf("/var/lib/minikube/binaries/%s/kubectl", k8sVersion) + cmd := exec.Command("sudo", kubectlPath, "wait", + "--kubeconfig=/var/lib/minikube/kubeconfig", + "--for=condition=Established", + fmt.Sprintf("--timeout=%ds", int(to.Seconds())), + "crd/"+crd, + ) + _, err := r.RunCmd(cmd) + return err +} + +// renderCalicoIPPools returns a small manifest for IPv4/IPv6 IPPools based on IPFamily and PodCIDRs. +func renderCalicoIPPools(k config.KubernetesConfig) string { + fam := strings.ToLower(k.IPFamily) + var b strings.Builder + emit := func(name, cidr string, nat bool, isV6 bool) { + if cidr == "" { + return + } + if b.Len() > 0 { + b.WriteString("\n---\n") + } + + // IPPool instances are served under projectcalico.org/v3. + fmt.Fprintf(&b, + "apiVersion: projectcalico.org/v3\nkind: IPPool\nmetadata:\n name: %s\nspec:\n cidr: %q\n disabled: false\n natOutgoing: %t\n nodeSelector: \"all()\"\n", + name, cidr, nat) + if !isV6 { + // Match calico-node env (IPIP Always, VXLAN Never) for IPv4. + b.WriteString(" ipipMode: Always\n vxlanMode: Never\n") + } else { + // For IPv6, IPIP is IPv4-only; keep VXLAN off to match DS env. + b.WriteString(" vxlanMode: Never\n") + } + } + + // IPv4 pool unless explicitly ipv6-only + if fam != "ipv6" { + cidr := k.PodCIDR + if cidr == "" { + cidr = DefaultPodCIDR + } + emit("default-ipv4-ippool", cidr, true, false) + } + // IPv6 pool for ipv6 or dual + if fam == "ipv6" || fam == "dual" { + cidr := k.PodCIDRv6 + if cidr == "" { + // default provided by your constants if you prefer: + // cidr = constants.DefaultPodCIDRv6 + // but safer to require user/normalizer to have set it. + cidr = constants.DefaultPodCIDRv6 + } + emit("default-ipv6-ippool", cidr, false, true) + } + return b.String() } // CIDR returns the default CIDR used by this CNI func (c Calico) CIDR() string { // Calico docs specify 192.168.0.0/16 - but we do this for compatibility with other CNI's. + k := c.cc.KubernetesConfig + fam := strings.ToLower(k.IPFamily) + // Prefer explicitly-set CIDRs; for ipv6-only prefer v6 + if k.PodCIDRv6 != "" && (fam == "ipv6" || k.PodCIDR == "") { + return k.PodCIDRv6 + } + if k.PodCIDR != "" { + return k.PodCIDR + } + // fallback for legacy behavior return DefaultPodCIDR } diff --git a/pkg/minikube/cni/calico.yaml b/pkg/minikube/cni/calico.yaml index 0f6b3a661f8a..82b879509e07 100644 --- a/pkg/minikube/cni/calico.yaml +++ b/pkg/minikube/cni/calico.yaml @@ -69,7 +69,11 @@ data: "nodename": "__KUBERNETES_NODE_NAME__", "mtu": __CNI_MTU__, "ipam": { - "type": "calico-ipam" + "type": "calico-ipam", + "assign_ipv4": {{ if .EnableIPv4 }}"true"{{ else }}"false"{{ end }}, + "assign_ipv6": {{ if .EnableIPv6 }}"true"{{ else }}"false"{{ end }}{{ if .EnableIPv4 }}, + "ipv4_pools": ["default-ipv4-ippool"]{{ end }}{{ if .EnableIPv6 }}, + "ipv6_pools": ["default-ipv6-ippool"]{{ end }} }, "policy": { "type": "k8s" @@ -78,6 +82,10 @@ data: "kubeconfig": "__KUBECONFIG_FILEPATH__" } }, + { + "type": "bandwidth", + "capabilities": { "bandwidth": true } + }, { "type": "portmap", "snat": true, @@ -7052,6 +7060,24 @@ spec: terminationGracePeriodSeconds: 0 priorityClassName: system-node-critical initContainers: + + # Ensure the CNI can read the node name immediately. + # The Calico CNI looks for /var/lib/calico/nodename when adding a pod; + # this seeds it before any workload is scheduled. + - name: seed-nodename + image: {{ .DaemonSetImageName }} + imagePullPolicy: IfNotPresent + command: ["sh", "-c", "echo -n ${KUBERNETES_NODE_NAME} > /var/lib/calico/nodename || true"] + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /var/lib/calico + name: var-lib-calico + securityContext: + privileged: true # This container performs upgrade from host-local IPAM to calico-ipam. # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. @@ -7065,6 +7091,12 @@ spec: name: kubernetes-services-endpoint optional: true env: + # No pool creation here; calico-node handles pools, and we apply CRDs explicitly. + # (Keep this init strictly for host-local -> calico-ipam upgrade.) + - name: IP + value: {{ if .EnableIPv4 }}"autodetect"{{ else }}"none"{{ end }} + - name: IP6 + value: {{ if .EnableIPv6 }}"autodetect"{{ else }}"none"{{ end }} - name: KUBERNETES_NODE_NAME valueFrom: fieldRef: @@ -7185,15 +7217,23 @@ spec: # Auto-detect the BGP IP address. - name: IP value: "autodetect" + - name: IP6 + value: '{{ if or (eq .IPFamily "ipv6") (eq .IPFamily "dual") }}autodetect{{ else }}none{{ end }}' # Enable IPIP - name: CALICO_IPV4POOL_IPIP - value: "Always" + value: "Never" # Enable or Disable VXLAN on the default IP pool. - name: CALICO_IPV4POOL_VXLAN value: "Never" # Enable or Disable VXLAN on the default IPv6 IP pool. - name: CALICO_IPV6POOL_VXLAN value: "Never" + # Don’t let calico-node create default pools; we apply IPPool CRDs explicitly. + - name: NO_DEFAULT_POOLS + value: "false" + # Let Calico know desired v6 capability for Felix. + - name: FELIX_IPV6SUPPORT + value: {{ if .EnableIPv6 }}"true"{{ else }}"false"{{ end }} # Set MTU for tunnel device used if ipip is enabled - name: FELIX_IPINIPMTU valueFrom: @@ -7223,9 +7263,6 @@ spec: # Set Felix endpoint to host default action to ACCEPT. - name: FELIX_DEFAULTENDPOINTTOHOSTACTION value: "ACCEPT" - # Disable IPv6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - name: FELIX_HEALTHENABLED value: "true" securityContext: @@ -7360,6 +7397,10 @@ spec: labels: k8s-app: calico-kube-controllers spec: + {{- if eq .IPFamily "ipv6" }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + {{- end }} nodeSelector: kubernetes.io/os: linux tolerations: @@ -7382,9 +7423,17 @@ spec: env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: node,loadbalancer + value: "node" - name: DATASTORE_TYPE value: kubernetes + {{- if eq .IPFamily "ipv6" }} + # Avoid DNS and Service VIPs during IPv6 bootstrap: + # kube-controllers runs with hostNetwork, so loopback reaches the local apiserver. + - name: KUBERNETES_SERVICE_HOST + value: "::1" + - name: KUBERNETES_SERVICE_PORT + value: "{{ .APIServerPort }}" + {{- end }} livenessProbe: exec: command: diff --git a/pkg/minikube/cni/cilium.go b/pkg/minikube/cni/cilium.go index f02728f6b902..7e20d118696a 100644 --- a/pkg/minikube/cni/cilium.go +++ b/pkg/minikube/cni/cilium.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "os/exec" + "strings" "text/template" "github.com/blang/semver/v4" @@ -33,6 +34,10 @@ import ( "k8s.io/minikube/pkg/util" ) +// Default IPv6 pod CIDR when running Cilium with IPv6 enabled +// (Each node will get a /64 slice from this /48 by default.) +const DefaultPodCIDRv6 = "fd01::/48" + // Generated by running `make update-cilium-version` // //go:embed cilium.yaml @@ -64,15 +69,72 @@ func (c Cilium) GenerateCiliumYAML() ([]byte, error) { } } - podCIDR := DefaultPodCIDR + // Decide IPv4/IPv6 family. Prefer explicit v6 fields if provided, otherwise + // fall back to parsing a comma-separated ServiceCIDR string. + svcCIDR := c.cc.KubernetesConfig.ServiceCIDR + svcCIDRv6 := c.cc.KubernetesConfig.ServiceCIDRv6 + partHasV6 := func(s string) bool { return strings.Contains(s, ":") } + partHasV4 := func(s string) bool { return strings.Contains(s, ".") } + enableV6 := false + enableV4 := true + if svcCIDR != "" { + if strings.Contains(svcCIDR, ",") { + for _, p := range strings.Split(svcCIDR, ",") { + p = strings.TrimSpace(p) + if partHasV6(p) { + enableV6 = true + } + if partHasV4(p) { + enableV4 = true + } + } + } else { + enableV6 = partHasV6(svcCIDR) + if enableV6 && !partHasV4(svcCIDR) { + enableV4 = false + } + } + } - klog.Infof("Using pod CIDR: %s", podCIDR) + // If the cluster was configured with a dedicated IPv6 service range, honor it. + if svcCIDRv6 != "" && partHasV6(svcCIDRv6) { + enableV6 = true + // If no IPv4 service range was provided at all, assume IPv6-only + if svcCIDR == "" { + enableV4 = false + } + } + + podCIDRv4 := DefaultPodCIDR + podCIDRv6 := DefaultPodCIDRv6 + // If a specific v6 pod subnet was provided (e.g. via --subnet-v6), prefer it. + if v := c.cc.Subnetv6; v != "" { + podCIDRv6 = v + } + // Valid values in Cilium v1.18: disabled | partial | strict + // Safe default (kube-proxy addon is typically enabled in minikube): partial + kprMode := "partial" + // If you later decide to disable the kube-proxy addon, flip this to "strict". + // if enableV6 { kprMode = "strict" } + + klog.Infof("Cilium IP family: enableIPv4=%t enableIPv6=%t", enableV4, enableV6) + klog.Infof("Using pod CIDRs: v4=%s v6=%s", podCIDRv4, podCIDRv6) + klog.Infof("Cilium kube-proxy-replacement: %s", kprMode) opts := struct { - PodSubnet string + PodSubnet string + PodSubnetV6 string + EnableIPv4 string + EnableIPv6 string + KubeProxyReplacement string }{ - PodSubnet: podCIDR, + PodSubnet: podCIDRv4, + PodSubnetV6: podCIDRv6, + EnableIPv4: fmt.Sprintf("%t", enableV4), + EnableIPv6: fmt.Sprintf("%t", enableV6), + KubeProxyReplacement: kprMode, } + ciliumTmpl := template.Must(template.New("name").Parse(ciliumYaml)) b := bytes.Buffer{} configTmpl := ciliumTmpl @@ -114,7 +176,7 @@ func removeAppArmorProfile(ciliumConfig string) (string, error) { } else if err != nil { return "", fmt.Errorf("failed to unmarshal yaml: %v", err) } - if err := dyno.Delete(obj, "appArmorProfile", "spec", "template", "spec", "securityContext"); err != nil { + if err := dyno.Delete(obj, "spec", "template", "spec", "securityContext", "appArmorProfile"); err != nil { return "", fmt.Errorf("failed to remove securityContext yaml: %v", err) } if err := encoder.Encode(obj); err != nil { diff --git a/pkg/minikube/cni/cilium.yaml b/pkg/minikube/cni/cilium.yaml index 38cf0bf99678..e4401f4844e1 100644 --- a/pkg/minikube/cni/cilium.yaml +++ b/pkg/minikube/cni/cilium.yaml @@ -79,11 +79,11 @@ data: # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 # address. - enable-ipv4: "true" + enable-ipv4: "{{ .EnableIPv4 }}" # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 # address. - enable-ipv6: "false" + enable-ipv6: "{{ .EnableIPv6 }}" # Users who wish to specify their own custom CNI configuration file must set # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. custom-cni-conf: "false" @@ -163,10 +163,10 @@ data: # Enables L7 proxy for L7 policy enforcement and visibility enable-l7-proxy: "true" - enable-ipv4-masquerade: "true" + enable-ipv4-masquerade: "{{ if eq .EnableIPv4 "true" }}true{{ else }}false{{ end }}" enable-ipv4-big-tcp: "false" enable-ipv6-big-tcp: "false" - enable-ipv6-masquerade: "true" + enable-ipv6-masquerade: "{{ if eq .EnableIPv6 "true" }}true{{ else }}false{{ end }}" enable-tcx: "true" datapath-mode: "veth" enable-masquerade-to-route-source: "false" @@ -219,8 +219,14 @@ data: hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt ipam: "cluster-pool" ipam-cilium-node-update-rate: "15s" + {{- if eq .EnableIPv4 "true" }} cluster-pool-ipv4-cidr: "{{ .PodSubnet }}" cluster-pool-ipv4-mask-size: "24" + {{- end }} + {{- if eq .EnableIPv6 "true" }} + cluster-pool-ipv6-cidr: "{{ .PodSubnetV6 }}" + cluster-pool-ipv6-mask-size: "64" + {{- end }} default-lb-service-ipam: "lbipam" egress-gateway-reconciliation-trigger-interval: "1s" diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 1077ac3ca255..7372ecdd3e0d 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -50,6 +50,7 @@ type ClusterConfig struct { InsecureRegistry []string RegistryMirror []string HostOnlyCIDR string // Only used by the virtualbox driver + HostOnlyCIDRv6 string // IPv6 CIDR for the virtualbox driver HypervVirtualSwitch string HypervUseExternalSwitch bool HypervExternalAdapter string @@ -85,6 +86,7 @@ type ClusterConfig struct { ListenAddress string // Only used by the docker and podman driver Network string // only used by docker driver Subnet string // only used by the docker and podman driver + Subnetv6 string // IPv6 subnet for docker and podman driver MultiNodeRequested bool ExtraDisks int // currently only implemented for hyperkit and kvm2 CertExpiration time.Duration @@ -105,6 +107,7 @@ type ClusterConfig struct { SocketVMnetClientPath string SocketVMnetPath string StaticIP string + StaticIPv6 string // Static IPv6 address for the cluster SSHAuthSock string SSHAgentPID int GPUs string @@ -126,6 +129,10 @@ type KubernetesConfig struct { NetworkPlugin string FeatureGates string // https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ ServiceCIDR string // the subnet which Kubernetes services will be deployed to + ServiceCIDRv6 string // the IPv6 subnet which Kubernetes services will be deployed to + PodCIDR string // the IPv4 subnet which Kubernetes pods will be deployed to + PodCIDRv6 string // the IPv6 subnet which Kubernetes pods will be deployed to + IPFamily string // IP family mode: ipv4, ipv6, or dual ImageRepository string LoadBalancerStartIP string // currently only used by MetalLB addon LoadBalancerEndIP string // currently only used by MetalLB addon @@ -143,6 +150,7 @@ type KubernetesConfig struct { type Node struct { Name string IP string + IPv6 string // IPv6 address of the node Port int KubernetesVersion string ContainerRuntime string diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 90e19ee41177..a75eaea5060d 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -80,6 +80,10 @@ const ( ClusterDNSDomain = "cluster.local" // DefaultServiceCIDR is The CIDR to be used for service cluster IPs DefaultServiceCIDR = "10.96.0.0/12" + // DefaultServiceCIDRv6 is The IPv6 CIDR to be used for service cluster IPs + DefaultServiceCIDRv6 = "fd00::/108" + // DefaultPodCIDRv6 is The IPv6 CIDR to be used for pod IPs + DefaultPodCIDRv6 = "fd01::/64" // HostAlias is a DNS alias to the container/VM host IP HostAlias = "host.minikube.internal" // ControlPlaneAlias is a DNS alias pointing to the apiserver frontend diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 85699fa0d375..57d9e8a96b01 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -159,20 +159,25 @@ func (r *CRIO) Active() bool { // enableIPForwarding configures IP forwarding, which is handled normally by Docker // Context: https://github.com/kubernetes/kubeadm/issues/1062 func enableIPForwarding(cr CommandRunner) error { - // The bridge-netfilter module enables iptables rules to work on Linux bridges - // NOTE: br_netfilter isn't available in WSL2, but forwarding works fine there anyways - c := exec.Command("sudo", "sysctl", "net.bridge.bridge-nf-call-iptables") - if rr, err := cr.RunCmd(c); err != nil { - klog.Infof("couldn't verify netfilter by %q which might be okay. error: %v", rr.Command(), err) - c = exec.Command("sudo", "modprobe", "br_netfilter") - if _, err := cr.RunCmd(c); err != nil { - klog.Warningf("%q failed, which may be ok: %v", rr.Command(), err) + // The bridge-netfilter module enables (ip|ip6)tables rules to apply on Linux bridges. + // NOTE: br_netfilter isn't available everywhere (e.g., some WSL2 kernels) – treat as best-effort. + if _, err := cr.RunCmd(exec.Command("sudo", "modprobe", "br_netfilter")); err != nil { + klog.Warningf("modprobe br_netfilter failed (may be OK on this kernel): %v", err) + } + + // Enable bridge netfilter hooks for both IPv4 and IPv6, and enable forwarding. + // Best-effort: warn but don't fail hard if a sysctl isn't present. + sysctls := []string{ + "sysctl -w net.bridge.bridge-nf-call-iptables=1", + "sysctl -w net.bridge.bridge-nf-call-ip6tables=1", + "sysctl -w net.ipv4.ip_forward=1", + "sysctl -w net.ipv6.conf.all.forwarding=1", + } + for _, s := range sysctls { + if _, err := cr.RunCmd(exec.Command("sudo", "sh", "-c", s)); err != nil { + klog.Warningf("failed to run %q (continuing): %v", s, err) } } - c = exec.Command("sudo", "sh", "-c", "echo 1 > /proc/sys/net/ipv4/ip_forward") - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrapf(err, "ip_forward") - } return nil } diff --git a/pkg/minikube/driver/endpoint.go b/pkg/minikube/driver/endpoint.go index d5e86a746e57..016c1518d5cb 100644 --- a/pkg/minikube/driver/endpoint.go +++ b/pkg/minikube/driver/endpoint.go @@ -19,6 +19,7 @@ package driver import ( "fmt" "net" + "strings" "k8s.io/klog/v2" "k8s.io/minikube/pkg/drivers/kic/oci" @@ -27,7 +28,6 @@ import ( "k8s.io/minikube/pkg/network" ) -// ControlPlaneEndpoint returns the location where callers can reach this cluster. func ControlPlaneEndpoint(cc *config.ClusterConfig, cp *config.Node, driverName string) (string, net.IP, int, error) { if NeedsPortForward(driverName) { port, err := oci.ForwardedPort(cc.Driver, cc.Name, cp.Port) @@ -35,35 +35,56 @@ func ControlPlaneEndpoint(cc *config.ClusterConfig, cp *config.Node, driverName klog.Warningf("failed to get forwarded control plane port %v", err) } - hostname := oci.DaemonHost(driverName) - - ips, err := net.LookupIP(hostname) - if err != nil || len(ips) == 0 { - return hostname, nil, port, fmt.Errorf("failed to lookup ip for %q", hostname) + // Start with daemon host (docker/podman), tweak for IPv6, then honor APIServerName override. + host := oci.DaemonHost(driverName) + // If the cluster/node IP is IPv6 and daemon host is localhost on IPv4, + // force IPv6 loopback so we hit the port that’s actually listening. + if strings.Contains(cp.IP, ":") && (host == "127.0.0.1" || host == "localhost") { + host = "::1" } - - // https://github.com/kubernetes/minikube/issues/3878 if cc.KubernetesConfig.APIServerName != constants.APIServerName { - hostname = cc.KubernetesConfig.APIServerName + host = cc.KubernetesConfig.APIServerName + } + + // Resolve final host -> IPs. Allow literal IPv4/IPv6 without DNS. + var ips []net.IP + if ip := net.ParseIP(host); ip != nil { + ips = []net.IP{ip} + } else { + ips, err = net.LookupIP(host) + if err != nil || len(ips) == 0 { + return host, nil, port, fmt.Errorf("failed to lookup ip for %q", host) + } } - return hostname, ips[0], port, nil + return host, ips[0], port, nil } if IsQEMU(driverName) && network.IsBuiltinQEMU(cc.Network) { - return "localhost", net.IPv4(127, 0, 0, 1), cc.APIServerPort, nil + if strings.Contains(cp.IP, ":") { + return "::1", net.IPv6loopback, cc.APIServerPort, nil + } + return "127.0.0.1", net.IPv4(127, 0, 0, 1), cc.APIServerPort, nil } - // https://github.com/kubernetes/minikube/issues/3878 - hostname := cp.IP + // Default: use the node IP (literal or resolvable name) + host := cp.IP if cc.KubernetesConfig.APIServerName != constants.APIServerName { - hostname = cc.KubernetesConfig.APIServerName + host = cc.KubernetesConfig.APIServerName } - ips, err := net.LookupIP(cp.IP) - if err != nil || len(ips) == 0 { - return hostname, nil, cp.Port, fmt.Errorf("failed to lookup ip for %q", cp.IP) + + var ips []net.IP + if ip := net.ParseIP(cp.IP); ip != nil { + ips = []net.IP{ip} + } else { + var err error + ips, err = net.LookupIP(cp.IP) + if err != nil || len(ips) == 0 { + return host, nil, cp.Port, fmt.Errorf("failed to lookup ip for %q", cp.IP) + } } - return hostname, ips[0], cp.Port, nil + + return host, ips[0], cp.Port, nil } // AutoPauseProxyEndpoint returns the endpoint for the auto-pause (reverse proxy to api-sever) diff --git a/pkg/minikube/kubeconfig/kubeconfig.go b/pkg/minikube/kubeconfig/kubeconfig.go index 29a9419f0f6b..fe5a1cdf670c 100644 --- a/pkg/minikube/kubeconfig/kubeconfig.go +++ b/pkg/minikube/kubeconfig/kubeconfig.go @@ -23,6 +23,7 @@ import ( "path" "path/filepath" "strconv" + "strings" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" @@ -52,7 +53,13 @@ func UpdateEndpoint(contextName string, host string, port int, configPath string return false, errors.Wrap(err, "get kubeconfig") } - address := "https://" + host + ":" + strconv.Itoa(port) + // Bracket IPv6 literals for a valid URL (e.g. https://[::1]:8443) + hostForURL := host + if strings.Contains(hostForURL, ":") && !strings.HasPrefix(hostForURL, "[") { + hostForURL = "[" + hostForURL + "]" + } + + address := "https://" + hostForURL + ":" + strconv.Itoa(port) // check & fix kubeconfig if the cluster or context setting is missing, or server address needs updating errs := configIssues(cfg, contextName, address) diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index 12ca77f21c9c..4e84ed9eb9a5 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -24,6 +24,7 @@ import ( "os/exec" "path" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -399,28 +400,51 @@ func showHostInfo(h *host.Host, cfg config.ClusterConfig) { out.Step(style.StartingVM, "Creating {{.driver_name}} {{.machine_type}} (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"driver_name": cfg.Driver, "number_of_cpus": cfg.CPUs, "memory_size": cfg.Memory, "disk_size": cfg.DiskSize, "machine_type": machineType}) } -// AddHostAlias makes fine adjustments to pod resources that aren't possible via kubeadm config. +// AddHostAlias ensures /etc/hosts contains an entry for name -> ip. +// It preserves an existing record of the other IP family (so dual-stack can have both A and AAAA). func AddHostAlias(c command.Runner, name string, ip net.IP) error { - record := fmt.Sprintf("%s\t%s", ip, name) - if _, err := c.RunCmd(exec.Command("grep", record+"$", "/etc/hosts")); err == nil { + if ip == nil || ip.IsUnspecified() || ip.String() == "" { + klog.Warningf("skipping AddHostAlias for %q: empty/unspecified IP", name) return nil } - if _, err := c.RunCmd(addHostAliasCommand(name, record, true, "/etc/hosts")); err != nil { + // Exact line we want to ensure exists. + record := fmt.Sprintf("%s\t%s", ip.String(), name) + + // Fast path: if the exact line already exists, do nothing. + if _, err := c.RunCmd(exec.Command("grep", "-Fxq", record, "/etc/hosts")); err == nil { + return nil + } + + // Remove only existing lines for *this* name and *this* IP family, keep the other family. + dropRegex := hostAliasDropRegex(name, ip) + + if _, err := c.RunCmd(addHostAliasCommand(dropRegex, record, true, "/etc/hosts")); err != nil { return errors.Wrap(err, "hosts update") } return nil } -func addHostAliasCommand(name string, record string, sudo bool, destPath string) *exec.Cmd { +// hostAliasDropRegex builds a regex that matches lines mapping the given name with the same IP family. +func hostAliasDropRegex(name string, ip net.IP) string { + qName := regexp.QuoteMeta(name) + if ip.To4() != nil { + // IPv4 lines like: 1.2.3.4 + return fmt.Sprintf(`^[[:space:]]*[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+[[:space:]]+%s$`, qName) + } + // IPv6 lines like: 2001:db8::1 (very permissive IPv6 matcher) + return fmt.Sprintf(`^[[:space:]]*[:0-9A-Fa-f]+[[:space:]]+%s$`, qName) +} + +func addHostAliasCommand(dropRegex, record string, sudo bool, destPath string) *exec.Cmd { sudoCmd := "sudo" if !sudo { // for testing sudoCmd = "" } script := fmt.Sprintf( - `{ grep -v $'\t%s$' "%s"; echo "%s"; } > /tmp/h.$$; %s cp /tmp/h.$$ "%s"`, - name, + `{ grep -v -F "%s" "%s"; echo "%s"; } > /tmp/h.$$; %s cp /tmp/h.$$ "%s"`, + record, destPath, record, sudoCmd, diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index f62799853e7d..3bdb2a45cf47 100755 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -632,12 +632,19 @@ func setupKubeconfig(h host.Host, cc config.ClusterConfig, n config.Node, cluste if hostIP, _, port, err = driver.ControlPlaneEndpoint(&cc, &n, h.DriverName); err != nil { exit.Message(reason.DrvCPEndpoint, fmt.Sprintf("failed to construct cluster server address: %v", err), out.V{"profileArg": fmt.Sprintf("--profile=%s", clusterName)}) } + + if hostIP == "" { + hostIP = "localhost" + } + } - addr := fmt.Sprintf("https://%s", net.JoinHostPort(hostIP, strconv.Itoa(port))) + // Build address *after* picking the final host (don’t string-replace inside a bracketed IPv6 literal). + hostForAddr := hostIP if cc.KubernetesConfig.APIServerName != constants.APIServerName { - addr = strings.ReplaceAll(addr, hostIP, cc.KubernetesConfig.APIServerName) + hostForAddr = cc.KubernetesConfig.APIServerName } + addr := "https://" + net.JoinHostPort(hostForAddr, strconv.Itoa(port)) kcs := &kubeconfig.Settings{ ClusterName: clusterName, @@ -674,6 +681,30 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool, return runner, preExists, m, hostInfo, errors.Wrap(err, "Failed to validate network") } + if driver.IsKIC(cfg.Driver) { + containerName := config.MachineName(*cfg, *node) + + _, ipv6, cipErr := oci.ContainerIPs(oci.Docker, containerName) + if cipErr != nil { + // Not fatal – behaviour stays as today if we can't read the IPv6. + klog.Warningf("failed to get container IPv6 for %q: %v", containerName, cipErr) + } else if ipv6 != "" { + // Update the node pointer we were given … + node.IPv6 = ipv6 + + // …and also update the copy inside cfg.Nodes, in case 'node' + // is a separate copy taken by the caller. + for i := range cfg.Nodes { + if cfg.Nodes[i].Name == node.Name { + cfg.Nodes[i].IPv6 = ipv6 + break + } + } + + klog.Infof("updated node %q IPv6 from container: ipv6=%q", node.Name, ipv6) + } + } + if driver.IsQEMU(hostInfo.Driver.DriverName()) && network.IsBuiltinQEMU(cfg.Network) { apiServerPort, err := getPort() if err != nil { diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index 766791306b1c..c6101e7f31d6 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -90,7 +90,10 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { ExtraArgs: extraArgs, Network: cc.Network, Subnet: cc.Subnet, + Subnetv6: cc.Subnetv6, StaticIP: cc.StaticIP, + StaticIPv6: cc.StaticIPv6, + IPFamily: cc.KubernetesConfig.IPFamily, ListenAddress: cc.ListenAddress, GPUs: cc.GPUs, }), nil diff --git a/pkg/util/constants.go b/pkg/util/constants.go index db1a39c0568e..7ec2d5b9a820 100644 --- a/pkg/util/constants.go +++ b/pkg/util/constants.go @@ -17,9 +17,12 @@ limitations under the License. package util import ( + "fmt" "net" + "strings" "github.com/pkg/errors" + mkconstants "k8s.io/minikube/pkg/minikube/constants" ) // DefaultAdmissionControllers are admission controllers we default to @@ -35,29 +38,110 @@ var DefaultAdmissionControllers = []string{ "ResourceQuota", } -// ServiceClusterIP returns the first IP of the ServiceCIDR +// ServiceClusterIP returns a "well-known" Service IP from the Service CIDR. +// +// - If serviceCIDR is empty, DefaultServiceCIDR is used. +// - If multiple CIDRs are provided (comma-separated), IPv4 is preferred for +// backward compatibility; otherwise the first non-empty CIDR is used. +// - For IPv4, we return the first usable IP after the network address. +// - For IPv6, we return the first IP in the subnet (or +1 on the last byte – +// the exact value is not important as long as it's inside the CIDR). func ServiceClusterIP(serviceCIDR string) (net.IP, error) { - ip, _, err := net.ParseCIDR(serviceCIDR) + if serviceCIDR == "" { + serviceCIDR = mkconstants.DefaultServiceCIDR + } + + cidr := pickPrimaryServiceCIDR(serviceCIDR) + cidr = strings.TrimSpace(cidr) + if cidr == "" { + return nil, fmt.Errorf("empty service CIDR") + } + + _, ipNet, err := net.ParseCIDR(cidr) if err != nil { - return nil, errors.Wrap(err, "parsing default service cidr") + return nil, err } - ip = ip.To4() - ip[3]++ + + // Copy to avoid mutating ipNet.IP + ip := make(net.IP, len(ipNet.IP)) + copy(ip, ipNet.IP) + + // IPv4: bump last octet (avoid the network address itself). + if v4 := ip.To4(); v4 != nil { + v4[3]++ + return v4, nil + } + + // IPv6: just bump the last byte – stays within the subnet for normal masks. + if len(ip) == 0 { + return nil, fmt.Errorf("parsed service CIDR %q has empty IP", cidr) + } + ip[len(ip)-1]++ return ip, nil } -// DNSIP returns x.x.x.10 of the service CIDR +// pickPrimaryServiceCIDR chooses which CIDR to use when a comma-separated list +// is provided. It prefers IPv4 when present, otherwise the first non-empty part. +func pickPrimaryServiceCIDR(serviceCIDR string) string { + parts := strings.Split(serviceCIDR, ",") + if len(parts) == 1 { + return strings.TrimSpace(parts[0]) + } + + var firstNonEmpty string + var firstV4 string + + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + if firstNonEmpty == "" { + firstNonEmpty = p + } + if strings.Contains(p, ".") && firstV4 == "" { + firstV4 = p + } + } + + if firstV4 != "" { + return firstV4 + } + return firstNonEmpty +} + func DNSIP(serviceCIDR string) (net.IP, error) { - ip, _, err := net.ParseCIDR(serviceCIDR) + base, _, err := net.ParseCIDR(serviceCIDR) if err != nil { return nil, errors.Wrap(err, "parsing default service cidr") } - ip = ip.To4() - ip[3] = 10 - return ip, nil + ip := normalizeIP(base) + return addToIP(ip, 10), nil } // AlternateDNS returns a list of alternate names for a domain func AlternateDNS(domain string) []string { return []string{"kubernetes.default.svc." + domain, "kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"} } + +// normalizeIP returns a 4-byte slice for v4 or 16-byte slice for v6. +func normalizeIP(ip net.IP) net.IP { + if v4 := ip.To4(); v4 != nil { + return v4 + } + return ip.To16() +} + +// addToIP returns ip + n, preserving length (v4/v6) with carry. +func addToIP(ip net.IP, n uint64) net.IP { + out := make(net.IP, len(ip)) + copy(out, ip) + i := len(out) - 1 + for n > 0 && i >= 0 { + sum := uint64(out[i]) + (n & 0xff) + out[i] = byte(sum & 0xff) + n = (n >> 8) + (sum >> 8) + i-- + } + return out +}