From 7b7c86fc15573ed58c03393bba0d6e0f6f27fb24 Mon Sep 17 00:00:00 2001 From: Boris Litvinenko Date: Thu, 16 Jan 2025 08:39:30 +0300 Subject: [PATCH] fix issues --- contrib/terraform/terraform.py | 1 + .../group_vars/k8s_cluster/k8s-cluster.yml | 389 ------------------ .../templates/cri-dockerd.service.j2 | 2 +- .../control-plane/tasks/kubeadm-setup.yml | 5 +- .../templates/kubeadm-config.v1beta3.yaml.j2 | 14 +- .../templates/kubeadm-config.v1beta4.yaml.j2 | 14 +- .../kubespray-defaults/defaults/main/main.yml | 36 +- roles/kubespray-defaults/tasks/no_proxy.yml | 2 +- roles/network_plugin/calico/tasks/install.yml | 2 +- .../kube-ovn/templates/cni-kube-ovn.yml.j2 | 6 +- 10 files changed, 55 insertions(+), 416 deletions(-) delete mode 100644 inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml diff --git a/contrib/terraform/terraform.py b/contrib/terraform/terraform.py index c22eb9f41c8..9f6132711ed 100755 --- a/contrib/terraform/terraform.py +++ b/contrib/terraform/terraform.py @@ -273,6 +273,7 @@ def openstack_host(resource, module_name): 'access_ip_v4': raw_attrs['access_ip_v4'], 'access_ip_v6': raw_attrs['access_ip_v6'], 'access_ip': raw_attrs['access_ip_v4'], + 'access_ip6': raw_attrs['access_ip_v6'], 'ip': raw_attrs['network.0.fixed_ip_v4'], 'flavor': parse_dict(raw_attrs, 'flavor', sep='_'), diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml deleted file mode 100644 index 0195a4943a7..00000000000 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ /dev/null @@ -1,389 +0,0 @@ ---- -# Kubernetes configuration dirs and system namespace. -# Those are where all the additional config stuff goes -# the kubernetes normally puts in /srv/kubernetes. -# This puts them in a sane location and namespace. -# Editing those values will almost surely break something. -kube_config_dir: /etc/kubernetes -kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" -kube_manifest_dir: "{{ kube_config_dir }}/manifests" - -# This is where all the cert scripts and certs will be located -kube_cert_dir: "{{ kube_config_dir }}/ssl" - -# This is where all of the bearer tokens will be stored -kube_token_dir: "{{ kube_config_dir }}/tokens" - -kube_api_anonymous_auth: true - -## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.31.3 - -# Where the binaries will be downloaded. -# Note: ensure that you've enough disk space (about 1G) -local_release_dir: "/tmp/releases" -# Random shifts for retrying failed ops like pushing/downloading -retry_stagger: 5 - -# This is the user that owns tha cluster installation. -kube_owner: kube - -# This is the group that the cert creation scripts chgrp the -# cert files to. Not really changeable... -kube_cert_group: kube-cert - -# Cluster Loglevel configuration -kube_log_level: 2 - -# Directory where credentials will be stored -credentials_dir: "{{ inventory_dir }}/credentials" - -## It is possible to activate / deactivate selected authentication methods (oidc, static token auth) -# kube_oidc_auth: false -# kube_token_auth: false - - -## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/ -## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...) - -# kube_oidc_url: https:// ... -# kube_oidc_client_id: kubernetes -## Optional settings for OIDC -# kube_oidc_ca_file: "{{ kube_cert_dir }}/ca.pem" -# kube_oidc_username_claim: sub -# kube_oidc_username_prefix: 'oidc:' -# kube_oidc_groups_claim: groups -# kube_oidc_groups_prefix: 'oidc:' - -## Variables to control webhook authn/authz -# kube_webhook_token_auth: false -# kube_webhook_token_auth_url: https://... -# kube_webhook_token_auth_url_skip_tls_verify: false - -## For webhook authorization, authorization_modes must include Webhook -# kube_webhook_authorization: false -# kube_webhook_authorization_url: https://... -# kube_webhook_authorization_url_skip_tls_verify: false - -# Choose network plugin (cilium, calico, kube-ovn, weave or flannel. Use cni for generic cni plugin) -# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing -kube_network_plugin: calico - -# Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni -kube_network_plugin_multus: false - -# Kubernetes internal network for services, unused block of space. -kube_service_addresses: 10.233.0.0/18 - -# internal network. When used, it will assign IP -# addresses from this range to individual pods. -# This network must be unused in your network infrastructure! -kube_pods_subnet: 10.233.64.0/18 - -# internal network node size allocation (optional). This is the size allocated -# to each node for pod IP address allocation. Note that the number of pods per node is -# also limited by the kubelet_max_pods variable which defaults to 110. -# -# Example: -# Up to 64 nodes and up to 254 or kubelet_max_pods (the lowest of the two) pods per node: -# - kube_pods_subnet: 10.233.64.0/18 -# - kube_network_node_prefix: 24 -# - kubelet_max_pods: 110 -# -# Example: -# Up to 128 nodes and up to 126 or kubelet_max_pods (the lowest of the two) pods per node: -# - kube_pods_subnet: 10.233.64.0/18 -# - kube_network_node_prefix: 25 -# - kubelet_max_pods: 110 -kube_network_node_prefix: 24 - -# Configure Dual Stack networking (i.e. both IPv4 and IPv6) -enable_dual_stack_networks: false - -# Configure IPv6 only -enable_ipv6only_stack_networks: false - -# Kubernetes internal network for IPv6 services, unused block of space. -# This is used if enable_dual_stack_networks or enable_ipv6only_stack_networks is set to true -# This provides 4096 IPv6 IPs -kube_service_addresses_ipv6: fd85:ee78:d8a6:8607::1000/116 - -# Internal network. When used, it will assign IPv6 addresses from this range to individual pods. -# This network must not already be in your network infrastructure! -# This is used if enable_dual_stack_networks or enable_ipv6only_stack_networks is set to true -# This provides room for 256 nodes with 254 pods per node. -kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112 - -# IPv6 subnet size allocated to each for pods. -# This is used if enable_dual_stack_networks or enable_ipv6only_stack_networks is set to true -# This provides room for 254 pods per node. -kube_network_node_prefix_ipv6: 120 - -# The port the API Server will be listening on. -kube_apiserver_ip: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}" -kube_apiserver_port: 6443 # (https) - -# Kube-proxy proxyMode configuration. -# Can be ipvs, iptables -kube_proxy_mode: ipvs - -# configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface -# must be set to true for MetalLB, kube-vip(ARP enabled) to work -kube_proxy_strict_arp: false - -# A string slice of values which specify the addresses to use for NodePorts. -# Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32). -# The default empty string slice ([]) means to use all local addresses. -# kube_proxy_nodeport_addresses_cidr is retained for legacy config -kube_proxy_nodeport_addresses: >- - {%- if kube_proxy_nodeport_addresses_cidr is defined -%} - [{{ kube_proxy_nodeport_addresses_cidr }}] - {%- else -%} - [] - {%- endif -%} - -# If non-empty, will use this string as identification instead of the actual hostname -# kube_override_hostname: {{ inventory_hostname }} - -## Encrypting Secret Data at Rest -kube_encrypt_secret_data: false - -# Graceful Node Shutdown (Kubernetes >= 1.21.0), see https://kubernetes.io/blog/2021/04/21/graceful-node-shutdown-beta/ -# kubelet_shutdown_grace_period had to be greater than kubelet_shutdown_grace_period_critical_pods to allow -# non-critical podsa to also terminate gracefully -# kubelet_shutdown_grace_period: 60s -# kubelet_shutdown_grace_period_critical_pods: 20s - -# DNS configuration. -# Kubernetes cluster name, also will be used as DNS domain -cluster_name: cluster.local -# Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods -ndots: 2 -# dns_timeout: 2 -# dns_attempts: 2 -# Custom search domains to be added in addition to the default cluster search domains -# searchdomains: -# - svc.{{ cluster_name }} -# - default.svc.{{ cluster_name }} -# Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``). -# remove_default_searchdomains: false -# Can be coredns, coredns_dual, manual or none -dns_mode: coredns -# Set manual server if using a custom cluster DNS server -# manual_dns_server: 10.x.x.x -# Enable nodelocal dns cache -enable_nodelocaldns: true -enable_nodelocaldns_secondary: false -nodelocaldns_ip: 169.254.25.10 -nodelocaldns_health_port: 9254 -nodelocaldns_second_health_port: 9256 -nodelocaldns_bind_metrics_host_ip: false -nodelocaldns_secondary_skew_seconds: 5 -# nodelocaldns_external_zones: -# - zones: -# - example.com -# - example.io:1053 -# nameservers: -# - 1.1.1.1 -# - 2.2.2.2 -# cache: 5 -# - zones: -# - https://mycompany.local:4453 -# nameservers: -# - 192.168.0.53 -# cache: 0 -# - zones: -# - mydomain.tld -# nameservers: -# - 10.233.0.3 -# cache: 5 -# rewrite: -# - name website.tld website.namespace.svc.cluster.local -# Enable k8s_external plugin for CoreDNS -enable_coredns_k8s_external: false -coredns_k8s_external_zone: k8s_external.local -# Enable endpoint_pod_names option for kubernetes plugin -enable_coredns_k8s_endpoint_pod_names: false -# Set forward options for upstream DNS servers in coredns (and nodelocaldns) config -# dns_upstream_forward_extra_opts: -# policy: sequential -# Apply extra options to coredns kubernetes plugin -# coredns_kubernetes_extra_opts: -# - 'fallthrough example.local' -# Forward extra domains to the coredns kubernetes plugin -# coredns_kubernetes_extra_domains: '' - -# Can be docker_dns, host_resolvconf or none -resolvconf_mode: host_resolvconf -# Deploy netchecker app to verify DNS resolve as an HTTP service -deploy_netchecker: false -# Ip address of the kubernetes skydns service -skydns_server: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}" -skydns_server_secondary: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}" -dns_domain: "{{ cluster_name }}" - -## Container runtime -## docker for docker, crio for cri-o and containerd for containerd. -## Default: containerd -container_manager: containerd - -# Additional container runtimes -kata_containers_enabled: false - -kubeadm_certificate_key: "{{ lookup('password', credentials_dir + '/kubeadm_certificate_key.creds length=64 chars=hexdigits') | lower }}" - -# K8s image pull policy (imagePullPolicy) -k8s_image_pull_policy: IfNotPresent - -# audit log for kubernetes -kubernetes_audit: false - -# define kubelet config dir for dynamic kubelet -# kubelet_config_dir: -default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir" - -# Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts -# kubeconfig_localhost: false -# Use ansible_host as external api ip when copying over kubeconfig. -# kubeconfig_localhost_ansible_host: false -# Download kubectl onto the host that runs Ansible in {{ bin_dir }} -# kubectl_localhost: false - -# A comma separated list of levels of node allocatable enforcement to be enforced by kubelet. -# Acceptable options are 'pods', 'system-reserved', 'kube-reserved' and ''. Default is "". -# kubelet_enforce_node_allocatable: pods - -## Set runtime and kubelet cgroups when using systemd as cgroup driver (default) -# kubelet_runtime_cgroups: "/{{ kube_service_cgroups }}/{{ container_manager }}.service" -# kubelet_kubelet_cgroups: "/{{ kube_service_cgroups }}/kubelet.service" - -## Set runtime and kubelet cgroups when using cgroupfs as cgroup driver -# kubelet_runtime_cgroups_cgroupfs: "/system.slice/{{ container_manager }}.service" -# kubelet_kubelet_cgroups_cgroupfs: "/system.slice/kubelet.service" - -# Whether to run kubelet and container-engine daemons in a dedicated cgroup. -# kube_reserved: false -## Uncomment to override default values -## The following two items need to be set when kube_reserved is true -# kube_reserved_cgroups_for_service_slice: kube.slice -# kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}" -# kube_memory_reserved: 256Mi -# kube_cpu_reserved: 100m -# kube_ephemeral_storage_reserved: 2Gi -# kube_pid_reserved: "1000" -# Reservation for control plane hosts -# kube_master_memory_reserved: 512Mi -# kube_master_cpu_reserved: 200m -# kube_master_ephemeral_storage_reserved: 2Gi -# kube_master_pid_reserved: "1000" - -## Optionally reserve resources for OS system daemons. -# system_reserved: true -## Uncomment to override default values -## The following two items need to be set when system_reserved is true -# system_reserved_cgroups_for_service_slice: system.slice -# system_reserved_cgroups: "/{{ system_reserved_cgroups_for_service_slice }}" -# system_memory_reserved: 512Mi -# system_cpu_reserved: 500m -# system_ephemeral_storage_reserved: 2Gi -## Reservation for master hosts -# system_master_memory_reserved: 256Mi -# system_master_cpu_reserved: 250m -# system_master_ephemeral_storage_reserved: 2Gi - -## Eviction Thresholds to avoid system OOMs -# https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds -# eviction_hard: {} -# eviction_hard_control_plane: {} - -# An alternative flexvolume plugin directory -# kubelet_flexvolumes_plugins_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec - -## Supplementary addresses that can be added in kubernetes ssl keys. -## That can be useful for example to setup a keepalived virtual IP -# supplementary_addresses_in_ssl_keys: [10.0.0.1, 10.0.0.2, 10.0.0.3] - -## Running on top of openstack vms with cinder enabled may lead to unschedulable pods due to NoVolumeZoneConflict restriction in kube-scheduler. -## See https://github.com/kubernetes-sigs/kubespray/issues/2141 -## Set this variable to true to get rid of this issue -volume_cross_zone_attachment: false -## Add Persistent Volumes Storage Class for corresponding cloud provider (supported: in-tree OpenStack, Cinder CSI, -## AWS EBS CSI, Azure Disk CSI, GCP Persistent Disk CSI) -persistent_volumes_enabled: false - -## Container Engine Acceleration -## Enable container acceleration feature, for example use gpu acceleration in containers -# nvidia_accelerator_enabled: true -## Nvidia GPU driver install. Install will by done by a (init) pod running as a daemonset. -## Important: if you use Ubuntu then you should set in all.yml 'docker_storage_options: -s overlay2' -## Array with nvida_gpu_nodes, leave empty or comment if you don't want to install drivers. -## Labels and taints won't be set to nodes if they are not in the array. -# nvidia_gpu_nodes: -# - kube-gpu-001 -# nvidia_driver_version: "384.111" -## flavor can be tesla or gtx -# nvidia_gpu_flavor: gtx -## NVIDIA driver installer images. Change them if you have trouble accessing gcr.io. -# nvidia_driver_install_centos_container: atzedevries/nvidia-centos-driver-installer:2 -# nvidia_driver_install_ubuntu_container: gcr.io/google-containers/ubuntu-nvidia-driver-installer@sha256:7df76a0f0a17294e86f691c81de6bbb7c04a1b4b3d4ea4e7e2cccdc42e1f6d63 -## NVIDIA GPU device plugin image. -# nvidia_gpu_device_plugin_container: "registry.k8s.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e" - -## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. -# tls_min_version: "" - -## Support tls cipher suites. -# tls_cipher_suites: {} -# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA -# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 -# - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 -# - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA -# - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 -# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 -# - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA -# - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA -# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA -# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 -# - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 -# - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA -# - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 -# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 -# - TLS_ECDHE_RSA_WITH_RC4_128_SHA -# - TLS_RSA_WITH_3DES_EDE_CBC_SHA -# - TLS_RSA_WITH_AES_128_CBC_SHA -# - TLS_RSA_WITH_AES_128_CBC_SHA256 -# - TLS_RSA_WITH_AES_128_GCM_SHA256 -# - TLS_RSA_WITH_AES_256_CBC_SHA -# - TLS_RSA_WITH_AES_256_GCM_SHA384 -# - TLS_RSA_WITH_RC4_128_SHA - -## Amount of time to retain events. (default 1h0m0s) -event_ttl_duration: "1h0m0s" - -## Automatically renew K8S control plane certificates on first Monday of each month -auto_renew_certificates: false -# First Monday of each month -# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube_control_plane'].index(inventory_hostname) }}0:00" - -kubeadm_patches_dir: "{{ kube_config_dir }}/patches" -kubeadm_patches: [] -# See https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches -# Correspondance with this link -# patchtype = type -# target = target -# suffix -> managed automatically -# extension -> always "yaml" -# kubeadm_patches: -# - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration -# type: strategic(default)|json|merge -# patch: -# metadata: -# annotations: -# example.com/test: "true" -# labels: -# example.com/prod_level: "{{ prod_level }}" -# - ... -# Patches are applied in the order they are specified. - -# Set to true to remove the role binding to anonymous users created by kubeadm -remove_anonymous_access: false diff --git a/roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2 b/roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2 index 79950fb2bb1..c5158bd94c4 100644 --- a/roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2 +++ b/roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2 @@ -7,7 +7,7 @@ Requires=cri-dockerd.socket [Service] Type=notify -ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint {{ cri_socket }} --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_version }} --log-level {{ cri_dockerd_log_level }} {% if enable_dual_stack_networks %}--ipv6-dual-stack=True{% endif %} +ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint {{ cri_socket }} --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ kube_pods_subnet_range }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_version }} --log-level {{ cri_dockerd_log_level }} {% if enable_dual_stack_networks %}--ipv6-dual-stack=True{% endif %} ExecReload=/bin/kill -s HUP $MAINPID TimeoutSec=0 diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml index dd13038d649..84f8038cc05 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml @@ -25,16 +25,17 @@ - name: Kubeadm | aggregate all SANs set_fact: - apiserver_sans: "{{ (sans_base + groups['kube_control_plane'] + sans_apiserver_ip + sans_lb + sans_lb_ip + sans_supp + sans_access_ip + sans_ip + sans_address + sans_override + sans_hostname + sans_fqdn + sans_kube_vip_address) | unique }}" + apiserver_sans: "{{ (sans_base + groups['kube_control_plane'] + sans_lb + sans_lb_ip + sans_supp + sans_access_ip + sans_ip + sans_address + sans_override + sans_hostname + sans_fqdn + sans_kube_vip_address) | unique }}" vars: sans_base: - "kubernetes" - "kubernetes.default" - "kubernetes.default.svc" - "kubernetes.default.svc.{{ dns_domain }}" + - "{{ kube_apiserver_ip }}" - "localhost" - "127.0.0.1" - sans_apiserver_ip: "{{ [kube_apiserver_ip] if not enable_ipv6only_stack_networks else [] }}" + - "::1" sans_lb: "{{ [apiserver_loadbalancer_domain_name] if apiserver_loadbalancer_domain_name is defined else [] }}" sans_lb_ip: "{{ [loadbalancer_apiserver.address] if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined else [] }}" sans_supp: "{{ supplementary_addresses_in_ssl_keys if supplementary_addresses_in_ssl_keys is defined else [] }}" diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 index 7d29ff0c8ed..3ff240b4268 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 @@ -94,9 +94,9 @@ dns: imageTag: {{ coredns_image_tag }} networking: dnsDomain: {{ dns_domain }} - serviceSubnet: "{{ [kube_service_addresses if not enable_ipv6only_stack_networks, kube_service_addresses_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}" + serviceSubnet: "{{ kube_service_addresses_range }}" {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %} - podSubnet: "{{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}" + podSubnet: "{{ kube_pods_subnet_range }}" {% endif %} {% if kubeadm_feature_gates %} featureGates: @@ -143,7 +143,7 @@ apiServer: etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}" {% endif %} service-node-port-range: {{ kube_apiserver_node_port_range }} - service-cluster-ip-range: "{{ [kube_service_addresses if not enable_ipv6only_stack_networks, kube_service_addresses_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}" + service-cluster-ip-range: "{{ kube_service_addresses_range }}" kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" profiling: "{{ kube_profiling }}" request-timeout: "{{ kube_apiserver_request_timeout }}" @@ -285,7 +285,7 @@ apiServer: {% endif %} certSANs: {% for san in apiserver_sans %} - - "{{ san }}" + - {{ san }} {% endfor %} timeoutForControlPlane: 5m0s controllerManager: @@ -293,9 +293,9 @@ controllerManager: node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} node-monitor-period: {{ kube_controller_node_monitor_period }} {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %} - cluster-cidr: "{{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}" + cluster-cidr: "{{ kube_pods_subnet_range }}" {% endif %} - service-cluster-ip-range: "{{ [kube_service_addresses if not enable_ipv6only_stack_networks, kube_service_addresses_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}" + service-cluster-ip-range: "{{ kube_service_addresses_range }}" {% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %} allocate-node-cidrs: "false" {% elif enable_ipv6only_stack_networks %} @@ -383,7 +383,7 @@ clientConnection: kubeconfig: {{ kube_proxy_client_kubeconfig }} qps: {{ kube_proxy_client_qps }} {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %} -clusterCIDR: "{{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}" +clusterCIDR: "{{ kube_pods_subnet_range }}" {% endif %} configSyncPeriod: {{ kube_proxy_config_sync_period }} conntrack: diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 index 02b0fe6c1e7..300094ab9e6 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 @@ -106,9 +106,9 @@ dns: imageTag: {{ coredns_image_tag }} networking: dnsDomain: {{ dns_domain }} - serviceSubnet: "{{ [kube_service_addresses if not enable_ipv6only_stack_networks, kube_service_addresses_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}" + serviceSubnet: "{{ kube_service_addresses_range }}" {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %} - podSubnet: "{{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}" + podSubnet: "{{ kube_pods_subnet_range }}" {% endif %} {% if kubeadm_feature_gates %} featureGates: @@ -169,7 +169,7 @@ apiServer: - name: service-node-port-range value: "{{ kube_apiserver_node_port_range }}" - name: service-cluster-ip-range - value: "{{ [kube_service_addresses if not enable_ipv6only_stack_networks, kube_service_addresses_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}" + value: "{{ kube_service_addresses_range }}" - name: kubelet-preferred-address-types value: "{{ kubelet_preferred_address_types }}" - name: profiling @@ -341,7 +341,7 @@ apiServer: {% endif %} certSANs: {% for san in apiserver_sans %} - - "{{ san }}" + - {{ san }} {% endfor %} controllerManager: extraArgs: @@ -351,10 +351,10 @@ controllerManager: value: "{{ kube_controller_node_monitor_period }}" {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %} - name: cluster-cidr - value: "{{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}" + value: "{{ kube_pods_subnet_range }}" {% endif %} - name: service-cluster-ip-range - value: "{{ [kube_service_addresses if not enable_ipv6only_stack_networks, kube_service_addresses_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}" + value: "{{ kube_service_addresses_range }}" {% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %} - name: allocate-node-cidrs value: "false" @@ -479,7 +479,7 @@ clientConnection: kubeconfig: {{ kube_proxy_client_kubeconfig }} qps: {{ kube_proxy_client_qps }} {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %} -clusterCIDR: "{{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}" +clusterCIDR: "{{ kube_pods_subnet_range }}" {% endif %} configSyncPeriod: {{ kube_proxy_config_sync_period }} conntrack: diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index f557e422da6..9e2fcddd4a8 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -131,8 +131,8 @@ resolvconf_mode: host_resolvconf # Deploy netchecker app to verify DNS resolve as an HTTP service deploy_netchecker: false # Ip address of the kubernetes DNS service (called skydns for historical reasons) -skydns_server: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}" -skydns_server_secondary: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}" +skydns_server: "{{ (kube_service_addresses_ipv6 if enable_ipv6only_stack_networks else kube_service_addresses) | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}" +skydns_server_secondary: "{{ (kube_service_addresses_ipv6 if enable_ipv6only_stack_networks else kube_service_addresses) | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}" dns_domain: "{{ cluster_name }}" docker_dns_search_domains: - 'default.svc.{{ dns_domain }}' @@ -246,11 +246,37 @@ kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112 # This provides room for 254 pods per node. kube_network_node_prefix_ipv6: 120 +# Configure all of service addresses in one variable. +# Merge all of different stack. +kube_service_addresses_range: >- + {%- if enable_ipv6only_stack_networks -%} + {{ kube_service_addresses_ipv6 }} + {%- elif enable_dual_stack_networks -%} + {{ kube_service_addresses }},{{ kube_service_addresses_ipv6 }} + {%- else -%} + {{ kube_service_addresses }} + {%- endif -%} + +# Configure all of pods subnets in one variable. +# Merge all of different stack. +kube_pods_subnet_range: >- + {%- if enable_ipv6only_stack_networks -%} + {{ kube_pods_subnet_ipv6 }} + {%- elif enable_dual_stack_networks -%} + {{ kube_pods_subnet }},{{ kube_pods_subnet_ipv6 }} + {%- else -%} + {{ kube_pods_subnet }} + {%- endif -%} + +# Configure variable with default stack for ip. +# IPv6 get more priority only with enable_ipv6only_stack_networks options. +default_net_mode: "{{ '6' if enable_ipv6only_stack_networks else '' }}" + # The virtual cluster IP, real host IPs and ports the API Server will be # listening on. # NOTE: loadbalancer_apiserver_localhost somewhat alters the final API enpdoint # access IP value (automatically evaluated below) -kube_apiserver_ip: "{{ kube_service_addresses_ipv6 if enable_ipv6only_stack_networks else kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}" +kube_apiserver_ip: "{{ (kube_service_addresses_ipv6 if enable_ipv6only_stack_networks else kube_service_addresses) | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}" # NOTE: If you specific address/interface and use loadbalancer_apiserver_localhost # loadbalancer_apiserver_localhost (nginx/haproxy) will deploy on control plane nodes on 127.0.0.1:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} too. @@ -551,9 +577,9 @@ ssl_ca_dirs: |- # Vars for pointing to kubernetes api endpoints kube_apiserver_count: "{{ groups['kube_control_plane'] | length }}" -kube_apiserver_address: "{{ ip6 | default(hostvars[inventory_hostname]['fallback_ip6']) if enable_ipv6only_stack_networks else ip | default(hostvars[inventory_hostname]['fallback_ip']) }}" +kube_apiserver_address: "{{ vars['ip' + default_net_mode] | default(hostvars[inventory_hostname]['fallback_ip' + default_net_mode]) }}" kube_apiserver_access_address: "{{ access_ip | default(kube_apiserver_address) }}" -first_kube_control_plane_address: "{{ hostvars[groups['kube_control_plane'][0]]['access_ip'] | default(hostvars[groups['kube_control_plane'][0]]['ip'] | default(hostvars[groups['kube_control_plane'][0]]['fallback_ip'])) if not enable_ipv6only_stack_networks else hostvars[groups['kube_control_plane'][0]]['access_ip_v6'] | default(hostvars[groups['kube_control_plane'][0]]['ip6'] | default(hostvars[groups['kube_control_plane'][0]]['fallback_ip6'])) }}" +first_kube_control_plane_address: "{{ hostvars[groups['kube_control_plane'][0]]['access_ip' + default_net_mode] | default(hostvars[groups['kube_control_plane'][0]]['ip' + default_net_mode] | default(hostvars[groups['kube_control_plane'][0]]['fallback_ip' + default_net_mode])) }}" loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}" loadbalancer_apiserver_type: "nginx" # applied if only external loadbalancer_apiserver is defined, otherwise ignored diff --git a/roles/kubespray-defaults/tasks/no_proxy.yml b/roles/kubespray-defaults/tasks/no_proxy.yml index 4aa85f7f232..e2430b032bf 100644 --- a/roles/kubespray-defaults/tasks/no_proxy.yml +++ b/roles/kubespray-defaults/tasks/no_proxy.yml @@ -23,7 +23,7 @@ {%- if additional_no_proxy is defined -%} {{ additional_no_proxy }}, {%- endif -%} - 127.0.0.1,localhost,{{ kube_service_addresses }},{{ kube_pods_subnet }},svc,svc.{{ dns_domain }} + 127.0.0.1,::1,localhost,{{ kube_service_addresses_range }},{{ kube_pods_subnet }},svc,svc.{{ dns_domain }} delegate_to: localhost connection: local delegate_facts: true diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index c7cdf342f07..b2beb3f0a80 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -350,7 +350,7 @@ {% if not calico_no_global_as_num | default(false) %}"asNumber": {{ global_as_num }},{% endif %} "nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled | default('true') }} , {% if calico_advertise_cluster_ips | default(false) %} - "serviceClusterIPs": [{{ ['{"cidr": "' + kube_service_addresses + '"}' if not enable_ipv6only_stack_networks, '{"cidr": "' + kube_service_addresses_ipv6 + '"}' if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}],{% endif %} + "serviceClusterIPs": [{% for cidr in kube_service_addresses_range.split(",") %}{{ "," if not loop.first }}{"cidr": "{{ cidr }}"}{% endfor %}],{% endif %} {% if calico_advertise_service_loadbalancer_ips | length > 0 %}"serviceLoadBalancerIPs": {{ _service_loadbalancer_ips }},{% endif %} "serviceExternalIPs": {{ _service_external_ips | default([]) }} } diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 index 9f85e0db405..9dce9fd9d47 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 @@ -240,14 +240,14 @@ spec: imagePullPolicy: {{ k8s_image_pull_policy }} args: - /kube-ovn/start-controller.sh - - --default-cidr={{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }} + - --default-cidr={{ kube_pods_subnet_range }} - --default-gateway={% if kube_ovn_default_gateway is defined %}{{ kube_ovn_default_gateway }}{% endif %}{{ '' }} - --default-gateway-check={{ kube_ovn_default_gateway_check | string }} - --default-logical-gateway={{ kube_ovn_default_logical_gateway | string }} - --default-u2o-interconnection={{ kube_ovn_u2o_interconnection }} - --default-exclude-ips={% if kube_ovn_default_exclude_ips is defined %}{{ kube_ovn_default_exclude_ips }}{% endif %}{{ '' }} - --node-switch-cidr={{ [kube_ovn_node_switch_cidr if not enable_ipv6only_stack_networks, kube_ovn_node_switch_cidr_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }} - - --service-cluster-ip-range={{ [kube_service_addresses if not enable_ipv6only_stack_networks, kube_service_addresses_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }} + - --service-cluster-ip-range={{ kube_service_addresses_range }} - --network-type={{ kube_ovn_network_type }} - --default-interface-name={{ kube_ovn_default_interface_name | default('') }} - --default-vlan-id={{ kube_ovn_default_vlan_id }} @@ -403,7 +403,7 @@ spec: args: - --enable-mirror={{ kube_ovn_traffic_mirror | lower }} - --encap-checksum={{ kube_ovn_encap_checksum | lower }} - - --service-cluster-ip-range={{ [kube_service_addresses if not enable_ipv6only_stack_networks, kube_service_addresses_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }} + - --service-cluster-ip-range={{ kube_service_addresses_range }} - --iface={{ kube_ovn_iface | default('') }} - --dpdk-tunnel-iface={{ kube_ovn_dpdk_tunnel_iface }} - --network-type={{ kube_ovn_network_type }}