feat: allow customization of flannel port

.github/workflows/main.yaml

@@ -16,6 +16,8 @@ jobs:
     with:
       # Defaults to 6443
       kube_apiserver_port: "8080"
+      # Defaults to 8472
+      flannel_port: "9072"
       # Defaults to 10250
       kubelet_port: "20250"
       # Defaults to 2379
@@ -32,6 +34,8 @@ jobs:
     with:
       # Defaults to 6443
       kube_apiserver_port: "8080"
+      # Defaults to 8472
+      flannel_port: "9072"
       # Defaults to 10250
       kubelet_port: "20250"
       # Defaults to 2379

.github/workflows/reusable-multi-node.yaml

@@ -7,6 +7,10 @@ on:
         description: kubelet serving port
         type: string
         default: "10250"
+      flannel_port:
+        description: flannel vxlan port
+        type: string
+        default: "8472"
       etcd_port:
         description: etcd service port
         type: string
@@ -42,6 +46,7 @@ jobs:
       LIMA_TEMPLATE: "${{ matrix.lima-template }}"
       CONTAINER_ENGINE: "${{ matrix.engine }}"
       U7S_PORT_KUBE_APISERVER: "${{ inputs.kube_apiserver_port }}"
+      U7S_PORT_FLANNEL: "${{ inputs.flannel_port }}"
       U7S_PORT_KUBELET: "${{ inputs.kubelet_port }}"
       U7S_PORT_ETCD: "${{ inputs.etcd_port }}"
       DOCKER_BUILDKIT: 1

.github/workflows/reusable-single-node.yaml

@@ -6,6 +6,10 @@ on:
         description: kubelet serving port
         type: string
         default: "10250"
+      flannel_port:
+        description: flannel vxlan port
+        type: string
+        default: "8472"
       etcd_port:
         description: etcd service port
         type: string
@@ -32,6 +36,7 @@ jobs:
     env:
       CONTAINER_ENGINE: "${{ matrix.engine }}"
       U7S_PORT_KUBE_APISERVER: "${{ inputs.kube_apiserver_port }}"
+      U7S_PORT_FLANNEL: "${{ inputs.flannel_port }}"
       U7S_PORT_KUBELET: "${{ inputs.kubelet_port }}"
       U7S_PORT_ETCD: "${{ inputs.etcd_port }}"
       DOCKER_BUILDKIT: 1

Dockerfile

@@ -9,14 +9,17 @@ RUN arch="$(uname -m | sed -e s/x86_64/amd64/ -e s/aarch64/arm64/)" && \
   grep "${fname}" "/tmp/SHA256SUMS.d/cni-plugins-${CNI_PLUGINS_VERSION}" | sha256sum -c && \
   mkdir -p /opt/cni/bin && \
   tar xzf "${fname}" -C /opt/cni/bin && \
-  rm -f "${fname}"
+  rm -f "${fname}" && \
+  curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \
+  chmod 700 get_helm.sh && ./get_helm.sh
 # gettext-base: for `envsubst`
 # moreutils: for `sponge`
 # socat: for `socat` (to silence "[WARNING FileExisting-socat]" from kubeadm)
 RUN apt-get update && apt-get install -y --no-install-recommends \
   gettext-base \
   moreutils \
-  socat
+  socat \
+  git
 ADD Dockerfile.d/etc_udev_rules.d_90-flannel.rules /etc/udev/rules.d/90-flannel.rules
 ADD Dockerfile.d/u7s-entrypoint.sh /
 ENTRYPOINT ["/u7s-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init"]

Makefile

@@ -8,7 +8,7 @@ export HOSTNAME := $(HOSTNAME)
 # Change ports for different kubernetes services
 export U7S_PORT_ETCD ?= 2379
 export U7S_PORT_KUBELET ?= 10250
-export U7S_PORT_FLANNEL := 8472
+export U7S_PORT_FLANNEL ?= 8472
 export U7S_PORT_KUBE_APISERVER ?= 6443
 
 HOST_IP ?= $(shell ip --json route get 1 | jq -r .[0].prefsrc)
@@ -41,6 +41,7 @@ NODE_SHELL := $(COMPOSE) exec \
 	-e U7S_NODE_SUBNET=$(U7S_NODE_SUBNET) \
 	-e U7S_NODE_IP=$(U7S_NODE_IP) \
 	-e U7S_PORT_KUBE_APISERVER=$(U7S_PORT_KUBE_APISERVER) \
+        -e U7S_PORT_FLANNEL=$(U7S_PORT_FLANNEL) \
         -e U7S_PORT_KUBELET=$(U7S_PORT_KUBELET) \
         -e U7S_PORT_ETCD=$(U7S_PORT_ETCD) \
 	$(NODE_SERVICE_NAME)
@@ -157,4 +158,4 @@ kubeadm-reset:
 
 .PHONY: install-flannel
 install-flannel:
-	$(NODE_SHELL) kubectl apply -f https://github.com/flannel-io/flannel/releases/download/v0.26.1/kube-flannel.yml
+	$(NODE_SHELL) /usernetes/Makefile.d/install-flannel.sh

Makefile.d/install-flannel.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+set -eu -o pipefail
+
+function INFO() {
+	echo >&2 -e "\e[104m\e[97m[INFO]\e[49m\e[39m $@"
+}
+function WARNING() {
+	echo >&2 -e "\e[101m\e[97m[WARNING]\e[49m\e[39m $@"
+}
+
+function ERROR() {
+	echo >&2 -e "\e[101m\e[97m[ERROR]\e[49m\e[39m $@"
+}
+
+# See chart values, 0 indicates default for platform
+# https://github.com/flannel-io/flannel/blob/v0.26.1/chart/kube-flannel/values.yaml
+: "${U7S_PORT_FLANNEL:='0'}"
+: "${U7S_PORT_ETCD:='2379'}"
+
+INFO "Flannel port: ${U7S_PORT_FLANNEL}"
+INFO "ETCD port: ${U7S_PORT_ETCD}"
+
+# Check hard dependency commands
+for cmd in helm kubectl git; do
+	if ! command -v "${cmd}" >/dev/null 2>&1; then
+		ERROR "Command \"${cmd}\" is not installed"
+		exit 1
+	fi
+done
+
+# We need to customize the values.yaml to expose the backendPort and args
+flannel_root=$(mktemp -d -u -t flannel-XXXXXXX)
+git clone --quiet --depth 1 --branch v0.26.1 https://github.com/flannel-io/flannel $flannel_root
+cd $flannel_root/chart
+
+# Write a new values.yaml that exposes what we need
+cat <<EOF > ./new-values.yaml
+---
+global:
+  imagePullSecrets:
+# - name: "a-secret-name"
+
+# The IPv4 cidr pool to create on startup if none exists. Pod IPs will be
+# chosen from this range.
+podCidr: "10.244.0.0/16"
+podCidrv6: ""
+flannel:
+  # kube-flannel image
+  image:
+    repository: docker.io/flannel/flannel
+    tag: v0.26.1
+  image_cni:
+    repository: docker.io/flannel/flannel-cni-plugin
+    tag: v1.5.1-flannel2
+  # flannel command arguments
+  enableNFTables: false
+  args:
+  - "--ip-masq"
+  - "--kube-subnet-mgr"
+  - "--etcd-endpoints=\"http://127.0.0.1:4001,http://127.0.0.1:${U7S_PORT_ETCD}\""
+  # Backend for kube-flannel. Backend should not be changed
+  # at runtime. (vxlan, host-gw, wireguard, udp)
+  # Documentation at https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md
+  backend: "vxlan"
+  # Port used by the backend 0 means default value (VXLAN: 8472, Wireguard: 51821, UDP: 8285)
+  backendPort: ${U7S_PORT_FLANNEL}
+  tolerations:
+  - effect: NoExecute
+    operator: Exists
+  - effect: NoSchedule
+    operator: Exists
+
+netpol:
+  enabled: false
+EOF
+
+mv ./new-values.yaml ./kube-flannel/values.yaml
+
+kubectl create namespace kube-flannel || WARNING "kube-flannel namespace is already created"
+kubectl label --overwrite namespace kube-flannel pod-security.kubernetes.io/enforce=privileged || true
+# If the command is issued again, this cleanup is needed
+helm delete flannel --namespace kube-flannel kube-flannel || true
+# We could also do --set flannel.backendPort=<value> but it's the same to set as the default
+helm install flannel --namespace kube-flannel kube-flannel
+cd -
+rm -rf $flannel_root

hack/create-cluster-lima.sh

@@ -6,6 +6,9 @@ set -eux -o pipefail
 : "${CONTAINER_ENGINE:=docker}"
 : "${LOCKDOWN_SUDO:=1}"
 : "${U7S_PORT_KUBE_APISERVER:=6443}"
+: "${U7S_PORT_ETCD:=2379}"
+: "${U7S_PORT_FLANNEL:=8472}"
+: "${U7S_PORT_KUBELET:=10250}"
 
 guest_home="/home/${USER}.linux"
 
@@ -25,21 +28,20 @@ for host in host0 host1; do
 	${LIMACTL} shell "${host}" CONTAINER_ENGINE="${CONTAINER_ENGINE}" "${guest_home}/usernetes/init-host/init-host.rootless.sh"
 done
 
+U7S_SERVICE_PORTS="U7S_PORT_KUBE_APISERVER=${U7S_PORT_KUBE_APISERVER} U7S_PORT_ETCD=${U7S_PORT_ETCD} U7S_PORT_FLANNEL=${U7S_PORT_FLANNEL} U7S_PORT_KUBELET=${U7S_PORT_KUBELET}"
+
 # Launch a Kubernetes node inside a Rootless Docker host
 for host in host0 host1; do
-	${LIMACTL} shell "${host}" U7S_PORT_KUBE_APISERVER=${U7S_PORT_KUBE_APISERVER} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" up
+	${LIMACTL} shell "${host}" ${U7S_SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" up
 done
 
 # Bootstrap a cluster with host0
-${LIMACTL} shell host0 U7S_PORT_KUBE_APISERVER=${U7S_PORT_KUBE_APISERVER} \
-                       CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-init install-flannel kubeconfig join-command
+${LIMACTL} shell host0 ${U7S_SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-init install-flannel kubeconfig join-command
 
 # Let host1 join the cluster
 ${LIMACTL} copy host0:~/usernetes/join-command host1:~/usernetes/join-command
-${LIMACTL} shell host1 U7S_PORT_KUBE_APISERVER=${U7S_PORT_KUBE_APISERVER} \
-                       CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-join
-${LIMACTL} shell host0 U7S_PORT_KUBE_APISERVER=${U7S_PORT_KUBE_APISERVER} \
-                       CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" sync-external-ip
+${LIMACTL} shell host1 ${U7S_SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-join
+${LIMACTL} shell host0 ${U7S_SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" sync-external-ip
 
 # Enable kubectl
 ssh -q -f -N -L ${U7S_PORT_KUBE_APISERVER}:127.0.0.1:${U7S_PORT_KUBE_APISERVER} -F ~/.lima/host0/ssh.config lima-host0