kata-deploy: Add k0s support

Add k0s support to kata-deploy, in the very same way kata-containers
already supports k3s, and rke2.

k0s support requires v1.27.1, which is noted as part of the kata-deploy
documentation, as it's the way to use dynamic configuration on
containerd CRI runtimes.

This support will only be part of the `main` branch, as it's not a bug
fix that can be backported to the `stable-3.2` branch, and this is also
noted as part of the documentation.

Fixes: kata-containers#7548
Signed-off-by: Steve Fan <[email protected]>

tools/packaging/kata-deploy/README.md

@@ -43,6 +43,36 @@ $ kubectl apply -f kata-rbac/base/kata-rbac.yaml
 $ kubectl apply -k kata-deploy/overlays/rke2
 ```
 
+#### [k0s] cluster
+
+For your [k0s](https://k0sproject.io/) cluster, run:
+
+```sh
+$ git clone https://github.com/kata-containers/kata-containers.git
+```
+
+Check and switch to "main", and then run:
+
+```bash
+$ cd kata-containers/tools/packaging/kata-deploy
+$ kubectl apply -f kata-rbac/base/kata-rbac.yaml
+$ kubectl apply -k kata-deploy/overlays/k0s
+```
+
+##### Note
+
+The supported version of k0s is **v1.27.1+k0s** and above, since the k0s support leverages a special dynamic containerd configuration mode:
+
+> From 1.27.1 onwards k0s enables dynamic configuration on containerd CRI runtimes. This works by k0s creating a special directory in /etc/k0s/containerd.d/ where user can drop-in partial containerd configuration snippets.
+> 
+> k0s will automatically pick up these files and adds these in containerd configuration imports list. If k0s sees the configuration drop-ins are CRI related configurations k0s will automatically collect all these into a single file and adds that as a single import file. This is to overcome some hard limitation on containerd 1.X versions. Read more at containerd#8056
+
+However, this would also require a magic string set in the beginning of the line for `/etc/k0s/containerd.toml`:
+
+```
+# k0s_managed=true
+```
+
 #### Vanilla Kubernetes cluster
 
 ##### Installing the latest image

tools/packaging/kata-deploy/kata-deploy/overlays/k0s/kustomization.yaml

@@ -0,0 +1,5 @@
+bases:
+- ../../base
+
+patchesStrategicMerge:
+- mount_k0s_conf.yaml

tools/packaging/kata-deploy/kata-deploy/overlays/k0s/mount_k0s_conf.yaml

@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kata-deploy
+  namespace: kube-system
+spec:
+  template:
+    spec:
+      volumes:
+        - name: containerd-conf
+          hostPath:
+            path: /etc/k0s/containerd.d/

tools/packaging/kata-deploy/scripts/kata-deploy.sh

@@ -74,6 +74,7 @@ function get_container_runtime() {
 	if [ "$?" -ne 0 ]; then
                 die "invalid node name"
 	fi
+
 	if echo "$runtime" | grep -qE 'containerd.*-k3s'; then
 		if host_systemctl is-active --quiet rke2-agent; then
 			echo "rke2-agent"
@@ -84,6 +85,12 @@ function get_container_runtime() {
 		else
 			echo "k3s"
 		fi
+	# Note: we assumed you used a conventional k0s setup and k0s will generate a systemd entry k0scontroller.service and k0sworker.service respectively    
+	# and it is impossible to run this script without a kubelet, so this k0s controller must also have worker mode enabled 
+	elif host_systemctl is-active --quiet k0scontroller; then
+		echo "k0s-controller"
+	elif host_systemctl is-active --quiet k0sworker; then
+		echo "k0s-worker"
 	else
 		echo "$runtime" | awk -F '[:]' '{print $1}'
 	fi
@@ -136,12 +143,17 @@ function configure_cri_runtime() {
 	crio)
 		configure_crio
 		;;
-	containerd | k3s | k3s-agent | rke2-agent | rke2-server)
-		configure_containerd
+	containerd | k3s | k3s-agent | rke2-agent | rke2-server | k0s-controller | k0s-worker)
+		configure_containerd "$1"
 		;;
 	esac
-	host_systemctl daemon-reload
-	host_systemctl restart "$1"
+	if [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then
+		# do nothing, k0s will automatically load the config on the fly
+		:
+	else
+		host_systemctl daemon-reload
+		host_systemctl restart "$1"
+	fi
 
 	wait_till_node_is_ready
 }
@@ -274,12 +286,15 @@ EOF
 function configure_containerd_runtime() {
 	local runtime="kata"
 	local configuration="configuration"
-	if [ -n "${1-}" ]; then
-		runtime+="-$1"
-		configuration+="-$1"
+	if [ -n "${2-}" ]; then
+		runtime+="-$2"
+		configuration+="-$2"
 	fi
 	local pluginid=cri
-	if grep -q "version = 2\>" $containerd_conf_file; then
+
+	# if we are running k0s auto containerd.toml generation, the base template is by default version 2
+	# we can safely assume to reference the older version of cri
+	if grep -q "version = 2\>" $containerd_conf_file || [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then
 		pluginid=\"io.containerd.grpc.v1.cri\"
 	fi
 	local runtime_table="plugins.${pluginid}.containerd.runtimes.$runtime"
@@ -333,10 +348,10 @@ function configure_containerd() {
 	fi
 
 	# Add default Kata runtime configuration
-	configure_containerd_runtime
+	configure_containerd_runtime "$1" 
 
 	for shim in "${shims[@]}"; do
-		configure_containerd_runtime $shim
+		configure_containerd_runtime "$1" $shim
 	done
 }
 
@@ -352,7 +367,7 @@ function cleanup_cri_runtime() {
 	crio)
 		cleanup_crio
 		;;
-	containerd | k3s | k3s-agent | rke2-agent | rke2-server)
+	containerd | k3s | k3s-agent | rke2-agent | rke2-server | k0s-controller | k0s-worker)
 		cleanup_containerd
 		;;
 	esac
@@ -375,8 +390,14 @@ function cleanup_containerd() {
 
 function reset_runtime() {
 	kubectl label node "$NODE_NAME" katacontainers.io/kata-runtime-
-	host_systemctl daemon-reload
-	host_systemctl restart "$1"
+	if [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then
+		# do nothing, k0s will auto restart
+		:
+	else
+		host_systemctl daemon-reload
+		host_systemctl restart "$1"
+	fi
+
 	if [ "$1" == "crio" ] || [ "$1" == "containerd" ]; then
 		host_systemctl restart kubelet
 	fi
@@ -412,6 +433,11 @@ function main() {
 
 		containerd_conf_file="${containerd_conf_tmpl_file}"
 		containerd_conf_file_backup="${containerd_conf_file}.bak"
+	elif [ "$runtime" == "k0s-worker" ] || [ "$runtime" == "k0s-controller" ]; then
+		# From 1.27.1 onwards k0s enables dynamic configuration on containerd CRI runtimes. 
+		# This works by k0s creating a special directory in /etc/k0s/containerd.d/ where user can drop-in partial containerd configuration snippets.
+		# k0s will automatically pick up these files and adds these in containerd configuration imports list.
+		containerd_conf_file="/etc/containerd/kata-containers.toml"
 	else
 		# runtime == containerd
 		if [ ! -f "$containerd_conf_file" ] && [ -d $(dirname "$containerd_conf_file") ] && \
@@ -427,7 +453,7 @@ function main() {
 	fi
 
 	# only install / remove / update if we are dealing with CRIO or containerd
-	if [[ "$runtime" =~ ^(crio|containerd|k3s|k3s-agent|rke2-agent|rke2-server)$ ]]; then
+	if [[ "$runtime" =~ ^(crio|containerd|k3s|k3s-agent|rke2-agent|rke2-server|k0s-worker|k0s-controller)$ ]]; then
 
 		case "$action" in
 		install)