From 72cbcf040bb179a2723b7e270da9762cbbc17cb3 Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Thu, 3 Nov 2022 00:06:56 +0800 Subject: [PATCH] kata-deploy: Add k0s support Add k0s support to kata-deploy, in the very same way kata-containers already supports k3s, and rke2. k0s support requires v1.27.1, which is noted as part of the kata-deploy documentation, as it's the way to use dynamic configuration on containerd CRI runtimes. This support will only be part of the `main` branch, as it's not a bug fix that can be backported to the `stable-3.2` branch, and this is also noted as part of the documentation. Fixes: #7548 Signed-off-by: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> --- tools/packaging/kata-deploy/README.md | 30 +++++++++++ .../overlays/k0s/kustomization.yaml | 5 ++ .../overlays/k0s/mount_k0s_conf.yaml | 12 +++++ .../kata-deploy/scripts/kata-deploy.sh | 54 ++++++++++++++----- 4 files changed, 87 insertions(+), 14 deletions(-) create mode 100644 tools/packaging/kata-deploy/kata-deploy/overlays/k0s/kustomization.yaml create mode 100644 tools/packaging/kata-deploy/kata-deploy/overlays/k0s/mount_k0s_conf.yaml diff --git a/tools/packaging/kata-deploy/README.md b/tools/packaging/kata-deploy/README.md index 9e546e4f5ece..6ef7cd6a98fb 100644 --- a/tools/packaging/kata-deploy/README.md +++ b/tools/packaging/kata-deploy/README.md @@ -43,6 +43,36 @@ $ kubectl apply -f kata-rbac/base/kata-rbac.yaml $ kubectl apply -k kata-deploy/overlays/rke2 ``` +#### [k0s] cluster + +For your [k0s](https://k0sproject.io/) cluster, run: + +```sh +$ git clone https://github.com/kata-containers/kata-containers.git +``` + +Check and switch to "main", and then run: + +```bash +$ cd kata-containers/tools/packaging/kata-deploy +$ kubectl apply -f kata-rbac/base/kata-rbac.yaml +$ kubectl apply -k kata-deploy/overlays/k0s +``` + +##### Note + +The supported version of k0s is **v1.27.1+k0s** and above, since the k0s support leverages a special dynamic containerd configuration mode: + +> From 1.27.1 onwards k0s enables dynamic configuration on containerd CRI runtimes. This works by k0s creating a special directory in /etc/k0s/containerd.d/ where user can drop-in partial containerd configuration snippets. +> +> k0s will automatically pick up these files and adds these in containerd configuration imports list. If k0s sees the configuration drop-ins are CRI related configurations k0s will automatically collect all these into a single file and adds that as a single import file. This is to overcome some hard limitation on containerd 1.X versions. Read more at containerd#8056 + +However, this would also require a magic string set in the beginning of the line for `/etc/k0s/containerd.toml`: + +``` +# k0s_managed=true +``` + #### Vanilla Kubernetes cluster ##### Installing the latest image diff --git a/tools/packaging/kata-deploy/kata-deploy/overlays/k0s/kustomization.yaml b/tools/packaging/kata-deploy/kata-deploy/overlays/k0s/kustomization.yaml new file mode 100644 index 000000000000..14904f5601ff --- /dev/null +++ b/tools/packaging/kata-deploy/kata-deploy/overlays/k0s/kustomization.yaml @@ -0,0 +1,5 @@ +bases: +- ../../base + +patchesStrategicMerge: +- mount_k0s_conf.yaml diff --git a/tools/packaging/kata-deploy/kata-deploy/overlays/k0s/mount_k0s_conf.yaml b/tools/packaging/kata-deploy/kata-deploy/overlays/k0s/mount_k0s_conf.yaml new file mode 100644 index 000000000000..c10f938f85f0 --- /dev/null +++ b/tools/packaging/kata-deploy/kata-deploy/overlays/k0s/mount_k0s_conf.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kata-deploy + namespace: kube-system +spec: + template: + spec: + volumes: + - name: containerd-conf + hostPath: + path: /etc/k0s/containerd.d/ \ No newline at end of file diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index fc26921df247..338834161274 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -74,6 +74,7 @@ function get_container_runtime() { if [ "$?" -ne 0 ]; then die "invalid node name" fi + if echo "$runtime" | grep -qE 'containerd.*-k3s'; then if host_systemctl is-active --quiet rke2-agent; then echo "rke2-agent" @@ -84,6 +85,12 @@ function get_container_runtime() { else echo "k3s" fi + # Note: we assumed you used a conventional k0s setup and k0s will generate a systemd entry k0scontroller.service and k0sworker.service respectively + # and it is impossible to run this script without a kubelet, so this k0s controller must also have worker mode enabled + elif host_systemctl is-active --quiet k0scontroller; then + echo "k0s-controller" + elif host_systemctl is-active --quiet k0sworker; then + echo "k0s-worker" else echo "$runtime" | awk -F '[:]' '{print $1}' fi @@ -136,12 +143,17 @@ function configure_cri_runtime() { crio) configure_crio ;; - containerd | k3s | k3s-agent | rke2-agent | rke2-server) - configure_containerd + containerd | k3s | k3s-agent | rke2-agent | rke2-server | k0s-controller | k0s-worker) + configure_containerd "$1" ;; esac - host_systemctl daemon-reload - host_systemctl restart "$1" + if [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then + # do nothing, k0s will automatically load the config on the fly + : + else + host_systemctl daemon-reload + host_systemctl restart "$1" + fi wait_till_node_is_ready } @@ -274,12 +286,15 @@ EOF function configure_containerd_runtime() { local runtime="kata" local configuration="configuration" - if [ -n "${1-}" ]; then - runtime+="-$1" - configuration+="-$1" + if [ -n "${2-}" ]; then + runtime+="-$2" + configuration+="-$2" fi local pluginid=cri - if grep -q "version = 2\>" $containerd_conf_file; then + + # if we are running k0s auto containerd.toml generation, the base template is by default version 2 + # we can safely assume to reference the older version of cri + if grep -q "version = 2\>" $containerd_conf_file || [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then pluginid=\"io.containerd.grpc.v1.cri\" fi local runtime_table="plugins.${pluginid}.containerd.runtimes.$runtime" @@ -333,10 +348,10 @@ function configure_containerd() { fi # Add default Kata runtime configuration - configure_containerd_runtime + configure_containerd_runtime "$1" for shim in "${shims[@]}"; do - configure_containerd_runtime $shim + configure_containerd_runtime "$1" $shim done } @@ -352,7 +367,7 @@ function cleanup_cri_runtime() { crio) cleanup_crio ;; - containerd | k3s | k3s-agent | rke2-agent | rke2-server) + containerd | k3s | k3s-agent | rke2-agent | rke2-server | k0s-controller | k0s-worker) cleanup_containerd ;; esac @@ -375,8 +390,14 @@ function cleanup_containerd() { function reset_runtime() { kubectl label node "$NODE_NAME" katacontainers.io/kata-runtime- - host_systemctl daemon-reload - host_systemctl restart "$1" + if [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then + # do nothing, k0s will auto restart + : + else + host_systemctl daemon-reload + host_systemctl restart "$1" + fi + if [ "$1" == "crio" ] || [ "$1" == "containerd" ]; then host_systemctl restart kubelet fi @@ -412,6 +433,11 @@ function main() { containerd_conf_file="${containerd_conf_tmpl_file}" containerd_conf_file_backup="${containerd_conf_file}.bak" + elif [ "$runtime" == "k0s-worker" ] || [ "$runtime" == "k0s-controller" ]; then + # From 1.27.1 onwards k0s enables dynamic configuration on containerd CRI runtimes. + # This works by k0s creating a special directory in /etc/k0s/containerd.d/ where user can drop-in partial containerd configuration snippets. + # k0s will automatically pick up these files and adds these in containerd configuration imports list. + containerd_conf_file="/etc/containerd/kata-containers.toml" else # runtime == containerd if [ ! -f "$containerd_conf_file" ] && [ -d $(dirname "$containerd_conf_file") ] && \ @@ -427,7 +453,7 @@ function main() { fi # only install / remove / update if we are dealing with CRIO or containerd - if [[ "$runtime" =~ ^(crio|containerd|k3s|k3s-agent|rke2-agent|rke2-server)$ ]]; then + if [[ "$runtime" =~ ^(crio|containerd|k3s|k3s-agent|rke2-agent|rke2-server|k0s-worker|k0s-controller)$ ]]; then case "$action" in install)