From 9ec28708cf967183782fced79e4b1e96641b4a42 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Wed, 20 Feb 2019 14:50:21 +0300 Subject: [PATCH] Fix kublelet killing VMs upon Virtlet pod restart The culprit were cgroups that aren't handled by libvirt. Of those, we already handle hugetlb by moving the emulator process out of it. Still, need to do the same for systemd (name=systemd) and pids cgroup controllers. The problem manifested itself when cgroup-per-qos is enabled for kubelet. This is the default, but in current kdc it may be disabled as a workaround for old kubelet bug. This bug is already fixed, so the workaround is to be removed soon. --- cmd/vmwrapper/vmwrapper.go | 16 ++++++++++------ pkg/utils/cgroups/controllers.go | 9 ++++++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/cmd/vmwrapper/vmwrapper.go b/cmd/vmwrapper/vmwrapper.go index f113533a9..5e0f300e9 100644 --- a/cmd/vmwrapper/vmwrapper.go +++ b/cmd/vmwrapper/vmwrapper.go @@ -71,13 +71,17 @@ func main() { } } - // FIXME: move the pid of qemu instance out of /kubepods/podxxxxxxx - // for some cases it will be killed by kubelet after the virtlet pod is deleted/recreated + // FIXME: move the pid of qemu instance out of kubelet-managed + // for cgroups that aren't managed by libvirt. + // If we don't do this, the VM pod will be killed by kubelet when Virtlet pod + // is removed dnd cgroup-per-qos is enabled in kubelet settings. cm := cgroups.NewManager(os.Getpid(), nil) - if _, err := cm.GetProcessController("hugetlb"); err == nil { - err = cm.MoveProcess("hugetlb", "/") - if err != nil { - glog.Warningf("failed to move pid into hugetlb path /: %v", err) + for _, ctl := range []string{"hugetlb", "systemd", "pids"} { + if _, err := cm.GetProcessController(ctl); err == nil { + err = cm.MoveProcess(ctl, "/") + if err != nil { + glog.Warningf("failed to move pid into cgroup %q path /: %v", ctl, err) + } } } diff --git a/pkg/utils/cgroups/controllers.go b/pkg/utils/cgroups/controllers.go index 6d56c27f3..b39256fad 100644 --- a/pkg/utils/cgroups/controllers.go +++ b/pkg/utils/cgroups/controllers.go @@ -94,8 +94,15 @@ func (c *RealManager) GetProcessControllers() (map[string]string, error) { // "6:memory:/user.slice/user-xxx.slice/session-xx.scope" parts := strings.SplitN(line, ":", 3) + name := parts[1] + if strings.HasPrefix(name, "name=") { + // Handle named cgroup hierarchies like name=systemd + // The corresponding directory tree will be /sys/fs/cgroup/systemd + name = name[5:] + } + // use second part as controller name and third as its path - ctrls[parts[1]] = parts[2] + ctrls[name] = parts[2] if err == io.EOF { break