Merge pull request #895 from Mirantis/ivan4th/fix-pod-recovery

Fix pod recovery after node reboot

.codeclimate.yml

@@ -15,7 +15,8 @@ checks:
   method-count:
     config:
       # CRI runtime implementation has >20 methods
-      threshold: 25
+      # Also, VirtualizationTool has plenty
+      threshold: 30
   method-lines:
     config:
       threshold: 200

pkg/libvirttools/virtualization.go

@@ -33,6 +33,7 @@ import (
 	"github.com/Mirantis/virtlet/pkg/fs"
 	"github.com/Mirantis/virtlet/pkg/metadata"
 	"github.com/Mirantis/virtlet/pkg/metadata/types"
+	"github.com/Mirantis/virtlet/pkg/network"
 	"github.com/Mirantis/virtlet/pkg/utils"
 	"github.com/Mirantis/virtlet/pkg/virt"
 )
@@ -412,6 +413,51 @@ func (v *VirtualizationTool) CreateContainer(config *types.VMConfig, netFdKey st
 	return settings.domainUUID, nil
 }
 
+func (v *VirtualizationTool) updateDiskImages(containerID string) error {
+	domain, err := v.domainConn.LookupDomainByUUIDString(containerID)
+	if err != nil {
+		return fmt.Errorf("failed to look up domain %q: %v", containerID, err)
+	}
+
+	config, _, err := v.getVMConfigFromMetadata(containerID)
+	if err != nil {
+		return err
+	}
+
+	if config == nil {
+		glog.Warningf("No info found for domain %q in the metadata store. Not updating disk images", containerID)
+		return nil
+	}
+
+	diskList, err := newDiskList(config, v.volumeSource, v)
+	if err != nil {
+		return err
+	}
+
+	return diskList.writeImages(domain)
+}
+
+// UpdateContainerNetwork updates network info for the container
+func (v *VirtualizationTool) UpdateContainerNetwork(containerID string, csn *network.ContainerSideNetwork) error {
+	if err := v.metadataStore.Container(containerID).Save(
+		func(c *types.ContainerInfo) (*types.ContainerInfo, error) {
+			// make sure the container is not removed during the call
+			if c != nil {
+				c.Config.ContainerSideNetwork = csn
+			}
+			return c, nil
+		}); err != nil {
+		return fmt.Errorf("error updating container info: %v", err)
+	}
+
+	// propagate network config to cloud-init
+	if err := v.updateDiskImages(containerID); err != nil {
+		return fmt.Errorf("domain %q: error updating disk images: %v", containerID, err)
+	}
+
+	return nil
+}
+
 func (v *VirtualizationTool) startContainer(containerID string) error {
 	domain, err := v.domainConn.LookupDomainByUUIDString(containerID)
 	if err != nil {

pkg/manager/runtime.go

@@ -291,29 +291,35 @@ func (v *VirtletRuntimeService) CreateContainer(ctx context.Context, in *kubeapi
 	podSandboxID := in.PodSandboxId
 	name := config.GetMetadata().Name
 
+	sandboxInfo, err := v.metadataStore.PodSandbox(podSandboxID).Retrieve()
+	if err != nil {
+		return nil, err
+	}
+	if sandboxInfo == nil {
+		return nil, fmt.Errorf("sandbox %q not in Virtlet metadata store", podSandboxID)
+	}
+
 	// Was a container already started in this sandbox?
 	// NOTE: there is no distinction between lack of key and other types of
 	// errors when accessing boltdb. This will be changed when we switch to
 	// storing whole marshaled sandbox metadata as json.
-	remainingContainers, err := v.metadataStore.ListPodContainers(podSandboxID)
+	curContainers, err := v.metadataStore.ListPodContainers(podSandboxID)
 	if err != nil {
 		glog.V(3).Infof("Error retrieving pod %q containers", podSandboxID)
 	} else {
-		for _, container := range remainingContainers {
+		for _, container := range curContainers {
+			// TODO: check container name; if it's the same, update the network config
 			glog.V(3).Infof("CreateContainer: there's already a container in the sandbox (id: %s)", container.GetID())
+			//err := v.updateContainer(sandboxInfo, container.GetID())
+			err := v.virtTool.UpdateContainerNetwork(container.GetID(), sandboxInfo.ContainerSideNetwork)
+			if err != nil {
+				return nil, err
+			}
 			response := &kubeapi.CreateContainerResponse{ContainerId: container.GetID()}
 			return response, nil
 		}
 	}
 
-	sandboxInfo, err := v.metadataStore.PodSandbox(podSandboxID).Retrieve()
-	if err != nil {
-		return nil, err
-	}
-	if sandboxInfo == nil {
-		return nil, fmt.Errorf("sandbox %q not in Virtlet metadata store", podSandboxID)
-	}
-
 	fdKey := podSandboxID
 	vmConfig, err := GetVMConfig(in, sandboxInfo.ContainerSideNetwork)
 	if err != nil {