verify pvcs for ownerreferences pointing to VM

Signed-off-by: pruthvitd <prd@redhat.com>

Author: pruthvitd

cmd/main.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
@@ -42,6 +43,7 @@ import (
 	controllers "github.com/ramendr/ramen/internal/controller"
 	argocdv1alpha1hack "github.com/ramendr/ramen/internal/controller/argocd"
 	rmnutil "github.com/ramendr/ramen/internal/controller/util"
+
 	// +kubebuilder:scaffold:imports
 	_ "github.com/ramendr/ramen/internal/dummy"
 )

config/rbac/role.yaml

@@ -144,6 +144,14 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - cdi.kubevirt.io
+  resources:
+  - datavolumes
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - cluster.open-cluster-management.io
   resources:
@@ -232,8 +240,10 @@ rules:
   resources:
   - virtualmachines
   verbs:
+  - delete
   - get
   - list
+  - watch
 - apiGroups:
   - multicluster.x-k8s.io
   resources:

internal/controller/suite_test.go

@@ -51,6 +51,7 @@ import (
 	argocdv1alpha1hack "github.com/ramendr/ramen/internal/controller/argocd"
 	testutils "github.com/ramendr/ramen/internal/controller/testutils"
 	"github.com/ramendr/ramen/internal/controller/util"
+
 	// +kubebuilder:scaffold:imports
 	_ "github.com/ramendr/ramen/internal/dummy"
 )

internal/controller/util/vm_util.go

@@ -6,13 +6,20 @@ import (
 
 	"github.com/go-logr/logr"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	virtv1 "kubevirt.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/ramendr/ramen/internal/controller/core"
 )
 
+const (
+	KindVirtualMachine = "VirtualMachine"
+	KubeVirtAPIVersion = "kubevirt.io/v1"
+)
+
 func ListVMsByLabelSelector(
 	ctx context.Context,
 	apiReader client.Reader,
@@ -89,3 +96,91 @@ func ListVMsByVMNamespace(
 
 	return nil, notFoundErr
 }
+
+func IsVMDeletionInProgress(ctx context.Context,
+	k8sclient client.Client,
+	vmList []string,
+	vmNamespaceList []string,
+	log logr.Logger,
+) bool {
+	log.Info("Checking if VirtualMachines are being deleted",
+		"vmCount", len(vmList),
+		"vmNames", vmList)
+
+	foundVM := &virtv1.VirtualMachine{}
+
+	for _, ns := range vmNamespaceList {
+		for _, vm := range vmList {
+			vmLookUp := types.NamespacedName{Namespace: ns, Name: vm}
+			if err := k8sclient.Get(ctx, vmLookUp, foundVM); err != nil {
+				// Continuing with remaining list of VMs as the current one might already have been deleted
+				continue
+			}
+
+			if !foundVM.GetDeletionTimestamp().IsZero() {
+				// Deletion of vm has been requested
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+func DeleteVMs(
+	ctx context.Context,
+	k8sclient client.Client,
+	vmList []string,
+	vmNamespaceList []string,
+	log logr.Logger,
+) error {
+	for _, ns := range vmNamespaceList {
+		for _, vmName := range vmList {
+			vm := &virtv1.VirtualMachine{}
+			key := client.ObjectKey{Name: vmName, Namespace: ns}
+
+			if err := k8sclient.Get(ctx, key, vm); err != nil {
+				log.Error(err, "Failed to get VM", "namespace", ns, "name", vmName)
+
+				return fmt.Errorf("failed to get VM %s/%s: %w", ns, vmName, err)
+			}
+
+			if err := k8sclient.Delete(ctx, vm); err != nil {
+				log.Error(err, "Failed to delete VM", "namespace", ns, "name", vmName)
+
+				return fmt.Errorf("failed to delete VM %s/%s: %w", ns, vmName, err)
+			}
+
+			log.Info("Deleted VM successfully", "namespace", ns, "name", vmName)
+		}
+	}
+
+	return nil
+}
+
+// IsOwnedByVM recursively traverses ownerReferences until it finds a VirtualMachine.
+func IsOwnedByVM(ctx context.Context, c client.Client, obj client.Object, log logr.Logger) (string, error) {
+	for {
+		owners := obj.GetOwnerReferences()
+		if len(owners) == 0 {
+			return "", fmt.Errorf("no VM found in ownership chain")
+		}
+
+		for _, owner := range owners {
+			if owner.Kind == KindVirtualMachine && owner.APIVersion == KubeVirtAPIVersion {
+				return owner.Name, nil // Found VM root
+			}
+
+			// Fetch only metadata of the owner
+			ownerMeta := &metav1.PartialObjectMetadata{}
+			ownerMeta.SetGroupVersionKind(schema.FromAPIVersionAndKind(owner.APIVersion, owner.Kind))
+
+			if err := c.Get(ctx, client.ObjectKey{Namespace: obj.GetNamespace(), Name: owner.Name}, ownerMeta); err != nil {
+				return "", err
+			}
+
+			// Continue traversal with the owner
+			obj = ownerMeta
+		}
+	}
+}

internal/controller/volumereplicationgroup_controller.go

@@ -397,7 +397,8 @@ func filterPVC(reader client.Reader, pvc *corev1.PersistentVolumeClaim, log logr
 // +kubebuilder:rbac:groups="",resources=configmaps,verbs=list;watch
 // +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=get;list;watch
 // +kubebuilder:rbac:groups=core,resources=pods/exec,verbs=create
-// +kubebuilder:rbac:groups="kubevirt.io",resources=virtualmachines,verbs=get;list
+// +kubebuilder:rbac:groups="kubevirt.io",resources=virtualmachines,verbs=get;list;watch;delete
+// +kubebuilder:rbac:groups="cdi.kubevirt.io",resources=datavolumes,verbs=get;list;watch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
@@ -1660,6 +1661,35 @@ func (v *VRGInstance) processAsSecondary() ctrl.Result {
 
 func (v *VRGInstance) reconcileAsSecondary() ctrl.Result {
 	result := ctrl.Result{}
+
+	if v.isVMRecipeProtection() {
+		v.log.Info("Checking VM cleanup and cross-cluster resource conflicts",
+			"name", v.instance.GetName(),
+			"namespace", v.instance.GetNamespace(),
+			"recipeName", "vm-recipe")
+
+		if err := v.validateVMsForStandaloneProtection(); err != nil {
+			result.Requeue = true
+		}
+
+		if !result.Requeue {
+			v.log.Info("VRG status observed",
+				"name", v.instance.GetName(),
+				"namespace", v.instance.GetNamespace(),
+				"replicationState", v.instance.Spec.ReplicationState,
+				"statusState", v.instance.Status.State,
+				"generation", v.instance.GetGeneration(),
+				"resourceVersion", v.instance.GetResourceVersion(),
+			)
+
+			if v.ShouldCleanupVMForSecondary() {
+				v.log.Info("Requeuing until VM cleanup is complete")
+
+				result.Requeue = true
+			}
+		}
+	}
+
 	result.Requeue = v.reconcileVolSyncAsSecondary() || result.Requeue
 	result.Requeue = v.reconcileVolRepsAsSecondary() || result.Requeue
 
@@ -1678,12 +1708,6 @@ func (v *VRGInstance) reconcileAsSecondary() ctrl.Result {
 		v.instance.Status.Conditions = []metav1.Condition{}
 	}
 
-	if !result.Requeue && v.isVMRecipeProtection() {
-		if err := v.validateVMsForStandaloneProtection(); err != nil {
-			result.Requeue = true
-		}
-	}
-
 	return result
 }
 
@@ -1791,6 +1815,8 @@ func (v *VRGInstance) updateVRGStatus(result ctrl.Result) ctrl.Result {
 
 	v.updateStatusState()
 
+	result.Requeue = v.instance.Status.State == ramendrv1alpha1.UnknownState
+
 	v.instance.Status.ObservedGeneration = v.instance.Generation
 
 	if !reflect.DeepEqual(v.savedInstanceStatus, v.instance.Status) {

internal/controller/vrg_volrep.go

@@ -23,6 +23,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	ramendrv1alpha1 "github.com/ramendr/ramen/api/v1alpha1"
+	recipecore "github.com/ramendr/ramen/internal/controller/core"
 	rmnutil "github.com/ramendr/ramen/internal/controller/util"
 )
 
@@ -2895,3 +2896,102 @@ func PruneAnnotations(annotations map[string]string) map[string]string {
 
 	return result
 }
+
+func (v *VRGInstance) ShouldCleanupVMForSecondary() bool {
+	if !v.IsDRActionInProgress() {
+		v.log.Info("Skip VM cleanup; reconcile as secondary")
+
+		return false
+	}
+
+	v.log.Info(
+		"DR action progressing",
+		"component", "VRGController",
+		"action", "reconcile",
+		"vrgName", v.instance.GetName(),
+		"namespace", v.instance.GetNamespace(),
+		"desiredState", v.instance.Spec.ReplicationState,
+		"currentState", v.instance.Status.State,
+	)
+
+	vmNamespaceList := v.instance.Spec.KubeObjectProtection.RecipeParameters[recipecore.ProtectedVMNamespace]
+	vmList := v.instance.Spec.KubeObjectProtection.RecipeParameters[recipecore.VMList]
+
+	if len(vmList) > 0 {
+		if rmnutil.IsVMDeletionInProgress(v.ctx, v.reconciler.Client, vmList, vmNamespaceList, v.log) {
+			v.log.Info("VM deletion is in progress, skipping ownerreferences check")
+
+			return true
+		}
+
+		var foundVMs []string
+
+		foundVMs, err := rmnutil.ListVMsByVMNamespace(v.ctx, v.reconciler.APIReader,
+			v.log, vmNamespaceList, vmList)
+		if len(foundVMs) == 0 || err != nil {
+			v.log.Info(
+				"No VirtualMachines found for cleanup; deletion appears complete",
+				"vmList", vmList,
+				"namespaceList", vmNamespaceList,
+			)
+
+			return false
+		}
+	}
+
+	if v.IsAllProtectedPVCsOwnedByProtectedVMs() {
+		v.log.Info("all protected PVCs have ownerreferences to protected list of VMs")
+		// Cleanup VM resources
+		err := rmnutil.DeleteVMs(v.ctx, v.reconciler.Client, vmList, vmNamespaceList, v.log)
+		if err != nil {
+			v.log.Error(err, "Failed to delete VMs",
+				"vmList", vmList,
+			)
+
+			return false
+		}
+
+		return true
+	}
+
+	v.log.Info("not all protected PVCs have ownerReferences to the protected list of VMs")
+
+	return false
+}
+
+func (v *VRGInstance) IsAllProtectedPVCsOwnedByProtectedVMs() bool {
+	yes := true
+	vmList := v.instance.Spec.KubeObjectProtection.RecipeParameters[recipecore.VMList]
+
+	allPVCs := make([]corev1.PersistentVolumeClaim, 0, len(v.volRepPVCs)+len(v.volSyncPVCs))
+	allPVCs = append(allPVCs, v.volRepPVCs...)
+	allPVCs = append(allPVCs, v.volSyncPVCs...)
+
+	for idx := range allPVCs {
+		pvc := &allPVCs[idx]
+		log := logWithPvcName(v.log, pvc)
+
+		vmName, err := rmnutil.IsOwnedByVM(v.ctx, v.reconciler.Client, pvc, log)
+		if err != nil {
+			log.Error(err, "Skipping cleanup",
+				"pvc", pvc.Name,
+				"vm", vmName,
+				"reason", "invalid ownerReferences",
+				"action", "manual cleanup required")
+			return false
+		}
+
+		v.log.Info("PVC is owned by VM", "pvc", pvc.Name, "vm", vmName)
+
+		if !slices.Contains(vmList, vmName) {
+			v.log.Info("PVC is owned by a VM that is not in the protected VM list",
+				"pvc", pvc.Name,
+				"vm", vmName,
+				"protectedVMs", vmList)
+
+			return false
+		}
+	}
+
+	return yes
+}