Remove Dependency on cluster-scoped resources (#173)

The commit introduces following changes-

- Replaced clusterrolebinding with rolebinding resource.
- The fsGroup: 1001 in the pod security context allows to set proper volume permissions in storage class
- The anyuid SCC allows the container to run as any user ID, removing the need to explicitly set RunAsUser
- Cleanup existing crb


Approved-by: rhdedgar

Approved-by: leseb

Author: VaishnaviHire

config/rbac/role.yaml

@@ -92,13 +92,9 @@ rules:
   resources:
   - clusterrolebindings
   verbs:
-  - create
   - delete
   - get
   - list
-  - patch
-  - update
-  - watch
 - apiGroups:
   - rbac.authorization.k8s.io
   resources:
@@ -107,6 +103,18 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - security.openshift.io
   resources:

controllers/kubebuilder_rbac.go

@@ -14,9 +14,12 @@ package controllers
 // ServiceAccount permissions - controller creates and manages service accounts for PVC permissions
 //+kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete
 
-//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;list;delete
 //+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles,verbs=get;list;watch
 
+// RoleBinding permissions - controller creates and manages role bindings for PVC permissions
+//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete
+
 //+kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=use
 //+kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,resourceNames=anyuid,verbs=use
 

controllers/manifests/base/kustomization.yaml

@@ -4,10 +4,10 @@ kind: Kustomization
 resources:
 - pvc.yaml
 - serviceaccount.yaml
-- scc-binding.yaml
 - service.yaml
 - networkpolicy.yaml
 - deployment.yaml
+- rolebinding.yaml
 
 labels:
 - includeSelectors: false

controllers/manifests/base/scc-binding.yaml renamed to controllers/manifests/base/rolebinding.yaml

@@ -1,7 +1,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
+kind: RoleBinding
 metadata:
-  name: crb
+  name: rb
 subjects:
 - kind: ServiceAccount
   name: sa

controllers/manifests/base/serviceaccount.yaml

@@ -4,7 +4,3 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: sa
-  annotations:
-    # This annotation is used by OpenShift to assign the anyuid SCC
-    # which allows the container to run as any user ID
-    openshift.io/scc: anyuid

controllers/resource_helper.go

@@ -375,7 +375,7 @@ done`, CABundleTempPath, CABundleSourceDir, fileList)
 		},
 		SecurityContext: &corev1.SecurityContext{
 			AllowPrivilegeEscalation: ptr.To(false),
-			RunAsNonRoot:             ptr.To(false),
+			RunAsNonRoot:             ptr.To(true),
 			Capabilities: &corev1.Capabilities{
 				Drop: []corev1.Capability{"ALL"},
 			},
@@ -390,7 +390,7 @@ func configurePodStorage(ctx context.Context, r *LlamaStackDistributionReconcile
 	}
 
 	// Configure storage volumes and init containers
-	configureStorage(instance, &podSpec, container.Image)
+	configureStorage(instance, &podSpec)
 
 	// Configure TLS CA bundle (with auto-detection support)
 	configureTLSCABundle(ctx, r, instance, &podSpec, container.Image)
@@ -405,16 +405,16 @@ func configurePodStorage(ctx context.Context, r *LlamaStackDistributionReconcile
 }
 
 // configureStorage handles storage volume configuration.
-func configureStorage(instance *llamav1alpha1.LlamaStackDistribution, podSpec *corev1.PodSpec, image string) {
+func configureStorage(instance *llamav1alpha1.LlamaStackDistribution, podSpec *corev1.PodSpec) {
 	if instance.Spec.Server.Storage != nil {
-		configurePersistentStorage(instance, podSpec, image)
+		configurePersistentStorage(instance, podSpec)
 	} else {
 		configureEmptyDirStorage(podSpec)
 	}
 }
 
 // configurePersistentStorage sets up PVC-based storage with init container for permissions.
-func configurePersistentStorage(instance *llamav1alpha1.LlamaStackDistribution, podSpec *corev1.PodSpec, image string) {
+func configurePersistentStorage(instance *llamav1alpha1.LlamaStackDistribution, podSpec *corev1.PodSpec) {
 	// Use PVC for persistent storage
 	podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{
 		Name: "lls-storage",
@@ -424,47 +424,6 @@ func configurePersistentStorage(instance *llamav1alpha1.LlamaStackDistribution,
 			},
 		},
 	})
-
-	// Add init container to fix permissions on the PVC mount.
-	mountPath := llamav1alpha1.DefaultMountPath
-	if instance.Spec.Server.Storage.MountPath != "" {
-		mountPath = instance.Spec.Server.Storage.MountPath
-	}
-
-	commands := []string{
-		fmt.Sprintf("mkdir -p %s 2>&1 || echo 'Warning: Could not create directory'", mountPath),
-		fmt.Sprintf("(chown 1001:0 %s 2>&1 || echo 'Warning: Could not change ownership')", mountPath),
-		fmt.Sprintf("ls -la %s 2>&1", mountPath),
-	}
-	command := strings.Join(commands, " && ")
-
-	initContainer := corev1.Container{
-		Name:            "update-pvc-permissions",
-		Image:           image,
-		ImagePullPolicy: corev1.PullAlways,
-		Command: []string{
-			"/bin/sh",
-			"-c",
-			// Try to set permissions, but don't fail if we can't
-			command,
-		},
-		VolumeMounts: []corev1.VolumeMount{
-			{
-				Name:      "lls-storage",
-				MountPath: mountPath,
-			},
-		},
-		SecurityContext: &corev1.SecurityContext{
-			RunAsUser:                ptr.To(int64(0)), // Run as root to be able to change ownership
-			RunAsGroup:               ptr.To(int64(0)),
-			AllowPrivilegeEscalation: ptr.To(false),
-			Capabilities: &corev1.Capabilities{
-				Drop: []corev1.Capability{"ALL"},
-			},
-		},
-	}
-
-	podSpec.InitContainers = append(podSpec.InitContainers, initContainer)
 }
 
 // configureEmptyDirStorage sets up temporary storage using emptyDir.
@@ -620,8 +579,9 @@ func configurePodOverrides(instance *llamav1alpha1.LlamaStackDistribution, podSp
 	}
 
 	// Set fsGroup to allow write access to mounted volumes
+	const defaultFSGroup = 1001
 	if podSpec.SecurityContext.FSGroup == nil {
-		podSpec.SecurityContext.FSGroup = ptr.To(int64(0))
+		podSpec.SecurityContext.FSGroup = ptr.To(int64(defaultFSGroup))
 	}
 
 	// Apply other pod overrides if specified

main.go

@@ -146,6 +146,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Perform one-time upgrade cleanup operations
+	if err := cluster.PerformUpgradeCleanup(ctx, setupClient); err != nil {
+		setupLog.Error(err, "failed to perform upgrade cleanup")
+		os.Exit(1)
+	}
+
 	if err := setupReconciler(ctx, setupClient, mgr, clusterInfo); err != nil {
 		setupLog.Error(err, "failed to set up reconciler")
 		os.Exit(1)

pkg/cluster/cluster.go

@@ -5,8 +5,12 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/go-logr/logr"
 	"github.com/llamastack/llama-stack-k8s-operator/pkg/deploy"
+	rbacv1 "k8s.io/api/rbac/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
 type ClusterInfo struct {
@@ -31,3 +35,70 @@ func NewClusterInfo(ctx context.Context, client client.Client, embeddedDistribut
 		DistributionImages: distributionImages,
 	}, nil
 }
+
+// PerformUpgradeCleanup performs one-time cleanup operations for seamless upgrades.
+func PerformUpgradeCleanup(ctx context.Context, client client.Client) error {
+	logger := log.FromContext(ctx).WithName("upgrade-cleanup")
+	logger.Info("Starting upgrade cleanup operations")
+
+	// Cleanup legacy ClusterRoleBindings from cluster-scoped to namespace-scoped migration
+	cleanupLegacyClusterRoleBindings(ctx, client, logger)
+	logger.Info("Upgrade cleanup operations completed successfully")
+	return nil
+}
+
+// cleanupLegacyClusterRoleBindings removes ClusterRoleBindings from previous operator versions.
+func cleanupLegacyClusterRoleBindings(ctx context.Context, client client.Client, logger logr.Logger) {
+	// List all ClusterRoleBindings
+	clusterRoleBindingList := &rbacv1.ClusterRoleBindingList{}
+	if err := client.List(ctx, clusterRoleBindingList); err != nil {
+		logger.V(1).Info("Unable to list ClusterRoleBindings for cleanup, skipping legacy cleanup", "error", err)
+		return
+	}
+
+	var crbsToDelete []*rbacv1.ClusterRoleBinding
+	for i := range clusterRoleBindingList.Items {
+		crb := &clusterRoleBindingList.Items[i]
+
+		if shouldDeleteLegacyClusterRoleBinding(crb) {
+			crbsToDelete = append(crbsToDelete, crb)
+		}
+	}
+
+	// Delete the identified ClusterRoleBindings
+	for _, crb := range crbsToDelete {
+		logger.Info("Cleaning up legacy ClusterRoleBinding from previous operator version",
+			"clusterRoleBinding", crb.Name)
+
+		if err := client.Delete(ctx, crb); err != nil {
+			if k8serrors.IsNotFound(err) {
+				// Already deleted, continue
+				continue
+			}
+			// Log the error but don't fail startup - the cleanup is best-effort
+			logger.Error(err, "Failed to delete legacy ClusterRoleBinding, continuing with startup",
+				"clusterRoleBinding", crb.Name)
+		}
+	}
+
+	if len(crbsToDelete) > 0 {
+		logger.Info("Successfully cleaned up legacy ClusterRoleBindings", "count", len(crbsToDelete))
+	}
+}
+
+// shouldDeleteLegacyClusterRoleBinding determines if a ClusterRoleBinding should be deleted.
+func shouldDeleteLegacyClusterRoleBinding(crb *rbacv1.ClusterRoleBinding) bool {
+	// Only delete ClusterRoleBindings that were created by our operator
+	if managedBy, exists := crb.Labels["app.kubernetes.io/managed-by"]; !exists || managedBy != "llama-stack-operator" {
+		return false
+	}
+
+	// Check if any subjects are ServiceAccounts in namespaces (namespace-scoped)
+	for _, subject := range crb.Subjects {
+		if subject.Kind == "ServiceAccount" && subject.Namespace != "" {
+			return true
+		}
+	}
+
+	return false
+}