diff --git a/controllers/actions.github.com/autoscalinglistener_controller.go b/controllers/actions.github.com/autoscalinglistener_controller.go index 5509946c85..51525a83ab 100644 --- a/controllers/actions.github.com/autoscalinglistener_controller.go +++ b/controllers/actions.github.com/autoscalinglistener_controller.go @@ -41,7 +41,6 @@ import ( const ( autoscalingListenerContainerName = "autoscaler" - autoscalingListenerOwnerKey = ".metadata.controller" autoscalingListenerFinalizerName = "autoscalinglistener.actions.github.com/finalizer" ) @@ -246,65 +245,6 @@ func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl. return ctrl.Result{}, nil } -// SetupWithManager sets up the controller with the Manager. -func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager) error { - groupVersionIndexer := func(rawObj client.Object) []string { - groupVersion := v1alpha1.GroupVersion.String() - owner := metav1.GetControllerOf(rawObj) - if owner == nil { - return nil - } - - // ...make sure it is owned by this controller - if owner.APIVersion != groupVersion || owner.Kind != "AutoscalingListener" { - return nil - } - - // ...and if so, return it - return []string{owner.Name} - } - - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Pod{}, autoscalingListenerOwnerKey, groupVersionIndexer); err != nil { - return err - } - - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.ServiceAccount{}, autoscalingListenerOwnerKey, groupVersionIndexer); err != nil { - return err - } - - labelBasedWatchFunc := func(obj client.Object) []reconcile.Request { - var requests []reconcile.Request - labels := obj.GetLabels() - namespace, ok := labels["auto-scaling-listener-namespace"] - if !ok { - return nil - } - - name, ok := labels["auto-scaling-listener-name"] - if !ok { - return nil - } - requests = append(requests, - reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: name, - Namespace: namespace, - }, - }, - ) - return requests - } - - return ctrl.NewControllerManagedBy(mgr). - For(&v1alpha1.AutoscalingListener{}). - Owns(&corev1.Pod{}). - Owns(&corev1.ServiceAccount{}). - Watches(&source.Kind{Type: &rbacv1.Role{}}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). - Watches(&source.Kind{Type: &rbacv1.RoleBinding{}}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). - WithEventFilter(predicate.ResourceVersionChangedPredicate{}). - Complete(r) -} - func (r *AutoscalingListenerReconciler) cleanupResources(ctx context.Context, autoscalingListener *v1alpha1.AutoscalingListener, logger logr.Logger) (done bool, err error) { logger.Info("Cleaning up the listener pod") listenerPod := new(corev1.Pod) @@ -615,3 +555,62 @@ func (r *AutoscalingListenerReconciler) createRoleBindingForListener(ctx context "serviceAccount", serviceAccount.Name) return ctrl.Result{Requeue: true}, nil } + +// SetupWithManager sets up the controller with the Manager. +func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager) error { + groupVersionIndexer := func(rawObj client.Object) []string { + groupVersion := v1alpha1.GroupVersion.String() + owner := metav1.GetControllerOf(rawObj) + if owner == nil { + return nil + } + + // ...make sure it is owned by this controller + if owner.APIVersion != groupVersion || owner.Kind != "AutoscalingListener" { + return nil + } + + // ...and if so, return it + return []string{owner.Name} + } + + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Pod{}, resourceOwnerKey, groupVersionIndexer); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.ServiceAccount{}, resourceOwnerKey, groupVersionIndexer); err != nil { + return err + } + + labelBasedWatchFunc := func(obj client.Object) []reconcile.Request { + var requests []reconcile.Request + labels := obj.GetLabels() + namespace, ok := labels["auto-scaling-listener-namespace"] + if !ok { + return nil + } + + name, ok := labels["auto-scaling-listener-name"] + if !ok { + return nil + } + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: name, + Namespace: namespace, + }, + }, + ) + return requests + } + + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.AutoscalingListener{}). + Owns(&corev1.Pod{}). + Owns(&corev1.ServiceAccount{}). + Watches(&source.Kind{Type: &rbacv1.Role{}}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). + Watches(&source.Kind{Type: &rbacv1.RoleBinding{}}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). + WithEventFilter(predicate.ResourceVersionChangedPredicate{}). + Complete(r) +} diff --git a/controllers/actions.github.com/autoscalinglistener_controller_test.go b/controllers/actions.github.com/autoscalinglistener_controller_test.go index d493279761..882f4b231a 100644 --- a/controllers/actions.github.com/autoscalinglistener_controller_test.go +++ b/controllers/actions.github.com/autoscalinglistener_controller_test.go @@ -213,7 +213,7 @@ var _ = Describe("Test AutoScalingListener controller", func() { Eventually( func() error { podList := new(corev1.PodList) - err := k8sClient.List(ctx, podList, client.InNamespace(autoscalingListener.Namespace), client.MatchingFields{autoscalingRunnerSetOwnerKey: autoscalingListener.Name}) + err := k8sClient.List(ctx, podList, client.InNamespace(autoscalingListener.Namespace), client.MatchingFields{resourceOwnerKey: autoscalingListener.Name}) if err != nil { return err } @@ -231,7 +231,7 @@ var _ = Describe("Test AutoScalingListener controller", func() { Eventually( func() error { serviceAccountList := new(corev1.ServiceAccountList) - err := k8sClient.List(ctx, serviceAccountList, client.InNamespace(autoscalingListener.Namespace), client.MatchingFields{autoscalingRunnerSetOwnerKey: autoscalingListener.Name}) + err := k8sClient.List(ctx, serviceAccountList, client.InNamespace(autoscalingListener.Namespace), client.MatchingFields{resourceOwnerKey: autoscalingListener.Name}) if err != nil { return err } diff --git a/controllers/actions.github.com/autoscalingrunnerset_controller.go b/controllers/actions.github.com/autoscalingrunnerset_controller.go index 903acb06e3..16c20442b8 100644 --- a/controllers/actions.github.com/autoscalingrunnerset_controller.go +++ b/controllers/actions.github.com/autoscalingrunnerset_controller.go @@ -42,13 +42,10 @@ import ( ) const ( - // TODO: Replace with shared image. - autoscalingRunnerSetOwnerKey = ".metadata.controller" - LabelKeyRunnerSpecHash = "runner-spec-hash" - autoscalingRunnerSetFinalizerName = "autoscalingrunnerset.actions.github.com/finalizer" - runnerScaleSetIdAnnotationKey = "runner-scale-set-id" - runnerScaleSetNameAnnotationKey = "runner-scale-set-name" - autoscalingRunnerSetCleanupFinalizerName = "actions.github.com/cleanup-protection" + labelKeyRunnerSpecHash = "runner-spec-hash" + autoscalingRunnerSetFinalizerName = "autoscalingrunnerset.actions.github.com/finalizer" + runnerScaleSetIdAnnotationKey = "runner-scale-set-id" + runnerScaleSetNameAnnotationKey = "runner-scale-set-name" ) // AutoscalingRunnerSetReconciler reconciles a AutoscalingRunnerSet object @@ -201,10 +198,10 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl desiredSpecHash := autoscalingRunnerSet.RunnerSetSpecHash() for _, runnerSet := range existingRunnerSets.all() { - log.Info("Find existing ephemeral runner set", "name", runnerSet.Name, "specHash", runnerSet.Labels[LabelKeyRunnerSpecHash]) + log.Info("Find existing ephemeral runner set", "name", runnerSet.Name, "specHash", runnerSet.Labels[labelKeyRunnerSpecHash]) } - if desiredSpecHash != latestRunnerSet.Labels[LabelKeyRunnerSpecHash] { + if desiredSpecHash != latestRunnerSet.Labels[labelKeyRunnerSpecHash] { log.Info("Latest runner set spec hash does not match the current autoscaling runner set. Creating a new runner set") return r.createEphemeralRunnerSet(ctx, autoscalingRunnerSet, log) } @@ -232,7 +229,7 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl } // Our listener pod is out of date, so we need to delete it to get a new recreate. - if listener.Labels[LabelKeyRunnerSpecHash] != autoscalingRunnerSet.ListenerSpecHash() { + if listener.Labels[labelKeyRunnerSpecHash] != autoscalingRunnerSet.ListenerSpecHash() { log.Info("RunnerScaleSetListener is out of date. Deleting it so that it is recreated", "name", listener.Name) if err := r.Delete(ctx, listener); err != nil { if kerrors.IsNotFound(err) { @@ -601,7 +598,7 @@ func (r *AutoscalingRunnerSetReconciler) createAutoScalingListenerForRunnerSet(c func (r *AutoscalingRunnerSetReconciler) listEphemeralRunnerSets(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet) (*EphemeralRunnerSets, error) { list := new(v1alpha1.EphemeralRunnerSetList) - if err := r.List(ctx, list, client.InNamespace(autoscalingRunnerSet.Namespace), client.MatchingFields{autoscalingRunnerSetOwnerKey: autoscalingRunnerSet.Name}); err != nil { + if err := r.List(ctx, list, client.InNamespace(autoscalingRunnerSet.Namespace), client.MatchingFields{resourceOwnerKey: autoscalingRunnerSet.Name}); err != nil { return nil, fmt.Errorf("failed to list ephemeral runner sets: %v", err) } @@ -694,7 +691,7 @@ func (r *AutoscalingRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) erro return []string{owner.Name} } - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &v1alpha1.EphemeralRunnerSet{}, autoscalingRunnerSetOwnerKey, groupVersionIndexer); err != nil { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &v1alpha1.EphemeralRunnerSet{}, resourceOwnerKey, groupVersionIndexer); err != nil { return err } @@ -754,12 +751,12 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeRol err := c.client.Get(ctx, types.NamespacedName{Name: roleBindingName, Namespace: c.autoscalingRunnerSet.Namespace}, roleBinding) switch { case err == nil: - if !controllerutil.ContainsFinalizer(roleBinding, autoscalingRunnerSetCleanupFinalizerName) { + if !controllerutil.ContainsFinalizer(roleBinding, AutoscalingRunnerSetCleanupFinalizerName) { c.logger.Info("Kubernetes mode role binding finalizer has already been removed", "name", roleBindingName) return } err = patch(ctx, c.client, roleBinding, func(obj *rbacv1.RoleBinding) { - controllerutil.RemoveFinalizer(obj, autoscalingRunnerSetCleanupFinalizerName) + controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName) }) if err != nil { c.err = fmt.Errorf("failed to patch kubernetes mode role binding without finalizer: %w", err) @@ -797,12 +794,12 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeRol err := c.client.Get(ctx, types.NamespacedName{Name: roleName, Namespace: c.autoscalingRunnerSet.Namespace}, role) switch { case err == nil: - if !controllerutil.ContainsFinalizer(role, autoscalingRunnerSetCleanupFinalizerName) { + if !controllerutil.ContainsFinalizer(role, AutoscalingRunnerSetCleanupFinalizerName) { c.logger.Info("Kubernetes mode role finalizer has already been removed", "name", roleName) return } err = patch(ctx, c.client, role, func(obj *rbacv1.Role) { - controllerutil.RemoveFinalizer(obj, autoscalingRunnerSetCleanupFinalizerName) + controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName) }) if err != nil { c.err = fmt.Errorf("failed to patch kubernetes mode role without finalizer: %w", err) @@ -841,12 +838,12 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeKubernetesModeSer err := c.client.Get(ctx, types.NamespacedName{Name: serviceAccountName, Namespace: c.autoscalingRunnerSet.Namespace}, serviceAccount) switch { case err == nil: - if !controllerutil.ContainsFinalizer(serviceAccount, autoscalingRunnerSetCleanupFinalizerName) { + if !controllerutil.ContainsFinalizer(serviceAccount, AutoscalingRunnerSetCleanupFinalizerName) { c.logger.Info("Kubernetes mode service account finalizer has already been removed", "name", serviceAccountName) return } err = patch(ctx, c.client, serviceAccount, func(obj *corev1.ServiceAccount) { - controllerutil.RemoveFinalizer(obj, autoscalingRunnerSetCleanupFinalizerName) + controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName) }) if err != nil { c.err = fmt.Errorf("failed to patch kubernetes mode service account without finalizer: %w", err) @@ -885,12 +882,12 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeNoPermissionServi err := c.client.Get(ctx, types.NamespacedName{Name: serviceAccountName, Namespace: c.autoscalingRunnerSet.Namespace}, serviceAccount) switch { case err == nil: - if !controllerutil.ContainsFinalizer(serviceAccount, autoscalingRunnerSetCleanupFinalizerName) { + if !controllerutil.ContainsFinalizer(serviceAccount, AutoscalingRunnerSetCleanupFinalizerName) { c.logger.Info("No permission service account finalizer has already been removed", "name", serviceAccountName) return } err = patch(ctx, c.client, serviceAccount, func(obj *corev1.ServiceAccount) { - controllerutil.RemoveFinalizer(obj, autoscalingRunnerSetCleanupFinalizerName) + controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName) }) if err != nil { c.err = fmt.Errorf("failed to patch service account without finalizer: %w", err) @@ -929,12 +926,12 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeGitHubSecretFinal err := c.client.Get(ctx, types.NamespacedName{Name: githubSecretName, Namespace: c.autoscalingRunnerSet.Namespace}, githubSecret) switch { case err == nil: - if !controllerutil.ContainsFinalizer(githubSecret, autoscalingRunnerSetCleanupFinalizerName) { + if !controllerutil.ContainsFinalizer(githubSecret, AutoscalingRunnerSetCleanupFinalizerName) { c.logger.Info("GitHub secret finalizer has already been removed", "name", githubSecretName) return } err = patch(ctx, c.client, githubSecret, func(obj *corev1.Secret) { - controllerutil.RemoveFinalizer(obj, autoscalingRunnerSetCleanupFinalizerName) + controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName) }) if err != nil { c.err = fmt.Errorf("failed to patch GitHub secret without finalizer: %w", err) @@ -973,12 +970,12 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleBindin err := c.client.Get(ctx, types.NamespacedName{Name: managerRoleBindingName, Namespace: c.autoscalingRunnerSet.Namespace}, roleBinding) switch { case err == nil: - if !controllerutil.ContainsFinalizer(roleBinding, autoscalingRunnerSetCleanupFinalizerName) { + if !controllerutil.ContainsFinalizer(roleBinding, AutoscalingRunnerSetCleanupFinalizerName) { c.logger.Info("Manager role binding finalizer has already been removed", "name", managerRoleBindingName) return } err = patch(ctx, c.client, roleBinding, func(obj *rbacv1.RoleBinding) { - controllerutil.RemoveFinalizer(obj, autoscalingRunnerSetCleanupFinalizerName) + controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName) }) if err != nil { c.err = fmt.Errorf("failed to patch manager role binding without finalizer: %w", err) @@ -1017,12 +1014,12 @@ func (c *autoscalingRunnerSetFinalizerDependencyCleaner) removeManagerRoleFinali err := c.client.Get(ctx, types.NamespacedName{Name: managerRoleName, Namespace: c.autoscalingRunnerSet.Namespace}, role) switch { case err == nil: - if !controllerutil.ContainsFinalizer(role, autoscalingRunnerSetCleanupFinalizerName) { + if !controllerutil.ContainsFinalizer(role, AutoscalingRunnerSetCleanupFinalizerName) { c.logger.Info("Manager role finalizer has already been removed", "name", managerRoleName) return } err = patch(ctx, c.client, role, func(obj *rbacv1.Role) { - controllerutil.RemoveFinalizer(obj, autoscalingRunnerSetCleanupFinalizerName) + controllerutil.RemoveFinalizer(obj, AutoscalingRunnerSetCleanupFinalizerName) }) if err != nil { c.err = fmt.Errorf("failed to patch manager role without finalizer: %w", err) diff --git a/controllers/actions.github.com/autoscalingrunnerset_controller_test.go b/controllers/actions.github.com/autoscalingrunnerset_controller_test.go index 6ad2f18a1e..a53732e392 100644 --- a/controllers/actions.github.com/autoscalingrunnerset_controller_test.go +++ b/controllers/actions.github.com/autoscalingrunnerset_controller_test.go @@ -280,10 +280,10 @@ var _ = Describe("Test AutoScalingRunnerSet controller", func() { return "", fmt.Errorf("We should have only 1 EphemeralRunnerSet, but got %v", len(runnerSetList.Items)) } - return runnerSetList.Items[0].Labels[LabelKeyRunnerSpecHash], nil + return runnerSetList.Items[0].Labels[labelKeyRunnerSpecHash], nil }, autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).ShouldNot(BeEquivalentTo(runnerSet.Labels[LabelKeyRunnerSpecHash]), "New EphemeralRunnerSet should be created") + autoscalingRunnerSetTestInterval).ShouldNot(BeEquivalentTo(runnerSet.Labels[labelKeyRunnerSpecHash]), "New EphemeralRunnerSet should be created") // We should create a new listener Eventually( @@ -1160,7 +1160,7 @@ var _ = Describe("Test external permissions cleanup", func() { ObjectMeta: metav1.ObjectMeta{ Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeRoleName], Namespace: autoscalingRunnerSet.Namespace, - Finalizers: []string{autoscalingRunnerSetCleanupFinalizerName}, + Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName}, }, } @@ -1171,7 +1171,7 @@ var _ = Describe("Test external permissions cleanup", func() { ObjectMeta: metav1.ObjectMeta{ Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeServiceAccountName], Namespace: autoscalingRunnerSet.Namespace, - Finalizers: []string{autoscalingRunnerSetCleanupFinalizerName}, + Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName}, }, } @@ -1182,7 +1182,7 @@ var _ = Describe("Test external permissions cleanup", func() { ObjectMeta: metav1.ObjectMeta{ Name: autoscalingRunnerSet.Annotations[AnnotationKeyKubernetesModeRoleBindingName], Namespace: autoscalingRunnerSet.Namespace, - Finalizers: []string{autoscalingRunnerSetCleanupFinalizerName}, + Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName}, }, Subjects: []rbacv1.Subject{ { @@ -1317,7 +1317,7 @@ var _ = Describe("Test external permissions cleanup", func() { ObjectMeta: metav1.ObjectMeta{ Name: autoscalingRunnerSet.Annotations[AnnotationKeyGitHubSecretName], Namespace: autoscalingRunnerSet.Namespace, - Finalizers: []string{autoscalingRunnerSetCleanupFinalizerName}, + Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName}, }, Data: map[string][]byte{ "github_token": []byte(defaultGitHubToken), @@ -1333,7 +1333,7 @@ var _ = Describe("Test external permissions cleanup", func() { ObjectMeta: metav1.ObjectMeta{ Name: autoscalingRunnerSet.Annotations[AnnotationKeyManagerRoleName], Namespace: autoscalingRunnerSet.Namespace, - Finalizers: []string{autoscalingRunnerSetCleanupFinalizerName}, + Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName}, }, } @@ -1344,7 +1344,7 @@ var _ = Describe("Test external permissions cleanup", func() { ObjectMeta: metav1.ObjectMeta{ Name: autoscalingRunnerSet.Annotations[AnnotationKeyManagerRoleBindingName], Namespace: autoscalingRunnerSet.Namespace, - Finalizers: []string{autoscalingRunnerSetCleanupFinalizerName}, + Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName}, }, RoleRef: rbacv1.RoleRef{ APIGroup: rbacv1.GroupName, @@ -1360,7 +1360,7 @@ var _ = Describe("Test external permissions cleanup", func() { ObjectMeta: metav1.ObjectMeta{ Name: autoscalingRunnerSet.Annotations[AnnotationKeyNoPermissionServiceAccountName], Namespace: autoscalingRunnerSet.Namespace, - Finalizers: []string{autoscalingRunnerSetCleanupFinalizerName}, + Finalizers: []string{AutoscalingRunnerSetCleanupFinalizerName}, }, } diff --git a/controllers/actions.github.com/constants.go b/controllers/actions.github.com/constants.go index 613db79c07..339c39d9e0 100644 --- a/controllers/actions.github.com/constants.go +++ b/controllers/actions.github.com/constants.go @@ -1,5 +1,7 @@ package actionsgithubcom +import corev1 "k8s.io/api/core/v1" + const ( LabelKeyRunnerTemplateHash = "runner-template-hash" LabelKeyPodTemplateHash = "pod-template-hash" @@ -16,3 +18,47 @@ const ( EnvVarHTTPSProxy = "https_proxy" EnvVarNoProxy = "no_proxy" ) + +// Labels applied to resources +const ( + // Kubernetes labels + LabelKeyKubernetesPartOf = "app.kubernetes.io/part-of" + LabelKeyKubernetesComponent = "app.kubernetes.io/component" + LabelKeyKubernetesVersion = "app.kubernetes.io/version" + + // Github labels + LabelKeyGitHubScaleSetName = "actions.github.com/scale-set-name" + LabelKeyGitHubScaleSetNamespace = "actions.github.com/scale-set-namespace" + LabelKeyGitHubEnterprise = "actions.github.com/enterprise" + LabelKeyGitHubOrganization = "actions.github.com/organization" + LabelKeyGitHubRepository = "actions.github.com/repository" +) + +// Finalizer used to protect resources from deletion while AutoscalingRunnerSet is running +const AutoscalingRunnerSetCleanupFinalizerName = "actions.github.com/cleanup-protection" + +const AnnotationKeyGitHubRunnerGroupName = "actions.github.com/runner-group-name" + +// Labels applied to listener roles +const ( + labelKeyListenerName = "auto-scaling-listener-name" + labelKeyListenerNamespace = "auto-scaling-listener-namespace" +) + +// Annotations applied for later cleanup of resources +const ( + AnnotationKeyManagerRoleBindingName = "actions.github.com/cleanup-manager-role-binding" + AnnotationKeyManagerRoleName = "actions.github.com/cleanup-manager-role-name" + AnnotationKeyKubernetesModeRoleName = "actions.github.com/cleanup-kubernetes-mode-role-name" + AnnotationKeyKubernetesModeRoleBindingName = "actions.github.com/cleanup-kubernetes-mode-role-binding-name" + AnnotationKeyKubernetesModeServiceAccountName = "actions.github.com/cleanup-kubernetes-mode-service-account-name" + AnnotationKeyGitHubSecretName = "actions.github.com/cleanup-github-secret-name" + AnnotationKeyNoPermissionServiceAccountName = "actions.github.com/cleanup-no-permission-service-account-name" +) + +// DefaultScaleSetListenerImagePullPolicy is the default pull policy applied +// to the listener when ImagePullPolicy is not specified +const DefaultScaleSetListenerImagePullPolicy = corev1.PullIfNotPresent + +// ownerKey is field selector matching the owner name of a particular resource +const resourceOwnerKey = ".metadata.controller" diff --git a/controllers/actions.github.com/ephemeralrunnerset_controller.go b/controllers/actions.github.com/ephemeralrunnerset_controller.go index f4850cf5e3..dcc6a525aa 100644 --- a/controllers/actions.github.com/ephemeralrunnerset_controller.go +++ b/controllers/actions.github.com/ephemeralrunnerset_controller.go @@ -40,8 +40,7 @@ import ( ) const ( - ephemeralRunnerSetReconcilerOwnerKey = ".metadata.controller" - ephemeralRunnerSetFinalizerName = "ephemeralrunner.actions.github.com/finalizer" + ephemeralRunnerSetFinalizerName = "ephemeralrunner.actions.github.com/finalizer" ) // EphemeralRunnerSetReconciler reconciles a EphemeralRunnerSet object @@ -147,7 +146,7 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R ctx, ephemeralRunnerList, client.InNamespace(req.Namespace), - client.MatchingFields{ephemeralRunnerSetReconcilerOwnerKey: req.Name}, + client.MatchingFields{resourceOwnerKey: req.Name}, ) if err != nil { log.Error(err, "Unable to list child ephemeral runners") @@ -243,7 +242,7 @@ func (r *EphemeralRunnerSetReconciler) cleanUpProxySecret(ctx context.Context, e func (r *EphemeralRunnerSetReconciler) cleanUpEphemeralRunners(ctx context.Context, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, log logr.Logger) (bool, error) { ephemeralRunnerList := new(v1alpha1.EphemeralRunnerList) - err := r.List(ctx, ephemeralRunnerList, client.InNamespace(ephemeralRunnerSet.Namespace), client.MatchingFields{ephemeralRunnerSetReconcilerOwnerKey: ephemeralRunnerSet.Name}) + err := r.List(ctx, ephemeralRunnerList, client.InNamespace(ephemeralRunnerSet.Namespace), client.MatchingFields{resourceOwnerKey: ephemeralRunnerSet.Name}) if err != nil { return false, fmt.Errorf("failed to list child ephemeral runners: %v", err) } @@ -522,7 +521,7 @@ func (r *EphemeralRunnerSetReconciler) actionsClientOptionsFor(ctx context.Conte // SetupWithManager sets up the controller with the Manager. func (r *EphemeralRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) error { // Index EphemeralRunner owned by EphemeralRunnerSet so we can perform faster look ups. - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &v1alpha1.EphemeralRunner{}, ephemeralRunnerSetReconcilerOwnerKey, func(rawObj client.Object) []string { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &v1alpha1.EphemeralRunner{}, resourceOwnerKey, func(rawObj client.Object) []string { groupVersion := v1alpha1.GroupVersion.String() // grab the job object, extract the owner... diff --git a/controllers/actions.github.com/resourcebuilder.go b/controllers/actions.github.com/resourcebuilder.go index 2ddba11b13..4a5bccc46b 100644 --- a/controllers/actions.github.com/resourcebuilder.go +++ b/controllers/actions.github.com/resourcebuilder.go @@ -20,40 +20,6 @@ const ( jitTokenKey = "jitToken" ) -// Labels applied to resources -const ( - // Kubernetes labels - LabelKeyKubernetesPartOf = "app.kubernetes.io/part-of" - LabelKeyKubernetesComponent = "app.kubernetes.io/component" - LabelKeyKubernetesVersion = "app.kubernetes.io/version" - - // Github labels - LabelKeyGitHubScaleSetName = "actions.github.com/scale-set-name" - LabelKeyGitHubScaleSetNamespace = "actions.github.com/scale-set-namespace" - LabelKeyGitHubEnterprise = "actions.github.com/enterprise" - LabelKeyGitHubOrganization = "actions.github.com/organization" - LabelKeyGitHubRepository = "actions.github.com/repository" -) - -const AnnotationKeyGitHubRunnerGroupName = "actions.github.com/runner-group-name" - -// Labels applied to listener roles -const ( - labelKeyListenerName = "auto-scaling-listener-name" - labelKeyListenerNamespace = "auto-scaling-listener-namespace" -) - -// Annotations applied for later cleanup of resources -const ( - AnnotationKeyManagerRoleBindingName = "actions.github.com/cleanup-manager-role-binding" - AnnotationKeyManagerRoleName = "actions.github.com/cleanup-manager-role-name" - AnnotationKeyKubernetesModeRoleName = "actions.github.com/cleanup-kubernetes-mode-role-name" - AnnotationKeyKubernetesModeRoleBindingName = "actions.github.com/cleanup-kubernetes-mode-role-binding-name" - AnnotationKeyKubernetesModeServiceAccountName = "actions.github.com/cleanup-kubernetes-mode-service-account-name" - AnnotationKeyGitHubSecretName = "actions.github.com/cleanup-github-secret-name" - AnnotationKeyNoPermissionServiceAccountName = "actions.github.com/cleanup-no-permission-service-account-name" -) - var commonLabelKeys = [...]string{ LabelKeyKubernetesPartOf, LabelKeyKubernetesComponent, @@ -67,8 +33,6 @@ var commonLabelKeys = [...]string{ const labelValueKubernetesPartOf = "gha-runner-scale-set" -const DefaultScaleSetListenerImagePullPolicy = corev1.PullIfNotPresent - // scaleSetListenerImagePullPolicy is applied to all listeners var scaleSetListenerImagePullPolicy = DefaultScaleSetListenerImagePullPolicy @@ -84,6 +48,62 @@ func SetListenerImagePullPolicy(pullPolicy string) bool { type resourceBuilder struct{} +func (b *resourceBuilder) newAutoScalingListener(autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, namespace, image string, imagePullSecrets []corev1.LocalObjectReference) (*v1alpha1.AutoscalingListener, error) { + runnerScaleSetId, err := strconv.Atoi(autoscalingRunnerSet.Annotations[runnerScaleSetIdAnnotationKey]) + if err != nil { + return nil, err + } + + effectiveMinRunners := 0 + effectiveMaxRunners := math.MaxInt32 + if autoscalingRunnerSet.Spec.MaxRunners != nil { + effectiveMaxRunners = *autoscalingRunnerSet.Spec.MaxRunners + } + if autoscalingRunnerSet.Spec.MinRunners != nil { + effectiveMinRunners = *autoscalingRunnerSet.Spec.MinRunners + } + + githubConfig, err := actions.ParseGitHubConfigFromURL(autoscalingRunnerSet.Spec.GitHubConfigUrl) + if err != nil { + return nil, fmt.Errorf("failed to parse github config from url: %v", err) + } + + autoscalingListener := &v1alpha1.AutoscalingListener{ + ObjectMeta: metav1.ObjectMeta{ + Name: scaleSetListenerName(autoscalingRunnerSet), + Namespace: namespace, + Labels: map[string]string{ + LabelKeyGitHubScaleSetNamespace: autoscalingRunnerSet.Namespace, + LabelKeyGitHubScaleSetName: autoscalingRunnerSet.Name, + LabelKeyKubernetesPartOf: labelValueKubernetesPartOf, + LabelKeyKubernetesComponent: "runner-scale-set-listener", + LabelKeyKubernetesVersion: autoscalingRunnerSet.Labels[LabelKeyKubernetesVersion], + LabelKeyGitHubEnterprise: githubConfig.Enterprise, + LabelKeyGitHubOrganization: githubConfig.Organization, + LabelKeyGitHubRepository: githubConfig.Repository, + labelKeyRunnerSpecHash: autoscalingRunnerSet.ListenerSpecHash(), + }, + }, + Spec: v1alpha1.AutoscalingListenerSpec{ + GitHubConfigUrl: autoscalingRunnerSet.Spec.GitHubConfigUrl, + GitHubConfigSecret: autoscalingRunnerSet.Spec.GitHubConfigSecret, + RunnerScaleSetId: runnerScaleSetId, + AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, + AutoscalingRunnerSetName: autoscalingRunnerSet.Name, + EphemeralRunnerSetName: ephemeralRunnerSet.Name, + MinRunners: effectiveMinRunners, + MaxRunners: effectiveMaxRunners, + Image: image, + ImagePullPolicy: scaleSetListenerImagePullPolicy, + ImagePullSecrets: imagePullSecrets, + Proxy: autoscalingRunnerSet.Spec.Proxy, + GitHubServerTLS: autoscalingRunnerSet.Spec.GitHubServerTLS, + }, + } + + return autoscalingListener, nil +} + func (b *resourceBuilder) newScaleSetListenerPod(autoscalingListener *v1alpha1.AutoscalingListener, serviceAccount *corev1.ServiceAccount, secret *corev1.Secret, envs ...corev1.EnvVar) *corev1.Pod { listenerEnv := []corev1.EnvVar{ { @@ -207,54 +227,6 @@ func (b *resourceBuilder) newScaleSetListenerPod(autoscalingListener *v1alpha1.A return newRunnerScaleSetListenerPod } -func (b *resourceBuilder) newEphemeralRunnerSet(autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet) (*v1alpha1.EphemeralRunnerSet, error) { - runnerScaleSetId, err := strconv.Atoi(autoscalingRunnerSet.Annotations[runnerScaleSetIdAnnotationKey]) - if err != nil { - return nil, err - } - runnerSpecHash := autoscalingRunnerSet.RunnerSetSpecHash() - - newLabels := map[string]string{ - LabelKeyRunnerSpecHash: runnerSpecHash, - LabelKeyKubernetesPartOf: labelValueKubernetesPartOf, - LabelKeyKubernetesComponent: "runner-set", - LabelKeyKubernetesVersion: autoscalingRunnerSet.Labels[LabelKeyKubernetesVersion], - LabelKeyGitHubScaleSetName: autoscalingRunnerSet.Name, - LabelKeyGitHubScaleSetNamespace: autoscalingRunnerSet.Namespace, - } - - if err := applyGitHubURLLabels(autoscalingRunnerSet.Spec.GitHubConfigUrl, newLabels); err != nil { - return nil, fmt.Errorf("failed to apply GitHub URL labels: %v", err) - } - - newAnnotations := map[string]string{ - AnnotationKeyGitHubRunnerGroupName: autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerGroupName], - } - - newEphemeralRunnerSet := &v1alpha1.EphemeralRunnerSet{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: autoscalingRunnerSet.ObjectMeta.Name + "-", - Namespace: autoscalingRunnerSet.ObjectMeta.Namespace, - Labels: newLabels, - Annotations: newAnnotations, - }, - Spec: v1alpha1.EphemeralRunnerSetSpec{ - Replicas: 0, - EphemeralRunnerSpec: v1alpha1.EphemeralRunnerSpec{ - RunnerScaleSetId: runnerScaleSetId, - GitHubConfigUrl: autoscalingRunnerSet.Spec.GitHubConfigUrl, - GitHubConfigSecret: autoscalingRunnerSet.Spec.GitHubConfigSecret, - Proxy: autoscalingRunnerSet.Spec.Proxy, - GitHubServerTLS: autoscalingRunnerSet.Spec.GitHubServerTLS, - PodTemplateSpec: autoscalingRunnerSet.Spec.Template, - }, - }, - } - - return newEphemeralRunnerSet, nil -} - func (b *resourceBuilder) newScaleSetListenerServiceAccount(autoscalingListener *v1alpha1.AutoscalingListener) *corev1.ServiceAccount { return &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ @@ -344,60 +316,52 @@ func (b *resourceBuilder) newScaleSetListenerSecretMirror(autoscalingListener *v return newListenerSecret } -func (b *resourceBuilder) newAutoScalingListener(autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, namespace, image string, imagePullSecrets []corev1.LocalObjectReference) (*v1alpha1.AutoscalingListener, error) { +func (b *resourceBuilder) newEphemeralRunnerSet(autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet) (*v1alpha1.EphemeralRunnerSet, error) { runnerScaleSetId, err := strconv.Atoi(autoscalingRunnerSet.Annotations[runnerScaleSetIdAnnotationKey]) if err != nil { return nil, err } + runnerSpecHash := autoscalingRunnerSet.RunnerSetSpecHash() - effectiveMinRunners := 0 - effectiveMaxRunners := math.MaxInt32 - if autoscalingRunnerSet.Spec.MaxRunners != nil { - effectiveMaxRunners = *autoscalingRunnerSet.Spec.MaxRunners + newLabels := map[string]string{ + labelKeyRunnerSpecHash: runnerSpecHash, + LabelKeyKubernetesPartOf: labelValueKubernetesPartOf, + LabelKeyKubernetesComponent: "runner-set", + LabelKeyKubernetesVersion: autoscalingRunnerSet.Labels[LabelKeyKubernetesVersion], + LabelKeyGitHubScaleSetName: autoscalingRunnerSet.Name, + LabelKeyGitHubScaleSetNamespace: autoscalingRunnerSet.Namespace, } - if autoscalingRunnerSet.Spec.MinRunners != nil { - effectiveMinRunners = *autoscalingRunnerSet.Spec.MinRunners + + if err := applyGitHubURLLabels(autoscalingRunnerSet.Spec.GitHubConfigUrl, newLabels); err != nil { + return nil, fmt.Errorf("failed to apply GitHub URL labels: %v", err) } - githubConfig, err := actions.ParseGitHubConfigFromURL(autoscalingRunnerSet.Spec.GitHubConfigUrl) - if err != nil { - return nil, fmt.Errorf("failed to parse github config from url: %v", err) + newAnnotations := map[string]string{ + AnnotationKeyGitHubRunnerGroupName: autoscalingRunnerSet.Annotations[AnnotationKeyGitHubRunnerGroupName], } - autoscalingListener := &v1alpha1.AutoscalingListener{ + newEphemeralRunnerSet := &v1alpha1.EphemeralRunnerSet{ + TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ - Name: scaleSetListenerName(autoscalingRunnerSet), - Namespace: namespace, - Labels: map[string]string{ - LabelKeyGitHubScaleSetNamespace: autoscalingRunnerSet.Namespace, - LabelKeyGitHubScaleSetName: autoscalingRunnerSet.Name, - LabelKeyKubernetesPartOf: labelValueKubernetesPartOf, - LabelKeyKubernetesComponent: "runner-scale-set-listener", - LabelKeyKubernetesVersion: autoscalingRunnerSet.Labels[LabelKeyKubernetesVersion], - LabelKeyGitHubEnterprise: githubConfig.Enterprise, - LabelKeyGitHubOrganization: githubConfig.Organization, - LabelKeyGitHubRepository: githubConfig.Repository, - LabelKeyRunnerSpecHash: autoscalingRunnerSet.ListenerSpecHash(), - }, + GenerateName: autoscalingRunnerSet.ObjectMeta.Name + "-", + Namespace: autoscalingRunnerSet.ObjectMeta.Namespace, + Labels: newLabels, + Annotations: newAnnotations, }, - Spec: v1alpha1.AutoscalingListenerSpec{ - GitHubConfigUrl: autoscalingRunnerSet.Spec.GitHubConfigUrl, - GitHubConfigSecret: autoscalingRunnerSet.Spec.GitHubConfigSecret, - RunnerScaleSetId: runnerScaleSetId, - AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, - AutoscalingRunnerSetName: autoscalingRunnerSet.Name, - EphemeralRunnerSetName: ephemeralRunnerSet.Name, - MinRunners: effectiveMinRunners, - MaxRunners: effectiveMaxRunners, - Image: image, - ImagePullPolicy: scaleSetListenerImagePullPolicy, - ImagePullSecrets: imagePullSecrets, - Proxy: autoscalingRunnerSet.Spec.Proxy, - GitHubServerTLS: autoscalingRunnerSet.Spec.GitHubServerTLS, + Spec: v1alpha1.EphemeralRunnerSetSpec{ + Replicas: 0, + EphemeralRunnerSpec: v1alpha1.EphemeralRunnerSpec{ + RunnerScaleSetId: runnerScaleSetId, + GitHubConfigUrl: autoscalingRunnerSet.Spec.GitHubConfigUrl, + GitHubConfigSecret: autoscalingRunnerSet.Spec.GitHubConfigSecret, + Proxy: autoscalingRunnerSet.Spec.Proxy, + GitHubServerTLS: autoscalingRunnerSet.Spec.GitHubServerTLS, + PodTemplateSpec: autoscalingRunnerSet.Spec.Template, + }, }, } - return autoscalingListener, nil + return newEphemeralRunnerSet, nil } func (b *resourceBuilder) newEphemeralRunner(ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet) *v1alpha1.EphemeralRunner { diff --git a/controllers/actions.github.com/resourcebuilder_test.go b/controllers/actions.github.com/resourcebuilder_test.go index e41d798006..925ba5cf43 100644 --- a/controllers/actions.github.com/resourcebuilder_test.go +++ b/controllers/actions.github.com/resourcebuilder_test.go @@ -36,7 +36,7 @@ func TestLabelPropagation(t *testing.T) { assert.Equal(t, labelValueKubernetesPartOf, ephemeralRunnerSet.Labels[LabelKeyKubernetesPartOf]) assert.Equal(t, "runner-set", ephemeralRunnerSet.Labels[LabelKeyKubernetesComponent]) assert.Equal(t, autoscalingRunnerSet.Labels[LabelKeyKubernetesVersion], ephemeralRunnerSet.Labels[LabelKeyKubernetesVersion]) - assert.NotEmpty(t, ephemeralRunnerSet.Labels[LabelKeyRunnerSpecHash]) + assert.NotEmpty(t, ephemeralRunnerSet.Labels[labelKeyRunnerSpecHash]) assert.Equal(t, autoscalingRunnerSet.Name, ephemeralRunnerSet.Labels[LabelKeyGitHubScaleSetName]) assert.Equal(t, autoscalingRunnerSet.Namespace, ephemeralRunnerSet.Labels[LabelKeyGitHubScaleSetNamespace]) assert.Equal(t, "", ephemeralRunnerSet.Labels[LabelKeyGitHubEnterprise]) @@ -49,7 +49,7 @@ func TestLabelPropagation(t *testing.T) { assert.Equal(t, labelValueKubernetesPartOf, listener.Labels[LabelKeyKubernetesPartOf]) assert.Equal(t, "runner-scale-set-listener", listener.Labels[LabelKeyKubernetesComponent]) assert.Equal(t, autoscalingRunnerSet.Labels[LabelKeyKubernetesVersion], listener.Labels[LabelKeyKubernetesVersion]) - assert.NotEmpty(t, ephemeralRunnerSet.Labels[LabelKeyRunnerSpecHash]) + assert.NotEmpty(t, ephemeralRunnerSet.Labels[labelKeyRunnerSpecHash]) assert.Equal(t, autoscalingRunnerSet.Name, listener.Labels[LabelKeyGitHubScaleSetName]) assert.Equal(t, autoscalingRunnerSet.Namespace, listener.Labels[LabelKeyGitHubScaleSetNamespace]) assert.Equal(t, "", listener.Labels[LabelKeyGitHubEnterprise])