Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions docs/docs/architecture/gateway.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,17 @@ After configuring a Route or Consumer, the operator deletes plugins it no longer

Plugins without this proof, including legacy plugins that were never adopted, are skipped. Deleting a Route or Consumer in Kong still deletes all plugins bound to it, including adopted legacy plugins.

## Consumer changes across zones

A cross-zone subscriber's ConsumeRoute references the proxy Route in the subscriber's zone. The JumperConfig with the subscriber's credentials, however, is built by the primary Route in the provider's zone and by failover secondary Routes. These Routes read all ConsumeRoutes with the same route name.

When a ConsumeRoute is created, changed or deleted, the Route controller therefore reconciles:

- the Route the ConsumeRoute references, and
- all primary and failover secondary Routes with the same name in the ConsumeRoute's environment.

Other proxy Routes with that name are not reconciled, because they only list their own consumers. Routes of other environments are never reconciled. A changed subscriber credential, such as a rotated client secret, reaches the provider zone's JumperConfig without waiting for the periodic resync.

## Domain Interactions

- **Admin domain** — Zones define which gateway instance to use.
Expand Down
13 changes: 13 additions & 0 deletions gateway/internal/controller/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
var (
IndexFieldSpecRoute = "spec.route"
IndexFieldSpecRouteName = "spec.route.name"
IndexFieldMetadataName = "metadata.name"
)

func RegisterIndecesOrDie(ctx context.Context, mgr ctrl.Manager) {
Expand Down Expand Up @@ -66,6 +67,18 @@ func RegisterIndecesOrDie(ctx context.Context, mgr ctrl.Manager) {
os.Exit(1)
}

// Index the route by its name so ConsumeRoutes on a proxy Route can be mapped to the
// same-named Routes in other zones that read them for their JumperConfig
filterNameOnRoute := func(obj client.Object) []string {
return []string{obj.GetName()}
}

err = mgr.GetFieldIndexer().IndexField(ctx, &gatewayv1.Route{}, IndexFieldMetadataName, filterNameOnRoute)
if err != nil {
ctrl.Log.Error(err, "unable to create fieldIndex for Route", "FieldIndex", IndexFieldMetadataName)
os.Exit(1)
}

err = index.SetOwnerIndex(ctx, mgr.GetFieldIndexer(), &gatewayv1.Route{})
if err != nil {
ctrl.Log.Error(err, "unable to create field-indexer")
Expand Down
38 changes: 32 additions & 6 deletions gateway/internal/controller/route_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"context"
"errors"

"github.com/go-logr/logr"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
Expand Down Expand Up @@ -71,19 +72,44 @@ func (r *RouteReconciler) SetupWithManager(mgr ctrl.Manager) error {
}

func (r *RouteReconciler) mapConsumeRouteToRoute(ctx context.Context, obj client.Object) []reconcile.Request {
// ensure its actually a ConsumeRoute
consumeRoute, ok := obj.(*gatewayv1.ConsumeRoute)
if !ok {
return nil
}
log := logr.FromContextOrDiscard(ctx)

// get the Route
route := &gatewayv1.Route{}
if err := r.Get(ctx, consumeRoute.Spec.Route.K8s(), route); err != nil {
return nil
direct := consumeRoute.Spec.Route.K8s()
requests := []reconcile.Request{{NamespacedName: direct}}

env := consumeRoute.Labels[cconfig.EnvironmentLabelKey]
if env == "" {
return requests
}

return []reconcile.Request{{NamespacedName: client.ObjectKey{Name: route.Name, Namespace: route.Namespace}}}
// Primary and failover-secondary Routes build their JumperConfig from all ConsumeRoutes
// with the same route name, including cross-zone ones that reference a proxy Route.
// Mirror that lookup, restricted to the ConsumeRoute's environment.
routes := &gatewayv1.RouteList{}
if err := r.List(ctx, routes,
client.MatchingFields{IndexFieldMetadataName: consumeRoute.Spec.Route.Name},
client.MatchingLabels{cconfig.EnvironmentLabelKey: env},
Comment thread
lukas016 marked this conversation as resolved.
Resolved
); err != nil {
log.Error(err, "Failed to list Routes by name", "routeName", consumeRoute.Spec.Route.Name)
return requests
}

for i := range routes.Items {
route := &routes.Items[i]
if route.IsProxy() && !route.IsFailoverSecondary() {
continue
}
key := client.ObjectKeyFromObject(route)
if key == direct {
continue
}
requests = append(requests, reconcile.Request{NamespacedName: key})
}
return requests
}

func (r *RouteReconciler) mapRouteListenerToRoute(ctx context.Context, obj client.Object) []reconcile.Request {
Expand Down
153 changes: 153 additions & 0 deletions gateway/internal/controller/route_controller_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
// Copyright 2025 Deutsche Telekom IT GmbH
//
// SPDX-License-Identifier: Apache-2.0

package controller

import (
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"

"github.com/telekom/controlplane/common/pkg/condition"
"github.com/telekom/controlplane/common/pkg/config"
"github.com/telekom/controlplane/common/pkg/types"
gatewayv1 "github.com/telekom/controlplane/gateway/api/v1"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var _ = Describe("RouteReconciler mapConsumeRouteToRoute", Ordered, func() {
const (
routeName = "cross-zone-api-v1"
zoneA = "map-zone-a"
zoneB = "map-zone-b"
zoneC = "map-zone-c"
zoneD = "map-zone-d"
otherEnvZone = "map-other-env"
otherEnvLabel = "other"
gatewayName = "map-gateway"
consumerPrefix = "map-consumer"
)

var reconciler *RouteReconciler
var gateways []*gatewayv1.Gateway
var created []client.Object

gatewayNamespace := map[string]string{testEnvironment: zoneA, otherEnvLabel: otherEnvZone}

newRoute := func(namespace, env string, routeType gatewayv1.RouteType) *gatewayv1.Route {
return &gatewayv1.Route{
ObjectMeta: metav1.ObjectMeta{
Name: routeName,
Namespace: namespace,
Labels: map[string]string{config.EnvironmentLabelKey: env},
},
Spec: gatewayv1.RouteSpec{
GatewayRef: types.ObjectRef{Name: gatewayName, Namespace: gatewayNamespace[env]},
Type: routeType,
Backend: gatewayv1.Backend{
Upstreams: []gatewayv1.Upstream{{Scheme: "https", Hostname: "backend.example.com", Port: 443}},
},
},
}
}

newConsumeRoute := func(name string, labels map[string]string) *gatewayv1.ConsumeRoute {
return &gatewayv1.ConsumeRoute{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: zoneB,
Labels: labels,
},
Spec: gatewayv1.ConsumeRouteSpec{
Route: types.ObjectRef{Name: routeName, Namespace: zoneB},
ConsumerName: "subscriber-app",
},
}
}

request := func(namespace string) reconcile.Request {
return reconcile.Request{NamespacedName: client.ObjectKey{Name: routeName, Namespace: namespace}}
}

BeforeAll(func() {
reconciler = &RouteReconciler{Client: mgrClient}

for _, ns := range []string{zoneA, zoneB, zoneC, zoneD, otherEnvZone} {
createNamespace(ns)
}

// Routes need a ready Gateway of their environment, otherwise their finalizer blocks deletion.
for env, ns := range gatewayNamespace {
gateway := newGateway(gatewayName, ns)
gateway.Labels[config.EnvironmentLabelKey] = env
Expect(k8sClient.Create(ctx, gateway)).To(Succeed())
gateways = append(gateways, gateway)
}
for _, gateway := range gateways {
Eventually(func(g Gomega) {
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(gateway), gateway)).To(Succeed())
g.Expect(meta.IsStatusConditionTrue(gateway.GetConditions(), condition.ConditionTypeReady)).To(BeTrue())
}, timeout, interval).Should(Succeed())
}

created = []client.Object{
newRoute(zoneA, testEnvironment, gatewayv1.RouteTypePrimary),
newRoute(zoneB, testEnvironment, gatewayv1.RouteTypeProxy),
newRoute(zoneC, testEnvironment, gatewayv1.RouteTypeProxy),
newRoute(zoneD, testEnvironment, gatewayv1.RouteTypeSecondary),
newRoute(otherEnvZone, otherEnvLabel, gatewayv1.RouteTypePrimary),
}
for _, obj := range created {
Expect(k8sClient.Create(ctx, obj)).To(Succeed())
}
})

AfterAll(func() {
for _, obj := range created {
Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, obj))).To(Succeed())
}
for _, obj := range created {
Eventually(func() bool {
return errors.IsNotFound(k8sClient.Get(ctx, client.ObjectKeyFromObject(obj), &gatewayv1.Route{}))
}, timeout, interval).Should(BeTrue())
}
for _, gateway := range gateways {
Expect(k8sClient.Delete(ctx, gateway)).To(Succeed())
Eventually(func() bool {
return errors.IsNotFound(k8sClient.Get(ctx, client.ObjectKeyFromObject(gateway), &gatewayv1.Gateway{}))
}, timeout, interval).Should(BeTrue())
}
})

It("enqueues the direct Route and same-named primary and failover Routes of the same environment", func() {
consumeRoute := newConsumeRoute(consumerPrefix+"-env", map[string]string{config.EnvironmentLabelKey: testEnvironment})

Eventually(func(g Gomega) {
g.Expect(reconciler.mapConsumeRouteToRoute(ctx, consumeRoute)).To(ConsistOf(
request(zoneB),
request(zoneA),
request(zoneD),
))
}, timeout, interval).Should(Succeed())
})

It("enqueues only the direct Route when the ConsumeRoute has no environment label", func() {
consumeRoute := newConsumeRoute(consumerPrefix+"-no-env", nil)

Expect(reconciler.mapConsumeRouteToRoute(ctx, consumeRoute)).To(ConsistOf(request(zoneB)))
})

It("enqueues the direct Route even if it does not exist", func() {
consumeRoute := newConsumeRoute(consumerPrefix+"-missing", map[string]string{config.EnvironmentLabelKey: testEnvironment})
consumeRoute.Spec.Route = types.ObjectRef{Name: "missing-route", Namespace: zoneB}

Expect(reconciler.mapConsumeRouteToRoute(ctx, consumeRoute)).To(ConsistOf(
reconcile.Request{NamespacedName: client.ObjectKey{Name: "missing-route", Namespace: zoneB}},
))
})
})
3 changes: 3 additions & 0 deletions gateway/internal/controller/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ const (
var (
cfg *rest.Config
k8sClient client.Client
// mgrClient is the manager's cached client, as used by the reconcilers and their map functions.
mgrClient client.Client
testEnv *envtest.Environment

ctx context.Context
Expand Down Expand Up @@ -94,6 +96,7 @@ var _ = BeforeSuite(func() {
},
})
Expect(err).ToNot(HaveOccurred())
mgrClient = k8sManager.GetClient()

// Register field indices needed by controllers
RegisterIndecesOrDie(ctx, k8sManager)
Expand Down
Loading