Skip to content

Commit

Permalink
Add EastWest Range feature (#1676)
Browse files Browse the repository at this point in the history 
* Add EastWest Range feature

The current east-west communication feature has some problems as the
lack of support for cluster-only routes and the fact that these internal
routes would still be accessible from outside if changing the Host
header. More details of #1526

This commit adds the concept of EastWest Range feature, what makes
possible to define internal domains and custom predicates to Skipper.
With this information, Skipper's Kubernetes dataclient adds the given
predicates on every route identified as part of one internal domain.
Differently from the `-enable-kubernetes-east-west` and the
`-kubernetes-east-west-domain=.ingress.cluster.local` flags this feature
will not automatically create routes and both features shouldn't be used
in combination.

It requires changing the current way we convert routegroups to eskip
routes, splitting the hostnames in internal and external hosts. All the
hosts identified as internals require a separated route once it will
have their own `ClientIP` predicate.

Also, it adds two new flags to skipper configuration,
`-kubernetes-east-west-range-domains` and
`-kubernetes-east-west-range-predicates`. It also include the basic
documentation regarding their usage.

Signed-off-by: Jonathan Juares Beber <[email protected]>

* Add Kubernetes Opt-in features subsections

This commit creates multiple subsections in the Kubernetes operations
documentation page.

Signed-off-by: Jonathan Juares Beber <[email protected]>
  • Loading branch information
@jonathanbeber
jonathanbeber authored Jan 15, 2021
1 parent 3c992ba commit eee6e59
Show file tree
Hide file tree
Showing 70 changed files with 1,523 additions and 106 deletions.
96 changes: 56 additions & 40 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,20 +120,23 @@ type Config struct {
WaitFirstRouteLoad bool `yaml:"wait-first-route-load"`

// Kubernetes:
KubernetesIngress bool `yaml:"kubernetes"`
KubernetesInCluster bool `yaml:"kubernetes-in-cluster"`
KubernetesURL string `yaml:"kubernetes-url"`
KubernetesHealthcheck bool `yaml:"kubernetes-healthcheck"`
KubernetesHTTPSRedirect bool `yaml:"kubernetes-https-redirect"`
KubernetesHTTPSRedirectCode int `yaml:"kubernetes-https-redirect-code"`
KubernetesIngressClass string `yaml:"kubernetes-ingress-class"`
KubernetesRouteGroupClass string `yaml:"kubernetes-routegroup-class"`
WhitelistedHealthCheckCIDR string `yaml:"whitelisted-healthcheck-cidr"`
KubernetesPathModeString string `yaml:"kubernetes-path-mode"`
KubernetesPathMode kubernetes.PathMode `yaml:"-"`
KubernetesNamespace string `yaml:"kubernetes-namespace"`
KubernetesEnableEastWest bool `yaml:"enable-kubernetes-east-west"`
KubernetesEastWestDomain string `yaml:"kubernetes-east-west-domain"`
KubernetesIngress bool `yaml:"kubernetes"`
KubernetesInCluster bool `yaml:"kubernetes-in-cluster"`
KubernetesURL string `yaml:"kubernetes-url"`
KubernetesHealthcheck bool `yaml:"kubernetes-healthcheck"`
KubernetesHTTPSRedirect bool `yaml:"kubernetes-https-redirect"`
KubernetesHTTPSRedirectCode int `yaml:"kubernetes-https-redirect-code"`
KubernetesIngressClass string `yaml:"kubernetes-ingress-class"`
KubernetesRouteGroupClass string `yaml:"kubernetes-routegroup-class"`
WhitelistedHealthCheckCIDR string `yaml:"whitelisted-healthcheck-cidr"`
KubernetesPathModeString string `yaml:"kubernetes-path-mode"`
KubernetesPathMode kubernetes.PathMode `yaml:"-"`
KubernetesNamespace string `yaml:"kubernetes-namespace"`
KubernetesEnableEastWest bool `yaml:"enable-kubernetes-east-west"`
KubernetesEastWestDomain string `yaml:"kubernetes-east-west-domain"`
KubernetesEastWestRangeDomains *listFlag `yaml:"kubernetes-east-west-range-domains"`
KubernetesEastWestRangePredicatesString string `yaml:"kubernetes-east-west-range-predicates"`
KubernetesEastWestRangePredicates []*eskip.Predicate `yaml:"-"`

// Default filters
DefaultFiltersDir string `yaml:"default-filters-dir"`
Expand Down Expand Up @@ -350,19 +353,21 @@ const (
waitFirstRouteLoadUsage = "prevent starting the listener before the first batch of routes were loaded"

// Kubernetes:
kubernetesUsage = "enables skipper to generate routes for ingress resources in kubernetes cluster"
kubernetesInClusterUsage = "specify if skipper is running inside kubernetes cluster"
kubernetesURLUsage = "kubernetes API base URL for the ingress data client; requires kubectl proxy running; omit if kubernetes-in-cluster is set to true"
kubernetesHealthcheckUsage = "automatic healthcheck route for internal IPs with path /kube-system/healthz; valid only with kubernetes"
kubernetesHTTPSRedirectUsage = "automatic HTTP->HTTPS redirect route; valid only with kubernetes"
kubernetesHTTPSRedirectCodeUsage = "overrides the default redirect code (308) when used together with -kubernetes-https-redirect"
kubernetesIngressClassUsage = "ingress class regular expression used to filter ingress resources for kubernetes"
kubernetesRouteGroupClassUsage = "route group class regular expression used to filter route group resources for kubernetes"
whitelistedHealthCheckCIDRUsage = "sets the iprange/CIDRS to be whitelisted during healthcheck"
kubernetesPathModeUsage = "controls the default interpretation of Kubernetes ingress paths: <kubernetes-ingress|path-regexp|path-prefix>"
kubernetesNamespaceUsage = "watch only this namespace for ingresses"
kubernetesEnableEastWestUsage = "enables east-west communication, which automatically adds routes for Ingress objects with hostname <name>.<namespace>.skipper.cluster.local"
kubernetesEastWestDomainUsage = "set the east-west domain, defaults to .skipper.cluster.local"
kubernetesUsage = "enables skipper to generate routes for ingress resources in kubernetes cluster"
kubernetesInClusterUsage = "specify if skipper is running inside kubernetes cluster"
kubernetesURLUsage = "kubernetes API base URL for the ingress data client; requires kubectl proxy running; omit if kubernetes-in-cluster is set to true"
kubernetesHealthcheckUsage = "automatic healthcheck route for internal IPs with path /kube-system/healthz; valid only with kubernetes"
kubernetesHTTPSRedirectUsage = "automatic HTTP->HTTPS redirect route; valid only with kubernetes"
kubernetesHTTPSRedirectCodeUsage = "overrides the default redirect code (308) when used together with -kubernetes-https-redirect"
kubernetesIngressClassUsage = "ingress class regular expression used to filter ingress resources for kubernetes"
kubernetesRouteGroupClassUsage = "route group class regular expression used to filter route group resources for kubernetes"
whitelistedHealthCheckCIDRUsage = "sets the iprange/CIDRS to be whitelisted during healthcheck"
kubernetesPathModeUsage = "controls the default interpretation of Kubernetes ingress paths: <kubernetes-ingress|path-regexp|path-prefix>"
kubernetesNamespaceUsage = "watch only this namespace for ingresses"
kubernetesEnableEastWestUsage = "enables east-west communication, which automatically adds routes for Ingress objects with hostname <name>.<namespace>.skipper.cluster.local"
kubernetesEastWestDomainUsage = "set the east-west domain, defaults to .skipper.cluster.local"
kubernetesEastWestRangeDomainsUsage = "set the the cluster internal domains for east west traffic. Identified routes to such domains will include the -kubernetes-east-west-range-predicates"
kubernetesEastWestRangePredicatesUsage = "set the predicates that will be appended to routes identified as to -kubernetes-east-west-range-domains"

// Auth:
oauth2GrantFlowEnableUsage = "enables OAuth2 Grant Flow filter"
Expand Down Expand Up @@ -459,6 +464,7 @@ func NewConfig() *Config {
cfg.SwarmRedisURLs = commaListFlag()
cfg.AppendFilters = &defaultFiltersFlags{}
cfg.PrependFilters = &defaultFiltersFlags{}
cfg.KubernetesEastWestRangeDomains = commaListFlag()

flag.StringVar(&cfg.ConfigFile, "config-file", "", configFileUsage)

Expand Down Expand Up @@ -566,6 +572,8 @@ func NewConfig() *Config {
flag.StringVar(&cfg.KubernetesNamespace, "kubernetes-namespace", "", kubernetesNamespaceUsage)
flag.BoolVar(&cfg.KubernetesEnableEastWest, "enable-kubernetes-east-west", false, kubernetesEnableEastWestUsage)
flag.StringVar(&cfg.KubernetesEastWestDomain, "kubernetes-east-west-domain", "", kubernetesEastWestDomainUsage)
flag.Var(cfg.KubernetesEastWestRangeDomains, "kubernetes-east-west-range-domains", kubernetesEastWestRangeDomainsUsage)
flag.StringVar(&cfg.KubernetesEastWestRangePredicatesString, "kubernetes-east-west-range-predicates", "", kubernetesEastWestRangePredicatesUsage)

// Auth:
flag.BoolVar(&cfg.EnableOAuth2GrantFlow, "enable-oauth2-grant-flow", false, oauth2GrantFlowEnableUsage)
Expand Down Expand Up @@ -688,13 +696,19 @@ func (c *Config) Parse() error {
return err
}

kubernetesEastWestRangePredicates, err := eskip.ParsePredicates(c.KubernetesEastWestRangePredicatesString)
if err != nil {
return fmt.Errorf("invalid east-west-range-predicates: %w", err)
}

histogramBuckets, err := c.parseHistogramBuckets()
if err != nil {
return err
}

c.ApplicationLogLevel = logLevel
c.KubernetesPathMode = kubernetesPathMode
c.KubernetesEastWestRangePredicates = kubernetesEastWestRangePredicates
c.HistogramMetricBuckets = histogramBuckets

if c.ClientKeyFile != "" && c.ClientCertFile != "" {
Expand Down Expand Up @@ -815,19 +829,21 @@ func (c *Config) ToOptions() skipper.Options {
WaitFirstRouteLoad: c.WaitFirstRouteLoad,

// Kubernetes:
Kubernetes: c.KubernetesIngress,
KubernetesInCluster: c.KubernetesInCluster,
KubernetesURL: c.KubernetesURL,
KubernetesHealthcheck: c.KubernetesHealthcheck,
KubernetesHTTPSRedirect: c.KubernetesHTTPSRedirect,
KubernetesHTTPSRedirectCode: c.KubernetesHTTPSRedirectCode,
KubernetesIngressClass: c.KubernetesIngressClass,
KubernetesRouteGroupClass: c.KubernetesRouteGroupClass,
WhitelistedHealthCheckCIDR: whitelistCIDRS,
KubernetesPathMode: c.KubernetesPathMode,
KubernetesNamespace: c.KubernetesNamespace,
KubernetesEnableEastWest: c.KubernetesEnableEastWest,
KubernetesEastWestDomain: c.KubernetesEastWestDomain,
Kubernetes: c.KubernetesIngress,
KubernetesInCluster: c.KubernetesInCluster,
KubernetesURL: c.KubernetesURL,
KubernetesHealthcheck: c.KubernetesHealthcheck,
KubernetesHTTPSRedirect: c.KubernetesHTTPSRedirect,
KubernetesHTTPSRedirectCode: c.KubernetesHTTPSRedirectCode,
KubernetesIngressClass: c.KubernetesIngressClass,
KubernetesRouteGroupClass: c.KubernetesRouteGroupClass,
WhitelistedHealthCheckCIDR: whitelistCIDRS,
KubernetesPathMode: c.KubernetesPathMode,
KubernetesNamespace: c.KubernetesNamespace,
KubernetesEnableEastWest: c.KubernetesEnableEastWest,
KubernetesEastWestDomain: c.KubernetesEastWestDomain,
KubernetesEastWestRangeDomains: c.KubernetesEastWestRangeDomains.values,
KubernetesEastWestRangePredicates: c.KubernetesEastWestRangePredicates,

// API Monitoring:
ApiUsageMonitoringEnable: c.ApiUsageMonitoringEnable,
Expand Down
1 change: 1 addition & 0 deletions config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ func Test_NewConfig(t *testing.T) {
AppendFilters: &defaultFiltersFlags{},
PrependFilters: &defaultFiltersFlags{},
SourcePollTimeout: 3000,
KubernetesEastWestRangeDomains: commaListFlag(),
KubernetesHealthcheck: true,
KubernetesHTTPSRedirect: true,
KubernetesHTTPSRedirectCode: 308,
Expand Down
14 changes: 14 additions & 0 deletions dataclients/kubernetes/eastwest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,17 @@ func createEastWestRouteRG(name, ns, postfix string, r *eskip.Route) *eskip.Rout
ewr.Predicates = p
return ewr
}

func applyEastWestRange(domains []string, predicates []*eskip.Predicate, host string, routes []*eskip.Route) {
for _, d := range domains {
if strings.HasSuffix(host, d) {
applyEastWestRangePredicates(routes, predicates)
}
}
}

func applyEastWestRangePredicates(routes []*eskip.Route, predicates []*eskip.Predicate) {
for _, route := range routes {
route.Predicates = append(route.Predicates, predicates...)
}
}
6 changes: 0 additions & 6 deletions dataclients/kubernetes/hosts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,3 @@ func hostCatchAllRoutes(hostRoutes map[string][]*eskip.Route, createID func(stri

return catchAll
}

func mergeHostRoutes(to, from map[string][]*eskip.Route) {
for host, routes := range from {
to[host] = append(to[host], routes...)
}
}
5 changes: 5 additions & 0 deletions dataclients/kubernetes/ingress.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ type ingress struct {
pathMode PathMode
kubernetesEnableEastWest bool
eastWestDomainRegexpPostfix string
eastWestRangeDomains []string
eastWestRangePredicates []*eskip.Predicate
}

var nonWord = regexp.MustCompile(`\W`)
Expand All @@ -67,6 +69,8 @@ func newIngress(o Options) *ingress {
pathMode: o.PathMode,
kubernetesEnableEastWest: o.KubernetesEnableEastWest,
eastWestDomainRegexpPostfix: ewPostfix,
eastWestRangeDomains: o.KubernetesEastWestRangeDomains,
eastWestRangePredicates: o.KubernetesEastWestRangePredicates,
}
}

Expand Down Expand Up @@ -753,6 +757,7 @@ func (ing *ingress) convert(state *clusterState, df defaultFilters) ([]*eskip.Ro
continue
}

applyEastWestRange(ing.eastWestRangeDomains, ing.eastWestRangePredicates, host, rs)
routes = append(routes, rs...)

// if routes were configured, but there is no catchall route
Expand Down
1 change: 1 addition & 0 deletions dataclients/kubernetes/ingress_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,6 @@ func TestIngressFixtures(t *testing.T) {
kubernetestest.FixturesToTest(t, "testdata/ingress/named-ports")
kubernetestest.FixturesToTest(t, "testdata/ingress/ingress-data")
kubernetestest.FixturesToTest(t, "testdata/ingress/eastwest")
kubernetestest.FixturesToTest(t, "testdata/ingress/eastwestrange")
kubernetestest.FixturesToTest(t, "testdata/ingress/service-ports")
}
9 changes: 9 additions & 0 deletions dataclients/kubernetes/kube.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,15 @@ type Options struct {
// KubernetesEastWestDomain sets the DNS domain to be used for east west traffic, defaults to "skipper.cluster.local"
KubernetesEastWestDomain string

// KubernetesEastWestRangeDomains set the the cluster internal domains for
// east west traffic. Identified routes to such domains will include
// the KubernetesEastWestRangePredicates.
KubernetesEastWestRangeDomains []string

// KubernetesEastWestRangePredicates set the Predicates that will be
// appended to routes identified as to KubernetesEastWestRangeDomains.
KubernetesEastWestRangePredicates []*eskip.Predicate

// DefaultFiltersDir enables default filters mechanism and sets the location of the default filters.
// The provided filters are then applied to all routes.
DefaultFiltersDir string
Expand Down
14 changes: 9 additions & 5 deletions dataclients/kubernetes/kubernetestest/fixtures.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,13 @@ type fixtureSet struct {
}

type kubeOptionsParser struct {
EastWest bool `yaml:"eastWest"`
EastWestDomain string `yaml:"eastWestDomain"`
HTTPSRedirect bool `yaml:"httpsRedirect"`
HTTPSRedirectCode int `yaml:"httpsRedirectCode"`
BackendNameTracingTag bool `yaml:"backendNameTracingTag"`
EastWest bool `yaml:"eastWest"`
EastWestDomain string `yaml:"eastWestDomain"`
EastWestRangeDomains []string `yaml:"eastWestRangeDomains"`
EastWestRangePredicates []*eskip.Predicate `yaml:"eastWestRangePredicatesAppend"`
HTTPSRedirect bool `yaml:"httpsRedirect"`
HTTPSRedirectCode int `yaml:"httpsRedirectCode"`
BackendNameTracingTag bool `yaml:"backendNameTracingTag"`
}

func baseNoExt(n string) string {
Expand Down Expand Up @@ -191,6 +193,8 @@ func testFixture(t *testing.T, f fixtureSet) {

o.KubernetesEnableEastWest = kop.EastWest
o.KubernetesEastWestDomain = kop.EastWestDomain
o.KubernetesEastWestRangeDomains = kop.EastWestRangeDomains
o.KubernetesEastWestRangePredicates = kop.EastWestRangePredicates
o.ProvideHTTPSRedirect = kop.HTTPSRedirect
o.HTTPSRedirectCode = kop.HTTPSRedirectCode
o.BackendNameTracingTag = kop.BackendNameTracingTag
Expand Down
Loading

0 comments on commit eee6e59

Please sign in to comment.