From 430a20617f2a6c48e552a6606615455f72068d1a Mon Sep 17 00:00:00 2001 From: Vivek Lohiya Date: Mon, 16 Dec 2024 10:06:17 +0530 Subject: [PATCH] Documentation for multi cluster support --- docs/RELEASE-NOTES.rst | 12 + .../Policy/policy-with-ab-persistence.yaml | 22 - docs/config_examples/multicluster/README.md | 611 +++--------------- .../ServiceTypeLB/sample-policy2.yaml | 18 - .../multicluster/ServiceTypeLB/svc-lb2.yaml | 19 - .../customResource/transportServer/README.md | 32 - ...s-with-weights-in-pool-service-and-ab.yaml | 31 - .../ts-without-multiClusterServices.yaml | 19 - .../multicluster/default-mode/README.md | 268 ++++++++ .../ServiceTypeLB/README.md | 0 .../sample-multi-cluster-svc-lb.yaml | 0 .../ServiceTypeLB/sample-policy.yaml | 0 .../ServiceTypeLB/svc-lb.yaml} | 0 ...default-mode-high-availability-svcLB.yaml} | 0 ...luster-default-mode-high-availablity.yaml} | 0 ...cluster-default-mode-standalone-svcLB.yaml | 19 + ...multicluster-default-mode-standalone.yaml} | 13 +- ...ulticluster-service-annotation-and-ab.yaml | 0 ...-with-multicluster-service-annotation.yaml | 0 .../ts-multiClusterServices-no-weights.yaml | 39 ++ .../ts-with-multiClusterServices.yaml | 0 ...-with-multiClusterServices-no-weights.yaml | 35 + .../vs-with-multiClusterServices.yaml | 39 ++ .../multicluster/non-default-mode/README.md | 532 +++++++++++++++ ...A-CIS-and-cluster-admin-state-no-pool.yaml | 0 ...e-CIS-and-cluster-admin-state-no-pool.yaml | 0 ...fig-with-primary-cluster-health-probe.yaml | 2 +- ...g-for-multicluster-with-active-active.yaml | 0 ...multicluster-with-cluster-admin-state.yaml | 0 .../global-spec-config-for-multicluster.yaml | 0 ...ulticluster-service-annotation-and-ab.yaml | 35 + ...-with-multicluster-service-annotation.yaml | 28 + .../virtualServer/vs.yaml} | 10 +- docs/multicluster_userguides/README.md | 1 - docs/upgradeProcess.md | 10 +- 35 files changed, 1102 insertions(+), 693 deletions(-) delete mode 100644 docs/config_examples/multicluster/Policy/policy-with-ab-persistence.yaml delete mode 100644 docs/config_examples/multicluster/ServiceTypeLB/sample-policy2.yaml delete mode 100644 docs/config_examples/multicluster/ServiceTypeLB/svc-lb2.yaml delete mode 100644 docs/config_examples/multicluster/customResource/transportServer/README.md delete mode 100644 docs/config_examples/multicluster/customResource/transportServer/ts-with-weights-in-pool-service-and-ab.yaml delete mode 100644 docs/config_examples/multicluster/customResource/transportServer/ts-without-multiClusterServices.yaml create mode 100644 docs/config_examples/multicluster/default-mode/README.md rename docs/config_examples/multicluster/{ => default-mode}/ServiceTypeLB/README.md (100%) rename docs/config_examples/multicluster/{ => default-mode}/ServiceTypeLB/sample-multi-cluster-svc-lb.yaml (100%) rename docs/config_examples/multicluster/{ => default-mode}/ServiceTypeLB/sample-policy.yaml (100%) rename docs/config_examples/multicluster/{ServiceTypeLB/svc-lb1.yaml => default-mode/ServiceTypeLB/svc-lb.yaml} (100%) rename docs/config_examples/multicluster/{extendedConfigmap/global-spec-config-for-multicluster-default-mode-svcLB.yaml => default-mode/extendedConfigmap/global-spec-config-for-multicluster-default-mode-high-availability-svcLB.yaml} (100%) rename docs/config_examples/multicluster/{extendedConfigmap/global-spec-config-for-multicluster-default-mode.yaml => default-mode/extendedConfigmap/global-spec-config-for-multicluster-default-mode-high-availablity.yaml} (100%) create mode 100644 docs/config_examples/multicluster/default-mode/extendedConfigmap/global-spec-config-for-multicluster-default-mode-standalone-svcLB.yaml rename docs/config_examples/multicluster/{extendedConfigmap/extendedConfigmap-with-admin-state-no-pool/global-spec-config-for-multicluster-with-HA-CIS-default-mode-and-cluster.yaml => default-mode/extendedConfigmap/global-spec-config-for-multicluster-default-mode-standalone.yaml} (51%) rename docs/config_examples/multicluster/{ => default-mode}/routes/route-with-multicluster-service-annotation-and-ab.yaml (100%) rename docs/config_examples/multicluster/{ => default-mode}/routes/route-with-multicluster-service-annotation.yaml (100%) create mode 100644 docs/config_examples/multicluster/default-mode/transportServer/ts-multiClusterServices-no-weights.yaml rename docs/config_examples/multicluster/{customResource => default-mode}/transportServer/ts-with-multiClusterServices.yaml (100%) create mode 100644 docs/config_examples/multicluster/default-mode/virtualServer/vs-with-multiClusterServices-no-weights.yaml create mode 100644 docs/config_examples/multicluster/default-mode/virtualServer/vs-with-multiClusterServices.yaml create mode 100644 docs/config_examples/multicluster/non-default-mode/README.md rename docs/config_examples/multicluster/{ => non-default-mode}/extendedConfigmap/extendedConfigmap-with-admin-state-no-pool/global-spec-config-for-multicluster-with-HA-CIS-and-cluster-admin-state-no-pool.yaml (100%) rename docs/config_examples/multicluster/{ => non-default-mode}/extendedConfigmap/extendedConfigmap-with-admin-state-no-pool/global-spec-config-for-multicluster-with-standalone-CIS-and-cluster-admin-state-no-pool.yaml (100%) rename docs/config_examples/multicluster/{ => non-default-mode}/extendedConfigmap/global-config-with-primary-cluster-health-probe.yaml (90%) rename docs/config_examples/multicluster/{ => non-default-mode}/extendedConfigmap/global-spec-config-for-multicluster-with-active-active.yaml (100%) rename docs/config_examples/multicluster/{ => non-default-mode}/extendedConfigmap/global-spec-config-for-multicluster-with-cluster-admin-state.yaml (100%) rename docs/config_examples/multicluster/{ => non-default-mode}/extendedConfigmap/global-spec-config-for-multicluster.yaml (100%) create mode 100644 docs/config_examples/multicluster/non-default-mode/routes/route-with-multicluster-service-annotation-and-ab.yaml create mode 100644 docs/config_examples/multicluster/non-default-mode/routes/route-with-multicluster-service-annotation.yaml rename docs/config_examples/multicluster/{customResource/virtualServer/vs-with-pool-service-and-ab.yaml => non-default-mode/virtualServer/vs.yaml} (53%) diff --git a/docs/RELEASE-NOTES.rst b/docs/RELEASE-NOTES.rst index 4acde2cc9..ddf709f7e 100644 --- a/docs/RELEASE-NOTES.rst +++ b/docs/RELEASE-NOTES.rst @@ -8,6 +8,15 @@ Added Functionality ``````````````````` **What's new:** * Multi Cluster + * *local-cluster-name* parameter is a new and mandatory parameter for multi-cluster mode, + * Introducing the new *default* mode for MultiCluster topologies which supports the ServiceType LoadBalancer, VirtualServer CR and Transport Server CR. See `Documentation <./config_examples/multicluster/default-mode>`_ + * CIS now supports the serviceType Load balancer discovery in remote clusters as well using the default mode. See `Documentation <./config_examples/multicluster/default-mode>`_ + * Support for the MultiCluster serviceType load balancer in the default mode. See `Example <./config_examples/multicluster/default-mode/ServiceTypeLB/sample-multi-cluster-svc-lb.yaml>`_ + * `Issue 3494 `_: make service discovery equal for all clusters by eliminating the extendedServiceReferences attribute + * CIS now does the service discovery for VS/TS CR in all the clusters when Active-Active or Ratio mode is configured. + * extendedServiceReferences have been deprecated for VS/TS CR in the Active-Active and Ratio mode. + * Active-standby mode is not supported any more. + * CRD * `Issue 3523 `_: Support for HTTP Compression profile in VS CR. See `Example <./config_examples/customResource/VirtualServer/httpCompressionProfile/>`_ * `Issue 3637 `_: Support for TLS in transport server. See `Example <./config_examples/customResource/TransportServer/transport-server-with-tls>`_ @@ -20,6 +29,9 @@ Bug Fixes * `Issue 3570 `_: tls irule fails if pool has no active members * Support dots and dashes in object names aligned to AS3 + + + 2.18.1 ------------- diff --git a/docs/config_examples/multicluster/Policy/policy-with-ab-persistence.yaml b/docs/config_examples/multicluster/Policy/policy-with-ab-persistence.yaml deleted file mode 100644 index 49af35692..000000000 --- a/docs/config_examples/multicluster/Policy/policy-with-ab-persistence.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# multiPoolPersistence is for a/b persistence -# supported in both CRD, and nextGen mode -# supported in cluster mode only -apiVersion: cis.f5.com/v1 -kind: Policy -metadata: - labels: - f5cr: "true" - name: cr-policy1 - namespace: foo -spec: - iRuleList: [] - iRules: {} - l3Policies: {} - l7Policies: - waf: /Common/WAF_Policy1 - poolSettings: - multiPoolPersistence: - # supported values for method: [uieSourceAddress, hashSourceAddress] - method: uieSourceAddress - # default time out is 180 seconds - timeOut: 2 \ No newline at end of file diff --git a/docs/config_examples/multicluster/README.md b/docs/config_examples/multicluster/README.md index e8602f21f..b4503ce14 100644 --- a/docs/config_examples/multicluster/README.md +++ b/docs/config_examples/multicluster/README.md @@ -1,6 +1,6 @@ # OpenShift/Kubernetes Multi-Cluster -This page documents the multi cluster support in CIS. Check the Known Issues section for more information on features not supported. +This page documents the multi cluster support in CIS. ## Contents @@ -10,257 +10,73 @@ This page documents the multi cluster support in CIS. Check the Known Issues sec [Configuration](#configuration) -[ExtendedSpecConfigMap](#extendedspecconfigmap) - -[Known Issues](#known-issues) - [FAQ](#faq) ## Overview -Multi-Cluster Support in CIS allows users to expose multiple apps spread across OpenShift/Kubernetes clusters using a single BIG-IP Virtual Server. An app can be deployed in different OpenShift/Kubernetes clusters exposing them using a Route/VS/TS CR resource. Using a Multi-Cluster implementation the CIS can be deployed in a HA topology, or Standalone CIS, to expose the apps spread across OpenShift/Kubernetes clusters. +Multi-Cluster Support in CIS allows users to expose multiple apps spread across OpenShift/Kubernetes clusters using a single BIG-IP Virtual Server. An app can be deployed in different OpenShift/Kubernetes clusters exposing them using a ServiceTypeLB/Route/VS/TS CR resource. Using a Multi-Cluster implementation the CIS can be deployed in a HA topology, or Standalone CIS, to expose the apps spread across OpenShift/Kubernetes clusters. -**Note**: +**Note**: * CIS supports processing of routes in traditional way as well as with NextGen Controller and with Multi-Cluster support. * At present, nodePort mode is supported and Cluster mode is available with static route configuration on BIGIP(No tunnels) ## Prerequisites * Cluster node, where CIS is deployed, should be able to reach the API server of all OpenShift/Kubernetes clusters. -* extendedConfigMap needs to be created to run CIS in Multi-Cluster mode. -* kube-config files for each cluster should be available for CIS to access resources such as Pods/Services/Endpoints/Nodes. +* ExtendedConfigMap needs to be created to run CIS in Multi-Cluster mode. +* Kube-config files for each cluster should be available for CIS to access resources such as Pods/Services/Endpoints/Nodes. ## Topologies ### Standalone CIS -In a Standalone deployment of CIS, CIS is only deployed in one cluster, then create a route resource with a Multi-Cluster annotation or CRD resource with multiClusterServices to expose the apps in different OpenShift/K8s clusters. +* In a Standalone deployment of CIS, CIS is only deployed in one cluster, +* There are two modes supported in the Standalone CIS, namely default and ratio. + * The default mode is supported with VS/TS CRs, routes and ServiceType LB. See [Documentation](default-mode/README.md). + * The ratio mode is supported with VS/TS CRs and routes.See [Documentation](non-default-mode/README.md#ratio-mode). ![architecture](images/standaloneMultiCluster.png) -Below is the sample Multi-Cluster Config in an Extended ConfigMap. -``` - extendedSpec: | - externalClustersConfig: -------------------------------------|----------------------------| | - - clusterName: cluster3 | | - secret: default/kubeconfig3 |---> Cluster configs for | - - clusterName: cluster4 | all other clusters | - secret: default/kubeconfig4 | except HA clusters | - - clusterName: cluster5 | | - secret: default/kubeconfig5 ------------------------------|----------------------------| - extendedRouteSpec: - - namespace: foo -------------------------------------| - vserverAddr: 10.8.0.4 | - vserverName: nextgenroutes |----------------> RouteGroup with namespace - allowOverride: true | - bigIpPartition: MultiTenant | - policyCR: default/sample-policy _____________________| - - namespace: bar -------------------------------------| - vserverAddr: 10.8.0.5 |----------------> RouteGroup with namespace - allowOverride: false _____________________| -``` -**Note**: extendedRouteSpec is only applicable in case of openshift route resources not for CRD resources. - ### High Availability CIS #### Prerequisites - * A pair of High Availability OpenShift/Kubernetes clusters should be available, which have the same applications running in both clusters. - * HealthCheck endpoint should be available to check the health of the primary cluster. Currently, TCP/HTTP Health endpoints are supported. - - -In HA deployment of CIS, CIS needs to be deployed in both the primary and secondary cluster. Also, the same extendedConfigMap needs to be deployed in both the primary and secondary cluster. -CIS will look for the same service name in both primary and secondary clusters to expose the application via routes. Additionally, a Multi-Cluster annotation is created in the route definition exposing the applications in the other clusters. +* A pair of High Availability OpenShift/Kubernetes clusters should be available, which may have the same applications running in both clusters. +* HealthCheck endpoint should be available to check the health of the primary cluster. Currently, TCP/HTTP Health endpoints are supported. -Deploying CIS HA in Two Modes: - * active-active and active-standby mode are used in case of CIS running in High Availability mode. - * Based on the provided value the CIS running on Primary cluster decides whether to monitor the workloads running on Secondary Cluster or not. - * In case of active-active mode CIS running on Primary cluster updates the pool members for Virtual Servers from both the clusters those are part of HA Cluster(Primary and Secondary Clusters) as well as pool members from all other remotely monitored clusters. - * Whereas in case of active-standby mode CIS running on Primary cluster updates the pool members for Virtual Servers only from Primary Cluster and from all the other remotely monitored clusters except Secondary cluster. - * However, in case the Primary cluster is down and CIS on the Secondary cluster has taken control then pool member from the Secondary Cluster as well as all other remotely monitored clusters are populated for the Virtual Servers irrespective of the value of HA mode. -**Note:** - * Additionally, there is another supported mode called "ratio", in which pool members from all the Kubernetes/Openshift clusters are update for the Virtual Server, however the traffic distribution is done based on the ratio values defined for each cluster. - * Ratio doesn't require CIS to be running in HA environment. It is supported in both CIS HA and non-HA environment. +* In HA deployment of CIS, CIS needs to be deployed in both the primary and secondary cluster. Also, the same extendedConfigMap needs to be deployed in both the primary and secondary cluster. +* There are three modes supported in the High Availability CIS, namely default, active-active and ratio. + * The default mode is supported with VS/TS CRs, routes and ServiceType LB. See [Documentation](default-mode/README.md). + * The Active-Active mode is supported with VS/TS CRs and routes.See [Documentation](non-default-mode/README.md#active-active-mode). + * The ratio mode is supported with VS/TS CRs and routes.See [Documentation](non-default-mode/README.md#ratio-mode). **Note**: -* For HA mode [namely Active-Standby, Active-Active, Ratio], CIS monitored resource manifests(such as routes, CRDs, extendedConfigmaps) must be available in both the clusters. -* The CIS monitored resource manifests must be identical on both primary and Secondary Clusters +* For HA mode [namely default, active-active, ratio], CIS monitored resource manifests(such as routes, CRDs, extendedConfigmaps, multiCluster serviceTypeLB) must be available in both the clusters. +* The CIS monitored resource manifests should be identical on both primary and Secondary Clusters * So, In case of CIS failover, CIS on Secondary Cluster will take control and will start processing the CIS monitored resource manifests. * CIS on Secondary Cluster will not process the CIS monitored resource manifests if they are not available in Secondary Cluster. -* MakeSure to have identical resource manifests in both the clusters to avoid any issues during CIS failover. - -Below is a tabular representation of the scenarios mentioned above: - -###### When Primary Cluster is healthy and CIS is running on it - -| HA Mode values | Primary Cluster Pool Members | Secondary Cluster Pool Members | Other remotely monitored clusters Pool Members | Traffic distribution based on ratio | -|----------------|------------------------------|--------------------------------|------------------------------------------------|-------------------------------------| -| active-active | Yes | Yes | Yes | No | -| active-standby | Yes | No | Yes | No | -| ratio | Yes | Yes | Yes | Yes | - - -###### When Primary Cluster is down, which means CIS in Secondary cluster has the control - -| HA Mode values | Primary Cluster Pool Members | Secondary Cluster Pool Members | Other remotely monitored clusters Pool Members | Traffic distribution based on ratio | -|----------------|------------------------------|--------------------------------|------------------------------------------------|-------------------------------------| -| active-active | No | Yes | Yes | No | -| active-standby | No | Yes | Yes | No | -| ratio | Yes | Yes | Yes | Yes | +* Make sure to have identical resource manifests in both the clusters to avoid any issues during CIS fail-over. + ![architecture](images/haMultiCluster.png) -Below is the sample Multi-Cluster Configs with HA in Extended ConfigMap. -``` - extendedSpec: | - mode: active-active -----------------------------------|----> HA Mode | - highAvailabilityCIS: --------------------------------------|----------------------------| - primaryEndPoint: http://10.145.72.114:8001 | | - probeInterval: 30 | | - retryInterval: 3 | | - primaryCluster: |---> Cluster configs for | - clusterName: cluster1 | High availability | - secret: default/kubeconfig1 | clusters |---> Multi-Cluster configs - secondaryCluster: | | - clusterName: cluster2 | | - secret: default/kubeconfig2 | | - externalClustersConfig: -------------------------------------| | - - clusterName: cluster3 | | - secret: default/kubeconfig3 |---> Cluster configs for | - - clusterName: cluster4 | all other clusters | - secret: default/kubeconfig4 | except HA clusters | - - clusterName: cluster5 | | - secret: default/kubeconfig5 ------------------------------|----------------------------| - extendedRouteSpec: - - namespace: foo -------------------------------------| - vserverAddr: 10.8.0.4 | - vserverName: nextgenroutes |----------------> RouteGroup with namespace - allowOverride: true | - bigIpPartition: MultiTenant | - policyCR: default/sample-policy _____________________| - - namespace: bar -------------------------------------| - vserverAddr: 10.8.0.5 |----------------> RouteGroup with namespace - allowOverride: false _____________________| -``` - -Below is the sample Multi-Cluster Configs with HA and Ratio in Extended ConfigMap. -``` - extendedSpec: | - mode: ratio ------------------------------------------|----------------------------| - localClusterRatio: 4 | | - highAvailabilityCIS: --------------------------------------| | - primaryEndPoint: http://10.145.72.114:8001 | | - probeInterval: 30 | | - retryInterval: 3 | | - primaryCluster: |---> Cluster configs for | - clusterName: cluster1 | High availability | - secret: default/kubeconfig1 | clusters |---> Multi-Cluster configs - ratio: 3 | | - secondaryCluster: | | - clusterName: cluster2 | | - secret: default/kubeconfig2 | | - ratio: 2 | | - externalClustersConfig: -------------------------------------| | - - clusterName: cluster3 | | - secret: default/kubeconfig3 |---> Cluster configs for | - ratio: 2 | all other clusters | - - clusterName: cluster4 | except HA clusters | - secret: default/kubeconfig4 | | - - clusterName: cluster5 | | - secret: default/kubeconfig5 | | - ratio: 1 ------------------------------|----------------------------| - extendedRouteSpec: - - namespace: foo -------------------------------------| - vserverAddr: 10.8.0.4 | - vserverName: nextgenroutes |----------------> RouteGroup with namespace - allowOverride: true | - bigIpPartition: MultiTenant | - policyCR: default/sample-policy _____________________| - - namespace: bar -------------------------------------| - vserverAddr: 10.8.0.5 |----------------> RouteGroup with namespace - allowOverride: false _____________________| -``` -**Note**: extendedRouteSpec is only applicable in case of openshift route resources not for CRD resources. - -Below is the sample Multi-Cluster Configs with HA and cluster AdminState in Extended ConfigMap. -``` - extendedSpec: | - mode: active-active --------------------------------------|----------------------------| - highAvailabilityCIS: --------------------------------------| | - primaryEndPoint: http://10.145.72.114:8001 | | - probeInterval: 30 | | - retryInterval: 3 | | - primaryCluster: |---> Cluster configs for | - clusterName: cluster1 | High availability | - secret: default/kubeconfig1 | clusters |---> Multi-Cluster configs - secondaryCluster: | | - clusterName: cluster2 | | - secret: default/kubeconfig2 | | - adminState: enable | | - externalClustersConfig: ----------------------------------| | - - clusterName: cluster3 | | - secret: default/kubeconfig3 |---> Cluster configs for | - adminState: disable | all other clusters | - - clusterName: cluster4 | except HA clusters | - secret: default/kubeconfig4 | | - - clusterName: cluster5 | | - secret: default/kubeconfig5 | | - adminState: offline ------------------------------|----------------------------| - extendedRouteSpec: - - namespace: foo -------------------------------------| - vserverAddr: 10.8.0.4 | - vserverName: nextgenroutes |----------------> RouteGroup with namespace - allowOverride: true | - bigIpPartition: MultiTenant | - policyCR: default/sample-policy _____________________| - - namespace: bar -------------------------------------| - vserverAddr: 10.8.0.5 |----------------> RouteGroup with namespace - allowOverride: false _____________________| -``` -**Note**: extendedRouteSpec is only applicable in case of openshift route resources not for CRD resources. - -Below is the sample Multi-Cluster Configs with standalone CIS and cluster AdminState in Extended ConfigMap. -``` - extendedSpec: | - localClusterAdminState: disable ----------------------------|AdminState for local cluster| - externalClustersConfig: ----------------------------------|----------------------------| - - clusterName: cluster3 | | - secret: default/kubeconfig3 |---> Cluster configs for | - adminState: enable | all other clusters | - - clusterName: cluster4 | except HA clusters | - secret: default/kubeconfig4 | | - - clusterName: cluster5 | | - secret: default/kubeconfig5 | | - adminState: offline ------------------------------|----------------------------| - extendedRouteSpec: - - namespace: foo -------------------------------------| - vserverAddr: 10.8.0.4 | - vserverName: nextgenroutes |----------------> RouteGroup with namespace - allowOverride: true | - bigIpPartition: MultiTenant | - policyCR: default/sample-policy _____________________| - - namespace: bar -------------------------------------| - vserverAddr: 10.8.0.5 |----------------> RouteGroup with namespace - allowOverride: false _____________________| -``` -**Note**: localClusterAdminState is only applicable in case of standalone CIS. It's ignored if specified in HA CIS mode. - - ## Configuration -### Openshift Routes with multi-cluster - ### CIS Deployment Parameter -If you are using multi-cluster mode, ```--multi-cluster-mode``` parameter is a required parameter. +If you are using multi-cluster mode, ```--multi-cluster-mode``` and ```--local-cluster-name``` are required parameters. -| Parameter | Type | Required | Description | Allowed Values | -|--------------------|--------|----------|----------------------------------------------------------------------------------------------------------------|------------------------------------| -| multi-cluster-mode | String | Required | Specify whether CIS is running standalone or as primary/secondary in the case of a high availability topology. | standalone or primary or secondary | +| Parameter | Type | Required | Description | Allowed Values | +|-------------------------|--------|----------|----------------------------------------------------------------------------------------------------------------|-----------------------------------------------------| +| multi-cluster-mode | String | Required | Specify whether CIS is running standalone or as primary/secondary in the case of a high availability topology. | standalone or primary or secondary | +| local-cluster-name | String | Required | Specify the cluster name where CIS is running. | valid cluster name defined in the extendedConfigMap | +| extended-spec-configmap | String | Required | extendedSpecConfigMap is required to configure the multi-cluster configuration | / | + +**Note**: -**Note**: Here **standalone** refers to standalone topology of CIS deployment, See [Standalone CIS](#standalone-cis). +* Here **standalone** refers to standalone topology of CIS deployment, See [Standalone CIS](#standalone-cis). Following is the sample deployment for primary CIS deployment: @@ -285,127 +101,7 @@ Following is the sample deployment for primary CIS deployment: - --pool-member-type - nodeport - --multi-cluster-mode=primary - command: - - /app/bin/k8s-bigip-ctlr - image: -``` - -**Note:** -1. Update the ```multi-cluster-mode``` to *secondary* for secondary CIS deployment in high availablility topology, See [High Availability CIS](#high-availability-cis). -2. Update the ```multi-cluster-mode``` to *standalone* for standalone topology, See [Standalone CIS](#standalone-cis). - -**Note**: -* For HA mode [namely Active-Standby, Active-Active, Ratio], CIS monitored resource manifests(such as routes, CRDs, extendedConfigmaps) must be available in both the clusters. -* The CIS monitored resource manifests must be identical on both primary and Secondary Clusters -* So, In case of CIS failover, CIS on Secondary Cluster will take control and will start processing the CIS monitored resource manifests. -* CIS on Secondary Cluster will not process the CIS monitored resource manifests if they are not available in Secondary Cluster. -* MakeSure to have identical resource manifests in both the clusters to avoid any issues during CIS failover. - - -### extended ConfigMap Parameters - -#### externalClustersConfig Parameters - -| Parameter | Type | Required | Description | Default | Examples | -|-------------|--------|-----------|---------------------------------------------------------------------------|---------|-------------------------| -| clusterName | String | Mandatory | Name of the cluster | - | cluster1 | -| secret | String | Mandatory | Name of the secret created for kubeconfig (format: namespace/secret-name) | - | test/secret-kubeconfig1 | -| ratio | int | Optional | Ratio at which the traffic has to be distributed over clusters | 1 | 3 | -| adminState | String | Optional | adminState can be used to disable/enable/offline/no-pool clusters | enable | disable | - - -**Note:** Avoid specifying HA cluster(Primary/Secondary cluster) configs in externalClustersConfig. - -#### High Availability Mode (Optional parameter) -| Parameter | Type | Required | Description | Default | Examples | -|------------------------|---------|-----------|---------------------------------------------------------------------|----------------|----------------| -| mode | Object | Optional | Type of high availability mode (active-active/active-standby/ratio) | active-standby | active-active | - -Specifies whether the CIS HA cluster is configured with active-active mode, active-standby mode or ratio mode. -* If mode Type: active-active, CIS fetches service from both the HA clusters whenever it's referenced in Route Spec. -* If mode Type: active-standby (default), CIS fetches service from only the local cluster whenever it's referenced in a Route Spec. -* If mode Type: ratio, CIS works in active-active mode and, it splits traffic according to the ratio specified for each cluster. - -#### Local cluster ratio (Optional parameter) -| Parameter | Type | Required | Description | Default | Examples | -|------------------------|------|-----------|-----------------------------------------------------------------------------------------------------------|---------|----------| -| localClusterRatio | Int | Optional | Ratio for the local cluster where CIS is running(specify only when using ratio in CIS non-HA environment) | 1 | 3 | -**Note:** It is not needed in case of using ratio in CIS HA environment, as ratio of Primary cluster does the same thing. If specified in this scenario then it will be ignored. - -#### highAvailabilityCIS Parameters - -| Parameter | Type | Required | Description | Default | Examples | -|------------------------|---------|-----------|-------------------------------------------------------------------------|---------|---------------------------| -| primaryClusterEndPoint | String | Mandatory | Endpoint to check health of primary cluster | - | http://10.145.72.114:8001 | -| probeInterval | Integer | Optional | Time interval between health check (in seconds) | 60 | 30 | -| retryInterval | Integer | Optional | Time interval between recheck when primary cluster is down (in seconds) | 15 | 3 | -| primaryCluster | Object | Mandatory | Primary cluster config | - | - | -| secondaryCluster | Object | Mandatory | Secondary cluster config | - | - | - - -##### primaryCluster/secondaryCluster Parameters - -| Parameter | Type | Required | Description | Default | Examples | -|-------------|--------|-----------|---------------------------------------------------------------------------|---------|-------------------------| -| clusterName | String | Mandatory | Name of the cluster | - | cluster1 | -| secret | String | Mandatory | Name of the secret created for kubeconfig (format: namespace/secret-name) | - | test/secret-kubeconfig1 | -| ratio | int | Optional | Ratio at which the traffic has to be distributed over clusters | 1 | 3 | -| adminState | String | Optional | adminState can be used to disable/enable/offline/no-pool clusters | enable | disable | - - -**Note**: In order to run CIS in high availability mode, multi-cluster-mode parameter (primary/secondary) needs to be set in the CIS deployment arguments. -* It's recommended to provide both primaryCluster and secondaryCluster configs in the extendedConfigMap. -* If no traffic has to be forwarded to a specific cluster then set the ratio field to 0. - -##### CIS Primary Cluster Endpoint - -Health probe parameters are provided in highAvailabilityCIS in extended configmap, helping to ensure high availability of CIS. CIS running in secondary cluster continuously monitors the health of the primary cluster. If it's down, then the secondary CIS takes the responsibility of posting declarations to BIG-IP. - -**Note**: primaryEndPoint is a mandatory parameter if CIS is intended to run in Multi-Cluster HA mode. If this is not specified the secondary CIS will not run. - - -### Route Annotation for Multi-ClusterServices -Services running in any other OpenShift clusters, as mentioned below: -``` -virtual-server.f5.com/multiClusterServices: -'[ - { - "clusterName": "cluster2", - "service": "svc-pytest-foo-1-com", - "namespace": "foo", - "servicePort": 80, - "weight": 30, - } -]' -``` -### CRD Resources with Multi-Cluster - -### CIS Deployment Parameter - -**Note**: Here **standalone** refers to standalone topology of CIS deployment, See [Standalone CIS](#standalone-cis). - -Following is the sample deployment for primary CIS deployment: - -```yaml - spec: - containers: - - args: - - --bigip-partition - - - - --bigip-url - - - - --bigip-username - - - - --bigip-password - - - - --log-level - - DEBUG - - --insecure - - --custom-resource-mode=true - - --extended-spec-configmap=kube-system/extended-spec-config - - --pool-member-type - - nodeport - - --multi-cluster-mode=primary + - --local-cluster-name=cluster1 command: - /app/bin/k8s-bigip-ctlr image: @@ -414,143 +110,31 @@ Following is the sample deployment for primary CIS deployment: **Note:** 1. Update the ```multi-cluster-mode``` to *secondary* for secondary CIS deployment in high availablility topology, See [High Availability CIS](#high-availability-cis). 2. Update the ```multi-cluster-mode``` to *standalone* for standalone topology, See [Standalone CIS](#standalone-cis). +3. Update the ```local-cluster-name``` to the cluster name where CIS is running. -**Note**: _weight_ needs to be specified only in [A/B](#ab-or-alternate-backends) scenario +### Extended ConfigMap -**Note**: -* For HA mode [namely Active-Standby, Active-Active, Ratio], CIS monitored resource manifests(such as routes, CRDs, extendedConfigmaps) must be available in both the clusters. -* The CIS monitored resource manifests must be identical on both primary and Secondary Clusters -* So, In case of CIS failover, CIS on Secondary Cluster will take control and will start processing the CIS monitored resource manifests. -* CIS on Secondary Cluster will not process the CIS monitored resource manifests if they are not available in Secondary Cluster. -* MakeSure to have identical resource manifests in both the clusters to avoid any issues during CIS failover. - - -### Virutal Server Pool with Multi-ClusterServices -This is not supported as of now. It will be supported soon. - -### Transport Server Pool with Multi-ClusterServices -Services running in any other OpenShift/Kubernetes clusters those are monitored by CIS, can be referenced in the TS Pool as mentioned below: -``` - pool: - multiClusterServices: - - clusterName: cluster2 - service: svc-1 - namespace: ns1 - servicePort: 8181 - - clusterName: cluster3 - service: svc-ext-1 - namespace: ns2 - servicePort: 8282 -``` - -## Static Routing Mode -CIS supports configuring static routes in BIG-IP with node subnets assigned for the nodes in the OpenShift/k8s cluster.This enables direct routing from BIGIP to k8s Pods in cluster mode without vxaln tunnel configuration on BIGIP. +* See How to configure the extendedConfigMap in default mode in standalone topology, [Documentation](default-mode/README.md#configuring-extendedconfigmapforstandalone). +* See How to configure the extendedConfigMap in default mode in high availability topology, [Documentation](default-mode/README.md#configuring-extendedconfigmapforhighavailability). +* See How to configure the extendedConfigMap in active-active mode, [Documentation](non-default-mode/README.md#configuring-extendedconfigmapforstandalone). +* See How to configure the extendedConfigMap in ratio mode in standalone topology, [Documentation](non-default-mode/README.md#configuring-extended-configmap-for-ratio-mode-standalone-cis). +* See How to configure the extendedConfigMap in ratio mode in high availability topology, [Documentation](non-default-mode/README.md#configuring-extended-configmap-for-ratio-mode-high-availability-cis). -### Configuration -To enable the static route configuration, set --static-routing-mode to true and --orchestration-cni to CNI configured in the cluster. +### VS CR Support +* See How to configure the VS CR in default mode [Documentation](default-mode/README.md#virtual-server-support). +* See How to configure the VS CR in active-active and ratio mode [Documentation](non-default-mode/README.md#virtual-server-support). -| Parameter | Type | Required | Description | Allowed values | Default | Agent | -|----------------------|---------|-----------|------------------------------------------------------------------------------------------------------|------------------------------------|---------|-------| -| static-routing-mode | Boolean | Optional | Adds Static Routes on the BIGIP so that traffic can be directly route to the pods. (Without tunnels) | true,false | false | AS3 | -| orchestration-cni | string | Optional | Kubernetes Cluster CNI Name | cilium-k8s, flannel,ovn-k8s,antrea | flannel | AS3 | -| shared-static-routes | Boolean | Optional | flag to enable configuration of static routes on bigip in common partition | true,false | false | AS3 | +### TS CR Support +* See How to configure the TS CR in default mode [Documentation](default-mode/README.md#transport-server-support). +* See How to configure the TS CR in active-active and ratio mode [Documentation](non-default-mode/README.md#transport-server-support). -### CIS Deployment Parameter - -**Note**: Here **standalone** refers to standalone topology of CIS deployment, See [Standalone CIS](#standalone-cis). +### Routes Support +* See How to configure the routes in default mode [Documentation](default-mode/README.md#routes-support). +* See How to configure the routes in active-active and ratio mode [Documentation](non-default-mode/README.md#routes-support). -Following is the sample deployment for primary CIS deployment: - -```yaml - spec: - containers: - - args: - - --bigip-partition - - - - --bigip-url - - - - --bigip-username - - - - --bigip-password - - - - --log-level - - DEBUG - - --insecure - - --custom-resource-mode=true - - --static-routing-mode=true - - --orchestration-cni=ovn-k8s - - --shared-static-routes=true - - --extended-spec-configmap=kube-system/extended-spec-config - - --pool-member-type - - cluster - - --multi-cluster-mode=primary - command: - - /app/bin/k8s-bigip-ctlr - image: -``` - -#### Route Annotation / VS or TS MultiClusterServices Parameters - -| Parameter | Type | Required | Description | Default | Examples | -|-------------|------------|------------|---------------------------------------------------------|---------|----------| -| clusterName | String | Mandatory | Name of the cluster | - | cluster1 | -| service | String | Mandatory | Name of the service | - | svc-1 | -| namespace | String | Mandatory | Namespace where the service is created | - | test | -| servicePort | String/Int | Mandatory | port of the service (for named port use string value ) | - | 80 | -| weight | Int | Optional | weight to be used for traffic splitting | 0 | 20 | - -### Cluster wise Ratio for traffic distribution -CIS supports distribution of traffic across clusters as per the ratio configured for each cluster in the extended ConfigMap.
-It works even along with [A/B](#ab-or-alternate-backends) where different weights are defined for each service. In such a case the ratio of traffic -distribution is computed taking into consideration both the service weights and cluster ratio.
-However, the ratio of the clusters those haven't hosted any services linked to the concerned route are not taken into consideration -while computing the final ratio.
- -**Note:** -* Cluster wise ratio for traffic distribution is supported in HA as well as non-HA CIS environment. -* Ratio is only supported for NextGen Routes and Virtual and Transport Server CR. -* Setting cluster adminState in conjunction with cluster ratio will affect the overall traffic distribution across clusters. - As the clusters marked as disable or offline will not receive traffic, so any ratio defined for these clusters will be rendered ineffective. - Thus, in such a scenario it's recommended to set the cluster ratio to 0 for all the clusters marked with disable/offline. - -### A/B or Alternate Backends -What it is? - -* A/B or Alternate Backends is a deployment strategy that allows you to release a new version of an application (version B) to a subset of users, while the majority still uses the old version (version A). -* It helps in comparing two versions of a service or application (referred to as A and B) to determine which one performs better based on specific metrics, such as response time, error rates, or user engagement. -* It allows you to gradually release changes to a subset of users and gather data to make informed decisions about whether to fully roll out the new version. -* This services defined in Alternate Backends exist in either of the HA peer clusters or in both of the HA clusters. Since HA clusters usually hold similar configurations, ideally these services exist in both the HA clusters. - -What it isn't? - -* Services defined as Alternate Backends don't have to be created only in the other HA peer cluster(by other HA peer cluster it means if CIS is running in Primary cluster then Secondary cluster is the other HA peer cluster and vice versa). -* Alternate Backends are not primarily used for failover scenarios, however BIGIP does forward the traffic to any of the available backend service if any service goes down or fails health check. - -### Cluster adminState to enable/disable/offline a cluster -adminState can be provided for a cluster to dictate the state of a particular cluster. -Supported values for adminState are [enable, disable, offline, no-pool]
-By default clusters are in enabled state.
-**adminState: enable**, all new connections are allowed to the pool members from the cluster.
-**adminState: disable**, all new connections except those which match an existing persistence session are not allowed for the pool members from the cluster.
-**adminState: offline**, no new connections are allowed to the pool members from the cluster, even if they match an existing persistence session.
-**adminState: no-pool**, in ratio mode, a service pool is not created for the affected cluster. For all other modes, pool members from the cluster are not added to the service pool. This configuration is helpful when we don't want to add pool or pool members from a particular cluster due to any reasons(for example cluster is under maintenance).
- - -**Note**: -* For HA mode [namely Active-Standby, Active-Active, Ratio], CIS monitored resource manifests(such as routes, CRDs, extendedConfigmaps) must be available in both the clusters. -* The CIS monitored resource manifests must be identical on both primary and Secondary Clusters -* So, In case of CIS failover, CIS on Secondary Cluster will take control and will start processing the CIS monitored resource manifests. -* CIS on Secondary Cluster will not process the CIS monitored resource manifests if they are not available in Secondary Cluster. -* MakeSure to have identical resource manifests in both the clusters to avoid any issues during CIS failover. - - -## Known issues -* Pool members are not getting populated for extended service in ratio mode -* CIS doesn't update pool members if service doesn't exist in primary cluster but exists in secondary cluster for Route. -* CIS on start up in multiCluster mode, if any external cluster kube-api server is down/not reachable, CIS is struck and not processing any valid clusters config also.Workaround to remove unreachable cluster config from configmap and restart CIS -* CIS fails to post declaration with VS with health monitors in ratio mode.Issue is observed intermittently -* Route status is not updated in other HA cluster. For eg: Active Primary CIS cluster doesn't update the route status in Secondary HA cluster and vice-versa. -* CIS on switch over from ratio to active-standby mode, doesn't add the external cluster services. Any change to mode, it is always recommended to restart the CIS +### ServiceTypeLB Support +* See How to configure the ServiceTypeLB in default mode [Documentation](default-mode/README.md#service-type-load-balancer-support). +* ServiceTypeLB is not supported in active-active and ratio mode. ## FAQ @@ -560,11 +144,14 @@ Yes. Multi-Cluster support only works if --multi-cluster-mode is defined in CIS ### Is extended configMap mandatory for Multi-Cluster support? Yes. Multi-Cluster support only works with extended configmap. +### Is local-cluster-name mandatory for Multi-Cluster support? +Yes. Multi-Cluster support only works if --local-cluster-name is defined in CIS deployment. + ### Does extended configmap update require CIS restart? -No. It's recommended to restart CIS if any HA configuration or external cluster configurations are updated in extended Configmap. However CIS restart is not required when updating ratio in the extended Configmap. +No. It's recommended to restart CIS if any HA configuration or external cluster configurations are updated in extended Configmap. However, CIS restart is not required when updating ratio in the extended Configmap. ### Does mode update require CIS restart? -Yes. CIS has to be restarted when there is a change in the mode. +Yes. CIS has to be restarted when there is a change in the mode. ### How do you add a new cluster? To add a new cluster, create a kube-config file with read only permissions. Then create a Kubernetes secret using the kube-config file. Refer this in secret in the extended ConfigMap to add the new cluster. @@ -577,7 +164,7 @@ Manifests or Configuration objects are managed centralized in Primary Cluster an Currently, NodePort mode is supported.For cluster mode, static routing mode is supported to enable configuration of static routes on bigip for pod network subnets for direct routing from BIGIP to k8s Pods ### What kind of providers are supported? -CIS supports Hybrid Cloud, any public Cloud providers such as; AWS, Azure, GCP, On-Prem, VmWare, Tanzu etc. which is in same network/datacenter and can communicate with each other. +CIS supports Hybrid Cloud, any public Cloud providers such as; AWS, Azure, GCP, On-Prem, VmWare, Tanzu etc. which is in same network/datacenter and can communicate with each other. ### What kind of clusters are supported? CIS multicluster solution is currently validated with openshift clusters and K8s clusters @@ -597,52 +184,43 @@ CIS requires read-only permission in Kubeconfig of external clusters to access r ### Can CIS manage multiple BIG-IPs? No. CIS can manage only Standalone BIG-IP or HA BIG-IP. In other words, CIS acts as a single point of BIG-IP Orchestrator and supports Multi-Cluster. -### Is traffic splitting with cluster ratio supported? -Yes. CIS supports traffic splitting as per the ratio specified for each cluster and also works with [A/B](#ab-or-alternate-backends) as well. - -### Is A/B supported in multiCluster mode? -Yes. CIS supports [A/B](#ab-or-alternate-backends) with multiCluster. - -### Is A/B custom persistence supported in all the modes? -No. [A/B](#ab-or-alternate-backends) persistence is supported in ratio mode and pool member type as cluster. - ### Does Secondary CIS require resource manifests existing in Primary Cluster? -Yes. CIS on Secondary Cluster will not process the CIS monitored resource manifests[NextGen Routes, CRDs, extendedConfigmap] if they are not available in Primary Cluster. +Yes. CIS on Secondary Cluster will not process the CIS monitored resource manifests[NextGen Routes, CRDs, ServiveTypeLB, extendedConfigmap] if they are not available in Primary Cluster. This is required in case of CIS failover, CIS on Secondary Cluster will take control and will start processing the CIS monitored resource manifests. It is suggested to maintain identical CIS monitored resource manifests in both the clusters to avoid any issues during CIS failover. -This requirement is applicable in case of CIS HA mode [namely Active-Standby, Active-Active, Ratio]. +This requirement is applicable in case of CIS HA mode [namely default, active-active, ratio]. -### How to configure primaryEndPoint in HA mode? -primaryEndPoint is a mandatory parameter if CIS is intended to run in Multi-Cluster HA mode[namely Active-Standby, Active-Active, Ratio]. If this is not specified the secondary CIS will not run. +### How to configure the primaryEndPoint in HA mode? +The primaryEndPoint is a mandatory parameter if CIS is intended to run in Multi-Cluster HA mode[namely default, active-active, ratio]. If this is not specified the secondary CIS will not run. Secondary CIS will continuously monitor the health of the primary cluster based on the primaryEndPoint value. If it's down, then the secondary CIS takes the responsibility of posting declarations to BIG-IP. -Supported Protocols for primaryEndPoint are HTTP and TCP. +Supported Protocols for the primaryEndPoint are HTTP and TCP. Generally, it's suggested to use the primaryEndPoint as - a) any available endpoint to check the health of the primary cluster. - b) Primary CIS cluster's health check endpoint (/ready) if accessible. - c) Primary CIS cluster's kube-api server endpoint if accessible. +a) any available endpoint to check the health of the primary cluster. +b) Primary CIS cluster's health check endpoint (/health) if accessible. +c) Primary CIS cluster's kube-api server endpoint if accessible. Response code 200 OK is expected from the primaryEndPoint in case of HTTP Protocol. Successful TCP connection is expected from the primaryEndPoint in case of TCP Protocol. Secondary CIS become active if the primaryEndPoint is not accessible. PrimaryEndPoint is optional to configure for Primary CIS. Note: Primary CIS will not monitor the health of the Secondary CIS cluster. -### How to configure primaryEndPoint in Standalone mode? -primaryEndPoint is not applicable in Standalone mode. It's only applicable in HA mode. +### How to configure the primaryEndPoint in Standalone mode? +The primaryEndPoint is not applicable in Standalone mode. It's only applicable in HA mode. -### How to use CIS /ready endpoint to check the health of CIS? +### How to use CIS /health endpoint to check the health of CIS? Fetch the CIS PodIP and use it in the curl command as shown below from any of the cluster nodes: ``` -curl http://:8080/ready +curl http://:8080/health ``` -Response code 200 OK is expected from the CIS /ready endpoint if kube-api server is accessible. +Response code 200 OK is expected from the CIS /health endpoint if kube-api server is accessible. Example: ``` -[root@cluster-1-worker0 ~]# curl http://10.244.1.213:8080/ready -Ok[root@cluster-1-worker0 ~]# curl http://10.244.1.213:8080/ready -v +[root@cluster-1-worker0 ~]# curl http://10.244.1.213:8080/health +Ok[root@cluster-1-worker0 ~]# curl http://10.244.1.213:8080/health -v * About to connect() to 10.244.1.213 port 8080 (#0) * Trying 10.244.1.213... * Connected to 10.244.1.213 (10.244.1.213) port 8080 (#0) -> GET /ready HTTP/1.1 +> GET /health HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 10.244.1.213:8080 > Accept: */* @@ -657,44 +235,13 @@ Ok[root@cluster-1-worker0 ~]# ``` where 10.244.1.213 is the CIS PodIP. +### Are the custom resources like TS and VS are same in all modes(default, active-active, ratio)? +No, Custom resources like TS and VS are different in default mode and non-default mode. See [Documentation](default-mode/README.md) for default mode and [Documentation](non-default-mode/README.md) for active-active and ratio mode. -### Which services can be provided as multiClusterServices? -Any service running in any OpenShift/Kubernetes clusters which are part of the multiCluster setup can be provided as multiClusterServices. - -### How to configure multiClusterServices in Route annotation? -multiClusterServices is a Route annotation. Below is the sample Route annotation with multiClusterServices: -``` -virtual-server.f5.com/multiClusterServices: -'[ - { - "clusterName": "cluster2", - "service": "svc-pytest-foo-1-com", - "namespace": "foo", - "servicePort": 80, - "weight": 30, - } -]' -``` -where clusterName is the name of the cluster where the service is running, namespace is the namespace where the service is running, servicePort is the port of the service and service is the name of the service. -where cluster2 is the external cluster apart from the HA cluster pair. -Note: External Clusters doesn't need to install CIS - -### How to configure multiClusterServices in Virtual Server CR or Transport Server CR? -multiClusterServices is not supported in VirutalServer CR yet. It's supported in Transport Server CR only when CIS is running in "default" mode. Below is the sample Transport Server CR with multiClusterServices: -``` - pools: - multiClusterServices: - - clusterName: cluster3 - namespace: ns1 - servicePort: 8080 - service: svc-1 - - clusterName: cluster4 - namespace: ns2 - servicePort: 80 - service: svc-ext-1 -``` -where clusterName is the name of the cluster where the service is running, namespace is the namespace where the service is running, servicePort is the port of the service and service is the name of the service. -Note: External Clusters doesn't need to install CIS +### Are the routes supported in all modes(default, active-active, ratio)? +Yes, Routes are supported in all modes. +See [Documentation](non-default-mode/README.md#routes-support) for active-active and ratio mode. +See [Documentation](default-mode/README.md#routes-support) for default mode. -### Can I specify the services running in CIS HA cluster in multiClusterServices? -Yes. multiClusterServices is applicable to refer the services running in K8S/Openshift clusters which are part of the HA cluster(Primary/Secondary Cluster) as well. \ No newline at end of file +### Are policy CR supported in all modes(default, active-active, ratio)? +Yes, Policy CR is supported in all modes same as non-multi-cluster mode. diff --git a/docs/config_examples/multicluster/ServiceTypeLB/sample-policy2.yaml b/docs/config_examples/multicluster/ServiceTypeLB/sample-policy2.yaml deleted file mode 100644 index fb7b8d297..000000000 --- a/docs/config_examples/multicluster/ServiceTypeLB/sample-policy2.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: cis.f5.com/v1 -kind: Policy -metadata: - labels: - f5cr: "true" - name: test-policy2 - namespace: default -spec: - poolSettings: - # reselectTries specifies the maximum number of attempts to find a responsive member for a connection - # Supported values: [0, 65535] - reselectTries: 1 - # serviceDownAction specifies connection handling when member is non-responsive - # Supported values: “drop”, “none”, “reselect”, “reset” - serviceDownAction: none - # BIG-IP AS3 sets the connection rate to a newly-active member slowly during this interval (seconds) - # Supported values: [0, 900] - slowRampTime: 20 \ No newline at end of file diff --git a/docs/config_examples/multicluster/ServiceTypeLB/svc-lb2.yaml b/docs/config_examples/multicluster/ServiceTypeLB/svc-lb2.yaml deleted file mode 100644 index 52679ab64..000000000 --- a/docs/config_examples/multicluster/ServiceTypeLB/svc-lb2.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - annotations: - cis.f5.com/ip: 10.1.1.1 - cis.f5.com/policyName: test-policy2 - labels: - app: svc-lb2 - name: svc-lb2 - namespace: default -spec: - ports: - - name: svc-lb2-80 - port: 80 - protocol: TCP - targetPort: 80 - selector: - app: svc-lb2 - type: LoadBalancer \ No newline at end of file diff --git a/docs/config_examples/multicluster/customResource/transportServer/README.md b/docs/config_examples/multicluster/customResource/transportServer/README.md deleted file mode 100644 index 5dd6068cf..000000000 --- a/docs/config_examples/multicluster/customResource/transportServer/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# Unsecured Transport Server - - -## Weight support in Active-Active, Active-Passive and ratio mode -In multi cluster Active-Active, Active-Passive or ratio mode, weight is supported with local, alternate and extended services. -CIS process the weight and populate each pool member on the bigIP with the specific ratio. -This allows to distribute specific percentage of traffic to a specific service and cluster. - - -`config` - -``` -multiClusterServices: - - clusterName: cluster3 - namespace: default - service: svc-1-external-service - servicePort: 1344 - weight: 70 - - clusterName: cluster4 - namespace: default - service: svc-1-external-service - servicePort: 1344 - weight: 70 -``` - - -**Note:** -* In Active-Active or Active-Passive mode, CIS populates the bigIP with pool members with the weights specified for each pool -* In Ratio mode CIS calculates ratio of each member by taking cluster ratio and service ratio into consideration -* Unlike VS, transport server supports alternate backend and external services(all modes) with single pool. -* To support alternate backend with transport server, CIS by default configures load balancing method to "ratio(member)" at pool level - \ No newline at end of file diff --git a/docs/config_examples/multicluster/customResource/transportServer/ts-with-weights-in-pool-service-and-ab.yaml b/docs/config_examples/multicluster/customResource/transportServer/ts-with-weights-in-pool-service-and-ab.yaml deleted file mode 100644 index 26e0588bc..000000000 --- a/docs/config_examples/multicluster/customResource/transportServer/ts-with-weights-in-pool-service-and-ab.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: cis.f5.com/v1 -kind: TransportServer -metadata: - labels: - f5cr: "true" - name: cr-transport-server - namespace: default -spec: - allowVlans: [] - iRules: - - /Common/test_rule2 - mode: standard - pool: - alternateBackends: - - service: svc-1-external-service - serviceNamespace: default - weight: 20 - - service: svc-2-external-service - serviceNamespace: default - weight: 60 - monitor: - interval: 20 - timeout: 10 - type: udp - service: pytest-svc-1 - servicePort: 1344 - weight: 20 - snat: auto - type: udp - virtualServerAddress: 10.8.0.72 - virtualServerPort: 1344 \ No newline at end of file diff --git a/docs/config_examples/multicluster/customResource/transportServer/ts-without-multiClusterServices.yaml b/docs/config_examples/multicluster/customResource/transportServer/ts-without-multiClusterServices.yaml deleted file mode 100644 index 55d02ae70..000000000 --- a/docs/config_examples/multicluster/customResource/transportServer/ts-without-multiClusterServices.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: "cis.f5.com/v1" -kind: TransportServer -metadata: - labels: - f5cr: "true" - name: transport-server-1 - namespace: default -spec: - virtualServerAddress: "172.16.3.9" - virtualServerPort: 8544 - virtualServerName: ts1 - pool: - service: svc-1 - servicePort: 8181 - loadBalancingMethod: fastest-node - monitor: - type: tcp - interval: 10 - timeout: 10 diff --git a/docs/config_examples/multicluster/default-mode/README.md b/docs/config_examples/multicluster/default-mode/README.md new file mode 100644 index 000000000..3dc4299b7 --- /dev/null +++ b/docs/config_examples/multicluster/default-mode/README.md @@ -0,0 +1,268 @@ +# default mode in Multi-Cluster + +This page documents the default mode for multi cluster support in CIS. + +## Contents + +[Overview](#overview) + +[Default Mode](#default-mode) + +[Supported ExtendedConfigMap Parameters](#extended-configmap-parameters) + +[Transport Server Support](#transport-server-support) + +[Virtual Server Support](#virtual-server-support) + +[Routes Support](#routes-support) + +[ServiceTypeLB Support](#service-type-load-balancer-support) + +[Known Issues](#known-issues) + +[FAQ](#faq) + +## Default Mode + +This is the default mode for multi-cluster support in CIS. This mode is supported with both standalone and HA deployments of CIS. + +In this mode, you need to explicitly define the list of services from all the clusters in the TS and VS CR that you want to expose through CIS. + +In this mode, routes are processed same as any other mode. If multiClusterServices annotation is not provided in the Route, then CIS will fetch the services from primary & secondary clusters and add the pool-members. If multiClusterServices annotation is provided in the Route, then CIS will fetch the services from the primary & secondary clusters as well as from the clusters mentioned in the annotation and add the pool-members. + +This mode also supports the discovery of serviceType LB resources from external clusters including HA peer cluster. + +MultiCluster LoadBalancer services are also supported in this mode. Where you can expose the same serviceType LB from the multiple clusters. + + +#### Configuring extendedConfigMapForStandAlone + +Below is the sample Multi-Cluster Config in an Extended ConfigMap with default mode in standalone topology. +``` + extendedSpec: | + mode: default + externalClustersConfig: ----------------------------------|---------------------------------------------------| | + - clusterName: cluster3 | | + secret: default/kubeconfig3 |---> Cluster configs for | + serviceTypeLBDiscovery: true |---> CIS discovers the LB services in this cluster | + - clusterName: cluster4 | all clusters | + secret: default/kubeconfig4 | | + - clusterName: cluster5 | | + secret: default/kubeconfig5 ------------------------------|---------------------------------------------------| +``` + + +#### Configuring extendedConfigMapForHighAvailability + +Below is the sample Multi-Cluster Config in an Extended ConfigMap with default mode in high availability topology. + +``` + extendedSpec: | + mode: default ----------------------------------------|----------------------------| + highAvailabilityCIS: --------------------------------------| | + primaryEndPoint: http://10.145.72.114:8001 | | + probeInterval: 30 | | + retryInterval: 3 | | + primaryCluster: |---> Cluster configs for | + clusterName: cluster1 | High availability | + secret: default/kubeconfig1 | clusters |---> Multi-Cluster configs + secondaryCluster: | | + clusterName: cluster2 | | + secret: default/kubeconfig2 | | + externalClustersConfig: ----------------------------------| | + - clusterName: cluster3 | | + secret: default/kubeconfig3 |---> Cluster configs for | + - clusterName: cluster4 | all other clusters | + secret: default/kubeconfig4 | except HA clusters | + - clusterName: cluster5 | | + secret: default/kubeconfig5 | | +``` + + +### extended ConfigMap Parameters + +**externalClustersConfig Parameters** + +| Parameter | Type | Required | Description | Default | Examples | +|-------------|--------|-----------|---------------------------------------------------------------------------|---------|-------------------------| +| clusterName | String | Mandatory | Name of the cluster | - | cluster1 | +| secret | String | Mandatory | Name of the secret created for kubeconfig (format: namespace/secret-name) | - | test/secret-kubeconfig1 | + + +Note: Avoid specifying HA cluster(Primary/Secondary cluster) configs in externalClustersConfig. + +**Mode Parameter** + +| Parameter | Type | Required | Description | Default | Examples | +|-----------|--------|----------|------------------------------------------------------|---------|----------| +| mode | Object | Optional | Type of high availability mode (active-active/ratio) | default | default | + +Specifies whether the CIS HA cluster is configured with active-active mode, default mode or ratio mode. +* active-active: CIS fetches service from both the HA clusters whenever it's referenced in Route Spec. +* ratio: CIS works in active-active mode and, it splits traffic according to the ratio specified for each cluster. +* default: See Documentation for more [details](../default-mode/README.md) + +**highAvailabilityCIS Parameters** + +| Parameter | Type | Required | Description | Default | Examples | +|------------------------|---------|-----------|-------------------------------------------------------------------------|---------|---------------------------| +| primaryClusterEndPoint | String | Mandatory | Endpoint to check health of primary cluster | - | http://10.145.72.114:8001 | +| probeInterval | Integer | Optional | Time interval between health check (in seconds) | 60 | 30 | +| retryInterval | Integer | Optional | Time interval between recheck when primary cluster is down (in seconds) | 15 | 3 | +| primaryCluster | Object | Mandatory | Primary cluster config | - | - | +| secondaryCluster | Object | Mandatory | Secondary cluster config | - | - | + +Health probe parameters are provided in highAvailabilityCIS in extended configmap, helping to ensure high availability of CIS. CIS running in secondary cluster continuously monitors the health of the primary cluster. If it's down, then the secondary CIS takes the responsibility of posting declarations to BIG-IP. + +Note: The primaryEndPoint is a mandatory parameter if CIS is intended to run in Multi-Cluster HA mode. If this is not specified the secondary CIS will not run. + + +**primaryCluster/secondaryCluster Parameters** + +| Parameter | Type | Required | Description | Default | Examples | +|-------------|--------|-----------|---------------------------------------------------------------------------|---------|-------------------------| +| clusterName | String | Mandatory | Name of the cluster | - | cluster1 | +| secret | String | Mandatory | Name of the secret created for kubeconfig (format: namespace/secret-name) | - | test/secret-kubeconfig1 | + + +Note: In order to run CIS in high availability mode, multi-cluster-mode parameter (primary/secondary) needs to be set in the CIS deployment arguments. +* It's recommended to provide both primaryCluster and secondaryCluster configs in the extendedConfigMap. + +### Transport Server Support +* CIS does not support the service and alternateBackend service parameters in this mode, you need to use the spec.Pool.multiClusterServices properties to define the services from all the clusters. +* See [Examples](transportServer) + +### Virtual Server Support +* CIS does not support the service and alternateBackend service parameters in this mode, you need to use the spec.Pools[*].multiClusterServices properties to define the services from all the clusters. +* See [Examples](virtualServer) + +### Routes Support +* If multiClusterServices annotation is not provided in the Route, then CIS will fetch the services from primary & secondary clusters and add the pool-members. +* If multiClusterServices annotation is provided in the Route, then CIS will fetch the services from the primary & secondary clusters as well as from the clusters mentioned in the annotation and add the pool-members. +* See Examples [here](routes) +### Route Annotation for Multi-ClusterServices +Services running in any other OpenShift clusters, as mentioned below: +``` +virtual-server.f5.com/multiClusterServices: +'[ + { + "clusterName": "cluster2", + "service": "svc-pytest-foo-1-com", + "namespace": "foo", + "servicePort": 80, + "weight": 30, + } +]' +``` + +| Parameter | Type | Required | Description | Default | Examples | +|-------------|------------|------------|---------------------------------------------------------|---------|----------| +| clusterName | String | Mandatory | Name of the cluster | - | cluster1 | +| service | String | Mandatory | Name of the service | - | svc-1 | +| namespace | String | Mandatory | Namespace where the service is created | - | test | +| servicePort | String/Int | Mandatory | port of the service (for named port use string value ) | - | 80 | +| weight | Int | Optional | weight to be used for traffic splitting | 0 | 20 | + +### Service Type Load Balancer Support +In multiCluster environment CIS offers following two solutions for supporting ServiceTypeLBs: + +1) Non-multiCluster ServiceTypeLB - In this case, CIS will discover the ServiceTypeLBs from the local cluster where CIS is running. The ServiceTypeLBs can be from the same cluster or from the external clusters. The ServiceTypeLBs from the external clusters can be discovered by setting the serviceTypeLBDiscovery to true in the extendedConfigMap. See [Example](ServiceTypeLB/svc-lb.yaml) + +2) MultiCluster ServiceTypeLB - In this case the LB services must be created in all the clusters where CIS is running, and the multiClusterServices annotation must be added to the LB services. The multiClusterServices annotation must contain the clusterNames and Weights. The clusterNames represent the names of the K8S/Openshift clusters where services are created. The Weights represent the weightage for the traffic distribution for the service in a particular cluster. See [Example](ServiceTypeLB/sample-multi-cluster-svc-lb.yaml) + +## Known Issues +* CIS is not able to clear serviceTypeLB status for some external clusters when TS/VS is deleted or label is removed in default mode +* CIS is unable to add the pool-members for external cluster intermittently in default mode when using namespace-labels in the CIS deployment. + +## FAQ + +### Which services can be provided as multiClusterServices? +Any service running in any OpenShift/Kubernetes clusters which are part of the multiCluster setup can be provided as multiClusterServices. + +### Can I specify the services running in CIS HA cluster in multiClusterServices? +Yes. multiClusterServices is applicable to refer the services running in K8S/Openshift clusters which are part of the HA cluster(Primary/Secondary Cluster) as well. + +### Is cluster-level traffic splitting supported in default mode? +No, cluster-level traffic splitting is not supported in default mode. + +### Can I put the services of a cluster in the maintenance mode? +Yes, you can put the services of a cluster in the maintenance mode, to do so you can update the weights of the services to 0 for that cluster. + +### adminState is supported in default mode for a cluster? +No, adminState is not supported in default mode for a cluster and adminState field will be ignored in extendedConfigMap. + +### Is ratio or localClusterRatio properties are supported in default mode for extendedConfigMap? +No, ratio or localClusterRatio properties are not supported in default mode for extendedConfigMap. + +### Is serviceTypeLBDiscovery supported for modes other than default mode? +No, serviceTypeLBDiscovery is supported only for default mode. + +### Are serviceTypeLBDiscovery enabled for all the clusters? +No, serviceTypeLBDiscovery is not enabled for all the clusters. It is enabled only for the cluster where active CIS is running. + +### How to enable the ServiceTypeLBDiscovery for external clusters? +To enable the ServiceTypeLBDiscovery for external clusters, set the serviceTypeLBDiscovery to true in the extendedConfigMap for the respective cluster. + +### How to enable the ServiceTypeLBDiscovery for secondary cluster? +To enable the ServiceTypeLBDiscovery for secondary clusters, set the serviceTypeLBDiscovery to true in the extendedConfigMap for the respective cluster. + +### Do I need to enable the ServiceTypeLBDiscovery if I am using only multiCluster ServicetypeLB? +No, you don't need to enable the ServiceTypeLBDiscovery if you are using only multiCluster ServiceTypeLB. + +### When to enable the ServiceTypeLBDiscovery for a cluster? +Enable the ServiceTypeLBDiscovery for a cluster when you want to expose the ServiceTypeLBs from that cluster to the CIS. + +### Does CIS merge the serviceTypeLBs from all the clusters? +No, CIS does not merge the serviceTypeLBs from all the clusters. CIS discovers the serviceTypeLBs from the local cluster where CIS is running and from the external/secondary clusters where serviceTypeLBDiscovery is enabled. + +### How to convert Non-MultiCluster ServiceTypeLB to MultiCluster ServiceTypeLB +In order to convert Non-MultiCluster ServiceTypeLB to MultiCluster ServiceTypeLB follow the steps mentioned below: + +1) Add the multiClusterServices annotations to the LB services. + + In multiClusterServices annotations you need to provide the clusterNames and Weights. + + clusterNames represent names of the K8S/Openshift clusters where this ServiceTypeLBs are created. + + Weights represent weightage for the traffic distribution for the serviceTypeLB in a particular cluster. + + Example: + + cis.f5.com/multiClusterServices: | + [ + {"clusterName": "cluster2", "weight": 50}, + {"clusterName": "cluster3", "weight": 30}, + {"clusterName": "cluster4", "weight": 20} + ] +2) Make sure to add these annotations to all the LB services which you want to convert to multiCluster ServiceTypeLB. + +3) Make sure to create the multiCluster ServiceTypeLB in each of the clusters where CIS runs. + + These multiCluster ServiceTypeLBs may or may not expose any application in these clusters, however these are required. CIS will read these multiCluster LB Services from the local cluster and discover the associated LB services referenced in the multiClusterServices annotation. + +4) All the multiCluster services with the same IP and Port must have the same Specs. + +5) The ServiceTypeLBDiscovery can be disabled or removed if you don't want to use non-multiCluster ServiceTypeLB. + + +### How to configure multiClusterServices in Route annotation? +multiClusterServices is a Route annotation. Below is the sample Route annotation with multiClusterServices: +``` +virtual-server.f5.com/multiClusterServices: +'[ + { + "clusterName": "cluster2", + "service": "svc-pytest-foo-1-com", + "namespace": "foo", + "servicePort": 80, + "weight": 30, + } +]' +``` +where clusterName is the name of the cluster where the service is running, namespace is the namespace where the service is running, servicePort is the port of the service and service is the name of the service. +where cluster2 is the external cluster apart from the HA cluster pair. +Note: External Clusters doesn't need to install CIS + + +### Are policy CR supported in default mode? +Yes, Policy CR is supported in all modes same as non-multi-cluster mode. diff --git a/docs/config_examples/multicluster/ServiceTypeLB/README.md b/docs/config_examples/multicluster/default-mode/ServiceTypeLB/README.md similarity index 100% rename from docs/config_examples/multicluster/ServiceTypeLB/README.md rename to docs/config_examples/multicluster/default-mode/ServiceTypeLB/README.md diff --git a/docs/config_examples/multicluster/ServiceTypeLB/sample-multi-cluster-svc-lb.yaml b/docs/config_examples/multicluster/default-mode/ServiceTypeLB/sample-multi-cluster-svc-lb.yaml similarity index 100% rename from docs/config_examples/multicluster/ServiceTypeLB/sample-multi-cluster-svc-lb.yaml rename to docs/config_examples/multicluster/default-mode/ServiceTypeLB/sample-multi-cluster-svc-lb.yaml diff --git a/docs/config_examples/multicluster/ServiceTypeLB/sample-policy.yaml b/docs/config_examples/multicluster/default-mode/ServiceTypeLB/sample-policy.yaml similarity index 100% rename from docs/config_examples/multicluster/ServiceTypeLB/sample-policy.yaml rename to docs/config_examples/multicluster/default-mode/ServiceTypeLB/sample-policy.yaml diff --git a/docs/config_examples/multicluster/ServiceTypeLB/svc-lb1.yaml b/docs/config_examples/multicluster/default-mode/ServiceTypeLB/svc-lb.yaml similarity index 100% rename from docs/config_examples/multicluster/ServiceTypeLB/svc-lb1.yaml rename to docs/config_examples/multicluster/default-mode/ServiceTypeLB/svc-lb.yaml diff --git a/docs/config_examples/multicluster/extendedConfigmap/global-spec-config-for-multicluster-default-mode-svcLB.yaml b/docs/config_examples/multicluster/default-mode/extendedConfigmap/global-spec-config-for-multicluster-default-mode-high-availability-svcLB.yaml similarity index 100% rename from docs/config_examples/multicluster/extendedConfigmap/global-spec-config-for-multicluster-default-mode-svcLB.yaml rename to docs/config_examples/multicluster/default-mode/extendedConfigmap/global-spec-config-for-multicluster-default-mode-high-availability-svcLB.yaml diff --git a/docs/config_examples/multicluster/extendedConfigmap/global-spec-config-for-multicluster-default-mode.yaml b/docs/config_examples/multicluster/default-mode/extendedConfigmap/global-spec-config-for-multicluster-default-mode-high-availablity.yaml similarity index 100% rename from docs/config_examples/multicluster/extendedConfigmap/global-spec-config-for-multicluster-default-mode.yaml rename to docs/config_examples/multicluster/default-mode/extendedConfigmap/global-spec-config-for-multicluster-default-mode-high-availablity.yaml diff --git a/docs/config_examples/multicluster/default-mode/extendedConfigmap/global-spec-config-for-multicluster-default-mode-standalone-svcLB.yaml b/docs/config_examples/multicluster/default-mode/extendedConfigmap/global-spec-config-for-multicluster-default-mode-standalone-svcLB.yaml new file mode 100644 index 000000000..80337b099 --- /dev/null +++ b/docs/config_examples/multicluster/default-mode/extendedConfigmap/global-spec-config-for-multicluster-default-mode-standalone-svcLB.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + f5nr: "true" + name: extended-spec-config + namespace: kube-system +data: + extendedSpec: | + mode: default + externalClustersConfig: + - clusterName: cluster1 + secret: default/kubeconfig1 + serviceTypeLBDiscovery: true # If set to true then CIS will watch for serviceTypeLB in this cluster.default is false + - clusterName: cluster2 + secret: default/kubeconfig2 + serviceTypeLBDiscovery: true # If set to true then CIS will watch for serviceTypeLB in this cluster.default is false + - clusterName: cluster3 + secret: default/kubeconfig3 diff --git a/docs/config_examples/multicluster/extendedConfigmap/extendedConfigmap-with-admin-state-no-pool/global-spec-config-for-multicluster-with-HA-CIS-default-mode-and-cluster.yaml b/docs/config_examples/multicluster/default-mode/extendedConfigmap/global-spec-config-for-multicluster-default-mode-standalone.yaml similarity index 51% rename from docs/config_examples/multicluster/extendedConfigmap/extendedConfigmap-with-admin-state-no-pool/global-spec-config-for-multicluster-with-HA-CIS-default-mode-and-cluster.yaml rename to docs/config_examples/multicluster/default-mode/extendedConfigmap/global-spec-config-for-multicluster-default-mode-standalone.yaml index cf22e4845..d8e4a2f59 100644 --- a/docs/config_examples/multicluster/extendedConfigmap/extendedConfigmap-with-admin-state-no-pool/global-spec-config-for-multicluster-with-HA-CIS-default-mode-and-cluster.yaml +++ b/docs/config_examples/multicluster/default-mode/extendedConfigmap/global-spec-config-for-multicluster-default-mode-standalone.yaml @@ -9,14 +9,9 @@ data: extendedSpec: | mode: default externalClustersConfig: + - clusterName: cluster1 + secret: default/kubeconfig1 + - clusterName: cluster2 + secret: default/kubeconfig2 - clusterName: cluster3 secret: default/kubeconfig3 - adminState: offline - - clusterName: cluster4 - secret: default/kubeconfig4 - extendedRouteSpec: - - allowOverride: false - namespace: foo - policyCR: foo/cr-policy1 - vserverAddr: 10.8.0.4 - vserverName: vs-foo diff --git a/docs/config_examples/multicluster/routes/route-with-multicluster-service-annotation-and-ab.yaml b/docs/config_examples/multicluster/default-mode/routes/route-with-multicluster-service-annotation-and-ab.yaml similarity index 100% rename from docs/config_examples/multicluster/routes/route-with-multicluster-service-annotation-and-ab.yaml rename to docs/config_examples/multicluster/default-mode/routes/route-with-multicluster-service-annotation-and-ab.yaml diff --git a/docs/config_examples/multicluster/routes/route-with-multicluster-service-annotation.yaml b/docs/config_examples/multicluster/default-mode/routes/route-with-multicluster-service-annotation.yaml similarity index 100% rename from docs/config_examples/multicluster/routes/route-with-multicluster-service-annotation.yaml rename to docs/config_examples/multicluster/default-mode/routes/route-with-multicluster-service-annotation.yaml diff --git a/docs/config_examples/multicluster/default-mode/transportServer/ts-multiClusterServices-no-weights.yaml b/docs/config_examples/multicluster/default-mode/transportServer/ts-multiClusterServices-no-weights.yaml new file mode 100644 index 000000000..cd3a22ccf --- /dev/null +++ b/docs/config_examples/multicluster/default-mode/transportServer/ts-multiClusterServices-no-weights.yaml @@ -0,0 +1,39 @@ +apiVersion: cis.f5.com/v1 +kind: TransportServer +metadata: + labels: + f5cr: "true" + name: cr-transport-server + namespace: default +spec: + allowVlans: [] + iRules: + - /Common/test_rule2 + mode: standard + pool: + multiClusterServices: + # CIS supports to refer svs from local cluster and ha cluster + - clusterName: cluster1 + namespace: default + service: svc-12-external-service + servicePort: 1344 + - clusterName: cluster2 + namespace: default + service: svc-21-external-service + servicePort: 1344 + - clusterName: cluster3 + namespace: default + service: svc-1-external-service + servicePort: 1344 + - clusterName: cluster4 + namespace: default + service: svc-1-external-service + servicePort: 1344 + monitor: + interval: 20 + timeout: 10 + type: udp + snat: auto + type: udp + virtualServerAddress: 10.8.0.72 + virtualServerPort: 1344 \ No newline at end of file diff --git a/docs/config_examples/multicluster/customResource/transportServer/ts-with-multiClusterServices.yaml b/docs/config_examples/multicluster/default-mode/transportServer/ts-with-multiClusterServices.yaml similarity index 100% rename from docs/config_examples/multicluster/customResource/transportServer/ts-with-multiClusterServices.yaml rename to docs/config_examples/multicluster/default-mode/transportServer/ts-with-multiClusterServices.yaml diff --git a/docs/config_examples/multicluster/default-mode/virtualServer/vs-with-multiClusterServices-no-weights.yaml b/docs/config_examples/multicluster/default-mode/virtualServer/vs-with-multiClusterServices-no-weights.yaml new file mode 100644 index 000000000..ef50f285e --- /dev/null +++ b/docs/config_examples/multicluster/default-mode/virtualServer/vs-with-multiClusterServices-no-weights.yaml @@ -0,0 +1,35 @@ +apiVersion: cis.f5.com/v1 +kind: VirtualServer +metadata: + labels: + f5cr: "true" + name: tea-virtual-server-edge + namespace: default +spec: + host: tea.example.com + virtualServerAddress: 10.8.0.71 + pools: + - path: /neam + multiClusterServices: + # CIS supports to refer svs from local cluster and ha cluster + - clusterName: cluster1 + namespace: default + service: svc-12-external-service + servicePort: 80 + - clusterName: cluster2 + namespace: default + service: svc-21-external-service + servicePort: 80 + - clusterName: cluster3 + namespace: default + service: svc-1-external-service + servicePort: 80 + - clusterName: cluster4 + namespace: default + service: svc-1-external-service + servicePort: 80 + monitor: + interval: 20 + timeout: 10 + type: http + send: GET /health HTTP/1.0 \ No newline at end of file diff --git a/docs/config_examples/multicluster/default-mode/virtualServer/vs-with-multiClusterServices.yaml b/docs/config_examples/multicluster/default-mode/virtualServer/vs-with-multiClusterServices.yaml new file mode 100644 index 000000000..1cae311f7 --- /dev/null +++ b/docs/config_examples/multicluster/default-mode/virtualServer/vs-with-multiClusterServices.yaml @@ -0,0 +1,39 @@ +apiVersion: cis.f5.com/v1 +kind: VirtualServer +metadata: + labels: + f5cr: "true" + name: tea-virtual-server-edge + namespace: default +spec: + host: tea.example.com + virtualServerAddress: 10.8.0.71 + pools: + - path: /neam + multiClusterServices: + # CIS supports to refer svs from local cluster and ha cluster + - clusterName: cluster1 + namespace: default + service: svc-12-external-service + servicePort: 80 + weight: 70 + - clusterName: cluster2 + namespace: default + service: svc-21-external-service + servicePort: 80 + weight: 70 + - clusterName: cluster3 + namespace: default + service: svc-1-external-service + servicePort: 80 + weight: 70 + - clusterName: cluster4 + namespace: default + service: svc-1-external-service + servicePort: 80 + weight: 70 + monitor: + interval: 20 + timeout: 10 + type: http + send: GET /health HTTP/1.0 \ No newline at end of file diff --git a/docs/config_examples/multicluster/non-default-mode/README.md b/docs/config_examples/multicluster/non-default-mode/README.md new file mode 100644 index 000000000..ffc323a25 --- /dev/null +++ b/docs/config_examples/multicluster/non-default-mode/README.md @@ -0,0 +1,532 @@ +# Ratio and Active-Active mode in Multi-Cluster + +This page documents the ratio and active-active mode for multi cluster support in CIS. + +## Contents + +[Overview](#overview) + +[Active-Active Mode](#active-active-mode) + +[Ratio Mode](#ratio-mode) + +[Supported ExtendedConfigMap Parameters](#extended-configmap-parameters) + +[Transport Server Support](#transport-server-support) + +[Virtual Server Support](#virtual-server-support) + +[Routes Support](#routes-support) + +[Known Issues](#known-issues) + +[FAQ](#faq) + + +## Overview + +CIS supports ratio and active-active modes as well for multi-cluster. These modes are used to distribute traffic across multiple clusters based on the ratio defined for each cluster. + +## Active-Active Mode +* In case of active-active mode CIS running on Primary cluster updates the pool members for Virtual Servers from both the clusters those are part of HA Cluster(Primary and Secondary Clusters) as well as pool members from all other remotely monitored clusters. +* However, in case the Primary cluster is down and CIS on the Secondary cluster has taken control then pool member from the Secondary Cluster as well as all other remotely monitored clusters are populated for the Virtual Servers irrespective of the value of HA mode. + +**Note**: +* For HA mode [namely default, active-active, ratio], CIS monitored resource manifests(such as routes, CRDs, extendedConfigmaps) must be available in both the clusters. +* The CIS monitored resource manifests must be identical on both primary and Secondary Clusters +* So, In case of CIS fail-over, CIS on Secondary Cluster will take control and will start processing the CIS monitored resource manifests. +* CIS on Secondary Cluster will not process the CIS monitored resource manifests if they are not available in Secondary Cluster. +* MakeSure to have identical resource manifests in both the clusters to avoid any issues during CIS fail-over. + +![architecture](../images/haMultiCluster.png) + + +### Configuring Extended ConfigMap for Active-Active High Availability CIS +Below is the sample Multi-Cluster Configs with HA in Extended ConfigMap. +``` + extendedSpec: | + mode: active-active -----------------------------------|----> HA Mode | + highAvailabilityCIS: --------------------------------------|----------------------------| + primaryEndPoint: http://10.145.72.114:8001 | | + probeInterval: 30 | | + retryInterval: 3 | | + primaryCluster: |---> Cluster configs for | + clusterName: cluster1 | High availability | + secret: default/kubeconfig1 | clusters |---> Multi-Cluster configs + secondaryCluster: | | + clusterName: cluster2 | | + secret: default/kubeconfig2 | | + externalClustersConfig: -------------------------------------| | + - clusterName: cluster3 | | + secret: default/kubeconfig3 |---> Cluster configs for | + - clusterName: cluster4 | all other clusters | + secret: default/kubeconfig4 | except HA clusters | + - clusterName: cluster5 | | + secret: default/kubeconfig5 ------------------------------|----------------------------| + extendedRouteSpec: + - namespace: foo -------------------------------------| + vserverAddr: 10.8.0.4 | + vserverName: nextgenroutes |----------------> RouteGroup with namespace + allowOverride: true | + bigIpPartition: MultiTenant | + policyCR: default/sample-policy _____________________| + - namespace: bar -------------------------------------| + vserverAddr: 10.8.0.5 |----------------> RouteGroup with namespace + allowOverride: false _____________________| +``` +**Note**: extendedRouteSpec is only applicable in case of openshift route resources not for CRD resources. + +### Configuring Extended ConfigMap for Active-Active High Availability CIS with Cluster AdminState +Below is the sample Multi-Cluster Configs with HA and cluster AdminState in Extended ConfigMap. +``` + extendedSpec: | + mode: active-active --------------------------------------|----------------------------| + highAvailabilityCIS: --------------------------------------| | + primaryEndPoint: http://10.145.72.114:8001 | | + probeInterval: 30 | | + retryInterval: 3 | | + primaryCluster: |---> Cluster configs for | + clusterName: cluster1 | High availability | + secret: default/kubeconfig1 | clusters |---> Multi-Cluster configs + secondaryCluster: | | + clusterName: cluster2 | | + secret: default/kubeconfig2 | | + adminState: enable | | + externalClustersConfig: ----------------------------------| | + - clusterName: cluster3 | | + secret: default/kubeconfig3 |---> Cluster configs for | + adminState: disable | all other clusters | + - clusterName: cluster4 | except HA clusters | + secret: default/kubeconfig4 | | + - clusterName: cluster5 | | + secret: default/kubeconfig5 | | + adminState: offline ------------------------------|----------------------------| + extendedRouteSpec: + - namespace: foo -------------------------------------| + vserverAddr: 10.8.0.4 | + vserverName: nextgenroutes |----------------> RouteGroup with namespace + allowOverride: true | + bigIpPartition: MultiTenant | + policyCR: default/sample-policy _____________________| + - namespace: bar -------------------------------------| + vserverAddr: 10.8.0.5 |----------------> RouteGroup with namespace + allowOverride: false _____________________| +``` +**Note**: extendedRouteSpec is only applicable in case of openshift route resources not for CRD resources. + + +## Ratio Mode +Ratio mode is a feature in CIS that allows users to distribute traffic across multiple clusters based on the ratio defined for each cluster. This feature is supported in both CIS HA and standalone environment. In ratio mode, CIS works in active-active mode and, it splits traffic according to the ratio values defined for each cluster. + +### Configuring Extended ConfigMap for Ratio Mode Standalone CIS +Below is the sample Multi-Cluster Config in an Extended ConfigMap with ratio mode in standalone topology. +``` + extendedSpec: | + mode: ratio + localClusterRatio: 4 | | + externalClustersConfig: ----------------------------------|----------------------------| | + - clusterName: cluster3 | | + secret: default/kubeconfig3 |---> Cluster configs for | + ratio: 3 | all clusters | + - clusterName: cluster4 | | + secret: default/kubeconfig4 | | + ratio: 4 | | + - clusterName: cluster5 | | + secret: default/kubeconfig5 ------------------------------|----------------------------| + ratio: 5 | | + extendedRouteSpec: + - namespace: foo -------------------------------------| + vserverAddr: 10.8.0.4 | + vserverName: nextgenroutes |----------------> RouteGroup with namespace + allowOverride: true | + bigIpPartition: MultiTenant | + policyCR: default/sample-policy _____________________| + - namespace: bar -------------------------------------| + vserverAddr: 10.8.0.5 |----------------> RouteGroup with namespace + allowOverride: false _____________________| +``` +**Note**: extendedRouteSpec is only applicable in case of openshift route resources not for CRD resources. + +### Configuring Extended ConfigMap for Cluster AdminState in Ratio Mode Standalone CIS +Below is the sample Multi-Cluster Configs with standalone CIS and cluster AdminState in Extended ConfigMap. +``` + extendedSpec: | + localClusterAdminState: disable ----------------------------|AdminState for local cluster| + externalClustersConfig: ----------------------------------|----------------------------| + - clusterName: cluster3 | | + secret: default/kubeconfig3 |---> Cluster configs for | + adminState: enable | all clusters | + - clusterName: cluster4 | | + secret: default/kubeconfig4 | | + - clusterName: cluster5 | | + secret: default/kubeconfig5 | | + adminState: offline ------------------------------|----------------------------| + extendedRouteSpec: + - namespace: foo -------------------------------------| + vserverAddr: 10.8.0.4 | + vserverName: nextgenroutes |----------------> RouteGroup with namespace + allowOverride: true | + bigIpPartition: MultiTenant | + policyCR: default/sample-policy _____________________| + - namespace: bar -------------------------------------| + vserverAddr: 10.8.0.5 |----------------> RouteGroup with namespace + allowOverride: false _____________________| +``` +**Note**: localClusterAdminState is only applicable in case of standalone CIS. It's ignored if specified in HA CIS mode. + +### Configuring Extended ConfigMap for Ratio Mode High Availability CIS + +Below is the sample Multi-Cluster Configs with HA and Ratio in Extended ConfigMap. +``` + extendedSpec: | + mode: ratio ------------------------------------------|----------------------------| + highAvailabilityCIS: --------------------------------------| | + primaryEndPoint: http://10.145.72.114:8001 | | + probeInterval: 30 | | + retryInterval: 3 | | + primaryCluster: |---> Cluster configs for | + clusterName: cluster1 | High availability | + secret: default/kubeconfig1 | clusters |---> Multi-Cluster configs + ratio: 3 | | + adminState: enable | | + secondaryCluster: | | + clusterName: cluster2 | | + secret: default/kubeconfig2 | | + ratio: 2 | | + adminState: enable | all other clusters | + externalClustersConfig: ----------------------------------| | + - clusterName: cluster3 | | + secret: default/kubeconfig3 |---> Cluster configs for | + ratio: 2 | all other clusters | + adminState: disable | except HA clusters | + - clusterName: cluster4 | | + secret: default/kubeconfig4 | | + - clusterName: cluster5 | | + secret: default/kubeconfig5 | | + adminState: offline | | + ratio: 1 ------------------------------|----------------------------| + extendedRouteSpec: + - namespace: foo -------------------------------------| + vserverAddr: 10.8.0.4 | + vserverName: nextgenroutes |----------------> RouteGroup with namespace + allowOverride: true | + bigIpPartition: MultiTenant | + policyCR: default/sample-policy _____________________| + - namespace: bar -------------------------------------| + vserverAddr: 10.8.0.5 |----------------> RouteGroup with namespace + allowOverride: false _____________________| +``` +**Note**: extendedRouteSpec is only applicable in case of openshift route resources not for CRD resources. + +### Configuring Extended ConfigMap for Cluster AdminState in Ratio Mode High Availability CIS + +``` + extendedSpec: | + mode: ratio ------------------------------------------|----------------------------| + highAvailabilityCIS: --------------------------------------| | + primaryEndPoint: http://10.145.72.114:8001 | | + probeInterval: 30 | | + retryInterval: 3 | | + primaryCluster: |---> Cluster configs for | + clusterName: cluster1 | High availability | + secret: default/kubeconfig1 | clusters |---> Multi-Cluster configs + ratio: 3 | | + secondaryCluster: | | + clusterName: cluster2 | | + secret: default/kubeconfig2 | | + ratio: 2 | | + externalClustersConfig: -------------------------------------| | + - clusterName: cluster3 | | + secret: default/kubeconfig3 |---> Cluster configs for | + ratio: 2 | all other clusters | + - clusterName: cluster4 | except HA clusters | + secret: default/kubeconfig4 | | + - clusterName: cluster5 | | + secret: default/kubeconfig5 | | + ratio: 1 ------------------------------|----------------------------| + extendedRouteSpec: + - namespace: foo -------------------------------------| + vserverAddr: 10.8.0.4 | + vserverName: nextgenroutes |----------------> RouteGroup with namespace + allowOverride: true | + bigIpPartition: MultiTenant | + policyCR: default/sample-policy _____________________| + - namespace: bar -------------------------------------| + vserverAddr: 10.8.0.5 |----------------> RouteGroup with namespace + allowOverride: false _____________________| +``` + +**Note**: extendedRouteSpec is only applicable in case of openshift route resources not for CRD resources. + +### extended ConfigMap Parameters + +**externalClustersConfig Parameters** + +| Parameter | Type | Required | Description | Default | Examples | +|-------------|--------|-----------|---------------------------------------------------------------------------|---------|-------------------------| +| clusterName | String | Mandatory | Name of the cluster | - | cluster1 | +| secret | String | Mandatory | Name of the secret created for kubeconfig (format: namespace/secret-name) | - | test/secret-kubeconfig1 | +| ratio | int | Optional | Ratio at which the traffic has to be distributed over clusters | 1 | 3 | +| adminState | String | Optional | adminState can be used to disable/enable/offline/no-pool clusters | enable | disable | + + +Note: Avoid specifying HA cluster(Primary/Secondary cluster) configs in externalClustersConfig. + +**Mode Parameter** + +| Parameter | Type | Required | Description | Default | Examples | +|-----------|--------|----------|------------------------------------------------------|---------|----------| +| mode | Object | Optional | Type of high availability mode (active-active/ratio) | default | default | + +Specifies whether the CIS HA cluster is configured with active-active mode, default mode or ratio mode. +* active-active: CIS fetches service from both the HA clusters whenever it's referenced in Route Spec. +* ratio: CIS works in active-active mode and, it splits traffic according to the ratio specified for each cluster. +* default: See Documentation for more [details](../default-mode/README.md) + +**Local cluster ratio Parameter** + +| Parameter | Type | Required | Description | Default | Examples | +|------------------------|------|-----------|-----------------------------------------------------------------------------------------------------------|---------|----------| +| localClusterRatio | Int | Optional | Ratio for the local cluster where CIS is running(specify only when using ratio in CIS non-HA environment) | 1 | 3 | + +Note: It is not needed in case of using ratio in CIS HA environment, as ratio of Primary cluster does the same thing. If specified in this scenario then it will be ignored. + +**highAvailabilityCIS Parameters** + +| Parameter | Type | Required | Description | Default | Examples | +|------------------------|---------|-----------|-------------------------------------------------------------------------|---------|---------------------------| +| primaryClusterEndPoint | String | Mandatory | Endpoint to check health of primary cluster | - | http://10.145.72.114:8001 | +| probeInterval | Integer | Optional | Time interval between health check (in seconds) | 60 | 30 | +| retryInterval | Integer | Optional | Time interval between recheck when primary cluster is down (in seconds) | 15 | 3 | +| primaryCluster | Object | Mandatory | Primary cluster config | - | - | +| secondaryCluster | Object | Mandatory | Secondary cluster config | - | - | + +Health probe parameters are provided in highAvailabilityCIS in extended configmap, helping to ensure high availability of CIS. CIS running in secondary cluster continuously monitors the health of the primary cluster. If it's down, then the secondary CIS takes the responsibility of posting declarations to BIG-IP. + +Note: The primaryEndPoint is a mandatory parameter if CIS is intended to run in Multi-Cluster HA mode. If this is not specified the secondary CIS will not run. + + +**primaryCluster/secondaryCluster Parameters** + +| Parameter | Type | Required | Description | Default | Examples | +|-------------|--------|-----------|---------------------------------------------------------------------------|---------|-------------------------| +| clusterName | String | Mandatory | Name of the cluster | - | cluster1 | +| secret | String | Mandatory | Name of the secret created for kubeconfig (format: namespace/secret-name) | - | test/secret-kubeconfig1 | +| ratio | int | Optional | Ratio at which the traffic has to be distributed over clusters | 1 | 3 | +| adminState | String | Optional | adminState can be used to disable/enable/offline/no-pool clusters | enable | disable | + + +Note: In order to run CIS in high availability mode, multi-cluster-mode parameter (primary/secondary) needs to be set in the CIS deployment arguments. +* It's recommended to provide both primaryCluster and secondaryCluster configs in the extendedConfigMap. +* If no traffic has to be forwarded to a specific cluster then set the ratio field to 0. + +### Transport Server Support +* CIS does the service and alternateBackend service discovery in all the clusters specified via the extended ConfigMap. +* See [Examples](transportServer) + +### Virtual Server Support +* CIS does the service and alternateBackend service discovery in all the clusters specified via the extended ConfigMap. +* See [Examples](virtualServer) + +### Routes Support +* If multiClusterServices annotation is not provided in the Route, then CIS will fetch the services from primary & secondary clusters and add the pool-members. +* If multiClusterServices annotation is provided in the Route, then CIS will fetch the services from the primary & secondary clusters as well as from the clusters mentioned in the annotation and add the pool-members. +* See Examples [here](routes) +* +### Route Annotation for Multi-ClusterServices +Services running in any other OpenShift clusters, as mentioned below: +``` +virtual-server.f5.com/multiClusterServices: +'[ + { + "clusterName": "cluster2", + "service": "svc-pytest-foo-1-com", + "namespace": "foo", + "servicePort": 80, + "weight": 30, + } +]' +``` + +| Parameter | Type | Required | Description | Default | Examples | +|-------------|------------|------------|---------------------------------------------------------|---------|----------| +| clusterName | String | Mandatory | Name of the cluster | - | cluster1 | +| service | String | Mandatory | Name of the service | - | svc-1 | +| namespace | String | Mandatory | Namespace where the service is created | - | test | +| servicePort | String/Int | Mandatory | port of the service (for named port use string value ) | - | 80 | +| weight | Int | Optional | weight to be used for traffic splitting | 0 | 20 | + +### Cluster wise Ratio for traffic distribution +CIS supports distribution of traffic across clusters as per the ratio configured for each cluster in the extended ConfigMap.
+It works even along with [A/B](#ab-or-alternate-backends) where different weights are defined for each service. In such a case the ratio of traffic +distribution is computed taking into consideration both the service weights and cluster ratio.
+However, the ratio of the clusters those haven't hosted any services linked to the concerned route are not taken into consideration +while computing the final ratio.
+ +**Note:** +* Cluster wise ratio for traffic distribution is supported in HA as well as non-HA CIS environment. +* Ratio is only supported for NextGen Routes and Virtual and Transport Server CR. +* Setting cluster adminState in conjunction with cluster ratio will affect the overall traffic distribution across clusters. + As the clusters marked as disable or offline will not receive traffic, so any ratio defined for these clusters will be rendered ineffective. + Thus, in such a scenario it's recommended to set the cluster ratio to 0 for all the clusters marked with disable/offline. + +### A/B or Alternate Backends +What it is? + +* A/B or Alternate Backends is a deployment strategy that allows you to release a new version of an application (version B) to a subset of users, while the majority still uses the old version (version A). +* It helps in comparing two versions of a service or application (referred to as A and B) to determine which one performs better based on specific metrics, such as response time, error rates, or user engagement. +* It allows you to gradually release changes to a subset of users and gather data to make informed decisions about whether to fully roll out the new version. +* This services defined in Alternate Backends exist in either of the HA peer clusters or in both of the HA clusters. Since HA clusters usually hold similar configurations, ideally these services exist in both the HA clusters. + +What it isn't? + +* Services defined as Alternate Backends don't have to be created only in the other HA peer cluster(by other HA peer cluster it means if CIS is running in Primary cluster then Secondary cluster is the other HA peer cluster and vice versa). +* Alternate Backends are not primarily used for failover scenarios, however BIGIP does forward the traffic to any of the available backend service if any service goes down or fails health check. + +### Cluster adminState to enable/disable/offline a cluster +adminState can be provided for a cluster to dictate the state of a particular cluster. +Supported values for adminState are [enable, disable, offline, no-pool]
+By default clusters are in enabled state.
+**adminState: enable**, all new connections are allowed to the pool members from the cluster.
+**adminState: disable**, all new connections except those which match an existing persistence session are not allowed for the pool members from the cluster.
+**adminState: offline**, no new connections are allowed to the pool members from the cluster, even if they match an existing persistence session.
+**adminState: no-pool**, in ratio mode, a service pool is not created for the affected cluster. For all other modes, pool members from the cluster are not added to the service pool. This configuration is helpful when we don't want to add pool or pool members from a particular cluster due to any reasons(for example cluster is under maintenance).
+ + +**Note**: +* For HA mode [namely default, active-active, ratio], CIS monitored resource manifests(such as routes, CRDs, extendedConfigmaps) must be available in both the clusters. +* The CIS monitored resource manifests must be identical on both primary and Secondary Clusters +* So, In case of CIS failover, CIS on Secondary Cluster will take control and will start processing the CIS monitored resource manifests. +* CIS on Secondary Cluster will not process the CIS monitored resource manifests if they are not available in Secondary Cluster. +* MakeSure to have identical resource manifests in both the clusters to avoid any issues during CIS failover. + + +## Known issues +* Pool members are not getting populated for extended service in ratio mode +* CIS doesn't update pool members if service doesn't exist in primary cluster but exists in secondary cluster for Route. +* CIS on start up in multiCluster mode, if any external cluster kube-api server is down/not reachable, CIS is struck and not processing any valid clusters config also.Workaround to remove unreachable cluster config from configmap and restart CIS +* CIS fails to post declaration with VS with health monitors in ratio mode.Issue is observed intermittently +* Route status is not updated in other HA cluster. For eg: Active Primary CIS cluster doesn't update the route status in Secondary HA cluster and vice-versa. + +## FAQ + +### Does extended configmap update require CIS restart? +No. It's recommended to restart CIS if any HA configuration or external cluster configurations are updated in extended Configmap. However CIS restart is not required when updating ratio in the extended Configmap. + +### Does mode update require CIS restart? +Yes. CIS has to be restarted when there is a change in the mode. + +### How do you add a new cluster? +To add a new cluster, create a kube-config file with read only permissions. Then create a Kubernetes secret using the kube-config file. Refer this in secret in the extended ConfigMap to add the new cluster. +CIS dynamically reads the new kube-config of the new cluster and starts listening to the services and endpoints in the new cluster when a route refers this new cluster. + +### Where do you manage the manifest or Configuration Objects like Routes, CRDs, ExtendedConfigmaps etc.? +Manifests or Configuration objects are managed centralized in Primary Cluster and if HA is desired the same manifests are expected to be in Secondary Cluster. + +### What are the supported CNIs? +Currently, NodePort mode is supported.For cluster mode, static routing mode is supported to enable configuration of static routes on bigip for pod network subnets for direct routing from BIGIP to k8s Pods + +### What kind of providers are supported? +CIS supports Hybrid Cloud, any public Cloud providers such as; AWS, Azure, GCP, On-Prem, VmWare, Tanzu etc. which is in same network/datacenter and can communicate with each other. + +### What kind of clusters are supported? +CIS multicluster solution is currently validated with openshift clusters and K8s clusters + +### How does CIS start as a secondary Cluster? +CIS recognizes as Secondary when it starts with a deployment parameter i.e. --multi-cluster-mode=secondary + +### How does Secondary CIS learn about the Primary Cluster endpoint state in HA mode? +Both of the CIS will communicate with both K8s API servers and prepares the AS3 declaration, but the secondary CIS only sends the declaration when the Primary cluster's health probe fails. As soon as primary cluster comes up, secondary CIS stops sending the declaration. + +### What kind of permission is required for HA or StandAlone deployment of CIS? +No RBAC change for CIS deployment with multiCluster support. Only additional kube-config configuration with read only permission is required to access the endpoints from external cluster. + +### What kind of permission is required to access external clusters (apart from HA and StandAlone)? +CIS requires read-only permission in Kubeconfig of external clusters to access resources like Pods/Services/Endpoints/Nodes. + +### Can CIS manage multiple BIG-IPs? +No. CIS can manage only Standalone BIG-IP or HA BIG-IP. In other words, CIS acts as a single point of BIG-IP Orchestrator and supports Multi-Cluster. + +### Is serviceTypeLBDiscovery supported for modes other than default mode? +No + +### Is traffic splitting with cluster ratio supported? +Yes. CIS supports traffic splitting as per the ratio specified for each cluster and also works with [A/B](#ab-or-alternate-backends) as well. + +### Is A/B supported in multiCluster mode? +Yes. CIS supports [A/B](#ab-or-alternate-backends) with active-active and ratio mode in multiCluster. + +### Is A/B custom persistence supported in all the modes? +No. [A/B](#ab-or-alternate-backends) persistence is supported in ratio mode and pool member type as cluster. + +### Does Secondary CIS require resource manifests existing in Primary Cluster? +Yes. CIS on Secondary Cluster will not process the CIS monitored resource manifests[NextGen Routes, CRDs, extendedConfigmap] if they are not available in Primary Cluster. +This is required in case of CIS failover, CIS on Secondary Cluster will take control and will start processing the CIS monitored resource manifests. +It is suggested to maintain identical CIS monitored resource manifests in both the clusters to avoid any issues during CIS failover. +This requirement is applicable in case of CIS HA mode [namely default, active-active, ratio]. + +### How to configure the primaryEndPoint in HA mode? +The primaryEndPoint is a mandatory parameter if CIS is intended to run in Multi-Cluster HA mode[namely default, active-active, ratio]. If this is not specified the secondary CIS will not run. +Secondary CIS will continuously monitor the health of the primary cluster based on the primaryEndPoint value. If it's down, then the secondary CIS takes the responsibility of posting declarations to BIG-IP. +Supported Protocols for the primaryEndPoint are HTTP and TCP. +Generally, it's suggested to use the primaryEndPoint as + a) any available endpoint to check the health of the primary cluster. + b) Primary CIS cluster's health check endpoint (/health) if accessible. + c) Primary CIS cluster's kube-api server endpoint if accessible. +Response code 200 OK is expected from the primaryEndPoint in case of HTTP Protocol. +Successful TCP connection is expected from the primaryEndPoint in case of TCP Protocol. +Secondary CIS become active if the primaryEndPoint is not accessible. +PrimaryEndPoint is optional to configure for Primary CIS. +Note: Primary CIS will not monitor the health of the Secondary CIS cluster. + +### How to configure the primaryEndPoint in Standalone mode? +The primaryEndPoint is not applicable in Standalone mode. It's only applicable in HA mode. + +### How to use CIS /health endpoint to check the health of CIS? +Fetch the CIS PodIP and use it in the curl command as shown below from any of the cluster nodes: +``` +curl http://:8080/health +``` +Response code 200 OK is expected from the CIS /health endpoint if kube-api server is accessible. +Example: +``` +[root@cluster-1-worker0 ~]# curl http://10.244.1.213:8080/health +Ok[root@cluster-1-worker0 ~]# curl http://10.244.1.213:8080/health -v +* About to connect() to 10.244.1.213 port 8080 (#0) +* Trying 10.244.1.213... +* Connected to 10.244.1.213 (10.244.1.213) port 8080 (#0) +> GET /health HTTP/1.1 +> User-Agent: curl/7.29.0 +> Host: 10.244.1.213:8080 +> Accept: */* +> +< HTTP/1.1 200 OK +< Date: Thu, 07 Dec 2023 08:28:57 GMT +< Content-Length: 2 +< Content-Type: text/plain; charset=utf-8 +< +* Connection #0 to host 10.244.1.213 left intact +Ok[root@cluster-1-worker0 ~]# +``` +where 10.244.1.213 is the CIS PodIP. + +### How to configure multiClusterServices in Route annotation? +multiClusterServices is a Route annotation. Below is the sample Route annotation with multiClusterServices: +``` +virtual-server.f5.com/multiClusterServices: +'[ + { + "clusterName": "cluster2", + "service": "svc-pytest-foo-1-com", + "namespace": "foo", + "servicePort": 80, + "weight": 30, + } +]' +``` +where clusterName is the name of the cluster where the service is running, namespace is the namespace where the service is running, servicePort is the port of the service and service is the name of the service. +where cluster2 is the external cluster apart from the HA cluster pair. +Note: External Clusters doesn't need to install CIS + +### Are policy CR supported in Active-Active and Ratio mode? +Yes, Policy CR is supported in all modes same as non-multi-cluster mode. diff --git a/docs/config_examples/multicluster/extendedConfigmap/extendedConfigmap-with-admin-state-no-pool/global-spec-config-for-multicluster-with-HA-CIS-and-cluster-admin-state-no-pool.yaml b/docs/config_examples/multicluster/non-default-mode/extendedConfigmap/extendedConfigmap-with-admin-state-no-pool/global-spec-config-for-multicluster-with-HA-CIS-and-cluster-admin-state-no-pool.yaml similarity index 100% rename from docs/config_examples/multicluster/extendedConfigmap/extendedConfigmap-with-admin-state-no-pool/global-spec-config-for-multicluster-with-HA-CIS-and-cluster-admin-state-no-pool.yaml rename to docs/config_examples/multicluster/non-default-mode/extendedConfigmap/extendedConfigmap-with-admin-state-no-pool/global-spec-config-for-multicluster-with-HA-CIS-and-cluster-admin-state-no-pool.yaml diff --git a/docs/config_examples/multicluster/extendedConfigmap/extendedConfigmap-with-admin-state-no-pool/global-spec-config-for-multicluster-with-standalone-CIS-and-cluster-admin-state-no-pool.yaml b/docs/config_examples/multicluster/non-default-mode/extendedConfigmap/extendedConfigmap-with-admin-state-no-pool/global-spec-config-for-multicluster-with-standalone-CIS-and-cluster-admin-state-no-pool.yaml similarity index 100% rename from docs/config_examples/multicluster/extendedConfigmap/extendedConfigmap-with-admin-state-no-pool/global-spec-config-for-multicluster-with-standalone-CIS-and-cluster-admin-state-no-pool.yaml rename to docs/config_examples/multicluster/non-default-mode/extendedConfigmap/extendedConfigmap-with-admin-state-no-pool/global-spec-config-for-multicluster-with-standalone-CIS-and-cluster-admin-state-no-pool.yaml diff --git a/docs/config_examples/multicluster/extendedConfigmap/global-config-with-primary-cluster-health-probe.yaml b/docs/config_examples/multicluster/non-default-mode/extendedConfigmap/global-config-with-primary-cluster-health-probe.yaml similarity index 90% rename from docs/config_examples/multicluster/extendedConfigmap/global-config-with-primary-cluster-health-probe.yaml rename to docs/config_examples/multicluster/non-default-mode/extendedConfigmap/global-config-with-primary-cluster-health-probe.yaml index 4758721a2..4265186c8 100644 --- a/docs/config_examples/multicluster/extendedConfigmap/global-config-with-primary-cluster-health-probe.yaml +++ b/docs/config_examples/multicluster/non-default-mode/extendedConfigmap/global-config-with-primary-cluster-health-probe.yaml @@ -1,7 +1,7 @@ # primaryEndPoint -> allowed values are http/tcp endpoint # http endpoint format : http://10.145.72.114:8001 # tcp endpoint format : tcp://10.145.72.114:8000 -# Note: when configuring primaryEndPoint, endPoint should contain protocol i.e http://ip:port , tcp://ip:port +# Note: when configuring the primaryEndPoint, endPoint should contain protocol i.e http://ip:port , tcp://ip:port # probeInterval -> time interval between health check # retryInterval -> time interval between recheck when primary cluster is down. apiVersion: v1 diff --git a/docs/config_examples/multicluster/extendedConfigmap/global-spec-config-for-multicluster-with-active-active.yaml b/docs/config_examples/multicluster/non-default-mode/extendedConfigmap/global-spec-config-for-multicluster-with-active-active.yaml similarity index 100% rename from docs/config_examples/multicluster/extendedConfigmap/global-spec-config-for-multicluster-with-active-active.yaml rename to docs/config_examples/multicluster/non-default-mode/extendedConfigmap/global-spec-config-for-multicluster-with-active-active.yaml diff --git a/docs/config_examples/multicluster/extendedConfigmap/global-spec-config-for-multicluster-with-cluster-admin-state.yaml b/docs/config_examples/multicluster/non-default-mode/extendedConfigmap/global-spec-config-for-multicluster-with-cluster-admin-state.yaml similarity index 100% rename from docs/config_examples/multicluster/extendedConfigmap/global-spec-config-for-multicluster-with-cluster-admin-state.yaml rename to docs/config_examples/multicluster/non-default-mode/extendedConfigmap/global-spec-config-for-multicluster-with-cluster-admin-state.yaml diff --git a/docs/config_examples/multicluster/extendedConfigmap/global-spec-config-for-multicluster.yaml b/docs/config_examples/multicluster/non-default-mode/extendedConfigmap/global-spec-config-for-multicluster.yaml similarity index 100% rename from docs/config_examples/multicluster/extendedConfigmap/global-spec-config-for-multicluster.yaml rename to docs/config_examples/multicluster/non-default-mode/extendedConfigmap/global-spec-config-for-multicluster.yaml diff --git a/docs/config_examples/multicluster/non-default-mode/routes/route-with-multicluster-service-annotation-and-ab.yaml b/docs/config_examples/multicluster/non-default-mode/routes/route-with-multicluster-service-annotation-and-ab.yaml new file mode 100644 index 000000000..ad719b4fd --- /dev/null +++ b/docs/config_examples/multicluster/non-default-mode/routes/route-with-multicluster-service-annotation-and-ab.yaml @@ -0,0 +1,35 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + annotations: + virtual-server.f5.com/multiClusterServices: '[ + {"clusterName": "cluster2", "service": "svc1", "namespace": "default", "servicePort": "8080", "weight": 20}, + {"clusterName": "cluster3", "service": "svc2", "namespace": "default", "servicePort": "8080", "weight": 10} + ]' + # you can define either service port or target port in the port value + labels: + name: svc1 + f5type: systest + name: svc1-route-edge +spec: + host: foo.com + path: "/test" + port: + targetPort: 443 + tls: + certificate: | + -----BEGIN CERTIFICATE----- + -----END CERTIFICATE----- + key: | + -----BEGIN PRIVATE KEY----- + -----END PRIVATE KEY----- + termination: edge + insecureEdgeTerminationPolicy: Allow + to: + kind: Service + name: foo-svc1 + weight: 40 + alternateBackends: + - kind: Service + name: foo-svc2 + weight: 30 diff --git a/docs/config_examples/multicluster/non-default-mode/routes/route-with-multicluster-service-annotation.yaml b/docs/config_examples/multicluster/non-default-mode/routes/route-with-multicluster-service-annotation.yaml new file mode 100644 index 000000000..39ad000ee --- /dev/null +++ b/docs/config_examples/multicluster/non-default-mode/routes/route-with-multicluster-service-annotation.yaml @@ -0,0 +1,28 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + annotations: + virtual-server.f5.com/multiClusterServices: '[{"clusterName": "cluster3", "service": + "svc-pytest-foo-1-com-default", "namespace": "default", "servicePort": "80" }]' + # you can define either service port or target port in the port value + labels: + name: svc1 + f5type: systest + name: svc1-route-edge +spec: + host: svc1-edge-route.local + path: "/test" + port: + targetPort: 443 + tls: + certificate: | + -----BEGIN CERTIFICATE----- + -----END CERTIFICATE----- + key: | + -----BEGIN PRIVATE KEY----- + -----END PRIVATE KEY----- + termination: edge + insecureEdgeTerminationPolicy: Allow + to: + kind: Service + name: svc1 diff --git a/docs/config_examples/multicluster/customResource/virtualServer/vs-with-pool-service-and-ab.yaml b/docs/config_examples/multicluster/non-default-mode/virtualServer/vs.yaml similarity index 53% rename from docs/config_examples/multicluster/customResource/virtualServer/vs-with-pool-service-and-ab.yaml rename to docs/config_examples/multicluster/non-default-mode/virtualServer/vs.yaml index 9cdc8c8c6..6868e6f64 100644 --- a/docs/config_examples/multicluster/customResource/virtualServer/vs-with-pool-service-and-ab.yaml +++ b/docs/config_examples/multicluster/non-default-mode/virtualServer/vs.yaml @@ -7,14 +7,8 @@ metadata: namespace: default spec: host: tea.example.com - httpTraffic: redirect + virtualServerAddress: 10.8.0.71 pools: - path: /neam service: svc-edge-a - servicePort: 80 - weight: 50 - alternateBackends: - - service: svc-edge-b - weight: 30 - - service: svc-edge-c - weight: 20 \ No newline at end of file + servicePort: 80 \ No newline at end of file diff --git a/docs/multicluster_userguides/README.md b/docs/multicluster_userguides/README.md index 542ab025a..52011977f 100644 --- a/docs/multicluster_userguides/README.md +++ b/docs/multicluster_userguides/README.md @@ -6,7 +6,6 @@ For guides on using CIS with multiple clusters, see the following: * [Standalone CIS to manage Multicluster services](https://github.com/f5devcentral/f5-cis-docs/blob/main/multicluster_user_guides/Standalone/README.md) * [CIS HA use-cases for MultiCluster Environment](https://github.com/f5devcentral/f5-cis-docs/tree/main/multicluster_user_guides/CIS%20HA/README.md) * [Multicluster HA CIS with active-active mode](https://github.com/f5devcentral/f5-cis-docs/blob/main/multicluster_user_guides/CIS%20HA/Active-Active/README.md) -* [Multicluster HA CIS with active-standby mode](https://github.com/f5devcentral/f5-cis-docs/blob/main/multicluster_user_guides/CIS%20HA/Active-Standby/README.md) * [Multicluster HA CIS with ratio mode](https://github.com/f5devcentral/f5-cis-docs/blob/main/multicluster_user_guides/CIS%20HA/ratio/README.md) diff --git a/docs/upgradeProcess.md b/docs/upgradeProcess.md index d36ef31a7..4879bdb35 100644 --- a/docs/upgradeProcess.md +++ b/docs/upgradeProcess.md @@ -408,4 +408,12 @@ Refer Release Notes for [CIS v2.17.0](https://github.com/F5Networks/k8s-bigip-ct ### **Upgrading from 2.18.0 to 2.18.1:** -Refer Release Notes for [CIS v2.18.1](https://github.com/F5Networks/k8s-bigip-ctlr/blob/2.x-master/docs/RELEASE-NOTES.rst) \ No newline at end of file +Refer Release Notes for [CIS v2.18.1](https://github.com/F5Networks/k8s-bigip-ctlr/blob/2.x-master/docs/RELEASE-NOTES.rst) + +### **Upgrading from 2.18.1 to 2.19.0:** +**_Functionality Changes:_** + * --local-cluster-name parameter is a new and mandatory parameter for multi-cluster mode, It’s required for all the modes(default, active-active and ratio). + * Default mode is the default mode for multi-cluster, if no mode is specified in the extended configMap. + * CIS now does the service discovery for VS/TS CR in all the clusters defined via extended configMap when active-active or ratio mode is configured. + * Active-standby mode is deprecated and not supported anymore. + * extendedServiceReferences have been deprecated for VS/TS CR in the active-active and ratio mode. \ No newline at end of file