Merge pull request #1810 from oracle-devrel/oke-rm

oke-rm-1.1.1 - Add-ons, cluster-autoscaler support, CoreDNS

Author: martatolosa

app-dev/devops-and-containers/oke/oke-rm/README.md

@@ -16,13 +16,13 @@ This stack is used to create the initial network infrastructure for OKE. When co
 * By default, everything is private, but there is the possibility to create public subnets
 * Be careful when modifying the default values, as inputs are not validated
 
-[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/technology-engineering/releases/download/oke-rm-1.1.0/infra.zip)
+[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/technology-engineering/releases/download/oke-rm-1.1.1/infra.zip)
 
 ## Step 2: Create the OKE control plane
 
 This stack is used to create the OKE control plane ONLY.
 
-[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/technology-engineering/releases/download/oke-rm-1.1.0/oke.zip)
+[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/technology-engineering/releases/download/oke-rm-1.1.1/oke.zip)
 
 Also note that if the network infrastructure is located in a different compartment than the OKE cluster AND you are planning to use the OCI_VCN_NATIVE CNI,
 you must add these policies:

app-dev/devops-and-containers/oke/oke-rm/infra/local.tf

@@ -1,3 +1,5 @@
 locals {
   create_bastion = var.create_bastion_subnet && var.create_bastion
+  # VCN_NATIVE_CNI internally it is mapped as npn
+  cni = var.cni_type == "vcn_native" ? "npn" : var.cni_type
 }

app-dev/devops-and-containers/oke/oke-rm/infra/main.tf

@@ -3,7 +3,7 @@ module "network" {
   source = "./modules/network"
   network_compartment_id = var.network_compartment_id
   region = var.region
-  cni_type = var.cni_type
+  cni_type = local.cni
   # VCN
   create_vcn = var.create_vcn
   vcn_id = var.vcn_id

app-dev/devops-and-containers/oke/oke-rm/infra/schema.yaml

@@ -1,6 +1,6 @@
-title: OCI Native - Base Infra
-description: Basic infrastructure and network rules to start with a Cloud Native project
-informationalText: Basic infrastructure
+title: OKE RM - Base Network Infrastructure
+description: Basic network infrastructure to start with OKE
+informationalText: Basic network infrastructure
 schemaVersion: 1.1.0
 version: "20190304"
 locale: "en"
@@ -88,7 +88,7 @@ variables:
     description: "CNI to use for the OKE cluster"
     type: enum
     enum:
-      - npn
+      - vcn_native
       - flannel
     required: true
 
@@ -210,7 +210,7 @@ variables:
     visible:
       eq:
         - ${cni_type}
-        - npn
+        - vcn_native
 
   pod_subnet_cidr:
     title: "Pod subnet CIDR"
@@ -222,7 +222,7 @@ variables:
         - ${create_pod_subnet}
         - eq:
             - ${cni_type}
-            - npn
+            - vcn_native
 
   pod_subnet_dns_label:
     title: "Pod subnet DNS label"
@@ -234,7 +234,7 @@ variables:
         - ${create_pod_subnet}
         - eq:
             - ${cni_type}
-            - npn
+            - vcn_native
 
   pod_subnet_name:
     title: "Pod subnet name"
@@ -246,7 +246,7 @@ variables:
         - ${create_pod_subnet}
         - eq:
             - ${cni_type}
-            - npn
+            - vcn_native
 
 # SERVICE SUBNET
 

app-dev/devops-and-containers/oke/oke-rm/infra/variable.tf

@@ -18,7 +18,7 @@ variable "vcn_id" {
 }
 
 variable "vcn_name" {
-  default = "vcn-spoke-1"
+  default = "vcn-oke-1"
 }
 
 variable "vcn_cidr_blocks" {
@@ -27,7 +27,7 @@ variable "vcn_cidr_blocks" {
 }
 
 variable "vcn_dns_label" {
-  default = "spoke1"
+  default = "oke1"
 }
 
 # CP SUBNET

app-dev/devops-and-containers/oke/oke-rm/oke/addons.tf

@@ -0,0 +1,143 @@
+# Add on section, you can also manage addons through Terraform
+# To find out add-ons available and configurations, run: oci ce addon-option list --kubernetes-version <OKE_VERSION > addons.json
+# See also https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringclusteraddons-configurationarguments.htm
+
+locals {
+
+  # SET THIS TO TRUE IF YOU WANT TO OVERRIDE THE COREDNS PLUGIN AND MANAGE IT THROUGH TERRAFORM
+  # REQUIRES AT LEAST 1 NODE IN THE CLUSTER. THAT NODE MUST BE FROM THE SYSTEM NODE POOL IF CLUSTER AUTOSCALER IS ENABLED!
+  override_coredns = false
+
+  coredns_addon_configs_base = {
+    # Distribute replicas on nodes belonging to different ADs, if possible
+    topologySpreadConstraints = jsonencode(
+      yamldecode(
+        <<-YAML
+          - maxSkew: "1"
+            topologyKey: topology.kubernetes.io/zone
+            whenUnsatisfiable: ScheduleAnyway
+            labelSelector:
+              matchLabels:
+                k8s-app: kube-dns
+          YAML
+      )
+    )
+    # Try to spread CoreDNS pods across different nodes
+    affinity = jsonencode(
+      yamldecode(
+        <<-YAML
+          podAntiAffinity:
+            preferredDuringSchedulingIgnoredDuringExecution:
+              - podAffinityTerm:
+                  labelSelector:
+                    matchLabels:
+                      k8s-app: "kube-dns"
+                  topologyKey: "kubernetes.io/hostname"
+                weight: 100
+          YAML
+      )
+    )
+    # Rolling update configurations for CoreDNS
+    rollingUpdate = "{\"maxSurge\": \"50%\", \"maxUnavailable\":\"25%\"}"
+    # For large clusters, it's better to increase this value. The default behaviour is to create a new CoreDNS for every new node. Also, resources for single CoreDNS pods should be increased
+    nodesPerReplica = "1"
+    # In case you need to customize the coredns ConfigMap in kube-system
+    customizeCoreDNSConfigMap = "false"
+  }
+
+  # COREDNS MUST be scheduled to the system node pool in case cluster autoscaler is enabled
+  coredns_addon_configs = merge(local.coredns_addon_configs_base, local.enable_cluster_autoscaler ? {
+    nodeSelectors = "{\"role\": \"system\"}"
+  } : null)
+
+  metrics_server_addon_configs_base = {
+    # At least 3 replicas for high availability
+    numOfReplicas = "3"
+    # Spread the replicas across ADs if possible
+    topologySpreadConstraints = jsonencode(
+      yamldecode(
+        <<-YAML
+          - maxSkew: "1"
+            topologyKey: topology.kubernetes.io/zone
+            whenUnsatisfiable: ScheduleAnyway
+            labelSelector:
+              matchLabels:
+                k8s-app: metrics-server
+          YAML
+      )
+    )
+  }
+
+  # METRICS-SERVER MUST be scheduled to the system node pool in case cluster autoscaler is enabled
+  metrics_server_addon_configs = merge(local.metrics_server_addon_configs_base, local.enable_cluster_autoscaler ? {
+    nodeSelectors = "{\"role\": \"system\"}"
+  } : null)
+
+  cluster_autoscaler_addon_configs = {
+    authType = "workload"
+    # Enable balancing of similar node groups
+    balanceSimilarNodeGroups = "true"
+    # We should never group by fault domain when balancing for similarity, only by AD
+    balancingIgnoreLabel = "oci.oraclecloud.com/fault-domain"
+    # Supported from OKE v1.30.10, autoscale based on freeform or defined tags in the node pools
+    # DEFINE HERE YOUR AUTOSCALER POLICY, DEFAULT IS MIN: 0, MAX: 5
+    nodeGroupAutoDiscovery = "compartmentId:${var.oke_compartment_id},nodepoolTags:cluster_autoscaler=enabled,min:0,max:5"
+    # Make sure to schedule the cluster autoscaler in a node that it is NOT autoscaled, in the system node pool
+    nodeSelectors = "{\"role\": \"system\"}"
+  }
+}
+
+
+resource "oci_containerengine_addon" "oke_cert_manager" {
+  addon_name                       = "CertManager"
+  cluster_id                       = module.oke.cluster_id
+  remove_addon_resources_on_delete = true
+  depends_on = [module.oke]
+  count = local.enable_cert_manager ? 1 : 0
+}
+
+resource "oci_containerengine_addon" "oke_metrics_server" {
+  addon_name                       = "KubernetesMetricsServer"
+  cluster_id                       = module.oke.cluster_id
+  remove_addon_resources_on_delete = true
+  dynamic "configurations" {
+    for_each = local.metrics_server_addon_configs
+    content {
+      key = configurations.key
+      value = configurations.value
+    }
+  }
+  depends_on = [module.oke, oci_containerengine_addon.oke_cert_manager]
+  count = local.enable_metrics_server ? 1 : 0
+}
+
+resource "oci_containerengine_addon" "oke_coredns" {
+  addon_name                       = "CoreDNS"
+  cluster_id                       = module.oke.cluster_id
+  remove_addon_resources_on_delete = false
+  override_existing = true
+  dynamic "configurations" {
+    for_each = local.coredns_addon_configs
+    content {
+      key = configurations.key
+      value = configurations.value
+    }
+  }
+  depends_on = [module.oke]
+  count = var.cluster_type == "enhanced" && local.override_coredns ? 1 : 0
+}
+
+resource "oci_containerengine_addon" "oke_cluster_autoscaler" {
+  addon_name = "ClusterAutoscaler"
+  cluster_id = module.oke.cluster_id
+  remove_addon_resources_on_delete = true
+  dynamic "configurations" {
+    for_each = local.cluster_autoscaler_addon_configs
+    content {
+      key = configurations.key
+      value = configurations.value
+    }
+  }
+  depends_on = [module.oke]
+  count = local.enable_cluster_autoscaler ? 1 : 0
+}

app-dev/devops-and-containers/oke/oke-rm/oke/locals.tf

@@ -0,0 +1,10 @@
+locals {
+  is_cp_subnet_private = data.oci_core_subnet.cp_subnet_data.prohibit_public_ip_on_vnic
+  is_lb_subnet_private = data.oci_core_subnet.lb_subnet_data.prohibit_public_ip_on_vnic
+  cni = var.cni_type == "vcn_native" ? "npn" : var.cni_type
+  is_flannel = var.cni_type == "flannel"
+  enable_cert_manager = var.cluster_type == "enhanced" && var.enable_cert_manager
+  enable_metrics_server = var.cluster_type == "enhanced" && var.enable_cert_manager && var.enable_metrics_server
+  enable_cluster_autoscaler = var.cluster_type == "enhanced" && var.enable_cluster_autoscaler
+  create_autoscaler_policies = var.cluster_type == "enhanced"&& var.enable_cluster_autoscaler && var.create_autoscaler_policies
+}