1
+ # Add on section, you can also manage addons through Terraform
2
+ # To find out add-ons available and configurations, run: oci ce addon-option list --kubernetes-version <OKE_VERSION > addons.json
3
+ # See also https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringclusteraddons-configurationarguments.htm
4
+
5
+ locals {
6
+
7
+ # SET THIS TO TRUE IF YOU WANT TO OVERRIDE THE COREDNS PLUGIN AND MANAGE IT THROUGH TERRAFORM
8
+ # REQUIRES AT LEAST 1 NODE IN THE CLUSTER. THAT NODE MUST BE FROM THE SYSTEM NODE POOL IF CLUSTER AUTOSCALER IS ENABLED!
9
+ override_coredns = false
10
+
11
+ coredns_addon_configs_base = {
12
+ # Distribute replicas on nodes belonging to different ADs, if possible
13
+ topologySpreadConstraints = jsonencode (
14
+ yamldecode (
15
+ <<- YAML
16
+ - maxSkew: "1"
17
+ topologyKey: topology.kubernetes.io/zone
18
+ whenUnsatisfiable: ScheduleAnyway
19
+ labelSelector:
20
+ matchLabels:
21
+ k8s-app: kube-dns
22
+ YAML
23
+ )
24
+ )
25
+ # Try to spread CoreDNS pods across different nodes
26
+ affinity = jsonencode (
27
+ yamldecode (
28
+ <<- YAML
29
+ podAntiAffinity:
30
+ preferredDuringSchedulingIgnoredDuringExecution:
31
+ - podAffinityTerm:
32
+ labelSelector:
33
+ matchLabels:
34
+ k8s-app: "kube-dns"
35
+ topologyKey: "kubernetes.io/hostname"
36
+ weight: 100
37
+ YAML
38
+ )
39
+ )
40
+ # Rolling update configurations for CoreDNS
41
+ rollingUpdate = " {\" maxSurge\" : \" 50%\" , \" maxUnavailable\" :\" 25%\" }"
42
+ # For large clusters, it's better to increase this value. The default behaviour is to create a new CoreDNS for every new node. Also, resources for single CoreDNS pods should be increased
43
+ nodesPerReplica = " 1"
44
+ # In case you need to customize the coredns ConfigMap in kube-system
45
+ customizeCoreDNSConfigMap = " false"
46
+ }
47
+
48
+ # COREDNS MUST be scheduled to the system node pool in case cluster autoscaler is enabled
49
+ coredns_addon_configs = merge (local. coredns_addon_configs_base , local. enable_cluster_autoscaler ? {
50
+ nodeSelectors = " {\" role\" : \" system\" }"
51
+ } : null )
52
+
53
+ metrics_server_addon_configs_base = {
54
+ # At least 3 replicas for high availability
55
+ numOfReplicas = " 3"
56
+ # Spread the replicas across ADs if possible
57
+ topologySpreadConstraints = jsonencode (
58
+ yamldecode (
59
+ <<- YAML
60
+ - maxSkew: "1"
61
+ topologyKey: topology.kubernetes.io/zone
62
+ whenUnsatisfiable: ScheduleAnyway
63
+ labelSelector:
64
+ matchLabels:
65
+ k8s-app: metrics-server
66
+ YAML
67
+ )
68
+ )
69
+ }
70
+
71
+ # METRICS-SERVER MUST be scheduled to the system node pool in case cluster autoscaler is enabled
72
+ metrics_server_addon_configs = merge (local. metrics_server_addon_configs_base , local. enable_cluster_autoscaler ? {
73
+ nodeSelectors = " {\" role\" : \" system\" }"
74
+ } : null )
75
+
76
+ cluster_autoscaler_addon_configs = {
77
+ authType = " workload"
78
+ # Enable balancing of similar node groups
79
+ balanceSimilarNodeGroups = " true"
80
+ # We should never group by fault domain when balancing for similarity, only by AD
81
+ balancingIgnoreLabel = " oci.oraclecloud.com/fault-domain"
82
+ # Supported from OKE v1.30.10, autoscale based on freeform or defined tags in the node pools
83
+ # DEFINE HERE YOUR AUTOSCALER POLICY, DEFAULT IS MIN: 0, MAX: 5
84
+ nodeGroupAutoDiscovery = " compartmentId:${ var . oke_compartment_id } ,nodepoolTags:cluster_autoscaler=enabled,min:0,max:5"
85
+ # Make sure to schedule the cluster autoscaler in a node that it is NOT autoscaled, in the system node pool
86
+ nodeSelectors = " {\" role\" : \" system\" }"
87
+ }
88
+ }
89
+
90
+
91
+ resource "oci_containerengine_addon" "oke_cert_manager" {
92
+ addon_name = " CertManager"
93
+ cluster_id = module. oke . cluster_id
94
+ remove_addon_resources_on_delete = true
95
+ depends_on = [module . oke ]
96
+ count = local. enable_cert_manager ? 1 : 0
97
+ }
98
+
99
+ resource "oci_containerengine_addon" "oke_metrics_server" {
100
+ addon_name = " KubernetesMetricsServer"
101
+ cluster_id = module. oke . cluster_id
102
+ remove_addon_resources_on_delete = true
103
+ dynamic "configurations" {
104
+ for_each = local. metrics_server_addon_configs
105
+ content {
106
+ key = configurations. key
107
+ value = configurations. value
108
+ }
109
+ }
110
+ depends_on = [module . oke , oci_containerengine_addon . oke_cert_manager ]
111
+ count = local. enable_metrics_server ? 1 : 0
112
+ }
113
+
114
+ resource "oci_containerengine_addon" "oke_coredns" {
115
+ addon_name = " CoreDNS"
116
+ cluster_id = module. oke . cluster_id
117
+ remove_addon_resources_on_delete = false
118
+ override_existing = true
119
+ dynamic "configurations" {
120
+ for_each = local. coredns_addon_configs
121
+ content {
122
+ key = configurations. key
123
+ value = configurations. value
124
+ }
125
+ }
126
+ depends_on = [module . oke ]
127
+ count = var. cluster_type == " enhanced" && local. override_coredns ? 1 : 0
128
+ }
129
+
130
+ resource "oci_containerengine_addon" "oke_cluster_autoscaler" {
131
+ addon_name = " ClusterAutoscaler"
132
+ cluster_id = module. oke . cluster_id
133
+ remove_addon_resources_on_delete = true
134
+ dynamic "configurations" {
135
+ for_each = local. cluster_autoscaler_addon_configs
136
+ content {
137
+ key = configurations. key
138
+ value = configurations. value
139
+ }
140
+ }
141
+ depends_on = [module . oke ]
142
+ count = local. enable_cluster_autoscaler ? 1 : 0
143
+ }
0 commit comments