diff --git a/cmd/operator/app/thread_chi.go b/cmd/operator/app/thread_chi.go index 5e863e124..2cd3286e5 100644 --- a/cmd/operator/app/thread_chi.go +++ b/cmd/operator/app/thread_chi.go @@ -61,7 +61,7 @@ func initClickHouse(ctx context.Context) { // Create operator instance chop.New(kubeClient, chopClient, chopConfigFile) log.V(1).F().Info("Config parsed:") - log.Info(chop.Config().String(true)) + log.Info("\n" + chop.Config().String(true)) // Create Informers kubeInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions( diff --git a/config/templates.d/001-templates.json.example b/config/templates.d/001-templates.json.example index abb1d747b..34468ecd2 100644 --- a/config/templates.d/001-templates.json.example +++ b/config/templates.d/001-templates.json.example @@ -29,7 +29,7 @@ "containers" : [ { "name": "clickhouse", - "image": "clickhouse/clickhouse-server:22.3", + "image": "clickhouse/clickhouse-server:23.8", "ports": [ { "name": "http", diff --git a/deploy/builder/operatorhub.sh b/deploy/builder/operatorhub.sh index a84c08671..4dfb3493b 100755 --- a/deploy/builder/operatorhub.sh +++ b/deploy/builder/operatorhub.sh @@ -8,7 +8,13 @@ SRC_ROOT="$(realpath "${CUR_DIR}/../..")" # 0.9.3 VERSION=$(cd "${SRC_ROOT}"; cat release) -PREVIOUS_VERSION="${PREVIOUS_VERSION:-0.18.0}" +PREVIOUS_VERSION="${PREVIOUS_VERSION}" + +if [[ -z "${PREVIOUS_VERSION}" ]]; then + echo "PREVIOUS_VERSION is not specified. Unable to proceed." + echo "Abort." + exit 1 +fi echo "PREVIOUS_VERSION: ${PREVIOUS_VERSION}" echo "VERSION: ${VERSION}" @@ -81,7 +87,7 @@ rm "${CVV_FILE_TEMPLATE}" CHI="clickhouseinstallations.clickhouse.altinity.com" CHIT="clickhouseinstallationtemplates.clickhouse.altinity.com" CONF="clickhouseoperatorconfigurations.clickhouse.altinity.com" - +CHK="clickhousekeeperinstallations.clickhouse-keeper.altinity.com" # Build partial .yaml manifest(s) MANIFEST_PRINT_CRD="yes" \ @@ -107,6 +113,14 @@ MANIFEST_PRINT_DEPLOYMENT="no" \ MANIFEST_PRINT_SERVICE_METRICS="no" \ "${CUR_DIR}/cat-clickhouse-operator-install-yaml.sh" | yq "select(.metadata.name == \"${CONF}\")" > "${MANIFESTS_DIR}/${CONF}.crd.yaml" +# Build partial .yaml manifest(s) +MANIFEST_PRINT_CRD="yes" \ +MANIFEST_PRINT_RBAC_CLUSTERED="no" \ +MANIFEST_PRINT_RBAC_NAMESPACED="no" \ +MANIFEST_PRINT_DEPLOYMENT="no" \ +MANIFEST_PRINT_SERVICE_METRICS="no" \ +"${CUR_DIR}/cat-clickhouse-operator-install-yaml.sh" | yq "select(.metadata.name == \"${CHK}\")" > "${MANIFESTS_DIR}/${CHK}.crd.yaml" + # TODO # Package file not used any more? #cat < "${OPERATORHUB_DIR}/clickhouse.package.yaml" diff --git a/deploy/builder/templates-config/templates.d/001-templates.json.example b/deploy/builder/templates-config/templates.d/001-templates.json.example index abb1d747b..34468ecd2 100644 --- a/deploy/builder/templates-config/templates.d/001-templates.json.example +++ b/deploy/builder/templates-config/templates.d/001-templates.json.example @@ -29,7 +29,7 @@ "containers" : [ { "name": "clickhouse", - "image": "clickhouse/clickhouse-server:22.3", + "image": "clickhouse/clickhouse-server:23.8", "ports": [ { "name": "http", diff --git a/deploy/builder/templates-operatorhub/clickhouse-operator.vVERSION.clusterserviceversion-template.yaml b/deploy/builder/templates-operatorhub/clickhouse-operator.vVERSION.clusterserviceversion-template.yaml index 40a8fc9b0..a3d350cf6 100644 --- a/deploy/builder/templates-operatorhub/clickhouse-operator.vVERSION.clusterserviceversion-template.yaml +++ b/deploy/builder/templates-operatorhub/clickhouse-operator.vVERSION.clusterserviceversion-template.yaml @@ -118,6 +118,34 @@ spec: - kind: PersistentVolumeClaim name: '' version: v1 + - description: ClickHouse Keeper Installation - ClickHouse Keeper cluster instance + displayName: ClickHouseKeeperInstallation + group: clickhouse-keeper.altinity.com + kind: ClickHouseKeeperInstallation + name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 description: |- ## ClickHouse [ClickHouse](https://clickhouse.yandex) is an open source column-oriented database management system capable of real time generation of analytical data reports. diff --git a/deploy/devspace/dev-config.sh b/deploy/devspace/dev-config.sh index eb94ab1e6..5a0f32981 100755 --- a/deploy/devspace/dev-config.sh +++ b/deploy/devspace/dev-config.sh @@ -1,17 +1,17 @@ #!/bin/bash -[ -n "${CONFIG_LOADED}" ] && return; CONFIG_LOADED=1; +[[ -n "${CONFIG_LOADED}" ]] && return; CONFIG_LOADED=1; CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" PROJECT_ROOT="$(realpath "${CUR_DIR}/../..")" MANIFEST_ROOT="$(realpath ${PROJECT_ROOT}/deploy)" -OPERATOR_NAMESPACE="${OPERATOR_NAMESPACE:-dev}" +OPERATOR_NAMESPACE="${OPERATOR_NAMESPACE:-"dev"}" METRICS_EXPORTER_NAMESPACE="${OPERATOR_NAMESPACE}" # yes, no, none, dev, release, prod, latest DEPLOY_OPERATOR="${DEPLOY_OPERATOR:-"no"}" -CUR_RELEASE=$(cat ${PROJECT_ROOT}/release) +CUR_RELEASE=$(cat "${PROJECT_ROOT}/release") MINIKUBE="${MINIKUBE:-"yes"}" echo "DEPLOY_OPERATOR=${DEPLOY_OPERATOR}" diff --git a/deploy/devspace/dev-install.sh b/deploy/devspace/dev-install.sh index af380f8ac..ec58fcb6f 100755 --- a/deploy/devspace/dev-install.sh +++ b/deploy/devspace/dev-install.sh @@ -4,10 +4,10 @@ CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" source "${CUR_DIR}/dev-config.sh" -echo "Create ${OPERATOR_NAMESPACE} namespace" +echo "Create namespace to deploy the operator into: ${OPERATOR_NAMESPACE}" kubectl create namespace "${OPERATOR_NAMESPACE}" -echo "Install operator requirements" +echo "Install operator requirements with the following options:" echo "OPERATOR_NAMESPACE=${OPERATOR_NAMESPACE}" echo "OPERATOR_VERSION=${OPERATOR_VERSION}" echo "OPERATOR_IMAGE=${OPERATOR_IMAGE}" diff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yaml index 4ed942999..c2ef5e22c 100644 --- a/deploy/helm/Chart.yaml +++ b/deploy/helm/Chart.yaml @@ -12,8 +12,8 @@ description: |- kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml ``` type: application -version: 0.23.0 -appVersion: 0.23.0 +version: 0.23.1 +appVersion: 0.23.1 home: https://github.com/Altinity/clickhouse-operator icon: https://logosandtypes.com/wp-content/uploads/2020/12/altinity.svg maintainers: diff --git a/deploy/helm/README.md b/deploy/helm/README.md index 24d27c0ac..f8ed72a8c 100644 --- a/deploy/helm/README.md +++ b/deploy/helm/README.md @@ -1,6 +1,6 @@ # altinity-clickhouse-operator -![Version: 0.23.0](https://img.shields.io/badge/Version-0.23.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.23.0](https://img.shields.io/badge/AppVersion-0.23.0-informational?style=flat-square) +![Version: 0.23.1](https://img.shields.io/badge/Version-0.23.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.23.1](https://img.shields.io/badge/AppVersion-0.23.1-informational?style=flat-square) Helm chart to deploy [altinity-clickhouse-operator](https://github.com/Altinity/clickhouse-operator). @@ -10,7 +10,6 @@ For upgrade please install CRDs separately: ```bash kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml - kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml ``` diff --git a/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml b/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml index 7ed41338b..f9f6b42f6 100644 --- a/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml +++ b/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced diff --git a/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml b/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml index 44af53f6b..7175710cc 100644 --- a/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml +++ b/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced diff --git a/deploy/helm/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml b/deploy/helm/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml index 3b9d3afe6..3831166b9 100644 --- a/deploy/helm/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml +++ b/deploy/helm/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml @@ -1,13 +1,13 @@ # Template Parameters: # -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.0 + clickhouse-keeper.altinity.com/chop: 0.23.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced diff --git a/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml b/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml index c0a8a65eb..26ccdc89a 100644 --- a/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml +++ b/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml @@ -7,7 +7,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced diff --git a/deploy/helm/templates/generated/Deployment-clickhouse-operator.yaml b/deploy/helm/templates/generated/Deployment-clickhouse-operator.yaml index eaa68e82b..1dcf5bf02 100644 --- a/deploy/helm/templates/generated/Deployment-clickhouse-operator.yaml +++ b/deploy/helm/templates/generated/Deployment-clickhouse-operator.yaml @@ -2,9 +2,9 @@ # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.0 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.1 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.0 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.1 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator diff --git a/deploy/helm/templates/generated/Secret-clickhouse-operator.yaml b/deploy/helm/templates/generated/Secret-clickhouse-operator.yaml index 60492f845..47a5ddfe9 100644 --- a/deploy/helm/templates/generated/Secret-clickhouse-operator.yaml +++ b/deploy/helm/templates/generated/Secret-clickhouse-operator.yaml @@ -3,7 +3,7 @@ # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index f2a8a1fc0..59e44140e 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -524,7 +524,7 @@ configs: "containers" : [ { "name": "clickhouse", - "image": "clickhouse/clickhouse-server:22.3", + "image": "clickhouse/clickhouse-server:23.8", "ports": [ { "name": "http", diff --git a/deploy/operator/clickhouse-operator-install-ansible.yaml b/deploy/operator/clickhouse-operator-install-ansible.yaml index 59e29c5ef..5b3649aff 100644 --- a/deploy/operator/clickhouse-operator-install-ansible.yaml +++ b/deploy/operator/clickhouse-operator-install-ansible.yaml @@ -11,14 +11,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1228,14 +1228,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2448,7 +2448,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2863,14 +2863,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.0 + clickhouse-keeper.altinity.com/chop: 0.23.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3139,7 +3139,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 --- # Template Parameters: # @@ -3165,7 +3165,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 rules: # @@ -3384,7 +3384,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -3406,7 +3406,7 @@ metadata: name: etc-clickhouse-operator-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: config.yaml: | @@ -3767,7 +3767,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: --- @@ -3783,7 +3783,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -3882,7 +3882,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 001-templates.json.example: | @@ -3917,7 +3917,7 @@ data: "containers" : [ { "name": "clickhouse", - "image": "clickhouse/clickhouse-server:22.3", + "image": "clickhouse/clickhouse-server:23.8", "ports": [ { "name": "http", @@ -3982,7 +3982,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4037,7 +4037,7 @@ data: # Template parameters available: # NAMESPACE={{ namespace }} # COMMENT= -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN={{ password }} # @@ -4047,7 +4047,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator type: Opaque stringData: @@ -4058,9 +4058,9 @@ stringData: # # NAMESPACE={{ namespace }} # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.0 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.1 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.0 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.1 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -4071,7 +4071,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator spec: replicas: 1 @@ -4107,7 +4107,7 @@ spec: name: etc-clickhouse-operator-usersd-files containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.23.0 + image: altinity/clickhouse-operator:0.23.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4175,7 +4175,7 @@ spec: name: metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.23.0 + image: altinity/metrics-exporter:0.23.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4258,7 +4258,7 @@ metadata: name: clickhouse-operator-metrics namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml index d1b11e455..13bce59d3 100644 --- a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1211,14 +1211,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2419,7 +2419,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2824,14 +2824,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.0 + clickhouse-keeper.altinity.com/chop: 0.23.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3099,7 +3099,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 # Template Parameters: # @@ -3124,7 +3124,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 rules: # # Core API group @@ -3333,7 +3333,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -3355,7 +3355,7 @@ metadata: name: etc-clickhouse-operator-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: config.yaml: | @@ -3715,7 +3715,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: --- @@ -3731,7 +3731,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -3825,7 +3825,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 001-templates.json.example: | @@ -3860,7 +3860,7 @@ data: "containers" : [ { "name": "clickhouse", - "image": "clickhouse/clickhouse-server:22.3", + "image": "clickhouse/clickhouse-server:23.8", "ports": [ { "name": "http", @@ -3923,7 +3923,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -3977,7 +3977,7 @@ data: # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -3987,7 +3987,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator type: Opaque stringData: @@ -3998,9 +3998,9 @@ stringData: # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.0 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.1 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.0 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.1 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -4011,7 +4011,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator spec: replicas: 1 @@ -4047,7 +4047,7 @@ spec: name: etc-clickhouse-operator-usersd-files containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.23.0 + image: altinity/clickhouse-operator:0.23.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4113,7 +4113,7 @@ spec: - containerPort: 9999 name: metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.23.0 + image: altinity/metrics-exporter:0.23.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4195,7 +4195,7 @@ metadata: name: clickhouse-operator-metrics namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-bundle.yaml b/deploy/operator/clickhouse-operator-install-bundle.yaml index e1e100e44..ab3a6748f 100644 --- a/deploy/operator/clickhouse-operator-install-bundle.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1221,14 +1221,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2441,7 +2441,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2856,14 +2856,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.0 + clickhouse-keeper.altinity.com/chop: 0.23.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3132,7 +3132,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 --- # Template Parameters: # @@ -3158,7 +3158,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 rules: # @@ -3377,7 +3377,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -3399,7 +3399,7 @@ metadata: name: etc-clickhouse-operator-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: config.yaml: | @@ -3760,7 +3760,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: --- @@ -3776,7 +3776,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -3875,7 +3875,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 001-templates.json.example: | @@ -3910,7 +3910,7 @@ data: "containers" : [ { "name": "clickhouse", - "image": "clickhouse/clickhouse-server:22.3", + "image": "clickhouse/clickhouse-server:23.8", "ports": [ { "name": "http", @@ -3975,7 +3975,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4030,7 +4030,7 @@ data: # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -4040,7 +4040,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator type: Opaque stringData: @@ -4051,9 +4051,9 @@ stringData: # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.0 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.1 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.0 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.1 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -4064,7 +4064,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator spec: replicas: 1 @@ -4100,7 +4100,7 @@ spec: name: etc-clickhouse-operator-usersd-files containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.23.0 + image: altinity/clickhouse-operator:0.23.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4168,7 +4168,7 @@ spec: name: metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.23.0 + image: altinity/metrics-exporter:0.23.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4251,7 +4251,7 @@ metadata: name: clickhouse-operator-metrics namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml index 80b4430ba..3b0111a4a 100644 --- a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1211,14 +1211,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2419,7 +2419,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2824,14 +2824,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.0 + clickhouse-keeper.altinity.com/chop: 0.23.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3099,7 +3099,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 # Template Parameters: # @@ -3124,7 +3124,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 rules: # # Core API group @@ -3333,7 +3333,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -3355,7 +3355,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: config.yaml: | @@ -3715,7 +3715,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: --- @@ -3731,7 +3731,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -3825,7 +3825,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 001-templates.json.example: | @@ -3860,7 +3860,7 @@ data: "containers" : [ { "name": "clickhouse", - "image": "clickhouse/clickhouse-server:22.3", + "image": "clickhouse/clickhouse-server:23.8", "ports": [ { "name": "http", @@ -3923,7 +3923,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -3977,7 +3977,7 @@ data: # Template parameters available: # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -3987,7 +3987,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator type: Opaque stringData: @@ -4011,7 +4011,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator spec: replicas: 1 @@ -4195,7 +4195,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-template.yaml b/deploy/operator/clickhouse-operator-install-template.yaml index 3d8468d8d..f93792da3 100644 --- a/deploy/operator/clickhouse-operator-install-template.yaml +++ b/deploy/operator/clickhouse-operator-install-template.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1221,14 +1221,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2441,7 +2441,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2856,14 +2856,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.0 + clickhouse-keeper.altinity.com/chop: 0.23.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3132,7 +3132,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 --- # Template Parameters: # @@ -3158,7 +3158,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 rules: # @@ -3377,7 +3377,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -3399,7 +3399,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: config.yaml: | @@ -3760,7 +3760,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: --- @@ -3776,7 +3776,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -3875,7 +3875,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 001-templates.json.example: | @@ -3910,7 +3910,7 @@ data: "containers" : [ { "name": "clickhouse", - "image": "clickhouse/clickhouse-server:22.3", + "image": "clickhouse/clickhouse-server:23.8", "ports": [ { "name": "http", @@ -3975,7 +3975,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4030,7 +4030,7 @@ data: # Template parameters available: # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -4040,7 +4040,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator type: Opaque stringData: @@ -4064,7 +4064,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator spec: replicas: 1 @@ -4251,7 +4251,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-tf.yaml b/deploy/operator/clickhouse-operator-install-tf.yaml index 7f4ff6d02..c3db45f3c 100644 --- a/deploy/operator/clickhouse-operator-install-tf.yaml +++ b/deploy/operator/clickhouse-operator-install-tf.yaml @@ -11,14 +11,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1228,14 +1228,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2448,7 +2448,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2863,14 +2863,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.0 + clickhouse-keeper.altinity.com/chop: 0.23.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3139,7 +3139,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 --- # Template Parameters: # @@ -3165,7 +3165,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 rules: # @@ -3384,7 +3384,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -3406,7 +3406,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: config.yaml: | @@ -3767,7 +3767,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: --- @@ -3783,7 +3783,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -3882,7 +3882,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 001-templates.json.example: | @@ -3917,7 +3917,7 @@ data: "containers" : [ { "name": "clickhouse", - "image": "clickhouse/clickhouse-server:22.3", + "image": "clickhouse/clickhouse-server:23.8", "ports": [ { "name": "http", @@ -3982,7 +3982,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4037,7 +4037,7 @@ data: # Template parameters available: # NAMESPACE=${namespace} # COMMENT= -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=${password} # @@ -4047,7 +4047,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator type: Opaque stringData: @@ -4058,9 +4058,9 @@ stringData: # # NAMESPACE=${namespace} # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.0 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.1 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.0 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.1 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -4071,7 +4071,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator spec: replicas: 1 @@ -4107,7 +4107,7 @@ spec: name: etc-clickhouse-operator-usersd-files containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.23.0 + image: altinity/clickhouse-operator:0.23.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4175,7 +4175,7 @@ spec: name: metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.23.0 + image: altinity/metrics-exporter:0.23.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4258,7 +4258,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/parts/crd.yaml b/deploy/operator/parts/crd.yaml index 741feb10c..a29a7caf4 100644 --- a/deploy/operator/parts/crd.yaml +++ b/deploy/operator/parts/crd.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1221,14 +1221,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2441,7 +2441,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.0 + clickhouse.altinity.com/chop: 0.23.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2856,14 +2856,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.23.0 +# OPERATOR_VERSION=0.23.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.0 + clickhouse-keeper.altinity.com/chop: 0.23.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced diff --git a/deploy/operatorhub/0.23.1/clickhouse-operator.v0.23.1.clusterserviceversion.yaml b/deploy/operatorhub/0.23.1/clickhouse-operator.v0.23.1.clusterserviceversion.yaml new file mode 100644 index 000000000..1843b253b --- /dev/null +++ b/deploy/operatorhub/0.23.1/clickhouse-operator.v0.23.1.clusterserviceversion.yaml @@ -0,0 +1,1636 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + name: clickhouse-operator.v0.23.1 + namespace: placeholder + annotations: + capabilities: Full Lifecycle + categories: Database + containerImage: docker.io/altinity/clickhouse-operator:0.23.1 + createdAt: '2024-02-07T23:43:27Z' + support: Altinity Ltd. https://altinity.com + description: ClickHouse Operator manages full lifecycle of ClickHouse clusters. + repository: https://github.com/altinity/clickhouse-operator + certified: 'false' + alm-examples: | + [ + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseInstallation", + "metadata": { + "name": "simple-01" + }, + "spec": { + "configuration": { + "users": { + "test_user/password_sha256_hex": "10a6e6cc8311a3e2bcc09bf6c199adecd5dd59408c343e926b129c4914f3cb01", + "test_user/password": "test_password", + "test_user/networks/ip": [ + "0.0.0.0/0" + ] + }, + "clusters": [ + { + "name": "simple" + } + ] + } + } + }, + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseInstallation", + "metadata": { + "name": "use-templates-all", + "labels": { + "target-chi-label-manual": "target-chi-label-manual-value", + "target-chi-label-auto": "target-chi-label-auto-value" + } + }, + "spec": { + "useTemplates": [ + { + "name": "chit-01" + }, + { + "name": "chit-02" + } + ], + "configuration": { + "clusters": [ + { + "name": "c1" + } + ] + } + } + }, + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseOperatorConfiguration", + "metadata": { + "name": "chop-config-01" + }, + "spec": { + "watch": { + "namespaces": [] + }, + "clickhouse": { + "configuration": { + "file": { + "path": { + "common": "config.d", + "host": "conf.d", + "user": "users.d" + } + }, + "user": { + "default": { + "profile": "default", + "quota": "default", + "networksIP": [ + "::1", + "127.0.0.1" + ], + "password": "default" + } + }, + "network": { + "hostRegexpTemplate": "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$" + } + }, + "access": { + "username": "clickhouse_operator", + "password": "clickhouse_operator_password", + "secret": { + "namespace": "", + "name": "" + }, + "port": 8123 + } + }, + "template": { + "chi": { + "path": "templates.d" + } + }, + "reconcile": { + "runtime": { + "reconcileCHIsThreadsNumber": 10, + "reconcileShardsThreadsNumber": 1, + "reconcileShardsMaxConcurrencyPercent": 50 + }, + "statefulSet": { + "create": { + "onFailure": "ignore" + }, + "update": { + "timeout": 300, + "pollInterval": 5, + "onFailure": "rollback" + } + }, + "host": { + "wait": { + "exclude": "true", + "include": "false" + } + } + }, + "annotation": { + "include": [], + "exclude": [] + }, + "label": { + "include": [], + "exclude": [], + "appendScope": "no" + }, + "statefulSet": { + "revisionHistoryLimit": 0 + }, + "pod": { + "terminationGracePeriod": 30 + }, + "logger": { + "logtostderr": "true", + "alsologtostderr": "false", + "v": "1", + "stderrthreshold": "", + "vmodule": "", + "log_backtrace_at": "" + } + } + } + ] +spec: + version: 0.23.1 + minKubeVersion: 1.12.6 + maturity: alpha + replaces: clickhouse-operator.v0.23.0 + maintainers: + - email: support@altinity.com + name: Altinity + provider: + name: Altinity + displayName: Altinity Operator for ClickHouse + keywords: + - "clickhouse" + - "database" + - "oltp" + - "timeseries" + - "time series" + - "altinity" + customresourcedefinitions: + owned: + - description: ClickHouse Installation - set of ClickHouse Clusters + displayName: ClickHouseInstallation + group: clickhouse.altinity.com + kind: ClickHouseInstallation + name: clickhouseinstallations.clickhouse.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + - description: ClickHouse Installation Template - template for ClickHouse Installation + displayName: ClickHouseInstallationTemplate + group: clickhouse.altinity.com + kind: ClickHouseInstallationTemplate + name: clickhouseinstallationtemplates.clickhouse.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + - description: ClickHouse Operator Configuration - configuration of ClickHouse operator + displayName: ClickHouseOperatorConfiguration + group: clickhouse.altinity.com + kind: ClickHouseOperatorConfiguration + name: clickhouseoperatorconfigurations.clickhouse.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + - description: ClickHouse Keeper Installation - ClickHouse Keeper cluster instance + displayName: ClickHouseKeeperInstallation + group: clickhouse-keeper.altinity.com + kind: ClickHouseKeeperInstallation + name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + description: |- + ## ClickHouse + [ClickHouse](https://clickhouse.yandex) is an open source column-oriented database management system capable of real time generation of analytical data reports. + Check [ClickHouse documentation](https://clickhouse.yandex/docs/en) for more complete details. + ## The Altinity Operator for ClickHouse + The [Altinity Operator for ClickHouse](https://github.com/altinity/clickhouse-operator) automates the creation, alteration, or deletion of nodes in your ClickHouse cluster environment. + Check [operator documentation](https://github.com/Altinity/clickhouse-operator/tree/master/docs) for complete details and examples. + links: + - name: Altinity + url: https://altinity.com/ + - name: Operator homepage + url: https://www.altinity.com/kubernetes-operator + - name: Github + url: https://github.com/altinity/clickhouse-operator + - name: Documentation + url: https://github.com/Altinity/clickhouse-operator/tree/master/docs + icon: + - mediatype: image/png + base64data: |- + iVBORw0KGgoAAAANSUhEUgAAASwAAAEsCAYAAAB5fY51AAAAAXNSR0IArs4c6QAAQABJREFUeAHs + vQmgZ2lVH3j/r6p676abpSNLE2TrRlwSQBoVtVFHiUGFLKOEKCQTTUzGmTExhmhiSJBoTMZEs2KQ + BsWNREWdyCTOGOMEQZbI0i0NyCaIxAB203tXvfef33LO+c697/+qqqu3V1Xvvq773fOd3/md5Tvf + 9+57/erVajq4DiqwoQLnvf4PH3Pe4ekx29P0oNVqfTEgl6xWE8eLV6uVxgnjCjLHab0TI+ZW00PW + 6zWgq0+up/XNkG+edjiubt6Zppu31tOnd1brm7fwvL1e3by1Bf165+b1+vDN02r7Jhh+6MZnXfYh + jAfXQQVmFUC/HVxnbQV+aX3Bg875xJOOracrD21NV00705U4XK6aVtMTVqut81WX6BAcNpOOIIye + x4iJFMfDhmry8CIwRh5mZBHfppH61XT7znp672pavQeH3g07W1s3rLbX77npQQ+6YXra6rYNXg6m + zoIKRPedBZmerSmu16vzfvnGRx9a3XXlar2+CscEDqXVVTgSrtzaWj2SZRmHURRJh0uf9/EycNE2 + cVpxPg+jLLMOJczPRiiPd0j1Q634eMgtr/X693CIvWe9OnQD+G/goXbXzs4Nt3/FZR8BxwaDJcGB + fLpW4ODAOl1Xbq+4cUBd+Mv/4/Om9bFrcF5cs1qvvgSvM5fpMIDN8tCxzDcaHjoCiFkyn+psur/f + sOaHneLfdHgxRs7zcNxZfwrjryOPX5u2pl+78VmXvgsyvgo9uM6UChwcWKf7Sq7XWxf+8u997s72 + 1jWHpvU169X6S/Dl3GVKi4cQrjp8JO11aGnPGxGHlw+ztPeh5jOtTjHhfdj50AgX8zcrHib8Mg9K + 2W8a49DJw2c2Jmkf85Aib/LnGPxw9ik8/joyODjAeu1O4+eDA+t0WzwcUBf84keeslof4uF0DRbw + mTgJ8I1xHArIRYdHjD4c4piAntdm3JnxhjU75BaHF7PH98Q+hfHgAFMnnJ43d/HpGfvZEXUcUOtt + fHm3tb4Gp9Izp62tBzF5HU4+pVQLnkn90AIg5ufLvPnQIp/gfgDRHHf6vWExHR/abWxvcsgIB9hO + HGBbB19CxvLv5yFbdD/HeFbGdu7PffSJ07T9l7ZWqxegAI/YdJioMFLsPkzqsMkvxNrh1Q81486O + N6xNh5fyzy8rd3Y+tl5NP74znfPKm7/ikveelY23z5M+OLD20wL9Xx++7Nw7tl+wNW29EBvnadxM + vOrwydVq8/FKFbiDN6xT+l6Zqje/gectWINXr849/JM3ffGlfzjXHkgPVAVyCzxQ/g/8vnZ9zjmr + D/0JLMQ34bh5ztbW1jkqSjuU/EYUpeI8JvIw89dxB29YqkP7co/yyRxeszcs2vcLMip7Fwr+Szs7 + 61fffM5DXz89a3WsQw6e798KHBxY92+9y9uRf//Bq7d2dl6IDfH1+HmoB0uhw+g4h0+uVjvMDt6w + ol44XOrwQTF3ffmHOZaPh9iuw03FX9wC1w89PP8BiH9ie3341bc++7J3LCwOxPuhArkF7gdXBy6m + 137wM86Zdl6EHfNN+N7Uk+JVaVaY+ZuT36S0+XKlDt6wZvWSsOkQupfesDa+qcEfjsd3Y0lefezI + zqtvfdblH98d1MHMfVGB3Ab3BfcBZ1bgtR958Dnru/43NP//ii9UHqI3AehYfB9GsQwHb1h8Bbr7 + b0B5OOWYdd00ngp/8GBwfHrwbWe988nVtPXPp/U5//zTz34Qf+7r4LoPK3BwYN2HxZ1+9n2POnJs + 62+gyN+MQ+pCHk/jsIrjiodUuw7esHgmtC/v4hCqL+Narepx0yF0koeX1qP5K04+BG//slCrxvn6 + dBO4abp1Z1r/yLHtwz94+1c/5KM0P7ju/QrMd8u9z39WMp772g9cubOz/bfwd9z+PPr6SB1C0eTj + 0OIR5i/7VCgeXrl52nzhc7XikBOvCYZ5s9Mm77JQ9tf9buQHYMxrmy5kEYvRccggPGw+dMwytvpM + jsMhD4nZWKztoR8meTjlCJjy2zRu8tNo67HzB490nG+XDzP+0C4OWczvrNdHkeGPT+vVD9z8Jx72 + ngY9eLwXKsAaH1z3UgWO/NT78aMI67+Nln4uvkeF357SN7G3Zx0C+Rk6Dp8MQZufQjuUtPlypTgv + 2pgQrr25Le0Wfsr/DGd78na/iqnczH+SXrhZehmgrOa3xSGx642FbvFH7jkCrzjbaH9EbLgW/HnY + nZKfTh+8u3g4XxHjMeQ8tCjiz860Wv/89tbW99/2lQ97a6c9eD71Chyny06d9GyzPPxT7//y1Wr7 + b+OM+vI8o9zS8Zk3Dods8jo0UCjhUs8RnV762aFSZ0k9EDc/ZFKMZW32fU1Oih+BzXG749IhAmLH + IYNys+nQYVSuy4aRuzzy3zUWa3sI/L3ip9HWY+fHJOPWxfl2+TAbb1iUkQj+GBc0/w9+LOL7bvnq + z/jVZnrweAoViM4+Bcuz3eQl661DV33guVvrHRxU09O8yWLPoTb4chB3NG0cGtnEdQjs0rug2vx8 + bIeNtkCulDY11TGhcfdhspefmp/x296niXkH/4jLcTS/s/QyQONn99i19+jNR3kzgg3Xgv8e+en0 + wetDqR2ynM/1Iz7k/oZlmog34Ov1zlumaev7bn725b+ABTz4LRIu0t26H6fL7hbPWQU+8tPv+Tz8 + FpeX48u+q3OvqADaVD5r3KMHb1izNyAUqW91Nl/JWchN46buCtyMH/XfdbjA9oR+TsQfcQpGv+2y + vxO+YUVcjg8B/cb26tC33fbsh/23RnXweBIVODiwTqJIBcGPJxzZvvNl+Ez5Lfhhz63abQRsOKy0 + HTmvy9um3nByG5U+UCnHWPiiwQP2zHgDWvAu7RZ+Bp8JLR+8YakOi8Nozzc14Vx3rVrIJ37DQp3x + wUMOF355xPrlq3Mu/Lv4e4uf9Oof3E9UAXftiVBnux5f/h258n3fggZ7GX4h3oN5JujNAA/svTgj + Nh5aauIBQCXbl2+SFocPCDcfKgs/sCUuAtEKDTGWNfwKJ4RvJ8Ufh2LmOYs78+n8s0IAnXn0Ee7F + t2lM+01ji70eA3ev+CnS9tD5Mc24dXG+Xaf4hsUCgQXrtLPzyenQ9F03P/vhr8CCHnyZ2Gq76TE6 + e5PqYI4VOPJT770azfVyNBN+i6cPiTqEoudUqTgtYtBnUrV5bm42JwjqsAgZEzLPWx0uMV/4hIWD + Oa7xLu0WfgafCS3b3qfJmHdejmxpp7hVj4h8kUfmozE2f57u3uQ+BOgty1gj8PLXRvsjYsO14L9H + fjp98O6Kl/NZV+JDVl+kyHllFgMSrcMNeC1j8mDE7zb7b9s7h77t9uf8kd+Q6cFtYwXcnRtVZ/nk + j/3O5UcOb/9jvLd/I75XxXbjadWH2FSeVrWWejR1HW4G4N4OF0k+BId905MP1zgsJJbDDEtxCafw + hBey2YdlTDOu4Xcjv9LtuN1xDb+sS9QnHGlzwv9shE5+N41pv2kMztkQuBl/+tvEjzlWk3jF3ccZ + cQidn3aJ4Xy75D/XGfPib4fZcIP6EacJAXHLutFOpFCvf2xaHX7xrX/y4K/7qCKLm3fEYvKsFv/z + +vDWx977bfghqpegOJd4U2aTe5PXIcQmywrycBgwTFMREyqo5TocdulddR1CfGyHjdzs8hMTwu0+ + TPbyU/Mzftv7NDHviGPE5Tia31l6GaDxs/vYtcqLm5Zo8W0aqUd8wsVYh8yMOIQFv3Z/2m/ix5z8 + b/LT+YN3V7ycrwzowPI9ecMindyRblp/Gs9/79YLH/4vpoPfDtFXRFWfTZzNwuGffN+XTjvb/wab + Bf+qTG4a7rHYXhxjk6pFtSm0B122pR7lTZ4AYAhePAVr8HPCXavNKpEI+7c/ieVQcTVFuJ/zhX1Y + ajgpfuXJ+O1/Fjcd8YrRccjA87j3w0b+sAMrX+pp3kftVsxsGoHbdQXuXvGzixwTnZ9iYjjfLh9m + sc6YpzwyKxrXg/0gXgGNC7mm05Pd/Pb2tP7m25/zyIMvE1EtXtF5Fs7a+2vwd/5W2z+Ipvmz3sw+ + VGK3oizonjgNduujaqV3cx+8YbVu0m4ch5E3edZpwwh8HXKoqzd52Dfaelzww0DrdUp+ihQPe/Fw + vo7bwPEwwgc3lNQYnVkMCp9656N2SR75CXeCDx54orODLxNRBF6s79l9/eT1Tz60fegX0ECPc7ex + 16P5tFksq2/UZZTdRd5UllXEpZ7NySbmvAG4W+4tX3rZN33YOZ6FHzDJTkTmD/fDX7O3f98HX9ox + zgU/Jua43XEBIELHIYNyo8MC+tkIrfxsGrVpwbdpLNb2ELgZf/rbxI85Rku84u5jo63Hzk+7VHC+ + XfKf64x58WcjSU53qB9x4g0FcSHXdHoqN3yA3c76evxWiK+/5Wsffj0mztoL36o5e69DP/7ubzm0 + c+gt6PPHae+hN7xJvTnZO9qMflDXzjax9FE/EoSMQc3JCdsTo+0S/KlP/uBA1ya+j+KjOa/0o2YP + WX7kfre9cGTwNYsfU5bpF4JgmYcdaj7ycBwRSMVBO2gMtJPgpaA8FnmJB7rZGPYzfLNb8qe8F955 + wf/J+Mk4Mdal/LweGSd18idQ1scedBht1GNS5eEnhVjfkEPRB8SbvHJCRhGstlZPXk/bb7nglz72 + zak5G0dX52zL/LXXX3ToztWPo/me610TZXBvqCmzubwXSzFvLjT1bK+qydnUgqn5ksclNs+uzUQD + XjJsmyTmCx8w4QRPR1aU386XOPLHNfjSjpvJ7gUJojlud1zzOLQL04XeJGQfh47fRLIuG8Ys5Kax + WNtD4MSLeGcjYMpn03gq/MEj77Rvl/OKwwjzlPOQIWy4Q/3wIT3LnjgBAsdpRa4H3HiZz36sB8/r + brnoyDdOz7r8FmPOnrs79uzJd5p+4t1P3dpe/wx+Uv1x7oXctNE0bH58cLMNPZpmtom7PopX+mwx + dWU/BQBsmz4+c+amzyWQXwrk08B4SpzFEQAMjXdpt/AzP4RItylfz5tf98D1edcn3DkuQ3ffx64V + bmw+iED3LS4ZeMXVRtWDPJuuBX+eEqfkp/MH7y4exZGRwyBk9U2K4ol4I0Hz+NBi3SirAvJjGrMi + /108MghW6d+/s9r6+tue84i3afIsuZ09XxLiL/9t/fi7//rWsZ034ueqHpe94marnlMT9c+Eu5sq + mwnNNnqoui16D5vQzWh7dtOQsyk1q0Ci23h4hNzHWfOGA/GXgnax+Zf29FunCsNs8TMqybbXLhHe + 846P99hkgedMOmRWmw6THj/1XXb+ES/NRScm4xKfI31EXnzU1fNMXI4AVJ54nvnbZBd48eaNuOBL + e6oyDzwJaRn54aPnMfSAQeF4og4hh6IP4rEf0eNGP61+8kMZ33Pd3n7j+b/40W9P5NkwssZn/vUT + H77s0M5tP41V/srxhoG01QsuAXqTTYAejebTBDAcCQx5tz7KV/pssWYniOXyn9tI/MGBgXrDPRY+ + pnscA4fNAjuns7Bb+NmMs/28HhlHhjPnH3FYLzkedw2x+aPAGGJzA0gviruP1DOfNtofkRuuBf89 + 8tPpg3dXvJyvyPEY8ji0kiTiDXg/tLjMlMUjP6ZxPaI+YV4VEp4S9a4PPUF+3W1bF//F6Tln/j9H + Fl2ZBT4Dxx+7/plb6+knsQGuyB4bm1arzVPCibtbvFnYFNo0VJdCvaNmg8XQR91CUXo2VfB0B+Uf + k2pZ8YsQE+FXouMqfIQJx6JTXCYIeQNf4xEo5O53Iz8AY975z2URidJxyCBdaDMKn/lwhFZ+N43c + jCrchrFY20Pg+6FW/jbxY07+N/lptPXY+YNPOs63qw4hrjPm6xBSplrOSIv6OGQMBBoP8lMDrIK/ + 3PAhDm/y46N4IOeF37f1kdXhQ887079EPHO/JOSXgK+6/nu2ptWvoZGv8KbinuAhwhG3ehiy9WgK + bR7jBWyyzdhsYceuKX3QshnZXHIkQMmylz75qceF5k18H1uYww/tS0G7FOl38LK5tSk063mbDZwP + VQCoCN7hn3OOq9ulQ7HE5iYyr2Fv/10WD4CzMeokXNYhR5JWHcPDCfDkmfGnv012onc9gt3+wn/y + UJd5qE4loz74EK7izPoCBIXjifUNORR9EI/98M6LPGEniX6GTAQvfE/2iunYMX+JiN737Jl3PzMT + ++X3nbv1B3e+arW19Q3+DBZpeu2jedhEWFB3mVd2pndTeC+WYt5c3BRqvmgMEoYsWjVxby7z7NpM + 2eSyD7+gzM1ReJrzCrz0Lf5wX3YznC3DfL65NvIrj47bHdeMf1YIlCE3ex/h3XXZMKb9prHFXo+B + u1f8FGl76PyYZty6ON+u/fKG5Tqw7jzi1j9965ErXjR99erOFuoZ8Zhb4IxIRkn82DsuXK0P/RJ+ + XdWz+mmiTYnVnG1O7R6XIM6K0MchU3p2AXAauIm7PkpX+tyM5A07QSyX//jMmZs+WOTf8IwrD41A + lB/rbd/zWtgt/JR/8uCy3PMZ8wboHjjjPQ/cLD3bGb24x+bP9fAmzzptGHOd2qg8ybPpWvDfIz+d + P3h3xas4mG/EE7K/XMNshZl6QFkuKPLLOS1j8nCUXgNYiVvyBEBeBw/Ecc15/vOt5x35mumrPuPW + ATj9n86sLwnxGxZWO4d+FV8GPovN0ZvccjZNNEPritE7rRnUI2y6aJaQbTaaTG0wbxY1He3k1wDx + uGnhHx+8hp7qgc/5whvuwBkH7R0IDbVJKq7Gaxw1vgbf8O9NBL1g5h3+aee4On86FIt3nx3EfdiT + b56XeMTKPOx2I77ZyV/3oAWL+iUuR+Aqz+TPcZNd4DGMS+vZ8g5NxunIwz/rg49ZfZQZ9TCEwvFE + HUIORR/EQ1cRJp9EkH7tZ9STWF7Si7fq+azzbz/6q9PPffxyI86MO2t8ZlyvfNcVq0MrHlaP1xKj + ebXqHHHpTQJdMN4wMGlg6BMezSd7GRoY8u43EpnTgfjKzO3reXsQT/nfpTcP9Ya3uDGR08NPx/W8 + FnYLP+U/CHfnM+wdR4bDuriOngcOE7O4DN19j82f6+HNG+UHWrx9zHVqo/MmcsO14L9Hfjp98O6K + l/NZV+JDHodWkkS8ATePDxvWTYcMeeSnBtQjD7/OEzi6k373obWLx/H/znq99WW3P/eRH0m203k8 + M96wrn3356AB3oA3hcdXc3mx3AVc5GqK1gzoAS1yDtFE+druXqpuU4/O+cvcD6P31FQ0cFOyRcxT + 9pJpFk1NCJJIuY8tzOGH9qWgXYrmK3vhCPTFeZsN3MEb1mIdWCoV1OuherX6RSU1uM7A4aPXFQSh + xwCF6x7rG3Io+iAeGo62IE/YcV5+hkwsL8UhXtGFrInHT9P2G8553Uc+x8jT+46MTvPr1dc/fbXe + +Y/4ntWltZhISW8q0XTMUG8S3Kw8FCTjVg/Uu0k2v3EAKEDwsGlCVpfagfjGNEsbdtS7nYf/kMUr + vW/iFdxLM4s350UbS0eHEY/TWdgt/Ay+juv5jHlFZHHELX/QRKIpLvOQbd5yHWKsQ5M0+KO4+5jr + 1Eb7I3LDteDXbkdgp+Sn0+8VL+crcjyGPA6tJIl4A94PLZUveeTHNK5HHn6dBxrhWS/qdx9azrvx + AK/1lt36xqPrra86+rwr3pysp+N4er9hvfpdz8Fh9atYvEv1GSYWUYuuJmiLLJmL2ZrBQK2b1lTw + 1gwhqzl32aMLB311ScDUVAQ4Lrpw15Z/yY6nGic2mdAKaBGvFfZL+4qfflKkn8HL5u6HSfmf8dte + fMGbcRvmPOzO/OlQknef/OZt2Nt/l8UD4Gxs8TBe4XMkaeiTP+UZruE5P+NPf5v8iD7ySgfEBV/y + UJV54ElIy4gXH8JVnKkHDArHk3kljxShz3o0O3sQQfrd67CSftA5zlk9pksPTzu/et7PfvRZoj1N + b6fvgXXtdd+wWk8/j8PqQtbebyZoGi5SytF0pYdi6DFrINXsTcnWm4f65BVgZs/uCLskCBkDetSE + thdADsq/9MlPPS7Em/g+io90vBgo/dC+FLRL0cCyF45AX+VfCdu/NgHNBDNR2YtuUVdSVRy0Y0D2 + SxWvYU++eV4KG5jZ2OIpfLNb8qcsP4nLMfzP+NPfJj8tXjz6Un4t75rOPD06T+DwIX9Vh9TDEAri + XGfWI+ujh5CzHs1OPim3+snPkCMs13vQhTxwsR4XrraOvf78n/vgN6Td6TaengfWq971HVj+n0ST + HM6C12c6Ni0m/RmHzYFVLNnz1mOyHgImOO3BTruQ46Fkm6Ve9CYQPmjZlckjyJBlL73jCQbAwy8m + nI9H+wtUxqWuFxA32oXfni/VwpHBF3nNx3iS3/ZSBO/wT5Tj6nbpUCyxuYnMa9gzsHle4gFwNva4 + E58jSUOf/CnLT+JyFDzyxPMJ/QS+uPlAf8GX9p523VjZISM/fAhXcaYeKChc96hDyKHog3jMK3rc + yNPqJz9DLpTijbBp1eInxv75sDoXP1L9k+f93IdPy19Tc9odWFuvetc/xv8J/Mf4jMH+qMsimoZN + hlnJsWgEWfZoPSbrgfOWbW8e6mXnB+Gt73xkx0UC4YM2tontBcCtxSd98lOPC/Emvo8tzOFHuwA2 + EXi4320vHBl8zeLHlGX6hSBY5sGJoO9xCxgK5ktQqzNFXj3+ZV7KB5jZGLyyyzrkaELex3UCfOUJ + ixP6ASbjLQfkD/9pT93AZX04on74EC7r48pUXR1PrC+A5tGDApQ78nc7OpTc+gJyvalJ75v4Bp35 + W/3sn37xBz8Jj+FHzvvZD31XozgtHk+rA2vr2uu+H2vwHaosHvqlzyhcTC4SFPkZhk3Hy7JH6zFZ + D5y3bHvzUC87PwhvfecjOy4SCB+0sU1sLwBuLT7pk596XGijxPexhTn80L4UtEuRiQzegzesqAvL + 2+uigu2uN2unq+llV9OuLyocMI7gwUfnH3qpta7Hf3PPOJM3HMpPi1N+hlyoZf+1+IkZfQuBxxV5 + VtPLzvvZD34/9afLxchPj+va6/4P/FPL/9TF3h1yfQZT83BNkFosGtHSQ+6fadwLLsGAR/NpQoa4 + oYlCLj+lJzuu0guNKNwUmjcA9+Z/l14gxWd4xsV4TF/zCsd68zdexiF82C38jPw7zvbdEXEmigGy + 48h5jFEW42Le8Pk916FtIqLFt2nMdWqj60uLDdeCP9e9NilMNvpr/KoLefoVvLt4hEtGGIQ8Dq0k + Cb5I1DyoI/Bql+ThCLoYUBfok1ZURRAS9eZJT1YseORn4CqP9KsV4D9Bvf72O//0Z/6zGdc+FViX + /X+96rrno6qv4T8P78XO1V2E7y7QptcSh+xNjTTVBdz8ufkwZ6BrMNN7E6uJyEOg+GowT0yLYKln + U+GjNn8EUP7LfeQh+/ALXdoVPmCKI/Ut/nBfdjOcAvRt8JlwyAGqPLJOzn/gWrwVRy8EN1/Wr43A + KtxNowtNwyhwGyOs2RC4e8XPjDiEzo8pxq2L8+3yIRDrjHnKSIBPQgWN68F+0ISAxoVc0+mp3JjP + fsiKeiaPPPgmPdwWjx5iQu5oZ/14oAz+Nf796Wn9wrv+zGNf0yj35SNz2N/Xte+6Bl9z/wr+IvNh + r8bmkL2ZYjGREeWOlx6LMzYdQOoF8w14O2RKj4fZJjaP7aN8pQ9ad4XtBKGf5n+X3jyKm4/k05CH + hsRZHAHA0HiXdgs/I//O3/MZ8+a336XdqIf1GW9I86FvHsQ3Nh/SAVJl7mOuUxvtj8gN14I/1/2U + /HT64N3Fw/mKHI8hb34zot5w80T+SJyyeOTHNK4H1iNpMTaCkKg3jybytuSBrHWTH/sz7zx+x7E+ + Nq22vuqOP/WYX026/Tju7+9hXXvd52Oxf5GHlYoaza5CxiJkUXPxOdaicPO2xaJi6GFp4KArfTRD + yPOm6vxF7wfhs5XdFI6LLoYst5IdjwLgreWXdrN4iWE+9EN7ESVvipQHr3EE+hp8A+fmh14w8w7/ + tGuHTNa94qCaAZmPaF7Dnnxjc5V/YBR+jmEvu8TnaELex3UC/N3yA9aMtxwov5Z3KAZu1E/1wXoo + n6pD6mGo8lB/vL7KejQ7+aTc6ic/Q46wHH8sQ8XR6lf1EJ0QZke8lEB7eL2z87ojr/3A05NzP46K + dT8GNr3ynY9Fo78Zv874IdoMKH6NGwL2G0A0BfSUOz4/0xhHPUC1eJbdo2gGNoXsRWRgyLbveqh5 + lT5o1QZwIEcC4GY7t2AEUHpiCOe8HmJgPJ2GeS1xjTfsK89FHDU/w/V87H/EoTAUl+No8SkO60ee + IfchNn+uR20eYMgm3j5yEyE+4WIUP3k2XQv+e+Sn8wfvrngVR0YOg5BP5zcsZaM8pk+uDx2++s7n + Pfr9vRT75Xl/vmH96A2PQIF+BcE9hM3CZp2NrJ6LW3WUHu1fzRV2ibM9zbAZ0rwequdCn/6M17YS + X7dnXGGX8YQs2tiGjksA3Fp80ic/9bgyTzym3SxeYjIOnVohi1fhzOwEF44R+Rp83vyWGRf0gmVe + qaddi1tATFUcVEfihMbV41/m5fpEvMBLDl7ZZR1yJGf6Df6U98I7L5glf46b/Ije+Sa982t5hyLz + MjPDsodxWCWPx1ALt5/fsCqP9fSQ6ejR/zj9wu9yD+67Sy26r6L60Rsunrbuwj8UsfVkNV1uhj5u + CNhvDjyM0GTQ6w2B3UK7kJPPekzWg2GG0375xhE8GpZ60QdBDYpDDsI/NHLoOClZzviCxXFT6HEP + sRwov8LFG4lwLV/KCz/lf8a/zJduzCPz4HW5ch5jlIVhZLx6Xt5yHWLUOgAjvk0jcIqzjeKn/aYr + ePOQyfGU/HT+4N3FozhYh4gn5HFoJUnqIbNcwOWhxfJSDkUfwEoc9Z1HBpqw3odpIqyAQcBsP+pI + feWRfiP++TyBwl5357mXfOH0dQ+7Wdz75La/3rCu/eB509bRX8QPhvqwiiKzWbW4ObJ4YzVVylz8 + WfHdFUOvxWzN4FUNPYbSpz8vcu+COX8Lg/HIXoOajk+Oiy7YfZbltjULtbpafmk3/AUm/dC+4idv + ivSTcdMrFQT6GnwD500EvWAmGv5p1+LOulccVMMw5wnHNeztv8sKG5jZGPbCZR1yNCHv4zoBnjwz + flhK3mQHXcZXDogL/8lD3cCN+pF5HDKeB1JUcgcCx+M6MhDz6EGByR35u50ZcA+70OehJ3XcxDfo + zN/qZ//0SzoAy0/uBymsJ+dq+uxz7/z06ybuyX107Z8D6yVr/JjVLf8OAV3jmrIJWFuvwmxkAVX0 + UUm/EcRnlGaXONubb7ZmwZNrSJyajxMAJq94Qh5xtTAK71bINxvbK2DcWnzsWlxDT7X1fd7xbPCj + rhcBDTO84ktexaEuJav9Vfwl2575AqEEyl5htrhVKBkaRxPtNudDkdewJ988L/kHZjYGr+wSn6MJ + eR/XCfDkmfHDUvImO+gy3nJAXPhPHuoGzvlaRn746Pyqo/C4QeF4og4hh6IP4rEf3nnRT6uf/AxZ + EKIUL0fRhTxw9m+91ku8thtxE29G9+/0ZedcuP0zE/fmPrn2TSDTFe/8l1ur6TkqHoozPiNodSWz + mv7MJMCshJ63XkuMZvPqkdF8XKPBi0kDQ2/Z+vATeAHF1+3JT1nmfghZtG7fEW+Th958waB4M78+ + Cr/0o10ASwXAeDMdA8teODL4cn7Dr2Xbu4dNVPaiW9SVVFkPPmuX2C9FXsOefFHPmHf+ES/mnJ/t + ZZf4HE3I+7ii8HvhOX/SfsCa8ZYD5dfyDsXAtXjh6Ux7w9KbHgqI/7723Cd94IerLg/wA9f0gb+u + fddfwcn5r9kC1WTRrP7M4ab3Jtkccn0GU/OAB/YdLz2bmLz0Mxwp/wGP5tNEA4ZcfkovcxNmmDTL + TIjTZYflf5c+UImPsfBFgwf5iQnhel6eL7uFn5qf8dt+FCbqx5DKTdatJiIOx+2CxvNyiM2f67Hn + YRLupEd8fRQ/eTZdC/575KfzB++ueBUH6xDxhDwOrSRJPWTAzZN5WQ5FH8Aah22lywcRiNh686Qn + K4ALmKJDXFpvxZf+m9+Iv/IL3EgrA8BPlq5X33r06x/3b2b+HgCBeT2w1yuv/yz8RsS3IJALajMh + olgij4vmJS6bchZ8zBfPEqdVxJqSD4ZUjwdOWLY+F9t4AcVnO9uzaYIHpt5UNYCOfuIwoD4clH/M + JI/Vzqv0CpB8LV7RMFDP9/gjPOEDMHB68m3wDZ6qByGVZ/p1YZZ2wgnuuFMmRT9sZMfNw3niN41Z + yE0j8LuuwN0rfnaRY6LzU0wM59tVhxAQykv6zLBoXA8giA8gWIALuabTU7nhg+3MSj/Bs4xj0IWf + mADOcdod3crvbH63H9ErH/71nem21aH10+/6M0+4XvMP0O2B/ZLw5R+7YDXt/Ds09O7Dqjapi86m + Z5E1sliU2+V561X6wCfO9jTLTQhjA8UiOsjWh5+QBRRf6j3SXnZkKH3QuivEJwdNllvJwWPALD/n + Y73wmW76oX0pGG+KBpa9cAT6GvkPnA9V6AUzUdkLtqgrqSoO2sGQcruGPfmintCXfzwr/BzDXnaJ + z5G8C/6U98LfLT+in8cv/vCfcTqMxHmUf2TiT07Ojzg8+c5B5WHfRR1CDkUfxCPrdCOeVj/IxSMP + vrkOLpPi1fo0O8ieZzx6kmHVSfUlPvjolzD5x797uJouwM/C/8z0Sx+7wIgH5v7AHlhHPvGvUBW8 + YbGGKBZH/NGYMovLYvfRBrzXJT0siyfwXhzzk3joYWpH4iB86NNf2FnR9J0vQpA/zgdtZOK4iLGD + 8u9uUDzBAMPwS7QC8tjCLAds2hE/7VKkn2YvHBl8lf8Zv+3FF7zDP+0cl+MwfzqUxC4PvnAzi3+Z + l3jECrMcWzyFb/VY8qesOBOXIzgrz+TPcZOfwGMYF3HBl/FSmXVx5CmjPviY1UeZUQ8jlYd61zHl + UPRBPPbDOy8TpF/7CR4DjFK89ldxbKqH6EZGVacIVIO8Mn9S08C8+CHuJx+55Rbs2QfueuAOrGvf + +SLU44X+DIXasLioQ5ayZMxz1YXLkfXifLvMY5zXxHaJs33zMxyJRXRcI8WR/ixr0RRH6mNe+Aii + 9F5iNiefHBcxQ3aelJNfjyPPNu94iAtM+iGfE4WCflKc8yqO7MLgtdnA+TO2aIgQUcZtv4u6ApUO + xUL+CpDKaTrv0Gr6zsecO7356RdPz3vYYai7P3nZtd60Ey7XOUcreB9X49vUH1U3WLjeG+rT+DO+ + ckD+0Kc9dQM38iHzmf6GpXIgz62trRce+an3vqjqdD8/uOr3s9OJ37daH3sL/o7gBfoUtKE5skk0 + Qq/PBG0sux77kifk2kzMFoTVzCFv1scmDXwY9sE80kcQ8kd+waKJzWOEHZZ/hyMe6TfFD0Xhac6L + OA3pKOXwG/oZTha+Db60i08WFiuBOW6PQ6viGIX42ocdmf7BY8+drjhvfD78zZuOTX/zvbdP1926 + wyNxfohQxuGgeDeNDnt+D9yyL+qNAeiT9jNnttT5MYPoxnw+Y7Q/1o+HVuRRnilnWtS7jwMINCwE + qEE8oh8OhRt5NR4BfXMdGs9wbDrIjk90BuKxeImP+Fy3CEDzc15obju6Pvzk6fmf+SEY3a8XY7t/ + r5e/9ch05Jy34/Xys1zk2CyIwiWLEZsu9W3VvfobIvbmisWEnnLZhZx88uNVKb4Bj+bThAxxg0XI + 5af0UPMqfeYRDjivy3IdAm6f8h8gx00h7ApfNHhQODEhHPLG6Lw8X/LCT83P+G0fBI429DJXOHP+ + nm8YTE84f2v6oSvPm65+UP3maqnyhm/cTq/5+F3TSz9wx/Spo9ziSBN/do3cXMynjfZH5IaLm4rx + LkbZA76LH3Py2/jT34w9+HbxcL4ipwPL49BKFs7jCrh5Mi+alUJ0cic4+zhoaT8IQqLePJrIWxAU + Dx56XpUHAS3++TxUGbZwJPdEjx/Pbz566OgXT//zk+8i4v66xqfA+8vjkSP89cY6rNhkLAKvKhqe + q5livnAN7yahpS/zRDNgSnI2ccnNT61Z+gdIPd+aIeTeTRUn45be/hVPyBiwxHbguIgZ8tBHnEFR + eRK9rIvDpMJ+ySei5E3RwLIXjkBfs/gxZZl1gyCYectedIu6kqrimKZLjmxNP/D4c6c3fP5Fex5W + NME3bqdvevg509uuvnj61ivOnY5ATrfDH/1HX+SY/jjm1eqzCV95As+0juuH+uBLeufX8g7FwKkw + YQccPuSneFIPQ+aJ+TpkQg5FH8RDV0UTkadf+4n6REzGs262qzha/ew/eLkvxEu5x01ek6p/CSsc + 6Wd+n37k2JEfMPr+uyuk+83dK9/5dXizep2KFMXUZwAEkCWsselVRRaZ1VSxd0dsnmgK8i3w0nNx + yCs9bvVgWtNTb9zQh18NS33EIn/BQ/7MqOJ1ZuV/l948ipuPYVd4mvMqPzEhXM/L82W38FPzM/5l + vnST/Ok265bzq4mf7f7CI8+d/ja+V/Vgnj5383r/bdvTd+DLxP9y43ZG6THXqY3Omwu24fLCjf4I + uTYjTBidlruPjV91oV2/9uIRLhlJaOZxaCVJ8IVjx4M6Aq9lTB6OoIvB/Ze0oiqCkGCPD/LMriUP + 5J5X1UN2I/75PBiDdvB7wrjmFzzb0/pPbT//yp+fxXEfCoz6/rmuffdjpvVdv4VCX1rN51WLTRjF + RTRZyhqBU7Ha6NVdhB98s02pRQxcEA49nKkXNulzsRleGQovN4xT8Th8FTEUpWdT4cP2RJin/GPG + 7tM/xmwyokkUYw8Tiprv8Zff0M9wsvCt/DeeXfyYmOPa+sDu6Zccmv7ZVRdMV114qDGf2uOvfPLo + 9F2/c/v0/tvhNNerj5toQ7/sC2+qqCvslFcfO68KRsSGq/NDXSjOt6s2MRBcFcq5zoQNd9THZjfQ + OAECR3x6KjfmG3k1HjqIS3rwBp3jaPmVfYVnB2N+tx9RK58FLxRph1+vfOOx1Tl//P76fpY7P7O+ + r0Z+3+qcc9+A3xz6+doEKMJshF+tYR9RbC9CrkIbN8TpzRWLSZ5cLI4hJ5+WphymPhaFftl8spch + brAIufyUnuy4Si+0mrPsDMA98iY8M6Zdu+SXcszbX4nNT9gJ13iXdgs/gy/z3pQv3SS/g0u7R553 + aPrex583Pffyc1rU9/wR39KafuSjd04/8KE7p5uP4Rvz8F/rpV2IeGLz7PLW9Yw75NxU3opajayG + R+B2+enke/FwvpjwGPLZ+IbFcu3srN987H1P/ILpJasdyvfldf98D4vft8JhxUTYRNyMszHmuUWq + FYCTvAfeTUJGX+KDRTVp2CXO/uzfvLCrh+q5sM/4Il4CxdftmUfYMYTSZys7E8clgBxWfMrUfNTq + yrpASLvCk45X+tEhGrLyznQMLHvhmKivwTdw/swPvWCYV16pp91qOhdzL/7M86e3PuPie/2wogd+ + RfnX8H2tt+L7W9/4iHMjz1gHAph3v0JWnlm3HAWP/sEz06K1xk12gccwLtW59VNosq5mZFhmHodV + xukx1ML1N6y0q/UE3HE2O/mkPOpgP0POgF0Hl6ny3FQP0WVFHH/h5ceM9KOAVLnkHX6zDopna/X0 + w0947/dkLPflyFjv2+tH3/W0abXzptVq6xAXZ/kZLT9zZwlrjGJT70Vt44aIzcMmRVGhn9mFzCKX + v+FIbMMN7Y0LIui5ePRv3rle5k0vmOIoO3sQT/lXNwRvUHBQ3H7QbOEZL68WR8kRb+UtWNRh4Wfw + mdDyMt8exzR9Hd6mXvqEC2Y/piDf9+Htulu2p2+/4bbpv92Cfx6Buz4XaJPPrk8cRq038KrLpjH7 + oY0z+uDdxcP5rCsNQlZfpMh5eY6B7UM/+NAYsnjkxzRclc08MiBp6M2jibwteegv6iA7+XccPf7K + j/YCxpBy5NHjN854zuNp+9g0ffb05668wdb3zf1+eMNa/2sskg4rpsDk2HyzMea9WFFK4CTvgQfB + rCLiy2ZofhJnf/ZvXoDqgfOWicumoj55BQjZZswj7BiJ4qxBTUWA7QWQA/MD15qAWl1ZFwhpV/hM + N/3Q3oEATT8pGlj2whHoa/ANnPOFXjATEcfvT73+qRdPr/qci+7Xw4qRfvZFh6ZfedrF07990vnT + I85FmzLBfoWsPLNuOQJXeeJZZcpxk13gMYyLuOBLeyrlT6hRP3oYh0zGmXqAQeB4su+TR4rQaxCP + /cgJbuQZfWQ/Qy6U4iVv8LT4ial6iG5kNObTjxnVn4TJf/IOv1kH49hB06HD6+lf2Pq+uyuk+4z+ + R/HT7KvpWvHjgdXME382AsBAXLIYs1nCjs3j1dgccr0pcHHJt8Dv8lcOzTfgtHecQeTIBDDvXA81 + r9JnHuGA87osO07mu9QHKvExFr5o8IAElR9NhIu6SjSw7BZ+an7GP8/3ksOr6Xsef8H0Fx513oQf + WH/Ar9u319MPfeSu6Yc+fMd0J3+Ya3nF5qz+aJuV4c/6qssb+nFGvRcP57OuNAhZfZEi5+U5Bi4b + /eFDY8jikR/TOF7igpY0mYH8UBo8UudtyUN/WOc6XCiTTTz2RFPHlfOc4GyTY8K4wQfDBY5/SXr1 + F7Zf8MRXSXEf3Bj1fXPxVx1Pd34ABXvoLgfcLEx2MdZmgkFviSx6H8u+kwdf8XR+4mKNhj4dRRlm + +lxshlkKBSZa0uEh0yC986kB7qDHh+0FwC14CZdkHmpt3/Tya73qEWEKJzgmmqLHVXyJ04RvI38T + pnwILzJ/8VHnT9/9uAumy07hxxSai/vk8aN37Ezf8/47ptf9AX5WMQqvTaR1yPXCCO+z/ulyLlgf + N0Xb+cNeMM63qzYx1xXzlMfKVpiaz8MqgMbJT+Bor8j1gBsv89mP9cVjgFHg0foDXnHkBFmo14hb + PfT53X6CuIbiFU/WOfJV3vpHWT+xfdH02OnrrrpPfrXyffcl4erOl2IjjMOKxcOlzaviQs4x5iN1 + LxGb8Dj43LTk5CVeWOTmSz+Js2yceWFUD5y3bHvzUC87PwhvfcxLT++41Bw1oCdMaHsBcGvxSZ/8 + 1ONa1INTwx8lXOmH9hU/eVOk38GrOKKZct5mA8dN8oxLD09veMal0z+56sJ9eVgx9kfhr/q88skX + TK9/ykXTlfHjFKpv1i3HyF954nk2aqGjPgs8fdSlgvb1sma+nlln4PDR6wqNDOQOCq+j+Qg0jx4U + YK1ftzMD7mGHJ/sZsiCcD4Li0cPA2T9xpFOkMh3zUlhPPsZBWIunf/J1/IkTUHz4OcuHHr5leqmE + ++Dmqt7bxPxG+7Ttb7Rv4o5iqjosXitulnI2Qq/PEG1U0WnXryVPyG0VojmyuWAsR8HDQeFQ78V2 + eKUIfZoxLoevMOQv0sHE7uYyTzVJuU//GEFY+shvyPJSDtU0Lf5wL/uKh3EET1jP+B+BQ+D7rrxo + eu5n4P/MnUYXvzL88d+/c3rp+2+fPoXv9s76A3moLJvGXLA+bso79PVmkhjOt8v66BfMU+bKOwLK + Xq7CaUJA40KuaUVe5vHAPkNfSOJoGWJd0rt9jBNhTABV9hWe8xjzVpRccSQusiJv58tKt3mot4+t + VvfJN+DvozcsfKOd/1ewX23zcRW92WIErjYlnlU64HPchIdBZzcfLIqH+ly04Cfh0Jcj8Yiu9Bmf + 8YpIfN2e/JQjjNJH/NFeytMe5LD8S5/8wZF1gZh2hV/6ob0LFLwpGlj2whHoK/nOx9d/L378hdNv + ffGDT7vDipnwr/m8ED/+8LYveND0rY/CX/PhxKJ+Kg+wszEWTPVZ4F2huGs9Wz/VtOuLFdKM6wwc + PrwcSz1gULjux+urjDN5w6H8hB2mdn8SNM750E/wtPiJsH/rtS/E2+fpl36Cj3omVDiKLY4AMh7z + hZ3nDx3eWd8n34CP8OzsXrnnN9rbYbGLN4rp6mo1lXQVFQZa/ByB92eQMapIUbTiD97i6X4IYrbV + PLl4nI8yzPReHKdRirCP+GA3S1P+TOf42cRjkTOAis/hqJnwGIbhV6LjKnyEmfFy3oVKXIoBjLyE + kwPfnvvw86eXXXnhdMX5888pDXLaPfKv+fwN/jWfTx3NZd485oL1cVO2oR9vHAHifLusj3XGPOVc + Z8KGG6wrPqTn8iROgMBxWguqB9x4mW/E0XgMMAo8agfATc8HPMkPB9qFWA99frefIK6heMUTfBHf + bj/IZDXhG/BPepUI7qUbY7j3rpe/56HT4TvejQ0yvneV7FE8bT4V14snGRgVo4/Aq8hZ9D4mZxvN + E4tJngV+5ld63LRGLsGAR/NpogFDLj+lB4ZX6YM2M+K8Lmdoe+YbGZc+UCnHWPiiwYN6MSaEY7My + 7sg7Rrfg8HMVflTghz/7kukZl927P6XuyPfH/T/hr/m8+L23TR+6AxsmNynrgT+uRxuXfQh5dlFm + XZc8wiUjic2sT04piij4rA6e7GualUIBUjRrO1yKBxrhGX/0eciC8BYExYMH9UXgKg/JI/75PHnM + 6PhErAnjHH9M1Lwe4pZ8+Gs7n9i+c/XY6X+5974Bf+9+SXjkzu9FzD6sokiVSCtaNkGNAGWSsYRD + pl00TY0kXfC7uK25wi5x0mvNWzNozbw6oit9LErIvZsqTvG3MJos2tgejksB49bii64YeqpHM+T8 + 8EcOXOmH9hU/7VLMfGIEjr9N4Z981sXTG5/50DP6sGJ5vvIhR6Y3Xf2g6e8/7rzpYvxMBqvg9dhQ + nw31JocuFbSvV067rliIgNnDOKyWesCij/obltdXitBnnMlrf/bT+gJ+iychGMU36EJudshHdVC4 + epL16K/II8JnPjKoPCnO+UhgHPl8dT7MPnTrnJ179Rvww1N6PNXxJL/Rnif+bITPLGGN+ZkNo3dj + GzfE6DeMWEzydbuQVUzySsatHohPN9SDRxMNGHL5KT0wvEoftJkR53VxNK/c7tIHKvEx2p/pTcNA + Iz9OCNd4mx0f/9KjL5z+zhP5f/7u3c9NjnZ/3//grp3pe/G7t17zsTsVqOue64ORmxhF6uMsow2H + llaR87l+NAh5HFrJQhyugHszpz/7F4/8mMZdgrhoFuaNIOioN48m8rbkgZz5EWL/Hnv883kCTcj5 + eNLQ44+JmjfO984Hhu3tra177Rvw92IX7+gb7ZVkJRuphCx9NAk3W+IrScCzKXIsXMO31ZQD80Qz + YCb9JM5yX7RyFPaWHUfEhQCSVzwhj7iol7kfpM9WoGLkB0DJso+uMH9wtPxy3vFs8EN7BxK8KTqg + Z1x2RG9U/+eTLz4rDytW9PJztqYfxm+U+LXPv2S6Gj+2cR6+Md/rWn0FbM7TThcXNtbD65XTrq/X + M+3Aiw8vx1IPOyi8jsfrq1i/6gv7s58Wt/wMuVCKl35aHyz6abTLyGj0F+Mmrxn95sRnT5h3+M16 + GUc+X3M+zK2nQ4e2t++1b8APT+nxVMZXvPO5q63p512tPShRPOrzxJ+N8EkrlyzGbJawY/Mcj198 + uZjkW+B3+SuHfCA+6bEo5NFEKBhZyOWn9DJv+swjHBCny7LtAc+MSx+olGMsfNHgQeHEhHBRV1D8 + 0QsOTy97Ev6CMr6xfnDNK/CxO3emv48fg3gtfuupVmNDP84stEu9SYWHMu3iyXDioBmHVrJwHpfV + aF8easEHIm96PYhO7gQnjnoa8yqCkAaPJvIWBMWDB/VPEM0PE3kwH3F4cjx8MGHJMWF7xx+GMVSg + Jc/5WJlDz9v+xitfZ+ZTv9/zN6w1/l/A1vrvKjlsnkoyilShtaJx8y/xLkYtjfSV9AY8AEXNB/uN + ZkiZm3nm1zjzAlQPARO8NUPIvZsqTvJKT++4mizaaDLHJQBuLb7WBNTqyjwhpN3wF5j0Q3s5Yh1W + 0/l4e/h7V148ve2ahx0cVlGq5cC/k/jyz7oQP3h68XTlBWj9DfUuG9W5r5c1uS5YIU1YBg4fXg7P + Dz1g6pMT9VUsZ/FmJF7f9Gs/jisRHKWXn+Bp8ad+tIueZE47z6cfTSsfKVo8edgmn0bquc/imvMx + LiqgX2+/ODH3ZByeTpXl2nc8G78F5/UKOoq0kSoOjzzxZyMMVLQ+RjMR58OgjRsciA/FU1HJ0+1C + VjHJKxm3enDNHX40n+wbMOTyU3pgeJU+aDMjzutyhrZnvpb7YhOmuP1gK9i3MJsf8xL/9fjrNC99 + 0iXTw/G7qg6uk6sA/nri9GO/d8f0vR/E75fH97q0Lt5dgyD6uTYhNFo14WL9iA55HFpJwZXDpQUk + bBxaapfk4Qi6GADPQ8TmjSDoBk8irFjw0B/7R37Sv8fIJMzSX4sXmrSz/7QffAp4hhOd7EadBMBt + NeHne581feOTfs2oU7vf8zesnenF3HRKLkfGEkWqsFrRNuG9mC5NJpvjJvyS38V1HN0ucY4vix7h + GagQFR5kx5H5WO7dVHHSQPjIsMmijS51XMSwGVp8kpM/OFr90m74C0z6gf0fu/TI9Otf/NDpFX/8 + soPDKspzsgP/Ujf/cvfbnvGg6ZuvOA+/7jk2axKozn29rMh18Xrm+gGHD6077XR5lKg+aYdMyNVA + kiWJh+ZFo7iiHzkvP0OWK+Hhr/O0+ImZ9RH6jEzzecrk1bT8KKHCUTv8Zh0YD/dnXsNP8lMDO3wc + Wq9fkrhTHYenU2H4t++6Bv989X9Wlgw6irSRKvR54s9GGGQJawReyXfe4/CLD0U5G96wHoZvJr/s + yZdOf+6KC1S3jfU+mLxbFXjvrfz9W7dOv3Ej3gPyapu++hI6b9ac0QRu3pSajU2fhwL3NBfKmzn7 + uvHID+XcB3n4wU5XEYTUDr9AWGGC4sFD7jPq54fJiH8+T6DYIk9ZaqLHHxM1r4e47eZz/JzfXq3u + 0VvWPXvD2tp5MYM4mTcgrQYS2gtfSQLjRc9FOzl+8bJpuEjNz9xvX7RyhAfOW7a9ebLJotti6Pxh + lwRqyog/utRxCSAHFV90xdBTHX7xmPOFR3z86yd/4wkXT9d9xcOnFxwcVizqvXY9EX+Z+j889ZLp + 1fm7v1pfj6091gVP8u11wrrho/rOmtBjUF9Qf7y+6n3T+kp+Wl/Iz5DlBDfF0fuvxU9M7yO/EY34 + R9zkNaPfnGSpCdFl/MFnLQz4UhHX8JP8VICXeQB26B5+L2t4So8nO77yHU9BDG/FCe4s+5vQJo7Q + 54k/G5WSW0DFoyxaLHLnjUXYTO+inKlvWF/7iPOnf4i3qsdeuPnf/NtUk4O5U6sAf+fWD+N3b/3g + B2+f8APzY7ODTv2pXZ2dismQx6GVfr1puVdp6M3M7cK+thyKPmhz20/nkYEm7Mc8ibDCh4K2CSbs + Z+Dmh4k8hNnisM2wlRchnujxh2HZ6yFucz+YzPpgxP+iW++st54+fdNVb+02J/t86m9Y+NVESBm/ + qp1FiqLkSO+VbIQS8l74ShJwL1YrYvLmuIFfvLAsnogr47DfWMQMz44UoMKDbPvMx3Lvpjl/S1P+ + LIs2ulR+7QH3Fl9rAql5a/ml3RPwmf8/PvNh008//aEHh1UV6r59OBdvsn+Tv7/+Cy+dnodfD+31 + tM9cF6y0JixjXfHhdordXnrATqqvBBMPiWO78EkE6Xevw0p6+Qme6P+yg+z4SKcnPNBPzqcfTTsO + wloe9YYYdtbCTnyUlnyUOev6MAAcGPj1w9t/h7Oncimku23It6tp9VY6V7BRHEXXgp/xcp7Fwagi + 9RFABuKSxdj0xZt+ZsQWxAuGM+UN65JzDk0veRK+IfzYi/fFb/3cUPKzZupNNx6dvv3dt07vxve5 + 1KfahdmxKEPI49DK0mi3VmP7cMj+p1l0vPraNGbNQ6TzQCM86aLPQ06U90njgT73GzHLw4lMu+cJ + 1HTEJ4QmevwxUfN6iNvcDyYVp/NmAXiYnWQAAEAASURBVKHnW9ZV0wuf9N5udzLPp/aGtbP+Tnhl + bZ1UHC48vBisrhwtRdB74ytJ4LVoLDaexZe8OZJzwW+/0QzNLnHmMZ95AaqHoIPsOGbFNZD+Sm+c + Zah5lT4PX9ah1UNd0OJrzSJ73pAf/2/VX37cxdO7v/IR01/BuB9+RXHFd5Y+PAP/N5a/3PCf4pcb + 4gfm43KfV99h3dxO0f+1voBX3xyvr3rfRD/K07yP9jqsRn8Hj/px9J/7OnjRZ3gyO3AjbuI1DS0e + qCgcxTmftcQJKMPhJ/k5Dbvksz/8nPn2d8ngbt6Gp5M1vPa3nzBt3/Xbq62tw96koIji1LiJi0kx + WIxKqo/AMxCmWGPTF2/62cB/JrxhPfOh507/4ikPna68+MiGDA+m9kMFbsQ/oPh9H7htesVH7pi2 + 2Y+8NHpTqn9j2h1NPf6o/Xk4ZP/TrBShz/7PQ4TkvBpO0uCROm/aH+QNHjzkfiNkfpjkTlvOE2hC + xydLTdje8cdEzeshbnM/mIw8xRcFwgF2bGd96Ml39y3r7r9hbR/9bhThcCbjICKJOGQii55DBO3i + 8ESe2SmnWCQ8O6dcNFZ/N95FGC4cj3FpT7vE2Z/9R83SkUiipoormyqbTMDoAvIM/qL3Q7iTPprM + cdGFm67soysoPxp/nea1X3D59Ctf+vCDw2os6b58uhS/6/4f4XeJvRG/OPCL8Lrl9UXf4aP6QpF7 + 1598X0U7Vl9k+qNvOGM/sR8Swvnqz2zrtm9C7/ggcF+Unx43eaGSlvZ+0l3i8Ou8HY/5iHIcvQ7m + c32iQAGcDm9NO3/dwsnfFdJJw1/+2w+fDt31uwjwME9uZbdp3EQYuDzxZyPwSrKPwLMoG/1s4Bef + mobF4Zq0+EJOPusxWQ9eQ605/ZJH9jI0MOTyU3qoeZU+aDMjzutyhrZnvquJ/8PvxU+6bPq2J1wy + 8Ru9B9fpV4Ff+oM7p+96z63TR27Hv5/IvkEKuendCZzAHyjUf3hwH1oORR/24BGBCrTXoSXHAXMc + 7mP642X/za8CW84TKLjw8aShxx8TNW+c73M/mJN/590LtLPeuX197Mjj8fuyPtbtj/d8996wDh39 + W9iY+EegMukYsSlVlBzpcaya/beicXMv8ZUk0Mmf4yb8kl982Qxyj6rzsJj5dbzmLUeKL2qquO7r + NywW/QWPuWi67tlXTN9x5YMODiutwOl5+5rLz53e8kWXTd/9+POnC/j7t6rv85BAXmpDHmbZ97Fv + rNh1WLESRROnXfLudVhJH+3u/nb/lx0IR9/rSQWnvvCKT9M6NKWoQ41hjvyKl3rus7jmfJkH7IQD + KBIjbmvaOn9ra/s70/ZkxuHpROiXv/XIdPjIR2FwOZdi9gbDIPJwaMHPKEOvNwwWCbKS40g+/JmN + Ta8kw74Xp/P7zSWagnwL/C5/5dAlGHAX1/YicmQCmFdNE7IXNXBZBoqZEXG67PApl+H7VE996MTx + 4DqzKvAx/DNkf/e9t07//vfviMTY0biisb2Zs++5d0uhDaBtJHgeIrLuBJrY69DyPvGZoG5r+4yG + 9t/8KrDlfLgLPO2A8J18+HDcnB7zAsRt7idxYefAhBTPev3fd+6884rpLz/taOfY6/nk37AOHfqT + 8OXDSjG0YLEp5TxHeotkynFPLnE5Bp+3tNZOfFl0HlIn4pc+itntMg7b98WBUwMVosKDTFwtSsi9 + m6w3jvaVJh9CxoAlZn3G4n4G/nWaa59++fSGr3jkwWGlip95N/4LRD/6uRdP/+/Vl+JfzfbWOvm+ + inaswyHrM++jvQ6r0d/Bo34c/Tf6lm3pDqWHMZ9+7Ff9S1iLp/ZF2FnLvheQ4oKPMmcRB3nkVhOB + 48Tqj0znXPAniTqZ6+QPrGn1DXQlnxwjyHyTmY303JKgmPIMh2w6z5JfMnkSl2PjE7foHRn5ut3c + r+O2Hkb1EOFBtn3EFbKAiiP1HrUGdMur9EEblToP/zrN38L3qa7/E4+env9HL9KsDQ7uZ2oFnoYf + g3jjFz1k+iH8WuoH4+99uk+8ad3/0T/ZQOqz3jduJ9dn9DVlHRqwy32TNRy8wdP3De1qX0DQKeLG + HfPpx4z0w/DoUXeJw2/6N07AwLX9R2uZwy75PBHxkA//AtK0/Q0yPombozkR8F9ef9F0ztHfX22t + LmJoNJqNLEYcJjmqKBHcLnrOs2iLsYp3qvzBVzydn0FE4EOfjqIMM70Xx2GWQomLlnR4yDRI73xq + mJ73qIum7/+8B0+PufDgxxRUn7PwdhN+DOIfvv/W6Uc+fCv/GXftE/dh7CA1UGwH1Mdv5nqIahlX + b0I8rPBBuV/ed40nGzNwZR9u1bh0A726W7jYx5zXDhdAbkQXeE6UnXBg2OXH8ZXdLr6RBzA3r3cO + P/Jk/rGKk3vDOm/7z+w6rLhrcXHTMtjZaIX0dTsBvg4RGKimPAzwPONNPyQNPj7yEg4WxRNxJc48 + xpkXRvUQdJBtn/kMXvGUPuYly70JQr7qkiPTr1zziOmnv/CPHBxWUZ6zdXgQfwziqoum38Qb1xde + dlj9xb7Lfh19Fe2o48Pt5JrpOAk85nlYRZ/3mo7+zrZu+xJA93Xw8hQpP22fiTe90p7P9G+77tfx + U2s/AgnX+WxHouO9YWGrXjxtbf/p5DjeeHIH1rR+0ThRnQJlXhp5uFDO0Qrp63YC/CZ+lmrGexx+ + 4VCW4ol4MKEQzGM+82K6Hjhv2faZT/jPRdHasPgZV9jRAwgefO6h6Yfwg59v+6pHT19y+cGvKGZZ + Di5X4IkXHZ5ef/WDp9f88UumKy44pD5VA6pP3UfqK58S2bYwdmO6vyl586ec9R39nW0934/ua/vR + IVl+Wj9r/5hRb1gMqHDkjX3B2dxX1GNf5jX8MO7wRzvhakL25hMMf8dw56S+LByebLf7/mPvuHw6 + On0cwJVL5xR8okewKjoQOe5mcVLQ66RfjsAzkCW/kmcxkjfHDfz+DBKLSb5uF3LyyU855EOF5/gQ + ie1DoWIzDvPWZ5oMOAg+9DV/dHr4+fV3Nzh7Vl//7HfvnF7z8aPT33z0OdPXf8aZ+28h3t1F/oWP + 3zF909tvik2f/R1tDjIfSpbNHY3G/i/9ODyMoYINWoP4c7/JDnp2uw8bPcl0eciwz3kZp6eS+6El + fzOcYPbb5xX3eAnodo5Hvo6uj01XTN/8Of/dLJvvJ37DunPnz4FUv5UhU7STKJ6KlMG0IirI5jRk + FSHflHKM5Jb85SdxOZJ2wZ+LMCs+8ImzXy+CecFRDwETPA495WV874I5f9Hr4ZIjB7+imEvznz55 + bHrKb948vfSDd04fwL/I/Fffc8f0rLfePL395m2qz/prHCLRn2PQYcUCjfbmPhv7yofZkLOYo7+z + rXl4DdzoW9Kp8WU65tOPGeuNKE4vb4c5H5HGkc/XnC/zgB155JZ+OJ/7zHZQ4cemVs+3tPf9xAfW + 1upFSoXJgydT1RsI5TgUZiP9qSh8iCvkGY5Bt/klf/rl6s3sSLngN49x3S5xtne81oOjHoIOMnFq + CvKHHA8l2yz18/xCOiuHD+GnvZ/3jlun51932/Rh/iIpXLlu77x1PX3F226ZvvWG26eP41+vOduv + 7Ff1p/os2lE7LPpRRfKOMx7z+Kj+bEV0nW03+nPD/hKdEGZXv8c6gTm3Ff2w/+lRd4lzPmupEDBw + cU4EkQfYJV/NZx4yU+B4K3pRSHsOxz+wrr3+jyHoz2PIdXLiOWWycp5ZzkYreB9XJDXDpR1Qm/jL + T+JyJGsrkkUvQvFEXImz3+bH8OIRnWrv4na8MhZft2feZd4eGM3Zdd2Cf9HhJfgHS5/xllumX7/R + b1GqH8rgOro/WPKfwT+x9bQ33zL9U/yCvLP53HJ9ooHGoMOI3TPae/S15nmo4CPryzlerrPtQDer + e+o9Dwn7CAhOC1d48WracVBROLb78Jv+7603LATCsD5vuvadVzmCzffjH1jbR7+BJ7dKlmOkUCc+ + k5ezNtIX5/sVsj8TMLg5vvzAZuav49IPeRf8jse8ac+4Emd/FDOfckQ2wwSnPuMzXhEpjm7P+MOu + CER11txY55/GAfRUfPn3zz9y14T/g1+X1yPq19aN87fhgPtefLn49DffPP2H/3FSP+BcvGfKg+sT + DTQGHQrMcbQ391H0I+d5aDSZWF7i6zzq12YHmcsjXu4L8PCinefTj6YdBxWFYxRzPmu5DwSU4Zwv + /NGOPHbUcOSTmED8TNbquN983/vA8i/n+7M8SZVKjuBPma58siNpBs3kM/gcI55MaoZreM6Lt/GX + n8TlaMfJrNF+7b/bzf06XuvLUdhbdhyZT+SnYjO/bp+yzKmIh7NjePvNx6Yvw/el/iq+xPsEz5xF + /l6PqF9bt77OH8VfZfnG62+bvua3bpn4j0CcTVf2q+qmvop9pV3dy8m+GvtKh0aTs2bi6zxcj011 + F512gExrPbR+9GNG+mG/I5LAUWxxBNA4AQM3zgtZyxx2yZd2GM0ns0p4Z71+AYIYhKHOYe8D6xXX + fSkMH9tPTPrOVHWiU0ZRmOVsJDvn+xXyDJd2gjOpOb/kk+R3PI6j22Uc9ut4rYezeohwIRPH4na8 + gIoj9R5pH2m1h570mff8P/Bv+H3bDbdNX/62W6d33Ix/z48psgmrEM7Z65F1inpC5fpipBll/HnD + TdvTM996y/Sd77tj+sP+mgbdmXq5PlG3MaAe3jejnK5U1RP66s9WHOk7j/p1Q91Fl5Vv6yGHxJu0 + 3ohaPN1vj0frH7FwXuxB5AG85LFCSOOGv3S8tVo9fnrVO69uqc0e9z6wVseeS5J+Art00WxsUlw+ + 2Y1LfCg01O0E+PIDA9eUSZ48v+KARfHkJpr5NZ95y5FCFAwK22c+4V/FRr6lz7g4yrw9hHyGDTxH + /tXv3jE95Y2fnn7i9++KdUfazJNdWYVw4l4PTrNuUU+oan1oRjlG/Da36RW/d+f0tN/8tEb+Q6dn + 8uX6RN3GgHo48VFOV6jqCb3fTOYFcp29DKrrXnUXXVa+rYcccp1cdcVBWIun++3xaP1jsThf/mkt + PvCSxwohjRv+0nHkv+eXhcc5sFbXkKSfmMo1UqgTNppVcuAVEef7FfIM1/DlBzbywybH8174XiS6 + EQ4WxRNxJc48xpm3HNHcMChsbx7VWHFLocDm/GFXBKI6427/5VNHpy94003Td//O7dOt+T/4VF/k + z2zZlarTSF31hui6Rz1DVv1pRjnHsOc/C/id7719+qK33Dy9of8bgcCdSZfr0/sq6+F9M8rpClU9 + UTFu6pSzJq6zl0F11foMHPWehwXXyysX/R7rJF4z1htR4RjfnI9I48jna/jpecCOPBWA/ZmvDPUg + 3A7Onj2uzQfWtb91Kcg/l03YT0yXzqnWCRvNKjnw8qWiNK8hz3ANX35gIj/A55hx1EjaBb/jmcdL + fOLsl2LyliOyGSa4i9vxioj+Sm8eyzIvPyGdEcMHb9+env+OW6bn4ntM78fPU9V6MLusB59bnSny + 8npknbwuOS8eCLMx1tN1X+l7Wl/z9lunb7ru1unD+HGJM+1yfdRQvb10KLhOmbEPl6onDw18pFyo + XA+3qfWb9lcdGnzw+ox1Ja8Z9aZDhU4Z4hjm8Jv+jRNQhpwffLajZfGFA+OGv3QcuM+ZXnH9gx3J + /L75wNqersFJKbZ+YirXSIHzvDQyyGjanFcTCxG3E+Bpt+SX3HnTjx13dvvnYiZP2GUcjs/xmhfm + 9cB5y7bPfCI/AsXX7WEQ0wpEBLOQTluB/xfvH7z/9unq37hp+r8/cZcKM6srM8t68JlNuMhf9RaM + dYp6hqyy04xyjmHvdcr6r6f/8Ilj0zPe/Gn9X0XGdaZcrk/UbQyoByvSy+kKVT1j86ec9XDdalmw + HHvUXXRZeeJj36n+rLsZFYfK7QnRYcXSb42Ml+sf15wv84CdcACFA+OGv5o3cms6tP0lydnHzQfW + tHWNTlCQ9xNTudIn/kifY+DoNOfxAG27Qu68HV9+YJL8ORbuOPz2O49XRZj5ddzmLUcKUjAoHEfk + EbIiIqD0kb/kyDH8hHTaDq/9/Tunp/zGjdMP4h9a4PetnNairswu68FnrMtyvb0eWafRF7vWmeai + c7/ILtc5Rv681g9+6PbpqW+6WT/HtegsRnDaXa5P1G0MOBKyDpkS5VY/HhpNLlSuB+BVz6wjQFV3 + 0Qkh0zGffsxYb0Qtnu7X8YOXeq5/XHO+bAvELxxAsU+MY15lqIeGuyY0s2GPA2vnGp2gYOsnplNi + kIzRQXYcvec8HmaOUt4Lz/klf/lJ3hzJvOC333m8qsYsTsdtXnDUQ9BBdhyRR8gCkqf0xlmONBfx + xOxpM7wLP6bw5b954/Qt190yfRw/buB6Rl2QaK1P5pn1YIZYl83rkXUafVE8NMMfdonG4JXfXOcc + gSH/f8f/ofzWd982fRm+v8UfqzidL9c36jYGHUaRbqTnChnPennzp5w1cN1UplHPVj/qXWdYcL1U + eeJzPv2Ysd6ICsd1mq8jkcaRz9ecz/HQsvjaOpuvDPVQuGnz97F2H1j8/tVq63N1gjIZJg2qGvGc + Mj10HJtKshW8j0tF2hu/ib/8JG+OZA2+dGC/83i1iWZ+Wx65ZqUHExw6jsgj5FA0febRwljEk3Ht + 9/ETOAT+99++ZfriN944ve3TPARQmMo781vUlUkxX+L4jHXZvB6cJm70xa51prnoxDTHN7vO/45b + tqcvx6H113B4/QHiPx2v7FflFeVTHVzRli7r0uoXm9/2I3PXuZZlXkfAqu6isydaj/n0Y85602nx + 8DBJvzVSz/WPa87neBS/cADVfvMhVtsm5ws3bfw+1u4Da/vQNSDFQYkgQNJPTKeEafiVPsfAJV6x + tyS63Hk7vvwAnPw5Fi79kHDB73jm8dIucfbruM1bjshmmOBYFHx0vCJSPbo961P07UF0+/52DLH/ + 6w/fNj3lv/7h9KqP3o74ETILw4eogyTORz2odp31YJzUMuBTXYlzHaOe0FIWD55nY6znDH+c9UaH + Tj+Fn7J/Cr5M/CH8NZ/T7dxyfaJuY0D1VXD3o6rphTGeq9P6s6od69J51K8b6i66rHxbD9WfeJO2 + Nx1HgfnaF5jp8WifRSycF3sQeQAv87Ii+DKPMvR84PAvreJs2v19rN0H1rR9DaPWCRqjc6TT1mQ8 + DCjHoTAbrZC+bifA18kMg5m/k+SXf1gWT9hlMR2f4zV/OVKICg8K22f+kZ+KiHxLn3lzjAzrIeR9 + PPzXT901Xf1fPzm9+IZbp5uO8oc/mS8CZmHYVZGnJOW3qGsqiOMzu3KRv9eD0+SLegLq+tqO7miv + Mexn+Ga35E/51mM709/H32O8+jdvml6vH7kH4WlwuT5RtzGgHqpopodMXCHjKXnzp5ypum7Qs9y0 + 2qvuohNCpsQVHk+xDPIjhfwn73wdSaB4uf5xzflsx4iMo0Hml3mUoR4GDuLW+prQ1rD7wFqNn78i + eT8xXToGyR50kBoDl3ixtyS6vBe+/ACc/DkWb/oh4YLf8czjVXFmcTpu85YjspkOCsdhHgaSvAKU + PuYlyzwI4nmfDr+LH1P487910/TVb75x+p3b/KXUyBdBIx8kotF5Z1qLugoWOD6zCTeuB6eJi3qS + HTLdsHtm42ydsv7DbsmfcvLzt0O84J34EYy3nx5/zcf1jbqNAXVhZXo5XSnjWTdv/pQFFp51tp3q + ulfdRSeETMlTeDzFMjgOKlo8Oiz7OklLvwJu4Ms8wEseO2q44S8dDxxg6xMdWIufvyJJPzGVq4Ik + l4PUGLjEK6KWRJf3wpefxu/aws9J8DueebwqwixOx23ecjTCY02BV1PQLuR4KHnERbzM20PI+2i4 + HT8O8LL33To99dc/Of0ifnlcxY8YR74QqFDXe16S8lvUNRWsD5/ZxFUITnR71tH2OS//EGZj2Gsd + E5+jDXkf1x54/pDrF+HHIF78vtunG/l17z69sl9Vtyif6+GYRzkpt/rxUGlypue6eRnEQ4JWP69z + LBPm8STTMZ9+zFhvOoVjFC2OrD/14gs7zJd/epEb2AlXE/O+o+mMjzInt3Z9H2v+hrX4+SuS9BPY + KZkrT3iNgUs8XfUkurwXvvwALD8stmiQ/knwO555vCpCFNN+GVbylqMRHhxabx7VWPZSKLBhz7ha + muFHZPvo9rP4N/Ke8l8+Mf2j37kFv85lUR/EOfKFwIKz3aIOkth9mJjlnQripO6F4IR5a8z1i3m6 + od1snK1T1j9GE/E+ruPg+eNaP/JR/HjGG2+arsVf99mPP77Fekah+4C6qKKod6ZqnPGsmzd/yoWi + QSyD6ip51G+sH91m5b1Ohdc6m1FxUNHiub/fsFar9a7vY80PrGm6xic1isKkkHQ/gV06pyC9ch+4 + xCtlFUVPvoXceTu+/AAtP8DnWLiIZxO/45nHSzsvDgfG6dG85WjQlT7zDzsbLuyTT+blJ6QHfHj3 + zUen/+mNn5xe9Fs3Tr/XfvFU1bmthz9zImQWRl2feWdai7oKFvnzudWZIi+vR4xt3co/MFqHHFs8 + J7Pe83XN9Yox/N+IHyT76++5bXrmm2+a3rTP/pqP66OGi77KeqCuil8Dn/Cn5cVDpcmFqv4OHsnN + DrLqLbqsvNfH8+nHjPVGJP/EkXfOR6RxZPBV60sDXB5gR54KIP2Sz3b5MHCcl8E1gdCw68DSyQ2W + PpKzTmg8p0yGjqNTyVbwPi42Na698Jv4y0/y5mgi8eXNfu2/28GhIPbb8lAtoCo9nmHoODL/iNeK + po954SMC8dDzA3+95Q/xTfX/75PTm/7wqJosm4GRVZ0rbzfTCB+FiToYz/uirqkgTupeCE5k3bJO + oy/KPzCs1lgGMSm+6qPjrPdYN8ab67XZzw34C5DPx/e39sulTZ0F5xjlcz2yDhkt5ZYXKsZNrX5O + CEbJnUe8A1d1F11W3naSIh4N5OPKUKEVIo7inM9aKgSkqDgGn+0Uf/KFA8dDPpklcPjVPJj4PfV2 + jQOL37+aVrOfvyJbPzHJoWA4RpAaA5d48bckurwXvvwALD9swvRzEvyOZx6vqjGL03GbtxyN8KBw + HOZRjWUvhQKrODkf0zMCCQ/s7SZ878b14Hpp0SugWfyYHflCYGGE97wkLjgUSztMOH+peyE40e3t + v+KBHd2QdjaqzmF3Eust/+lnA77ibX7wuC8ubVblG3Ubg9cLUUY5+IQ/0Y+SWL8hY0qX6tt5SJB1 + AaLqIbqsfJ9PP8EXfWD/xDGK4bfWUzjyhR2AYo8EPMAu+Wo+8yhDPQwcRSU0+z7WOLCOHf5jRLCY + jK6PzpFOReGROBvswodCQ91OgNcikg9/Zv42xCPO4Et+xQvL4gk7Lhov5+PR/JisB85btn3mH3YE + iq/bwyCmyZ9+9LwPbq4HwvKiV0Sz+ihsN1PmTwvnlXWj6aKunMp6SN0LwQmq0558Uc+YV9nxPBs3 + 4ZvdrvqeAF95hh8M++pyfaJuY/B6IdJIj0/40+oHmZs665tJSe48JGj1q3qILivvdZIkh+Q14355 + w8K/h7o1rY89MfMcB9bWscdw0ic1ioJkGf04mZEM9fjjnPm0GR8KDXUjH67Om/w5v+QvPxFHx4NI + fHkTLxeTi9T8JM5+7d96gOqB85Ztn/lHvAQSoKHzhx2DWMTDqQfycj0QFlcsuxABzepTMvOFwHoI + b5wkzkOxtJMB6yF11IfPcZV/1S3qCV3x4Jnu0u1GfK47OZf1DVl2ictR8Hm/kmI/Xc436jYG1EMV + bem6QlUf6PubTubkOrhMqutedRddVr6th+oZfQDSetNp8XS/PZ7j9xcjBC95HJhCpr35JNb6DlzY + Ma6tVf2e93FgrTzpkxpkbHKSsgnoMkc8pyzKhks853sSXe68Hb+Jv/xEHB2/5BcvIiueiCtx9tvy + yDUjLsOFQ9tn/paVsfhSH/PCy5wT8bA/BtcDYbFLuOhxzeqDuZEvBMGIj/ykp+GirpzKekjdC8GJ + bm//FQ/s6IbVmo1RP+FOYr2z3nvhndfww5j20+V6RN3G4PVCoKOdXCnjmY83f8qZk+TOo/WJPhZf + 7mMI6gfy0k/Opx9NOw4uEJ50x8DDJP3WSP1x+4vWsBMOj22dzUc9rpwvHCfD32pnw4G1vdakT2ok + wSCYDJuHpjmSO2QMM1ziOQ+FhrqF3Hk7fhN/+Yk4On7JL15EVjz0xyLM/LY8GJ4dKETBBHdxHWfk + R6D4uj35PT0IKtsH/MH1QHz4UB0ioll9MGeZdYPAegjveUmch2JpV/WQuheCE93e/iseOKKbdFej + Agi7k1hvB7w3vuKFL6XFoPbR5XpE3cbg9UKcUQ4+4Y/rz/C1yZvMOV7i6zwkyDqGnnUQL/eFeG3n + +fQDlbS095PuElscuV7kER9RS77wx3iFE6DhyCcxgQ3H+fC3s+kNa2v1GEHoHCw+sT2Ss05iPKe8 + F57zPYkud97yIziTYoiDv/ws4ul8eqZdLELFGXlkHPbb8hiOBh0c2j7zH7ziKX3MS44I2qJlTA/k + 6HogTla0uiLzy7xSZr6IlgUXvus5N+8DzlQ9+Ez+Rf7lP9ahy3ST7moMe+FOYr3T3154znc/DHM/ + Xa5H1G0MXi8EOsrpChnPujGv6M+WkOtgO+W9V91Fl5UhPuokh9EH9I8PFZAjZYnDb4/n+P1Fa9gl + n/yk3+EvEx64sCN+tX4MJV7+kvC160PTzvQ4TvikRhLRhBwZco14TnkvPOd7El3uvAxSsuDhB8/J + n2PhGn7Jbx7zdbvE2W/LI9eMecq/HROnplD+UQ9G1OTB39IMHpHtg1vVlV0XTcKwnF/mlTLrBiUT + E77rObeoq2DkFdr8i/zLv+p2nHUWe/fneGV/nPWer2vwN3zlGfwMeT9drk/vK5dfbyIIdJTTC1P1 + jM2fcuYkedBpnWvfiC/3MQT1A3lddy57xpN+642I/SA945uvo+ZP2F9EwU64INKQ+4x6XOF44DhZ + /j5z4hmFywfWrW+/At/YukgQJgNjn9geVTI2gyig5qikY1zgyeOi6Mm3E+DrpAc6+XNcxiPC4EsP + jmcer4ow8+t4zVuOBh0UjiPzj/xUbNYl9TEvOSJYxJNxPVBjrQ8WPZuBsVSdW13YJBJZGOEjP+E5 + t6grp2jA/KXuheBEt7f/igd2dEO72djiOZn1dsDhZ0P/VZ7hB8O+ulyPqNsYUBdVNNNDzK5U1Q+y + 18u4TEr6zqP1iT4mS9ZddFn5Pp9+zFhvOi2e7rfHc/z+Ih/iII/cOm7HE31HSK5/4cJO86tLJp5R + uHxgbW9dlU59UsfJxiTZDADWiOeUSbAJz/nk03OT98Jv4i8/EQeTkn3jS37Pz+NVERC/4VzNlkeu + WemBKn3mH/lZ0fSZd0szeDKeB3rMOvkzFivpq+pcebuZJAo26kQLwxZ1TQXrxWc21SL/8s/5tm7l + n2b4M5ZBTF7fxOdIHwv+lOUncTkKHn2LZ6VFjn10uT5RtzGgHlmHDNYVqnpC3990CqU6u0yq6151 + F50QMq31UH25zmZ03/B5xNP99ni0/jbT+pV/WsscvOSxQkj7Hf7S8cARNvpuOjY9hjM+sNb4LjwW + m5dPahSFMrz5JPQ8fctn4nIMXOIxTQMNdTsOv+FMas4vucVxPH7FC4Yer4ow89vyGIkoRMFYUzyw + aOILWYumOFLvkQEHfXuojB/QB9cDYXHFsgtZX+XnkQGOfCGw4MJ3PecWdRWMvEKbvwpBZbe3/4on + /QNDd2MZ+BR290E/iXwf3VyPaKAxeL1UhwzWFar6AVH9mRDhWWfWL+qqh+jj0KveotOTrL3+uV7E + m7TeiLRCyTvnI/LE/UUU7MhTAdif86AeVzgeOE7aH83yRxt8YK2mx2RT6+SEcR+dI52GT46xCTqO + TnM++ehL1wnwtFvyp9/iPQ6//dp/t8s4HKfjth5R1QPnLTuOzD/zBJAADRFnyU4v/YT0gA+uB8Jm + l6BueTm/zMsjmyTzp0XmSRvNS9/yTgXrwWfyG0hJV/lXnUZflH+gGBXtNYa97HKdcyTjgj/lvfBL + P6TYT5fidqGj3lkPVTTTQ8iukPGUuA6jnpmT6+AyVT1b/aoeosvKE9/XNfpAXgEkTP6Td/jt8Ry/ + v8gBO/I4ME6E3+EvEx44ouzPYfinGOINC//bkE1HCEcm0UbnSKfhM3E5LvCYJpGGuh2H3/Dd/Ol3 + GY84F/z+DOS4u13G4Xycn/VgqYcIFzJxagryhxwPJdss9ZHhIp6YfcAG1wP5sEvaZnd+zpPBjXwh + MDHhu55zi7oKFvlLrULxqa7yrzpGP0Fb/vGsOuYY9ZPdfdBPFdg+eXB9om5j8HohxtFOqHPUn6Hr + sGoy53i5brZTXfequ+iy8rYrvHiDL/qAHs3PKObrqHnhyOCr1rfWk/OwS76az31WhnoYuLADXuwr + fNsKV7xh4fSKpvZJDRBlgtk8ANaI55RJ0HGJ5zwUGup2HH7Dww+E5M+xeCMecS74FQcsK07qWZyZ + 35YHw7ODQSe4i+u8Ij8Cxdftye/pQaCnfXFzPRAfl5t1iGtWH8xZZt0gCJZ5cSLzW9Q1Fcyfz63O + FHmV/1iHLtNNuqtxtk72V+tuQt7HdQJ85QmLkf0wf6CfXA8WEBUYg9cLwUV6fMKfqIek1p+Q8xJf + 5xFvs4PMOohX/UBeyjmffjTtOGgg/7bTYSkC21nL+AWU4ZzPdoqfPBWA7c0nswQOvwpv9N3Eb1vh + 2pr+5fX8v4OPSqebPsMpFQSlEeCUSbAJz/nk03OT98LXyQxs8ufIKs/sGl/ySw/L4slNFMW0veM1 + bzkShWBQ2D79RX6MSHypj/mYHgQZzQM/uh6Ik10STcaoZvUpmflCYGGEj/yk59yirpzKekjdC8GJ + bm//FQ/s6Cbd1agAwu4k1tsB742vPOFLaTGofXS5HlG3MXi9EGeUg0/4E/0oifUbMqZ0ia/zaH0G + ruohOlaED66fJDkkXtOOg4rCMYo5n7UwOG5/EQU78tgRJ+An85DICc8XjqL9OYzVo/ijDVvTkbse + D/ShdOqTGmQMgqRsHprmiOeURdlwied88um5yZ234zfxl5+Io+OX/OJFZMUTcSXOflseKt6Ik3DV + Snlm/saHouljHgHKruXHx/1wuR6Ij10SzcC4ZvUpmflCYMGFN04S55d1TQXzl7oXghPd3v4rHtXX + dnSXbru+1jnX3YS8jysKL7vE5QhU5YlnpTUs98WT8426jQH1UEVHX0WFqj6Q+5tOJuM6MG/nazn6 + GCDKqrcKrieZjnmvRJTVcRDW4ul+xS8tHQq4gc/xqH/IUwFkPNF3ckP/9JY4So7//2fvXYB2za6y + wPc/nU5CEjAoXoay8IIZCAm5QBDRGhFrlKmxykJriDrKCASJiKMU4zhT5eCoU17GKRwZLEdrnBKm + RDOiI1rlBS1jB+SWEGIIBJAAIUAg5Nb3TtLd55/nsp611/ue7z//6aS7z59071Pn23vt9axnrfXs + /b3f13+fPl1l3LG9/y0vwP+b4tqvoitJ/aT2k43dryczgkXhFoRTGOgK1/Pg41KjmjrFT3/nwVp5 + ePlqv3mTxwF87eF69vUybt/XyCPxEN51YS0486Z/48sx/LW/6JunC7rNiz4f3hLqUKN17r7TLwCC + EZ++09ZBV3Ixnjiuh840OTq/cKVn7TMN43bzqOdWznudG+vIeV2cB+mu1Mh9bR0tJ3SRomkPNVup + 1hN238/Rkfw6t9L1It1FF+V9TrKkP/UzaX8jGvXMvLOem98v8oGXPE6kBIw3n/Ml8cJVnHB1n+44 + /7X4Gdb1ZyukLvXuyUwwLwMAPWMdW5R1WXdxdvB1jZvwE3SKv/NUHWxKeRywuCuelTVP1YUN4Vzf + yJMzaz9g1BQ2RZv4cgy/ccZXGcVT1m2fVD+q4KXIZWBR7q/qb5v9wqDgwk8/9w66CkZeoc1/6L/z + c3+cW+cXq+P7nLGnuOAzJx/njMp3Ef6YJ2FXZVbduUC6d5LT54Uil5w+GOOp17ifoxnr4LjWc+jX + eohOCEWv/eQxqe8N19wPb70vZNc+/ciTsedzHDtrvnFufp9VZPbFxwTcP96768/GA+uaH1iV1E9q + iEIbJH4SsiYmJYW55Kc9cMFjmw5N/VL2RfhT/Mp3i/yuZ1+vTn2Xd/SxGlGJgkl7i+s6qz+JSD1m + fOzqsPJ0v7d5YT1QL09s3f51jkMXvQmih/DpmzMbOejKLTqoh9wShqsenV+4uk/wch9oxe1mJ5K/ + 71HdP5GWfySobdZR/JnJf8jTcVdkYX1KtzVBFykqeV0q7aEf/D4v49KO+CYP9Tqlh+ii/NBJ+jJP + sjKea2+IbtZRwMvvFznASx6lDV/6oB9jx0ebm+7bYdy4hgfW9XP8TaP0qTpMmBE8Z0L7yYl1bIfd + iOd++LQe9uTtPIKzKZa4+DvPoR7SHfnFC4aus/oIznlHHyvRokNCx6d/26pIfPHXvvAK50YtrsZk + PVAWFa3LwMp2+rTNfmFQcOGNk6W2DrrGwf65Jv+h/84v3UpPQDs/w2hnrnjF3cJ5J99F+GMepLlS + Q3VTAenTE/SQomkPNVsh42n5zR87TcledNL55PtLdFGeeet9J/3rHigrgISNevjQSd6e6b/p/SIH + 4oQjHQtI3pWv9xtHlPOpWsbhWXUNe35gVVI/qf1kI0l/UvESicIt+BOCtVql3axc6pYrj5vwE9B5 + sFae5LtFftezr1ci7PKOPFJBiZnemiOx60j/tlWR6oi/9oVXODdqcTUm64Gy8CuXgZW1zkMXvQmi + h/DVn/CMOujKregh9xSCGzPe+bsexOl8gdnNo57cu55NyNc1LsF3n4hgnqs2rEfptiafF4qt9rjC + 77qPsqjfsrGlIb7JQ4J6/xLQeoiOinAx95NH266DsMaxipXX9dPrPI468tGmB3HCkW7mJV9FZr9x + FYd9VUs/nlXrgTWefGTxE9szOftJzJxli5JxBzz3KdZu3ITfcDbF1hZ/57kFfj/x9/VKjV3e0cdK + pDIFQ0L3mf5tqyIC2l/7shXOjVpcjcl6oCwqysOu0ec4dPEnJwCCrT4ZYthB1zjYP9fkP/Tf+bmf + 8xPf4ZwZXvuYpH/jR9yRP7byBJe5eMSLNeerNqxP6bYmnxeKXXJS4bqP3Mcvn5eU77asg+PU90W6 + i04IxTKu8cpjSt8brp1HdLOOKvDy+0UO1E8eJ+KGzrnvnTe83ziah3unB9b29DcsSVNn6E8iHCIu + v7Tj7EXbPvP4GY0hnJdX4fXWPwF9mdKmbhXvVvXjyXp03+m37qDeXYf+Vzx1Kj0lE/Mpy37ufAM/ + 4m7Q9xI88888V+FMZg3WhwKue+R6YWNUe1zh99Cv3vzRl1gO2Yuu7BEXPUQXZRwnSwmJLz7mpUP5 + iaO557OXDgEV2LoXkSfEha/3x70TkRMvHDedb9THH7qfP/1vCSmNVOFscdcnVjt0drtPpHlW49Ao + 9e0eqh9FXP4JmH4B1p1Bv+qLfZcu2Nj1HQdxXOs2G0+To/NTF/inzTRE7+bST7jgM5uQr2tcgu96 + EcE8V21YDwpIfXryeaHYao8r/B76wX7KfsPazvDAOj97+t8S8krUu8efEH6y8674k0wL3Z3dJ0ht + I7wItLoSL66bV73eDVXVrn6VPT7p+N4QPn2nLetBd3glGPtniN50WtHSCE4z/NMWD1C7WQdQ/MFn + JmP5RT7sHf/Ac3/yd9wVWVgPCsjz6cnnhRpXu9R16AfE/KaTdqzD4BHviIseoosyxJdOSki8Gfub + jk+4ytzzEXn5/SIKceRRWidw3pUviReu4mZ95/wZVv6REIctCGeCxuwembRycr4Jvog09csl+OMn + Ytujjq6LpMUXftfjulOvVZ59uW77EdmLooPtvOnftoCqI/7aF74qONSTum7XbD1QJ29JbiGK2ena + NvuFQT2Er/7k595BV25FD7mnENyY8eM+1T7TJF3PpZ/qPtw/8rlArfxyCb77BFptjdCrsPT5lG5r + 8nmhwGqPK/yu+yjLb37HY6OGdXOc9NX5jDjY3keA7gN5ic9+8mjbdTBA+cO757MXcTe9X0Qhjjxd + QPKSj36MWiwcN52v+8Gzav3QvZL6SY0maLMZzGolM7lJdRM8U80mpj15w2945YER/syNq3omn9Z4 + cT37eiXCrk7jzNuJFh0c7jf9L1710/7al10VVJ6ybvvU58Nb0rci/aWv2OwXJVMY4aefewddBSOv + 0OY/9N/5uT/Ozfo6jumS9iR+xLlAJq5R+RQXXGZAjnkSdlVm91sXaE3Qg4qw/lRqhVof+J+y37B2 + P3THYXP4Se0nG1WbT+C+XMFlLlzw5Jlvkmmf4jccbwqG4bdmXj7arOsW+IXjYR7iUod5zGfeToRF + lQuH49N/5WclqiP+2q/tRaDVlXixHqhTTxV27LHTB1urXxiCUe/qT37GHXTlVvSQewrBjRk/zq/2 + mabPGWvZ5OO+eKN/zXbI3y+X4LtPBJD/qg31aaFL79JBylhe12yljKdufF8MXaox6+a41jPvG2Ba + D9HlBOZ+8iQrbAnH/fCuvLOe8XQdeVYcidY3p+ynD+fTfWIe/FJewZyv+9E3LPxzoULQHMepTyzG + Hj+x+onPOFyeXZyJ+LrGTfgJOsWfvLfC73pcx4wDsWpwfSOPVFDi8mNCoOtIP7bLMfy1L7zCuVGL + qzFZD5TF0683Nytzf1V/2+wXBoUTfvq5d9BVMPIKbf5D/52f+7kfZIfNNEnXc8UrLvjMycc54xL8 + MU/CrspsfeoCrcnnhSKrPa7wu+6jLL/5HY+NGtbNcdKXBEO/1kN0OQHi6zyUsO4BOPtnScof3lFH + FXj5/WKBiCOPC1PFzrvypeGFq7h9fU//W8K8mXG2PCWYFnd9YrVj+I0zXvpzoxZXY7r1T8D06/51 + q0oHduK2bviks6PuoC7bof/Oz31euvJzRtiND63hb/yIu0HfS/DHPOzlKg3rIaHrXmmCLr5H1R5K + 9v1r/WDzTR07PcledPYP/eiX7qLLCfAYs588ZuxvOqOemTf5jSNfxe34fE3YWfNVY87LPjpQi4Wj + 6T5dN4D4F4T4ofvT/5ZQ0tQZric/NqAR7Vq0raPlvvyMxhDOy6vw6rp55KyTFXu4v/TlmZckbTJi + 9U0/4+jnpVtxcrB/uacQ3Fg41cFLXPo0DzDiyzz8rPcYp3wkzrgEf8yTsKsyW4/SbU0+LxRZ7XGF + 30M/2D4vKd/tWC/HSVcSREfx5fxIF+V9To1XHlP2Nx2fsOqZeV0/q3OeFNK69/nQg/qFw7L300dF + Zr9xFYf9VZ//LaEj1AR7gZugMUsy2JqZE7/7CTtwHadcTDMGcRiTd+K5f+TvPId6ikhTXlyP655x + SFhw9uX89mO7F9y37TrSf/pU4CE+fFVB5Snrtk/WA/XzuOsysKjWeejCy5T+GRGdjOfrQdc4KIvc + pQ/XNTo/iXN+8HV+rCV/5lFP40ecCyxyTpfgj3lG5JVYWp95r6KHFE17bBS/6z7K8ps/+qYZ2YtO + OreOjINe0lt0UX7uJ48ZfW+4XvX4nsSuWfeFfB4rT3DcR/3CYTnOre8dIdlvHDdvvHf8Yw1305VL + 7Sc1muMlZ5O8NHLXjHVshy1c8NwPn9bDnrwT33mADX/mxlU94mR9Y4gXkc1T9acO52VZ6QfBTiAW + 0cG2P/3bFlB88de+8FWECEZBt3mpflGDP9nYqMfq3/qtfuEXjOeZvjkz7qArt6KH3FMIbsx48pWe + tc80pN3NpZ/qDj4zsFUIVx6X4LtPoJnnqg31aaFVYMnp80Kx1R5X+D30gz2/6aQv6+Y46SrCEQfb + +6SL8sRnP3nM6HvDNffDu+fTPv3iE+zA5zjVL1wRaUofjkvDnVdpna/7wbMKf6zh/AMKqaR+UoOM + NpvBrFYyMxl+y5+5cMFPPq35chN+uysPjPBnbt7kGXxccriefb0SYZfXOPMiqBdVHmz3m/4Xr+pv + f+3LVvoiqPUVmPp88qboMtd5cmv1C4N6CF/9yc+9g67coq7sX+4pBDdm/LpH2Wcaxu3m3TlF/5od + yNc1LsG7r5VnBV6Nlc+ndFsTdJGikteVWql5nv5mYly6kX/y6HyWfq2H6KK8z0mW9CQ+WbGgY9Qz + 88569D6rQlYeE5kPvORxIiGNW/mSeOEIc/1dH55V/BmWH1h4GAjCGVn8xPbM1P0kxjr2RXjuzyam + PXk7j+BsiiUu/s5zqGfyac041b+vVyLs+hp9rESLDgndZ/pfvOqn/bUvuyqoPKnnds/WA3VS0dxC + FOX+0lds9gsnBRd++rl30FUw8gpt/kP/nZ/7OT+yw2aapOu54hUXfObk45xxCf6YJ2FXZbY+dYHW + 5PNCkdUeV/hd91EW9Vs2tjSsm+OkLwmGfq2H6HICxNd5KGHdAzD2Nx3lD+/K6/qDI5/Hns9xqp88 + LkxA41a+NNx5WWf12f3gWbX+kRDNCcKZTYzZPYJcFCySWlyMLyJN/XIJvp/MCAh/5mM9p/hdj+ue + cTw0Dvfj2X5s9oL7tl1H+k+fABKgqXRom+wYlcfG7X+1HiiLt0SX0TW5v/Tl2Z+c8FMP4aefewdd + BSs95C59uK7R+aVT6Qlf58ea6Up27TNUcYf7x/0b9C29L8If84jjCr2obl8oCVHXyee1a9cKGU+9 + eP+WnmnJOlgm6XqR7qKL8ta78eI1Y3/T0QmFd+Wd9dz8fpEPceRxIiVgfN877uQ8G8dN51v18R8J + z5/+GZakqTNcT35sUOP65NGhlO0zj5/RGOOh4I3b++q6eeSskxV7uL/05VlvguqfEatv+hmHy8WH + CFbhlYN6yM2FVrQ0gtPMy1l+zuIBajcPP+s9xh35Y+9wiQP3MY+rujqvqpsKsO81+bxUf2qlrkM/ + IHxeJ/SePOIdcdFddFF+6CT9iXde3xuuvSG6WUcBL79f5AAveZQ2fOmDfowdH21uun6HKR4PrO3p + n2FJmjrD9eTHBjXSm12Ltqml9mub8fOhIPs2v7hulMVbkluoMnl50pdnXqa0yYjVN/1shP59nBzs + X+4pBDfM2zPydz3hgVN1ZHYi44LPbCK+rnEJvutFBPNctWE9Src1+bxQbLXHFX4P/WD7vKR8tyW+ + yUOCoV/rIToq4vi1L0fn7W9EjWMVo47oTz/yZOz50gfihGPamZd8FZn9xnHf+VSt/XxgPf0zLElT + Z+hPZj/ZfRbt0Bn3JzcPaZ7VOLQ6gts6+RPcl2zdCt4tXh7P7tuXSeXr8qQv9k0cX63HjJOD/cvN + hfE0OTo/93HZpi0eYHZzxQsXfGYT8nWNS/DdJyKY56oN61G6rUkPBda65KSuQz++iYedvqyb46Tr + RbqLTgiFtk5KyDxm7G9EPmHtz7yuH/no98PkBF/6AK9wgFQC5135er9xpHPf3Y9+6H5W/0hYSf2k + 9pONJPOJaelYJGskTc2FC74cmvrlEnznQUD4Mzdv8pB0iGTTh9A89FOcXV7Xa95OxHDDBLe46q9s + VSS+GU/+ph8L0d32lz4fHLp0qIp2+qjv9AuDwgjvPmVRVjiOcRKM/cs9heDGjHf+rgc6Mg3jdvPu + nPb3j3w5R62HLd7ci8yC7/N03BVZWI/SbU3QRYqOdq1U6wf//KaTdqyDZZKuuq+l49RDdFHe59R4 + nbMZ+xvRqGfmnfXc/H6RD3WQx4mUgPHmc740vHAVl/ui+zF/6M43N4af1CCrN7ufhN63dHXZboIv + Ik39cgm+8yDAmrLJ0/WIs/jCr3oR0TxVf8SUn5ph37ydaNG1P/1XfkaIb8ZDr9reEaSg2zi/9BPu + 3H7nr6i/5oy3RIftglb/67x5SSQnhRE+fbttNnqMaz0YQn4R0PCQ3lhq5qUrf/PAx3SsQvPwk+8Y + d+SPvcMlrvLu+LF3VYberBGcc8nnenMuqdYKtX5QzOdlXKOOPLJP6C66KOPzkVX1aAJpfyPSCRHH + Mvd8zG0cGTz6fIvIE+LI40QCGke+DvR+42g636pv/tCdl44QzmCZs3tk0soZXOYDHtsk0tQvN+E3 + /Eb+5D3WI84Dv+pFhboMAKT+1GHb++YFqBdVLmzHp//ikYjUJf7al61quFGEZd/G6Zc/69r2jz/n + E7dvfcUnbr/mOfynfjbqsdMHW6tfGIKtPhlhmQ+6xsH+uSb/yfMwf5+f+A7nzPDax6R6Gp97ZQdf + 16h8PtecV81AdZ9Yi39F3vaV3qy5L5xLPtcpRYectEdffBMPO81YBx9D6zn0az1EtxRZ+8ljxv6m + 4xNWPTOv8gFqHPk89nyuR/WTx4UJaBz76kDvN46m++5+8C8I1w/d61LrSQ6WObtHBIuCRYLqJnim + 4qXbjUvwOkSG4Xf4M7OrWY94D/yux7gZlzoc77rt70SLDg7XkXy2VRHztb/2ZSucG7W4OtN/8Sue + tb3xt37S9uf+0+duH3eH63N/6cuzPzlRN4XhCVSfshR20DUO4rjWm878NDl8HjXn/GqfaYjezZfc + jxv0vQTffVYeTFdqWB8JXXpHD+u4rpOVaj2h3Pymk6bkX3TW/5TuoovyPh9ZSshzNmN/I/IJa3/m + nfWsp86Rjzb5wEseJ1ICxve9404lXjhuHu6dHljn9UN3XjpC6vLN2T0yaeUMLjOTzzjszyZoxt7h + Eif3jfzJy2Z2cYOPSw75USHnGbfPa5z9COpFlQfb8cm3eMXT/tqXrfRFUOsrND3z2tn2tZ/6vO1N + v/WXbV/8yc/e64M6V78wqAdvVfUpS9fioGscxHHNy1b3hyaHz6PmnF/tMw3jdnPFKy74zMAe+WNf + hHdfKw8prtKwPqXbmqCLFE17KNlKGU+L97vu52jIOlgm6Xp4P7YeoovyxI/3i3hNqjoIG/XMvLOe + PGyE3vG5Hp50841zNp/zpeGF4/7h3t3BP+l+7ezp/5aQ0tQZric/NnC4/iTRom2fefwluAhqfQWn + T372Hdv//dJfsr32837p9mnPu6P6cn96E1T/vCSr79IFG9YlepRglIW91ptjtm3dCs9LXPo0D8Pw + O2mnn3yyM5P4qO/gO4U/5iHFVRrud96r6CFFR7tWqPXhm7jOY/ZjvSzTup8ndBddlPf5NF68Zu1v + Oj5h1TPzznrysGEk9xdfjg11kMcOJTCO9TlfFgvHfdfffOfXnv6T7hGb7zlpyjcJfq1PrHYMP6Rk + AM+Abo5e2Lyqr694/p3b9/6WX7Z9/Quft33infhzw90vKtblSV9uzG0dPunYHB11B32bI4Q7709g + 4UpPhVFfhOP3bnYi1UM+67vilM/Ufr0E775Wnhl6FdbWhwKWjp6gi3Ws9lCqlWo9Yff9HI1Yr6a7 + Qb/WQ3RRnvg6DyWk3iZVHYSNembeWU8eNkLv+FwPT7r5KoHzrnxJvHBk8/mrWsfhZ1jXr/8CXUnq + JzWa4KOPyccT09K5BfkVtnDBTz6t+VKP0sk78Z0HUGvKJhl2a/yuZ1+vRNjlNZ95OxEWVR4criP9 + V35Wojrir/3aXgRaXfkX/FPi9hWf8nHbm/6zT8T8HLTGflE2heFjpHSQxX1s9PmUnq2H3FMIbpQ+ + meseZZ9pkq7n3TlF/5odyNc1LsF3vYhQWyvySqxyX1tHyw5dJLjPQ5VaIeOpm9/8sdOM7DoG6Ut9 + TukuupyAz6nxOmczqg46Rj18mCRvz/T7YaJA7i8+8nMbccKRThviMZ/C1n7jKm7y4Vl1bXvO834S + hI8mqZ6cBLGImt0jkzI1tjlXkRMXPFOFT+thX4Tn/pG/8xzqEecQyfSurHmq/tThvK7bvIjqRZUL + 2/Hp37aA4ou/9oVXNdyoxUfPxG9YX//C527f/XmfuH3WJzzDevCESwd24rash+WqPqMHQTifY//S + W/HforHqAABAAElEQVTkKz3LFg/DaGcu/XxO0X/FHfljX4T3OS5+pLlSw/qUbmuCHtZ3XScrZDz7 + 8Zs/dpqSPXl0Pks/+qW36LRS6NpPHjOqDsJGPXpYznOSF3E8/xp7PkSTFpmbb8SbrwO1WLiK67qv + P7o9/My3Xdv+0Avuhetnk9RPajRXl3A+Md1SXYIqcuJYnWzlWk2okkvwnQdg5QE+c/PehN95nX/G + 7fuituHtRKs8BNqf/m2rItbf/tqXrXBu1OKjb3rh856xvfZzn7/9nRd//ParnnVH98lO3NZB1zjY + P9e8hIf+fR7cpm7rXrT+DMNvxmuu+B1+xB35Y1+EP+ZBmis1VLcvlASQTKhQ30Q493WyQsbbr4fW + Aqgv6+C41nPo13qILsoTn/dD8lim/kbkE1Y9M++sR+fvsANf+sD5k8eFdb3m60DvN47mundg+tnt + q190P/5Yg/bfnqR+UqOJuoTziemWEMoQ+jOz6YEvh6Z+uQTfeRAQ/sxU6zJ+12PcjENglYMZjs4j + 8eBqvxPbn3zpU4GH+PCJvnnK+qic/qv/5FnbD/yW529f++s/bnuGblfaOujK7nTemLjmm6d0pMnh + 86g551f7Oh+sd3OfA3WN/jWbkK9rXILvc0YE81y1YX1KtzVBTyk65KS9dNCbfNjpS3yTR+cz4mBT + B8nG8+o82U8eM6oOwhrHKvZ89vq8HOXzdp7ZB+LI0wUER76KrMXCcd/5XMb527njB9bZ2Y/q0hFS + l2/O5OwnMdaxsTyJ5374tB725GW1sgVnUwjD7/BnbtzAH/nNs57IyROc7dHHSsTqDENC4nQppINt + VTTsVVfFNYGoPqpfnoM/r/VnP/U52/f95udvX/hJzyz5DrqmX+rFNS8b9RnD51H6jXOzvo6TjgzH + 75P4EXfkj6244DIX3+QfpV2Jpfst3dYEPa3jktMXtfWBv+/n6MQ6+Bhaz1N6iC7KWPfGgzl5+xvR + qGfmnfWsp86Rz/XwhJuvEjDefNVE9pmvy1v3DoX9KJH1Dev8R5PUT2qQoVlWz9k91oyg2CSYuOC5 + Hz6th30RvvMAG/7MzVv1iJP1jSFeRDZP1Z86nNf1mhfBvahyYTs+/dsWUHzx177wVcShnlHaR+Xy + 1z/nju01L3ve9m0v//jt1+CPROx0ZUfRg2tetkP/0luwdY8IbR6G0c5c8YrLOWcG5sgf+yL8MQ8p + rtJQ3VQgOlImFMg3MceS0woZb//8piOw8CYoOsTTrntcfvGLLsozz3p/s4LkVR2EjXpm3lmPzp9Q + jD0fbe6ClzxdQHArXxIvXMVVffCPB9bZtae/YdUZric/NnTm7WjbZx4/hcXgm/ZjcHz+L71ze/3n + /ZLtz/+G5+hPy+eT1Q8p30FdtkP/wWnmpSs/Z+kHrXbz8JPvGHeDvpfgj3mu2tFYD12wulfRw/eo + 2kPZvn/G0/KbP3b6kr3obtCPfuktuijPa5v95DFjfyNSfuJY3/4ciTSOfB57PscxsvmqMePI14Fa + LBxN5xM7n1EY/oZ1x/Wnv2HVGfoTAofIjwYeEmcv2tbRtp8yYgjn5cfa6zNxS/7Er3n29sbf9Anb + F//KO91e+qel26xr1a1bt9IP/mlLP4bhd6m785PP+q+4G/QtvXe4xJEX/snfhV2RhfVAhdHR1w16 + WMdqrxUynnr5zR877chedDfoR7/0kOBRZuikhNTbjP1NZ9TDh0ny9kw/z7/GyjP7QJxwAFUC41a+ + 3m8cCZ1P7HxGYfiB9dyX/Qx8jwrC5GxuzO6RScclqyInruNMxNc1LsH3kxkRu3yjjpvxqw5ENk/F + RUzXSW2rj5xZ1+XE9qd/41WR+GY8daJdLfZitfyxtvpV+I+q//ZnPHf715/9vO1F+NPy6p9N8hIe + +pfecFn30rPs3T1ieHCZD/cP2zfwJ9+OP3HFM/OI4wq9WJ/SbU14G/pCLTl9UVtPvonxK3Zasg6W + qfU8pYfooozPp/HiNWN/0xn1zLzJbxwZKg6FL74cG+oljx0CMt58Hej9xtF0n2jj/o3PKAw/sF55 + hj+Htb1NEHh5GfQErdk9Mmnl5Exc5gO+HJr65RJ8P5kRsMt3op5T/K7HdSdel3qX13Xb34kWHRyu + I/2nTzlUWNepuuhX+FiU/TE8vQJ/Zut1r/j47a992sdtz78T2uD8lxBu3OdR+uV+wNX6YY0o3+Ha + xyT/8f5x/8gfW3nCn7l4Jr84rtCL9Snd1qQ3Mcvse1UKtZ6w+eaPnZasg+PUt+7nwtHvfUTwvMRL + fPb1NOm8/Y2ocTyvPR9zG0c+jz2f62Fk81VjxpGvA7VYOJqV7+z8JzY+ozD8wOJq80/h/aRGE2yK + zfAS0JsZ69iKGrjguW9RtPILcRiTd+JP8XeeqmPij/ziRWXNU3UF57yjj5xZ14XikNDx6b/qtWP4 + 0wdntTUWZX+MT/zT8l/6yc/c3vi5H7+9CvM1bozh84hOpSf8fT5Y63wz9zmA5xbOO8L7XHNeF+cZ + pV2JpfXRhat7FT2sY98rvGnpaT3rzR87zVgHX8Mb3jcA0e990kX5uZ88ZuxvRMof3lFHzot+8VVc + 55l9IE44FSKg6yGf47JYOO5XvvOztxdqPrD8U3g++Rg8Z7XCSyQKuDlXkRPXcWQfTdCMfRGe+0f+ + znOoZ/JpLXofQvMwP9XY1em6zYugXhRMcNaR/o0XUHwznvwVxyIqD5dPpfH8Z5xtf/UFz96+87Of + u/3GT8A/JtbQOWO9O++yJTvWu3l3TtG/ZnIe9b0E3/eg8pDiKg3rUxdoTXiL8h7Pdte91j7fxLmf + QvrFOjtOulKfvG/EV+8v0UV5n0/jxVt8rIOOUc/M6/rpdR5HHflo04NzFI502tC9MF9FZr9xFcf9 + a342cWd9w6qfwvtJjebqzb6ezEw6Lhn9tAeOxci2Q/5+uQTfeRCgPMBnbt6b8Duv8884HhqH6/Rs + fycqv23Xkf4rTiKCB4Fdp/qmrfCxKPspNn36c+/Y/uXLn7v9nRd+3PYrn4mruNN93YvWz3LrPdHn + hT3F5ZwzU8sWmgbGKf6BP+Zx0NV5tT66UHWv6t77KZH22Ch+D/3qze/41Y91syyt5yk9RCeEglsn + 6ck85uTDRG/AUQ8fOsnbs3DkqzgQdH5smQ9x4asEzrvyJfHCka/ynfvfEHJnPbCuP/p2bvhJjaRo + liSc3WPNwMS+CM99PmR2o+zJG37Db+TvPFXHxB/5xYvKZr0SYZfX/ZkXWXtR5cJ2fPq3LSB52l/7 + sqvLY7+75p86xu/5FXfiHxOft33tpzxrw8/opec8tz6fyJ95d07Rv2bKd9T3EvwxDymu0mB9+3tV + 19FPidGuccZDBr6J657PfuSv+4jpYt1FJ4TCW6eqJzL3N6JRz8w769H7rIrZ8+XYcI7kcWEjL8+3 + A73fOJp1/tdPfcN6xjP1rw315ATLnN0jk0ZUzONJyawTr8zl15ovl+AZf+RP3lvhdz2uY8bt87pu + +1FTL6o82K4j/dgWkPW3v/ZlV4fw11/qWRtP3enj8POsP/PrnrV962fyb4KgbqUnJLnhnLHnY+Bt + jq7Rf8XlHAUyUMsd/yV5OvY2L/Smpi6+UHPSw4jlya06jVOf3OebGL9iCyI8dXZc63lKD9EJYXYk + arx4zdjfdJCPQ8c48ia/cWTw4P7iSx+olzx2CGgc++hA7zeOZvX5DP+RBu6sb1hf9iL8NTPnb9aT + k0nRLNnmE1O9Ikgz/RgTF3w5NPXLJfjOg4DwZ27e1EXS4gu/6kBk81T9wblO12veTiQK0cHh+PRf + /UlE6hF/7cuuCkDwqCVJSU/Z+QEI8b/81Ae3L37Lg9Kzzw+K9PlgrXPIXOfpc4r+NVPJ8nOpcQn+ + VJ6E3u5Zb1bVXxdoTXiL5n2VKmkvHfSwGnajyDd5ZI842NJbdFoptHWqejTB09+IRj16WBZA59Q4 + 8nns+XJsqIM8XUDuAevrQC0Wjib919+8fdnL/ZeMYmc9sATf7tKTEyxzdo9MWjk516Nx4pg9+1iY + Ma+X4Bl35E/e5r0Jv/M6/4xLHa7TdduPwnrBfduuI/2nTwAJ0FR1tl0NiiDNPjVnyvkP3/Xw9jmv + v3/76+/44PbB66XfOLcbzhkxPgbfF59T9H/87tNVOxH16QtV96p00Lu67qOK9sU0Hvt8E+NX7PRl + 3fqa2n9Kd9FJcbPj3i79yWvG/kY06pl5k984MlTcji99gJc8TiQg483Xgd5vHE0EnJ3dJUe93PDA + 0pOTZGiW1c8npqXDNqnoz3zAl0NTv1yC7zwICH/m1NEzSYsv/K5nXy/xwckvk+LVdi+W7TrSP/el + sgG7eOpTcSfqSV1PlfmH7n90+50/8MD2VT/60PauD+Eqnjpv6V76Yy35M5/C515RxMN5x/a55rxq + FnyfhxRXadx4r6JH3lep1vev9cSbWA+tgx7Woa+p9R/60d/Xne8LPT2Iz37yOC8fJgpoHM29vkQa + Rz6PPZ/rYWTzVd3Gka8DtVg4muC9frMH1p3P/k6ArpPFT2zPaoXNm8Kzmgaa8wGvzOXXmi+X4P3E + VYnNn7y3wq86ENk8Vdc+r+s1L2rqRZUH2/Hp37aA4ou/9oWvDo/91vbH+vQePJz+5I89tP22Nz6w + /cB9+rN9atnnEZ1KT3j6fLCW/JkvuR85R5Hz5RL8MU/HXZGF9akLtCY9FEZ7bBS/h3715o++aUf2 + 5NF9HXGwpbfotFJo6yQ9iTdjfyNSfsvNh0ny9kx/P3XG+RaRJ8QJB+7e90Ms+Xq/cewaxHc8epcr + 8uv+G9YffuF78Uh7C4P5BMzsHpl0XLIqcuKCF/VoYtoX4f3E3fMnb/OmLhIe+MWLCpun6g/OeRlW + feTMikeTtLe4Ew8hnK/95sn27E/rp8DLw9Dvb/7MB7fP+r77tr/3Cw9L19m29MOGdaz7VLbOFevd + 3Oew7l2fO4nLz6XGJfg+Z4CZ56oN66MLNa8X36QqdbXri9p68qGBX7HTl+xFd7HuoovyPh9ZSkhe + M/Y3nVHPzJv8xpGh4kCw+HJs4CWPHQIy3nwd6P3GwTw/+8H58ysC9g8shVy7i1X7ie3ZPTLpuGTj + SXnEi6YfnbIQ6KYmb8eRlw+jA3/yNi51kfLA7yf+vl7GBee8I4/EWzyiE9ziTjyUM0/7zZNtlpM8 + Wn+Mv7zufQ9vn/f6+7b/6W0f2PgD9qlzWvd5RKe6T5LpcM7YA4POn7HWfX//uH+Dvo/xPonjCr1Y + n3mvSge9q2e7vqitJ/x888dOS9bNca3neL/Q731E8H3RebKfPGbsb0SNY30rb/IbR76K6zx5v3Mf + ceTpAlhn+uhALRYO5tn5XeXt6cQD6/pdvBx+YntWK2weYWm1n7D1UJh4sUuUzoNAN7XDJQ95T/An + 77GeU/yuZ18v4/Z5R57VyKIT3OK6TuN1uOpzxoNg0Xee0fHH3PInH3p0+wM/eP/2e978wPaTD+If + /9g/uxw6p2mfR+l3s3NmOH6fxI+4nGP4Y/uc9veVGO6LF2vOV224Xwq47pHrlaJpD2XTrv5kjfs5 + mrIOTaf++33DuOghuigz95PHpP2NyCdcZY46WDd56ef511h5yq8JccIxIPvpowO1WDiYh59fEXDj + A4s/x7p+ft1PbJDy0gDYM9axSTBxLEa2HXxdo5q6CH+Kv/OENzNZh0g2fQjNQz/F2eUdfeTM2g8W + wS2u66z+7Bj+2l/0nYe1fKwNfov68z/x0Pabvvfe7dvf+4jbk75om9bQOb1LPxjWcd2LPh+G0Z+5 + z8HndozLOQLucQn+mCdhV2VWf7t7FT2k6GjXChlPvcb9HM1YL+pdPFqc0F10Ud7nI0t6El/y8mTo + 0AmFd89nL3ECKrB1LyJPiAtf76cPhTmBsjnvOd2Hn18ReeMDiz/Hura9xU9sRPEhAWDPIrVNgolj + t7Lt4Osa1dRF+FP8ydu8N+F33n29Pj2L6byjD50Syuu6sJb2FnfiyzH86bvDx2K1/NG+okSv+fkP + bi//7nu3v/72h7ZHsGGd4aBuvluYJdyu3eCs47oXtHWuQO/mPgfyFj4zmcvfSS7BH/N03BVZWJ/S + bU14l1L12S7toR/8emgd9LDOfSw+p6Ff6yG6KE/8en87j9K7DsJGPTOv66fX5+WoIx9telC/cKTT + RuVlXxWZ/eCubzf8/IrIGx9Y3D0/v8tPbDTDprHVM91lY9I+s04897GhqV/K3uESJ3jlwTr8mW+F + X7yI7DqZjyLs8o4+cmbtd2LHp5/qjxWJb8aT39vqsXi0/hh4+Q/3PrJ9wffds33VWx/Y3v2hR/e6 + sr/owfXQmSaHz6Pmm50zsH3OiQs+swn5ukafm89Z+Qa+7wEiyH/VhvWZ96p00GN83CvZdR/RxFPl + Gxb+g+e7Tp3Z6QfWdsddfmLzCQixENmzRLNNwonjJZZtB1/XAA/HRfhT/MnbvDfhd959vX5Tzbyj + D247QdVl23VUH3ovFJD1l73qIp/Cx6Lsj9LpFz90fftqPKT4sHoz/piC+zvoyt6iB9d6eEUIbljn + nse53XDOALWeiQs+s4n4ukYJr3MPLjNQxzwr8GqsVLcvVN2r0qEer32v/DhXP6yc31TmN510Yx36 + WIw/pYeusxRXaOukhDxnM/Y3olHPzOv6XY/OvwrZ87kedtZ849zM14Guh/lY3qP7P39VqAu+Yd15 + 5+7PY7lHJo2omCEGx+6TDcVkfzZRwJviGXfkT16quMvjxOLLi/MaN+NSh+Ndr/2I7AX3bbuO5Euf + ABKgqepsuyoQQar56Jv5xxS+8e0Pbi/793dv3/JzHyhBMh10ZXvpn2tewkP/Pg9uU7fSU2GHc2Z4 + 7WPa40fckT/2jn/gfY7gK35yX6VhfUq3NelhxDqXnO7AePZD/Zae6ck6OK71PKWH6IRQaOukhOQ1 + I/PoYDhj6BhH3llPP+WEG++PiiNR81UC5135krgeYid/fsU6Tn/DOvx5LPfIpHW5OEMMDs0oYjfb + IX+/XILvJzMCdvkYdwv8rsd1JN4qzzpdr/2dSCWqPDhcR/pJn3KosK5TddFfHfai7I+i6XXv/dD2 + G7/rfdvX/fiD24OP8r+nSd/p76Are0v/XOs2RwhuzPhxfrUv/bHezaWfzvEWzjvCX4Tvc6o8mK7U + UN0WuvSOHrmvKZd23UesnhLfsLbTP7+iIqcfWPRs689jSTJeIuz2JeMlpV2XdTfbIX+/XIL3E3fP + n7y8nJfxy1+fADNOb6au0/Xaj81ecN+260i+6o9A9TnjEVDb6rH60/qj5OUn8EcTft+b7t1+9xvv + 2X7qITyoqIfeROnbbdPR55M+owdD9PDyfaDJ4fOoOedX+5Id691cvIoLPrMJ+brGJfiuFxFqa0Ve + iZX1QWXR0dcJ6lvHag+10q77KIvvw2WnGevWdNZ/6Nd6iC7K+3xkKSF5zdjfiEY9M6/rZ3UIQJ6M + lcdE5gOvcEBVAuNWvt4nbjv98yvmuPiBdX7920jSn1RsnlT4rbmKvOgTbjbBRLEvwneewd95qo7U + M/m0Fr0rax7WR3F2ddKsPlYji05wi+s6jVfH4pvx5G/6sUhFV3e+H/+67+t+7IHtc//9+7Z/9Ysf + RBs8Z9RLwXnCpYMs7ssf3bThfomTewrBDbqNs46+R9lnGnp38yl8zt2BfF3jEnyfMyKY56oN61O6 + rQm6RLdUbKVaT/h9XsY1inpMHtkndBcdFXF86yQ96x7Ia76FI/2ej7lVL99nNfZ88CsN4phPaWfe + la+Ahbv2beE7zhc/sL7iZa/bzq+/bT4xmSqt9hO2HgqyUV329bCY2aqpHW7guX/kl32L/M7r/DMu + dTgv6k+e1YiqVHkItL/6KFuHpjri96wzyFmJYDZ89dbU5e//3EPbS7/jvdv/gZ9X8edWq184CeAJ + d9/0c++gq2CFk1sBXPWQ3rA03+ycgWHak/gRV4U0f+wd/8BzX7zFvwKvxsr9lm5rgvq+UOs60V7v + Kz00hp1urIPPq/U8pYfooox1b7x4zag66Bj1PBnfsJDwp7YvffHrXMWNrxc/sM7OzrdrZ/94PjEt + nVvoJz5E4eWRnZl5uD9H2TvcwHcexFhTHBLWF+GP/MIhonmqruDMYz7zdiJVqfLgcHz6qfysRHzx + 135tLwKtruTLm+55ePv873rv9kd/8N7tPfg3gdYr/bBflE1h+DYvHWTpGA+6xkEc13x3iYCGx+Tv + +wGX9XUc0yXtSfy4H0f+2IoLLvOJPK7q6ry639JtTdBDiqa9Vqj1gX9+00lH1sHHIF11X+sekwW2 + 92HwvDpP9n0SOcb+RtQ4Xos9H3MbRz6PlWf2gTjydAGph3wdqAVw/w/qy24513TxA4uYR85f059U + vAzYSqv+hGDv2KEYc2asROGiRtk7XOIA4f6RX/bkHfgjv3jB0DwVF5zzjjyrERWo8pDQ8enHtg5X + fPHXvvD7/sq6MtO78JdS/dEfvGf7bd/9vo1/tmqnD6pc/cKg4Dzh0kEWdTrqGgdxck8huFH6ZB7n + 1vnhY7p1DGJSPU/EfUKqKzWoQwk9J+gRHVKuccZTL7/5Yzeq72fpKrvuMUCtu+ii/NxPHjOqDsJG + PXzoJG/P9ON8M1Ye8pGfr4gTrjeqHvLRj7EWr/HG6debP7C+8uX/gX/jHzn7yUnusknpJzuflCga + STXbwdc1qqkdbuC5L15EhD9z8w48Ei1urJzX+WdccM47+mC4geIRHWzXkX4Wr3jaX/uyq4xDPbV7 + 26YPXT/f/vpP3L+95K534x8DP6CHA4tpnave1S+c1IPI6lOWZD7oGgdxXPOyHfqX3nBpHufW+RlG + f+ZRz62cd/Lt+G+SB2mu1LA+pduaoIcUTXuo2QoZT8tv/thpyjpQ79JVi/V+pF96iy7KE5/95DGj + 6iBs1KOHJXm5m5l+nn+NPZ/rYUXNlzjlZX0dyDX+dtGX6K9qD99xvvkDi+iza9+kVngZaOJ3bLlZ + LJPP2Q6+rlFN7XCJA6qfzFiHP/Ot8IsXkc1T9URM5x15ViOqUeUhoePTj21VJL74a1/4ajEEZd6u + 6UH8d39/9x0Pbq/4jvfgB+v3b/jvlXU+qWenDzZXv8QRxfOs/uTn3kFXbkUPuacQ3Jjx437UPtO0 + /FjLln4Vl3uRGZicI5cal+C7T4DJfy/+RcOrfviB7R0fwL8Nvc1Db+rcl+ho2aELlZntWin2o334 + 9dAqW5vCm6DodK79vim/dYYBXX0C1nvpz3OGS17zLRx1rPcF/aMe8ylM+4uPOO4jjn11Acm78hF4 + fnb2TWa5+PXyB9b1a/+AGfvJCYM10ObQzGS0M9shf79cgj/F33nCm/kEv+txHTMOhakE1+d67cd2 + L7hv23Wkn/QJoPqb8ey34roeEt6ewfK/5Wce3F702ndtf+It+GMKD/g/UvYn26qrdR66+JMTBIKl + LzKmv4OucbB/rnEu0Zkmh8+j5nFunR8YpmO85lFP36MRd+SPrTzBZa78O37s/ZN34c+bfc8921/4 + yfprcbB3O4YepuvCSYC6XtBj6s7qrFDrCXt+00n91oF6Dz1P6SG6KOPzWfrznM3oe5P84a33Basq + 4OX3ixyIYx9OxA3F972jff06/lK1Z32LnDd5ufyB9Uc+812I/47jJ1Y/YSEKu5SdmQm5P0fZO9zA + n+K3trfG73pcx4xLHc7LsiheldeLZduffoxHgwZoSjzrqjj2eex39v4Er99094e23/qd795e/ea7 + 8QN1NpW6MfOW1OXKPhHWyzMvk8pXaPrCLBxfD7rGwf7l5sJ4mhyTv+9H7Ss/1ru54hWXe5HZhHxd + 4xK8z9H1zTyU53/Hf8j9Wd9z7/YP8B9203c7hvVBdvaxJp8XClpyUte6j9zHL5/XCb0nj3hHHGz2 + Kl7dB8e3TnZ0Xt8bBPiEq8w9n72sn8wee770gTjydAHcTx8JvPba7ctf8O7wXDRf/sBS5Nk38YnK + FpWTcxXpJzuS02YRKT5zMl+CZ9yRXzbjwpuZnAd+53X+GRec63Pd9oOjF0UH23WkH9sCqo74a1/4 + avBQT+0+odMvfODR7Svf9H48rN6zvenuh6sdnlDqxswTg24ZrXPVu/oFQjDiR7zoDrqSLHpwTf5D + /9JbMOefNtOQdjePem7lvJNPvLkXmSvvjj/5Ks8vfvDR7Y/9yIPbF7z+nu3776m/MgeYJ2tYj9Jt + TT4vFLHktFKtHxB6aC2ASrYOfSyIv0B30UUZ4ut9Jz6esxXob0Q6qfDW+0L1GXj5/SIf4sijtBWn + vDPf9Zv+sN1V3ewPjgah+Zn/GJfoPqZKq/6EgF2XdTczhvtzlL3Dseixf+SXfYv85jHfjEsdzut6 + 7UdxvahyYRNHcSdeQNURv2edQdqsPmbLT9SaP1D/az9+3/bSf/uu7e//7ENVb9pxQdYDdfLEcgtR + kPur+ttmvzCoh/DTz72DroKRV2jzH/rv/NKt9CQ7bMkuVsfLrnjF5V5kTj7OGZfgbzXPD95/ffsd + 33/v9pX4D77fyf/Nz5M0rA8FLB09+bxQQ7XHFX4P/WD3/Ry1Wremk848d+fhfukuupzA3E8ek/Y3 + IuUP756PyMvvF1GII48PmhtVD/m4ff7Adsf5t8pxycutfcN61affh7/U71+4JRaJJPUm2D3JKUrt + zzeJargE3096gMOfmV3t8pAweUSeeoybccE53jj7O5EYRAeH60i+xSue9te+7C6gFk/s9C9+Af84 + g59Tfd2P3LM9gAeX+2O9SzdWoH4585bwVtRwf8OPQF6m9M8IHkDHw+TGMU4BxMk9hXCiFe/802Y1 + jNvNKqDy3sJ5u+CL8V3vreQB5lvxj4evwM+3/upPPbR9gLo+wcN6lG5rgi7OXXKgCivV+sH2ee1r + lH/ykCA6kgW29BadVupw7SePG/e94dp5RFf3QLtV4OX3i2jcH/J0Aamn7u352T/a/puXPkDkZePW + HlhkObv2GrdUEtabwE92PimtVp7o802iIi7B9ycAwMpDsZl28lL04jnye9/+GRececxnfyda5cFB + HMWdeFWkOuJPXZwVPhZlP87T2+5/ePtd3/WL2xe//r36gfr+E4v1Lt2Y2npg5i2py5X91qdw7hcG + HcKPePV30FUwJ7RbwnG3R+dXYaUnvNZXWXy+2DvWw3qt/4pTg80uIlk7XOLkrvuDtfgz14Ht4gr/ + EP4N61/+yYe2z/7ue7b/Dz+gfyKH8rOyOrg+P72rve38VHjpoIfVsFOj+2m6G/Sj3zqTLooQn/3k + SVafLxDacH2jDm5gXH6/iEIcebqA5CUf/ibk7eyW/nGQTLf+wHr0Q/8cCd+VVvuJz+bZ9JxVI5Fj + SCRqdRrfT3qEUIq2L8Bb9MUvXkQe44KTn5qBz/ydSCQqr/3pp+qV2Kx7xseuGkSw6nm8Vvc+fH37 + H95y9/ZZ//YXtrve80EW4Dp4+qOR2lZ/zG09MAtHoMfqH/EYttkvDMGKv/rxdNDVgVUH4yQMd3t0 + fhVWesLb+bFW+Zk7H/NH/xXnApueRDKU5wT+MeUBU+olKf/R8FVvuW/7z99wz/aW+56Yn285X+m2 + Jp+X6mElHD6Y1KcPFygX25iqf/JcpLvoorzjZElP8prR9yb5ieN5rbzJf/n9IgfidA+xrASMJ9/1 + 6+fv3j700L8l6lbGrT+wXv2Kh0H4mrTqTwjeLau0m5mZ+3OUvcOx6LHvo0FPCmeTt85vHvPNuNTh + vOazH+S9qHJhE6dLwbrKrkXbDou/mqw+yvqIJ/5Tyd99+/3bi//Nz2/f+BP3bf1//EtdOOxVP/uO + iQVG6yocK/Zwf8OPQPcLv2DpKzyMO+jKra6DbgTSHqPzC3eTc2Y4fp/Ej/tx5I+tuOAyF594w5+5 + 6tzFjfxYegD3Rvww/vNff+/2x/HzLf7Fho/ncL+l25r0JmaeJSd1HfoB0fdzFOR++lis5yk9RBdl + rLssJfQ9Uv7cL84YdM+8rh/7wpHBg/uLz3GqP3zVmHHId+3a39/8bAnFTedbf2CR5vr538ST8RH3 + 7CL1pGWREIddaSaW9hxl73ADz33xIib8mZt34I/8zuv8My4452VZlUeqIlnX5cT2px/jVRFxIF7x + savJ4inrI5re8L4Pbr/pte/cvvpN79veg3+b5X5GHtbBa9EO1hsTCwz1y1k4Aj129WNr9QtDsPQV + HsYddOVW9JCbBRlPk6PzC1d61r7Kxno3V7zics6ZTcjXNS7Bd5+IuDQPMKm3E1Td1zF/yzs/sL3s + u+7evgF/HOLxem45X+m2Jp+X6kkl1HXoB8T8ptMo1dvH4n6Gfq2H6KKI+5YlPX2PyNn3i/eHNqaZ + N3pdfr8YDV7yOBE3ovcj16+d/a/auMWXx/bA+sqX/Udk/Rb3zOywIAq72c12yN8vxGHscImrffFi + Hf7Mt8IvXkRynnHYYNrK69n+TlR+245PPxUnsdnnjI+tcDpq8eFPP/fQI9uXvuE92+ff9QvbD+G/ + ++NY/RQv87AOnv5opLbdZ8UpXjgCPRaf6139wi9Y8bdujDvoyq2ug24WZD66OMjb883OGSC3cQI/ + 4o78sZUnuMyVX7zhz3yTugBZQ/2tvvnzrf/5bQ9un/s9d2//4hc/8p9vWZ/SbU0+V1Sx5KQudR+5 + j19880ffFGwdHNd6ntJDdFHG59R48Zqx7xfvD4bkGHmT3zgyeHB/8TlO9ZPHDgEr/lu2L/mMn0/s + rcyP7YFFxuvnfwmveMC6SM0sknZm4srPpcYleMb7aECDgLYn7034lR+Rx7jUIT81Sx6Jp0SrvPan + H+NVkeqY8eyXtsLHouzHMH2AP+z9kbu3l/zrd27/7zvur3pMvKuXntTB07dQ2GS9MVec4MIR6LH4 + Fk5vApqCmUh6Ycv9HXQlVdfBuCkEnXQv/nkvOj8wKj/zKfxNzrsKc57gMlf+Hf/N8hQe0xrqb/Rd + np968JHtv37zvdvvesPd23+s/5pgBd36yvqUbmvCaUW3cPlgWk/4fV7GNSrngW31PeonpnUXXZSZ + +8ljxv5GNOqZeWc9Ov8qZOVxfT5W6EgeFyYkvrniD7df+4sVdsvTY39g8VvW+fbP+glbl1U2qsv+ + bELVEIexww089y0ZRCxcZl7OXZyJ+NrDeY2bcanD8c5vP0J7wX3briP5ql4C1eeMR0Btq4jqrwu6 + xcW3/dwD20v/9c9tf+Gtd28PPoJ/X1I8cx5lrjp4+u1gvTHZSOrGLByBHu5v+BHoT2z4BUtf4WHc + QVduVUKheCurbro4Zv19frWvsrHezRWvuFs47+S7CN993kqeUS+WHupv9N3b1uW77n50+zx82/pT + P/rA9j78i5HHOqxP6bYmnxfIlpzMV/eR+/jl83IdyWsdHCddR/3EtB6ii/JzP3nM6HujSG2IbtZR + BV5+vxiO+vELkwvEdO3s7J9tX/YZP07vYxmP/YFF9vPtr/QTti6rbDSRfV7S3Sh7hxt47lsy9MQU + sRkXXGYSF19yOK/zJ16nXjjnnbyINFAUgsF23uqjbAFVR/yedQZp81BP6rpo/rF7P7T9jrt+fvv9 + 3/OL2zvwqa0x+nM/M18xpQ6eftfPemO6oI4XjkAP91f1Y2v1C0MwE3W86A66kqrrYJyE4m6PFU++ + 0hPezo+1ys9c+iku+MxkLT+XGpfgH1MeEKbeYq/+Rt/lWLjzDV+Mt/8L/6H5y77z/dvf+mn/fxs7 + /pKFeUq3NeG0JPhol/bQD349tA56iG/y0D/0o196i04rVbj2k8eF9zeiUc/M6/ohE/3Ik7Hny7Gh + fuGAqrofffTsryTmscwf3gPrj7z0e1Hk65jIT3aIwaIpSorPnGrK3uEGnvuWjCKYN3PzDjwShVmz + 8zr/jAvOeScvwgyseNuuI/1UfxKb/c342AqnoxY3n96Pn9p+zX947/bZ/+ad279/zweWXgwb/bmf + ma94mYd18Pp1/aw3puvoeOEI9HB/5uXO6heGYMVf/Xg66OrAqoNxLMh56eLo/Cqs9Kx9lY31bu58 + zB/9V9yRP7bynMB3n7eSZ9SLpceoO3XSkb6wEo72vfhm/D/ir5z+HPyPPPg/9LiVYZ7SbU0+V+UJ + C/MsHfTQGHajVK+PQfWO+olpPUS3Olr7yWPGvl/dJ6sYdZCfvPTz/Gvs+VyP6heOAYp7Hb5dfW9i + Hsv84T2wmOH6uZ6QetKiiN1M/2iCZuwdLnFyQwzO+K2Zl7D22eQuDvvh45JDfkRwnnHBOd44+xHU + i6KD7fjkW7ziaX/ty1b6Iqj1iYmfxn/rbfduL/qX79j+NuZHSh/XXQHpE2b2Xc+gZxzzUinM7o/1 + xsRixgtHoMfiWzi9CWgKZqKVn3EHXbnVddDNgsxHF8eKJ1/pWfsqG+vdXPGKCz6zCfm6xiX47hMR + l+YBJvV2AvU3+i7Hwrlf28Dh10/hf+rxu7//7u2VP3DP9rYH+g+iNOVcJK51tOziIW7JyTxDPyB8 + Xs4fTvHVMajfUb/58r4gXRRx341XHjP2/UI+x7OKUUf0p198FYf9xZc+rE858L8cvPbnjH7srx/+ + A+tVL/l2lPr9+qRgkSw6M+sYTaissne4ge8nM8CUqO3JO/BHfvHyMIFPvC8D5TOftY0fmwaW37bj + 00/FOVD4Pf9o89ivWP3yHe9+aHv5t//s9jVves/2fvxF6kqr0wt/gUd/7sf+UaYTYsOfbIhTXtab + dka/dDMPeDN29Svcl6loFBGdGOO2DrrGoTpgkN9AejRm/fNedH6gWBWr1VzxiosOmcl44I99Ef4x + 5RG9dWMqDear/KmT+8pnQMHcgd7M5f/2d39o+1x82/oz+PvI+PdwnRrmgU95evJ5iSdRxR99gOAN + WnUYJ3vR2T/0o199iE4rBa795Ck+3RuuuZ/6Vt7kv/x+MRpxi+/7ti954V3c/XDGh//A4t+7fL59 + vZ/sEAPiUHx/cqAU2nOUvcMNPPctmSVqe/IO/JHfeZ3fZ+J6gnNellV5cmZdF4pFoP3px3Y5hr/2 + ha8mi2e2/NP4t0i//7vftf1O/KzqP97Hf1QY9en6hL+iRn/uJ/VwHnmYl/FutHhjGtjxwhHosfpf + OL3ZaApm3o4XbNSdQjirDsZxIWBlcd00xHPoi2mI3s0Vv8OPuCN/7Ivw3N/xJ9+pPKkTcw/iKn94 + 6FM+gdyvbeiDX8IVP79B/w38jz74P/z4pp95CP9A0syOFg4RytOTeAgoGq7wu+6jLOZZNrY0rMPg + Ee/CtR6iWx2t/eQpPuYlTPnDu+ezlzgBFbjnc5zqL75Hz7YP62dXIsfLh//AIsOnvuQf4RX/1hAi + sujM9I0maMbe4Qa+n/SAWlOIo7DBO/DhIzWHn/iuY8YF57zG2Y+gXlR5sF1H+lm84ml/7ctW+iLw + mn/r51/4ofdvL8E//vHfAnrwUEd9vg1Vd0FGf+4n9Qx66Qyb8V0/eWMyj+M0C0egh/sbfgT6Ext+ + wUy08jNu1J1z7TronkIQv+ef96LzA6PyMxev8kaHzCbk6xqX4B9THrCm306g/kbf5Vi4qTNw+KV+ + qi4wKoL/BvFP/vB92+fhfwDyve9fP98yT+m2JvEwsGnE4zq0rzzLVhLhc26l66hfcbBdHyyeV9XX + OikheYmm13wLR96VNzoYRz6PPV/6QBx+nV/ffnz7Qy/8p8F+OPNH9sD6gjP8662zv8wu/YSvmZVI + lFFS2Ttc4gRnUyX2tBkXXGbSHvjFS1GwL56KC855GRY/OAwkm2E8I/nTj20BxRd/7QuvcG6QZfuH + +HNUn/HP37H9xbe+b+PfVqK8gviSdH5dn+kHaPSXuMbnTqQOxnf9rDemgR0vHIEei2/hdJloCmai + jhfsoCupug7GTSHoXH2J59CXygZmN0u/igs+swn5usYl+O4TEZfmASb9dgL1N/oux8It/Zjh+A0L + jIpQmSjgR+5/ZPvC77t7+5I33bP9NP6tsHngUJ6exMPAao8r/HYd2oft8zI/9zjEt+jKHnEglA6i + 06rjvJ882nYddCi/65l5owP71vk7THkXn+Nan7OzvwTsvvCKu9VJJd0q+CTu350/Y/vJN//02dm1 + T5bKvLynBvcpWl3C3Qy8mpzzwDWvDvc0v5/sdZjkqXwR84Z8ndB8C16XTxsiwksdiqajH24O4D/n + lz57+/73+X/4wMPtOANku072e/QL5Lq5ZH5NzNemF1VHATCVroXn/kV5en/H7/iZiDiNTLBdR28c + 2qt9R+1fc24187ITLb5TM/3MN2bpwfhTg/us9zB/WHkmf/HdwKM60gECyl4PrZBUvdWoedIXw9qB + +qt80uGX2CucOw1ov3mSSbPqHTywoyP93Ufyive4T6DYqj5FamPWXxu9r0W97PNgE/lQyc8/+qs/ + /VM2fcmZ6Me2/si+YTGXv2X9bxK/LplKkCijmLJ3uIHvJhHiw8qhQb3gMpP2wC9eRDYP/cAH57w0 + w9uJyGaY4PTXZSi7L0v7zcNCuwws3sCHFdOST6c+LxVvwaivboXrZgUYo7/s7+olphKIX4nCW3mr + oI5nHupQY/Exzn24XxiCEe99+/k66i7+VQfdCiCwR+dXvUuHzg8k07n6mc/1Kn7osYSuFLPP4DKT + F/4df/Kdiit8MXsadYeHjvTlymOjP/wSrviXH0FwuJ7SoexyzEk8zsNXDiuUvM6z9DSm6hBv6Trq + Fwts1weL5yXe1FXxQKT8vl+NI+/KO+u5+f1idsX91Y/0YUWmj/yBRZYHnvN/oqgfY7frk5mijCGR + qJVV3c2A6ZOAM37riHj5ar95b8LvvM4/4yKm8408KxGycB8vCHQd1UfZ5Rh+44xXuAmEr/qrE9el + DEpgfsDlL56i6D5hJ67xrI9D+lW8G8Um6628amTEM09uocKHrm07XoILP+KV96Ar4pLQ7mqc+zVm + /ce+VDZwu3nWnXPOTM7yF33byhNcZsGrT6wvzVP45uZCgo6+y5m+ANCObeDwS3m6zvgBg4M4PWzE + a7sccxIPiZtGeVyH9pVn2dzjsA6O6zpO6cGysA8kw6quikeFyav7SVjjWObKGx2ME5DgAx9t7p7/ + 2PVPuhPPiI98PD4PrD/xAvxt/udfw+ry5LUoo0CJRK3QXHCZAeO+tMTampZ9Af7I77zOn3iptcs7 + 8uTM2u/ErqP6AFF4la/sxU9/9ag6bcuPw2Unjidm2csf/uI46KEo8DpfYZKHfO1gnpjMs3j9SUmg + h/sbfvE7XnzFm7rJmz6czvxJaDc8BlaWPX+fN7ydX6ywM1e88kaHzGQ98Me+CP+Y8oi++mIuDuar + /Orbu6rfS+OVn+eMXzt91FmVLXnor/tQtk5MeSod0844JbJCzmN/81RNnKzD4Bn1x+/6YKEvMu33 + k0fbroOwxrHaqp+7dR6X3y9yXPua7b/EM+JxGI/PA4uFfMVL/9X59fN/mievRRkVSiRqpdPaz4Bx + 35JZorYvwB/5nReXIjwVF5z8SLD8SKqEOhXD2m8e+sMrQPvTB+fqUflsixaHywSOJ2bZy188RcE3 + SfBzFv6Yh3ztYFxMAzteOAI9uO+whdObjaZgJup4wVzXjEtCu+FpIVYersRz6Es8lS5pVz7mLx0y + m4iva1S+Hf/Ac/+W84A1+TsB+YsvPPQtnDovG/Xil3CtQ/wIgsP1pK/wyFF+TeJxHr5ykKfiZDHP + songUF2LruyFc37iAEZf5nWcLDvslxdAOpTfcTNvdGDf5iP2yMeN83/6yJd8+r+S83F4efweWCzm + 2rX/Fk/e+1SXRBkVlu1PAohOGyL1kxq2jwbbCON+5sYN/BSJWcxjvhkXnPNNXgQZyHDDYDtv6lu8 + ArS/9mUr3ARlY9KlYgLXpQx4GfX5Ngw/3QufONdjerJ0HYxXIl23bDdfxwtHoMfi020U3p+c8Atm + 3o4XbNRd55iEdiMw+yMPl+I59KWyKx3jZVf8Dj/ijvyxL8Jz/5bzpE7MPVhP5Q8PfconkDovG/rg + 1+wDSKOqQddT5wugebSQAEpH/hnXeca9UJ5lC8K4ImieUT8xzs8ZBvpiJo6170Lllxc2YY2jufK6 + fnqJE5DgHR989z+63fnH5XicXh7fB9aXf+bPoK4/q9pGE9P2JwEOl36o009q2JbMEnFf9sQN/BSJ + /OYx34wLzvmMsx9BvSjNYTtv6lu84ml/7ctWdyYoG5MOlwlcFzG+JOaPn2HrsI96KKp1oIUhPSpe + icKbdswX3ss+AV0P6wQ3+Vin+ggP9w66ClY4uRXAVY/Or3qXDs6nLEqXtCfxNzlvF0w5WEfxZ0YV + jylP4bt4LgavZC5n6nQHlZ/64JdwElIEipBZeuqhJt4V5zyVjmmpP2dP2on+2leepSf3OKzD4FGe + hWs9JPjqaO3L0Xl9b8Rc/Kxiz2cv4qB7xuTD7tdtf+gFPxvf4zE/vg8sVvTxL/lG1K8fwO8KrKb8 + SVBPaojaT2peNgREyt2TP7jMJB4i2XTkMS4452VY5VmJGG6YtKc/9RnPayRA+2u/tptgwXS4jFNe + A/BqG7Dyh18A3bbg5yw86+Vg38xDpdpB3pgGdrxwBHpw32EL537hF8xEHS/YqJuJOLoOrOvNof16 + WfHkWzp0fobhN9mO9TR+xClfcWuqOpQnuMzkTZ/hz3wqrvCY1lB/o+/ypC9X7jzsQG9mYG70YxMN + up7SoexyzEk8TFVlciWC8DrP0pNYDvnFK7qyF875ixc6mddxspSQeNG5DjqU33H9vuBuAX0PBVTg + ynP+xuvPfOE3avNxfHn8H1ivPMN/9Xntq/pRnWIlEsTkjGZ3MzD9ZMZaR8TLV/un8Ed+8SGieSpP + cM438uiUlAAv3McLEjo+9dkux/DXvvAKN0HZqrvejq5LGZSg65M//MURXWAmrvG5E+oLfr0LBCze + pRvZOl44VuSx+Exom/3CLxgW6iN+xh105VbXQbcCuNuj8wtXesLb+bFmuqQ9iR96uMCmd/7iO3U/ + HlOe8Ax69zf6Ll/qdOXuR/qgE/UjIQke+pWeethEN+FKtzUhasQppxVKXj006jyqJKOat3SVfUJ3 + 0anSjpNV9WiCp+/XqKfrp7+AxpHBo3U/21696VkQz+MzP/4PLNb1qhf/O/wA/pt3JfJSY/iToJ78 + aLqf1Lyc9OO3NS273gy7OBPxtYd5zJf4XDqCHO/Zfmz2gvu2idOlUN6KI3DYi7/inEB8BUMfJnRd + AiiB+REnf/jpxzjowa3GWz4nZDmM7/rZd0wDk9c4Aj0W38K5X/gFM1HHC3bQlVTRg2teXtpjrHjy + 3eScGY7fJ/Ej7sgfW3HBZS4+8YY/c9W5ixv5sfRQf6Pv3k6fnl03cPg1++AJcSid5LnsXpUOM84M + eB36Kc+yBVEe6ux8XccpPVgW9ld9s27ymrHv16in3xfJJxbmJZ8H9Tg7P//mh//gC9+YvcdzfmIe + WKzw0Wf/KRT/ni62mtKTmU3RzgxQP5mxtqa8BN5v3MBPkZhDfDxM8M644Jxv5MmZdV1O7PjUt3jF + o7OZ/PQzO4b66Ql3xwlclwB4GfXJH376MUZ/iXM9J/LodiJGBZB36UaqjheOingsPhdu2/ESrng7 + XrBRdxpOv6RF3a6jkmBa8SzM8fR2fqx1TpmLV3HBZ3YgX9e4BP+Y8oA19XYC9Tf6LsfCLf3Yid7M + O574sdn3pnQouxxzEg9TVXtciSB5nWfpSSyH/HUM0nXUH7/3YfG8xOu4xrMPl+066Ggcy1x5Zz3m + I1bjvQ8/uv13MR7v+Yl7YL360/CwuvZ1XbBEolZQAarsZoD0ZOaM3zoiXtbaP4U/iGQ+RDRP5QnO + +UaelQhZuI8XJHR86rNdjuGvfeEVboKyVXd1oryCOEHXJ3/4iyO6wExc4xnOob4wMV6JwhvTwI4X + jkCPxbdw/uSEXzDzdrxgB11J1XUwDoESkA6PFU++0hOuzo+1ys9c8YoLPjMpD/yxL8I/pjyitx5M + paH+Rt+9HZxn5UcnT3/DKtm28z+1/eEXvrfketyndZMfd2oQ/sPzO7b73vK9uHevyKXmk1mXac6A + 7i4v7eHX5YTd84lahce12X3S1aUjfPLxqpGO72UvPBlel0/54gew7M7TfmA42l+06Yj7Gk7oeMBv + 8Bcq+Job3zRYqJzaEM6ffG7H+x13yNP7O/765NzpEf5V1+Sf/bq9whu+f805nHgIMEq8cwZOdY7Z + +Yg8MQ78uScXPrRAobyDP/l27MV7Aw/3oysDyl4PrbBUvQU3T+4/w9ohOqUjHX65vsmDHeHjN08Q + mougebCYfXUf4onyriN6mMesro9r9zHrN27tl/3Gh//gp79C6yfo5Yn7hsWC9UO3s6/CATzaYkvN + EhtvmojSYiIs4mXmZRUuM7klOhce5jFuxgXn+Hk4iDNQBDlD15F8xs/bZH/tI77LUF+2RVtvQ9fF + FL6cHT8ugQrgy+gvcY333egEvNSrftYb08COF44VeSy+hXv6GxZltR7RyYKO+zT0KyU1OQ44/PJ1 + Ck/0BUz3hP6b3as6P54XxiqHdsVxX3mWTSyH6lCecQ8O98n1AYx9MnHs7wN5ta08rHvhyLvyRi/f + Q+zjh0Db9oxXO/qJe31iH1is+1Wf+f14/RsWiVpZ1d0MgD4JOOO3johi1z5VPOLDB4iG/IhonsoT + nONHnpWo4jEhoeOTz3Y5hr/2hXd+5SlbdVcnrosYJ+z6ZIe/ONIn0bpUnsXHcA71hYnx7WC9MQ3s + eOEI9Oj8O37Hi694O150B11J1XVgzVtefM5Cc9Rx6EtlM4w0mU/hR9yRP7byBJeZvODb8d8sT+Ex + rcF6ii88dKYvVx4b+uCXcNXH8iMIDtdjHWOXY07icR6+clih5HWe4jHAKNXLPKJznaf0EN3qqHVS + 3eQ1ad8v5Q/vyjvr8VPu/G88/Ad/wxPyg/bR5uP0Hz9PxlPr5z3jT+N/QvZ6uvxJgMOFmFSnn9QU + l378tqZlT9zAWyQyepjHfIn36ZExeT3bj81ecN82eXQplLfiCBz24q84JxBfwdCHCdMfkEpgflqj + LsZzjP4S13jDVx2M7/rZd8w9bz4BnYDhQ1ds2na8+Ip35VdhN8QlobLxlrOAMVY8C7vJOSOm9cRa + ccFnJu+BP/ZFeO6LN/yZi2cXJ/p9/eKv/OEBzPVxsTs/9Idfs4/lBxQO11M6lF2OOYlH7F0OF0M/ + 5Vk2sRzuh7Poyl641kN0q6O1nzzFhzxqiLP4ae75tM+Kz7Y3PHLfp/33Aj7BL0/8Nyw28MoXfWi7 + 9szfh9bu1pOZlwmXger2k5qXA9BIyX3ZEzfwfBPMYR7zzbjgnA/8zYtoA0UjOtj2pz7bAqqO+Gtf + +Kqi/UVbnbguYtxZ55cd/uIY/SWu8Wk3eRjf9bPemAZ2vHAEeiy+hdObjaZgJup4wQ66kqrrYNwU + gs7Vl3gOfalsYHYz+RIXfGY75O+XS/DdJwIuzQNM+t3xV/7E07dwo15k8DefU34EgcD13Oxepc7w + phLaFcf8+OXzMq5ROQ9sq17ZIw6290m3OmqdpCfxZmQeBXDGEB020n9mPK3ufmS7/srt1WcPO/KJ + fX1yHljs4cte+Pbt/NqX+5MA4lE0iijxqCHEACxStj1xA2/RlzjmMZ94Ki44+XkGybMSiYRwnZH8 + qc/4cgx/7YuvalA+7guGPkzouohZtvukHX4tEbjXI373U5jkIV87GBdzz6s6eNs6vHRmAIb1cLzv + polSt2Guy+kcl4SyyF98lUa84T/2JR44d/Oop/FDjyN/bNUZXObuK6qP+VSewrPeHsQVX+qkL7qY + MTb0wS/hWgfrJBOOpTN5V5z6kB09RpyKoW39lV95li0IXqwD5+IZ9SsOtuuDhb6A5HbVVfHKo231 + o4DGkXfljQ7Xr13/0u0P4L39JI0n74HFhl71mf8ET+ZvoKp6QmeGnLO7BAAAKHpJREFUi7aPxlK2 + TXGDy0wuic6Fh/jAcIwLTn4eZvLkzIpHU/tTn/GqiID2177sLmDCdLjccF3EOGHnlx3+4hj9Ja7x + DOdIHYxHfvfHPDEN7HjhCPRYfAvnT2z4BTNRxwt20JVUXQfjphB0rr7Ec+hLZQOzm8mXuOAz2yF/ + v1yC7z4RcGkeYNLvjr/yJ56+hRv1IsNT8RsW5PmGR3//p39Ef0d7632Liyf3gYWirj/32p8+Pzt7 + vT8R9k9sXoFcDn8i8b2AHVzO3czmuD+G/Lg2x7jgHG8+5VmJxCI6OByffJU/b/r2177sKkJ1ch/l + Yuvpb1g+H+t+k3OGVtKrznOHz7lT4vJzqXEJ3ufIc8h53CQPMMprZr/qIMd9Kt/Cjf547/Br9uHM + VbbuST3UxJt8cqiw2q57U3HK6Q6S13mWnilZ/kXnfoZ+9Ls+ROhhv+r3fvKYUfeXDinoeuY3rOvn + 19/w8H0veFJ+buWK/KqS5saTsv67P/Jrz7aH3wQtnn+zT0KJTHHr8vR8okjzHD7pEge8/Dw0HqJs + vPTCZ2h4XT7lHcCyO0/7geFof9H6etTlEAAvI/8NfmJcZy3aHmWOPHV0zDt5ZZvHV9B+X9Kxv8O5 + rr0e4VcZQ7fsY0aCohkL43evOYea+80DUFW3n3NOY7a+7OjEOPDnnnxYeSb9RfVyvyvGsuz10ApJ + 1Vtw1+OHDXWjLR7lMY314D0uWlENHPd53vjleAH8cuSBnXtPQOuRvDrw4z6BoavFDneG/2XZ+d0P + nz/j5dsf+HVvN/LJe33Sv2GpNfw86/xR/wfSEb3FBMCHlUODaDhd4TKTRKKLTS/mqcsgt+OCc/w8 + HIDGrcgZuo7kM35/qWZdowwSgK8mHDEPe16qZSvtuARqgC+jP/dzqJeY5GF81888MZnHcZqFI9DD + /Q0/An354RfMRCs/4w66cqvroBuBtMdY8eRbOnR+huE3ozRXvOKCz0zeA3/si/CPKY/o9/W7v9E3 + a8BQPq+GDRx+zT7cWZUteaIz9QiPFmVHD9ex2rVCyXvRw8o6OF/XMfRrPUQnRNffeBSWvLq/dOiE + wms98D+Dws+tnvyHFau5PQ8sZv6KF78GIn4zPwE4/M0lhzZs+qGicJkdoLi8mMc4n4nj9GYqfmrf + eZi2D4/7tu1PPuMFVB0znvwVxyLaX7S+vspHNwD4PeqTbT65+TL6cz/2jzJXHsa3g7wxmWfx8mHU + t7D2HbZwehPQpKN4V37ujbolFGHkFdr82SccY8U7/7SVH5jdXPHCRYfMJuTrGpfgybPjR6TsU3Hw + pb5OQFzlDw99C0fBYkMf/Jr88JQfExyuxzrGLsecxGNeheOFPBUni3mW3SjVyzyic51Dv9ZDdKuj + tZ88ZvS94Zr74VVm/NzqBU/qz61UQL3cvgcWC3j4k/4Y/ln4h7nsTwCsrSkvgfd5Cv4EqdkBfO0h + PyKaB4fl0+PhhGfkyZkRJz9ekNDxyVdxdgz/5FM4N8qvSZeKK9elDErQ9QGh3covI33CSFzjDV95 + GG+BgGaemHtef1IS6LH4Fu7pb1hL7+hkQcd9GvqVkpp8TsDhV99Xe8qP6ZbuVZ1f3wuF44XntO6R + 8yy7Ucf7J3vh1rmTTpUqdO0njxl9b7jOPdH+Wx++51Of9J9buSK/3t4H1qs/+cHt2jN+H0R7sJ/0 + JVHb+sioTxQcAvc1MtuqffslfcX5cHhGvDWe7e9Ei679yVdxDjzEh68LKL+PWJ9QiOt6fZ1lK7/s + 8BfH6C9xnLtewtKH3gVlK0/lpV/bNQtHBo/Ft3B6E9AUDAvMKz/jDrpyq+ugWwHc7bHiybd06PxA + Ml3SnsSPOOVrdgYysuoMLnPt7/ixJ/tUXHgw91B/o+9ypE5XXvmpD35N/uVHIBzuu3QouxxzEg9T + VZlciSB5nWfpSSyH/OIdfZ7SQ3SqtONkKSF5te066FB+TQ9i+cVP1p+3chU3vt7eBxbr+dIX/TCE + +Op+0mPLmvISQHyIThV3M+O4P4b8iGieigvO8ebbnVnxaILD8clX+VmJ+OKv/dpWGe2v+n19xecy + dS2KH/HyF0/6SJ+w3Y/nrpe45NG7oGz1vXQzzPr4k5IMHjt9sLX6hSEY9Z75GXfQlVtdB90K4G6P + WX+fH7ydH2umsyozH/NH/5rJynxzlK08J/CPKY/oT/AXb+pk+vTlymOjTvwSrus0n0zJQ3/6WnGt + I9smvxShn9k4uFg6OM+yBSGKAcpTPLIXrvUQnTMxdu0njxl9b4TQBrxf/aFXvuCt9t6+19v/wGLv + X/aSb8InxDdYsjoiXha4/MnBJ79PQzNjaI/hfeNmXHCON5/9CO5F0cEmTpdC+WwLOOzFX3Gso/1F + W9ev6x224mWHvxrBJQt+zqPMlUe3s/Ki4kp/Y7xwZPBwfyuvbcdLj+Jd+Rl30JVb6Vdu8NMeY8Xv + z63zA2sdaq54xUWHzOQ98Me+CP+Y8oh+X7/7G32zBgzl82rYwOGXzyk8nlW25LnsXkWPEdd5XAfN + 2/ENCz9k/4aHX/kbvknl3OaXq/HAggjXf/rFX3t2vv29vsS8rNj3JwcOGzYvkWaKRnsM79s/44Jz + vPnsR3Avig42cTf/JJx1jTJUn23R1tux6x328jtftzH6S5zrOZGHfF0/+45pXTpeOAI9Ft/CuV/4 + BTNRxwt20JVU6Zdrvitpj7HiybfOrfMzDL8ZpbniFRd8ZvIe+GNfhH9MeUS/r9/9jb5ZA4byeTVs + 4PBr9uHOqmzJc9m9ih6uY7VrhZL3yf6GdX5+/e89/NZP/Vo1ewVeqPHVGf/u/Blnb3/Lt+Pofrs+ + IXFq+QTtNwXfHCeGP1HrUsC/iytbl5hvAtl46QXxvlzNo40BLPtGPzAc7S/avB25r8HZ/fgK2lZc + ITipbi+063ymrw0l2OMGb+XruEMdvb/DRWcV0Hm90KvqslzVz+i3Agw89cp3n/CeL3yYIFaq1Ln3 + ec34W+A/9bAptaPGxXkmf+W9oV7uNxOWZa+HVkiIwyi4eeohiIJoi0d5TOM6eY+LlvGLoCz6zaON + vBx5YOu8lcf5zLuvv/srnNIx6/n5az/4y3/9Fz4e/4v5lPiRzlfmG5Ya+YKzR86vP/OLsNafhG9x + cdl1uJkJjrgK9GHw8Hfi500iOA4JhMuPzXErRNf+5Fu8ytf+2pfdBcy7hzP3pVDdgixbaetWLD/r + WZcw+7t6ycNCmZfxIgpvTNqpOzgCPRbfwj39DWvpFZ2s87hPQ79SUpPPCTj88nFYVyrPcev3qs5v + xpkBr+NeKM+yBVEe5OO98ISZi4Vb5046VarQtc96idc2/3Do6z/4rE/4oqv0sGJl6ya7zqvx+s0/ + 8svOzh/9Pqj3qfomUeL7NE6X7G8O9cnDxvKw0uHY1uHwEOXHSy/or8OW359Myw+gAMXDS1N2K9j+ + opWj4qSqL4nrBM8NfoHMyyX5NI16vaEEyh+76nE7h7hDns6/45/9rHjSK5wT8JNf9e3ac5xiji+H + 8+s3SdGLF+uegVe+MTsfESfGgT/35MPKM+mL9wYe7kdX4steD62QVL0FNw8fCuyPYe0QHU2qeJpH + ASK23zzJZIcJmkd5Fq77SF4p7jqUV/uwr5//xAef8azP3X7vr37C/qrjXd2Pwbha37BSOP5OaBzK + F+JE36lD5ZtFp7DEx0bQmnP4u0PxrVj+uiQ+HGz3godm2/HJ58Oct2nPX3HMoPp60qVjnOsSQAk6 + flwWejXSJ4zENT7tJg/ju37miWlgxwtHoMfiW7inv2EtvaOTBfX5SeahXympyToDh18+DusKxvJj + gsO6130ouxxzEg8DdR/NIIJ5nqf+cVB+8Y57cLhPro90WpkdibzPelHf9e2d2513fuFVfFixYNZ6 + dcffeeuLz+64/t2Q8eP7E1Zi31iyvwFQfF4engla07vYLe4+seXHi4EiW3DG8xOQ8QNYdudpv8J9 + CYBf28xbG84g2/EU/ug3j/JySSJN6UdmJ9jjXK/LPcQd8nT+Hf+xX6Yxj8J3dWQf86692q8yd1PO + oWa/eUteAFX3nPkmQn7harawRJ4YB/6c+4eVZ9JfVC/3oyvxZa+HVkiq3oK7nvTFsHaITulIh19U + U25RDVz7zSN3XoqgebCIjoS0HiJWBkVmH3+I+77za3f+5g/93k/5oVBetflqfsOKSl/xGT90fn72 + RfgbDR/R4eby0i/RA/Rh8NQjfvDB2Z6HhthxK3KGjq/LAL/iCCSgbIfFrhraX7T1NnQ8Mb505qdF + O/xaInBdwsQ13vBVB+NdCIIZF3PPqzzgzVh8C/f0N6zDOVAsCTruUwmYcwFAO7aBw6++F/aUHxMc + 1r3Ot+xyzEk8DGR6Dy4qDivnWXaBxO88oit74da5k06VKjT716+ff+Ds2h1fdJUfViz4aj+wWOGX + v+i1+Jr6ZVD5Ok9xffJT9DW8X58o/397VxtraVWdzzn3zgxQgQEKSP1ATSWhlaaJ/dNUrYxNTEYL + CRoxVRtpy4/GtJo2TfnRxFT7o/5ppLRNjKlaCtoMatKG4h/LxJg2/eE/EezwVQlWKylcYGBm7syc + t896nrX2Xvs97713mLngPTPvvtyz9seznrX22muv+56Zwx1Mc+xJZyiNJZkCcWZ+meMMDceksAkA + gxedMpZ+jN2Hsk4YckcGpE8P8JL843rwO0faX+jJH8P17DA7MccF43W7Diz6xJnHapVPhBpL3/YL + wrJPjghLfocjblDLDJQM+GuxT5z0xWfxpZVWZr8jDiGl6MwutsCXfQLe2BvSAyb8LUaS36FvaxVX + 48dzRdyIc37t0PBQwoL88Tj42BeyAEvwhiciCLv84QLCGBcU/ZW94keKn+yHP0RQ1ebn3WSOSnDz + 0ZvecH/w7VS58wuWRe53rrPPZ/2xnb5+omEOh5Gb5rXOI7Z1HqJwXMdC/EShuoCkibHW3Y7jQSR7 + jb7xJzdor8CQUmY3+ZvGUPN1w8s/OpH2F/ONvwYKO8ZHorATQ/EVfeIMqFb5Km58wuqdg4WKcU75 + lOLnkaRQnIHDl45DcQWBr0NgQXHfLK/8/LKeGEiQz1PnFXYIUh7RTsqDXj7JP6NjT+zMv+kfHnvf + m/5FTDv7dTkKlsXwlrd8Fn8i+Jnyk4VBr8HVvH7y2FFy7ElnKI0ltY7J0qlnaLjxCcvjxbhZ9Hpx + tSmLP+JnqR+X27rRdB7Og3PIY4YdwEb6eeqcHJ/0aC/ITW6B1znKvy3tkI47qRa4v7RvX4l9+M59 + X8Dhi3bcr7oORSzIn9iXxr6QBXnMVKFhhF3P5mmnjg1rTXGTXvEjxa/Eg7UqIkLVPz/2/jf8LXtL + 8NI7pSXw+AsP/NlsNvn00CXhKfsh5UMrp8/D0uFqHfstHfSbdSUF6JQMyro2uZBVWve4MclT0iwk + lwyU5Cnm/Rio73bNHc/agndY7IfryX83X/QanLtoovKJsI4dVPaRL+GiXw1/EwiEKc4hS9DT3SEZ + +kPS3WqE47bFTkPsg8yPKfObzeZTK09OQPB0uc5egetYbd3yBfoCYh0dH5fpsFTMWEd6Yk08fT8q + ndvxCeDkp8yFfTB98uhNV38q0ez4rsVg+doXH7gVSfA5OF/816Xzw8SOdJktOQThOrKiuZzMhVj3 + 3AE+P2FJ33ko+usePmalzJFWWVHsowNgsr+wLh76bd3sdx0WAy0u8fb1enbq/mPf/f3UeXqkYYpb + mbDthJupQ632hbcRei7L5bF94VvxSjLOKUkaMv2h5rzBH/K07GR+513goR/hORR8XN8WBon76xsU + jxcfhqMsMBA0Z3SWJ0FLqoQr6+Lhcrw4QeFBh+dtE2ixD3woFBOz3zt60+u/EKrLIi0uy9m++MAH + 4fyXcD/3lA2waPQuL0/Pt8kssLtll9TvWOnYBL4x1noctsa+kIUng/OYE7RfBOhSMbB1N1DsY0bm + wz/ISDJDG5/L7CbtxHpacPNFr8GRSS/F/mb8JQ7hRy+uRpX0o0iEGV4OrDcSi3R3SMY5DckgzdJx + DX/YG+LHHI/3dPidj+ZNPzXaj3PGvI3dElHVHOJnOE4QKJyPyzQjZOtU947HkaPEExCbB4EdR+Fh + xydiHRLF6hj8+MiRm66+J6kvTVc3Ymnc7Tn6Dw9cP+26r+MA/HfD+2ECxsseh+ZjHaoVEb9rpaOx + 4LZuh2+HnYA+1mXP68BYK+uupuvhhgjAi/RIu7BuGIP7kbiUvUwjv1pc4u3r9exUPtnROO+nzssh + vtIvhSP5h4lwt3aEb17jHFzyHABQHAYkcPQrSfKb/lBzXt1Wi48cOy07mX8jHvphcXB/fDz8ZOQw + uoV94asWF48A7bjbhBtOYwjr4dv3xVHlsdXS+jwYRxwNg3/M+NlusnLj0Zte962is2Qdi8tyty8+ + dN10euKbSIQrIqmbS8lD9G2aYC4PFy1liYpGFC2pF0XX9xTCbc30sq9LrBRLxY9RFk/xT+4wqbSM + 9UgyTBjOWsFrKAOxTkOBq365ootQ5DDxhV6Kh0HMbhMnw3nyc9n5kn9tIGwb9bLQfxuTBfRDMgI5 + JIFfaI7bFjsL5JjI/DYMjM2npqLo54x5G9cdFhrFAzvnOgPgONpxnOmHpWJGONnRehS95Ib4wet0 + bscnjHc+f+rk6vT69Ruu5m/4zbrL1F+evyXcKKq3XPvdbmX2a7hkj9hx1ssdp2fZYXfQxpJMAZsu + HZvXWPriCbwr9vSDz9jRCr/T+rWkXQHwmvzjuvvFdVt2u+iGnvxx/wwXdky/+G96MbSNJH3iDKhW + +SpOxRnrhImo2je95DcDhanihy1DMeYNjlb1ja/dF90GppGuT73AhxShvda2Bb7sExpb2gEm/C0G + uL+0b1+ouBo/s8Ai0vDEOiYZnvTDy8e+kAV5zFQNp/Gk+AERP0wNF01xkx73m/3vJo/Op7NfXfZi + ZXtVVGPXyyzvfOSK6fwYnrQ6PHHZoXoS5suks2dyah2g0jEFjaXvyYp1JSk7bXKZHZ+GpgHb9YXk + koHin8w5f+inS2J8nI79cCg7Pp/9d/MtX+Bc1USxvxm/7xvC+Rb9wgJZGZ8mEAiDF5tGAk2+IRn6 + Q5JWei+Oa/jhT30Sgd+naqdHzWHmd54yn/CyZ+eD+BjO9IplG1v8PB6WD5wgUDgfl2lGyNbxzSa+ + ui+z4zwBgeQ67bh1N4z57xzZvWv/ZP9VTyX40naX/wkrQv/bP/+TrrvgHUiOgzxiZYmyBRhdKkmt + Y7J0HIaxLrMup61TTx3ite7zXHcHaE88pPWklb5hdH2KPsfB7xxIssBnST5TtxZ2TL8smF4MBSz6 + xBlQrdg3BTSNpU8+5y36hMkvmZNeGNQyVpxPVuq+yNPbF3kAbGTyx255X6/PH+MGF3plX/DjVOw4 + HqI2BjTt21ciLmKOfQKHryY+tGzrUGR4vKiRt+oRwPWIh+Lr4YCyCMKu7Hh8qregAS7zYIzfuvCt + Iyurv362FCvbrqKTNr703fse3jP7ybEv4QQ/GMmsrPHdluSxpMEcxsPrnqyOz9nguQE1+0lX1cnj + Y9IyiXNy1eTTepj3YyCx26Vbmqed5Gb4G0laxwN8iQddtsq3CT/3kS/hol/VLniaQNgw4pckrDf7 + zuPQH5Jyu3113LbYaZk1yvyYMb/ZbD412o9zxryNkRHWI8ppFA/DcYJA4XxcpsNSMSM+2TFWxDN4 + aEEvXI9jwBT+R+Z7jux+/Ucm+6fHEmzpu8rYpd9GbwP4P6Zn//jQ5/A3iLfmS6VDjUsIHeaCQuC1 + AnBb1yWr6wASYKK/7rbLutNG0to8m0m/vOhZ0hFZ1gkiP3s+X4uL1rMfFZd4+3o9O5Uv9t3fT50X + v+z29aofyS/vLoh6axnHevksDhGvJIGnvSRlr9zi1kSPH1Xh9O1k5o14bL54bhvQuL4tDBL31+Ha + N+INvB2TSfKY5DhYsR60pEo4m7c8wpf0CdCL8+CjC589csPr/ghGTPGsahaXs7at3Png7+JQ70Dy + n89NMguQFMgWpoCPmT0GaNbj0gjP68QsszFH4rFcMz1r/XVLKkuuCgDIeQ3Okak5AfXTus83/kIn + DFKvbKSYb/kIDwdNWfakpvlBfu4r4iRPKy7563y8tLEPzOlS6lJRzy6pzRt+SPKyYWVIAr/QHLct + dhbIMZH5bRgYm0+N9uOcMW/jusNCo3gYjrwECufjMh2WihnxyY75YXF0HjBEk93JEfyDER8/euPr + Ph/zZ5v0zDvbtpX2c9dD181Odl+bzaZv5qHjUjEFyq1RCLxW8LKXIpOBBMRlV1ERkdsq60ruxSco + GSyXHklHpOmlZutsLgs+YMVOxm1S5Hp2Kp/0Nc77qfPyI9yJuCW7iE+4WzvCN6+8jdBzWS9fvdrG + qqtpMN9PkuQ3/aHmvMEf8rTsZH7nXeChH+ExFHysYuJD8ri/vjHxqNjwGIOHdqQnVuw/aAsPZoiH + wFe/aIH70RPT1RvXb7hqqT+2wO1u8nL2/KH7Rpv88LXfnZ+YvRWfQ/mqZUFJvno7lBIDSRV4XquS + VJFMUKg5VLLNYUwqA5g9NRko9nk95U9xHVkc+Cyb5A0/TL8smF4MZa/oE2dAtWLf/dJY+uRz3qJP + Ovklc+IPg1rGStlntWM98vT2RR6sNTL5Y8Wpr9fnj3GDCz232/CHvSE74SdkaYZzvuCxtYgLeoRq + DH/xRVyJQ6wDhgXDlSLjY1/IgjyyQ3q8GI/Hg6PEE5DJ5K4Xj+355bO9WNl2LcbnTLO3iEjCO7Dh + 85GLngsKgY2Vo558nAAmgD5efCLx8JX1SDE3QH3DaFyecHwsfucwVOBdFrzclD+8Sz5BHJIYkqnd + 1+vZGcZJ3wnoTPVDvvX16Df9cN/dro9a4ZffA+yXN+I0IO1y236SlD3b4UDr8Z+RnUzvvCo2yU+b + j7ga3se1aAWJ++vwXLQsXDYmD+2IRlliRclpSZVwNo8vK374g/Wj+F1Wt+It4F2EnQMvnvXnwE5j + i3iLuNp1X8ORv5l3jLngYVC2+OWPSxNFBEBmWRF+qTQmfX8dScXkoiFDyEC5/JiR+bAPieQt665X + x7RSDNq8E3ChDhOfWS32pV/5hKvjxM/t5iK46BfjEfy8dG4Xc7nYkN/2ZfOGH5KhPySBX2iO2xY7 + C+SYyPw2DIzNp1aKEBDcF9djh4VG8QCCRUpAsKBDO0WAxfmLGet48UavFKv5/Jx4C5hCze7Z/5aw + v2O8RTyxPnsrcuBLzI1SFJgNXix0OW1dl50dyxZf93mfpgmvFqVoWDJCQfqGqGOoYWRj52EPL0je + wGdJvOCmID9MvyyYXgwFLPrEGVDN5qVWcSqqWCdMREWfMPmV9cKglrFiDqRW9Y2v3Rd53JxpZV7q + BT6k8fb4Y7wR3uZP2Q7pW/+1v7Rv8wEt9oVeGgOHr7yPug4YFuSPx8HHvpAFeWSH9HhRhMKuv628 + 68WLzo23gBGFkL1TiulzQ67c/dBH8cth/w7JMPC3iJ6smyWXXQque7zsUvkYwpPYk5SQmnxaJ7xe + Auq7XdO3sUvi47TSvN8Sxw3wuT4B/qLLswU/95Ev4aJfcNDtQjaBsGHEL0mgm33ncegPSfe7EY7b + FjsNsQ8yP6bMbzabT+2VfMKCF0dPTrtbj7733HkLmELNblyB/vy5M7a3iJPJ13CJ8beIusTlbRxv + l4WICy7sEusSMosjgiw2pq/k1hOU65VQp0vvPznj0kfAo0jFfC0ujih23DCLRuLlOPYB2bNT+aSv + cd5PnS9uo9PXi426OW08NtGXfvmjqOmSR5wGJPC0l6Ts2YEMtB7/GdnJ9M674K/NR1wN72O9XfMh + edxfh5fiBjyPMXhMWt5IKL+C1nnw0cLvTbvpzYfP8r8F5HY3efGs3wRxLiz9/fcvXNkz/Wt8zu6j + VoTyJWYSeVFQluluMgcxr3UPErMwrSMLS3EjxMKdigtHsqdlZW2/ONQxUTKArs2bv/TLRfhFJP1O + /JzUOOMG+QGo8z2/nbfald8xNjO8nMA10ubxLbaejEAOSSPsN8c1/GEP2Jdkp89t48zvfITZfGql + CNm5Yt7GdWeFRnGwfCAvgcL5uEzTc1vHNxsz6K4XfuaqW/GvMB+N2XNVWozH5hFYvfP7+6az6R3d + tPuFthgge+ySUqQiFLlp+mVdudZ/sokkLkUgrq3ppUa7Nvb5gg9YseMTxKUi2Nfr2al80tdY+rwk + SZ9uFTO2b3OrTHg8iCr++qgVvI3Qc6lLHnEakMDRryQVX/NgoPX4z8hOpt/IX5uPuBrexy/DE9aD + k9nsDw7vf/WO/9dscthezr5n38tpYsm4D3aru/7nvz6Gvy7+FD5sepFy1sKEJLXLWoVfqnRX++v2 + ExVf5ZJ7kqtI1JQv69TfpPjEaXnRoB7vjhbcfLWXcekYiv20nmjSPnORWvSL8QCv/PDAuB0WJfA3 + UlEsV92jqrEXhyg2jUy+l67jG/6w91LtFNLUyfzOx1WbT03F188Z8zauJ6taZmEuOPISKJyPy7Ql + 2GTyHMafxFPV3+Cp6oRNjE0RiCswxqMfgQOPv3rX+vrtk9n0A7lK6bJ7EYrcNF1mZRG4hH4dbZ5N + 41IsFtYdFXiXBV9o0IFdFglTIW6xmBS9np0y3/Dn/chQ5a9+cbvZP/qhdfnh/b7gbTS/paDLy22E + d60Ejn4mSX7TH2rOG/whT8tO5nfeBR76YXFyf3y8HU9YID2A3wr68Rfec8WPsytjXxFQdo7R2DAC + q1/G28RudscEbxN1aVJRwuWNu0ICu8x+iS2VrWiNT1ip+CAmisuAjEAOyaHTcRyLCc/hDOxsxe9+ + E2Z2U1Mx25YnrAeRX+PbvxTboe5YsIai0p+zt4n/+/DHupP+NhHXjk8gcfsMP1CsSvUin34ilyec + eKYwvdTqk43mCz5gxY5PUN8vK3hCv+j17JR5t6tx3k+1S7eKGbuUlT/vV7hwkKP2pVeEFp5YjBff + Ec6hIiR7bbEoRnr88VPktOwUUnMI9hCnBR4WrfDYcfHDKYbkcX99Y+JxPqjbGAaem8/n49s/xmvr + l02ybGvlcw5hbxNPrN+OS/4BphqTmTmtUFgRwIILpLD/5LUJNpOpuHBkeF+nYlr3+VpkxEID6FJP + jnCB6jFvM0mfAH+pfLJbxwXg+8hFatGvhj+KhlPwcsJ+I7FGd4dk6A9J52yE4xr+sDfEjzlG/3T4 + nY/2TT812o9zxryN3RJR1RziZzhOENfBnzvn3cpt49u/FNAtun5TtkCNy00EVr/8yL7ZrLsDtw9v + E33Jq4ULJqff+gBApktPRSS3KaTGImRjn9+smFCNuMTb1+vZqXyyq7H0VU3qvPj5Cndy8XL/svtu + V+jea7213JcuOaMT3rUSeNpLkvEwnqHW40dVOH07mX8jHvphcXJ/fHyqf4aF/wfwwenK7Nbn333l + f2RzY3/rCLS3ZWv8iIgIHOhWdp98+Gbk6p/ioxC/xGm7tH6JLZXHJ6zxCctrnp6suul38fuqPvPC + 4Sv+afKB6clIpVGeegTGgnXqsdoQufKVR9+DJ67b8Psd3+Y1i8WqVC9q6idyeVKJZ4rek4mtCy5Z + 8HFSpShm3PiEVZ7AGJ+d9YTVzSf/PlmZ/eXz77783g2TaFw4pQjENTgl8AjaPAKrBx59G35Z4G3T + 2eQ94xPW4ts6SzbFZUDWRxG9Hc7jobD7Ot9eokg18qXa2Yrf+Qgzu6np7a29XcZ+MW9je7a2HaJ/ + 34lu8ukj+1/9n0ll7J5BBMaCdQbB20h194FHr8P/VH0bPgpxM56QVoRTEpcnpvEJa9PixTjh8ocs + T1BDQffihQrRFDsVk4HiCI4oLsEfsqF3vgUem4/zMwUfW9HCf3ir1x2Yz1f/4vD+yx5s+MbBGUdg + LFhnHMJNCA789xt3zY//yXQ6uwUPAedFkvNyQE0lDNLexljztzNl3efrWDDiCIce7470qR7zBk36 + NoxW+UIv/kDdEU7U4hbfdjb8USycgpccPI3EWr7qZr2MQ39IOmcjHNfwh73Mi/6WdhpiH2R+5+OK + zaemYmbxm78AS59fX9312aPvuuQHCTJ2tzECythtJBypBiLw5ceu3L3afQIrvz+bTi/WJfVr5EUl + tKxIsLmsRcMRNg+CFrdYTIpeXNcFPtkRTvqqHnVefoQ7UdSSf/Qj+eXdBdErQrrkqVhBwVgVF0jg + 6VeSLI69YlHs9PjjSeu07BRSc0gbXOChH/IY3afxj2rdMb1gz+3Pvn3vM1l97G9/BDz7tp94ZByI + wIFHL95tRWsy/QQ+hHNluaS5SOEGlGKzUGScM823RcYvfeaDivHldkr8XhRZRKi/6BeLSPD75Q47 + Q0Wn7BegUpxM38ahPySDNEvHbYudzBv9zI8585fN5q11kyfR+6vn1n/2c5PfnL6oyfH15Y5Am8kv + t7WRv0Rg99efeMvsxPEP4dL/Fr5fHwuluKSixMsdJ2XzmGhxi8WkFKV4dlngE6Fw0lcVqfP0ye0W + PudhsaIf7nnMx0ay7BWhhScWYM0M92kSeNpLUvYMMdB6/Bs+GUF1SzuZ3nmTv0+A4CsnT87uOvzu + Sx/I0LH/ykTA0/GVMTZaGYgAfjPbeV99Yt98Mv8Q7vz7cCAX9YtDHbt+Kj5tkfFLH8Uj45Lpyqfj + r+PE70WRRYQ8i0WRRQQqph9FIswMFR2zVoqS6eXxBkWHvEGaZS4msN/Yy7zob2kn80Y/+OfdM/jL + k3tOzlbvPrxv77ex2Q2qZiiO8uWMgJ3l2HZKBA4+ft7up2fvXem6D+Et434Ugt2bFRO6vUExKXpx + XYlTcalFKMbjExbjZUUKDWIdJfAb+Fly97OvufSfJ784XefC+PJTj8BYsH7qR7CBA/f+4JLz1ifv + xyPJh/FJ+rcDZXdKLRWf8Qlr8e1j80RlgcN3ebLzJ6fyRBhFChDgvg22uyd7ZveMf4DuubbDRFyB + HebW6E4Tgft+dPn568fxa266fd1sdj3elOCfKPOjo1x8uzY+YfnbRASyFCv0c/HCb0l4ZDad3Y8P + Th2c7971b4ffceFTTdzHwY6LwFiwdtyRbO3Q+fc9+dru+ORd+Dfa9uE22q91fq1pWe3Kb/c4wfn2 + mEsxS09q0nPbTtTiFotiwx9PLk7R/JkS+NIfXDdFoxST0B+SztkIx700O92T3by7fzJbOXhsNv/m + kesve7LhHAc7PgJtJu94d0cHhyKw51+fvGblJJ68Jp0Vr+uBubwUmygPC8VJRy+cipFXO5qweXVc + WNFBt85jHRMBqx3hm9deETqV4kW/oBeS/MYz1Hr88Xavm89/BP2D80l3cNKt3P/sb+x9bEh9nFue + CHhWLo/Do6dbRAB/Uvyqb/z4WvwPItdPZngCm3TvxCftL81aLAKYMGmtjjlUFWIxykVq5z9hwfGn + 8Ynzb+Fzbvcfn3YHD++77Hu+o1GcJREYC9ZZcpAbbgMF7Px7/+/nVqbHr5lMZ9fgf8y+Bk8g7OPh + 60345P0q/8ddK156hCJVFDM+oGGmFjVPGce/0k9YcPEE/Hwcny44hBJ6CNs7BPcOrc92HzryzvN/ + CEc3eAzbMELjwhJFYCxYS3RY2+4qfvXzRcefeuNkPkUBQyHDNxLiGpSna3DrX4Nixtuv4vXKPWF1 + MDvt5j/sJrNDkIcwftgK1ImTq4eeX7nwsfFfktn2TFgawrFgLc1RvcKOfqfbdeHzz188Pd5dMpkd + 3zudrO6dTE/unUxWICeXwJu9+EN/jPGNMYqa+jEnd9cg1vBEt4aiszaxD2HOpmvzebcG3TX8Mru1 + ldnkGfwt3RrK4Rp41ubT2drhC161NvmV6XFRjK9jBGoE/h//xb5CiJqhJQAAAABJRU5ErkJggg== + installModes: + - supported: true + type: OwnNamespace + - supported: false + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: false + type: AllNamespaces + install: + strategy: deployment + spec: + deployments: + - name: clickhouse-operator + spec: + replicas: 1 + selector: + matchLabels: + app: clickhouse-operator + template: + metadata: + labels: + app: clickhouse-operator + spec: + containers: + - env: + - name: OPERATOR_POD_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPERATOR_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: OPERATOR_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: OPERATOR_CONTAINER_CPU_REQUEST + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: requests.cpu + - name: OPERATOR_CONTAINER_CPU_LIMIT + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: limits.cpu + - name: OPERATOR_CONTAINER_MEM_REQUEST + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: requests.memory + - name: OPERATOR_CONTAINER_MEM_LIMIT + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: limits.memory + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: docker.io/altinity/clickhouse-operator:0.23.1 + imagePullPolicy: Always + name: clickhouse-operator + - image: docker.io/altinity/metrics-exporter:0.23.1 + imagePullPolicy: Always + name: metrics-exporter + serviceAccountName: clickhouse-operator + permissions: + - serviceAccountName: clickhouse-operator + rules: + # + # Core API group + # + - apiGroups: + - "" + resources: + - configmaps + - services + - persistentvolumeclaims + - secrets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch + - delete + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + # + # apps.* resources + # + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - patch + - update + - delete + # The operator deployment personally, identified by name + - apiGroups: + - apps + resources: + - deployments + resourceNames: + - clickhouse-operator + verbs: + - get + - patch + - update + - delete + # + # policy.* resources + # + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + # + # apiextensions + # + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + # clickhouse - related resources + - apiGroups: + - clickhouse.altinity.com + # + # The operator's specific Custom Resources + # + + resources: + - clickhouseinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallationtemplates + - clickhouseoperatorconfigurations + verbs: + - get + - list + - watch + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/finalizers + - clickhouseinstallationtemplates/finalizers + - clickhouseoperatorconfigurations/finalizers + verbs: + - update + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/status + - clickhouseinstallationtemplates/status + - clickhouseoperatorconfigurations/status + verbs: + - get + - update + - patch + - create + - delete + # clickhouse-keeper - related resources + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/finalizers + verbs: + - update + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/status + verbs: + - get + - update + - patch + - create + - delete diff --git a/deploy/operatorhub/0.23.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml new file mode 100644 index 000000000..f9f6b42f6 --- /dev/null +++ b/deploy/operatorhub/0.23.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -0,0 +1,1210 @@ +# Template Parameters: +# +# KIND=ClickHouseInstallation +# SINGULAR=clickhouseinstallation +# PLURAL=clickhouseinstallations +# SHORT=chi +# OPERATOR_VERSION=0.23.1 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseinstallations.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.23.1 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseInstallation + singular: clickhouseinstallation + plural: clickhouseinstallations + shortNames: + - chi + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID + - name: status + type: string + description: CHI status + jsonPath: .status.status + - name: hosts-updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUpdated + - name: hosts-added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsAdded + - name: hosts-completed + type: integer + description: Completed hosts count + jsonPath: .status.hostsCompleted + - name: hosts-deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDeleted + - name: hosts-delete + type: integer + description: Hosts to be deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDelete + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + schema: + openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters" + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + type: object + description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other" + properties: + chop-version: + type: string + description: "ClickHouse operator version" + chop-commit: + type: string + description: "ClickHouse operator git commit SHA" + chop-date: + type: string + description: "ClickHouse operator build date" + chop-ip: + type: string + description: "IP address of the operator's pod which managed this CHI" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" + status: + type: string + description: "Status" + taskID: + type: string + description: "Current task id" + taskIDsStarted: + type: array + description: "Started task ids" + nullable: true + items: + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + nullable: true + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + nullable: true + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + nullable: true + items: + type: string + hostsUpdated: + type: integer + minimum: 0 + description: "Updated Hosts count" + hostsAdded: + type: integer + minimum: 0 + description: "Added Hosts count" + hostsCompleted: + type: integer + minimum: 0 + description: "Completed Hosts count" + hostsDeleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + hostsDelete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + nullable: true + items: + type: string + pod-ips: + type: array + description: "Pod IPs" + nullable: true + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + nullable: true + items: + type: string + endpoint: + type: string + description: "Endpoint" + generation: + type: integer + minimum: 0 + description: "Generation" + normalized: + type: object + description: "Normalized CHI requested" + x-kubernetes-preserve-unknown-fields: true + normalizedCompleted: + type: object + description: "Normalized CHI completed" + x-kubernetes-preserve-unknown-fields: true + hostsWithTablesCreated: + type: array + description: "List of hosts with tables created by the operator" + nullable: true + items: + type: string + usedTemplates: + type: array + description: "List of templates used to build this CHI" + nullable: true + x-kubernetes-preserve-unknown-fields: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md + properties: + taskID: + type: string + description: | + Allows to define custom taskID for CHI update and watch status of this update execution. + Displayed in all .status.taskID* fields. + By default (if not filled) every update of CHI manifest will generate random taskID + stop: &TypeStringBool + type: string + description: | + Allows to stop all ClickHouse clusters defined in a CHI. + Works as the following: + - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. + - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + restart: + type: string + description: | + In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. + This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. + enum: + - "" + - "RollingUpdate" + troubleshoot: + !!merge <<: *TypeStringBool + description: | + Allows to troubleshoot Pods during CrashLoopBack state. + This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start. + Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts + and give time to troubleshoot via CLI. + Liveness and Readiness probes are disabled as well. + namespaceDomainPattern: + type: string + description: | + Custom domain pattern which will be used for DNS names of `Service` or `Pod`. + Typical use scenario - custom cluster domain in Kubernetes cluster + Example: %s.svc.my.test + templating: + type: object + # nullable: true + description: | + Optional, applicable inside ClickHouseInstallationTemplate only. + Defines current ClickHouseInstallationTemplate application options to target ClickHouseInstallation(s)." + properties: + policy: + type: string + description: | + When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate + will be auto-added into ClickHouseInstallation, selectable by `chiSelector`. + Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. + enum: + - "" + - "auto" + - "manual" + chiSelector: + type: object + description: "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + reconciling: + type: object + description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: | + DISCUSSED TO BE DEPRECATED + Syntax sugar + Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config + Possible values: + - wait - should wait to exclude host, complete queries and include host back into the cluster + - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + enum: + - "" + - "wait" + - "nowait" + configMapPropagationTimeout: + type: integer + description: | + Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` + More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: | + Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, + but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. + Default behavior is `Delete`" + # nullable: true + properties: + statefulSet: &TypeObjectsCleanup + type: string + description: "Behavior policy for unknown StatefulSet, `Delete` by default" + enum: + # List ObjectsCleanupXXX constants from model + - "" + - "Retain" + - "Delete" + pvc: + type: string + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown PVC, `Delete` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown ConfigMap, `Delete` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown Service, `Delete` by default" + reconcileFailedObjects: + type: object + description: | + Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. + Default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed StatefulSet, `Retain` by default" + pvc: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed PVC, `Retain` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed ConfigMap, `Retain` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed Service, `Retain` by default" + defaults: + type: object + description: | + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + replicasUseFQDN: + !!merge <<: *TypeStringBool + description: | + define should replicas be specified by FQDN in ``. + In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup + "yes" by default + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + storageManagement: + type: object + description: default storage management options + properties: + provisioner: &TypePVCProvisioner + type: string + description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + enum: + - "" + - "StatefulSet" + - "Operator" + reclaimPolicy: &TypePVCReclaimPolicy + type: string + description: | + defines behavior of `PVC` deletion. + `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet + enum: + - "" + - "Retain" + - "Delete" + templates: &TypeTemplateNames + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + zookeeper: &TypeZookeeperConfig + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ + currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` + More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper + # nullable: true + properties: + nodes: + type: array + description: "describe every available zookeeper cluster node for interaction" + # nullable: true + items: + type: object + #required: + # - host + properties: + host: + type: string + description: "dns name or ip address for Zookeeper node" + port: + type: integer + description: "TCP port which used to connect to Zookeeper node" + minimum: 0 + maximum: 65535 + secure: + !!merge <<: *TypeStringBool + description: "if a secure connection to Zookeeper is required" + session_timeout_ms: + type: integer + description: "session timeout during connect to Zookeeper" + operation_timeout_ms: + type: integer + description: "one operation timeout during Zookeeper transactions" + root: + type: string + description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" + identity: + type: string + description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" + users: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure password hashed, authorization restrictions, database level security row filters etc. + More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers + # nullable: true + x-kubernetes-preserve-unknown-fields: true + profiles: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of settings profile + More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles + # nullable: true + x-kubernetes-preserve-unknown-fields: true + quotas: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of resource quotas + More details: https://clickhouse.tech/docs/en/operations/quotas/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas + # nullable: true + x-kubernetes-preserve-unknown-fields: true + settings: &TypeSettings + type: object + description: | + allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: &TypeFiles + type: object + description: | + allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + every key in this object is the file name + every value in this object is the file content + you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html + each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored + More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml + # nullable: true + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level + every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` + all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` + Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zookeeper: + !!merge <<: *TypeZookeeperConfig + description: | + optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.zookeeper` settings + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` + schemaPolicy: + type: object + description: | + describes how schema is propagated within replicas and shards + properties: + replica: + type: string + description: "how schema is propagated within a replica" + enum: + # List SchemaPolicyReplicaXXX constants from model + - "" + - "None" + - "All" + shard: + type: string + description: "how schema is propagated between shards" + enum: + # List SchemaPolicyShardXXX constants from model + - "" + - "None" + - "All" + - "DistributedTablesOnly" + insecure: + !!merge <<: *TypeStringBool + description: optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: optional, open secure ports for cluster + secret: + type: object + description: "optional, shared secret value to secure cluster communications" + properties: + auto: + !!merge <<: *TypeStringBool + description: "Auto-generate shared secret value to secure cluster communications" + value: + description: "Cluster shared secret value in plain text" + type: string + valueFrom: + description: "Cluster shared secret source" + type: object + properties: + secretKeyRef: + description: | + Selects a key of a secret in the clickhouse installation namespace. + Should not be used if value is not empty. + type: object + properties: + name: + description: | + Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - name + - key + layout: + type: object + description: | + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly + # nullable: true + properties: + type: + type: string + description: "DEPRECATED - to be removed soon" + shardsCount: + type: integer + description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" + replicasCount: + type: integer + description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" + shards: + type: array + description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + definitionType: + type: string + description: "DEPRECATED - to be removed soon" + weight: + type: integer + description: | + optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + internalReplication: + !!merge <<: *TypeStringBool + description: | + optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise + allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard + override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` + replicasCount: + type: integer + description: | + optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + shard contains 1 replica by default + override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` + minimum: 1 + replicas: + type: array + description: | + optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants + - "" + - "Unspecified" + - "ClusterScopeIndex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` + More info: https://clickhouse.tech/docs/en/interfaces/tcp/ + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` + More info: https://clickhouse.tech/docs/en/interfaces/http/ + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + templates: + !!merge <<: *TypeTemplateNames + description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + - "" + - "Unspecified" + - "OnePerHost" + podDistribution: + type: array + description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants + - "" + - "Unspecified" + - "ClickHouseAntiAffinity" + - "ShardAntiAffinity" + - "ReplicaAntiAffinity" + - "AnotherNamespaceAntiAffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "AnotherClusterAntiAffinity" + - "MaxNumberPerNode" + - "NamespaceAffinity" + - "ClickHouseInstallationAffinity" + - "ClusterAffinity" + - "ShardAffinity" + - "ReplicaAffinity" + - "PreviousTailAffinity" + - "CircularReplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants + - "" + - "Unspecified" + - "Shard" + - "Replica" + - "Cluster" + - "ClickHouseInstallation" + - "Namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + provisioner: *TypePVCProvisioner + reclaimPolicy: *TypePVCReclaimPolicy + metadata: + type: object + description: | + allows to pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + useTemplates: + type: array + description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "name of `ClickHouseInstallationTemplate` (chit) resource" + namespace: + type: string + description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" + useType: + type: string + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + enum: + # List useTypeXXX constants from model + - "" + - "merge" diff --git a/deploy/operatorhub/0.23.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml new file mode 100644 index 000000000..7175710cc --- /dev/null +++ b/deploy/operatorhub/0.23.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -0,0 +1,1210 @@ +# Template Parameters: +# +# KIND=ClickHouseInstallationTemplate +# SINGULAR=clickhouseinstallationtemplate +# PLURAL=clickhouseinstallationtemplates +# SHORT=chit +# OPERATOR_VERSION=0.23.1 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseinstallationtemplates.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.23.1 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseInstallationTemplate + singular: clickhouseinstallationtemplate + plural: clickhouseinstallationtemplates + shortNames: + - chit + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID + - name: status + type: string + description: CHI status + jsonPath: .status.status + - name: hosts-updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUpdated + - name: hosts-added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsAdded + - name: hosts-completed + type: integer + description: Completed hosts count + jsonPath: .status.hostsCompleted + - name: hosts-deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDeleted + - name: hosts-delete + type: integer + description: Hosts to be deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDelete + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + schema: + openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters" + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + type: object + description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other" + properties: + chop-version: + type: string + description: "ClickHouse operator version" + chop-commit: + type: string + description: "ClickHouse operator git commit SHA" + chop-date: + type: string + description: "ClickHouse operator build date" + chop-ip: + type: string + description: "IP address of the operator's pod which managed this CHI" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" + status: + type: string + description: "Status" + taskID: + type: string + description: "Current task id" + taskIDsStarted: + type: array + description: "Started task ids" + nullable: true + items: + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + nullable: true + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + nullable: true + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + nullable: true + items: + type: string + hostsUpdated: + type: integer + minimum: 0 + description: "Updated Hosts count" + hostsAdded: + type: integer + minimum: 0 + description: "Added Hosts count" + hostsCompleted: + type: integer + minimum: 0 + description: "Completed Hosts count" + hostsDeleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + hostsDelete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + nullable: true + items: + type: string + pod-ips: + type: array + description: "Pod IPs" + nullable: true + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + nullable: true + items: + type: string + endpoint: + type: string + description: "Endpoint" + generation: + type: integer + minimum: 0 + description: "Generation" + normalized: + type: object + description: "Normalized CHI requested" + x-kubernetes-preserve-unknown-fields: true + normalizedCompleted: + type: object + description: "Normalized CHI completed" + x-kubernetes-preserve-unknown-fields: true + hostsWithTablesCreated: + type: array + description: "List of hosts with tables created by the operator" + nullable: true + items: + type: string + usedTemplates: + type: array + description: "List of templates used to build this CHI" + nullable: true + x-kubernetes-preserve-unknown-fields: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md + properties: + taskID: + type: string + description: | + Allows to define custom taskID for CHI update and watch status of this update execution. + Displayed in all .status.taskID* fields. + By default (if not filled) every update of CHI manifest will generate random taskID + stop: &TypeStringBool + type: string + description: | + Allows to stop all ClickHouse clusters defined in a CHI. + Works as the following: + - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. + - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + restart: + type: string + description: | + In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. + This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. + enum: + - "" + - "RollingUpdate" + troubleshoot: + !!merge <<: *TypeStringBool + description: | + Allows to troubleshoot Pods during CrashLoopBack state. + This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start. + Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts + and give time to troubleshoot via CLI. + Liveness and Readiness probes are disabled as well. + namespaceDomainPattern: + type: string + description: | + Custom domain pattern which will be used for DNS names of `Service` or `Pod`. + Typical use scenario - custom cluster domain in Kubernetes cluster + Example: %s.svc.my.test + templating: + type: object + # nullable: true + description: | + Optional, applicable inside ClickHouseInstallationTemplate only. + Defines current ClickHouseInstallationTemplate application options to target ClickHouseInstallation(s)." + properties: + policy: + type: string + description: | + When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate + will be auto-added into ClickHouseInstallation, selectable by `chiSelector`. + Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. + enum: + - "" + - "auto" + - "manual" + chiSelector: + type: object + description: "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + reconciling: + type: object + description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: | + DISCUSSED TO BE DEPRECATED + Syntax sugar + Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config + Possible values: + - wait - should wait to exclude host, complete queries and include host back into the cluster + - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + enum: + - "" + - "wait" + - "nowait" + configMapPropagationTimeout: + type: integer + description: | + Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` + More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: | + Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, + but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. + Default behavior is `Delete`" + # nullable: true + properties: + statefulSet: &TypeObjectsCleanup + type: string + description: "Behavior policy for unknown StatefulSet, `Delete` by default" + enum: + # List ObjectsCleanupXXX constants from model + - "" + - "Retain" + - "Delete" + pvc: + type: string + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown PVC, `Delete` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown ConfigMap, `Delete` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown Service, `Delete` by default" + reconcileFailedObjects: + type: object + description: | + Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. + Default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed StatefulSet, `Retain` by default" + pvc: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed PVC, `Retain` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed ConfigMap, `Retain` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed Service, `Retain` by default" + defaults: + type: object + description: | + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + replicasUseFQDN: + !!merge <<: *TypeStringBool + description: | + define should replicas be specified by FQDN in ``. + In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup + "yes" by default + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + storageManagement: + type: object + description: default storage management options + properties: + provisioner: &TypePVCProvisioner + type: string + description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + enum: + - "" + - "StatefulSet" + - "Operator" + reclaimPolicy: &TypePVCReclaimPolicy + type: string + description: | + defines behavior of `PVC` deletion. + `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet + enum: + - "" + - "Retain" + - "Delete" + templates: &TypeTemplateNames + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + zookeeper: &TypeZookeeperConfig + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ + currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` + More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper + # nullable: true + properties: + nodes: + type: array + description: "describe every available zookeeper cluster node for interaction" + # nullable: true + items: + type: object + #required: + # - host + properties: + host: + type: string + description: "dns name or ip address for Zookeeper node" + port: + type: integer + description: "TCP port which used to connect to Zookeeper node" + minimum: 0 + maximum: 65535 + secure: + !!merge <<: *TypeStringBool + description: "if a secure connection to Zookeeper is required" + session_timeout_ms: + type: integer + description: "session timeout during connect to Zookeeper" + operation_timeout_ms: + type: integer + description: "one operation timeout during Zookeeper transactions" + root: + type: string + description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" + identity: + type: string + description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" + users: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure password hashed, authorization restrictions, database level security row filters etc. + More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers + # nullable: true + x-kubernetes-preserve-unknown-fields: true + profiles: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of settings profile + More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles + # nullable: true + x-kubernetes-preserve-unknown-fields: true + quotas: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of resource quotas + More details: https://clickhouse.tech/docs/en/operations/quotas/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas + # nullable: true + x-kubernetes-preserve-unknown-fields: true + settings: &TypeSettings + type: object + description: | + allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: &TypeFiles + type: object + description: | + allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + every key in this object is the file name + every value in this object is the file content + you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html + each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored + More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml + # nullable: true + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level + every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` + all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` + Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zookeeper: + !!merge <<: *TypeZookeeperConfig + description: | + optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.zookeeper` settings + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` + schemaPolicy: + type: object + description: | + describes how schema is propagated within replicas and shards + properties: + replica: + type: string + description: "how schema is propagated within a replica" + enum: + # List SchemaPolicyReplicaXXX constants from model + - "" + - "None" + - "All" + shard: + type: string + description: "how schema is propagated between shards" + enum: + # List SchemaPolicyShardXXX constants from model + - "" + - "None" + - "All" + - "DistributedTablesOnly" + insecure: + !!merge <<: *TypeStringBool + description: optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: optional, open secure ports for cluster + secret: + type: object + description: "optional, shared secret value to secure cluster communications" + properties: + auto: + !!merge <<: *TypeStringBool + description: "Auto-generate shared secret value to secure cluster communications" + value: + description: "Cluster shared secret value in plain text" + type: string + valueFrom: + description: "Cluster shared secret source" + type: object + properties: + secretKeyRef: + description: | + Selects a key of a secret in the clickhouse installation namespace. + Should not be used if value is not empty. + type: object + properties: + name: + description: | + Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - name + - key + layout: + type: object + description: | + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly + # nullable: true + properties: + type: + type: string + description: "DEPRECATED - to be removed soon" + shardsCount: + type: integer + description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" + replicasCount: + type: integer + description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" + shards: + type: array + description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + definitionType: + type: string + description: "DEPRECATED - to be removed soon" + weight: + type: integer + description: | + optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + internalReplication: + !!merge <<: *TypeStringBool + description: | + optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise + allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard + override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` + replicasCount: + type: integer + description: | + optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + shard contains 1 replica by default + override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` + minimum: 1 + replicas: + type: array + description: | + optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants + - "" + - "Unspecified" + - "ClusterScopeIndex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` + More info: https://clickhouse.tech/docs/en/interfaces/tcp/ + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` + More info: https://clickhouse.tech/docs/en/interfaces/http/ + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + templates: + !!merge <<: *TypeTemplateNames + description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + - "" + - "Unspecified" + - "OnePerHost" + podDistribution: + type: array + description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants + - "" + - "Unspecified" + - "ClickHouseAntiAffinity" + - "ShardAntiAffinity" + - "ReplicaAntiAffinity" + - "AnotherNamespaceAntiAffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "AnotherClusterAntiAffinity" + - "MaxNumberPerNode" + - "NamespaceAffinity" + - "ClickHouseInstallationAffinity" + - "ClusterAffinity" + - "ShardAffinity" + - "ReplicaAffinity" + - "PreviousTailAffinity" + - "CircularReplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants + - "" + - "Unspecified" + - "Shard" + - "Replica" + - "Cluster" + - "ClickHouseInstallation" + - "Namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + provisioner: *TypePVCProvisioner + reclaimPolicy: *TypePVCReclaimPolicy + metadata: + type: object + description: | + allows to pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + useTemplates: + type: array + description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "name of `ClickHouseInstallationTemplate` (chit) resource" + namespace: + type: string + description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" + useType: + type: string + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + enum: + # List useTypeXXX constants from model + - "" + - "merge" diff --git a/deploy/operatorhub/0.23.1/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml b/deploy/operatorhub/0.23.1/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml new file mode 100644 index 000000000..3831166b9 --- /dev/null +++ b/deploy/operatorhub/0.23.1/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml @@ -0,0 +1,263 @@ +# Template Parameters: +# +# OPERATOR_VERSION=0.23.1 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com + labels: + clickhouse-keeper.altinity.com/chop: 0.23.1 +spec: + group: clickhouse-keeper.altinity.com + scope: Namespaced + names: + kind: ClickHouseKeeperInstallation + singular: clickhousekeeperinstallation + plural: clickhousekeeperinstallations + shortNames: + - chk + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: status + type: string + description: CHK status + jsonPath: .status.status + - name: replicas + type: integer + description: Replica count + priority: 1 # show in wide view + jsonPath: .status.replicas + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: + - spec + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one ClickHouse Keeper cluster" + properties: + apiVersion: + type: string + description: | + APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + kind: + type: string + description: | + Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + metadata: + type: object + status: + type: object + description: | + Current ClickHouseKeeperInstallation status, contains many fields like overall status, desired replicas and ready replica list with their endpoints + properties: + chop-version: + type: string + description: "ClickHouse operator version" + chop-commit: + type: string + description: "ClickHouse operator git commit SHA" + chop-date: + type: string + description: "ClickHouse operator build date" + chop-ip: + type: string + description: "IP address of the operator's pod which managed this CHI" + status: + type: string + description: "Status" + replicas: + type: integer + format: int32 + description: Replicas is the number of number of desired replicas in the cluster + readyReplicas: + type: array + description: ReadyReplicas is the array of endpoints of those ready replicas in the cluster + items: + type: object + properties: + host: + type: string + description: dns name or ip address for Keeper node + port: + type: integer + minimum: 0 + maximum: 65535 + description: TCP port which used to connect to Keeper node + secure: + type: string + description: if a secure connection to Keeper is required + normalized: + type: object + description: "Normalized CHK requested" + x-kubernetes-preserve-unknown-fields: true + normalizedCompleted: + type: object + description: "Normalized CHK completed" + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: KeeperSpec defines the desired state of a Keeper cluster + properties: + namespaceDomainPattern: + type: string + description: | + Custom domain pattern which will be used for DNS names of `Service` or `Pod`. + Typical use scenario - custom cluster domain in Kubernetes cluster + Example: %s.svc.my.test + replicas: + type: integer + format: int32 + description: | + Replicas is the expected size of the keeper cluster. + The valid range of size is from 1 to 7. + minimum: 1 + maximum: 7 + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + settings: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-keeper` instance" + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes ClickHouseKeeper clusters layout and allows change settings on cluster-level and replica-level + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of ClickHouseKeeper servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + layout: + type: object + description: | + describe current cluster layout, how many replicas + # nullable: true + properties: + replicasCount: + type: integer + description: "how many replicas in ClickHouseKeeper cluster" + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + metadata: + type: object + description: | + allows to pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true diff --git a/deploy/operatorhub/0.23.1/clickhouseoperatorconfigurations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.1/clickhouseoperatorconfigurations.clickhouse.altinity.com.crd.yaml new file mode 100644 index 000000000..26ccdc89a --- /dev/null +++ b/deploy/operatorhub/0.23.1/clickhouseoperatorconfigurations.clickhouse.altinity.com.crd.yaml @@ -0,0 +1,415 @@ +# Template Parameters: +# +# NONE +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseoperatorconfigurations.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.23.1 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseOperatorConfiguration + singular: clickhouseoperatorconfiguration + plural: clickhouseoperatorconfigurations + shortNames: + - chopconf + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: namespaces + type: string + description: Watch namespaces + jsonPath: .status + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + description: "allows customize `clickhouse-operator` settings, need restart clickhouse-operator pod after adding, more details https://github.com/Altinity/clickhouse-operator/blob/master/docs/operator_configuration.md" + x-kubernetes-preserve-unknown-fields: true + properties: + status: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + Allows to define settings of the clickhouse-operator. + More info: https://github.com/Altinity/clickhouse-operator/blob/master/config/config.yaml + Check into etc-clickhouse-operator* ConfigMaps if you need more control + x-kubernetes-preserve-unknown-fields: true + properties: + watch: + type: object + description: "Parameters for watch kubernetes resources which used by clickhouse-operator deployment" + properties: + namespaces: + type: array + description: "List of namespaces where clickhouse-operator watches for events." + items: + type: string + clickhouse: + type: object + description: "Clickhouse related parameters used by clickhouse-operator" + properties: + configuration: + type: object + properties: + file: + type: object + properties: + path: + type: object + description: | + Each 'path' can be either absolute or relative. + In case path is absolute - it is used as is. + In case path is relative - it is relative to the folder where configuration file you are reading right now is located. + properties: + common: + type: string + description: | + Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. + Default value - config.d + host: + type: string + description: | + Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. + Default value - conf.d + user: + type: string + description: | + Path to the folder where ClickHouse configuration files with users settings are located. + Files are common for all instances within a CHI. + Default value - users.d + user: + type: object + description: "Default parameters for any user which will create" + properties: + default: + type: object + properties: + profile: + type: string + description: "ClickHouse server configuration `...` for any " + quota: + type: string + description: "ClickHouse server configuration `...` for any " + networksIP: + type: array + description: "ClickHouse server configuration `...` for any " + items: + type: string + password: + type: string + description: "ClickHouse server configuration `...` for any " + network: + type: object + description: "Default network parameters for any user which will create" + properties: + hostRegexpTemplate: + type: string + description: "ClickHouse server configuration `...` for any " + configurationRestartPolicy: + type: object + description: "Configuration restart policy describes what configuration changes require ClickHouse restart" + properties: + rules: + type: array + description: "Array of set of rules per specified ClickHouse versions" + items: + type: object + properties: + version: + type: string + description: "ClickHouse version expression" + rules: + type: array + description: "Set of configuration rules for specified ClickHouse version" + items: + type: object + description: "setting: value pairs for configuration restart policy" + access: + type: object + description: "parameters which use for connect to clickhouse from clickhouse-operator deployment" + properties: + scheme: + type: string + description: "The scheme to user for connecting to ClickHouse. Possible values: http, https, auto" + username: + type: string + description: "ClickHouse username to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" + password: + type: string + description: "ClickHouse password to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" + rootCA: + type: string + description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + secret: + type: object + properties: + namespace: + type: string + description: "Location of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" + name: + type: string + description: "Name of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" + port: + type: integer + minimum: 1 + maximum: 65535 + description: "Port to be used by operator to connect to ClickHouse instances" + timeouts: + type: object + description: "Timeouts used to limit connection and queries from the operator to ClickHouse instances, In seconds" + properties: + connect: + type: integer + minimum: 1 + maximum: 10 + description: "Timout to setup connection from the operator to ClickHouse instances. In seconds." + query: + type: integer + minimum: 1 + maximum: 600 + description: "Timout to perform SQL query from the operator to ClickHouse instances. In seconds." + metrics: + type: object + description: "parameters which use for connect to fetch metrics from clickhouse by clickhouse-operator" + properties: + timeouts: + type: object + description: | + Timeouts used to limit connection and queries from the metrics exporter to ClickHouse instances + Specified in seconds. + properties: + collect: + type: integer + minimum: 1 + maximum: 600 + description: | + Timeout used to limit metrics collection request. In seconds. + Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle. + All collected metrics are returned. + template: + type: object + description: "Parameters which are used if you want to generate ClickHouseInstallationTemplate custom resources from files which are stored inside clickhouse-operator deployment" + properties: + chi: + type: object + properties: + policy: + type: string + description: | + CHI template updates handling policy + Possible policy values: + - ReadOnStart. Accept CHIT updates on the operators start only. + - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI + enum: + - "" + - "ReadOnStart" + - "ApplyOnNextReconcile" + path: + type: string + description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." + reconcile: + type: object + description: "allow tuning reconciling process" + properties: + runtime: + type: object + description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle" + properties: + reconcileCHIsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileShardsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default" + reconcileShardsMaxConcurrencyPercent: + type: integer + minimum: 0 + maximum: 100 + description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default." + statefulSet: + type: object + description: "Allow change default behavior for reconciling StatefulSet which generated by clickhouse-operator" + properties: + create: + type: object + description: "Behavior during create StatefulSet" + properties: + onFailure: + type: string + description: | + What to do in case created StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds + Possible options: + 1. abort - do nothing, just break the process and wait for admin. + 2. delete - delete newly created problematic StatefulSet. + 3. ignore (default) - ignore error, pretend nothing happened and move on to the next StatefulSet. + update: + type: object + description: "Behavior during update StatefulSet" + properties: + timeout: + type: integer + description: "How many seconds to wait for created/updated StatefulSet to be Ready" + pollInterval: + type: integer + description: "How many seconds to wait between checks for created/updated StatefulSet status" + onFailure: + type: string + description: | + What to do in case updated StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds + Possible options: + 1. abort - do nothing, just break the process and wait for admin. + 2. rollback (default) - delete Pod and rollback StatefulSet to previous Generation. Pod would be recreated by StatefulSet based on rollback-ed configuration. + 3. ignore - ignore error, pretend nothing happened and move on to the next StatefulSet. + host: + type: object + description: | + Whether the operator during reconcile procedure should wait for a ClickHouse host: + - to be excluded from a ClickHouse cluster + - to complete all running queries + - to be included into a ClickHouse cluster + respectfully before moving forward + properties: + wait: + type: object + properties: + exclude: &TypeStringBool + type: string + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be excluded from a ClickHouse cluster" + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + queries: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to complete all running queries" + include: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be included into a ClickHouse cluster" + annotation: + type: object + description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" + properties: + include: + type: array + description: | + When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, + include annotations with names from the following list + items: + type: string + exclude: + type: array + description: | + When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, + exclude annotations with names from the following list + items: + type: string + label: + type: object + description: "defines which metadata.labels will include or exclude during render StatefulSet, Pod, PVC resources" + properties: + include: + type: array + description: | + When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, + include labels from the following list + items: + type: string + exclude: + type: array + items: + type: string + description: | + When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, + exclude labels from the following list + appendScope: + !!merge <<: *TypeStringBool + description: | + Whether to append *Scope* labels to StatefulSet and Pod + - "LabelShardScopeIndex" + - "LabelReplicaScopeIndex" + - "LabelCHIScopeIndex" + - "LabelCHIScopeCycleSize" + - "LabelCHIScopeCycleIndex" + - "LabelCHIScopeCycleOffset" + - "LabelClusterScopeIndex" + - "LabelClusterScopeCycleSize" + - "LabelClusterScopeCycleIndex" + - "LabelClusterScopeCycleOffset" + statefulSet: + type: object + description: "define StatefulSet-specific parameters" + properties: + revisionHistoryLimit: + type: integer + description: "revisionHistoryLimit is the maximum number of revisions that will be\nmaintained in the StatefulSet's revision history. \nLook details in `statefulset.spec.revisionHistoryLimit`\n" + pod: + type: object + description: "define pod specific parameters" + properties: + terminationGracePeriod: + type: integer + description: "Optional duration in seconds the pod needs to terminate gracefully. \nLook details in `pod.spec.terminationGracePeriodSeconds`\n" + logger: + type: object + description: "allow setup clickhouse-operator logger behavior" + properties: + logtostderr: + type: string + description: "boolean, allows logs to stderr" + alsologtostderr: + type: string + description: "boolean allows logs to stderr and files both" + v: + type: string + description: "verbosity level of clickhouse-operator log, default - 1 max - 9" + stderrthreshold: + type: string + vmodule: + type: string + description: | + Comma-separated list of filename=N, where filename (can be a pattern) must have no .go ext, and N is a V level. + Ex.: file*=2 sets the 'V' to 2 in all files with names like file*. + log_backtrace_at: + type: string + description: | + It can be set to a file and line number with a logging line. + Ex.: file.go:123 + Each time when this line is being executed, a stack trace will be written to the Info log. diff --git a/dev/build_operatorhub_release.sh b/dev/build_operatorhub_release.sh index 7d6ae8883..3bb3719a1 100755 --- a/dev/build_operatorhub_release.sh +++ b/dev/build_operatorhub_release.sh @@ -5,16 +5,21 @@ CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" source "${CUR_DIR}/go_build_config.sh" # Where community-operators sources are located -DEFAULT_CO_PATH=~/dev/community-operators/operators/clickhouse -CO_PATH=${CO_PATH:-${DEFAULT_CO_PATH}} + +DEFAULT_CO_REPO_PATH=~/dev/community-operators +CO_REPO_PATH="${CO_REPO_PATH:-"${DEFAULT_CO_REPO_PATH}"}" +DEFAULT_CO_FOLDER_PATH="${CO_REPO_PATH}/operators/clickhouse" +CO_FOLDER_PATH="${CO_FOLDER_PATH:-"${DEFAULT_CO_FOLDER_PATH}"}" # Ask to prepare copy of required files -echo "Please ensure new clickhouse folder in operatorhub repo is available :" -echo "${CO_PATH}" +echo "Please check operatorhub repo and clickhouse folder within the repo are available :" +echo "Repo: ${CO_REPO_PATH}" +echo "Folder:${CO_FOLDER_PATH}" read -n 1 -r -s -p $'Press enter to continue...\n' -if [[ ! -d "${CO_PATH}" ]]; then - echo "Folder ${CO_PATH} is not available! Abort." +if [[ ! -d "${CO_FOLDER_PATH}" ]]; then + echo "Folder ${CO_FOLDER_PATH} is not available." + echo "Abort." exit 1 fi @@ -29,8 +34,10 @@ else fi if [[ -z "${PREVIOUS_VERSION}" ]]; then - echo "Please specify PREVIOUS_VERSION earlier published on operatorhub, like" + echo "No PREVIOUS_VERSION available." + echo "Please specify PREVIOUS_VERSION earlier published on operatorhub, like:" echo "PREVIOUS_VERSION=0.18.1" + echo "Abort." exit 1 else echo "Going to use" @@ -43,8 +50,8 @@ read -n 1 -r -s -p $'Press enter to continue...\n' PREVIOUS_VERSION="${PREVIOUS_VERSION}" ${SRC_ROOT}/deploy/builder/operatorhub.sh OPERATORHUB_DIR="${SRC_ROOT}/deploy/operatorhub" -DST_MANIFESTS_DIR="${CO_PATH}/${VERSION}/manifests/" -DST_METADATA_DIR="${CO_PATH}/${VERSION}/metadata/" +DST_MANIFESTS_DIR="${CO_FOLDER_PATH}/${VERSION}/manifests/" +DST_METADATA_DIR="${CO_FOLDER_PATH}/${VERSION}/metadata/" mkdir -p "${DST_MANIFESTS_DIR}" mkdir -p "${DST_METADATA_DIR}" cp -r "${OPERATORHUB_DIR}/${VERSION}/"* "${DST_MANIFESTS_DIR}" diff --git a/docs/chi-examples/02-templates-01-pod-template.yaml b/docs/chi-examples/02-templates-01-pod-template.yaml index bdce026e9..715044a7b 100644 --- a/docs/chi-examples/02-templates-01-pod-template.yaml +++ b/docs/chi-examples/02-templates-01-pod-template.yaml @@ -29,7 +29,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 # Container has explicitly specified resource requests resources: requests: diff --git a/docs/chi-examples/02-templates-04-host-template-volume-claim-and-pod-resources-limit.yaml b/docs/chi-examples/02-templates-04-host-template-volume-claim-and-pod-resources-limit.yaml index 2d1a2d231..5b3f121ef 100644 --- a/docs/chi-examples/02-templates-04-host-template-volume-claim-and-pod-resources-limit.yaml +++ b/docs/chi-examples/02-templates-04-host-template-volume-claim-and-pod-resources-limit.yaml @@ -29,7 +29,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: clickhouse-data-storage mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/02-templates-05-sidecar.yaml b/docs/chi-examples/02-templates-05-sidecar.yaml index 75b397521..604b23366 100644 --- a/docs/chi-examples/02-templates-05-sidecar.yaml +++ b/docs/chi-examples/02-templates-05-sidecar.yaml @@ -39,7 +39,7 @@ spec: containers: # Main container - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: sidecar-configmap-volume mountPath: /configmap-volume diff --git a/docs/chi-examples/02-templates-06-bootstrap-schema.yaml b/docs/chi-examples/02-templates-06-bootstrap-schema.yaml index 10f4f4772..f17b7f5b9 100644 --- a/docs/chi-examples/02-templates-06-bootstrap-schema.yaml +++ b/docs/chi-examples/02-templates-06-bootstrap-schema.yaml @@ -41,7 +41,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: bootstrap-configmap-volume mountPath: /docker-entrypoint-initdb.d diff --git a/docs/chi-examples/03-persistent-volume-02-pod-template.yaml b/docs/chi-examples/03-persistent-volume-02-pod-template.yaml index f112eec2d..bdc563467 100644 --- a/docs/chi-examples/03-persistent-volume-02-pod-template.yaml +++ b/docs/chi-examples/03-persistent-volume-02-pod-template.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: data-storage-vc-template mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/03-persistent-volume-03-custom-labels-and-annotations.yaml b/docs/chi-examples/03-persistent-volume-03-custom-labels-and-annotations.yaml index 9a8d7216e..41da5a2bb 100644 --- a/docs/chi-examples/03-persistent-volume-03-custom-labels-and-annotations.yaml +++ b/docs/chi-examples/03-persistent-volume-03-custom-labels-and-annotations.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: data-storage-vc-template mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/03-persistent-volume-07-multiple-resizable-volumes-1.yaml b/docs/chi-examples/03-persistent-volume-07-multiple-resizable-volumes-1.yaml index 31a53f82d..77c34fff8 100644 --- a/docs/chi-examples/03-persistent-volume-07-multiple-resizable-volumes-1.yaml +++ b/docs/chi-examples/03-persistent-volume-07-multiple-resizable-volumes-1.yaml @@ -32,7 +32,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: data-storage-vc-template-1 mountPath: /data/clickhouse-01 diff --git a/docs/chi-examples/03-persistent-volume-07-multiple-resizable-volumes-2.yaml b/docs/chi-examples/03-persistent-volume-07-multiple-resizable-volumes-2.yaml index 91e0d618e..51977d7ee 100644 --- a/docs/chi-examples/03-persistent-volume-07-multiple-resizable-volumes-2.yaml +++ b/docs/chi-examples/03-persistent-volume-07-multiple-resizable-volumes-2.yaml @@ -32,7 +32,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: data-storage-vc-template-1 mountPath: /data/clickhouse-01 diff --git a/docs/chi-examples/03-persistent-volume-07-multiple-resizable-volumes-3.yaml b/docs/chi-examples/03-persistent-volume-07-multiple-resizable-volumes-3.yaml index 4ac9dabb9..ff5e307b6 100644 --- a/docs/chi-examples/03-persistent-volume-07-multiple-resizable-volumes-3.yaml +++ b/docs/chi-examples/03-persistent-volume-07-multiple-resizable-volumes-3.yaml @@ -32,7 +32,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: data-storage-vc-template-1 mountPath: /data/clickhouse-01 diff --git a/docs/chi-examples/03-persistent-volume-07-security-context.yaml b/docs/chi-examples/03-persistent-volume-07-security-context.yaml index e3cdc173f..6c104f5a6 100644 --- a/docs/chi-examples/03-persistent-volume-07-security-context.yaml +++ b/docs/chi-examples/03-persistent-volume-07-security-context.yaml @@ -46,7 +46,7 @@ spec: add: [ "CAP_NICE", "CAP_IPC_LOCK" ] containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 imagePullPolicy: IfNotPresent volumeMounts: - name: data-storage-vc-template-1 diff --git a/docs/chi-examples/04-replication-zookeeper-03-minimal-AWS-persistent-volume.yaml b/docs/chi-examples/04-replication-zookeeper-03-minimal-AWS-persistent-volume.yaml index 2a55780a7..06fe5c07e 100644 --- a/docs/chi-examples/04-replication-zookeeper-03-minimal-AWS-persistent-volume.yaml +++ b/docs/chi-examples/04-replication-zookeeper-03-minimal-AWS-persistent-volume.yaml @@ -21,7 +21,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: clickhouse-storage-template mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/04-replication-zookeeper-04-medium-AWS-persistent-volume.yaml b/docs/chi-examples/04-replication-zookeeper-04-medium-AWS-persistent-volume.yaml index cf4a3c50b..c21b85041 100644 --- a/docs/chi-examples/04-replication-zookeeper-04-medium-AWS-persistent-volume.yaml +++ b/docs/chi-examples/04-replication-zookeeper-04-medium-AWS-persistent-volume.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: clickhouse-storage-template mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/04-replication-zookeeper-05-simple-PV.yaml b/docs/chi-examples/04-replication-zookeeper-05-simple-PV.yaml index 9c02e5b89..534184645 100644 --- a/docs/chi-examples/04-replication-zookeeper-05-simple-PV.yaml +++ b/docs/chi-examples/04-replication-zookeeper-05-simple-PV.yaml @@ -34,4 +34,4 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/06-advanced-layout-01-one-shard-many-replicas.yaml b/docs/chi-examples/06-advanced-layout-01-one-shard-many-replicas.yaml index 510643bd1..516133c40 100644 --- a/docs/chi-examples/06-advanced-layout-01-one-shard-many-replicas.yaml +++ b/docs/chi-examples/06-advanced-layout-01-one-shard-many-replicas.yaml @@ -10,21 +10,21 @@ spec: shards: - replicas: - templates: - podTemplate: clickhouse:22.1 + podTemplate: clickhouse:23.8 - templates: podTemplate: clickhouse:nonexist templates: podTemplates: - - name: clickhouse:22.1 + - name: clickhouse:23.8 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.1 + image: clickhouse/clickhouse-server:23.8 - name: clickhouse:nonexist spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.122 + image: clickhouse/clickhouse-server:23.822 diff --git a/docs/chi-examples/06-advanced-layout-02-shards.yaml b/docs/chi-examples/06-advanced-layout-02-shards.yaml index 73bc3499d..82be10895 100644 --- a/docs/chi-examples/06-advanced-layout-02-shards.yaml +++ b/docs/chi-examples/06-advanced-layout-02-shards.yaml @@ -10,17 +10,17 @@ spec: shards: - replicas: - templates: - podTemplate: clickhouse:22.1 + podTemplate: clickhouse:22.8 httpPort: 8000 tcpPort: 8001 interserverHTTPPort: 8002 - templates: - podTemplate: clickhouse:22.2 + podTemplate: clickhouse:23.3 httpPort: 9000 tcpPort: 9001 interserverHTTPPort: 9002 - templates: - podTemplate: clickhouse:22.3 + podTemplate: clickhouse:23.8 httpPort: 10000 tcpPort: 10001 interserverHTTPPort: 10002 @@ -28,21 +28,21 @@ spec: templates: podTemplates: - - name: clickhouse:22.1 + - name: clickhouse:22.8 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.1 + image: clickhouse/clickhouse-server:22.8 - - name: clickhouse:22.2 + - name: clickhouse:23.3 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.2 + image: clickhouse/clickhouse-server:23.3 - - name: clickhouse:22.3 + - name: clickhouse:23.8 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/06-advanced-layout-03-replicas.yaml b/docs/chi-examples/06-advanced-layout-03-replicas.yaml index 85ba5ce69..5af781e0b 100644 --- a/docs/chi-examples/06-advanced-layout-03-replicas.yaml +++ b/docs/chi-examples/06-advanced-layout-03-replicas.yaml @@ -10,17 +10,17 @@ spec: shardsCount: 4 replicas: - templates: - podTemplate: clickhouse:22.1 + podTemplate: clickhouse:22.8 httpPort: 8000 tcpPort: 8001 interserverHTTPPort: 8002 - templates: - podTemplate: clickhouse:22.2 + podTemplate: clickhouse:23.3 httpPort: 9000 tcpPort: 9001 interserverHTTPPort: 9002 - templates: - podTemplate: clickhouse:22.3 + podTemplate: clickhouse:23.8 httpPort: 10000 tcpPort: 10001 interserverHTTPPort: 10002 @@ -28,21 +28,21 @@ spec: templates: podTemplates: - - name: clickhouse:22.1 + - name: clickhouse:22.8 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.1 + image: clickhouse/clickhouse-server:22.8 - - name: clickhouse:22.3 + - name: clickhouse:23.3 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.2 + image: clickhouse/clickhouse-server:23.3 - - name: clickhouse:22.3 + - name: clickhouse:23.8 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/06-advanced-layout-04-multiple-clusters.yaml b/docs/chi-examples/06-advanced-layout-04-multiple-clusters.yaml index 14ed27eac..b0dcf3450 100644 --- a/docs/chi-examples/06-advanced-layout-04-multiple-clusters.yaml +++ b/docs/chi-examples/06-advanced-layout-04-multiple-clusters.yaml @@ -48,16 +48,16 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.1 + image: clickhouse/clickhouse-server:22.8 - name: t2 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.2 + image: clickhouse/clickhouse-server:23.3 - name: t3 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/08-clickhouse-version-update-01-initial-position.yaml b/docs/chi-examples/08-clickhouse-version-update-01-initial-position.yaml index afa2c34f9..528e23074 100644 --- a/docs/chi-examples/08-clickhouse-version-update-01-initial-position.yaml +++ b/docs/chi-examples/08-clickhouse-version-update-01-initial-position.yaml @@ -7,7 +7,7 @@ spec: clusters: - name: update templates: - podTemplate: clickhouse:22.2 + podTemplate: clickhouse:23.3 layout: shards: - replicas: @@ -17,8 +17,8 @@ spec: templates: podTemplates: - - name: clickhouse:22.2 + - name: clickhouse:23.3 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.2 + image: clickhouse/clickhouse-server:23.3 diff --git a/docs/chi-examples/08-clickhouse-version-update-02-apply-update-one.yaml b/docs/chi-examples/08-clickhouse-version-update-02-apply-update-one.yaml index 09294801d..68ae27418 100644 --- a/docs/chi-examples/08-clickhouse-version-update-02-apply-update-one.yaml +++ b/docs/chi-examples/08-clickhouse-version-update-02-apply-update-one.yaml @@ -7,7 +7,7 @@ spec: clusters: - name: update templates: - podTemplate: clickhouse:22.2 + podTemplate: clickhouse:23.3 layout: shards: - replicas: @@ -15,18 +15,18 @@ spec: - tcpPort: 9000 - tcpPort: 9000 templates: - podTemplate: clickhouse:22.3 + podTemplate: clickhouse:23.8 templates: podTemplates: - - name: clickhouse:22.2 + - name: clickhouse:23.3 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.2 + image: clickhouse/clickhouse-server:23.3 - - name: clickhouse:22.3 + - name: clickhouse:23.8 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/08-clickhouse-version-update-03-apply-update-all.yaml b/docs/chi-examples/08-clickhouse-version-update-03-apply-update-all.yaml index 26ae0ac72..b62644237 100644 --- a/docs/chi-examples/08-clickhouse-version-update-03-apply-update-all.yaml +++ b/docs/chi-examples/08-clickhouse-version-update-03-apply-update-all.yaml @@ -7,7 +7,7 @@ spec: clusters: - name: update templates: - podTemplate: clickhouse:22.3 + podTemplate: clickhouse:23.8 layout: shards: - replicas: @@ -17,8 +17,8 @@ spec: templates: podTemplates: - - name: clickhouse:22.3 + - name: clickhouse:23.8 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/09-rolling-update-emptydir-01-initial-position.yaml b/docs/chi-examples/09-rolling-update-emptydir-01-initial-position.yaml index b5a78057c..e73929760 100644 --- a/docs/chi-examples/09-rolling-update-emptydir-01-initial-position.yaml +++ b/docs/chi-examples/09-rolling-update-emptydir-01-initial-position.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: clickhouse-storage mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/09-rolling-update-emptydir-02-apply-update.yaml b/docs/chi-examples/09-rolling-update-emptydir-02-apply-update.yaml index 0841e758d..7d92f614b 100644 --- a/docs/chi-examples/09-rolling-update-emptydir-02-apply-update.yaml +++ b/docs/chi-examples/09-rolling-update-emptydir-02-apply-update.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: clickhouse-storage mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/10-zones-01-simple-01-aws-pods-in-availability-zones.yaml b/docs/chi-examples/10-zones-01-simple-01-aws-pods-in-availability-zones.yaml index fa8c2b097..a21bf7a79 100644 --- a/docs/chi-examples/10-zones-01-simple-01-aws-pods-in-availability-zones.yaml +++ b/docs/chi-examples/10-zones-01-simple-01-aws-pods-in-availability-zones.yaml @@ -49,7 +49,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 - name: clickhouse-in-zone-us-east-1b zone: @@ -58,4 +58,4 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/10-zones-01-simple-02-aws-pod-per-host.yaml b/docs/chi-examples/10-zones-01-simple-02-aws-pod-per-host.yaml index db97013f1..437bc7fee 100644 --- a/docs/chi-examples/10-zones-01-simple-02-aws-pod-per-host.yaml +++ b/docs/chi-examples/10-zones-01-simple-02-aws-pod-per-host.yaml @@ -40,4 +40,4 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/10-zones-02-advanced-01-aws-pods-in-availability-zones.yaml b/docs/chi-examples/10-zones-02-advanced-01-aws-pods-in-availability-zones.yaml index 1b3392dff..12a77cac1 100644 --- a/docs/chi-examples/10-zones-02-advanced-01-aws-pods-in-availability-zones.yaml +++ b/docs/chi-examples/10-zones-02-advanced-01-aws-pods-in-availability-zones.yaml @@ -56,7 +56,7 @@ spec: - "us-east-1a" containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 - name: clickhouse-in-zone-us-east-1b spec: @@ -72,4 +72,4 @@ spec: - "us-east-1b" containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/10-zones-02-advanced-02-aws-pod-per-host.yaml b/docs/chi-examples/10-zones-02-advanced-02-aws-pod-per-host.yaml index 6a67baf67..e65e7d5a7 100644 --- a/docs/chi-examples/10-zones-02-advanced-02-aws-pod-per-host.yaml +++ b/docs/chi-examples/10-zones-02-advanced-02-aws-pod-per-host.yaml @@ -55,4 +55,4 @@ spec: topologyKey: "kubernetes.io/hostname" containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/10-zones-03-advanced-03-pod-per-host-default-storage-class.yaml b/docs/chi-examples/10-zones-03-advanced-03-pod-per-host-default-storage-class.yaml index eb47b28ce..3704698b4 100644 --- a/docs/chi-examples/10-zones-03-advanced-03-pod-per-host-default-storage-class.yaml +++ b/docs/chi-examples/10-zones-03-advanced-03-pod-per-host-default-storage-class.yaml @@ -76,7 +76,7 @@ spec: topologyKey: "kubernetes.io/hostname" containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: storage-vc-template mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/10-zones-04-advanced-04-pod-per-host-local-storage.yaml b/docs/chi-examples/10-zones-04-advanced-04-pod-per-host-local-storage.yaml index 3cbaf3212..5a0379d02 100644 --- a/docs/chi-examples/10-zones-04-advanced-04-pod-per-host-local-storage.yaml +++ b/docs/chi-examples/10-zones-04-advanced-04-pod-per-host-local-storage.yaml @@ -57,7 +57,7 @@ spec: type: DirectoryOrCreate containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: local-path mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/11-local-storage-01-simple-host-path.yaml b/docs/chi-examples/11-local-storage-01-simple-host-path.yaml index b01ff3fe4..2e77b1875 100644 --- a/docs/chi-examples/11-local-storage-01-simple-host-path.yaml +++ b/docs/chi-examples/11-local-storage-01-simple-host-path.yaml @@ -48,7 +48,7 @@ spec: type: DirectoryOrCreate containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: # Specify reference to volume on local filesystem - name: local-path diff --git a/docs/chi-examples/11-local-storage-02-advanced-host-path.yaml b/docs/chi-examples/11-local-storage-02-advanced-host-path.yaml index 7c6caff12..737f577ac 100644 --- a/docs/chi-examples/11-local-storage-02-advanced-host-path.yaml +++ b/docs/chi-examples/11-local-storage-02-advanced-host-path.yaml @@ -62,7 +62,7 @@ spec: type: DirectoryOrCreate containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: # Specify reference to volume on local filesystem - name: local-path diff --git a/docs/chi-examples/12-troubleshooting-01.yaml b/docs/chi-examples/12-troubleshooting-01.yaml index 0fcc53076..68cffbec6 100644 --- a/docs/chi-examples/12-troubleshooting-01.yaml +++ b/docs/chi-examples/12-troubleshooting-01.yaml @@ -18,7 +18,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 command: - "/bin/bash" - "-c" diff --git a/docs/chi-examples/13-distribution-02-3x3-circular-replication.yaml b/docs/chi-examples/13-distribution-02-3x3-circular-replication.yaml index f2c2226ff..ffd58dbde 100644 --- a/docs/chi-examples/13-distribution-02-3x3-circular-replication.yaml +++ b/docs/chi-examples/13-distribution-02-3x3-circular-replication.yaml @@ -20,7 +20,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 --- apiVersion: "clickhouse.altinity.com/v1" kind: "ClickHouseInstallation" @@ -44,4 +44,4 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/13-distribution-03-3x3-distribution-detailed.yaml b/docs/chi-examples/13-distribution-03-3x3-distribution-detailed.yaml index 96a532447..27b97ddbc 100644 --- a/docs/chi-examples/13-distribution-03-3x3-distribution-detailed.yaml +++ b/docs/chi-examples/13-distribution-03-3x3-distribution-detailed.yaml @@ -27,4 +27,4 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/15-hostNetwork-01-simple.yaml b/docs/chi-examples/15-hostNetwork-01-simple.yaml index b0da96425..79191166d 100644 --- a/docs/chi-examples/15-hostNetwork-01-simple.yaml +++ b/docs/chi-examples/15-hostNetwork-01-simple.yaml @@ -24,4 +24,4 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/15-hostNetwork-02-simple-port-distribution.yaml b/docs/chi-examples/15-hostNetwork-02-simple-port-distribution.yaml index ceff8642e..3391b93f4 100644 --- a/docs/chi-examples/15-hostNetwork-02-simple-port-distribution.yaml +++ b/docs/chi-examples/15-hostNetwork-02-simple-port-distribution.yaml @@ -34,4 +34,4 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/15-hostNetwork-03-expanded-port-distribution.yaml b/docs/chi-examples/15-hostNetwork-03-expanded-port-distribution.yaml index 9bd002184..92886f058 100644 --- a/docs/chi-examples/15-hostNetwork-03-expanded-port-distribution.yaml +++ b/docs/chi-examples/15-hostNetwork-03-expanded-port-distribution.yaml @@ -41,4 +41,4 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/15-hostNetwork-04-simple-fixed-replicas.yaml b/docs/chi-examples/15-hostNetwork-04-simple-fixed-replicas.yaml index b1099ff10..bee7d74d4 100644 --- a/docs/chi-examples/15-hostNetwork-04-simple-fixed-replicas.yaml +++ b/docs/chi-examples/15-hostNetwork-04-simple-fixed-replicas.yaml @@ -50,4 +50,4 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/15-hostNetwork-05-expanded-fixed-replicas.yaml b/docs/chi-examples/15-hostNetwork-05-expanded-fixed-replicas.yaml index 7d097fe4e..e6ba0cb72 100644 --- a/docs/chi-examples/15-hostNetwork-05-expanded-fixed-replicas.yaml +++ b/docs/chi-examples/15-hostNetwork-05-expanded-fixed-replicas.yaml @@ -51,4 +51,4 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 diff --git a/docs/chi-examples/17-monitoring-cluster-01.yaml b/docs/chi-examples/17-monitoring-cluster-01.yaml index 6ef645345..149ab682d 100644 --- a/docs/chi-examples/17-monitoring-cluster-01.yaml +++ b/docs/chi-examples/17-monitoring-cluster-01.yaml @@ -21,7 +21,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: clickhouse-storage-template mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/19-pod-generate-name.yaml b/docs/chi-examples/19-pod-generate-name.yaml index 3ff26620d..a3889d8b8 100644 --- a/docs/chi-examples/19-pod-generate-name.yaml +++ b/docs/chi-examples/19-pod-generate-name.yaml @@ -18,7 +18,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: clickhouse-storage-template mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/22-secure-ssl-01-files-plaintext.yaml b/docs/chi-examples/22-secure-ssl-01-files-plaintext.yaml index 73e6a5722..3a3f8c996 100644 --- a/docs/chi-examples/22-secure-ssl-01-files-plaintext.yaml +++ b/docs/chi-examples/22-secure-ssl-01-files-plaintext.yaml @@ -134,7 +134,7 @@ metadata: spec: containers: - name: clickhouse-client - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 command: [ "/bin/sh", "-c", "sleep 3600" ] volumeMounts: - name: config diff --git a/docs/chi-examples/22-secure-ssl-02-files-secret-ref.yaml b/docs/chi-examples/22-secure-ssl-02-files-secret-ref.yaml index 9d28c3a4b..a1ff47d98 100644 --- a/docs/chi-examples/22-secure-ssl-02-files-secret-ref.yaml +++ b/docs/chi-examples/22-secure-ssl-02-files-secret-ref.yaml @@ -156,7 +156,7 @@ metadata: spec: containers: - name: clickhouse-client - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 command: [ "/bin/sh", "-c", "sleep 3600" ] volumeMounts: - name: config diff --git a/docs/chi-examples/22-secure-ssl-03-files-multi-secrets-ref.yaml b/docs/chi-examples/22-secure-ssl-03-files-multi-secrets-ref.yaml index 126d5876e..40d23340e 100644 --- a/docs/chi-examples/22-secure-ssl-03-files-multi-secrets-ref.yaml +++ b/docs/chi-examples/22-secure-ssl-03-files-multi-secrets-ref.yaml @@ -170,7 +170,7 @@ metadata: spec: containers: - name: clickhouse-client - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 command: [ "/bin/sh", "-c", "sleep 3600" ] volumeMounts: - name: config diff --git a/docs/chi-examples/99-clickhouseinstallation-max.yaml b/docs/chi-examples/99-clickhouseinstallation-max.yaml index cf484d240..29569903d 100644 --- a/docs/chi-examples/99-clickhouseinstallation-max.yaml +++ b/docs/chi-examples/99-clickhouseinstallation-max.yaml @@ -123,7 +123,7 @@ spec: templates: hostTemplate: host-template-custom-ports - podTemplate: clickhouse-v22.2 + podTemplate: clickhouse-v23.8 dataVolumeClaimTemplate: default-volume-claim logVolumeClaimTemplate: default-volume-claim serviceTemplate: chi-service-template @@ -203,7 +203,7 @@ spec: - name: all-counts templates: - podTemplate: clickhouse-v22.2 + podTemplate: clickhouse-v23.8 dataVolumeClaimTemplate: default-volume-claim logVolumeClaimTemplate: default-volume-claim schemaPolicy: @@ -215,7 +215,7 @@ spec: - name: shards-only templates: - podTemplate: clickhouse-v22.2 + podTemplate: clickhouse-v23.8 dataVolumeClaimTemplate: default-volume-claim logVolumeClaimTemplate: default-volume-claim layout: @@ -224,7 +224,7 @@ spec: - name: replicas-only templates: - podTemplate: clickhouse-v22.2 + podTemplate: clickhouse-v23.8 dataVolumeClaimTemplate: default-volume-claim logVolumeClaimTemplate: default-volume-claim layout: @@ -233,7 +233,7 @@ spec: - name: customized templates: - podTemplate: clickhouse-v22.2 + podTemplate: clickhouse-v23.8 dataVolumeClaimTemplate: default-volume-claim logVolumeClaimTemplate: default-volume-claim schemaPolicy: @@ -246,13 +246,13 @@ spec: weight: 1 internalReplication: Disabled templates: - podTemplate: clickhouse-v22.2 + podTemplate: clickhouse-v23.8 dataVolumeClaimTemplate: default-volume-claim logVolumeClaimTemplate: default-volume-claim - name: shard1 templates: - podTemplate: clickhouse-v22.2 + podTemplate: clickhouse-v23.8 dataVolumeClaimTemplate: default-volume-claim logVolumeClaimTemplate: default-volume-claim replicas: @@ -263,7 +263,7 @@ spec: - name: shard2 replicasCount: 3 templates: - podTemplate: clickhouse-v22.2 + podTemplate: clickhouse-v23.8 dataVolumeClaimTemplate: default-volume-claim logVolumeClaimTemplate: default-volume-claim replicaServiceTemplate: replica-service-template @@ -273,7 +273,7 @@ spec: httpPort: 8123 interserverHTTPPort: 9009 templates: - podTemplate: clickhouse-v22.3 + podTemplate: clickhouse-v23.8 dataVolumeClaimTemplate: default-volume-claim logVolumeClaimTemplate: default-volume-claim replicaServiceTemplate: replica-service-template @@ -486,7 +486,7 @@ spec: podTemplates: # multiple pod templates makes possible to update version smoothly # pod template for ClickHouse v18.16.1 - - name: clickhouse-v22.3 + - name: clickhouse-v23.8 # We may need to label nodes with clickhouse=allow label for this example to run # See ./label_nodes.sh for this purpose zone: @@ -569,7 +569,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: default-volume-claim mountPath: /var/lib/clickhouse @@ -581,7 +581,7 @@ spec: memory: "64Mi" cpu: "100m" - name: clickhouse-log - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 command: - "/bin/sh" - "-c" @@ -589,8 +589,8 @@ spec: args: - "while true; do sleep 30; done;" - # pod template for ClickHouse v22.2 - - name: clickhouse-v22.2 + # pod template for ClickHouse v23.8 + - name: clickhouse-v23.8 # type ObjectMeta struct {} from k8s.io/meta/v1 metadata: labels: @@ -599,7 +599,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.2 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: default-volume-claim mountPath: /var/lib/clickhouse @@ -611,7 +611,7 @@ spec: memory: "64Mi" cpu: "100m" - name: clickhouse-log - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 command: - "/bin/sh" - "-c" diff --git a/docs/chi-examples/evolution/01-persistent-volume.yaml b/docs/chi-examples/evolution/01-persistent-volume.yaml index 4d0bcd5c0..c5b162fa2 100644 --- a/docs/chi-examples/evolution/01-persistent-volume.yaml +++ b/docs/chi-examples/evolution/01-persistent-volume.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: storage-vc-template mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/evolution/02-introduce-replication.yaml b/docs/chi-examples/evolution/02-introduce-replication.yaml index d9f747aeb..6f14bd9f7 100644 --- a/docs/chi-examples/evolution/02-introduce-replication.yaml +++ b/docs/chi-examples/evolution/02-introduce-replication.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: storage-vc-template mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/evolution/03-introduce-more-shards-and-zones.yaml b/docs/chi-examples/evolution/03-introduce-more-shards-and-zones.yaml index f5b0733c8..7a6947e0b 100644 --- a/docs/chi-examples/evolution/03-introduce-more-shards-and-zones.yaml +++ b/docs/chi-examples/evolution/03-introduce-more-shards-and-zones.yaml @@ -28,7 +28,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: storage-vc-template mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/evolution/04-update-introduce-canary.yaml b/docs/chi-examples/evolution/04-update-introduce-canary.yaml index 2e9c7506a..1bcaca634 100644 --- a/docs/chi-examples/evolution/04-update-introduce-canary.yaml +++ b/docs/chi-examples/evolution/04-update-introduce-canary.yaml @@ -21,7 +21,7 @@ spec: replicas: - name: "1" templates: - podTemplate: pod-template-with-volume-22.3 + podTemplate: pod-template-with-volume-23.3 templates: podTemplates: @@ -35,12 +35,12 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:21.8 + image: clickhouse/clickhouse-server:23.3 volumeMounts: - name: storage-vc-template mountPath: /var/lib/clickhouse - - name: pod-template-with-volume-22.3 + - name: pod-template-with-volume-23.3 zone: key: "clickhouse" values: @@ -50,7 +50,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: storage-vc-template mountPath: /var/lib/clickhouse diff --git a/docs/chi-examples/evolution/05-update-propagate-update.yaml b/docs/chi-examples/evolution/05-update-propagate-update.yaml index f5b0733c8..7a6947e0b 100644 --- a/docs/chi-examples/evolution/05-update-propagate-update.yaml +++ b/docs/chi-examples/evolution/05-update-propagate-update.yaml @@ -28,7 +28,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: storage-vc-template mountPath: /var/lib/clickhouse diff --git a/docs/chit-examples/103-templates.yaml b/docs/chit-examples/103-templates.yaml index 67d98740a..721bd8541 100644 --- a/docs/chit-examples/103-templates.yaml +++ b/docs/chit-examples/103-templates.yaml @@ -9,7 +9,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 ports: - name: http containerPort: 8123 diff --git a/docs/chk-examples/02-extended-1-node.yaml b/docs/chk-examples/02-extended-1-node.yaml index 9a1393d22..b3d85d5e8 100644 --- a/docs/chk-examples/02-extended-1-node.yaml +++ b/docs/chk-examples/02-extended-1-node.yaml @@ -3,8 +3,11 @@ kind: "ClickHouseKeeperInstallation" metadata: name: chk-1-node spec: - replicas: 1 configuration: + clusters: + - name: "simple-1" + layout: + replicasCount: 1 settings: logger/level: "trace" logger/console: "true" @@ -22,24 +25,8 @@ spec: prometheus/status_info: "false" templates: podTemplates: - - name: pod1 - metadata: - labels: - app: clickhouse-keeper - what: node + - name: default spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 50 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: "app" - operator: In - values: - - clickhouse-keeper - topologyKey: "kubernetes.io/hostname" containers: - name: clickhouse-keeper imagePullPolicy: IfNotPresent @@ -52,7 +39,7 @@ spec: memory: "4Gi" cpu: "2" volumeClaimTemplates: - - name: t1 + - name: default metadata: name: both-paths spec: diff --git a/docs/chk-examples/02-extended-3-nodes.yaml b/docs/chk-examples/02-extended-3-nodes.yaml index a84d997a6..ee404be1a 100644 --- a/docs/chk-examples/02-extended-3-nodes.yaml +++ b/docs/chk-examples/02-extended-3-nodes.yaml @@ -3,64 +3,64 @@ kind: "ClickHouseKeeperInstallation" metadata: name: chk-3-nodes spec: - replicas: 3 - settings: - logger/level: "trace" - logger/console: "true" - listen_host: "0.0.0.0" - keeper_server/storage_path: /var/lib/clickhouse-keeper - keeper_server/tcp_port: "2181" - keeper_server/four_letter_word_white_list: "*" - keeper_server/coordination_settings/raft_logs_level: "information" - keeper_server/raft_configuration/server/port: "9444" - prometheus/endpoint: "/metrics" - prometheus/port: "7000" - prometheus/metrics: "true" - prometheus/events: "true" - prometheus/asynchronous_metrics: "true" - prometheus/status_info: "false" - podTemplate: - metadata: - labels: - app: clickhouse-keeper - what: node - spec: - # affinity removed to allow use in single node test environment - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: "app" - operator: In - values: - - clickhouse-keeper - topologyKey: "kubernetes.io/hostname" - containers: - - name: clickhouse-keeper - imagePullPolicy: IfNotPresent - image: "clickhouse/clickhouse-keeper:head-alpine" + configuration: + clusters: + - name: "simple-3" + layout: + replicasCount: 3 + settings: + logger/level: "trace" + logger/console: "true" + listen_host: "0.0.0.0" + keeper_server/storage_path: /var/lib/clickhouse-keeper + keeper_server/tcp_port: "2181" + keeper_server/four_letter_word_white_list: "*" + keeper_server/coordination_settings/raft_logs_level: "information" + keeper_server/raft_configuration/server/port: "9444" + prometheus/endpoint: "/metrics" + prometheus/port: "7000" + prometheus/metrics: "true" + prometheus/events: "true" + prometheus/asynchronous_metrics: "true" + prometheus/status_info: "false" + templates: + podTemplates: + - name: default + spec: + # affinity removed to allow use in single node test environment + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - clickhouse-keeper + topologyKey: "kubernetes.io/hostname" + containers: + - name: clickhouse-keeper + imagePullPolicy: IfNotPresent + image: "clickhouse/clickhouse-keeper:head-alpine" + resources: + requests: + memory: "256M" + cpu: "1" + limits: + memory: "4Gi" + cpu: "2" + volumeClaimTemplates: + - name: default + spec: + accessModes: + - ReadWriteOnce resources: requests: - memory: "256M" - cpu: "1" - limits: - memory: "4Gi" - cpu: "2" - volumeClaimTemplates: - - metadata: - name: log-storage-path - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - - metadata: - name: snapshot-storage-path - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi + storage: 10Gi + - name: snapshot-storage-path + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/docs/custom_resource_explained.md b/docs/custom_resource_explained.md index 69fc8d807..60944e9c8 100644 --- a/docs/custom_resource_explained.md +++ b/docs/custom_resource_explained.md @@ -561,7 +561,7 @@ with additional sections, such as: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: default-volume-claim mountPath: /var/lib/clickhouse diff --git a/docs/operator_configuration.md b/docs/operator_configuration.md index 4a3be8a58..d7c56a3c3 100644 --- a/docs/operator_configuration.md +++ b/docs/operator_configuration.md @@ -149,7 +149,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 ``` Template needs to be deployed to some namespace, and later on used in the installation: diff --git a/docs/quick_start.md b/docs/quick_start.md index 0d104a047..48bb176f4 100644 --- a/docs/quick_start.md +++ b/docs/quick_start.md @@ -285,7 +285,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: data-storage-vc-template mountPath: /var/lib/clickhouse diff --git a/docs/replication_setup.md b/docs/replication_setup.md index 255235527..e5096a6a3 100644 --- a/docs/replication_setup.md +++ b/docs/replication_setup.md @@ -48,7 +48,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 ``` diff --git a/docs/security_hardening.md b/docs/security_hardening.md index 75ecebba7..a0ee867b8 100644 --- a/docs/security_hardening.md +++ b/docs/security_hardening.md @@ -512,7 +512,7 @@ spec: - name: default containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 ports: - name: http containerPort: 8123 diff --git a/generator/github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go b/generator/github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go index da0cbe61c..e3d73185c 100644 --- a/generator/github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go +++ b/generator/github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go @@ -1270,6 +1270,22 @@ func (in *ComparableAttributes) DeepCopy() *ComparableAttributes { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigCRSource) DeepCopyInto(out *ConfigCRSource) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigCRSource. +func (in *ConfigCRSource) DeepCopy() *ConfigCRSource { + if in == nil { + return nil + } + out := new(ConfigCRSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Configuration) DeepCopyInto(out *Configuration) { *out = *in @@ -1552,7 +1568,7 @@ func (in *ObjectAddress) DeepCopy() *ObjectAddress { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OperatorConfig) DeepCopyInto(out *OperatorConfig) { *out = *in - out.Runtime = in.Runtime + in.Runtime.DeepCopyInto(&out.Runtime) in.Watch.DeepCopyInto(&out.Watch) in.ClickHouse.DeepCopyInto(&out.ClickHouse) in.Template.DeepCopyInto(&out.Template) @@ -1971,6 +1987,27 @@ func (in OperatorConfigRestartPolicyRuleSet) DeepCopy() OperatorConfigRestartPol return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OperatorConfigRuntime) DeepCopyInto(out *OperatorConfigRuntime) { + *out = *in + if in.ConfigCRSources != nil { + in, out := &in.ConfigCRSources, &out.ConfigCRSources + *out = make([]ConfigCRSource, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorConfigRuntime. +func (in *OperatorConfigRuntime) DeepCopy() *OperatorConfigRuntime { + if in == nil { + return nil + } + out := new(OperatorConfigRuntime) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OperatorConfigTemplate) DeepCopyInto(out *OperatorConfigTemplate) { *out = *in diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_config_chop.go b/pkg/apis/clickhouse.altinity.com/v1/type_config_chop.go index d3fa027d3..50bd41821 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_config_chop.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_config_chop.go @@ -63,8 +63,8 @@ const ( // 2. Schema maintenance // User credentials can be specified in additional ClickHouse config files located in `chUsersConfigsPath` folder defaultChScheme = ChSchemeAuto - defaultChUsername = "" - defaultChPassword = "" + defaultChUsername = "clickhouse_operator" + defaultChPassword = "clickhouse_operator_password" defaultChPort = 8123 defaultChRootCA = "" @@ -143,6 +143,22 @@ const ( // Do not forget to update func (config *OperatorConfig) String() // Do not forget to update CRD spec +// OperatorConfigRuntime specifies runtime config +type OperatorConfigRuntime struct { + // Full path to the config file and folder where file part of this OperatorConfig originates from + ConfigFilePath string `json:"configFilePath" yaml:"configFilePath"` + ConfigFolderPath string `json:"configFolderPath" yaml:"configFolderPath"` + // Namespace and Name of the config Custom Resource + ConfigCRNamespace string `json:"configCRNamespace" yaml:"configCRNamespace"` + ConfigCRName string `json:"configCRName" yaml:"configCRName"` + + // ConfigCRSources specifies list of Custom Resource-based configuration sources + ConfigCRSources []ConfigCRSource `json:"configCRSources" yaml:"configCRSources"` + + // Namespace specifies namespace where the operator runs + Namespace string `json:"namespace" yaml:"namespace"` +} + // OperatorConfigWatch specifies watch section type OperatorConfigWatch struct { // Namespaces where operator watches for events @@ -375,15 +391,14 @@ type OperatorConfigLabel struct { } `json:"runtime" yaml:"runtime"` } +type ConfigCRSource struct { + Namespace string + Name string +} + // OperatorConfig specifies operator config type OperatorConfig struct { - Runtime struct { - // Full path to the config file and folder where this OperatorConfig originates from - ConfigFilePath string - ConfigFolderPath string - // Namespace specifies namespace where operator runs - Namespace string - } + Runtime OperatorConfigRuntime `json:"runtime" yaml:"runtime"` Watch OperatorConfigWatch `json:"watch" yaml:"watch"` ClickHouse OperatorConfigClickHouse `json:"clickhouse" yaml:"clickhouse"` Template OperatorConfigTemplate `json:"template" yaml:"template"` @@ -693,7 +708,6 @@ func (c *OperatorConfig) normalizeSectionClickHouseConfigurationFile() { util.PreparePath(&c.ClickHouse.Config.File.Path.Common, c.Runtime.ConfigFolderPath, CommonConfigDir) util.PreparePath(&c.ClickHouse.Config.File.Path.Host, c.Runtime.ConfigFolderPath, HostConfigDir) util.PreparePath(&c.ClickHouse.Config.File.Path.User, c.Runtime.ConfigFolderPath, UsersConfigDir) - } func (c *OperatorConfig) normalizeSectionTemplate() { diff --git a/pkg/apis/metrics/prometheus_writer.go b/pkg/apis/metrics/prometheus_writer.go index 72849cd0e..22551ba4e 100644 --- a/pkg/apis/metrics/prometheus_writer.go +++ b/pkg/apis/metrics/prometheus_writer.go @@ -18,12 +18,13 @@ import ( "fmt" "sort" "strconv" - "strings" "time" log "github.com/golang/glog" // log "k8s.io/klog" "github.com/prometheus/client_golang/prometheus" + + "github.com/altinity/clickhouse-operator/pkg/util" ) const ( @@ -64,6 +65,11 @@ func (w *CHIPrometheusWriter) WriteMetrics(data [][]string) { if len(metric) < 2 { continue } + + // Build metric from data row + name := metric[0] + value := metric[1] + desc := metric[2] var metricType prometheus.ValueType if metric[3] == "counter" { metricType = prometheus.CounterValue @@ -71,14 +77,9 @@ func (w *CHIPrometheusWriter) WriteMetrics(data [][]string) { metricType = prometheus.GaugeValue } w.writeSingleMetricToPrometheus( - w.out, - convertMetricName(metric[0]), - metric[2], - metric[1], - metricType, - nil, - nil, - ) + name, desc, + metricType, value, + nil, nil) } } @@ -90,23 +91,28 @@ func (w *CHIPrometheusWriter) WriteTableSizes(data [][]string) { if len(metric) < 2 { continue } - labels := []string{"database", "table", "active"} - values := []string{metric[0], metric[1], metric[2]} - w.writeSingleMetricToPrometheus(w.out, - "table_partitions", "Number of partitions of the table", metric[3], prometheus.GaugeValue, - labels, values) - w.writeSingleMetricToPrometheus(w.out, - "table_parts", "Number of parts of the table", metric[4], prometheus.GaugeValue, - labels, values) - w.writeSingleMetricToPrometheus(w.out, - "table_parts_bytes", "Table size in bytes", metric[5], prometheus.GaugeValue, - labels, values) - w.writeSingleMetricToPrometheus(w.out, - "table_parts_bytes_uncompressed", "Table size in bytes uncompressed", metric[6], prometheus.GaugeValue, - labels, values) - w.writeSingleMetricToPrometheus(w.out, - "table_parts_rows", "Number of rows in the table", metric[7], prometheus.GaugeValue, - labels, values) + labelNames := []string{"database", "table", "active"} + labelValues := []string{metric[0], metric[1], metric[2]} + w.writeSingleMetricToPrometheus( + "table_partitions", "Number of partitions of the table", + prometheus.GaugeValue, metric[3], + labelNames, labelValues) + w.writeSingleMetricToPrometheus( + "table_parts", "Number of parts of the table", + prometheus.GaugeValue, metric[4], + labelNames, labelValues) + w.writeSingleMetricToPrometheus( + "table_parts_bytes", "Table size in bytes", + prometheus.GaugeValue, metric[5], + labelNames, labelValues) + w.writeSingleMetricToPrometheus( + "table_parts_bytes_uncompressed", "Table size in bytes uncompressed", + prometheus.GaugeValue, metric[6], + labelNames, labelValues) + w.writeSingleMetricToPrometheus( + "table_parts_rows", "Number of rows in the table", + prometheus.GaugeValue, metric[7], + labelNames, labelValues) } } @@ -142,73 +148,77 @@ func (w *CHIPrometheusWriter) WriteSystemParts(data [][]string) { // WriteSystemReplicas writes system replicas func (w *CHIPrometheusWriter) WriteSystemReplicas(data [][]string) { for _, metric := range data { - w.writeSingleMetricToPrometheus(w.out, - "system_replicas_is_session_expired", "Number of expired Zookeeper sessions of the table", metric[2], prometheus.GaugeValue, - // Label names - []string{"database", "table"}, - // Label values - []string{metric[0], metric[1]}) + labelNames := []string{"database", "table"} + labelValues := []string{metric[0], metric[1]} + w.writeSingleMetricToPrometheus( + "system_replicas_is_session_expired", "Number of expired Zookeeper sessions of the table", + prometheus.GaugeValue, metric[2], + labelNames, labelValues) } } // WriteMutations writes mutations func (w *CHIPrometheusWriter) WriteMutations(data [][]string) { for _, metric := range data { - labels := []string{"database", "table"} - values := []string{metric[0], metric[1]} - w.writeSingleMetricToPrometheus(w.out, - "table_mutations", "Number of active mutations for the table", metric[2], prometheus.GaugeValue, - labels, values) - w.writeSingleMetricToPrometheus(w.out, - "table_mutations_parts_to_do", "Number of data parts that need to be mutated for the mutation to finish", metric[3], prometheus.GaugeValue, - labels, values) + labelNames := []string{"database", "table"} + labelValues := []string{metric[0], metric[1]} + w.writeSingleMetricToPrometheus( + "table_mutations", "Number of active mutations for the table", + prometheus.GaugeValue, metric[2], + labelNames, labelValues) + w.writeSingleMetricToPrometheus( + "table_mutations_parts_to_do", "Number of data parts that need to be mutated for the mutation to finish", + prometheus.GaugeValue, metric[3], + labelNames, labelValues) } } // WriteSystemDisks writes system disks func (w *CHIPrometheusWriter) WriteSystemDisks(data [][]string) { for _, metric := range data { - labels := []string{"disk"} - values := []string{metric[0]} - w.writeSingleMetricToPrometheus(w.out, - "metric_DiskFreeBytes", "Free disk space available from system.disks", metric[1], prometheus.GaugeValue, - labels, values) - w.writeSingleMetricToPrometheus(w.out, - "metric_DiskTotalBytes", "Total disk space available from system.disks", metric[2], prometheus.GaugeValue, - labels, values) + labelNames := []string{"disk"} + labelValues := []string{metric[0]} + w.writeSingleMetricToPrometheus( + "metric_DiskFreeBytes", "Free disk space available from system.disks", + prometheus.GaugeValue, metric[1], + labelNames, labelValues) + w.writeSingleMetricToPrometheus( + "metric_DiskTotalBytes", "Total disk space available from system.disks", + prometheus.GaugeValue, metric[2], + labelNames, labelValues) } } // WriteDetachedParts writes detached parts func (w *CHIPrometheusWriter) WriteDetachedParts(data [][]string) { for _, metric := range data { - w.writeSingleMetricToPrometheus(w.out, - "metric_DetachedParts", "Count of currently detached parts from system.detached_parts", metric[0], prometheus.GaugeValue, - // Label names - []string{"database", "table", "disk", "reason"}, - // Label values - []string{metric[1], metric[2], metric[3], metric[4]}) + labelNames := []string{"database", "table", "disk", "reason"} + labelValues := []string{metric[1], metric[2], metric[3], metric[4]} + w.writeSingleMetricToPrometheus( + "metric_DetachedParts", "Count of currently detached parts from system.detached_parts", + prometheus.GaugeValue, metric[0], + labelNames, labelValues) } } // WriteErrorFetch writes error fetch func (w *CHIPrometheusWriter) WriteErrorFetch(fetchType string) { - w.writeSingleMetricToPrometheus(w.out, - "metric_fetch_errors", "status of fetching metrics from ClickHouse 1 - unsuccessful, 0 - successful", "1", prometheus.GaugeValue, - // Label names - []string{"fetch_type"}, - // Label values - []string{fetchType}) + labelNames := []string{"fetch_type"} + labelValues := []string{fetchType} + w.writeSingleMetricToPrometheus( + "metric_fetch_errors", "status of fetching metrics from ClickHouse 1 - unsuccessful, 0 - successful", + prometheus.GaugeValue, "1", + labelNames, labelValues) } // WriteOKFetch writes successful fetch func (w *CHIPrometheusWriter) WriteOKFetch(fetchType string) { - w.writeSingleMetricToPrometheus(w.out, - "metric_fetch_errors", "status of fetching metrics from ClickHouse 1 - unsuccessful, 0 - successful", "0", prometheus.GaugeValue, - // Label names - []string{"fetch_type"}, - // Label values - []string{fetchType}) + labelNames := []string{"fetch_type"} + labelValues := []string{fetchType} + w.writeSingleMetricToPrometheus( + "metric_fetch_errors", "status of fetching metrics from ClickHouse 1 - unsuccessful, 0 - successful", + prometheus.GaugeValue, "0", + labelNames, labelValues) } func (w *CHIPrometheusWriter) getCHILabels() (labels []string) { @@ -258,59 +268,44 @@ func (w *CHIPrometheusWriter) getMandatoryLabelsAndValues() (mandatoryLabels []s } func (w *CHIPrometheusWriter) writeSingleMetricToPrometheus( - out chan<- prometheus.Metric, name string, desc string, - value string, metricType prometheus.ValueType, + value string, optionalLabels []string, optionalLabelValues []string, ) { - labels, labelValues := w.getMandatoryLabelsAndValues() - labels = append(labels, optionalLabels...) + // Prepare mandatory set of labels + labelNames, labelValues := w.getMandatoryLabelsAndValues() + // Append optional labels + labelNames = append(labelNames, optionalLabels...) labelValues = append(labelValues, optionalLabelValues...) floatValue, _ := strconv.ParseFloat(value, 64) - m, err := prometheus.NewConstMetric( - newDescription(name, desc, labels), + metric, err := prometheus.NewConstMetric( + newMetricDescriptor(name, desc, labelNames), metricType, floatValue, labelValues..., ) if err != nil { - log.Warningf("Error creating metric %s: %s", name, err) + log.Warningf("Error creating metric: %s err: %s", name, err) return } + // Send metric into channel select { - case out <- m: + case w.out <- metric: case <-time.After(writeMetricWaitTimeout): - log.Warningf("Error sending metric to the channel %s", name) + log.Warningf("Error sending metric to the channel: %s", name) } } -// newDescription creates a new prometheus.Desc object -func newDescription(name, help string, labels []string) *prometheus.Desc { +// newMetricDescriptor creates a new prometheus.Desc object +func newMetricDescriptor(name, help string, labels []string) *prometheus.Desc { return prometheus.NewDesc( - prometheus.BuildFQName(namespace, subsystem, name), + prometheus.BuildFQName(namespace, subsystem, util.BuildPrometheusMetricName(name)), help, - labels, + util.BuildPrometheusLabels(labels...), nil, ) } - -// convertMetricName converts the given string to snake case following the Golang format: -// acronyms are converted to lower-case and preceded by an underscore. -func convertMetricName(in string) string { - /*runes := []rune(in) - length := len(runes) - - var out []rune - for i := 0; i < length; i++ { - if i > 0 && unicode.IsUpper(runes[i]) && ((i+1 < length && unicode.IsLower(runes[i+1])) || unicode.IsLower(runes[i-1])) { - out = append(out, '_') - } - out = append(out, unicode.ToLower(runes[i])) - }*/ - - return strings.NewReplacer("-", "_", ".", "_").Replace(in) -} diff --git a/pkg/chop/config_manager.go b/pkg/chop/config_manager.go index b835fcee9..598dad2f9 100644 --- a/pkg/chop/config_manager.go +++ b/pkg/chop/config_manager.go @@ -88,14 +88,14 @@ func (cm *ConfigManager) Init() error { if err != nil { return err } - log.V(1).Info("File-based CHOP config start:") - log.V(1).Info(cm.fileConfig.String(true)) - log.V(1).Info("File-based CHOP config end.") + log.V(1).Info("File-based CHOP config:") + log.V(1).Info("\n" + cm.fileConfig.String(true)) - // Get configs from all config Custom Resources - watchedNamespace := cm.fileConfig.GetInformerNamespace() - cm.getCRBasedConfigs(watchedNamespace) - cm.logCRBasedConfigs() + // Get configs from all config Custom Resources that are located in the namespace where Operator is running + if namespace, ok := cm.GetRuntimeParam(deployment.OPERATOR_POD_NAMESPACE); ok { + cm.getAllCRBasedConfigs(namespace) + cm.logAllCRBasedConfigs() + } // Prepare one unified config from all available config pieces cm.buildUnifiedConfig() @@ -103,17 +103,15 @@ func (cm *ConfigManager) Init() error { cm.fetchSecretCredentials() // From now on we have one unified CHOP config - log.V(1).Info("Unified CHOP config (but not post-processed yet) start:") - log.V(1).Info(cm.config.String(true)) - log.V(1).Info("Unified CHOP config (but not post-processed yet) end.") + log.V(1).Info("Unified CHOP config - with secret data fetched (but not post-processed yet):") + log.V(1).Info("\n" + cm.config.String(true)) // Finalize config by post-processing cm.Postprocess() // OperatorConfig is ready - log.V(1).Info("Final CHOP config start:") - log.V(1).Info(cm.config.String(true)) - log.V(1).Info("Final CHOP config end.") + log.V(1).Info("Final CHOP config:") + log.V(1).Info("\n" + cm.config.String(true)) return nil } @@ -123,17 +121,19 @@ func (cm *ConfigManager) Config() *api.OperatorConfig { return cm.config } -// getCRBasedConfigs reads all ClickHouseOperatorConfiguration objects in specified namespace -func (cm *ConfigManager) getCRBasedConfigs(namespace string) { +// getAllCRBasedConfigs reads all ClickHouseOperatorConfiguration objects in specified namespace +func (cm *ConfigManager) getAllCRBasedConfigs(namespace string) { // We need to have chop kube client available in order to fetch ClickHouseOperatorConfiguration objects if cm.chopClient == nil { return } + log.V(1).F().Info("Looking for ClickHouseOperatorConfigurations in namespace '%s'.", namespace) + // Get list of ClickHouseOperatorConfiguration objects var err error if cm.chopConfigList, err = cm.chopClient.ClickhouseV1().ClickHouseOperatorConfigurations(namespace).List(context.TODO(), controller.NewListOptions()); err != nil { - log.V(1).F().Error("Error read ClickHouseOperatorConfigurations %v", err) + log.V(1).F().Error("Error read ClickHouseOperatorConfigurations in namespace '%s'. Err: %v", namespace, err) return } @@ -156,21 +156,23 @@ func (cm *ConfigManager) getCRBasedConfigs(namespace string) { chOperatorConfiguration := &cm.chopConfigList.Items[i] if chOperatorConfiguration.Name == name { // Save location info into OperatorConfig itself - chOperatorConfiguration.Spec.Runtime.ConfigFolderPath = namespace - chOperatorConfiguration.Spec.Runtime.ConfigFilePath = name + chOperatorConfiguration.Spec.Runtime.ConfigCRNamespace = namespace + chOperatorConfiguration.Spec.Runtime.ConfigCRName = name cm.crConfigs = append(cm.crConfigs, &chOperatorConfiguration.Spec) + + log.V(1).F().Error("Append ClickHouseOperatorConfigurations '%s/%s'.", namespace, name) continue } } } } -// logCRBasedConfigs writes all ClickHouseOperatorConfiguration objects into log -func (cm *ConfigManager) logCRBasedConfigs() { +// logAllCRBasedConfigs writes all ClickHouseOperatorConfiguration objects into log +func (cm *ConfigManager) logAllCRBasedConfigs() { for _, chOperatorConfiguration := range cm.crConfigs { - log.V(1).Info("chop config %s/%s :", chOperatorConfiguration.Runtime.ConfigFolderPath, chOperatorConfiguration.Runtime.ConfigFilePath) - log.V(1).Info(chOperatorConfiguration.String(true)) + log.V(1).Info("CR-based chop config: %s/%s :", chOperatorConfiguration.Runtime.ConfigCRNamespace, chOperatorConfiguration.Runtime.ConfigCRName) + log.V(1).Info("\n" + chOperatorConfiguration.String(true)) } } @@ -183,6 +185,10 @@ func (cm *ConfigManager) buildUnifiedConfig() { // Merge all the rest CR-based configs into base config for _, chOperatorConfiguration := range cm.crConfigs { _ = cm.config.MergeFrom(chOperatorConfiguration, api.MergeTypeOverrideByNonEmptyValues) + cm.config.Runtime.ConfigCRSources = append(cm.config.Runtime.ConfigCRSources, api.ConfigCRSource{ + Namespace: chOperatorConfiguration.Runtime.ConfigCRNamespace, + Name: chOperatorConfiguration.Runtime.ConfigCRName, + }) } } @@ -207,14 +213,14 @@ const ( fileIsMandatory = false ) -// getFileBasedConfig creates OperatorConfig object based on file specified +// getFileBasedConfig creates one OperatorConfig object based on the first available configuration file func (cm *ConfigManager) getFileBasedConfig(configFilePath string) (*api.OperatorConfig, error) { - // Check config files in the following order: + // Check config files availability in the following order: // 1. Explicitly specified config file as CLI option // 2. Explicitly specified config file as ENV var // 3. Well-known config file in home dir // 4. Well-known config file in /etc - // In case no file found fallback to default config + // In case no file found fallback to the default config // 1. Check config file as explicitly specified CLI option if len(configFilePath) > 0 { @@ -385,12 +391,12 @@ func (cm *ConfigManager) GetRuntimeParam(name string) (string, bool) { // fetchSecretCredentials func (cm *ConfigManager) fetchSecretCredentials() { - // Secret name where to look for credentials + // Secret name where to look for ClickHouse access credentials name := cm.config.ClickHouse.Access.Secret.Name // Do we need to fetch credentials from the secret? if name == "" { - // No name specified, no need to read secret + // No secret name specified, no need to read it return } @@ -405,6 +411,8 @@ func (cm *ConfigManager) fetchSecretCredentials() { } } + log.V(1).Info("Going to search for username/password in the secret '%s/%s'", namespace, name) + // Sanity check if namespace == "" { // We've already checked that name is not empty @@ -415,18 +423,22 @@ func (cm *ConfigManager) fetchSecretCredentials() { secret, err := cm.kubeClient.CoreV1().Secrets(namespace).Get(context.TODO(), name, controller.NewGetOptions()) if err != nil { cm.config.ClickHouse.Access.Secret.Runtime.Error = err.Error() + log.V(1).Warning("Unable to fetch secret '%s/%s'", namespace, name) return } cm.config.ClickHouse.Access.Secret.Runtime.Fetched = true + log.V(1).Info("Secret fetched %s/%s :", namespace, name) // Find username and password from credentials for key, value := range secret.Data { switch key { case "username": cm.config.ClickHouse.Access.Secret.Runtime.Username = string(value) + log.V(1).Info("Username read from the secret '%s/%s'", namespace, name) case "password": cm.config.ClickHouse.Access.Secret.Runtime.Password = string(value) + log.V(1).Info("Password read from the secret '%s/%s'", namespace, name) } } } diff --git a/pkg/controller/chk/reconciler.go b/pkg/controller/chk/reconciler.go index bc74c6df5..6cb60ffcd 100644 --- a/pkg/controller/chk/reconciler.go +++ b/pkg/controller/chk/reconciler.go @@ -70,17 +70,17 @@ func (r *ChkReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R } if new.HasAncestor() { - log.V(1).M(new).F().Info("has ancestor, use it as a base for reconcile. CHK: %s/%s", new.Namespace, new.Name) + log.V(2).M(new).F().Info("has ancestor, use it as a base for reconcile. CHK: %s/%s", new.Namespace, new.Name) old = new.GetAncestor() } else { - log.V(1).M(new).F().Info("has NO ancestor, use empty CHK as a base for reconcile. CHK: %s/%s", new.Namespace, new.Name) + log.V(2).M(new).F().Info("has NO ancestor, use empty CHK as a base for reconcile. CHK: %s/%s", new.Namespace, new.Name) old = nil } - log.V(1).M(new).F().Info("Normalized OLD CHK: %s/%s", new.Namespace, new.Name) + log.V(2).M(new).F().Info("Normalized OLD CHK: %s/%s", new.Namespace, new.Name) old = r.normalize(old) - log.V(1).M(new).F().Info("Normalized NEW CHK %s/%s", new.Namespace, new.Name) + log.V(2).M(new).F().Info("Normalized NEW CHK %s/%s", new.Namespace, new.Name) new = r.normalize(new) new.SetAncestor(old) @@ -279,7 +279,7 @@ func (r *ChkReconciler) reconcileClusterStatus(chk *apiChk.ClickHouseKeeperInsta }) } - log.V(1).Info("ReadyReplicas: " + fmt.Sprintf("%v", cur.Status.ReadyReplicas)) + log.V(2).Info("ReadyReplicas: " + fmt.Sprintf("%v", cur.Status.ReadyReplicas)) if len(readyMembers) == model.GetReplicasCount(chk) { cur.Status.Status = "Completed" diff --git a/pkg/model/chi/normalizer.go b/pkg/model/chi/normalizer.go index 873da0270..bbf05e353 100644 --- a/pkg/model/chi/normalizer.go +++ b/pkg/model/chi/normalizer.go @@ -1183,7 +1183,7 @@ func (n *Normalizer) fetchSecretFieldValue(secretAddress api.ObjectAddress) (str func (n *Normalizer) normalizeUsersList(users *api.Settings, extraUsernames ...string) (usernames []string) { usernames = append(usernames, users.Groups()...) usernames = append(usernames, extraUsernames...) - usernames = util.Unique(usernames) + usernames = util.NonEmpty(util.Unique(usernames)) sort.Strings(usernames) return usernames diff --git a/pkg/util/array.go b/pkg/util/array.go index 8cc4161d9..46d364dfa 100644 --- a/pkg/util/array.go +++ b/pkg/util/array.go @@ -150,3 +150,12 @@ func Unique(slice []string) (unique []string) { return unique } + +func NonEmpty(slice []string) (nonEmpty []string) { + for _, str := range slice { + if str != "" { + nonEmpty = append(nonEmpty, str) + } + } + return nonEmpty +} diff --git a/pkg/util/dir.go b/pkg/util/dir.go index 836a4bac4..a3877d538 100644 --- a/pkg/util/dir.go +++ b/pkg/util/dir.go @@ -35,16 +35,22 @@ func RelativeToBasePath(basePath, relativePath string) string { // PreparePath - prepares path absolute/relative with default relative value func PreparePath(path *string, basePath, defaultRelativePath string) { - if *path == "" { - // Path is not specified, try to build it relative to specified base + switch { + case *path == "": + // Path is not specified at all. + // Build path as 'default' path relative to the specified base *path = RelativeToBasePath(basePath, defaultRelativePath) - } else if filepath.IsAbs(*path) { - // Absolute path explicitly specified - nothing to do here - } else { - // Relative path is specified - make relative path relative to base path + case filepath.IsAbs(*path): + // Path is specified as an absolute path. + // Absolute paths are considered to be prepared already. + // Nothing to do here. + default: + // Path is specified as relative path. + // Build result path as provided relative path relative to the base path *path = RelativeToBasePath(basePath, *path) } + // Specified path should exist // In case of incorrect/unavailable path - make it empty if (*path != "") && !IsDirOk(*path) { *path = "" diff --git a/pkg/util/prometheus.go b/pkg/util/prometheus.go new file mode 100644 index 000000000..bd130b23a --- /dev/null +++ b/pkg/util/prometheus.go @@ -0,0 +1,76 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "regexp" +) + +// Labels may contain ASCII letters, numbers, as well as underscores. +// They must match the regex [a-zA-Z_][a-zA-Z0-9_]* +// For more details check: +// https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels + +const prometheusLabelFormat string = "[a-zA-Z_][a-zA-Z0-9_]*" +const prometheusLabelNotAllowedCharsFormat string = "[^a-zA-Z0-9_]" + +var prometheusLabelRegexp = regexp.MustCompile("^" + prometheusLabelFormat + "$") +var prometheusLabelNotAllowedCharsRegexp = regexp.MustCompile(prometheusLabelNotAllowedCharsFormat) + +// Metric names may contain ASCII letters, digits, underscores, and colons. +// It must match the regex [a-zA-Z_:][a-zA-Z0-9_:]* +// For more details check: +// https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels + +const prometheusMetricFormat string = "[a-zA-Z_:][a-zA-Z0-9_:]*" +const prometheusMetricNotAllowedCharsFormat string = "[^a-zA-Z0-9_:]" + +var prometheusMetricRegexp = regexp.MustCompile("^" + prometheusMetricFormat + "$") +var prometheusMetricNotAllowedCharsRegexp = regexp.MustCompile(prometheusMetricNotAllowedCharsFormat) + +func BuildPrometheusLabel(label string) string { + // Replace not allowed chars + return prometheusLabelNotAllowedCharsRegexp.ReplaceAllString(label, "_") +} + +func BuildPrometheusLabels(labels ...string) []string { + var res []string + for _, label := range labels { + res = append(res, BuildPrometheusLabel(label)) + } + return res +} + +// IsValidPrometheusLabel tests for a string that conforms to the definition of a label in Prometheus +func IsValidPrometheusLabel(value string) bool { + return prometheusLabelRegexp.MatchString(value) +} + +func IsValidPrometheusLabelValue(value string) bool { + // Label values may contain any Unicode characters + // For more details check: + // https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels + return true +} + +func BuildPrometheusMetricName(name string) string { + // Replace not allowed chars + return prometheusMetricNotAllowedCharsRegexp.ReplaceAllString(name, "_") +} + +// IsValidPrometheusMetricName tests for a string that conforms to the definition of a metric in Prometheus +func IsValidPrometheusMetricName(value string) bool { + return prometheusMetricRegexp.MatchString(value) +} diff --git a/pkg/util/rfc1035.go b/pkg/util/rfc1035.go index 739b6b1ec..c35d4aef0 100644 --- a/pkg/util/rfc1035.go +++ b/pkg/util/rfc1035.go @@ -19,12 +19,14 @@ import ( "strings" ) -const dns1035LabelFmt string = "[a-z]([-a-z0-9]*[a-z0-9])?" const dns1035LabelMaxLength int = 63 +const dns1035LabelFormat string = "[a-z]([-a-z0-9]*[a-z0-9])?" +const dns1035LabelStartFormat string = "[a-z]" +const dns1035LabelNotAllowedCharsFormat string = "[^-a-z0-9]" -var dns1035LabelRegexp = regexp.MustCompile("^" + dns1035LabelFmt + "$") -var dns1035LabelStartRegexp = regexp.MustCompile("^" + "[a-z]") -var dns1035LabelNotAllowedCharsRegexp = regexp.MustCompile("[^-a-z0-9]") +var dns1035LabelRegexp = regexp.MustCompile("^" + dns1035LabelFormat + "$") +var dns1035LabelStartRegexp = regexp.MustCompile("^" + dns1035LabelStartFormat) +var dns1035LabelNotAllowedCharsRegexp = regexp.MustCompile(dns1035LabelNotAllowedCharsFormat) func BuildRFC1035Label(str string) (string, bool) { // Must be lowercase @@ -44,15 +46,15 @@ func BuildRFC1035Label(str string) (string, bool) { str = str[0:dns1035LabelMaxLength] } - if IsDNS1035Label(str) { + if IsValidDNS1035Label(str) { return str, true } return "", false } -// IsDNS1035Label tests for a string that conforms to the definition of a label in DNS (RFC 1035). -func IsDNS1035Label(value string) bool { +// IsValidDNS1035Label tests for a string that conforms to the definition of a label in DNS (RFC 1035). +func IsValidDNS1035Label(value string) bool { if len(value) > dns1035LabelMaxLength { return false } diff --git a/release b/release index ca222b7cf..610e28725 100644 --- a/release +++ b/release @@ -1 +1 @@ -0.23.0 +0.23.1 diff --git a/releases b/releases index 90bd9ef7b..a43064501 100644 --- a/releases +++ b/releases @@ -1,3 +1,4 @@ +0.23.0 0.22.2 0.22.1 0.22.0 diff --git a/tests/e2e/install_olm.sh b/tests/e2e/install_olm.sh new file mode 100755 index 000000000..3780a8ac7 --- /dev/null +++ b/tests/e2e/install_olm.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +kubectl create -f https://raw.githubusercontent.com/operator-framework/operator-lifecycle-manager/master/deploy/upstream/quickstart/crds.yaml +kubectl create -f https://raw.githubusercontent.com/operator-framework/operator-lifecycle-manager/master/deploy/upstream/quickstart/olm.yaml + +kubectl create ns mynamespace +kubectl apply -f olm.yaml diff --git a/tests/e2e/kubectl.py b/tests/e2e/kubectl.py index e66cb064d..7896f0129 100644 --- a/tests/e2e/kubectl.py +++ b/tests/e2e/kubectl.py @@ -93,7 +93,10 @@ def delete_all_chi(ns=None): if "items" in chis: for chi in chis["items"]: # kubectl(f"patch chi {chi} --type=merge -p '\{\"metadata\":\{\"finalizers\": [null]\}\}'", ns = ns) - delete_chi(chi["metadata"]["name"], ns) + delete_chi(chi["metadata"]["name"], ns, wait = False) + for chi in chis["items"]: + wait_object("chi", chi["metadata"]["name"], ns=ns, count=0) + def delete_all_keeper(ns=None): diff --git a/tests/e2e/manifests/chi/test-005-acm.yaml b/tests/e2e/manifests/chi/test-005-acm.yaml index d9f413105..e5fbfb419 100644 --- a/tests/e2e/manifests/chi/test-005-acm.yaml +++ b/tests/e2e/manifests/chi/test-005-acm.yaml @@ -13,7 +13,7 @@ spec: fsGroup: 101 containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 ports: - name: http containerPort: 8123 diff --git a/tests/e2e/manifests/chi/test-006-ch-upgrade-1.yaml b/tests/e2e/manifests/chi/test-006-ch-upgrade-1.yaml index a147a9a24..88ca38049 100644 --- a/tests/e2e/manifests/chi/test-006-ch-upgrade-1.yaml +++ b/tests/e2e/manifests/chi/test-006-ch-upgrade-1.yaml @@ -9,12 +9,12 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.3 defaults: templates: podTemplate: clickhouse-old configuration: clusters: - - name: shards + - name: default layout: replicasCount: 2 diff --git a/tests/e2e/manifests/chi/test-006-ch-upgrade-2.yaml b/tests/e2e/manifests/chi/test-006-ch-upgrade-2.yaml index 73cc45cd1..042318b0a 100644 --- a/tests/e2e/manifests/chi/test-006-ch-upgrade-2.yaml +++ b/tests/e2e/manifests/chi/test-006-ch-upgrade-2.yaml @@ -9,12 +9,12 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 defaults: templates: podTemplate: clickhouse-new configuration: clusters: - - name: shards + - name: default layout: replicasCount: 2 diff --git a/tests/e2e/manifests/chi/test-006-ch-upgrade-3.yaml b/tests/e2e/manifests/chi/test-006-ch-upgrade-3.yaml index 698f5587a..2a43785a5 100644 --- a/tests/e2e/manifests/chi/test-006-ch-upgrade-3.yaml +++ b/tests/e2e/manifests/chi/test-006-ch-upgrade-3.yaml @@ -9,12 +9,12 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.3 defaults: templates: podTemplate: clickhouse-new configuration: clusters: - - name: shards + - name: default layout: replicasCount: 2 diff --git a/tests/e2e/manifests/chi/test-008-operator-restart-2.yaml b/tests/e2e/manifests/chi/test-008-operator-restart-2.yaml index fdb47b44f..15a499ac1 100644 --- a/tests/e2e/manifests/chi/test-008-operator-restart-2.yaml +++ b/tests/e2e/manifests/chi/test-008-operator-restart-2.yaml @@ -64,7 +64,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 ports: - name: http containerPort: 8123 diff --git a/tests/e2e/manifests/chi/test-008-operator-restart-3-1.yaml b/tests/e2e/manifests/chi/test-008-operator-restart-3-1.yaml index e3b2d9fe5..17e846afa 100644 --- a/tests/e2e/manifests/chi/test-008-operator-restart-3-1.yaml +++ b/tests/e2e/manifests/chi/test-008-operator-restart-3-1.yaml @@ -30,4 +30,4 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 \ No newline at end of file + image: clickhouse/clickhouse-server:23.3 \ No newline at end of file diff --git a/tests/e2e/manifests/chi/test-011-secured-cluster-1.yaml b/tests/e2e/manifests/chi/test-011-secured-cluster-1.yaml index 8678a5424..9dc7cc489 100644 --- a/tests/e2e/manifests/chi/test-011-secured-cluster-1.yaml +++ b/tests/e2e/manifests/chi/test-011-secured-cluster-1.yaml @@ -7,33 +7,35 @@ spec: - name: clickhouse-version configuration: users: - # default user should be security to localhost and interhost connections + # `default` user should be security to localhost and interhost connections # operator adds IP addresses of pods # operator creates host_regxp expression for that # default/networks/host_regexp: \.chi-test-011-secure-user-[^.]+-\d+-\d+\.test.svc.cluster.local$ default/profile: default default/quota: default default/networks/ip: - - 127.0.0.1 - - 127.0.0.2 + - 127.0.0.1 + - 127.0.0.2 + # user1 with a password user1/password: topsecret user1/networks/ip: "::/0" + # user2 should get default password automatically user2/networks/ip: "::/0" - # user3 should get sha256 correctly + + # user3 should get sha256 correctly and plain password should be ignored user3/password: abcd user3/password_sha256_hex: 716b36073a90c6fe1d445ac1af85f4777c5b7a155cea359961826a030513e448 + # restricted user with row-level security restricted/password: secret restricted/databases/system/numbers/filter: "number=1000" + # User with access control enabled user4/password: secret user4/access_management: 1 - # User with google.com as a host filter can not login - user5/password: secret - user5/networks/ip: "::/0" - user5/networks/host: www.google.com + # clickhouse_operator password should be accepted and encrypted clickhuse_operator/password: operator_secret clusters: diff --git a/tests/e2e/manifests/chi/test-011-secured-cluster-2.yaml b/tests/e2e/manifests/chi/test-011-secured-cluster-2.yaml index af875c99c..30b287efc 100644 --- a/tests/e2e/manifests/chi/test-011-secured-cluster-2.yaml +++ b/tests/e2e/manifests/chi/test-011-secured-cluster-2.yaml @@ -7,33 +7,35 @@ spec: - name: clickhouse-version configuration: users: - # default user should be security to localhost and interhost connections + # `default` user should be security to localhost and interhost connections # operator adds IP addresses of pods # operator will not create host_regxp expression for that - disabled default/networks/host_regexp: "disabled" default/profile: default default/quota: default default/networks/ip: - - 127.0.0.1 - - 127.0.0.2 + - 127.0.0.1 + - 127.0.0.2 + # user1 with a password user1/password: topsecret user1/networks/ip: "::/0" + # user2 should get default password automatically user2/networks/ip: "::/0" - # user3 should get sha256 correctly + + # user3 should get sha256 correctly and plain password should be ignored user3/password: abcd user3/password_sha256_hex: 716b36073a90c6fe1d445ac1af85f4777c5b7a155cea359961826a030513e448 + # restricted user with row-level security restricted/password: secret restricted/databases/system/numbers/filter: "number=1000" + # User with access control enabled user4/password: secret user4/access_management: 1 - # User with google.com as a host filter can not login - user5/password: secret - user5/networks/ip: "::/0" - user5/networks/host: www.google.com + # clickhouse_operator password should be accepted and encrypted clickhuse_operator/password: operator_secret clusters: diff --git a/tests/e2e/manifests/chi/test-015-host-network.yaml b/tests/e2e/manifests/chi/test-015-host-network.yaml index 42eda30f1..d44a8ca88 100644 --- a/tests/e2e/manifests/chi/test-015-host-network.yaml +++ b/tests/e2e/manifests/chi/test-015-host-network.yaml @@ -41,7 +41,7 @@ spec: hostNetwork: true containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 # dnsPolicy: ClusterFirstWithHostNet diff --git a/tests/e2e/manifests/chi/test-017-multi-version.yaml b/tests/e2e/manifests/chi/test-017-multi-version.yaml index 2cd2137f5..215c42595 100644 --- a/tests/e2e/manifests/chi/test-017-multi-version.yaml +++ b/tests/e2e/manifests/chi/test-017-multi-version.yaml @@ -2,28 +2,30 @@ apiVersion: clickhouse.altinity.com/v1 kind: ClickHouseInstallation metadata: name: test-017-multi-version + labels: + clickhouse.altinity.com/chi: test-017-multi-version spec: templates: podTemplates: - - name: v22.3 + - name: v23.3 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 - - name: v22.8 + image: clickhouse/clickhouse-server:23.3 + - name: v23.8 spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 configuration: clusters: - name: default layout: shards: - templates: - podTemplate: v22.3 + podTemplate: v23.3 - templates: - podTemplate: v22.8 + podTemplate: v23.8 files: users.d/remove_database_ordinary.xml: | diff --git a/tests/e2e/manifests/chi/test-020-1-multi-volume.yaml b/tests/e2e/manifests/chi/test-020-1-multi-volume.yaml index b18caf8c4..f2cd7a6d5 100644 --- a/tests/e2e/manifests/chi/test-020-1-multi-volume.yaml +++ b/tests/e2e/manifests/chi/test-020-1-multi-volume.yaml @@ -38,7 +38,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: disk1 mountPath: /var/lib/clickhouse diff --git a/tests/e2e/manifests/chi/test-020-2-multi-volume.yaml b/tests/e2e/manifests/chi/test-020-2-multi-volume.yaml index 9abb5adbb..3356476ae 100644 --- a/tests/e2e/manifests/chi/test-020-2-multi-volume.yaml +++ b/tests/e2e/manifests/chi/test-020-2-multi-volume.yaml @@ -38,7 +38,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: disk1 mountPath: /var/lib/clickhouse diff --git a/tests/e2e/manifests/chi/test-021-1-rescale-volume-01.yaml b/tests/e2e/manifests/chi/test-021-1-rescale-volume-01.yaml index d284d5c30..3b4e82b12 100644 --- a/tests/e2e/manifests/chi/test-021-1-rescale-volume-01.yaml +++ b/tests/e2e/manifests/chi/test-021-1-rescale-volume-01.yaml @@ -27,7 +27,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: disk1 mountPath: /var/lib/clickhouse diff --git a/tests/e2e/manifests/chi/test-021-1-rescale-volume-02-enlarge-disk.yaml b/tests/e2e/manifests/chi/test-021-1-rescale-volume-02-enlarge-disk.yaml index a47756837..0ee08d344 100644 --- a/tests/e2e/manifests/chi/test-021-1-rescale-volume-02-enlarge-disk.yaml +++ b/tests/e2e/manifests/chi/test-021-1-rescale-volume-02-enlarge-disk.yaml @@ -27,7 +27,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: disk1 mountPath: /var/lib/clickhouse diff --git a/tests/e2e/manifests/chi/test-021-1-rescale-volume-03-add-disk.yaml b/tests/e2e/manifests/chi/test-021-1-rescale-volume-03-add-disk.yaml index 01be63c2b..bad8d951f 100644 --- a/tests/e2e/manifests/chi/test-021-1-rescale-volume-03-add-disk.yaml +++ b/tests/e2e/manifests/chi/test-021-1-rescale-volume-03-add-disk.yaml @@ -38,7 +38,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: disk1 mountPath: /var/lib/clickhouse diff --git a/tests/e2e/manifests/chi/test-021-1-rescale-volume-04-decrease-disk.yaml b/tests/e2e/manifests/chi/test-021-1-rescale-volume-04-decrease-disk.yaml index ab2ca5cc4..6543edd6c 100644 --- a/tests/e2e/manifests/chi/test-021-1-rescale-volume-04-decrease-disk.yaml +++ b/tests/e2e/manifests/chi/test-021-1-rescale-volume-04-decrease-disk.yaml @@ -38,7 +38,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: disk1 mountPath: /var/lib/clickhouse diff --git a/tests/e2e/manifests/chi/test-021-2-rescale-volume-01.yaml b/tests/e2e/manifests/chi/test-021-2-rescale-volume-01.yaml index ab6041fb5..5032f6744 100644 --- a/tests/e2e/manifests/chi/test-021-2-rescale-volume-01.yaml +++ b/tests/e2e/manifests/chi/test-021-2-rescale-volume-01.yaml @@ -29,7 +29,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: disk1 mountPath: /var/lib/clickhouse diff --git a/tests/e2e/manifests/chi/test-021-2-rescale-volume-02-enlarge-disk.yaml b/tests/e2e/manifests/chi/test-021-2-rescale-volume-02-enlarge-disk.yaml index e52b53d28..f1233f268 100644 --- a/tests/e2e/manifests/chi/test-021-2-rescale-volume-02-enlarge-disk.yaml +++ b/tests/e2e/manifests/chi/test-021-2-rescale-volume-02-enlarge-disk.yaml @@ -29,7 +29,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: disk1 mountPath: /var/lib/clickhouse diff --git a/tests/e2e/manifests/chi/test-021-2-rescale-volume-03-add-disk.yaml b/tests/e2e/manifests/chi/test-021-2-rescale-volume-03-add-disk.yaml index 2bc6ee6e4..83f4c0696 100644 --- a/tests/e2e/manifests/chi/test-021-2-rescale-volume-03-add-disk.yaml +++ b/tests/e2e/manifests/chi/test-021-2-rescale-volume-03-add-disk.yaml @@ -40,7 +40,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: disk1 mountPath: /var/lib/clickhouse diff --git a/tests/e2e/manifests/chi/test-021-2-rescale-volume-04-decrease-disk.yaml b/tests/e2e/manifests/chi/test-021-2-rescale-volume-04-decrease-disk.yaml index 9c7111b88..aba2d2401 100644 --- a/tests/e2e/manifests/chi/test-021-2-rescale-volume-04-decrease-disk.yaml +++ b/tests/e2e/manifests/chi/test-021-2-rescale-volume-04-decrease-disk.yaml @@ -40,7 +40,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: disk1 mountPath: /var/lib/clickhouse diff --git a/tests/e2e/manifests/chi/test-022-broken-image.yaml b/tests/e2e/manifests/chi/test-022-broken-image.yaml index 0211f871c..7b2a0d3b5 100644 --- a/tests/e2e/manifests/chi/test-022-broken-image.yaml +++ b/tests/e2e/manifests/chi/test-022-broken-image.yaml @@ -12,7 +12,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8-broken + image: clickhouse/clickhouse-server:23.8-broken configuration: clusters: - name: default diff --git a/tests/e2e/manifests/chi/test-023-auto-templates.yaml b/tests/e2e/manifests/chi/test-023-auto-templates.yaml index d93a4aecb..ebdfde075 100644 --- a/tests/e2e/manifests/chi/test-023-auto-templates.yaml +++ b/tests/e2e/manifests/chi/test-023-auto-templates.yaml @@ -14,4 +14,4 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 \ No newline at end of file + image: clickhouse/clickhouse-server:23.8 \ No newline at end of file diff --git a/tests/e2e/manifests/chi/test-024-template-annotations.yaml b/tests/e2e/manifests/chi/test-024-template-annotations.yaml index 4fd6ca7a2..bd91b847e 100644 --- a/tests/e2e/manifests/chi/test-024-template-annotations.yaml +++ b/tests/e2e/manifests/chi/test-024-template-annotations.yaml @@ -14,7 +14,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 volumeClaimTemplates: - name: default-volumeclaim-template reclaimPolicy: Delete diff --git a/tests/e2e/manifests/chi/test-025-rescaling-2.yaml b/tests/e2e/manifests/chi/test-025-rescaling-2.yaml index 6b3308793..f10a071d3 100644 --- a/tests/e2e/manifests/chi/test-025-rescaling-2.yaml +++ b/tests/e2e/manifests/chi/test-025-rescaling-2.yaml @@ -27,7 +27,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 command: - "/bin/bash" - "-c" diff --git a/tests/e2e/manifests/chi/test-025-rescaling.yaml b/tests/e2e/manifests/chi/test-025-rescaling.yaml index 33d291997..2872fc989 100644 --- a/tests/e2e/manifests/chi/test-025-rescaling.yaml +++ b/tests/e2e/manifests/chi/test-025-rescaling.yaml @@ -27,7 +27,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 command: - "/bin/bash" - "-c" diff --git a/tests/e2e/manifests/chi/test-026-mixed-replicas.yaml b/tests/e2e/manifests/chi/test-026-mixed-replicas.yaml index 339f60b87..bd25eb95f 100644 --- a/tests/e2e/manifests/chi/test-026-mixed-replicas.yaml +++ b/tests/e2e/manifests/chi/test-026-mixed-replicas.yaml @@ -54,7 +54,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 - name: multi-volume spec: securityContext: @@ -63,7 +63,7 @@ spec: fsGroup: 101 containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 volumeMounts: - name: disk1 mountPath: /var/lib/clickhouse diff --git a/tests/e2e/manifests/chi/test-029-distribution.yaml b/tests/e2e/manifests/chi/test-029-distribution.yaml index 58cb99c5d..0275e1fd5 100644 --- a/tests/e2e/manifests/chi/test-029-distribution.yaml +++ b/tests/e2e/manifests/chi/test-029-distribution.yaml @@ -9,7 +9,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 podDistribution: - scope: ClickHouseInstallation type: ClickHouseAntiAffinity @@ -17,7 +17,7 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 podDistribution: - scope: ClickHouseInstallation type: ReplicaAntiAffinity diff --git a/tests/e2e/manifests/chi/test-032-rescaling.yaml b/tests/e2e/manifests/chi/test-032-rescaling.yaml index a578804a6..2c168710f 100644 --- a/tests/e2e/manifests/chi/test-032-rescaling.yaml +++ b/tests/e2e/manifests/chi/test-032-rescaling.yaml @@ -30,4 +30,4 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 diff --git a/tests/e2e/manifests/chi/test-043-0-logs-container-customizing.yaml b/tests/e2e/manifests/chi/test-043-0-logs-container-customizing.yaml index 203adb94d..aa108734f 100644 --- a/tests/e2e/manifests/chi/test-043-0-logs-container-customizing.yaml +++ b/tests/e2e/manifests/chi/test-043-0-logs-container-customizing.yaml @@ -9,7 +9,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 - name: clickhouse-log image: registry.access.redhat.com/ubi8/ubi-minimal:latest command: diff --git a/tests/e2e/manifests/chi/test-043-1-logs-container-customizing.yaml b/tests/e2e/manifests/chi/test-043-1-logs-container-customizing.yaml index 906a02a52..27f809407 100644 --- a/tests/e2e/manifests/chi/test-043-1-logs-container-customizing.yaml +++ b/tests/e2e/manifests/chi/test-043-1-logs-container-customizing.yaml @@ -9,7 +9,7 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:22.3 + image: clickhouse/clickhouse-server:23.8 volumeClaimTemplates: - name: log-volume-template spec: diff --git a/tests/e2e/manifests/chi/test-044-0-slow-propagation.yaml b/tests/e2e/manifests/chi/test-044-0-slow-propagation.yaml index 6c582e116..94a0e0df9 100644 --- a/tests/e2e/manifests/chi/test-044-0-slow-propagation.yaml +++ b/tests/e2e/manifests/chi/test-044-0-slow-propagation.yaml @@ -24,4 +24,4 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 \ No newline at end of file + image: clickhouse/clickhouse-server:23.8 \ No newline at end of file diff --git a/tests/e2e/manifests/chi/test-044-1-slow-propagation.yaml b/tests/e2e/manifests/chi/test-044-1-slow-propagation.yaml index 0cff47f71..ad65b63d9 100644 --- a/tests/e2e/manifests/chi/test-044-1-slow-propagation.yaml +++ b/tests/e2e/manifests/chi/test-044-1-slow-propagation.yaml @@ -26,12 +26,12 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 - name: slow-replica spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 command: - "/bin/bash" - "-c" diff --git a/tests/e2e/manifests/chi/test-044-2-slow-propagation.yaml b/tests/e2e/manifests/chi/test-044-2-slow-propagation.yaml index a7eb003a6..cf15b83b3 100644 --- a/tests/e2e/manifests/chi/test-044-2-slow-propagation.yaml +++ b/tests/e2e/manifests/chi/test-044-2-slow-propagation.yaml @@ -27,12 +27,12 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 - name: slow-replica spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 command: - "/bin/bash" - "-c" diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.6.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.6.yaml index c37bbb627..619c6ea7d 100644 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.6.yaml +++ b/tests/e2e/manifests/chit/tpl-clickhouse-22.6.yaml @@ -14,4 +14,4 @@ spec: containers: - name: clickhouse-pod image: clickhouse/clickhouse-server:22.6 - imagePullPolicy: IfNotPresent \ No newline at end of file + imagePullPolicy: IfNotPresent diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.7.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.7.yaml index faced663d..c04ecbc93 100644 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.7.yaml +++ b/tests/e2e/manifests/chit/tpl-clickhouse-22.7.yaml @@ -14,4 +14,4 @@ spec: containers: - name: clickhouse-pod image: clickhouse/clickhouse-server:22.7 - imagePullPolicy: IfNotPresent \ No newline at end of file + imagePullPolicy: IfNotPresent diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.8.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.8.yaml index a80e38edb..54e52c932 100644 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.8.yaml +++ b/tests/e2e/manifests/chit/tpl-clickhouse-22.8.yaml @@ -14,4 +14,4 @@ spec: containers: - name: clickhouse-pod image: clickhouse/clickhouse-server:22.8 - imagePullPolicy: IfNotPresent \ No newline at end of file + imagePullPolicy: IfNotPresent diff --git a/tests/e2e/manifests/chit/tpl-test-031.yaml b/tests/e2e/manifests/chit/tpl-test-031.yaml index 1647727b0..d248ba8aa 100644 --- a/tests/e2e/manifests/chit/tpl-test-031.yaml +++ b/tests/e2e/manifests/chit/tpl-test-031.yaml @@ -16,4 +16,4 @@ spec: spec: containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.8 + image: clickhouse/clickhouse-server:23.8 diff --git a/tests/e2e/olm.yaml b/tests/e2e/olm.yaml new file mode 100644 index 000000000..503c3f253 --- /dev/null +++ b/tests/e2e/olm.yaml @@ -0,0 +1,20 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: operatorgroup + namespace: mynamespace +spec: + targetNamespaces: + - mynamespace +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: clickhouse + namespace: mynamespace +spec: + channel: latest + name: clickhouse + source: operatorhubio-catalog + sourceNamespace: olm + installPlanApproval: Automatic diff --git a/tests/e2e/test_examples.py b/tests/e2e/test_examples.py index a36ae9e0e..1c8d07d19 100644 --- a/tests/e2e/test_examples.py +++ b/tests/e2e/test_examples.py @@ -55,7 +55,7 @@ def test_examples02_2(self): manifest="../../docs/chi-examples/03-persistent-volume-02-pod-template.yaml", check={ "pod_count": 1, - "pod_image": "clickhouse/clickhouse-server:22.3", + "pod_image": "clickhouse/clickhouse-server:23.8", "pod_volumes": { "/var/lib/clickhouse", "/var/log/clickhouse-server", diff --git a/tests/e2e/test_metrics_exporter.py b/tests/e2e/test_metrics_exporter.py index 3bedb5421..6234d49e1 100644 --- a/tests/e2e/test_metrics_exporter.py +++ b/tests/e2e/test_metrics_exporter.py @@ -21,8 +21,8 @@ def test_metrics_exporter_setup(self): @TestScenario -@Name("Check metrics server state after reboot") -def test_metrics_exporter_reboot(self): +@Name("Test basic metrics exporter functionality") +def test_metrics_exporter_chi(self): def check_monitoring_chi(operator_namespace, operator_pod, expect_result, max_retries=10): with Then(f"metrics-exporter /chi endpoint result should return {expect_result}"): for i in range(1, max_retries): @@ -41,9 +41,30 @@ def check_monitoring_chi(operator_namespace, operator_pod, expect_result, max_re out = json.loads(out) if out == expect_result: break + print(out) + print(expect_result) with Then("Not ready. Wait for " + str(i * 5) + " seconds"): time.sleep(i * 5) assert out == expect_result, error() + def check_monitoring_metrics(operator_namespace, operator_pod, expect_result, max_retries=10): + with Then(f"metrics-exporter /metrics endpoint result should match with {expect_result}"): + found = 0 + for i in range(1, max_retries): + url_cmd = util.make_http_get_request("127.0.0.1", "8888", "/metrics") + out = kubectl.launch( + f"exec {operator_pod} -c metrics-exporter -- {url_cmd}", + ns=operator_namespace, + ) + found = 0 + for string, exists in expect_result.items(): + if exists == (string in out): + found = found + 1 + print(f"FOUND: {string}") + if found == len(expect_result.items()): + break + with Then("Not ready. Wait for " + str(i * 5) + " seconds"): + time.sleep(i * 5) + assert found == len(expect_result.items()), error() with Given("clickhouse-operator is installed"): kubectl.wait_field( @@ -60,14 +81,14 @@ def check_monitoring_chi(operator_namespace, operator_pod, expect_result, max_re kubectl.delete_all_chi(self.context.test_namespace) check_monitoring_chi(self.context.operator_namespace, operator_pod, []) with And("created simple clickhouse installation"): - manifest = "../../docs/chi-examples/01-simple-layout-01-1shard-1repl.yaml" + manifest = "manifests/chi/test-017-multi-version.yaml" kubectl.create_and_check( manifest=manifest, check={ "object_counts": { - "statefulset": 1, - "pod": 1, - "service": 2, + "statefulset": 2, + "pod": 2, + "service": 3, }, "do_not_delete": True, }, @@ -75,14 +96,22 @@ def check_monitoring_chi(operator_namespace, operator_pod, expect_result, max_re expected_chi = [ { "namespace": "test", - "name": "simple-01", + "name": "test-017-multi-version", + "labels": {"clickhouse.altinity.com/chi": "test-017-multi-version"}, + "annotations": {}, "clusters": [ { - "name": "simple", + "name": "default", "hosts": [ { "name": "0-0", - "hostname": "chi-simple-01-simple-0-0.test.svc.cluster.local", + "hostname": "chi-test-017-multi-version-default-0-0.test.svc.cluster.local", + "tcpPort": 9000, + "httpPort": 8123 + }, + { + "name": "1-0", + "hostname": "chi-test-017-multi-version-default-1-0.test.svc.cluster.local", "tcpPort": 9000, "httpPort": 8123 } @@ -91,85 +120,35 @@ def check_monitoring_chi(operator_namespace, operator_pod, expect_result, max_re ] } ] + with Then("Check both pods are monitored"): check_monitoring_chi(self.context.operator_namespace, operator_pod, expected_chi) - with When("reboot metrics exporter"): - kubectl.launch(f"exec -n {self.context.operator_namespace} {operator_pod} -c metrics-exporter -- bash -c 'kill 1'") - time.sleep(15) - kubectl.wait_field( - "pods", - util.operator_label, - ".status.containerStatuses[*].ready", - "true,true", - ns=self.context.operator_namespace, - ) - with Then("check metrics exporter still contains chi objects"): - check_monitoring_chi(self.context.operator_namespace, operator_pod, expected_chi) - kubectl.delete(util.get_full_path(manifest, lookup_in_host=False), timeout=600) - check_monitoring_chi(self.context.operator_namespace, operator_pod, []) - - -@TestScenario -@Name("Check metrics server help with different clickhouse version") -def test_metrics_exporter_with_multiple_clickhouse_version(self): - def check_monitoring_metrics(operator_namespace, operator_pod, expect_result, max_retries=10): - with Then(f"metrics-exporter /metrics endpoint result should match with {expect_result}"): - for i in range(1, max_retries): - url_cmd = util.make_http_get_request("127.0.0.1", "8888", "/metrics") - out = kubectl.launch( - f"exec {operator_pod} -c metrics-exporter -- {url_cmd}", - ns=operator_namespace, - ) - all_strings_expected_done = True - for string, exists in expect_result.items(): - all_strings_expected_done = exists == (string in out) - if not all_strings_expected_done: - break - - if all_strings_expected_done: - break - with Then("Not ready. Wait for " + str(i * 5) + " seconds"): - time.sleep(i * 5) - assert all_strings_expected_done, error() - - with Given("clickhouse-operator pod exists"): - out = kubectl.launch("get pods -l app=clickhouse-operator", ns=self.context.operator_namespace).splitlines()[1] - operator_pod = re.split(r"[\t\r\n\s]+", out)[0] - - with Then("check empty /metrics"): - kubectl.delete_all_chi(self.context.test_namespace) - check_monitoring_metrics( - self.context.operator_namespace, - operator_pod, - expect_result={ - "chi_clickhouse_metric_VersionInteger": False, - }, - ) - - with Then("Install multiple clickhouse version"): - manifest = "manifests/chi/test-017-multi-version.yaml" - kubectl.create_and_check( - manifest=manifest, - check={ - "object_counts": { - "statefulset": 2, - "pod": 2, - "service": 3, - }, - "do_not_delete": True, - }, - ) - with And("Check not empty /metrics"): + with Then("Check not empty /metrics"): check_monitoring_metrics( self.context.operator_namespace, operator_pod, expect_result={ "# HELP chi_clickhouse_metric_VersionInteger": True, "# TYPE chi_clickhouse_metric_VersionInteger gauge": True, - 'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-0-0': True, - 'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-1-0': True, + 'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",clickhouse_altinity_com_chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-0-0': True, + 'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",clickhouse_altinity_com_chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-1-0': True, }, ) + with When("reboot metrics exporter"): + kubectl.launch(f"exec -n {self.context.operator_namespace} {operator_pod} -c metrics-exporter -- bash -c 'kill 1'") + time.sleep(15) + kubectl.wait_field( + "pods", + util.operator_label, + ".status.containerStatuses[*].ready", + "true,true", + ns=self.context.operator_namespace, + ) + with Then("check metrics exporter still contains chi objects"): + check_monitoring_chi(self.context.operator_namespace, operator_pod, expected_chi) + kubectl.delete(util.get_full_path(manifest, lookup_in_host=False), timeout=600) + check_monitoring_chi(self.context.operator_namespace, operator_pod, []) + with Then("check empty /metrics after delete namespace"): kubectl.delete_all_chi(self.context.test_namespace) check_monitoring_metrics( @@ -196,8 +175,7 @@ def test(self): util.install_operator_if_not_exist() test_cases = [ test_metrics_exporter_setup, - test_metrics_exporter_reboot, - test_metrics_exporter_with_multiple_clickhouse_version, + test_metrics_exporter_chi, ] for t in test_cases: Scenario(test=t)() diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index c8a0d4995..50bb1fd01 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -150,8 +150,8 @@ def test_005(self): def test_006(self): create_shell_namespace_clickhouse_template() - old_version = "clickhouse/clickhouse-server:22.3" - new_version = "clickhouse/clickhouse-server:22.8" + old_version = "clickhouse/clickhouse-server:23.3" + new_version = "clickhouse/clickhouse-server:23.8" with Then("Create initial position"): kubectl.create_and_check( manifest="manifests/chi/test-006-ch-upgrade-1.yaml", @@ -662,6 +662,9 @@ def test_011_1(self): kubectl.create_and_check( manifest="manifests/chi/test-011-insecured-cluster.yaml", check={ + "apply_templates": { + current().context.clickhouse_template, + }, "chi_status": "InProgress", "do_not_delete": 1, }, @@ -673,14 +676,12 @@ def test_011_1(self): # Tests default user security def test_default_user(): with Then("Default user should have 5 allowed ips"): - print(f"Give the config time to propagate") - time.sleep(90) ips = get_user_xml_from_configmap("test-011-secured-cluster", "default").findall("networks/ip") ips_l = [] for ip in ips: ips_l.append(ip.text) # Expected output: ['::1', '127.0.0.1', '127.0.0.2', , ] - print(f"users.xml: {ips_l}") + print(f"default user's IPs: {ips_l}") assert len(ips) == 5 clickhouse.query("test-011-secured-cluster", "SYSTEM RELOAD CONFIG") @@ -712,7 +713,9 @@ def test_default_user(): with When("Remove host_regexp for default user"): kubectl.create_and_check( manifest="manifests/chi/test-011-secured-cluster-2.yaml", - check={"do_not_delete": 1}, + check={ + "do_not_delete": 1, + }, ) with Then("Make sure host_regexp is disabled"): @@ -722,6 +725,9 @@ def test_default_user(): print(f"users.xml: {regexp}") assert regexp == "disabled" + print(f"Give ClickHouse time to recongize the config change") + time.sleep(30) + test_default_user() with And("Connection from insecured to secured host should fail for user 'user1' with no password"): @@ -792,18 +798,9 @@ def test_default_user(): ) assert "ACCESS_DENIED" not in out - with And("User 'user5' with google.com as a host filter can not login"): + with And("User 'clickhouse_operator' can login with custom password"): out = clickhouse.query_with_error( - "test-011-insecured-cluster", - "select 'OK'", - user="user5", - pwd="secret", - ) - assert out != "OK" - - with And("User 'clickhouse_operator' with can login with custom password"): - out = clickhouse.query_with_error( - "test-011-insecured-cluster", + "test-011-secured-cluster", "select 'OK'", user="clickhouse_operator", pwd="operator_secret", @@ -814,7 +811,6 @@ def test_default_user(): with By("deleting test namespace"): delete_test_namespace() - @TestScenario @Name("test_011_2. Test default user security") @Requirements(RQ_SRS_026_ClickHouseOperator_DefaultUsers("1.0")) @@ -861,11 +857,9 @@ def test_011_2(self): out = clickhouse.query_with_error("test-011-secured-default", "select 'OK'") assert out == "OK" - with Then("I delete namespace"): - shell = get_shell() - self.context.shell = shell - util.delete_namespace(namespace=self.context.test_namespace, delete_chi=True) - shell.close() + with Finally("I clean up"): + with By("deleting test namespace"): + delete_test_namespace() @TestScenario @@ -3403,7 +3397,7 @@ def test_034(self): operator_namespace = current().context.operator_namespace def check_metrics_monitoring(operator_namespace, operator_pod, expect_pattern, max_retries=7): - with Then(f"metrics-exporter /metrics endpoint result should contain {expect_pattern}"): + with Then(f"metrics-exporter /metrics endpoint result should contain pattern: '{expect_pattern}'"): for i in range(1, max_retries): url_cmd = util.make_http_get_request("127.0.0.1", "8888", "/metrics") out = kubectl.launch( @@ -3452,10 +3446,10 @@ def check_metrics_monitoring(operator_namespace, operator_pod, expect_pattern, m expect_pattern="^chi_clickhouse_metric_fetch_errors{(.*?)} 0$", ) - with And(f"apply ClickHouseOperatorConfiguration {chopconf_file} with https connection"): + with And(f"apply ClickHouseOperatorConfiguration with https connection: {chopconf_file}"): kubectl.apply(util.get_full_path(chopconf_file, lookup_in_host=False), operator_namespace) - with And("reboot metrics exporter to update the configuration [1]"): + with And("Re-create operator pod in order to restart metrics exporter to update the configuration [1]"): util.restart_operator() out = kubectl.launch("get pods -l app=clickhouse-operator", ns=current().context.operator_namespace).splitlines()[1] operator_pod = re.split(r"[\t\r\n\s]+", out)[0] @@ -3470,7 +3464,7 @@ def check_metrics_monitoring(operator_namespace, operator_pod, expect_pattern, m with When("Reset ClickHouseOperatorConfiguration to default"): kubectl.delete(util.get_full_path(chopconf_file, lookup_in_host=False), operator_namespace) - with And("reboot metrics exporter to update the configuration [2]"): + with And("Re-create operator pod in order to restart metrics exporter to update the configuration [2]"): util.restart_operator() out = kubectl.launch("get pods -l app=clickhouse-operator", ns=current().context.operator_namespace).splitlines()[1] operator_pod = re.split(r"[\t\r\n\s]+", out)[0] @@ -3510,7 +3504,7 @@ def check_metrics_monitoring(operator_namespace, operator_pod, expect_pattern, m ) client_pod = "test-034-client" - with And(f"Start {client_pod} pod"): + with And(f"Start pod: {client_pod}"): kubectl.apply(util.get_full_path("manifests/chi/test-034-client.yaml")) kubectl.wait_pod_status(client_pod, "Running") @@ -3529,10 +3523,10 @@ def check_metrics_monitoring(operator_namespace, operator_pod, expect_pattern, m print(out) assert "NETWORK_ERROR" in out, out - with And(f"apply ClickHouseOperatorConfiguration {chopconf_file} with https connection"): + with And(f"apply ClickHouseOperatorConfiguration with https connection: {chopconf_file}"): kubectl.apply(util.get_full_path(chopconf_file, lookup_in_host=False), operator_namespace) - with And("reboot metrics exporter to update the configuration [3]"): + with And("Re-create operator pod in order to restart metrics exporter to update the configuration [3]"): util.restart_operator() out = kubectl.launch("get pods -l app=clickhouse-operator", ns=current().context.operator_namespace).splitlines()[1] operator_pod = re.split(r"[\t\r\n\s]+", out)[0] @@ -3547,7 +3541,7 @@ def check_metrics_monitoring(operator_namespace, operator_pod, expect_pattern, m with When("Reset ClickHouseOperatorConfiguration to default"): kubectl.delete(util.get_full_path(chopconf_file, lookup_in_host=False), operator_namespace) - with And("reboot metrics exporter to update the configuration [4]"): + with And("Re-create operator pod in order to restart metrics exporter to update the configuration [4]"): util.restart_operator() out = kubectl.launch("get pods -l app=clickhouse-operator", ns=current().context.operator_namespace).splitlines()[1] operator_pod = re.split(r"[\t\r\n\s]+", out)[0] @@ -3754,7 +3748,7 @@ def check_data_is_recovered(reconcile_task_id): delete_pvc() check_data_is_recovered("reconcile-after-PVC-deleted") - + delete_pv() check_data_is_recovered("reconcile-after-PV-deleted") @@ -4004,7 +3998,7 @@ def test_040(self): "pod_volumes": { "/var/lib/clickhouse", }, - "pod_image": "clickhouse/clickhouse-server:22.8", + "pod_image": "clickhouse/clickhouse-server:23.8", "do_not_delete": 1, "chi_status": "InProgress", },