diff --git a/charts/spark-hs-3.1.1/README.md b/charts/spark-hs-3.3.1/README.md similarity index 100% rename from charts/spark-hs-3.1.1/README.md rename to charts/spark-hs-3.3.1/README.md diff --git a/charts/spark-hs-3.1.1/spark-hs-chart/.helmignore b/charts/spark-hs-3.3.1/spark-hs-chart/.helmignore similarity index 100% rename from charts/spark-hs-3.1.1/spark-hs-chart/.helmignore rename to charts/spark-hs-3.3.1/spark-hs-chart/.helmignore diff --git a/charts/spark-hs-3.1.1/spark-hs-chart/Chart.yaml b/charts/spark-hs-3.3.1/spark-hs-chart/Chart.yaml similarity index 100% rename from charts/spark-hs-3.1.1/spark-hs-chart/Chart.yaml rename to charts/spark-hs-3.3.1/spark-hs-chart/Chart.yaml diff --git a/charts/spark-hs-3.1.1/spark-hs-chart/files/example-imagepullsecret b/charts/spark-hs-3.3.1/spark-hs-chart/files/example-imagepullsecret similarity index 100% rename from charts/spark-hs-3.1.1/spark-hs-chart/files/example-imagepullsecret rename to charts/spark-hs-3.3.1/spark-hs-chart/files/example-imagepullsecret diff --git a/charts/spark-hs-3.1.1/spark-hs-chart/templates/NOTES.txt b/charts/spark-hs-3.3.1/spark-hs-chart/templates/NOTES.txt similarity index 100% rename from charts/spark-hs-3.1.1/spark-hs-chart/templates/NOTES.txt rename to charts/spark-hs-3.3.1/spark-hs-chart/templates/NOTES.txt diff --git a/charts/spark-hs-3.1.1/spark-hs-chart/templates/_helpers.tpl b/charts/spark-hs-3.3.1/spark-hs-chart/templates/_helpers.tpl similarity index 100% rename from charts/spark-hs-3.1.1/spark-hs-chart/templates/_helpers.tpl rename to charts/spark-hs-3.3.1/spark-hs-chart/templates/_helpers.tpl diff --git a/charts/spark-hs-3.1.1/spark-hs-chart/templates/configmap.yaml b/charts/spark-hs-3.3.1/spark-hs-chart/templates/configmap.yaml similarity index 100% rename from charts/spark-hs-3.1.1/spark-hs-chart/templates/configmap.yaml rename to charts/spark-hs-3.3.1/spark-hs-chart/templates/configmap.yaml diff --git a/charts/spark-hs-3.1.1/spark-hs-chart/templates/deployment.yaml b/charts/spark-hs-3.3.1/spark-hs-chart/templates/deployment.yaml similarity index 80% rename from charts/spark-hs-3.1.1/spark-hs-chart/templates/deployment.yaml rename to charts/spark-hs-3.3.1/spark-hs-chart/templates/deployment.yaml index 4ac659eb6..3baf13b12 100644 --- a/charts/spark-hs-3.1.1/spark-hs-chart/templates/deployment.yaml +++ b/charts/spark-hs-3.3.1/spark-hs-chart/templates/deployment.yaml @@ -12,19 +12,23 @@ metadata: helm.sh/chart: {{ include "spark-hs-chart.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} spec: + progressDeadlineSeconds: {{ .Values.progressDeadlineSeconds }} replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} selector: matchLabels: app.kubernetes.io/name: {{ include "spark-hs-chart.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate template: metadata: labels: - {{- include "common.labels" (dict "componentName" $componentName "namespace" .Release.Namespace ) | nindent 8 }} - {{- if .Values.podAnnotations }} - annotations: - {{ toYaml .Values.podAnnotations | indent 8 }} - {{- end }} + app.kubernetes.io/name: {{ include "spark-hs-chart.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} spec: serviceAccountName: {{ include "spark-hs-chart.serviceAccountName" . }} imagePullSecrets: @@ -63,13 +67,21 @@ spec: - configMapRef: name: {{ include "spark-hs-chart.fullname" . }} livenessProbe: - httpGet: - path: / - port: historyport + tcpSocket: + port: {{ .Values.ports.httpPort }} + initialDelaySeconds: 5 + periodSeconds: 10 readinessProbe: - httpGet: - path: / - port: historyport + tcpSocket: + port: {{ .Values.ports.httpPort }} + initialDelaySeconds: 5 + periodSeconds: 10 + startupProbe: + tcpSocket: + port: {{ .Values.ports.httpPort }} + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 30 {{- if .Values.pvc.enablePVC }} volumeMounts: - name: sparkhs-eventlog-storage diff --git a/charts/spark-hs-3.1.1/spark-hs-chart/templates/imagepull.yaml b/charts/spark-hs-3.3.1/spark-hs-chart/templates/imagepull.yaml similarity index 100% rename from charts/spark-hs-3.1.1/spark-hs-chart/templates/imagepull.yaml rename to charts/spark-hs-3.3.1/spark-hs-chart/templates/imagepull.yaml diff --git a/charts/spark-hs-3.1.1/spark-hs-chart/templates/pv.yaml b/charts/spark-hs-3.3.1/spark-hs-chart/templates/pv.yaml similarity index 100% rename from charts/spark-hs-3.1.1/spark-hs-chart/templates/pv.yaml rename to charts/spark-hs-3.3.1/spark-hs-chart/templates/pv.yaml diff --git a/charts/spark-hs-3.1.1/spark-hs-chart/templates/pvc.yaml b/charts/spark-hs-3.3.1/spark-hs-chart/templates/pvc.yaml similarity index 100% rename from charts/spark-hs-3.1.1/spark-hs-chart/templates/pvc.yaml rename to charts/spark-hs-3.3.1/spark-hs-chart/templates/pvc.yaml diff --git a/charts/spark-hs-3.1.1/spark-hs-chart/templates/rbac.yaml b/charts/spark-hs-3.3.1/spark-hs-chart/templates/rbac.yaml similarity index 100% rename from charts/spark-hs-3.1.1/spark-hs-chart/templates/rbac.yaml rename to charts/spark-hs-3.3.1/spark-hs-chart/templates/rbac.yaml diff --git a/charts/spark-hs-3.1.1/spark-hs-chart/templates/service.yaml b/charts/spark-hs-3.3.1/spark-hs-chart/templates/service.yaml similarity index 60% rename from charts/spark-hs-3.1.1/spark-hs-chart/templates/service.yaml rename to charts/spark-hs-3.3.1/spark-hs-chart/templates/service.yaml index d8b88a69c..61e7ba1c7 100644 --- a/charts/spark-hs-3.1.1/spark-hs-chart/templates/service.yaml +++ b/charts/spark-hs-3.3.1/spark-hs-chart/templates/service.yaml @@ -11,20 +11,13 @@ metadata: app.kubernetes.io/name: {{ include "spark-hs-chart.name" . }} helm.sh/chart: {{ include "spark-hs-chart.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} - {{- if .Values.service.annotations }} - annotations: - {{ toYaml .Values.service.annotations | trim | indent 4 }} - {{- end }} spec: type: {{ .Values.service.type }} ports: - - port: {{ .Values.service.port.number }} - targetPort: historyport - protocol: TCP - name: {{ .Values.service.port.name }} - {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} - nodePort: {{ .Values.service.nodePort }} - {{- end }} + - name: https-sparkhs + port: {{ .Values.ports.httpsPort }} + targetPort: {{ .Values.ports.httpsPort }} + protocol: TCP selector: app.kubernetes.io/name: {{ include "spark-hs-chart.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/spark-hs-3.1.1/spark-hs-chart/templates/serviceaccount.yaml b/charts/spark-hs-3.3.1/spark-hs-chart/templates/serviceaccount.yaml similarity index 100% rename from charts/spark-hs-3.1.1/spark-hs-chart/templates/serviceaccount.yaml rename to charts/spark-hs-3.3.1/spark-hs-chart/templates/serviceaccount.yaml diff --git a/charts/spark-hs-3.1.1/spark-hs-chart/values.yaml b/charts/spark-hs-3.3.1/spark-hs-chart/values.yaml similarity index 91% rename from charts/spark-hs-3.1.1/spark-hs-chart/values.yaml rename to charts/spark-hs-3.3.1/spark-hs-chart/values.yaml index c8f1ee0a0..eec86d5b3 100644 --- a/charts/spark-hs-3.1.1/spark-hs-chart/values.yaml +++ b/charts/spark-hs-3.3.1/spark-hs-chart/values.yaml @@ -5,8 +5,8 @@ replicaCount: 1 image: - repository: gcr.io/mapr-252711/apache-spark-hs-3.1.1 - tag: "202108282015" + repository: gcr.io/mapr-252711/apache-spark-hs-3.3.1 + tag: "202302021308-AJ" pullPolicy: IfNotPresent createDefaultPullSecret: true @@ -16,11 +16,17 @@ nameOverride: "" fullnameOverride: "" service: type: NodePort -# nodePort: 32000 - port: - number: 18080 - name: http-historyport - annotations: {} + + +#container ports +ports: + sshPort: 22 + httpPort: "18080" + httpsPort: "18480" + +progressDeadlineSeconds: 600 + +revisionHistoryLimit: 10 rbac: create: true @@ -42,7 +48,6 @@ environment: # SPARK_DAEMON_CLASSPATH: ... # SPARK_PUBLIC_DNS: ... -podAnnotations: {} resources: ## We usually recommend not to specify default resources and to leave this as a conscious diff --git a/dockerfiles/spark-hs-3.1.1/Dockerfile b/dockerfiles/spark-hs-3.1.1/Dockerfile deleted file mode 100644 index 72fb3fd67..000000000 --- a/dockerfiles/spark-hs-3.1.1/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v3.1.1 -FROM ${SPARK_IMAGE} - -ENTRYPOINT ["/opt/entrypoint.sh"] \ No newline at end of file diff --git a/dockerfiles/spark-hs-3.3.1/Dockerfile b/dockerfiles/spark-hs-3.3.1/Dockerfile new file mode 100644 index 000000000..586365a1c --- /dev/null +++ b/dockerfiles/spark-hs-3.3.1/Dockerfile @@ -0,0 +1,6 @@ +ARG SPARK_IMAGE=apache/spark:v3.3.1 +FROM ${SPARK_IMAGE} + +COPY entrypoint.sh /opt + +ENTRYPOINT ["/opt/entrypoint.sh"] diff --git a/dockerfiles/spark-hs-3.1.1/build.sh b/dockerfiles/spark-hs-3.3.1/build.sh similarity index 77% rename from dockerfiles/spark-hs-3.1.1/build.sh rename to dockerfiles/spark-hs-3.3.1/build.sh index acb062abf..2f1f78f0e 100755 --- a/dockerfiles/spark-hs-3.1.1/build.sh +++ b/dockerfiles/spark-hs-3.3.1/build.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash +chmod +x entrypoint.sh docker build --no-cache --network=host -t apache-spark-hs . -VERSION="3.1.1" +VERSION="3.3.1" -tag=$(date +%Y%m%d%H%M) +tag=$(date +%Y%m%d%H%M)-AJ docker tag apache-spark-hs gcr.io/mapr-252711/apache-spark-hs-${VERSION}:$tag docker push gcr.io/mapr-252711/apache-spark-hs-${VERSION}:$tag \ No newline at end of file diff --git a/dockerfiles/spark-hs-3.1.1/entrypoint.sh b/dockerfiles/spark-hs-3.3.1/entrypoint.sh old mode 100644 new mode 100755 similarity index 85% rename from dockerfiles/spark-hs-3.1.1/entrypoint.sh rename to dockerfiles/spark-hs-3.3.1/entrypoint.sh index 6ad251650..22db65e43 --- a/dockerfiles/spark-hs-3.1.1/entrypoint.sh +++ b/dockerfiles/spark-hs-3.3.1/entrypoint.sh @@ -80,10 +80,14 @@ fi if [[ "$enablePVC" == "true" ]]; then export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS \ - -Dspark.history.fs.logDirectory=file:/mnt/$eventsDir"; + -Dspark.eventLog.enabled=true \ + -Dspark.eventLog.dir=file:$eventsDir \ + -Dspark.history.fs.logDirectory=file:$eventsDir"; else export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS \ + -Dspark.eventLog.enabled=true \ + -Dspark.eventLog.dir=file:$eventsDir \ -Dspark.history.fs.logDirectory=$eventsDir"; fi; -exec /bin/bash -s -- /opt/spark/bin/spark-class org.apache.spark.deploy.history.HistoryServer \ No newline at end of file +exec /usr/bin/tini -s -- /opt/spark/bin/spark-class org.apache.spark.deploy.history.HistoryServer \ No newline at end of file