Update open source image of spark-hs to 3.3.1

charts/spark-hs-3.1.1/spark-hs-chart/templates/deployment.yaml → charts/spark-hs-3.3.1/spark-hs-chart/templates/deployment.yaml

@@ -12,19 +12,23 @@ metadata:
     helm.sh/chart: {{ include "spark-hs-chart.chart" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
 spec:
+  progressDeadlineSeconds: {{ .Values.progressDeadlineSeconds }}
   replicas: {{ .Values.replicaCount }}
+  revisionHistoryLimit: {{ .Values.revisionHistoryLimit }}
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ include "spark-hs-chart.name" . }}
       app.kubernetes.io/instance: {{ .Release.Name }}
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
   template:
     metadata:
       labels:
-      {{- include "common.labels" (dict "componentName" $componentName "namespace" .Release.Namespace ) | nindent 8 }}
-      {{- if .Values.podAnnotations }}
-      annotations:
-      {{ toYaml .Values.podAnnotations | indent 8 }}
-      {{- end }}
+       app.kubernetes.io/name: {{ include "spark-hs-chart.name" . }}
+       app.kubernetes.io/instance: {{ .Release.Name }}
     spec:
       serviceAccountName: {{ include "spark-hs-chart.serviceAccountName" . }}
       imagePullSecrets:
@@ -63,13 +67,21 @@ spec:
             - configMapRef:
                 name: {{ include "spark-hs-chart.fullname" . }}
           livenessProbe:
-            httpGet:
-              path: /
-              port: historyport
+            tcpSocket:
+              port: {{ .Values.ports.httpPort }}
+            initialDelaySeconds: 5
+            periodSeconds: 10
           readinessProbe:
-            httpGet:
-              path: /
-              port: historyport
+            tcpSocket:
+              port: {{ .Values.ports.httpPort }}
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          startupProbe:
+            tcpSocket:
+              port: {{ .Values.ports.httpPort }}
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            failureThreshold: 30
           {{- if .Values.pvc.enablePVC }}
           volumeMounts:
             - name: sparkhs-eventlog-storage

charts/spark-hs-3.1.1/spark-hs-chart/templates/service.yaml → charts/spark-hs-3.3.1/spark-hs-chart/templates/service.yaml

@@ -11,20 +11,13 @@ metadata:
     app.kubernetes.io/name: {{ include "spark-hs-chart.name" . }}
     helm.sh/chart: {{ include "spark-hs-chart.chart" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
-  {{- if .Values.service.annotations }}
-  annotations:
-  {{ toYaml .Values.service.annotations | trim | indent 4 }}
-  {{- end }}
 spec:
   type: {{ .Values.service.type }}
   ports:
-  - port: {{ .Values.service.port.number }}
-    targetPort: historyport
-    protocol: TCP
-    name: {{ .Values.service.port.name }}
-    {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }}
-    nodePort: {{ .Values.service.nodePort }}
-    {{- end }}
+    - name: https-sparkhs
+      port: {{ .Values.ports.httpsPort }}
+      targetPort: {{ .Values.ports.httpsPort }}
+      protocol: TCP
   selector:
     app.kubernetes.io/name: {{ include "spark-hs-chart.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}

charts/spark-hs-3.1.1/spark-hs-chart/values.yaml → charts/spark-hs-3.3.1/spark-hs-chart/values.yaml

@@ -5,8 +5,8 @@
 replicaCount: 1
 
 image:
-  repository: gcr.io/mapr-252711/apache-spark-hs-3.1.1
-  tag: "202108282015"
+  repository: gcr.io/mapr-252711/apache-spark-hs-3.3.1
+  tag: "202302021308-AJ"
   pullPolicy: IfNotPresent
 
 createDefaultPullSecret: true
@@ -16,11 +16,17 @@ nameOverride: ""
 fullnameOverride: ""
 service:
   type: NodePort
-#  nodePort: 32000
-  port:
-    number: 18080
-    name: http-historyport
-  annotations: {}
+
+
+#container ports
+ports:
+  sshPort: 22
+  httpPort: "18080"
+  httpsPort: "18480"
+
+progressDeadlineSeconds: 600
+
+revisionHistoryLimit: 10
 
 rbac:
   create: true
@@ -42,7 +48,6 @@ environment:
 # SPARK_DAEMON_CLASSPATH: ...
 # SPARK_PUBLIC_DNS: ...
 
-podAnnotations: {}
 
 resources:
   ## We usually recommend not to specify default resources and to leave this as a conscious

dockerfiles/spark-hs-3.3.1/Dockerfile

@@ -0,0 +1,6 @@
+ARG SPARK_IMAGE=apache/spark:v3.3.1
+FROM ${SPARK_IMAGE}
+
+COPY entrypoint.sh /opt
+
+ENTRYPOINT ["/opt/entrypoint.sh"]

dockerfiles/spark-hs-3.1.1/build.sh → dockerfiles/spark-hs-3.3.1/build.sh

@@ -1,9 +1,10 @@
 #!/usr/bin/env bash
 
+chmod +x entrypoint.sh
 docker build --no-cache --network=host -t apache-spark-hs .
 
-VERSION="3.1.1"
+VERSION="3.3.1"
 
-tag=$(date +%Y%m%d%H%M)
+tag=$(date +%Y%m%d%H%M)-AJ
 docker tag apache-spark-hs gcr.io/mapr-252711/apache-spark-hs-${VERSION}:$tag
 docker push gcr.io/mapr-252711/apache-spark-hs-${VERSION}:$tag

dockerfiles/spark-hs-3.1.1/entrypoint.sh → dockerfiles/spark-hs-3.3.1/entrypoint.sh

@@ -80,10 +80,14 @@ fi
 
 if [[ "$enablePVC" == "true" ]]; then
     export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS \
-    -Dspark.history.fs.logDirectory=file:/mnt/$eventsDir";
+    -Dspark.eventLog.enabled=true \
+    -Dspark.eventLog.dir=file:$eventsDir \
+    -Dspark.history.fs.logDirectory=file:$eventsDir";
 else
     export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS \
+    -Dspark.eventLog.enabled=true \
+    -Dspark.eventLog.dir=file:$eventsDir \
     -Dspark.history.fs.logDirectory=$eventsDir";
 fi;
 
-exec /bin/bash -s -- /opt/spark/bin/spark-class org.apache.spark.deploy.history.HistoryServer
+exec /usr/bin/tini -s -- /opt/spark/bin/spark-class org.apache.spark.deploy.history.HistoryServer