linera-io · ndr-ds · Nov 25, 2025
diff --git a/kubernetes/linera-validator/templates/otel-collector-router-config.yaml b/kubernetes/linera-validator/templates/otel-collector-router-config.yaml
@@ -0,0 +1,54 @@
+{{- if .Values.otelCollector.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: otel-collector-router-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: otel-collector-router
+data:
+  otel-collector-config.yaml: |
+    extensions:
+      health_check:
+        endpoint: 0.0.0.0:13133
+
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+          http:
+            endpoint: 0.0.0.0:4318
+
+    processors:
+      batch:
+        timeout: 200ms
+        send_batch_size: 512
+
+      memory_limiter:
+        check_interval: 1s
+        limit_mib: 400
+        spike_limit_mib: 100
+
+    exporters:
+      loadbalancing:
+        routing_key: traceID
+        protocol:
+          otlp:
+            tls:
+              insecure: true
+        resolver:
+          dns:
+            hostname: otel-collector-sampler-headless.{{ .Release.Namespace }}.svc.cluster.local
+            port: "4317"
+            interval: 30s
+            timeout: 10s
+
+    service:
+      extensions: [health_check]
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: [memory_limiter, batch]
+          exporters: [loadbalancing]
+{{- end }}
diff --git a/kubernetes/linera-validator/templates/otel-collector-router-deployment.yaml b/kubernetes/linera-validator/templates/otel-collector-router-deployment.yaml
@@ -0,0 +1,71 @@
+{{- if .Values.otelCollector.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: otel-collector-router
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: otel-collector-router
+spec:
+  replicas: {{ .Values.otelCollector.router.replicas }}
+  selector:
+    matchLabels:
+      app: otel-collector-router
+  template:
+    metadata:
+      labels:
+        app: otel-collector-router
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/otel-collector-router-config.yaml") . | sha256sum }}
+    spec:
+      {{- if .Values.otelCollector.router.nodeSelector }}
+      nodeSelector:
+        {{- toYaml .Values.otelCollector.router.nodeSelector | nindent 8 }}
+      {{- end }}
+      {{- if .Values.otelCollector.router.tolerations }}
+      tolerations:
+        {{- toYaml .Values.otelCollector.router.tolerations | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: otel-collector-router
+          image: {{ .Values.otelCollector.router.image }}:{{ .Values.otelCollector.router.imageTag }}
+          imagePullPolicy: {{ .Values.otelCollector.router.imagePullPolicy | default "IfNotPresent" }}
+          args:
+            - "--config=/conf/otel-collector-config.yaml"
+          ports:
+            - name: otlp-grpc
+              containerPort: 4317
+              protocol: TCP
+            - name: otlp-http
+              containerPort: 4318
+              protocol: TCP
+            - name: health
+              containerPort: 13133
+              protocol: TCP
+          volumeMounts:
+            - name: config
+              mountPath: /conf
+          resources:
+            requests:
+              memory: {{ .Values.otelCollector.router.resources.requests.memory }}
+              cpu: {{ .Values.otelCollector.router.resources.requests.cpu }}
+            limits:
+              memory: {{ .Values.otelCollector.router.resources.limits.memory }}
+              cpu: {{ .Values.otelCollector.router.resources.limits.cpu }}
+          livenessProbe:
+            httpGet:
+              path: /
+              port: health
+            initialDelaySeconds: 15
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /
+              port: health
+            initialDelaySeconds: 5
+            periodSeconds: 5
+      volumes:
+        - name: config
+          configMap:
+            name: otel-collector-router-config
+{{- end }}
diff --git a/kubernetes/linera-validator/templates/otel-collector-router-service.yaml b/kubernetes/linera-validator/templates/otel-collector-router-service.yaml
@@ -0,0 +1,26 @@
+{{- if .Values.otelCollector.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: otel-collector-router
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: otel-collector-router
+spec:
+  type: ClusterIP
+  ports:
+    - name: otlp-grpc
+      port: 4317
+      targetPort: 4317
+      protocol: TCP
+    - name: otlp-http
+      port: 4318
+      targetPort: 4318
+      protocol: TCP
+    - name: health
+      port: 13133
+      targetPort: 13133
+      protocol: TCP
+  selector:
+    app: otel-collector-router
+{{- end }}
diff --git a/kubernetes/linera-validator/templates/otel-collector-sampler-config.yaml b/kubernetes/linera-validator/templates/otel-collector-sampler-config.yaml
@@ -0,0 +1,64 @@
+{{- if .Values.otelCollector.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: otel-collector-sampler-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: otel-collector-sampler
+data:
+  otel-collector-config.yaml: |
+    extensions:
+      health_check:
+        endpoint: 0.0.0.0:13133
+
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+          http:
+            endpoint: 0.0.0.0:4318
+
+    processors:
+      memory_limiter:
+        check_interval: 1s
+        limit_mib: 3000
+        spike_limit_mib: 500
+
+      batch:
+        timeout: 1s
+        send_batch_size: 1024
+
+      tail_sampling:
+        decision_wait: {{ .Values.otelCollector.sampler.decisionWait }}
+        num_traces: {{ .Values.otelCollector.sampler.numTraces }}
+        expected_new_traces_per_sec: {{ .Values.otelCollector.sampler.expectedTracesPerSec }}
+        policies:
+          - name: errors
+            type: status_code
+            status_code:
+              status_codes: [ERROR]
+          - name: slow-requests
+            type: latency
+            latency:
+              threshold_ms: {{ .Values.otelCollector.sampler.latencyThresholdMs }}
+          - name: baseline
+            type: probabilistic
+            probabilistic:
+              sampling_percentage: {{ .Values.otelCollector.sampler.baselineSamplingPercent }}
+
+    exporters:
+      otlp:
+        endpoint: {{ .Values.otelCollector.tempoEndpoint | quote }}
+        tls:
+          insecure: {{ .Values.otelCollector.tempoInsecure }}
+
+    service:
+      extensions: [health_check]
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: [memory_limiter, batch, tail_sampling]
+          exporters: [otlp]
+{{- end }}
diff --git a/kubernetes/linera-validator/templates/otel-collector-sampler-service.yaml b/kubernetes/linera-validator/templates/otel-collector-sampler-service.yaml
@@ -0,0 +1,27 @@
+{{- if .Values.otelCollector.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: otel-collector-sampler-headless
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: otel-collector-sampler
+spec:
+  type: ClusterIP
+  clusterIP: None
+  ports:
+    - name: otlp-grpc
+      port: 4317
+      targetPort: 4317
+      protocol: TCP
+    - name: otlp-http
+      port: 4318
+      targetPort: 4318
+      protocol: TCP
+    - name: health
+      port: 13133
+      targetPort: 13133
+      protocol: TCP
+  selector:
+    app: otel-collector-sampler
+{{- end }}
diff --git a/kubernetes/linera-validator/templates/otel-collector-sampler-statefulset.yaml b/kubernetes/linera-validator/templates/otel-collector-sampler-statefulset.yaml
@@ -0,0 +1,72 @@
+{{- if .Values.otelCollector.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: otel-collector-sampler
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: otel-collector-sampler
+spec:
+  serviceName: otel-collector-sampler-headless
+  replicas: {{ .Values.otelCollector.sampler.replicas }}
+  selector:
+    matchLabels:
+      app: otel-collector-sampler
+  template:
+    metadata:
+      labels:
+        app: otel-collector-sampler
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/otel-collector-sampler-config.yaml") . | sha256sum }}
+    spec:
+      {{- if .Values.otelCollector.sampler.nodeSelector }}
+      nodeSelector:
+        {{- toYaml .Values.otelCollector.sampler.nodeSelector | nindent 8 }}
+      {{- end }}
+      {{- if .Values.otelCollector.sampler.tolerations }}
+      tolerations:
+        {{- toYaml .Values.otelCollector.sampler.tolerations | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: otel-collector-sampler
+          image: {{ .Values.otelCollector.sampler.image }}:{{ .Values.otelCollector.sampler.imageTag }}
+          imagePullPolicy: {{ .Values.otelCollector.sampler.imagePullPolicy | default "IfNotPresent" }}
+          args:
+            - "--config=/conf/otel-collector-config.yaml"
+          ports:
+            - name: otlp-grpc
+              containerPort: 4317
+              protocol: TCP
+            - name: otlp-http
+              containerPort: 4318
+              protocol: TCP
+            - name: health
+              containerPort: 13133
+              protocol: TCP
+          volumeMounts:
+            - name: config
+              mountPath: /conf
+          resources:
+            requests:
+              memory: {{ .Values.otelCollector.sampler.resources.requests.memory }}
+              cpu: {{ .Values.otelCollector.sampler.resources.requests.cpu }}
+            limits:
+              memory: {{ .Values.otelCollector.sampler.resources.limits.memory }}
+              cpu: {{ .Values.otelCollector.sampler.resources.limits.cpu }}
+          livenessProbe:
+            httpGet:
+              path: /
+              port: health
+            initialDelaySeconds: 30
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /
+              port: health
+            initialDelaySeconds: 10
+            periodSeconds: 5
+      volumes:
+        - name: config
+          configMap:
+            name: otel-collector-sampler-config
+{{- end }}
diff --git a/kubernetes/linera-validator/templates/proxy.yaml b/kubernetes/linera-validator/templates/proxy.yaml
@@ -91,8 +91,10 @@ spec:
               value: {{ .Values.logLevel }}
             - name: RUST_BACKTRACE
               value: "1"
+            {{- if .Values.otlpExporterEndpoint }}
             - name: LINERA_OTLP_EXPORTER_ENDPOINT
               value: {{ .Values.otlpExporterEndpoint }}
+            {{- end }}
       containers:
         - name: linera-proxy
           imagePullPolicy: {{ .Values.lineraImagePullPolicy }}
@@ -119,8 +121,10 @@ spec:
           env:
             - name: RUST_LOG
               value: {{ .Values.logLevel }}
+            {{- if .Values.otlpExporterEndpoint }}
             - name: LINERA_OTLP_EXPORTER_ENDPOINT
               value: {{ .Values.otlpExporterEndpoint }}
+            {{- end }}
           livenessProbe:
             tcpSocket:
               port: {{ .Values.proxyPort }}

diff --git a/kubernetes/linera-validator/templates/shards.yaml b/kubernetes/linera-validator/templates/shards.yaml
@@ -79,8 +79,10 @@ spec:
               value: {{ .Values.logLevel }}
             - name: RUST_BACKTRACE
               value: "1"
+            {{- if .Values.otlpExporterEndpoint }}
             - name: LINERA_OTLP_EXPORTER_ENDPOINT
               value: {{ .Values.otlpExporterEndpoint }}
+            {{- end }}
           volumeMounts:
             - name: config
               mountPath: "/config"
@@ -122,8 +124,10 @@ spec:
           env:
             - name: RUST_LOG
               value: {{ .Values.logLevel }}
+            {{- if .Values.otlpExporterEndpoint }}
             - name: LINERA_OTLP_EXPORTER_ENDPOINT
               value: {{ .Values.otlpExporterEndpoint }}
+            {{- end }}
             {{- if .Values.serverTokioThreads }}
             - name: LINERA_SERVER_TOKIO_THREADS
               value: "{{ .Values.serverTokioThreads }}"

diff --git a/kubernetes/linera-validator/values-local.yaml.gotmpl b/kubernetes/linera-validator/values-local.yaml.gotmpl
@@ -179,3 +179,20 @@ environment: "kind"
 validator:
   serverConfig: {{ env "LINERA_HELMFILE_SET_SERVER_CONFIG" | default "working/server_1.json" }}
   genesisConfig: {{ env "LINERA_HELMFILE_SET_GENESIS_CONFIG" | default "working/genesis.json" }}
+
+# OpenTelemetry Collector - Tail-Based Sampling
+otelCollector:
+  enabled: {{ env "LINERA_HELMFILE_SET_OTEL_COLLECTOR_ENABLED" | default (env "LINERA_HELMFILE_SET_TEMPO_ENABLED" | default "false") }}
+  tempoEndpoint: {{ env "LINERA_HELMFILE_SET_TEMPO_ENDPOINT" | default "tempo.tempo.svc.cluster.local:4317" }}
+  latencyThresholdMs: {{ env "LINERA_HELMFILE_SET_OTEL_LATENCY_THRESHOLD" | default "500" }}
+  baselineSamplingPercent: {{ env "LINERA_HELMFILE_SET_OTEL_BASELINE_SAMPLING" | default "5.0" }}
+  decisionWait: {{ env "LINERA_HELMFILE_SET_OTEL_DECISION_WAIT" | default "200ms" }}
+  numTraces: {{ env "LINERA_HELMFILE_SET_OTEL_NUM_TRACES" | default "10000" }}
+  expectedTracesPerSec: {{ env "LINERA_HELMFILE_SET_OTEL_EXPECTED_TRACES_PER_SEC" | default "500" }}
+  resources:
+    requests:
+      memory: {{ env "LINERA_HELMFILE_SET_OTEL_MEMORY_REQUEST" | default "512Mi" }}
+      cpu: {{ env "LINERA_HELMFILE_SET_OTEL_CPU_REQUEST" | default "500m" }}
+    limits:
+      memory: {{ env "LINERA_HELMFILE_SET_OTEL_MEMORY_LIMIT" | default "4Gi" }}
+      cpu: {{ env "LINERA_HELMFILE_SET_OTEL_CPU_LIMIT" | default "4000m" }}