Skip to content

Commit 95c5cb3

Browse files
simonswinesimonswine
committed
WIP: Create pyroscope-monitoring
This contains a fully fledged observability deployment for pyroscope-monitoring Over time I would like to consoldidate our internal dashboards into this helm chart. This also allows us to to tests at least features of OSS in a local kind deployment.
1 parent b1824a1 commit 95c5cb3

File tree

900 files changed

+17413
-115058
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

900 files changed

+17413
-115058
lines changed

‎Makefile‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ fmt: $(BIN)/golangci-lint $(BIN)/buf $(BIN)/tk ## Automatically fix some lint er
209209
$(BIN)/golangci-lint run --fix
210210
cd api/ && $(BIN)/buf format -w .
211211
cd pkg && $(BIN)/buf format -w .
212-
$(BIN)/tk fmt ./operations/pyroscope/jsonnet/ tools/monitoring/
212+
$(BIN)/tk fmt ./operations/pyroscope/jsonnet/
213213

214214
.PHONY: check/unstaged-changes
215215
check/unstaged-changes:
@@ -236,10 +236,15 @@ define deploy
236236
--set pyroscope.image.repository=$(IMAGE_PREFIX)pyroscope \
237237
--set pyroscope.podAnnotations.image-digest=$(shell cat .docker-image-digest-pyroscope) \
238238
--set pyroscope.service.port_name=http-metrics \
239-
--set pyroscope.podAnnotations."profiles\.grafana\.com\/memory\.port_name"=http-metrics \
240-
--set pyroscope.podAnnotations."profiles\.grafana\.com\/cpu\.port_name"=http-metrics \
241-
--set pyroscope.podAnnotations."profiles\.grafana\.com\/goroutine\.port_name"=http-metrics \
242-
--set pyroscope.extraEnvVars.JAEGER_AGENT_HOST=jaeger.monitoring.svc.cluster.local. \
239+
--set-string pyroscope.podAnnotations."profiles\.grafana\.com/memory\.port_name"=http-metrics \
240+
--set-string pyroscope.podAnnotations."profiles\.grafana\.com/cpu\.port_name"=http-metrics \
241+
--set-string pyroscope.podAnnotations."profiles\.grafana\.com/goroutine\.port_name"=http-metrics \
242+
--set-string pyroscope.podAnnotations."k8s\.grafana\.com/scrape"=true \
243+
--set-string pyroscope.podAnnotations."k8s\.grafana\.com/metrics\.portName"=http-metrics \
244+
--set-string pyroscope.podAnnotations."k8s\.grafana\.com/metrics\.scrapeInterval"=15s \
245+
--set-string pyroscope.extraEnvVars.JAEGER_AGENT_HOST=pyroscope-monitoring-alloy-receiver \
246+
--set pyroscope.extraEnvVars.JAEGER_SAMPLER_TYPE=const \
247+
--set pyroscope.extraEnvVars.JAEGER_SAMPLER_PARAM=1 \
243248
--set pyroscope.extraArgs."pyroscopedb\.max-block-duration"=5m
244249
endef
245250

@@ -394,15 +399,17 @@ $(BIN)/helm-docs: Makefile go.mod
394399

395400
.PHONY: cve/check
396401
cve/check:
397-
docker run -t -i --rm --volume "$(CURDIR)/:/repo" -u "$(shell id -u)" aquasec/trivy:0.45.1 filesystem --cache-dir /repo/.cache/trivy --scanners vuln --skip-dirs .tmp/ --skip-dirs node_modules/ --skip-dirs tools/monitoring/vendor/ /repo
402+
docker run -t -i --rm --volume "$(CURDIR)/:/repo" -u "$(shell id -u)" aquasec/trivy:0.45.1 filesystem --cache-dir /repo/.cache/trivy --scanners vuln --skip-dirs .tmp/ --skip-dirs node_modules/ /repo
398403

399404
.PHONY: helm/lint
400405
helm/lint: $(BIN)/helm
401-
$(BIN)/helm lint ./operations/pyroscope/helm/pyroscope/
406+
$(BIN)/helm lint ./operations/pyroscope/helm/pyroscope
407+
$(BIN)/helm lint ./operations/pyroscope/helm/pyroscope-monitoring
402408

403409
.PHONY: helm/docs
404410
helm/docs: $(BIN)/helm-docs
405411
$(BIN)/helm-docs -c operations/pyroscope/helm/pyroscope
412+
$(BIN)/helm-docs -c operations/pyroscope/helm/pyroscope-monitoring
406413

407414
.PHONY: goreleaser/lint
408415
goreleaser/lint: $(BIN)/goreleaser
@@ -460,16 +467,9 @@ deploy-micro-services-v1: $(BIN)/kind $(BIN)/helm docker-image/pyroscope/build
460467
$(call deploy,pyroscope-micro-services,$(HELM_FLAGS_V1_MICROSERVICES))
461468

462469
.PHONY: deploy-monitoring
463-
deploy-monitoring: $(BIN)/tk $(BIN)/kind tools/monitoring/environments/default/spec.json
464-
kubectl --context="kind-$(KIND_CLUSTER)" create namespace monitoring --dry-run=client -o yaml | kubectl --context="kind-$(KIND_CLUSTER)" apply -f -
465-
$(BIN)/tk apply tools/monitoring/environments/default/main.jsonnet
466-
467-
.PHONY: tools/monitoring/environments/default/spec.json # This is a phony target for now as the cluster might be not already created.
468-
tools/monitoring/environments/default/spec.json: $(BIN)/tk $(BIN)/kind
470+
deploy-monitoring: $(BIN)/kind $(BIN)/helm
469471
$(BIN)/kind export kubeconfig --name $(KIND_CLUSTER) || $(BIN)/kind create cluster --name $(KIND_CLUSTER)
470-
pushd tools/monitoring/ && rm -Rf vendor/ lib/ environments/default/spec.json && PATH=$(BIN):$(PATH) $(BIN)/tk init -f
471-
echo "import 'monitoring.libsonnet'" > tools/monitoring/environments/default/main.jsonnet
472-
$(BIN)/tk env set tools/monitoring/environments/default --server=$(shell $(BIN)/kind get kubeconfig --name pyroscope-dev | grep server: | sed 's/server://g' | xargs) --namespace=monitoring
472+
$(BIN)/helm upgrade --install pyroscope-monitoring ./operations/pyroscope/helm/pyroscope-monitoring
473473

474474
include Makefile.examples
475475

‎operations/pyroscope/helm/pyroscope-monitoring/.helmignore‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Patterns to ignore when building packages.
2+
# This supports shell glob matching, relative path matching, and
3+
# negation (prefixed with !). Only one pattern per line.
4+
.DS_Store
5+
# Common VCS dirs
6+
.git/
7+
.gitignore
8+
.bzr/
9+
.bzrignore
10+
.hg/
11+
.hgignore
12+
.svn/
13+
# Common backup files
14+
*.swp
15+
*.bak
16+
*.tmp
17+
*.orig
18+
*~
19+
# Various IDEs
20+
.project
21+
.idea/
22+
*.tmproj
23+
.vscode/

‎operations/pyroscope/helm/pyroscope-monitoring/Chart.lock‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
dependencies:
2+
- name: k8s-monitoring
3+
repository: https://grafana.github.io/helm-charts
4+
version: 3.5.3
5+
digest: sha256:08e8b457ff645e35c4193f93f51fb59b89246698756ab1df764a3133f8854969
6+
generated: "2025-10-13T16:40:28.118524+01:00"

‎operations/pyroscope/helm/pyroscope-monitoring/Chart.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
apiVersion: v2
2+
name: pyroscope-monitoring
3+
description: A Helm chart for monitoring Grafana Pyroscope
4+
type: application
5+
6+
version: 0.1.0
7+
appVersion: "0.0.0"
8+
9+
dependencies:
10+
- name: k8s-monitoring
11+
alias: monitoring
12+
version: 3.5.3
13+
repository: https://grafana.github.io/helm-charts
14+
condition: monitoring.enabled

‎operations/pyroscope/helm/pyroscope-monitoring/README.md‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
# pyroscope-monitoring
2+
3+
![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.0.0](https://img.shields.io/badge/AppVersion-0.0.0-informational?style=flat-square)
4+
5+
A Helm chart for monitoring Grafana Pyroscope
6+
7+
## Requirements
8+
9+
| Repository | Name | Version |
10+
|------------|------|---------|
11+
| https://grafana.github.io/helm-charts | monitoring(k8s-monitoring) | 3.5.3 |
12+
13+
## Values
14+
15+
| Key | Type | Default | Description |
16+
|-----|------|---------|-------------|
17+
| affinity | object | `{}` | |
18+
| dashboards.cloudBackendGateway | bool | `false` | |
19+
| dashboards.cloudBackendGatewaySelector | string | `"container=~\"cortex-gw(-internal)?\""` | |
20+
| dashboards.cluster | string | `"pyroscope-dev"` | |
21+
| dashboards.ingestSelector | string | `"container=~\"pyroscope|distributor|query-frontend\""` | |
22+
| dashboards.namespace | string | `"default"` | |
23+
| dashboards.namespaceRegex | string | `".*"` | |
24+
| env[0].name | string | `"GF_PLUGINS_PREINSTALL"` | |
25+
| env[0].value | string | `"grafana-exploretraces-app"` | |
26+
| env[1].name | string | `"ENABLE_LOGS_GRAFANA"` | |
27+
| env[1].value | string | `"true"` | |
28+
| fullnameOverride | string | `""` | |
29+
| image.pullPolicy | string | `"IfNotPresent"` | |
30+
| image.repository | string | `"grafana/otel-lgtm"` | |
31+
| image.tag | string | `"0.11.0"` | |
32+
| imagePullSecrets | list | `[]` | |
33+
| monitoring.alloy-logs.enabled | bool | `true` | |
34+
| monitoring.alloy-metrics.enabled | bool | `true` | |
35+
| monitoring.alloy-metrics.image.tag | string | `"v1.10.2"` | |
36+
| monitoring.alloy-receiver.alloy.extraPorts[0].name | string | `"thrift-compact"` | |
37+
| monitoring.alloy-receiver.alloy.extraPorts[0].port | int | `6831` | |
38+
| monitoring.alloy-receiver.alloy.extraPorts[0].protocol | string | `"UDP"` | |
39+
| monitoring.alloy-receiver.alloy.extraPorts[0].targetPort | int | `6831` | |
40+
| monitoring.alloy-receiver.alloy.extraPorts[1].name | string | `"jaeger-binary"` | |
41+
| monitoring.alloy-receiver.alloy.extraPorts[1].port | int | `6832` | |
42+
| monitoring.alloy-receiver.alloy.extraPorts[1].protocol | string | `"UDP"` | |
43+
| monitoring.alloy-receiver.alloy.extraPorts[1].targetPort | int | `6832` | |
44+
| monitoring.alloy-receiver.alloy.extraPorts[2].name | string | `"jaeger-http"` | |
45+
| monitoring.alloy-receiver.alloy.extraPorts[2].port | int | `14268` | |
46+
| monitoring.alloy-receiver.alloy.extraPorts[2].protocol | string | `"TCP"` | |
47+
| monitoring.alloy-receiver.alloy.extraPorts[2].targetPort | int | `14268` | |
48+
| monitoring.alloy-receiver.enabled | bool | `true` | |
49+
| monitoring.alloy-singleton.enabled | bool | `true` | |
50+
| monitoring.annotationAutodiscovery.enabled | bool | `true` | |
51+
| monitoring.applicationObservability.enabled | bool | `true` | |
52+
| monitoring.applicationObservability.receivers.jaeger.thriftBinary.enabled | bool | `true` | |
53+
| monitoring.applicationObservability.receivers.jaeger.thriftCompact.enabled | bool | `true` | |
54+
| monitoring.applicationObservability.receivers.jaeger.thriftHttp.enabled | bool | `true` | |
55+
| monitoring.cluster.name | string | `"pyroscope-dev"` | |
56+
| monitoring.clusterEvents.enabled | bool | `true` | |
57+
| monitoring.clusterMetrics.enabled | bool | `true` | |
58+
| monitoring.clusterMetrics.kepler.enabled | bool | `false` | |
59+
| monitoring.clusterMetrics.opencost.enabled | bool | `false` | |
60+
| monitoring.destinations[0].logs.enabled | bool | `true` | |
61+
| monitoring.destinations[0].metrics.enabled | bool | `true` | |
62+
| monitoring.destinations[0].name | string | `"otlp-gateway"` | |
63+
| monitoring.destinations[0].protocol | string | `"http"` | |
64+
| monitoring.destinations[0].traces.enabled | bool | `true` | |
65+
| monitoring.destinations[0].type | string | `"otlp"` | |
66+
| monitoring.destinations[0].url | string | `"http://pyroscope-monitoring:4318"` | |
67+
| monitoring.destinations[1].metrics.enabled | bool | `true` | |
68+
| monitoring.destinations[1].name | string | `"prometheus"` | |
69+
| monitoring.destinations[1].sendNativeHistograms | bool | `true` | |
70+
| monitoring.destinations[1].type | string | `"prometheus"` | |
71+
| monitoring.destinations[1].url | string | `"http://pyroscope-monitoring:9090/api/v1/write"` | |
72+
| monitoring.enabled | bool | `true` | |
73+
| monitoring.global.scrapeClassicHistograms | bool | `true` | |
74+
| monitoring.global.scrapeProtocols[0] | string | `"PrometheusProto"` | |
75+
| monitoring.global.scrapeProtocols[1] | string | `"OpenMetricsText1.0.0"` | |
76+
| monitoring.global.scrapeProtocols[2] | string | `"OpenMetricsText0.0.1"` | |
77+
| monitoring.global.scrapeProtocols[3] | string | `"PrometheusText0.0.4"` | |
78+
| monitoring.podLogs.enabled | bool | `true` | |
79+
| nameOverride | string | `""` | |
80+
| nodeSelector | object | `{}` | |
81+
| podAnnotations | object | `{}` | |
82+
| podLabels | object | `{}` | |
83+
| podSecurityContext | object | `{}` | |
84+
| replicaCount | int | `1` | |
85+
| resources | object | `{}` | |
86+
| securityContext | object | `{}` | |
87+
| service.ports[0].name | string | `"grafana"` | |
88+
| service.ports[0].port | int | `3000` | |
89+
| service.ports[0].protocol | string | `"TCP"` | |
90+
| service.ports[0].targetPort | int | `3000` | |
91+
| service.ports[1].name | string | `"otel-grpc"` | |
92+
| service.ports[1].port | int | `4317` | |
93+
| service.ports[1].protocol | string | `"TCP"` | |
94+
| service.ports[1].targetPort | int | `4317` | |
95+
| service.ports[2].name | string | `"otel-http"` | |
96+
| service.ports[2].port | int | `4318` | |
97+
| service.ports[2].protocol | string | `"TCP"` | |
98+
| service.ports[2].targetPort | int | `4318` | |
99+
| service.ports[3].name | string | `"prometheus"` | |
100+
| service.ports[3].port | int | `9090` | |
101+
| service.ports[3].protocol | string | `"TCP"` | |
102+
| service.ports[3].targetPort | int | `9090` | |
103+
| service.type | string | `"ClusterIP"` | |
104+
| tolerations | list | `[]` | |
105+
| volumeMounts[0].mountPath | string | `"/data/tempo"` | |
106+
| volumeMounts[0].name | string | `"tempo-data"` | |
107+
| volumeMounts[1].mountPath | string | `"/data/grafana"` | |
108+
| volumeMounts[1].name | string | `"grafana-data"` | |
109+
| volumeMounts[2].mountPath | string | `"/data/loki"` | |
110+
| volumeMounts[2].name | string | `"loki-data"` | |
111+
| volumeMounts[3].mountPath | string | `"/loki"` | |
112+
| volumeMounts[3].name | string | `"loki-storage"` | |
113+
| volumeMounts[4].mountPath | string | `"/data/prometheus"` | |
114+
| volumeMounts[4].name | string | `"p8s-storage"` | |
115+
| volumeMounts[5].mountPath | string | `"/data/pyroscope"` | |
116+
| volumeMounts[5].name | string | `"pyroscope-storage"` | |
117+
| volumeMounts[6].mountPath | string | `"/otel-lgtm/grafana/conf/provisioning/dashboards"` | |
118+
| volumeMounts[6].name | string | `"dashboards"` | |
119+
| volumes[0].emptyDir | object | `{}` | |
120+
| volumes[0].name | string | `"tempo-data"` | |
121+
| volumes[1].emptyDir | object | `{}` | |
122+
| volumes[1].name | string | `"loki-data"` | |
123+
| volumes[2].emptyDir | object | `{}` | |
124+
| volumes[2].name | string | `"grafana-data"` | |
125+
| volumes[3].emptyDir | object | `{}` | |
126+
| volumes[3].name | string | `"loki-storage"` | |
127+
| volumes[4].emptyDir | object | `{}` | |
128+
| volumes[4].name | string | `"p8s-storage"` | |
129+
| volumes[5].emptyDir | object | `{}` | |
130+
| volumes[5].name | string | `"pyroscope-storage"` | |
131+

‎operations/pyroscope/helm/pyroscope-monitoring/charts/k8s-monitoring-3.5.3.tgz‎

659 KB
Binary file not shown.

‎operations/pyroscope/helm/pyroscope-monitoring/convert-dashboard.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env bash
2+
3+
# Goal of that script is to convert a grafana dashboard from internal GL environments so it can be used with this pyroscope-monitoring chart.
4+
5+
set -eux
6+
set -o pipefail
7+
8+
# loop over inputs
9+
for dashboard in "$@"
10+
do
11+
filename="$(basename $dashboard)"
12+
name="${filename%.*}"
13+
destination="templates/_dashboard_${name}.yaml"
14+
15+
# create basic dashboard
16+
echo "{{- define \"pyroscope-monitoring.dashboards.${name}\" -}}" > "${destination}"
17+
cat "$dashboard" | js-yaml | sed 's/{{.*}}/{{"\0\"}}/g' >> "${destination}"
18+
echo "{{- end }}" >> "${destination}"
19+
20+
# replace specific namespace
21+
sed -i 's/profiles-ops-002/{{.Values.dashboards.namespace | quote}}/g' "${destination}"
22+
sed -i 's/fire|profiles-\.\*/{{.Values.dashboards.namespaceRegex | quote}}/g' "${destination}"
23+
sed -i 's/fire-\.\*|profiles-\.\*/{{.Values.dashboards.namespaceRegex | quote}}/g' "${destination}"
24+
25+
# replace default cluster
26+
sed -i 's/ops-eu-south-0/{{.Values.dashboards.cluster | quote}}/g' "${destination}"
27+
28+
# replace cortex-gw selectors
29+
sed -i 's/{job=~"$namespace\/cortex-gw(-internal)?"/{ {{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace"/g' "${destination}"
30+
sed -i 's/job=~"$namespace\/cortex-gw(-internal)?"/{{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace"/g' "${destination}"
31+
32+
done

‎operations/pyroscope/helm/pyroscope-monitoring/lgtm.yaml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# this is intended for demo / testing purposes only, not for production usage
2+
apiVersion: v1
3+
kind: Service
4+
metadata:
5+
name: lgtm
6+
spec:
7+
selector:
8+
app: lgtm
9+
ports:
10+
- name: grafana
11+
protocol: TCP
12+
port: 3000
13+
targetPort: 3000
14+
- name: otel-grpc
15+
protocol: TCP
16+
port: 4317
17+
targetPort: 4317
18+
- name: otel-http
19+
protocol: TCP
20+
port: 4318
21+
targetPort: 4318
22+
---
23+
apiVersion: apps/v1
24+
kind: Deployment
25+
metadata:
26+
name: lgtm
27+
spec:
28+
replicas: 1
29+
selector:
30+
matchLabels:
31+
app: lgtm
32+
template:
33+
metadata:
34+
labels:
35+
app: lgtm
36+
spec:
37+
containers:
38+
- name: lgtm
39+
image: grafana/otel-lgtm:latest
40+
ports:
41+
- containerPort: 3000
42+
- containerPort: 4317
43+
- containerPort: 4318
44+
readinessProbe:
45+
exec:
46+
command:
47+
- cat
48+
- /tmp/ready
49+
# NOTE: By default OpenShift does not allow writing the root directory.
50+
# Thats why the data dirs for grafana, prometheus and loki can not be
51+
# created and the pod never becomes ready.
52+
# See: https://github.com/grafana/docker-otel-lgtm/issues/132
53+
volumeMounts:
54+
- name: tempo-data
55+
mountPath: /data/tempo
56+
- name: grafana-data
57+
mountPath: /data/grafana
58+
- name: loki-data
59+
mountPath: /data/loki
60+
- name: loki-storage
61+
mountPath: /loki
62+
- name: p8s-storage
63+
mountPath: /data/prometheus
64+
- name: pyroscope-storage
65+
mountPath: /data/pyroscope
66+
volumes:
67+
- name: tempo-data
68+
emptyDir: {}
69+
- name: loki-data
70+
emptyDir: {}
71+
- name: grafana-data
72+
emptyDir: {}
73+
- name: loki-storage
74+
emptyDir: {}
75+
- name: p8s-storage
76+
emptyDir: {}
77+
- name: pyroscope-storage
78+
emptyDir: {}

‎operations/pyroscope/helm/pyroscope-monitoring/templates/NOTES.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# TODO: Pyroscope Monitoring helm chart

0 commit comments

Comments
 (0)