From d75e37d29b5840e41d4cbe4c5caff9b9e908e3e3 Mon Sep 17 00:00:00 2001 From: Pi Lanningham Date: Fri, 27 Dec 2024 04:51:48 -0500 Subject: [PATCH 1/2] fix: Grafana / Tempo communication Grafana was no longer able to communicate with the Tempo backend after some config changes recently. Checking the Grafana logs, I found ``` 2024-12-27 04:52:35 logger=grafana-apiserver t=2024-12-27T09:52:35.291900754Z level=info msg="[core] [Channel #2 SubChannel #3]grpc: addrConn.createTransport failed to connect to {Addr: \"tempo:3200\", ServerName: \"tempo:3200\", }. Err: connection error: desc = \"error reading server preface: http2: frame too large\"" ``` After some googling, this seemed to be an indicator that the client (Grafana) was trying to connect to the backend (Tempo) with http2, but the backend only supporting http1.1. In particular, Tempo uses grpc for internal communication, but by default, exposes its grpc capabilities on the grpc port, 4317; However, we had Grafana connecting to the http port, 3200. One option would be to expose the grpc port and use that, but it seems this is common enough that there's a built in [flag](https://github.com/grafana/tempo/blob/main/example/docker-compose/shared/tempo.yaml#L1) for it. This causes tempo to expose its grpc API over http; in particular, this means that the http2 request to initiate the grpc connection now succeeds. I chose to go with this approach, since that's the way the [examples](https://github.com/grafana/tempo/blob/main/example/docker-compose/shared/tempo.yaml#L1) I saw were structured, and it's the first thing that worked. See also this github discussion for context: https://github.com/open-telemetry/opentelemetry-collector/issues/7680 --- monitoring/grafana-tempo/tempo.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/monitoring/grafana-tempo/tempo.yml b/monitoring/grafana-tempo/tempo.yml index 9cf0117..57c2d6b 100644 --- a/monitoring/grafana-tempo/tempo.yml +++ b/monitoring/grafana-tempo/tempo.yml @@ -1,3 +1,4 @@ +stream_over_http_enabled: true server: http_listen_port: 3200 From 27a3517ece21078d2c95e23453e37e2528d1398f Mon Sep 17 00:00:00 2001 From: KtorZ Date: Sun, 29 Dec 2024 10:14:31 +0100 Subject: [PATCH 2/2] chore: add missing Prometheus service to grafana-tempo monitoring example. Prometheus is needed to collect metrics and serve them to grafana. While we don't have many metrics yet emitted directly by Amaru, we are able to automatically derive some from spans using the OpenTelemetry collector with the 'spanmetrics' collector/plugin. --- monitoring/grafana-tempo/docker-compose.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/monitoring/grafana-tempo/docker-compose.yml b/monitoring/grafana-tempo/docker-compose.yml index b14326c..3b121f2 100644 --- a/monitoring/grafana-tempo/docker-compose.yml +++ b/monitoring/grafana-tempo/docker-compose.yml @@ -10,6 +10,17 @@ services: - "55679:55679" - "13133:13133" + prometheus: + image: prom/prometheus:v3.0.1 + command: + [ "--web.enable-otlp-receiver" + , "--config.file=/etc/prometheus/prometheus.yml" + ] + volumes: + - "./prometheus.yml:/etc/prometheus/prometheus.yml" + ports: + - "9090:9090" + tempo: image: grafana/tempo:latest ports: