feat(otlp): support metrics

pyproject.toml

@@ -173,6 +173,11 @@ opentelemetry-log-handler = [
     "opentelemetry-sdk==1.40.0",
     "opentelemetry-exporter-otlp==1.40.0",
 ]
+opentelemetry-metrics = [
+    "opentelemetry-api==1.40.0",
+    "opentelemetry-sdk==1.40.0",
+    "opentelemetry-exporter-otlp==1.40.0",
+]
 
 # The duplication here is awful.
 #

synapse/api/auth/__init__.py

@@ -20,13 +20,12 @@
 #
 from typing import TYPE_CHECKING, Protocol
 
-from prometheus_client import Histogram
-
 from twisted.web.server import Request
 
 from synapse.appservice import ApplicationService
 from synapse.http.site import SynapseRequest
 from synapse.metrics import SERVER_NAME_LABEL
+from synapse.metrics.instruments import Histogram
 from synapse.types import Requester
 
 if TYPE_CHECKING:

synapse/app/_base.py

@@ -337,9 +337,21 @@ def listen_metrics(
     Twisted reactor thread between bytecode boundaries and the metrics thread gets
     scheduled with roughly equal priority to the Twisted reactor thread.
 
+    When ``SYNAPSE_METRICS_BACKEND=otlp`` the Prometheus scrape endpoint is not
+    needed – metrics are pushed via OTLP instead – so this is a no-op.
+
     Returns:
         List of WSGIServer with the thread they are running on.
     """
+    from synapse.metrics.instruments import METRICS_BACKEND
+
+    if METRICS_BACKEND == "otlp":
+        logger.info(
+            "Not starting Prometheus metrics listener because "
+            "SYNAPSE_METRICS_BACKEND=otlp; metrics are exported via OTLP"
+        )
+        return []
+
     from prometheus_client import start_http_server as start_http_server_prometheus
 
     from synapse.metrics import RegistryProxy
@@ -764,6 +776,16 @@ def log_shutdown() -> None:
     setup_sentry(hs)
     setup_sdnotify(hs)
 
+    # Flush pending OTLP metrics on shutdown.
+    from synapse.metrics.instruments import METRICS_BACKEND
+
+    if METRICS_BACKEND == "otlp":
+        from synapse.metrics._otel import shutdown as shutdown_otel_metrics
+
+        hs.get_clock().add_system_event_trigger(
+            "during", "shutdown", shutdown_otel_metrics
+        )
+
     # Register background tasks required by this server. This must be done
     # somewhat manually due to the background tasks not being registered
     # unless handlers are instantiated.

synapse/app/generic_worker.py

@@ -193,7 +193,15 @@ def _listen_http(self, listener_config: ListenerConfig) -> None:
         for res in listener_config.http_options.resources:
             for name in res.names:
                 if name == "metrics":
-                    resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
+                    from synapse.metrics.instruments import METRICS_BACKEND
+
+                    if METRICS_BACKEND == "otlp":
+                        logger.info(
+                            "Not exposing Prometheus /metrics resource because "
+                            "SYNAPSE_METRICS_BACKEND=otlp; metrics are exported via OTLP"
+                        )
+                    else:
+                        resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
                 elif name == "client":
                     resource: Resource = ClientRestResource(self)
 

synapse/app/homeserver.py

@@ -257,10 +257,18 @@ def _configure_named_resource(
             resources[SERVER_KEY_PREFIX] = KeyResource(self)
 
         if name == "metrics" and self.config.metrics.enable_metrics:
-            metrics_resource: Resource = MetricsResource(RegistryProxy)
-            if compress:
-                metrics_resource = gz_wrap(metrics_resource)
-            resources[METRICS_PREFIX] = metrics_resource
+            from synapse.metrics.instruments import METRICS_BACKEND
+
+            if METRICS_BACKEND == "otlp":
+                logger.info(
+                    "Not exposing Prometheus /metrics resource because "
+                    "SYNAPSE_METRICS_BACKEND=otlp; metrics are exported via OTLP"
+                )
+            else:
+                metrics_resource: Resource = MetricsResource(RegistryProxy)
+                if compress:
+                    metrics_resource = gz_wrap(metrics_resource)
+                resources[METRICS_PREFIX] = metrics_resource
 
         if name == "replication":
             resources[REPLICATION_PREFIX] = ReplicationRestResource(self)

synapse/app/phone_stats_home.py

@@ -24,11 +24,10 @@
 import sys
 from typing import TYPE_CHECKING, Mapping, Sized
 
-from prometheus_client import Gauge
-
 from twisted.internet import defer
 
 from synapse.metrics import SERVER_NAME_LABEL
+from synapse.metrics.instruments import Gauge
 from synapse.types import JsonDict
 from synapse.util.duration import Duration
 

synapse/appservice/api.py

@@ -29,7 +29,6 @@
     TypeVar,
 )
 
-from prometheus_client import Counter
 from typing_extensions import ParamSpec, TypeGuard
 
 from synapse.api.constants import ThirdPartyEntityKind
@@ -44,6 +43,7 @@
 from synapse.http.client import SimpleHttpClient, is_unknown_endpoint
 from synapse.logging import opentracing
 from synapse.metrics import SERVER_NAME_LABEL
+from synapse.metrics.instruments import Counter
 from synapse.types import DeviceListUpdates, JsonDict, JsonMapping, ThirdPartyInstanceID
 from synapse.util.caches.response_cache import ResponseCache
 from synapse.util.duration import Duration

synapse/config/metrics.py

@@ -20,6 +20,7 @@
 #
 #
 
+import os
 from typing import Any
 
 import attr
@@ -50,6 +51,25 @@ class MetricsConfig(Config):
     def read_config(self, config: JsonDict, **kwargs: Any) -> None:
         self.enable_metrics = config.get("enable_metrics", False)
 
+        # The metrics backend is authoritative from the env var because metric
+        # objects are created at import time (before config is read).  The yaml
+        # value is accepted for documentation / validation purposes.
+        env_backend = os.environ.get("SYNAPSE_METRICS_BACKEND", "prometheus").lower()
+        yaml_backend = config.get("metrics_backend", "prometheus").lower()
+        if yaml_backend not in ("prometheus", "otlp"):
+            raise ConfigError(
+                "Invalid metrics_backend %r (must be 'prometheus' or 'otlp')"
+                % yaml_backend
+            )
+        if yaml_backend == "otlp" and env_backend != "otlp":
+            raise ConfigError(
+                "metrics_backend is set to 'otlp' in config but the "
+                "SYNAPSE_METRICS_BACKEND env var is %r. The env var must also "
+                "be set to 'otlp' because metrics are initialised at import "
+                "time (before config is read)." % env_backend
+            )
+        self.metrics_backend: str = env_backend
+
         self.report_stats = config.get("report_stats", None)
         self.report_stats_endpoint = config.get(
             "report_stats_endpoint", "https://matrix.org/report-usage-stats/push"

synapse/federation/federation_client.py

@@ -40,7 +40,6 @@
 )
 
 import attr
-from prometheus_client import Counter
 
 from synapse.api.constants import Direction, EventContentFields, EventTypes, Membership
 from synapse.api.errors import (
@@ -71,6 +70,7 @@
 from synapse.http.types import QueryParams
 from synapse.logging.opentracing import SynapseTags, log_kv, set_tag, tag_args, trace
 from synapse.metrics import SERVER_NAME_LABEL
+from synapse.metrics.instruments import Counter
 from synapse.types import JsonDict, StrCollection, UserID, get_domain_from_id
 from synapse.util.async_helpers import concurrently_execute
 from synapse.util.caches.expiringcache import ExpiringCache

synapse/federation/federation_server.py

@@ -31,8 +31,6 @@
     Mapping,
 )
 
-from prometheus_client import Counter, Gauge, Histogram
-
 from twisted.python import failure
 
 from synapse.api.constants import (
@@ -80,6 +78,7 @@
 )
 from synapse.metrics import SERVER_NAME_LABEL
 from synapse.metrics.background_process_metrics import wrap_as_background_process
+from synapse.metrics.instruments import Counter, Gauge, Histogram
 from synapse.replication.http.federation import (
     ReplicationFederationSendEduRestServlet,
 )

synapse/federation/sender/__init__.py

@@ -141,7 +141,6 @@
 )
 
 import attr
-from prometheus_client import Counter
 
 from twisted.internet import defer
 
@@ -166,6 +165,7 @@
 from synapse.metrics.background_process_metrics import (
     wrap_as_background_process,
 )
+from synapse.metrics.instruments import Counter
 from synapse.types import (
     JsonDict,
     ReadReceipt,

synapse/federation/sender/per_destination_queue.py

@@ -26,7 +26,6 @@
 from typing import TYPE_CHECKING, Hashable, Iterable
 
 import attr
-from prometheus_client import Counter
 
 from twisted.internet import defer
 
@@ -44,6 +43,7 @@
 from synapse.logging.context import PreserveLoggingContext
 from synapse.logging.opentracing import SynapseTags, set_tag
 from synapse.metrics import SERVER_NAME_LABEL, sent_transactions_counter
+from synapse.metrics.instruments import Counter
 from synapse.types import JsonDict, ReadReceipt
 from synapse.util.retryutils import NotRetryingDestination, get_retry_limiter
 from synapse.visibility import filter_events_for_server

synapse/federation/sender/transaction_manager.py

@@ -20,8 +20,6 @@
 import logging
 from typing import TYPE_CHECKING
 
-from prometheus_client import Gauge
-
 from synapse.api.constants import EduTypes
 from synapse.api.errors import HttpResponseException
 from synapse.events import EventBase
@@ -35,6 +33,7 @@
     whitelisted_homeserver,
 )
 from synapse.metrics import SERVER_NAME_LABEL
+from synapse.metrics.instruments import Gauge
 from synapse.types import JsonDict
 from synapse.util.json import json_decoder
 from synapse.util.metrics import measure_func

synapse/handlers/appservice.py

@@ -26,8 +26,6 @@
     Mapping,
 )
 
-from prometheus_client import Counter
-
 from twisted.internet import defer
 
 import synapse
@@ -44,6 +42,7 @@
 from synapse.metrics.background_process_metrics import (
     wrap_as_background_process,
 )
+from synapse.metrics.instruments import Counter
 from synapse.storage.databases.main.directory import RoomAliasMapping
 from synapse.types import (
     DeviceListUpdates,