Merge pull request #193 from asherf/labels2

Extensibility model for requests/response metrics.

Author: asherf

README.md

@@ -204,6 +204,16 @@ that will export the metrics (replace myapp by your project name).
 
 Then we inject the wrapper in settings:
 
-```python 
+```python
 ROOT_URLCONF = "graphite.urls_prometheus_wrapper"
 ```
+
+## Adding custom labels to middleware (request/response) metrics
+
+You can add application specific labels to metrics reported by the django-prometheus middleware.
+This involves extending the classes defined in middleware.py.
+
+* Extend the Metrics class and override the `register_metric` method to add the application specific labels.
+* Extend middleware classes, set the metrics_cls class attribute to the the extended metric class and override the label_metric method to attach custom metrics.
+
+See implementation example in [the test app](django_prometheus/tests/end2end/testapp/test_middleware_custom_labels.py#L19-L46)

django_prometheus/middleware.py

@@ -25,7 +25,7 @@
 )
 
 
-class Metrics:
+class Metrics(object):
     _instance = None
 
     @classmethod
@@ -216,15 +216,22 @@ def _method(self, request):
             return "<invalid method>"
         return m
 
+    def label_metric(self, metric, request, response=None, **labels):
+        return metric.labels(**labels) if labels else metric
+
     def process_request(self, request):
         transport = self._transport(request)
         method = self._method(request)
-        self.metrics.requests_by_method.labels(method=method).inc()
-        self.metrics.requests_by_transport.labels(transport=transport).inc()
+        self.label_metric(self.metrics.requests_by_method, request, method=method).inc()
+        self.label_metric(
+            self.metrics.requests_by_transport, request, transport=transport
+        ).inc()
         if request.is_ajax():
-            self.metrics.requests_ajax.inc()
+            self.label_metric(self.metrics.requests_ajax, request).inc()
         content_length = int(request.META.get("CONTENT_LENGTH") or 0)
-        self.metrics.requests_body_bytes.observe(content_length)
+        self.label_metric(self.metrics.requests_body_bytes, request).observe(
+            content_length
+        )
         request.prometheus_after_middleware_event = Time()
 
     def _get_view_name(self, request):
@@ -240,49 +247,79 @@ def process_view(self, request, view_func, *view_args, **view_kwargs):
         method = self._method(request)
         if hasattr(request, "resolver_match"):
             name = request.resolver_match.view_name or "<unnamed view>"
-            self.metrics.requests_by_view_transport_method.labels(
-                view=name, transport=transport, method=method
+            self.label_metric(
+                self.metrics.requests_by_view_transport_method,
+                request,
+                view=name,
+                transport=transport,
+                method=method,
             ).inc()
 
     def process_template_response(self, request, response):
         if hasattr(response, "template_name"):
-            self.metrics.responses_by_templatename.labels(
-                templatename=str(response.template_name)
+            self.label_metric(
+                self.metrics.responses_by_templatename,
+                request,
+                response=response,
+                templatename=str(response.template_name),
             ).inc()
         return response
 
     def process_response(self, request, response):
         method = self._method(request)
         name = self._get_view_name(request)
         status = str(response.status_code)
-        self.metrics.responses_by_status.labels(status=status).inc()
-        self.metrics.responses_by_status_view_method.labels(
-            status=status, view=name, method=method
+        self.label_metric(
+            self.metrics.responses_by_status, request, response, status=status
+        ).inc()
+        self.label_metric(
+            self.metrics.responses_by_status_view_method,
+            request,
+            response,
+            status=status,
+            view=name,
+            method=method,
         ).inc()
         if hasattr(response, "charset"):
-            self.metrics.responses_by_charset.labels(
-                charset=str(response.charset)
+            self.label_metric(
+                self.metrics.responses_by_charset,
+                request,
+                response,
+                charset=str(response.charset),
             ).inc()
         if hasattr(response, "streaming") and response.streaming:
-            self.metrics.responses_streaming.inc()
+            self.label_metric(self.metrics.responses_streaming, request, response).inc()
         if hasattr(response, "content"):
-            self.metrics.responses_body_bytes.observe(len(response.content))
+            self.label_metric(
+                self.metrics.responses_body_bytes, request, response
+            ).observe(len(response.content))
         if hasattr(request, "prometheus_after_middleware_event"):
-            self.metrics.requests_latency_by_view_method.labels(
-                view=self._get_view_name(request), method=request.method
+            self.label_metric(
+                self.metrics.requests_latency_by_view_method,
+                request,
+                response,
+                view=self._get_view_name(request),
+                method=request.method,
             ).observe(TimeSince(request.prometheus_after_middleware_event))
         else:
-            self.metrics.requests_unknown_latency.inc()
+            self.label_metric(
+                self.metrics.requests_unknown_latency, request, response
+            ).inc()
         return response
 
     def process_exception(self, request, exception):
-        self.metrics.exceptions_by_type.labels(type=type(exception).__name__).inc()
+        self.label_metric(
+            self.metrics.exceptions_by_type, request, type=type(exception).__name__
+        ).inc()
         if hasattr(request, "resolver_match"):
             name = request.resolver_match.view_name or "<unnamed view>"
-            self.metrics.exceptions_by_view.labels(view=name).inc()
+            self.label_metric(self.metrics.exceptions_by_view, request, view=name).inc()
         if hasattr(request, "prometheus_after_middleware_event"):
-            self.metrics.requests_latency_by_view_method.labels(
-                view=self._get_view_name(request), method=request.method
+            self.label_metric(
+                self.metrics.requests_latency_by_view_method,
+                request,
+                view=self._get_view_name(request),
+                method=request.method,
             ).observe(TimeSince(request.prometheus_after_middleware_event))
         else:
-            self.metrics.requests_unknown_latency.inc()
+            self.label_metric(self.metrics.requests_unknown_latency, request).inc()

django_prometheus/tests/end2end/testapp/test_middleware_custom_labels.py

@@ -0,0 +1,121 @@
+from prometheus_client import REGISTRY
+from prometheus_client.metrics import MetricWrapperBase
+
+from django.test import SimpleTestCase, override_settings
+from django_prometheus.middleware import (
+    Metrics,
+    PrometheusAfterMiddleware,
+    PrometheusBeforeMiddleware,
+)
+from django_prometheus.testutils import PrometheusTestCaseMixin
+from testapp.helpers import get_middleware
+from testapp.test_middleware import M, T
+
+EXTENDED_METRICS = [
+    M("requests_latency_seconds_by_view_method"),
+    M("responses_total_by_status_view_method"),
+]
+
+
+class CustomMetrics(Metrics):
+    def register_metric(
+        self, metric_cls, name, documentation, labelnames=tuple(), **kwargs
+    ):
+        if name in EXTENDED_METRICS:
+            labelnames.extend(("view_type", "user_agent_type"))
+        return super(CustomMetrics, self).register_metric(
+            metric_cls, name, documentation, labelnames=labelnames, **kwargs
+        )
+
+
+class AppMetricsBeforeMiddleware(PrometheusBeforeMiddleware):
+    metrics_cls = CustomMetrics
+
+
+class AppMetricsAfterMiddleware(PrometheusAfterMiddleware):
+    metrics_cls = CustomMetrics
+
+    def label_metric(self, metric, request, response=None, **labels):
+        new_labels = labels
+        if metric._name in EXTENDED_METRICS:
+            new_labels = {"view_type": "foo", "user_agent_type": "browser"}
+            new_labels.update(labels)
+        return super(AppMetricsAfterMiddleware, self).label_metric(
+            metric, request, response=response, **new_labels
+        )
+
+
+@override_settings(
+    MIDDLEWARE=get_middleware(
+        "testapp.test_middleware_custom_labels.AppMetricsBeforeMiddleware",
+        "testapp.test_middleware_custom_labels.AppMetricsAfterMiddleware",
+    )
+)
+class TestMiddlewareMetricsWithCustomLabels(PrometheusTestCaseMixin, SimpleTestCase):
+    @classmethod
+    def setUpClass(cls):
+        super(TestMiddlewareMetricsWithCustomLabels, cls).setUpClass()
+        # Allow CustomMetrics to be used
+        for metric in Metrics._instance.__dict__.values():
+            if isinstance(metric, MetricWrapperBase):
+                REGISTRY.unregister(metric)
+        Metrics._instance = None
+
+    def test_request_counters(self):
+        registry = self.saveRegistry()
+        self.client.get("/")
+        self.client.get("/")
+        self.client.get("/help")
+        self.client.post("/", {"test": "data"})
+
+        self.assertMetricDiff(registry, 4, M("requests_before_middlewares_total"))
+        self.assertMetricDiff(registry, 4, M("responses_before_middlewares_total"))
+        self.assertMetricDiff(registry, 3, T("requests_total_by_method"), method="GET")
+        self.assertMetricDiff(registry, 1, T("requests_total_by_method"), method="POST")
+        self.assertMetricDiff(
+            registry, 4, T("requests_total_by_transport"), transport="http"
+        )
+        self.assertMetricDiff(
+            registry,
+            2,
+            T("requests_total_by_view_transport_method"),
+            view="testapp.views.index",
+            transport="http",
+            method="GET",
+        )
+        self.assertMetricDiff(
+            registry,
+            1,
+            T("requests_total_by_view_transport_method"),
+            view="testapp.views.help",
+            transport="http",
+            method="GET",
+        )
+        self.assertMetricDiff(
+            registry,
+            1,
+            T("requests_total_by_view_transport_method"),
+            view="testapp.views.index",
+            transport="http",
+            method="POST",
+        )
+        self.assertMetricDiff(
+            registry,
+            2.0,
+            T("responses_total_by_status_view_method"),
+            status="200",
+            view="testapp.views.index",
+            method="GET",
+            view_type="foo",
+            user_agent_type="browser",
+        )
+        self.assertMetricDiff(
+            registry,
+            1.0,
+            T("responses_total_by_status_view_method"),
+            status="200",
+            view="testapp.views.help",
+            method="GET",
+            view_type="foo",
+            user_agent_type="browser",
+        )