Add error metrics tracking for workflow incidents

src/apps/Elsa.Server.Web/Program.cs

@@ -29,6 +29,7 @@
 using Elsa.MongoDb.Modules.Management;
 using Elsa.MongoDb.Modules.Runtime;
 using Elsa.MongoDb.Modules.Tenants;
+using Elsa.OpenTelemetry.Metrics;
 using Elsa.OpenTelemetry.Middleware;
 using Elsa.Retention.Extensions;
 using Elsa.Retention.Models;
@@ -146,15 +147,16 @@
                 .AddAspNetCoreInstrumentation()
                 .AddHttpClientInstrumentation()
                 .AddSqlClientInstrumentation()
-                .AddConsoleExporter()
+                //.AddConsoleExporter()
                 .AddOtlpExporter()
                 ;
         })
         .WithMetrics(metrics =>
         {
             metrics
-                .AddAspNetCoreInstrumentation()
-                .AddHttpClientInstrumentation()
+                .AddMeter(ErrorMetrics.MeterName)
+                //.AddAspNetCoreInstrumentation()
+                //.AddHttpClientInstrumentation()
                 .AddConsoleExporter()
                 .AddOtlpExporter()
                 ;

src/modules/Elsa.OpenTelemetry/Contracts/IErrorMetricHandler.cs

@@ -0,0 +1,10 @@
+using Elsa.OpenTelemetry.Models;
+
+namespace Elsa.OpenTelemetry.Contracts;
+
+public interface IErrorMetricHandler
+{
+    float Order { get; }
+    bool CanHandle(ErrorMetricContext context);
+    void Handle(ErrorMetricContext context);
+}

src/modules/Elsa.OpenTelemetry/Features/OpenTelemetryFeature.cs

@@ -2,6 +2,7 @@
 using Elsa.Features.Services;
 using Elsa.OpenTelemetry.Contracts;
 using Elsa.OpenTelemetry.Handlers;
+using Elsa.OpenTelemetry.Metrics;
 using Elsa.OpenTelemetry.Options;
 using Microsoft.Extensions.DependencyInjection;
 
@@ -23,10 +24,13 @@ public class OpenTelemetryFeature(IModule module) : FeatureBase(module)
     public override void Configure()
     {
         Services
-            .AddScoped<IActivityErrorSpanHandler, DefaultErrorSpanHandler>()
-            .AddScoped<IActivityErrorSpanHandler, FaultExceptionErrorSpanHandler>()
-            .AddScoped<IWorkflowErrorSpanHandler, DefaultErrorSpanHandler>()
-            .AddScoped<IWorkflowErrorSpanHandler, FaultExceptionErrorSpanHandler>();
+            .AddScoped<IActivityErrorSpanHandler, DefaultExceptionHandler>()
+            .AddScoped<IActivityErrorSpanHandler, FaultExceptionHandler>()
+            .AddScoped<IWorkflowErrorSpanHandler, DefaultExceptionHandler>()
+            .AddScoped<IWorkflowErrorSpanHandler, FaultExceptionHandler>()
+            .AddScoped<IErrorMetricHandler, DefaultExceptionHandler>()
+            .AddScoped<IErrorMetricHandler, FaultExceptionHandler>()
+            .AddScoped<ErrorMetrics>();
 
         Services.Configure<OpenTelemetryOptions>(options =>
         {

src/modules/Elsa.OpenTelemetry/Handlers/DefaultExceptionHandler.cs

@@ -0,0 +1,27 @@
+using Elsa.OpenTelemetry.Contracts;
+using Elsa.OpenTelemetry.Models;
+
+namespace Elsa.OpenTelemetry.Handlers;
+
+public class DefaultExceptionHandler : IActivityErrorSpanHandler, IWorkflowErrorSpanHandler, IErrorMetricHandler
+{
+    public float Order => 100000;
+    public bool CanHandle(ErrorMetricContext context) => true;
+    public bool CanHandle(WorkflowErrorSpanContext context) => true;
+    public bool CanHandle(ActivityErrorSpanContext context) => context.Exception != null;
+
+    public void Handle(WorkflowErrorSpanContext context)
+    {
+        // No-op.
+    }
+
+    public void Handle(ActivityErrorSpanContext context)
+    {
+        context.Span.AddException(context.Exception!);
+    }
+
+    public void Handle(ErrorMetricContext context)
+    {
+        context.Tags["error.exception.type"] = context.Exception.GetType().Name;
+    }
+}

src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionErrorSpanHandler.cs → src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionHandler.cs

@@ -4,12 +4,13 @@
 
 namespace Elsa.OpenTelemetry.Handlers;
 
-public class FaultExceptionErrorSpanHandler : IActivityErrorSpanHandler, IWorkflowErrorSpanHandler
+public class FaultExceptionHandler : IActivityErrorSpanHandler, IWorkflowErrorSpanHandler, IErrorMetricHandler
 {
     public float Order => 0;
-
+    
     public bool CanHandle(ActivityErrorSpanContext context) => context.Exception is FaultException;
     public bool CanHandle(WorkflowErrorSpanContext context) => context.Exception is FaultException;
+    public bool CanHandle(ErrorMetricContext context) => context.Exception is FaultException;
 
     public void Handle(ActivityErrorSpanContext context)
     {
@@ -36,4 +37,12 @@ public void Handle(WorkflowErrorSpanContext context)
         // Datadog will ignore unknown attributes, so we'll set them on a different object.
         span.SetTag("error_details.type", faultException.Type);
     }
+
+    public void Handle(ErrorMetricContext context)
+    {
+        var faultException = (FaultException)context.Exception!;
+        context.Tags["error.code"] = faultException.Code;
+        context.Tags["error.category"] = faultException.Category;
+        context.Tags["error.type"] = faultException.Type;
+    }
 }

src/modules/Elsa.OpenTelemetry/Metrics/ErrorMetrics.cs

@@ -0,0 +1,59 @@
+using System.Diagnostics.Metrics;
+using Elsa.OpenTelemetry.Contracts;
+using Elsa.OpenTelemetry.Models;
+using Elsa.Workflows;
+
+namespace Elsa.OpenTelemetry.Metrics;
+
+/// Tracks and records metrics related to workflow incidents.
+public class ErrorMetrics
+{
+    private readonly IEnumerable<IErrorMetricHandler> _errorMetricHandlers;
+    private readonly Counter<long> _errorCounter;
+
+    /// Tracks and records metrics related to workflow incidents.
+    public ErrorMetrics(IMeterFactory meterFactory, IEnumerable<IErrorMetricHandler> errorMetricHandlers)
+    {
+        var meter = meterFactory.Create(MeterName);
+        _errorMetricHandlers = errorMetricHandlers;
+        _errorCounter = meter.CreateCounter<long>(
+            name: "workflow_incident_count",
+            description: "Counts workflow incidents"
+        );
+    }
+
+    public const string MeterName = "Elsa.OpenTelemetry.Incidents";
+
+    /// Track 
+    public void TrackError(ActivityExecutionContext context)
+    {
+        var exception = context.Exception;
+
+        if (exception == null)
+            return;
+
+        var workflow = context.WorkflowExecutionContext.Workflow;
+        var tags = new Dictionary<string, object?>()
+        {
+            ["activity.type"] = context.Activity.Type,
+            ["workflow.definition.id"] = workflow.Identity.DefinitionId,
+            ["workflow.definition.version"] = workflow.Identity.Version
+        };
+
+        if (!string.IsNullOrWhiteSpace(workflow.WorkflowMetadata.Name))
+            tags["workflow.definition.name"] = workflow.WorkflowMetadata.Name;
+
+        if (!string.IsNullOrWhiteSpace(workflow.Identity.TenantId))
+            tags["tenant.id"] = workflow.Identity.TenantId;
+
+        var errorMetricContext = new ErrorMetricContext(_errorCounter, exception, tags);
+        var errorMetricHandlers = _errorMetricHandlers
+            .OrderBy(x => x.Order)
+            .Where(x => x.CanHandle(errorMetricContext));
+
+        foreach (var handler in errorMetricHandlers) 
+            handler.Handle(errorMetricContext);
+
+        _errorCounter.Add(1, tags.ToArray());
+    }
+}

src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs

@@ -3,6 +3,7 @@
 using Elsa.Extensions;
 using Elsa.OpenTelemetry.Contracts;
 using Elsa.OpenTelemetry.Helpers;
+using Elsa.OpenTelemetry.Metrics;
 using Elsa.OpenTelemetry.Models;
 using Elsa.Workflows;
 using Elsa.Workflows.Pipelines.ActivityExecution;
@@ -14,7 +15,7 @@ namespace Elsa.OpenTelemetry.Middleware;
 
 /// <inheritdoc />
 [UsedImplicitly]
-public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareDelegate next, ISystemClock systemClock) : IActivityExecutionMiddleware
+public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareDelegate next, ISystemClock systemClock, ErrorMetrics errorMetrics) : IActivityExecutionMiddleware
 {
     /// <inheritdoc />
     public async ValueTask InvokeAsync(ActivityExecutionContext context)
@@ -29,7 +30,7 @@ public async ValueTask InvokeAsync(ActivityExecutionContext context)
         }
 
         span.SetTag("operation.name", "elsa.activity.execution");
-        span.SetTag("activity.id", activity.NodeId);
+        span.SetTag("activity.id", activity.Id);
         span.SetTag("activity.node.id", activity.NodeId);
         span.SetTag("activity.type", activity.Type);
         span.SetTag("activity.name", activity.Name);
@@ -56,6 +57,7 @@ public async ValueTask InvokeAsync(ActivityExecutionContext context)
                 .FirstOrDefault(x => x.CanHandle(errorSpanHandlerContext));
 
             errorSpanHandler?.Handle(errorSpanHandlerContext);
+            errorMetrics.TrackError(context);
         }
         else if (context.Status == ActivityStatus.Canceled)
         {

src/modules/Elsa.OpenTelemetry/Models/ErrorMetricContext.cs

@@ -0,0 +1,10 @@
+using System.Diagnostics.Metrics;
+
+namespace Elsa.OpenTelemetry.Models;
+
+public class ErrorMetricContext(Counter<long> errorCounter, Exception exception, IDictionary<string, object?> tags)
+{
+    public Counter<long> ErrorCounter { get; } = errorCounter;
+    public Exception Exception { get; } = exception;
+    public IDictionary<string, object?> Tags { get; } = tags;
+}