Add error metrics tracking for workflow incidents

Elsa.sln

@@ -84,6 +84,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docker", "docker", "{986E54
 		docker\otel-collector-config.yaml = docker\otel-collector-config.yaml
 		docker\init-db-postgres.sh = docker\init-db-postgres.sh
 		docker\docker-compose-datadog+otel-collector.yml = docker\docker-compose-datadog+otel-collector.yml
+		docker\docker-compose-datadog.yml = docker\docker-compose-datadog.yml
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elsa.Elasticsearch", "src\modules\Elsa.Elasticsearch\Elsa.Elasticsearch.csproj", "{3246883E-2FA7-4B4A-BDC5-99039A2869BC}"

src/apps/Elsa.Server.Web/Program.cs

@@ -29,6 +29,7 @@
 using Elsa.MongoDb.Modules.Management;
 using Elsa.MongoDb.Modules.Runtime;
 using Elsa.MongoDb.Modules.Tenants;
+using Elsa.OpenTelemetry.Metrics;
 using Elsa.OpenTelemetry.Middleware;
 using Elsa.Retention.Extensions;
 using Elsa.Retention.Models;
@@ -105,7 +106,7 @@
 const bool useSecrets = false;
 const bool disableVariableWrappers = false;
 const bool disableVariableCopying = false;
-const bool useManualOtelInstrumentation = false;
+const bool useManualOtelInstrumentation = true;
 
 ObjectConverter.StrictMode = false;
 
@@ -137,7 +138,7 @@
 if (useManualOtelInstrumentation)
 {
     services.AddOpenTelemetry()
-        .ConfigureResource(resource => resource.AddService("elsa-workflows", serviceVersion: "3.4.0").AddTelemetrySdk())
+        .ConfigureResource(resource => resource.AddService("elsa-workflows", serviceVersion: "3.5.0").AddTelemetrySdk())
         .WithTracing(tracing =>
         {
             tracing
@@ -146,15 +147,16 @@
                 .AddAspNetCoreInstrumentation()
                 .AddHttpClientInstrumentation()
                 .AddSqlClientInstrumentation()
-                .AddConsoleExporter()
+                //.AddConsoleExporter()
                 .AddOtlpExporter()
                 ;
         })
         .WithMetrics(metrics =>
         {
             metrics
-                .AddAspNetCoreInstrumentation()
-                .AddHttpClientInstrumentation()
+                .AddMeter(ErrorMetrics.MeterName)
+                //.AddAspNetCoreInstrumentation()
+                //.AddHttpClientInstrumentation()
                 .AddConsoleExporter()
                 .AddOtlpExporter()
                 ;
@@ -536,7 +538,7 @@
                     alterations.UseMassTransitDispatcher();
                 }
             })
-            .UseOpenTelemetry()
+            .UseOpenTelemetry(otel => otel.UseNewRootActivityForRemoteParent = true)
             .UseWorkflowContexts();
 
         if (useQuartz)

src/modules/Elsa.OpenTelemetry/Abstractions/ActivityErrorSpanHandlerBase.cs

@@ -0,0 +1,11 @@
+using Elsa.OpenTelemetry.Contracts;
+using Elsa.OpenTelemetry.Models;
+
+namespace Elsa.OpenTelemetry.Abstractions;
+
+public abstract class ActivityErrorSpanHandlerBase : IActivityErrorSpanHandler
+{
+    public virtual float Order => 0;
+    public abstract bool CanHandle(ActivityErrorSpanContext context);
+    public abstract void Handle(ActivityErrorSpanContext context);
+}

src/modules/Elsa.OpenTelemetry/Abstractions/WorkflowErrorSpanHandlerBase.cs

@@ -0,0 +1,11 @@
+using Elsa.OpenTelemetry.Contracts;
+using Elsa.OpenTelemetry.Models;
+
+namespace Elsa.OpenTelemetry.Abstractions;
+
+public abstract class WorkflowErrorSpanHandlerBase : IWorkflowErrorSpanHandler
+{
+    public virtual float Order => 0;
+    public abstract bool CanHandle(WorkflowErrorSpanContext context);
+    public abstract void Handle(WorkflowErrorSpanContext context);
+}

src/modules/Elsa.OpenTelemetry/Contracts/IActivityErrorSpanHandler.cs

@@ -0,0 +1,10 @@
+using Elsa.OpenTelemetry.Models;
+
+namespace Elsa.OpenTelemetry.Contracts;
+
+public interface IActivityErrorSpanHandler
+{
+    float Order { get; }
+    bool CanHandle(ActivityErrorSpanContext context);
+    void Handle(ActivityErrorSpanContext context);
+}

src/modules/Elsa.OpenTelemetry/Contracts/IErrorMetricHandler.cs

@@ -0,0 +1,10 @@
+using Elsa.OpenTelemetry.Models;
+
+namespace Elsa.OpenTelemetry.Contracts;
+
+public interface IErrorMetricHandler
+{
+    float Order { get; }
+    bool CanHandle(ErrorMetricContext context);
+    void Handle(ErrorMetricContext context);
+}

src/modules/Elsa.OpenTelemetry/Contracts/IWorkflowErrorSpanHandler.cs

@@ -0,0 +1,10 @@
+using Elsa.OpenTelemetry.Models;
+
+namespace Elsa.OpenTelemetry.Contracts;
+
+public interface IWorkflowErrorSpanHandler
+{
+    float Order { get; }
+    bool CanHandle(WorkflowErrorSpanContext context);
+    void Handle(WorkflowErrorSpanContext context);
+}

src/modules/Elsa.OpenTelemetry/Features/OpenTelemetryFeature.cs

@@ -2,6 +2,7 @@
 using Elsa.Features.Services;
 using Elsa.OpenTelemetry.Contracts;
 using Elsa.OpenTelemetry.Handlers;
+using Elsa.OpenTelemetry.Metrics;
 using Elsa.OpenTelemetry.Options;
 using Microsoft.Extensions.DependencyInjection;
 
@@ -23,8 +24,13 @@ public class OpenTelemetryFeature(IModule module) : FeatureBase(module)
     public override void Configure()
     {
         Services
-            .AddScoped<IErrorSpanHandler, DefaultErrorSpanHandler>()
-            .AddScoped<IErrorSpanHandler, FaultExceptionErrorSpanHandler>();
+            .AddScoped<IActivityErrorSpanHandler, DefaultExceptionHandler>()
+            .AddScoped<IActivityErrorSpanHandler, FaultExceptionHandler>()
+            .AddScoped<IWorkflowErrorSpanHandler, DefaultExceptionHandler>()
+            .AddScoped<IWorkflowErrorSpanHandler, FaultExceptionHandler>()
+            .AddScoped<IErrorMetricHandler, DefaultExceptionHandler>()
+            .AddScoped<IErrorMetricHandler, FaultExceptionHandler>()
+            .AddScoped<ErrorMetrics>();
 
         Services.Configure<OpenTelemetryOptions>(options =>
         {

src/modules/Elsa.OpenTelemetry/Handlers/DefaultExceptionHandler.cs

@@ -0,0 +1,27 @@
+using Elsa.OpenTelemetry.Contracts;
+using Elsa.OpenTelemetry.Models;
+
+namespace Elsa.OpenTelemetry.Handlers;
+
+public class DefaultExceptionHandler : IActivityErrorSpanHandler, IWorkflowErrorSpanHandler, IErrorMetricHandler
+{
+    public float Order => 100000;
+    public bool CanHandle(ErrorMetricContext context) => true;
+    public bool CanHandle(WorkflowErrorSpanContext context) => true;
+    public bool CanHandle(ActivityErrorSpanContext context) => context.Exception != null;
+
+    public void Handle(WorkflowErrorSpanContext context)
+    {
+        // No-op.
+    }
+
+    public void Handle(ActivityErrorSpanContext context)
+    {
+        context.Span.AddException(context.Exception!);
+    }
+
+    public void Handle(ErrorMetricContext context)
+    {
+        context.Tags["error.exception.type"] = context.Exception.GetType().Name;
+    }
+}

src/modules/Elsa.OpenTelemetry/Handlers/FaultExceptionHandler.cs

@@ -0,0 +1,45 @@
+using Elsa.OpenTelemetry.Contracts;
+using Elsa.OpenTelemetry.Models;
+using Elsa.Workflows.Exceptions;
+
+namespace Elsa.OpenTelemetry.Handlers;
+
+public class FaultExceptionHandler : IActivityErrorSpanHandler, IWorkflowErrorSpanHandler, IErrorMetricHandler
+{
+    public float Order => 0;
+
+    public bool CanHandle(ActivityErrorSpanContext context) => context.Exception is FaultException;
+    public bool CanHandle(WorkflowErrorSpanContext context) => context.Exception is FaultException;
+    public bool CanHandle(ErrorMetricContext context) => context.Exception is FaultException;
+
+    public void Handle(ActivityErrorSpanContext context)
+    {
+        var faultException = (FaultException)context.Exception!;
+        var span = context.Span;
+        var tags = new Dictionary<string, object?>()
+        {
+            ["exception.code"] = faultException.Code,
+            ["exception.category"] = faultException.Category,
+            ["exception.type"] = faultException.Type
+        };
+        span.AddException(faultException, new(tags.ToArray()));
+    }
+
+    public void Handle(WorkflowErrorSpanContext context)
+    {
+        var faultException = (FaultException)context.Exception!;
+        var span = context.Span;
+
+        span.SetTag("error.code", faultException.Code);
+        span.SetTag("error.category", faultException.Category);
+        span.SetTag("error_details.type", faultException.Type);
+    }
+
+    public void Handle(ErrorMetricContext context)
+    {
+        var faultException = (FaultException)context.Exception!;
+        context.Tags["error.code"] = faultException.Code;
+        context.Tags["error.category"] = faultException.Category;
+        context.Tags["error.type"] = faultException.Type;
+    }
+}

src/modules/Elsa.OpenTelemetry/Metrics/ErrorMetrics.cs

@@ -0,0 +1,59 @@
+using System.Diagnostics.Metrics;
+using Elsa.OpenTelemetry.Contracts;
+using Elsa.OpenTelemetry.Models;
+using Elsa.Workflows;
+
+namespace Elsa.OpenTelemetry.Metrics;
+
+/// Tracks and records metrics related to workflow incidents.
+public class ErrorMetrics
+{
+    private readonly IEnumerable<IErrorMetricHandler> _errorMetricHandlers;
+    private readonly Counter<long> _errorCounter;
+
+    /// Tracks and records metrics related to workflow incidents.
+    public ErrorMetrics(IMeterFactory meterFactory, IEnumerable<IErrorMetricHandler> errorMetricHandlers)
+    {
+        var meter = meterFactory.Create(MeterName);
+        _errorMetricHandlers = errorMetricHandlers;
+        _errorCounter = meter.CreateCounter<long>(
+            name: "workflow_incident_count",
+            description: "Counts workflow incidents"
+        );
+    }
+
+    public const string MeterName = "Elsa.OpenTelemetry.Incidents";
+
+    /// Track 
+    public void TrackError(ActivityExecutionContext context)
+    {
+        var exception = context.Exception;
+
+        if (exception == null)
+            return;
+
+        var workflow = context.WorkflowExecutionContext.Workflow;
+        var tags = new Dictionary<string, object?>()
+        {
+            ["activity.type"] = context.Activity.Type,
+            ["workflow.definition.id"] = workflow.Identity.DefinitionId,
+            ["workflow.definition.version"] = workflow.Identity.Version
+        };
+
+        if (!string.IsNullOrWhiteSpace(workflow.WorkflowMetadata.Name))
+            tags["workflow.definition.name"] = workflow.WorkflowMetadata.Name;
+
+        if (!string.IsNullOrWhiteSpace(workflow.Identity.TenantId))
+            tags["tenant.id"] = workflow.Identity.TenantId;
+
+        var errorMetricContext = new ErrorMetricContext(_errorCounter, exception, tags);
+        var errorMetricHandlers = _errorMetricHandlers
+            .OrderBy(x => x.Order)
+            .Where(x => x.CanHandle(errorMetricContext));
+
+        foreach (var handler in errorMetricHandlers) 
+            handler.Handle(errorMetricContext);
+
+        _errorCounter.Add(1, tags.ToArray());
+    }
+}

src/modules/Elsa.OpenTelemetry/Middleware/OpenTelemetryTracingActivityExecutionMiddleware.cs

@@ -3,6 +3,7 @@
 using Elsa.Extensions;
 using Elsa.OpenTelemetry.Contracts;
 using Elsa.OpenTelemetry.Helpers;
+using Elsa.OpenTelemetry.Metrics;
 using Elsa.OpenTelemetry.Models;
 using Elsa.Workflows;
 using Elsa.Workflows.Pipelines.ActivityExecution;
@@ -14,7 +15,7 @@ namespace Elsa.OpenTelemetry.Middleware;
 
 /// <inheritdoc />
 [UsedImplicitly]
-public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareDelegate next, ISystemClock systemClock) : IActivityExecutionMiddleware
+public class OpenTelemetryTracingActivityExecutionMiddleware(ActivityMiddlewareDelegate next, ISystemClock systemClock, ErrorMetrics errorMetrics) : IActivityExecutionMiddleware
 {
     /// <inheritdoc />
     public async ValueTask InvokeAsync(ActivityExecutionContext context)
@@ -29,7 +30,7 @@ public async ValueTask InvokeAsync(ActivityExecutionContext context)
         }
 
         span.SetTag("operation.name", "elsa.activity.execution");
-        span.SetTag("activity.id", activity.NodeId);
+        span.SetTag("activity.id", activity.Id);
         span.SetTag("activity.node.id", activity.NodeId);
         span.SetTag("activity.type", activity.Type);
         span.SetTag("activity.name", activity.Name);
@@ -50,12 +51,14 @@ public async ValueTask InvokeAsync(ActivityExecutionContext context)
             span.AddEvent(new("faulted"));
             span.SetStatus(ActivityStatusCode.Error);
 
-            var errorSpanHandlerContext = new ErrorSpanContext(span, context.Exception);
-            var errorSpanHandler = context.GetServices<IErrorSpanHandler>()
+            var exception = context.Exception;
+            var errorSpanHandlerContext = new ActivityErrorSpanContext(span, context.Exception);
+            var errorSpanHandler = context.GetServices<IActivityErrorSpanHandler>()
                 .OrderBy(x => x.Order)
                 .FirstOrDefault(x => x.CanHandle(errorSpanHandlerContext));
 
             errorSpanHandler?.Handle(errorSpanHandlerContext);
+            errorMetrics.TrackError(context);
         }
         else if (context.Status == ActivityStatus.Canceled)
         {