[Instrumentation.Hangfire] Add metrics support

Implements OpenTelemetry workflow semantic conventions for Hangfire metrics.

Fixes #2075

Author: gorbach

src/OpenTelemetry.Instrumentation.Hangfire/.publicApi/PublicAPI.Unshipped.txt

@@ -1,3 +1,4 @@
+OpenTelemetry.Metrics.MeterProviderBuilderExtensions
 OpenTelemetry.Trace.HangfireInstrumentationOptions
 OpenTelemetry.Trace.HangfireInstrumentationOptions.DisplayNameFunc.get -> System.Func<Hangfire.BackgroundJob!, string!>!
 OpenTelemetry.Trace.HangfireInstrumentationOptions.DisplayNameFunc.set -> void
@@ -6,7 +7,12 @@ OpenTelemetry.Trace.HangfireInstrumentationOptions.Filter.set -> void
 OpenTelemetry.Trace.HangfireInstrumentationOptions.HangfireInstrumentationOptions() -> void
 OpenTelemetry.Trace.HangfireInstrumentationOptions.RecordException.get -> bool
 OpenTelemetry.Trace.HangfireInstrumentationOptions.RecordException.set -> void
+OpenTelemetry.Trace.HangfireInstrumentationOptions.RecordQueueLatency.get -> bool
+OpenTelemetry.Trace.HangfireInstrumentationOptions.RecordQueueLatency.set -> void
 OpenTelemetry.Trace.TracerProviderBuilderExtensions
+static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Metrics.MeterProviderBuilder! builder) -> OpenTelemetry.Metrics.MeterProviderBuilder!
+static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Metrics.MeterProviderBuilder! builder, string? name, System.Action<OpenTelemetry.Trace.HangfireInstrumentationOptions!>? configure) -> OpenTelemetry.Metrics.MeterProviderBuilder!
+static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Metrics.MeterProviderBuilder! builder, System.Action<OpenTelemetry.Trace.HangfireInstrumentationOptions!>? configure) -> OpenTelemetry.Metrics.MeterProviderBuilder!
 static OpenTelemetry.Trace.TracerProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Trace.TracerProviderBuilder! builder) -> OpenTelemetry.Trace.TracerProviderBuilder!
 static OpenTelemetry.Trace.TracerProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Trace.TracerProviderBuilder! builder, string? name, System.Action<OpenTelemetry.Trace.HangfireInstrumentationOptions!>? configure) -> OpenTelemetry.Trace.TracerProviderBuilder!
 static OpenTelemetry.Trace.TracerProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Trace.TracerProviderBuilder! builder, System.Action<OpenTelemetry.Trace.HangfireInstrumentationOptions!>? configure) -> OpenTelemetry.Trace.TracerProviderBuilder!

src/OpenTelemetry.Instrumentation.Hangfire/HangfireInstrumentationOptions.cs

@@ -45,4 +45,14 @@ public class HangfireInstrumentationOptions
     /// </list>
     /// </remarks>
     public Func<BackgroundJob, bool>? Filter { get; set; }
+
+    /// <summary>
+    /// Gets or sets a value indicating whether to record queue latency metrics.
+    /// </summary>
+    /// <remarks>
+    /// When enabled, records the time jobs spend waiting in the queue before execution.
+    /// This requires an additional database call per job execution to retrieve the enqueue timestamp.
+    /// Default is <see langword="false"/> to avoid performance impact in high-throughput scenarios.
+    /// </remarks>
+    public bool RecordQueueLatency { get; set; }
 }

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetrics.cs

@@ -0,0 +1,72 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using System.Diagnostics.Metrics;
+using System.Reflection;
+using OpenTelemetry.Internal;
+
+namespace OpenTelemetry.Instrumentation.Hangfire.Implementation;
+
+/// <summary>
+/// Centralized metrics definitions for Hangfire instrumentation.
+/// </summary>
+internal static class HangfireMetrics
+{
+    private static readonly Assembly Assembly = typeof(HangfireMetrics).Assembly;
+    private static readonly AssemblyName AssemblyName = Assembly.GetName();
+    internal static readonly string MeterName = AssemblyName.Name!;
+    private static readonly string InstrumentationVersion = Assembly.GetPackageVersion();
+
+    // Metric name constants
+    internal const string ExecutionCountMetricName = "workflow.execution.count";
+    internal const string ExecutionDurationMetricName = "workflow.execution.duration";
+    internal const string ExecutionStatusMetricName = "workflow.execution.status";
+    internal const string ExecutionErrorsMetricName = "workflow.execution.errors";
+
+    internal const string QueueLatencyMetricName = "hangfire.queue.latency";
+
+    /// <summary>
+    /// The meter instance for all Hangfire metrics.
+    /// </summary>
+    public static readonly Meter Meter = new(MeterName, InstrumentationVersion);
+
+    /// <summary>
+    /// Counter for the number of task executions which have been initiated.
+    /// Follows OpenTelemetry workflow semantic conventions.
+    /// </summary>
+    public static readonly Counter<long> ExecutionCount =
+        Meter.CreateCounter<long>(ExecutionCountMetricName, unit: "{executions}",
+            description: "The number of task executions which have been initiated.");
+
+    /// <summary>
+    /// Histogram for duration of an execution grouped by task, type and result.
+    /// Follows OpenTelemetry workflow semantic conventions.
+    /// </summary>
+    public static readonly Histogram<double> ExecutionDuration =
+        Meter.CreateHistogram<double>(ExecutionDurationMetricName, unit: "s",
+            description: "Duration of an execution grouped by task, type and result.");
+
+    /// <summary>
+    /// UpDownCounter for the number of actively running tasks grouped by task and state.
+    /// Follows OpenTelemetry workflow semantic conventions.
+    /// </summary>
+    public static readonly UpDownCounter<long> ExecutionStatus =
+        Meter.CreateUpDownCounter<long>(ExecutionStatusMetricName, unit: "{executions}",
+            description: "The number of actively running tasks grouped by task, type and the current state.");
+
+    /// <summary>
+    /// Counter for the number of errors encountered in task runs (eg. compile, test failures).
+    /// Follows OpenTelemetry workflow semantic conventions.
+    /// </summary>
+    public static readonly Counter<long> ExecutionErrors =
+        Meter.CreateCounter<long>(ExecutionErrorsMetricName, unit: "{error}",
+            description: "The number of errors encountered in task runs (eg. compile, test failures).");
+
+    /// <summary>
+    /// Histogram for time tasks spend waiting in queue before execution.
+    /// Hangfire-specific metric (not part of standard workflow conventions).
+    /// </summary>
+    public static readonly Histogram<double> QueueLatency =
+        Meter.CreateHistogram<double>(QueueLatencyMetricName, unit: "s",
+            description: "Time tasks spend waiting in queue before execution starts.");
+}

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsErrorFilterAttribute.cs

@@ -0,0 +1,30 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using Hangfire.Common;
+using Hangfire.Server;
+
+namespace OpenTelemetry.Instrumentation.Hangfire.Implementation;
+
+/// <summary>
+/// Hangfire filter that records OpenTelemetry error metrics for job execution failures.
+/// Follows OpenTelemetry workflow semantic conventions for workflow.execution.errors metric.
+/// </summary>
+internal sealed class HangfireMetricsErrorFilterAttribute : JobFilterAttribute, IServerFilter
+{
+    public void OnPerforming(PerformingContext performingContext)
+    {
+    }
+
+    public void OnPerformed(PerformedContext performedContext)
+    {
+        if (performedContext.Exception != null)
+        {
+            var errorTags = HangfireTagBuilder.BuildErrorTags(
+                performedContext.BackgroundJob.Job,
+                performedContext.Exception);
+
+            HangfireMetrics.ExecutionErrors.Add(1, errorTags);
+        }
+    }
+}

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsInstrumentation.cs

@@ -0,0 +1,42 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using Hangfire;
+using OpenTelemetry.Trace;
+
+namespace OpenTelemetry.Instrumentation.Hangfire.Implementation;
+
+/// <summary>
+/// Hangfire metrics instrumentation following OpenTelemetry workflow semantic conventions.
+/// </summary>
+internal sealed class HangfireMetricsInstrumentation : IDisposable
+{
+    private readonly List<object> filters = new();
+
+    public HangfireMetricsInstrumentation(HangfireInstrumentationOptions options)
+    {
+        this.AddFilter(new HangfireMetricsJobFilterAttribute());
+        this.AddFilter(new HangfireMetricsStateFilter());
+        this.AddFilter(new HangfireMetricsErrorFilterAttribute());
+
+        // Only register queue latency filter if enabled (requires DB call per job)
+        if (options.RecordQueueLatency)
+        {
+            this.AddFilter(new HangfireQueueLatencyFilterAttribute());
+        }
+    }
+
+    public void Dispose()
+    {
+        foreach (var filter in this.filters)
+        {
+            GlobalJobFilters.Filters.Remove(filter);
+        }
+    }
+
+    private void AddFilter(object filter)
+    {
+        this.filters.Add(filter);
+        GlobalJobFilters.Filters.Add(filter);
+    }
+}

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsJobFilterAttribute.cs

@@ -0,0 +1,36 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using System.Diagnostics;
+using Hangfire.Common;
+using Hangfire.Server;
+
+namespace OpenTelemetry.Instrumentation.Hangfire.Implementation;
+
+/// <summary>
+/// Hangfire filter that records OpenTelemetry metrics for job execution.
+/// </summary>
+internal sealed class HangfireMetricsJobFilterAttribute : JobFilterAttribute, IServerFilter
+{
+    private const string StopwatchKey = "OpenTelemetry.Metrics.Stopwatch";
+
+    public void OnPerforming(PerformingContext performingContext)
+    {
+        performingContext.Items[StopwatchKey] = Stopwatch.StartNew();
+    }
+
+    public void OnPerformed(PerformedContext performedContext)
+    {
+        var executionTags = HangfireTagBuilder.BuildExecutionTags(performedContext.BackgroundJob.Job, performedContext.Exception);
+
+        HangfireMetrics.ExecutionCount.Add(1, executionTags);
+
+        if (performedContext.Items.TryGetValue(StopwatchKey, out var stopwatchObj) && stopwatchObj is Stopwatch stopwatch)
+        {
+            stopwatch.Stop();
+            var duration = stopwatch.Elapsed.TotalSeconds;
+
+            HangfireMetrics.ExecutionDuration.Record(duration, executionTags);
+        }
+    }
+}

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsStateFilter.cs

@@ -0,0 +1,107 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using Hangfire.Common;
+using Hangfire.States;
+using Hangfire.Storage;
+
+namespace OpenTelemetry.Instrumentation.Hangfire.Implementation;
+
+/// <summary>
+/// Hangfire state change filter responsible for emitting workflow execution status metrics.
+/// </summary>
+internal sealed class HangfireMetricsStateFilter : JobFilterAttribute, IApplyStateFilter
+{
+    public void OnStateApplied(ApplyStateContext context, IWriteOnlyTransaction transaction)
+    {
+        var workflowState = HangfireTagBuilder.MapWorkflowState(context.NewState.Name);
+        if (workflowState == null)
+        {
+            return;
+        }
+
+        var errorType = GetErrorTypeFromNewState(context.NewState);
+        var recurringJobId = GetRecurringJobId(context);
+        var tags = HangfireTagBuilder.BuildStateTags(
+            context.BackgroundJob.Job,
+            workflowState,
+            errorType,
+            recurringJobId);
+
+        HangfireMetrics.ExecutionStatus.Add(1, tags);
+    }
+
+    public void OnStateUnapplied(ApplyStateContext context, IWriteOnlyTransaction transaction)
+    {
+        var workflowState = HangfireTagBuilder.MapWorkflowState(context.OldStateName);
+        if (workflowState == null)
+        {
+            return;
+        }
+
+        var errorType = GetErrorTypeFromOldState(context);
+        var recurringJobId = GetRecurringJobId(context);
+        var tags = HangfireTagBuilder.BuildStateTags(
+            context.BackgroundJob.Job,
+            workflowState,
+            errorType,
+            recurringJobId);
+
+        HangfireMetrics.ExecutionStatus.Add(-1, tags);
+    }
+
+    private static string? GetErrorTypeFromNewState(IState state)
+    {
+        if (!string.Equals(state.Name, FailedState.StateName, StringComparison.Ordinal))
+        {
+            return null;
+        }
+
+        if (state is FailedState failedState && failedState.Exception != null)
+        {
+            var exceptionType = failedState.Exception.GetType();
+            return exceptionType.FullName ?? exceptionType.Name;
+        }
+
+        return TryGetExceptionTypeFromSerializedData(state.SerializeData());
+    }
+
+    private static string? GetErrorTypeFromOldState(ApplyStateContext context)
+    {
+        if (!string.Equals(context.OldStateName, FailedState.StateName, StringComparison.Ordinal))
+        {
+            return null;
+        }
+
+        StateData? stateData = context.Connection.GetStateData(context.BackgroundJob.Id);
+        return stateData != null ? TryGetExceptionTypeFromSerializedData(stateData.Data) : null;
+    }
+
+    private static string? TryGetExceptionTypeFromSerializedData(IDictionary<string, string>? data)
+    {
+        if (data == null)
+        {
+            return null;
+        }
+
+        if (data.TryGetValue("ExceptionType", out var exceptionType) && !string.IsNullOrWhiteSpace(exceptionType))
+        {
+            return exceptionType;
+        }
+
+        return null;
+    }
+
+    private static string? GetRecurringJobId(ApplyStateContext context)
+    {
+        try
+        {
+            return context.Connection.GetJobParameter(context.BackgroundJob.Id, "RecurringJobId");
+        }
+        catch
+        {
+            // Parameter doesn't exist or couldn't be retrieved
+            return null;
+        }
+    }
+}

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireQueueLatencyFilterAttribute.cs

@@ -0,0 +1,71 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using Hangfire.Common;
+using Hangfire.Server;
+using Hangfire.States;
+using DateTime = System.DateTime;
+
+namespace OpenTelemetry.Instrumentation.Hangfire.Implementation;
+
+/// <summary>
+/// Hangfire filter that records queue latency metrics.
+/// </summary>
+/// <remarks>
+/// This filter captures the EnqueuedAt timestamp when a job enters the Enqueued state
+/// and calculates queue latency when the job starts executing.
+/// </remarks>
+internal sealed class HangfireQueueLatencyFilterAttribute : JobFilterAttribute, IServerFilter, IElectStateFilter
+{
+    private const string EnqueuedAtParameter = "OpenTelemetry.EnqueuedAt";
+
+    public void OnStateElection(ElectStateContext context)
+    {
+        // When a job transitions to Enqueued state, capture the EnqueuedAt timestamp
+        if (context.CandidateState is EnqueuedState enqueuedState)
+        {
+            try
+            {
+                var enqueuedAt = enqueuedState.EnqueuedAt;
+                context.Connection.SetJobParameter(
+                    context.BackgroundJob.Id,
+                    EnqueuedAtParameter,
+                    JobHelper.SerializeDateTime(enqueuedAt));
+            }
+            catch
+            {
+                // Skip storing timestamp if parameter write fails
+                // Instrumentation must never break Hangfire's scheduling pipeline
+            }
+        }
+    }
+
+    public void OnPerforming(PerformingContext performingContext)
+    {
+        try
+        {
+            // Retrieve the EnqueuedAt timestamp that was stored when the job was enqueued
+            var enqueuedAtStr = performingContext.Connection.GetJobParameter(
+                performingContext.BackgroundJob.Id,
+                EnqueuedAtParameter);
+
+            if (!string.IsNullOrEmpty(enqueuedAtStr))
+            {
+                var enqueuedAt = JobHelper.DeserializeDateTime(enqueuedAtStr);
+                var queueLatency = (DateTime.UtcNow - enqueuedAt).TotalSeconds;
+
+                var tags = HangfireTagBuilder.BuildCommonTags(performingContext.BackgroundJob.Job);
+                HangfireMetrics.QueueLatency.Record(queueLatency, tags);
+            }
+        }
+        catch
+        {
+            // Skip recording if parameter retrieval fails
+        }
+    }
+
+    public void OnPerformed(PerformedContext performedContext)
+    {
+        // No-op: This filter only handles queue latency in OnPerforming
+    }
+}