[Instrumentation.Hangfire] Add metrics support

src/OpenTelemetry.Instrumentation.Hangfire/.publicApi/PublicAPI.Unshipped.txt

@@ -1,3 +1,4 @@
+OpenTelemetry.Metrics.MeterProviderBuilderExtensions
 OpenTelemetry.Trace.HangfireInstrumentationOptions
 OpenTelemetry.Trace.HangfireInstrumentationOptions.DisplayNameFunc.get -> System.Func<Hangfire.BackgroundJob!, string!>!
 OpenTelemetry.Trace.HangfireInstrumentationOptions.DisplayNameFunc.set -> void
@@ -7,6 +8,15 @@ OpenTelemetry.Trace.HangfireInstrumentationOptions.HangfireInstrumentationOption
 OpenTelemetry.Trace.HangfireInstrumentationOptions.RecordException.get -> bool
 OpenTelemetry.Trace.HangfireInstrumentationOptions.RecordException.set -> void
 OpenTelemetry.Trace.TracerProviderBuilderExtensions
+static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Metrics.MeterProviderBuilder! builder) -> OpenTelemetry.Metrics.MeterProviderBuilder!
+static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Metrics.MeterProviderBuilder! builder, string? name, System.Action<OpenTelemetry.Metrics.HangfireMetricsInstrumentationOptions!>? configure) -> OpenTelemetry.Metrics.MeterProviderBuilder!
+static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Metrics.MeterProviderBuilder! builder, System.Action<OpenTelemetry.Metrics.HangfireMetricsInstrumentationOptions!>? configure) -> OpenTelemetry.Metrics.MeterProviderBuilder!
+OpenTelemetry.Metrics.HangfireMetricsInstrumentationOptions
+OpenTelemetry.Metrics.HangfireMetricsInstrumentationOptions.DisplayNameFunc.get -> System.Func<Hangfire.BackgroundJob!, string!>!
+OpenTelemetry.Metrics.HangfireMetricsInstrumentationOptions.DisplayNameFunc.set -> void
+OpenTelemetry.Metrics.HangfireMetricsInstrumentationOptions.HangfireMetricsInstrumentationOptions() -> void
+OpenTelemetry.Metrics.HangfireMetricsInstrumentationOptions.RecordQueueLatency.get -> bool
+OpenTelemetry.Metrics.HangfireMetricsInstrumentationOptions.RecordQueueLatency.set -> void
 static OpenTelemetry.Trace.TracerProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Trace.TracerProviderBuilder! builder) -> OpenTelemetry.Trace.TracerProviderBuilder!
 static OpenTelemetry.Trace.TracerProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Trace.TracerProviderBuilder! builder, string? name, System.Action<OpenTelemetry.Trace.HangfireInstrumentationOptions!>? configure) -> OpenTelemetry.Trace.TracerProviderBuilder!
 static OpenTelemetry.Trace.TracerProviderBuilderExtensions.AddHangfireInstrumentation(this OpenTelemetry.Trace.TracerProviderBuilder! builder, System.Action<OpenTelemetry.Trace.HangfireInstrumentationOptions!>? configure) -> OpenTelemetry.Trace.TracerProviderBuilder!

src/OpenTelemetry.Instrumentation.Hangfire/HangfireMetricsInstrumentationOptions.cs

@@ -0,0 +1,24 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using Hangfire;
+
+namespace OpenTelemetry.Metrics;
+
+/// <summary>
+/// Options for Hangfire metrics instrumentation.
+/// </summary>
+public sealed class HangfireMetricsInstrumentationOptions
+{
+    /// <summary>
+    /// Gets or sets a value indicating whether to record the pending state duration in metrics.
+    /// </summary>
+    /// <remarks>
+    /// When enabled, records workflow.execution.duration with state="pending", representing
+    /// the time jobs spend waiting in the queue before execution starts.
+    /// This requires an additional database call per job execution to retrieve the enqueue timestamp.
+    /// Default is <see langword="false"/> to avoid performance impact in high-throughput scenarios.
+    /// When disabled, only execution duration (state="executing") is recorded.
+    /// </remarks>
+    public bool RecordQueueLatency { get; set; }
+}

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetrics.cs

@@ -0,0 +1,90 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using System.Diagnostics.Metrics;
+using System.Reflection;
+using OpenTelemetry.Internal;
+
+namespace OpenTelemetry.Instrumentation.Hangfire.Implementation;
+
+/// <summary>
+/// Centralized metrics definitions for Hangfire instrumentation.
+/// </summary>
+internal static class HangfireMetrics
+{
+    internal static readonly Assembly Assembly = typeof(HangfireMetrics).Assembly;
+    internal static readonly AssemblyName AssemblyName = Assembly.GetName();
+    internal static readonly string MeterName = AssemblyName.Name!;
+
+    private static readonly string InstrumentationVersion = Assembly.GetPackageVersion();
+
+    /// <summary>
+    /// The meter instance for all Hangfire metrics.
+    /// </summary>
+    private static readonly Meter Meter = new(MeterName, InstrumentationVersion);
+
+    /// <summary>
+    /// Counter for the number of task executions which have been initiated.
+    /// Follows OpenTelemetry workflow semantic conventions.
+    /// </summary>
+    public static readonly Counter<long> ExecutionCount =
+        Meter.CreateCounter<long>(
+            WorkflowMetricNames.ExecutionCount,
+            unit: "{executions}",
+            description: "The number of task executions which have been initiated.");
+
+    /// <summary>
+    /// Histogram for duration of an execution grouped by task, type and result.
+    /// Follows OpenTelemetry workflow semantic conventions.
+    /// Records duration for different execution phases using workflow.execution.state attribute:
+    /// - state=pending: Time spent waiting in queue before execution.
+    /// - state=executing: Time spent in actual execution.
+    /// </summary>
+    public static readonly Histogram<double> ExecutionDuration =
+        Meter.CreateHistogram<double>(
+            WorkflowMetricNames.ExecutionDuration,
+            unit: "s",
+            description: "Duration of an execution grouped by task, type and result.");
+
+    /// <summary>
+    /// UpDownCounter for the number of actively running tasks grouped by task and state.
+    /// Follows OpenTelemetry workflow semantic conventions.
+    /// </summary>
+    public static readonly UpDownCounter<long> ExecutionStatus =
+        Meter.CreateUpDownCounter<long>(
+            WorkflowMetricNames.ExecutionStatus,
+            unit: "{executions}",
+            description: "The number of actively running tasks grouped by task, type and the current state.");
+
+    /// <summary>
+    /// Counter for the number of errors encountered in task runs (eg. compile, test failures).
+    /// Follows OpenTelemetry workflow semantic conventions.
+    /// </summary>
+    public static readonly Counter<long> ExecutionErrors =
+        Meter.CreateCounter<long>(
+            WorkflowMetricNames.ExecutionErrors,
+            unit: "{error}",
+            description: "The number of errors encountered in task runs (eg. compile, test failures).");
+
+    /// <summary>
+    /// Counter for the number of workflow instances which have been initiated.
+    /// Follows OpenTelemetry workflow semantic conventions.
+    /// In Hangfire, this tracks individual job completions. For batch workflows, this would track batch completion.
+    /// </summary>
+    public static readonly Counter<long> WorkflowCount =
+        Meter.CreateCounter<long>(
+            WorkflowMetricNames.WorkflowCount,
+            unit: "{workflows}",
+            description: "The number of workflow instances which have been initiated.");
+
+    /// <summary>
+    /// UpDownCounter for the number of actively running workflows grouped by definition and state.
+    /// Follows OpenTelemetry workflow semantic conventions.
+    /// In Hangfire, this tracks workflows that haven't entered the execution pipeline yet (e.g., scheduled jobs).
+    /// </summary>
+    public static readonly UpDownCounter<long> WorkflowStatus =
+        Meter.CreateUpDownCounter<long>(
+            WorkflowMetricNames.WorkflowStatus,
+            unit: "{workflows}",
+            description: "The number of actively running workflows grouped by definition and the current state.");
+}

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsErrorFilterAttribute.cs

@@ -0,0 +1,40 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using Hangfire.Common;
+using Hangfire.Server;
+using OpenTelemetry.Metrics;
+
+namespace OpenTelemetry.Instrumentation.Hangfire.Implementation;
+
+/// <summary>
+/// Hangfire filter that records OpenTelemetry error metrics for job execution failures.
+/// Follows OpenTelemetry workflow semantic conventions for workflow.execution.errors metric.
+/// </summary>
+internal sealed class HangfireMetricsErrorFilterAttribute : JobFilterAttribute, IServerFilter
+{
+    private readonly HangfireMetricsInstrumentationOptions options;
+
+#pragma warning disable CA1019 // Define accessors for attribute arguments
+    public HangfireMetricsErrorFilterAttribute(HangfireMetricsInstrumentationOptions options)
+#pragma warning restore CA1019 // Define accessors for attribute arguments
+    {
+        this.options = options ?? throw new ArgumentNullException(nameof(options));
+    }
+
+    public void OnPerforming(PerformingContext performingContext)
+    {
+    }
+
+    public void OnPerformed(PerformedContext performedContext)
+    {
+        if (performedContext.Exception != null)
+        {
+            var errorTags = HangfireTagBuilder.BuildErrorTags(
+                performedContext.BackgroundJob,
+                performedContext.Exception);
+
+            HangfireMetrics.ExecutionErrors.Add(1, errorTags);
+        }
+    }
+}

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsInstrumentation.cs

@@ -0,0 +1,42 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using Hangfire;
+using OpenTelemetry.Metrics;
+
+namespace OpenTelemetry.Instrumentation.Hangfire.Implementation;
+
+/// <summary>
+/// Hangfire metrics instrumentation following OpenTelemetry workflow semantic conventions.
+/// </summary>
+internal sealed class HangfireMetricsInstrumentation : IDisposable
+{
+    private readonly List<object> filters = new();
+
+    public HangfireMetricsInstrumentation(HangfireMetricsInstrumentationOptions options)
+    {
+        this.AddFilter(new HangfireMetricsJobFilterAttribute());
+        this.AddFilter(new HangfireMetricsStateFilter());
+        this.AddFilter(new HangfireMetricsErrorFilterAttribute(options));
+
+        // Only register pending duration filter if enabled (requires DB call per job)
+        if (options.RecordQueueLatency)
+        {
+            this.AddFilter(new HangfirePendingDurationFilterAttribute());
+        }
+    }
+
+    public void Dispose()
+    {
+        foreach (var filter in this.filters)
+        {
+            GlobalJobFilters.Filters.Remove(filter);
+        }
+    }
+
+    private void AddFilter(object filter)
+    {
+        this.filters.Add(filter);
+        GlobalJobFilters.Filters.Add(filter);
+    }
+}

src/OpenTelemetry.Instrumentation.Hangfire/Implementation/HangfireMetricsJobFilterAttribute.cs

@@ -0,0 +1,68 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using System.Diagnostics;
+using Hangfire.Common;
+using Hangfire.Server;
+
+namespace OpenTelemetry.Instrumentation.Hangfire.Implementation;
+
+/// <summary>
+/// Hangfire filter that records OpenTelemetry metrics for job execution.
+/// </summary>
+internal sealed class HangfireMetricsJobFilterAttribute : JobFilterAttribute, IServerFilter
+{
+    private const string StopwatchKey = "OpenTelemetry.Metrics.Stopwatch";
+
+    public void OnPerforming(PerformingContext performingContext)
+    {
+        performingContext.Items[StopwatchKey] = Stopwatch.StartNew();
+    }
+
+    public void OnPerformed(PerformedContext performedContext)
+    {
+        // Get recurring job ID if this job was triggered by a recurring job
+        string? recurringJobId = null;
+        try
+        {
+            recurringJobId = performedContext.Connection.GetJobParameter(
+                performedContext.BackgroundJob.Id,
+                "RecurringJobId");
+        }
+        catch
+        {
+            // If we can't get the recurring job ID, treat it as a non-recurring job
+        }
+
+        var backgroundJob = performedContext.BackgroundJob;
+
+        // Record execution count (without state attribute per semantic conventions)
+        var countTags = HangfireTagBuilder.BuildExecutionCountTags(
+            backgroundJob,
+            performedContext.Exception);
+
+        HangfireMetrics.ExecutionCount.Add(1, countTags);
+
+        // Record execution duration (with state="executing" to differentiate from pending phase)
+        if (performedContext.Items.TryGetValue(StopwatchKey, out var stopwatchObj) && stopwatchObj is Stopwatch stopwatch)
+        {
+            stopwatch.Stop();
+            var duration = stopwatch.Elapsed.TotalSeconds;
+
+            var durationTags = HangfireTagBuilder.BuildExecutionTags(
+                backgroundJob,
+                performedContext.Exception,
+                workflowState: WorkflowAttributes.WorkflowStateValues.Executing);
+
+            HangfireMetrics.ExecutionDuration.Record(duration, durationTags);
+        }
+
+        // Record workflow-level metrics (includes trigger type)
+        var workflowTags = HangfireTagBuilder.BuildWorkflowTags(
+            backgroundJob,
+            performedContext.Exception,
+            recurringJobId);
+
+        HangfireMetrics.WorkflowCount.Add(1, workflowTags);
+    }
+}