Add detailed failure attributes to exporter send_failed metrics

Signed-off-by: Israel Blancas <[email protected]>

Author: iblancasa

.chloggen/13956.yaml

@@ -0,0 +1,32 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/otlp)
+component: all
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add detailed failure attributes to exporter send_failed metrics at detailed telemetry level.
+
+# One or more tracking issues or pull requests related to the change
+issues: [13956]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |-
+  The `otelcol_exporter_send_failed_{spans,metric_points,log_records}` metrics now include
+  failure attributes when telemetry level is Detailed: `error.type` (OpenTelemetry semantic convention
+  describing the error class) and `failure.permanent` (indicates if error is permanent).
+  The `error.type` attribute captures gRPC status codes (e.g., "Unavailable", "ResourceExhausted"),
+  HTTP status codes (e.g., "404", "503"), standard Go errors (e.g., "Canceled", "DeadlineExceeded"),
+  and collector-specific errors (e.g., "RetryExhausted", "Shutdown").
+  This enables better alerting and debugging by providing standardized error classification.
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/exporterhelper/documentation.md

@@ -64,23 +64,23 @@ Current size of the retry queue (in batches). [Alpha]
 
 ### otelcol_exporter_send_failed_log_records
 
-Number of log records in failed attempts to send to destination. [Alpha]
+Number of log records in failed attempts to send to destination. At detailed telemetry level, includes attributes: error.type (semantic convention), failure.permanent. [Alpha]
 
 | Unit | Metric Type | Value Type | Monotonic | Stability |
 | ---- | ----------- | ---------- | --------- | --------- |
 | {records} | Sum | Int | true | Alpha |
 
 ### otelcol_exporter_send_failed_metric_points
 
-Number of metric points in failed attempts to send to destination. [Alpha]
+Number of metric points in failed attempts to send to destination. At detailed telemetry level, includes attributes: error.type (semantic convention), failure.permanent. [Alpha]
 
 | Unit | Metric Type | Value Type | Monotonic | Stability |
 | ---- | ----------- | ---------- | --------- | --------- |
 | {datapoints} | Sum | Int | true | Alpha |
 
 ### otelcol_exporter_send_failed_spans
 
-Number of spans in failed attempts to send to destination. [Alpha]
+Number of spans in failed attempts to send to destination. At detailed telemetry level, includes attributes: error.type (semantic convention), failure.permanent. [Alpha]
 
 | Unit | Metric Type | Value Type | Monotonic | Stability |
 | ---- | ----------- | ---------- | --------- | --------- |

exporter/exporterhelper/go.mod

@@ -33,6 +33,7 @@ require (
 	go.uber.org/goleak v1.3.0
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.27.1
+	google.golang.org/grpc v1.77.0
 	google.golang.org/protobuf v1.36.10
 )
 
@@ -63,7 +64,6 @@ require (
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/sys v0.37.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect
-	google.golang.org/grpc v1.77.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 

exporter/exporterhelper/internal/metadata/generated_telemetry.go

@@ -5,14 +5,21 @@ package internal // import "go.opentelemetry.io/collector/exporter/exporterhelpe
 
 import (
 	"context"
+	"errors"
+	"strconv"
 
 	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/codes"
+	otelcodes "go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/metric"
+	semconv "go.opentelemetry.io/otel/semconv/v1.37.0"
 	"go.opentelemetry.io/otel/trace"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/consumer/consumererror"
 	"go.opentelemetry.io/collector/exporter"
+	"go.opentelemetry.io/collector/exporter/exporterhelper/internal/experr"
 	"go.opentelemetry.io/collector/exporter/exporterhelper/internal/metadata"
 	"go.opentelemetry.io/collector/exporter/exporterhelper/internal/queuebatch"
 	"go.opentelemetry.io/collector/exporter/exporterhelper/internal/request"
@@ -21,7 +28,6 @@ import (
 )
 
 const (
-	// spanNameSep is duplicate between receiver and exporter.
 	spanNameSep = "/"
 
 	// ExporterKey used to identify exporters in metrics and traces.
@@ -34,6 +40,9 @@ const (
 	ItemsSent = "items.sent"
 	// ItemsFailed used to track number of items that failed to be sent by exporters.
 	ItemsFailed = "items.failed"
+
+	// FailurePermanentKey indicates whether the error is permanent (non-retryable).
+	FailurePermanentKey = "failure.permanent"
 )
 
 type obsReportSender[K request.Request] struct {
@@ -46,6 +55,7 @@ type obsReportSender[K request.Request] struct {
 	metricAttr      metric.MeasurementOption
 	itemsSentInst   metric.Int64Counter
 	itemsFailedInst metric.Int64Counter
+	exporterID      string
 	next            sender.Sender[K]
 }
 
@@ -63,6 +73,7 @@ func newObsReportSender[K request.Request](set exporter.Settings, signal pipelin
 		tracer:     metadata.Tracer(set.TelemetrySettings),
 		spanAttrs:  trace.WithAttributes(expAttr, attribute.String(DataTypeKey, signal.String())),
 		metricAttr: metric.WithAttributeSet(attribute.NewSet(expAttr)),
+		exporterID: idStr,
 		next:       next,
 	}
 
@@ -88,7 +99,6 @@ func (ors *obsReportSender[K]) Send(ctx context.Context, req K) error {
 	// be modified by the downstream components like the batcher.
 	c := ors.startOp(ctx)
 	items := req.ItemsCount()
-	// Forward the data to the next consumer (this pusher is the next).
 	err := ors.next.Send(c, req)
 	ors.endOp(c, items, err)
 	return err
@@ -112,21 +122,22 @@ func (ors *obsReportSender[K]) endOp(ctx context.Context, numLogRecords int, err
 	if ors.itemsSentInst != nil {
 		ors.itemsSentInst.Add(ctx, numSent, ors.metricAttr)
 	}
-	// No metrics recorded for profiles.
-	if ors.itemsFailedInst != nil {
-		ors.itemsFailedInst.Add(ctx, numFailedToSend, ors.metricAttr)
+	if ors.itemsFailedInst != nil && numFailedToSend > 0 {
+		failedAttrs := extractFailureAttributes(err)
+		baseAttrs := attribute.NewSet(attribute.String(ExporterKey, ors.exporterID))
+		combinedAttrs := attribute.NewSet(append(baseAttrs.ToSlice(), failedAttrs.ToSlice()...)...)
+		ors.itemsFailedInst.Add(ctx, numFailedToSend, metric.WithAttributeSet(combinedAttrs))
 	}
 
 	span := trace.SpanFromContext(ctx)
 	defer span.End()
-	// End the span according to errors.
 	if span.IsRecording() {
 		span.SetAttributes(
 			attribute.Int64(ItemsSent, numSent),
 			attribute.Int64(ItemsFailed, numFailedToSend),
 		)
 		if err != nil {
-			span.SetStatus(codes.Error, err.Error())
+			span.SetStatus(otelcodes.Error, err.Error())
 		}
 	}
 }
@@ -137,3 +148,65 @@ func toNumItems(numExportedItems int, err error) (int64, int64) {
 	}
 	return int64(numExportedItems), 0
 }
+
+func extractFailureAttributes(err error) attribute.Set {
+	if err == nil {
+		return attribute.NewSet()
+	}
+
+	attrs := []attribute.KeyValue{}
+
+	errorType := determineErrorType(err)
+	attrs = append(attrs, attribute.String(string(semconv.ErrorTypeKey), errorType))
+
+	isPermanent := consumererror.IsPermanent(err)
+	attrs = append(attrs, attribute.Bool(FailurePermanentKey, isPermanent))
+
+	return attribute.NewSet(attrs...)
+}
+
+func determineErrorType(err error) string {
+	if err == nil {
+		return ""
+	}
+
+	if IsRetryExhaustedErr(err) {
+		return "RetryExhausted"
+	}
+
+	if experr.IsShutdownErr(err) {
+		return "Shutdown"
+	}
+
+	if errors.Is(err, context.Canceled) {
+		return "Canceled"
+	}
+
+	if errors.Is(err, context.DeadlineExceeded) {
+		return "DeadlineExceeded"
+	}
+
+	if st, ok := status.FromError(err); ok && st.Code() != codes.OK {
+		return st.Code().String()
+	}
+
+	if httpCode := extractHTTPStatusCode(err); httpCode > 0 {
+		return strconv.Itoa(httpCode)
+	}
+
+	return "Unknown"
+}
+
+// extractHTTPStatusCode attempts to extract an HTTP status code from the error.
+func extractHTTPStatusCode(err error) int {
+	type httpStatusCoder interface {
+		HTTPStatusCode() int
+	}
+
+	var statusCoder httpStatusCoder
+	if errors.As(err, &statusCoder) {
+		return statusCoder.HTTPStatusCode()
+	}
+
+	return 0
+}