Skip to content

Commit a330ae2

Browse files
mx-psimx-psi
authored
[chore] Skip or bump some benchmarks to reduce noise (#14258)
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue. Ex. Adding a feature - Explain what this achieves.--> #### Description <!-- Issue number if applicable --> Bumps size to 2k on some benchmarks to hopefully make those benchmarks more reliable. Skips a bunch of other benchmarks that are allocation-heavy and currently are more noise than signal. #### Link to tracking issue Relates to #14257
1 parent a0cbea7 commit a330ae2

File tree

10 files changed

+40
-12
lines changed

10 files changed

+40
-12
lines changed

‎config/confighttp/compressor_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@ import (
1616
"github.com/stretchr/testify/require"
1717

1818
"go.opentelemetry.io/collector/config/configcompression"
19+
"go.opentelemetry.io/collector/internal/testutil"
1920
)
2021

2122
func BenchmarkCompression(b *testing.B) {
23+
testutil.SkipGCHeavyBench(b)
2224
benchmarks := []struct {
2325
codec configcompression.Type
2426
name string

‎config/confighttp/go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ require (
2323
go.opentelemetry.io/collector/extension/extensionmiddleware v0.141.0
2424
go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.141.0
2525
go.opentelemetry.io/collector/featuregate v1.47.0
26+
go.opentelemetry.io/collector/internal/testutil v0.141.0
2627
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0
2728
go.opentelemetry.io/otel v1.38.0
2829
go.uber.org/goleak v1.3.0

‎exporter/exporterhelper/go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ require (
2020
go.opentelemetry.io/collector/extension/extensiontest v0.141.0
2121
go.opentelemetry.io/collector/extension/xextension v0.141.0
2222
go.opentelemetry.io/collector/featuregate v1.47.0
23+
go.opentelemetry.io/collector/internal/testutil v0.141.0
2324
go.opentelemetry.io/collector/pdata v1.47.0
2425
go.opentelemetry.io/collector/pdata/pprofile v0.141.0
2526
go.opentelemetry.io/collector/pdata/testdata v0.141.0

‎exporter/exporterhelper/internal/queuebatch/logs_batch_test.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212

1313
"go.opentelemetry.io/collector/exporter/exporterhelper/internal/request"
1414
"go.opentelemetry.io/collector/exporter/exporterhelper/internal/sizer"
15+
"go.opentelemetry.io/collector/internal/testutil"
1516
"go.opentelemetry.io/collector/pdata/plog"
1617
"go.opentelemetry.io/collector/pdata/testdata"
1718
)
@@ -325,6 +326,7 @@ func TestLogsMergeSplitUnknownSizerType(t *testing.T) {
325326
}
326327

327328
func BenchmarkSplittingBasedOnItemCountManySmallLogs(b *testing.B) {
329+
testutil.SkipGCHeavyBench(b)
328330
// All requests merge into a single batch.
329331
b.ReportAllocs()
330332
for b.Loop() {
@@ -339,6 +341,7 @@ func BenchmarkSplittingBasedOnItemCountManySmallLogs(b *testing.B) {
339341
}
340342

341343
func BenchmarkSplittingBasedOnByteSizeManySmallLogs(b *testing.B) {
344+
testutil.SkipGCHeavyBench(b)
342345
// All requests merge into a single batch.
343346
b.ReportAllocs()
344347
for b.Loop() {
@@ -353,6 +356,7 @@ func BenchmarkSplittingBasedOnByteSizeManySmallLogs(b *testing.B) {
353356
}
354357

355358
func BenchmarkSplittingBasedOnItemCountManyLogsSlightlyAboveLimit(b *testing.B) {
359+
testutil.SkipGCHeavyBench(b)
356360
// Every incoming request results in a split.
357361
b.ReportAllocs()
358362
for b.Loop() {
@@ -367,6 +371,7 @@ func BenchmarkSplittingBasedOnItemCountManyLogsSlightlyAboveLimit(b *testing.B)
367371
}
368372

369373
func BenchmarkSplittingBasedOnByteSizeManyLogsSlightlyAboveLimit(b *testing.B) {
374+
testutil.SkipGCHeavyBench(b)
370375
// Every incoming request results in a split.
371376
b.ReportAllocs()
372377
for b.Loop() {
@@ -382,6 +387,7 @@ func BenchmarkSplittingBasedOnByteSizeManyLogsSlightlyAboveLimit(b *testing.B) {
382387
}
383388

384389
func BenchmarkSplittingBasedOnItemCountHugeLogs(b *testing.B) {
390+
testutil.SkipGCHeavyBench(b)
385391
// One request splits into many batches.
386392
b.ReportAllocs()
387393
for b.Loop() {
@@ -394,6 +400,7 @@ func BenchmarkSplittingBasedOnItemCountHugeLogs(b *testing.B) {
394400
}
395401

396402
func BenchmarkSplittingBasedOnByteSizeHugeLogs(b *testing.B) {
403+
testutil.SkipGCHeavyBench(b)
397404
// One request splits into many batches.
398405
b.ReportAllocs()
399406
for b.Loop() {

‎exporter/exporterhelper/internal/queuebatch/metrics_batch_test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212

1313
"go.opentelemetry.io/collector/exporter/exporterhelper/internal/request"
1414
"go.opentelemetry.io/collector/exporter/exporterhelper/internal/sizer"
15+
"go.opentelemetry.io/collector/internal/testutil"
1516
"go.opentelemetry.io/collector/pdata/pmetric"
1617
"go.opentelemetry.io/collector/pdata/testdata"
1718
)
@@ -264,6 +265,7 @@ func TestMergeSplitManySmallMetrics(t *testing.T) {
264265
}
265266

266267
func BenchmarkSplittingBasedOnItemCountManySmallMetrics(b *testing.B) {
268+
testutil.SkipGCHeavyBench(b)
267269
// All requests merge into a single batch.
268270
b.ReportAllocs()
269271
for b.Loop() {
@@ -278,6 +280,7 @@ func BenchmarkSplittingBasedOnItemCountManySmallMetrics(b *testing.B) {
278280
}
279281

280282
func BenchmarkSplittingBasedOnItemCountManyMetricsSlightlyAboveLimit(b *testing.B) {
283+
testutil.SkipGCHeavyBench(b)
281284
// Every incoming request results in a split.
282285
b.ReportAllocs()
283286
for b.Loop() {
@@ -292,6 +295,7 @@ func BenchmarkSplittingBasedOnItemCountManyMetricsSlightlyAboveLimit(b *testing.
292295
}
293296

294297
func BenchmarkSplittingBasedOnItemCountHugeMetrics(b *testing.B) {
298+
testutil.SkipGCHeavyBench(b)
295299
// One request splits into many batches.
296300
b.ReportAllocs()
297301
for b.Loop() {

‎exporter/exporterhelper/internal/queuebatch/traces_batch_test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212

1313
"go.opentelemetry.io/collector/exporter/exporterhelper/internal/request"
1414
"go.opentelemetry.io/collector/exporter/exporterhelper/internal/sizer"
15+
"go.opentelemetry.io/collector/internal/testutil"
1516
"go.opentelemetry.io/collector/pdata/ptrace"
1617
"go.opentelemetry.io/collector/pdata/testdata"
1718
)
@@ -338,6 +339,7 @@ func TestTracesMergeSplitUnknownSizerType(t *testing.T) {
338339
}
339340

340341
func BenchmarkSplittingBasedOnItemCountManySmallTraces(b *testing.B) {
342+
testutil.SkipGCHeavyBench(b)
341343
// All requests merge into a single batch.
342344
b.ReportAllocs()
343345
for b.Loop() {
@@ -352,6 +354,7 @@ func BenchmarkSplittingBasedOnItemCountManySmallTraces(b *testing.B) {
352354
}
353355

354356
func BenchmarkSplittingBasedOnItemCountManyTracesSlightlyAboveLimit(b *testing.B) {
357+
testutil.SkipGCHeavyBench(b)
355358
// Every incoming request results in a split.
356359
b.ReportAllocs()
357360
for b.Loop() {
@@ -366,6 +369,7 @@ func BenchmarkSplittingBasedOnItemCountManyTracesSlightlyAboveLimit(b *testing.B
366369
}
367370

368371
func BenchmarkSplittingBasedOnItemCountHugeTraces(b *testing.B) {
372+
testutil.SkipGCHeavyBench(b)
369373
// One request splits into many batches.
370374
b.ReportAllocs()
371375
for b.Loop() {

‎internal/testutil/benchmarks.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,12 @@ func SkipMemoryBench(b *testing.B) {
1515
b.Skip("Skipping since the 'MEMBENCH' environment variable was not set")
1616
}
1717
}
18+
19+
// SkipGCHeavyBench will skip GC-heavy benchmarks on CI.
20+
// These benchmarks tend to be flaky with the current settings since garbage
21+
// collection pauses can take ~50ms which is significant with the current benchmark times.
22+
func SkipGCHeavyBench(b *testing.B) {
23+
if os.Getenv("GCHEAVYBENCH") == "" {
24+
b.Skip("Skipping since the 'GCHEAVYBENCH' environment variable was not set. See https://github.com/open-telemetry/opentelemetry-collector/issues/14257.")
25+
}
26+
}

‎pdata/plog/pb_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,9 @@ func TestProtoSizerEmptyLogs(t *testing.T) {
7373
assert.Equal(t, 0, sizer.LogsSize(NewLogs()))
7474
}
7575

76-
func BenchmarkLogsToProto(b *testing.B) {
76+
func BenchmarkLogsToProto2k(b *testing.B) {
7777
marshaler := &ProtoMarshaler{}
78-
logs := generateBenchmarkLogs(128)
78+
logs := generateBenchmarkLogs(2_000)
7979

8080
for b.Loop() {
8181
buf, err := marshaler.MarshalLogs(logs)
@@ -84,10 +84,10 @@ func BenchmarkLogsToProto(b *testing.B) {
8484
}
8585
}
8686

87-
func BenchmarkLogsFromProto(b *testing.B) {
87+
func BenchmarkLogsFromProto2k(b *testing.B) {
8888
marshaler := &ProtoMarshaler{}
8989
unmarshaler := &ProtoUnmarshaler{}
90-
baseLogs := generateBenchmarkLogs(128)
90+
baseLogs := generateBenchmarkLogs(2_000)
9191
buf, err := marshaler.MarshalLogs(baseLogs)
9292
require.NoError(b, err)
9393
assert.NotEmpty(b, buf)

‎pdata/pmetric/pb_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,9 @@ func TestProtoSizerEmptyMetrics(t *testing.T) {
7373
assert.Equal(t, 0, sizer.MetricsSize(NewMetrics()))
7474
}
7575

76-
func BenchmarkMetricsToProto(b *testing.B) {
76+
func BenchmarkMetricsToProto2k(b *testing.B) {
7777
marshaler := &ProtoMarshaler{}
78-
metrics := generateBenchmarkMetrics(128)
78+
metrics := generateBenchmarkMetrics(2_000)
7979

8080
for b.Loop() {
8181
buf, err := marshaler.MarshalMetrics(metrics)
@@ -84,10 +84,10 @@ func BenchmarkMetricsToProto(b *testing.B) {
8484
}
8585
}
8686

87-
func BenchmarkMetricsFromProto(b *testing.B) {
87+
func BenchmarkMetricsFromProto10k(b *testing.B) {
8888
marshaler := &ProtoMarshaler{}
8989
unmarshaler := &ProtoUnmarshaler{}
90-
baseMetrics := generateBenchmarkMetrics(128)
90+
baseMetrics := generateBenchmarkMetrics(2_000)
9191
buf, err := marshaler.MarshalMetrics(baseMetrics)
9292
require.NoError(b, err)
9393
assert.NotEmpty(b, buf)

‎pdata/ptrace/pb_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,9 +74,9 @@ func TestProtoSizerEmptyTraces(t *testing.T) {
7474
assert.Equal(t, 0, sizer.TracesSize(NewTraces()))
7575
}
7676

77-
func BenchmarkTracesToProto(b *testing.B) {
77+
func BenchmarkTracesToProto2k(b *testing.B) {
7878
marshaler := &ProtoMarshaler{}
79-
traces := generateBenchmarkTraces(128)
79+
traces := generateBenchmarkTraces(2_000)
8080

8181
for b.Loop() {
8282
buf, err := marshaler.MarshalTraces(traces)
@@ -85,10 +85,10 @@ func BenchmarkTracesToProto(b *testing.B) {
8585
}
8686
}
8787

88-
func BenchmarkTracesFromProto(b *testing.B) {
88+
func BenchmarkTracesFromProto2k(b *testing.B) {
8989
marshaler := &ProtoMarshaler{}
9090
unmarshaler := &ProtoUnmarshaler{}
91-
baseTraces := generateBenchmarkTraces(128)
91+
baseTraces := generateBenchmarkTraces(2_000)
9292
buf, err := marshaler.MarshalTraces(baseTraces)
9393
require.NoError(b, err)
9494
assert.NotEmpty(b, buf)

0 commit comments

Comments
 (0)