Skip to content

Commit 8029751

Browse files
cijothomascijothomas
authored
Metrics hotpath benchmark updates (#2571)
1 parent 7d6a22f commit 8029751

File tree

2 files changed

+38
-99
lines changed

2 files changed

+38
-99
lines changed

‎test/Benchmarks/Benchmarks.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
<ItemGroup>
2020
<ProjectReference Include="$(RepoRoot)\src\OpenTelemetry\OpenTelemetry.csproj" />
21+
<ProjectReference Aliases="InMemory" Include="$(RepoRoot)\src\OpenTelemetry.Exporter.InMemory\OpenTelemetry.Exporter.InMemory.csproj" />
2122
<ProjectReference Aliases="Jaeger" Include="$(RepoRoot)\src\OpenTelemetry.Exporter.Jaeger\OpenTelemetry.Exporter.Jaeger.csproj" />
2223
<ProjectReference Aliases="OpenTelemetryProtocol" Include="$(RepoRoot)\src\OpenTelemetry.Exporter.OpenTelemetryProtocol\OpenTelemetry.Exporter.OpenTelemetryProtocol.csproj" />
2324
<ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Exporter.Prometheus\OpenTelemetry.Exporter.Prometheus.csproj" />

‎test/Benchmarks/Metrics/MetricsBenchmarks.cs‎

Lines changed: 37 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -14,47 +14,42 @@
1414
// limitations under the License.
1515
// </copyright>
1616

17+
extern alias InMemory;
18+
1719
using System;
1820
using System.Collections.Generic;
1921
using System.Diagnostics;
2022
using System.Diagnostics.Metrics;
2123
using BenchmarkDotNet.Attributes;
24+
using InMemory::OpenTelemetry.Exporter;
2225
using OpenTelemetry;
2326
using OpenTelemetry.Metrics;
2427
using OpenTelemetry.Tests;
2528

2629
/*
27-
BenchmarkDotNet=v0.12.1, OS=Windows 10.0.19043
28-
Intel Core i7-9700 CPU 3.00GHz, 1 CPU, 8 logical and 8 physical cores
29-
.NET Core SDK=6.0.100-rc.1.21458.32
30-
[Host] : .NET Core 3.1.19 (CoreCLR 4.700.21.41101, CoreFX 4.700.21.41603), X64 RyuJIT
31-
DefaultJob : .NET Core 3.1.19 (CoreCLR 4.700.21.41101, CoreFX 4.700.21.41603), X64 RyuJIT
32-
33-
34-
| Method | WithSDK | Mean | Error | StdDev | Gen 0 | Gen 1 | Gen 2 | Allocated |
35-
|-------------------------------------- |-------- |----------:|---------:|---------:|-------:|------:|------:|----------:|
36-
| CounterHotPath | False | 12.87 ns | 0.086 ns | 0.080 ns | - | - | - | - |
37-
| CounterWith1LabelsHotPath | False | 19.36 ns | 0.083 ns | 0.069 ns | - | - | - | - |
38-
| CounterWith3LabelsHotPath | False | 53.71 ns | 0.336 ns | 0.314 ns | - | - | - | - |
39-
| CounterWith5LabelsHotPath | False | 79.82 ns | 0.525 ns | 0.439 ns | 0.0166 | - | - | 104 B |
40-
| CounterWith6LabelsHotPath | False | 91.35 ns | 0.827 ns | 0.733 ns | 0.0191 | - | - | 120 B |
41-
| CounterWith7LabelsHotPath | False | 104.40 ns | 0.924 ns | 0.865 ns | 0.0216 | - | - | 136 B |
42-
| CounterWith1LabelsHotPathUsingTagList | False | 50.43 ns | 0.415 ns | 0.388 ns | - | - | - | - |
43-
| CounterWith3LabelsHotPathUsingTagList | False | 88.48 ns | 0.402 ns | 0.376 ns | - | - | - | - |
44-
| CounterWith5LabelsHotPathUsingTagList | False | 121.61 ns | 0.472 ns | 0.418 ns | - | - | - | - |
45-
| CounterWith6LabelsHotPathUsingTagList | False | 139.11 ns | 0.593 ns | 0.554 ns | - | - | - | - |
46-
| CounterWith7LabelsHotPathUsingTagList | False | 154.07 ns | 0.773 ns | 0.685 ns | - | - | - | - |
47-
| CounterHotPath | True | 38.23 ns | 0.235 ns | 0.220 ns | - | - | - | - |
48-
| CounterWith1LabelsHotPath | True | 102.48 ns | 0.942 ns | 0.881 ns | - | - | - | - |
49-
| CounterWith3LabelsHotPath | True | 417.00 ns | 2.904 ns | 2.716 ns | - | - | - | - |
50-
| CounterWith5LabelsHotPath | True | 578.45 ns | 5.287 ns | 4.946 ns | 0.0162 | - | - | 104 B |
51-
| CounterWith6LabelsHotPath | True | 665.56 ns | 3.716 ns | 3.476 ns | 0.0191 | - | - | 120 B |
52-
| CounterWith7LabelsHotPath | True | 778.88 ns | 5.482 ns | 4.578 ns | 0.0210 | - | - | 136 B |
53-
| CounterWith1LabelsHotPathUsingTagList | True | 135.55 ns | 1.012 ns | 0.947 ns | - | - | - | - |
54-
| CounterWith3LabelsHotPathUsingTagList | True | 457.96 ns | 4.242 ns | 3.968 ns | - | - | - | - |
55-
| CounterWith5LabelsHotPathUsingTagList | True | 631.81 ns | 3.423 ns | 2.858 ns | - | - | - | - |
56-
| CounterWith6LabelsHotPathUsingTagList | True | 719.81 ns | 4.704 ns | 4.400 ns | - | - | - | - |
57-
| CounterWith7LabelsHotPathUsingTagList | True | 828.88 ns | 4.321 ns | 3.830 ns | - | - | - | - |
30+
// * Summary *
31+
32+
BenchmarkDotNet=v0.13.1, OS=Windows 10.0.19043.1288 (21H1/May2021Update)
33+
Intel Core i7-4790 CPU 3.60GHz (Haswell), 1 CPU, 8 logical and 4 physical cores
34+
.NET SDK=6.0.100-rc.2.21505.57
35+
[Host] : .NET 5.0.11 (5.0.1121.47308), X64 RyuJIT
36+
DefaultJob : .NET 5.0.11 (5.0.1121.47308), X64 RyuJIT
37+
38+
39+
| Method | AggregationTemporality | Mean | Error | StdDev | Allocated |
40+
|-------------------------- |----------------------- |------------:|----------:|----------:|----------:|
41+
| CounterHotPath | Cumulative | 22.12 ns | 0.233 ns | 0.218 ns | - |
42+
| CounterWith1LabelsHotPath | Cumulative | 101.78 ns | 0.283 ns | 0.265 ns | - |
43+
| CounterWith3LabelsHotPath | Cumulative | 488.20 ns | 6.580 ns | 7.577 ns | - |
44+
| CounterWith5LabelsHotPath | Cumulative | 748.28 ns | 14.936 ns | 22.355 ns | - |
45+
| CounterWith6LabelsHotPath | Cumulative | 857.56 ns | 6.008 ns | 4.691 ns | - |
46+
| CounterWith7LabelsHotPath | Cumulative | 1,023.17 ns | 11.500 ns | 9.603 ns | - |
47+
| CounterHotPath | Delta | 20.93 ns | 0.244 ns | 0.216 ns | - |
48+
| CounterWith1LabelsHotPath | Delta | 104.09 ns | 0.432 ns | 0.404 ns | - |
49+
| CounterWith3LabelsHotPath | Delta | 477.48 ns | 2.646 ns | 2.066 ns | - |
50+
| CounterWith5LabelsHotPath | Delta | 732.90 ns | 8.778 ns | 7.782 ns | - |
51+
| CounterWith6LabelsHotPath | Delta | 868.00 ns | 4.503 ns | 3.760 ns | - |
52+
| CounterWith7LabelsHotPath | Delta | 986.40 ns | 9.503 ns | 7.419 ns | - |
5853
*/
5954

6055
namespace Benchmarks.Metrics
@@ -68,20 +63,21 @@ public class MetricsBenchmarks
6863
private Random random = new Random();
6964
private string[] dimensionValues = new string[] { "DimVal1", "DimVal2", "DimVal3", "DimVal4", "DimVal5", "DimVal6", "DimVal7", "DimVal8", "DimVal9", "DimVal10" };
7065

71-
[Params(false, true)]
72-
public bool WithSDK { get; set; }
66+
[Params(AggregationTemporality.Cumulative, AggregationTemporality.Delta)]
67+
public AggregationTemporality AggregationTemporality { get; set; }
7368

7469
[GlobalSetup]
7570
public void Setup()
7671
{
7772
this.meter = new Meter(Utils.GetCurrentMethodName());
7873

79-
if (this.WithSDK)
80-
{
81-
this.provider = Sdk.CreateMeterProviderBuilder()
82-
.AddMeter(this.meter.Name) // All instruments from this meter are enabled.
83-
.Build();
84-
}
74+
var exportedItems = new List<Metric>();
75+
var reader = new PeriodicExportingMetricReader(new InMemoryExporter<Metric>(exportedItems), 1000);
76+
reader.PreferredAggregationTemporality = this.AggregationTemporality;
77+
this.provider = Sdk.CreateMeterProviderBuilder()
78+
.AddMeter(this.meter.Name) // All instruments from this meter are enabled.
79+
.AddReader(reader)
80+
.Build();
8581

8682
this.counter = this.meter.CreateCounter<long>("counter");
8783
}
@@ -117,64 +113,6 @@ public void CounterWith3LabelsHotPath()
117113

118114
[Benchmark]
119115
public void CounterWith5LabelsHotPath()
120-
{
121-
var tag1 = new KeyValuePair<string, object>("DimName1", this.dimensionValues[this.random.Next(0, 2)]);
122-
var tag2 = new KeyValuePair<string, object>("DimName2", this.dimensionValues[this.random.Next(0, 2)]);
123-
var tag3 = new KeyValuePair<string, object>("DimName3", this.dimensionValues[this.random.Next(0, 5)]);
124-
var tag4 = new KeyValuePair<string, object>("DimName4", this.dimensionValues[this.random.Next(0, 5)]);
125-
var tag5 = new KeyValuePair<string, object>("DimName5", this.dimensionValues[this.random.Next(0, 10)]);
126-
this.counter?.Add(100, tag1, tag2, tag3, tag4, tag5);
127-
}
128-
129-
[Benchmark]
130-
public void CounterWith6LabelsHotPath()
131-
{
132-
var tag1 = new KeyValuePair<string, object>("DimName1", this.dimensionValues[this.random.Next(0, 2)]);
133-
var tag2 = new KeyValuePair<string, object>("DimName2", this.dimensionValues[this.random.Next(0, 2)]);
134-
var tag3 = new KeyValuePair<string, object>("DimName3", this.dimensionValues[this.random.Next(0, 5)]);
135-
var tag4 = new KeyValuePair<string, object>("DimName4", this.dimensionValues[this.random.Next(0, 5)]);
136-
var tag5 = new KeyValuePair<string, object>("DimName5", this.dimensionValues[this.random.Next(0, 5)]);
137-
var tag6 = new KeyValuePair<string, object>("DimName6", this.dimensionValues[this.random.Next(0, 2)]);
138-
this.counter?.Add(100, tag1, tag2, tag3, tag4, tag5, tag6);
139-
}
140-
141-
[Benchmark]
142-
public void CounterWith7LabelsHotPath()
143-
{
144-
var tag1 = new KeyValuePair<string, object>("DimName1", this.dimensionValues[this.random.Next(0, 2)]);
145-
var tag2 = new KeyValuePair<string, object>("DimName2", this.dimensionValues[this.random.Next(0, 2)]);
146-
var tag3 = new KeyValuePair<string, object>("DimName3", this.dimensionValues[this.random.Next(0, 5)]);
147-
var tag4 = new KeyValuePair<string, object>("DimName4", this.dimensionValues[this.random.Next(0, 5)]);
148-
var tag5 = new KeyValuePair<string, object>("DimName5", this.dimensionValues[this.random.Next(0, 5)]);
149-
var tag6 = new KeyValuePair<string, object>("DimName6", this.dimensionValues[this.random.Next(0, 2)]);
150-
var tag7 = new KeyValuePair<string, object>("DimName7", this.dimensionValues[this.random.Next(0, 1)]);
151-
this.counter?.Add(100, tag1, tag2, tag3, tag4, tag5, tag6, tag7);
152-
}
153-
154-
[Benchmark]
155-
public void CounterWith1LabelsHotPathUsingTagList()
156-
{
157-
var tags = new TagList
158-
{
159-
{ "DimName1", this.dimensionValues[this.random.Next(0, 2)] },
160-
};
161-
this.counter?.Add(100, tags);
162-
}
163-
164-
[Benchmark]
165-
public void CounterWith3LabelsHotPathUsingTagList()
166-
{
167-
var tags = new TagList
168-
{
169-
{ "DimName1", this.dimensionValues[this.random.Next(0, 10)] },
170-
{ "DimName2", this.dimensionValues[this.random.Next(0, 10)] },
171-
{ "DimName3", this.dimensionValues[this.random.Next(0, 10)] },
172-
};
173-
this.counter?.Add(100, tags);
174-
}
175-
176-
[Benchmark]
177-
public void CounterWith5LabelsHotPathUsingTagList()
178116
{
179117
var tags = new TagList
180118
{
@@ -188,7 +126,7 @@ public void CounterWith5LabelsHotPathUsingTagList()
188126
}
189127

190128
[Benchmark]
191-
public void CounterWith6LabelsHotPathUsingTagList()
129+
public void CounterWith6LabelsHotPath()
192130
{
193131
var tags = new TagList
194132
{
@@ -203,7 +141,7 @@ public void CounterWith6LabelsHotPathUsingTagList()
203141
}
204142

205143
[Benchmark]
206-
public void CounterWith7LabelsHotPathUsingTagList()
144+
public void CounterWith7LabelsHotPath()
207145
{
208146
var tags = new TagList
209147
{

0 commit comments

Comments
 (0)