Skip to content

Add more System.Net.Http counters #38686

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions src/libraries/System.Net.Http/src/System/Net/Http/HttpTelemetry.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ internal sealed class HttpTelemetry : EventSource
private PollingCounter? _currentRequestsCounter;
private PollingCounter? _abortedRequestsCounter;

#if !TARGETS_BROWSER
private PollingCounter? _maxHttp11ConnectionsPerPoolCounter;
private PollingCounter? _maxHttp20StreamsPerConnectionCounter;
#endif

private long _startedRequests;
private long _stoppedRequests;
private long _abortedRequests;
Expand Down Expand Up @@ -96,6 +101,18 @@ protected override void OnEventCommand(EventCommandEventArgs command)
{
DisplayName = "Current Requests"
};

#if !TARGETS_BROWSER
_maxHttp11ConnectionsPerPoolCounter ??= new PollingCounter("http11-connections-single-pool-max", this, () => HttpConnectionPoolManager.GetMaxHttp11ConnectionsPerPool())
{
DisplayName = "Maximum Http 1.1 Connections per Connection Pool"
};

_maxHttp20StreamsPerConnectionCounter ??= new PollingCounter("http20-streams-single-connection-max", this, () => HttpConnectionPoolManager.GetMaxHttp20StreamsPerConnection())
{
DisplayName = "Maximum Http Streams per Http 2.0 Connection"
};
#endif
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1559,6 +1559,8 @@ public void Dispose()
}
}

public int HttpStreamCount => _httpStreams.Count;

private enum FrameType : byte
{
Data = 0,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1553,6 +1553,10 @@ private void Trace(string? message, [CallerMemberName] string? memberName = null
memberName, // method name
message); // message

public int Http11ConnectionCount => _associatedConnectionCount;

public int Http20StreamCount => _http2Connection?.HttpStreamCount ?? 0;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we some how make it more friendly to upcoming multiple HTTP/2 connection API?


/// <summary>A cached idle connection and metadata about it.</summary>
[StructLayout(LayoutKind.Auto)]
private readonly struct CachedConnection : IEquatable<CachedConnection>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ namespace System.Net.Http
/// <summary>Provides a set of connection pools, each for its own endpoint.</summary>
internal sealed class HttpConnectionPoolManager : IDisposable
{
// Used by HttpTelemetry to calculate statistics over all connection pools
private static readonly ConcurrentDictionary<WeakReference<HttpConnectionPoolManager>, object?> s_allConnectionPools =
new ConcurrentDictionary<WeakReference<HttpConnectionPoolManager>, object?>();

/// <summary>How frequently an operation should be initiated to clean out old pools and connections in those pools.</summary>
private readonly TimeSpan _cleanPoolTimeout;
/// <summary>The pools, indexed by endpoint.</summary>
Expand All @@ -44,6 +48,8 @@ internal sealed class HttpConnectionPoolManager : IDisposable
private readonly IWebProxy? _proxy;
private readonly ICredentials? _proxyCredentials;

private readonly WeakReference<HttpConnectionPoolManager>? _weakThisRef;

private NetworkChangeCleanup? _networkChangeCleanup;

/// <summary>
Expand Down Expand Up @@ -103,14 +109,19 @@ public HttpConnectionPoolManager(HttpConnectionSettings settings)
// Create the timer. Ensure the Timer has a weak reference to this manager; otherwise, it
// can introduce a cycle that keeps the HttpConnectionPoolManager rooted by the Timer
// implementation until the handler is Disposed (or indefinitely if it's not).
_weakThisRef = new WeakReference<HttpConnectionPoolManager>(this);

_cleaningTimer = new Timer(s =>
{
var wr = (WeakReference<HttpConnectionPoolManager>)s!;
if (wr.TryGetTarget(out HttpConnectionPoolManager? thisRef))
{
thisRef.RemoveStalePools();
}
}, new WeakReference<HttpConnectionPoolManager>(this), Timeout.Infinite, Timeout.Infinite);
}, _weakThisRef, Timeout.Infinite, Timeout.Infinite);

bool success = s_allConnectionPools.TryAdd(_weakThisRef, null);
Debug.Assert(success);
}
finally
{
Expand All @@ -121,6 +132,10 @@ public HttpConnectionPoolManager(HttpConnectionSettings settings)
}
}
}
else
{
GC.SuppressFinalize(this);
}

// Figure out proxy stuff.
if (settings._useProxy)
Expand All @@ -133,6 +148,14 @@ public HttpConnectionPoolManager(HttpConnectionSettings settings)
}
}

~HttpConnectionPoolManager()
{
if (_weakThisRef != null)
{
s_allConnectionPools.TryRemove(_weakThisRef, out _);
}
}

/// <summary>
/// Starts monitoring for network changes. Upon a change, <see cref="HttpConnectionPool.OnNetworkChanged"/> will be
/// called for every <see cref="HttpConnectionPool"/> in the <see cref="HttpConnectionPoolManager"/>.
Expand Down Expand Up @@ -455,6 +478,12 @@ public void Dispose()
{
_cleaningTimer?.Dispose();

if (_weakThisRef != null)
{
s_allConnectionPools.TryRemove(_weakThisRef, out _);
GC.SuppressFinalize(this);
}

foreach (KeyValuePair<HttpConnectionKey, HttpConnectionPool> pool in _pools)
{
pool.Value.Dispose();
Expand Down Expand Up @@ -527,6 +556,38 @@ private static string GetIdentityIfDefaultCredentialsUsed(bool defaultCredential
return defaultCredentialsUsed ? CurrentUserIdentityProvider.GetIdentity() : string.Empty;
}

public static int GetMaxHttp11ConnectionsPerPool()
{
int max = 0;
foreach ((WeakReference<HttpConnectionPoolManager> weakConnectionPoolManagerRef, _) in s_allConnectionPools)
{
if (weakConnectionPoolManagerRef.TryGetTarget(out HttpConnectionPoolManager? connectionPoolManager))
{
foreach ((_, HttpConnectionPool connectionPool) in connectionPoolManager._pools)
{
max = Math.Max(max, connectionPool.Http11ConnectionCount);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd suggest to consider using an associative MaxHeap (i.e. MaxHeap combined with Dictionary to access elements for O(1)) where each node stores open connection count for a single HttpConnectionPool.

On connection open or close, the pool will increase or decrease the respective counter. Since each pool will change only their own counter, it can be easily done thread safely. When the polling counter _maxHttp11ConnectionsPerPoolCounter triggers, it will call Heapify method to rebuild the heap and get the max counter on the top. Heapify runs in O(logn) in the worst case so it seems fast enough.

There is one more issue though, that is to ensure Heapify can run concurrently with heap nodes changes. To solve that, I'd suggest to guard Heapify call with lock, but allow concurrent changes to counters on heap as explained above.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for the idea! While having a better O() when collecting the maximum on polling intervals, I am doubtful it would be a measurable perf improvement at the end, especially considering the added complexity.

The current approach tries to minimize the overhead when telemetry is disabled. The proposed one would incur an overhead on every connection even when telemetry is disabled.

While the O(N) for this collection might seem scary, it only runs once per polling interval, and the N is effectively capped at the port limit, so I doubt it would show up as a significant overhead.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The proposed one would incur an overhead on every connection even when telemetry is disabled.

That's not exactly so. If telemetry is disabled increase/decrease counter methods will be no-op and Heapify will never be called since timer will be disabled as well.

However, I do agree it adds more complexity.

}
}
}
return max;
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of maintaining a collection and looping, can you modify HttpConnectionPool to call a HttpTelemetry.SetNewMaximum(x) whenever it opens up a new connection, and Http2Connection to do similar when it opens a new stream? It should be much cheaper this way.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I still question the value of these max counters. Wouldn't a current count be more useful? You could approximate a max on top of it, but you can't do the inverse.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree. A current count seems a lot more useful.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you modify HttpConnectionPool to call a HttpTelemetry.SetNewMaximum(x) whenever it opens up a new connection, and Http2Connection to do similar when it opens a new stream?

The maximum reported here can fluctuate up and down, from what I understand that approach would only allow growing the max over time.

I also don't see the value of this information. @davidsh if you have the time, could you share what use case you had in mind?

By "current count", I assume you mean a sum of all opened http11 connections / all active Http2 streams? (in which case this global list mess isn't needed since we can just increment/decrement a global counter)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

By "current count", I assume you mean a sum of all opened http11 connections / all active Http2 streams?

Yup

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

By "current count", I assume you mean a sum of all opened http11 connections / all active Http2 streams

Does it really make sense to count total number of streams on all connections? I think it would be better to count HTTP/2 connections only. HTTP/2 stream is a logical entity in the nutshell whereas HTTP/2 connection is a physical one consuming limited resources (in example SNAT ports).


public static int GetMaxHttp20StreamsPerConnection()
{
int max = 0;
foreach ((WeakReference<HttpConnectionPoolManager> weakConnectionPoolManagerRef, _) in s_allConnectionPools)
{
if (weakConnectionPoolManagerRef.TryGetTarget(out HttpConnectionPoolManager? connectionPoolManager))
{
foreach ((_, HttpConnectionPool connectionPool) in connectionPoolManager._pools)
{
max = Math.Max(max, connectionPool.Http20StreamCount);
}
}
}
return max;
}

internal readonly struct HttpConnectionKey : IEquatable<HttpConnectionKey>
{
public readonly HttpConnectionKind Kind;
Expand Down