Make HttpLogging middleware endpoint aware (#47595)

Author: BrennanConroy

src/Middleware/HttpLogging/src/HttpLoggingAttribute.cs

@@ -0,0 +1,44 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.AspNetCore.HttpLogging;
+
+/// <summary>
+/// Metadata that provides endpoint-specific settings for the HttpLogging middleware.
+/// </summary>
+[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method, AllowMultiple = false, Inherited = true)]
+public sealed class HttpLoggingAttribute : Attribute
+{
+    /// <summary>
+    /// Initializes an instance of the <see cref="HttpLoggingAttribute"/> class.
+    /// </summary>
+    /// <param name="loggingFields">Specifies what fields to log for the endpoint.</param>
+    /// <param name="requestBodyLogLimit">Specifies the maximum number of bytes to be logged for the request body. A value of <c>-1</c> means use the default setting in <see cref="HttpLoggingOptions.RequestBodyLogLimit"/>.</param>
+    /// <param name="responseBodyLogLimit">Specifies the maximum number of bytes to be logged for the response body. A value of <c>-1</c> means use the default setting in <see cref="HttpLoggingOptions.ResponseBodyLogLimit"/>.</param>
+    /// <exception cref="ArgumentOutOfRangeException">Thrown when <paramref name="requestBodyLogLimit"/> or <paramref name="responseBodyLogLimit"/> is less than <c>-1</c>.</exception>
+    public HttpLoggingAttribute(HttpLoggingFields loggingFields, int requestBodyLogLimit = -1, int responseBodyLogLimit = -1)
+    {
+        LoggingFields = loggingFields;
+
+        ArgumentOutOfRangeException.ThrowIfLessThan(requestBodyLogLimit, -1);
+        ArgumentOutOfRangeException.ThrowIfLessThan(responseBodyLogLimit, -1);
+
+        RequestBodyLogLimit = requestBodyLogLimit;
+        ResponseBodyLogLimit = responseBodyLogLimit;
+    }
+
+    /// <summary>
+    /// Specifies what fields to log.
+    /// </summary>
+    public HttpLoggingFields LoggingFields { get; }
+
+    /// <summary>
+    /// Specifies the maximum number of bytes to be logged for the request body.
+    /// </summary>
+    public int RequestBodyLogLimit { get; }
+
+    /// <summary>
+    /// Specifies the maximum number of bytes to be logged for the response body.
+    /// </summary>
+    public int ResponseBodyLogLimit { get; }
+}

src/Middleware/HttpLogging/src/HttpLoggingEndpointConventionBuilderExtensions.cs

@@ -0,0 +1,34 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.AspNetCore.HttpLogging;
+
+namespace Microsoft.AspNetCore.Builder;
+
+/// <summary>
+/// HttpLogging middleware extension methods for <see cref="IEndpointConventionBuilder"/>.
+/// </summary>
+public static class HttpLoggingEndpointConventionBuilderExtensions
+{
+    /// <summary>
+    /// Adds endpoint specific settings for the HttpLogging middleware.
+    /// </summary>
+    /// <typeparam name="TBuilder">The type of endpoint convention builder.</typeparam>
+    /// <param name="builder">The endpoint convention builder.</param>
+    /// <param name="loggingFields">The <see cref="HttpLoggingFields"/> to apply to this endpoint.</param>
+    /// <param name="requestBodyLogLimit">Sets the <see cref="HttpLoggingOptions.RequestBodyLogLimit"/> for this endpoint. A value of <c>-1</c> means use the default setting in <see cref="HttpLoggingOptions.RequestBodyLogLimit"/>.</param>
+    /// <param name="responseBodyLogLimit">Sets the <see cref="HttpLoggingOptions.ResponseBodyLogLimit"/> for this endpoint. A value of <c>-1</c> means use the default setting in <see cref="HttpLoggingOptions.ResponseBodyLogLimit"/>.</param>
+    /// <returns>The original convention builder parameter.</returns>
+    /// <exception cref="ArgumentOutOfRangeException">Thrown when <paramref name="requestBodyLogLimit"/> or <paramref name="responseBodyLogLimit"/> is less than <c>-1</c>.</exception>
+    public static TBuilder WithHttpLogging<TBuilder>(this TBuilder builder, HttpLoggingFields loggingFields, int requestBodyLogLimit = -1, int responseBodyLogLimit = -1) where TBuilder : IEndpointConventionBuilder
+    {
+        // Construct outside build.Add lambda to allow exceptions to be thrown immediately
+        var metadata = new HttpLoggingAttribute(loggingFields, requestBodyLogLimit, responseBodyLogLimit);
+
+        builder.Add(endpointBuilder =>
+        {
+            endpointBuilder.Metadata.Add(metadata);
+        });
+        return builder;
+    }
+}

src/Middleware/HttpLogging/src/HttpLoggingMiddleware.cs

@@ -59,44 +59,47 @@ private async Task InvokeInternal(HttpContext context)
         RequestBufferingStream? requestBufferingStream = null;
         Stream? originalBody = null;
 
-        if ((HttpLoggingFields.Request & options.LoggingFields) != HttpLoggingFields.None)
+        var loggingAttribute = context.GetEndpoint()?.Metadata.GetMetadata<HttpLoggingAttribute>();
+        var loggingFields = loggingAttribute?.LoggingFields ?? options.LoggingFields;
+
+        if ((HttpLoggingFields.Request & loggingFields) != HttpLoggingFields.None)
         {
             var request = context.Request;
             var list = new List<KeyValuePair<string, object?>>(
                 request.Headers.Count + DefaultRequestFieldsMinusHeaders);
 
-            if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestProtocol))
+            if (loggingFields.HasFlag(HttpLoggingFields.RequestProtocol))
             {
                 AddToList(list, nameof(request.Protocol), request.Protocol);
             }
 
-            if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestMethod))
+            if (loggingFields.HasFlag(HttpLoggingFields.RequestMethod))
             {
                 AddToList(list, nameof(request.Method), request.Method);
             }
 
-            if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestScheme))
+            if (loggingFields.HasFlag(HttpLoggingFields.RequestScheme))
             {
                 AddToList(list, nameof(request.Scheme), request.Scheme);
             }
 
-            if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestPath))
+            if (loggingFields.HasFlag(HttpLoggingFields.RequestPath))
             {
                 AddToList(list, nameof(request.PathBase), request.PathBase);
                 AddToList(list, nameof(request.Path), request.Path);
             }
 
-            if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestQuery))
+            if (loggingFields.HasFlag(HttpLoggingFields.RequestQuery))
             {
                 AddToList(list, nameof(request.QueryString), request.QueryString.Value);
             }
 
-            if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestHeaders))
+            if (loggingFields.HasFlag(HttpLoggingFields.RequestHeaders))
             {
                 FilterHeaders(list, request.Headers, options._internalRequestHeaders);
             }
 
-            if (options.LoggingFields.HasFlag(HttpLoggingFields.RequestBody))
+            if (loggingFields.HasFlag(HttpLoggingFields.RequestBody))
             {
                 if (request.ContentType is null)
                 {
@@ -106,10 +109,16 @@ private async Task InvokeInternal(HttpContext context)
                     options.MediaTypeOptions.MediaTypeStates,
                     out var encoding))
                 {
+                    var requestBodyLogLimit = options.RequestBodyLogLimit;
+                    if (loggingAttribute?.RequestBodyLogLimit is int)
+                    {
+                        requestBodyLogLimit = loggingAttribute.RequestBodyLogLimit;
+                    }
+
                     originalBody = request.Body;
                     requestBufferingStream = new RequestBufferingStream(
                         request.Body,
-                        options.RequestBodyLogLimit,
+                        requestBodyLogLimit,
                         _logger,
                         encoding);
                     request.Body = requestBufferingStream;
@@ -135,29 +144,36 @@ private async Task InvokeInternal(HttpContext context)
         {
             var response = context.Response;
 
-            if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseStatusCode) || options.LoggingFields.HasFlag(HttpLoggingFields.ResponseHeaders))
+            if (loggingFields.HasFlag(HttpLoggingFields.ResponseStatusCode) || loggingFields.HasFlag(HttpLoggingFields.ResponseHeaders))
             {
                 originalUpgradeFeature = context.Features.Get<IHttpUpgradeFeature>();
 
                 if (originalUpgradeFeature != null && originalUpgradeFeature.IsUpgradableRequest)
                 {
-                    loggableUpgradeFeature = new UpgradeFeatureLoggingDecorator(originalUpgradeFeature, response, options, _logger);
+                    loggableUpgradeFeature = new UpgradeFeatureLoggingDecorator(originalUpgradeFeature, response, options._internalResponseHeaders, loggingFields, _logger);
 
                     context.Features.Set<IHttpUpgradeFeature>(loggableUpgradeFeature);
                 }
             }
 
-            if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseBody))
+            if (loggingFields.HasFlag(HttpLoggingFields.ResponseBody))
             {
                 originalBodyFeature = context.Features.Get<IHttpResponseBodyFeature>()!;
 
+                var responseBodyLogLimit = options.ResponseBodyLogLimit;
+                if (loggingAttribute?.ResponseBodyLogLimit is int)
+                {
+                    responseBodyLogLimit = loggingAttribute.ResponseBodyLogLimit;
+                }
+
                 // TODO pool these.
                 responseBufferingStream = new ResponseBufferingStream(originalBodyFeature,
-                    options.ResponseBodyLogLimit,
+                    responseBodyLogLimit,
                     _logger,
                     context,
                     options.MediaTypeOptions.MediaTypeStates,
-                    options);
+                    options._internalResponseHeaders,
+                    loggingFields);
                 response.Body = responseBufferingStream;
                 context.Features.Set<IHttpResponseBodyFeature>(responseBufferingStream);
             }
@@ -174,7 +190,7 @@ private async Task InvokeInternal(HttpContext context)
             if (ResponseHeadersNotYetWritten(responseBufferingStream, loggableUpgradeFeature))
             {
                 // No body, not an upgradable request or request not upgraded, write headers here.
-                LogResponseHeaders(response, options, _logger);
+                LogResponseHeaders(response, loggingFields, options._internalResponseHeaders, _logger);
             }
 
             if (responseBufferingStream != null)
@@ -216,7 +232,7 @@ private static bool ResponseHeadersNotYetWritten(ResponseBufferingStream? respon
 
     private static bool BodyNotYetWritten(ResponseBufferingStream? responseBufferingStream)
     {
-        return responseBufferingStream == null || responseBufferingStream.FirstWrite == false;
+        return responseBufferingStream == null || responseBufferingStream.HeadersWritten == false;
     }
 
     private static bool NotUpgradeableRequestOrRequestNotUpgraded(UpgradeFeatureLoggingDecorator? upgradeFeatureLogging)
@@ -229,19 +245,19 @@ private static void AddToList(List<KeyValuePair<string, object?>> list, string k
         list.Add(new KeyValuePair<string, object?>(key, value));
     }
 
-    public static void LogResponseHeaders(HttpResponse response, HttpLoggingOptions options, ILogger logger)
+    public static void LogResponseHeaders(HttpResponse response, HttpLoggingFields loggingFields, HashSet<string> allowedResponseHeaders, ILogger logger)
     {
         var list = new List<KeyValuePair<string, object?>>(
             response.Headers.Count + DefaultResponseFieldsMinusHeaders);
 
-        if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseStatusCode))
+        if (loggingFields.HasFlag(HttpLoggingFields.ResponseStatusCode))
         {
             list.Add(new KeyValuePair<string, object?>(nameof(response.StatusCode), response.StatusCode));
         }
 
-        if (options.LoggingFields.HasFlag(HttpLoggingFields.ResponseHeaders))
+        if (loggingFields.HasFlag(HttpLoggingFields.ResponseHeaders))
         {
-            FilterHeaders(list, response.Headers, options._internalResponseHeaders);
+            FilterHeaders(list, response.Headers, allowedResponseHeaders);
         }
 
         if (list.Count > 0)

src/Middleware/HttpLogging/src/PublicAPI.Unshipped.txt

@@ -1 +1,8 @@
 #nullable enable
+Microsoft.AspNetCore.Builder.HttpLoggingEndpointConventionBuilderExtensions
+Microsoft.AspNetCore.HttpLogging.HttpLoggingAttribute
+Microsoft.AspNetCore.HttpLogging.HttpLoggingAttribute.HttpLoggingAttribute(Microsoft.AspNetCore.HttpLogging.HttpLoggingFields loggingFields, int requestBodyLogLimit = -1, int responseBodyLogLimit = -1) -> void
+Microsoft.AspNetCore.HttpLogging.HttpLoggingAttribute.LoggingFields.get -> Microsoft.AspNetCore.HttpLogging.HttpLoggingFields
+Microsoft.AspNetCore.HttpLogging.HttpLoggingAttribute.RequestBodyLogLimit.get -> int
+Microsoft.AspNetCore.HttpLogging.HttpLoggingAttribute.ResponseBodyLogLimit.get -> int
+static Microsoft.AspNetCore.Builder.HttpLoggingEndpointConventionBuilderExtensions.WithHttpLogging<TBuilder>(this TBuilder builder, Microsoft.AspNetCore.HttpLogging.HttpLoggingFields loggingFields, int requestBodyLogLimit = -1, int responseBodyLogLimit = -1) -> TBuilder

src/Middleware/HttpLogging/src/ResponseBufferingStream.cs

@@ -22,7 +22,8 @@ internal sealed class ResponseBufferingStream : BufferingStream, IHttpResponseBo
 
     private readonly HttpContext _context;
     private readonly List<MediaTypeState> _encodings;
-    private readonly HttpLoggingOptions _options;
+    private readonly HashSet<string> _allowedResponseHeaders;
+    private readonly HttpLoggingFields _loggingFields;
     private Encoding? _encoding;
 
     private static readonly StreamPipeWriterOptions _pipeWriterOptions = new StreamPipeWriterOptions(leaveOpen: true);
@@ -32,18 +33,20 @@ internal ResponseBufferingStream(IHttpResponseBodyFeature innerBodyFeature,
         ILogger logger,
         HttpContext context,
         List<MediaTypeState> encodings,
-        HttpLoggingOptions options)
+        HashSet<string> allowedResponseHeaders,
+        HttpLoggingFields loggingFields)
         : base(innerBodyFeature.Stream, logger)
     {
         _innerBodyFeature = innerBodyFeature;
         _innerStream = innerBodyFeature.Stream;
         _limit = limit;
         _context = context;
         _encodings = encodings;
-        _options = options;
+        _allowedResponseHeaders = allowedResponseHeaders;
+        _loggingFields = loggingFields;
     }
 
-    public bool FirstWrite { get; private set; }
+    public bool HeadersWritten { get; private set; }
 
     public Stream Stream => this;
 
@@ -68,24 +71,7 @@ public override void EndWrite(IAsyncResult asyncResult)
 
     public override void Write(ReadOnlySpan<byte> span)
     {
-        var remaining = _limit - _bytesBuffered;
-        var innerCount = Math.Min(remaining, span.Length);
-
-        OnFirstWrite();
-
-        if (innerCount > 0)
-        {
-            if (span.Slice(0, innerCount).TryCopyTo(_tailMemory.Span))
-            {
-                _tailBytesBuffered += innerCount;
-                _bytesBuffered += innerCount;
-                _tailMemory = _tailMemory.Slice(innerCount);
-            }
-            else
-            {
-                BuffersExtensions.Write(this, span.Slice(0, innerCount));
-            }
-        }
+        CommonWrite(span);
 
         _innerStream.Write(span);
     }
@@ -96,39 +82,44 @@ public override async Task WriteAsync(byte[] buffer, int offset, int count, Canc
     }
 
     public override async ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
+    {
+        CommonWrite(buffer.Span);
+
+        await _innerStream.WriteAsync(buffer, cancellationToken);
+    }
+
+    private void CommonWrite(ReadOnlySpan<byte> span)
     {
         var remaining = _limit - _bytesBuffered;
-        var innerCount = Math.Min(remaining, buffer.Length);
+        var innerCount = Math.Min(remaining, span.Length);
 
         OnFirstWrite();
 
         if (innerCount > 0)
         {
-            if (_tailMemory.Length - innerCount > 0)
+            var slice = span.Slice(0, innerCount);
+            if (slice.TryCopyTo(_tailMemory.Span))
             {
-                buffer.Slice(0, innerCount).CopyTo(_tailMemory);
                 _tailBytesBuffered += innerCount;
                 _bytesBuffered += innerCount;
                 _tailMemory = _tailMemory.Slice(innerCount);
             }
             else
             {
-                BuffersExtensions.Write(this, buffer.Span);
+                BuffersExtensions.Write(this, slice);
             }
         }
-
-        await _innerStream.WriteAsync(buffer, cancellationToken);
     }
 
     private void OnFirstWrite()
     {
-        if (!FirstWrite)
+        if (!HeadersWritten)
         {
             // Log headers as first write occurs (headers locked now)
-            HttpLoggingMiddleware.LogResponseHeaders(_context.Response, _options, _logger);
+            HttpLoggingMiddleware.LogResponseHeaders(_context.Response, _loggingFields, _allowedResponseHeaders, _logger);
 
             MediaTypeHelpers.TryGetEncodingForMediaType(_context.Response.ContentType, _encodings, out _encoding);
-            FirstWrite = true;
+            HeadersWritten = true;
         }
     }
 

src/Middleware/HttpLogging/src/UpgradeFeatureLoggingDecorator.cs

@@ -11,17 +11,19 @@ internal sealed class UpgradeFeatureLoggingDecorator : IHttpUpgradeFeature
 {
     private readonly IHttpUpgradeFeature _innerUpgradeFeature;
     private readonly HttpResponse _response;
-    private readonly HttpLoggingOptions _options;
+    private readonly HashSet<string> _allowedResponseHeaders;
     private readonly ILogger _logger;
+    private readonly HttpLoggingFields _loggingFields;
 
     private bool _isUpgraded;
 
-    public UpgradeFeatureLoggingDecorator(IHttpUpgradeFeature innerUpgradeFeature, HttpResponse response, HttpLoggingOptions options, ILogger logger)
+    public UpgradeFeatureLoggingDecorator(IHttpUpgradeFeature innerUpgradeFeature, HttpResponse response, HashSet<string> allowedResponseHeaders, HttpLoggingFields loggingFields, ILogger logger)
     {
         _innerUpgradeFeature = innerUpgradeFeature ?? throw new ArgumentNullException(nameof(innerUpgradeFeature));
         _response = response ?? throw new ArgumentNullException(nameof(response));
-        _options = options ?? throw new ArgumentNullException(nameof(options));
+        _allowedResponseHeaders = allowedResponseHeaders ?? throw new ArgumentNullException(nameof(allowedResponseHeaders));
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        _loggingFields = loggingFields;
     }
 
     public bool IsUpgradableRequest => _innerUpgradeFeature.IsUpgradableRequest;
@@ -34,7 +36,7 @@ public async Task<Stream> UpgradeAsync()
 
         _isUpgraded = true;
 
-        HttpLoggingMiddleware.LogResponseHeaders(_response, _options, _logger);
+        HttpLoggingMiddleware.LogResponseHeaders(_response, _loggingFields, _allowedResponseHeaders, _logger);
 
         return upgradeStream;
     }