Support tab characters in headers and unify validation code between servers. (#40671)

* Support tab characters in headers and unify validation code between servers.

* Preserve non-ASCII header value char treatment in http.sys.

Author: adityamandaleeka

src/Servers/HttpSys/src/Microsoft.AspNetCore.Server.HttpSys.csproj

@@ -18,6 +18,7 @@
     <Compile Include="$(SharedSourceRoot)HttpSys\**\*.cs" />
     <Compile Include="$(SharedSourceRoot)Buffers.MemoryPool\*.cs" LinkBase="MemoryPool" />
     <Compile Include="$(SharedSourceRoot)ServerInfrastructure\StringUtilities.cs" LinkBase="ServerInfrastructure\StringUtilities.cs" />
+    <Compile Include="$(SharedSourceRoot)ServerInfrastructure\HttpCharacters.cs" LinkBase="ServerInfrastructure\HttpCharacters.cs" />
     <Compile Include="$(SharedSourceRoot)TaskToApm.cs" />
   </ItemGroup>
 

src/Servers/HttpSys/test/FunctionalTests/ResponseHeaderTests.cs

@@ -113,6 +113,27 @@ public async Task ResponseHeaders_ServerSendsCustomHeaders_Success()
             }
         }
 
+        [ConditionalFact]
+        public async Task ResponseHeaders_ServerSendsNonAsciiHeaders_Success()
+        {
+            string address;
+            using (Utilities.CreateHttpServer(out address, httpContext =>
+            {
+                var responseInfo = httpContext.Features.Get<IHttpResponseFeature>();
+                var responseHeaders = responseInfo.Headers;
+                responseHeaders["Custom-Header1"] = new string[] { "Dašta" };
+                return Task.FromResult(0);
+            }))
+            {
+                var socketsHttpHandler = new SocketsHttpHandler() { ResponseHeaderEncodingSelector = (_, _) => Encoding.UTF8 };
+                var httpClient = new HttpClient(socketsHttpHandler);
+                var response = await httpClient.GetAsync(address);
+                response.EnsureSuccessStatusCode();
+                Assert.True(response.Headers.TryGetValues("Custom-Header1", out var header));
+                Assert.Equal("Dašta", header.Single());
+            }
+        }
+
         [ConditionalFact]
         public async Task ResponseHeaders_ServerSendsConnectionClose_Closed()
         {

src/Servers/Kestrel/Core/src/Internal/Http/Http1Connection.cs

@@ -8,6 +8,7 @@
 using System.IO.Pipelines;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Connections;
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 

src/Servers/Kestrel/Core/src/Internal/Http/HttpHeaders.cs

@@ -10,7 +10,6 @@
 using System.Runtime.InteropServices;
 using System.Text;
 using Microsoft.AspNetCore.Http;
-using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 using Microsoft.Extensions.Primitives;
 using Microsoft.Net.Http.Headers;
 

src/Servers/Kestrel/Core/src/Internal/Http/HttpParser.cs

@@ -6,6 +6,7 @@
 using System.Diagnostics;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http

src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs

@@ -16,6 +16,7 @@
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
 using Microsoft.Extensions.Primitives;
 using Microsoft.Net.Http.Headers;
+using HttpCharacters = Microsoft.AspNetCore.Http.HttpCharacters;
 using HttpMethods = Microsoft.AspNetCore.Http.HttpMethods;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2

src/Servers/Kestrel/Core/src/Internal/Http3/Http3Stream.cs

@@ -15,6 +15,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Primitives;
 using Microsoft.Net.Http.Headers;
+using HttpCharacters = Microsoft.AspNetCore.Http.HttpCharacters;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http3
 {

src/Servers/Kestrel/Core/src/Internal/KestrelServerImpl.cs

@@ -11,6 +11,7 @@
 using Microsoft.AspNetCore.Connections;
 using Microsoft.AspNetCore.Hosting.Server;
 using Microsoft.AspNetCore.Hosting.Server.Features;
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;

src/Servers/Kestrel/Core/test/HttpResponseHeadersTests.cs

@@ -241,6 +241,18 @@ public void AddingNonAsciiCharactersWithCustomEncoderWorks(string value)
             ((IDictionary<string, StringValues>)responseHeaders).Add("Unknown", value);
         }
 
+        [Fact]
+        public void AddingTabCharactersToHeaderPropertyWorks()
+        {
+            var responseHeaders = (IHeaderDictionary)new HttpResponseHeaders();
+
+            // Known special header
+            responseHeaders.Allow = "Da\tta";
+
+            // Unknown header fallback
+            responseHeaders.Accept = "Da\tta";
+        }
+
         [Fact]
         public void ThrowsWhenAddingHeaderAfterReadOnlyIsSet()
         {

src/Shared/HttpSys/RequestProcessing/HeaderCollection.cs

@@ -272,12 +272,10 @@ public static void ValidateHeaderCharacters(string headerCharacters)
         {
             if (headerCharacters != null)
             {
-                foreach (var ch in headerCharacters)
+                var invalid = HttpCharacters.IndexOfInvalidFieldValueCharExtended(headerCharacters);
+                if (invalid >= 0)
                 {
-                    if (ch < 0x20)
-                    {
-                        throw new InvalidOperationException(string.Format(CultureInfo.CurrentCulture, "Invalid control character in header: 0x{0:X2}", (byte)ch));
-                    }
+                    throw new InvalidOperationException(string.Format(CultureInfo.CurrentCulture, "Invalid control character in header: 0x{0:X2}", headerCharacters[invalid]));
                 }
             }
         }