[Android] Improvements to remote certificate verification in SslStream (#77386)

* Extract existing validation code into a separate class

* Implement AndroidDexBuilderTask

* Implement TrustManager proxy

* Integrate the trust manager proxy with SslStream on Android

* Update tests

* Update System.Net.Http tests

* Update System.Net.Security tests

* Fix packaging

* Propagate caught exceptions

* Build and pack .jar

* Optimize allocation and deallocation of memory for certificate data

* Fix building .jar

* Cleanup

* Remove complicated certificate copying

* Remove unnecessary JNI classes and methods

* Simplify and fix the core implementation

* Update enabled and disabled tests

* Cleanup

* Renaming

* Remove unnecessary changes

* Fix invoking validation even when the Java callbacks aren't called (no peer certificate to validate)

* Minor refactoring

* Enable more unnecessarily disabled tests

* Refactor exception handling

* Update disabled tests

* Renaming

* Remove network security config workarounds

* Keep existing active issue

* Remove unnecessary changes

* Remove unnecessary code

* Enable more disabled tests

* Fix throwing exception

* Fix intptr_t cast to Java

* Remove initialization lock

* Update naming

* Fix type casting

* Improve throwing validation exception

* Experiment with code structure

* Fix repeated calls to beginHandshake

* Make SslStream proxy mandatory

* Add missing attributes

* Free temporary buffer

* Update src/native/libs/System.Security.Cryptography.Native.Android/pal_sslstream.c

Co-authored-by: Elinor Fung <[email protected]>

* Refactor creating array of trust managers

* Add comments and clean up pal_sslstream.c

* Revert experimental change

* Remove special case for IPv6 addresses as hostnames and disable affected tests

* Fix duplicate variable after merge

* Improve code formatting

* Remove the hack with SafeDeleteContextStub

* Enable passing test

* Remove unnecessary factory

* Move clearing selected client certificate out of the remote certificate verification method

* Fix typo in comment

* Add comment with java equivalent

* Move Android specific runtime files into a separate item group

* Apply suggestions from code review

Co-authored-by: Alexander Köplinger <[email protected]>

* Update src/native/libs/build-native.proj

Co-authored-by: Alexander Köplinger <[email protected]>

* Disable test that fails on Android emualtors

Co-authored-by: Elinor Fung <[email protected]>
Co-authored-by: Alexander Köplinger <[email protected]>

Author: 3 people

eng/liveBuilds.targets

@@ -179,6 +179,11 @@
         $(LibrariesNativeArtifactsPath)*.pdb"
         IsNative="true"
         Exclude="@(ExcludeNativeLibrariesRuntimeFiles)" />
+      <LibrariesRuntimeFiles Condition="'$(TargetOS)' == 'android'"
+                             Include="
+        $(LibrariesNativeArtifactsPath)*.dex;
+        $(LibrariesNativeArtifactsPath)*.jar;"
+        IsNative="true" />
       <LibrariesRuntimeFiles Condition="'$(TargetOS)' == 'browser'"
                              Include="
         $(LibrariesNativeArtifactsPath)dotnet.js;

src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props

@@ -95,6 +95,8 @@
     <PlatformManifestFileEntry Include="libSystem.Security.Cryptography.Native.Apple.dylib" IsNative="true" />
     <PlatformManifestFileEntry Include="libSystem.Security.Cryptography.Native.Android.a" IsNative="true" />
     <PlatformManifestFileEntry Include="libSystem.Security.Cryptography.Native.Android.so" IsNative="true" />
+    <PlatformManifestFileEntry Include="libSystem.Security.Cryptography.Native.Android.dex" IsNative="true" />
+    <PlatformManifestFileEntry Include="libSystem.Security.Cryptography.Native.Android.jar" IsNative="true" />
     <PlatformManifestFileEntry Include="libSystem.Security.Cryptography.Native.OpenSsl.a" IsNative="true" />
     <PlatformManifestFileEntry Include="libSystem.Security.Cryptography.Native.OpenSsl.dylib" IsNative="true" />
     <PlatformManifestFileEntry Include="libSystem.Security.Cryptography.Native.OpenSsl.so" IsNative="true" />

src/libraries/Common/src/Interop/Android/System.Security.Cryptography.Native.Android/Interop.Ssl.cs

@@ -29,25 +29,37 @@ internal enum PAL_SSLStreamStatus
         };
 
         [LibraryImport(Interop.Libraries.AndroidCryptoNative, EntryPoint = "AndroidCryptoNative_SSLStreamCreate")]
-        internal static partial SafeSslHandle SSLStreamCreate();
+        private static partial SafeSslHandle SSLStreamCreate(IntPtr sslStreamProxyHandle);
+        internal static SafeSslHandle SSLStreamCreate(SslStream.JavaProxy sslStreamProxy)
+            => SSLStreamCreate(sslStreamProxy.Handle);
 
         [LibraryImport(Interop.Libraries.AndroidCryptoNative, EntryPoint = "AndroidCryptoNative_SSLStreamCreateWithCertificates")]
         private static partial SafeSslHandle SSLStreamCreateWithCertificates(
+            IntPtr sslStreamProxyHandle,
             ref byte pkcs8PrivateKey,
             int pkcs8PrivateKeyLen,
             PAL_KeyAlgorithm algorithm,
             IntPtr[] certs,
             int certsLen);
-        internal static SafeSslHandle SSLStreamCreateWithCertificates(ReadOnlySpan<byte> pkcs8PrivateKey, PAL_KeyAlgorithm algorithm, IntPtr[] certificates)
+        internal static SafeSslHandle SSLStreamCreateWithCertificates(
+            SslStream.JavaProxy sslStreamProxy,
+            ReadOnlySpan<byte> pkcs8PrivateKey,
+            PAL_KeyAlgorithm algorithm,
+            IntPtr[] certificates)
         {
             return SSLStreamCreateWithCertificates(
+                sslStreamProxy.Handle,
                 ref MemoryMarshal.GetReference(pkcs8PrivateKey),
                 pkcs8PrivateKey.Length,
                 algorithm,
                 certificates,
                 certificates.Length);
         }
 
+        [LibraryImport(Interop.Libraries.AndroidCryptoNative, EntryPoint = "AndroidCryptoNative_RegisterRemoteCertificateValidationCallback")]
+        internal static unsafe partial void RegisterRemoteCertificateValidationCallback(
+            delegate* unmanaged<IntPtr, bool> verifyRemoteCertificate);
+
         [LibraryImport(Interop.Libraries.AndroidCryptoNative, EntryPoint = "AndroidCryptoNative_SSLStreamInitialize")]
         private static unsafe partial int SSLStreamInitializeImpl(
             SafeSslHandle sslHandle,

src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.AcceptAllCerts.cs

@@ -96,7 +96,6 @@ await TestHelper.WhenAllCompletedOrAnyFailed(
         [OuterLoop]
         [ConditionalTheory(nameof(ClientSupportsDHECipherSuites))]
         [MemberData(nameof(InvalidCertificateServers))]
-        [SkipOnPlatform(TestPlatforms.Android, "Android rejects the certificate, the custom validation callback in .NET cannot override OS behavior in the current implementation")]
         public async Task InvalidCertificateServers_CertificateValidationDisabled_Succeeds(string url)
         {
             using (HttpClientHandler handler = CreateHttpClientHandler(allowAllCertificates: true))

src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.ServerCertificates.cs

@@ -286,7 +286,6 @@ private async Task UseCallback_BadCertificate_ExpectedPolicyErrors_Helper(string
         [OuterLoop("Uses external servers")]
         [Theory]
         [MemberData(nameof(CertificateValidationServersAndExpectedPolicies))]
-        [SkipOnPlatform(TestPlatforms.Android, "Android rejects the certificate, the custom validation callback in .NET cannot override OS behavior in the current implementation")]
         public async Task UseCallback_BadCertificate_ExpectedPolicyErrors(string url, SslPolicyErrors expectedErrors)
         {
             const int SEC_E_BUFFER_TOO_SMALL = unchecked((int)0x80090321);
@@ -310,7 +309,6 @@ public async Task UseCallback_BadCertificate_ExpectedPolicyErrors(string url, Ss
         }
 
         [Fact]
-        [SkipOnPlatform(TestPlatforms.Android, "Android rejects the certificate, the custom validation callback in .NET cannot override OS behavior in the current implementation")]
         public async Task UseCallback_SelfSignedCertificate_ExpectedPolicyErrors()
         {
             using (HttpClientHandler handler = CreateHttpClientHandler())

src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.cs

@@ -153,7 +153,7 @@ public void Properties_AddItemToDictionary_ItemPresent()
 
         [ConditionalFact]
         [SkipOnPlatform(TestPlatforms.Browser, "ServerCertificateCustomValidationCallback not supported on Browser")]
-        [SkipOnPlatform(TestPlatforms.Android, "IPv6 loopback with SSL doesn't work on Android")]
+        [SkipOnPlatform(TestPlatforms.Android, "TargetHost cannot be set to an IPv6 address on Android because the string doesn't conform to the STD 3 ASCII rules")]
         public async Task GetAsync_IPv6LinkLocalAddressUri_Success()
         {
             if (IsWinHttpHandler && UseVersion >= HttpVersion20.Value)
@@ -205,7 +205,7 @@ public async Task GetAsync_IPBasedUri_Success(IPAddress address)
 
             if (PlatformDetection.IsAndroid && options.UseSsl && address == IPAddress.IPv6Loopback)
             {
-                throw new SkipTestException("IPv6 loopback with SSL doesn't work on Android");
+                throw new SkipTestException("TargetHost cannot be set to an IPv6 address on Android because the string doesn't conform to the STD 3 ASCII rules");
             }
 
             await LoopbackServerFactory.CreateServerAsync(async (server, url) =>
@@ -287,7 +287,7 @@ public async Task GetAsync_SecureAndNonSecureIPBasedUri_CorrectlyFormatted(IPAdd
 
             if (PlatformDetection.IsAndroid && useSsl && address == IPAddress.IPv6Loopback)
             {
-                throw new SkipTestException("IPv6 loopback with SSL doesn't work on Android");
+                throw new SkipTestException("TargetHost cannot be set to an IPv6 address on Android because the string doesn't conform to the STD 3 ASCII rules");
             }
 
             var options = new LoopbackServer.Options { Address = address, UseSsl = useSsl };

src/libraries/Common/tests/System/Net/Http/TestHelper.cs

@@ -173,9 +173,6 @@ public static SocketsHttpHandler CreateSocketsHttpHandler(bool allowAllCertifica
             // Browser doesn't support ServerCertificateCustomValidationCallback
             if (allowAllCertificates && PlatformDetection.IsNotBrowser)
             {
-                // On Android, it is not enough to set the custom validation callback, the certificates also need to be trusted by the OS.
-                // See HttpClientHandlerTestBase.SocketsHttpHandler.cs:CreateHttpClientHandler for more details.
-
                 handler.SslOptions.RemoteCertificateValidationCallback = delegate { return true; };
             }
 

src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTestBase.SocketsHttpHandler.cs

@@ -37,13 +37,6 @@ protected static HttpClientHandler CreateHttpClientHandler(Version useVersion =
             // Browser doesn't support ServerCertificateCustomValidationCallback
             if (allowAllCertificates && PlatformDetection.IsNotBrowser)
             {
-                // On Android, it is not enough to set the custom validation callback, the certificates also need to be trusted by the OS.
-                // The public keys of our self-signed certificates that are used by the loopback server are part of the System.Net.TestData
-                // package and they can be included in a the Android test apk by adding the following property to the test's .csproj:
-                //
-                //    <IncludeNetworkSecurityConfig Condition="'$(TargetOS)' == 'android'">true</IncludeNetworkSecurityConfig>
-                //
-
                 handler.ServerCertificateCustomValidationCallback = TestHelper.AllowAllCertificates;
             }
 

src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs

@@ -4076,7 +4076,6 @@ public abstract class SocketsHttpHandler_SecurityTest : HttpClientHandlerTestBas
         public SocketsHttpHandler_SecurityTest(ITestOutputHelper output) : base(output) { }
 
         [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindows7))]
-        [SkipOnPlatform(TestPlatforms.Android, "Self-signed certificates are rejected by Android before the .NET validation is reached")]
         public async Task SslOptions_CustomTrust_Ok()
         {
             X509Certificate2Collection caCerts = new X509Certificate2Collection();
@@ -4113,7 +4112,6 @@ await LoopbackServerFactory.CreateClientAndServerAsync(
         }
 
         [Fact]
-        [SkipOnPlatform(TestPlatforms.Android, "Self-signed certificates are rejected by Android before the .NET validation is reached")]
         public async Task SslOptions_InvalidName_Throws()
         {
             X509Certificate2Collection caCerts = new X509Certificate2Collection();
@@ -4144,7 +4142,6 @@ await LoopbackServerFactory.CreateClientAndServerAsync(
         }
 
         [Fact]
-        [SkipOnPlatform(TestPlatforms.Android, "Self-signed certificates are rejected by Android before the .NET validation is reached")]
         public async Task SslOptions_CustomPolicy_IgnoresNameMismatch()
         {
             X509Certificate2Collection caCerts = new X509Certificate2Collection();

src/libraries/System.Net.Http/tests/FunctionalTests/SocksProxyTest.cs

@@ -37,7 +37,7 @@ public async Task TestLoopbackAsync(string scheme, bool useSsl, bool useAuth, st
 
             if (PlatformDetection.IsAndroid && useSsl && host == "::1")
             {
-                throw new SkipTestException("IPv6 loopback with SSL doesn't work on Android");
+                throw new SkipTestException("TargetHost cannot be set to an IPv6 address on Android because the string doesn't conform to the STD 3 ASCII rules");
             }
 
             await LoopbackServerFactory.CreateClientAndServerAsync(