[WIP] Run helix tests on master

.azure/pipelines/helix-test.yml

@@ -0,0 +1,20 @@
+# Don't run CI for this config yet. We're not ready to move official builds on to Azure Pipelines
+trigger: none
+
+# Run PR validation on all branches
+pr:
+  branches:
+    include:
+    - '*'
+
+jobs:
+- job: Helix
+  timeoutInMinutes: 240
+  pool:
+    name: Hosted VS2017
+    vmImage: vs2017-win2016
+  steps:
+  - checkout: self
+    clean: true
+  - script: .\build.cmd -all -ci /t:Helix /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog
+    displayName: Run build.cmd helix target

Directory.Build.props

@@ -1,4 +1,4 @@
-<Project>
+<Project>
   <Import Project="version.props" />
 
   <PropertyGroup>

Directory.Build.targets

@@ -60,6 +60,8 @@
     <IsPackable Condition="'$(IsPackable)' == '' ">false</IsPackable>
 
     <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp3.0' ">$(MicrosoftNETCoreAppPackageVersion)</RuntimeFrameworkVersion>
+
+    <BuildHelixPayload Condition="'$(BuildHelixPayload)' == '' AND '$(IsTestProject)' == 'true'">true</BuildHelixPayload> 
   </PropertyGroup>
 
   <Import Project="eng\targets\ResolveIisReferences.targets" Condition=" '$(MSBuildProjectExtension)' != '.vcxproj' " />
@@ -68,4 +70,70 @@
   <Import Project="eng\targets\FSharp.Common.targets"  Condition="'$(MSBuildProjectExtension)' == '.fsproj'" />
   <Import Project="eng\targets\Wix.Common.targets"  Condition="'$(MSBuildProjectExtension)' == '.wixproj'" />
   <Import Project="eng\targets\Npm.Common.targets"  Condition="'$(MSBuildProjectExtension)' == '.npmproj'" />
+
+  <ItemGroup>
+    <PackageReference Include="xunit.runner.console" Version="2.4.1" Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'" />
+  </ItemGroup>
+
+  <Target Name="CreateHelixPayload" Returns="@(HelixPayload)">
+    <ItemGroup>
+      <_TargetFrameworks Remove="@(_TargetFrameworks)" />
+      <_TargetFrameworks Include="$(TargetFrameworks);$(TargetFramework)" />
+    </ItemGroup>
+
+    <MSBuild Projects="$(MSBuildProjectFullPath)"
+             Targets="_CreateHelixPayloadInner"
+             Properties="TargetFramework=%(_TargetFrameworks.Identity);IsWindowsHelixQueue=$(HelixTargetQueue.Contains('Windows'))">
+      <Output TaskParameter="TargetOutputs" ItemName="HelixPayload" />
+    </MSBuild>
+  </Target>
+
+  <Target Name="CollectXunitConsoleRunner" Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">
+    <PropertyGroup>
+      <XunitConsoleRunnerDir>$([System.IO.Path]::GetDirectoryName($(XunitConsole472Path)))</XunitConsoleRunnerDir>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <XunitConsoleRunnerFiles Include="$(XunitConsoleRunnerDir)/**/*" />
+    </ItemGroup>
+
+    <Copy SourceFiles="@(XunitConsoleRunnerFiles)" DestinationFolder="$(PublishDir)" />
+  </Target>
+
+  <Target Name="_CreateHelixPayloadInner" 
+    DependsOnTargets="Publish;CollectXunitConsoleRunner" 
+    Condition="'$(BuildHelixPayload)' == 'true' AND ($(IsWindowsHelixQueue) OR '$(TargetFrameworkIdentifier)' != '.NETFramework')" 
+    Returns="@(HelixPayload)">
+
+    <ConvertToAbsolutePath Paths="$(PublishDir)">
+      <Output TaskParameter="AbsolutePaths" PropertyName="PublishAbsoluteDir" />
+    </ConvertToAbsolutePath>
+
+    <!-- Windows NetCore -->
+    <ItemGroup Condition="'$(TargetFrameworkIdentifier)' != '.NETFramework' AND $(IsWindowsHelixQueue)" >
+        <_CopyItems Include="$(MSBuildThisFileDirectory)test\vstest\runtests.cmd" />
+    </ItemGroup>
+
+    <!-- Windows NetFramework -->
+    <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework' AND $(IsWindowsHelixQueue)" >
+        <_CopyItems Include="$(MSBuildThisFileDirectory)test\xunit\runtests.cmd" />
+    </ItemGroup>
+
+    <!-- NonWindows -->
+    <ItemGroup Condition="!$(IsWindowsHelixQueue)"  >
+        <_CopyItems Include="$(MSBuildThisFileDirectory)test\vstest\runtests.sh" />
+    </ItemGroup>
+
+    <Copy SourceFiles="@(_CopyItems)" DestinationFolder="$(PublishAbsoluteDir)" />
+
+    <ItemGroup>
+      <HelixPayload Include="$(PublishAbsoluteDir)">
+        <TestAssembly>$(TargetFileName)</TestAssembly>
+        <TestName>$(MSBuildProjectName)-$(TargetFramework)</TestName>
+        <Command Condition="$(IsWindowsHelixQueue)">runtests.cmd $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion)</Command>
+        <Command Condition="!$(IsWindowsHelixQueue)">./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion)</Command>
+        <TestTimeout>00:30:00</TestTimeout>
+      </HelixPayload>
+    </ItemGroup>
+  </Target>
 </Project>

NuGet.config

@@ -7,6 +7,7 @@
       The only allowed feed here is myget.org/aspnet-tools which is required to work around
       https://github.com/Microsoft/msbuild/issues/2914
     -->
+    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
     <add key="myget.org aspnetcore-tools" value="https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json" />
   </packageSources>
 </configuration>

build/repo.targets

@@ -101,4 +101,18 @@
       FrameworkOnlyPackages="@(SharedFrameworkOnlyPackage)" />
   </Target>
 
+  <!-- will move into korebuild -->
+  <Target Name="Helix" DependsOnTargets="Restore">
+    <ItemGroup>
+      <CsProjects Include="@(ProjectToBuild)" Condition="'%(Extension)' == '.csproj'" />
+    </ItemGroup>
+    <PropertyGroup>
+      <HelixTestProjects>@(CsProjects)</HelixTestProjects>
+    </PropertyGroup>
+    <MSBuild Projects="$(MSBuildThisFileDirectory)..\test\helix.proj"
+             Targets="Test"
+             Properties="ProjectsToTest=$(HelixTestProjects)"
+             ContinueOnError="ErrorAndStop"
+             BuildInParallel="false" />
+  </Target>
 </Project>

src/Azure/AzureAD/test/FunctionalTests/Microsoft.AspNetCore.Authentication.AzureAD.FunctionalTests.csproj

@@ -2,6 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Components/Blazor/Build/test/Microsoft.AspNetCore.Blazor.Build.Tests.csproj

@@ -30,6 +30,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\testassets\StandaloneApp\StandaloneApp.csproj" />
+    <Compile Include="..\..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />
 
     <Compile Include="$(ComponentsSharedSourceRoot)test\**\*.cs" LinkBase="Helpers" />
   </ItemGroup>

src/Components/Blazor/Build/test/RuntimeDependenciesResolverTest.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using Microsoft.AspNetCore.Testing.xunit;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -19,7 +20,8 @@ public RuntimeDependenciesResolverTest(ITestOutputHelper output)
             _output = output;
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6549
         public void FindsReferenceAssemblyGraph_ForStandaloneApp()
         {
             // Arrange

src/Components/Server/test/Microsoft.AspNetCore.Components.Server.Tests.csproj

@@ -2,6 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
+    <!-- https://github.com/aspnet/AspNetCore/issues/7202 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj

@@ -10,6 +10,8 @@
       Temporarily disabled until this runs on macOS
     -->
     <SkipTests Condition="'$(BlazorAllTests)' != 'true'">true</SkipTests>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6857 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/DataProtection/Extensions/test/DataProtectionProviderTests.cs

@@ -113,8 +113,9 @@ public void System_UsesProvidedDirectory_WithConfigurationCallback()
             });
         }
 
-        [ConditionalFact]
+        [ConditionalFact] 
         [X509StoreIsAvailable(StoreName.My, StoreLocation.CurrentUser)]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public void System_UsesProvidedDirectoryAndCertificate()
         {
             var filePath = Path.Combine(GetTestFilesPath(), "TestCert.pfx");
@@ -164,6 +165,7 @@ public void System_UsesProvidedDirectoryAndCertificate()
 
         [ConditionalFact]
         [X509StoreIsAvailable(StoreName.My, StoreLocation.CurrentUser)]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public void System_UsesProvidedCertificateNotFromStore()
         {
             using (var store = new X509Store(StoreName.My, StoreLocation.CurrentUser))

src/DataProtection/Extensions/test/Microsoft.AspNetCore.DataProtection.Extensions.Tests.csproj

@@ -7,6 +7,7 @@
   <ItemGroup>
     <Compile Include="..\..\shared\test\*.cs" />
     <Content Include="TestFiles\**\*" CopyToOutputDirectory="PreserveNewest" />
+    <Compile Include="..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />
   </ItemGroup>
 
   <ItemGroup>

src/DefaultBuilder/test/Microsoft.AspNetCore.FunctionalTests/Microsoft.AspNetCore.FunctionalTests.csproj

@@ -2,6 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Hosting/test/FunctionalTests/Microsoft.AspNetCore.Hosting.FunctionalTests.csproj

@@ -2,6 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Http/Routing/test/UnitTests/EndpointNameAddressSchemeTest.cs

@@ -173,17 +173,17 @@ public void AddressScheme_ThrowsExceptionForDuplicateEndpoints()
             var ex = Assert.Throws<InvalidOperationException>(() => addressScheme.FindEndpoints("any name"));
 
             // Assert
-            Assert.Equal(@"The following endpoints with a duplicate endpoint name were found.
-
-Endpoints with endpoint name 'name1':
-a
-b
-c
-
-Endpoints with endpoint name 'name2':
-e
-f
-", ex.Message);
+            Assert.Equal(String.Join(Environment.NewLine, @"The following endpoints with a duplicate endpoint name were found.",
+"",
+"Endpoints with endpoint name 'name1':",
+"a",
+"b",
+"c",
+"",
+"Endpoints with endpoint name 'name2':",
+"e",
+"f",
+""), ex.Message);
         }
 
         private EndpointNameAddressScheme CreateAddressScheme(params Endpoint[] endpoints)

src/Http/Routing/test/UnitTests/Internal/DfaGraphWriterTest.cs

@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using System.IO;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Routing.Patterns;
@@ -33,10 +34,9 @@ public void Write_ExcludeNonRouteEndpoint()
             graphWriter.Write(endpointsDataSource, writer);
 
             // Assert
-            Assert.Equal(@"digraph DFA {
-0 [label=""/""]
-}
-", writer.ToString());
+            Assert.Equal(String.Join(Environment.NewLine, @"digraph DFA {",
+@"0 [label=""/""]",
+"}") + Environment.NewLine, writer.ToString());
         }
 
         [Fact]
@@ -57,10 +57,9 @@ public void Write_ExcludeRouteEndpointWithSuppressMatchingMetadata()
             graphWriter.Write(endpointsDataSource, writer);
 
             // Assert
-            Assert.Equal(@"digraph DFA {
-0 [label=""/""]
-}
-", writer.ToString());
+            Assert.Equal(String.Join(Environment.NewLine, @"digraph DFA {",
+@"0 [label=""/""]",
+@"}") + Environment.NewLine, writer.ToString());
         }
 
         [Fact]
@@ -82,14 +81,13 @@ public void Write_IncludeRouteEndpointWithPolicy()
 
             // Assert
             var sdf = writer.ToString();
-            Assert.Equal(@"digraph DFA {
-0 [label=""/ HTTP: GET""]
-1 [label=""/ HTTP: *""]
-2 -> 0 [label=""HTTP: GET""]
-2 -> 1 [label=""HTTP: *""]
-2 [label=""/""]
-}
-", sdf);
+            Assert.Equal(String.Join(Environment.NewLine, @"digraph DFA {",
+@"0 [label=""/ HTTP: GET""]",
+@"1 [label=""/ HTTP: *""]",
+@"2 -> 0 [label=""HTTP: GET""]",
+@"2 -> 1 [label=""HTTP: *""]",
+@"2 [label=""/""]",
+@"}") + Environment.NewLine, sdf);
         }
     }
 }

src/Http/Routing/test/UnitTests/Matching/DefaultEndpointSelectorTest.cs

@@ -167,10 +167,8 @@ public async Task SelectAsync_MultipleValidCandidatesInGroup_ReportsAmbiguity()
 
             // Assert
             Assert.Equal(
-@"The request matched multiple endpoints. Matches: 
-
-test: /test2
-test: /test3", ex.Message);
+@"The request matched multiple endpoints. Matches: " + Environment.NewLine + Environment.NewLine +
+"test: /test2" + Environment.NewLine + "test: /test3", ex.Message);
             Assert.Null(context.Endpoint);
         }
 
@@ -204,4 +202,4 @@ private static DefaultEndpointSelector CreateSelector()
             return new DefaultEndpointSelector();
         }
     }
-}
+}

src/Identity/ApiAuthorization.IdentityServer/test/Configuration/ConfigureSigningCredentialsTests.cs

@@ -22,7 +22,8 @@ public class ConfigureSigningCredentialsTests
             UnsafeEphemeralKeySet : (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? X509KeyStorageFlags.PersistKeySet :
             X509KeyStorageFlags.DefaultKeySet);
 
-        [ConditionalFact]
+        [ConditionalFact] 
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         [FrameworkSkipCondition(RuntimeFrameworks.CLR)]
         public void Configure_AddsDevelopmentKeyFromConfiguration()
         {
@@ -62,7 +63,8 @@ public void Configure_AddsDevelopmentKeyFromConfiguration()
             }
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public void Configure_LoadsPfxCertificateCredentialFromConfiguration()
         {
             // Arrange
@@ -91,7 +93,8 @@ public void Configure_LoadsPfxCertificateCredentialFromConfiguration()
             Assert.Equal("AC8FDF4BD4C10841BD24DC88D983225D10B43BB2", key.Certificate.Thumbprint);
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public void Configure_LoadsCertificateStoreCertificateCredentialFromConfiguration()
         {
             try

src/Identity/ApiAuthorization.IdentityServer/test/Configuration/SigningKeysLoaderTests.cs

@@ -58,7 +58,8 @@ public static void LoadFromStoreCert_ThrowsIfThereIsNoCertificateAvailable()
             Assert.Equal("Couldn't find a valid certificate with subject 'Invalid' on the 'CurrentUser\\My'", exception.Message);
         }
 
-        [Fact]
+        [ConditionalFact] 
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public static void LoadFromStoreCert_SkipsCertificatesNotYetValid()
         {
             try
@@ -80,7 +81,8 @@ public static void LoadFromStoreCert_SkipsCertificatesNotYetValid()
             }
         }
 
-        [Fact]
+        [ConditionalFact] 
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public static void LoadFromStoreCert_PrefersCertificatesCloserToExpirationDate()
         {
             try
@@ -102,7 +104,8 @@ public static void LoadFromStoreCert_PrefersCertificatesCloserToExpirationDate()
             }
         }
 
-        [Fact]
+        [ConditionalFact] 
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public static void LoadFromStoreCert_SkipsExpiredCertificates()
         {
             try