[WIP] Run helix tests on master

.azure/pipelines/helix-test.yml

@@ -0,0 +1,28 @@
+# Don't run CI for this config yet. We're not ready to move official builds on to Azure Pipelines
+trigger: none
+
+# Run PR validation on all branches
+pr:
+  branches:
+    include:
+    - '*'
+
+jobs:
+- job: Helix
+  timeoutInMinutes: 240
+  pool:
+    name: Hosted VS2017
+    vmImage: vs2017-win2016
+  steps:
+  - checkout: self
+    clean: true
+  - script: .\build.cmd -all -ci /t:Helix /p:IsHelixJob=true /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\SendToHelix.binlog
+    displayName: Run build.cmd helix target
+  - task: PublishBuildArtifacts@1
+    displayName: Publish Logs to VSTS
+    inputs:
+      PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log'
+      PublishLocation: Container
+      ArtifactName: $(Agent.Os)_$(Agent.JobName)
+    continueOnError: true
+    condition: always() 

Directory.Build.props

@@ -1,4 +1,4 @@
-<Project>
+<Project>
   <Import Project="version.props" />
 
   <PropertyGroup>

Directory.Build.targets

@@ -60,6 +60,8 @@
     <IsPackable Condition="'$(IsPackable)' == '' ">false</IsPackable>
 
     <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp3.0' ">$(MicrosoftNETCoreAppPackageVersion)</RuntimeFrameworkVersion>
+
+    <BuildHelixPayload Condition="'$(BuildHelixPayload)' == '' AND '$(IsTestProject)' == 'true'">true</BuildHelixPayload> 
   </PropertyGroup>
 
   <Import Project="eng\targets\ResolveIisReferences.targets" Condition=" '$(MSBuildProjectExtension)' != '.vcxproj' " />
@@ -68,4 +70,96 @@
   <Import Project="eng\targets\FSharp.Common.targets"  Condition="'$(MSBuildProjectExtension)' == '.fsproj'" />
   <Import Project="eng\targets\Wix.Common.targets"  Condition="'$(MSBuildProjectExtension)' == '.wixproj'" />
   <Import Project="eng\targets\Npm.Common.targets"  Condition="'$(MSBuildProjectExtension)' == '.npmproj'" />
+
+  <!-- Move to CSharp.Common.props -->
+  <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework' AND '$(IsHelixJob)' == 'true' ">
+    <PackageReference Include="xunit.runner.console" Version="2.4.1" />
+  </ItemGroup>  
+
+  <!-- Prepares the test projects for helix by including xunit and publishing -->
+  <Target Name="PrepareHelixPayload" Returns="@(HelixDirectory)">
+    <ItemGroup>
+      <_TargetFrameworks Remove="@(_TargetFrameworks)" />
+      <_TargetFrameworks Include="$(TargetFrameworks);$(TargetFramework)" />
+    </ItemGroup>
+
+    <MSBuild Projects="$(MSBuildProjectFullPath)"
+             Targets="_PrepareHelixPayloadInner"
+             Properties="TargetFramework=%(_TargetFrameworks.Identity);IsWindowsHelixQueue=$(HelixTargetQueue.Contains('Windows'))">
+      <Output TaskParameter="TargetOutputs" ItemName="HelixDirectory" />
+    </MSBuild>
+  </Target>
+
+  <Target Name="_PrepareHelixPayloadInner" 
+    DependsOnTargets="Publish" 
+    Condition="'$(BuildHelixPayload)' == 'true'" 
+    Returns="@(HelixDirectory)">
+
+    <ItemGroup>
+      <HelixDirectory Include="$(MSBuildProjectFullPath)" />
+    </ItemGroup>
+  </Target>
+
+  <!-- Build the actual helix work items to send to helix queues -->
+  <Target Name="CreateHelixPayload" Returns="@(HelixPayload)">
+    <ItemGroup>
+      <_TargetFrameworks Remove="@(_TargetFrameworks)" />
+      <_TargetFrameworks Include="$(TargetFrameworks);$(TargetFramework)" />
+    </ItemGroup>
+
+    <MSBuild Projects="$(MSBuildProjectFullPath)"
+             Targets="_CreateHelixPayloadInner"
+             Properties="TargetFramework=%(_TargetFrameworks.Identity);IsWindowsHelixQueue=$(HelixTargetQueue.Contains('Windows'))">
+      <Output TaskParameter="TargetOutputs" ItemName="HelixPayload" />
+    </MSBuild>
+  </Target>
+
+  <Target Name="CollectXunitConsoleRunner" Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">
+    <PropertyGroup>
+      <XunitConsoleRunnerDir>$([System.IO.Path]::GetDirectoryName($(XunitConsole472Path)))</XunitConsoleRunnerDir>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <XunitConsoleRunnerFiles Include="$(XunitConsoleRunnerDir)/**/*" />
+    </ItemGroup>
+
+    <Copy SourceFiles="@(XunitConsoleRunnerFiles)" DestinationFolder="$(PublishDir)" />
+  </Target>
+
+  <Target Name="_CreateHelixPayloadInner" 
+    DependsOnTargets="CollectXunitConsoleRunner" 
+    Condition="($(IsWindowsHelixQueue) OR '$(TargetFrameworkIdentifier)' != '.NETFramework')" 
+    Returns="@(HelixPayload)">
+
+    <ConvertToAbsolutePath Paths="$(PublishDir)">
+      <Output TaskParameter="AbsolutePaths" PropertyName="PublishAbsoluteDir" />
+    </ConvertToAbsolutePath>
+
+    <!-- Windows NetCore -->
+    <ItemGroup Condition="'$(TargetFrameworkIdentifier)' != '.NETFramework' AND $(IsWindowsHelixQueue)" >
+        <_CopyItems Include="$(MSBuildThisFileDirectory)test\vstest\runtests.cmd" />
+    </ItemGroup>
+
+    <!-- Windows NetFramework -->
+    <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework' AND $(IsWindowsHelixQueue)" >
+        <_CopyItems Include="$(MSBuildThisFileDirectory)test\xunit\runtests.cmd" />
+    </ItemGroup>
+
+    <!-- NonWindows -->
+    <ItemGroup Condition="!$(IsWindowsHelixQueue)"  >
+        <_CopyItems Include="$(MSBuildThisFileDirectory)test\vstest\runtests.sh" />
+    </ItemGroup>
+
+    <Copy SourceFiles="@(_CopyItems)" DestinationFolder="$(PublishAbsoluteDir)" />
+
+    <ItemGroup>
+      <HelixPayload Include="$(PublishAbsoluteDir)">
+        <TestAssembly>$(TargetFileName)</TestAssembly>
+        <TestName>$(MSBuildProjectName)-$(TargetFramework)</TestName>
+        <Command Condition="$(IsWindowsHelixQueue)">runtests.cmd $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion)</Command>
+        <Command Condition="!$(IsWindowsHelixQueue)">./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion)</Command>
+        <TestTimeout>00:30:00</TestTimeout>
+      </HelixPayload>
+    </ItemGroup>
+  </Target>
 </Project>

NuGet.config

@@ -7,6 +7,7 @@
       The only allowed feed here is myget.org/aspnet-tools which is required to work around
       https://github.com/Microsoft/msbuild/issues/2914
     -->
+    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
     <add key="myget.org aspnetcore-tools" value="https://dotnet.myget.org/F/aspnetcore-tools/api/v3/index.json" />
   </packageSources>
 </configuration>

build/repo.targets

@@ -111,4 +111,30 @@
              Properties="$(BuildProperties);__DummyTarget=GenerateBuildAssetManifest" />
   </Target>
 
+  <!-- will move into korebuild -->
+  <Target Name="HelixPublish" DependsOnTargets="Restore">
+    <ItemGroup>
+      <CsProjects Include="@(ProjectToBuild)" Condition="'%(Extension)' == '.csproj'" />
+    </ItemGroup>
+    <MSBuild Projects="@(CsProjects)"
+              Targets="PrepareHelixPayload"
+              BuildInParallel="false">
+      <Output TaskParameter="TargetOutputs" ItemName="HelixDirectory" />
+    </MSBuild>
+
+    <ItemGroup>
+      <HelixTestProject Include="@(HelixDirectory)" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="Helix" DependsOnTargets="HelixPublish">
+    <PropertyGroup>
+      <HelixTestProjects>@(HelixTestProject)</HelixTestProjects>
+    </PropertyGroup>
+    <MSBuild Projects="$(MSBuildThisFileDirectory)..\test\helix.proj"
+             Targets="Test"
+             Properties="ProjectsToTest=$(HelixTestProjects)"
+             ContinueOnError="ErrorAndStop"
+             BuildInParallel="false" />
+  </Target>
 </Project>

eng/targets/CSharp.Common.targets

@@ -5,7 +5,7 @@
     <TargetFramework>net$(TargetFrameworkVersion.Substring(1).Replace('.',''))</TargetFramework>
     <TargetFrameworkIdentifier>.NETFramework</TargetFrameworkIdentifier>
   </PropertyGroup>
-
+  
   <Import Project="Packaging.targets" />
   <Import Project="ResolveReferences.targets" />
 </Project>

src/Azure/AzureAD/test/FunctionalTests/Microsoft.AspNetCore.Authentication.AzureAD.FunctionalTests.csproj

@@ -2,6 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Components/Blazor/Build/test/Microsoft.AspNetCore.Blazor.Build.Tests.csproj

@@ -30,6 +30,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\testassets\StandaloneApp\StandaloneApp.csproj" />
+    <Compile Include="..\..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />
 
     <Compile Include="$(ComponentsSharedSourceRoot)test\**\*.cs" LinkBase="Helpers" />
   </ItemGroup>

src/Components/Blazor/Build/test/RuntimeDependenciesResolverTest.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using Microsoft.AspNetCore.Testing.xunit;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -19,7 +20,8 @@ public RuntimeDependenciesResolverTest(ITestOutputHelper output)
             _output = output;
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6549
         public void FindsReferenceAssemblyGraph_ForStandaloneApp()
         {
             // Arrange

src/Components/Server/test/Microsoft.AspNetCore.Components.Server.Tests.csproj

@@ -2,6 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
+    <!-- https://github.com/aspnet/AspNetCore/issues/7202 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj

@@ -10,6 +10,8 @@
       Temporarily disabled until this runs on macOS
     -->
     <SkipTests Condition="'$(BlazorAllTests)' != 'true'">true</SkipTests>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6857 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/DataProtection/Extensions/test/DataProtectionProviderTests.cs

@@ -113,8 +113,9 @@ public void System_UsesProvidedDirectory_WithConfigurationCallback()
             });
         }
 
-        [ConditionalFact]
+        [ConditionalFact] 
         [X509StoreIsAvailable(StoreName.My, StoreLocation.CurrentUser)]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public void System_UsesProvidedDirectoryAndCertificate()
         {
             var filePath = Path.Combine(GetTestFilesPath(), "TestCert.pfx");
@@ -164,6 +165,7 @@ public void System_UsesProvidedDirectoryAndCertificate()
 
         [ConditionalFact]
         [X509StoreIsAvailable(StoreName.My, StoreLocation.CurrentUser)]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
         public void System_UsesProvidedCertificateNotFromStore()
         {
             using (var store = new X509Store(StoreName.My, StoreLocation.CurrentUser))

src/DataProtection/Extensions/test/Microsoft.AspNetCore.DataProtection.Extensions.Tests.csproj

@@ -7,6 +7,7 @@
   <ItemGroup>
     <Compile Include="..\..\shared\test\*.cs" />
     <Content Include="TestFiles\**\*" CopyToOutputDirectory="PreserveNewest" />
+    <Compile Include="..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />
   </ItemGroup>
 
   <ItemGroup>

src/DefaultBuilder/test/Microsoft.AspNetCore.FunctionalTests/Microsoft.AspNetCore.FunctionalTests.csproj

@@ -2,6 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Hosting/Hosting/test/Microsoft.AspNetCore.Hosting.Tests.csproj

@@ -5,6 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Include="..\..\..\..\test\xunit\SkipOnHelixAttribute.cs" />    
     <Content Include="testroot\**\*" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="PreserveNewest" />
   </ItemGroup>
 

src/Hosting/Hosting/test/WebHostTests.cs

@@ -14,6 +14,7 @@
 using Microsoft.AspNetCore.Hosting.Server.Features;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Features;
+using Microsoft.AspNetCore.Testing.xunit;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
@@ -194,7 +195,8 @@ public async Task WebHostShutsDownWhenTokenTriggers()
             }
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7291
         public async Task WebHostStopAsyncUsesDefaultTimeoutIfGivenTokenDoesNotFire()
         {
             var data = new Dictionary<string, string>
@@ -233,7 +235,8 @@ public async Task WebHostStopAsyncUsesDefaultTimeoutIfGivenTokenDoesNotFire()
             }
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7291
         public async Task WebHostStopAsyncUsesDefaultTimeoutIfNoTokenProvided()
         {
             var data = new Dictionary<string, string>
@@ -308,7 +311,8 @@ public async Task WebHostStopAsyncCanBeCancelledEarly()
             }
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7291
         public void WebHostApplicationLifetimeEventsOrderedCorrectlyDuringShutdown()
         {
             using (var host = CreateBuilder()
@@ -1325,4 +1329,4 @@ public static IWebHostBuilder UseFakeServer(this IWebHostBuilder builder)
             return builder.ConfigureServices(services => services.AddSingleton<IServer, WebHostTests.FakeServer>());
         }
     }
-}
+}

src/Hosting/test/FunctionalTests/Microsoft.AspNetCore.Hosting.FunctionalTests.csproj

@@ -2,6 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.0</TargetFramework>
+    <!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
+    <BuildHelixPayload>false</BuildHelixPayload>
   </PropertyGroup>
 
   <ItemGroup>

src/Http/Routing/test/UnitTests/EndpointNameAddressSchemeTest.cs

@@ -173,17 +173,17 @@ public void AddressScheme_ThrowsExceptionForDuplicateEndpoints()
             var ex = Assert.Throws<InvalidOperationException>(() => addressScheme.FindEndpoints("any name"));
 
             // Assert
-            Assert.Equal(@"The following endpoints with a duplicate endpoint name were found.
-
-Endpoints with endpoint name 'name1':
-a
-b
-c
-
-Endpoints with endpoint name 'name2':
-e
-f
-", ex.Message);
+            Assert.Equal(String.Join(Environment.NewLine, @"The following endpoints with a duplicate endpoint name were found.",
+"",
+"Endpoints with endpoint name 'name1':",
+"a",
+"b",
+"c",
+"",
+"Endpoints with endpoint name 'name2':",
+"e",
+"f",
+""), ex.Message);
         }
 
         private EndpointNameAddressScheme CreateAddressScheme(params Endpoint[] endpoints)