Split integration tests to single tfm and multi tfm project

Directory.Build.props

@@ -134,10 +134,12 @@
   <!-- Test config -->
   <PropertyGroup>
     <TestRunnerName>MSTest</TestRunnerName>
+    <!-- Avoid making the TestRunnerAdditionalArguments a global property that we cannot change in the run, by passing TestRunnerExternalArguments externally instead. -->
+    <TestRunnerAdditionalArguments Condition=" '$(TestRunnerExternalArguments)' != ''">$(TestRunnerExternalArguments)</TestRunnerAdditionalArguments>
     <!-- Skip windows only tests on non-windows systems. -->
     <TestRunnerAdditionalArguments Condition=" '$(OS)' != 'Windows_NT' ">$(TestRunnerAdditionalArguments) --filter "TestCategory!=Windows&amp;TestCategory!=Windows-Review"</TestRunnerAdditionalArguments>
-    <!-- Do not capture test output on CI to help community see errors -->
-    <TestCaptureOutput Condition=" '$(ContinuousIntegrationBuild)' == 'true' ">false</TestCaptureOutput>
+    <!-- Allow us to see errors instead of capturing everything into files. -->
+    <TestCaptureOutput>false</TestCaptureOutput>
   </PropertyGroup>
 
 </Project>

TestPlatform.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.0.31519.33
+# Visual Studio Version 18
+VisualStudioVersion = 18.5.11605.296
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{ED0C35EB-7F31-4841-A24F-8EB708FFA959}"
 EndProject
@@ -200,6 +200,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.TestPlatform.Acce
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.TestPlatform.CommunicationUtilities.Platform.UnitTests", "test\Microsoft.TestPlatform.CommunicationUtilities.Platform.UnitTests\Microsoft.TestPlatform.CommunicationUtilities.Platform.UnitTests.csproj", "{E602AB2B-048F-4317-8A6B-A225C00173FA}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.TestPlatform.Library.IntegrationTests", "test\Microsoft.TestPlatform.Library.IntegrationTests\Microsoft.TestPlatform.Library.IntegrationTests.csproj", "{E01BC153-8BE9-95E9-723E-2788114A37B6}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -470,6 +472,10 @@ Global
 		{E602AB2B-048F-4317-8A6B-A225C00173FA}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{E602AB2B-048F-4317-8A6B-A225C00173FA}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{E602AB2B-048F-4317-8A6B-A225C00173FA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E01BC153-8BE9-95E9-723E-2788114A37B6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E01BC153-8BE9-95E9-723E-2788114A37B6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E01BC153-8BE9-95E9-723E-2788114A37B6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E01BC153-8BE9-95E9-723E-2788114A37B6}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -552,6 +558,7 @@ Global
 		{4454139C-174A-4DD8-8E76-657D9F03BA09} = {D27E1CB4-C641-4C6C-A140-EF5F6215AE29}
 		{450D5371-32A1-4667-987E-40D75F035233} = {46250E12-4CF1-4051-B4A7-80C8C06E0068}
 		{E602AB2B-048F-4317-8A6B-A225C00173FA} = {376C19DE-31E2-4FF6-88FC-0D0D6233C999}
+		{E01BC153-8BE9-95E9-723E-2788114A37B6} = {46250E12-4CF1-4051-B4A7-80C8C06E0068}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {0541B30C-FF51-4E28-B172-83F5F3934BCD}

azure-pipelines-official.yml

@@ -190,14 +190,25 @@ extends:
             - ${{ if eq(parameters.SkipTests, False) }}:
               # -ci is allowing to import some environment variables and some required configurations
               # -nobl avoid overwriting binlog of the main Build
+              - script: Test.cmd
+                  -configuration Release
+                  -ci
+                  -nobl
+                name: Test
+                displayName: Unit Test
+                env:
+                  DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet
+              # -ci is allowing to import some environment variables and some required configurations
+              # -nobl avoid overwriting binlog of the main Build
               - script: Test.cmd
                   -configuration Release
                   -ci
                   -nobl
                   -integrationTest
+                  -compatibilityTest
                   -performanceTest
-                name: Test
-                displayName: Test
+                name: IntegrationTest
+                displayName: Integration Test
                 env:
                   DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet
 

azure-pipelines.yml

@@ -103,6 +103,18 @@ stages:
             /p:SourceBranchName=$(Build.SourceBranchName)
           name: Build
           displayName: Build
+
+        # -ci is allowing to import some environment variables and some required configurations
+        # -nobl avoid overwriting binlog of the main Build
+        - script: Test.cmd
+            -configuration $(_BuildConfig)
+            -ci
+            -nobl
+            /bl:$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/UnitTest.binlog
+          name: Test
+          displayName: Unit Test
+          env:
+            DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet
 
         # -ci is allowing to import some environment variables and some required configurations
         # -nobl avoid overwriting binlog of the main Build
@@ -111,10 +123,9 @@ stages:
             -ci
             -nobl
             -integrationTest
-            -performanceTest
-            /bl:$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Test.binlog
-          name: Test
-          displayName: Test
+            /bl:$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/IntegrationTest.binlog
+          name: IntegrationTest
+          displayName: Integration Test
           env:
             DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet
 

eng/build.ps1

@@ -1,4 +1,4 @@
-[CmdletBinding(PositionalBinding=$false)]
+[CmdletBinding(PositionalBinding = $false)]
 Param(
   [string][Alias('c')]$configuration = "Debug",
   [string]$platform = $null,
@@ -14,8 +14,15 @@ Param(
   [switch] $rebuild,
   [switch] $deploy,
   [switch][Alias('t')]$test,
+  [string] $filter,
+  [switch] $smokeTest,
   [switch] $integrationTest,
   [switch] $performanceTest,
+  [switch] $compatibilityTest,
+  # Skip the build when running multiple categories from the integration tests. This is useful mostly in CI where we want to split the runs to different jobs, but
+  # they all fall back to the same project and initialization.
+  [switch] $skipIntegrationTestBuild,
+  [switch] $compatibilityTestBuild,
   [switch] $sign,
   [switch] $pack,
   [switch] $publish,
@@ -31,13 +38,103 @@ Param(
   [switch] $excludePrereleaseVS,
   [switch] $nativeToolsOnMachine,
   [switch] $help,
-  [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
+  [Parameter(ValueFromRemainingArguments = $true)][String[]]$properties
 )
 
 # Add steps that need to happen before build here
 
+if ($properties -like "*TestRunnerAdditionalArguments*--filter*") {
+  throw "Use --filter instead of passing filter as an additional argument to TestRunnerAdditionalArguments."
+}
+
+if ($smokeTest -and $integrationTest) {
+  throw "Cannot specify both smoke and integration tests. Smoke tests are a subset of integration tests, so specifying both is redundant and will run all integration tests."
+}
+
+$filters = @()
+# This translates to properties on test context, the only way MSTest allows us to pass info dynamically to AssemblyInitialize
+$testParameters = @{}
+if ($skipIntegrationTestBuild) {
+  $testParameters['SkipIntegrationTestBuild'] = $true
+}
+
+if ($filter) {
+  $filters += $filter
+}
+
+if ([System.Environment]::OSVersion.Platform -notlike "Win*") {
+  $filters += "TestCategory!=Windows&TestCategory!=Windows-Review"
+}
+
+if ($smokeTest) {
+  $filters += "TestCategory=Smoke"
+}
+else {
+  # Don't exclude smoke tests if not specified explicitly, those are integration tests that should
+  # run by default when running integration tests. We want to make sure we can run just Smoke tests,
+  # but should not skip them when not specified.
+  # $filters += "TestCategory!=Smoke"
+}
+
+if ($compatibilityTestBuild) {
+  $testParameters['CompatibilityTestBuild'] = $true
+}
+
+if ($compatibilityTest) {
+  $testParameters['BuildCompatibility'] = $true
+  if (-not $integrationTest) {
+    # We specified just compatibility so run just compatibility tests. If there are both
+    # we need to run all, but we don't have a filter for uncategorized tests, https://github.com/microsoft/testfx/issues/5136
+    # so we simply don't provide an include filter.
+    $filters += "TestCategory=Compatibility"
+  }
+}
+else {
+  $filters += "TestCategory!=Compatibility"
+}
+
+if ($performanceTest) {
+  # We don't have any perf tests in the library.integrationtest.csproj, so providing this alone will fail
+  # but we will use it together with compatibility tests in a nightly run, so good enough for now.
+  if (-not $integrationTest) {
+    # We specified just perf tests so run just perf tests. If there are both
+    # we need to run all, but we don't have a filter for uncategorized tests, https://github.com/microsoft/testfx/issues/5136
+    # so we simply don't provide an include filter.
+    $filters += "TestCategory=TelemetryPerf"
+  }
+
+}
+else {
+  $filters += "TestCategory!=TelemetryPerf"
+}
+
+$null = $PSBoundParameters.Remove("filter")
+$null = $PSBoundParameters.Remove("smokeTest")
+$null = $PSBoundParameters.Remove("compatibilityTest")
+$null = $PSBoundParameters.Remove("performanceTest")
+$null = $PSBoundParameters.Remove("skipIntegrationTestBuild")
+$null = $PSBoundParameters.Remove("compatibilityTestBuild")
+
+if ($integrationTest -or $performanceTest -or $compatibilityTest -or $smokeTest) {
+  # Rest of the infra knows nothing about or additional categories for tests. They simply consider them
+  # integration tests, so mark that.
+  $PSBoundParameters['integrationTest'] = $true
+  # This is also non-default, normally we would run also unit tests, but if we filter anything that matches 0 tests in a project
+  # the project will fail.
+  $PSBoundParameters['test'] = $false
+}
+
+if ($filters.Count -gt 0 -or $testParameters.Count -gt 0) {
+  # We have to double escape, otherwise the filter is passed as string with & in it and interpreted directly as a separate command to run.
+  $filterString = "--filter \`"$($filters -join '&')\`""
+  $testParameterString = ($testParameters.GetEnumerator() | ForEach-Object { "--test-parameter $($_.Key)=$($_.Value)" }) -join ' '
+
+  $PSBoundParameters['properties'] += "/p:TestRunnerExternalArguments=$filterString $testParameterString"
+  Write-Host "Running tests with filter: $filterString and additional parameters: $testParameterString"
+}
 
 # Call the build script provided by Arcade
 & $PSScriptRoot/common/build.ps1 @PSBoundParameters
+
 # Forward exit code of the parent script
 exit $LastExitCode

src/Microsoft.TestPlatform.Common/Friends.cs

@@ -27,6 +27,7 @@
 [assembly: InternalsVisibleTo("Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
 [assembly: InternalsVisibleTo("Microsoft.TestPlatform.TestUtilities, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
 [assembly: InternalsVisibleTo("Microsoft.TestPlatform.Acceptance.IntegrationTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
+[assembly: InternalsVisibleTo("Microsoft.TestPlatform.Library.IntegrationTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
 [assembly: InternalsVisibleTo("vstest.ProgrammerTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
 [assembly: InternalsVisibleTo("TranslationLayer.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
 [assembly: InternalsVisibleTo("Microsoft.TestPlatform.Perf.IntegrationTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]

src/vstest.console/Friends.cs

@@ -17,5 +17,6 @@
 [assembly: InternalsVisibleTo("vstest.ProgrammerTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
 [assembly: InternalsVisibleTo("TestPlatform.Playground, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
 [assembly: InternalsVisibleTo("Microsoft.TestPlatform.Acceptance.IntegrationTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
+[assembly: InternalsVisibleTo("Microsoft.TestPlatform.Library.IntegrationTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
 
 #endregion

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ArgumentProcessorTests.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using Microsoft.TestPlatform.TestUtilities;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
 namespace Microsoft.TestPlatform.AcceptanceTests;