Onboard to new dotnet test experience (#5111)

Co-authored-by: Youssef Victor <youssefvictor00@gmail.com>

Author: Evangelink

NonWindowsTests.slnf

@@ -0,0 +1,14 @@
+{
+  "solution": {
+    "path": "TestFx.sln",
+    "projects": [
+      "test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests.csproj",
+      "test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSTest.Acceptance.IntegrationTests.csproj",
+      "test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj",
+      "test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests.csproj",
+      "test/UnitTests/Microsoft.Testing.Platform.UnitTests/Microsoft.Testing.Platform.UnitTests.csproj",
+      "test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests.csproj",
+      "test/UnitTests/MSTest.Analyzers.UnitTests/MSTest.Analyzers.UnitTests.csproj"
+    ]
+  }
+}

azure-pipelines.yml

@@ -94,14 +94,17 @@ stages:
           displayName: Build
 
         - ${{ if eq(parameters.SkipTests, False) }}:
-          # -ci is allowing to import some environment variables and some required configurations
-          # -nobl avoids overwriting build binlog with binlog from tests
-          - script: Test.cmd
-              -configuration $(_BuildConfig)
-              -ci
-              -nobl
+
+          # Because the build step is using -ci flag, restore is done in a local .packages directory.
+          # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them.
+          # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set.
+          - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog
             name: Test
             displayName: Test
+            env:
+              DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet
+              NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages
+
 
           # This step is only helpful for diagnosing some issues with vstest/test host that would not appear
           # through the console or trx
@@ -157,13 +160,15 @@ stages:
           displayName: Build
 
         - ${{ if eq(parameters.SkipTests, False) }}:
-          # --ci is allowing to import some environment variables and some required configurations
-          # --nobl avoids overwriting build binlog with binlog from tests
-          - script: |
-              chmod +x ./test.sh
-              ./test.sh --configuration $(_BuildConfig) --ci --test --integrationTest --nobl
+          # Because the build step is using -ci flag, restore is done in a local .packages directory.
+          # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them.
+          # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set.
+          - script: $(Build.SourcesDirectory)/.dotnet/dotnet test --solution NonWindowsTests.slnf -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog
             name: Test
-            displayName: Tests
+            displayName: Test
+            env:
+              DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet
+              NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages
 
           # This step is only helpful for diagnosing some issues with vstest/test host that would not appear
           # through the console or trx
@@ -212,13 +217,15 @@ stages:
           displayName: Build
 
         - ${{ if eq(parameters.SkipTests, False) }}:
-          # --ci is allowing to import some environment variables and some required configurations
-          # --nobl avoids overwriting build binlog with binlog from tests
-          - script: |
-              chmod +x ./test.sh
-              ./test.sh --configuration $(_BuildConfig) --ci --test --integrationTest --nobl
+          # Because the build step is using -ci flag, restore is done in a local .packages directory.
+          # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them.
+          # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set.
+          - script: $(Build.SourcesDirectory)/.dotnet/dotnet test --solution NonWindowsTests.slnf -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog
             name: Test
-            displayName: Tests
+            displayName: Test
+            env:
+              DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet
+              NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages
 
           # This step is only helpful for diagnosing some issues with vstest/test host that would not appear
           # through the console or trx

dotnet.config

@@ -0,0 +1,2 @@
+[dotnet.test:runner]
+name= "Microsoft.Testing.Platform"

global.json

@@ -1,6 +1,6 @@
 {
   "tools": {
-    "dotnet": "10.0.100-preview.3.25125.5",
+    "dotnet": "10.0.100-preview.3.25166.18",
     "runtimes": {
       "dotnet": [
         "3.1.32",
@@ -23,7 +23,7 @@
     }
   },
   "sdk": {
-    "version": "10.0.100-preview.3.25125.5",
+    "version": "10.0.100-preview.3.25166.18",
     "allowPrerelease": true,
     "rollForward": "latestFeature"
   },

samples/Playground/Playground.csproj

@@ -6,6 +6,8 @@
     <Nullable>enable</Nullable>
     <GenerateProgramFile>false</GenerateProgramFile>
     <NoWarn>$(NoWarn);NETSDK1023</NoWarn>
+
+    <IsTestingPlatformApplication>true</IsTestingPlatformApplication>
   </PropertyGroup>
 
   <ItemGroup Condition=" '$(TargetFramework)' == 'net462' ">

samples/Playground/Tests.cs

@@ -15,7 +15,8 @@ public class TestClass
     [TestMethod]
     [DynamicData(nameof(Data))]
     public void Test3(int a, int b)
-        => throw new Exception("aaaa");
+    {
+    }
 
     public static IEnumerable<(int A, int B)> Data
     {

test/Directory.Build.targets

@@ -17,6 +17,17 @@
     <TestRunnerAdditionalArguments>$(TestRunnerAdditionalArguments) --hangdump --hangdump-timeout 15m</TestRunnerAdditionalArguments>
     <TestRunnerAdditionalArguments Condition=" '$(EnableCodeCoverage)' == 'True' ">$(TestRunnerAdditionalArguments) --coverage --coverage-settings $(RepoRoot)test/coverage.config --coverage-output $(ModuleName).coverage</TestRunnerAdditionalArguments>
     <!-- /Arcade runner settings -->
+
+    <!-- dotnet test runner settings - most duplicate from Arcade to have parity -->
+    <_TestArchitecture>$(PlatformTarget)</_TestArchitecture>
+    <_TestArchitecture Condition="'$(PlatformTarget)' == '' or '$(PlatformTarget)' == 'AnyCpu'">x64</_TestArchitecture>
+
+    <_ResultFileNameNoExt>$(MSBuildProjectName)_$(TargetFramework)_$(_TestArchitecture)</_ResultFileNameNoExt>
+    <ResultsTrxPath>$(ArtifactsTestResultsDir)$(_ResultFileNameNoExt).trx</ResultsTrxPath>
+    <_TestResultTrxFileName>$([System.IO.Path]::GetFileName('$(ResultsTrxPath)'))</_TestResultTrxFileName>
+    <_TestResultDirectory>$([System.IO.Path]::GetDirectoryName('$(ResultsTrxPath)'))</_TestResultDirectory>
+    <TestingPlatformCommandLineArguments>$(TestingPlatformCommandLineArguments) --report-trx --report-trx-filename "$(_TestResultTrxFileName)" --results-directory "$(_TestResultDirectory)" $(TestRunnerAdditionalArguments)</TestingPlatformCommandLineArguments>
+    <!-- /dotnet test runner settings -->
   </PropertyGroup>
 
   <ItemGroup Condition=" '$(EnableMSTestRunner)' == 'true' OR '$(UseInternalTestFramework)' == 'true' ">
@@ -32,6 +43,7 @@
 
   <!-- Import the capabilities for the Microsoft.Testing.Platform -->
   <Import Project="$(RepoRoot)src\Platform\Microsoft.Testing.Platform\buildMultiTargeting\Microsoft.Testing.Platform.props" Condition=" '$(EnableMSTestRunner)' == 'true' OR '$(UseInternalTestFramework)' == 'true' " />
+  <Import Project="$(RepoRoot)src\Platform\Microsoft.Testing.Platform\buildMultiTargeting\Microsoft.Testing.Platform.targets" Condition=" '$(EnableMSTestRunner)' == 'true' OR '$(UseInternalTestFramework)' == 'true' " />
 
   <ItemGroup Condition=" '$(EnableMSTestRunner)' == 'true' AND '$(UseInternalTestFramework)' != 'true' AND '$(UseMSTestFromSource)' != 'true' ">
     <PackageReference Include="MSTest.Analyzers" />

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DotnetTestCliTests.cs

@@ -25,7 +25,7 @@ public async Task DotnetTest_Should_Execute_Tests(string tfm, BuildConfiguration
             .PatchCodeWithReplace("$OutputType$", string.Empty)
             .PatchCodeWithReplace("$Extra$", string.Empty));
 
-        DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -m:1 -nodeReuse:false {generator.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path);
+        DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -m:1 -nodeReuse:false {generator.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: generator.TargetAssetPath);
 
         // There is whitespace difference in output in parent and public repo that depends on the version of the dotnet SDK used.
         compilationResult.AssertOutputMatchesRegex(@"Passed!\s+-\s+Failed:\s+0,\s+Passed:\s+1,\s+Skipped:\s+0,\s+Total:\s+1");

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs

@@ -66,7 +66,8 @@ public async Task MSBuildTestTarget_SingleAndMultiTfm_Should_Run_Solution_Tests(
         // Build the solution
         DotnetMuxerResult restoreResult = await DotnetCli.RunAsync($"restore -m:1 -nodeReuse:false {solution.SolutionFile} --configfile {nugetFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path);
         restoreResult.AssertOutputDoesNotContain("An approximate best match of");
-        DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -m:1 -nodeReuse:false {solution.SolutionFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path);
+
+        DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -m:1 -nodeReuse:false {solution.SolutionFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: generator.TargetAssetPath);
 
         if (isMultiTfm)
         {

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/PublishAotNonNativeTests.cs

@@ -24,7 +24,7 @@ public async Task RunTests_ThatEnablePublishAOT_ButDontBuildToNative()
             .PatchCodeWithReplace("$TargetFramework$", $"<TargetFramework>{TargetFrameworks.NetCurrent}</TargetFramework>")
             .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion));
 
-        DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -c Debug {generator.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, failIfReturnValueIsNotZero: false);
+        DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -c Debug {generator.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: generator.TargetAssetPath, failIfReturnValueIsNotZero: false);
 
         // In the real-world issue, access to path C:\Program Files\dotnet\ is denied, but we run this from a local .dotnet folder, where we have write access.
         // So instead of relying on the test run failing because of AccessDenied, we check the output, and see where TestResults were placed.
@@ -68,6 +68,10 @@ we end up with a -dev or -ci version which will lose resolution over -preview de
     </ItemGroup>
 </Project>
 
+#file dotnet.config
+[dotnet.test:runner]
+name= "VSTest"
+
 #file UnitTest1.cs
 
 using Microsoft.VisualStudio.TestTools.UnitTesting;