-
-
Notifications
You must be signed in to change notification settings - Fork 1k
Add a VSTest Adapter #2438
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
adamsitnik
merged 11 commits into
dotnet:master
from
caaavik-msft:caaavik/vstest-adapter
Dec 26, 2023
Merged
Add a VSTest Adapter #2438
Changes from 6 commits
Commits
Show all changes
11 commits
Select commit
Hold shift + click to select a range
b68e111
Add VSTest Adapter
caaavik-msft 29ba8ac
Merge from master
caaavik-msft 7bce341
Address PR comments
caaavik-msft 5317384
Address further PR comments
caaavik-msft b2e45a9
Put histogram first in test result output
caaavik-msft 494f2ce
Add Auto-Generated Entry Point
caaavik-msft 7c8fb19
Add comments to samples csproj
caaavik-msft f8fa9ad
Change default for GenerateBDNEntryPoint
caaavik-msft 0c1b275
Reuse GenerateProgramFile property
caaavik-msft c432f88
Add documentation of VSTest
caaavik-msft fa711e3
Update docs to use Config attribute instead
caaavik-msft File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
94 changes: 94 additions & 0 deletions
94
src/BenchmarkDotNet.TestAdapter/BenchmarkCaseExtensions.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| using BenchmarkDotNet.Attributes; | ||
| using BenchmarkDotNet.Characteristics; | ||
| using BenchmarkDotNet.Exporters; | ||
| using BenchmarkDotNet.Extensions; | ||
| using BenchmarkDotNet.Running; | ||
| using Microsoft.TestPlatform.AdapterUtilities; | ||
| using Microsoft.VisualStudio.TestPlatform.ObjectModel; | ||
| using System; | ||
|
|
||
| namespace BenchmarkDotNet.TestAdapter | ||
| { | ||
| /// <summary> | ||
| /// A set of extensions for BenchmarkCase to support converting to VSTest TestCase objects. | ||
| /// </summary> | ||
| internal static class BenchmarkCaseExtensions | ||
| { | ||
| /// <summary> | ||
| /// Converts a BDN BenchmarkCase to a VSTest TestCase. | ||
| /// </summary> | ||
| /// <param name="benchmarkCase">The BenchmarkCase to convert.</param> | ||
| /// <param name="assemblyPath">The dll or exe of the benchmark project.</param> | ||
| /// <param name="includeJobInName">Whether or not the display name should include the job name.</param> | ||
| /// <returns>The VSTest TestCase.</returns> | ||
| internal static TestCase ToVsTestCase(this BenchmarkCase benchmarkCase, string assemblyPath, bool includeJobInName = false) | ||
| { | ||
| var benchmarkMethod = benchmarkCase.Descriptor.WorkloadMethod; | ||
| var fullClassName = benchmarkCase.Descriptor.Type.GetCorrectCSharpTypeName(); | ||
| var benchmarkMethodName = benchmarkCase.Descriptor.WorkloadMethod.Name; | ||
| var benchmarkFullMethodName = $"{fullClassName}.{benchmarkMethodName}"; | ||
|
|
||
| // Display name has arguments as well. | ||
| var displayMethodName = FullNameProvider.GetMethodName(benchmarkCase); | ||
| if (includeJobInName) | ||
| displayMethodName += $" [{benchmarkCase.GetUnrandomizedJobDisplayInfo()}]"; | ||
|
|
||
| var displayName = $"{fullClassName}.{displayMethodName}"; | ||
|
|
||
| var vsTestCase = new TestCase(benchmarkFullMethodName, VsTestAdapter.ExecutorUri, assemblyPath) | ||
| { | ||
| DisplayName = displayName, | ||
| Id = GetTestCaseId(benchmarkCase) | ||
| }; | ||
|
|
||
| var benchmarkAttribute = benchmarkMethod.ResolveAttribute<BenchmarkAttribute>(); | ||
adamsitnik marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| if (benchmarkAttribute != null) | ||
| { | ||
| vsTestCase.CodeFilePath = benchmarkAttribute.SourceCodeFile; | ||
| vsTestCase.LineNumber = benchmarkAttribute.SourceCodeLineNumber; | ||
| } | ||
|
|
||
| var categories = DefaultCategoryDiscoverer.Instance.GetCategories(benchmarkMethod); | ||
| foreach (var category in categories) | ||
| vsTestCase.Traits.Add("Category", category); | ||
|
|
||
| vsTestCase.Traits.Add("", "BenchmarkDotNet"); | ||
|
|
||
| return vsTestCase; | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// If an ID is not provided, a random string is used for the ID. This method will identify if randomness was | ||
| /// used for the ID and return the Job's DisplayInfo with that randomness removed so that the same benchmark | ||
| /// can be referenced across multiple processes. | ||
| /// </summary> | ||
| /// <param name="benchmarkCase">The benchmark case.</param> | ||
| /// <returns>The benchmark case' job's DisplayInfo without randomness.</returns> | ||
| internal static string GetUnrandomizedJobDisplayInfo(this BenchmarkCase benchmarkCase) | ||
| { | ||
| var jobDisplayInfo = benchmarkCase.Job.DisplayInfo; | ||
| if (!benchmarkCase.Job.HasValue(CharacteristicObject.IdCharacteristic) && benchmarkCase.Job.ResolvedId.StartsWith("Job-", StringComparison.OrdinalIgnoreCase)) | ||
| { | ||
| // Replace Job-ABCDEF with Job | ||
| jobDisplayInfo = "Job" + jobDisplayInfo.Substring(benchmarkCase.Job.ResolvedId.Length); | ||
| } | ||
|
|
||
| return jobDisplayInfo; | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Gets an ID for a given BenchmarkCase that is uniquely identifiable from discovery to execution phase. | ||
| /// </summary> | ||
| /// <param name="benchmarkCase">The benchmark case.</param> | ||
| /// <returns>The test case ID.</returns> | ||
| internal static Guid GetTestCaseId(this BenchmarkCase benchmarkCase) | ||
| { | ||
| var testIdProvider = new TestIdProvider(); | ||
| testIdProvider.AppendString(VsTestAdapter.ExecutorUriString); | ||
| testIdProvider.AppendString(benchmarkCase.Descriptor.DisplayInfo); | ||
| testIdProvider.AppendString(benchmarkCase.GetUnrandomizedJobDisplayInfo()); | ||
| testIdProvider.AppendString(benchmarkCase.Parameters.DisplayInfo); | ||
| return testIdProvider.GetId(); | ||
| } | ||
| } | ||
| } | ||
28 changes: 28 additions & 0 deletions
28
src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
| <Import Project="..\..\build\common.props" /> | ||
| <PropertyGroup> | ||
| <TargetFrameworks>netstandard2.0;net462</TargetFrameworks> | ||
caaavik-msft marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| <AssemblyTitle>BenchmarkDotNet.TestAdapter</AssemblyTitle> | ||
| <AssemblyName>BenchmarkDotNet.TestAdapter</AssemblyName> | ||
| <PackageId>BenchmarkDotNet.TestAdapter</PackageId> | ||
| <ProduceReferenceAssembly>True</ProduceReferenceAssembly> | ||
| <Nullable>enable</Nullable> | ||
caaavik-msft marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Microsoft.TestPlatform.AdapterUtilities" Version="17.7.2" /> | ||
| <PackageReference Include="Microsoft.TestPlatform.ObjectModel" Version="17.7.2" /> | ||
| <PackageReference Include="Microsoft.TestPlatform.TranslationLayer" Version="17.7.2" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\BenchmarkDotNet\BenchmarkDotNet.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| <!-- Include files in nuget package for generating entry point --> | ||
| <ItemGroup> | ||
| <Compile Remove="Package\EntryPoint.*" /> | ||
| <None Include="Package\EntryPoint.*" Pack="true" PackagePath="entrypoints\" /> | ||
| <None Include="Package\BenchmarkDotNet.TestAdapter.props" Pack="true" PackagePath="build\" /> | ||
| </ItemGroup> | ||
| </Project> | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| using BenchmarkDotNet.Extensions; | ||
| using BenchmarkDotNet.Helpers; | ||
| using BenchmarkDotNet.Running; | ||
| using BenchmarkDotNet.Toolchains; | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Reflection; | ||
|
|
||
| namespace BenchmarkDotNet.TestAdapter | ||
| { | ||
| /// <summary> | ||
| /// A class used for enumerating all the benchmarks in an assembly. | ||
| /// </summary> | ||
| internal static class BenchmarkEnumerator | ||
| { | ||
| /// <summary> | ||
| /// Returns all the BenchmarkRunInfo objects from a given assembly. | ||
| /// </summary> | ||
| /// <param name="assemblyPath">The dll or exe of the benchmark project.</param> | ||
| /// <returns>The benchmarks inside the assembly.</returns> | ||
| public static BenchmarkRunInfo[] GetBenchmarksFromAssemblyPath(string assemblyPath) | ||
| { | ||
| var assembly = Assembly.LoadFrom(assemblyPath); | ||
|
|
||
| var isDebugAssembly = assembly.IsJitOptimizationDisabled() ?? false; | ||
|
|
||
| return GenericBenchmarksBuilder.GetRunnableBenchmarks(assembly.GetRunnableBenchmarks()) | ||
| .Select(type => | ||
| { | ||
| var benchmarkRunInfo = BenchmarkConverter.TypeToBenchmarks(type); | ||
| if (isDebugAssembly) | ||
| { | ||
| // If the assembly is a debug assembly, then only display them if they will run in-process | ||
| // This will allow people to debug their benchmarks using VSTest if they wish. | ||
| benchmarkRunInfo = new BenchmarkRunInfo( | ||
| benchmarkRunInfo.BenchmarksCases.Where(c => c.GetToolchain().IsInProcess).ToArray(), | ||
| benchmarkRunInfo.Type, | ||
| benchmarkRunInfo.Config); | ||
| } | ||
|
|
||
| return benchmarkRunInfo; | ||
| }) | ||
| .Where(runInfo => runInfo.BenchmarksCases.Length > 0) | ||
| .ToArray(); | ||
| } | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| using BenchmarkDotNet.Configs; | ||
| using BenchmarkDotNet.Running; | ||
| using BenchmarkDotNet.TestAdapter.Remoting; | ||
| using Microsoft.VisualStudio.TestPlatform.ObjectModel; | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Threading; | ||
|
|
||
| namespace BenchmarkDotNet.TestAdapter | ||
| { | ||
| /// <summary> | ||
| /// A class used for executing benchmarks | ||
| /// </summary> | ||
| internal class BenchmarkExecutor | ||
| { | ||
| private readonly CancellationTokenSource cts = new (); | ||
|
|
||
| /// <summary> | ||
| /// Runs all the benchmarks in the given assembly, updating the TestExecutionRecorder as they get run. | ||
| /// </summary> | ||
| /// <param name="assemblyPath">The dll or exe of the benchmark project.</param> | ||
| /// <param name="recorder">The interface used to record the current test execution progress.</param> | ||
| /// <param name="benchmarkIds"> | ||
| /// An optional list of benchmark IDs specifying which benchmarks to run. | ||
| /// These IDs are the same as the ones generated for the VSTest TestCase. | ||
| /// </param> | ||
| public void RunBenchmarks(string assemblyPath, TestExecutionRecorderWrapper recorder, HashSet<Guid>? benchmarkIds = null) | ||
| { | ||
| var benchmarks = BenchmarkEnumerator.GetBenchmarksFromAssemblyPath(assemblyPath); | ||
| var testCases = new List<TestCase>(); | ||
|
|
||
| var filteredBenchmarks = new List<BenchmarkRunInfo>(); | ||
| foreach (var benchmark in benchmarks) | ||
| { | ||
| var needsJobInfo = benchmark.BenchmarksCases.Select(c => c.Job.DisplayInfo).Distinct().Count() > 1; | ||
| var filteredCases = new List<BenchmarkCase>(); | ||
| foreach (var benchmarkCase in benchmark.BenchmarksCases) | ||
| { | ||
| var testId = benchmarkCase.GetTestCaseId(); | ||
| if (benchmarkIds != null && benchmarkIds.Contains(testId)) | ||
| { | ||
| filteredCases.Add(benchmarkCase); | ||
| testCases.Add(benchmarkCase.ToVsTestCase(assemblyPath, needsJobInfo)); | ||
| } | ||
| } | ||
|
|
||
| if (filteredCases.Count > 0) | ||
| { | ||
| filteredBenchmarks.Add(new BenchmarkRunInfo(filteredCases.ToArray(), benchmark.Type, benchmark.Config)); | ||
| } | ||
| } | ||
|
|
||
| benchmarks = filteredBenchmarks.ToArray(); | ||
|
|
||
| if (benchmarks.Length == 0) | ||
| return; | ||
|
|
||
| // Create an event processor which will subscribe to events and push them to VSTest | ||
| var eventProcessor = new VsTestEventProcessor(testCases, recorder, cts.Token); | ||
|
|
||
| // Create a logger which will forward all log messages in BDN to the VSTest logger. | ||
| var logger = new VsTestLogger(recorder.GetLogger()); | ||
|
|
||
| // Modify all the benchmarks so that the event process and logger is added. | ||
| benchmarks = benchmarks | ||
| .Select(b => new BenchmarkRunInfo( | ||
| b.BenchmarksCases, | ||
| b.Type, | ||
| b.Config.AddEventProcessor(eventProcessor).AddLogger(logger).CreateImmutableConfig())) | ||
| .ToArray(); | ||
|
|
||
| // Run all the benchmarks, and ensure that any tests that don't have a result yet are sent. | ||
| BenchmarkRunner.Run(benchmarks); | ||
| eventProcessor.SendUnsentTestResults(); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Stop the benchmarks when next able. | ||
caaavik-msft marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| /// </summary> | ||
| public void Cancel() | ||
| { | ||
| cts.Cancel(); | ||
| } | ||
| } | ||
| } | ||
12 changes: 12 additions & 0 deletions
12
src/BenchmarkDotNet.TestAdapter/Package/BenchmarkDotNet.TestAdapter.props
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> | ||
| <PropertyGroup> | ||
| <!-- Disables generation of MSTest entrypoint --> | ||
| <GenerateProgramFile>false</GenerateProgramFile> | ||
| <EntryPointSourceDirectory>$(MSBuildThisFileDirectory)..\entrypoints\</EntryPointSourceDirectory> | ||
| </PropertyGroup> | ||
| <ItemGroup Condition="'$(GenerateBDNEntryPoint)' == 'true'"> | ||
timcassell marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| <Compile Include="$(EntryPointSourceDirectory)EntryPoint.cs" Condition="'$(Language)' == 'C#'" Visible="false"/> | ||
| <CompileAfter Include="$(EntryPointSourceDirectory)EntryPoint.fs" Condition="'$(Language)' == 'F#'" Visible="false"/> | ||
| <Compile Include="$(EntryPointSourceDirectory)EntryPoint.vb" Condition="'$(Language)' == 'VB'" Visible="false"/> | ||
| </ItemGroup> | ||
| </Project> | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| using BenchmarkDotNet.Running; | ||
| using System.Reflection; | ||
|
|
||
| public class __AutoGeneratedEntryPointClass | ||
| { | ||
| public static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(__AutoGeneratedEntryPointClass).Assembly).Run(args); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| module __AutoGeneratedEntryPointClass | ||
| open System.Reflection; | ||
| open BenchmarkDotNet.Running | ||
|
|
||
| [<EntryPoint>] | ||
| let main argv = | ||
| BenchmarkSwitcher.FromAssembly(Assembly.GetEntryAssembly()).Run(argv) |> ignore | ||
| 0 // return an integer exit code |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| Imports System.Reflection | ||
| Imports BenchmarkDotNet.Running | ||
|
|
||
| Namespace Global | ||
| Module __AutoGeneratedEntryPointClass | ||
| Sub Main(args As String()) | ||
| Dim summary = BenchmarkSwitcher.FromAssembly(Assembly.GetEntryAssembly()).Run(args) | ||
| End Sub | ||
| End Module | ||
| End Namespace |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.