-
Notifications
You must be signed in to change notification settings - Fork 426
Add ability to use custom runsettings for tests #1710
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
844a100
67f7080
a2fa1b3
57b88c8
d7bf954
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,8 +26,13 @@ public VSTestManager(Project project, string workingDirectory, IDotNetCliService | |
| { | ||
| } | ||
|
|
||
| private object GetDefaultRunSettings(string targetFrameworkVersion) | ||
| private object LoadRunSettingsOrDefault(string runSettingsPath, string targetFrameworkVersion) | ||
| { | ||
| if (runSettingsPath != null) | ||
| { | ||
| return File.ReadAllText(runSettingsPath); | ||
| } | ||
|
|
||
| if (!string.IsNullOrWhiteSpace(targetFrameworkVersion)) | ||
| { | ||
| return $@" | ||
|
|
@@ -101,18 +106,18 @@ private static void VerifyTestFramework(string testFrameworkName) | |
| } | ||
| } | ||
|
|
||
| public override GetTestStartInfoResponse GetTestStartInfo(string methodName, string testFrameworkName, string targetFrameworkVersion) | ||
| public override GetTestStartInfoResponse GetTestStartInfo(string methodName, string runSettings, string testFrameworkName, string targetFrameworkVersion) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. normally we do not break the public APIs but this particular package is no published on nuget, so it's OK
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Even though these functions are public, TestManager and VSTestManager are actually internal classes, so there probably isn't any worry.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I was more concerned about the changes to the over-the-wire interface. At least the stdio version of the server seems to treat arguments as optional, in which case I think everything remains compatible. |
||
| { | ||
| VerifyTestFramework(testFrameworkName); | ||
|
|
||
| var testCases = DiscoverTests(new string[] { methodName }, targetFrameworkVersion); | ||
| var testCases = DiscoverTests(new string[] { methodName }, runSettings, targetFrameworkVersion); | ||
|
|
||
| SendMessage(MessageType.GetTestRunnerProcessStartInfoForRunSelected, | ||
| new | ||
| { | ||
| TestCases = testCases, | ||
| DebuggingEnabled = true, | ||
| RunSettings = GetDefaultRunSettings(targetFrameworkVersion) | ||
| RunSettings = LoadRunSettingsOrDefault(runSettings, targetFrameworkVersion) | ||
| }); | ||
|
|
||
| var message = ReadMessage(); | ||
|
|
@@ -126,21 +131,21 @@ public override GetTestStartInfoResponse GetTestStartInfo(string methodName, str | |
| }; | ||
| } | ||
|
|
||
| public override async Task<DebugTestGetStartInfoResponse> DebugGetStartInfoAsync(string methodName, string testFrameworkName, string targetFrameworkVersion, CancellationToken cancellationToken) | ||
| => await DebugGetStartInfoAsync(new string[] { methodName }, testFrameworkName, targetFrameworkVersion, cancellationToken); | ||
| public override async Task<DebugTestGetStartInfoResponse> DebugGetStartInfoAsync(string methodName, string runSettings, string testFrameworkName, string targetFrameworkVersion, CancellationToken cancellationToken) | ||
| => await DebugGetStartInfoAsync(new string[] { methodName }, runSettings, testFrameworkName, targetFrameworkVersion, cancellationToken); | ||
|
|
||
| public override async Task<DebugTestGetStartInfoResponse> DebugGetStartInfoAsync(string[] methodNames, string testFrameworkName, string targetFrameworkVersion, CancellationToken cancellationToken) | ||
| public override async Task<DebugTestGetStartInfoResponse> DebugGetStartInfoAsync(string[] methodNames, string runSettings, string testFrameworkName, string targetFrameworkVersion, CancellationToken cancellationToken) | ||
| { | ||
| VerifyTestFramework(testFrameworkName); | ||
|
|
||
| var testCases = await DiscoverTestsAsync(methodNames, targetFrameworkVersion, cancellationToken); | ||
| var testCases = await DiscoverTestsAsync(methodNames, runSettings, targetFrameworkVersion, cancellationToken); | ||
|
|
||
| SendMessage(MessageType.GetTestRunnerProcessStartInfoForRunSelected, | ||
| new | ||
| { | ||
| TestCases = testCases, | ||
| DebuggingEnabled = true, | ||
| RunSettings = GetDefaultRunSettings(targetFrameworkVersion) | ||
| RunSettings = LoadRunSettingsOrDefault(runSettings, targetFrameworkVersion) | ||
| }); | ||
|
|
||
| var message = await ReadMessageAsync(cancellationToken); | ||
|
|
@@ -186,14 +191,14 @@ public override async Task DebugLaunchAsync(CancellationToken cancellationToken) | |
| } | ||
| } | ||
|
|
||
| public override RunTestResponse RunTest(string methodName, string testFrameworkName, string targetFrameworkVersion) | ||
| => RunTest(new string[] { methodName }, testFrameworkName, targetFrameworkVersion); | ||
| public override RunTestResponse RunTest(string methodName, string runSettings, string testFrameworkName, string targetFrameworkVersion) | ||
| => RunTest(new string[] { methodName }, runSettings, testFrameworkName, targetFrameworkVersion); | ||
|
|
||
| public override RunTestResponse RunTest(string[] methodNames, string testFrameworkName, string targetFrameworkVersion) | ||
| public override RunTestResponse RunTest(string[] methodNames, string runSettings, string testFrameworkName, string targetFrameworkVersion) | ||
| { | ||
| VerifyTestFramework(testFrameworkName); | ||
|
|
||
| var testCases = DiscoverTests(methodNames, targetFrameworkVersion); | ||
| var testCases = DiscoverTests(methodNames, runSettings, targetFrameworkVersion); | ||
|
|
||
| var testResults = new List<TestResult>(); | ||
|
|
||
|
|
@@ -204,7 +209,7 @@ public override RunTestResponse RunTest(string[] methodNames, string testFramewo | |
| new | ||
| { | ||
| TestCases = testCases, | ||
| RunSettings = GetDefaultRunSettings(targetFrameworkVersion) | ||
| RunSettings = LoadRunSettingsOrDefault(runSettings, targetFrameworkVersion) | ||
| }); | ||
|
|
||
| var done = false; | ||
|
|
@@ -244,10 +249,10 @@ public override RunTestResponse RunTest(string[] methodNames, string testFramewo | |
| ErrorMessage = testResult.ErrorMessage, | ||
| ErrorStackTrace = testResult.ErrorStackTrace, | ||
| StandardOutput = testResult.Messages | ||
| .Where(message => message.Category == TestResultMessage.StandardOutCategory) | ||
| .Select(message => message.Text).ToArray(), | ||
| .Where(message => message.Category == TestResultMessage.StandardOutCategory) | ||
| .Select(message => message.Text).ToArray(), | ||
| StandardError = testResult.Messages.Where(message => message.Category == TestResultMessage.StandardErrorCategory) | ||
| .Select(message => message.Text).ToArray() | ||
| .Select(message => message.Text).ToArray() | ||
| }); | ||
|
|
||
| return new RunTestResponse | ||
|
|
@@ -257,7 +262,7 @@ public override RunTestResponse RunTest(string[] methodNames, string testFramewo | |
| }; | ||
| } | ||
|
|
||
| private async Task<TestCase[]> DiscoverTestsAsync(string[] methodNames, string targetFrameworkVersion, CancellationToken cancellationToken) | ||
| private async Task<TestCase[]> DiscoverTestsAsync(string[] methodNames, string runSettings, string targetFrameworkVersion, CancellationToken cancellationToken) | ||
| { | ||
| SendMessage(MessageType.StartDiscovery, | ||
| new | ||
|
|
@@ -266,7 +271,7 @@ private async Task<TestCase[]> DiscoverTestsAsync(string[] methodNames, string t | |
| { | ||
| Project.OutputFilePath | ||
| }, | ||
| RunSettings = GetDefaultRunSettings(targetFrameworkVersion) | ||
| RunSettings = LoadRunSettingsOrDefault(runSettings, targetFrameworkVersion) | ||
| }); | ||
|
|
||
| var testCases = new List<TestCase>(); | ||
|
|
@@ -322,9 +327,9 @@ bool isInRequestedMethods(TestCase testCase) | |
| }; | ||
| } | ||
|
|
||
| private TestCase[] DiscoverTests(string[] methodNames, string targetFrameworkVersion) | ||
| private TestCase[] DiscoverTests(string[] methodNames, string runSettings, string targetFrameworkVersion) | ||
| { | ||
| return DiscoverTestsAsync(methodNames, targetFrameworkVersion, CancellationToken.None).Result; | ||
| return DiscoverTestsAsync(methodNames, runSettings, targetFrameworkVersion, CancellationToken.None).Result; | ||
| } | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| <RunSettings> | ||
| <TestRunParameters> | ||
| <Parameter name="TestRunSetting" value="CorrectValue" /> | ||
| </TestRunParameters> | ||
| </RunSettings> |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -223,5 +223,27 @@ public async Task RunMSTestStandardOutputIsReturned() | |
| Assert.Single(response.Results); | ||
| Assert.NotEmpty(response.Results[0].StandardOutput); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task RunMSTestWithRunSettings() | ||
| { | ||
| await RunDotNetTestAsync( | ||
| MSTestProject, | ||
| methodName: "Main.Test.MainTest.CheckRunSettings", | ||
| testFramework: "mstest", | ||
| shouldPass: true, | ||
| useRunSettings: true); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task RunMSTestWithoutRunSettings() | ||
| { | ||
| await RunDotNetTestAsync( | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if this is supposed to fail, could you assert on some failure conditions to distinguish a "failure by design" from "failure because something got broken"?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Added some additional assertions. |
||
| MSTestProject, | ||
| methodName: "Main.Test.MainTest.CheckRunSettings", | ||
| testFramework: "mstest", | ||
| shouldPass: false, | ||
| useRunSettings: false); | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
you should check if the file exists, if not, I'd fallback to defaults to not crash?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Added an error message if runsettings load fails, and it will continue with default settings.