Skip to content

Commit 4d117c4

Browse files
rolfbjarnerolfbjarne
committed
[tests] Improve test summary to include up to 3 failing unit tests.
1 parent 699b780 commit 4d117c4

File tree

6 files changed

+164
-54
lines changed

6 files changed

+164
-54
lines changed

‎scripts/create-windows-html-report/create-windows-html-report.cs‎

Lines changed: 31 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
using System.Text;
1313
using System.Xml;
1414

15+
using Xamarin.Utils;
16+
1517
public class Program {
1618
static string GetOutcomeColor (string outcome)
1719
{
@@ -120,50 +122,40 @@ public static int Main (string [] args)
120122
foreach (var trx in trxFiles) {
121123
var name = trx.Name;
122124
var path = trx.TestResults;
123-
string? outcome;
124125
var messageLines = new List<string> ();
125126

126-
try {
127-
var xml = new XmlDocument ();
128-
xml.Load (path);
129-
outcome = xml.SelectSingleNode ("/*[local-name() = 'TestRun']/*[local-name() = 'ResultSummary']")?.Attributes? ["outcome"]?.Value;
130-
if (outcome is null) {
131-
outcome = $"Could not find outcome in trx file {path}";
132-
} else {
133-
var failedTests = xml.SelectNodes ("/*[local-name() = 'TestRun']/*[local-name() = 'Results']/*[local-name() = 'UnitTestResult'][@outcome != 'Passed']")?.Cast<XmlNode> ();
134-
if (failedTests?.Any () == true) {
135-
messageLines.Add (" <ul>");
136-
foreach (var node in failedTests) {
137-
var testName = node.Attributes? ["testName"]?.Value ?? "<unknown test name>";
138-
var testOutcome = node.Attributes? ["outcome"]?.Value ?? "<unknown test outcome>";
139-
var testMessage = node.SelectSingleNode ("*[local-name() = 'Output']/*[local-name() = 'ErrorInfo']/*[local-name() = 'Message']")?.InnerText;
140-
141-
var testId = node.Attributes? ["testId"]?.Value;
142-
if (!string.IsNullOrEmpty (testId)) {
143-
var testMethod = xml.SelectSingleNode ($"/*[local-name() = 'TestRun']/*[local-name() = 'TestDefinitions']/*[local-name() = 'UnitTest'][@id='{testId}']/*[local-name() = 'TestMethod']");
144-
var className = testMethod?.Attributes? ["className"]?.Value ?? string.Empty;
145-
if (!string.IsNullOrEmpty (className))
146-
testName = className + "." + testName;
147-
}
148-
149-
if (string.IsNullOrEmpty (testMessage)) {
150-
messageLines.Add ($" <li>{testName} (<span style='color: {GetOutcomeColor (testOutcome)}'>{testOutcome}</span>)</li>");
151-
} else if (testMessage.Split ('\n').Length == 1) {
152-
messageLines.Add ($" <li>{testName} (<span style='color: {GetOutcomeColor (testOutcome)}'>{testOutcome}</span>): {FormatHtml (testMessage)}</li>");
153-
} else {
154-
messageLines.Add ($" <li>{testName} (<span style='color: {GetOutcomeColor (testOutcome)}'>{testOutcome}</span>)");
155-
messageLines.Add ($" <div class='pdiv' style='margin-left: 20px;'>");
156-
messageLines.Add (FormatHtml (testMessage));
157-
messageLines.Add ($" </div>");
158-
messageLines.Add ($" </li>");
159-
}
127+
if (TrxParser.TryParseTrxFile (path, out var failedTests, out var outcome, out allTestsSucceeded, out var ex)) {
128+
if (failedTests?.Any () == true) {
129+
messageLines.Add (" <ul>");
130+
foreach (var ft in failedTests) {
131+
var testName = ft.Name;
132+
var testOutcome = ft.Outcome;
133+
var testMessage = ft.Message;
134+
135+
if (string.IsNullOrEmpty (testMessage)) {
136+
messageLines.Add ($" <li>{testName} (<span style='color: {GetOutcomeColor (testOutcome)}'>{testOutcome}</span>)</li>");
137+
} else if (testMessage.Split ('\n').Length == 1) {
138+
messageLines.Add ($" <li>{testName} (<span style='color: {GetOutcomeColor (testOutcome)}'>{testOutcome}</span>): {FormatHtml (testMessage)}</li>");
139+
} else {
140+
messageLines.Add ($" <li>{testName} (<span style='color: {GetOutcomeColor (testOutcome)}'>{testOutcome}</span>)");
141+
messageLines.Add ($" <div class='pdiv' style='margin-left: 20px;'>");
142+
messageLines.Add (FormatHtml (testMessage));
143+
messageLines.Add ($" </div>");
144+
messageLines.Add ($" </li>");
160145
}
161-
messageLines.Add (" </ul>");
162-
allTestsSucceeded = false;
163-
} else if (outcome != "Completed" && outcome != "Passed") {
164-
messageLines.Add ($" Failed to find any test failures in the trx file {path}");
165146
}
147+
messageLines.Add (" </ul>");
148+
} else if (outcome != "Completed" && outcome != "Passed") {
149+
messageLines.Add ($" Failed to find any test failures in the trx file {path}");
166150
}
151+
} else {
152+
outcome = "Failed to parse test results";
153+
if (ex is not null)
154+
messageLines.Add ($"<div>{FormatHtml (ex.ToString ())}</div>");
155+
allTestsSucceeded = false;
156+
}
157+
158+
try {
167159
var htmlPath = Path.ChangeExtension (path, "html");
168160
if (File.Exists (htmlPath)) {
169161
var relativeHtmlPath = Path.GetRelativePath (outputDirectory, htmlPath);

‎scripts/create-windows-html-report/create-windows-html-report.csproj‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,7 @@
55
<ImplicitUsings>enable</ImplicitUsings>
66
<Nullable>enable</Nullable>
77
</PropertyGroup>
8+
<ItemGroup>
9+
<Compile Include="../../tests/common/ParseTrxFile.cs" />
10+
</ItemGroup>
811
</Project>

‎tests/common/ParseTrxFile.cs‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
namespace Xamarin.Utils;
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Diagnostics.CodeAnalysis;
6+
using System.IO;
7+
using System.Linq;
8+
using System.Xml;
9+
10+
#nullable enable
11+
12+
public class TrxParser {
13+
public class TrxTestResult {
14+
public required string Name;
15+
public required string Outcome;
16+
public required string Message;
17+
18+
public bool Passed { get => Outcome == "Passed"; }
19+
}
20+
21+
public static bool TryParseTrxFile (string path, [NotNullWhen (true)] out IList<TrxTestResult>? failedTests, [NotNullWhen (true)] out string? outcome, out bool allTestsSucceeded, out Exception? exception)
22+
{
23+
allTestsSucceeded = false;
24+
failedTests = null;
25+
outcome = null;
26+
exception = null;
27+
28+
if (!File.Exists (path))
29+
return false;
30+
31+
var rv = new List<TrxTestResult> ();
32+
try {
33+
var xml = new XmlDocument ();
34+
xml.Load (path);
35+
outcome = xml.SelectSingleNode ("/*[local-name() = 'TestRun']/*[local-name() = 'ResultSummary']")?.Attributes? ["outcome"]?.Value;
36+
if (outcome is null) {
37+
outcome = $"Could not find outcome in trx file {path}";
38+
} else {
39+
var failedTestsQuery = xml.SelectNodes ("/*[local-name() = 'TestRun']/*[local-name() = 'Results']/*[local-name() = 'UnitTestResult'][@outcome != 'Passed']")?.Cast<XmlNode> ();
40+
if (failedTestsQuery?.Any () == true) {
41+
foreach (var node in failedTestsQuery) {
42+
var testName = node.Attributes? ["testName"]?.Value ?? "<unknown test name>";
43+
var testOutcome = node.Attributes? ["outcome"]?.Value ?? "<unknown test outcome>";
44+
var testMessage = node.SelectSingleNode ("*[local-name() = 'Output']/*[local-name() = 'ErrorInfo']/*[local-name() = 'Message']")?.InnerText ?? "";
45+
46+
var testId = node.Attributes? ["testId"]?.Value;
47+
if (!string.IsNullOrEmpty (testId)) {
48+
var testMethod = xml.SelectSingleNode ($"/*[local-name() = 'TestRun']/*[local-name() = 'TestDefinitions']/*[local-name() = 'UnitTest'][@id='{testId}']/*[local-name() = 'TestMethod']");
49+
var className = testMethod?.Attributes? ["className"]?.Value ?? string.Empty;
50+
if (!string.IsNullOrEmpty (className))
51+
testName = className + "." + testName;
52+
}
53+
54+
rv.Add (new TrxTestResult () {
55+
Name = testName,
56+
Outcome = testOutcome,
57+
Message = testMessage,
58+
});
59+
}
60+
allTestsSucceeded = false;
61+
} else if (outcome != "Completed" && outcome != "Passed") {
62+
// failed, but no test failures?
63+
allTestsSucceeded = false;
64+
} else {
65+
allTestsSucceeded = true;
66+
}
67+
}
68+
failedTests = rv;
69+
return true;
70+
} catch (Exception e) {
71+
outcome = "Failed to parse test results";
72+
exception = e;
73+
allTestsSucceeded = false;
74+
return false;
75+
}
76+
}
77+
}

‎tests/xharness/Jenkins/Reports/MarkdownReportWriter.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
using System.Collections.Generic;
2+
using System.Diagnostics.CodeAnalysis;
23
using System.IO;
34
using System.Linq;
5+
using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
46
using Xharness.Jenkins.TestTasks;
57

68
#nullable enable
@@ -30,6 +32,15 @@ void WriteFailedTestsDetails (IEnumerable<ITestTask> failedTests, StreamWriter w
3032
if (test.KnownFailure is not null)
3133
writer.Write ($" Known issue: [{test.KnownFailure.HumanMessage}]({test.KnownFailure.IssueLink})");
3234
writer.WriteLine ();
35+
var trxLog = test.Logs.FirstOrDefault (v => v.Description == LogType.TrxLog.ToString ());
36+
if (trxLog is not null && Xamarin.Utils.TrxParser.TryParseTrxFile (trxLog.FullPath, out var failedTrxTests, out var outcome, out var allTestsSucceeded, out var exception)) {
37+
foreach (var failedTrxTest in failedTrxTests.OrderBy (v => v.Name).Take (3)) {
38+
writer.WriteLine ($" * {failedTrxTest.Name.Cap (64)}: {failedTrxTest.Message.Cap (128)}");
39+
}
40+
if (failedTrxTests.Count > 3) {
41+
writer.WriteLine ($" * ... and {failedTrxTests.Count - 3} more");
42+
}
43+
}
3344
}
3445
continue;
3546
}
@@ -93,4 +104,18 @@ public void Write (IList<ITestTask> allTasks, StreamWriter writer)
93104
writer.Flush ();
94105
}
95106
}
107+
108+
static class StringExtensions {
109+
[return: NotNullIfNotNull (nameof (value))]
110+
public static string? Cap (this string? value, int length)
111+
{
112+
if (string.IsNullOrEmpty (value))
113+
return value;
114+
115+
if (value.Length <= length)
116+
return value;
117+
118+
return value [0..length] + "...";
119+
}
120+
}
96121
}

‎tests/xharness/xharness.csproj‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,9 @@
117117
<Link>ApplePlatform.cs</Link>
118118
</Compile>
119119
<Compile Include="IAppBundleInformationParserExtensions.cs" />
120+
<Compile Include="..\common\ParseTrxFile.cs">
121+
<Link>ParseTrxFile.cs</Link>
122+
</Compile>
120123
</ItemGroup>
121124
<ItemGroup>
122125
<Compile Include="..\..\tools\common\SdkVersions.cs">

‎tools/devops/automation/scripts/TestResults.psm1‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -392,26 +392,36 @@ class ParallelTestsResults {
392392
$stringBuilder.AppendLine($this.GetDownloadLinks($r))
393393
$stringBuilder.AppendLine("")
394394
} else {
395-
# create a detail per test result with the name of the test and will contain the exact summary
396-
$stringBuilder.AppendLine("<summary>$($result.Failed) tests failed, $($result.Passed) tests passed.</summary>")
397-
$stringBuilder.AppendLine("<details>")
395+
$addSummary = $true
396+
$startLine = 0
398397
if (Test-Path -Path $r.ResultsPath -PathType Leaf) {
399-
$stringBuilder.AppendLine("")
400-
$foundTests = $false
401-
foreach ($line in Get-Content -Path $r.ResultsPath)
402-
{
403-
if (-not $foundTests) {
404-
$foundTests = $line.Contains("## Failed tests")
405-
} else {
406-
if (-not [string]::IsNullOrEmpty($line)) {
407-
$stringBuilder.AppendLine("$line") # the extra space is needed for the multiline list item
408-
}
398+
$resultLines = Get-Content -Path $r.ResultsPath
399+
for ($i = 0; $i -lt $resultLines.Length; $i++) {
400+
$line = $resultLines[$i]
401+
if ($line.Contains ("<details>")) {
402+
startLine = $i
403+
addSummary = $false
404+
break
405+
} elseif ($line.Contains("## Failed tests")) {
406+
startLine = $i + 1
407+
break
409408
}
410409
}
411410
} else {
412-
$stringBuilder.AppendLine(" Test has no summary file.")
411+
$resultLines = @("Test has no summary file.")
413412
}
414-
$stringBuilder.AppendLine("</details>")
413+
414+
if ($addSummary) {
415+
$stringBuilder.AppendLine("<summary>$($result.Failed) tests failed, $($result.Passed) tests passed.</summary>")
416+
$stringBuilder.AppendLine("<details>")
417+
}
418+
for ($i = 0; $i -lt $resultLines.Length; $i++) {
419+
$stringBuilder.AppendLine($resultLines[$i])
420+
}
421+
if ($addSummary) {
422+
$stringBuilder.AppendLine("</details>")
423+
}
424+
415425
$stringBuilder.AppendLine("")
416426
$stringBuilder.AppendLine($this.GetDownloadLinks($r))
417427
$stringBuilder.AppendLine("")

0 commit comments

Comments
 (0)