Skip to content

Exclude some not run tests from console report #1722

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
CharliePoole merged 1 commit into from
Jul 3, 2025
Merged

Exclude some not run tests from console report #1722

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/NUnitCommon/nunit.agent.core/AgentDirectRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public void ExecuteTestsDirectly()
var xmlResult = runner.Run(null, TestFilter.Empty).Xml;
var summary = new ResultSummary(xmlResult);

ResultReporter.ReportResults(summary, OutWriter);
new ResultReporter().ReportResults(summary, OutWriter);

var pathToResultFile = Path.Combine(_options.WorkDirectory, "TestResult.xml");
WriteResultFile(xmlResult, pathToResultFile);
Expand Down
46 changes: 33 additions & 13 deletions src/NUnitCommon/nunit.common.tests/TextDisplay/ResultReporterTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Xml;
using NUnit.Common;
Expand Down Expand Up @@ -71,25 +72,20 @@ public void SetUp()
[TearDown]
public void TearDown() => _writer.Dispose();

[Test]
public void ReportSequenceTest()
[TestCase(false, "Tests Not Run", "Errors, Failures and Warnings", "Test Run Summary")]
[TestCase(true, "Errors, Failures and Warnings", "Test Run Summary")]
public void ReportSequenceTest(bool omitNotRunReport, params string[] reportSequence)
{
ResultReporter.ReportResults(new ResultSummary(_result), _writer);

var reportSequence = new[]
{
"Tests Not Run",
"Errors, Failures and Warnings",
"Test Run Summary"
};
var settings = new ResultReporterSettings() { OmitNotRunReport = omitNotRunReport };
new ResultReporter(settings).ReportResults(new ResultSummary(_result), _writer);

int last = -1;

string reportOutput = _reportBuilder.ToString();
foreach (string title in reportSequence)
{
var index = reportOutput.IndexOf(title);
Assert.That(index > 0, "Report not found: " + title);
Assert.That(index >= 0, "Report not found: " + title);
Assert.That(index > last, "Report out of sequence: " + title);
last = index;
}
Expand Down Expand Up @@ -139,7 +135,7 @@ public void ErrorsFailuresAndWarningsReportTest()
"No suitable constructor was found"
};

ResultReporter.WriteErrorsFailuresAndWarningsReport(_result, _writer);
new ResultReporter().WriteErrorsFailuresAndWarningsReport(_result, _writer);

string reportOutput = _reportBuilder.ToString();
foreach (var item in expected)
Expand Down Expand Up @@ -175,10 +171,34 @@ public void TestsNotRunTest()
string.Empty
};

ResultReporter.WriteNotRunReport(_result, _writer);
new ResultReporter().WriteNotRunReport(_result, _writer);
Assert.That(ReportLines, Is.EqualTo(expected));
}

[Test]
public void TestsNotRun_OmitExplicit()
{
var settings = new ResultReporterSettings() { OmitExplicitTests = true };
new ResultReporter(settings).WriteNotRunReport(_result, _writer);
Assert.That(ReportLines.Where(line => line.Contains("Explicit")), Is.Empty);
}

[Test]
public void TestsNotRun_OmitIgnored()
{
var settings = new ResultReporterSettings() { OmitIgnoredTests = true };
new ResultReporter(settings).WriteNotRunReport(_result, _writer);
Assert.That(ReportLines.Where(line => line.Contains("Ignored")), Is.Empty);
}

[Test]
public void TestsNotRun_OmitExplicitAndIgnored()
{
var settings = new ResultReporterSettings() { OmitExplicitTests = true, OmitIgnoredTests = true };
new ResultReporter(settings).WriteNotRunReport(_result, _writer);
Assert.That(ReportLines.Where(line => line.Contains("Explicit") || line.Contains("Ignored")), Is.Empty);
}

[Test]
public void TestParameterSettingsWrittenCorrectly()
{
Expand Down
51 changes: 39 additions & 12 deletions src/NUnitCommon/nunit.common/TextDisplay/ResultReporter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,31 @@
// Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt

using System;
using System.Configuration;
using System.Diagnostics;
using System.Globalization;
using System.Reflection;
using System.Reflection.Emit;
using System.Runtime.CompilerServices;
using System.Xml;
using NUnit.Common;

namespace NUnit.TextDisplay
{
public static class ResultReporter
public class ResultReporter
{
private ResultReporterSettings _settings;

public ResultReporter()
{
_settings = new ResultReporterSettings();
}

public ResultReporter(ResultReporterSettings settings)
{
_settings = settings;
}

public static void WriteHeader(ExtendedTextWriter writer)
{
Assembly entryAssembly = Assembly.GetEntryAssembly() ?? Assembly.GetExecutingAssembly();
Expand All @@ -27,24 +42,22 @@ public static void WriteHeader(ExtendedTextWriter writer)
/// <summary>
/// Reports the results to the console
/// </summary>
public static void ReportResults(ResultSummary summary, ExtendedTextWriter writer, bool stopOnError = false)
public void ReportResults(ResultSummary summary, ExtendedTextWriter writer)
{
writer.WriteLine();

var topLevelResult = summary.ResultNode;

if (summary.ExplicitCount + summary.SkipCount + summary.IgnoreCount > 0)
WriteNotRunReport(topLevelResult, writer);

if (summary.OverallResult == "Failed" || summary.WarningCount > 0)
WriteErrorsFailuresAndWarningsReport(topLevelResult, writer, stopOnError);
WriteErrorsFailuresAndWarningsReport(topLevelResult, writer);

WriteRunSettingsReport(topLevelResult, writer);

WriteSummaryReport(summary, writer);
}

public static void WriteRunSettingsReport(XmlNode topLevelResult, ExtendedTextWriter writer)
internal static void WriteRunSettingsReport(XmlNode topLevelResult, ExtendedTextWriter writer)
{
var firstSuite = topLevelResult.SelectSingleNode("test-suite");
if (firstSuite is not null)
Expand Down Expand Up @@ -84,7 +97,7 @@ private static void WriteSettingsNode(XmlNode node, ExtendedTextWriter writer)
}
}

public static void WriteSummaryReport(ResultSummary summary, ExtendedTextWriter writer)
internal static void WriteSummaryReport(ResultSummary summary, ExtendedTextWriter writer)
{
const string INDENT4 = " ";
const string INDENT8 = " ";
Expand Down Expand Up @@ -135,15 +148,15 @@ public static void WriteSummaryReport(ResultSummary summary, ExtendedTextWriter
writer.WriteLine();
}

public static void WriteErrorsFailuresAndWarningsReport(XmlNode resultNode, ExtendedTextWriter writer, bool stopOnError = false)
internal void WriteErrorsFailuresAndWarningsReport(XmlNode resultNode, ExtendedTextWriter writer)
{
int reportIndex = 0;
writer.WriteLine(ColorStyle.SectionHeader, "Errors, Failures and Warnings");
writer.WriteLine();

WriteErrorsFailuresAndWarnings(resultNode, writer, ref reportIndex);

if (stopOnError)
if (_settings.StopOnError)
{
writer.WriteLine(ColorStyle.Failure, "Execution terminated after first error");
writer.WriteLine();
Expand Down Expand Up @@ -211,23 +224,37 @@ private static void WriteErrorsFailuresAndWarnings(XmlNode resultNode, ExtendedT
}
}

public static void WriteNotRunReport(XmlNode resultNode, ExtendedTextWriter writer)
internal void WriteNotRunReport(XmlNode resultNode, ExtendedTextWriter writer)
{
int reportIndex = 0;
writer.WriteLine(ColorStyle.SectionHeader, "Tests Not Run");
writer.WriteLine();
WriteNotRunResults(resultNode, writer, ref reportIndex);
}

private static void WriteNotRunResults(XmlNode resultNode, ExtendedTextWriter writer, ref int reportIndex)
private void WriteNotRunResults(XmlNode resultNode, ExtendedTextWriter writer, ref int reportIndex)
{
switch (resultNode.Name)
{
case "test-case":
string? status = resultNode.GetAttribute("result");

if (status == "Skipped")
new ClientTestResult(resultNode, ++reportIndex).WriteResult(writer);
{
bool display = true;
string? label = resultNode.GetAttribute("label");
switch (label)
{
case "Explicit":

@veleek veleek Jul 3, 2025

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should these attribute values all be case sensitive, or is it worth doing a case insensitive check?

@CharliePoole CharliePoole Jul 3, 2025

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I remember asking myself the same question as I coded this. What I think is that they are exact here, because we create them exactly. Both the names and the values are case sensitive. They are also exact when used as attributes in the test code. We may allow (I can't remember) case-insensitive values at the command-line, which isn't relevant here.

I think the bottom line is whether we want to accept XML input from non-nunit sources, which might not be completely correct from our point of view. I'd say no to that.

display = !_settings.OmitExplicitTests;
break;
case "Ignored":
display = !_settings.OmitIgnoredTests;
break;
}
if (display)
new ClientTestResult(resultNode, ++reportIndex).WriteResult(writer);
}

break;

Expand Down
18 changes: 18 additions & 0 deletions src/NUnitCommon/nunit.common/TextDisplay/ResultReporterSettings.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
// Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace NUnit.TextDisplay
{
public class ResultReporterSettings
{
public bool StopOnError { get; init; } = false;
public bool OmitExplicitTests { get; init; } = false;
public bool OmitIgnoredTests { get; init; } = false;
public bool OmitNotRunReport { get; init; } = false;
}
}
5 changes: 5 additions & 0 deletions src/NUnitCommon/nunit.common/nunit.common.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@
<TargetFrameworks>net462;netstandard2.0</TargetFrameworks>
</PropertyGroup>

<ItemGroup>
<InternalsVisibleTo Include="nunit.common.tests, PublicKey=$(NUnitPublicKey)" />
<InternalsVisibleTo Include="nunit4-console.tests, PublicKey=$(NUnitPublicKey)" />
</ItemGroup>

<ItemGroup Condition="'$(TargetFramework)'=='netstandard2.0'">
<PackageReference Include="Microsoft.Win32.Registry" Version="4.3.0" />
</ItemGroup>
Expand Down
2 changes: 1 addition & 1 deletion src/NUnitConsole/nunit4-console.tests/BadFileTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public void MissingFileTest(string filename, string message)
var sb = new StringBuilder();
var writer = new ExtendedTextWrapper(new StringWriter(sb));

ResultReporter.WriteErrorsFailuresAndWarningsReport(result, writer);
new ResultReporter().WriteErrorsFailuresAndWarningsReport(result, writer);
var report = sb.ToString();

Assert.That(report, Contains.Substring($"1) Invalid : {fullname}"));
Expand Down
18 changes: 18 additions & 0 deletions src/NUnitConsole/nunit4-console.tests/CommandLineTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using NUnit.ConsoleRunner.Options;

using Spec = NUnit.ConsoleRunner.Options.OutputSpecification;
using NUnit.TextDisplay;

namespace NUnit.ConsoleRunner
{
Expand Down Expand Up @@ -141,6 +142,9 @@ public void NoInputFiles()
[TestCase(FrameworkPackageSettings.StopOnError, "stoponerror")]
[TestCase("WaitBeforeExit", "wait")]
[TestCase("NoHeader", "noheader|noh")]
[TestCase("OmitExplicitTests", "omitexplicit")]
[TestCase("OmitIgnoredTests", "omitignored")]
[TestCase("OmitNotRunReport", "omitnotrunreport")]
[TestCase("DisposeRunners", "dispose-runners")]
[TestCase("SkipNonTestAssemblies", "skipnontestassemblies")]
[TestCase("NoResultSpecified", "noresult")]
Expand Down Expand Up @@ -727,6 +731,20 @@ public void DisableExtensionTest()
ConsoleOptions options = ConsoleMocks.Options("--disable=NUnit.Engine.Listeners.TeamCityEventListener");
Assert.That(options.DisableExtensions.Contains("NUnit.Engine.Listeners.TeamCityEventListener"));
}

[TestCase("--omitexplicit", true, false, false)]
[TestCase("--omitignored", false, true, false)]
[TestCase("--omitignored --omitexplicit", true, true, false)]
[TestCase("--omitnotrunreport", false, false, true)]
public void ResultReporterSettingsTest(string arglist, bool explicitOmitted, bool ignoredOmitted, bool reportOmitted)
{
ConsoleOptions options = ConsoleMocks.Options(arglist.Split(' '));

Assert.That(options.ResultReporterSettings.OmitExplicitTests, Is.EqualTo(explicitOmitted));
Assert.That(options.ResultReporterSettings.OmitIgnoredTests, Is.EqualTo(ignoredOmitted));
Assert.That(options.ResultReporterSettings.OmitNotRunReport, Is.EqualTo(reportOmitted));
}

private static VirtualFileSystem GetFileSystemContainingFile(string fileName)
{
var fileSystem = new VirtualFileSystem();
Expand Down
25 changes: 25 additions & 0 deletions src/NUnitConsole/nunit4-console/ConsoleOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using NUnit.ConsoleRunner.Options;
using NUnit.TextDisplay;

namespace NUnit.ConsoleRunner
{
Expand Down Expand Up @@ -34,6 +36,13 @@ internal ConsoleOptions(IFileSystem fileSystem, params string[] args)
ConfigureOptions();
if (args is not null)
Parse(args);

ResultReporterSettings = new ResultReporterSettings()
{
OmitExplicitTests = OmitExplicitTests,
OmitIgnoredTests = OmitIgnoredTests,
OmitNotRunReport = OmitNotRunReport
};
}

// Action to Perform
Expand Down Expand Up @@ -97,6 +106,13 @@ public bool NumberOfTestWorkersSpecified

public bool NoHeader { get; private set; }

public bool OmitIgnoredTests { get; private set; }
public bool OmitExplicitTests { get; private set; }

public bool OmitNotRunReport { get; private set; }

public ResultReporterSettings ResultReporterSettings { get; private set; }

public bool NoColor { get; private set; }

public string? OutFile { get; private set; }
Expand Down Expand Up @@ -327,6 +343,15 @@ private void ConfigureOptions()
this.Add("noheader|noh", "Suppress display of program information at start of run.",
v => NoHeader = !string.IsNullOrEmpty(v));

this.Add("omitignored", "Suppress listing of tests not run because they were ignored.",
v => OmitIgnoredTests = !string.IsNullOrEmpty(v));

this.Add("omitexplicit", "Suppress listing of tests not run because they are explicit.",
v => OmitExplicitTests = !string.IsNullOrEmpty(v));

this.Add("omitnotrunreport", "Suppress the entire report of tests not run.",
v => OmitNotRunReport = !string.IsNullOrEmpty(v));

this.Add("nocolor|noc", "Displays console output without color.",
v => NoColor = !string.IsNullOrEmpty(v));

Expand Down
4 changes: 3 additions & 1 deletion src/NUnitConsole/nunit4-console/ConsoleRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,9 @@ private int RunTests(TestPackage package, TestFilter filter)
if (result is not null)
{
var summary = new ResultSummary(result);
ResultReporter.ReportResults(summary, writer, _options.StopOnError);
var settings = _options.ResultReporterSettings;

new ResultReporter(settings).ReportResults(summary, writer);

foreach (var spec in _options.ResultOutputSpecifications)
{
Expand Down
Loading