-
Notifications
You must be signed in to change notification settings - Fork 293
Fix O(n²) output accumulation in data-driven tests causing large TRX files #7926
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from 3 commits
6806b99
facdab3
28103bb
8c86c59
7ac5236
ccb4fc2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,8 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
|
||
| using System.Xml.Linq; | ||
|
|
||
| using Microsoft.Testing.Platform.Acceptance.IntegrationTests; | ||
| using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; | ||
| using Microsoft.Testing.Platform.Helpers; | ||
|
|
@@ -89,3 +91,117 @@ public void FailingTest() | |
|
|
||
| public TestContext TestContext { get; set; } = null!; | ||
| } | ||
|
|
||
| [TestClass] | ||
| public sealed class TrxReportDataDrivenOutputTests : AcceptanceTestBase<TrxReportDataDrivenOutputTests.TestAssetFixture> | ||
| { | ||
| /// <summary> | ||
| /// Regression test for https://github.com/microsoft/testfx/issues/7908. | ||
| /// Verifies that data-driven test output does not accumulate across data rows in the TRX file. | ||
| /// Each data row's StdOut in the TRX should contain only that row's output, not output from previous rows. | ||
| /// </summary> | ||
| [TestMethod] | ||
| [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] | ||
| public async Task TrxReport_DataDrivenTestOutput_DoesNotAccumulateAcrossRows(string tfm) | ||
| { | ||
| string fileName = Guid.NewGuid().ToString("N"); | ||
| var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, TestAssetFixture.ProjectName, tfm); | ||
| TestHostResult testHostResult = await testHost.ExecuteAsync($"--report-trx --report-trx-filename {fileName}.trx", cancellationToken: TestContext.CancellationToken); | ||
|
|
||
| testHostResult.AssertExitCodeIs(ExitCodes.Success); | ||
|
Evangelink marked this conversation as resolved.
Outdated
|
||
|
|
||
| string trxFile = Directory.GetFiles(testHost.DirectoryName, $"{fileName}.trx", SearchOption.AllDirectories).Single(); | ||
| string trxContent = File.ReadAllText(trxFile); | ||
|
|
||
| XNamespace ns = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"; | ||
| var trxDoc = XDocument.Parse(trxContent); | ||
| var results = trxDoc.Descendants(ns + "UnitTestResult").ToList(); | ||
|
|
||
| // We have 3 data rows, each writing a unique marker like "UNIQUE_ROW_0_MARKER", "UNIQUE_ROW_1_MARKER", "UNIQUE_ROW_2_MARKER" | ||
| Assert.IsGreaterThanOrEqualTo(3, results.Count, $"Expected at least 3 test results but found {results.Count}. TRX content:\n{trxContent}"); | ||
|
|
||
| int resultsWithOutput = 0; | ||
| foreach (XElement result in results) | ||
| { | ||
| string? stdOut = result.Descendants(ns + "StdOut").FirstOrDefault()?.Value; | ||
| if (stdOut is null) | ||
| { | ||
| continue; | ||
| } | ||
|
Comment on lines
+124
to
+130
|
||
|
|
||
| resultsWithOutput++; | ||
|
|
||
| // Count how many unique row markers appear in this single result's StdOut. | ||
| // Each result should contain exactly ONE marker (its own row's output). | ||
| int markerCount = 0; | ||
| for (int i = 0; i < 3; i++) | ||
| { | ||
| if (stdOut.Contains($"UNIQUE_ROW_{i}_MARKER")) | ||
| { | ||
| markerCount++; | ||
| } | ||
|
Evangelink marked this conversation as resolved.
|
||
| } | ||
|
|
||
| Assert.AreEqual( | ||
| 1, | ||
| markerCount, | ||
| $"Test result '{result.Attribute("testName")?.Value}' contains output from {markerCount} data rows. " + | ||
| $"Each row should only contain its own output. StdOut:\n{stdOut}"); | ||
| } | ||
|
|
||
| Assert.IsGreaterThanOrEqualTo(3, resultsWithOutput, $"Expected at least 3 test results to have StdOut output but only {resultsWithOutput} did. TRX content:\n{trxContent}"); | ||
| } | ||
|
Evangelink marked this conversation as resolved.
|
||
|
|
||
| public sealed class TestAssetFixture() : TestAssetFixtureBase() | ||
| { | ||
| public const string ProjectName = "MSTestTrxDataDriven"; | ||
|
|
||
| public string TargetAssetPath => GetAssetPath(ProjectName); | ||
|
|
||
| public override (string ID, string Name, string Code) GetAssetsToGenerate() => (ProjectName, ProjectName, | ||
| SourceCode | ||
| .PatchTargetFrameworks(TargetFrameworks.All) | ||
| .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) | ||
| .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); | ||
|
|
||
| private const string SourceCode = """ | ||
| #file MSTestTrxDataDriven.csproj | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <OutputType>Exe</OutputType> | ||
| <EnableMSTestRunner>true</EnableMSTestRunner> | ||
| <TargetFrameworks>$TargetFrameworks$</TargetFrameworks> | ||
| <LangVersion>latest</LangVersion> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Microsoft.Testing.Platform" Version="$MicrosoftTestingPlatformVersion$" /> | ||
| <PackageReference Include="MSTest" Version="$MSTestVersion$" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> | ||
|
|
||
| #file UnitTest1.cs | ||
| using System; | ||
| using Microsoft.VisualStudio.TestTools.UnitTesting; | ||
|
|
||
| namespace MSTestTrxDataDriven; | ||
|
|
||
| [TestClass] | ||
| public class UnitTest1 | ||
| { | ||
| [TestMethod] | ||
| [DataRow(0)] | ||
| [DataRow(1)] | ||
| [DataRow(2)] | ||
| public void DataDrivenTestWithOutput(int row) | ||
| { | ||
| Console.WriteLine($"UNIQUE_ROW_{row}_MARKER"); | ||
| } | ||
| } | ||
| """; | ||
| } | ||
|
|
||
| public TestContext TestContext { get; set; } = null!; | ||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.