From 6c528cd5f5d3b4603e2c689b457548516a30b86e Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Sun, 12 Apr 2020 17:45:14 +0200 Subject: [PATCH 01/11] add integration tests --- .gitignore | 3 + .../DeepThought.cs | 10 ++ .../TemplateTest.cs | 14 +++ ...verlet.integration.determisticbuild.csproj | 28 +++++ test/coverlet.integration.tests/BaseTest.cs | 13 ++- test/coverlet.integration.tests/Collectors.cs | 1 - .../DeterministicBuild.cs | 107 ++++++++++++++++++ test/coverlet.integration.tests/Msbuild.cs | 1 + 8 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 test/coverlet.integration.determisticbuild/DeepThought.cs create mode 100644 test/coverlet.integration.determisticbuild/TemplateTest.cs create mode 100644 test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj create mode 100644 test/coverlet.integration.tests/DeterministicBuild.cs diff --git a/.gitignore b/.gitignore index 7a005aa76..177132965 100644 --- a/.gitignore +++ b/.gitignore @@ -301,3 +301,6 @@ __pycache__/ # DeterministicSourcePaths workaround generated file DeterministicBuild.targets .AssemblyAttributes +DeterministicTest.props +test/coverlet.integration.determisticbuild/*.txt +test/coverlet.integration.determisticbuild/runsettings \ No newline at end of file diff --git a/test/coverlet.integration.determisticbuild/DeepThought.cs b/test/coverlet.integration.determisticbuild/DeepThought.cs new file mode 100644 index 000000000..bd7ac7a9d --- /dev/null +++ b/test/coverlet.integration.determisticbuild/DeepThought.cs @@ -0,0 +1,10 @@ +namespace Coverlet.Integration.DeterministicBuild +{ + public class DeepThought + { + public int AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything() + { + return 42; + } + } +} diff --git a/test/coverlet.integration.determisticbuild/TemplateTest.cs b/test/coverlet.integration.determisticbuild/TemplateTest.cs new file mode 100644 index 000000000..3abca1a40 --- /dev/null +++ b/test/coverlet.integration.determisticbuild/TemplateTest.cs @@ -0,0 +1,14 @@ +using Xunit; + +namespace Coverlet.Integration.DeterministicBuild +{ + public class TemplateTest + { + [Fact] + public void Answer() + { + DeepThought dt = new DeepThought(); + Assert.Equal(42, dt.AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything()); + } + } +} diff --git a/test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj b/test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj new file mode 100644 index 000000000..cd8044ac8 --- /dev/null +++ b/test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj @@ -0,0 +1,28 @@ + + + + + + netcoreapp3.1 + false + coverletsample.integration.determisticbuild + + https://api.nuget.org/v3/index.json; + $(RepoRoot)bin\$(Configuration)\Packages + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + \ No newline at end of file diff --git a/test/coverlet.integration.tests/BaseTest.cs b/test/coverlet.integration.tests/BaseTest.cs index e174ae486..a0c14aa6f 100644 --- a/test/coverlet.integration.tests/BaseTest.cs +++ b/test/coverlet.integration.tests/BaseTest.cs @@ -15,7 +15,7 @@ namespace Coverlet.Integration.Tests { [Flags] - enum BuildConfiguration + public enum BuildConfiguration { Debug = 1, Release = 2 @@ -24,7 +24,8 @@ enum BuildConfiguration public abstract class BaseTest { private static int _folderSuffix = 0; - private BuildConfiguration GetAssemblyBuildConfiguration() + + protected BuildConfiguration GetAssemblyBuildConfiguration() { var configurationAttribute = Assembly.GetExecutingAssembly().GetCustomAttribute(); @@ -63,7 +64,7 @@ private protected string GetPackageVersion(string filter) private protected ClonedTemplateProject CloneTemplateProject(bool cleanupOnDispose = true, string testSDKVersion = "16.5.0") { - DirectoryInfo finalRoot = Directory.CreateDirectory($"template{Interlocked.Increment(ref _folderSuffix)}"); + DirectoryInfo finalRoot = Directory.CreateDirectory($"tmpprj{Interlocked.Increment(ref _folderSuffix)}"); foreach (string file in (Directory.GetFiles($"../../../../coverlet.integration.template", "*.cs") .Union(Directory.GetFiles($"../../../../coverlet.integration.template", "*.csproj") .Union(Directory.GetFiles($"../../../../coverlet.integration.template", "nuget.config"))))) @@ -189,17 +190,17 @@ private protected void AddCoverletCollectosRef(string projectPath) xml.Save(csproj); } - private protected string AddCollectorRunsettingsFile(string projectPath) + private protected string AddCollectorRunsettingsFile(string projectPath,string includeFilter = "[coverletsamplelib.integration.template]*DeepThought") { string runSettings = -@" +$@" json,cobertura - [coverletsamplelib.integration.template]*DeepThought + {includeFilter} true diff --git a/test/coverlet.integration.tests/Collectors.cs b/test/coverlet.integration.tests/Collectors.cs index 2ba8a4f37..21c345175 100644 --- a/test/coverlet.integration.tests/Collectors.cs +++ b/test/coverlet.integration.tests/Collectors.cs @@ -68,7 +68,6 @@ private protected virtual void AssertCollectorsInjection(ClonedTemplateProject c public void TestVsTest_Test() { using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); - string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath!); Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); // We don't have any result to check because tests and code to instrument are in same assembly so we need to pass // IncludeTestAssembly=true we do it in other test diff --git a/test/coverlet.integration.tests/DeterministicBuild.cs b/test/coverlet.integration.tests/DeterministicBuild.cs new file mode 100644 index 000000000..d013da712 --- /dev/null +++ b/test/coverlet.integration.tests/DeterministicBuild.cs @@ -0,0 +1,107 @@ +using Coverlet.Core; +using Newtonsoft.Json; +using System; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +using Xunit; + +namespace Coverlet.Integration.Tests +{ + public class DeterministicBuild : BaseTest, IDisposable + { + private string _testProjectPath = Path.GetFullPath("../../../../coverlet.integration.determisticbuild"); + private string? _testProjectTfm; + private const string PropsFileName = "DeterministicTest.props"; + + private void CreateDeterministicTestPropsFile() + { + XDocument deterministicTestProps = new XDocument(); + deterministicTestProps.Add( + new XElement("Project", + new XElement("PropertyGroup", + new XElement("coverletMsbuilVersion", GetPackageVersion("*msbuild*.nupkg")), + new XElement("coverletCollectorsVersion", GetPackageVersion("*collector*.nupkg"))))); + _testProjectTfm = XElement.Load(Path.Combine(_testProjectPath, "coverlet.integration.determisticbuild.csproj")). + Descendants("PropertyGroup").Single().Element("TargetFramework").Value; + + deterministicTestProps.Save(Path.Combine(_testProjectPath, PropsFileName)); + } + + private protected void AssertCoverage(string standardOutput = "") + { + bool coverageChecked = false; + string reportFilePath = ""; + foreach (string coverageFile in Directory.GetFiles(_testProjectPath, "coverage.json", SearchOption.AllDirectories)) + { + JsonConvert.DeserializeObject(File.ReadAllText(coverageFile)) + .Document("DeepThought.cs") + .Class("Coverlet.Integration.DeterministicBuild.DeepThought") + .Method("System.Int32 Coverlet.Integration.DeterministicBuild.DeepThought::AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything()") + .AssertLinesCovered((6, 1), (7, 1), (8, 1)); + coverageChecked = true; + reportFilePath = coverageFile; + } + Assert.True(coverageChecked, $"Coverage check fail\n{standardOutput}"); + File.Delete(reportFilePath); + Assert.False(File.Exists(reportFilePath)); + } + + [Fact] + public void Msbuild() + { + CreateDeterministicTestPropsFile(); + DotnetCli("build /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); + Assert.Contains("Build succeeded.", standardOutput); + string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "Bin", GetAssemblyBuildConfiguration().ToString(), _testProjectTfm!, "CoverletSourceRootsMapping"); + Assert.True(File.Exists(sourceRootMappingFilePath)); + Assert.NotEmpty(File.ReadAllText(sourceRootMappingFilePath)); + Assert.Equal(2, File.ReadAllLines(sourceRootMappingFilePath).Length); + + DotnetCli($"test --no-build /p:CollectCoverage=true /p:Include=\"[coverletsample.integration.determisticbuild]*DeepThought\" /p:IncludeTestAssembly=true", out standardOutput, out standardError, _testProjectPath); + Assert.Contains("Test Run Successful.", standardOutput); + Assert.Contains("| coverletsample.integration.determisticbuild | 100% | 100% | 100% |", standardOutput); + Assert.True(File.Exists(Path.Combine(_testProjectPath, "coverage.json"))); + AssertCoverage(standardOutput); + + // Process exits hang on clean seem that process doesn't close, maybe some mbuild node reuse? btw manually tested + // DotnetCli("clean", out standardOutput, out standardError, _fixture.TestProjectPath); + // Assert.False(File.Exists(sourceRootMappingFilePath)); + RunCommand("git", "clean -fdx", out _, out _, _testProjectPath); + } + + [Fact] + public void Collectors() + { + CreateDeterministicTestPropsFile(); + DotnetCli("build /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); + Assert.Contains("Build succeeded.", standardOutput); + string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "Bin", GetAssemblyBuildConfiguration().ToString(), _testProjectTfm!, "CoverletSourceRootsMapping"); + Assert.True(File.Exists(sourceRootMappingFilePath)); + Assert.NotEmpty(File.ReadAllText(sourceRootMappingFilePath)); + Assert.Equal(2, File.ReadAllLines(sourceRootMappingFilePath).Length); + + string runSettingsPath = AddCollectorRunsettingsFile(_testProjectPath, "[coverletsample.integration.determisticbuild]*DeepThought"); + Assert.True(DotnetCli($"test --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out standardError), standardOutput); + Assert.Contains("Test Run Successful.", standardOutput); + AssertCoverage(standardOutput); + + // Check out/in process collectors injection + string dataCollectorLogContent = File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.datacollector.*.txt").Single()); + Assert.Contains("[coverlet]Initializing CoverletCoverageDataCollector with configuration:", dataCollectorLogContent); + Assert.Contains("[coverlet]Initialize CoverletInProcDataCollector", File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.host.*.txt").Single())); + Assert.Contains("[coverlet]Mapping resolved", dataCollectorLogContent); + + // Process exits hang on clean seem that process doesn't close, maybe some mbuild node reuse? btw manually tested + // DotnetCli("clean", out standardOutput, out standardError, _fixture.TestProjectPath); + // Assert.False(File.Exists(sourceRootMappingFilePath)); + RunCommand("git", "clean -fdx", out _, out _, _testProjectPath); + } + + public void Dispose() + { + File.Delete(Path.Combine(_testProjectPath, PropsFileName)); + } + } +} diff --git a/test/coverlet.integration.tests/Msbuild.cs b/test/coverlet.integration.tests/Msbuild.cs index e6c9be57e..834473c8a 100644 --- a/test/coverlet.integration.tests/Msbuild.cs +++ b/test/coverlet.integration.tests/Msbuild.cs @@ -1,5 +1,6 @@ using System.IO; using System.Linq; +using System.Xml.Linq; using Xunit; namespace Coverlet.Integration.Tests From 2843922914ad8ff9347435c18474868dcca5f65c Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Sun, 12 Apr 2020 17:46:27 +0200 Subject: [PATCH 02/11] cleanup --- test/coverlet.integration.tests/Msbuild.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/coverlet.integration.tests/Msbuild.cs b/test/coverlet.integration.tests/Msbuild.cs index 834473c8a..e6c9be57e 100644 --- a/test/coverlet.integration.tests/Msbuild.cs +++ b/test/coverlet.integration.tests/Msbuild.cs @@ -1,6 +1,5 @@ using System.IO; using System.Linq; -using System.Xml.Linq; using Xunit; namespace Coverlet.Integration.Tests From c32711a85dcac2d936fecf7068c6db2d09e104a3 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 13 Apr 2020 09:40:18 +0200 Subject: [PATCH 03/11] ci test --- .../DeterministicBuild.cs | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/test/coverlet.integration.tests/DeterministicBuild.cs b/test/coverlet.integration.tests/DeterministicBuild.cs index d013da712..3ce26e230 100644 --- a/test/coverlet.integration.tests/DeterministicBuild.cs +++ b/test/coverlet.integration.tests/DeterministicBuild.cs @@ -68,36 +68,36 @@ public void Msbuild() // Process exits hang on clean seem that process doesn't close, maybe some mbuild node reuse? btw manually tested // DotnetCli("clean", out standardOutput, out standardError, _fixture.TestProjectPath); // Assert.False(File.Exists(sourceRootMappingFilePath)); - RunCommand("git", "clean -fdx", out _, out _, _testProjectPath); + // RunCommand("git", "clean -fdx", out _, out _, _testProjectPath); } - [Fact] - public void Collectors() - { - CreateDeterministicTestPropsFile(); - DotnetCli("build /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); - Assert.Contains("Build succeeded.", standardOutput); - string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "Bin", GetAssemblyBuildConfiguration().ToString(), _testProjectTfm!, "CoverletSourceRootsMapping"); - Assert.True(File.Exists(sourceRootMappingFilePath)); - Assert.NotEmpty(File.ReadAllText(sourceRootMappingFilePath)); - Assert.Equal(2, File.ReadAllLines(sourceRootMappingFilePath).Length); + //[Fact] + //public void Collectors() + //{ + // CreateDeterministicTestPropsFile(); + // DotnetCli("build /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); + // Assert.Contains("Build succeeded.", standardOutput); + // string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "Bin", GetAssemblyBuildConfiguration().ToString(), _testProjectTfm!, "CoverletSourceRootsMapping"); + // Assert.True(File.Exists(sourceRootMappingFilePath)); + // Assert.NotEmpty(File.ReadAllText(sourceRootMappingFilePath)); + // Assert.Equal(2, File.ReadAllLines(sourceRootMappingFilePath).Length); - string runSettingsPath = AddCollectorRunsettingsFile(_testProjectPath, "[coverletsample.integration.determisticbuild]*DeepThought"); - Assert.True(DotnetCli($"test --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out standardError), standardOutput); - Assert.Contains("Test Run Successful.", standardOutput); - AssertCoverage(standardOutput); + // string runSettingsPath = AddCollectorRunsettingsFile(_testProjectPath, "[coverletsample.integration.determisticbuild]*DeepThought"); + // Assert.True(DotnetCli($"test --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out standardError), standardOutput); + // Assert.Contains("Test Run Successful.", standardOutput); + // AssertCoverage(standardOutput); - // Check out/in process collectors injection - string dataCollectorLogContent = File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.datacollector.*.txt").Single()); - Assert.Contains("[coverlet]Initializing CoverletCoverageDataCollector with configuration:", dataCollectorLogContent); - Assert.Contains("[coverlet]Initialize CoverletInProcDataCollector", File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.host.*.txt").Single())); - Assert.Contains("[coverlet]Mapping resolved", dataCollectorLogContent); + // // Check out/in process collectors injection + // string dataCollectorLogContent = File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.datacollector.*.txt").Single()); + // Assert.Contains("[coverlet]Initializing CoverletCoverageDataCollector with configuration:", dataCollectorLogContent); + // Assert.Contains("[coverlet]Initialize CoverletInProcDataCollector", File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.host.*.txt").Single())); + // Assert.Contains("[coverlet]Mapping resolved", dataCollectorLogContent); - // Process exits hang on clean seem that process doesn't close, maybe some mbuild node reuse? btw manually tested - // DotnetCli("clean", out standardOutput, out standardError, _fixture.TestProjectPath); - // Assert.False(File.Exists(sourceRootMappingFilePath)); - RunCommand("git", "clean -fdx", out _, out _, _testProjectPath); - } + // // Process exits hang on clean seem that process doesn't close, maybe some mbuild node reuse? btw manually tested + // // DotnetCli("clean", out standardOutput, out standardError, _fixture.TestProjectPath); + // // Assert.False(File.Exists(sourceRootMappingFilePath)); + // RunCommand("git", "clean -fdx", out _, out _, _testProjectPath); + //} public void Dispose() { From a5e8785ac20d52097b96dcd01f787cdebca18a8c Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 13 Apr 2020 09:47:30 +0200 Subject: [PATCH 04/11] ci test --- test/coverlet.integration.tests/DeterministicBuild.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/coverlet.integration.tests/DeterministicBuild.cs b/test/coverlet.integration.tests/DeterministicBuild.cs index 3ce26e230..59af669db 100644 --- a/test/coverlet.integration.tests/DeterministicBuild.cs +++ b/test/coverlet.integration.tests/DeterministicBuild.cs @@ -55,8 +55,8 @@ public void Msbuild() DotnetCli("build /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); Assert.Contains("Build succeeded.", standardOutput); string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "Bin", GetAssemblyBuildConfiguration().ToString(), _testProjectTfm!, "CoverletSourceRootsMapping"); - Assert.True(File.Exists(sourceRootMappingFilePath)); - Assert.NotEmpty(File.ReadAllText(sourceRootMappingFilePath)); + Assert.True(File.Exists(sourceRootMappingFilePath), sourceRootMappingFilePath); + Assert.True(!string.IsNullOrEmpty(File.ReadAllText(sourceRootMappingFilePath)), "Empty CoverletSourceRootsMapping file"); Assert.Equal(2, File.ReadAllLines(sourceRootMappingFilePath).Length); DotnetCli($"test --no-build /p:CollectCoverage=true /p:Include=\"[coverletsample.integration.determisticbuild]*DeepThought\" /p:IncludeTestAssembly=true", out standardOutput, out standardError, _testProjectPath); From fe209803aa99158abae86d667f0ea37523eab761 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 13 Apr 2020 10:00:15 +0200 Subject: [PATCH 05/11] ci test --- eng/build.yml | 2 +- test/coverlet.integration.tests/BaseTest.cs | 28 ++++++--------------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/eng/build.yml b/eng/build.yml index dbf368942..957579706 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -15,7 +15,7 @@ steps: - script: dotnet build -c $(BuildConfiguration) --no-restore displayName: Build -- script: dotnet pack -c $(BuildConfiguration) +- script: dotnet pack -c $(BuildConfiguration) --no-restore --no-build displayName: Pack - task: DotNetCoreCLI@2 diff --git a/test/coverlet.integration.tests/BaseTest.cs b/test/coverlet.integration.tests/BaseTest.cs index a0c14aa6f..76638c826 100644 --- a/test/coverlet.integration.tests/BaseTest.cs +++ b/test/coverlet.integration.tests/BaseTest.cs @@ -27,25 +27,13 @@ public abstract class BaseTest protected BuildConfiguration GetAssemblyBuildConfiguration() { - var configurationAttribute = Assembly.GetExecutingAssembly().GetCustomAttribute(); - - if (configurationAttribute is null) - { - throw new ArgumentNullException("AssemblyConfigurationAttribute not found"); - } - - if (configurationAttribute.Configuration.Equals("Debug", StringComparison.InvariantCultureIgnoreCase)) - { - return BuildConfiguration.Debug; - } - else if (configurationAttribute.Configuration.Equals("Release", StringComparison.InvariantCultureIgnoreCase)) - { - return BuildConfiguration.Release; - } - else - { - throw new NotSupportedException($"Build configuration '{configurationAttribute.Configuration}' not supported"); - } +#if DEBUG + return BuildConfiguration.Debug; +#endif +#if RELEASE + return BuildConfiguration.Release; +#endif + throw new NotSupportedException($"Build configuration not supported"); } private protected string GetPackageVersion(string filter) @@ -190,7 +178,7 @@ private protected void AddCoverletCollectosRef(string projectPath) xml.Save(csproj); } - private protected string AddCollectorRunsettingsFile(string projectPath,string includeFilter = "[coverletsamplelib.integration.template]*DeepThought") + private protected string AddCollectorRunsettingsFile(string projectPath, string includeFilter = "[coverletsamplelib.integration.template]*DeepThought") { string runSettings = $@" From 48c138dddcfdb3050a4a2a298d726644c38cf832 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 13 Apr 2020 10:03:51 +0200 Subject: [PATCH 06/11] ci test --- eng/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/build.yml b/eng/build.yml index 957579706..06a9ae7ed 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -15,7 +15,7 @@ steps: - script: dotnet build -c $(BuildConfiguration) --no-restore displayName: Build -- script: dotnet pack -c $(BuildConfiguration) --no-restore --no-build +- script: dotnet pack -c $(BuildConfiguration) --no-restore displayName: Pack - task: DotNetCoreCLI@2 From d344e57a783b3e6b24c4fc4f438ebe0b5deb012f Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 13 Apr 2020 10:06:42 +0200 Subject: [PATCH 07/11] ci test --- test/coverlet.integration.tests/DeterministicBuild.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/coverlet.integration.tests/DeterministicBuild.cs b/test/coverlet.integration.tests/DeterministicBuild.cs index 59af669db..e55cc3d3e 100644 --- a/test/coverlet.integration.tests/DeterministicBuild.cs +++ b/test/coverlet.integration.tests/DeterministicBuild.cs @@ -51,15 +51,16 @@ private protected void AssertCoverage(string standardOutput = "") [Fact] public void Msbuild() { + string buildConfiguration = GetAssemblyBuildConfiguration().ToString(); CreateDeterministicTestPropsFile(); - DotnetCli("build /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); + DotnetCli($"build -c {buildConfiguration} /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); Assert.Contains("Build succeeded.", standardOutput); - string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "Bin", GetAssemblyBuildConfiguration().ToString(), _testProjectTfm!, "CoverletSourceRootsMapping"); + string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "Bin", buildConfiguration, _testProjectTfm!, "CoverletSourceRootsMapping"); Assert.True(File.Exists(sourceRootMappingFilePath), sourceRootMappingFilePath); Assert.True(!string.IsNullOrEmpty(File.ReadAllText(sourceRootMappingFilePath)), "Empty CoverletSourceRootsMapping file"); Assert.Equal(2, File.ReadAllLines(sourceRootMappingFilePath).Length); - DotnetCli($"test --no-build /p:CollectCoverage=true /p:Include=\"[coverletsample.integration.determisticbuild]*DeepThought\" /p:IncludeTestAssembly=true", out standardOutput, out standardError, _testProjectPath); + DotnetCli($"test -c {buildConfiguration} --no-build /p:CollectCoverage=true /p:Include=\"[coverletsample.integration.determisticbuild]*DeepThought\" /p:IncludeTestAssembly=true", out standardOutput, out standardError, _testProjectPath); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsample.integration.determisticbuild | 100% | 100% | 100% |", standardOutput); Assert.True(File.Exists(Path.Combine(_testProjectPath, "coverage.json"))); From 90f6d33be05dfe4062259166737d8400697a51e3 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 13 Apr 2020 10:58:11 +0200 Subject: [PATCH 08/11] improve tests --- .../Coverage/InstrumenterHelper.Assertions.cs | 20 +++----- .../coverlet.integration.template.csproj | 1 + test/coverlet.integration.tests/BaseTest.cs | 47 ++++++++++++++----- test/coverlet.integration.tests/Collectors.cs | 15 ++++-- .../DeterministicBuild.cs | 42 ++++++++++------- test/coverlet.integration.tests/Msbuild.cs | 31 +++++++----- 6 files changed, 99 insertions(+), 57 deletions(-) diff --git a/test/coverlet.core.tests/Coverage/InstrumenterHelper.Assertions.cs b/test/coverlet.core.tests/Coverage/InstrumenterHelper.Assertions.cs index e8cc928ef..24d9b6f4a 100644 --- a/test/coverlet.core.tests/Coverage/InstrumenterHelper.Assertions.cs +++ b/test/coverlet.core.tests/Coverage/InstrumenterHelper.Assertions.cs @@ -275,19 +275,13 @@ public static Document AssertNonInstrumentedLines(this Document document, BuildC private static BuildConfiguration GetAssemblyBuildConfiguration() { - var configurationAttribute = Assembly.GetExecutingAssembly().GetCustomAttribute(); - if (configurationAttribute.Configuration.Equals("Debug", StringComparison.InvariantCultureIgnoreCase)) - { - return Tests.BuildConfiguration.Debug; - } - else if (configurationAttribute.Configuration.Equals("Release", StringComparison.InvariantCultureIgnoreCase)) - { - return Tests.BuildConfiguration.Release; - } - else - { - throw new NotSupportedException($"Build configuration '{configurationAttribute.Configuration}' not supported"); - } +#if DEBUG + return BuildConfiguration.Debug; +#endif +#if RELEASE + return BuildConfiguration.Release; +#endif + throw new NotSupportedException($"Build configuration not supported"); } } } diff --git a/test/coverlet.integration.template/coverlet.integration.template.csproj b/test/coverlet.integration.template/coverlet.integration.template.csproj index 301a09754..ddd56538f 100644 --- a/test/coverlet.integration.template/coverlet.integration.template.csproj +++ b/test/coverlet.integration.template/coverlet.integration.template.csproj @@ -4,6 +4,7 @@ netcoreapp3.1 false coverletsamplelib.integration.template + false diff --git a/test/coverlet.integration.tests/BaseTest.cs b/test/coverlet.integration.tests/BaseTest.cs index 76638c826..5d7b3717c 100644 --- a/test/coverlet.integration.tests/BaseTest.cs +++ b/test/coverlet.integration.tests/BaseTest.cs @@ -52,7 +52,7 @@ private protected string GetPackageVersion(string filter) private protected ClonedTemplateProject CloneTemplateProject(bool cleanupOnDispose = true, string testSDKVersion = "16.5.0") { - DirectoryInfo finalRoot = Directory.CreateDirectory($"tmpprj{Interlocked.Increment(ref _folderSuffix)}"); + DirectoryInfo finalRoot = Directory.CreateDirectory($"{Guid.NewGuid().ToString("N").Substring(0, 6)}{Interlocked.Increment(ref _folderSuffix)}"); foreach (string file in (Directory.GetFiles($"../../../../coverlet.integration.template", "*.cs") .Union(Directory.GetFiles($"../../../../coverlet.integration.template", "*.csproj") .Union(Directory.GetFiles($"../../../../coverlet.integration.template", "nuget.config"))))) @@ -73,6 +73,8 @@ private protected ClonedTemplateProject CloneTemplateProject(bool cleanupOnDispo AddMicrosoftNETTestSdkRef(finalRoot.FullName, testSDKVersion); + SetIsTestProjectTrue(finalRoot.FullName); + return new ClonedTemplateProject(finalRoot.FullName, cleanupOnDispose); } @@ -120,6 +122,26 @@ private protected void UpdateNugeConfigtWithLocalPackageFolder(string projectPat xml.Save(nugetFile); } + private void SetIsTestProjectTrue(string projectPath) + { + string csproj = Path.Combine(projectPath, "coverlet.integration.template.csproj"); + if (!File.Exists(csproj)) + { + throw new FileNotFoundException("coverlet.integration.template.csproj not found", "coverlet.integration.template.csproj"); + } + XDocument xml; + using (var csprojStream = File.OpenRead(csproj)) + { + xml = XDocument.Load(csprojStream); + } + + xml.Element("Project") + .Element("PropertyGroup") + .Element("IsTestProject").Value = "true"; + + xml.Save(csproj); + } + private protected void AddMicrosoftNETTestSdkRef(string projectPath, string version) { string csproj = Path.Combine(projectPath, "coverlet.integration.template.csproj"); @@ -204,18 +226,21 @@ private protected string AddCollectorRunsettingsFile(string projectPath, string private protected void AssertCoverage(ClonedTemplateProject clonedTemplateProject, string filter = "coverage.json", string standardOutput = "") { - bool coverageChecked = false; - foreach (string coverageFile in clonedTemplateProject.GetFiles(filter)) + if (GetAssemblyBuildConfiguration() == BuildConfiguration.Debug) { - JsonConvert.DeserializeObject(File.ReadAllText(coverageFile)) - .Document("DeepThought.cs") - .Class("Coverlet.Integration.Template.DeepThought") - .Method("System.Int32 Coverlet.Integration.Template.DeepThought::AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything()") - .AssertLinesCovered((6, 1), (7, 1), (8, 1)); - coverageChecked = true; - } + bool coverageChecked = false; + foreach (string coverageFile in clonedTemplateProject.GetFiles(filter)) + { + JsonConvert.DeserializeObject(File.ReadAllText(coverageFile)) + .Document("DeepThought.cs") + .Class("Coverlet.Integration.Template.DeepThought") + .Method("System.Int32 Coverlet.Integration.Template.DeepThought::AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything()") + .AssertLinesCovered((6, 1), (7, 1), (8, 1)); + coverageChecked = true; + } - Assert.True(coverageChecked, $"Coverage check fail\n{standardOutput}"); + Assert.True(coverageChecked, $"Coverage check fail\n{standardOutput}"); + } } private protected void UpdateProjectTargetFramework(ClonedTemplateProject project, params string[] targetFrameworks) diff --git a/test/coverlet.integration.tests/Collectors.cs b/test/coverlet.integration.tests/Collectors.cs index 21c345175..b260741d6 100644 --- a/test/coverlet.integration.tests/Collectors.cs +++ b/test/coverlet.integration.tests/Collectors.cs @@ -42,6 +42,13 @@ public TestSDK_Preview() public abstract class Collectors : BaseTest { + private string _buildConfiguration; + + public Collectors() + { + _buildConfiguration = GetAssemblyBuildConfiguration().ToString(); + } + protected string? TestSDKVersion { get; set; } private ClonedTemplateProject PrepareTemplateProject() @@ -68,7 +75,7 @@ private protected virtual void AssertCollectorsInjection(ClonedTemplateProject c public void TestVsTest_Test() { using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); - Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); + Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); // We don't have any result to check because tests and code to instrument are in same assembly so we need to pass // IncludeTestAssembly=true we do it in other test Assert.Contains("Test Run Successful.", standardOutput); @@ -80,7 +87,7 @@ public void TestVsTest_Test_Settings() { using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath!); - Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError), standardOutput); + Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError), standardOutput); Assert.Contains("Test Run Successful.", standardOutput); AssertCoverage(clonedTemplateProject); AssertCollectorsInjection(clonedTemplateProject); @@ -91,7 +98,7 @@ public void TestVsTest_VsTest() { using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath!); - Assert.True(DotnetCli($"publish {clonedTemplateProject.ProjectRootPath}", out string standardOutput, out string standardError), standardOutput); + Assert.True(DotnetCli($"publish -c {_buildConfiguration} {clonedTemplateProject.ProjectRootPath}", out string standardOutput, out string standardError), standardOutput); string publishedTestFile = clonedTemplateProject.GetFiles("*" + ClonedTemplateProject.AssemblyName + ".dll").Single(f => f.Contains("publish")); Assert.NotNull(publishedTestFile); Assert.True(DotnetCli($"vstest \"{publishedTestFile}\" --collect:\"XPlat Code Coverage\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out standardOutput, out standardError), standardOutput); @@ -106,7 +113,7 @@ public void TestVsTest_VsTest_Settings() { using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath!); - Assert.True(DotnetCli($"publish \"{clonedTemplateProject.ProjectRootPath}\"", out string standardOutput, out string standardError), standardOutput); + Assert.True(DotnetCli($"publish -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\"", out string standardOutput, out string standardError), standardOutput); string publishedTestFile = clonedTemplateProject.GetFiles("*" + ClonedTemplateProject.AssemblyName + ".dll").Single(f => f.Contains("publish")); Assert.NotNull(publishedTestFile); Assert.True(DotnetCli($"vstest \"{publishedTestFile}\" --collect:\"XPlat Code Coverage\" --ResultsDirectory:\"{clonedTemplateProject.ProjectRootPath}\" /settings:\"{runSettingsPath}\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out standardOutput, out standardError), standardOutput); diff --git a/test/coverlet.integration.tests/DeterministicBuild.cs b/test/coverlet.integration.tests/DeterministicBuild.cs index e55cc3d3e..4c432c7ce 100644 --- a/test/coverlet.integration.tests/DeterministicBuild.cs +++ b/test/coverlet.integration.tests/DeterministicBuild.cs @@ -14,6 +14,12 @@ public class DeterministicBuild : BaseTest, IDisposable private string _testProjectPath = Path.GetFullPath("../../../../coverlet.integration.determisticbuild"); private string? _testProjectTfm; private const string PropsFileName = "DeterministicTest.props"; + private string _buildConfiguration; + + public DeterministicBuild() + { + _buildConfiguration = GetAssemblyBuildConfiguration().ToString(); + } private void CreateDeterministicTestPropsFile() { @@ -31,36 +37,38 @@ private void CreateDeterministicTestPropsFile() private protected void AssertCoverage(string standardOutput = "") { - bool coverageChecked = false; - string reportFilePath = ""; - foreach (string coverageFile in Directory.GetFiles(_testProjectPath, "coverage.json", SearchOption.AllDirectories)) + if (_buildConfiguration == "Debug") { - JsonConvert.DeserializeObject(File.ReadAllText(coverageFile)) - .Document("DeepThought.cs") - .Class("Coverlet.Integration.DeterministicBuild.DeepThought") - .Method("System.Int32 Coverlet.Integration.DeterministicBuild.DeepThought::AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything()") - .AssertLinesCovered((6, 1), (7, 1), (8, 1)); - coverageChecked = true; - reportFilePath = coverageFile; + bool coverageChecked = false; + string reportFilePath = ""; + foreach (string coverageFile in Directory.GetFiles(_testProjectPath, "coverage.json", SearchOption.AllDirectories)) + { + JsonConvert.DeserializeObject(File.ReadAllText(coverageFile)) + .Document("DeepThought.cs") + .Class("Coverlet.Integration.DeterministicBuild.DeepThought") + .Method("System.Int32 Coverlet.Integration.DeterministicBuild.DeepThought::AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything()") + .AssertLinesCovered((6, 1), (7, 1), (8, 1)); + coverageChecked = true; + reportFilePath = coverageFile; + } + Assert.True(coverageChecked, $"Coverage check fail\n{standardOutput}"); + File.Delete(reportFilePath); + Assert.False(File.Exists(reportFilePath)); } - Assert.True(coverageChecked, $"Coverage check fail\n{standardOutput}"); - File.Delete(reportFilePath); - Assert.False(File.Exists(reportFilePath)); } [Fact] public void Msbuild() { - string buildConfiguration = GetAssemblyBuildConfiguration().ToString(); CreateDeterministicTestPropsFile(); - DotnetCli($"build -c {buildConfiguration} /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); + DotnetCli($"build -c {_buildConfiguration} /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); Assert.Contains("Build succeeded.", standardOutput); - string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "Bin", buildConfiguration, _testProjectTfm!, "CoverletSourceRootsMapping"); + string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "Bin", _buildConfiguration, _testProjectTfm!, "CoverletSourceRootsMapping"); Assert.True(File.Exists(sourceRootMappingFilePath), sourceRootMappingFilePath); Assert.True(!string.IsNullOrEmpty(File.ReadAllText(sourceRootMappingFilePath)), "Empty CoverletSourceRootsMapping file"); Assert.Equal(2, File.ReadAllLines(sourceRootMappingFilePath).Length); - DotnetCli($"test -c {buildConfiguration} --no-build /p:CollectCoverage=true /p:Include=\"[coverletsample.integration.determisticbuild]*DeepThought\" /p:IncludeTestAssembly=true", out standardOutput, out standardError, _testProjectPath); + DotnetCli($"test -c {_buildConfiguration} --no-build /p:CollectCoverage=true /p:Include=\"[coverletsample.integration.determisticbuild]*DeepThought\" /p:IncludeTestAssembly=true", out standardOutput, out standardError, _testProjectPath); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsample.integration.determisticbuild | 100% | 100% | 100% |", standardOutput); Assert.True(File.Exists(Path.Combine(_testProjectPath, "coverage.json"))); diff --git a/test/coverlet.integration.tests/Msbuild.cs b/test/coverlet.integration.tests/Msbuild.cs index e6c9be57e..983effc7a 100644 --- a/test/coverlet.integration.tests/Msbuild.cs +++ b/test/coverlet.integration.tests/Msbuild.cs @@ -6,6 +6,13 @@ namespace Coverlet.Integration.Tests { public class Msbuild : BaseTest { + private string _buildConfiguration; + + public Msbuild() + { + _buildConfiguration = GetAssemblyBuildConfiguration().ToString(); + } + private ClonedTemplateProject PrepareTemplateProject() { ClonedTemplateProject clonedTemplateProject = CloneTemplateProject(); @@ -18,7 +25,7 @@ private ClonedTemplateProject PrepareTemplateProject() public void TestMsbuild() { using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); - Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\", out string standardOutput, out string standardError), standardOutput); + Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\", out string standardOutput, out string standardError), standardOutput); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput); Assert.True(File.Exists(Path.Combine(clonedTemplateProject.ProjectRootPath, "coverage.json"))); @@ -29,7 +36,7 @@ public void TestMsbuild() public void TestMsbuild_NoCoverletOutput() { using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); - Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true", out string standardOutput, out string standardError), standardOutput); + Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true", out string standardOutput, out string standardError), standardOutput); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput); Assert.True(File.Exists(Path.Combine(clonedTemplateProject.ProjectRootPath, "coverage.json"))); @@ -40,7 +47,7 @@ public void TestMsbuild_NoCoverletOutput() public void TestMsbuild_CoverletOutput_Folder_FileNameWithoutExtension() { using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); - Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file", out string standardOutput, out string standardError), standardOutput); + Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file", out string standardOutput, out string standardError), standardOutput); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput); Assert.True(File.Exists(Path.Combine(clonedTemplateProject.ProjectRootPath, "file.json"))); @@ -51,7 +58,7 @@ public void TestMsbuild_CoverletOutput_Folder_FileNameWithoutExtension() public void TestMsbuild_CoverletOutput_Folder_FileNameExtension() { using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); - Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext", out string standardOutput, out string standardError), standardOutput); + Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext", out string standardOutput, out string standardError), standardOutput); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput); Assert.True(File.Exists(Path.Combine(clonedTemplateProject.ProjectRootPath, "file.ext"))); @@ -64,7 +71,7 @@ public void TestMsbuild_CoverletOutput_Folder_FileNameExtension_SpecifyFramework using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); Assert.False(clonedTemplateProject.IsMultipleTargetFramework()); string framework = clonedTemplateProject.GetTargetFrameworks().Single(); - Assert.True(DotnetCli($"test -f {framework} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext", out string standardOutput, out string standardError), standardOutput); + Assert.True(DotnetCli($"test -c {_buildConfiguration} -f {framework} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext", out string standardOutput, out string standardError), standardOutput); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput); Assert.True(File.Exists(Path.Combine(clonedTemplateProject.ProjectRootPath, "file.ext"))); @@ -75,7 +82,7 @@ public void TestMsbuild_CoverletOutput_Folder_FileNameExtension_SpecifyFramework public void TestMsbuild_CoverletOutput_Folder_FileNameWithDoubleExtension() { using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); - Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext1.ext2", out string standardOutput, out string standardError), standardOutput); + Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext1.ext2", out string standardOutput, out string standardError), standardOutput); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput); Assert.True(File.Exists(Path.Combine(clonedTemplateProject.ProjectRootPath, "file.ext1.ext2"))); @@ -88,7 +95,7 @@ public void Test_MultipleTargetFrameworkReport_NoCoverletOutput() using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); string[] targetFrameworks = new string[] { "netcoreapp2.2", "netcoreapp2.1" }; UpdateProjectTargetFramework(clonedTemplateProject, targetFrameworks); - Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); + Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput); @@ -106,7 +113,7 @@ public void Test_MultipleTargetFrameworkReport_CoverletOutput_Folder() using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); string[] targetFrameworks = new string[] { "netcoreapp2.2", "netcoreapp2.1" }; UpdateProjectTargetFramework(clonedTemplateProject, targetFrameworks); - Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); + Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput); @@ -125,7 +132,7 @@ public void Test_MultipleTargetFrameworkReport_CoverletOutput_Folder_FileNameWit using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); string[] targetFrameworks = new string[] { "netcoreapp2.2", "netcoreapp2.1" }; UpdateProjectTargetFramework(clonedTemplateProject, targetFrameworks); - Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); + Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput); @@ -147,7 +154,7 @@ public void Test_MultipleTargetFrameworkReport_CoverletOutput_Folder_FileNameWit string[] frameworks = clonedTemplateProject.GetTargetFrameworks(); Assert.Equal(2, frameworks.Length); string framework = frameworks.FirstOrDefault(); - Assert.True(DotnetCli($"test -f {framework} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); + Assert.True(DotnetCli($"test -c {_buildConfiguration} -f {framework} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput); @@ -172,7 +179,7 @@ public void Test_MultipleTargetFrameworkReport_CoverletOutput_Folder_FileNameWit using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); string[] targetFrameworks = new string[] { "netcoreapp2.2", "netcoreapp2.1" }; UpdateProjectTargetFramework(clonedTemplateProject, targetFrameworks); - Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); + Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput); @@ -190,7 +197,7 @@ public void Test_MultipleTargetFrameworkReport_CoverletOutput_Folder_FileNameWit using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject(); string[] targetFrameworks = new string[] { "netcoreapp2.2", "netcoreapp2.1" }; UpdateProjectTargetFramework(clonedTemplateProject, targetFrameworks); - Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext1.ext2", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); + Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext1.ext2", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput); From 4f997ef4028fc840b3e54b1248c24ef7b5b07cc0 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 13 Apr 2020 11:46:41 +0200 Subject: [PATCH 09/11] ci test --- test/coverlet.integration.tests/DeterministicBuild.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/coverlet.integration.tests/DeterministicBuild.cs b/test/coverlet.integration.tests/DeterministicBuild.cs index 4c432c7ce..f6219c41d 100644 --- a/test/coverlet.integration.tests/DeterministicBuild.cs +++ b/test/coverlet.integration.tests/DeterministicBuild.cs @@ -3,6 +3,7 @@ using System; using System.IO; using System.Linq; +using System.Text; using System.Xml.Linq; using Xunit; @@ -64,7 +65,12 @@ public void Msbuild() DotnetCli($"build -c {_buildConfiguration} /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); Assert.Contains("Build succeeded.", standardOutput); string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "Bin", _buildConfiguration, _testProjectTfm!, "CoverletSourceRootsMapping"); - Assert.True(File.Exists(sourceRootMappingFilePath), sourceRootMappingFilePath); + StringBuilder buidler = new StringBuilder(); + foreach (var item in Directory.GetFiles(Path.Combine(_testProjectPath, "Bin", _buildConfiguration, _testProjectTfm!))) + { + buidler.AppendLine(item); + } + Assert.True(File.Exists(sourceRootMappingFilePath), buidler.ToString()); Assert.True(!string.IsNullOrEmpty(File.ReadAllText(sourceRootMappingFilePath)), "Empty CoverletSourceRootsMapping file"); Assert.Equal(2, File.ReadAllLines(sourceRootMappingFilePath).Length); From 2c720a1592b8263876251f0469ab5a2be97a8fc9 Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 13 Apr 2020 12:01:56 +0200 Subject: [PATCH 10/11] ci test --- test/coverlet.integration.tests/DeterministicBuild.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/coverlet.integration.tests/DeterministicBuild.cs b/test/coverlet.integration.tests/DeterministicBuild.cs index f6219c41d..4909a51f8 100644 --- a/test/coverlet.integration.tests/DeterministicBuild.cs +++ b/test/coverlet.integration.tests/DeterministicBuild.cs @@ -64,9 +64,9 @@ public void Msbuild() CreateDeterministicTestPropsFile(); DotnetCli($"build -c {_buildConfiguration} /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); Assert.Contains("Build succeeded.", standardOutput); - string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "Bin", _buildConfiguration, _testProjectTfm!, "CoverletSourceRootsMapping"); + string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "bin", _buildConfiguration, _testProjectTfm!, "CoverletSourceRootsMapping"); StringBuilder buidler = new StringBuilder(); - foreach (var item in Directory.GetFiles(Path.Combine(_testProjectPath, "Bin", _buildConfiguration, _testProjectTfm!))) + foreach (var item in Directory.GetFiles(Path.Combine(_testProjectPath, "bin", _buildConfiguration, _testProjectTfm!))) { buidler.AppendLine(item); } From 123205a646c1f479669599571a898c7eb317311b Mon Sep 17 00:00:00 2001 From: Marco Rossignoli Date: Mon, 13 Apr 2020 12:16:02 +0200 Subject: [PATCH 11/11] fix tests --- .../DeterministicBuild.cs | 72 +++++++++---------- 1 file changed, 33 insertions(+), 39 deletions(-) diff --git a/test/coverlet.integration.tests/DeterministicBuild.cs b/test/coverlet.integration.tests/DeterministicBuild.cs index 4909a51f8..26880becd 100644 --- a/test/coverlet.integration.tests/DeterministicBuild.cs +++ b/test/coverlet.integration.tests/DeterministicBuild.cs @@ -1,21 +1,20 @@ -using Coverlet.Core; -using Newtonsoft.Json; -using System; +using System; using System.IO; using System.Linq; -using System.Text; using System.Xml.Linq; +using Coverlet.Core; +using Newtonsoft.Json; using Xunit; namespace Coverlet.Integration.Tests { public class DeterministicBuild : BaseTest, IDisposable { - private string _testProjectPath = Path.GetFullPath("../../../../coverlet.integration.determisticbuild"); + private readonly string _testProjectPath = Path.GetFullPath("../../../../coverlet.integration.determisticbuild"); private string? _testProjectTfm; private const string PropsFileName = "DeterministicTest.props"; - private string _buildConfiguration; + private readonly string _buildConfiguration; public DeterministicBuild() { @@ -62,19 +61,14 @@ private protected void AssertCoverage(string standardOutput = "") public void Msbuild() { CreateDeterministicTestPropsFile(); - DotnetCli($"build -c {_buildConfiguration} /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); + DotnetCli($"build -c {_buildConfiguration} /p:DeterministicSourcePaths=true", out string standardOutput, out string _, _testProjectPath); Assert.Contains("Build succeeded.", standardOutput); string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "bin", _buildConfiguration, _testProjectTfm!, "CoverletSourceRootsMapping"); - StringBuilder buidler = new StringBuilder(); - foreach (var item in Directory.GetFiles(Path.Combine(_testProjectPath, "bin", _buildConfiguration, _testProjectTfm!))) - { - buidler.AppendLine(item); - } - Assert.True(File.Exists(sourceRootMappingFilePath), buidler.ToString()); + Assert.True(File.Exists(sourceRootMappingFilePath), sourceRootMappingFilePath); Assert.True(!string.IsNullOrEmpty(File.ReadAllText(sourceRootMappingFilePath)), "Empty CoverletSourceRootsMapping file"); Assert.Equal(2, File.ReadAllLines(sourceRootMappingFilePath).Length); - DotnetCli($"test -c {_buildConfiguration} --no-build /p:CollectCoverage=true /p:Include=\"[coverletsample.integration.determisticbuild]*DeepThought\" /p:IncludeTestAssembly=true", out standardOutput, out standardError, _testProjectPath); + DotnetCli($"test -c {_buildConfiguration} --no-build /p:CollectCoverage=true /p:Include=\"[coverletsample.integration.determisticbuild]*DeepThought\" /p:IncludeTestAssembly=true", out standardOutput, out _, _testProjectPath); Assert.Contains("Test Run Successful.", standardOutput); Assert.Contains("| coverletsample.integration.determisticbuild | 100% | 100% | 100% |", standardOutput); Assert.True(File.Exists(Path.Combine(_testProjectPath, "coverage.json"))); @@ -83,36 +77,36 @@ public void Msbuild() // Process exits hang on clean seem that process doesn't close, maybe some mbuild node reuse? btw manually tested // DotnetCli("clean", out standardOutput, out standardError, _fixture.TestProjectPath); // Assert.False(File.Exists(sourceRootMappingFilePath)); - // RunCommand("git", "clean -fdx", out _, out _, _testProjectPath); + RunCommand("git", "clean -fdx", out _, out _, _testProjectPath); } - //[Fact] - //public void Collectors() - //{ - // CreateDeterministicTestPropsFile(); - // DotnetCli("build /p:DeterministicSourcePaths=true", out string standardOutput, out string standardError, _testProjectPath); - // Assert.Contains("Build succeeded.", standardOutput); - // string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "Bin", GetAssemblyBuildConfiguration().ToString(), _testProjectTfm!, "CoverletSourceRootsMapping"); - // Assert.True(File.Exists(sourceRootMappingFilePath)); - // Assert.NotEmpty(File.ReadAllText(sourceRootMappingFilePath)); - // Assert.Equal(2, File.ReadAllLines(sourceRootMappingFilePath).Length); + [Fact] + public void Collectors() + { + CreateDeterministicTestPropsFile(); + DotnetCli($"build -c {_buildConfiguration} /p:DeterministicSourcePaths=true", out string standardOutput, out string _, _testProjectPath); + Assert.Contains("Build succeeded.", standardOutput); + string sourceRootMappingFilePath = Path.Combine(_testProjectPath, "bin", GetAssemblyBuildConfiguration().ToString(), _testProjectTfm!, "CoverletSourceRootsMapping"); + Assert.True(File.Exists(sourceRootMappingFilePath), sourceRootMappingFilePath); + Assert.NotEmpty(File.ReadAllText(sourceRootMappingFilePath)); + Assert.Equal(2, File.ReadAllLines(sourceRootMappingFilePath).Length); - // string runSettingsPath = AddCollectorRunsettingsFile(_testProjectPath, "[coverletsample.integration.determisticbuild]*DeepThought"); - // Assert.True(DotnetCli($"test --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out standardError), standardOutput); - // Assert.Contains("Test Run Successful.", standardOutput); - // AssertCoverage(standardOutput); + string runSettingsPath = AddCollectorRunsettingsFile(_testProjectPath, "[coverletsample.integration.determisticbuild]*DeepThought"); + Assert.True(DotnetCli($"test -c {_buildConfiguration} --no-build \"{_testProjectPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(_testProjectPath, "log.txt")}", out standardOutput, out _), standardOutput); + Assert.Contains("Test Run Successful.", standardOutput); + AssertCoverage(standardOutput); - // // Check out/in process collectors injection - // string dataCollectorLogContent = File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.datacollector.*.txt").Single()); - // Assert.Contains("[coverlet]Initializing CoverletCoverageDataCollector with configuration:", dataCollectorLogContent); - // Assert.Contains("[coverlet]Initialize CoverletInProcDataCollector", File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.host.*.txt").Single())); - // Assert.Contains("[coverlet]Mapping resolved", dataCollectorLogContent); + // Check out/in process collectors injection + string dataCollectorLogContent = File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.datacollector.*.txt").Single()); + Assert.Contains("[coverlet]Initializing CoverletCoverageDataCollector with configuration:", dataCollectorLogContent); + Assert.Contains("[coverlet]Initialize CoverletInProcDataCollector", File.ReadAllText(Directory.GetFiles(_testProjectPath, "log.host.*.txt").Single())); + Assert.Contains("[coverlet]Mapping resolved", dataCollectorLogContent); - // // Process exits hang on clean seem that process doesn't close, maybe some mbuild node reuse? btw manually tested - // // DotnetCli("clean", out standardOutput, out standardError, _fixture.TestProjectPath); - // // Assert.False(File.Exists(sourceRootMappingFilePath)); - // RunCommand("git", "clean -fdx", out _, out _, _testProjectPath); - //} + // Process exits hang on clean seem that process doesn't close, maybe some mbuild node reuse? btw manually tested + // DotnetCli("clean", out standardOutput, out standardError, _fixture.TestProjectPath); + // Assert.False(File.Exists(sourceRootMappingFilePath)); + RunCommand("git", "clean -fdx", out _, out _, _testProjectPath); + } public void Dispose() {