Add integration tests for deterministic build support

.gitignore

@@ -301,3 +301,6 @@ __pycache__/
 
 # DeterministicSourcePaths workaround generated file DeterministicBuild.targets
 .AssemblyAttributes
+DeterministicTest.props
+test/coverlet.integration.determisticbuild/*.txt
+test/coverlet.integration.determisticbuild/runsettings

eng/build.yml

@@ -15,7 +15,7 @@ steps:
 - script: dotnet build -c $(BuildConfiguration) --no-restore
   displayName: Build
 
-- script: dotnet pack -c $(BuildConfiguration)
+- script: dotnet pack -c $(BuildConfiguration) --no-restore
   displayName: Pack
 
 - task: DotNetCoreCLI@2

test/coverlet.core.tests/Coverage/InstrumenterHelper.Assertions.cs

@@ -275,19 +275,13 @@ public static Document AssertNonInstrumentedLines(this Document document, BuildC
 
         private static BuildConfiguration GetAssemblyBuildConfiguration()
         {
-            var configurationAttribute = Assembly.GetExecutingAssembly().GetCustomAttribute<AssemblyConfigurationAttribute>();
-            if (configurationAttribute.Configuration.Equals("Debug", StringComparison.InvariantCultureIgnoreCase))
-            {
-                return Tests.BuildConfiguration.Debug;
-            }
-            else if (configurationAttribute.Configuration.Equals("Release", StringComparison.InvariantCultureIgnoreCase))
-            {
-                return Tests.BuildConfiguration.Release;
-            }
-            else
-            {
-                throw new NotSupportedException($"Build configuration '{configurationAttribute.Configuration}' not supported");
-            }
+#if DEBUG
+            return BuildConfiguration.Debug;
+#endif
+#if RELEASE
+            return BuildConfiguration.Release;
+#endif
+            throw new NotSupportedException($"Build configuration not supported");
         }
     }
 }

test/coverlet.integration.determisticbuild/DeepThought.cs

@@ -0,0 +1,10 @@
+namespace Coverlet.Integration.DeterministicBuild
+{
+    public class DeepThought
+    {
+        public int AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything()
+        {
+            return 42;
+        }
+    }
+}

test/coverlet.integration.determisticbuild/TemplateTest.cs

@@ -0,0 +1,14 @@
+using Xunit;
+
+namespace Coverlet.Integration.DeterministicBuild
+{
+    public class TemplateTest
+    {
+        [Fact]
+        public void Answer()
+        {
+            DeepThought dt = new DeepThought();
+            Assert.Equal(42, dt.AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything());
+        }
+    }
+}

test/coverlet.integration.determisticbuild/coverlet.integration.determisticbuild.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <!-- Import coverlet version-->
+  <Import Project="$(MSBuildThisFileDirectory)\DeterministicTest.props" />
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <IsPackable>false</IsPackable>
+    <AssemblyName>coverletsample.integration.determisticbuild</AssemblyName>
+    <RestoreSources>
+      https://api.nuget.org/v3/index.json;
+      $(RepoRoot)bin\$(Configuration)\Packages
+    </RestoreSources>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="coverlet.msbuild" Version="$(coverletMsbuilVersion)">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="coverlet.collector" Version="$(coverletCollectorsVersion)">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="xunit" />
+    <PackageReference Include="xunit.runner.visualstudio" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+  </ItemGroup>
+</Project>

test/coverlet.integration.template/coverlet.integration.template.csproj

@@ -4,6 +4,7 @@
     <TargetFramework>netcoreapp3.1</TargetFramework>
     <IsPackable>false</IsPackable>
     <AssemblyName>coverletsamplelib.integration.template</AssemblyName>
+    <IsTestProject>false</IsTestProject>
   </PropertyGroup>
 
   <ItemGroup>

test/coverlet.integration.tests/BaseTest.cs

@@ -15,7 +15,7 @@
 namespace Coverlet.Integration.Tests
 {
     [Flags]
-    enum BuildConfiguration
+    public enum BuildConfiguration
     {
         Debug = 1,
         Release = 2
@@ -24,27 +24,16 @@ enum BuildConfiguration
     public abstract class BaseTest
     {
         private static int _folderSuffix = 0;
-        private BuildConfiguration GetAssemblyBuildConfiguration()
-        {
-            var configurationAttribute = Assembly.GetExecutingAssembly().GetCustomAttribute<AssemblyConfigurationAttribute>();
 
-            if (configurationAttribute is null)
-            {
-                throw new ArgumentNullException("AssemblyConfigurationAttribute not found");
-            }
-
-            if (configurationAttribute.Configuration.Equals("Debug", StringComparison.InvariantCultureIgnoreCase))
-            {
-                return BuildConfiguration.Debug;
-            }
-            else if (configurationAttribute.Configuration.Equals("Release", StringComparison.InvariantCultureIgnoreCase))
-            {
-                return BuildConfiguration.Release;
-            }
-            else
-            {
-                throw new NotSupportedException($"Build configuration '{configurationAttribute.Configuration}' not supported");
-            }
+        protected BuildConfiguration GetAssemblyBuildConfiguration()
+        {
+#if DEBUG
+            return BuildConfiguration.Debug;
+#endif
+#if RELEASE
+            return BuildConfiguration.Release;
+#endif
+            throw new NotSupportedException($"Build configuration not supported");
         }
 
         private protected string GetPackageVersion(string filter)
@@ -63,7 +52,7 @@ private protected string GetPackageVersion(string filter)
 
         private protected ClonedTemplateProject CloneTemplateProject(bool cleanupOnDispose = true, string testSDKVersion = "16.5.0")
         {
-            DirectoryInfo finalRoot = Directory.CreateDirectory($"template{Interlocked.Increment(ref _folderSuffix)}");
+            DirectoryInfo finalRoot = Directory.CreateDirectory($"{Guid.NewGuid().ToString("N").Substring(0, 6)}{Interlocked.Increment(ref _folderSuffix)}");
             foreach (string file in (Directory.GetFiles($"../../../../coverlet.integration.template", "*.cs")
                     .Union(Directory.GetFiles($"../../../../coverlet.integration.template", "*.csproj")
                     .Union(Directory.GetFiles($"../../../../coverlet.integration.template", "nuget.config")))))
@@ -84,6 +73,8 @@ private protected ClonedTemplateProject CloneTemplateProject(bool cleanupOnDispo
 
             AddMicrosoftNETTestSdkRef(finalRoot.FullName, testSDKVersion);
 
+            SetIsTestProjectTrue(finalRoot.FullName);
+
             return new ClonedTemplateProject(finalRoot.FullName, cleanupOnDispose);
         }
 
@@ -131,6 +122,26 @@ private protected void UpdateNugeConfigtWithLocalPackageFolder(string projectPat
             xml.Save(nugetFile);
         }
 
+        private void SetIsTestProjectTrue(string projectPath)
+        {
+            string csproj = Path.Combine(projectPath, "coverlet.integration.template.csproj");
+            if (!File.Exists(csproj))
+            {
+                throw new FileNotFoundException("coverlet.integration.template.csproj not found", "coverlet.integration.template.csproj");
+            }
+            XDocument xml;
+            using (var csprojStream = File.OpenRead(csproj))
+            {
+                xml = XDocument.Load(csprojStream);
+            }
+
+            xml.Element("Project")
+               .Element("PropertyGroup")
+               .Element("IsTestProject").Value = "true";
+
+            xml.Save(csproj);
+        }
+
         private protected void AddMicrosoftNETTestSdkRef(string projectPath, string version)
         {
             string csproj = Path.Combine(projectPath, "coverlet.integration.template.csproj");
@@ -189,17 +200,17 @@ private protected void AddCoverletCollectosRef(string projectPath)
             xml.Save(csproj);
         }
 
-        private protected string AddCollectorRunsettingsFile(string projectPath)
+        private protected string AddCollectorRunsettingsFile(string projectPath, string includeFilter = "[coverletsamplelib.integration.template]*DeepThought")
         {
             string runSettings =
-@"<?xml version=""1.0"" encoding=""utf-8"" ?>
+$@"<?xml version=""1.0"" encoding=""utf-8"" ?>
   <RunSettings>
     <DataCollectionRunSettings>  
       <DataCollectors>  
         <DataCollector friendlyName=""XPlat code coverage"" >
            <Configuration>
             <Format>json,cobertura</Format>
-            <Include>[coverletsamplelib.integration.template]*DeepThought</Include>
+            <Include>{includeFilter}</Include>
             <!-- We need to include test assembly because test and code to cover are in same template project -->
             <IncludeTestAssembly>true</IncludeTestAssembly>
         </Configuration>
@@ -215,18 +226,21 @@ private protected string AddCollectorRunsettingsFile(string projectPath)
 
         private protected void AssertCoverage(ClonedTemplateProject clonedTemplateProject, string filter = "coverage.json", string standardOutput = "")
         {
-            bool coverageChecked = false;
-            foreach (string coverageFile in clonedTemplateProject.GetFiles(filter))
+            if (GetAssemblyBuildConfiguration() == BuildConfiguration.Debug)
             {
-                JsonConvert.DeserializeObject<Modules>(File.ReadAllText(coverageFile))
-                .Document("DeepThought.cs")
-                .Class("Coverlet.Integration.Template.DeepThought")
-                .Method("System.Int32 Coverlet.Integration.Template.DeepThought::AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything()")
-                .AssertLinesCovered((6, 1), (7, 1), (8, 1));
-                coverageChecked = true;
-            }
+                bool coverageChecked = false;
+                foreach (string coverageFile in clonedTemplateProject.GetFiles(filter))
+                {
+                    JsonConvert.DeserializeObject<Modules>(File.ReadAllText(coverageFile))
+                    .Document("DeepThought.cs")
+                    .Class("Coverlet.Integration.Template.DeepThought")
+                    .Method("System.Int32 Coverlet.Integration.Template.DeepThought::AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything()")
+                    .AssertLinesCovered((6, 1), (7, 1), (8, 1));
+                    coverageChecked = true;
+                }
 
-            Assert.True(coverageChecked, $"Coverage check fail\n{standardOutput}");
+                Assert.True(coverageChecked, $"Coverage check fail\n{standardOutput}");
+            }
         }
 
         private protected void UpdateProjectTargetFramework(ClonedTemplateProject project, params string[] targetFrameworks)

test/coverlet.integration.tests/Collectors.cs

@@ -42,6 +42,13 @@ public TestSDK_Preview()
 
     public abstract class Collectors : BaseTest
     {
+        private string _buildConfiguration;
+
+        public Collectors()
+        {
+            _buildConfiguration = GetAssemblyBuildConfiguration().ToString();
+        }
+
         protected string? TestSDKVersion { get; set; }
 
         private ClonedTemplateProject PrepareTemplateProject()
@@ -68,8 +75,7 @@ private protected virtual void AssertCollectorsInjection(ClonedTemplateProject c
         public void TestVsTest_Test()
         {
             using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
-            string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath!);
-            Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput);
+            Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath!), standardOutput);
             // We don't have any result to check because tests and code to instrument are in same assembly so we need to pass
             // IncludeTestAssembly=true we do it in other test
             Assert.Contains("Test Run Successful.", standardOutput);
@@ -81,7 +87,7 @@ public void TestVsTest_Test_Settings()
         {
             using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
             string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath!);
-            Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError), standardOutput);
+            Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError), standardOutput);
             Assert.Contains("Test Run Successful.", standardOutput);
             AssertCoverage(clonedTemplateProject);
             AssertCollectorsInjection(clonedTemplateProject);
@@ -92,7 +98,7 @@ public void TestVsTest_VsTest()
         {
             using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
             string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath!);
-            Assert.True(DotnetCli($"publish {clonedTemplateProject.ProjectRootPath}", out string standardOutput, out string standardError), standardOutput);
+            Assert.True(DotnetCli($"publish -c {_buildConfiguration} {clonedTemplateProject.ProjectRootPath}", out string standardOutput, out string standardError), standardOutput);
             string publishedTestFile = clonedTemplateProject.GetFiles("*" + ClonedTemplateProject.AssemblyName + ".dll").Single(f => f.Contains("publish"));
             Assert.NotNull(publishedTestFile);
             Assert.True(DotnetCli($"vstest \"{publishedTestFile}\" --collect:\"XPlat Code Coverage\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out standardOutput, out standardError), standardOutput);
@@ -107,7 +113,7 @@ public void TestVsTest_VsTest_Settings()
         {
             using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
             string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath!);
-            Assert.True(DotnetCli($"publish \"{clonedTemplateProject.ProjectRootPath}\"", out string standardOutput, out string standardError), standardOutput);
+            Assert.True(DotnetCli($"publish -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\"", out string standardOutput, out string standardError), standardOutput);
             string publishedTestFile = clonedTemplateProject.GetFiles("*" + ClonedTemplateProject.AssemblyName + ".dll").Single(f => f.Contains("publish"));
             Assert.NotNull(publishedTestFile);
             Assert.True(DotnetCli($"vstest \"{publishedTestFile}\" --collect:\"XPlat Code Coverage\" --ResultsDirectory:\"{clonedTemplateProject.ProjectRootPath}\" /settings:\"{runSettingsPath}\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out standardOutput, out standardError), standardOutput);