Code coverage support

README.md

@@ -42,6 +42,8 @@ Daily NuGet builds of the project are also available in our [MyGet](https://dotn
 
 To build ML.NET from source please visit our [developers guide](docs/project-docs/developer-guide.md).
 
+[![codecov](https://codecov.io/gh/dotnet/machinelearning/branch/master/graph/badge.svg)](https://codecov.io/gh/dotnet/machinelearning)
+
 |    | Debug | Release |
 |:---|----------------:|------------------:|
 |**Linux**|[![x64-debug](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/machinelearning/MachineLearning-CI?branchName=master&jobname=Linux&configuration=Build_Debug)](https://dnceng.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|[![x64-release](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/machinelearning/MachineLearning-CI?branchName=master&jobname=Linux&configuration=Build_Release)](https://dnceng.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|

build/Codecov.proj

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project>
+  <Import Project="..\Directory.Build.props" />
+
+  <Target Name="Codecov">
+    <PropertyGroup Condition="'$(UseCodecov)' == 'true'">
+      <_CodecovPath>$(RestorePackagesPath)codecov\$(CodecovVersion)\tools\Codecov.exe</_CodecovPath>
+
+      <_BranchName Condition="'$(_BranchName)' == ''">$(SYSTEM_PULLREQUEST_TARGETBRANCH)</_BranchName>
+      <_BranchName Condition="'$(_BranchName)' == ''">$(BUILD_SOURCEBRANCHNAME)</_BranchName>
+    </PropertyGroup>
+
+    <ItemGroup Condition="'$(UseCodecov)' == 'true'">
+      <_CoverageReports Include="$(BaseOutputPath)$(PlatformConfig)\coverage\*.xml" />
+
+      <_CodecovArgs Include="-f;@(_CoverageReports)" />
+      <_CodecovArgs Include="-r;$(BUILD_REPOSITORY_NAME)" Condition="'$(BUILD_REPOSITORY_NAME)' != ''" />
+      <_CodecovArgs Include="--pr;$(SYSTEM_PULLREQUEST_PULLREQUESTNUMBER)" Condition="'$(SYSTEM_PULLREQUEST_PULLREQUESTNUMBER)' != ''" />
+      <_CodecovArgs Include="-b;$(BUILD_BUILDNUMBER)" Condition="'$(BUILD_BUILDNUMBER)' != ''" />
+      <_CodecovArgs Include="--branch;$(_BranchName)" Condition="'$(_BranchName)' != ''" />
+      <_CodecovArgs Include="-c;$(BUILD_SOURCEVERSION)" Condition="'$(BUILD_SOURCEVERSION)' != ''" />
+      <_CodecovArgs Include="-n;$(BUILD_DEFINITIONNAME)" Condition="'$(BUILD_DEFINITIONNAME)' != ''" />
+      <_CodecovArgs Include="--flag;$(Configuration)" Condition="'$(Configuration)' != ''" />
+      <_CodecovArgs Include="-t;$(CODECOV_TOKEN)" Condition="'$(CODECOV_TOKEN)' != ''" />
+    </ItemGroup>
+
+    <Message Importance="high" Text="&quot;$(_CodecovPath)&quot; @(_CodecovArgs, ' ')" />
+    <Exec Condition="'$(UseCodecov)' == 'true'"
+          Command="&quot;$(_CodecovPath)&quot; @(_CodecovArgs, ' ')" />
+  </Target>
+
+</Project>

build/Dependencies.props

@@ -34,6 +34,8 @@
   <!-- Build/infrastructure Dependencies -->
   <PropertyGroup>
     <PublishSymbolsPackageVersion>1.0.0-beta-62824-02</PublishSymbolsPackageVersion>
+    <CodecovVersion>1.1.0</CodecovVersion>
+    <CoverletVersion>2.5.1</CoverletVersion>
   </PropertyGroup>
 
   <!-- Test-only Dependencies -->

build/ci/phase-template.yml

@@ -34,9 +34,18 @@ phases:
       displayName: Download Benchmark Data
     - ${{ if eq(parameters.name, 'MacOS') }}:
       - script: brew update && brew install libomp mono-libgdiplus gettext && brew link gettext --force
-        displayName: Install runtime dependencies
+        displayName: Install runtime dependencies     
+    - script: $(_buildScript) -$(_configuration) -runtests -coverage=true
+      displayName: Run Tests with code coverage.
+      condition: and(eq(variables._phaseName, 'Windows_x64'), eq(variables._configuration, 'Debug'))
     - script: $(_buildScript) -$(_configuration) -runtests
-      displayName: Run Tests
+      displayName: Run Tests without code coverage.
+      condition: not(and(eq(variables._phaseName, 'Windows_x64'), eq(variables._configuration, 'Debug')))
+    - script: dotnet msbuild build/Codecov.proj /t:Codecov /p:UseCodecov=true
+      env:
+        CODECOV_TOKEN: $(CODECOV_TOKEN)
+      displayName: Upload coverage to codecov.io
+      condition: and(eq(variables._phaseName, 'Windows_x64'), eq(variables._configuration, 'Debug'))
     - task: PublishTestResults@2
       displayName: Publish Test Results
       condition: succeededOrFailed()

config.json

@@ -78,6 +78,12 @@
       "values": [],
       "defaultValue": ""
     },
+    "Coverage": {
+      "description": "Turn on code coverge.",
+      "valueType": "property",
+      "values": ["false", "true"],
+      "defaultValue": "false"
+    },
     "CleanAllProjects": {
       "description": "MsBuild target that deletes the binary output directory.",
       "valueType": "target",

test/Directory.Build.props

@@ -31,6 +31,20 @@
     <PackageReference Include="xunit" Version="2.4.0" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.0" />
     <PackageReference Include="Microsoft.DotNet.XUnitExtensions" Version="2.4.0-prerelease-63213-02" />
+    <PackageReference Include="coverlet.msbuild" Version="$(CoverletVersion)" />
+    <PackageReference Include="Codecov" Version="$(CodecovVersion)" />
   </ItemGroup>
 
+  <PropertyGroup Condition="'$(Coverage)' == 'true'">
+      <CollectCoverage>true</CollectCoverage>
+      <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies> <!-- https://github.com/tonerdo/coverlet/issues/72 -->
+      <CoverletOutputFormat>opencover</CoverletOutputFormat>
+      <CoverletOutput>$(BaseOutputPath)$(PlatformConfig)\coverage\$(MSBuildProjectName).xml</CoverletOutput>
+      <Include>[Microsoft.ML.*]*</Include>
+      <!-- Excluding for perf reasons. These classes have tests that can be run conditionally
+           but they need to be migrated. Excluding these classes should have very minimal effect on code coverage.
+      -->
+      <Exclude>[*]Microsoft.ML.*Contracts*,[*]Microsoft.ML.Internal.Utilities*,[*]Microsoft.ML.Data.VBuffer*</Exclude>
+  </PropertyGroup>
+
 </Project>