[release/2.1] Detect package cache blocking source-built package uptake

dir.props

@@ -69,6 +69,7 @@
     <PackageReportDataFile>$(PackageReportDir)prebuilt-usage.json</PackageReportDataFile>
     <ProdConManifestFile>$(PackageReportDir)prodcon-build.xml</ProdConManifestFile>
     <PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>
+    <ConflictingPackageReportDir>$(BaseOutputPath)conflict-report/</ConflictingPackageReportDir>
   </PropertyGroup>
 
   <!-- Import Build tools common props file where repo-independent properties are found -->

repos/dir.targets

@@ -7,6 +7,7 @@
   </ItemGroup>
 
   <UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="AddSourceToNuGetConfig" />
+  <UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="GetSourceBuiltNupkgCacheConflicts" />
   <UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="ReadNuGetPackageInfos" />
   <UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="RemoveInternetSourcesFromNuGetConfig" />
   <UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="UpdateJson" />
@@ -155,7 +156,8 @@
   <Target Name="CreateBuildOutputProps"
           BeforeTargets="Build">
     <ItemGroup>
-      <_PreviouslySourceBuiltPackages Include="$(SourceBuiltPackagesPath)*.nupkg" />
+      <_PreviouslySourceBuiltPackages Include="$(SourceBuiltPackagesPath)*.nupkg"
+                                      Exclude="$(SourceBuiltPackagesPath)*.symbols.nupkg" />
     </ItemGroup>
     <WriteBuildOutputProps NuGetPackages="@(_PreviouslySourceBuiltPackages)"
                            ExtraPackageInfo="@(ExtraPackageVersionPropsPackageInfo)"
@@ -170,6 +172,34 @@
 
     <Message Importance="High" Text="$(RepositoryName) is using versions $(PackageVersionPropsPath)" />
     <Message Importance="High" Text="%(VersionProperties.Identity)" />
+
+    <ReadNuGetPackageInfos PackagePaths="@(_PreviouslySourceBuiltPackages)">
+      <Output TaskParameter="PackageInfoItems" ItemName="_PreviouslySourceBuiltPackageInfos" />
+    </ReadNuGetPackageInfos>
+
+    <GetSourceBuiltNupkgCacheConflicts SourceBuiltPackageInfos="@(_PreviouslySourceBuiltPackageInfos)"
+                                       PackageCacheDir="$(PackagesDir)">
+      <Output TaskParameter="ConflictingPackageInfos" ItemName="ConflictingPackageInfos" />
+    </GetSourceBuiltNupkgCacheConflicts>
+  </Target>
+
+  <Target Name="CheckSourceBuiltNupkgConflictUsages"
+          DependsOnTargets="GetAllProjectDirectories"
+          AfterTargets="CreateBuildOutputProps"
+          Condition="'@(ConflictingPackageInfos)' != ''">
+    <PropertyGroup>
+      <_ReportDir>$(ConflictingPackageReportDir)before-$(RepositoryName)/</_ReportDir>
+      <_ReportDataFile>$(_ReportDir)usage.json</_ReportDataFile>
+    </PropertyGroup>
+
+    <WritePackageUsageData NuGetPackageInfos="@(ConflictingPackageInfos)"
+                           ProjectDirectories="@(ProjectDirectories)"
+                           DataFile="$(_ReportDataFile)" />
+
+    <WriteUsageReports DataFile="$(_ReportDataFile)"
+                       OutputDirectory="$(_ReportDir)" />
+
+    <Warning Text="Detected package id/version(s) in the cache that were source-built, but contents don't match. They were probably downloaded. See $(_ReportDir) for usage details. @(ConflictingPackageInfos->'%(PackageId) %(PackageVersion)', ', ')" />
   </Target>
 
   <Target Name="CreateRestoreSourceProps"
@@ -318,18 +348,8 @@
     </ItemGroup>
   </Target>
 
-  <Target Name="WritePrebuiltUsageData">
-    <ItemGroup>
-      <AllRepoProjects Include="$(ProjectDir)repos\*.proj" />
-    </ItemGroup>
-
-    <Message Importance="High" Text="Finding project directories..." />
-
-    <MSBuild Projects="@(AllRepoProjects)"
-             Targets="GetProjectDirectory">
-      <Output TaskParameter="TargetOutputs" ItemName="ProjectDirectories" />
-    </MSBuild>
-
+  <Target Name="WritePrebuiltUsageData"
+          DependsOnTargets="GetAllProjectDirectories">
     <ItemGroup>
       <PrebuiltPackages Include="$(PrebuiltPackagesPath)*.nupkg" />
       <PrebuiltPackages Include="$(TarballPrebuiltPackagesPath)*.nupkg" Condition="'$(TarballPrebuiltPackagesPath)' != ''"/>
@@ -354,6 +374,19 @@
                            DataFile="$(PackageReportDataFile)" />
   </Target>
 
+  <Target Name="GetAllProjectDirectories">
+    <ItemGroup>
+      <AllRepoProjects Include="$(ProjectDir)repos\*.proj" />
+    </ItemGroup>
+
+    <Message Importance="High" Text="Finding project directories..." />
+
+    <MSBuild Projects="@(AllRepoProjects)"
+             Targets="GetProjectDirectory">
+      <Output TaskParameter="TargetOutputs" ItemName="ProjectDirectories" />
+    </MSBuild>
+  </Target>
+
   <Target Name="ReportPrebuiltUsage"
           DependsOnTargets="GetPreviousReleasePrebuiltPackageInfos">
     <ItemGroup>

tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks/GetSourceBuiltNupkgCacheConflicts.cs

@@ -0,0 +1,103 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using System;
+using System.IO;
+using System.Linq;
+
+namespace Microsoft.DotNet.SourceBuild.Tasks
+{
+    /// <summary>
+    /// For each source-built nupkg info given, ensure that if the package cache contains a package
+    /// with the same id and version, the cached nupkg is the same as the source-built one.
+    ///
+    /// If the package cache contains a package with the same package id and version as a
+    /// source-built one, nuget restore short-circuits and doesn't look for the source-built one.
+    /// This usually results in prebuilt packages being used, which can either break the build or
+    /// end up in the outputs.
+    /// </summary>
+    public class GetSourceBuiltNupkgCacheConflicts : Task
+    {
+        /// <summary>
+        /// Items containing package id and version of each source-built package.
+        /// ReadNuGetPackageInfos is recommended to generate these.
+        ///
+        /// %(Identity): Path to the original nupkg.
+        /// %(PackageId): Identity of the package.
+        /// %(PackageVersion): Version of the package.
+        /// </summary>
+        [Required]
+        public ITaskItem[] SourceBuiltPackageInfos { get; set; }
+
+        /// <summary>
+        /// Package cache dir containing nupkgs to compare. Path is expected to be like:
+        /// 
+        /// {PackageCacheDir}/{lowercase id}/{version}/{lowercase id}.{version}.nupkg
+        /// </summary>
+        [Required]
+        public string PackageCacheDir { get; set; }
+
+        [Output]
+        public ITaskItem[] ConflictingPackageInfos { get; set; }
+
+        public override bool Execute()
+        {
+            DateTime startTime = DateTime.Now;
+
+            ConflictingPackageInfos = SourceBuiltPackageInfos
+                .Where(item =>
+                {
+                    string sourceBuiltPath = item.ItemSpec;
+                    string id = item.GetMetadata("PackageId");
+                    string version = item.GetMetadata("PackageVersion");
+
+                    string packageCachePath = Path.Combine(
+                        PackageCacheDir,
+                        id.ToLowerInvariant(),
+                        version,
+                        $"{id.ToLowerInvariant()}.{version}.nupkg");
+
+                    if (!File.Exists(packageCachePath))
+                    {
+                        Log.LogMessage(
+                            MessageImportance.Low,
+                            $"OK: Package not found in package cache: {id} {version}");
+                        return false;
+                    }
+
+                    Log.LogMessage(
+                        MessageImportance.Low,
+                        $"Package id/version found in package cache, verifying: {id} {version}");
+
+                    bool identical = File.ReadAllBytes(sourceBuiltPath)
+                        .SequenceEqual(File.ReadAllBytes(packageCachePath));
+
+                    if (!identical)
+                    {
+                        Log.LogMessage(
+                            MessageImportance.Low,
+                            "BAD: Source-built nupkg is not byte-for-byte identical " +
+                                $"to nupkg in cache: {id} {version}");
+                        return true;
+                    }
+
+                    Log.LogMessage(
+                        MessageImportance.Low,
+                        $"OK: Package in cache is identical to source-built: {id} {version}");
+                    return false;
+                })
+                .ToArray();
+
+            // Tell the user about this task, in case it takes a while.
+            Log.LogMessage(
+                MessageImportance.High,
+                "Checked cache for conflicts with source-built nupkgs. " +
+                    $"Took {DateTime.Now - startTime}");
+
+            return !Log.HasLoggedErrors;
+        }
+    }
+}