[release/2.1] Add additional ProdCon updaters pointed at recent 2.1.1 build

BuildToolsVersion.txt

@@ -1 +1 @@
-2.1.0-rc1-02804-05
+2.1.0-rc1-02911-02

dependencies.props

@@ -7,7 +7,7 @@
   <PropertyGroup>
     <ProdConCurrentRef>79731c509a9d31b921f531ee63c48daa8e0c30d5</ProdConCurrentRef>
     <CoreClrCurrentRef>79731c509a9d31b921f531ee63c48daa8e0c30d5</CoreClrCurrentRef>
-    <BuildToolsCurrentRef>54227d85bec7093210eef26b6013d1435c6fc21e</BuildToolsCurrentRef>
+    <BuildToolsCurrentRef>ba7a90f8cbc824737d0fae5ff230635a9764a1d7</BuildToolsCurrentRef>
   </PropertyGroup>
 
   <!-- Package dependency verification/auto-upgrade configuration. -->
@@ -40,11 +40,17 @@
 
     <DependencyInfo Include="ProdCon">
       <DependencyType>Orchestrated build</DependencyType>
-      <BasePath>build-info/dotnet/product/cli/release/2.1</BasePath>
+      <BasePath>build-info/dotnet/product/cli/release/2.1.1</BasePath>
       <CurrentRef>$(ProdConCurrentRef)</CurrentRef>
       <VersionsRepoOwner>dotnet</VersionsRepoOwner>
       <VersionsRepo>versions</VersionsRepo>
     </DependencyInfo>
+
+    <UpdateStep Include="ProdCon">
+      <UpdaterType>Orchestrated blob feed attribute</UpdaterType>
+      <SingleLineFile>$(MSBuildThisFileDirectory)ProdConFeed.txt</SingleLineFile>
+      <AttributeName>Url</AttributeName>
+    </UpdateStep>
   </ItemGroup>
 
   <ItemGroup Condition="'$(SkipBuildToolsUpdate)' != 'true'">
@@ -53,7 +59,7 @@
       <Path>$(MSBuildThisFileDirectory)BuildToolsVersion.txt</Path>
       <PackageId>Microsoft.DotNet.BuildTools</PackageId>
     </UpdateStep>
-    <UpdateStep Include="BuildTools">
+    <UpdateStep Include="CoreClr">
       <UpdaterType>File</UpdaterType>
       <Path>$(MSBuildThisFileDirectory)tools-local/ILAsmVersion.txt</Path>
       <PackageId>Microsoft.NETCore.ILAsm</PackageId>

dependencies.targets

@@ -1,58 +1,90 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <Target Name="CreateOrchestratedBuildSubmoduleUpdaters"
+
+  <UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="JoinItems" />
+
+  <Target Name="CreateOrchestratedBuildUpdaters"
           BeforeTargets="CreateDefaultDependencyInfos"
-          Condition="'$(SkipOrchestratedBuildSubmoduleUpdate)' != 'true'">
+          Condition="'$(SkipOrchestratedBuildUpdate)' != 'true'">
 
     <ItemGroup>
-      <OrchestratedBuildRepoProjects Include="$(ProjectDir)repos\*.proj" />
+      <RepoProjectFile Include="$(ProjectDir)repos\*.proj" />
     </ItemGroup>
 
-    <MSBuild Projects="@(OrchestratedBuildRepoProjects)"
-             Targets="GetOrchestratedManifestBuildName">
+    <!--
+      Evaluate the dependency properties of each repo project. Use MSBuildSourceProjectFile as the
+      join key to make one combined item for each project with all available metadata.
+    -->
+    <MSBuild Projects="@(RepoProjectFile)" Targets="GetOrchestratedManifestBuildName">
       <Output TaskParameter="TargetOutputs" ItemName="OrchestratedManifestBuildNames" />
     </MSBuild>
+    <ItemGroup>
+      <_RepoProjectExtractedProperties
+        Include="@(OrchestratedManifestBuildNames)"
+        BuildName="%(Identity)"
+        KeepMetadata="MSBuildSourceProjectFile" />
+    </ItemGroup>
 
-    <MSBuild Projects="@(OrchestratedBuildRepoProjects)"
-             Targets="GetProjectDirectory">
+    <MSBuild Projects="@(RepoProjectFile)" Targets="GetProjectDirectory">
       <Output TaskParameter="TargetOutputs" ItemName="ProjectDirectories" />
     </MSBuild>
+    <ItemGroup>
+      <_RepoProjectExtractedProperties
+        Include="@(ProjectDirectories)"
+        ProjectDirectory="%(Identity)"
+        KeepMetadata="MSBuildSourceProjectFile" />
+    </ItemGroup>
+
+    <MSBuild Projects="@(RepoProjectFile)" Targets="GetOfficialBuildId">
+      <Output TaskParameter="TargetOutputs" ItemName="OfficialBuildIds" />
+    </MSBuild>
+    <ItemGroup>
+      <_RepoProjectExtractedProperties
+        Include="@(OfficialBuildIds)"
+        OfficialBuildId="%(Identity)"
+        KeepMetadata="MSBuildSourceProjectFile" />
+    </ItemGroup>
 
+    <JoinItems
+      Left="@(RepoProjectFile)" LeftMetadata="*"
+      Right="@(_RepoProjectExtractedProperties)" RightKey="MSBuildSourceProjectFile" RightMetadata="*">
+      <Output TaskParameter="JoinResult" ItemName="_JoinProjectFileProperties" />
+    </JoinItems>
+
+    <!-- Add the git submodule directory to each project item. -->
     <ReadGitConfigFile File="$(GitModulesPath)">
       <Output TaskParameter="SubmoduleConfiguration" ItemName="SubmoduleConfigurations" />
     </ReadGitConfigFile>
-
-    <!--
-      Perform joins to gather information about each repo into a single item each. Each A-B join
-      first creates all A*B "candidate" combinations. The candidate items are then filtered down to
-      the combinations where the desired metadata from A and B match.
-    -->
     <ItemGroup>
-      <!-- Join manifest build name to project directory. -->
-      <BuildNamePathCandidate Include="@(OrchestratedManifestBuildNames)">
-        <Path>%(ProjectDirectories.Identity)</Path>
-        <ProjectDirectorySourceProjectFile>%(ProjectDirectories.MSBuildSourceProjectFile)</ProjectDirectorySourceProjectFile>
-      </BuildNamePathCandidate>
-
-      <BuildNamePath Include="@(BuildNamePathCandidate)"
-                     Condition="'%(MSBuildSourceProjectFile)' == '%(ProjectDirectorySourceProjectFile)' AND
-                                '%(Identity)' != 'N/A'" />
-
-      <!-- Join project directory to submodule URL. -->
-      <BuildNamePathGitUrlCandidate Include="@(BuildNamePath)">
-        <Submodule>$(ProjectDir)%(SubmoduleConfigurations.Path)/</Submodule>
-        <GitUrl>%(SubmoduleConfigurations.Url)</GitUrl>
-      </BuildNamePathGitUrlCandidate>
-
-      <BuildNamePathGitUrl Include="@(BuildNamePathGitUrlCandidate)"
-                           Condition="'%(Path)' == '%(Submodule)'" />
+      <_ExtractedGitModuleProperties
+        Include="@(SubmoduleConfigurations)"
+        ProjectDirectory="$(ProjectDir)%(Path)/"
+        GitUrl="%(Url)"
+        KeepMetadata="Identity" />
     </ItemGroup>
 
+    <JoinItems
+      Left="@(_JoinProjectFileProperties)" LeftKey="ProjectDirectory" LeftMetadata="*" LeftItemSpec="Identity"
+      Right="@(_ExtractedGitModuleProperties)" RightKey="ProjectDirectory" RightMetadata="*">
+      <Output TaskParameter="JoinResult" ItemName="RepoProjectWithProperties" />
+    </JoinItems>
+
     <ItemGroup>
-      <UpdateStep Include="@(BuildNamePathGitUrl)">
-        <UpdaterType>Submodule from orchestrated build</UpdaterType>
-        <BuildName>%(BuildNamePathGitUrl.Identity)</BuildName>
-      </UpdateStep>
+      <UpdateStep
+        Include="@(RepoProjectWithProperties)"
+        Condition="'%(RepoProjectWithProperties.BuildName)' != 'N/A'"
+        UpdaterType="Submodule from orchestrated build"
+        Path="%(RepoProjectWithProperties.ProjectDirectory)" />
+
+      <UpdateStep
+        Include="@(RepoProjectWithProperties)"
+        Condition="
+          '%(RepoProjectWithProperties.BuildName)' != 'N/A' AND
+          '%(RepoProjectWithProperties.OfficialBuildId)' != ''"
+        UpdaterType="Build attribute from orchestrated build"
+        AttributeName="BuildId"
+        Path="%(RepoProjectWithProperties.Identity)"
+        ElementName="OfficialBuildId" />
     </ItemGroup>
   </Target>
 </Project>

repos/cli-migrate.proj

@@ -14,7 +14,11 @@
     <GlobalJsonFile>$(ProjectDirectory)global.json</GlobalJsonFile>
     <DependencyVersionInputRepoApiImplemented>false</DependencyVersionInputRepoApiImplemented>
     <UsesRepoToolset>true</UsesRepoToolset>
+    <!--
+      This repo is not currently built in ProdCon
     <OrchestratedManifestBuildName>dotnet/cli-migrate</OrchestratedManifestBuildName>
+    -->
+    <OrchestratedManifestBuildName>N/A</OrchestratedManifestBuildName>
   </PropertyGroup>
 
   <ItemGroup>

repos/clicommandlineparser.proj

@@ -11,7 +11,11 @@
     <DependencyVersionInputRepoApiImplemented>false</DependencyVersionInputRepoApiImplemented>
     <UsesRepoToolset>true</UsesRepoToolset>
     <GlobalJsonFile>$(ProjectDirectory)global.json</GlobalJsonFile>
+    <!--
+      This repo is not currently built in ProdCon
     <OrchestratedManifestBuildName>dotnet/CliCommandLineParser</OrchestratedManifestBuildName>
+    -->
+    <OrchestratedManifestBuildName>N/A</OrchestratedManifestBuildName>
   </PropertyGroup>
 
   <ItemGroup>

repos/dir.targets

@@ -371,6 +371,7 @@
 
   <Target Name="GetProjectDirectory" Outputs="$(ProjectDirectory)" />
   <Target Name="GetOrchestratedManifestBuildName" Outputs="$(OrchestratedManifestBuildName)" />
+  <Target Name="GetOfficialBuildId" Outputs="$(OfficialBuildId)" />
 
   <Import Project="$(ToolsDir)VersionTools.targets" />
 </Project>

repos/fsharp.proj

@@ -6,6 +6,8 @@
     <BuildRevision>171107</BuildRevision>
     <BuildCommand>$(ProjectDirectory)/src/buildfromsource$(ShellExtension)</BuildCommand>
     <RepoApiImplemented>false</RepoApiImplemented>
+    <!-- This repo is not currently built in ProdCon -->
+    <OrchestratedManifestBuildName>N/A</OrchestratedManifestBuildName>
   </PropertyGroup>
 
   <!-- Auto-detect the root NuGet.Config, but also update the build-from-source-specific one. -->

repos/nuget-client.proj

@@ -8,7 +8,11 @@
     <NuGetKeyFilePath>$(KeysDir)NuGet.Client.snk</NuGetKeyFilePath>
     <NuGetClientBuildNumber>5156</NuGetClientBuildNumber>
     <RepoApiImplemented>false</RepoApiImplemented>
+    <!--
+      This repo is not currently built in ProdCon
     <OrchestratedManifestBuildName>nuget.client</OrchestratedManifestBuildName>
+    -->
+    <OrchestratedManifestBuildName>N/A</OrchestratedManifestBuildName>
   </PropertyGroup>
 
   <ItemGroup>

repos/templating.proj

@@ -7,6 +7,8 @@
     <PackageDateTime>20170727</PackageDateTime>
     <PackageBuildQuality>beta2</PackageBuildQuality>
     <RepoApiImplemented>false</RepoApiImplemented>
+    <!-- This repo is not currently built in ProdCon -->
+    <OrchestratedManifestBuildName>N/A</OrchestratedManifestBuildName>
   </PropertyGroup>
 
   <ItemGroup>

tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks/RepoTasks/JoinItems.cs

@@ -0,0 +1,139 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+// Copied from https://github.com/aspnet/Universe/blob/1f8f30a1e834eff147ced0c669cef8828f9511c8/build/tasks/JoinItems.cs.
+// When this task is available in https://github.com/dotnet/Arcade, switch to use that version.
+// Modified to allow multiple Right matches using GroupJoin.
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using System;
+using System.Linq;
+
+namespace RepoTasks
+{
+    public class JoinItems : Task
+    {
+        [Required]
+        public ITaskItem[] Left { get; set; }
+
+        [Required]
+        public ITaskItem[] Right { get; set; }
+
+        // The metadata to use as the new item spec. If not specified, LeftKey is used.
+        public string LeftItemSpec { get; set; }
+
+        //  LeftKey and RightKey: The metadata to join on.  If not set, then use the ItemSpec
+        public string LeftKey { get; set; }
+
+        public string RightKey { get; set; }
+
+
+        //  LeftMetadata and RightMetadata: The metadata names to include in the result.  Specify "*" to include all metadata
+        public string[] LeftMetadata { get; set; }
+
+        public string[] RightMetadata { get; set; }
+
+
+        [Output]
+        public ITaskItem[] JoinResult { get; private set; }
+
+        public override bool Execute()
+        {
+            bool useAllLeftMetadata = LeftMetadata != null && LeftMetadata.Length == 1 && LeftMetadata[0] == "*";
+            bool useAllRightMetadata = RightMetadata != null && RightMetadata.Length == 1 && RightMetadata[0] == "*";
+            var newItemSpec = string.IsNullOrEmpty(LeftItemSpec)
+                ? LeftKey
+                : LeftItemSpec;
+
+            JoinResult = Left.GroupJoin(Right,
+                item => GetKeyValue(LeftKey, item),
+                item => GetKeyValue(RightKey, item),
+                (left, rights) =>
+                {
+                    //  If including all metadata from left items and none from right items, just return left items directly
+                    if (useAllLeftMetadata &&
+                        string.IsNullOrEmpty(LeftKey) &&
+                        string.IsNullOrEmpty(LeftItemSpec) &&
+                        (RightMetadata == null || RightMetadata.Length == 0))
+                    {
+                        return left;
+                    }
+
+                    //  If including all metadata from all right items and none from left items, just return the right items directly
+                    if (useAllRightMetadata &&
+                        string.IsNullOrEmpty(RightKey) &&
+                        string.IsNullOrEmpty(LeftItemSpec) &&
+                        (LeftMetadata == null || LeftMetadata.Length == 0))
+                    {
+                        return rights.Aggregate(
+                            new TaskItem(),
+                            (agg, next) =>
+                            {
+                                CopyAllMetadata(next, agg);
+                                return agg;
+                            });
+                    }
+
+                    var ret = new TaskItem(GetKeyValue(newItemSpec, left));
+
+                    //  Weird ordering here is to prefer left metadata in all cases, as CopyToMetadata doesn't overwrite any existing metadata
+                    if (useAllLeftMetadata)
+                    {
+                        CopyAllMetadata(left, ret);
+                    }
+
+                    if (!useAllRightMetadata && RightMetadata != null)
+                    {
+                        foreach (string name in RightMetadata)
+                        {
+                            foreach (var right in rights)
+                            {
+                                ret.SetMetadata(name, right.GetMetadata(name));
+                            }
+                        }
+                    }
+
+                    if (!useAllLeftMetadata && LeftMetadata != null)
+                    {
+                        foreach (string name in LeftMetadata)
+                        {
+                            ret.SetMetadata(name, left.GetMetadata(name));
+                        }
+                    }
+
+                    if (useAllRightMetadata)
+                    {
+                        foreach (var right in rights)
+                        {
+                            CopyAllMetadata(right, ret);
+                        }
+                    }
+
+                    return (ITaskItem)ret;
+                },
+                StringComparer.OrdinalIgnoreCase).ToArray();
+
+            return true;
+        }
+
+        static void CopyAllMetadata(ITaskItem source, ITaskItem dest)
+        {
+            //  CopyMetadata adds an OriginalItemSpec, which we don't want.  So we subsequently remove it
+            source.CopyMetadataTo(dest);
+            dest.RemoveMetadata("OriginalItemSpec");
+        }
+
+        static string GetKeyValue(string key, ITaskItem item)
+        {
+            if (string.IsNullOrEmpty(key))
+            {
+                return item.ItemSpec;
+            }
+            else
+            {
+                return item.GetMetadata(key);
+            }
+        }
+    }
+}