Switch agent queues used on Windows builds (#6627)

Author: natemcmaster

.azure/pipelines/jobs/default-build.yml

@@ -27,8 +27,6 @@
 #     A map of custom variables
 #   matrix: { string: { string: string } }
 #     A map of matrix configurations and variables. https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema?view=vsts#matrix
-#   demands: string | [ string ]
-#     A list of agent demands. https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema?view=vsts#demands
 #   dependsOn: string | [ string ]
 #     For fan-out/fan-in. https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema?view=vsts#phase
 #   codeSign: boolean
@@ -45,7 +43,6 @@ parameters:
   poolName: ''
   buildArgs: ''
   configuration: 'Release'
-  demands: []
   beforeBuild: []
   afterBuild: []
   codeSign: false
@@ -71,6 +68,7 @@ jobs:
       maxParallel: 8
       matrix: ${{ parameters.matrix }}
   # Map friendly OS names to the right queue
+  # See https://github.com/dotnet/arcade/blob/master/Documentation/ChoosingAMachinePool.md
   pool:
     ${{ if ne(parameters.poolName, '') }}:
       name: ${{ parameters.poolName }}
@@ -81,24 +79,25 @@ jobs:
       name: Hosted Ubuntu 1604
       vmImage: ubuntu-16.04
     ${{ if and(eq(parameters.poolName, ''), eq(parameters.agentOs, 'Windows')) }}:
-      ${{ if ne(parameters.codeSign, 'true') }}:
-        name: Hosted VS2017
-        vmImage: vs2017-win2016
-      ${{ if eq(parameters.codeSign, 'true') }}:
-        name: DotNetCore-Windows
+      ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+        name: dotnet-internal-temp
+      ${{ if ne(variables['System.TeamProject'], 'internal') }}:
+        name: dotnet-external-temp
   variables:
     AgentOsName: ${{ parameters.agentOs }}
     ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping
-    DOTNET_HOME: $(Agent.WorkFolder)/.dotnet
+    DOTNET_HOME: $(Agent.BuildDirectory)/.dotnet
+    BuildScript: ${{ parameters.buildScript }}
     BuildScriptArgs: ${{ parameters.buildArgs }}
     BuildConfiguration: ${{ parameters.configuration }}
     BuildDirectory: ${{ parameters.buildDirectory }}
-    VSTS_OVERWRITE_TEMP: false # Workaround for https://github.com/dotnet/core-eng/issues/2812
-    ${{ if eq(parameters.codeSign, 'true') }}:
+    ${{ if eq(parameters.agentOs, 'Windows') }}:
+      JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk
+    ${{ if or(ne(parameters.codeSign, 'true'), ne(variables['System.TeamProject'], 'internal')) }}:
+      _SignType:
+    ${{ if and(eq(parameters.codeSign, 'true'), eq(variables['System.TeamProject'], 'internal')) }}:
       TeamName: AspNetCore
       _SignType: real
-    ${{ if ne(parameters.codeSign, 'true') }}:
-      _SignType:
     ${{ insert }}: ${{ parameters.variables }}
   steps:
   - checkout: self
@@ -107,13 +106,17 @@ jobs:
     displayName: Install Node 10.x
     inputs:
       versionSpec: 10.x
-  - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}:
+  - ${{ if eq(parameters.agentOs, 'Windows') }}:
+    - powershell: ./eng/scripts/InstallJdk.ps1 '11.0.1'
+      displayName: Install JDK 11
+  - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}:
     - task: MicroBuildSigningPlugin@1
       displayName: Install MicroBuild Signing plugin
       condition: and(succeeded(), in(variables['_SignType'], 'test', 'real'))
       inputs:
         signType: $(_SignType)
         zipSources: false
+        feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
   - ${{ parameters.beforeBuild }}
   - ${{ if eq(parameters.agentOs, 'Windows') }}:
     - script: .\$(BuildDirectory)\build.cmd -ci /p:SignType=$(_SignType) /p:Configuration=$(BuildConfiguration) $(BuildScriptArgs)
@@ -124,6 +127,7 @@ jobs:
   - task: PublishTestResults@2
     displayName: Publish test results
     condition: always()
+    continueOnError: true
     inputs:
       testRunTitle: $(AgentOsName)-$(BuildConfiguration)
       testRunner: vstest
@@ -133,6 +137,7 @@ jobs:
     - task: PublishBuildArtifacts@1
       displayName: Upload artifacts
       condition: eq(variables['system.pullrequest.isfork'], false)
+      continueOnError: true
       inputs:
         ${{ if eq(parameters.buildDirectory, '') }}:
           pathtoPublish: ${{ parameters.artifacts.path }}
@@ -145,7 +150,7 @@ jobs:
         artifactType: Container
         parallel: true
   - ${{ parameters.afterBuild }}
-  - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}:
+  - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.agentOs, 'Windows')) }}:
     - task: MicroBuildCleanup@1
       displayName: Cleanup MicroBuild tasks
       condition: always()

eng/scripts/InstallJdk.ps1

@@ -0,0 +1,27 @@
+
+param(
+    [Parameter(Mandatory = $true)]
+    $JdkVersion
+    )
+
+$ErrorActionPreference = 'Stop'
+$ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
+
+Set-StrictMode -Version 1
+
+if (-not $env:JAVA_HOME) {
+    throw 'You must set the JAVA_HOME environment variable to the destination of the JDK.'
+}
+
+$repoRoot = Resolve-Path "$PSScriptRoot/../.."
+$tempDir = "$repoRoot/obj"
+mkdir $tempDir -ea Ignore | out-null
+Write-Host "Starting download of JDK ${JdkVersion}"
+Invoke-WebRequest -UseBasicParsing -Uri "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/java/jdk-${JdkVersion}_windows-x64_bin.zip" -Out "$tempDir/jdk.zip"
+Write-Host "Done downloading JDK ${JdkVersion}"
+Expand-Archive "$tempDir/jdk.zip" -d "$tempDir/jdk/"
+Write-Host "Expanded JDK to $tempDir"
+mkdir (split-path -parent $env:JAVA_HOME) -ea ignore | out-null
+Write-Host "Installing JDK to $env:JAVA_HOME"
+Move-Item "$tempDir/jdk/jdk-${jdkVersion}" $env:JAVA_HOME
+Write-Host "Done installing JDK to $env:JAVA_HOME"

eng/scripts/common.psm1

@@ -62,6 +62,7 @@ function Get-RemoteFile([string]$RemotePath, [string]$LocalPath) {
     while ($retries -gt 0) {
         $retries -= 1
         try {
+            $ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
             Invoke-WebRequest -UseBasicParsing -Uri $RemotePath -OutFile $LocalPath
             return
         }

run.ps1

@@ -172,6 +172,7 @@ function Get-RemoteFile([string]$RemotePath, [string]$LocalPath) {
     while ($retries -gt 0) {
         $retries -= 1
         try {
+            $ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
             Invoke-WebRequest -UseBasicParsing -Uri $RemotePath -OutFile $LocalPath
             return
         }

scripts/common.psm1

@@ -141,6 +141,7 @@ function Ensure-Hub() {
 
         mkdir -Path $tmpDir -ErrorAction Ignore | Out-Null
 
+        $ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
         Invoke-WebRequest -OutFile $zipLocation -Uri $source
 
         Expand-Archive -Path $zipLocation -DestinationPath $zipDir -Force

src/Middleware/WebSockets/setup-wstest.ps1

@@ -14,6 +14,7 @@ $WsTest = Join-Path $ScriptsDir "wstest.exe"
 $VCPythonMsi = Join-Path $VendorDir "VCForPython27.msi"
 if(!(Test-Path $VCPythonMsi)) {
     Write-Host "Downloading VCForPython27.msi"
+    $ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
     Invoke-WebRequest -Uri https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi -OutFile "$VCPythonMsi"
 }
 else {
@@ -49,4 +50,4 @@ virtualenv $VirtualEnvDir
 # Install autobahn into the virtualenv
 & "$ScriptsDir\pip" install autobahntestsuite
 
-Write-Host "Using wstest from: '$WsTest'"
+Write-Host "Using wstest from: '$WsTest'"

src/Servers/IIS/tools/SetupTestEnvironment.ps1

@@ -94,6 +94,7 @@ function Shutdown-Dumps()
     {
         $downloadedFile = [System.IO.Path]::GetTempFileName();
         $downloadedFile = "$downloadedFile.exe";
+        $ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
         Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?linkid=870807" -OutFile $downloadedFile;
         & $downloadedFile /features OptionId.WindowsDesktopDebuggers /norestart /q;
     }
@@ -132,4 +133,4 @@ if ($Mode -eq "Shutdown")
     Shutdown-Dumps;
 }
 
-Exit 0;
+Exit 0;

test/Cli.FunctionalTests/run-tests.ps1

@@ -139,6 +139,7 @@ try {
         $cliUrl = "$AssetRootUrl/Sdk/$sdkVersion/dotnet-sdk-$sdkVersion-$HostRid$archiveExt"
         $cliArchiveFile = "$PSScriptRoot/obj/dotnet$archiveExt"
         Write-Host "Downloading $cliUrl"
+        $ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
         Invoke-WebRequest -UseBasicParsing "${cliUrl}${AccessTokenSuffix}" -OutFile $cliArchiveFile
         if ($archiveExt -eq '.zip') {
             Expand-Archive $cliArchiveFile -DestinationPath $dotnetRoot