Create workload insertion process

Directory.Build.props

@@ -7,10 +7,10 @@
     <Copyright>$(CopyrightNetFoundation)</Copyright>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <TreatWarningsAsErrors>false</TreatWarningsAsErrors>
-    <!-- <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
-    <DebugType>embedded</DebugType>
-    <DebugSymbols>true</DebugSymbols> -->
     <LangVersion>Latest</LangVersion>
+    <NoSwixBuildPlugin>true</NoSwixBuildPlugin>
+    <!-- See: https://dev.azure.com/devdiv/DevDiv/_wiki/wikis/DevDiv.wiki/29608/How-to-Build-a-vsman-file?anchor=creating-your-.vsman-file -->
+    <GetBuildVersionTargetDefined>true</GetBuildVersionTargetDefined>
   </PropertyGroup>
 
 </Project>

Directory.Build.targets

@@ -6,4 +6,5 @@
     <!-- Need to update the workloads version after the arcade targets have run -->
     <WorkloadsVersion Condition="'$(VersionSuffix)' != ''">$(WorkloadsVersion)-$(VersionSuffix)</WorkloadsVersion>
   </PropertyGroup>
+
 </Project>

README.md

@@ -1,3 +1,20 @@
 ## Welcome to the .NET SDK Workload Versions repo
 
-This repository contains the version information for .NET SDK Workloads.
+This repository contains the version information for .NET SDK Workloads.
+
+### Pre-requisites for local VS insertion build
+
+1. Install the latest [Visual Studio](https://visualstudio.microsoft.com/downloads/) with the .NET Desktop workload
+  - Make sure to restart your PC after the installation is complete.
+2. [Install Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli-windows#install-or-update)
+3. Run `az login` to authenticate
+  - When it asks for a subscription to select, just press Enter. The default subscription selection does not affect DARC.
+4. [Install DARC](https://github.com/dotnet/arcade/blob/main/Documentation/Darc.md#setting-up-your-darc-client)
+5. [Add GitHub auth for DARC](https://github.com/dotnet/arcade/blob/main/Documentation/Darc.md#step-3-set-additional-pats-for-azure-devops-and-github-operations)
+  - Use the [darc authenticate](https://github.com/dotnet/arcade/blob/main/Documentation/Darc.md#authenticate) command.
+  - Generate the GitHub PAT [here](https://github.com/settings/tokens?type=beta). Create a fine-grained PAT instead of the classic PAT.
+  - **Do not** create an AzDO PAT. Leave that entry blank in the darc-authenticate file for it to use local machine auth.
+6. Request access to the [.NET Daily Internal Build Access](https://coreidentity.microsoft.com/manage/Entitlement/entitlement/netdailyinte-q2ql) entitlement
+  - This allows the local AzDO machine auth to gather internal assets from AzDO.
+  - **Send a message** to one of the primary owners on the entitlement page for approval after requesting access to the entitlement.
+  - Should take about 20 mins for the entitlement process to complete (will appear on your [entitlements list](https://coreidentity.microsoft.com/manage/entitlement)) and another 30 mins the access to propagate to DARC. Basically, after approval, wait an hour until you actually attempt to build.

build.cmd

@@ -1,3 +1,3 @@
 @echo off
-powershell -NoLogo -NoProfile -ExecutionPolicy ByPass -Command "& """%~dp0eng\common\build.ps1""" -build -restore %*"
+powershell -NoLogo -NoProfile -ExecutionPolicy ByPass -Command "& """%~dp0eng\common\build.ps1""" -restore -build -msbuildEngine vs %*"
 exit /b %ErrorLevel%

eng/Publishing.props

@@ -6,9 +6,4 @@
     <PublishingVersion>3</PublishingVersion>
   </PropertyGroup>
 
-  <ItemGroup>
-    <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.msi" />
-    <SdkAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.wixpack.zip" />
-  </ItemGroup>
-
 </Project>

eng/Signing.props

@@ -6,11 +6,6 @@
     <InternalCertificateId Condition="'$(InternalCertificateId)' == ''">MicrosoftDotNet500</InternalCertificateId>
   </PropertyGroup>
 
-  <ItemGroup>
-    <ItemsToSign Include="$(ArtifactsPackagesDir)**\*.msi" />
-    <ItemsToSign Include="$(ArtifactsPackagesDir)**\*.nupkg" />
-  </ItemGroup>
-
   <ItemGroup>
     <ItemsToSignPostBuild Remove="*.wixpack.zip" />
   </ItemGroup>

eng/Versions.props

@@ -56,7 +56,7 @@
   </PropertyGroup>
   <PropertyGroup>
     <WixPackageVersion>3.14.0-8606.20240208.1</WixPackageVersion>
-    <SwixPackageVersion>1.1.87-gba258badda</SwixPackageVersion>
+    <SwixPackageVersion>1.1.392</SwixPackageVersion>
   </PropertyGroup>
   <PropertyGroup Label="EmscriptenWorkloads">
     <!-- Workloads from dotnet/emsdk -->

eng/create-workload-drops.ps1

@@ -0,0 +1,91 @@
+# Using the downloaded workloads, this creates the VS drops to upload for VS insertion.
+# It builds the Microsoft.NET.Workloads.Vsman.vsmanproj per workload ZIP, which creates the appropriate VSMAN file.
+
+# $workloadPath: The path to the directory containing the workload ZIPs, usually the output path used by DARC in the download-workloads.ps1 script.
+# - Example Value: "$(RepoRoot)artifacts\workloads"
+# $msBuildToolsPath: The path to the MSBuild tools directory, generally $(MSBuildToolsPath) in MSBuild.
+# - Example Value: 'C:\Program Files\Microsoft Visual Studio\2022\Preview\MSBuild\Current\Bin'
+
+param ([Parameter(Mandatory=$true)] [string] $workloadPath, [Parameter(Mandatory=$true)] [string] $msBuildToolsPath)
+
+# Extracts the workload drop zips.
+$workloads = Get-ChildItem $workloadPath -Include 'Workload.VSDrop.*.zip' -Recurse
+$workloadDropPath = (New-Item "$workloadPath\drops" -Type Container -Force).FullName
+$null = $workloads | ForEach-Object { Expand-Archive -Path $_.FullName -DestinationPath "$workloadDropPath\$([IO.Path]::GetFileNameWithoutExtension($_.Name))" -Force }
+
+# Extracts the workload drop metadata from the drop name and builds the .vsmanproj project.
+# - full: The full drop name, excluding the 'Workload.VSDrop.' prefix.
+# - short: The short name of the drop. Only contains the first word after 'Workload.VSDrop.'.
+# - type: Either 'pre.components', 'components', or 'packs'.
+$dropInfoRegex = '^Workload\.VSDrop\.(?<full>(?<short>\w*)\..*?(?<type>(pre\.)?components$|packs$))'
+$primaryVSComponentJsonValues = ''
+$secondaryVSComponentJsonValues = ''
+Get-ChildItem -Path $workloadDropPath -Directory | ForEach-Object {
+  $null = $_.Name -match $dropInfoRegex
+  $assemblyName = "$($Matches.full)"
+  $dropDir = "$($_.FullName)\"
+
+  # Hash the files within the drop folder to create a unique identifier that represents this workload drop.
+  # Example: 1E3EA4FE202394037253F57436A6EAD5DE1359792B618B9072014A98563A30FB
+  # See: https://learn.microsoft.com/powershell/module/microsoft.powershell.utility/get-filehash#example-4-compute-the-hash-of-a-string
+  $contentStream = [System.IO.MemoryStream]::new()
+  $writer = [System.IO.StreamWriter]::new($contentStream)
+  $dropFiles = Get-ChildItem -Path $dropDir | Sort-Object
+  # Note: We're using ASCII because when testing between PS 5.1 and PS 7.5, this would result in the same hash. Other encodings arrived at different hashes.
+  $null = $dropFiles | Get-Content -Encoding ASCII -Raw | ForEach-Object { $writer.Write($_) }
+  $writer.Flush()
+  $contentStream.Position = 0
+  $dropHash = (Get-FileHash -InputStream $contentStream).Hash
+  $writer.Close()
+
+  $vsDropName = "Products/dotnet/workloads/$assemblyName/$dropHash"
+  # Reads the first line out of the .metadata file in the workload's output folder and sets it to the workload version.
+  $workloadVersion = Get-Content "$dropDir.metadata" -First 1
+  # This requires building via MSBuild.exe as there are .NET Framework dependencies necessary for building the .vsmanproj.
+  # Additionally, even using the MSBuild task won't work as '/restore' must be used for it to restore properly when building the .vsmanproj.
+  & "$msBuildToolsPath\MSBuild.exe" Microsoft.NET.Workloads.Vsman.vsmanproj /restore /t:Build `
+    /p:AssemblyName=$assemblyName `
+    /p:VstsDropNames=$vsDropName `
+    /p:VsixOutputPath=$dropDir `
+    /p:WorkloadVersion=$workloadVersion
+
+  # While in CI, set the variables necessary for uploading the VS drop.
+  if ($env:TF_BUILD) {
+    $shortName = "$($Matches.short)"
+    # Remove the '.' from 'pre.components'
+    $dropType = $Matches.type.Replace('.', '')
+    $dropUrl = "https://vsdrop.microsoft.com/file/v1/$vsDropName;$assemblyName.vsman"
+
+    Write-Host "##vso[task.setvariable variable=$($shortName)_$($dropType)_name]$vsDropName"
+    Write-Host "##vso[task.setvariable variable=$($shortName)_$($dropType)_dir]$dropDir"
+    Write-Host "##vso[task.setvariable variable=$($shortName)_$($dropType)_url]$dropUrl"
+
+    # Each vsman file is comma-separated. First .vsman is destination and the second is source.
+    $vsComponentValue = "$assemblyName.vsman{$workloadVersion}=$dropUrl,"
+    # All VS components are added to the primary VS component JSON string.
+    $primaryVSComponentJsonValues += $vsComponentValue
+
+    # Secondary VS components do not include (pre)components drop types.
+    if ($dropType -ne 'components' -and $dropType -ne 'precomponents') {
+      $secondaryVSComponentJsonValues += $vsComponentValue
+    }
+  }
+
+  Write-Host '❗ After upload, your workload drop will be available at:'
+  Write-Host "https://devdiv.visualstudio.com/_apps/hub/ms-vscs-artifact.build-tasks.drop-hub-group-explorer-hub?name=$vsDropName"
+}
+
+# Clean up intermediate build files in the workload drop folders.
+$null = Get-ChildItem -Path $workloadDropPath -Include *.json, *.vsmand, files.txt -Recurse | Remove-Item
+
+# Write the primary and secondary component strings for the vsman files to a variable for the pipeline to use for the VS insertion step.
+if ($primaryVSComponentJsonValues) {
+  # Remove the trailing comma.
+  $primaryVSComponentJsonValues = $primaryVSComponentJsonValues -replace '.$'
+  Write-Host "##vso[task.setvariable variable=PrimaryVSComponentJsonValues]$primaryVSComponentJsonValues"
+}
+if ($secondaryVSComponentJsonValues) {
+  # Remove the trailing comma.
+  $secondaryVSComponentJsonValues = $secondaryVSComponentJsonValues -replace '.$'
+  Write-Host "##vso[task.setvariable variable=SecondaryVSComponentJsonValues]$secondaryVSComponentJsonValues"
+}

eng/download-workloads.ps1

@@ -0,0 +1,88 @@
+# This downloads the workloads using DARC.
+# In CI, we need to pass PATs to this, so it runs in Azure Pipelines only (not through MSBuild).
+# For local builds, some preconfiguration is necessary. Check the README.md for details.
+
+# $workloadPath: The path to the directory as output for the workload ZIPs. This is --output-dir in the DARC command.
+# - Example Value: "$(RepoRoot)artifacts\workloads"
+# $gitHubPat: The GitHub PAT to use for DARC (CI build only). See workload-build.yml for converting the PAT to SecureString.
+# $azDOPat: The Azure DevOps PAT to use for DARC (CI build only). See workload-build.yml for converting the PAT to SecureString.
+# $workloadListJson: The JSON string of the list of workload drop names to download. If not provided, all workloads found in Version.Details.xml will be downloaded.
+# - See the workloadDropNames parameter in official.yml for the list generally passed to this script.
+# - Example Value: '{["emsdk","mono"]}'
+# $usePreComponents:
+# - If $true, includes *pre.components.zip drops and excludes *components.zip drops.
+# - If $false, excludes *pre.components.zip drops and includes *components.zip drops.
+
+param ([Parameter(Mandatory=$true)] [string] $workloadPath, [SecureString] $gitHubPat, [SecureString] $azDOPat, [string] $workloadListJson = '', [bool] $usePreComponents = $false)
+
+### Local Build ###
+# Local build requires the installation of DARC. See: https://github.com/dotnet/arcade/blob/main/Documentation/Darc.md#setting-up-your-darc-client
+$darc = 'darc'
+$ciArguments = @()
+$ci = $gitHubPat -and $azDOPat
+
+### CI Build ###
+if ($ci) {
+  # Darc access copied from: eng/common/post-build/publish-using-darc.ps1
+  $disableConfigureToolsetImport = $true
+  . $PSScriptRoot\common\tools.ps1
+
+  $darc = Get-Darc
+  $gitHubPatPlain = ConvertFrom-SecureString -SecureString $gitHubPat -AsPlainText
+  $azDOPatPlain = ConvertFrom-SecureString -SecureString $azDOPat -AsPlainText
+  $ciArguments = @(
+    '--ci'
+    '--github-pat'
+    $gitHubPatPlain
+    '--azdev-pat'
+    $azDOPatPlain
+  )
+}
+
+# Reads the Version.Details.xml file to get the workload builds.
+$versionDetailsPath = (Get-Item "$PSScriptRoot\Version.Details.xml").FullName
+$versionDetailsXml = [Xml.XmlDocument](Get-Content $versionDetailsPath)
+$versionDetails = $versionDetailsXml.Dependencies.ProductDependencies.Dependency | Select-Object -Property Uri, Sha -Unique
+
+# Construct the asset filter to only download the required workload drops.
+$workloadFilter = ''
+if ($workloadListJson) {
+  $workloadList = ConvertFrom-Json -InputObject $workloadListJson
+  if ($workloadList.Count -ne 0) {
+    $workloadFilter = "($($workloadList | Join-String -Separator '|'))"
+  }
+}
+# Note: The $ at the end of these filters are required for the positive/negative lookbehinds to function.
+# Exclude pre.components.zip.
+$componentFilter = '(?<!pre\.components\.zip)$'
+if ($usePreComponents) {
+  # Exclude .components.zip but include pre.components.zip.
+  $componentFilter = '((?<!\.components\.zip)|(?<=pre\.components\.zip))$'
+}
+$assetFilter = "Workload\.VSDrop\.$workloadFilter.*$componentFilter"
+Write-Host "assetFilter: $assetFilter"
+
+# Runs DARC against each workload build to download the drops (if applicable based on the filter).
+$versionDetails | ForEach-Object {
+  $darcArguments = @(
+    'gather-drop'
+    '--asset-filter'
+    $assetFilter
+    '--repo'
+    $_.Uri
+    '--commit'
+    $_.Sha
+    '--output-dir'
+    $workloadPath
+    '--include-released'
+    '--skip-existing'
+    '--continue-on-error'
+    '--use-azure-credential-for-blobs'
+  )
+
+  & $darc ($darcArguments + $ciArguments)
+}
+
+Write-Host 'Workload drops downloaded:'
+# https://stackoverflow.com/a/9570030/294804
+Get-ChildItem $workloadPath -File -Include 'Workload.VSDrop.*.zip' -Recurse | Select-Object -Expand FullName