diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml new file mode 100644 index 000000000..7d760e2cd --- /dev/null +++ b/.github/workflows/updateBuildYamls.yml @@ -0,0 +1,124 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +name: Update Channel Based ReleaseStage Yaml Files + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + stableVersion: + description: 'stable release version (i.e: 7.4.1)' + required: false + previewVersion: + description: 'preview release version (i.e: 7.4.0-preview.5)' + required: false + ltsVersion: + description: 'lts release version (i.e: 7.2.12)' + required: false + +defaults: + run: + shell: pwsh +jobs: + update-yamls: + name: Update ReleaseStageYaml for Stable Channel + timeout-minutes: 15 + runs-on: ubuntu-20.04 + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Validate Version Syntax + id: validate-version + run: | + $stableValue = '${{ inputs.stableVersion }}' + if (!($stableValue -eq "")) + { + if ($stableValue -notmatch '\d+\.\d+\.\d+$') { + throw "stable release tag is not for a stable build: '$stableValue'" + } + } + + $previewValue = '${{ inputs.previewVersion }}' + if (!($previewValue -eq "")) + { + if ($previewValue -notmatch '\d+\.\d+\.\d+-(preview|rc)\.\d+$') { + throw "preview release tag is not for a preview build: '$previewValue'" + } + } + + $ltsValue = '${{ inputs.ltsVersion }}' + if (!($ltsValue -eq "")) + { + if ($ltsValue -notmatch '\d+\.\d+\.\d+$') { + throw "lts release tag is not for a lts build: '$ltsValue'" + } + } + - name: Update ReleaseStageYaml file for Stable + run: | + $stableValue = '${{ inputs.stableVersion }}' + $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' + $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' + $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' + Import-Module $buildHelperModulePath + if (!($stableValue -eq "")) + { + Write-Verbose -Verbose "using stable version: $stableValue" + ./build.ps1 -UpdateBuildYaml -Channel stable -StableVersion $stableValue -Verbose -Acr All -OsFilter All + } + else + { + # Use the version from channels.json when no version is provided + ./build.ps1 -UpdateBuildYaml -Channel stable -Verbose -Acr All -OsFilter All + } + - name: Update ReleaseStageYaml file for Preview + run: | + $previewValue = '${{ inputs.previewVersion }}' + $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' + $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' + $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' + Import-Module $buildHelperModulePath + if (!($previewValue -eq "")) + { + Write-Verbose -Verbose "using preview version: $previewValue" + ./build.ps1 -UpdateBuildYaml -Channel preview -PreviewVersion $previewValue -Verbose -Acr All -OsFilter All + } + else + { + ./build.ps1 -UpdateBuildYaml -Channel preview -Verbose -Acr All -OsFilter All + } + - name: Update ReleaseStageYaml file for LTS + run: | + $ltsValue = '${{ inputs.ltsVersion }}' + $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' + $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' + $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' + Import-Module $buildHelperModulePath + if (!($ltsValue -eq "")) + { + Write-Verbose -Verbose "using lts version: $ltsValue" + ./build.ps1 -UpdateBuildYaml -Channel lts -LtsVersion $ltsValue -Verbose -Acr All -OsFilter All + } + else + { + # Use the version from channels.json when no version is provided + ./build.ps1 -UpdateBuildYaml -Channel lts -Verbose -Acr All -OsFilter All + } + - name: Create Pull Request + if: github.event_name == 'workflow_dispatch' + id: cpr + uses: peter-evans/create-pull-request@v4 + with: + token: "${{ secrets.PR_PAT }}" + commit-message: "Update the stableReleaseStage yaml file" + committer: PwshBot + author: PwshBot + title: "Update the stableReleaseStage json" + base: master + draft: false + branch: update-build-yaml-files + push-to-fork: pwshBot/PowerShell-Docker diff --git a/build.ps1 b/build.ps1 index b98e2a5fd..7f44828e3 100755 --- a/build.ps1 +++ b/build.ps1 @@ -34,6 +34,10 @@ param( [switch] $GenerateMatrixJson, + [Parameter(Mandatory, ParameterSetName="UpdateBuildYaml")] + [switch] + $UpdateBuildYaml, + [Parameter(ParameterSetName="GenerateMatrixJson")] [switch] $FullJson, @@ -108,6 +112,7 @@ param( $Version, [Parameter(ParameterSetName="GenerateMatrixJson")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateManifestLists")] [Parameter(ParameterSetName="GenerateTagsYaml")] [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')] @@ -115,6 +120,7 @@ param( $StableVersion, [Parameter(ParameterSetName="GenerateMatrixJson")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateManifestLists")] [Parameter(ParameterSetName="GenerateTagsYaml")] [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')] @@ -122,6 +128,7 @@ param( $LtsVersion, [Parameter(ParameterSetName="GenerateManifestLists")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateMatrixJson")] [Parameter(ParameterSetName="GenerateTagsYaml")] [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')] @@ -129,6 +136,7 @@ param( $PreviewVersion, [Parameter(ParameterSetName="GenerateManifestLists")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateMatrixJson")] [Parameter(ParameterSetName="GenerateTagsYaml")] [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')] @@ -147,12 +155,14 @@ param( $ForcePesterInstall, [Parameter(ParameterSetName="GenerateManifestLists")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateMatrixJson")] [string] [ValidateSet('All','OnlyAcr','NoAcr')] $Acr, [Parameter(ParameterSetName="GenerateManifestLists")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateMatrixJson")] [string] [ValidateSet('All','Linux','Windows')] @@ -203,6 +213,7 @@ DynamicParam { Add-ParameterAttribute -ParameterSetName 'GetTagsByChannel' -Attributes $channelAttributes Add-ParameterAttribute -ParameterSetName 'DupeCheck' -Attributes $channelAttributes Add-ParameterAttribute -ParameterSetName 'GenerateMatrixJson' -Attributes $channelAttributes -Mandatory $false + Add-ParameterAttribute -ParameterSetName "UpdateBuildYaml" -Attributes $channelAttributes -Mandatory $false Add-ParameterAttribute -ParameterSetName 'GenerateTagsYaml' -Attributes $channelAttributes Add-ParameterAttribute -ParameterSetName 'GenerateManifestLists' -Attributes $channelAttributes @@ -227,7 +238,7 @@ Begin { $ENV:DOCKER_BUILDKIT = 1 - if ($PSCmdlet.ParameterSetName -notin 'GenerateMatrixJson', 'GenerateTagsYaml', 'DupeCheck', 'GenerateManifestLists' -and $Channel.Count -gt 1) + if ($PSCmdlet.ParameterSetName -notin 'GenerateMatrixJson', 'UpdateBuildYaml', 'GenerateTagsYaml', 'DupeCheck', 'GenerateManifestLists' -and $Channel.Count -gt 1) { throw "Multiple Channels are not supported in this parameter set" } @@ -404,7 +415,7 @@ End { } } } - elseif ($GenerateTagsYaml.IsPresent -or $GenerateMatrixJson.IsPresent -or $GenerateManifestLists.IsPresent) { + elseif ($GenerateTagsYaml.IsPresent -or $GenerateMatrixJson.IsPresent -or $UpdateBuildYaml.IsPresent -or $GenerateManifestLists.IsPresent) { if($Acr -eq 'OnlyAcr' -and !$useAcr) { continue @@ -437,7 +448,7 @@ End { $osVersion = $allMeta.meta.osVersion $manifestLists = $allMeta.meta.ManifestLists - if($osVersion -or $GenerateMatrixJson.IsPresent -or $GenerateManifestLists.IsPresent) + if($osVersion -or $GenerateMatrixJson.IsPresent -or $UpdateBuildYaml.IsPresent -or $GenerateManifestLists.IsPresent) { if ($osVersion) { $osVersion = $osVersion.replace('${fromTag}', $actualTagData.fromTag) @@ -525,8 +536,8 @@ End { Invoke-PesterWrapper -Script $testsPath -OutputFile $logPath -ExtraParams $extraParams } - Write-Verbose "!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$GenerateManifestLists -and '$($PSCmdlet.ParameterSetName)' -notlike '*All'" -Verbose - if (!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$GenerateManifestLists -and $PSCmdlet.ParameterSetName -notlike '*All') { + Write-Verbose "!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$UpdateBuildYaml -and !$GenerateManifestLists -and '$($PSCmdlet.ParameterSetName)' -notlike '*All'" -Verbose + if (!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$UpdateBuildYaml -and !$GenerateManifestLists -and $PSCmdlet.ParameterSetName -notlike '*All') { $dockerImagesToScan = '' # print local image names # used only with the -Build @@ -665,6 +676,47 @@ End { } } + $channelsUsed = @{} + if ($UpdateBuildYaml.IsPresent) { + foreach ($repo in $tagGroups.Keys | Sort-Object) { + $channelGroups = $tagGroups.$repo | Group-Object -Property Channel + foreach($channelGroup in $channelGroups) + { + $channelName = $channelGroup.Name + $ciFolder = Join-Path -Path $PSScriptRoot -ChildPath '.vsts-ci' + $channelReleaseStagePath = Join-Path -Path $ciFolder -ChildPath "$($channelName)ReleaseStage.yml" + + if (!$channelsUsed.Contains($channelName)) + { + # Note: channelGroup contains entry for a channels' regular and channel's test-deps images. + # But, we only want 1 ReleaseStage.yml file to be created per channel (ie 1 stableReleaseStage.yml) so only populate start of yaml file once per channel. + $channelReleaseStageFileExists = Test-Path $channelReleaseStagePath + if ($channelReleaseStageFileExists) + { + Remove-Item -Path $channelReleaseStagePath + } + + New-Item -Type File -Path $channelReleaseStagePath + + Get-StartOfYamlPopulated -Channel $channelName -YamlFilePath $channelReleaseStagePath + $channelsUsed.Add($channelName, $channelGroup.Values) + } + + $osGroups = $channelGroup.Group | Group-Object -Property os + foreach ($osGroup in $osGroups) { + $architectureGroups = $osGroup.Group | Group-Object -Property Architecture + foreach ($architectureGroup in $architectureGroups) { + # Note: For each image to be built (including test-deps images) the ReleaseStage.yml file needs to have a template call for the image. + foreach ($tag in $architectureGroup.Group | Sort-Object -Property dockerfile) { + Write-Verbose -Verbose "calling method to populate template call in yaml file for channel: $channelName for image: $($tag.Name)" + Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag + } + } + } + } + } + } + if ($GenerateManifestLists.IsPresent) { $manifestLists = @() $tags = @() diff --git a/tools/buildHelper/buildHelper.psm1 b/tools/buildHelper/buildHelper.psm1 index 1d63eb98c..85bc26d1b 100644 --- a/tools/buildHelper/buildHelper.psm1 +++ b/tools/buildHelper/buildHelper.psm1 @@ -1030,3 +1030,89 @@ Function ConvertTo-SortedDictionary { } return $sortedDictionary } + +function Get-StartOfYamlPopulated { + param( + [string] + $Channel, + + [string] + $YamlFilePath + ) + + if (!$YamlFilePath) + { + throw "Yaml file $YamlFilePath provided as parameter cannot be found." + } + + $doubleSpace = " "*2 + + Add-Content -Path $YamlFilePath -Value "parameters:" + Add-Content -Path $YamlFilePath -Value "- name: channel" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: 'preview'" + Add-Content -Path $YamlFilePath -Value "- name: channelPath" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: ''" + Add-Content -Path $YamlFilePath -Value "- name: vmImage" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: PSMMSUbuntu20.04-Secure" + Add-Content -Path $YamlFilePath -Value "stages:" + Add-Content -Path $YamlFilePath -Value "- stage: StageGenerateBuild_$Channel" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)dependsOn: ['StageResolveVersionandYaml']" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)displayName: Build $Channel" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)jobs:" +} + +function Get-TemplatePopulatedYaml { + param( + [string] + $YamlFilePath, + + [psobject] + $ImageInfo + ) + + if (!$YamlFilePath) + { + throw "Yaml file $YamlFilePath provided as parameter cannot be found." + } + + $doubleSpace = " "*2 + $fourSpace = " "*4 + $sixSpace = " "*6 + + $imageName = $ImageInfo.Name + $artifactSuffix = $ImageInfo.Name.ToString().Replace("\", "_").Replace("-","_") + $architecture = $ImageInfo.Architecture + $poolOS = $ImageInfo.IsLinux ? "linux" : "windows" + $archBasedJobName = "Build_$($poolOS)_$($architecture)" + + if ($architecture -eq "arm32") + { + $architecture = "arm64" # we need to use hostArchicture arm64 for pool for arm32 + } + + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)- template: /.vsts-ci/releasePhase.yml@self" + Add-Content -Path $YamlFilePath -Value "$($fourSpace)parameters:" + Add-Content -Path $YamlFilePath -Value "$($sixSpace)archName: '$archBasedJobName'" # ie: Build_Linux_arm32 + Add-Content -Path $YamlFilePath -Value "$($sixSpace)imageName: $imageName" # ie. imageName: alpine317\test-deps (since this differs from artifactSuffix for test-deps images only, we have a separate entry as the yaml param) + Add-Content -Path $YamlFilePath -Value "$($sixSpace)artifactSuffix: $artifactSuffix" # i.e artifactSuffix: alpine317_test_deps + Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolOS: '$poolOS'" + Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolHostArchitecture: '$architecture'" + if ($poolOS -eq "linux") + { + # only need to specify buildKitValue=1 for linux + Add-Content -Path $YamlFilePath -Value "$($sixSpace)buildKitValue: 1" + } + else + { + if ($imageName -contains "2022") + { + Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolHostVersion: '1ESWindows2022'" + Add-Content -Path $YamlFilePath -Value "$($sixSpace)windowsContainerImageValue: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'" + } + + Add-Content -Path $YamlFilePath -Value "$($sixSpace)maxParallel: 3" + } + + Add-Content -Path $YamlFilePath -Value "$($sixSpace)channel: `${{ parameters.channel }}" + Add-Content -Path $YamlFilePath -Value "$($sixSpace)channelPath: `${{ parameters.channelPath }}" +}