Skip to content

Commit d45cd03

Browse files
ramsessanchezramsessanchez
authored
Separate weekly openApi-doc refresh and command-metadata refresh (#3467)
* Separate weekly open api -doc refresh and command metadata refresh * rename file * skip openApi download steps entirley on check * temp disable condition param * file rename ref * re introduce sign param * consolidate skipDownload parameters * move module matrix to independent script * consolidate conditionals and variables * move openapidocdiff to dedicated script file * remove duplicate diff script * branch variable scope * fix parameter passing
1 parent 3e13a51 commit d45cd03

File tree

6 files changed

+424
-88
lines changed

6 files changed

+424
-88
lines changed

‎.azure-pipelines/command-metadata-refresh.yml‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool.
2+
# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected.
3+
# This pipeline will be extended to the OneESPT template
4+
# If you are not using the E+D shared hosted pool with windows-2022, replace the pool section with your hosted pool, os, and image name. If you are using a Linux image, you must specify an additional windows image for SDL: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/sdlanalysis/overview#how-to-specify-a-windows-pool-for-the-sdl-source-analysis-stage
5+
# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish Module Artifacts' in the templateContext section.
6+
# The Task 'NuGetCommand@2' has been converted to an output named 'Publish NuGet to feed' in the templateContext section.
7+
8+
name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)
9+
parameters:
10+
- name: BuildAgent
11+
displayName: Build Agent
12+
default: 1es-windows-ps-compute-m
13+
- name: BaseBranch
14+
displayName: Base Branch
15+
default: main
16+
- name: Test
17+
type: boolean
18+
default: true
19+
- name: Pack
20+
type: boolean
21+
default: false
22+
- name: Sign
23+
type: boolean
24+
default: false
25+
- name: BumpModuleVersion
26+
type: boolean
27+
default: false
28+
- name: CreatePullRequest
29+
type: boolean
30+
default: true
31+
32+
variables:
33+
BuildAgent: ${{ parameters.BuildAgent }}
34+
Branch: "ModuleCommandMetadataRefresh"
35+
BaseBranch: ${{ parameters.BaseBranch }}
36+
37+
trigger:
38+
branches:
39+
include:
40+
- main
41+
42+
resources:
43+
repositories:
44+
- repository: 1ESPipelineTemplates
45+
type: git
46+
name: 1ESPipelineTemplates/1ESPipelineTemplates
47+
ref: refs/tags/release
48+
extends:
49+
template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
50+
parameters:
51+
pool: $(BuildAgent)
52+
sdl:
53+
binskim:
54+
enabled: false
55+
justificationForDisabling: "Binskim keeps on crashing and failing the weekly build pipeline. Disabling it for now because we are unable to publish the artifacts to internal feeds."
56+
credscan:
57+
suppressionsFile: $(Build.SourcesDirectory)/.azure-pipelines/config/credscan/credscan-suppressions.json
58+
policheck:
59+
exclusionFile: $(Build.SourcesDirectory)/.azure-pipelines/config/policheck/policheck-exclusions.xml
60+
customBuildTags:
61+
- ES365AIMigrationTooling
62+
stages:
63+
- stage: stage
64+
jobs:
65+
- job: CreateMetadataRefreshBranch
66+
displayName: Create Metadata Refresh Branch
67+
timeoutInMinutes: 1200
68+
steps:
69+
- template: .azure-pipelines/common-templates/checkout.yml@self
70+
parameters:
71+
TargetBranch: ${{ parameters.BaseBranch }}
72+
- task: PowerShell@2
73+
name: "ComputeBranch"
74+
displayName: "Compute Module Command Metadata Refresh Branch Name"
75+
inputs:
76+
targetType: inline
77+
script: |
78+
$branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm)
79+
Write-Host "##vso[task.setvariable variable=RefreshBranch;isOutput=true]$branch"
80+
- task: Bash@3
81+
displayName: "Create Module Command Metadata Refresh Branch"
82+
inputs:
83+
targetType: inline
84+
script: |
85+
git status
86+
git branch $(ComputeBranch.RefreshBranch)
87+
git checkout $(ComputeBranch.RefreshBranch)
88+
git status
89+
90+
- job: MsGraphPsSdkModuleCommandMetadataGeneration
91+
dependsOn: CreateMetadataRefreshBranch
92+
displayName: Microsoft Graph PowerShell SDK Generation
93+
condition: succeeded()
94+
timeoutInMinutes: 840
95+
variables:
96+
RefreshBranch: $[ dependencies.CreateMetadataRefreshBranch.outputs['ComputeBranch.RefreshBranch'] ]
97+
steps:
98+
- template: .azure-pipelines/common-templates/checkout.yml@self
99+
parameters:
100+
TargetBranch: $(RefreshBranch)
101+
- template: .azure-pipelines/common-templates/install-tools.yml@self
102+
- template: .azure-pipelines/common-templates/security-pre-checks.yml@self
103+
- template: .azure-pipelines/generation-templates/authentication-module.yml@self
104+
parameters:
105+
Test: ${{ parameters.Test }}
106+
Pack: ${{ parameters.Pack }}
107+
Sign: ${{ parameters.Sign }}
108+
- template: .azure-pipelines/generation-templates/workload-modules.yml@self
109+
parameters:
110+
Test: ${{ parameters.Test }}
111+
Pack: ${{ parameters.Pack }}
112+
Sign: ${{ parameters.Sign }}
113+
- template: .azure-pipelines/generation-templates/meta-module.yml@self
114+
parameters:
115+
Test: ${{ parameters.Test }}
116+
Pack: ${{ parameters.Pack }}
117+
Sign: ${{ parameters.Sign }}
118+
- template: .azure-pipelines/generation-templates/generate-command-metadata.yml@self
119+
- template: .azure-pipelines/common-templates/security-post-checks.yml@self
120+
121+
- ${{ if eq(parameters.BumpModuleVersion, true) }}:
122+
- task: PowerShell@2
123+
name: CalculateAndBumpModuleVersion
124+
displayName: Calculate and bump module version
125+
condition: succeeded()
126+
inputs:
127+
pwsh: true
128+
targetType: inline
129+
script: |
130+
. "$(System.DefaultWorkingDirectory)\tools\Versions\BumpModuleVersion.ps1" -BumpV1Module -BumpBetaModule -BumpAuthModule -Debug
131+
- task: Bash@3
132+
displayName: Push version bump changes
133+
env:
134+
GITHUB_TOKEN: $(GITHUB_TOKEN)
135+
inputs:
136+
targetType: inline
137+
script: |
138+
git status
139+
git add "$(System.DefaultWorkingDirectory)/config/ModuleMetadata.json"
140+
git commit -m 'Bump module versions after metadata generation.'
141+
git push "https://$(GITHUB_TOKEN)@github.com/microsoftgraph/msgraph-sdk-powershell.git"
142+
git status
143+
144+
- ${{ if eq(parameters.CreatePullRequest, true) }}:
145+
- template: .azure-pipelines/common-templates/create-pr.yml@self
146+
parameters:
147+
BaseBranch: $(BaseBranch)
148+
TargetBranch: $(RefreshBranch)
149+
Title: "[v2] Module Command Metadata Refresh"
150+
Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR."

‎.azure-pipelines/common-templates/download-openapi-docs.yml‎

Lines changed: 16 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,10 @@
22
# Licensed under the MIT License.
33

44
parameters:
5-
- name: Branch
6-
type: string
75
- name: BaseBranch
86
type: string
97
- name: BuildAgent
108
displayName: Build Agent
11-
- name: SkipForceRefresh
12-
type: string
13-
- name: SkipOpenAPIDocsDownload
14-
type: boolean
15-
default: false
169

1710
steps:
1811
- template: ./checkout.yml
@@ -23,74 +16,49 @@ steps:
2316

2417
- task: PowerShell@2
2518
name: "ComputeBranch"
26-
displayName: "Compute weekly branch name"
19+
displayName: "Compute weekly branch name and set Branch variable"
2720
inputs:
2821
targetType: inline
2922
script: |
30-
$branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm)
31-
Write-Host "##vso[task.setvariable variable=WeeklyBranch;isOutput=true]$branch"
23+
$branch = "WeeklyApiRefresh/{0}" -f (Get-Date -Format yyyyMMddHHmm)
24+
Write-Host "Computed weekly branch: $branch"
25+
# Set the Branch pipeline variable for current job
26+
Write-Host "##vso[task.setvariable variable=Branch;isOutput=false]$branch"
27+
# Set the Branch variable as output for subsequent jobs
28+
Write-Host "##vso[task.setvariable variable=Branch;isOutput=true]$branch"
3229
3330
- task: Bash@3
3431
displayName: "Create weekly branch"
3532
inputs:
3633
targetType: inline
3734
script: |
3835
git status
39-
git branch $(ComputeBranch.WeeklyBranch)
40-
git checkout $(ComputeBranch.WeeklyBranch)
36+
git branch $(Branch)
37+
git checkout $(Branch)
4138
git status
4239
4340
- task: PowerShell@2
4441
displayName: Download v1.0 OpenApi docs
45-
condition: and(succeeded(), eq('${{ parameters.SkipOpenAPIDocsDownload }}', false))
4642
continueOnError: false
4743
inputs:
4844
filePath: "$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1"
49-
arguments: -SkipForceRefresh:$$(SkipForceRefresh)
5045
pwsh: true
5146

5247
- task: PowerShell@2
5348
displayName: Download beta OpenApi docs
54-
condition: and(succeeded(), eq('${{ parameters.SkipOpenAPIDocsDownload }}', false))
5549
continueOnError: false
5650
inputs:
5751
filePath: "$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1"
58-
arguments: -BetaGraphVersion -SkipForceRefresh:$$(SkipForceRefresh)
52+
arguments: -BetaGraphVersion
5953
pwsh: true
6054

6155
- task: PowerShell@2
6256
name: OpenAPIDocDiff
63-
displayName: Get OpenAPI docs diff
57+
displayName: Get OpenAPI docs diff and set ModuleGenerationList variable
6458
inputs:
59+
filePath: "$(System.DefaultWorkingDirectory)/tools/Utilities/OpenApiDocDiff.ps1"
60+
arguments: -OutputVariable "ModuleGenerationList"
6561
pwsh: true
66-
targetType: "inline"
67-
script: |
68-
Write-Host "SkipOpenAPIDocsDownload: ${{ parameters.SkipOpenAPIDocsDownload }}"
69-
if ('${{ parameters.SkipOpenAPIDocsDownload }}' -eq 'True') {
70-
$ModuleNames = "Skipped"
71-
Write-Warning "Skipped OpenAPI Docs Download."
72-
} else {
73-
$diff = git diff --name-only
74-
$ModulesWithChanges = @{}
75-
$diff | %{
76-
if (($_ -match 'openApiDocs\/(v1.0|beta)\/(.*).yml') -and !$ModulesWithChanges.ContainsKey($matches.2))
77-
{
78-
$ModulesWithChanges.Add($matches.2, $matches.1)
79-
}
80-
}
81-
$ModuleNames = $ModulesWithChanges.Keys
82-
}
83-
Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames"
84-
85-
- task: PowerShell@2
86-
name: CalculateAndBumpModuleVersion
87-
displayName: Calculate and bump module version
88-
condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], ''))
89-
inputs:
90-
pwsh: true
91-
targetType: inline
92-
script: |
93-
. "$(System.DefaultWorkingDirectory)\tools\Versions\BumpModuleVersion.ps1" -BumpV1Module -BumpBetaModule -BumpAuthModule -Debug
9462

9563
- task: PowerShell@2
9664
name: BuildOpenApiMetadataDetectionTool
@@ -112,7 +80,7 @@ steps:
11280

11381
- task: Bash@3
11482
displayName: Commit downloaded files
115-
condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], ''))
83+
condition: and(succeeded(), ne(variables['ModuleGenerationList'], ''))
11684
env:
11785
GITHUB_TOKEN: $(GITHUB_TOKEN)
11886
inputs:
@@ -122,10 +90,10 @@ steps:
12290
git add .
12391
git commit -m 'Weekly OpenApiDocs Download.'
12492
git status
125-
git push --set-upstream "https://$(GITHUB_TOKEN)@github.com/microsoftgraph/msgraph-sdk-powershell.git" $(ComputeBranch.WeeklyBranch)
93+
git push --set-upstream "https://$(GITHUB_TOKEN)@github.com/microsoftgraph/msgraph-sdk-powershell.git" $(Branch)
12694
git status
12795
12896
# References
12997
# [0] https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables
13098
# [1] https://hub.github.com/hub-pull-request.1.html
131-
# https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token
99+
# https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token

‎.azure-pipelines/generation-templates/individual-workload-module.yml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Copyright (c) Microsoft Corporation. All rights reserved.
2+
# Licensed under the MIT License.
3+
4+
5+
parameters:
6+
- name: Test
7+
type: boolean
8+
default: true
9+
- name: ModuleName
10+
type: string
11+
default: ""
12+
- name: ModuleVersion
13+
type: string
14+
default: ""
15+
- name: Pack
16+
type: boolean
17+
default: true
18+
- name: Sign
19+
type: boolean
20+
default: true
21+
22+
steps:
23+
- task: PowerShell@2
24+
displayName: Generate Workload Modules
25+
inputs:
26+
targetType: inline
27+
pwsh: true
28+
script: |
29+
. $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -EnableSigning:$${{ parameters.Sign }} -Build -ExcludeExampleTemplates -ExcludeNotesSection -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }}
30+
- template: ../common-templates/guardian-analyzer.yml
31+
32+
- task: PowerShell@2
33+
displayName: Test Workload Modules
34+
enabled: false
35+
inputs:
36+
targetType: inline
37+
pwsh: true
38+
script: |
39+
. $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -SkipGeneration -Test -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }}
40+
41+
- task: PowerShell@2
42+
displayName: Find Duplicate Commands
43+
inputs:
44+
targetType: inline
45+
pwsh: true
46+
script: |
47+
. $(System.DefaultWorkingDirectory)/tools/PostGeneration/FindDuplicateCommand.ps1 -SourcePath "$(System.DefaultWorkingDirectory)/src/"
48+

0 commit comments

Comments
 (0)