sync main (#28116)

Author: YanaXu

.azure-pipelines/PipelineSteps/BatchGeneration/analyse-modules.ps1

@@ -82,9 +82,10 @@ foreach ($moduleName in $moduleGroup) {
         $result.Status = "Failed"
         $result.Error = $_.Exception.Message
     } finally {
-        $endTine = Get-Date
-        $result.DurationSeconds = ($endTine - $startTime).TotalSeconds
+        $endTime = Get-Date
+        $result.DurationSeconds = ($endTime - $startTime).TotalSeconds
         $results += $result
+        $result | ConvertTo-Json -Depth 5 | Write-Output
     }
 }
 

.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1

@@ -113,6 +113,7 @@ foreach ($moduleName in $sortedModuleNames) {
     $moduleEndTime = Get-Date
     $moduleResult.DurationSeconds = ($moduleEndTime - $moduleStartTime).TotalSeconds
     $results += $moduleResult
+    $moduleResult | ConvertTo-Json -Depth 5 | Write-Output
 }
 
 $ArtifactOutputDir = Join-Path $RepoRoot "artifacts"

.azure-pipelines/PipelineSteps/BatchGeneration/build-modules.ps1

@@ -30,9 +30,10 @@ foreach ($moduleName in $moduleGroup) {
         $result.Status = "Failed"
         $result.Error = $_.Exception.Message
     } finally {
-        $endTine = Get-Date
-        $result.DurationSeconds = ($endTine - $startTime).TotalSeconds
+        $endTime = Get-Date
+        $result.DurationSeconds = ($endTime - $startTime).TotalSeconds
         $results += $result
+        $result | ConvertTo-Json -Depth 5 | Write-Output
     }
 }
 

.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1

@@ -15,7 +15,21 @@ $artifactsDir = Join-Path $RepoRoot 'artifacts'
 
 $changedModulesDict = @{}
 $changedSubModulesDict = @{}
-if ($env:RUN_TEST_ON_ALL_MODULES -eq "True") {
+if ($env:TEST_CHANGED_MODULES_ONLY -eq "True") {
+    Write-Host "Run test on generated folder changed modules"
+    # Only generated folder change should trigger the test
+    for ($i = 0; $i -lt $ChangedFiles.Count; $i++) {
+        if ($ChangedFiles[$i] -match '^generated/([^/]+)/([^/]+\.Autorest)/') {
+            $moduleName = $Matches[1]
+            $subModuleName = $Matches[2]
+            $subModule = "$moduleName/$subModuleName"
+            
+            $changedModulesDict[$moduleName] = $true
+            $changedSubModulesDict[$subModule] = $true
+        }
+    }
+}
+else {
     Write-Host "Run test on all modules"
     $V4ModulesFile = Join-Path $artifactsDir "generationTargets.json"
     $V4ModuleMaps = Get-Content -Raw -Path $V4ModulesFile | ConvertFrom-Json
@@ -31,20 +45,6 @@ if ($env:RUN_TEST_ON_ALL_MODULES -eq "True") {
         }
     }
 }
-else {
-    Write-Host "Run test on generated folder changed modules"
-    # Only generated folder change should trigger the test 
-    for ($i = 0; $i -lt $ChangedFiles.Count; $i++) {
-        if ($ChangedFiles[$i] -match '^generated/([^/]+)/([^/]+\.autorest)/') {
-            $moduleName = $Matches[2]
-            $subModuleName = $Matches[3]
-            $subModule = "$moduleName/$subModuleName"
-            
-            $changedModulesDict[$moduleName] = $true
-            $changedSubModulesDict[$subModule] = $true
-        }
-    }
-}
 
 $changedModules = $changedModulesDict.Keys | Sort-Object
 $changedSubModules = $changedSubModulesDict.Keys | Sort-Object
@@ -63,6 +63,11 @@ foreach ($subModule in $changedSubModules) {
 Write-Host "##[endgroup]"
 Write-Host
 
+$changedModulesRecordFile = Join-Path $artifactsDir 'filteredChangedModules.txt'
+$changedModules | Set-Content -Path $changedModulesRecordFile -Encoding UTF8
+$changedSubModulesRecordFile = Join-Path $artifactsDir 'filteredChangedSubModules.txt'
+$changedSubModules | Set-Content -Path $changedSubModulesRecordFile -Encoding UTF8
+
 $groupedBuildModules = Group-Modules -Modules $changedModules -MaxParallelJobs $MaxParallelBuildJobs
 Write-Matrix -GroupedModules $groupedBuildModules -VariableName 'buildTargets' -RepoRoot $RepoRoot
 

.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1

@@ -8,7 +8,7 @@ $utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGen
 Import-Module $utilFilePath -Force
 
 $srcPath = Join-Path $RepoRoot 'src'
-$moduleMap = Get-AutorestV4ModuleMap -srcPath $srcPath
+$moduleMap = Get-BatchGenerationModuleMap -srcPath $srcPath
 Write-Host "Total matched modules: $($moduleMap.Count)"
 
 $modules = @($moduleMap.Keys | Sort-Object)
@@ -34,14 +34,17 @@ foreach ($moduleGroup in $groupedModules) {
     $index++
 }
 
-$generationTargetsOutputDir = Join-Path $RepoRoot "artifacts"
-if (-not (Test-Path -Path $generationTargetsOutputDir)) {
-    New-Item -ItemType Directory -Path $generationTargetsOutputDir
+$artifactsDir = Join-Path $RepoRoot "artifacts"
+if (-not (Test-Path -Path $artifactsDir)) {
+    New-Item -ItemType Directory -Path $artifactsDir
 }
-$generationTargetsOutputFile = Join-Path $generationTargetsOutputDir "generationTargets.json"
+$generationTargetsOutputFile = Join-Path $artifactsDir "generationTargets.json"
 $generationTargets | ConvertTo-Json -Depth 5 | Out-File -FilePath $generationTargetsOutputFile -Encoding utf8
 
 if ($MatrixStr -and $MatrixStr.Length -gt 1) {
     $MatrixStr = $MatrixStr.Substring(1)
 }
 Write-Host "##vso[task.setVariable variable=generationTargets;isOutput=true]{$MatrixStr}"
+
+$V4ModulesRecordFile = Join-Path $artifactsDir 'preparedV4Modules.txt'
+$modules | Set-Content -Path $V4ModulesRecordFile -Encoding UTF8

.azure-pipelines/PipelineSteps/BatchGeneration/test-modules.ps1

@@ -56,6 +56,7 @@ foreach ($subModule in $subModuleGroup) {
         $endTime = Get-Date
         $result.DurationSeconds = ($endTime - $startTime).TotalSeconds
         $results += $result
+        $result | ConvertTo-Json -Depth 5 | Write-Output
     }
 }
 

.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1

@@ -1,18 +1,26 @@
-function Get-AutorestV4ModuleMap {
+function Get-BatchGenerationModuleMap {
     param (
         [string]$srcPath
     )
-    
+    $skippedModules = $env:SKIPPED_MODULES -split ',' | ForEach-Object { $_.Trim() }
     $result = @{}
+    $modules = Get-ChildItem -Path $srcPath -Directory
 
-    Get-ChildItem -Path $srcPath -Directory | ForEach-Object {
-        $module = $_
-
-        Get-ChildItem -Path $module.FullName -Directory | Where-Object { 
+    foreach($module in $modules) {
+        if ($skippedModules -contains $module.Name) {
+            Write-Warning "Skipping module: $($module.Name) as it is in the skipped modules list."
+            continue
+        }
+        $subModules = Get-ChildItem -Path $module.FullName -Directory | Where-Object { 
             $_.Name -like '*.autorest'
-        } | ForEach-Object {
-            $subModule = $_
-            
+        }
+        foreach ($subModule in $subModules) {
+            $tspPath = Join-Path $subModule.FullName 'tsp-location.yaml'
+            if (Test-Path $tspPath){
+                Write-Warning "tsp-location.yaml found in $($subModule.FullName), skipping."
+                continue
+            }
+                       
             $readmePath = Join-Path $subModule.FullName 'README.md'
 
             if (Test-Path $readmePath) {
@@ -66,18 +74,19 @@ function Write-Matrix {
         [string]$RepoRoot
     )
 
-    Write-Host "$VariableName module groups: $($GroupedModules.Count)"
-    $GroupedModules | ForEach-Object { $_ -join ', ' } | ForEach-Object { Write-Host $_ }
-
+    Write-Host "##[group]$VariableName module groups: $($GroupedModules.Count)"
     $targets = @{}
     $MatrixStr = ""
     $index = 0
     foreach ($modules in $GroupedModules) {
         $key = ($index + 1).ToString() + "-" + $modules.Count
         $MatrixStr = "$MatrixStr,'$key':{'MatrixKey':'$key'}"
         $targets[$key] = $modules
+        $moduleNamesStr = $modules -join ', '
+        Write-Host "$key : $moduleNamesStr"
         $index++
     }
+    Write-Host "##[endgroup]"
 
     if ($MatrixStr -and $MatrixStr.Length -gt 1) {
         $MatrixStr = $MatrixStr.Substring(1)
@@ -91,6 +100,7 @@ function Write-Matrix {
     }
     $targetsOutputFile = Join-Path $targetsOutputDir "$VariableName.json"
     $targets | ConvertTo-Json -Depth 5 | Out-File -FilePath $targetsOutputFile -Encoding utf8
+    Write-Host
 }
 
 function Get-Targets {

.azure-pipelines/batch-generation.yml

@@ -1,3 +1,9 @@
+parameters:
+- name: TestChangedModulesOnly
+  displayName: 'Only run tests on modules that are changed by regeneration'
+  type: string
+  default: 'True'
+
 variables:
   IntermediateStepTimeoutInMinutes: 30
   GenerateTimeoutInMinutes: 120
@@ -78,6 +84,8 @@ stages:
           script: |
             $prepareModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'prepare.ps1'
             & $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParallelGenerateJobs }}"
+        env:
+          SKIPPED_MODULES: $(SkippedModules)
 
       - task: PublishPipelineArtifact@1
         displayName: 'Upload generated targets'
@@ -212,7 +220,7 @@ stages:
               -ChangedFiles $changedFiles `
               -RepoRoot "$(Build.SourcesDirectory)"
         env:
-            RUN_TEST_ON_ALL_MODULES: $(RUN_TEST_ON_ALL_MODULES)
+            TEST_CHANGED_MODULES_ONLY: ${{ parameters.TestChangedModulesOnly }}
 
       - task: PublishPipelineArtifact@1
         displayName: 'Upload filtered targets'

.azure-pipelines/live-test.yml

@@ -13,8 +13,7 @@ parameters:
     default: ' '
 
 variables:
-  os_windows_2019: windows-2019
-  os_windows_2022: windows-2022
+  os_windows: windows-latest
   os_linux: ubuntu-latest
   os_macos: macOS-latest
   win_ps_5_1: 5.1
@@ -36,8 +35,8 @@ stages:
   jobs:
   - template: util/live-test-steps.yml
     parameters:
-      name: 'win_ps_5_1_win_2019'
-      vmImage: ${{ variables.os_windows_2019 }}
+      name: 'ps_5_1_windows'
+      vmImage: ${{ variables.os_windows }}
       osType: 'Windows'
       psVersion: ${{ variables.win_ps_5_1 }}
       dotnetVersion: ${{ variables.dotnet_sdk_8 }}
@@ -46,18 +45,8 @@ stages:
 
   - template: util/live-test-steps.yml
     parameters:
-      name: 'win_ps_5_1_win_2022'
-      vmImage: ${{ variables.os_windows_2022 }}
-      osType: 'Windows'
-      psVersion: ${{ variables.win_ps_5_1 }}
-      dotnetVersion: ${{ variables.dotnet_sdk_8 }}
-      galleryName: ${{ parameters.GalleryName }}
-      pipelineBuildId: ${{ parameters.PipelineBuildId }}
-
-  - template: util/live-test-steps.yml
-    parameters:
-      name: 'ps_7_4_x_win_2019'
-      vmImage: ${{ variables.os_windows_2019 }}
+      name: 'ps_7_4_x_windows'
+      vmImage: ${{ variables.os_windows }}
       osType: 'Windows'
       psVersion: ${{ variables.ps_7_4_x }}
       dotnetVersion: ${{ variables.dotnet_sdk_8 }}
@@ -66,8 +55,8 @@ stages:
 
   - template: util/live-test-steps.yml
     parameters:
-      name: 'ps_7_5_x_win_2019'
-      vmImage: ${{ variables.os_windows_2019 }}
+      name: 'ps_7_5_x_windows'
+      vmImage: ${{ variables.os_windows }}
       osType: 'Windows'
       psVersion: ${{ variables.ps_7_5_x }}
       dotnetVersion: ${{ variables.dotnet_sdk_9 }}
@@ -76,8 +65,8 @@ stages:
 
   - template: util/live-test-steps.yml
     parameters:
-      name: 'ps_latest_win_2019'
-      vmImage: ${{ variables.os_windows_2019 }}
+      name: 'ps_latest_windows'
+      vmImage: ${{ variables.os_windows }}
       osType: 'Windows'
       psVersion: ${{ variables.ps_latest }}
       dotnetVersion: ${{ variables.dotnet_sdk_9 }}
@@ -86,88 +75,64 @@ stages:
 
   - template: util/live-test-steps.yml
     parameters:
-      name: 'ps_7_4_x_win_2022'
-      vmImage: ${{ variables.os_windows_2022 }}
-      osType: 'Windows'
+      name: 'ps_7_4_x_linux'
+      vmImage: ${{ variables.os_linux }}
+      osType: 'Linux'
       psVersion: ${{ variables.ps_7_4_x }}
       dotnetVersion: ${{ variables.dotnet_sdk_8 }}
       galleryName: ${{ parameters.GalleryName }}
       pipelineBuildId: ${{ parameters.PipelineBuildId }}
 
   - template: util/live-test-steps.yml
     parameters:
-      name: 'ps_7_5_x_win_2022'
-      vmImage: ${{ variables.os_windows_2022 }}
-      osType: 'Windows'
+      name: 'ps_7_5_x_linux'
+      vmImage: ${{ variables.os_linux }}
+      osType: 'Linux'
       psVersion: ${{ variables.ps_7_5_x }}
       dotnetVersion: ${{ variables.dotnet_sdk_9 }}
       galleryName: ${{ parameters.GalleryName }}
       pipelineBuildId: ${{ parameters.PipelineBuildId }}
 
   - template: util/live-test-steps.yml
     parameters:
-      name: 'ps_latest_win_2022'
-      vmImage: ${{ variables.os_windows_2022 }}
-      osType: 'Windows'
+      name: 'ps_latest_linux'
+      vmImage: ${{ variables.os_linux }}
+      osType: 'Linux'
       psVersion: ${{ variables.ps_latest }}
       dotnetVersion: ${{ variables.dotnet_sdk_9 }}
       galleryName: ${{ parameters.GalleryName }}
       pipelineBuildId: ${{ parameters.PipelineBuildId }}
 
   - template: util/live-test-steps.yml
     parameters:
-      name: 'ps_7_4_x_linux'
-      vmImage: ${{ variables.os_linux }}
-      osType: 'Linux'
+      name: 'ps_7_4_x_macos'
+      vmImage: ${{ variables.os_macos }}
+      osType: 'MacOS'
       psVersion: ${{ variables.ps_7_4_x }}
       dotnetVersion: ${{ variables.dotnet_sdk_8 }}
       galleryName: ${{ parameters.GalleryName }}
       pipelineBuildId: ${{ parameters.PipelineBuildId }}
 
   - template: util/live-test-steps.yml
     parameters:
-      name: 'ps_7_5_x_linux'
-      vmImage: ${{ variables.os_linux }}
-      osType: 'Linux'
+      name: 'ps_7_5_x_macos'
+      vmImage: ${{ variables.os_macos }}
+      osType: 'MacOS'
       psVersion: ${{ variables.ps_7_5_x }}
       dotnetVersion: ${{ variables.dotnet_sdk_9 }}
       galleryName: ${{ parameters.GalleryName }}
       pipelineBuildId: ${{ parameters.PipelineBuildId }}
 
   - template: util/live-test-steps.yml
     parameters:
-      name: 'ps_latest_linux'
-      vmImage: ${{ variables.os_linux }}
-      osType: 'Linux'
+      name: 'ps_latest_macos'
+      vmImage: ${{ variables.os_macos }}
+      osType: 'MacOS'
       psVersion: ${{ variables.ps_latest }}
       dotnetVersion: ${{ variables.dotnet_sdk_9 }}
       galleryName: ${{ parameters.GalleryName }}
       pipelineBuildId: ${{ parameters.PipelineBuildId }}
 
-  # - template: util/live-test-steps.yml
-  #   parameters:
-  #     name: 'ps_7_4_x_macOS'
-  #     vmImage: ${{ variables.os_macos }}
-  #     osType: 'MacOS'
-  #     psVersion: ${{ variables.ps_7_4_x }}
-  #     dotnetVersion: ${{ variables.dotnet_sdk_8 }}
-
-  # - template: util/live-test-steps.yml
-  #   parameters:
-  #     name: 'ps_7_5_x_macOS'
-  #     vmImage: ${{ variables.os_macos }}
-  #     osType: 'MacOS'
-  #     psVersion: ${{ variables.ps_7_5_x }}
-  #     dotnetVersion: ${{ variables.dotnet_sdk_9 }}
-
-  # - template: util/live-test-steps.yml
-  #   parameters:
-  #     name: 'ps_latest_macOS'
-  #     vmImage: ${{ variables.os_macos }}
-  #     osType: 'MacOS'
-  #     psVersion: ${{ variables.ps_latest }}
-  #     dotnetVersion: ${{ variables.dotnet_sdk_9 }}
-
 - stage: Report
   displayName: Send Report
   condition: always()