From da6c9aaefad283da8fd286339a909c98db4275ed Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 00:10:12 +1000
Subject: [PATCH 01/33] Batch generation
---
.../PipelineSteps/BatchGeneration/prepare.ps1 | 81 +++++++++++++++++++
.azure-pipelines/batch-generation.yml | 62 ++++++++++++++
.azure-pipelines/test.ps1 | 9 +++
3 files changed, 152 insertions(+)
create mode 100644 .azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
create mode 100644 .azure-pipelines/batch-generation.yml
create mode 100644 .azure-pipelines/test.ps1
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
new file mode 100644
index 000000000000..52a99ae90721
--- /dev/null
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
@@ -0,0 +1,81 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+ [string]$RepoRoot,
+ [int]$MaxParallelJobs = 3
+)
+
+$moduleRoot = Join-Path $RepoRoot 'src'
+$subModules = @()
+
+Get-ChildItem -Path $moduleRoot -Directory | ForEach-Object {
+ $module = $_
+ Get-ChildItem -Path $module.FullName -Directory | Where-Object {
+ $_.Name -like '*.autorest'
+ } | ForEach-Object {
+ $sub_module = $_
+ $subModules += ,@($module.Name, $sub_module.Name)
+ }
+}
+
+$subModules = @(
+ # V3
+ @("Cdn","Cdn.Autorest"),
+ @("ImageBuilder", "ImageBuilder.Autorest"),
+
+ # V4
+ @("Chaos", "Chaos.Autorest"),
+ @("DeviceRegistry", "DeviceRegistry.Autorest"),
+ @("Astro", "Astro.Autorest"),
+
+ # V4 Multi sub-modules
+ @("Communication","EmailService.Autorest"),
+ @("Communication", "EmailServicedata.Autorest")
+)
+
+Write-Host "Total matched sub modules: $($subModules.Count)"
+
+function Split-List {
+ param (
+ [array]$subModules,
+ [int]$maxParallelJobs
+ )
+
+ $count = $subModules.Count
+ $n = [Math]::Min($count, $maxParallelJobs)
+ if ($n -eq 0) {
+ return @()
+ }
+
+ $result = @()
+ $sizePerGroup = [Math]::Ceiling($count / $n)
+
+ for ($i = 0; $i -lt $count; $i += $sizePerGroup) {
+ $group = $subModules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
+ $result += ,$group
+ }
+
+ return $result
+}
+
+$devidedSubModules = Split-List -subModules $subModules -maxParallelJobs $MaxParallelJobs
+
+Write-Host "Total matched devides: $($devidedSubModules.Count)"
+
+$index = 0
+foreach ($subModules in $devidedSubModules) {
+ Write-Host "Outer Group ${index}:"
+ $subIndex = 0
+ foreach ($subModule in $subModules) {
+ Write-Host "Inner Group ${subIndex}: $($subModule -join ',')"
+ $subIndex++
+ }
+
+ $moduleNames = $subModules | ForEach-Object { $_[0] }
+
+ $MatrixStr="$MatrixStr,'" + $($index + 1) + "-" + $($subModules.Count) + "':{'Target':$($moduleNames -join ',')}"
+
+ $index++
+}
+
+$MatrixStr=$MatrixStr.Substring(1)
+Write-Host "##vso[task.setVariable variable=Targets;isOutput=true]{$MatrixStr}"
\ No newline at end of file
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
new file mode 100644
index 000000000000..e2ad2ccb4e60
--- /dev/null
+++ b/.azure-pipelines/batch-generation.yml
@@ -0,0 +1,62 @@
+variables:
+
+ BuildTimeoutInMinutes: 120
+ AnalysisTimeoutInMinutes: 120
+ MaxParallelJobs: 3
+
+trigger: none
+
+jobs:
+- job: prepare
+ displayName: Generate Matrix
+ pool: pool-ubuntu-2004
+ steps:
+ - checkout: self
+ persistCredentials: true
+ clean: true
+ ref: refs/heads/main
+
+ - task: PowerShell@2
+ displayName: 'Create Batch Generation Branch'
+ inputs:
+ targetType: inline
+ script: |
+ $newBranch = "batch-generation/branch-$(Build.BuildId)"
+ git checkout -b $newBranch
+ git push origin $newBranch
+
+ - task: PowerShell@2
+ name: mtrx
+ displayName: 'Generate Matrix'
+ inputs:
+ targetType: inline
+ script: |
+ $prepareModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'prepare.ps1'
+ & $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParallelJobs }}"
+
+
+- job: build
+ displayName: "Build:"
+ dependsOn: prepare
+ timeoutInMinutes: 120
+ pool: pool-windows-2019
+ strategy:
+ matrix: $[ dependencies.prepare.outputs['mtrx.Targets'] ]
+ maxParallel: 15
+
+ steps:
+ - checkout: self
+ persistCredentials: true
+ clean: true
+ ref: "batch-generation/branch-$(Build.BuildId)"
+
+ - task: PowerShell@2
+ name: build
+ displayName: 'Build Targets'
+ inputs:
+ targetType: inline
+ script: |
+ Write-Host "Building targets: ${Target}"
+ $buildModulesPath = Join-Path "$(Build.SourcesDirectory)" 'tools' 'BuildScripts' 'BuildModules.ps1'
+ & $buildModulesPath -TargetModule $($Target -split ',')
+
\ No newline at end of file
diff --git a/.azure-pipelines/test.ps1 b/.azure-pipelines/test.ps1
new file mode 100644
index 000000000000..450da14c3360
--- /dev/null
+++ b/.azure-pipelines/test.ps1
@@ -0,0 +1,9 @@
+
+$RepoRoot = Get-Location
+$buildProjPath = Join-Path $RepoRoot 'build.proj'
+
+$filesChangedOutputPath = Join-Path $RepoRoot 'artifacts' 'FilesChanged.txt'
+$subTasksFilePath = Join-Path $RepoRoot 'artifacts' 'SubTasksFile.txt'
+$IsSecurityCheck = $null
+
+dotnet msbuild $buildProjPath /t:FilterBuild "/p:FilesChangedOutputPath=$FilesChangedOutputPath;SubTasksFilePath=$SubTasksFilePath;IsSecurityCheck=$IsSecurityCheck"
From e6ad771f0469ce2dc24a4465cd0f9a46e9a81698 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 00:45:11 +1000
Subject: [PATCH 02/33] fix
---
.azure-pipelines/batch-generation.yml | 28 +++++++++++++++++++--------
1 file changed, 20 insertions(+), 8 deletions(-)
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index e2ad2ccb4e60..0fc6c960cdd2 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -3,6 +3,7 @@ variables:
BuildTimeoutInMinutes: 120
AnalysisTimeoutInMinutes: 120
MaxParallelJobs: 3
+ GenerationBranch: batch-generation/branch-$(Build.BuildId)
trigger: none
@@ -14,16 +15,14 @@ jobs:
- checkout: self
persistCredentials: true
clean: true
- ref: refs/heads/main
- task: PowerShell@2
displayName: 'Create Batch Generation Branch'
inputs:
targetType: inline
script: |
- $newBranch = "batch-generation/branch-$(Build.BuildId)"
- git checkout -b $newBranch
- git push origin $newBranch
+ git checkout -b "${{ variables.GenerationBranch }}"
+ git push origin "${{ variables.GenerationBranch }}"
- task: PowerShell@2
name: mtrx
@@ -48,7 +47,6 @@ jobs:
- checkout: self
persistCredentials: true
clean: true
- ref: "batch-generation/branch-$(Build.BuildId)"
- task: PowerShell@2
name: build
@@ -56,7 +54,21 @@ jobs:
inputs:
targetType: inline
script: |
+ git fetch origin "${{ variables.GenerationBranch }}"
+ git checkout "${{ variables.GenerationBranch }}"
+
Write-Host "Building targets: ${Target}"
- $buildModulesPath = Join-Path "$(Build.SourcesDirectory)" 'tools' 'BuildScripts' 'BuildModules.ps1'
- & $buildModulesPath -TargetModule $($Target -split ',')
-
\ No newline at end of file
+
+ $modules = $Target -split ','
+
+ foreach ($module in $modules) {
+ $folderPath = Join-Path $(Build.SourcesDirectory) 'src' $module
+ $filePath = Join-Path $folderPath "newfile.txt"
+ "This is a new file in $folder" | Out-File -FilePath $filePath -Encoding UTF8
+ }
+
+ git add .
+ git commit -m "Build ${Target}"
+ git push origin "${{ variables.GenerationBranch }}"
+
+ workingDirectory: $(Build.SourcesDirectory)
\ No newline at end of file
From fbbb8650ad972ea7a354b1a5943e5fcc686b3f08 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 00:47:11 +1000
Subject: [PATCH 03/33] fix
---
.azure-pipelines/batch-generation.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 0fc6c960cdd2..ad41eb24acb8 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -71,4 +71,4 @@ jobs:
git commit -m "Build ${Target}"
git push origin "${{ variables.GenerationBranch }}"
- workingDirectory: $(Build.SourcesDirectory)
\ No newline at end of file
+ workingDirectory: $(Build.SourcesDirectory)
\ No newline at end of file
From 203f4fa8f28fc5a06eb269e4ae3131100124ef83 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 01:21:16 +1000
Subject: [PATCH 04/33] fix
---
.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1 | 5 ++---
.azure-pipelines/batch-generation.yml | 4 +++-
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
index 52a99ae90721..c9ca5ba6e49f 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
@@ -71,9 +71,8 @@ foreach ($subModules in $devidedSubModules) {
}
$moduleNames = $subModules | ForEach-Object { $_[0] }
-
- $MatrixStr="$MatrixStr,'" + $($index + 1) + "-" + $($subModules.Count) + "':{'Target':$($moduleNames -join ',')}"
-
+ $moduleNamesStr = $moduleNames -join ','
+ $MatrixStr="$MatrixStr,'" + ($index + 1) + "-" + $subModules.Count + "':{'Target':'$moduleNamesStr'}"
$index++
}
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index ad41eb24acb8..7a67a6e284dd 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -10,7 +10,7 @@ trigger: none
jobs:
- job: prepare
displayName: Generate Matrix
- pool: pool-ubuntu-2004
+ pool: pool-windows-2019
steps:
- checkout: self
persistCredentials: true
@@ -21,6 +21,8 @@ jobs:
inputs:
targetType: inline
script: |
+ git checkout main
+ git pull origin main
git checkout -b "${{ variables.GenerationBranch }}"
git push origin "${{ variables.GenerationBranch }}"
From 437b916d9b2b81bcf911b3747a4220f01fe5a287 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 01:39:10 +1000
Subject: [PATCH 05/33] fix 2
---
.azure-pipelines/batch-generation.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 7a67a6e284dd..fb4f1ba0ecff 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -32,7 +32,8 @@ jobs:
inputs:
targetType: inline
script: |
- $prepareModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'prepare.ps1'
+ $pathParts = @("$(Build.SourcesDirectory)", ".azure-pipelines", "PipelineSteps", "BatchGeneration", "prepare.ps1")
+ $prepareModulesPath = $pathParts -join '\'
& $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParallelJobs }}"
From 84f30884de374fed01ea04af20f351f860c3a8f7 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 01:49:38 +1000
Subject: [PATCH 06/33] fix 3
---
.azure-pipelines/batch-generation.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index fb4f1ba0ecff..2ded77e3a8c4 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -31,9 +31,9 @@ jobs:
displayName: 'Generate Matrix'
inputs:
targetType: inline
+ pwsh: true
script: |
- $pathParts = @("$(Build.SourcesDirectory)", ".azure-pipelines", "PipelineSteps", "BatchGeneration", "prepare.ps1")
- $prepareModulesPath = $pathParts -join '\'
+ $prepareModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'prepare.ps1'
& $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParallelJobs }}"
From 9feb53b8db05f0ff727452a6b4610ed4431e7cf8 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 02:23:46 +1000
Subject: [PATCH 07/33] fix 4
---
.azure-pipelines/batch-generation.yml | 2 --
1 file changed, 2 deletions(-)
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 2ded77e3a8c4..0940f796b08f 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -21,8 +21,6 @@ jobs:
inputs:
targetType: inline
script: |
- git checkout main
- git pull origin main
git checkout -b "${{ variables.GenerationBranch }}"
git push origin "${{ variables.GenerationBranch }}"
From ff8cba4fe44049298a63418964b54961de45ca22 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 02:44:36 +1000
Subject: [PATCH 08/33] fix 5
---
.azure-pipelines/batch-generation.yml | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 0940f796b08f..f3d4e751e4e1 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -14,7 +14,7 @@ jobs:
steps:
- checkout: self
persistCredentials: true
- clean: true
+ fetchTags: false
- task: PowerShell@2
displayName: 'Create Batch Generation Branch'
@@ -47,20 +47,21 @@ jobs:
steps:
- checkout: self
persistCredentials: true
- clean: true
+ fetchTags: false
- task: PowerShell@2
name: build
displayName: 'Build Targets'
inputs:
targetType: inline
+ pwsh: true
script: |
git fetch origin "${{ variables.GenerationBranch }}"
git checkout "${{ variables.GenerationBranch }}"
- Write-Host "Building targets: ${Target}"
+ Write-Host "Building targets: $(Target)"
- $modules = $Target -split ','
+ $modules = $(Target) -split ','
foreach ($module in $modules) {
$folderPath = Join-Path $(Build.SourcesDirectory) 'src' $module
@@ -69,7 +70,7 @@ jobs:
}
git add .
- git commit -m "Build ${Target}"
+ git commit -m "Build $(Target)"
git push origin "${{ variables.GenerationBranch }}"
workingDirectory: $(Build.SourcesDirectory)
\ No newline at end of file
From afa7d3435b747e2ac32ee0b43fa3b7c141e958c9 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 13:55:07 +1000
Subject: [PATCH 09/33] fix 6
---
.azure-pipelines/batch-generation.yml | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index f3d4e751e4e1..3c979e84fb5b 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -59,9 +59,10 @@ jobs:
git fetch origin "${{ variables.GenerationBranch }}"
git checkout "${{ variables.GenerationBranch }}"
- Write-Host "Building targets: $(Target)"
+ $target = "$(Target)"
+ Write-Host "Building targets: $target"
- $modules = $(Target) -split ','
+ $modules = $target -split ','
foreach ($module in $modules) {
$folderPath = Join-Path $(Build.SourcesDirectory) 'src' $module
@@ -70,7 +71,7 @@ jobs:
}
git add .
- git commit -m "Build $(Target)"
+ git commit -m "Build $target"
git push origin "${{ variables.GenerationBranch }}"
workingDirectory: $(Build.SourcesDirectory)
\ No newline at end of file
From 3243b24cd6c9a14b15e5ed506ce9d8e191d15069 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 21:08:47 +1000
Subject: [PATCH 10/33] fix 7
---
.azure-pipelines/batch-generation.yml | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 3c979e84fb5b..20303b12efd5 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -49,6 +49,8 @@ jobs:
persistCredentials: true
fetchTags: false
+ - template: util/get-github-pat-steps.yml
+
- task: PowerShell@2
name: build
displayName: 'Build Targets'
@@ -69,6 +71,10 @@ jobs:
$filePath = Join-Path $folderPath "newfile.txt"
"This is a new file in $folder" | Out-File -FilePath $filePath -Encoding UTF8
}
+
+ git config user.email "65331932+azure-powershell-bot@users.noreply.github.com";
+ git config user.name "azure-powershell-bot";
+ git remote set-url origin 'https://azure-powershell-bot:$(GithubToken)@github.com/Azure/azure-powershell.git';
git add .
git commit -m "Build $target"
From 9e274af80c190758d85cf52426d327bdfecdd344 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 22:17:36 +1000
Subject: [PATCH 11/33] fix 8
---
.azure-pipelines/batch-generation.yml | 3 +++
1 file changed, 3 insertions(+)
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 20303b12efd5..4c4563c83450 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -48,6 +48,9 @@ jobs:
- checkout: self
persistCredentials: true
fetchTags: false
+
+ - checkout: azure-powershell-devops
+ path: "azure-powershell-devops
- template: util/get-github-pat-steps.yml
From 6f21d1cf40217ab52753ea7f1340d03b294f7053 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 22:33:04 +1000
Subject: [PATCH 12/33] fix 9
---
.azure-pipelines/batch-generation.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 4c4563c83450..1409fc50b306 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -50,7 +50,7 @@ jobs:
fetchTags: false
- checkout: azure-powershell-devops
- path: "azure-powershell-devops
+ path: "azure-powershell-devops"
- template: util/get-github-pat-steps.yml
From dd5761cfd3416be5b887d8541e5b36421c69053e Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 22:34:46 +1000
Subject: [PATCH 13/33] fix 10
---
.azure-pipelines/batch-generation.yml | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 1409fc50b306..902c53856c87 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -1,5 +1,10 @@
+resources:
+ repositories:
+ - repository: azure-powershell-devops
+ type: git
+ name: azure-powershell-devops
+
variables:
-
BuildTimeoutInMinutes: 120
AnalysisTimeoutInMinutes: 120
MaxParallelJobs: 3
From 039098f1ceffdd851b786760dacdd3a06bb839a3 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 22:52:16 +1000
Subject: [PATCH 14/33] token
---
.azure-pipelines/batch-generation.yml | 38 ++++++++++++++++++---------
1 file changed, 25 insertions(+), 13 deletions(-)
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 902c53856c87..7298e330eb00 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -1,9 +1,9 @@
-resources:
- repositories:
- - repository: azure-powershell-devops
- type: git
- name: azure-powershell-devops
-
+# resources:
+# repositories:
+# - repository: azure-powershell-devops
+# type: git
+# name: azure-powershell-devops
+
variables:
BuildTimeoutInMinutes: 120
AnalysisTimeoutInMinutes: 120
@@ -17,17 +17,29 @@ jobs:
displayName: Generate Matrix
pool: pool-windows-2019
steps:
- - checkout: self
- persistCredentials: true
- fetchTags: false
+ # - checkout: self
+ # persistCredentials: true
+ # fetchTags: false
+
+ # - task: PowerShell@2
+ # displayName: 'Create Batch Generation Branch'
+ # inputs:
+ # targetType: inline
+ # script: |
+ # git checkout -b "${{ variables.GenerationBranch }}"
+ # git push origin "${{ variables.GenerationBranch }}"
- task: PowerShell@2
displayName: 'Create Batch Generation Branch'
inputs:
targetType: inline
+ pwsh: true
script: |
- git checkout -b "${{ variables.GenerationBranch }}"
- git push origin "${{ variables.GenerationBranch }}"
+ $orgUrl = "$(System.CollectionUri)"
+ $project = "$(System.TeamProject)"
+ $repo = "$(Build.Repository.Name)"
+ $newBranch = "refs/heads/batch-generation"
+ $sourceBranch = "refs/heads/main"
- task: PowerShell@2
name: mtrx
@@ -54,8 +66,8 @@ jobs:
persistCredentials: true
fetchTags: false
- - checkout: azure-powershell-devops
- path: "azure-powershell-devops"
+ # - checkout: azure-powershell-devops
+ # path: "azure-powershell-devops"
- template: util/get-github-pat-steps.yml
From eea76cc6cbf952f3ee6e34748dba1b5895f0b50a Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 23:27:21 +1000
Subject: [PATCH 15/33] create branch
---
.../BatchGeneration/create-branch.ps1 | 22 ++++++++++++++
.../PipelineSteps/BatchGeneration/prepare.ps1 | 2 +-
.azure-pipelines/batch-generation.yml | 30 +++++--------------
3 files changed, 30 insertions(+), 24 deletions(-)
create mode 100644 .azure-pipelines/PipelineSteps/BatchGeneration/create-branch.ps1
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/create-branch.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/create-branch.ps1
new file mode 100644
index 000000000000..450921b95f1e
--- /dev/null
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/create-branch.ps1
@@ -0,0 +1,22 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+ [string]$Owner,
+ [string]$Repo,
+ [string]$BaseBranch,
+ [string]$NewBranch,
+ [string]$Token
+)
+
+$headers = @{ Authorization = "Bearer $Token"; "User-Agent" = "ADO-Pipeline" }
+$branchInfo = Invoke-RestMethod -Uri "https://api.github.com/repos/$Owner/$Repo/git/ref/heads/$BaseBranch" -Headers $headers
+$sha = $branchInfo.object.sha
+
+$body = @{
+ ref = "refs/heads/$NewBranch"
+ sha = $sha
+} | ConvertTo-Json
+
+Invoke-RestMethod -Uri "https://api.github.com/repos/$Owner/$Repo/git/refs" `
+ -Method Post -Headers $headers -Body $body -ContentType "application/json"
+
+Write-Host "Created branch '$NewBranch' from '$BBaseBranch'"
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
index c9ca5ba6e49f..9308ccde40fa 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
@@ -77,4 +77,4 @@ foreach ($subModules in $devidedSubModules) {
}
$MatrixStr=$MatrixStr.Substring(1)
-Write-Host "##vso[task.setVariable variable=Targets;isOutput=true]{$MatrixStr}"
\ No newline at end of file
+Write-Host "##vso[task.setVariable variable=Targets;isOutput=true]{$MatrixStr}"
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 7298e330eb00..a18a0bdecfb2 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -1,9 +1,3 @@
-# resources:
-# repositories:
-# - repository: azure-powershell-devops
-# type: git
-# name: azure-powershell-devops
-
variables:
BuildTimeoutInMinutes: 120
AnalysisTimeoutInMinutes: 120
@@ -17,17 +11,7 @@ jobs:
displayName: Generate Matrix
pool: pool-windows-2019
steps:
- # - checkout: self
- # persistCredentials: true
- # fetchTags: false
-
- # - task: PowerShell@2
- # displayName: 'Create Batch Generation Branch'
- # inputs:
- # targetType: inline
- # script: |
- # git checkout -b "${{ variables.GenerationBranch }}"
- # git push origin "${{ variables.GenerationBranch }}"
+ - template: util/get-github-pat-steps.yml
- task: PowerShell@2
displayName: 'Create Batch Generation Branch'
@@ -35,11 +19,11 @@ jobs:
targetType: inline
pwsh: true
script: |
- $orgUrl = "$(System.CollectionUri)"
- $project = "$(System.TeamProject)"
- $repo = "$(Build.Repository.Name)"
- $newBranch = "refs/heads/batch-generation"
- $sourceBranch = "refs/heads/main"
+ $newBranch = "$(GenerationBranch)"
+ $token = "$(GitHubToken)"
+
+ $createBranchPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'create-branch.ps1'
+ & $prepareModulesPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch 'main' -NewBranch $newBranch -Token $token
- task: PowerShell@2
name: mtrx
@@ -100,4 +84,4 @@ jobs:
git commit -m "Build $target"
git push origin "${{ variables.GenerationBranch }}"
- workingDirectory: $(Build.SourcesDirectory)
\ No newline at end of file
+ workingDirectory: $(Build.SourcesDirectory)
From b1fc7bae1b3ac49daef2e3867d6e2473a85e7c72 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 10 Apr 2025 23:41:19 +1000
Subject: [PATCH 16/33] create branch 2
---
.azure-pipelines/batch-generation.yml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index a18a0bdecfb2..70919b53c363 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -11,6 +11,8 @@ jobs:
displayName: Generate Matrix
pool: pool-windows-2019
steps:
+ - checkout: none
+
- template: util/get-github-pat-steps.yml
- task: PowerShell@2
@@ -23,7 +25,7 @@ jobs:
$token = "$(GitHubToken)"
$createBranchPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'create-branch.ps1'
- & $prepareModulesPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch 'main' -NewBranch $newBranch -Token $token
+ & $createBranchPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch 'main' -NewBranch $newBranch -Token $token
- task: PowerShell@2
name: mtrx
From 7a89243609db79fbf92c551b40161d2b294f6a62 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 17 Apr 2025 11:38:12 +1000
Subject: [PATCH 17/33] filter
---
.../PipelineSteps/BatchGeneration/filter.ps1 | 65 +++++++
.../PipelineSteps/BatchGeneration/prepare.ps1 | 9 +-
.azure-pipelines/batch-generation.yml | 177 ++++++++++++++++--
3 files changed, 231 insertions(+), 20 deletions(-)
create mode 100644 .azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
new file mode 100644
index 000000000000..30fa562990df
--- /dev/null
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
@@ -0,0 +1,65 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+ [int]$MaxParallelJobs = 3,
+ [string[]]$ChangedFiles
+)
+
+$autorestFolders = @{}
+
+foreach ($file in $changedFiles) {
+ if ($file -match '^(src|generated)/([^/]+)/([^/]+\.autorest)/') {
+ $parent = $Matches[2]
+ $child = $Matches[3]
+ $key = "$parent/$child"
+
+ $autorestFolders[$key] = $true
+ }
+}
+
+$subModules = $autorestFolders.Keys
+Write-Host "Outer Group ${subModules}:"
+
+function Split-List {
+ param (
+ [array]$subModules,
+ [int]$maxParallelJobs
+ )
+
+ $count = $subModules.Count
+ $n = [Math]::Min($count, $maxParallelJobs)
+ if ($n -eq 0) {
+ return @()
+ }
+
+ $result = @()
+ $sizePerGroup = [Math]::Ceiling($count / $n)
+
+ for ($i = 0; $i -lt $count; $i += $sizePerGroup) {
+ $group = $subModules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
+ $result += ,$group
+ }
+
+ return $result
+}
+
+$devidedSubModules = Split-List -subModules $subModules -maxParallelJobs $MaxParallelJobs
+
+$index = 0
+foreach ($subModules in $devidedSubModules) {
+ Write-Host "Outer Group ${index}:"
+ $subIndex = 0
+ foreach ($subModule in $subModules) {
+ Write-Host "Inner Group ${subIndex}: $($subModule -join ',')"
+ $subIndex++
+ }
+
+ $moduleNamesStr = $subModules -join ','
+ $key = ($index + 1).ToString() + "-" + $subModules.Count
+ $MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
+ $index++
+}
+
+if ($MatrixStr -and $MatrixStr.Length -gt 1) {
+ $MatrixStr = $MatrixStr.Substring(1)
+}
+Write-Host "##vso[task.setVariable variable=analyzeTargets;isOutput=true]{$MatrixStr}"
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
index 9308ccde40fa..7b1b5c67f273 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
@@ -72,9 +72,12 @@ foreach ($subModules in $devidedSubModules) {
$moduleNames = $subModules | ForEach-Object { $_[0] }
$moduleNamesStr = $moduleNames -join ','
- $MatrixStr="$MatrixStr,'" + ($index + 1) + "-" + $subModules.Count + "':{'Target':'$moduleNamesStr'}"
+ $key = ($index + 1).ToString() + "-" + $subModules.Count
+ $MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
$index++
}
-$MatrixStr=$MatrixStr.Substring(1)
-Write-Host "##vso[task.setVariable variable=Targets;isOutput=true]{$MatrixStr}"
+if ($MatrixStr -and $MatrixStr.Length -gt 1) {
+ $MatrixStr = $MatrixStr.Substring(1)
+}
+Write-Host "##vso[task.setVariable variable=buildTargets;isOutput=true]{$MatrixStr}"
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 70919b53c363..ae44887dbd9c 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -1,17 +1,23 @@
variables:
BuildTimeoutInMinutes: 120
AnalysisTimeoutInMinutes: 120
- MaxParallelJobs: 3
+ MaxParalleBuildJobs: 3
GenerationBranch: batch-generation/branch-$(Build.BuildId)
+ MaxParalleAnalyzeJobs: 3
+ MaxParalleWindowsTestJobs: 3
+ MaxParalleLinuxTestJobs: 3
+ MaxParalleMacTestJobs: 3
trigger: none
jobs:
- job: prepare
- displayName: Generate Matrix
+ displayName: Generate Build Targets Matrix
pool: pool-windows-2019
steps:
- - checkout: none
+
+ # TODO: (Bernard) Uncomment the no checkout step after automatically install repo into agnets
+ # - checkout: none
- template: util/get-github-pat-steps.yml
@@ -29,13 +35,13 @@ jobs:
- task: PowerShell@2
name: mtrx
- displayName: 'Generate Matrix'
+ displayName: 'Generate Build Targets Matrix'
inputs:
targetType: inline
pwsh: true
script: |
$prepareModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'prepare.ps1'
- & $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParallelJobs }}"
+ & $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParalleBuildJobs }}"
- job: build
@@ -44,8 +50,8 @@ jobs:
timeoutInMinutes: 120
pool: pool-windows-2019
strategy:
- matrix: $[ dependencies.prepare.outputs['mtrx.Targets'] ]
- maxParallel: 15
+ matrix: $[ dependencies.prepare.outputs['mtrx.buildTargets'] ]
+ maxParallel: ${{ variables.MaxParalleBuildJobs }}
steps:
- checkout: self
@@ -55,7 +61,7 @@ jobs:
# - checkout: azure-powershell-devops
# path: "azure-powershell-devops"
- - template: util/get-github-pat-steps.yml
+ # - template: util/get-github-pat-steps.yml
- task: PowerShell@2
name: build
@@ -70,20 +76,157 @@ jobs:
$target = "$(Target)"
Write-Host "Building targets: $target"
+ $patchDir = "$(Build.ArtifactStagingDirectory)/$(MatrixKey)"
+ New-Item -ItemType Directory -Force -Path $patchDir | Out-Null
+
$modules = $target -split ','
foreach ($module in $modules) {
- $folderPath = Join-Path $(Build.SourcesDirectory) 'src' $module
- $filePath = Join-Path $folderPath "newfile.txt"
- "This is a new file in $folder" | Out-File -FilePath $filePath -Encoding UTF8
- }
+ $modulePath = Join-Path $(Build.SourcesDirectory) 'src' $module
+ $subModulePath = Get-ChildItem -Path $modulePath -Directory |
+ Where-Object { $_.Name -like '*.autorest' }
+ foreach ($subModule in $subModulePath) {
+ $filePath = Join-Path $subModule "newfile.txt"
+ "This is a new file in $folder" | Out-File -FilePath $filePath -Encoding UTF8
+ }
+ }
- git config user.email "65331932+azure-powershell-bot@users.noreply.github.com";
- git config user.name "azure-powershell-bot";
- git remote set-url origin 'https://azure-powershell-bot:$(GithubToken)@github.com/Azure/azure-powershell.git';
+ # git config user.email "65331932+azure-powershell-bot@users.noreply.github.com";
+ # git config user.name "azure-powershell-bot";
+ # git remote set-url origin 'https://azure-powershell-bot:$(GithubToken)@github.com/Azure/azure-powershell.git';
+
+ $random = Get-Random -Minimum 10 -Maximum 121
+ Write-Host "Sleeping for $random seconds..."
+ Start-Sleep -Seconds $random
git add .
- git commit -m "Build $target"
- git push origin "${{ variables.GenerationBranch }}"
+ # git commit -m "Build $target"
+ # git push origin "${{ variables.GenerationBranch }}"
+
+
+ $patchPath = "$patchDir/changed.patch"
+ git diff --cached > $patchPath
+
+ Get-Content -Path $patchPath
workingDirectory: $(Build.SourcesDirectory)
+
+ - task: PublishPipelineArtifact@1
+ displayName: 'Save patch'
+ inputs:
+ targetPath: '$(Build.ArtifactStagingDirectory)/$(MatrixKey)'
+ artifact: 'patch-$(MatrixKey)'
+ condition: always()
+
+- job: filter
+ displayName: 'Filter Changed Modules'
+ dependsOn: build
+ steps:
+ - checkout: self
+ persistCredentials: true
+ fetchTags: false
+
+ - task: PowerShell@2
+ name: checkout
+ displayName: 'Checkout Generation Branch'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ git fetch origin "${{ variables.GenerationBranch }}"
+ git checkout "${{ variables.GenerationBranch }}"
+
+ - download: current
+ patterns: '**/*.patch'
+ displayName: 'Download all .patch artifacts'
+
+ - task: PowerShell@2
+ name: apply
+ displayName: 'Apply all patches'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ git config user.email "65331932+azure-powershell-bot@users.noreply.github.com"
+ git config user.name "azure-powershell-bot"
+
+ Write-Host "Applying all .patch files to ${{ variables.GenerationBranch }}..."
+ $patchFiles = Get-ChildItem -Path "$env:PIPELINE_WORKSPACE" -Recurse -Filter *.patch
+
+ # $patchFiles = Get-ChildItem -Path "$env:PIPELINE_WORKSPACE" -Recurse -Filter *.patch |
+ # Where-Object { $_.Directory.Parent.FullName -eq $env:PIPELINE_WORKSPACE }
+
+ foreach ($patch in $patchFiles) {
+ Write-Host "Applying patch: $($patch.FullName)"
+ git apply --whitespace=fix "$($patch.FullName)"
+ }
+
+ git add .
+ git commit -m "Apply matrix job patches to ${{ variables.GenerationBranch }}"
+ git push origin "${{ variables.GenerationBranch }}"
+
+ - task: PowerShell@2
+ name: mtrx
+ displayName: 'Generate Test Targets Matrix'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ $base = git merge-base HEAD origin/main
+ $changedFiles = git diff --name-only $base HEAD
+
+ $sourceBranchName = "$(Build.SourceBranch)".Replace('refs/heads/', '')
+ git checkout $sourceBranchName
+
+ $filterModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'filter.ps1'
+ & $filterModulesPath -MaxParallelJobs "${{ variables.MaxParalleAnalyzeJobs }}" -ChangedFiles $changedFiles
+
+ # - checkout: none
+
+ # - script: |
+ # git init
+ # git remote add origin https://github.com/Azure/azure-powershell.git
+ # git fetch origin main --depth=1
+ # git checkout FETCH_HEAD -- .azure-pipelines/
+
+ # displayName: 'Checkout Pipeline Templates'
+
+ # - template: util/get-github-pat-steps.yml
+
+ # - script: |
+ # $token = "$(GitHubToken)"
+ # changedModules=$(curl -s -H "Authorization: Bearer $token" \
+ # https://api.github.com/repos/Azure/azure-powershell/compare/main...${{ variables.GenerationBranch }} \
+ # | jq -r '.files[].filename' \
+ # | grep -E '^(src|generated)/[^/]+/[^/]+\.autorest/' \
+ # | sed -E 's#^(src|generated)/##' \
+ # | awk -F/ '{print $1 "/" $2}' \
+ # | sort -u)
+
+ # IFS=$'\n' read -r -d '' -a changedModulePath <<< "$changedModules"
+
+ # for path in "${changedModulePath[@]}"; do
+ # echo "$path"
+ # done
+
+- job: analyze
+ displayName: "Analyze:"
+ dependsOn: filter
+ timeoutInMinutes: 120
+ pool: pool-windows-2019
+ strategy:
+ matrix: $[ dependencies.filter.outputs['mtrx.analyzeTargets'] ]
+ maxParallel: ${{ variables.MaxParalleAnalyzeJobs }}
+
+ steps:
+ - checkout: none
+
+ - task: PowerShell@2
+ name: analyze
+ displayName: 'Analyze modules'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ Write-Host "Matrix Key: $(MatrixKey)"
+ Write-Host "Analyze Targets: $(Target)"
\ No newline at end of file
From d45661914321b4c561c1a2a68ed0c57f15af4442 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Tue, 29 Apr 2025 16:26:33 +1000
Subject: [PATCH 18/33] batch generate
---
.../batch-generate-modules.ps1 | 82 +++++++
.../PipelineSteps/BatchGeneration/filter.ps1 | 80 ++++---
.../PipelineSteps/BatchGeneration/prepare.ps1 | 119 +++++----
.../BatchGeneration/test-module.ps1 | 45 ++++
.azure-pipelines/batch-generation.yml | 226 +++++++++++++-----
.azure-pipelines/test.ps1 | 9 -
6 files changed, 418 insertions(+), 143 deletions(-)
create mode 100644 .azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
create mode 100644 .azure-pipelines/PipelineSteps/BatchGeneration/test-module.ps1
delete mode 100644 .azure-pipelines/test.ps1
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
new file mode 100644
index 000000000000..ad9ddebd6329
--- /dev/null
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
@@ -0,0 +1,82 @@
+param (
+ [string]$MatrixKey,
+ [string]$RepoRoot
+)
+
+Write-Host "Matrix Key: $MatrixKey"
+
+$generateTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generateTargets.json"
+$generateTargets = Get-Content -Path $generateTargetsOutPutFile -Raw | ConvertFrom-Json
+$moduleGroup = $generateTargets.$MatrixKey
+$sortedModuleNames = $moduleGroup.PSObject.Properties.Name | Sort-Object
+
+$AutorestOutputDir = Join-Path $RepoRoot "artifacts" "autorest"
+New-Item -ItemType Directory -Force -Path $AutorestOutputDir
+
+$sourceDirectory = Join-Path $RepoRoot "src"
+$generatedDirectory = Join-Path $RepoRoot "generated"
+$buildScriptsModulePath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildScripts.psm1'
+Import-Module $buildScriptsModulePath
+
+$results = @()
+
+foreach ($moduleName in $sortedModuleNames) {
+ Write-Host "=============================================================="
+ Write-Host "Regenerating Module: $moduleName"
+ $moduleStartTime = Get-Date
+ $moduleResult = @{
+ Module = $moduleName
+ DurationSeconds = 0
+ SubModules = @()
+ }
+
+ $subModuleNames = $moduleGroup.$moduleName
+ foreach ($subModuleName in $subModuleNames) {
+ Write-Host "Regenerating SubModule: $subModuleName"
+ $subModuleStartTime = Get-Date
+ $subModuleResult = @{
+ SubModule = $subModuleName
+ Status = "Success"
+ DurationSeconds = 0
+ Error = ""
+ }
+
+ try {
+ $generateLog = Join-Path $AutorestOutputDir $moduleName "$subModuleName.log"
+ if (Test-Path $generateLog) {
+ Remove-Item -Path $generateLog -Recurse -Force
+ }
+ New-Item -ItemType File -Force -Path $generateLog
+
+ if (-not (Update-GeneratedSubModule -ModuleRootName $moduleName -SubModuleName $subModuleName -SourceDirectory $sourceDirectory -GeneratedDirectory $generatedDirectory -GenerateLog $generateLog -IsInvokedByPipeline $true)) {
+ Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
+ Write-Warning "log can be found at $generateLog"
+ $subModuleResult.Status = "Failed"
+ $subModuleResult.Error = "Update-GeneratedSubModule function returned false."
+ }
+
+ } catch {
+ Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
+ $subModuleResult.Status = "Failed"
+ $subModuleResult.Error = $_.Exception.Message
+ } finally {
+ $subModuleEndTime = Get-Date
+ $subModuleResult.DurationSeconds = ($subModuleEndTime - $subModuleStartTime).TotalSeconds
+ $moduleResult.SubModules += $subModuleResult
+ }
+ }
+ $moduleEndTime = Get-Date
+ $moduleResult.DurationSeconds = ($moduleEndTime - $moduleStartTime).TotalSeconds
+ $results += $moduleResult
+}
+
+$ArtifactOutputDir = Join-Path $RepoRoot "artifacts"
+
+git add .
+$patchPath = Join-Path $ArtifactOutputDir "changed-$MatrixKey.patch"
+git diff --cached > $patchPath
+
+$reportPath = Join-Path $ArtifactOutputDir "GenerationReport-$MatrixKey.json"
+$results | ConvertTo-Json -Depth 5 | Out-File -FilePath $reportPath -Encoding utf8
+
+Write-Host "Build report written to $reportPath"
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
index 30fa562990df..2346e35a60af 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
@@ -1,13 +1,15 @@
[CmdletBinding(DefaultParameterSetName="AllSet")]
param (
- [int]$MaxParallelJobs = 3,
+ [int]$MaxParalleAnalyzeJobs = 3,
+ [int]$MaxParalleTestWindowsJobs = 3,
+ [int]$MaxParalleTestLinuxJobs = 3,
+ [int]$MaxParalleTestMacJobs = 3,
[string[]]$ChangedFiles
)
$autorestFolders = @{}
-
-foreach ($file in $changedFiles) {
- if ($file -match '^(src|generated)/([^/]+)/([^/]+\.autorest)/') {
+for ($i = 0; $i -lt $ChangedFiles.Count; $i++) {
+ if ($ChangedFiles[$i] -match '^(src|generated)/([^/]+)/([^/]+\.autorest)/') {
$parent = $Matches[2]
$child = $Matches[3]
$key = "$parent/$child"
@@ -16,8 +18,13 @@ foreach ($file in $changedFiles) {
}
}
-$subModules = $autorestFolders.Keys
-Write-Host "Outer Group ${subModules}:"
+$changedSubModules = $autorestFolders.Keys
+# TODO(Bernard) Remove test data after test
+# $changedSubModules = @("A", "B", "C", "D", "E", "F", "G")
+Write-Host "Chagned sub modules: "
+foreach ($subModule in $changedSubModules) {
+ Write-Host $subModule
+}
function Split-List {
param (
@@ -27,39 +34,56 @@ function Split-List {
$count = $subModules.Count
$n = [Math]::Min($count, $maxParallelJobs)
+
if ($n -eq 0) {
return @()
}
$result = @()
- $sizePerGroup = [Math]::Ceiling($count / $n)
- for ($i = 0; $i -lt $count; $i += $sizePerGroup) {
- $group = $subModules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
- $result += ,$group
+ for ($i = 0; $i -lt $n; $i++) {
+ $result += ,@()
+ }
+
+ for ($i = 0; $i -lt $count; $i++) {
+ $groupIndex = $i % $n
+ $result[$groupIndex] += $subModules[$i]
}
- return $result
+ return ,$result
}
-$devidedSubModules = Split-List -subModules $subModules -maxParallelJobs $MaxParallelJobs
+function Write-Matrix {
+ param (
+ [string]$variableName,
+ [array]$groupedSubModules
+ )
-$index = 0
-foreach ($subModules in $devidedSubModules) {
- Write-Host "Outer Group ${index}:"
- $subIndex = 0
- foreach ($subModule in $subModules) {
- Write-Host "Inner Group ${subIndex}: $($subModule -join ',')"
- $subIndex++
+ $index = 0
+ foreach ($subModules in $groupedSubModules) {
+ $moduleNamesStr = $subModules -join ','
+ $key = ($index + 1).ToString() + "-" + $subModules.Count
+ $MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
+ $index++
}
- $moduleNamesStr = $subModules -join ','
- $key = ($index + 1).ToString() + "-" + $subModules.Count
- $MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
- $index++
-}
+ if ($MatrixStr -and $MatrixStr.Length -gt 1) {
+ $MatrixStr = $MatrixStr.Substring(1)
+ }
+ Write-Host "##vso[task.setVariable variable=$variableName;isOutput=true]{$MatrixStr}"
+ Write-Host "variable=$variableName; value=$MatrixStr"
+ }
-if ($MatrixStr -and $MatrixStr.Length -gt 1) {
- $MatrixStr = $MatrixStr.Substring(1)
-}
-Write-Host "##vso[task.setVariable variable=analyzeTargets;isOutput=true]{$MatrixStr}"
+$groupedAnalyzeModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleAnalyzeJobs
+Write-Matrix -variableName 'AnalyzeTargets' -groupedSubModules $groupedAnalyzeModules
+
+$groupedTestWindowsModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestWindowsJobs
+Write-Matrix -variableName 'TestWindowsTargets' -groupedSubModules $groupedTestWindowsModules
+
+$groupedTestLinuxModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestLinuxJobs
+Write-Matrix -variableName 'TestLinuxTargets' -groupedSubModules $groupedTestLinuxModules
+
+# $groupedTestMacModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestMacJobs
+# Write-Matrix -variableName 'TestMacTargets' -groupedSubModules $groupedTestMacModules
+
+Write-Host "##vso[task.setVariable variable=TestMacTargets;isOutput=true]{}"
\ No newline at end of file
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
index 7b1b5c67f273..cf9dfd047182 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
@@ -4,43 +4,67 @@ param (
[int]$MaxParallelJobs = 3
)
-$moduleRoot = Join-Path $RepoRoot 'src'
-$subModules = @()
-
-Get-ChildItem -Path $moduleRoot -Directory | ForEach-Object {
- $module = $_
- Get-ChildItem -Path $module.FullName -Directory | Where-Object {
- $_.Name -like '*.autorest'
- } | ForEach-Object {
- $sub_module = $_
- $subModules += ,@($module.Name, $sub_module.Name)
- }
-}
+$srcPath = Join-Path $RepoRoot 'src'
+function Get-SubModuleWithAutorestV4 {
+ param (
+ [string]$srcPath
+ )
+
+ $result = @{}
-$subModules = @(
- # V3
- @("Cdn","Cdn.Autorest"),
- @("ImageBuilder", "ImageBuilder.Autorest"),
+ Get-ChildItem -Path $srcPath -Directory | ForEach-Object {
+ $module = $_
- # V4
- @("Chaos", "Chaos.Autorest"),
- @("DeviceRegistry", "DeviceRegistry.Autorest"),
- @("Astro", "Astro.Autorest"),
-
- # V4 Multi sub-modules
- @("Communication","EmailService.Autorest"),
- @("Communication", "EmailServicedata.Autorest")
-)
+ Get-ChildItem -Path $module.FullName -Directory | Where-Object {
+ $_.Name -like '*.autorest'
+ } | ForEach-Object {
+ $subModule = $_
+
+ $readmePath = Join-Path $subModule.FullName 'README.md'
+
+ if (Test-Path $readmePath) {
+ $readmeContent = Get-Content -Path $readmePath -Raw
+
+ if ($readmeContent -notmatch 'use-extension:\s+"@autorest/powershell":\s+"3.x"') {
+ if ($result.ContainsKey($module.Name)) {
+ $result[$module.Name] += $subModule.Name
+ } else {
+ $result[$module.Name] = @($subModule.Name)
+ }
+ }
+ }
+ }
+ }
-Write-Host "Total matched sub modules: $($subModules.Count)"
+ return $result
+}
+# TODO(Bernard): Use real function after test
+# $modules = Get-SubModuleWithAutorestV4 -srcPath $srcPath
+$modules = @{
+ "DeviceRegistry" = @("DeviceRegistry.Autorest")
+ "ArcGateway" = @("ArcGateway.Autorest")
+ "Chaos" = @("Chaos.Autorest")
+ "Cdn" = @("Cdn.Autorest")
+ "Communication" = @("EmailService.Autorest", "EmailServicedata.Autorest")
+ "Astro" = @("Astro.Autorest")
+ "ImageBuilder" = @("ImageBuilder.Autorest")
+}
+$modules = $modules.GetEnumerator() | ForEach-Object {
+ [PSCustomObject]@{
+ ModuleName = $_.Key
+ SubModules = ($_.Value | Sort-Object)
+ }
+} | Sort-Object -Property ModuleName
-function Split-List {
+Write-Host "Total matched modules: $($modules.Count)"
+
+function Group-List {
param (
- [array]$subModules,
+ [array]$modules,
[int]$maxParallelJobs
)
- $count = $subModules.Count
+ $count = $modules.Count
$n = [Math]::Min($count, $maxParallelJobs)
if ($n -eq 0) {
return @()
@@ -50,34 +74,43 @@ function Split-List {
$sizePerGroup = [Math]::Ceiling($count / $n)
for ($i = 0; $i -lt $count; $i += $sizePerGroup) {
- $group = $subModules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
+ $group = $modules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
$result += ,$group
}
return $result
}
-$devidedSubModules = Split-List -subModules $subModules -maxParallelJobs $MaxParallelJobs
-
-Write-Host "Total matched devides: $($devidedSubModules.Count)"
+$groupedModules = Group-List -modules $modules -maxParallelJobs $MaxParallelJobs
+Write-Host "Total module groups: $($groupedModules.Count)"
$index = 0
-foreach ($subModules in $devidedSubModules) {
- Write-Host "Outer Group ${index}:"
- $subIndex = 0
- foreach ($subModule in $subModules) {
- Write-Host "Inner Group ${subIndex}: $($subModule -join ',')"
+$generateTargets = @{}
+foreach ($moduleGroup in $groupedModules) {
+ Write-Host "##[group]Prepareing module group $($index + 1)"
+ $mergedModules = @{}
+ foreach ($moduleObj in $moduleGroup) {
+ Write-Host "Module $($moduleObj.ModuleName): $($moduleObj.SubModules -join ',')"
+ $mergedModules[$moduleObj.ModuleName] = @($moduleObj.SubModules)
$subIndex++
}
- $moduleNames = $subModules | ForEach-Object { $_[0] }
- $moduleNamesStr = $moduleNames -join ','
- $key = ($index + 1).ToString() + "-" + $subModules.Count
- $MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
+ $key = ($index + 1).ToString() + "-" + $moduleGroup.Count
+ $generateTargets[$key] = $mergedModules
+ $MatrixStr = "$MatrixStr,'$key':{'MatrixKey':'$key'}"
+ Write-Host "##[endgroup]"
+ Write-Host
$index++
}
+$generateTargetsOutputDir = Join-Path $RepoRoot "artifacts"
+if (-not (Test-Path -Path $generateTargetsOutputDir)) {
+ New-Item -ItemType Directory -Path $generateTargetsOutputDir
+}
+$generateTargetsOutputFile = Join-Path $generateTargetsOutputDir "generateTargets.json"
+$generateTargets | ConvertTo-Json -Depth 10 | Out-File -FilePath $generateTargetsOutputFile -Encoding utf8
+
if ($MatrixStr -and $MatrixStr.Length -gt 1) {
$MatrixStr = $MatrixStr.Substring(1)
}
-Write-Host "##vso[task.setVariable variable=buildTargets;isOutput=true]{$MatrixStr}"
+Write-Host "##vso[task.setVariable variable=generateTargets;isOutput=true]{$MatrixStr}"
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/test-module.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/test-module.ps1
new file mode 100644
index 000000000000..41d0c6fe7cb7
--- /dev/null
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/test-module.ps1
@@ -0,0 +1,45 @@
+param (
+ [string]$MatrixKey,
+ [string]$Target,
+ [string]$TestEnvName,
+ [string]$RepoRoot,
+ [string]$ArtifactRoot
+)
+
+Write-Host "Matrix Key: $MatrixKey"
+Write-Host "Test $($TestEnvName): $Target"
+
+$modules = $Target -split ','
+$results = @()
+
+foreach ($module in $modules) {
+ $startTime = Get-Date
+ $result = @{
+ Module = $module
+ Status = "Success"
+ DurationSeconds = 0
+ Error = ""
+ }
+
+ try {
+ Write-Host "Testing module: $module"
+ $subModulePath = Join-Path $RepoRoot 'src' $module
+ # TODO(Bernard) Remove log after test
+ Write-Host "Sub module path: $subModulePath"
+ Set-Location -Path $subModulePath
+ & ".\test-module.ps1"
+ }
+ catch {
+ Write-Warning "Failed to test module: $module"
+ $result.Status = "Failed"
+ $result.Error = $_.Exception.Message
+ }
+ finally {
+ $endTime = Get-Date
+ $result.DurationSeconds = ($endTime - $startTime).TotalSeconds
+ $results += $result
+ }
+}
+
+$reportPath = Join-Path $ArtifactRoot "Test$($TestEnvName)Report-$MatrixKey.json"
+$results | ConvertTo-Json -Depth 3 | Out-File -FilePath $reportPath -Encoding utf8
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index ae44887dbd9c..4fbeff6f6300 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -1,19 +1,28 @@
variables:
+ IntermediateStepTimeoutInMinutes: 30
+ GenerateTimeoutInMinutes: 120
BuildTimeoutInMinutes: 120
AnalysisTimeoutInMinutes: 120
+ TestTimeoutInMinutes: 180
+ MaxParalleGenerateJobs: 3
MaxParalleBuildJobs: 3
+ MaxParalleAnalyzeJobs: 4
+ MaxParalleTestWindowsJobs: 1
+ MaxParalleTestLinuxJobs: 2
+ MaxParalleTestMacJobs: 3
+ WindowsAgentPoolName: pool-windows-2019
+ LinuxAgentPoolName: pool-ubuntu-2004
+ MacOSAgentPoolName: 'Azure Pipelines'
+ MacOSAgentPoolVMImage: macOS-latest
GenerationBranch: batch-generation/branch-$(Build.BuildId)
- MaxParalleAnalyzeJobs: 3
- MaxParalleWindowsTestJobs: 3
- MaxParalleLinuxTestJobs: 3
- MaxParalleMacTestJobs: 3
trigger: none
jobs:
- job: prepare
displayName: Generate Build Targets Matrix
- pool: pool-windows-2019
+ timeoutInMinutes: ${{ variables.IntermediateStepTimeoutInMinutes }}
+ pool: ${{ variables.WindowsAgentPoolName }}
steps:
# TODO: (Bernard) Uncomment the no checkout step after automatically install repo into agnets
@@ -31,7 +40,8 @@ jobs:
$token = "$(GitHubToken)"
$createBranchPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'create-branch.ps1'
- & $createBranchPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch 'main' -NewBranch $newBranch -Token $token
+ # TODO: (Bernard) Change BaseBranch to main after test
+ & $createBranchPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch 'bernard-pipeline-generation' -NewBranch $newBranch -Token $token
- task: PowerShell@2
name: mtrx
@@ -42,30 +52,36 @@ jobs:
script: |
$prepareModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'prepare.ps1'
& $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParalleBuildJobs }}"
-
-- job: build
- displayName: "Build:"
+ - task: PublishPipelineArtifact@1
+ displayName: 'Upload generated targets'
+ inputs:
+ targetPath: artifacts
+ artifact: 'prepare'
+
+- job: generate
+ displayName: "Batch Generate: "
dependsOn: prepare
- timeoutInMinutes: 120
- pool: pool-windows-2019
+ condition: and(succeeded(), ne(dependencies.prepare.outputs['mtrx.generateTargets'], '{}'))
+ timeoutInMinutes: ${{ variables.GenerateTimeoutInMinutes }}
+ pool: ${{ variables.WindowsAgentPoolName }}
strategy:
- matrix: $[ dependencies.prepare.outputs['mtrx.buildTargets'] ]
- maxParallel: ${{ variables.MaxParalleBuildJobs }}
+ matrix: $[ dependencies.prepare.outputs['mtrx.generateTargets'] ]
+ maxParallel: ${{ variables.MaxParalleGenerateJobs }}
steps:
- checkout: self
persistCredentials: true
fetchTags: false
-
- # - checkout: azure-powershell-devops
- # path: "azure-powershell-devops"
-
- # - template: util/get-github-pat-steps.yml
+
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifactName: 'prepare'
+ targetPath: artifacts
- task: PowerShell@2
- name: build
- displayName: 'Build Targets'
+ name: generate
+ displayName: 'Regenerate Modules'
inputs:
targetType: inline
pwsh: true
@@ -73,54 +89,65 @@ jobs:
git fetch origin "${{ variables.GenerationBranch }}"
git checkout "${{ variables.GenerationBranch }}"
- $target = "$(Target)"
- Write-Host "Building targets: $target"
-
- $patchDir = "$(Build.ArtifactStagingDirectory)/$(MatrixKey)"
- New-Item -ItemType Directory -Force -Path $patchDir | Out-Null
+ $batchGenerateModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'batch-generate-modules.ps1'
+ & $batchGenerateModulesPath -MatrixKey "$(MatrixKey)" -RepoRoot "$(Build.SourcesDirectory)"
- $modules = $target -split ','
+ workingDirectory: $(Build.SourcesDirectory)
- foreach ($module in $modules) {
- $modulePath = Join-Path $(Build.SourcesDirectory) 'src' $module
- $subModulePath = Get-ChildItem -Path $modulePath -Directory |
- Where-Object { $_.Name -like '*.autorest' }
- foreach ($subModule in $subModulePath) {
- $filePath = Join-Path $subModule "newfile.txt"
- "This is a new file in $folder" | Out-File -FilePath $filePath -Encoding UTF8
- }
- }
-
- # git config user.email "65331932+azure-powershell-bot@users.noreply.github.com";
- # git config user.name "azure-powershell-bot";
- # git remote set-url origin 'https://azure-powershell-bot:$(GithubToken)@github.com/Azure/azure-powershell.git';
+ - task: PublishPipelineArtifact@1
+ displayName: 'Save patch and generation report'
+ inputs:
+ targetPath: artifacts
+ artifact: 'build-$(MatrixKey)'
+ condition: always()
- $random = Get-Random -Minimum 10 -Maximum 121
- Write-Host "Sleeping for $random seconds..."
- Start-Sleep -Seconds $random
+# - job: build
+# displayName: "Build:"
+# dependsOn: prepare
+# timeoutInMinutes: ${{ variables.BuildTimeoutInMinutes }}
+# pool: ${{ variables.WindowsAgentPoolName }}
+# strategy:
+# matrix: $[ dependencies.prepare.outputs['mtrx.buildTargets'] ]
+# maxParallel: ${{ variables.MaxParalleBuildJobs }}
+
+# steps:
+# - checkout: self
+# persistCredentials: true
+# fetchTags: false
+
+# # - checkout: azure-powershell-devops
+# # path: "azure-powershell-devops"
- git add .
- # git commit -m "Build $target"
- # git push origin "${{ variables.GenerationBranch }}"
-
+# # - template: util/get-github-pat-steps.yml
- $patchPath = "$patchDir/changed.patch"
- git diff --cached > $patchPath
+# - task: PowerShell@2
+# name: build
+# displayName: 'Build Targets'
+# inputs:
+# targetType: inline
+# pwsh: true
+# script: |
+# git fetch origin "${{ variables.GenerationBranch }}"
+# git checkout "${{ variables.GenerationBranch }}"
- Get-Content -Path $patchPath
+# $batchGenerateModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'batch-generate-modules.ps1'
+# & $batchGenerateModulesPath -MatrixKey "$(MatrixKey)" -Target "$(Target)" -RepoRoot "$(Build.SourcesDirectory)" -ArtifactRoot "$(Build.ArtifactStagingDirectory)"
- workingDirectory: $(Build.SourcesDirectory)
+# workingDirectory: $(Build.SourcesDirectory)
- - task: PublishPipelineArtifact@1
- displayName: 'Save patch'
- inputs:
- targetPath: '$(Build.ArtifactStagingDirectory)/$(MatrixKey)'
- artifact: 'patch-$(MatrixKey)'
- condition: always()
+# - task: PublishPipelineArtifact@1
+# displayName: 'Save patch'
+# inputs:
+# targetPath: '$(Build.ArtifactStagingDirectory)'
+# artifact: 'build-$(MatrixKey)'
+# condition: always()
- job: filter
displayName: 'Filter Changed Modules'
- dependsOn: build
+ dependsOn: generate
+ condition: always()
+ timeoutInMinutes: ${{ variables.IntermediateStepTimeoutInMinutes }}
+ pool: ${{ variables.WindowsAgentPoolName }}
steps:
- checkout: self
persistCredentials: true
@@ -137,7 +164,7 @@ jobs:
git checkout "${{ variables.GenerationBranch }}"
- download: current
- patterns: '**/*.patch'
+ patterns: '**/changed-*.patch'
displayName: 'Download all .patch artifacts'
- task: PowerShell@2
@@ -179,7 +206,7 @@ jobs:
git checkout $sourceBranchName
$filterModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'filter.ps1'
- & $filterModulesPath -MaxParallelJobs "${{ variables.MaxParalleAnalyzeJobs }}" -ChangedFiles $changedFiles
+ & $filterModulesPath -MaxParalleAnalyzeJobs "${{ variables.MaxParalleAnalyzeJobs }}" -MaxParalleTestWindowsJobs "${{ variables.MaxParalleTestWindowsJobs }}" -MaxParalleTestLinuxJobs "${{ variables.MaxParalleTestLinuxJobs }}" -MaxParalleTestMacJobs "${{ variables.MaxParalleTestMacJobs }}" -ChangedFiles $changedFiles
# - checkout: none
@@ -212,10 +239,11 @@ jobs:
- job: analyze
displayName: "Analyze:"
dependsOn: filter
- timeoutInMinutes: 120
- pool: pool-windows-2019
+ condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.AnalyzeTargets'], '{}'))
+ timeoutInMinutes: ${{ variables.AnalysisTimeoutInMinutes }}
+ pool: ${{ variables.WindowsAgentPoolName }}
strategy:
- matrix: $[ dependencies.filter.outputs['mtrx.analyzeTargets'] ]
+ matrix: $[ dependencies.filter.outputs['mtrx.AnalyzeTargets'] ]
maxParallel: ${{ variables.MaxParalleAnalyzeJobs }}
steps:
@@ -229,4 +257,76 @@ jobs:
pwsh: true
script: |
Write-Host "Matrix Key: $(MatrixKey)"
- Write-Host "Analyze Targets: $(Target)"
\ No newline at end of file
+ Write-Host "Analyze Targets: $(Target)"
+
+- job: test_windows
+ displayName: "Test Windows:"
+ dependsOn: filter
+ condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.TestWindowsTargets'], '{}'))
+ timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
+ pool: ${{ variables.WindowsAgentPoolName }}
+ strategy:
+ matrix: $[ dependencies.filter.outputs['mtrx.TestWindowsTargets'] ]
+ maxParallel: ${{ variables.MaxParalleTestWindowsJobs }}
+
+ steps:
+ - checkout: none
+
+ - task: PowerShell@2
+ name: test
+ displayName: 'Test Windows'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ $testModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'test-module.ps1'
+ & $testModulesPath -MatrixKey "$(MatrixKey)" -Target "$(Target)" -TestEnvName 'Windows' -RepoRoot "$(Build.SourcesDirectory)" -ArtifactRoot "$(Build.ArtifactStagingDirectory)"
+
+- job: test_linux
+ displayName: "Test Linux:"
+ dependsOn: filter
+ condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.TestLinuxTargets'], '{}'))
+ timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
+ pool: ${{ variables.LinuxAgentPoolName }}
+ strategy:
+ matrix: $[ dependencies.filter.outputs['mtrx.TestLinuxTargets'] ]
+ maxParallel: ${{ variables.MaxParalleTestLinuxJobs }}
+
+ steps:
+ - checkout: none
+
+ - task: PowerShell@2
+ name: test
+ displayName: 'Test Linux'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ $testModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'test-module.ps1'
+ & $testModulesPath -MatrixKey "$(MatrixKey)" -Target "$(Target)" -TestEnvName 'Linux' -RepoRoot "$(Build.SourcesDirectory)" -ArtifactRoot "$(Build.ArtifactStagingDirectory)"
+
+- job: test_mac
+ displayName: "Test Mac:"
+ dependsOn: filter
+ condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.TestMacTargets'], '{}'))
+ timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
+ pool:
+ name: ${{ variables.MacOSAgentPoolName }}
+ vmImage: ${{ variables.MacOSAgentPoolVMImage }}
+ strategy:
+ matrix: $[ dependencies.filter.outputs['mtrx.TestMacTargets'] ]
+ maxParallel: ${{ variables.MaxParalleTestMacJobs }}
+
+ steps:
+ - checkout: none
+
+ - task: PowerShell@2
+ name: test
+ displayName: 'Test Mac'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ $testModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'test-module.ps1'
+ & $testModulesPath -MatrixKey "$(MatrixKey)" -Target "$(Target)" -TestEnvName 'Mac' -RepoRoot "$(Build.SourcesDirectory)" -ArtifactRoot "$(Build.ArtifactStagingDirectory)"
+
diff --git a/.azure-pipelines/test.ps1 b/.azure-pipelines/test.ps1
deleted file mode 100644
index 450da14c3360..000000000000
--- a/.azure-pipelines/test.ps1
+++ /dev/null
@@ -1,9 +0,0 @@
-
-$RepoRoot = Get-Location
-$buildProjPath = Join-Path $RepoRoot 'build.proj'
-
-$filesChangedOutputPath = Join-Path $RepoRoot 'artifacts' 'FilesChanged.txt'
-$subTasksFilePath = Join-Path $RepoRoot 'artifacts' 'SubTasksFile.txt'
-$IsSecurityCheck = $null
-
-dotnet msbuild $buildProjPath /t:FilterBuild "/p:FilesChangedOutputPath=$FilesChangedOutputPath;SubTasksFilePath=$SubTasksFilePath;IsSecurityCheck=$IsSecurityCheck"
From 0c5d6921b7c2fdf7820e002c2f3a45f4e0d32851 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Wed, 30 Apr 2025 17:10:07 +1000
Subject: [PATCH 19/33] build
---
.../batch-generate-modules.ps1 | 2 +-
.../BatchGeneration/build-module.ps1 | 40 +++++
.../PipelineSteps/BatchGeneration/filter.ps1 | 104 +++++-------
.../PipelineSteps/BatchGeneration/prepare.ps1 | 6 +-
.../PipelineSteps/BatchGeneration/util.psm1 | 60 +++++++
.azure-pipelines/batch-generation.yml | 149 ++++++++----------
6 files changed, 207 insertions(+), 154 deletions(-)
create mode 100644 .azure-pipelines/PipelineSteps/BatchGeneration/build-module.ps1
create mode 100644 .azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
index ad9ddebd6329..8da03eb94249 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
@@ -16,7 +16,7 @@ New-Item -ItemType Directory -Force -Path $AutorestOutputDir
$sourceDirectory = Join-Path $RepoRoot "src"
$generatedDirectory = Join-Path $RepoRoot "generated"
$buildScriptsModulePath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildScripts.psm1'
-Import-Module $buildScriptsModulePath
+Import-Module $buildScriptsModulePath -Force
$results = @()
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/build-module.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/build-module.ps1
new file mode 100644
index 000000000000..46264c362763
--- /dev/null
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/build-module.ps1
@@ -0,0 +1,40 @@
+param (
+ [string]$MatrixKey,
+ [string]$RepoRoot
+)
+
+Write-Host "Matrix Key: $MatrixKey"
+
+$buildTargetsOutputFile = Join-Path $RepoRoot "artifacts" "buildTargets.json"
+$buildTargets = Get-Content -Path $buildTargetsOutputFile -Raw | ConvertFrom-Json
+$moduleGroup = $buildTargets.$MatrixKey
+$buildModulesPath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildModules.ps1'
+
+$results = @()
+foreach ($moduleName in $moduleGroup) {
+ Write-Host "=============================================================="
+ Write-Host "Building Module: $moduleName"
+
+ $startTime = Get-Date
+ $result = @{
+ Module = $moduleName
+ Status = "Success"
+ DurationSeconds = 0
+ Error = ""
+ }
+
+ try {
+ & $buildModulesPath -TargetModule $moduleName -InvokedByPipeline
+ } catch {
+ Write-Warning "Failed to build module: $moduleName"
+ $result.Status = "Failed"
+ $result.Error = $_.Exception.Message
+ } finally {
+ $endTine = Get-Date
+ $result.DurationSeconds = ($endTine - $startTime).TotalSeconds
+ $results += $result
+ }
+}
+
+$reportPath = Join-Path $RepoRoot "artifacts" "BuildReport-$MatrixKey.json"
+$results | ConvertTo-Json -Depth 5 | Out-File -FilePath $reportPath -Encoding utf8
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
index 2346e35a60af..93157bc51fa5 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
@@ -1,89 +1,57 @@
[CmdletBinding(DefaultParameterSetName="AllSet")]
param (
- [int]$MaxParalleAnalyzeJobs = 3,
- [int]$MaxParalleTestWindowsJobs = 3,
- [int]$MaxParalleTestLinuxJobs = 3,
- [int]$MaxParalleTestMacJobs = 3,
- [string[]]$ChangedFiles
+ [int]$MaxParallelBuildJobs = 3,
+ [int]$MaxParallelAnalyzeJobs = 3,
+ [int]$MaxParallelTestWindowsJobs = 3,
+ [int]$MaxParallelTestLinuxJobs = 3,
+ [int]$MaxParallelTestMacJobs = 3,
+ [string[]]$ChangedFiles,
+ [string]$RepoRoot
)
-$autorestFolders = @{}
+$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
+Import-Module $utilFilePath -Force
+
+$changedModulesDict = @{}
+$changedSubModulesDict = @{}
for ($i = 0; $i -lt $ChangedFiles.Count; $i++) {
if ($ChangedFiles[$i] -match '^(src|generated)/([^/]+)/([^/]+\.autorest)/') {
$parent = $Matches[2]
$child = $Matches[3]
$key = "$parent/$child"
-
- $autorestFolders[$key] = $true
+
+ $changedModulesDict[$parent] = $true
+ $changedSubModulesDict[$key] = $true
}
}
+$changedModules = $changedModulesDict.Keys | Sort-Object
+$changedSubModules = $changedSubModulesDict.Keys | Sort-Object
-$changedSubModules = $autorestFolders.Keys
-# TODO(Bernard) Remove test data after test
-# $changedSubModules = @("A", "B", "C", "D", "E", "F", "G")
-Write-Host "Chagned sub modules: "
-foreach ($subModule in $changedSubModules) {
- Write-Host $subModule
+Write-Host "##[group]Changed modules: $($changedModules.Count)"
+foreach ($module in $changedModules) {
+ Write-Host $module
}
+Write-Host "##[endgroup]"
+ Write-Host
-function Split-List {
- param (
- [array]$subModules,
- [int]$maxParallelJobs
- )
-
- $count = $subModules.Count
- $n = [Math]::Min($count, $maxParallelJobs)
-
- if ($n -eq 0) {
- return @()
- }
-
- $result = @()
-
- for ($i = 0; $i -lt $n; $i++) {
- $result += ,@()
- }
-
- for ($i = 0; $i -lt $count; $i++) {
- $groupIndex = $i % $n
- $result[$groupIndex] += $subModules[$i]
- }
-
- return ,$result
+Write-Host "##[group]Changed sub modules: $($changedSubModules.Count)"
+foreach ($subModule in $changedSubModules) {
+ Write-Host $subModule
}
+Write-Host "##[endgroup]"
+Write-Host
-function Write-Matrix {
- param (
- [string]$variableName,
- [array]$groupedSubModules
- )
-
- $index = 0
- foreach ($subModules in $groupedSubModules) {
- $moduleNamesStr = $subModules -join ','
- $key = ($index + 1).ToString() + "-" + $subModules.Count
- $MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
- $index++
- }
+$groupedBuildModules = Group-Modules -Modules $changedModules -MaxParallelJobs $MaxParallelBuildJobs
+Write-Matrix -GroupedModules $groupedBuildModules -VariableName 'buildTargets' -RepoRoot $RepoRoot
- if ($MatrixStr -and $MatrixStr.Length -gt 1) {
- $MatrixStr = $MatrixStr.Substring(1)
- }
- Write-Host "##vso[task.setVariable variable=$variableName;isOutput=true]{$MatrixStr}"
- Write-Host "variable=$variableName; value=$MatrixStr"
- }
+$groupedAnalyzeModules = Group-Modules -Modules $changedModules -MaxParallelJobs $MaxParallelAnalyzeJobs
+Write-Matrix -GroupedModules $groupedAnalyzeModules -variableName 'analyzeTargets' -RepoRoot $RepoRoot
-$groupedAnalyzeModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleAnalyzeJobs
-Write-Matrix -variableName 'AnalyzeTargets' -groupedSubModules $groupedAnalyzeModules
+# $groupedTestWindowsModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParallelTestWindowsJobs
+# Write-Matrix -variableName 'TestWindowsTargets' -groupedSubModules $groupedTestWindowsModules
-$groupedTestWindowsModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestWindowsJobs
-Write-Matrix -variableName 'TestWindowsTargets' -groupedSubModules $groupedTestWindowsModules
+# $groupedTestLinuxModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParallelTestLinuxJobs
+# Write-Matrix -variableName 'TestLinuxTargets' -groupedSubModules $groupedTestLinuxModules
-$groupedTestLinuxModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestLinuxJobs
-Write-Matrix -variableName 'TestLinuxTargets' -groupedSubModules $groupedTestLinuxModules
-
-# $groupedTestMacModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestMacJobs
+# $groupedTestMacModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParallelTestMacJobs
# Write-Matrix -variableName 'TestMacTargets' -groupedSubModules $groupedTestMacModules
-
-Write-Host "##vso[task.setVariable variable=TestMacTargets;isOutput=true]{}"
\ No newline at end of file
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
index cf9dfd047182..386b814071a7 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
@@ -58,7 +58,7 @@ $modules = $modules.GetEnumerator() | ForEach-Object {
Write-Host "Total matched modules: $($modules.Count)"
-function Group-List {
+function Group-Modules {
param (
[array]$modules,
[int]$maxParallelJobs
@@ -81,7 +81,7 @@ function Group-List {
return $result
}
-$groupedModules = Group-List -modules $modules -maxParallelJobs $MaxParallelJobs
+$groupedModules = Group-Modules -modules $modules -maxParallelJobs $MaxParallelJobs
Write-Host "Total module groups: $($groupedModules.Count)"
$index = 0
@@ -108,7 +108,7 @@ if (-not (Test-Path -Path $generateTargetsOutputDir)) {
New-Item -ItemType Directory -Path $generateTargetsOutputDir
}
$generateTargetsOutputFile = Join-Path $generateTargetsOutputDir "generateTargets.json"
-$generateTargets | ConvertTo-Json -Depth 10 | Out-File -FilePath $generateTargetsOutputFile -Encoding utf8
+$generateTargets | ConvertTo-Json -Depth 5 | Out-File -FilePath $generateTargetsOutputFile -Encoding utf8
if ($MatrixStr -and $MatrixStr.Length -gt 1) {
$MatrixStr = $MatrixStr.Substring(1)
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1 b/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
new file mode 100644
index 000000000000..22fb2fe361a6
--- /dev/null
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
@@ -0,0 +1,60 @@
+function Group-Modules {
+ param (
+ [array]$Modules,
+ [int]$MaxParallelJobs
+ )
+
+ $count = $Modules.Count
+ $n = [Math]::Min($count, $MaxParallelJobs)
+
+ if ($n -eq 0) {
+ return @()
+ }
+
+ $result = @()
+
+ for ($i = 0; $i -lt $n; $i++) {
+ $result += ,@()
+ }
+
+ for ($i = 0; $i -lt $count; $i++) {
+ $groupIndex = $i % $n
+ $result[$groupIndex] += $Modules[$i]
+ }
+
+ return ,$result
+}
+
+function Write-Matrix {
+ param (
+ [array]$GroupedModules,
+ [string]$VariableName,
+ [string]$RepoRoot
+ )
+
+ Write-Host "Module groups: $($GroupedModules.Count)"
+ $GroupedModules | ForEach-Object { $_ -join ', ' } | ForEach-Object { Write-Host $_ }
+
+ $targets = @{}
+ $MatrixStr = ""
+ $index = 0
+ foreach ($modules in $GroupedModules) {
+ $key = ($index + 1).ToString() + "-" + $modules.Count
+ $MatrixStr = "$MatrixStr,'$key':{'MatrixKey':'$key'}"
+ $targets[$key] = $modules
+ $index++
+ }
+
+ if ($MatrixStr -and $MatrixStr.Length -gt 1) {
+ $MatrixStr = $MatrixStr.Substring(1)
+ }
+ Write-Host "##vso[task.setVariable variable=$VariableName;isOutput=true]{$MatrixStr}"
+ Write-Host "variable=$VariableName; value=$MatrixStr"
+
+ $targetsOutputDir = Join-Path $RepoRoot "artifacts"
+ if (-not (Test-Path -Path $targetsOutputDir)) {
+ New-Item -ItemType Directory -Path $targetsOutputDir -Force | Out-Null
+ }
+ $targetsOutputFile = Join-Path $targetsOutputDir "$VariableName.json"
+ $targets | ConvertTo-Json -Depth 5 | Out-File -FilePath $targetsOutputFile -Encoding utf8
+}
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 4fbeff6f6300..d139c8cacb4c 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -4,12 +4,12 @@ variables:
BuildTimeoutInMinutes: 120
AnalysisTimeoutInMinutes: 120
TestTimeoutInMinutes: 180
- MaxParalleGenerateJobs: 3
- MaxParalleBuildJobs: 3
- MaxParalleAnalyzeJobs: 4
- MaxParalleTestWindowsJobs: 1
- MaxParalleTestLinuxJobs: 2
- MaxParalleTestMacJobs: 3
+ MaxParallelGenerateJobs: 3
+ MaxParallelBuildJobs: 15
+ MaxParallelAnalyzeJobs: 4
+ MaxParallelTestWindowsJobs: 1
+ MaxParallelTestLinuxJobs: 2
+ MaxParallelTestMacJobs: 3
WindowsAgentPoolName: pool-windows-2019
LinuxAgentPoolName: pool-ubuntu-2004
MacOSAgentPoolName: 'Azure Pipelines'
@@ -51,7 +51,7 @@ jobs:
pwsh: true
script: |
$prepareModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'prepare.ps1'
- & $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParalleBuildJobs }}"
+ & $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParallelGenerateJobs }}"
- task: PublishPipelineArtifact@1
displayName: 'Upload generated targets'
@@ -67,7 +67,7 @@ jobs:
pool: ${{ variables.WindowsAgentPoolName }}
strategy:
matrix: $[ dependencies.prepare.outputs['mtrx.generateTargets'] ]
- maxParallel: ${{ variables.MaxParalleGenerateJobs }}
+ maxParallel: ${{ variables.MaxParallelGenerateJobs }}
steps:
- checkout: self
@@ -98,50 +98,9 @@ jobs:
displayName: 'Save patch and generation report'
inputs:
targetPath: artifacts
- artifact: 'build-$(MatrixKey)'
+ artifact: 'generate-$(MatrixKey)'
condition: always()
-# - job: build
-# displayName: "Build:"
-# dependsOn: prepare
-# timeoutInMinutes: ${{ variables.BuildTimeoutInMinutes }}
-# pool: ${{ variables.WindowsAgentPoolName }}
-# strategy:
-# matrix: $[ dependencies.prepare.outputs['mtrx.buildTargets'] ]
-# maxParallel: ${{ variables.MaxParalleBuildJobs }}
-
-# steps:
-# - checkout: self
-# persistCredentials: true
-# fetchTags: false
-
-# # - checkout: azure-powershell-devops
-# # path: "azure-powershell-devops"
-
-# # - template: util/get-github-pat-steps.yml
-
-# - task: PowerShell@2
-# name: build
-# displayName: 'Build Targets'
-# inputs:
-# targetType: inline
-# pwsh: true
-# script: |
-# git fetch origin "${{ variables.GenerationBranch }}"
-# git checkout "${{ variables.GenerationBranch }}"
-
-# $batchGenerateModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'batch-generate-modules.ps1'
-# & $batchGenerateModulesPath -MatrixKey "$(MatrixKey)" -Target "$(Target)" -RepoRoot "$(Build.SourcesDirectory)" -ArtifactRoot "$(Build.ArtifactStagingDirectory)"
-
-# workingDirectory: $(Build.SourcesDirectory)
-
-# - task: PublishPipelineArtifact@1
-# displayName: 'Save patch'
-# inputs:
-# targetPath: '$(Build.ArtifactStagingDirectory)'
-# artifact: 'build-$(MatrixKey)'
-# condition: always()
-
- job: filter
displayName: 'Filter Changed Modules'
dependsOn: generate
@@ -206,45 +165,67 @@ jobs:
git checkout $sourceBranchName
$filterModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'filter.ps1'
- & $filterModulesPath -MaxParalleAnalyzeJobs "${{ variables.MaxParalleAnalyzeJobs }}" -MaxParalleTestWindowsJobs "${{ variables.MaxParalleTestWindowsJobs }}" -MaxParalleTestLinuxJobs "${{ variables.MaxParalleTestLinuxJobs }}" -MaxParalleTestMacJobs "${{ variables.MaxParalleTestMacJobs }}" -ChangedFiles $changedFiles
+ & $filterModulesPath -MaxParallelBuildJobs "${{ variables.MaxParallelBuildJobs }}" -MaxParallelAnalyzeJobs "${{ variables.MaxParallelAnalyzeJobs }}" -MaxParallelTestWindowsJobs "${{ variables.MaxParallelTestWindowsJobs }}" -MaxParallelTestLinuxJobs "${{ variables.MaxParallelTestLinuxJobs }}" -MaxParallelTestMacJobs "${{ variables.MaxParallelTestMacJobs }}" -ChangedFiles $changedFiles -RepoRoot "$(Build.SourcesDirectory)"
- # - checkout: none
+ - task: PublishPipelineArtifact@1
+ displayName: 'Upload filtered targets'
+ inputs:
+ targetPath: artifacts
+ artifact: 'filter'
- # - script: |
- # git init
- # git remote add origin https://github.com/Azure/azure-powershell.git
- # git fetch origin main --depth=1
- # git checkout FETCH_HEAD -- .azure-pipelines/
-
- # displayName: 'Checkout Pipeline Templates'
+- job: build
+ displayName: "Build:"
+ dependsOn: filter
+ condition: ne(dependencies.filter.outputs['mtrx.buildTargets'], '{}')
+ timeoutInMinutes: ${{ variables.BuildTimeoutInMinutes }}
+ pool: ${{ variables.WindowsAgentPoolName }}
+ strategy:
+ matrix: $[ dependencies.filter.outputs['mtrx.buildTargets'] ]
+ maxParallel: ${{ variables.MaxParallelBuildJobs }}
+
+ steps:
+ - checkout: self
+ persistCredentials: true
+ fetchTags: false
+
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifactName: 'filter'
+ targetPath: artifacts
- # - template: util/get-github-pat-steps.yml
+ - task: PowerShell@2
+ name: build
+ displayName: 'Build Targets'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ git fetch origin "${{ variables.GenerationBranch }}"
+ git checkout "${{ variables.GenerationBranch }}"
- # - script: |
- # $token = "$(GitHubToken)"
- # changedModules=$(curl -s -H "Authorization: Bearer $token" \
- # https://api.github.com/repos/Azure/azure-powershell/compare/main...${{ variables.GenerationBranch }} \
- # | jq -r '.files[].filename' \
- # | grep -E '^(src|generated)/[^/]+/[^/]+\.autorest/' \
- # | sed -E 's#^(src|generated)/##' \
- # | awk -F/ '{print $1 "/" $2}' \
- # | sort -u)
+ $buildModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'build-modules.ps1'
+ & $buildModulesPath -MatrixKey "$(MatrixKey)" -RepoRoot "$(Build.SourcesDirectory)"
- # IFS=$'\n' read -r -d '' -a changedModulePath <<< "$changedModules"
+ workingDirectory: $(Build.SourcesDirectory)
- # for path in "${changedModulePath[@]}"; do
- # echo "$path"
- # done
+ - task: PublishPipelineArtifact@1
+ displayName: 'Save build artifacts'
+ inputs:
+ targetPath: artifacts
+ artifact: 'build-$(MatrixKey)'
+ condition: always()
- job: analyze
displayName: "Analyze:"
- dependsOn: filter
- condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.AnalyzeTargets'], '{}'))
+ dependsOn:
+ - filter
+ - build
+ condition: ne(dependencies.filter.outputs['mtrx.analyzeTargets'], '{}')
timeoutInMinutes: ${{ variables.AnalysisTimeoutInMinutes }}
pool: ${{ variables.WindowsAgentPoolName }}
strategy:
- matrix: $[ dependencies.filter.outputs['mtrx.AnalyzeTargets'] ]
- maxParallel: ${{ variables.MaxParalleAnalyzeJobs }}
+ matrix: $[ dependencies.filter.outputs['mtrx.analyzeTargets'] ]
+ maxParallel: ${{ variables.MaxParallelAnalyzeJobs }}
steps:
- checkout: none
@@ -261,13 +242,15 @@ jobs:
- job: test_windows
displayName: "Test Windows:"
- dependsOn: filter
+ dependsOn:
+ - filter
+ - build
condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.TestWindowsTargets'], '{}'))
timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
pool: ${{ variables.WindowsAgentPoolName }}
strategy:
matrix: $[ dependencies.filter.outputs['mtrx.TestWindowsTargets'] ]
- maxParallel: ${{ variables.MaxParalleTestWindowsJobs }}
+ maxParallel: ${{ variables.MaxParallelTestWindowsJobs }}
steps:
- checkout: none
@@ -284,13 +267,15 @@ jobs:
- job: test_linux
displayName: "Test Linux:"
- dependsOn: filter
+ dependsOn:
+ - filter
+ - build
condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.TestLinuxTargets'], '{}'))
timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
pool: ${{ variables.LinuxAgentPoolName }}
strategy:
matrix: $[ dependencies.filter.outputs['mtrx.TestLinuxTargets'] ]
- maxParallel: ${{ variables.MaxParalleTestLinuxJobs }}
+ maxParallel: ${{ variables.MaxParallelTestLinuxJobs }}
steps:
- checkout: none
@@ -315,7 +300,7 @@ jobs:
vmImage: ${{ variables.MacOSAgentPoolVMImage }}
strategy:
matrix: $[ dependencies.filter.outputs['mtrx.TestMacTargets'] ]
- maxParallel: ${{ variables.MaxParalleTestMacJobs }}
+ maxParallel: ${{ variables.MaxParallelTestMacJobs }}
steps:
- checkout: none
From 57fe7df91cf6f42fca8dc9d2d16370d72264e12e Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Fri, 2 May 2025 13:21:58 +1000
Subject: [PATCH 20/33] analyse
---
.../BatchGeneration/analyse-modules.ps1 | 91 +++
.../batch-generate-modules.ps1 | 11 +-
.../{build-module.ps1 => build-modules.ps1} | 9 +-
.../PipelineSteps/BatchGeneration/util.psm1 | 17 +
.azure-pipelines/batch-generation.yml | 732 +++++++++++-------
5 files changed, 554 insertions(+), 306 deletions(-)
create mode 100644 .azure-pipelines/PipelineSteps/BatchGeneration/analyse-modules.ps1
rename .azure-pipelines/PipelineSteps/BatchGeneration/{build-module.ps1 => build-modules.ps1} (76%)
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/analyse-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/analyse-modules.ps1
new file mode 100644
index 000000000000..0ec348f6a58a
--- /dev/null
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/analyse-modules.ps1
@@ -0,0 +1,91 @@
+param (
+ [string]$MatrixKey,
+ [string]$RepoRoot
+)
+
+$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
+Import-Module $utilFilePath -Force
+$moduleGroup = Get-Targets -RepoRoot $RepoRoot -TargetsOutputFileName "analyzeTargets.json" -MatrixKey $MatrixKey
+$RepoArtifacts = Join-Path $RepoRoot 'artifacts'
+$StaticAnalysisOutputDirectory = Join-Path $RepoArtifacts 'StaticAnalysisResults'
+if (-not (Test-Path -Path $StaticAnalysisOutputDirectory)) {
+ New-Item -ItemType Directory -Path $StaticAnalysisOutputDirectory
+}
+$toolsDirectory = Join-Path $RepoRoot 'tools'
+
+$results = @()
+foreach ($moduleName in $moduleGroup) {
+ Write-Host "=============================================================="
+ Write-Host "Analysing Module: $moduleName"
+
+ $startTime = Get-Date
+ $result = @{
+ Module = $moduleName
+ Status = "Success"
+ DurationSeconds = 0
+ Error = ""
+ FailedTasks = @()
+ }
+ $Parameters = @{
+ RepoArtifacts = $RepoArtifacts
+ StaticAnalysisOutputDirectory = $StaticAnalysisOutputDirectory
+ Configuration = "Debug"
+ TargetModule = @($moduleName)
+ }
+ $FailedTasks = @()
+ $ErrorLogPath = "$StaticAnalysisOutputDirectory/error.log"
+
+ try {
+ .("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisBreakingChange @Parameters 2>$ErrorLogPath
+ If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
+ {
+ $FailedTasks += "BreakingChange"
+ }
+ .("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisDependency @Parameters 2>>$ErrorLogPath
+ If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
+ {
+ $FailedTasks += "Dependency"
+ }
+ .("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisSignature @Parameters 2>>$ErrorLogPath
+ If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
+ {
+ $FailedTasks += "Signature"
+ }
+ .("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisHelp @Parameters 2>>$ErrorLogPath
+ If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
+ {
+ $FailedTasks += "Help"
+ }
+ .("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisUX @Parameters 2>>$ErrorLogPath
+ If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
+ {
+ $FailedTasks += "UXMetadata"
+ }
+ .("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisCmdletDiff @Parameters 2>>$ErrorLogPath
+ If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
+ {
+ $FailedTasks += "CmdletDiff"
+ }
+ If ($FailedTasks.Length -ne 0)
+ {
+ Write-Host "There are failed tasks: $FailedTasks"
+ $ErrorLog = Get-Content -Path $ErrorLogPath | Join-String -Separator "`n"
+ Write-Error $ErrorLog
+ $result.Status = "Failed"
+ $result.Error = "Failed tasks: $($FailedTasks -join ', ')"
+ $result.FailedTasks = $FailedTasks
+ }
+ } catch {
+ Write-Warning "Failed to analyse module: $moduleName"
+ Write-Warning "Error message: $($_.Exception.Message)"
+ $result.Status = "Failed"
+ $result.Error = $_.Exception.Message
+ } finally {
+ $endTine = Get-Date
+ $result.DurationSeconds = ($endTine - $startTime).TotalSeconds
+ $results += $result
+ }
+}
+
+$reportPath = Join-Path $RepoRoot "artifacts" "AnalyseReport-$MatrixKey.json"
+$results | ConvertTo-Json -Depth 5 | Out-File -FilePath $reportPath -Encoding utf8
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
index 8da03eb94249..44bb6d427368 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
@@ -3,11 +3,16 @@ param (
[string]$RepoRoot
)
-Write-Host "Matrix Key: $MatrixKey"
-
$generateTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generateTargets.json"
$generateTargets = Get-Content -Path $generateTargetsOutPutFile -Raw | ConvertFrom-Json
$moduleGroup = $generateTargets.$MatrixKey
+Write-Host "##[group]Generating module group $MatrixKey"
+foreach ($key in $moduleGroup.PSObject.Properties.Name | Sort-Object) {
+ $values = $moduleGroup.$key -join ', '
+ Write-Output "$key : $values"
+}
+Write-Host "##[endgroup]"
+Write-Host
$sortedModuleNames = $moduleGroup.PSObject.Properties.Name | Sort-Object
$AutorestOutputDir = Join-Path $RepoRoot "artifacts" "autorest"
@@ -57,6 +62,7 @@ foreach ($moduleName in $sortedModuleNames) {
} catch {
Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
+ Write-Warning "Error message: $($_.Exception.Message)"
$subModuleResult.Status = "Failed"
$subModuleResult.Error = $_.Exception.Message
} finally {
@@ -71,6 +77,7 @@ foreach ($moduleName in $sortedModuleNames) {
}
$ArtifactOutputDir = Join-Path $RepoRoot "artifacts"
+Set-Location $RepoRoot
git add .
$patchPath = Join-Path $ArtifactOutputDir "changed-$MatrixKey.patch"
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/build-module.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/build-modules.ps1
similarity index 76%
rename from .azure-pipelines/PipelineSteps/BatchGeneration/build-module.ps1
rename to .azure-pipelines/PipelineSteps/BatchGeneration/build-modules.ps1
index 46264c362763..8952d31e8e3c 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/build-module.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/build-modules.ps1
@@ -3,11 +3,9 @@ param (
[string]$RepoRoot
)
-Write-Host "Matrix Key: $MatrixKey"
-
-$buildTargetsOutputFile = Join-Path $RepoRoot "artifacts" "buildTargets.json"
-$buildTargets = Get-Content -Path $buildTargetsOutputFile -Raw | ConvertFrom-Json
-$moduleGroup = $buildTargets.$MatrixKey
+$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
+Import-Module $utilFilePath -Force
+$moduleGroup = Get-Targets -RepoRoot $RepoRoot -TargetsOutputFileName "buildTargets.json" -MatrixKey $MatrixKey
$buildModulesPath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildModules.ps1'
$results = @()
@@ -27,6 +25,7 @@ foreach ($moduleName in $moduleGroup) {
& $buildModulesPath -TargetModule $moduleName -InvokedByPipeline
} catch {
Write-Warning "Failed to build module: $moduleName"
+ Write-Warning "Error message: $($_.Exception.Message)"
$result.Status = "Failed"
$result.Error = $_.Exception.Message
} finally {
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1 b/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
index 22fb2fe361a6..c183bdc8d5d0 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
@@ -58,3 +58,20 @@ function Write-Matrix {
$targetsOutputFile = Join-Path $targetsOutputDir "$VariableName.json"
$targets | ConvertTo-Json -Depth 5 | Out-File -FilePath $targetsOutputFile -Encoding utf8
}
+
+function Get-Targets {
+ param (
+ [string]$RepoRoot,
+ [string]$TargetsOutputFileName,
+ [string]$MatrixKey
+ )
+
+ $targetsOutputFile = Join-Path $RepoRoot "artifacts" $TargetsOutputFileName
+ $targetGroups = Get-Content -Path $targetsOutputFile -Raw | ConvertFrom-Json
+ $targetGroup = $targetGroups.$MatrixKey
+ Write-Host "##[group]Target group: $MatrixKey"
+ $targetGroup | ForEach-Object { Write-Host $_ }
+ Write-Host "##[endgroup]"
+ Write-Host
+ return $targetGroup
+}
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index d139c8cacb4c..093e38bb96bc 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -4,12 +4,12 @@ variables:
BuildTimeoutInMinutes: 120
AnalysisTimeoutInMinutes: 120
TestTimeoutInMinutes: 180
- MaxParallelGenerateJobs: 3
+ MaxParallelGenerateJobs: 15
MaxParallelBuildJobs: 15
- MaxParallelAnalyzeJobs: 4
- MaxParallelTestWindowsJobs: 1
- MaxParallelTestLinuxJobs: 2
- MaxParallelTestMacJobs: 3
+ MaxParallelAnalyzeJobs: 5
+ MaxParallelTestWindowsJobs: 5
+ MaxParallelTestLinuxJobs: 5
+ MaxParallelTestMacJobs: 5
WindowsAgentPoolName: pool-windows-2019
LinuxAgentPoolName: pool-ubuntu-2004
MacOSAgentPoolName: 'Azure Pipelines'
@@ -18,300 +18,434 @@ variables:
trigger: none
-jobs:
-- job: prepare
- displayName: Generate Build Targets Matrix
- timeoutInMinutes: ${{ variables.IntermediateStepTimeoutInMinutes }}
- pool: ${{ variables.WindowsAgentPoolName }}
- steps:
-
- # TODO: (Bernard) Uncomment the no checkout step after automatically install repo into agnets
- # - checkout: none
-
- - template: util/get-github-pat-steps.yml
-
- - task: PowerShell@2
- displayName: 'Create Batch Generation Branch'
- inputs:
- targetType: inline
- pwsh: true
- script: |
- $newBranch = "$(GenerationBranch)"
- $token = "$(GitHubToken)"
-
- $createBranchPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'create-branch.ps1'
- # TODO: (Bernard) Change BaseBranch to main after test
- & $createBranchPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch 'bernard-pipeline-generation' -NewBranch $newBranch -Token $token
-
- - task: PowerShell@2
- name: mtrx
- displayName: 'Generate Build Targets Matrix'
- inputs:
- targetType: inline
- pwsh: true
- script: |
- $prepareModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'prepare.ps1'
- & $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParallelGenerateJobs }}"
-
- - task: PublishPipelineArtifact@1
- displayName: 'Upload generated targets'
- inputs:
- targetPath: artifacts
- artifact: 'prepare'
-
-- job: generate
- displayName: "Batch Generate: "
- dependsOn: prepare
- condition: and(succeeded(), ne(dependencies.prepare.outputs['mtrx.generateTargets'], '{}'))
- timeoutInMinutes: ${{ variables.GenerateTimeoutInMinutes }}
- pool: ${{ variables.WindowsAgentPoolName }}
- strategy:
- matrix: $[ dependencies.prepare.outputs['mtrx.generateTargets'] ]
- maxParallel: ${{ variables.MaxParallelGenerateJobs }}
-
- steps:
- - checkout: self
- persistCredentials: true
- fetchTags: false
-
- - task: DownloadPipelineArtifact@2
- inputs:
- artifactName: 'prepare'
- targetPath: artifacts
-
- - task: PowerShell@2
- name: generate
- displayName: 'Regenerate Modules'
- inputs:
- targetType: inline
- pwsh: true
- script: |
- git fetch origin "${{ variables.GenerationBranch }}"
- git checkout "${{ variables.GenerationBranch }}"
-
- $batchGenerateModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'batch-generate-modules.ps1'
- & $batchGenerateModulesPath -MatrixKey "$(MatrixKey)" -RepoRoot "$(Build.SourcesDirectory)"
-
- workingDirectory: $(Build.SourcesDirectory)
-
- - task: PublishPipelineArtifact@1
- displayName: 'Save patch and generation report'
- inputs:
- targetPath: artifacts
- artifact: 'generate-$(MatrixKey)'
+stages:
+ - stage: Generate
+ jobs:
+ - job: prepare
+ displayName: Generate Build Targets Matrix
+ timeoutInMinutes: ${{ variables.IntermediateStepTimeoutInMinutes }}
+ pool: ${{ variables.WindowsAgentPoolName }}
+ steps:
+
+ # TODO: (Bernard) Uncomment the no checkout step after automatically install repo into agnets
+ # - checkout: none
+ - checkout: self
+ fetchDepth: 1
+ fetchTags: false
+
+ - template: util/get-github-pat-steps.yml
+
+ - task: PowerShell@2
+ displayName: 'Create Batch Generation Branch'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ $newBranch = "$(GenerationBranch)"
+ $token = "$(GitHubToken)"
+
+ $createBranchPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'create-branch.ps1'
+ # TODO: (Bernard) Change BaseBranch to main after test
+ & $createBranchPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch 'bernard-pipeline-generation' -NewBranch $newBranch -Token $token
+
+ - task: PowerShell@2
+ name: mtrx
+ displayName: 'Generate Build Targets Matrix'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ $prepareModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'prepare.ps1'
+ & $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParallelGenerateJobs }}"
+
+ - task: PublishPipelineArtifact@1
+ displayName: 'Upload generated targets'
+ inputs:
+ targetPath: artifacts
+ artifact: 'prepare'
+
+ - job: generate
+ displayName: "Batch Generate: "
+ dependsOn: prepare
+ condition: ne(dependencies.prepare.outputs['mtrx.generateTargets'], '{}')
+ timeoutInMinutes: ${{ variables.GenerateTimeoutInMinutes }}
+ pool: ${{ variables.WindowsAgentPoolName }}
+ strategy:
+ matrix: $[ dependencies.prepare.outputs['mtrx.generateTargets'] ]
+ maxParallel: ${{ variables.MaxParallelGenerateJobs }}
+
+ steps:
+ - checkout: self
+ persistCredentials: true
+ fetchTags: false
+
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifactName: 'prepare'
+ targetPath: artifacts
+
+ - task: PowerShell@2
+ name: generate
+ displayName: 'Regenerate Modules'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ git fetch origin "${{ variables.GenerationBranch }}"
+ git checkout "${{ variables.GenerationBranch }}"
+
+ $batchGenerateModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'batch-generate-modules.ps1'
+ & $batchGenerateModulesPath -MatrixKey "$(MatrixKey)" -RepoRoot "$(Build.SourcesDirectory)"
+
+ workingDirectory: $(Build.SourcesDirectory)
+
+ - task: PublishPipelineArtifact@1
+ displayName: 'Save patch and generation report'
+ inputs:
+ targetPath: artifacts
+ artifact: 'generate-$(MatrixKey)'
+ condition: always()
+
+ - stage: Build
condition: always()
-
-- job: filter
- displayName: 'Filter Changed Modules'
- dependsOn: generate
- condition: always()
- timeoutInMinutes: ${{ variables.IntermediateStepTimeoutInMinutes }}
- pool: ${{ variables.WindowsAgentPoolName }}
- steps:
- - checkout: self
- persistCredentials: true
- fetchTags: false
-
- - task: PowerShell@2
- name: checkout
- displayName: 'Checkout Generation Branch'
- inputs:
- targetType: inline
- pwsh: true
- script: |
- git fetch origin "${{ variables.GenerationBranch }}"
- git checkout "${{ variables.GenerationBranch }}"
-
- - download: current
- patterns: '**/changed-*.patch'
- displayName: 'Download all .patch artifacts'
-
- - task: PowerShell@2
- name: apply
- displayName: 'Apply all patches'
- inputs:
- targetType: inline
- pwsh: true
- script: |
- git config user.email "65331932+azure-powershell-bot@users.noreply.github.com"
- git config user.name "azure-powershell-bot"
-
- Write-Host "Applying all .patch files to ${{ variables.GenerationBranch }}..."
- $patchFiles = Get-ChildItem -Path "$env:PIPELINE_WORKSPACE" -Recurse -Filter *.patch
-
- # $patchFiles = Get-ChildItem -Path "$env:PIPELINE_WORKSPACE" -Recurse -Filter *.patch |
- # Where-Object { $_.Directory.Parent.FullName -eq $env:PIPELINE_WORKSPACE }
-
- foreach ($patch in $patchFiles) {
- Write-Host "Applying patch: $($patch.FullName)"
- git apply --whitespace=fix "$($patch.FullName)"
- }
-
- git add .
- git commit -m "Apply matrix job patches to ${{ variables.GenerationBranch }}"
- git push origin "${{ variables.GenerationBranch }}"
-
- - task: PowerShell@2
- name: mtrx
- displayName: 'Generate Test Targets Matrix'
- inputs:
- targetType: inline
- pwsh: true
- script: |
- $base = git merge-base HEAD origin/main
- $changedFiles = git diff --name-only $base HEAD
-
- $sourceBranchName = "$(Build.SourceBranch)".Replace('refs/heads/', '')
- git checkout $sourceBranchName
-
- $filterModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'filter.ps1'
- & $filterModulesPath -MaxParallelBuildJobs "${{ variables.MaxParallelBuildJobs }}" -MaxParallelAnalyzeJobs "${{ variables.MaxParallelAnalyzeJobs }}" -MaxParallelTestWindowsJobs "${{ variables.MaxParallelTestWindowsJobs }}" -MaxParallelTestLinuxJobs "${{ variables.MaxParallelTestLinuxJobs }}" -MaxParallelTestMacJobs "${{ variables.MaxParallelTestMacJobs }}" -ChangedFiles $changedFiles -RepoRoot "$(Build.SourcesDirectory)"
-
- - task: PublishPipelineArtifact@1
- displayName: 'Upload filtered targets'
- inputs:
- targetPath: artifacts
- artifact: 'filter'
-
-- job: build
- displayName: "Build:"
- dependsOn: filter
- condition: ne(dependencies.filter.outputs['mtrx.buildTargets'], '{}')
- timeoutInMinutes: ${{ variables.BuildTimeoutInMinutes }}
- pool: ${{ variables.WindowsAgentPoolName }}
- strategy:
- matrix: $[ dependencies.filter.outputs['mtrx.buildTargets'] ]
- maxParallel: ${{ variables.MaxParallelBuildJobs }}
-
- steps:
- - checkout: self
- persistCredentials: true
- fetchTags: false
-
- - task: DownloadPipelineArtifact@2
- inputs:
- artifactName: 'filter'
- targetPath: artifacts
-
- - task: PowerShell@2
- name: build
- displayName: 'Build Targets'
- inputs:
- targetType: inline
- pwsh: true
- script: |
- git fetch origin "${{ variables.GenerationBranch }}"
- git checkout "${{ variables.GenerationBranch }}"
-
- $buildModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'build-modules.ps1'
- & $buildModulesPath -MatrixKey "$(MatrixKey)" -RepoRoot "$(Build.SourcesDirectory)"
-
- workingDirectory: $(Build.SourcesDirectory)
-
- - task: PublishPipelineArtifact@1
- displayName: 'Save build artifacts'
- inputs:
- targetPath: artifacts
- artifact: 'build-$(MatrixKey)'
+ jobs:
+ - job: filter
+ displayName: 'Filter Changed Modules'
+ timeoutInMinutes: ${{ variables.IntermediateStepTimeoutInMinutes }}
+ pool: ${{ variables.WindowsAgentPoolName }}
+ steps:
+ - checkout: self
+ persistCredentials: true
+ fetchTags: false
+
+ - task: PowerShell@2
+ name: checkout
+ displayName: 'Checkout Generation Branch'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ git fetch origin "${{ variables.GenerationBranch }}"
+ git checkout "${{ variables.GenerationBranch }}"
+
+ - download: current
+ patterns: '**/changed-*.patch'
+ displayName: 'Download all .patch artifacts'
+
+ - task: PowerShell@2
+ name: apply
+ displayName: 'Apply all patches'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ git config user.email "65331932+azure-powershell-bot@users.noreply.github.com"
+ git config user.name "azure-powershell-bot"
+
+ Write-Host "Applying all .patch files to ${{ variables.GenerationBranch }}..."
+ $patchFiles = Get-ChildItem -Path "$env:PIPELINE_WORKSPACE" -Recurse -Filter *.patch
+
+ # $patchFiles = Get-ChildItem -Path "$env:PIPELINE_WORKSPACE" -Recurse -Filter *.patch |
+ # Where-Object { $_.Directory.Parent.FullName -eq $env:PIPELINE_WORKSPACE }
+
+ foreach ($patch in $patchFiles) {
+ Write-Host "Applying patch: $($patch.FullName)"
+ git apply --whitespace=fix "$($patch.FullName)"
+ }
+
+ git add .
+ git commit -m "Apply matrix job patches to ${{ variables.GenerationBranch }}"
+ git push origin "${{ variables.GenerationBranch }}"
+
+ - task: PowerShell@2
+ name: mtrx
+ displayName: 'Generate Test Targets Matrix'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ $base = git merge-base HEAD origin/main
+ $changedFiles = git diff --name-only $base HEAD
+
+ $sourceBranchName = "$(Build.SourceBranch)".Replace('refs/heads/', '')
+ git checkout $sourceBranchName
+
+ $filterModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'filter.ps1'
+ & $filterModulesPath -MaxParallelBuildJobs "${{ variables.MaxParallelBuildJobs }}" -MaxParallelAnalyzeJobs "${{ variables.MaxParallelAnalyzeJobs }}" -MaxParallelTestWindowsJobs "${{ variables.MaxParallelTestWindowsJobs }}" -MaxParallelTestLinuxJobs "${{ variables.MaxParallelTestLinuxJobs }}" -MaxParallelTestMacJobs "${{ variables.MaxParallelTestMacJobs }}" -ChangedFiles $changedFiles -RepoRoot "$(Build.SourcesDirectory)"
+
+ - task: PublishPipelineArtifact@1
+ displayName: 'Upload filtered targets'
+ inputs:
+ targetPath: artifacts
+ artifact: 'filter'
+
+ - job: build
+ displayName: "Build:"
+ dependsOn: filter
+ condition: ne(dependencies.filter.outputs['mtrx.buildTargets'], '{}')
+ timeoutInMinutes: ${{ variables.BuildTimeoutInMinutes }}
+ pool: ${{ variables.WindowsAgentPoolName }}
+ strategy:
+ matrix: $[ dependencies.filter.outputs['mtrx.buildTargets'] ]
+ maxParallel: ${{ variables.MaxParallelBuildJobs }}
+
+ steps:
+ - checkout: self
+ fetchDepth: 1
+ fetchTags: false
+
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifactName: 'filter'
+ targetPath: artifacts
+
+ - task: PowerShell@2
+ name: build
+ displayName: 'Build Targets'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ git fetch origin "${{ variables.GenerationBranch }}"
+ git checkout "${{ variables.GenerationBranch }}"
+
+ $buildModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'build-modules.ps1'
+ & $buildModulesPath -MatrixKey "$(MatrixKey)" -RepoRoot "$(Build.SourcesDirectory)"
+
+ workingDirectory: $(Build.SourcesDirectory)
+
+ - task: PublishPipelineArtifact@1
+ displayName: 'Save build artifacts'
+ inputs:
+ targetPath: artifacts
+ artifact: 'build-$(MatrixKey)'
+ condition: always()
+
+ - stage: Test
+ dependsOn: Build
condition: always()
-
-- job: analyze
- displayName: "Analyze:"
- dependsOn:
- - filter
- - build
- condition: ne(dependencies.filter.outputs['mtrx.analyzeTargets'], '{}')
- timeoutInMinutes: ${{ variables.AnalysisTimeoutInMinutes }}
- pool: ${{ variables.WindowsAgentPoolName }}
- strategy:
- matrix: $[ dependencies.filter.outputs['mtrx.analyzeTargets'] ]
- maxParallel: ${{ variables.MaxParallelAnalyzeJobs }}
-
- steps:
- - checkout: none
-
- - task: PowerShell@2
- name: analyze
- displayName: 'Analyze modules'
- inputs:
- targetType: inline
- pwsh: true
- script: |
- Write-Host "Matrix Key: $(MatrixKey)"
- Write-Host "Analyze Targets: $(Target)"
-
-- job: test_windows
- displayName: "Test Windows:"
- dependsOn:
- - filter
- - build
- condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.TestWindowsTargets'], '{}'))
- timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
- pool: ${{ variables.WindowsAgentPoolName }}
- strategy:
- matrix: $[ dependencies.filter.outputs['mtrx.TestWindowsTargets'] ]
- maxParallel: ${{ variables.MaxParallelTestWindowsJobs }}
-
- steps:
- - checkout: none
-
- - task: PowerShell@2
- name: test
- displayName: 'Test Windows'
- inputs:
- targetType: inline
- pwsh: true
- script: |
- $testModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'test-module.ps1'
- & $testModulesPath -MatrixKey "$(MatrixKey)" -Target "$(Target)" -TestEnvName 'Windows' -RepoRoot "$(Build.SourcesDirectory)" -ArtifactRoot "$(Build.ArtifactStagingDirectory)"
-
-- job: test_linux
- displayName: "Test Linux:"
- dependsOn:
- - filter
- - build
- condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.TestLinuxTargets'], '{}'))
- timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
- pool: ${{ variables.LinuxAgentPoolName }}
- strategy:
- matrix: $[ dependencies.filter.outputs['mtrx.TestLinuxTargets'] ]
- maxParallel: ${{ variables.MaxParallelTestLinuxJobs }}
-
- steps:
- - checkout: none
-
- - task: PowerShell@2
- name: test
- displayName: 'Test Linux'
- inputs:
- targetType: inline
- pwsh: true
- script: |
- $testModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'test-module.ps1'
- & $testModulesPath -MatrixKey "$(MatrixKey)" -Target "$(Target)" -TestEnvName 'Linux' -RepoRoot "$(Build.SourcesDirectory)" -ArtifactRoot "$(Build.ArtifactStagingDirectory)"
-
-- job: test_mac
- displayName: "Test Mac:"
- dependsOn: filter
- condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.TestMacTargets'], '{}'))
- timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
- pool:
- name: ${{ variables.MacOSAgentPoolName }}
- vmImage: ${{ variables.MacOSAgentPoolVMImage }}
- strategy:
- matrix: $[ dependencies.filter.outputs['mtrx.TestMacTargets'] ]
- maxParallel: ${{ variables.MaxParallelTestMacJobs }}
-
- steps:
- - checkout: none
-
- - task: PowerShell@2
- name: test
- displayName: 'Test Mac'
- inputs:
- targetType: inline
- pwsh: true
- script: |
- $testModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'test-module.ps1'
- & $testModulesPath -MatrixKey "$(MatrixKey)" -Target "$(Target)" -TestEnvName 'Mac' -RepoRoot "$(Build.SourcesDirectory)" -ArtifactRoot "$(Build.ArtifactStagingDirectory)"
+ jobs:
+ - job: collect
+ displayName: "Collect Artifacts"
+ timeoutInMinutes: ${{ variables.IntermediateStepTimeoutInMinutes }}
+ pool: ${{ variables.WindowsAgentPoolName }}
+
+ steps:
+ - checkout: none
+
+ - download: current
+ patterns: |
+ **/debug/**
+ **/StaticAnalysis/**
+ displayName: 'Download build artifacts'
+
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifactName: 'filter'
+ targetPath: artifacts
+
+ - task: PowerShell@2
+ name: collect
+ displayName: 'Collect modules artifacts'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+
+ $workspace = "$env:PIPELINE_WORKSPACE"
+ $repoRoot = "$(Build.SourcesDirectory)"
+ $artifactsRoot = Join-Path $repoRoot "artifacts"
+ $debugArtifactDestPath = Join-Path $artifactsRoot "debug"
+ New-Item -ItemType Directory -Force -Path $debugArtifactDestPath | Out-Null
+
+ $copiedModules = @{}
+ $StaticAnalysisCopied = $false
+
+ Get-ChildItem -Path $workspace -Directory | ForEach-Object {
+ $debugArtifactSrcPath = Join-Path $_.FullName "debug"
+ $StaticAnalysisSrcDirectory = Join-Path $_.FullName 'StaticAnalysis'
+
+ if (Test-Path $debugArtifactSrcPath) {
+ Get-ChildItem -Path $debugArtifactSrcPath -Directory | ForEach-Object {
+ $moduleName = $_.Name
+
+ if (-not $copiedModules.ContainsKey($moduleName)) {
+ $destPath = Join-Path $debugArtifactDestPath $moduleName
+ Copy-Item -Path $_.FullName -Destination $destPath -Recurse
+ $copiedModules[$moduleName] = $true
+ Write-Host "Copied $moduleName from $debugArtifactSrcPath"
+ } else {
+ Write-Host "Skipped $moduleName from $debugArtifactSrcPath (already copied)"
+ }
+ }
+ }
+
+ if (Test-Path $StaticAnalysisSrcDirectory and -not $StaticAnalysisCopied) {
+ $destPath = Join-Path $artifactsRoot 'StaticAnalysis'
+ Copy-Item -Path $StaticAnalysisSrcDirectory -Destination $destPath -Recurse
+ $StaticAnalysisCopied = $true
+ Write-Host "Copied StaticAnalysis from $StaticAnalysisSrcDirectory"
+ }
+
+ }
+
+ Get-ChildItem -Path $artifactsRoot -Directory | ForEach-Object {
+ Write-Host "Artifact Directory - $($_.Name)"
+ }
+
+ Get-ChildItem -Path $debugArtifactDestPath -Directory | ForEach-Object {
+ Write-Host "Debug Directory - $($_.Name)"
+ }
+
+
+ - task: PublishPipelineArtifact@1
+ displayName: 'Upload collected artifacts'
+ inputs:
+ targetPath: artifacts
+ artifact: 'collect'
+
+ - job: analyze
+ displayName: "Analyze:"
+ dependsOn:
+ - collect
+ condition: ne(stageDependencies.Build.filter.outputs['mtrx.analyzeTargets'], '{}')
+ timeoutInMinutes: ${{ variables.AnalysisTimeoutInMinutes }}
+ pool: ${{ variables.WindowsAgentPoolName }}
+ strategy:
+ matrix: $[ stageDependencies.Build.filter.outputs['mtrx.analyzeTargets'] ]
+ maxParallel: ${{ variables.MaxParallelAnalyzeJobs }}
+
+ steps:
+ - checkout: self
+ fetchDepth: 1
+ sparseCheckoutDirectories: tools/ .azure-pipelines/
+ fetchTags: false
+
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifact: collect
+ path: $(Build.SourcesDirectory)/artifacts
+ displayName: 'Download collected artifact to artifacts folder'
+
+ - task: NodeTool@0
+ displayName: Install autorest
+ inputs:
+ versionSpec: '14.17.1'
+ command: custom
+ verbose: false
+ customCommand: install autorest@latest
+
+ - task: PowerShell@2
+ displayName: Setup environment for autorest
+ inputs:
+ targetType: inline
+ script: "$env:NODE_OPTIONS=\"--max-old-space-size=65536\""
+ pwsh: true
+
+ - task: UseDotNet@2
+ displayName: 'Use .NET SDK for Static Analysis'
+ inputs:
+ packageType: sdk
+ version: 8.x
+
+ - task: PowerShell@2
+ name: analyze
+ displayName: 'Analyze modules'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ Write-Host "Matrix Key: $(MatrixKey)"
+
+ Write-Host "Top-level contents of $(Build.SourcesDirectory):"
+ Get-ChildItem -Path "$(Build.SourcesDirectory)" | ForEach-Object {
+ Write-Host $_.FullName
+ }
+
+ $analyseModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'analyse-modules.ps1'
+ & $analyseModulesPath -MatrixKey "$(MatrixKey)" -RepoRoot "$(Build.SourcesDirectory)"
+
+
+ - task: PublishPipelineArtifact@1
+ displayName: 'Save Analyse Report'
+ inputs:
+ targetPath: artifacts
+ artifact: 'analyse-$(MatrixKey)'
+ condition: always()
+
+
+ # - job: test_windows
+ # displayName: "Test Windows:"
+ # dependsOn:
+ # - filter
+ # - build
+ # condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.TestWindowsTargets'], '{}'))
+ # timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
+ # pool: ${{ variables.WindowsAgentPoolName }}
+ # strategy:
+ # matrix: $[ dependencies.filter.outputs['mtrx.TestWindowsTargets'] ]
+ # maxParallel: ${{ variables.MaxParallelTestWindowsJobs }}
+
+ # steps:
+ # - checkout: none
+
+ # - task: PowerShell@2
+ # name: test
+ # displayName: 'Test Windows'
+ # inputs:
+ # targetType: inline
+ # pwsh: true
+ # script: |
+ # $testModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'test-module.ps1'
+ # & $testModulesPath -MatrixKey "$(MatrixKey)" -Target "$(Target)" -TestEnvName 'Windows' -RepoRoot "$(Build.SourcesDirectory)" -ArtifactRoot "$(Build.ArtifactStagingDirectory)"
+
+ # - job: test_linux
+ # displayName: "Test Linux:"
+ # dependsOn:
+ # - filter
+ # - build
+ # condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.TestLinuxTargets'], '{}'))
+ # timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
+ # pool: ${{ variables.LinuxAgentPoolName }}
+ # strategy:
+ # matrix: $[ dependencies.filter.outputs['mtrx.TestLinuxTargets'] ]
+ # maxParallel: ${{ variables.MaxParallelTestLinuxJobs }}
+
+ # steps:
+ # - checkout: none
+
+ # - task: PowerShell@2
+ # name: test
+ # displayName: 'Test Linux'
+ # inputs:
+ # targetType: inline
+ # pwsh: true
+ # script: |
+ # $testModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'test-module.ps1'
+ # & $testModulesPath -MatrixKey "$(MatrixKey)" -Target "$(Target)" -TestEnvName 'Linux' -RepoRoot "$(Build.SourcesDirectory)" -ArtifactRoot "$(Build.ArtifactStagingDirectory)"
+
+ # - job: test_mac
+ # displayName: "Test Mac:"
+ # dependsOn: filter
+ # condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.TestMacTargets'], '{}'))
+ # timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
+ # pool:
+ # name: ${{ variables.MacOSAgentPoolName }}
+ # vmImage: ${{ variables.MacOSAgentPoolVMImage }}
+ # strategy:
+ # matrix: $[ dependencies.filter.outputs['mtrx.TestMacTargets'] ]
+ # maxParallel: ${{ variables.MaxParallelTestMacJobs }}
+
+ # steps:
+ # - checkout: none
+
+ # - task: PowerShell@2
+ # name: test
+ # displayName: 'Test Mac'
+ # inputs:
+ # targetType: inline
+ # pwsh: true
+ # script: |
+ # $testModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'test-module.ps1'
+ # & $testModulesPath -MatrixKey "$(MatrixKey)" -Target "$(Target)" -TestEnvName 'Mac' -RepoRoot "$(Build.SourcesDirectory)" -ArtifactRoot "$(Build.ArtifactStagingDirectory)"
From 9947f1854b1f804d35c9112792749667c6f1284e Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Fri, 2 May 2025 15:16:00 +1000
Subject: [PATCH 21/33] analyse 2
---
.azure-pipelines/batch-generation.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 093e38bb96bc..f508bed254b6 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -280,7 +280,7 @@ stages:
}
}
- if (Test-Path $StaticAnalysisSrcDirectory and -not $StaticAnalysisCopied) {
+ if ((Test-Path $StaticAnalysisSrcDirectory) -and (-not $StaticAnalysisCopied)) {
$destPath = Join-Path $artifactsRoot 'StaticAnalysis'
Copy-Item -Path $StaticAnalysisSrcDirectory -Destination $destPath -Recurse
$StaticAnalysisCopied = $true
From 019934cd6f17e7dc813a6bd044191ff93d36abae Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Tue, 6 May 2025 16:17:21 +1000
Subject: [PATCH 22/33] test
---
.../PipelineSteps/BatchGeneration/filter.ps1 | 14 +-
.../BatchGeneration/test-module.ps1 | 45 ------
.../BatchGeneration/test-modules.ps1 | 58 ++++++++
.azure-pipelines/batch-generation.yml | 132 ++++++------------
.../util/batch-generation-test-job.yml | 61 ++++++++
.../Get-AzAstroOrganization.Recording.json | 2 +-
6 files changed, 168 insertions(+), 144 deletions(-)
delete mode 100644 .azure-pipelines/PipelineSteps/BatchGeneration/test-module.ps1
create mode 100644 .azure-pipelines/PipelineSteps/BatchGeneration/test-modules.ps1
create mode 100644 .azure-pipelines/util/batch-generation-test-job.yml
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
index 93157bc51fa5..262ce709d966 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
@@ -45,13 +45,13 @@ $groupedBuildModules = Group-Modules -Modules $changedModules -MaxParallelJobs $
Write-Matrix -GroupedModules $groupedBuildModules -VariableName 'buildTargets' -RepoRoot $RepoRoot
$groupedAnalyzeModules = Group-Modules -Modules $changedModules -MaxParallelJobs $MaxParallelAnalyzeJobs
-Write-Matrix -GroupedModules $groupedAnalyzeModules -variableName 'analyzeTargets' -RepoRoot $RepoRoot
+Write-Matrix -GroupedModules $groupedAnalyzeModules -VariableName 'analyzeTargets' -RepoRoot $RepoRoot
-# $groupedTestWindowsModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParallelTestWindowsJobs
-# Write-Matrix -variableName 'TestWindowsTargets' -groupedSubModules $groupedTestWindowsModules
+$groupedTestWindowsModules = Group-Modules -Modules $changedSubModules -MaxParallelJobs $MaxParallelTestWindowsJobs
+Write-Matrix -GroupedModules $groupedTestWindowsModules -VariableName 'testWindowsTargets' -RepoRoot $RepoRoot
-# $groupedTestLinuxModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParallelTestLinuxJobs
-# Write-Matrix -variableName 'TestLinuxTargets' -groupedSubModules $groupedTestLinuxModules
+$groupedTestLinuxModules = Group-Modules -Modules $changedSubModules -MaxParallelJobs $MaxParallelTestLinuxJobs
+Write-Matrix -GroupedModules $groupedTestLinuxModules -VariableName 'testLinuxTargets' -RepoRoot $RepoRoot
-# $groupedTestMacModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParallelTestMacJobs
-# Write-Matrix -variableName 'TestMacTargets' -groupedSubModules $groupedTestMacModules
+$groupedTestMacModules = Group-Modules -Modules $changedSubModules -MaxParallelJobs $MaxParallelTestMacJobs
+Write-Matrix -GroupedModules $groupedTestMacModules -VariableName 'testMacOSTargets' -RepoRoot $RepoRoot
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/test-module.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/test-module.ps1
deleted file mode 100644
index 41d0c6fe7cb7..000000000000
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/test-module.ps1
+++ /dev/null
@@ -1,45 +0,0 @@
-param (
- [string]$MatrixKey,
- [string]$Target,
- [string]$TestEnvName,
- [string]$RepoRoot,
- [string]$ArtifactRoot
-)
-
-Write-Host "Matrix Key: $MatrixKey"
-Write-Host "Test $($TestEnvName): $Target"
-
-$modules = $Target -split ','
-$results = @()
-
-foreach ($module in $modules) {
- $startTime = Get-Date
- $result = @{
- Module = $module
- Status = "Success"
- DurationSeconds = 0
- Error = ""
- }
-
- try {
- Write-Host "Testing module: $module"
- $subModulePath = Join-Path $RepoRoot 'src' $module
- # TODO(Bernard) Remove log after test
- Write-Host "Sub module path: $subModulePath"
- Set-Location -Path $subModulePath
- & ".\test-module.ps1"
- }
- catch {
- Write-Warning "Failed to test module: $module"
- $result.Status = "Failed"
- $result.Error = $_.Exception.Message
- }
- finally {
- $endTime = Get-Date
- $result.DurationSeconds = ($endTime - $startTime).TotalSeconds
- $results += $result
- }
-}
-
-$reportPath = Join-Path $ArtifactRoot "Test$($TestEnvName)Report-$MatrixKey.json"
-$results | ConvertTo-Json -Depth 3 | Out-File -FilePath $reportPath -Encoding utf8
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/test-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/test-modules.ps1
new file mode 100644
index 000000000000..58d8f05c5806
--- /dev/null
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/test-modules.ps1
@@ -0,0 +1,58 @@
+param (
+ [string]$MatrixKey,
+ [string]$TestEnvName,
+ [string]$RepoRoot
+)
+
+$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
+Import-Module $utilFilePath -Force
+$subModuleGroup = Get-Targets -RepoRoot $RepoRoot -TargetsOutputFileName "test$($TestEnvName)Targets.json" -MatrixKey $MatrixKey
+
+if ($TestEnvName -eq 'Linux') {
+ $accountsModulePath = Join-Path $RepoRoot 'artifacts' 'Debug' "Az.Accounts" "Az.Accounts.psd1"
+ Import-Module $accountsModulePath -Force
+}
+
+$results = @()
+
+foreach ($subModule in $subModuleGroup) {
+ $startTime = Get-Date
+ $moduleName, $subModuleName = $subModule -split '/'
+ $result = @{
+ Module = $moduleName
+ SubModule = $subModuleName
+ Status = "Success"
+ DurationSeconds = 0
+ Error = ""
+ }
+
+ try {
+ Write-Host "Testing sub module: $subModule"
+ $subModulePath = Join-Path $RepoRoot 'artifacts' 'Debug' "Az.$ModuleName" $subModuleName
+ Push-Location $subModulePath
+
+ & ".\test-module.ps1"
+
+ if ($LASTEXITCODE -ne 0) {
+ Write-Warning "ā Test failed: $subModule (exit code $LASTEXITCODE)"
+ $result.Status = "Failed"
+ $result.Error = "Test failed with exit code $LASTEXITCODE"
+ }
+ }
+ catch {
+ Write-Warning "Failed to test module: $module"
+ Write-Warning "Error message: $($_.Exception.Message)"
+ $result.Status = "Failed"
+ $result.Error = $_.Exception.Message
+ }
+ finally {
+ $endTime = Get-Date
+ $result.DurationSeconds = ($endTime - $startTime).TotalSeconds
+ $results += $result
+ Pop-Location
+ }
+}
+
+$artifactRoot = Join-Path $RepoRoot 'artifacts'
+$reportPath = Join-Path $artifactRoot "Test$($TestEnvName)Report-$MatrixKey.json"
+$results | ConvertTo-Json -Depth 3 | Out-File -FilePath $reportPath -Encoding utf8
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index f508bed254b6..b1a8f46a6784 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -8,8 +8,8 @@ variables:
MaxParallelBuildJobs: 15
MaxParallelAnalyzeJobs: 5
MaxParallelTestWindowsJobs: 5
- MaxParallelTestLinuxJobs: 5
- MaxParallelTestMacJobs: 5
+ MaxParallelTestLinuxJobs: 4
+ MaxParallelTestMacJobs: 2
WindowsAgentPoolName: pool-windows-2019
LinuxAgentPoolName: pool-ubuntu-2004
MacOSAgentPoolName: 'Azure Pipelines'
@@ -67,7 +67,10 @@ stages:
- job: generate
displayName: "Batch Generate: "
dependsOn: prepare
- condition: ne(dependencies.prepare.outputs['mtrx.generateTargets'], '{}')
+ condition: and(
+ ne(dependencies.prepare.outputs['mtrx.generateTargets'], ''),
+ ne(dependencies.prepare.outputs['mtrx.generateTargets'], '{}')
+ )
timeoutInMinutes: ${{ variables.GenerateTimeoutInMinutes }}
pool: ${{ variables.WindowsAgentPoolName }}
strategy:
@@ -182,7 +185,10 @@ stages:
- job: build
displayName: "Build:"
dependsOn: filter
- condition: ne(dependencies.filter.outputs['mtrx.buildTargets'], '{}')
+ condition: and(
+ ne(dependencies.filter.outputs['mtrx.buildTargets'], ''),
+ ne(dependencies.filter.outputs['mtrx.buildTargets'], '{}')
+ )
timeoutInMinutes: ${{ variables.BuildTimeoutInMinutes }}
pool: ${{ variables.WindowsAgentPoolName }}
strategy:
@@ -235,7 +241,7 @@ stages:
- download: current
patterns: |
- **/debug/**
+ **/Debug/**
**/StaticAnalysis/**
displayName: 'Download build artifacts'
@@ -255,14 +261,14 @@ stages:
$workspace = "$env:PIPELINE_WORKSPACE"
$repoRoot = "$(Build.SourcesDirectory)"
$artifactsRoot = Join-Path $repoRoot "artifacts"
- $debugArtifactDestPath = Join-Path $artifactsRoot "debug"
+ $debugArtifactDestPath = Join-Path $artifactsRoot "Debug"
New-Item -ItemType Directory -Force -Path $debugArtifactDestPath | Out-Null
$copiedModules = @{}
$StaticAnalysisCopied = $false
Get-ChildItem -Path $workspace -Directory | ForEach-Object {
- $debugArtifactSrcPath = Join-Path $_.FullName "debug"
+ $debugArtifactSrcPath = Join-Path $_.FullName "Debug"
$StaticAnalysisSrcDirectory = Join-Path $_.FullName 'StaticAnalysis'
if (Test-Path $debugArtifactSrcPath) {
@@ -308,7 +314,10 @@ stages:
displayName: "Analyze:"
dependsOn:
- collect
- condition: ne(stageDependencies.Build.filter.outputs['mtrx.analyzeTargets'], '{}')
+ condition: and(
+ ne(stageDependencies.Build.filter.outputs['mtrx.analyzeTargets'], ''),
+ ne(stageDependencies.Build.filter.outputs['mtrx.analyzeTargets'], '{}')
+ )
timeoutInMinutes: ${{ variables.AnalysisTimeoutInMinutes }}
pool: ${{ variables.WindowsAgentPoolName }}
strategy:
@@ -318,7 +327,6 @@ stages:
steps:
- checkout: self
fetchDepth: 1
- sparseCheckoutDirectories: tools/ .azure-pipelines/
fetchTags: false
- task: DownloadPipelineArtifact@2
@@ -357,15 +365,9 @@ stages:
script: |
Write-Host "Matrix Key: $(MatrixKey)"
- Write-Host "Top-level contents of $(Build.SourcesDirectory):"
- Get-ChildItem -Path "$(Build.SourcesDirectory)" | ForEach-Object {
- Write-Host $_.FullName
- }
-
$analyseModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'analyse-modules.ps1'
& $analyseModulesPath -MatrixKey "$(MatrixKey)" -RepoRoot "$(Build.SourcesDirectory)"
-
- task: PublishPipelineArtifact@1
displayName: 'Save Analyse Report'
inputs:
@@ -373,79 +375,27 @@ stages:
artifact: 'analyse-$(MatrixKey)'
condition: always()
-
- # - job: test_windows
- # displayName: "Test Windows:"
- # dependsOn:
- # - filter
- # - build
- # condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.TestWindowsTargets'], '{}'))
- # timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
- # pool: ${{ variables.WindowsAgentPoolName }}
- # strategy:
- # matrix: $[ dependencies.filter.outputs['mtrx.TestWindowsTargets'] ]
- # maxParallel: ${{ variables.MaxParallelTestWindowsJobs }}
-
- # steps:
- # - checkout: none
-
- # - task: PowerShell@2
- # name: test
- # displayName: 'Test Windows'
- # inputs:
- # targetType: inline
- # pwsh: true
- # script: |
- # $testModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'test-module.ps1'
- # & $testModulesPath -MatrixKey "$(MatrixKey)" -Target "$(Target)" -TestEnvName 'Windows' -RepoRoot "$(Build.SourcesDirectory)" -ArtifactRoot "$(Build.ArtifactStagingDirectory)"
-
- # - job: test_linux
- # displayName: "Test Linux:"
- # dependsOn:
- # - filter
- # - build
- # condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.TestLinuxTargets'], '{}'))
- # timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
- # pool: ${{ variables.LinuxAgentPoolName }}
- # strategy:
- # matrix: $[ dependencies.filter.outputs['mtrx.TestLinuxTargets'] ]
- # maxParallel: ${{ variables.MaxParallelTestLinuxJobs }}
-
- # steps:
- # - checkout: none
-
- # - task: PowerShell@2
- # name: test
- # displayName: 'Test Linux'
- # inputs:
- # targetType: inline
- # pwsh: true
- # script: |
- # $testModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'test-module.ps1'
- # & $testModulesPath -MatrixKey "$(MatrixKey)" -Target "$(Target)" -TestEnvName 'Linux' -RepoRoot "$(Build.SourcesDirectory)" -ArtifactRoot "$(Build.ArtifactStagingDirectory)"
-
- # - job: test_mac
- # displayName: "Test Mac:"
- # dependsOn: filter
- # condition: and(succeeded(), ne(dependencies.filter.outputs['mtrx.TestMacTargets'], '{}'))
- # timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
- # pool:
- # name: ${{ variables.MacOSAgentPoolName }}
- # vmImage: ${{ variables.MacOSAgentPoolVMImage }}
- # strategy:
- # matrix: $[ dependencies.filter.outputs['mtrx.TestMacTargets'] ]
- # maxParallel: ${{ variables.MaxParallelTestMacJobs }}
-
- # steps:
- # - checkout: none
-
- # - task: PowerShell@2
- # name: test
- # displayName: 'Test Mac'
- # inputs:
- # targetType: inline
- # pwsh: true
- # script: |
- # $testModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'test-module.ps1'
- # & $testModulesPath -MatrixKey "$(MatrixKey)" -Target "$(Target)" -TestEnvName 'Mac' -RepoRoot "$(Build.SourcesDirectory)" -ArtifactRoot "$(Build.ArtifactStagingDirectory)"
-
+ - template: util/batch-generation-test-job.yml
+ parameters:
+ jobName: 'test_windows'
+ OSName: 'Windows'
+ timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
+ agentPoolName: ${{ variables.WindowsAgentPoolName }}
+ maxParallel: ${{ variables.MaxParallelTestWindowsJobs }}
+
+ - template: util/batch-generation-test-job.yml
+ parameters:
+ jobName: 'test_linux'
+ OSName: 'Linux'
+ timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
+ agentPoolName: ${{ variables.LinuxAgentPoolName }}
+ maxParallel: ${{ variables.MaxParallelTestLinuxJobs }}
+
+ - template: util/batch-generation-test-job.yml
+ parameters:
+ jobName: 'test_mac'
+ OSName: 'MacOS'
+ timeoutInMinutes: ${{ variables.TestTimeoutInMinutes }}
+ agentPoolName: ${{ variables.MacOSAgentPoolName }}
+ agentPoolVMImage: ${{ variables.MacOSAgentPoolVMImage }}
+ maxParallel: ${{ variables.MaxParallelTestMacJobs }}
diff --git a/.azure-pipelines/util/batch-generation-test-job.yml b/.azure-pipelines/util/batch-generation-test-job.yml
new file mode 100644
index 000000000000..05604f45408b
--- /dev/null
+++ b/.azure-pipelines/util/batch-generation-test-job.yml
@@ -0,0 +1,61 @@
+parameters:
+ jobName: 'test_windows'
+ OSName: 'Windows'
+ timeoutInMinutes: 60
+ agentPoolName: 'pool-windows-2019'
+ agentPoolVMImage: ''
+ maxParallel: 3
+
+jobs:
+- job: ${{ parameters.jobName }}
+ displayName: "Test ${{ parameters.OSName }}:"
+ dependsOn:
+ - collect
+ condition: and(
+ ne(stageDependencies.Build.filter.outputs['mtrx.test${{ parameters.OSName }}Targets'], ''),
+ ne(stageDependencies.Build.filter.outputs['mtrx.test${{ parameters.OSName }}Targets'], '{}')
+ )
+ timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+ pool:
+ name: ${{ parameters.agentPoolName }}
+ ${{ if ne(parameters.agentPoolVMImage, '') }}:
+ vmImage: ${{ parameters.agentPoolVMImage }}
+ strategy:
+ matrix: $[ stageDependencies.Build.filter.outputs['mtrx.test${{ parameters.OSName }}Targets'] ]
+ maxParallel: ${{ parameters.maxParallel }}
+
+ steps:
+ #TODO(Bernard) check if can skip checkout step after move this to template
+ - checkout: self
+ fetchDepth: 1
+ fetchTags: false
+
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifact: collect
+ path: $(Build.SourcesDirectory)/artifacts
+ displayName: 'Download collected artifact to artifacts folder'
+
+ - task: UseDotNet@2
+ displayName: 'Use .NET SDK for Test'
+ inputs:
+ packageType: sdk
+ version: 8.x
+
+ - task: PowerShell@2
+ name: test
+ displayName: 'Test ${{ parameters.OSName }}'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ $testModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'test-modules.ps1'
+ & $testModulesPath -MatrixKey "$(MatrixKey)" -TestEnvName ${{ parameters.OSName }} -RepoRoot "$(Build.SourcesDirectory)"
+
+ - task: PublishPipelineArtifact@1
+ displayName: 'Save Test ${{ parameters.OSName }} Report'
+ inputs:
+ targetPath: artifacts
+ artifact: 'test${{ parameters.OSName }}-$(MatrixKey)'
+ condition: always()
+
\ No newline at end of file
diff --git a/src/Astro/Astro.Autorest/test/Get-AzAstroOrganization.Recording.json b/src/Astro/Astro.Autorest/test/Get-AzAstroOrganization.Recording.json
index e02a62dcf540..7a81650e0073 100644
--- a/src/Astro/Astro.Autorest/test/Get-AzAstroOrganization.Recording.json
+++ b/src/Astro/Astro.Autorest/test/Get-AzAstroOrganization.Recording.json
@@ -18,7 +18,7 @@
}
},
"Response": {
- "StatusCode": 200,
+ "StatusCode": 800,
"Headers": {
"Cache-Control": [ "no-cache" ],
"Pragma": [ "no-cache" ],
From 5a09e98767b9c69244943b33fd9e542eb781f1ee Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Mon, 12 May 2025 15:53:24 +1000
Subject: [PATCH 23/33] report
---
.../BatchGeneration/analyse-modules.ps1 | 1 +
.../batch-generate-modules.ps1 | 15 ++-
.../BatchGeneration/build-modules.ps1 | 1 +
.../PipelineSteps/BatchGeneration/prepare.ps1 | 96 ++++---------------
.../PipelineSteps/BatchGeneration/report.ps1 | 62 ++++++++++++
.../BatchGeneration/test-modules.ps1 | 17 +++-
.../PipelineSteps/BatchGeneration/util.psm1 | 34 +++++++
.azure-pipelines/batch-generation.yml | 46 +++++++--
8 files changed, 180 insertions(+), 92 deletions(-)
create mode 100644 .azure-pipelines/PipelineSteps/BatchGeneration/report.ps1
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/analyse-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/analyse-modules.ps1
index 0ec348f6a58a..16c4bf3b168f 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/analyse-modules.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/analyse-modules.ps1
@@ -20,6 +20,7 @@ foreach ($moduleName in $moduleGroup) {
$startTime = Get-Date
$result = @{
+ MatrixKey = $MatrixKey
Module = $moduleName
Status = "Success"
DurationSeconds = 0
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
index 44bb6d427368..f4f027f7c9d6 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
@@ -3,9 +3,9 @@ param (
[string]$RepoRoot
)
-$generateTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generateTargets.json"
-$generateTargets = Get-Content -Path $generateTargetsOutPutFile -Raw | ConvertFrom-Json
-$moduleGroup = $generateTargets.$MatrixKey
+$generationTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generationTargets.json"
+$generationTargets = Get-Content -Path $generationTargetsOutPutFile -Raw | ConvertFrom-Json
+$moduleGroup = $generationTargets.$MatrixKey
Write-Host "##[group]Generating module group $MatrixKey"
foreach ($key in $moduleGroup.PSObject.Properties.Name | Sort-Object) {
$values = $moduleGroup.$key -join ', '
@@ -32,6 +32,7 @@ foreach ($moduleName in $sortedModuleNames) {
$moduleResult = @{
Module = $moduleName
DurationSeconds = 0
+ Status = "Success"
SubModules = @()
}
@@ -40,6 +41,7 @@ foreach ($moduleName in $sortedModuleNames) {
Write-Host "Regenerating SubModule: $subModuleName"
$subModuleStartTime = Get-Date
$subModuleResult = @{
+ MatrixKey = $MatrixKey
SubModule = $subModuleName
Status = "Success"
DurationSeconds = 0
@@ -52,10 +54,14 @@ foreach ($moduleName in $sortedModuleNames) {
Remove-Item -Path $generateLog -Recurse -Force
}
New-Item -ItemType File -Force -Path $generateLog
-
+ # TODO(Bernard) Remove exception for EmailService.Autorest after test
+ if ($subModuleName -eq "EmailService.Autorest") {
+ throw "Something went wrong"
+ }
if (-not (Update-GeneratedSubModule -ModuleRootName $moduleName -SubModuleName $subModuleName -SourceDirectory $sourceDirectory -GeneratedDirectory $generatedDirectory -GenerateLog $generateLog -IsInvokedByPipeline $true)) {
Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
Write-Warning "log can be found at $generateLog"
+ $moduleResult.Status = "Failed"
$subModuleResult.Status = "Failed"
$subModuleResult.Error = "Update-GeneratedSubModule function returned false."
}
@@ -63,6 +69,7 @@ foreach ($moduleName in $sortedModuleNames) {
} catch {
Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
Write-Warning "Error message: $($_.Exception.Message)"
+ $moduleResult.Status = "Failed"
$subModuleResult.Status = "Failed"
$subModuleResult.Error = $_.Exception.Message
} finally {
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/build-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/build-modules.ps1
index 8952d31e8e3c..ca212b900b2e 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/build-modules.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/build-modules.ps1
@@ -15,6 +15,7 @@ foreach ($moduleName in $moduleGroup) {
$startTime = Get-Date
$result = @{
+ MatrixKey = $MatrixKey
Module = $moduleName
Status = "Success"
DurationSeconds = 0
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
index 386b814071a7..3c072b44392b 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
@@ -4,43 +4,13 @@ param (
[int]$MaxParallelJobs = 3
)
-$srcPath = Join-Path $RepoRoot 'src'
-function Get-SubModuleWithAutorestV4 {
- param (
- [string]$srcPath
- )
-
- $result = @{}
-
- Get-ChildItem -Path $srcPath -Directory | ForEach-Object {
- $module = $_
-
- Get-ChildItem -Path $module.FullName -Directory | Where-Object {
- $_.Name -like '*.autorest'
- } | ForEach-Object {
- $subModule = $_
-
- $readmePath = Join-Path $subModule.FullName 'README.md'
+$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
+Import-Module $utilFilePath -Force
- if (Test-Path $readmePath) {
- $readmeContent = Get-Content -Path $readmePath -Raw
-
- if ($readmeContent -notmatch 'use-extension:\s+"@autorest/powershell":\s+"3.x"') {
- if ($result.ContainsKey($module.Name)) {
- $result[$module.Name] += $subModule.Name
- } else {
- $result[$module.Name] = @($subModule.Name)
- }
- }
- }
- }
- }
-
- return $result
-}
+$srcPath = Join-Path $RepoRoot 'src'
# TODO(Bernard): Use real function after test
-# $modules = Get-SubModuleWithAutorestV4 -srcPath $srcPath
-$modules = @{
+# $moduleMap = Get-AutorestV4ModuleMap -srcPath $srcPath
+$moduleMap = @{
"DeviceRegistry" = @("DeviceRegistry.Autorest")
"ArcGateway" = @("ArcGateway.Autorest")
"Chaos" = @("Chaos.Autorest")
@@ -49,68 +19,40 @@ $modules = @{
"Astro" = @("Astro.Autorest")
"ImageBuilder" = @("ImageBuilder.Autorest")
}
-$modules = $modules.GetEnumerator() | ForEach-Object {
- [PSCustomObject]@{
- ModuleName = $_.Key
- SubModules = ($_.Value | Sort-Object)
- }
-} | Sort-Object -Property ModuleName
-Write-Host "Total matched modules: $($modules.Count)"
-
-function Group-Modules {
- param (
- [array]$modules,
- [int]$maxParallelJobs
- )
-
- $count = $modules.Count
- $n = [Math]::Min($count, $maxParallelJobs)
- if ($n -eq 0) {
- return @()
- }
-
- $result = @()
- $sizePerGroup = [Math]::Ceiling($count / $n)
-
- for ($i = 0; $i -lt $count; $i += $sizePerGroup) {
- $group = $modules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
- $result += ,$group
- }
-
- return $result
-}
+Write-Host "Total matched modules: $($moduleMap.Count)"
+$modules = @($moduleMap.Keys | Sort-Object)
$groupedModules = Group-Modules -modules $modules -maxParallelJobs $MaxParallelJobs
Write-Host "Total module groups: $($groupedModules.Count)"
$index = 0
-$generateTargets = @{}
+$generationTargets = @{}
foreach ($moduleGroup in $groupedModules) {
- Write-Host "##[group]Prepareing module group $($index + 1)"
+ Write-Host "##[group]Prepareing module group $($index + 1) with $($moduleGroup.Count) modules"
$mergedModules = @{}
- foreach ($moduleObj in $moduleGroup) {
- Write-Host "Module $($moduleObj.ModuleName): $($moduleObj.SubModules -join ',')"
- $mergedModules[$moduleObj.ModuleName] = @($moduleObj.SubModules)
+ foreach ($moduleName in $moduleGroup) {
+ Write-Host "Module $($moduleName): $($moduleMap[$moduleName] -join ',')"
+ $mergedModules[$moduleName] = @($moduleMap[$moduleName])
$subIndex++
}
$key = ($index + 1).ToString() + "-" + $moduleGroup.Count
- $generateTargets[$key] = $mergedModules
+ $generationTargets[$key] = $mergedModules
$MatrixStr = "$MatrixStr,'$key':{'MatrixKey':'$key'}"
Write-Host "##[endgroup]"
Write-Host
$index++
}
-$generateTargetsOutputDir = Join-Path $RepoRoot "artifacts"
-if (-not (Test-Path -Path $generateTargetsOutputDir)) {
- New-Item -ItemType Directory -Path $generateTargetsOutputDir
+$generationTargetsOutputDir = Join-Path $RepoRoot "artifacts"
+if (-not (Test-Path -Path $generationTargetsOutputDir)) {
+ New-Item -ItemType Directory -Path $generationTargetsOutputDir
}
-$generateTargetsOutputFile = Join-Path $generateTargetsOutputDir "generateTargets.json"
-$generateTargets | ConvertTo-Json -Depth 5 | Out-File -FilePath $generateTargetsOutputFile -Encoding utf8
+$generationTargetsOutputFile = Join-Path $generationTargetsOutputDir "generationTargets.json"
+$generationTargets | ConvertTo-Json -Depth 5 | Out-File -FilePath $generationTargetsOutputFile -Encoding utf8
if ($MatrixStr -and $MatrixStr.Length -gt 1) {
$MatrixStr = $MatrixStr.Substring(1)
}
-Write-Host "##vso[task.setVariable variable=generateTargets;isOutput=true]{$MatrixStr}"
+Write-Host "##vso[task.setVariable variable=generationTargets;isOutput=true]{$MatrixStr}"
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/report.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/report.ps1
new file mode 100644
index 000000000000..8d2c64f6a1e3
--- /dev/null
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/report.ps1
@@ -0,0 +1,62 @@
+param (
+ [string]$RepoRoot,
+ [string]$PipelineWorkspace
+)
+
+$reportFilePattern = @{
+ Generation = "GenerationReport-*.json"
+ Build = "BuildReport-*.json"
+ Analyze = "AnalyseReport-*.json"
+ TestWindows = "TestWindowsReport-*.json"
+ TestLinux = "TestLinuxReport-*.json"
+ TestMacOS = "TestMacOSReport-*.json"
+}
+
+$artifactRoot = Join-Path $RepoRoot 'artifacts'
+if (-not (Test-Path -Path $artifactRoot)) {
+ New-Item -Path $artifactRoot -ItemType Directory
+}
+$failedReports = @{}
+
+foreach ($pattern in $reportFilePattern.GetEnumerator()) {
+ Write-Host "Filtering $($pattern.Key) Report with pattern: $($pattern.Value)"
+ $allResults = @()
+
+ foreach ($dir in Get-ChildItem -Path $PipelineWorkspace -Directory) {
+ $file = Get-ChildItem -Path $dir.FullName -Filter $pattern.Value -File | Select-Object -First 1
+ if ($file) {
+ Write-Host "Found file: $($file.FullName)"
+ $jsonContent = Get-Content $file.FullName -Raw | ConvertFrom-Json
+
+ if ($jsonContent -isnot [System.Collections.IEnumerable] -or $jsonContent -is [string]) {
+ $jsonContent = @($jsonContent)
+ }
+
+ $allResults += $jsonContent
+ }
+ }
+
+ Write-Host "$($pattern.Key): $($allResults.Count) result(s) found."
+ $reportPath = Join-Path $artifactRoot "$($pattern.Key)Report.json"
+ $allResults | ConvertTo-Json -Depth 10 | Set-Content -Path $reportPath -Encoding UTF8
+ Write-Host "Written report to $reportPath"
+
+ $failed = $allResults | Where-Object { $_.Status -ne "Success" }
+ if ($failed.Count -gt 0) {
+ $failedReports[$pattern.Key] = $failed
+ }
+}
+
+if ($failedReports.Count -eq 0) {
+ Write-Host "`nā
Exist Successfully: All reports passed."
+ exit 0
+} else {
+ Write-Host "`nā Exist with Errors: Some reports failed."
+ foreach ($key in $failedReports.Keys) {
+ Write-Host "##[group]Failed entries in $key"
+ $failedReports[$key] | ConvertTo-Json -Depth 10 | Write-Host
+ Write-Host "##[endgroup]"
+ Write-Host
+ }
+ exit 1
+}
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/test-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/test-modules.ps1
index 58d8f05c5806..ce1271bfa745 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/test-modules.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/test-modules.ps1
@@ -4,21 +4,22 @@ param (
[string]$RepoRoot
)
+# Install Az.Accounts module and Pester module
+Install-Module -Name Pester -Repository PSGallery -RequiredVersion 4.10.1 -Force -SkipPublisherCheck
+Install-Module -Name Az.Accounts -AllowClobber -Force -Repository PSGallery
+
$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
Import-Module $utilFilePath -Force
$subModuleGroup = Get-Targets -RepoRoot $RepoRoot -TargetsOutputFileName "test$($TestEnvName)Targets.json" -MatrixKey $MatrixKey
-if ($TestEnvName -eq 'Linux') {
- $accountsModulePath = Join-Path $RepoRoot 'artifacts' 'Debug' "Az.Accounts" "Az.Accounts.psd1"
- Import-Module $accountsModulePath -Force
-}
-
$results = @()
foreach ($subModule in $subModuleGroup) {
$startTime = Get-Date
$moduleName, $subModuleName = $subModule -split '/'
$result = @{
+ OSName = $TestEnvName
+ MatrixKey = $MatrixKey
Module = $moduleName
SubModule = $subModuleName
Status = "Success"
@@ -30,6 +31,12 @@ foreach ($subModule in $subModuleGroup) {
Write-Host "Testing sub module: $subModule"
$subModulePath = Join-Path $RepoRoot 'artifacts' 'Debug' "Az.$ModuleName" $subModuleName
Push-Location $subModulePath
+ # remove the integrated Az Accounts so that the installed latest one could be used for test
+ $integratedAzAccounts = Join-Path $subModulePath 'generated' 'modules' 'Az.Accounts'
+ If (Test-Path $integratedAzAccounts){
+ Write-Host "Removing integrated Az.Accounts module from $integratedAzAccounts"
+ Remove-Item -Path $integratedAzAccounts -Recurse -Force
+ }
& ".\test-module.ps1"
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1 b/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
index c183bdc8d5d0..ab040bbd463d 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
@@ -1,3 +1,37 @@
+function Get-AutorestV4ModuleMap {
+ param (
+ [string]$srcPath
+ )
+
+ $result = @{}
+
+ Get-ChildItem -Path $srcPath -Directory | ForEach-Object {
+ $module = $_
+
+ Get-ChildItem -Path $module.FullName -Directory | Where-Object {
+ $_.Name -like '*.autorest'
+ } | ForEach-Object {
+ $subModule = $_
+
+ $readmePath = Join-Path $subModule.FullName 'README.md'
+
+ if (Test-Path $readmePath) {
+ $readmeContent = Get-Content -Path $readmePath -Raw
+
+ if ($readmeContent -notmatch 'use-extension:\s+"@autorest/powershell":\s+"3.x"') {
+ if ($result.ContainsKey($module.Name)) {
+ $result[$module.Name] += $subModule.Name
+ } else {
+ $result[$module.Name] = @($subModule.Name)
+ }
+ }
+ }
+ }
+ }
+
+ return $result
+}
+
function Group-Modules {
param (
[array]$Modules,
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index b1a8f46a6784..14c23aa2a468 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -68,13 +68,13 @@ stages:
displayName: "Batch Generate: "
dependsOn: prepare
condition: and(
- ne(dependencies.prepare.outputs['mtrx.generateTargets'], ''),
- ne(dependencies.prepare.outputs['mtrx.generateTargets'], '{}')
+ ne(dependencies.prepare.outputs['mtrx.generationTargets'], ''),
+ ne(dependencies.prepare.outputs['mtrx.generationTargets'], '{}')
)
timeoutInMinutes: ${{ variables.GenerateTimeoutInMinutes }}
pool: ${{ variables.WindowsAgentPoolName }}
strategy:
- matrix: $[ dependencies.prepare.outputs['mtrx.generateTargets'] ]
+ matrix: $[ dependencies.prepare.outputs['mtrx.generationTargets'] ]
maxParallel: ${{ variables.MaxParallelGenerateJobs }}
steps:
@@ -148,9 +148,6 @@ stages:
Write-Host "Applying all .patch files to ${{ variables.GenerationBranch }}..."
$patchFiles = Get-ChildItem -Path "$env:PIPELINE_WORKSPACE" -Recurse -Filter *.patch
- # $patchFiles = Get-ChildItem -Path "$env:PIPELINE_WORKSPACE" -Recurse -Filter *.patch |
- # Where-Object { $_.Directory.Parent.FullName -eq $env:PIPELINE_WORKSPACE }
-
foreach ($patch in $patchFiles) {
Write-Host "Applying patch: $($patch.FullName)"
git apply --whitespace=fix "$($patch.FullName)"
@@ -399,3 +396,40 @@ stages:
agentPoolName: ${{ variables.MacOSAgentPoolName }}
agentPoolVMImage: ${{ variables.MacOSAgentPoolVMImage }}
maxParallel: ${{ variables.MaxParallelTestMacJobs }}
+
+ - stage: Report
+ condition: always()
+ jobs:
+ - job: report
+ displayName: 'Report'
+ timeoutInMinutes: ${{ variables.IntermediateStepTimeoutInMinutes }}
+ pool: ${{ variables.WindowsAgentPoolName }}
+ steps:
+ - checkout: self
+ fetchDepth: 1
+ fetchTags: false
+
+ - download: current
+ patterns: |
+ **/AnalyseReport-*.json
+ **/GenerationReport-*.json
+ **/BuildReport-*.json
+ **/Test*Report-*.json
+ displayName: 'Download build artifacts'
+
+ - task: PowerShell@2
+ name: report
+ displayName: 'Report Results'
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ $reportScriptPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'report.ps1'
+ & $reportScriptPath -RepoRoot "$(Build.SourcesDirectory)" -PipelineWorkspace "$env:PIPELINE_WORKSPACE"
+
+ - task: PublishPipelineArtifact@1
+ displayName: 'Save Reports'
+ inputs:
+ targetPath: artifacts
+ artifact: 'reports'
+ condition: always()
\ No newline at end of file
From f3cc572084f9f93373eced9f9a6b21f0b6cf2ca7 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Tue, 13 May 2025 15:32:12 +1000
Subject: [PATCH 24/33] Run test on all modules
---
.../PipelineSteps/BatchGeneration/filter.ps1 | 38 +++++++++++++++----
.azure-pipelines/batch-generation.yml | 7 ++++
2 files changed, 37 insertions(+), 8 deletions(-)
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
index 262ce709d966..5a8b26a8a0d6 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
@@ -11,19 +11,41 @@ param (
$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
Import-Module $utilFilePath -Force
+$artifactsDir = Join-Path $RepoRoot 'artifacts'
$changedModulesDict = @{}
$changedSubModulesDict = @{}
-for ($i = 0; $i -lt $ChangedFiles.Count; $i++) {
- if ($ChangedFiles[$i] -match '^(src|generated)/([^/]+)/([^/]+\.autorest)/') {
- $parent = $Matches[2]
- $child = $Matches[3]
- $key = "$parent/$child"
-
- $changedModulesDict[$parent] = $true
- $changedSubModulesDict[$key] = $true
+if ($env:RUN_TEST_ON_ALL_MODULES -eq "True") {
+ Write-Host "Run test on all modules"
+ $V4ModulesFile = Join-Path $artifactsDir "generationTargets.json"
+ $V4ModuleMaps = Get-Content -Raw -Path $V4ModulesFile | ConvertFrom-Json
+
+ foreach ($matrixKey in $V4ModuleMaps.PSObject.Properties.Name) {
+ Write-Host "##[group]Matrix key: $matrixKey"
+ $moduleMap = $V4ModuleMaps.$matrixKey
+ foreach ($moduleName in $moduleMap.PSObject.Properties.Name) {
+ foreach ($subModuleName in $moduleMap.$moduleName) {
+ $subModule = "$moduleName/$subModuleName"
+ $changedModulesDict[$moduleName] = $true
+ $changedSubModulesDict[$subModule] = $true
+ }
+ }
+ }
+}
+else {
+ Write-Host "Run test on changed modules"
+ for ($i = 0; $i -lt $ChangedFiles.Count; $i++) {
+ if ($ChangedFiles[$i] -match '^(src|generated)/([^/]+)/([^/]+\.autorest)/') {
+ $moduleName = $Matches[2]
+ $subModuleName = $Matches[3]
+ $subModule = "$moduleName/$subModuleName"
+
+ $changedModulesDict[$moduleName] = $true
+ $changedSubModulesDict[$subModule] = $true
+ }
}
}
+
$changedModules = $changedModulesDict.Keys | Sort-Object
$changedSubModules = $changedSubModulesDict.Keys | Sort-Object
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 14c23aa2a468..72a935f65358 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -131,6 +131,11 @@ stages:
git fetch origin "${{ variables.GenerationBranch }}"
git checkout "${{ variables.GenerationBranch }}"
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifactName: 'prepare'
+ targetPath: artifacts
+
- download: current
patterns: '**/changed-*.patch'
displayName: 'Download all .patch artifacts'
@@ -172,6 +177,8 @@ stages:
$filterModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'filter.ps1'
& $filterModulesPath -MaxParallelBuildJobs "${{ variables.MaxParallelBuildJobs }}" -MaxParallelAnalyzeJobs "${{ variables.MaxParallelAnalyzeJobs }}" -MaxParallelTestWindowsJobs "${{ variables.MaxParallelTestWindowsJobs }}" -MaxParallelTestLinuxJobs "${{ variables.MaxParallelTestLinuxJobs }}" -MaxParallelTestMacJobs "${{ variables.MaxParallelTestMacJobs }}" -ChangedFiles $changedFiles -RepoRoot "$(Build.SourcesDirectory)"
+ env:
+ RUN_TEST_ON_ALL_MODULES: $(RUN_TEST_ON_ALL_MODULES)
- task: PublishPipelineArtifact@1
displayName: 'Upload filtered targets'
From 15a526af5b63b73faddfdd87e7175453c1fa979d Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Wed, 14 May 2025 21:25:32 +1000
Subject: [PATCH 25/33] Add change log
---
.../batch-generate-modules.ps1 | 30 ++++++++++++++++++-
.../PipelineSteps/BatchGeneration/filter.ps1 | 1 -
.azure-pipelines/batch-generation.yml | 19 ++++++++++--
3 files changed, 45 insertions(+), 5 deletions(-)
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
index f4f027f7c9d6..a83ecd345464 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
@@ -1,6 +1,7 @@
param (
[string]$MatrixKey,
- [string]$RepoRoot
+ [string]$RepoRoot,
+ [string]$CommitTitle
)
$generationTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generationTargets.json"
@@ -33,6 +34,7 @@ foreach ($moduleName in $sortedModuleNames) {
Module = $moduleName
DurationSeconds = 0
Status = "Success"
+ Changed = "No"
SubModules = @()
}
@@ -78,6 +80,32 @@ foreach ($moduleName in $sortedModuleNames) {
$moduleResult.SubModules += $subModuleResult
}
}
+
+ Set-Location $RepoRoot
+ $srcFolderModuleRelativePath = ".\src\$moduleName"
+ $generatedFolderModuleRelativePath = ".\generated\$moduleName"
+ $diffSrc = git diff --name-only HEAD -- $srcFolderModuleRelativePath
+ $diffGenerated = git diff --name-only HEAD -- $generatedFolderModuleRelativePath
+ $diff = $diffSrc -or $diffGenerated
+ if ($diff) {
+ Write-Host "Changes detected in $moduleName, adding change log"
+ $moduleResult.Changed = "Yes"
+ $changeLogPath = Join-Path $RepoRoot "src" $moduleName $moduleName "ChangeLog.md"
+ $changeLogContent = Get-Content $changeLogPath
+ $newChangeLogEntry = "* Autorest Upgration: $CommitTitle"
+
+ $index = $changeLogContent.IndexOf("## Upcoming Release")
+ if ($index -ge 0) {
+ $newChangeLogContent = $changeLogContent[0..$index] + $newChangeLogEntry + $changeLogContent[($index + 1)..($changeLogContent.Count - 1)]
+ Set-Content $changelogPath -Value $newChangeLogContent
+ $moduleResult.Changed = "Yes, Change Log Updated"
+ Write-Host "New change log entry added to $changeLogPath"
+ } else {
+ $moduleResult.Changed = "Yes, Change Log Not Updated"
+ Write-Host "Can not find '## Upcoming Release'"
+ }
+ }
+
$moduleEndTime = Get-Date
$moduleResult.DurationSeconds = ($moduleEndTime - $moduleStartTime).TotalSeconds
$results += $moduleResult
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
index 5a8b26a8a0d6..8dbf9cfee74b 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
@@ -21,7 +21,6 @@ if ($env:RUN_TEST_ON_ALL_MODULES -eq "True") {
$V4ModuleMaps = Get-Content -Raw -Path $V4ModulesFile | ConvertFrom-Json
foreach ($matrixKey in $V4ModuleMaps.PSObject.Properties.Name) {
- Write-Host "##[group]Matrix key: $matrixKey"
$moduleMap = $V4ModuleMaps.$matrixKey
foreach ($moduleName in $moduleMap.PSObject.Properties.Name) {
foreach ($subModuleName in $moduleMap.$moduleName) {
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 72a935f65358..349a4a15f8dd 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -1,3 +1,9 @@
+parameters:
+- name: CommitTitle
+ displayName: 'Title of the commit that trigger batch generation'
+ type: string
+ default: 'Default commit title'
+
variables:
IntermediateStepTimeoutInMinutes: 30
GenerateTimeoutInMinutes: 120
@@ -98,7 +104,7 @@ stages:
git checkout "${{ variables.GenerationBranch }}"
$batchGenerateModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'batch-generate-modules.ps1'
- & $batchGenerateModulesPath -MatrixKey "$(MatrixKey)" -RepoRoot "$(Build.SourcesDirectory)"
+ & $batchGenerateModulesPath -MatrixKey "$(MatrixKey)" -RepoRoot "$(Build.SourcesDirectory)" -CommitTitle "${{ parameters.CommitTitle }}"
workingDirectory: $(Build.SourcesDirectory)
@@ -159,7 +165,7 @@ stages:
}
git add .
- git commit -m "Apply matrix job patches to ${{ variables.GenerationBranch }}"
+ git commit -m "Apply batch generation patches to ${{ variables.GenerationBranch }}"
git push origin "${{ variables.GenerationBranch }}"
- task: PowerShell@2
@@ -176,7 +182,14 @@ stages:
git checkout $sourceBranchName
$filterModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'filter.ps1'
- & $filterModulesPath -MaxParallelBuildJobs "${{ variables.MaxParallelBuildJobs }}" -MaxParallelAnalyzeJobs "${{ variables.MaxParallelAnalyzeJobs }}" -MaxParallelTestWindowsJobs "${{ variables.MaxParallelTestWindowsJobs }}" -MaxParallelTestLinuxJobs "${{ variables.MaxParallelTestLinuxJobs }}" -MaxParallelTestMacJobs "${{ variables.MaxParallelTestMacJobs }}" -ChangedFiles $changedFiles -RepoRoot "$(Build.SourcesDirectory)"
+ & $filterModulesPath `
+ -MaxParallelBuildJobs "${{ variables.MaxParallelBuildJobs }}" `
+ -MaxParallelAnalyzeJobs "${{ variables.MaxParallelAnalyzeJobs }}" `
+ -MaxParallelTestWindowsJobs "${{ variables.MaxParallelTestWindowsJobs }}" `
+ -MaxParallelTestLinuxJobs "${{ variables.MaxParallelTestLinuxJobs }}" `
+ -MaxParallelTestMacJobs "${{ variables.MaxParallelTestMacJobs }}" `
+ -ChangedFiles $changedFiles `
+ -RepoRoot "$(Build.SourcesDirectory)"
env:
RUN_TEST_ON_ALL_MODULES: $(RUN_TEST_ON_ALL_MODULES)
From a65934d51089f9bcbde6abb90d1b3b76d03111f0 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Thu, 15 May 2025 23:17:04 +1000
Subject: [PATCH 26/33] Add date on generation branch name
---
.../BatchGeneration/create-branch.ps1 | 2 +-
.azure-pipelines/batch-generation.yml | 32 +++++++++++--------
2 files changed, 20 insertions(+), 14 deletions(-)
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/create-branch.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/create-branch.ps1
index 450921b95f1e..6e08cd07ab2b 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/create-branch.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/create-branch.ps1
@@ -19,4 +19,4 @@ $body = @{
Invoke-RestMethod -Uri "https://api.github.com/repos/$Owner/$Repo/git/refs" `
-Method Post -Headers $headers -Body $body -ContentType "application/json"
-Write-Host "Created branch '$NewBranch' from '$BBaseBranch'"
+Write-Host "Created branch '$NewBranch' from '$BaseBranch'"
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 349a4a15f8dd..2d60adc972fe 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -20,7 +20,6 @@ variables:
LinuxAgentPoolName: pool-ubuntu-2004
MacOSAgentPoolName: 'Azure Pipelines'
MacOSAgentPoolVMImage: macOS-latest
- GenerationBranch: batch-generation/branch-$(Build.BuildId)
trigger: none
@@ -42,17 +41,20 @@ stages:
- template: util/get-github-pat-steps.yml
- task: PowerShell@2
+ name: checkout
displayName: 'Create Batch Generation Branch'
inputs:
targetType: inline
pwsh: true
script: |
- $newBranch = "$(GenerationBranch)"
+ $date = Get-Date -Format "yy-MM-dd"
+ $generationBranch = "batch-generation/branch-$date-$(Build.BuildId)"
$token = "$(GitHubToken)"
$createBranchPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'create-branch.ps1'
# TODO: (Bernard) Change BaseBranch to main after test
- & $createBranchPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch 'bernard-pipeline-generation' -NewBranch $newBranch -Token $token
+ & $createBranchPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch 'bernard-pipeline-generation' -NewBranch $generationBranch -Token $token
+ Write-Host "##vso[task.setvariable variable=GenerationBranch;isOutput=true]$generationBranch"
- task: PowerShell@2
name: mtrx
@@ -77,6 +79,8 @@ stages:
ne(dependencies.prepare.outputs['mtrx.generationTargets'], ''),
ne(dependencies.prepare.outputs['mtrx.generationTargets'], '{}')
)
+ variables:
+ GenerationBranch: $[dependencies.prepare.outputs['checkout.GenerationBranch']]
timeoutInMinutes: ${{ variables.GenerateTimeoutInMinutes }}
pool: ${{ variables.WindowsAgentPoolName }}
strategy:
@@ -100,12 +104,11 @@ stages:
targetType: inline
pwsh: true
script: |
- git fetch origin "${{ variables.GenerationBranch }}"
- git checkout "${{ variables.GenerationBranch }}"
+ git fetch origin "$(GenerationBranch)"
+ git checkout "$(GenerationBranch)"
$batchGenerateModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'batch-generate-modules.ps1'
& $batchGenerateModulesPath -MatrixKey "$(MatrixKey)" -RepoRoot "$(Build.SourcesDirectory)" -CommitTitle "${{ parameters.CommitTitle }}"
-
workingDirectory: $(Build.SourcesDirectory)
- task: PublishPipelineArtifact@1
@@ -116,7 +119,10 @@ stages:
condition: always()
- stage: Build
+ dependsOn: Generate
condition: always()
+ variables:
+ GenerationBranch: $[stageDependencies.Generate.prepare.outputs['checkout.GenerationBranch']]
jobs:
- job: filter
displayName: 'Filter Changed Modules'
@@ -134,8 +140,8 @@ stages:
targetType: inline
pwsh: true
script: |
- git fetch origin "${{ variables.GenerationBranch }}"
- git checkout "${{ variables.GenerationBranch }}"
+ git fetch origin "$(GenerationBranch)"
+ git checkout "$(GenerationBranch)"
- task: DownloadPipelineArtifact@2
inputs:
@@ -156,7 +162,7 @@ stages:
git config user.email "65331932+azure-powershell-bot@users.noreply.github.com"
git config user.name "azure-powershell-bot"
- Write-Host "Applying all .patch files to ${{ variables.GenerationBranch }}..."
+ Write-Host "Applying all .patch files to "$(GenerationBranch)"..."
$patchFiles = Get-ChildItem -Path "$env:PIPELINE_WORKSPACE" -Recurse -Filter *.patch
foreach ($patch in $patchFiles) {
@@ -165,8 +171,8 @@ stages:
}
git add .
- git commit -m "Apply batch generation patches to ${{ variables.GenerationBranch }}"
- git push origin "${{ variables.GenerationBranch }}"
+ git commit -m "Autorest Upgration - ${{ parameters.CommitTitle }}"
+ git push origin "$(GenerationBranch)"
- task: PowerShell@2
name: mtrx
@@ -229,8 +235,8 @@ stages:
targetType: inline
pwsh: true
script: |
- git fetch origin "${{ variables.GenerationBranch }}"
- git checkout "${{ variables.GenerationBranch }}"
+ git fetch origin "$(GenerationBranch)"
+ git checkout "$(GenerationBranch)"
$buildModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'build-modules.ps1'
& $buildModulesPath -MatrixKey "$(MatrixKey)" -RepoRoot "$(Build.SourcesDirectory)"
From a8a1b9ae1c04c86bc43c245993f010a69d05179a Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Fri, 16 May 2025 00:37:03 +1000
Subject: [PATCH 27/33] Failed job notification
---
.../FailedJobTeamsTemplate.html | 26 +++++++++++++
.../BatchGeneration/notify-failed-job.ps1 | 27 +++++++++++++
.../PipelineSteps/BatchGeneration/util.psm1 | 39 +++++++++++++++++++
.azure-pipelines/batch-generation.yml | 24 +++++++++++-
4 files changed, 115 insertions(+), 1 deletion(-)
create mode 100644 .azure-pipelines/PipelineSteps/BatchGeneration/FailedJobTeamsTemplate.html
create mode 100644 .azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/FailedJobTeamsTemplate.html b/.azure-pipelines/PipelineSteps/BatchGeneration/FailedJobTeamsTemplate.html
new file mode 100644
index 000000000000..52ff7a103c1e
--- /dev/null
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/FailedJobTeamsTemplate.html
@@ -0,0 +1,26 @@
+
+
+
+
+
+
+
+
+ Pipeline Failure Alert šØ
+ The pipeline **{{ pipelineName }}** has failed.
+ You can review the details at the following links:
+
+
+ Please check and address the issue as soon as possible.
+ Sincerely,
+ Your Azure CLI Tools Team
+
+
+
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1
new file mode 100644
index 000000000000..bf534195a9d7
--- /dev/null
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1
@@ -0,0 +1,27 @@
+param (
+ [string]$RepoRoot
+)
+
+$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
+Import-Module $utilFilePath -Force
+
+# TODO(Bernard): Add Xiaogang and Yabo to the list
+$batchGenSubscribers = @(
+ "bernardpan@microsoft.com"
+)
+$receivers = $batchGenSubscribers -join ","
+
+$pipelineName = $env:BUILD_DEFINITIONNAME
+$pipelineUrl = "$($env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI)$($env:SYSTEM_TEAMPROJECT)/_build?definitionId=$($env:SYSTEM_DEFINITIONID)"
+$runUrl = "$($env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI)$($env:SYSTEM_TEAMPROJECT)/_build/results?buildId=$($env:BUILD_BUILDID)"
+
+$templateFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'FailedJobTeamsTemplate.html'
+$notificationTemplate = Get-Content -Path $templateFilePath -Raw
+$notificationContent = $notificationTemplate -replace "{{ pipelineName }}", $pipelineName `
+ -replace "{{ pipelineUrl }}", $pipelineUrl `
+ -replace "{{ runUrl }}", $runUrl
+
+Send-Teams `
+ -to $receivers `
+ -title "Batch Generation Job Failed" `
+ -content $notificationContent
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1 b/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
index ab040bbd463d..1256595d2d15 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
@@ -109,3 +109,42 @@ function Get-Targets {
Write-Host
return $targetGroup
}
+
+function Send-Teams {
+ param (
+ [string]$to,
+ [string]$title,
+ [string]$content
+ )
+
+ $teamsUrl = $env:TEAMS_URL
+ if ([string]::IsNullOrEmpty($teamsUrl)) {
+ Write-Host "TEAMS_URL environment variable is not set." -ForegroundColor Red
+ exit 1
+ }
+
+ if ([string]::IsNullOrEmpty($to)) {
+ Write-Host "'to' parameter is empty, nothing to send." -ForegroundColor Yellow
+ return 0
+ }
+
+ $body = @{
+ to = $to
+ title = $title
+ content = $content
+ } | ConvertTo-Json -Depth 3
+
+ try {
+ Invoke-RestMethod -Uri $teamsUrl -Method Post -Headers @{
+ 'Accept' = 'application/json'
+ 'Content-Type' = 'application/json'
+ } -Body $body
+
+ Write-Host "Message sent successfully."
+ return 0
+ }
+ catch {
+ Write-Host "Failed to send message: $_" -ForegroundColor Red
+ exit 1
+ }
+}
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 2d60adc972fe..9fe348b94587 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -458,4 +458,26 @@ stages:
inputs:
targetPath: artifacts
artifact: 'reports'
- condition: always()
\ No newline at end of file
+ condition: always()
+
+ - stage: FailedJobNotification
+ dependsOn:
+ - Generate
+ - Build
+ - Test
+ - Report
+ condition: failed()
+ jobs:
+ - job: FailJobNotification
+ steps:
+ - task: PowerShell@2
+ name: notification
+ displayName: "teams notification on failed job"
+ inputs:
+ targetType: inline
+ pwsh: true
+ script: |
+ $notificationScriptPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'notify-failed-job.ps1'
+ & $notificationScriptPath -RepoRoot "$(Build.SourcesDirectory)"
+ env:
+ TEAMS_URL: $(TEAMS_URL)
From c2bda26ec8b16aae28a8182c325963c673a8c846 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Mon, 19 May 2025 13:22:08 +1000
Subject: [PATCH 28/33] Only generated folder change should trigger tests
---
.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 | 5 +++--
.../PipelineSteps/BatchGeneration/test-modules.ps1 | 3 +--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
index 8dbf9cfee74b..ec3d85208ecf 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
@@ -32,9 +32,10 @@ if ($env:RUN_TEST_ON_ALL_MODULES -eq "True") {
}
}
else {
- Write-Host "Run test on changed modules"
+ Write-Host "Run test on generated folder changed modules"
+ # Only generated filder change should trigger the test
for ($i = 0; $i -lt $ChangedFiles.Count; $i++) {
- if ($ChangedFiles[$i] -match '^(src|generated)/([^/]+)/([^/]+\.autorest)/') {
+ if ($ChangedFiles[$i] -match '^generated/([^/]+)/([^/]+\.autorest)/') {
$moduleName = $Matches[2]
$subModuleName = $Matches[3]
$subModule = "$moduleName/$subModuleName"
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/test-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/test-modules.ps1
index ce1271bfa745..d90f1bc8b9b6 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/test-modules.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/test-modules.ps1
@@ -30,7 +30,7 @@ foreach ($subModule in $subModuleGroup) {
try {
Write-Host "Testing sub module: $subModule"
$subModulePath = Join-Path $RepoRoot 'artifacts' 'Debug' "Az.$ModuleName" $subModuleName
- Push-Location $subModulePath
+ Set-Location $subModulePath
# remove the integrated Az Accounts so that the installed latest one could be used for test
$integratedAzAccounts = Join-Path $subModulePath 'generated' 'modules' 'Az.Accounts'
If (Test-Path $integratedAzAccounts){
@@ -56,7 +56,6 @@ foreach ($subModule in $subModuleGroup) {
$endTime = Get-Date
$result.DurationSeconds = ($endTime - $startTime).TotalSeconds
$results += $result
- Pop-Location
}
}
From 560f7f92627cfce6d5d6c6603de8ae01cc2639d4 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Mon, 19 May 2025 14:34:23 +1000
Subject: [PATCH 29/33] Add seperate Autorest change log
---
.../batch-generate-modules.ps1 | 27 +++++++++----------
.azure-pipelines/batch-generation.yml | 12 ++++-----
2 files changed, 19 insertions(+), 20 deletions(-)
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
index a83ecd345464..0461ec0c50fe 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
@@ -1,7 +1,7 @@
param (
[string]$MatrixKey,
[string]$RepoRoot,
- [string]$CommitTitle
+ [string]$AutorestVersion
)
$generationTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generationTargets.json"
@@ -81,6 +81,7 @@ foreach ($moduleName in $sortedModuleNames) {
}
}
+ # If the module is changed in either src or generated folder, add a change log entry
Set-Location $RepoRoot
$srcFolderModuleRelativePath = ".\src\$moduleName"
$generatedFolderModuleRelativePath = ".\generated\$moduleName"
@@ -90,20 +91,18 @@ foreach ($moduleName in $sortedModuleNames) {
if ($diff) {
Write-Host "Changes detected in $moduleName, adding change log"
$moduleResult.Changed = "Yes"
- $changeLogPath = Join-Path $RepoRoot "src" $moduleName $moduleName "ChangeLog.md"
- $changeLogContent = Get-Content $changeLogPath
- $newChangeLogEntry = "* Autorest Upgration: $CommitTitle"
-
- $index = $changeLogContent.IndexOf("## Upcoming Release")
- if ($index -ge 0) {
- $newChangeLogContent = $changeLogContent[0..$index] + $newChangeLogEntry + $changeLogContent[($index + 1)..($changeLogContent.Count - 1)]
- Set-Content $changelogPath -Value $newChangeLogContent
- $moduleResult.Changed = "Yes, Change Log Updated"
- Write-Host "New change log entry added to $changeLogPath"
- } else {
- $moduleResult.Changed = "Yes, Change Log Not Updated"
- Write-Host "Can not find '## Upcoming Release'"
+ $changeLogPath = Join-Path $RepoRoot "src" $moduleName $moduleName "AutorestUpgradeLog.md"
+ if (-not (Test-Path $changeLogPath)) {
+ New-Item -Path $changeLogPath -ItemType File -Force | Out-Null
+ Add-Content -Path $changeLogPath -Value "## Autorest upgrade log"
}
+ $changeLogContent = Get-Content -Path $changeLogPath
+ $date = Get-Date -Format "dd/MM/yy"
+ $newChangeLogEntry = "* Autorest version: $AutorestVersion - $date"
+ $changeLogContent.Insert(1, $newChangeLogEntry)
+ Set-Content $changelogPath -Value $changeLogContent
+ $moduleResult.Changed = "Yes, Autorest Change Log Updated"
+ Write-Host "New change log entry added to $changeLogPath"
}
$moduleEndTime = Get-Date
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 9fe348b94587..e69efed3bfcf 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -1,8 +1,8 @@
parameters:
-- name: CommitTitle
- displayName: 'Title of the commit that trigger batch generation'
+- name: AutorestVersion
+ displayName: 'Version of Autorest that trigger batch generation'
type: string
- default: 'Default commit title'
+ default: 'Default version'
variables:
IntermediateStepTimeoutInMinutes: 30
@@ -47,7 +47,7 @@ stages:
targetType: inline
pwsh: true
script: |
- $date = Get-Date -Format "yy-MM-dd"
+ $date = Get-Date -Format "dd/mm/yy"
$generationBranch = "batch-generation/branch-$date-$(Build.BuildId)"
$token = "$(GitHubToken)"
@@ -108,7 +108,7 @@ stages:
git checkout "$(GenerationBranch)"
$batchGenerateModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'batch-generate-modules.ps1'
- & $batchGenerateModulesPath -MatrixKey "$(MatrixKey)" -RepoRoot "$(Build.SourcesDirectory)" -CommitTitle "${{ parameters.CommitTitle }}"
+ & $batchGenerateModulesPath -MatrixKey "$(MatrixKey)" -RepoRoot "$(Build.SourcesDirectory)" -AutorestVersion "${{ parameters.AutorestVersion }}"
workingDirectory: $(Build.SourcesDirectory)
- task: PublishPipelineArtifact@1
@@ -171,7 +171,7 @@ stages:
}
git add .
- git commit -m "Autorest Upgration - ${{ parameters.CommitTitle }}"
+ git commit -m "Autorest Upgrade - ${{ parameters.AutorestVersion }}"
git push origin "$(GenerationBranch)"
- task: PowerShell@2
From 486807f38b6bb14d7275745e0b6574a45419e320 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Tue, 20 May 2025 21:37:52 +1000
Subject: [PATCH 30/33] Remove test
---
.../BatchGeneration/batch-generate-modules.ps1 | 4 ----
.../PipelineSteps/BatchGeneration/prepare.ps1 | 13 +------------
.azure-pipelines/util/batch-generation-test-job.yml | 1 -
3 files changed, 1 insertion(+), 17 deletions(-)
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
index 0461ec0c50fe..352966790526 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
@@ -56,10 +56,6 @@ foreach ($moduleName in $sortedModuleNames) {
Remove-Item -Path $generateLog -Recurse -Force
}
New-Item -ItemType File -Force -Path $generateLog
- # TODO(Bernard) Remove exception for EmailService.Autorest after test
- if ($subModuleName -eq "EmailService.Autorest") {
- throw "Something went wrong"
- }
if (-not (Update-GeneratedSubModule -ModuleRootName $moduleName -SubModuleName $subModuleName -SourceDirectory $sourceDirectory -GeneratedDirectory $generatedDirectory -GenerateLog $generateLog -IsInvokedByPipeline $true)) {
Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
Write-Warning "log can be found at $generateLog"
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
index 3c072b44392b..a8520e0e9463 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1
@@ -8,18 +8,7 @@ $utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGen
Import-Module $utilFilePath -Force
$srcPath = Join-Path $RepoRoot 'src'
-# TODO(Bernard): Use real function after test
-# $moduleMap = Get-AutorestV4ModuleMap -srcPath $srcPath
-$moduleMap = @{
- "DeviceRegistry" = @("DeviceRegistry.Autorest")
- "ArcGateway" = @("ArcGateway.Autorest")
- "Chaos" = @("Chaos.Autorest")
- "Cdn" = @("Cdn.Autorest")
- "Communication" = @("EmailService.Autorest", "EmailServicedata.Autorest")
- "Astro" = @("Astro.Autorest")
- "ImageBuilder" = @("ImageBuilder.Autorest")
-}
-
+$moduleMap = Get-AutorestV4ModuleMap -srcPath $srcPath
Write-Host "Total matched modules: $($moduleMap.Count)"
$modules = @($moduleMap.Keys | Sort-Object)
diff --git a/.azure-pipelines/util/batch-generation-test-job.yml b/.azure-pipelines/util/batch-generation-test-job.yml
index 05604f45408b..42d9e23c95cb 100644
--- a/.azure-pipelines/util/batch-generation-test-job.yml
+++ b/.azure-pipelines/util/batch-generation-test-job.yml
@@ -25,7 +25,6 @@ jobs:
maxParallel: ${{ parameters.maxParallel }}
steps:
- #TODO(Bernard) check if can skip checkout step after move this to template
- checkout: self
fetchDepth: 1
fetchTags: false
From 7a9829e0aa2e6577b1abedf5537af9bdacca6208 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Tue, 20 May 2025 23:44:38 +1000
Subject: [PATCH 31/33] Add report item
---
.../PipelineSteps/BatchGeneration/report.ps1 | 27 +++++++++++++++++++
.../PipelineSteps/BatchGeneration/util.psm1 | 2 +-
.azure-pipelines/batch-generation.yml | 6 +++--
3 files changed, 32 insertions(+), 3 deletions(-)
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/report.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/report.ps1
index 8d2c64f6a1e3..77d3791c00e8 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/report.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/report.ps1
@@ -3,6 +3,33 @@ param (
[string]$PipelineWorkspace
)
+# Check number of generation targets
+$generationTargetsFile = Join-Path $PipelineWorkspace "prepare" "generationTargets.json"
+$generationTargets = Get-Content -Raw -Path $generationTargetsFile | ConvertFrom-Json
+$totalGenerationModules = 0
+foreach ($outerKey in $generationTargets.PSObject.Properties.Name) {
+ $innerKeys = $generationTargets.$outerKey.PSObject.Properties.Name.Count
+ $totalGenerationModules += $innerKeys
+}
+Write-Output "Total regeneration target modules: $totalGenerationModules"
+
+# Check number of other targets
+$targetPatterns = @{
+ Analyse = "analyzeTargets.json"
+ Build = "buildTargets.json"
+ Test = "testWindowsTargets.json"
+}
+foreach ($pattern in $targetPatterns.GetEnumerator()) {
+ $targetFilePath = Join-Path $PipelineWorkspace "filter" $pattern.Value
+ $targetJson = Get-Content -Raw -Path $targetFilePath | ConvertFrom-Json
+ $total = ($targetJson.PSObject.Properties.Value | ForEach-Object {
+ $_.Count
+ }) | Measure-Object -Sum | Select-Object -ExpandProperty Sum
+
+ Write-Output "Total $($pattern.Key) targets : $total"
+}
+
+# Check all steps reports
$reportFilePattern = @{
Generation = "GenerationReport-*.json"
Build = "BuildReport-*.json"
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1 b/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
index 1256595d2d15..cc39536925ba 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1
@@ -66,7 +66,7 @@ function Write-Matrix {
[string]$RepoRoot
)
- Write-Host "Module groups: $($GroupedModules.Count)"
+ Write-Host "$VariableName module groups: $($GroupedModules.Count)"
$GroupedModules | ForEach-Object { $_ -join ', ' } | ForEach-Object { Write-Host $_ }
$targets = @{}
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index e69efed3bfcf..e4e2ab0de485 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -14,8 +14,8 @@ variables:
MaxParallelBuildJobs: 15
MaxParallelAnalyzeJobs: 5
MaxParallelTestWindowsJobs: 5
- MaxParallelTestLinuxJobs: 4
- MaxParallelTestMacJobs: 2
+ MaxParallelTestLinuxJobs: 5
+ MaxParallelTestMacJobs: 5
WindowsAgentPoolName: pool-windows-2019
LinuxAgentPoolName: pool-ubuntu-2004
MacOSAgentPoolName: 'Azure Pipelines'
@@ -437,6 +437,8 @@ stages:
- download: current
patterns: |
+ prepare/generationTargets.json
+ filter/**
**/AnalyseReport-*.json
**/GenerationReport-*.json
**/BuildReport-*.json
From 78261c5acfe16e872da1711b041d7af5003bdbf4 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Wed, 21 May 2025 16:44:46 +1000
Subject: [PATCH 32/33] Fix path
---
.../BatchGeneration/batch-generate-modules.ps1 | 2 +-
.../PipelineSteps/BatchGeneration/report.ps1 | 4 ++--
.azure-pipelines/batch-generation.yml | 12 ++++++++++--
.../test/Get-AzAstroOrganization.Recording.json | 2 +-
4 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
index 352966790526..96d51bef1aba 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
@@ -96,7 +96,7 @@ foreach ($moduleName in $sortedModuleNames) {
$date = Get-Date -Format "dd/MM/yy"
$newChangeLogEntry = "* Autorest version: $AutorestVersion - $date"
$changeLogContent.Insert(1, $newChangeLogEntry)
- Set-Content $changelogPath -Value $changeLogContent
+ Set-Content $changeLogPath -Value $changeLogContent
$moduleResult.Changed = "Yes, Autorest Change Log Updated"
Write-Host "New change log entry added to $changeLogPath"
}
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/report.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/report.ps1
index 77d3791c00e8..007ce3aa6a71 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/report.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/report.ps1
@@ -4,7 +4,7 @@ param (
)
# Check number of generation targets
-$generationTargetsFile = Join-Path $PipelineWorkspace "prepare" "generationTargets.json"
+$generationTargetsFile = Join-Path $RepoRoot "artifacts" "generationTargets.json"
$generationTargets = Get-Content -Raw -Path $generationTargetsFile | ConvertFrom-Json
$totalGenerationModules = 0
foreach ($outerKey in $generationTargets.PSObject.Properties.Name) {
@@ -20,7 +20,7 @@ $targetPatterns = @{
Test = "testWindowsTargets.json"
}
foreach ($pattern in $targetPatterns.GetEnumerator()) {
- $targetFilePath = Join-Path $PipelineWorkspace "filter" $pattern.Value
+ $targetFilePath = Join-Path $RepoRoot "artifacts" $pattern.Value
$targetJson = Get-Content -Raw -Path $targetFilePath | ConvertFrom-Json
$total = ($targetJson.PSObject.Properties.Value | ForEach-Object {
$_.Count
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index e4e2ab0de485..6e354c7f45a2 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -435,10 +435,18 @@ stages:
fetchDepth: 1
fetchTags: false
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifactName: 'prepare'
+ targetPath: artifacts
+
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifactName: 'filter'
+ targetPath: artifacts
+
- download: current
patterns: |
- prepare/generationTargets.json
- filter/**
**/AnalyseReport-*.json
**/GenerationReport-*.json
**/BuildReport-*.json
diff --git a/src/Astro/Astro.Autorest/test/Get-AzAstroOrganization.Recording.json b/src/Astro/Astro.Autorest/test/Get-AzAstroOrganization.Recording.json
index 7a81650e0073..e02a62dcf540 100644
--- a/src/Astro/Astro.Autorest/test/Get-AzAstroOrganization.Recording.json
+++ b/src/Astro/Astro.Autorest/test/Get-AzAstroOrganization.Recording.json
@@ -18,7 +18,7 @@
}
},
"Response": {
- "StatusCode": 800,
+ "StatusCode": 200,
"Headers": {
"Cache-Control": [ "no-cache" ],
"Pragma": [ "no-cache" ],
From 008f29f955ae4619dd2f7fb2ef9b4d2778aa6592 Mon Sep 17 00:00:00 2001
From: Qi Pan <43341456+Pan-Qi@users.noreply.github.com>
Date: Wed, 21 May 2025 23:59:41 +1000
Subject: [PATCH 33/33] remove test param
---
.../PipelineSteps/BatchGeneration/notify-failed-job.ps1 | 5 +++--
.azure-pipelines/batch-generation.yml | 3 +--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1
index bf534195a9d7..ff1ccc6f68ab 100644
--- a/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1
+++ b/.azure-pipelines/PipelineSteps/BatchGeneration/notify-failed-job.ps1
@@ -5,9 +5,10 @@ param (
$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
Import-Module $utilFilePath -Force
-# TODO(Bernard): Add Xiaogang and Yabo to the list
$batchGenSubscribers = @(
- "bernardpan@microsoft.com"
+ "bernardpan@microsoft.com",
+ "xidi@microsoft.com",
+ "yabhu@microsoft.com"
)
$receivers = $batchGenSubscribers -join ","
diff --git a/.azure-pipelines/batch-generation.yml b/.azure-pipelines/batch-generation.yml
index 6e354c7f45a2..e9a335f55848 100644
--- a/.azure-pipelines/batch-generation.yml
+++ b/.azure-pipelines/batch-generation.yml
@@ -52,8 +52,7 @@ stages:
$token = "$(GitHubToken)"
$createBranchPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'create-branch.ps1'
- # TODO: (Bernard) Change BaseBranch to main after test
- & $createBranchPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch 'bernard-pipeline-generation' -NewBranch $generationBranch -Token $token
+ & $createBranchPath -Owner 'Azure' -Repo 'azure-powershell' -BaseBranch 'main' -NewBranch $generationBranch -Token $token
Write-Host "##vso[task.setvariable variable=GenerationBranch;isOutput=true]$generationBranch"
- task: PowerShell@2