Skip to content

Commit 5a09e98

Browse files
Pan-QiPan-Qi
committed
report
1 parent 019934c commit 5a09e98

File tree

8 files changed

+180
-92
lines changed

8 files changed

+180
-92
lines changed

‎.azure-pipelines/PipelineSteps/BatchGeneration/analyse-modules.ps1

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ foreach ($moduleName in $moduleGroup) {
2020

2121
$startTime = Get-Date
2222
$result = @{
23+
MatrixKey = $MatrixKey
2324
Module = $moduleName
2425
Status = "Success"
2526
DurationSeconds = 0

‎.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ param (
33
[string]$RepoRoot
44
)
55

6-
$generateTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generateTargets.json"
7-
$generateTargets = Get-Content -Path $generateTargetsOutPutFile -Raw | ConvertFrom-Json
8-
$moduleGroup = $generateTargets.$MatrixKey
6+
$generationTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generationTargets.json"
7+
$generationTargets = Get-Content -Path $generationTargetsOutPutFile -Raw | ConvertFrom-Json
8+
$moduleGroup = $generationTargets.$MatrixKey
99
Write-Host "##[group]Generating module group $MatrixKey"
1010
foreach ($key in $moduleGroup.PSObject.Properties.Name | Sort-Object) {
1111
$values = $moduleGroup.$key -join ', '
@@ -32,6 +32,7 @@ foreach ($moduleName in $sortedModuleNames) {
3232
$moduleResult = @{
3333
Module = $moduleName
3434
DurationSeconds = 0
35+
Status = "Success"
3536
SubModules = @()
3637
}
3738

@@ -40,6 +41,7 @@ foreach ($moduleName in $sortedModuleNames) {
4041
Write-Host "Regenerating SubModule: $subModuleName"
4142
$subModuleStartTime = Get-Date
4243
$subModuleResult = @{
44+
MatrixKey = $MatrixKey
4345
SubModule = $subModuleName
4446
Status = "Success"
4547
DurationSeconds = 0
@@ -52,17 +54,22 @@ foreach ($moduleName in $sortedModuleNames) {
5254
Remove-Item -Path $generateLog -Recurse -Force
5355
}
5456
New-Item -ItemType File -Force -Path $generateLog
55-
57+
# TODO(Bernard) Remove exception for EmailService.Autorest after test
58+
if ($subModuleName -eq "EmailService.Autorest") {
59+
throw "Something went wrong"
60+
}
5661
if (-not (Update-GeneratedSubModule -ModuleRootName $moduleName -SubModuleName $subModuleName -SourceDirectory $sourceDirectory -GeneratedDirectory $generatedDirectory -GenerateLog $generateLog -IsInvokedByPipeline $true)) {
5762
Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
5863
Write-Warning "log can be found at $generateLog"
64+
$moduleResult.Status = "Failed"
5965
$subModuleResult.Status = "Failed"
6066
$subModuleResult.Error = "Update-GeneratedSubModule function returned false."
6167
}
6268

6369
} catch {
6470
Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
6571
Write-Warning "Error message: $($_.Exception.Message)"
72+
$moduleResult.Status = "Failed"
6673
$subModuleResult.Status = "Failed"
6774
$subModuleResult.Error = $_.Exception.Message
6875
} finally {

‎.azure-pipelines/PipelineSteps/BatchGeneration/build-modules.ps1

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ foreach ($moduleName in $moduleGroup) {
1515

1616
$startTime = Get-Date
1717
$result = @{
18+
MatrixKey = $MatrixKey
1819
Module = $moduleName
1920
Status = "Success"
2021
DurationSeconds = 0

‎.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1

Lines changed: 19 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -4,43 +4,13 @@ param (
44
[int]$MaxParallelJobs = 3
55
)
66

7-
$srcPath = Join-Path $RepoRoot 'src'
8-
function Get-SubModuleWithAutorestV4 {
9-
param (
10-
[string]$srcPath
11-
)
12-
13-
$result = @{}
14-
15-
Get-ChildItem -Path $srcPath -Directory | ForEach-Object {
16-
$module = $_
17-
18-
Get-ChildItem -Path $module.FullName -Directory | Where-Object {
19-
$_.Name -like '*.autorest'
20-
} | ForEach-Object {
21-
$subModule = $_
22-
23-
$readmePath = Join-Path $subModule.FullName 'README.md'
7+
$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
8+
Import-Module $utilFilePath -Force
249

25-
if (Test-Path $readmePath) {
26-
$readmeContent = Get-Content -Path $readmePath -Raw
27-
28-
if ($readmeContent -notmatch 'use-extension:\s+"@autorest/powershell":\s+"3.x"') {
29-
if ($result.ContainsKey($module.Name)) {
30-
$result[$module.Name] += $subModule.Name
31-
} else {
32-
$result[$module.Name] = @($subModule.Name)
33-
}
34-
}
35-
}
36-
}
37-
}
38-
39-
return $result
40-
}
10+
$srcPath = Join-Path $RepoRoot 'src'
4111
# TODO(Bernard): Use real function after test
42-
# $modules = Get-SubModuleWithAutorestV4 -srcPath $srcPath
43-
$modules = @{
12+
# $moduleMap = Get-AutorestV4ModuleMap -srcPath $srcPath
13+
$moduleMap = @{
4414
"DeviceRegistry" = @("DeviceRegistry.Autorest")
4515
"ArcGateway" = @("ArcGateway.Autorest")
4616
"Chaos" = @("Chaos.Autorest")
@@ -49,68 +19,40 @@ $modules = @{
4919
"Astro" = @("Astro.Autorest")
5020
"ImageBuilder" = @("ImageBuilder.Autorest")
5121
}
52-
$modules = $modules.GetEnumerator() | ForEach-Object {
53-
[PSCustomObject]@{
54-
ModuleName = $_.Key
55-
SubModules = ($_.Value | Sort-Object)
56-
}
57-
} | Sort-Object -Property ModuleName
5822

59-
Write-Host "Total matched modules: $($modules.Count)"
60-
61-
function Group-Modules {
62-
param (
63-
[array]$modules,
64-
[int]$maxParallelJobs
65-
)
66-
67-
$count = $modules.Count
68-
$n = [Math]::Min($count, $maxParallelJobs)
69-
if ($n -eq 0) {
70-
return @()
71-
}
72-
73-
$result = @()
74-
$sizePerGroup = [Math]::Ceiling($count / $n)
75-
76-
for ($i = 0; $i -lt $count; $i += $sizePerGroup) {
77-
$group = $modules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
78-
$result += ,$group
79-
}
80-
81-
return $result
82-
}
23+
Write-Host "Total matched modules: $($moduleMap.Count)"
8324

25+
$modules = @($moduleMap.Keys | Sort-Object)
8426
$groupedModules = Group-Modules -modules $modules -maxParallelJobs $MaxParallelJobs
8527
Write-Host "Total module groups: $($groupedModules.Count)"
8628

8729
$index = 0
88-
$generateTargets = @{}
30+
$generationTargets = @{}
8931
foreach ($moduleGroup in $groupedModules) {
90-
Write-Host "##[group]Prepareing module group $($index + 1)"
32+
Write-Host "##[group]Prepareing module group $($index + 1) with $($moduleGroup.Count) modules"
9133
$mergedModules = @{}
92-
foreach ($moduleObj in $moduleGroup) {
93-
Write-Host "Module $($moduleObj.ModuleName): $($moduleObj.SubModules -join ',')"
94-
$mergedModules[$moduleObj.ModuleName] = @($moduleObj.SubModules)
34+
foreach ($moduleName in $moduleGroup) {
35+
Write-Host "Module $($moduleName): $($moduleMap[$moduleName] -join ',')"
36+
$mergedModules[$moduleName] = @($moduleMap[$moduleName])
9537
$subIndex++
9638
}
9739

9840
$key = ($index + 1).ToString() + "-" + $moduleGroup.Count
99-
$generateTargets[$key] = $mergedModules
41+
$generationTargets[$key] = $mergedModules
10042
$MatrixStr = "$MatrixStr,'$key':{'MatrixKey':'$key'}"
10143
Write-Host "##[endgroup]"
10244
Write-Host
10345
$index++
10446
}
10547

106-
$generateTargetsOutputDir = Join-Path $RepoRoot "artifacts"
107-
if (-not (Test-Path -Path $generateTargetsOutputDir)) {
108-
New-Item -ItemType Directory -Path $generateTargetsOutputDir
48+
$generationTargetsOutputDir = Join-Path $RepoRoot "artifacts"
49+
if (-not (Test-Path -Path $generationTargetsOutputDir)) {
50+
New-Item -ItemType Directory -Path $generationTargetsOutputDir
10951
}
110-
$generateTargetsOutputFile = Join-Path $generateTargetsOutputDir "generateTargets.json"
111-
$generateTargets | ConvertTo-Json -Depth 5 | Out-File -FilePath $generateTargetsOutputFile -Encoding utf8
52+
$generationTargetsOutputFile = Join-Path $generationTargetsOutputDir "generationTargets.json"
53+
$generationTargets | ConvertTo-Json -Depth 5 | Out-File -FilePath $generationTargetsOutputFile -Encoding utf8
11254

11355
if ($MatrixStr -and $MatrixStr.Length -gt 1) {
11456
$MatrixStr = $MatrixStr.Substring(1)
11557
}
116-
Write-Host "##vso[task.setVariable variable=generateTargets;isOutput=true]{$MatrixStr}"
58+
Write-Host "##vso[task.setVariable variable=generationTargets;isOutput=true]{$MatrixStr}"

‎.azure-pipelines/PipelineSteps/BatchGeneration/report.ps1

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
param (
2+
[string]$RepoRoot,
3+
[string]$PipelineWorkspace
4+
)
5+
6+
$reportFilePattern = @{
7+
Generation = "GenerationReport-*.json"
8+
Build = "BuildReport-*.json"
9+
Analyze = "AnalyseReport-*.json"
10+
TestWindows = "TestWindowsReport-*.json"
11+
TestLinux = "TestLinuxReport-*.json"
12+
TestMacOS = "TestMacOSReport-*.json"
13+
}
14+
15+
$artifactRoot = Join-Path $RepoRoot 'artifacts'
16+
if (-not (Test-Path -Path $artifactRoot)) {
17+
New-Item -Path $artifactRoot -ItemType Directory
18+
}
19+
$failedReports = @{}
20+
21+
foreach ($pattern in $reportFilePattern.GetEnumerator()) {
22+
Write-Host "Filtering $($pattern.Key) Report with pattern: $($pattern.Value)"
23+
$allResults = @()
24+
25+
foreach ($dir in Get-ChildItem -Path $PipelineWorkspace -Directory) {
26+
$file = Get-ChildItem -Path $dir.FullName -Filter $pattern.Value -File | Select-Object -First 1
27+
if ($file) {
28+
Write-Host "Found file: $($file.FullName)"
29+
$jsonContent = Get-Content $file.FullName -Raw | ConvertFrom-Json
30+
31+
if ($jsonContent -isnot [System.Collections.IEnumerable] -or $jsonContent -is [string]) {
32+
$jsonContent = @($jsonContent)
33+
}
34+
35+
$allResults += $jsonContent
36+
}
37+
}
38+
39+
Write-Host "$($pattern.Key): $($allResults.Count) result(s) found."
40+
$reportPath = Join-Path $artifactRoot "$($pattern.Key)Report.json"
41+
$allResults | ConvertTo-Json -Depth 10 | Set-Content -Path $reportPath -Encoding UTF8
42+
Write-Host "Written report to $reportPath"
43+
44+
$failed = $allResults | Where-Object { $_.Status -ne "Success" }
45+
if ($failed.Count -gt 0) {
46+
$failedReports[$pattern.Key] = $failed
47+
}
48+
}
49+
50+
if ($failedReports.Count -eq 0) {
51+
Write-Host "`n✅ Exist Successfully: All reports passed."
52+
exit 0
53+
} else {
54+
Write-Host "`n❌ Exist with Errors: Some reports failed."
55+
foreach ($key in $failedReports.Keys) {
56+
Write-Host "##[group]Failed entries in $key"
57+
$failedReports[$key] | ConvertTo-Json -Depth 10 | Write-Host
58+
Write-Host "##[endgroup]"
59+
Write-Host
60+
}
61+
exit 1
62+
}

‎.azure-pipelines/PipelineSteps/BatchGeneration/test-modules.ps1

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,22 @@ param (
44
[string]$RepoRoot
55
)
66

7+
# Install Az.Accounts module and Pester module
8+
Install-Module -Name Pester -Repository PSGallery -RequiredVersion 4.10.1 -Force -SkipPublisherCheck
9+
Install-Module -Name Az.Accounts -AllowClobber -Force -Repository PSGallery
10+
711
$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
812
Import-Module $utilFilePath -Force
913
$subModuleGroup = Get-Targets -RepoRoot $RepoRoot -TargetsOutputFileName "test$($TestEnvName)Targets.json" -MatrixKey $MatrixKey
1014

11-
if ($TestEnvName -eq 'Linux') {
12-
$accountsModulePath = Join-Path $RepoRoot 'artifacts' 'Debug' "Az.Accounts" "Az.Accounts.psd1"
13-
Import-Module $accountsModulePath -Force
14-
}
15-
1615
$results = @()
1716

1817
foreach ($subModule in $subModuleGroup) {
1918
$startTime = Get-Date
2019
$moduleName, $subModuleName = $subModule -split '/'
2120
$result = @{
21+
OSName = $TestEnvName
22+
MatrixKey = $MatrixKey
2223
Module = $moduleName
2324
SubModule = $subModuleName
2425
Status = "Success"
@@ -30,6 +31,12 @@ foreach ($subModule in $subModuleGroup) {
3031
Write-Host "Testing sub module: $subModule"
3132
$subModulePath = Join-Path $RepoRoot 'artifacts' 'Debug' "Az.$ModuleName" $subModuleName
3233
Push-Location $subModulePath
34+
# remove the integrated Az Accounts so that the installed latest one could be used for test
35+
$integratedAzAccounts = Join-Path $subModulePath 'generated' 'modules' 'Az.Accounts'
36+
If (Test-Path $integratedAzAccounts){
37+
Write-Host "Removing integrated Az.Accounts module from $integratedAzAccounts"
38+
Remove-Item -Path $integratedAzAccounts -Recurse -Force
39+
}
3340

3441
& ".\test-module.ps1"
3542

‎.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,37 @@
1+
function Get-AutorestV4ModuleMap {
2+
param (
3+
[string]$srcPath
4+
)
5+
6+
$result = @{}
7+
8+
Get-ChildItem -Path $srcPath -Directory | ForEach-Object {
9+
$module = $_
10+
11+
Get-ChildItem -Path $module.FullName -Directory | Where-Object {
12+
$_.Name -like '*.autorest'
13+
} | ForEach-Object {
14+
$subModule = $_
15+
16+
$readmePath = Join-Path $subModule.FullName 'README.md'
17+
18+
if (Test-Path $readmePath) {
19+
$readmeContent = Get-Content -Path $readmePath -Raw
20+
21+
if ($readmeContent -notmatch 'use-extension:\s+"@autorest/powershell":\s+"3.x"') {
22+
if ($result.ContainsKey($module.Name)) {
23+
$result[$module.Name] += $subModule.Name
24+
} else {
25+
$result[$module.Name] = @($subModule.Name)
26+
}
27+
}
28+
}
29+
}
30+
}
31+
32+
return $result
33+
}
34+
135
function Group-Modules {
236
param (
337
[array]$Modules,

0 commit comments

Comments
 (0)