Skip to content

Commit 8bdd1e1

Browse files
Pan-QiPan-Qi
committed
report 8
1 parent 4344fc4 commit 8bdd1e1

File tree

5 files changed

+70
-88
lines changed

5 files changed

+70
-88
lines changed

‎.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ param (
33
[string]$RepoRoot
44
)
55

6-
$generateTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generateTargets.json"
7-
$generateTargets = Get-Content -Path $generateTargetsOutPutFile -Raw | ConvertFrom-Json
8-
$moduleGroup = $generateTargets.$MatrixKey
6+
$generationTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generationTargets.json"
7+
$generationTargets = Get-Content -Path $generationTargetsOutPutFile -Raw | ConvertFrom-Json
8+
$moduleGroup = $generationTargets.$MatrixKey
99
Write-Host "##[group]Generating module group $MatrixKey"
1010
foreach ($key in $moduleGroup.PSObject.Properties.Name | Sort-Object) {
1111
$values = $moduleGroup.$key -join ', '
@@ -32,6 +32,7 @@ foreach ($moduleName in $sortedModuleNames) {
3232
$moduleResult = @{
3333
Module = $moduleName
3434
DurationSeconds = 0
35+
Status = "Success"
3536
SubModules = @()
3637
}
3738

@@ -53,17 +54,22 @@ foreach ($moduleName in $sortedModuleNames) {
5354
Remove-Item -Path $generateLog -Recurse -Force
5455
}
5556
New-Item -ItemType File -Force -Path $generateLog
56-
57+
# TODO(Bernard) Remove exception for EmailService.Autorest after test
58+
if ($subModuleName -eq "EmailService.Autorest") {
59+
throw "Something went wrong"
60+
}
5761
if (-not (Update-GeneratedSubModule -ModuleRootName $moduleName -SubModuleName $subModuleName -SourceDirectory $sourceDirectory -GeneratedDirectory $generatedDirectory -GenerateLog $generateLog -IsInvokedByPipeline $true)) {
5862
Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
5963
Write-Warning "log can be found at $generateLog"
64+
$moduleResult.Status = "Failed"
6065
$subModuleResult.Status = "Failed"
6166
$subModuleResult.Error = "Update-GeneratedSubModule function returned false."
6267
}
6368

6469
} catch {
6570
Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
6671
Write-Warning "Error message: $($_.Exception.Message)"
72+
$moduleResult.Status = "Failed"
6773
$subModuleResult.Status = "Failed"
6874
$subModuleResult.Error = $_.Exception.Message
6975
} finally {

‎.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1

Lines changed: 19 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -4,43 +4,13 @@ param (
44
[int]$MaxParallelJobs = 3
55
)
66

7-
$srcPath = Join-Path $RepoRoot 'src'
8-
function Get-SubModuleWithAutorestV4 {
9-
param (
10-
[string]$srcPath
11-
)
12-
13-
$result = @{}
14-
15-
Get-ChildItem -Path $srcPath -Directory | ForEach-Object {
16-
$module = $_
17-
18-
Get-ChildItem -Path $module.FullName -Directory | Where-Object {
19-
$_.Name -like '*.autorest'
20-
} | ForEach-Object {
21-
$subModule = $_
22-
23-
$readmePath = Join-Path $subModule.FullName 'README.md'
7+
$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
8+
Import-Module $utilFilePath -Force
249

25-
if (Test-Path $readmePath) {
26-
$readmeContent = Get-Content -Path $readmePath -Raw
27-
28-
if ($readmeContent -notmatch 'use-extension:\s+"@autorest/powershell":\s+"3.x"') {
29-
if ($result.ContainsKey($module.Name)) {
30-
$result[$module.Name] += $subModule.Name
31-
} else {
32-
$result[$module.Name] = @($subModule.Name)
33-
}
34-
}
35-
}
36-
}
37-
}
38-
39-
return $result
40-
}
10+
$srcPath = Join-Path $RepoRoot 'src'
4111
# TODO(Bernard): Use real function after test
42-
# $modules = Get-SubModuleWithAutorestV4 -srcPath $srcPath
43-
$modules = @{
12+
# $moduleMap = Get-AutorestV4ModuleMap -srcPath $srcPath
13+
$moduleMap = @{
4414
"DeviceRegistry" = @("DeviceRegistry.Autorest")
4515
"ArcGateway" = @("ArcGateway.Autorest")
4616
"Chaos" = @("Chaos.Autorest")
@@ -49,68 +19,40 @@ $modules = @{
4919
"Astro" = @("Astro.Autorest")
5020
"ImageBuilder" = @("ImageBuilder.Autorest")
5121
}
52-
$modules = $modules.GetEnumerator() | ForEach-Object {
53-
[PSCustomObject]@{
54-
ModuleName = $_.Key
55-
SubModules = ($_.Value | Sort-Object)
56-
}
57-
} | Sort-Object -Property ModuleName
5822

59-
Write-Host "Total matched modules: $($modules.Count)"
60-
61-
function Group-Modules {
62-
param (
63-
[array]$modules,
64-
[int]$maxParallelJobs
65-
)
66-
67-
$count = $modules.Count
68-
$n = [Math]::Min($count, $maxParallelJobs)
69-
if ($n -eq 0) {
70-
return @()
71-
}
72-
73-
$result = @()
74-
$sizePerGroup = [Math]::Ceiling($count / $n)
75-
76-
for ($i = 0; $i -lt $count; $i += $sizePerGroup) {
77-
$group = $modules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
78-
$result += ,$group
79-
}
80-
81-
return $result
82-
}
23+
Write-Host "Total matched modules: $($moduleMap.Count)"
8324

25+
$modules = @($moduleMap.Keys | Sort-Object)
8426
$groupedModules = Group-Modules -modules $modules -maxParallelJobs $MaxParallelJobs
8527
Write-Host "Total module groups: $($groupedModules.Count)"
8628

8729
$index = 0
88-
$generateTargets = @{}
30+
$generationTargets = @{}
8931
foreach ($moduleGroup in $groupedModules) {
90-
Write-Host "##[group]Prepareing module group $($index + 1)"
32+
Write-Host "##[group]Prepareing module group $($index + 1) with $($moduleGroup.Count) modules"
9133
$mergedModules = @{}
92-
foreach ($moduleObj in $moduleGroup) {
93-
Write-Host "Module $($moduleObj.ModuleName): $($moduleObj.SubModules -join ',')"
94-
$mergedModules[$moduleObj.ModuleName] = @($moduleObj.SubModules)
34+
foreach ($moduleName in $moduleGroup) {
35+
Write-Host "Module $($moduleName): $($moduleMap[$moduleName] -join ',')"
36+
$mergedModules[$moduleName] = @($moduleMap[$moduleName])
9537
$subIndex++
9638
}
9739

9840
$key = ($index + 1).ToString() + "-" + $moduleGroup.Count
99-
$generateTargets[$key] = $mergedModules
41+
$generationTargets[$key] = $mergedModules
10042
$MatrixStr = "$MatrixStr,'$key':{'MatrixKey':'$key'}"
10143
Write-Host "##[endgroup]"
10244
Write-Host
10345
$index++
10446
}
10547

106-
$generateTargetsOutputDir = Join-Path $RepoRoot "artifacts"
107-
if (-not (Test-Path -Path $generateTargetsOutputDir)) {
108-
New-Item -ItemType Directory -Path $generateTargetsOutputDir
48+
$generationTargetsOutputDir = Join-Path $RepoRoot "artifacts"
49+
if (-not (Test-Path -Path $generationTargetsOutputDir)) {
50+
New-Item -ItemType Directory -Path $generationTargetsOutputDir
10951
}
110-
$generateTargetsOutputFile = Join-Path $generateTargetsOutputDir "generateTargets.json"
111-
$generateTargets | ConvertTo-Json -Depth 5 | Out-File -FilePath $generateTargetsOutputFile -Encoding utf8
52+
$generationTargetsOutputFile = Join-Path $generationTargetsOutputDir "generationTargets.json"
53+
$generationTargets | ConvertTo-Json -Depth 5 | Out-File -FilePath $generationTargetsOutputFile -Encoding utf8
11254

11355
if ($MatrixStr -and $MatrixStr.Length -gt 1) {
11456
$MatrixStr = $MatrixStr.Substring(1)
11557
}
116-
Write-Host "##vso[task.setVariable variable=generateTargets;isOutput=true]{$MatrixStr}"
58+
Write-Host "##vso[task.setVariable variable=generationTargets;isOutput=true]{$MatrixStr}"

‎.azure-pipelines/PipelineSteps/BatchGeneration/report.ps1

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ param (
44
)
55

66
$reportFilePattern = @{
7-
Generation = "AnalyseReport-*.json"
8-
Build = "GenerationReport-*.json"
9-
Analyze = "BuildReport-*.json"
7+
Generation = "GenerationReport-*.json"
8+
Build = "BuildReport-*.json"
9+
Analyze = "AnalyseReport-*.json"
1010
TestWindows = "TestWindowsReport-*.json"
1111
TestLinux = "TestLinuxReport-*.json"
1212
TestMacOS = "TestMacOSReport-*.json"
@@ -41,7 +41,7 @@ foreach ($pattern in $reportFilePattern.GetEnumerator()) {
4141
$allResults | ConvertTo-Json -Depth 10 | Set-Content -Path $reportPath -Encoding UTF8
4242
Write-Host "Written report to $reportPath"
4343

44-
$failed = $allResults | Where-Object { $_.Status -eq "Failed" }
44+
$failed = $allResults | Where-Object { $_.Status -ne "Success" }
4545
if ($failed.Count -gt 0) {
4646
$failedReports[$pattern.Key] = $failed
4747
}

‎.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,37 @@
1+
function Get-AutorestV4ModuleMap {
2+
param (
3+
[string]$srcPath
4+
)
5+
6+
$result = @{}
7+
8+
Get-ChildItem -Path $srcPath -Directory | ForEach-Object {
9+
$module = $_
10+
11+
Get-ChildItem -Path $module.FullName -Directory | Where-Object {
12+
$_.Name -like '*.autorest'
13+
} | ForEach-Object {
14+
$subModule = $_
15+
16+
$readmePath = Join-Path $subModule.FullName 'README.md'
17+
18+
if (Test-Path $readmePath) {
19+
$readmeContent = Get-Content -Path $readmePath -Raw
20+
21+
if ($readmeContent -notmatch 'use-extension:\s+"@autorest/powershell":\s+"3.x"') {
22+
if ($result.ContainsKey($module.Name)) {
23+
$result[$module.Name] += $subModule.Name
24+
} else {
25+
$result[$module.Name] = @($subModule.Name)
26+
}
27+
}
28+
}
29+
}
30+
}
31+
32+
return $result
33+
}
34+
135
function Group-Modules {
236
param (
337
[array]$Modules,

‎.azure-pipelines/batch-generation.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,13 @@ stages:
6868
displayName: "Batch Generate: "
6969
dependsOn: prepare
7070
condition: and(
71-
ne(dependencies.prepare.outputs['mtrx.generateTargets'], ''),
72-
ne(dependencies.prepare.outputs['mtrx.generateTargets'], '{}')
71+
ne(dependencies.prepare.outputs['mtrx.generationTargets'], ''),
72+
ne(dependencies.prepare.outputs['mtrx.generationTargets'], '{}')
7373
)
7474
timeoutInMinutes: ${{ variables.GenerateTimeoutInMinutes }}
7575
pool: ${{ variables.WindowsAgentPoolName }}
7676
strategy:
77-
matrix: $[ dependencies.prepare.outputs['mtrx.generateTargets'] ]
77+
matrix: $[ dependencies.prepare.outputs['mtrx.generationTargets'] ]
7878
maxParallel: ${{ variables.MaxParallelGenerateJobs }}
7979

8080
steps:

0 commit comments

Comments
 (0)