Skip to content

Commit 0c5d692

Browse files
Pan-QiPan-Qi
committed
build
1 parent d456619 commit 0c5d692

File tree

6 files changed

+207
-154
lines changed

6 files changed

+207
-154
lines changed

‎.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ New-Item -ItemType Directory -Force -Path $AutorestOutputDir
1616
$sourceDirectory = Join-Path $RepoRoot "src"
1717
$generatedDirectory = Join-Path $RepoRoot "generated"
1818
$buildScriptsModulePath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildScripts.psm1'
19-
Import-Module $buildScriptsModulePath
19+
Import-Module $buildScriptsModulePath -Force
2020

2121
$results = @()
2222

‎.azure-pipelines/PipelineSteps/BatchGeneration/build-module.ps1

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
param (
2+
[string]$MatrixKey,
3+
[string]$RepoRoot
4+
)
5+
6+
Write-Host "Matrix Key: $MatrixKey"
7+
8+
$buildTargetsOutputFile = Join-Path $RepoRoot "artifacts" "buildTargets.json"
9+
$buildTargets = Get-Content -Path $buildTargetsOutputFile -Raw | ConvertFrom-Json
10+
$moduleGroup = $buildTargets.$MatrixKey
11+
$buildModulesPath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildModules.ps1'
12+
13+
$results = @()
14+
foreach ($moduleName in $moduleGroup) {
15+
Write-Host "=============================================================="
16+
Write-Host "Building Module: $moduleName"
17+
18+
$startTime = Get-Date
19+
$result = @{
20+
Module = $moduleName
21+
Status = "Success"
22+
DurationSeconds = 0
23+
Error = ""
24+
}
25+
26+
try {
27+
& $buildModulesPath -TargetModule $moduleName -InvokedByPipeline
28+
} catch {
29+
Write-Warning "Failed to build module: $moduleName"
30+
$result.Status = "Failed"
31+
$result.Error = $_.Exception.Message
32+
} finally {
33+
$endTine = Get-Date
34+
$result.DurationSeconds = ($endTine - $startTime).TotalSeconds
35+
$results += $result
36+
}
37+
}
38+
39+
$reportPath = Join-Path $RepoRoot "artifacts" "BuildReport-$MatrixKey.json"
40+
$results | ConvertTo-Json -Depth 5 | Out-File -FilePath $reportPath -Encoding utf8

‎.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1

Lines changed: 36 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -1,89 +1,57 @@
11
[CmdletBinding(DefaultParameterSetName="AllSet")]
22
param (
3-
[int]$MaxParalleAnalyzeJobs = 3,
4-
[int]$MaxParalleTestWindowsJobs = 3,
5-
[int]$MaxParalleTestLinuxJobs = 3,
6-
[int]$MaxParalleTestMacJobs = 3,
7-
[string[]]$ChangedFiles
3+
[int]$MaxParallelBuildJobs = 3,
4+
[int]$MaxParallelAnalyzeJobs = 3,
5+
[int]$MaxParallelTestWindowsJobs = 3,
6+
[int]$MaxParallelTestLinuxJobs = 3,
7+
[int]$MaxParallelTestMacJobs = 3,
8+
[string[]]$ChangedFiles,
9+
[string]$RepoRoot
810
)
911

10-
$autorestFolders = @{}
12+
$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
13+
Import-Module $utilFilePath -Force
14+
15+
$changedModulesDict = @{}
16+
$changedSubModulesDict = @{}
1117
for ($i = 0; $i -lt $ChangedFiles.Count; $i++) {
1218
if ($ChangedFiles[$i] -match '^(src|generated)/([^/]+)/([^/]+\.autorest)/') {
1319
$parent = $Matches[2]
1420
$child = $Matches[3]
1521
$key = "$parent/$child"
16-
17-
$autorestFolders[$key] = $true
22+
23+
$changedModulesDict[$parent] = $true
24+
$changedSubModulesDict[$key] = $true
1825
}
1926
}
27+
$changedModules = $changedModulesDict.Keys | Sort-Object
28+
$changedSubModules = $changedSubModulesDict.Keys | Sort-Object
2029

21-
$changedSubModules = $autorestFolders.Keys
22-
# TODO(Bernard) Remove test data after test
23-
# $changedSubModules = @("A", "B", "C", "D", "E", "F", "G")
24-
Write-Host "Chagned sub modules: "
25-
foreach ($subModule in $changedSubModules) {
26-
Write-Host $subModule
30+
Write-Host "##[group]Changed modules: $($changedModules.Count)"
31+
foreach ($module in $changedModules) {
32+
Write-Host $module
2733
}
34+
Write-Host "##[endgroup]"
35+
Write-Host
2836

29-
function Split-List {
30-
param (
31-
[array]$subModules,
32-
[int]$maxParallelJobs
33-
)
34-
35-
$count = $subModules.Count
36-
$n = [Math]::Min($count, $maxParallelJobs)
37-
38-
if ($n -eq 0) {
39-
return @()
40-
}
41-
42-
$result = @()
43-
44-
for ($i = 0; $i -lt $n; $i++) {
45-
$result += ,@()
46-
}
47-
48-
for ($i = 0; $i -lt $count; $i++) {
49-
$groupIndex = $i % $n
50-
$result[$groupIndex] += $subModules[$i]
51-
}
52-
53-
return ,$result
37+
Write-Host "##[group]Changed sub modules: $($changedSubModules.Count)"
38+
foreach ($subModule in $changedSubModules) {
39+
Write-Host $subModule
5440
}
41+
Write-Host "##[endgroup]"
42+
Write-Host
5543

56-
function Write-Matrix {
57-
param (
58-
[string]$variableName,
59-
[array]$groupedSubModules
60-
)
61-
62-
$index = 0
63-
foreach ($subModules in $groupedSubModules) {
64-
$moduleNamesStr = $subModules -join ','
65-
$key = ($index + 1).ToString() + "-" + $subModules.Count
66-
$MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
67-
$index++
68-
}
44+
$groupedBuildModules = Group-Modules -Modules $changedModules -MaxParallelJobs $MaxParallelBuildJobs
45+
Write-Matrix -GroupedModules $groupedBuildModules -VariableName 'buildTargets' -RepoRoot $RepoRoot
6946

70-
if ($MatrixStr -and $MatrixStr.Length -gt 1) {
71-
$MatrixStr = $MatrixStr.Substring(1)
72-
}
73-
Write-Host "##vso[task.setVariable variable=$variableName;isOutput=true]{$MatrixStr}"
74-
Write-Host "variable=$variableName; value=$MatrixStr"
75-
}
47+
$groupedAnalyzeModules = Group-Modules -Modules $changedModules -MaxParallelJobs $MaxParallelAnalyzeJobs
48+
Write-Matrix -GroupedModules $groupedAnalyzeModules -variableName 'analyzeTargets' -RepoRoot $RepoRoot
7649

77-
$groupedAnalyzeModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleAnalyzeJobs
78-
Write-Matrix -variableName 'AnalyzeTargets' -groupedSubModules $groupedAnalyzeModules
50+
# $groupedTestWindowsModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParallelTestWindowsJobs
51+
# Write-Matrix -variableName 'TestWindowsTargets' -groupedSubModules $groupedTestWindowsModules
7952

80-
$groupedTestWindowsModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestWindowsJobs
81-
Write-Matrix -variableName 'TestWindowsTargets' -groupedSubModules $groupedTestWindowsModules
53+
# $groupedTestLinuxModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParallelTestLinuxJobs
54+
# Write-Matrix -variableName 'TestLinuxTargets' -groupedSubModules $groupedTestLinuxModules
8255

83-
$groupedTestLinuxModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestLinuxJobs
84-
Write-Matrix -variableName 'TestLinuxTargets' -groupedSubModules $groupedTestLinuxModules
85-
86-
# $groupedTestMacModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestMacJobs
56+
# $groupedTestMacModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParallelTestMacJobs
8757
# Write-Matrix -variableName 'TestMacTargets' -groupedSubModules $groupedTestMacModules
88-
89-
Write-Host "##vso[task.setVariable variable=TestMacTargets;isOutput=true]{}"

‎.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ $modules = $modules.GetEnumerator() | ForEach-Object {
5858

5959
Write-Host "Total matched modules: $($modules.Count)"
6060

61-
function Group-List {
61+
function Group-Modules {
6262
param (
6363
[array]$modules,
6464
[int]$maxParallelJobs
@@ -81,7 +81,7 @@ function Group-List {
8181
return $result
8282
}
8383

84-
$groupedModules = Group-List -modules $modules -maxParallelJobs $MaxParallelJobs
84+
$groupedModules = Group-Modules -modules $modules -maxParallelJobs $MaxParallelJobs
8585
Write-Host "Total module groups: $($groupedModules.Count)"
8686

8787
$index = 0
@@ -108,7 +108,7 @@ if (-not (Test-Path -Path $generateTargetsOutputDir)) {
108108
New-Item -ItemType Directory -Path $generateTargetsOutputDir
109109
}
110110
$generateTargetsOutputFile = Join-Path $generateTargetsOutputDir "generateTargets.json"
111-
$generateTargets | ConvertTo-Json -Depth 10 | Out-File -FilePath $generateTargetsOutputFile -Encoding utf8
111+
$generateTargets | ConvertTo-Json -Depth 5 | Out-File -FilePath $generateTargetsOutputFile -Encoding utf8
112112

113113
if ($MatrixStr -and $MatrixStr.Length -gt 1) {
114114
$MatrixStr = $MatrixStr.Substring(1)

‎.azure-pipelines/PipelineSteps/BatchGeneration/util.psm1

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
function Group-Modules {
2+
param (
3+
[array]$Modules,
4+
[int]$MaxParallelJobs
5+
)
6+
7+
$count = $Modules.Count
8+
$n = [Math]::Min($count, $MaxParallelJobs)
9+
10+
if ($n -eq 0) {
11+
return @()
12+
}
13+
14+
$result = @()
15+
16+
for ($i = 0; $i -lt $n; $i++) {
17+
$result += ,@()
18+
}
19+
20+
for ($i = 0; $i -lt $count; $i++) {
21+
$groupIndex = $i % $n
22+
$result[$groupIndex] += $Modules[$i]
23+
}
24+
25+
return ,$result
26+
}
27+
28+
function Write-Matrix {
29+
param (
30+
[array]$GroupedModules,
31+
[string]$VariableName,
32+
[string]$RepoRoot
33+
)
34+
35+
Write-Host "Module groups: $($GroupedModules.Count)"
36+
$GroupedModules | ForEach-Object { $_ -join ', ' } | ForEach-Object { Write-Host $_ }
37+
38+
$targets = @{}
39+
$MatrixStr = ""
40+
$index = 0
41+
foreach ($modules in $GroupedModules) {
42+
$key = ($index + 1).ToString() + "-" + $modules.Count
43+
$MatrixStr = "$MatrixStr,'$key':{'MatrixKey':'$key'}"
44+
$targets[$key] = $modules
45+
$index++
46+
}
47+
48+
if ($MatrixStr -and $MatrixStr.Length -gt 1) {
49+
$MatrixStr = $MatrixStr.Substring(1)
50+
}
51+
Write-Host "##vso[task.setVariable variable=$VariableName;isOutput=true]{$MatrixStr}"
52+
Write-Host "variable=$VariableName; value=$MatrixStr"
53+
54+
$targetsOutputDir = Join-Path $RepoRoot "artifacts"
55+
if (-not (Test-Path -Path $targetsOutputDir)) {
56+
New-Item -ItemType Directory -Path $targetsOutputDir -Force | Out-Null
57+
}
58+
$targetsOutputFile = Join-Path $targetsOutputDir "$VariableName.json"
59+
$targets | ConvertTo-Json -Depth 5 | Out-File -FilePath $targetsOutputFile -Encoding utf8
60+
}

0 commit comments

Comments
 (0)