Azure · Pan-Qi · Apr 9, 2025 · Apr 9, 2025 · Apr 9, 2025 · Apr 9, 2025
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/FailedJobTeamsTemplate.html b/.azure-pipelines/PipelineSteps/BatchGeneration/FailedJobTeamsTemplate.html
@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <meta charset="utf-8">
+</head>
+
+<body>
+    <h1 style="color: #FF0000; font-size: 24px; font-weight: bold;">Pipeline Failure Alert 🚨</h1>
+    <p style="font-size: 18px;">The pipeline **{{ pipelineName }}** has failed.</p>
+    <p style="font-size: 16px;">You can review the details at the following links:</p>
+    <ul style="list-style-type: none; padding-left: 0;">
+        <li style="font-size: 16px; margin: 10px 0;">
+            <a href="{{ pipelineUrl }}" style="color: #0078D4; text-decoration: none;">📝 Pipeline Overview</a>
+        </li>
+        <li style="font-size: 16px; margin: 10px 0;">
+            <a href="{{ runUrl }}" style="color: #0078D4; text-decoration: none;">❌ Failed Run Details</a>
+        </li>
+    </ul>
+    <hr style="border: none; height: 1px; background-color: #0078D4; margin: 20px 0;">
+    <p style="font-size: 16px;">Please check and address the issue as soon as possible.</p>
+    <p style="font-size: 16px; margin-top: 20px; font-style: italic;">Sincerely,</p>
+    <p style="font-size: 16px; font-weight: bold;">Your Azure CLI Tools Team</p>
+</body>
+
+</html>
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/analyse-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/analyse-modules.ps1
@@ -0,0 +1,92 @@
+param (
+    [string]$MatrixKey,
+    [string]$RepoRoot
+)
+
+$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
+Import-Module $utilFilePath -Force
+$moduleGroup = Get-Targets -RepoRoot $RepoRoot -TargetsOutputFileName "analyzeTargets.json" -MatrixKey $MatrixKey
+$RepoArtifacts = Join-Path $RepoRoot 'artifacts'
+$StaticAnalysisOutputDirectory = Join-Path $RepoArtifacts 'StaticAnalysisResults'
+if (-not (Test-Path -Path $StaticAnalysisOutputDirectory)) {
+    New-Item -ItemType Directory -Path $StaticAnalysisOutputDirectory
+}
+$toolsDirectory = Join-Path $RepoRoot 'tools'
+
+$results = @()  
+foreach ($moduleName in $moduleGroup) {
+    Write-Host "=============================================================="
+    Write-Host "Analysing Module: $moduleName"
+
+    $startTime = Get-Date
+    $result = @{
+        MatrixKey = $MatrixKey
+        Module = $moduleName
+        Status = "Success"
+        DurationSeconds = 0
+        Error = ""
+        FailedTasks = @()
+    }
+    $Parameters = @{
+        RepoArtifacts = $RepoArtifacts
+        StaticAnalysisOutputDirectory = $StaticAnalysisOutputDirectory
+        Configuration = "Debug"
+        TargetModule = @($moduleName)
+    }
+    $FailedTasks = @()
+    $ErrorLogPath = "$StaticAnalysisOutputDirectory/error.log"
+
+    try {
+        .("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisBreakingChange @Parameters 2>$ErrorLogPath
+        If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
+        {
+            $FailedTasks += "BreakingChange"
+        }
+        .("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisDependency @Parameters 2>>$ErrorLogPath
+        If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
+        {
+            $FailedTasks += "Dependency"
+        }
+        .("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisSignature @Parameters 2>>$ErrorLogPath
+        If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
+        {
+            $FailedTasks += "Signature"
+        }
+        .("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisHelp @Parameters 2>>$ErrorLogPath
+        If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
+        {
+            $FailedTasks += "Help"
+        }
+        .("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisUX @Parameters 2>>$ErrorLogPath
+        If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
+        {
+            $FailedTasks += "UXMetadata"
+        }
+        .("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisCmdletDiff @Parameters 2>>$ErrorLogPath
+        If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
+        {
+            $FailedTasks += "CmdletDiff"
+        }
+        If ($FailedTasks.Length -ne 0)
+        {
+            Write-Host "There are failed tasks: $FailedTasks"
+            $ErrorLog = Get-Content -Path $ErrorLogPath | Join-String -Separator "`n"
+            Write-Error $ErrorLog
+            $result.Status = "Failed"
+            $result.Error = "Failed tasks: $($FailedTasks -join ', ')"
+            $result.FailedTasks = $FailedTasks
+        }
+    } catch {
+        Write-Warning "Failed to analyse module: $moduleName"
+        Write-Warning "Error message: $($_.Exception.Message)"
+        $result.Status = "Failed"
+        $result.Error = $_.Exception.Message
+    } finally {
+        $endTine = Get-Date
+        $result.DurationSeconds = ($endTine - $startTime).TotalSeconds
+        $results += $result
+    }
+}
+
+$reportPath = Join-Path $RepoRoot "artifacts" "AnalyseReport-$MatrixKey.json"
+$results | ConvertTo-Json -Depth 5 | Out-File -FilePath $reportPath -Encoding utf8
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1
@@ -0,0 +1,119 @@
+param (
+    [string]$MatrixKey,
+    [string]$RepoRoot,
+    [string]$AutorestVersion
+)
+
+$generationTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generationTargets.json"
+$generationTargets = Get-Content -Path $generationTargetsOutPutFile -Raw | ConvertFrom-Json
+$moduleGroup = $generationTargets.$MatrixKey
+Write-Host "##[group]Generating module group $MatrixKey"
+foreach ($key in $moduleGroup.PSObject.Properties.Name | Sort-Object) {
+    $values = $moduleGroup.$key -join ', '
+    Write-Output "$key : $values"
+}
+Write-Host "##[endgroup]"
+Write-Host
+$sortedModuleNames = $moduleGroup.PSObject.Properties.Name | Sort-Object
+
+$AutorestOutputDir = Join-Path $RepoRoot "artifacts" "autorest"
+New-Item -ItemType Directory -Force -Path $AutorestOutputDir
+
+$sourceDirectory = Join-Path $RepoRoot "src"
+$generatedDirectory = Join-Path $RepoRoot "generated"
+$buildScriptsModulePath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildScripts.psm1'
+Import-Module $buildScriptsModulePath -Force
+
+$results = @()
+
+foreach ($moduleName in $sortedModuleNames) {
+    Write-Host "=============================================================="
+    Write-Host "Regenerating Module: $moduleName"
+    $moduleStartTime = Get-Date
+    $moduleResult = @{
+        Module = $moduleName
+        DurationSeconds = 0
+        Status = "Success"
+        Changed = "No"
+        SubModules = @()
+    }
+
+    $subModuleNames = $moduleGroup.$moduleName
+    foreach ($subModuleName in $subModuleNames) {
+        Write-Host "Regenerating SubModule: $subModuleName"
+        $subModuleStartTime = Get-Date
+        $subModuleResult = @{
+            MatrixKey = $MatrixKey
+            SubModule = $subModuleName
+            Status = "Success"
+            DurationSeconds = 0
+            Error = ""
+        }
+
+        try {
+            $generateLog = Join-Path $AutorestOutputDir $moduleName "$subModuleName.log"
+            if (Test-Path $generateLog) {
+                Remove-Item -Path $generateLog -Recurse -Force
+            }
+            New-Item -ItemType File -Force -Path $generateLog
+            if (-not (Update-GeneratedSubModule -ModuleRootName $moduleName -SubModuleName $subModuleName -SourceDirectory $sourceDirectory -GeneratedDirectory $generatedDirectory -GenerateLog $generateLog -IsInvokedByPipeline $true)) {
+                Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
+                Write-Warning "log can be found at $generateLog"
+                $moduleResult.Status = "Failed"
+                $subModuleResult.Status = "Failed"
+                $subModuleResult.Error = "Update-GeneratedSubModule function returned false."
+            }
+
+        } catch {
+            Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
+            Write-Warning "Error message: $($_.Exception.Message)"
+            $moduleResult.Status = "Failed"
+            $subModuleResult.Status = "Failed"
+            $subModuleResult.Error = $_.Exception.Message
+        } finally {
+            $subModuleEndTime = Get-Date
+            $subModuleResult.DurationSeconds = ($subModuleEndTime - $subModuleStartTime).TotalSeconds
+            $moduleResult.SubModules += $subModuleResult
+        }
+    }
+
+    # If the module is changed in either src or generated folder, add a change log entry
+    Set-Location $RepoRoot
+    $srcFolderModuleRelativePath = ".\src\$moduleName"
+    $generatedFolderModuleRelativePath = ".\generated\$moduleName"
+    $diffSrc = git diff --name-only HEAD -- $srcFolderModuleRelativePath
+    $diffGenerated = git diff --name-only HEAD -- $generatedFolderModuleRelativePath
+    $diff = $diffSrc -or $diffGenerated
+    if ($diff) {
+        Write-Host "Changes detected in $moduleName, adding change log"
+        $moduleResult.Changed = "Yes"
+        $changeLogPath = Join-Path $RepoRoot "src" $moduleName $moduleName "AutorestUpgradeLog.md"
+        if (-not (Test-Path $changeLogPath)) {
+            New-Item -Path $changeLogPath -ItemType File -Force | Out-Null
+            Add-Content -Path $changeLogPath -Value "## Autorest upgrade log"
+        }
+        $changeLogContent = Get-Content -Path $changeLogPath
+        $date = Get-Date -Format "dd/MM/yy"
+        $newChangeLogEntry = "* Autorest version: $AutorestVersion - $date"
+        $changeLogContent.Insert(1, $newChangeLogEntry)
+        Set-Content $changeLogPath -Value $changeLogContent
+        $moduleResult.Changed = "Yes, Autorest Change Log Updated"
+        Write-Host "New change log entry added to $changeLogPath"
+    }
+
+    $moduleEndTime = Get-Date
+    $moduleResult.DurationSeconds = ($moduleEndTime - $moduleStartTime).TotalSeconds
+    $results += $moduleResult
+}
+
+$ArtifactOutputDir = Join-Path $RepoRoot "artifacts"
+Set-Location $RepoRoot
+
+git add .
+$patchPath = Join-Path $ArtifactOutputDir "changed-$MatrixKey.patch"
+git diff --cached > $patchPath
+
+$reportPath = Join-Path $ArtifactOutputDir "GenerationReport-$MatrixKey.json"
+$results | ConvertTo-Json -Depth 5 | Out-File -FilePath $reportPath -Encoding utf8
+
+Write-Host "Build report written to $reportPath"
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/build-modules.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/build-modules.ps1
@@ -0,0 +1,40 @@
+param (
+    [string]$MatrixKey,
+    [string]$RepoRoot
+)
+
+$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
+Import-Module $utilFilePath -Force
+$moduleGroup = Get-Targets -RepoRoot $RepoRoot -TargetsOutputFileName "buildTargets.json" -MatrixKey $MatrixKey
+$buildModulesPath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildModules.ps1'
+
+$results = @()  
+foreach ($moduleName in $moduleGroup) {
+    Write-Host "=============================================================="
+    Write-Host "Building Module: $moduleName"
+
+    $startTime = Get-Date
+    $result = @{
+        MatrixKey = $MatrixKey
+        Module = $moduleName
+        Status = "Success"
+        DurationSeconds = 0
+        Error = ""
+    }
+
+    try {
+        & $buildModulesPath -TargetModule $moduleName -InvokedByPipeline
+    } catch {
+        Write-Warning "Failed to build module: $moduleName"
+        Write-Warning "Error message: $($_.Exception.Message)"
+        $result.Status = "Failed"
+        $result.Error = $_.Exception.Message
+    } finally {
+        $endTine = Get-Date
+        $result.DurationSeconds = ($endTine - $startTime).TotalSeconds
+        $results += $result
+    }
+}
+
+$reportPath = Join-Path $RepoRoot "artifacts" "BuildReport-$MatrixKey.json"
+$results | ConvertTo-Json -Depth 5 | Out-File -FilePath $reportPath -Encoding utf8
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/create-branch.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/create-branch.ps1
@@ -0,0 +1,22 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [string]$Owner,
+    [string]$Repo,
+    [string]$BaseBranch,
+    [string]$NewBranch,
+    [string]$Token
+)
+
+$headers = @{ Authorization = "Bearer $Token"; "User-Agent" = "ADO-Pipeline" }
+$branchInfo = Invoke-RestMethod -Uri "https://api.github.com/repos/$Owner/$Repo/git/ref/heads/$BaseBranch" -Headers $headers
+$sha = $branchInfo.object.sha
+
+$body = @{
+    ref = "refs/heads/$NewBranch"
+    sha = $sha
+} | ConvertTo-Json
+
+Invoke-RestMethod -Uri "https://api.github.com/repos/$Owner/$Repo/git/refs" `
+                -Method Post -Headers $headers -Body $body -ContentType "application/json"
+
+Write-Host "Created branch '$NewBranch' from '$BaseBranch'"
diff --git a/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1 b/.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1
@@ -0,0 +1,79 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [int]$MaxParallelBuildJobs = 3,
+    [int]$MaxParallelAnalyzeJobs = 3,
+    [int]$MaxParallelTestWindowsJobs = 3,
+    [int]$MaxParallelTestLinuxJobs = 3,
+    [int]$MaxParallelTestMacJobs = 3,
+    [string[]]$ChangedFiles,
+    [string]$RepoRoot
+)
+
+$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
+Import-Module $utilFilePath -Force
+$artifactsDir = Join-Path $RepoRoot 'artifacts'
+
+$changedModulesDict = @{}
+$changedSubModulesDict = @{}
+if ($env:RUN_TEST_ON_ALL_MODULES -eq "True") {
+    Write-Host "Run test on all modules"
+    $V4ModulesFile = Join-Path $artifactsDir "generationTargets.json"
+    $V4ModuleMaps = Get-Content -Raw -Path $V4ModulesFile | ConvertFrom-Json
+
+    foreach ($matrixKey in $V4ModuleMaps.PSObject.Properties.Name) {
+        $moduleMap = $V4ModuleMaps.$matrixKey
+        foreach ($moduleName in $moduleMap.PSObject.Properties.Name) {
+            foreach ($subModuleName in $moduleMap.$moduleName) {
+                $subModule = "$moduleName/$subModuleName"
+                $changedModulesDict[$moduleName] = $true
+                $changedSubModulesDict[$subModule] = $true
+            }
+        }
+    }
+}
+else {
+    Write-Host "Run test on generated folder changed modules"
+    # Only generated filder change should trigger the test 
+    for ($i = 0; $i -lt $ChangedFiles.Count; $i++) {
+        if ($ChangedFiles[$i] -match '^generated/([^/]+)/([^/]+\.autorest)/') {
+            $moduleName = $Matches[2]
+            $subModuleName = $Matches[3]
+            $subModule = "$moduleName/$subModuleName"
+
+            $changedModulesDict[$moduleName] = $true
+            $changedSubModulesDict[$subModule] = $true
+        }
+    }
+}
+
+$changedModules = $changedModulesDict.Keys | Sort-Object
+$changedSubModules = $changedSubModulesDict.Keys | Sort-Object
+
+Write-Host "##[group]Changed modules: $($changedModules.Count)"
+foreach ($module in $changedModules) {
+    Write-Host $module
+}
+Write-Host "##[endgroup]"
+    Write-Host
+
+Write-Host "##[group]Changed sub modules: $($changedSubModules.Count)"
+foreach ($subModule in $changedSubModules) {
+    Write-Host $subModule
+}
+Write-Host "##[endgroup]"
+Write-Host
+
+$groupedBuildModules = Group-Modules -Modules $changedModules -MaxParallelJobs $MaxParallelBuildJobs
+Write-Matrix -GroupedModules $groupedBuildModules -VariableName 'buildTargets' -RepoRoot $RepoRoot
+
+$groupedAnalyzeModules = Group-Modules -Modules $changedModules -MaxParallelJobs $MaxParallelAnalyzeJobs
+Write-Matrix -GroupedModules $groupedAnalyzeModules -VariableName 'analyzeTargets' -RepoRoot $RepoRoot
+
+$groupedTestWindowsModules = Group-Modules -Modules $changedSubModules -MaxParallelJobs $MaxParallelTestWindowsJobs
+Write-Matrix -GroupedModules $groupedTestWindowsModules -VariableName 'testWindowsTargets' -RepoRoot $RepoRoot
+
+$groupedTestLinuxModules = Group-Modules -Modules $changedSubModules -MaxParallelJobs $MaxParallelTestLinuxJobs
+Write-Matrix -GroupedModules $groupedTestLinuxModules -VariableName 'testLinuxTargets' -RepoRoot $RepoRoot
+
+$groupedTestMacModules = Group-Modules -Modules $changedSubModules -MaxParallelJobs $MaxParallelTestMacJobs
+Write-Matrix -GroupedModules $groupedTestMacModules -VariableName 'testMacOSTargets' -RepoRoot $RepoRoot