Incremental Builds

.github/dependabot.yml

@@ -6,6 +6,7 @@ updates:
   directories:
     - ".github/workflows/"
     - "/Templates/**/.github/workflows/"
+    - "/Actions/**/"
   groups:
     External-Dependencies:
       applies-to: version-updates

.github/workflows/CleanupTempRepos.yaml

@@ -73,7 +73,7 @@ jobs:
           Import-Module (Join-Path "." "e2eTests/e2eTestHelper.psm1") -DisableNameChecking
           $owner = '${{ needs.Check.outputs.githubowner }}'
           $e2epat = '${{ Secrets.E2EPAT }}'
-          SetTokenAndRepository -github -githubOwner $owner -e2epat $e2epat -repository ''
+          SetTokenAndRepository -github -githubOwner $owner -token $e2epat -repository ''
           @(invoke-gh repo list $owner --limit 1000 -silent -returnValue) | ForEach-Object { $_.Split("`t")[0] } | Where-Object { "$_" -like "$owner/tmp*" } | ForEach-Object {
               $repo = $_
               Write-Host "https://github.com/$repo"

Actions/AL-Go-Helper.ps1

@@ -97,6 +97,23 @@ function Copy-HashTable() {
     }
 }
 
+# Convert SecureString to plain text
+function Get-PlainText {
+    Param(
+        [parameter(ValueFromPipeline, Mandatory = $true)]
+        [System.Security.SecureString] $SecureString
+    )
+    Process {
+        $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($SecureString);
+        try {
+            return [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr);
+        }
+        finally {
+            [Runtime.InteropServices.Marshal]::FreeBSTR($bstr);
+        }
+    }
+}
+
 function IsPropertySecret {
     param (
         [string] $propertyName
@@ -406,7 +423,7 @@ function DownloadAndImportBcContainerHelper([string] $baseFolder = $ENV:GITHUB_W
         # Read Repository Settings file (without applying organization variables, repository variables or project settings files)
         # Override default BcContainerHelper version from AL-Go-Helper only if new version is specifically specified in repo settings file
         $repoSettings = Get-Content $repoSettingsPath -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable
-        if ($repoSettings.Keys -contains "BcContainerHelperVersion") {
+        if ($repoSettings.Keys -contains "BcContainerHelperVersion" -and $defaultBcContainerHelperVersion -notlike 'https://*') {
             $bcContainerHelperVersion = $repoSettings.BcContainerHelperVersion
             Write-Host "Using BcContainerHelper $bcContainerHelperVersion version"
             if ($bcContainerHelperVersion -like "https://*") {
@@ -647,6 +664,15 @@ function ReadSettings {
         "cacheImageName"                                = "my"
         "cacheKeepDays"                                 = 3
         "alwaysBuildAllProjects"                        = $false
+        "incrementalBuilds"                             = [ordered]@{
+            "onPush"                                    = $false
+            "onPull_Request"                            = $true
+            "onSchedule"                                = $false
+            "retentionDays"                             = 30
+            "mode"                                      = "modifiedApps" # modifiedProjects, modifiedApps
+        }
+        "cicdConcurrency"                               = "Allowed"
+        "createReleaseConcurrency"                      = "Wait"
         "microsoftTelemetryConnectionString"            = "InstrumentationKey=cd2cc63e-0f37-4968-b99a-532411a314b8;IngestionEndpoint=https://northeurope-2.in.applicationinsights.azure.com/"
         "partnerTelemetryConnectionString"              = ""
         "sendExtendedTelemetryToMicrosoft"              = $false
@@ -732,29 +758,13 @@ function ReadSettings {
                     if ("$conditionalSetting" -ne "") {
                         $conditionMet = $true
                         $conditions = @()
-                        if ($conditionalSetting.PSObject.Properties.Name -eq "buildModes") {
-                            $conditionMet = $conditionMet -and ($conditionalSetting.buildModes | Where-Object { $buildMode -like $_ })
-                            $conditions += @("buildMode: $buildMode")
-                        }
-                        if ($conditionalSetting.PSObject.Properties.Name -eq "branches") {
-                            $conditionMet = $conditionMet -and ($conditionalSetting.branches | Where-Object { $branchName -like $_ })
-                            $conditions += @("branchName: $branchName")
-                        }
-                        if ($conditionalSetting.PSObject.Properties.Name -eq "repositories") {
-                            $conditionMet = $conditionMet -and ($conditionalSetting.repositories | Where-Object { $repoName -like $_ })
-                            $conditions += @("repoName: $repoName")
-                        }
-                        if ($project -and $conditionalSetting.PSObject.Properties.Name -eq "projects") {
-                            $conditionMet = $conditionMet -and ($conditionalSetting.projects | Where-Object { $project -like $_ })
-                            $conditions += @("project: $project")
-                        }
-                        if ($workflowName -and $conditionalSetting.PSObject.Properties.Name -eq "workflows") {
-                            $conditionMet = $conditionMet -and ($conditionalSetting.workflows | Where-Object { $workflowName -like $_ })
-                            $conditions += @("workflowName: $workflowName")
-                        }
-                        if ($userName -and $conditionalSetting.PSObject.Properties.Name -eq "users") {
-                            $conditionMet = $conditionMet -and ($conditionalSetting.users | Where-Object { $userName -like $_ })
-                            $conditions += @("userName: $userName")
+                        @{"buildModes" = $buildMode; "branches" = $branchName; "repositories" = $repoName; "projects" = $project; "workflows" = $workflowName; "users" = $userName}.GetEnumerator() | ForEach-Object {
+                            $propName = $_.Key
+                            $propValue = $_.Value
+                            if ($conditionMet -and $conditionalSetting.PSObject.Properties.Name -eq $propName) {
+                                $conditionMet = $propValue -and $conditionMet -and ($conditionalSetting."$propName" | Where-Object { $propValue -like $_ })
+                                $conditions += @("$($propName): $propValue")
+                            }
                         }
                         if ($conditionMet) {
                             Write-Host "Applying conditional settings for $($conditions -join ", ")"
@@ -926,11 +936,6 @@ function AnalyzeRepo {
     )
 
     $settings = $settings | Copy-HashTable
-
-    if (!$runningLocal) {
-        Write-Host "::group::Analyzing repository"
-    }
-
     $projectPath = Join-Path $baseFolder $project -Resolve
 
     # Check applicationDependency
@@ -1124,17 +1129,12 @@ function AnalyzeRepo {
             if ($performanceToolkitApps.Contains($dep.id)) { $settings.installPerformanceToolkit = $true }
         }
     }
-
-    if (!$runningLocal) {
-        Write-Host "::endgroup::"
-    }
-
     if (!$settings.doNotRunBcptTests -and -not $settings.bcptTestFolders) {
-        Write-Host "No performance test apps found in bcptTestFolders in $ALGoSettingsFile"
+        if (!$settings.doNotBuildTests) { Write-Host "No performance test apps found in bcptTestFolders in $ALGoSettingsFile" }
         $settings.doNotRunBcptTests = $true
     }
     if (!$settings.doNotRunTests -and -not $settings.testFolders) {
-        if (!$doNotIssueWarnings) { OutputWarning -message "No test apps found in testFolders in $ALGoSettingsFile" }
+        if (-not ($doNotIssueWarnings -or $settings.doNotBuildTests)) { OutputWarning -message "No test apps found in testFolders in $ALGoSettingsFile" }
         $settings.doNotRunTests = $true
     }
     if (-not $settings.appFolders) {
@@ -2119,7 +2119,7 @@ Function AnalyzeProjectDependencies {
             Pop-Location
         }
 
-        Write-Host "Folders containing apps are $($folders -join ',' )"
+        OutputMessageAndArray -Message "Folders containing apps" -arrayOfStrings $folders
 
         $unknownDependencies = @()
         $apps = @()
@@ -2382,6 +2382,7 @@ function RetryCommand {
         try {
             Invoke-Command $Command -ArgumentList $argumentList
             if ($LASTEXITCODE -ne 0) {
+                $host.SetShouldExit(0);
                 throw "Command failed with exit code $LASTEXITCODE"
             }
             break
@@ -2437,6 +2438,22 @@ function GetProjectsFromRepository {
     return @(GetMatchingProjects -projects $projects -selectProjects $selectProjects)
 }
 
+function GetFoldersFromAllProjects {
+    Param(
+        [string] $baseFolder
+    )
+
+    $settings = ReadSettings -baseFolder $baseFolder
+    $projects = GetProjectsFromRepository -baseFolder $baseFolder -projectsFromSettings $settings.projects
+    $folders = @()
+    foreach($project in $projects) {
+        $projectSettings = ReadSettings -project $project -baseFolder $baseFolder
+        ResolveProjectFolders -baseFolder $baseFolder -project $project -projectSettings ([ref] $projectSettings)
+        $folders += @( @($projectSettings.appFolders) + @($projectSettings.testFolders) + @($projectSettings.bcptTestFolders) | ForEach-Object { Join-Path $project "$_".Substring(2) } )
+    }
+    return $folders
+}
+
 function GetPackageVersion($packageName) {
     $alGoPackages = Get-Content -Path "$PSScriptRoot\Packages.json" | ConvertFrom-Json
 
@@ -2520,3 +2537,36 @@ function ConnectAz {
         throw "Error trying to authenticate to Azure. Error was $($_.Exception.Message)"
     }
 }
+
+function OutputMessageAndArray {
+    Param(
+        [string] $message,
+        [string[]] $arrayOfStrings
+    )
+    Write-Host "$($message):"
+    if (!$arrayOfStrings) {
+        Write-Host "- None"
+    }
+    else {
+        $arrayOfStrings | ForEach-Object {
+            Write-Host "- $_"
+        }
+    }
+}
+
+<#
+.SYNOPSIS
+Run an executable and check the exit code
+.EXAMPLE
+RunAndCheck git checkout -b xxx
+#>
+function RunAndCheck {
+    $ErrorActionPreference = 'SilentlyContinue'
+    $rest = if ($args.Count -gt 1) { $args[1..($args.Count - 1)] } else { $null }
+    & $args[0] $rest
+    $ErrorActionPreference = 'STOP'
+    if ($LASTEXITCODE -ne 0) {
+        $host.SetShouldExit(0)
+        throw "$($args[0]) $($rest | ForEach-Object { $_ }) failed with exit code $LASTEXITCODE"
+    }
+}

Actions/AL-Go-TestRepoHelper.ps1

@@ -43,6 +43,23 @@ function Test-Shell {
     }
 }
 
+function Test-Deprecations {
+    Param(
+        [HashTable] $json,
+        [string] $settingsDescription
+    )
+
+    # cleanModePreprocessorSymbols is deprecated
+    if ($json.Keys -contains 'cleanModePreprocessorSymbols') {
+        OutputWarning -Message "cleanModePreprocessorSymbols in $settingsDescription is deprecated. See https://aka.ms/algodeprecations#cleanModePreprocessorSymbols"
+    }
+
+    # <workflowName>Schedule is deprecated
+    ($json.Keys | Where-Object {$_ -like '*Schedule' -and $_ -ne 'WorkflowSchedule'}) | ForEach-Object {
+        OutputWarning -Message "$_ in $settingsDescription is deprecated. See https://aka.ms/algodeprecations#_workflow_Schedule. This warning will become an error in the future."
+    }
+}
+
 function Test-SettingsJson {
     Param(
         [hashtable] $json,
@@ -51,6 +68,8 @@ function Test-SettingsJson {
         [string] $type
     )
 
+    Test-Deprecations -json $json -settingsDescription $settingsDescription
+
     Test-Shell -json $json -settingsDescription $settingsDescription -property 'shell'
     Test-Shell -json $json -settingsDescription $settingsDescription -property 'gitHubRunnerShell'
 
@@ -71,18 +90,17 @@ function Test-SettingsJson {
     if ($type -eq 'Workflow') {
         # Test for things that should / should not exist in a workflow settings file
     }
+    else {
+        # workflowSchedule and workflowConcurrency should only exist in workflow specific settings files (or conditional settings - not tested)
+        Test-Property -settingsDescription $settingsDescription -json $json -key 'workflowSchedule' -maynot
+        Test-Property -settingsDescription $settingsDescription -json $json -key 'workflowConcurrency' -maynot
+    }
     if ($type -eq 'Variable') {
         # Test for things that should / should not exist in a settings variable
     }
     if ($type -eq 'Project' -or $type -eq 'Workflow') {
         # templateUrl should not be in Project or Workflow settings
         Test-Property -settingsDescription $settingsDescription -json $json -key 'templateUrl' -maynot
-
-        # schedules and runs-on should not be in Project or Workflow settings
-        # These properties are used in Update AL-Go System Files, hence they should only be in Repo settings
-        'nextMajorSchedule','nextMinorSchedule','currentSchedule','runs-on' | ForEach-Object {
-            Test-Property -settingsDescription $settingsDescription -json $json -key $_ -shouldnot
-        }
     }
 }
 
@@ -100,7 +118,7 @@ function Test-JsonStr {
 
     try {
         $json = $jsonStr | ConvertFrom-Json | ConvertTo-HashTable
-        Test-SettingsJson -json $json -settingsDescription $settingsDescription -type:$type
+        Test-SettingsJson -json $json -settingsDescription $settingsDescription -type $type
     }
     catch {
         OutputError "$($_.Exception.Message.Replace("`r",'').Replace("`n",' ')) in $settingsDescription"

Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1

@@ -303,14 +303,50 @@ function GetWorkflowContentWithChangesFromSettings {
 
     $baseName = [System.IO.Path]::GetFileNameWithoutExtension($srcFile)
     $yaml = [Yaml]::Load($srcFile)
-    $workflowScheduleKey = "$($baseName)Schedule"
+    $yamlName = $yaml.get('name:')
+    if ($yamlName) {
+        $workflowName = $yamlName.content.SubString('name:'.Length).Trim().Trim('''"').Trim()
+    }
+    else {
+        $workflowName = $baseName
+    }
+
+    $workflowScheduleKey = "WorkflowSchedule"
+    $workflowConcurrencyKey = "WorkflowConcurrency"
+    foreach($key in @($workflowScheduleKey,$workflowConcurrencyKey)) {
+        if ($repoSettings.Keys -contains $key -and ($repoSettings."$key")) {
+            throw "The $key setting is not allowed in the global repository settings. Please use the workflow specific settings file or conditional settings."
+        }
+    }
+
+    # Re-read settings and this time include workflow specific settings
+    $repoSettings = ReadSettings -buildMode '' -project '' -workflowName $workflowName -userName '' -branchName '' | ConvertTo-HashTable -recurse
 
-    # Any workflow (except for the PullRequestHandler and reusable workflows (_*)) can have a RepoSetting called <workflowname>Schedule, which will be used to set the schedule for the workflow
+    # Old Schedule key is deprecated, but still supported
+    $oldWorkflowScheduleKey = "$($baseName)Schedule"
+    if ($repoSettings.Keys -contains $oldWorkflowScheduleKey) {
+        # DEPRECATION: REPLACE WITH ERROR AFTER April 1st 2025 --->
+        if ($repoSettings.Keys -contains $workflowScheduleKey) {
+            OutputWarning "Both $oldWorkflowScheduleKey and $workflowScheduleKey are defined in the settings file. $oldWorkflowScheduleKey will be ignored. This warning will become an error in the future"
+        }
+        else {
+            Trace-DeprecationWarning -Message "$oldWorkflowScheduleKey is deprecated" -DeprecationTag "_workflow_Schedule" -WillBecomeError
+            # Convert the old <workflow>Schedule setting to the new WorkflowSchedule setting
+            $repoSettings."$workflowScheduleKey" = $repoSettings."$oldWorkflowScheduleKey"
+        }
+        # <--- REPLACE WITH ERROR AFTER April 1st 2025
+    }
+
+    # Any workflow (except for the PullRequestHandler and reusable workflows (_*)) can have concurrency and schedule defined
     if ($baseName -ne "PullRequestHandler" -and $baseName -notlike '_*') {
+        # Add Schedule and Concurrency settings to the workflow
         if ($repoSettings.Keys -contains $workflowScheduleKey) {
-            # Read the section under the on: key and add the schedule section
-            $yamlOn = $yaml.Get('on:/')
-            $yaml.Replace('on:/', $yamlOn.content+@('schedule:', "  - cron: '$($repoSettings."$workflowScheduleKey")'"))
+            # Replace or add the schedule part under the on: key
+            $yaml.ReplaceOrAdd('on:/', 'schedule:', @("- cron: '$($repoSettings."$workflowScheduleKey")'"))
+        }
+        if ($repoSettings.Keys -contains $workflowConcurrencyKey) {
+            # Replace or add the concurrency part
+            $yaml.ReplaceOrAdd('', 'concurrency:', $repoSettings."$workflowConcurrencyKey")
         }
     }
 

Actions/CheckForUpdates/CheckForUpdates.ps1

@@ -16,6 +16,7 @@
 )
 
 . (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve)
+Import-Module (Join-Path -Path $PSScriptRoot -ChildPath '..\TelemetryHelper.psm1' -Resolve)
 . (Join-Path -Path $PSScriptRoot -ChildPath "yamlclass.ps1")
 . (Join-Path -Path $PSScriptRoot -ChildPath "CheckForUpdates.HelperFunctions.ps1")
 
@@ -63,7 +64,7 @@ $headers = GetHeaders -token $readToken
 # if $downloadLatest is set to true, CheckForUpdates will download the latest version of the template repository, else it will use the templateSha setting in the .github/AL-Go-Settings file
 
 # Get Repo settings as a hashtable (do NOT read any specific project settings, nor any specific workflow, user or branch settings)
-$repoSettings = ReadSettings -project '' -workflowName '' -userName '' -branchName '' | ConvertTo-HashTable -recurse
+$repoSettings = ReadSettings -buildMode '' -project '' -workflowName '' -userName '' -branchName '' | ConvertTo-HashTable -recurse
 $templateSha = $repoSettings.templateSha
 $unusedALGoSystemFiles = $repoSettings.unusedALGoSystemFiles
 $includeBuildPP = $repoSettings.type -eq 'PTE' -and $repoSettings.powerPlatformSolutionFolder -ne ''
@@ -212,7 +213,8 @@ else {
     # $update set, update the files
     try {
         # If a pull request already exists with the same REF, then exit
-        $commitMessage = "[$updateBranch] Update AL-Go System Files from $templateInfo -  $templateSha"
+        $branchSHA = RunAndCheck git rev-list -n 1 $updateBranch
+        $commitMessage = "[$($updateBranch)@$($branchSHA.SubString(0,7))] Update AL-Go System Files from $templateInfo - $($templateSha.SubString(0,7))"
 
         # Get Token with permissions to modify workflows in this repository
         $writeToken = GetAccessToken -token $token -permissions @{"actions"="read";"contents"="write";"pull_requests"="write";"workflows"="write"}

Actions/CheckForUpdates/yamlclass.ps1

@@ -144,9 +144,18 @@ class Yaml {
             $this.Remove($start, $count)
             $this.Insert($start, $yamlContent)
         }
-        else {
-            Write-Host -ForegroundColor Red "cannot locate $line"
-        }
+    }
+
+    # Add the lines in $content to the lines for the specified Yaml path, given by $line
+    [void] Add([string] $line, [string[]] $content) {
+        $this.Replace($line, $this.Get($line).content + $content)
+    }
+
+    # Replace or add a key and content to the lines for the specified Yaml path, given by $line
+    [void] ReplaceOrAdd([string] $line, [string] $key, [string[]]$content) {
+        # Remove the key part under the line
+        $this.Replace("$line$key",@())
+        $this.Add($line, @($key) + @($content | ForEach-Object { "  $_" }))
     }
 
     # Replace all occurrences of $from with $to throughout the Yaml content