Incremental Builds

.github/dependabot.yml

@@ -6,6 +6,7 @@ updates:
   directories:
     - ".github/workflows/"
     - "/Templates/**/.github/workflows/"
+    - "/Actions/**/"
   groups:
     External-Dependencies:
       applies-to: version-updates

Actions/AL-Go-Helper.ps1

@@ -97,6 +97,23 @@ function Copy-HashTable() {
     }
 }
 
+# Convert SecureString to plain text
+function Get-PlainText {
+    Param(
+        [parameter(ValueFromPipeline, Mandatory = $true)]
+        [System.Security.SecureString] $SecureString
+    )
+    Process {
+        $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($SecureString);
+        try {
+            return [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr);
+        }
+        finally {
+            [Runtime.InteropServices.Marshal]::FreeBSTR($bstr);
+        }
+    }
+}
+
 function IsPropertySecret {
     param (
         [string] $propertyName
@@ -406,7 +423,7 @@ function DownloadAndImportBcContainerHelper([string] $baseFolder = $ENV:GITHUB_W
         # Read Repository Settings file (without applying organization variables, repository variables or project settings files)
         # Override default BcContainerHelper version from AL-Go-Helper only if new version is specifically specified in repo settings file
         $repoSettings = Get-Content $repoSettingsPath -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable
-        if ($repoSettings.Keys -contains "BcContainerHelperVersion") {
+        if ($repoSettings.Keys -contains "BcContainerHelperVersion" -and $defaultBcContainerHelperVersion -notlike 'https://*') {
             $bcContainerHelperVersion = $repoSettings.BcContainerHelperVersion
             Write-Host "Using BcContainerHelper $bcContainerHelperVersion version"
             if ($bcContainerHelperVersion -like "https://*") {
@@ -647,6 +664,13 @@ function ReadSettings {
         "cacheImageName"                                = "my"
         "cacheKeepDays"                                 = 3
         "alwaysBuildAllProjects"                        = $false
+        "incrementalBuilds"                             = [ordered]@{
+            "enable"                                    = $false
+            "retentionDays"                             = 30
+            "mode"                                      = "modifiedApps" # modifiedProjects, modifiedApps
+        }
+        "cicdConcurrency"                               = "Allowed"
+        "createReleaseConcurrency"                      = "Wait"
         "microsoftTelemetryConnectionString"            = "InstrumentationKey=cd2cc63e-0f37-4968-b99a-532411a314b8;IngestionEndpoint=https://northeurope-2.in.applicationinsights.azure.com/"
         "partnerTelemetryConnectionString"              = ""
         "sendExtendedTelemetryToMicrosoft"              = $false
@@ -926,11 +950,6 @@ function AnalyzeRepo {
     )
 
     $settings = $settings | Copy-HashTable
-
-    if (!$runningLocal) {
-        Write-Host "::group::Analyzing repository"
-    }
-
     $projectPath = Join-Path $baseFolder $project -Resolve
 
     # Check applicationDependency
@@ -1124,17 +1143,12 @@ function AnalyzeRepo {
             if ($performanceToolkitApps.Contains($dep.id)) { $settings.installPerformanceToolkit = $true }
         }
     }
-
-    if (!$runningLocal) {
-        Write-Host "::endgroup::"
-    }
-
     if (!$settings.doNotRunBcptTests -and -not $settings.bcptTestFolders) {
-        Write-Host "No performance test apps found in bcptTestFolders in $ALGoSettingsFile"
+        if (!$settings.doNotBuildTests) { Write-Host "No performance test apps found in bcptTestFolders in $ALGoSettingsFile" }
         $settings.doNotRunBcptTests = $true
     }
     if (!$settings.doNotRunTests -and -not $settings.testFolders) {
-        if (!$doNotIssueWarnings) { OutputWarning -message "No test apps found in testFolders in $ALGoSettingsFile" }
+        if (!$doNotIssueWarnings -and !$settings.doNotBuildTests) { OutputWarning -message "No test apps found in testFolders in $ALGoSettingsFile" }
         $settings.doNotRunTests = $true
     }
     if (-not $settings.appFolders) {
@@ -2118,7 +2132,7 @@ Function AnalyzeProjectDependencies {
             Pop-Location
         }
 
-        Write-Host "Folders containing apps are $($folders -join ',' )"
+        OutputMessageAndArray -Message "Folders containing apps" -arrayOfStrings $folders
 
         $unknownDependencies = @()
         $apps = @()
@@ -2369,6 +2383,7 @@ function RetryCommand {
         try {
             Invoke-Command $Command -ArgumentList $argumentList
             if ($LASTEXITCODE -ne 0) {
+                $host.SetShouldExit(0);
                 throw "Command failed with exit code $LASTEXITCODE"
             }
             break
@@ -2424,6 +2439,22 @@ function GetProjectsFromRepository {
     return @(GetMatchingProjects -projects $projects -selectProjects $selectProjects)
 }
 
+function GetFoldersFromAllProjects {
+    Param(
+        [string] $baseFolder
+    )
+
+    $settings = ReadSettings -baseFolder $baseFolder
+    $projects = GetProjectsFromRepository -baseFolder $baseFolder -projectsFromSettings $settings.projects
+    $folders = @()
+    foreach($project in $projects) {
+        $projectSettings = ReadSettings -project $project -baseFolder $baseFolder
+        ResolveProjectFolders -baseFolder $baseFolder -project $project -projectSettings ([ref] $projectSettings)
+        $folders += @( @($projectSettings.appFolders) + @($projectSettings.testFolders) + @($projectSettings.bcptTestFolders) | ForEach-Object { Join-Path $project "$_".Substring(2) } )
+    }
+    return $folders
+}
+
 function GetPackageVersion($packageName) {
     $alGoPackages = Get-Content -Path "$PSScriptRoot\Packages.json" | ConvertFrom-Json
 
@@ -2507,3 +2538,19 @@ function ConnectAz {
         throw "Error trying to authenticate to Azure. Error was $($_.Exception.Message)"
     }
 }
+
+function OutputMessageAndArray {
+    Param(
+        [string] $message,
+        [string[]] $arrayOfStrings
+    )
+    Write-Host "$($message):"
+    if (!$arrayOfStrings) {
+        Write-Host "- None"
+    }
+    else {
+        $arrayOfStrings | ForEach-Object {
+            Write-Host "- $_"
+        }
+    }
+}

Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1

@@ -294,6 +294,7 @@ function GetWorkflowContentWithChangesFromSettings {
     $baseName = [System.IO.Path]::GetFileNameWithoutExtension($srcFile)
     $yaml = [Yaml]::Load($srcFile)
     $workflowScheduleKey = "$($baseName)Schedule"
+    $workflowConcurrencyKey = "$($baseName)Concurrency"
 
     # Any workflow (except for the PullRequestHandler and reusable workflows (_*)) can have a RepoSetting called <workflowname>Schedule, which will be used to set the schedule for the workflow
     if ($baseName -ne "PullRequestHandler" -and $baseName -notlike '_*') {
@@ -302,6 +303,38 @@ function GetWorkflowContentWithChangesFromSettings {
             $yamlOn = $yaml.Get('on:/')
             $yaml.Replace('on:/', $yamlOn.content+@('schedule:', "  - cron: '$($repoSettings."$workflowScheduleKey")'"))
         }
+        $concurrency = 'allowed'
+        if ($repoSettings.Keys -contains $workflowConcurrencyKey) {
+            $concurrency = $repoSettings."$workflowConcurrencyKey"
+        }
+        $start = 0
+        $count = 0
+        if ($yaml.Find('concurrency:/', [ref] $start, [ref] $count)) {
+            Write-Host "$start $count"
+            $yaml.Remove($start-1, $count+1)
+        }
+        if ($concurrency -ne 'allowed') {
+            if ($start -eq 0) {
+                if (-not $yaml.Find('on:', [ref] $start, [ref] $count)) { throw "Internal error, cannot find the on: key in the workflow" }
+            }
+            switch ($concurrency) {
+                'WaitRef' {
+                    $yaml.Insert($start-1, @("concurrency:", "  group: `${{ github.workflow }}-`${{ github.ref }}"))
+                }
+                'Wait' {
+                    $yaml.Insert($start-1, @("concurrency:", "  group: `${{ github.workflow }}"))
+                }
+                'CancelRef' {
+                    $yaml.Insert($start-1, @("concurrency:", "  group: `${{ github.workflow }}-`${{ github.ref }}", "  cancel-in-progress: true"))
+                }
+                'Cancel' {
+                    $yaml.Insert($start-1, @("concurrency:", "  group: `${{ github.workflow }}", "  cancel-in-progress: true"))
+                }
+                defult: {
+                    throw "Unknown concurrency setting: $concurrency"
+                }
+            }
+        }
     }
 
     if ($baseName -eq "CICD") {

Actions/Deliver/Deliver.ps1

@@ -250,13 +250,18 @@ foreach ($thisProject in $projectList) {
             }
             elseif ($folder.Count -eq 1) {
                 Get-Item -Path (Join-Path $folder[0] "*.app") | ForEach-Object {
-                    $parameters = @{
-                        "gitHubRepository" = "$ENV:GITHUB_SERVER_URL/$ENV:GITHUB_REPOSITORY"
-                        "preReleaseTag"    = $preReleaseTag
-                        "appFile"          = $_.FullName
+                    $appJson = Get-AppJsonFromAppFile -appFile $_.FullName
+                    $packageName = Get-BcNuGetPackageId -publisher $appJson.publisher -name $appJson.name -id $appJson.id -version $appJson.version
+                    $feed, $packageId, $packageVersion = Find-BcNugetPackage -nuGetServerUrl $nuGetServerUrl -nuGetToken $nuGetToken -packageName $packageName -version $appJson.version -select Exact -allowPrerelease
+                    if (-not $feed) {
+                        $parameters = @{
+                            "gitHubRepository" = "$ENV:GITHUB_SERVER_URL/$ENV:GITHUB_REPOSITORY"
+                            "preReleaseTag"    = $preReleaseTag
+                            "appFile"          = $_.FullName
+                        }
+                        $package = New-BcNuGetPackage @parameters
+                        Push-BcNuGetPackage -nuGetServerUrl $nuGetServerUrl -nuGetToken $nuGetToken -bcNuGetPackage $package
                     }
-                    $package = New-BcNuGetPackage @parameters
-                    Push-BcNuGetPackage -nuGetServerUrl $nuGetServerUrl -nuGetToken $nuGetToken -bcNuGetPackage $package
                 }
             }
         }

Actions/DetermineBuildProject/DetermineBuildProject.ps1

@@ -0,0 +1,51 @@
+Param(
+    [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)]
+    [string] $token,
+    [Parameter(HelpMessage = "Compressed JSON string containing the list of projects that should be skipped", Mandatory = $true)]
+    [string] $skippedProjectsJson,
+    [Parameter(HelpMessage = "Name of the project to build", Mandatory = $true)]
+    [string] $project,
+    [Parameter(HelpMessage = "Id of the baseline workflow run, from which to download artifacts if build is skipped", Mandatory = $false)]
+    [string] $baselineWorkflowRunId
+)
+
+. (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve)
+
+$settings = $env:Settings | ConvertFrom-Json | ConvertTo-HashTable -recurse
+$baseFolder = $ENV:GITHUB_WORKSPACE
+$projectPath = Join-Path $baseFolder $project
+$buildArtifactFolder = Join-Path $projectPath ".buildartifacts"
+$skippedProjects = $skippedProjectsJson | ConvertFrom-Json
+$buildIt = $skippedProjects -notcontains $project
+if (!$buildIt) {
+    # Download the artifacts from the baseline workflow run
+    # Set buildIt to true if the download isn't successful
+    New-Item $buildArtifactFolder -ItemType Directory | Out-Null
+    $buildIt = $true
+    foreach($mask in @('Apps','TestApps','Dependencies','PowerPlatformSolution')) {
+        Write-Host "test $mask"
+        Write-Host "GetArtifactsFromWorkflowRun -workflowRun '$baselineWorkflowRunId' -token '$token' -api_url $env:GITHUB_API_URL -repository $env:GITHUB_REPOSITORY -mask '$mask' -projects '$project'"
+        $artifact = GetArtifactsFromWorkflowRun -workflowRun $baselineWorkflowRunId -token $token -api_url $env:GITHUB_API_URL -repository $env:GITHUB_REPOSITORY -mask $mask -projects $project
+        if ($artifact) {
+            Write-Host "Artifact found for mask $mask"
+            if ($artifact -is [Array]) {
+                throw "Multiple artifacts found with mask $mask for project $project"
+            }
+            $file = DownloadArtifact -path $buildArtifactFolder -token $token -artifact $artifact
+            if ($file) {
+                Write-Host "Artifact downloaded for mask $mask"
+                $thisArtifactFolder = Join-Path $buildArtifactFolder $mask
+                Expand-Archive -Path $file -DestinationPath $thisArtifactFolder -Force
+                Remove-Item -Path $file -Force
+                $buildIt = $false
+            }
+        }
+    }
+    if ($buildIt) {
+        # No downloads succeeded - remove the build artifact folder and build the project
+        Remove-Item -Path $buildArtifactFolder -Recurse -Force
+    }
+}
+
+Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "BuildIt=$buildIt"
+Write-Host "BuildIt=$buildIt"

Actions/DetermineBuildProject/README.md

@@ -0,0 +1,33 @@
+# Determine whether to build project
+
+Determine whether to build project
+
+## INPUT
+
+### ENV variables
+
+| Name | Description |
+| :-- | :-- |
+| Settings | env.Settings must be set by a prior call to the ReadSettings Action |
+
+### Parameters
+
+| Name | Required | Description | Default value |
+| :-- | :-: | :-- | :-- |
+| shell | | The shell (powershell or pwsh) in which the PowerShell script in this action should run | powershell |
+| token | | The GitHub token running the action | github.token |
+| skippedProjectsJson | Yes | Compressed JSON string containing the list of projects that should be skipped | |
+| project | Yes | Name of the project to build | |
+| baselineWorkflowRunId | Yes | Id of the baseline workflow run, from which to download artifacts if build is skipped | |
+
+## OUTPUT
+
+### ENV variables
+
+none
+
+### OUTPUT variables
+
+| Name | Description |
+| :-- | :-- |
+| BuildIt | True if the project should be built |

Actions/DetermineBuildProject/action.yaml

@@ -0,0 +1,42 @@
+name: Determine whether to Build Project
+author: Microsoft Corporation
+inputs:
+  shell:
+    description: Shell in which you want to run the action (powershell or pwsh)
+    required: false
+    default: powershell
+  token:
+    description: The GitHub token running the action
+    required: false
+    default: ${{ github.token }}
+  skippedProjectsJson:
+    description: Compressed JSON string containing the list of projects that should be skipped
+    required: true
+  project:
+    description: Name of the project to build
+    required: true
+  baselineWorkflowRunId:
+    description: Id of the baseline workflow run, from which to download artifacts if build is skipped
+    required: true
+outputs:
+  BuildIt:
+    description: Determines whether the project needs to be built
+    value: ${{ steps.determineBuildProject.outputs.BuildIt }}
+runs:
+  using: composite
+  steps:
+    - name: run
+      shell: ${{ inputs.shell }}
+      id: determineBuildProject
+      env:
+        _token: ${{ inputs.token }}
+        _skippedProjectsJson: ${{ inputs.skippedProjectsJson }}
+        _project: ${{ inputs.project }}
+        _baselineWorkflowRunId: ${{ inputs.baselineWorkflowRunId }}
+      run: |
+        ${{ github.action_path }}/../Invoke-AlGoAction.ps1 -ActionName "DetermineBuildProject" -Action {
+          ${{ github.action_path }}/DetermineBuildProject.ps1 -token $env:_token -skippedProjectsJson $env:_skippedProjectsJson -project $env:_project -baselineWorkflowRunId $env:_baselineWorkflowRunId
+        }
+branding:
+  icon: terminal
+  color: blue