diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index c27715b7a..a4113ca9a 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -1553,7 +1553,8 @@ function CreateDevEnv { [Parameter(ParameterSetName = 'local')] [string] $containerName = "", [string] $licenseFileUrl = "", - [switch] $accept_insiderEula + [switch] $accept_insiderEula, + [switch] $clean ) if ($PSCmdlet.ParameterSetName -ne $kind) { @@ -1711,6 +1712,15 @@ function CreateDevEnv { Write-Host "Repository is empty" } + if ($clean) { + $appFolders = @() + $testFolders = @() + } + else { + $appFolders = $settings.appFolders + $testFolders = $settings.testFolders + } + $installApps = $settings.installApps $installTestApps = $settings.installTestApps @@ -1905,8 +1915,8 @@ function CreateDevEnv { -installApps $installApps ` -installTestApps $installTestApps ` -installOnlyReferencedApps:$settings.installOnlyReferencedApps ` - -appFolders $settings.appFolders ` - -testFolders $settings.testFolders ` + -appFolders $appFolders ` + -testFolders $testFolders ` -testResultsFile $testResultsFile ` -testResultsFormat 'JUnit' ` -customCodeCops $settings.customCodeCops ` diff --git a/Actions/Deliver/Deliver.ps1 b/Actions/Deliver/Deliver.ps1 index 374a5397b..760141a5d 100644 --- a/Actions/Deliver/Deliver.ps1 +++ b/Actions/Deliver/Deliver.ps1 @@ -130,7 +130,7 @@ try { else { $atypes.Split(',') | ForEach-Object { $atype = $_ - $allArtifacts = GetArtifacts -token $token -api_url $ENV:GITHUB_API_URL -repository $ENV:GITHUB_REPOSITORY -mask $atype -projects $project -version $artifacts -branch $refname + $allArtifacts = GetArtifacts -token $token -api_url $ENV:GITHUB_API_URL -repository $ENV:GITHUB_REPOSITORY -mask $atype -projects $project -version $artifacts -branch $ENV:GITHUB_REF_NAME if ($allArtifacts) { $allArtifacts | ForEach-Object { $artifactFile = DownloadArtifact -token $token -artifact $_ -path $artifactsFolder diff --git a/Actions/Deploy/Deploy.ps1 b/Actions/Deploy/Deploy.ps1 index 8e6b913c7..98415dc5e 100644 --- a/Actions/Deploy/Deploy.ps1 +++ b/Actions/Deploy/Deploy.ps1 @@ -127,8 +127,8 @@ try { if ($searchArtifacts) { New-Item $artifactsFolder -ItemType Directory | Out-Null $refname = "$ENV:GITHUB_REF_NAME".Replace('/','_') - $allArtifacts = @(GetArtifacts -token $token -api_url $ENV:GITHUB_API_URL -repository $ENV:GITHUB_REPOSITORY -mask "Apps" -projects $deploymentSettings.Projects -version $artifacts -branch $refname) - $allArtifacts += @(GetArtifacts -token $token -api_url $ENV:GITHUB_API_URL -repository $ENV:GITHUB_REPOSITORY -mask "Dependencies" -projects $deploymentSettings.Projects -version $artifacts -branch $refname) + $allArtifacts = @(GetArtifacts -token $token -api_url $ENV:GITHUB_API_URL -repository $ENV:GITHUB_REPOSITORY -mask "Apps" -projects $deploymentSettings.Projects -version $artifacts -branch $ENV:GITHUB_REF_NAME) + $allArtifacts += @(GetArtifacts -token $token -api_url $ENV:GITHUB_API_URL -repository $ENV:GITHUB_REPOSITORY -mask "Dependencies" -projects $deploymentSettings.Projects -version $artifacts -branch $ENV:GITHUB_REF_NAME) if ($allArtifacts) { $allArtifacts | ForEach-Object { $appFile = DownloadArtifact -token $token -artifact $_ -path $artifactsFolder diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index 411287d4b..b70160642 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -61,8 +61,7 @@ function InvokeWebRequest { [string] $body, [string] $outFile, [string] $uri, - [switch] $retry, - [switch] $ignoreErrors + [switch] $retry ) try { @@ -109,13 +108,8 @@ function InvokeWebRequest { Write-Host "Retry failed as well" } } - if ($ignoreErrors.IsPresent) { - Write-Host $message - } - else { - Write-Host "::Error::$message" - throw $message - } + Write-Host $message + throw $message } } @@ -579,6 +573,26 @@ function GetHeader { return $headers } +function WaitForRateLimit { + Param( + [hashtable] $headers, + [int] $percentage = 10, + [switch] $displayStatus + ) + + $rate = ((InvokeWebRequest -Headers $headers -Uri "https://api.github.com/rate_limit" -retry).Content | ConvertFrom-Json).rate + $percentRemaining = [int]($rate.remaining*100/$rate.limit) + if ($displayStatus) { + Write-Host "$($rate.remaining) API calls remaining out of $($rate.limit) ($percentRemaining%)" + } + if ($percentRemaining-lt $percentage) { + $resetTimeStamp = ([datetime] '1970-01-01Z').AddSeconds($rate.reset) + $waitTime = $resetTimeStamp.Subtract([datetime]::Now) + Write-Host "`nLess than 10% API calls left, waiting for $($waitTime.TotalSeconds) seconds for limits to reset." + Start-Sleep -seconds ($waitTime.TotalSeconds+1) + } +} + function GetReleaseNotes { Param( [string] $token, @@ -955,6 +969,7 @@ function GetArtifacts { [string] $baselineWorkflowID ) + $refname = $branch.Replace('/','_') $headers = GetHeader -token $token if ($version -eq 'latest') { $version = '*' } @@ -979,14 +994,14 @@ function GetArtifacts { # Download all artifacts matching branch and version # We might have results from multiple workflow runs, but we will have all artifacts from the workflow run that created the first matching artifact # Use the buildOutput artifact to determine the workflow run id (as that will always be there) - $artifactPattern = "*-$branch-*-$version" + $artifactPattern = "*-$refname-*-$version" # Use buildOutput artifact to determine the workflow run id to avoid excessive API calls # Reason: A project called xx-main will match the artifact pattern *-main-*-version, and there might not be any artifacts matching the mask - $buildOutputPattern = "*-$branch-BuildOutput-$version" + $buildOutputPattern = "*-$refname-BuildOutput-$version" # Old builds from PR runs are vresioned differently and should be ignored - $ignoreBuildOutputPattern1 = "*-$branch-BuildOutput-*.*.2147483647.0" + $ignoreBuildOutputPattern1 = "*-$refname-BuildOutput-*.*.2147483647.0" # Build Output from TestCurrent, TestNextMinor and TestNextMajor are named differently and should be ignored - $ignoreBuildOutputPattern2 = "*-$branch-BuildOutput-*-*" + $ignoreBuildOutputPattern2 = "*-$refname-BuildOutput-*-*" Write-Host "Analyzing artifacts matching $artifactPattern" while ($true) { if ($total_count -eq 0) { @@ -1036,7 +1051,7 @@ function GetArtifacts { $result = $matchingArtifacts | Where-Object { $_.workflow_run.id -eq $buildOutputArtifacts[0].workflow_run.id } | ForEach-Object { foreach($project in $projects.Split(',')) { $project = $project.Replace('\','_').Replace('/','_') - $artifactPattern = "$project-$branch-$mask-$version" + $artifactPattern = "$project-$refname-$mask-$version" if ($_.name -like $artifactPattern) { Write-Host "- $($_.name)" return $_ diff --git a/RELEASENOTES.md b/RELEASENOTES.md index d33409f58..6c1465287 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -13,10 +13,11 @@ Note that when using the preview version of AL-Go for GitHub, we recommend you U - Issue 997 'Deliver to AppSource' action fails for projects containing a space - Issue 987 Resource not accessible by integration when creating release from specific version - Issue 979 Publish to AppSource Documentation -- Issue 1018 Artifact setting - possiblity to read version from app.json +- Issue 1018 Artifact setting - possibility to read version from app.json - Issue 1008 Allow PullRequestHandler to use ubuntu or self hosted runners for all jobs except for pregateCheck - Issue 962 Finer control of "shell"-property -- Issue 1041 Harden the version comparisson when incrementing version number +- Issue 1041 Harden the version comparison when incrementing version number +- Issue 1042 Downloading artifacts from GitHub doesn't work with branch names which include forward slashes ### Better artifact selection @@ -39,6 +40,10 @@ The artifact setting in your project settings file can now contain a `*` instead - `appSourceMainAppFolder` is moved to the `deliverToAppSource` structure - `appSourceProductId` is moved to the `deliverToAppSource` structure +### New parameter -clean on localdevenv and clouddevenv + +Adding -clean when running localdevenv or clouddevenv will create a clean development environment without compiling and publishing your apps. + ## v5.0 ### Issues diff --git a/Scenarios/UpdateAlGoSystemFiles.md b/Scenarios/UpdateAlGoSystemFiles.md index c08a06048..f46c1eca3 100644 --- a/Scenarios/UpdateAlGoSystemFiles.md +++ b/Scenarios/UpdateAlGoSystemFiles.md @@ -2,23 +2,44 @@ *Prerequisites: A completed [scenario 5](RegisterProductionEnvironment.md)* 1. Every time a CI/CD pipeline runs, it checks whether there are updates to AL-Go system files. AL-Go System files are scripts in the **.AL-Go folder** and workflows in the **.github folder**. Looking into the details of the Check for updates to Al-Go system files, usually looks like this -![CI/CD](https://github.com/microsoft/AL-Go/assets/10775043/8322a06e-a270-4b6d-8d92-ccc547ca4555) -1. In VS Code, try to modify the **LocalDevEnv.ps1** file, **stage** the change, **commit** and **push**. -![localdevenv](https://github.com/microsoft/AL-Go/assets/10775043/9eb67bc0-5460-44c5-8ede-fc8f6545a821) -1. Now there is a difference. AL-Go doesn’t support that anybody changes the AL-Go system files and will warn about these changes. The CI/CD pipeline, which kicked off when pushing the change, tells me about this. -![summary](https://github.com/microsoft/AL-Go/assets/10775043/8b87cf1e-5f39-487d-9b39-4ebf9a39706a) -1. To update the AL-Go system files using the Update AL-Go System Files workflow, you need to provide a secret called GHTOKENWORKFLOW containing a Personal Access Token with permissions to modify workflows -1. In a browser, navigate to [New personal access token](https://github.com/settings/tokens/new) and create a new **personal access token**. Name it, set the expiration date and check the **workflow option** in the list of **scopes**. -![newPAT](https://github.com/microsoft/AL-Go/assets/10775043/1ab9978a-37e8-423a-8f8e-5c0203f7ae00) -1. Note that the above applies to **classic** tokens only. If you are creating a **Fine-grained** token, you need to specify which repositories to include and assign **Read and Write** permissions to **Contents**, **Pull Requests** and **Workflows**. -1. Generate the token and **copy it to the clipboard**. You won’t be able to see the token again. -1. On github.com, open **Settings** in your project and select **Secrets**. Choose the New repository secret button and create a secret called GHTOKENWORKFLOW and paste the personal access token in the value field and choose **Add secret**. -![PAT](https://github.com/microsoft/AL-Go/assets/10775043/7dcccca3-ec43-47ba-bffb-795332c890ad) -1. On github.com, under **Actions** in your project, select the **Update AL-Go system files** workflow and choose **Run workflow**. Leave the **Template Repository URL** blank and choose **Run workflow**. -![update](https://github.com/microsoft/AL-Go/assets/10775043/221e6aa1-27a8-47ea-b011-88bb6b7005b9) -1. Inspect the pull request and see that it indeed reverts your change to the `LocalDevEnv.ps1` file. -![update](https://github.com/microsoft/AL-Go/assets/10775043/c5811750-eeb2-4ce5-a8a6-9d7db620c81e) -1. By default, this workflow will apply any updates to the **workflow files (in .github\workflows)** or **system scripts (in .AL-Go)** from the template repository used to spin up the repository. If you want to change branch or template Url, you can specify the `templateUrl@branch` when you run the workflow. + + ![CI/CD](https://github.com/microsoft/AL-Go/assets/10775043/8322a06e-a270-4b6d-8d92-ccc547ca4555) + +2. In VS Code, try to modify the **LocalDevEnv.ps1** file, **stage** the change, **commit** and **push**. + + ![localdevenv](https://github.com/microsoft/AL-Go/assets/10775043/9eb67bc0-5460-44c5-8ede-fc8f6545a821) + +3. Now there is a difference. AL-Go doesn’t support that anybody changes the AL-Go system files and will warn about these changes. The CI/CD pipeline, which kicked off when pushing the change, tells me about this. + + ![summary](https://github.com/microsoft/AL-Go/assets/10775043/8b87cf1e-5f39-487d-9b39-4ebf9a39706a) + +4. To update the AL-Go system files using the Update AL-Go System Files workflow, you need to provide a secret called GHTOKENWORKFLOW containing a Personal Access Token with permissions to modify workflows. Personal access tokens are either fine-grained personal access tokens or classic personal access tokens. AL-Go for GitHub works with both if you have enabled these tokens in organization settings: + + ![orgimage](https://github.com/microsoft/AL-Go/assets/10775043/f3a26b6c-ddf6-4ab6-9e7e-2a4ac04a0828) + +5. To create a **classic** personal access token, navigate to [New personal access token](https://github.com/settings/tokens/new) in a browser. Name it, set the expiration date and check the **workflow option** in the list of **scopes**. The classic tokens have access to all repositories, which you as a user have access to. + + ![newPAT](https://github.com/microsoft/AL-Go/assets/10775043/1ab9978a-37e8-423a-8f8e-5c0203f7ae00) + +6. To create a **fine-grained** personal access token, navigate to [New fine-grained personal access token](https://github.com/settings/personal-access-tokens/new) in a browser. Name it, set the expiration date and specify which repositories to include. You need to and assign **Read and Write** permissions to **Contents**, **Pull Requests** and **Workflows**. You also need to assign **Read-only** permissions to **Actions**. + + ![newFineGrained](https://github.com/microsoft/AL-Go/assets/10775043/c4618a82-3f56-4423-9ab5-15b53b4b2e95) + +7. Generate the token and **copy it to the clipboard**. You won’t be able to see the token again. + +8. On github.com, open **Settings** in your project and select **Secrets**. Choose the New repository secret button and create a secret called GHTOKENWORKFLOW and paste the personal access token in the value field and choose **Add secret**. + + ![PAT](https://github.com/microsoft/AL-Go/assets/10775043/7dcccca3-ec43-47ba-bffb-795332c890ad) + +9. On github.com, under **Actions** in your project, select the **Update AL-Go system files** workflow and choose **Run workflow**. Leave the **Template Repository URL** blank and choose **Run workflow**. + + ![update](https://github.com/microsoft/AL-Go/assets/10775043/221e6aa1-27a8-47ea-b011-88bb6b7005b9) + +10. Inspect the pull request and see that it indeed reverts your change to the `LocalDevEnv.ps1` file. + + ![update](https://github.com/microsoft/AL-Go/assets/10775043/c5811750-eeb2-4ce5-a8a6-9d7db620c81e) + +11. By default, this workflow will apply any updates to the **workflow files (in .github\workflows)** or **system scripts (in .AL-Go)** from the template repository used to spin up the repository. If you want to change branch or template Url, you can specify the `templateUrl@branch` when you run the workflow. --- [back](../README.md) diff --git a/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 b/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 index 19e61a21d..eca0288fd 100644 --- a/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 +++ b/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 @@ -6,7 +6,8 @@ Param( [string] $environmentName = "", [bool] $reuseExistingEnvironment, - [switch] $fromVSCode + [switch] $fromVSCode, + [switch] $clean ) $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 @@ -78,7 +79,8 @@ CreateDevEnv ` -environmentName $environmentName ` -reuseExistingEnvironment:$reuseExistingEnvironment ` -baseFolder $baseFolder ` - -project $project + -project $project ` + -clean:$clean } catch { Write-Host -ForegroundColor Red "Error: $($_.Exception.Message)`nStacktrace: $($_.scriptStackTrace)" diff --git a/Templates/AppSource App/.AL-Go/localDevEnv.ps1 b/Templates/AppSource App/.AL-Go/localDevEnv.ps1 index 5abeb6515..af510e228 100644 --- a/Templates/AppSource App/.AL-Go/localDevEnv.ps1 +++ b/Templates/AppSource App/.AL-Go/localDevEnv.ps1 @@ -10,7 +10,8 @@ Param( [pscredential] $credential = $null, [string] $licenseFileUrl = "", [switch] $fromVSCode, - [switch] $accept_insiderEula + [switch] $accept_insiderEula, + [switch] $clean ) $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 @@ -105,8 +106,8 @@ if (-not $credential) { if (-not $licenseFileUrl) { if ($settings.type -eq "AppSource App") { - $description = "When developing AppSource Apps, your local development environment needs the developer licensefile with permissions to your AppSource app object IDs" - $default = "" + $description = "When developing AppSource Apps for Business Central versions prior to 22, your local development environment needs the developer licensefile with permissions to your AppSource app object IDs" + $default = "none" } else { $description = "When developing PTEs, you can optionally specify a developer licensefile with permissions to object IDs of your dependant apps" @@ -135,7 +136,8 @@ CreateDevEnv ` -auth $auth ` -credential $credential ` -licenseFileUrl $licenseFileUrl ` - -accept_insiderEula:$accept_insiderEula + -accept_insiderEula:$accept_insiderEula ` + -clean:$clean } catch { Write-Host -ForegroundColor Red "Error: $($_.Exception.Message)`nStacktrace: $($_.scriptStackTrace)" diff --git a/Templates/AppSource App/.github/workflows/PublishToAppSource.yaml b/Templates/AppSource App/.github/workflows/PublishToAppSource.yaml index 7065f214d..67ece3d28 100644 --- a/Templates/AppSource App/.github/workflows/PublishToAppSource.yaml +++ b/Templates/AppSource App/.github/workflows/PublishToAppSource.yaml @@ -1,4 +1,5 @@ name: ' Publish To AppSource' +run-name: 'Publish To AppSource - Version ${{ inputs.appVersion }}, Projects ${{ inputs.projects }}' on: workflow_dispatch: diff --git a/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 b/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 index 19e61a21d..eca0288fd 100644 --- a/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 +++ b/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 @@ -6,7 +6,8 @@ Param( [string] $environmentName = "", [bool] $reuseExistingEnvironment, - [switch] $fromVSCode + [switch] $fromVSCode, + [switch] $clean ) $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 @@ -78,7 +79,8 @@ CreateDevEnv ` -environmentName $environmentName ` -reuseExistingEnvironment:$reuseExistingEnvironment ` -baseFolder $baseFolder ` - -project $project + -project $project ` + -clean:$clean } catch { Write-Host -ForegroundColor Red "Error: $($_.Exception.Message)`nStacktrace: $($_.scriptStackTrace)" diff --git a/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 b/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 index f5d301b44..af510e228 100644 --- a/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 +++ b/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 @@ -10,7 +10,8 @@ Param( [pscredential] $credential = $null, [string] $licenseFileUrl = "", [switch] $fromVSCode, - [switch] $accept_insiderEula + [switch] $accept_insiderEula, + [switch] $clean ) $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 @@ -135,7 +136,8 @@ CreateDevEnv ` -auth $auth ` -credential $credential ` -licenseFileUrl $licenseFileUrl ` - -accept_insiderEula:$accept_insiderEula + -accept_insiderEula:$accept_insiderEula ` + -clean:$clean } catch { Write-Host -ForegroundColor Red "Error: $($_.Exception.Message)`nStacktrace: $($_.scriptStackTrace)" diff --git a/Workshop/Prerequisites.md b/Workshop/Prerequisites.md index f9ff981fc..9eb8a86d9 100644 --- a/Workshop/Prerequisites.md +++ b/Workshop/Prerequisites.md @@ -8,8 +8,9 @@ You will need: - Free should be enough to follow the workshop 1. An organizational **secret** called GHTOKENWORKFLOW containing a personal access token (classic or fine-grained) - Using classic tokens, the token should have **workflow** permissions (automatically includes repo permissions) - - Using Fine-grained tokens, the token should have **Read and Write** access to **Contents**, **Pull Requests** and **Workflows** (automatically includes Read-only access to Metadata) + - Using Fine-grained tokens, the token should have **Read and Write** access to **Contents**, **Pull Requests** and **Workflows** (automatically includes Read-only access to Metadata). You should also have **Read-only** access to **Actions** - The secret should be available to all public repositories (you cannot have organizational secret accessible for private repos in Free GitHub) + - Note that organizational settings must allow for personal access tokens to be used. For more information [read this](https://github.com/microsoft/AL-Go/blob/main/Scenarios/UpdateAlGoSystemFiles.md) 1. An organizational **variable** called ALGOORGSETTINGS - Containing the following JSON structure (for performance reasons) @@ -17,7 +18,7 @@ You will need: { "useCompilerFolder": true, "doNotPublishApps": true, - "artifact": "bcartifacts/sandbox/23.0.12034.13450/us/closest" + "artifact": "//*//first" } ``` diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 2d35029b0..405e710e4 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -126,15 +126,7 @@ function RunWorkflow { } $headers = GetHeader -token $token - $rate = ((InvokeWebRequest -Headers $headers -Uri "https://api.github.com/rate_limit" -retry).Content | ConvertFrom-Json).rate - $percent = [int]($rate.remaining*100/$rate.limit) - Write-Host "$($rate.remaining) API calls remaining out of $($rate.limit) ($percent%)" - if ($percent -lt 10) { - $resetTimeStamp = ([datetime] '1970-01-01Z').AddSeconds($rate.reset) - $waitTime = $resetTimeStamp.Subtract([datetime]::Now) - Write-Host "Less than 10% API calls left, waiting for $($waitTime.TotalSeconds) seconds for limits to reset." - Start-Sleep -seconds ($waitTime.TotalSeconds+1) - } + WaitForRateLimit -headers $headers -displayStatus Write-Host "Get Workflows" $url = "https://api.github.com/repos/$repository/actions/workflows" @@ -256,8 +248,9 @@ function WaitWorkflow { $status = "" do { if ($delay) { - Start-Sleep -Seconds 60 + Start-Sleep -Seconds 120 } + WaitForRateLimit -headers $headers $url = "https://api.github.com/repos/$repository/actions/runs/$runid" $run = ((InvokeWebRequest -Method Get -Headers $headers -Uri $url).Content | ConvertFrom-Json) if ($run.status -ne $status) {