From 2022f947827a23f7b7a5a96505d1c9d266035a7d Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 4 Nov 2024 10:39:03 +0100 Subject: [PATCH] reporting improvements. --- .../scripts/TestConfiguration.Tests.ps1 | 25 ++++- .../automation/scripts/TestConfiguration.psm1 | 4 +- .../automation/scripts/TestResults.Tests.ps1 | 32 +++--- .../automation/scripts/TestResults.psm1 | 97 ++++++++++--------- .../devops/automation/scripts/VSTS.Tests.ps1 | 44 ++++++--- .../automation/templates/main-stage.yml | 6 ++ .../pipelines/run-macos-tests-pipeline.yml | 6 ++ .../automation/templates/tests-stage.yml | 6 ++ 8 files changed, 144 insertions(+), 76 deletions(-) diff --git a/tools/devops/automation/scripts/TestConfiguration.Tests.ps1 b/tools/devops/automation/scripts/TestConfiguration.Tests.ps1 index b60750abc45c..ef4404f25019 100644 --- a/tools/devops/automation/scripts/TestConfiguration.Tests.ps1 +++ b/tools/devops/automation/scripts/TestConfiguration.Tests.ps1 @@ -15,6 +15,7 @@ Describe 'Get-TestConfiguration' { "label": "cecil", "splitByPlatforms": "false", "testPrefix": "test-prefix_", + "testStage": "simulator", }, { "label": "dotnettests", @@ -68,6 +69,7 @@ Describe 'Get-TestConfiguration' { "cecil": { "LABEL": "cecil", "TESTS_LABELS": "extra-test-labels,run-cecil-tests", + "TEST_STAGE": "simulator", "LABEL_WITH_PLATFORM": "cecil", "STATUS_CONTEXT": "status-context - cecil", "TEST_PREFIX": "test-prefix_cecil", @@ -76,6 +78,7 @@ Describe 'Get-TestConfiguration' { "dotnettests_iOS": { "LABEL": "dotnettests", "TESTS_LABELS": "extra-test-labels,run-dotnettests-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "dotnettests_iOS", "STATUS_CONTEXT": "status-context - dotnettests - iOS", "TEST_PREFIX": "test-prefix_dotnettests_iOS", @@ -85,6 +88,7 @@ Describe 'Get-TestConfiguration' { "dotnettests_macOS": { "LABEL": "dotnettests", "TESTS_LABELS": "extra-test-labels,run-dotnettests-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "dotnettests_macOS", "STATUS_CONTEXT": "status-context - dotnettests - macOS", "TEST_PREFIX": "test-prefix_dotnettests_macOS", @@ -94,6 +98,7 @@ Describe 'Get-TestConfiguration' { "dotnettests_MacCatalyst": { "LABEL": "dotnettests", "TESTS_LABELS": "extra-test-labels,run-dotnettests-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "dotnettests_MacCatalyst", "STATUS_CONTEXT": "status-context - dotnettests - MacCatalyst", "TEST_PREFIX": "test-prefix_dotnettests_MacCatalyst", @@ -103,6 +108,7 @@ Describe 'Get-TestConfiguration' { "dotnettests_tvOS": { "LABEL": "dotnettests", "TESTS_LABELS": "extra-test-labels,run-dotnettests-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "dotnettests_tvOS", "STATUS_CONTEXT": "status-context - dotnettests - tvOS", "TEST_PREFIX": "test-prefix_dotnettests_tvOS", @@ -112,6 +118,7 @@ Describe 'Get-TestConfiguration' { "dotnettests_Multiple": { "LABEL": "dotnettests", "TESTS_LABELS": "extra-test-labels,run-dotnettests-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "dotnettests_Multiple", "STATUS_CONTEXT": "status-context - dotnettests - Multiple", "TEST_PREFIX": "test-prefix_dotnettests_Multiple", @@ -121,6 +128,7 @@ Describe 'Get-TestConfiguration' { "monotouchtest_ios": { "LABEL": "monotouchtest", "TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "monotouchtest_iOS", "STATUS_CONTEXT": "status-context - monotouchtest - iOS", "TEST_PREFIX": "test-prefix_monotouchtest_ios", @@ -130,6 +138,7 @@ Describe 'Get-TestConfiguration' { "monotouchtest_macos": { "LABEL": "monotouchtest", "TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "monotouchtest_macOS", "STATUS_CONTEXT": "status-context - monotouchtest - macOS", "TEST_PREFIX": "test-prefix_monotouchtest_macos", @@ -139,6 +148,7 @@ Describe 'Get-TestConfiguration' { "monotouchtest_maccatalyst": { "LABEL": "monotouchtest", "TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "monotouchtest_MacCatalyst", "STATUS_CONTEXT": "status-context - monotouchtest - MacCatalyst", "TEST_PREFIX": "test-prefix_monotouchtest_maccatalyst", @@ -148,6 +158,7 @@ Describe 'Get-TestConfiguration' { "monotouchtest_tvos": { "LABEL": "monotouchtest", "TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "monotouchtest_tvOS", "STATUS_CONTEXT": "status-context - monotouchtest - tvOS", "TEST_PREFIX": "test-prefix_monotouchtest_tvos", @@ -174,6 +185,7 @@ Describe 'Get-TestConfiguration' { "cecil": { "LABEL": "cecil", "TESTS_LABELS": "extra-test-labels,run-cecil-tests", + "TEST_STAGE": "simulator", "LABEL_WITH_PLATFORM": "cecil", "STATUS_CONTEXT": "status-context - cecil", "TEST_PREFIX": "test-prefix_cecil", @@ -182,6 +194,7 @@ Describe 'Get-TestConfiguration' { "dotnettests_iOS": { "LABEL": "dotnettests", "TESTS_LABELS": "extra-test-labels,run-dotnettests-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "dotnettests_iOS", "STATUS_CONTEXT": "status-context - dotnettests - iOS", "TEST_PREFIX": "test-prefix_dotnettests_iOS", @@ -191,6 +204,7 @@ Describe 'Get-TestConfiguration' { "monotouchtest_ios": { "LABEL": "monotouchtest", "TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "monotouchtest_iOS", "STATUS_CONTEXT": "status-context - monotouchtest - iOS", "TEST_PREFIX": "test-prefix_monotouchtest_ios", @@ -202,7 +216,7 @@ Describe 'Get-TestConfiguration' { } - It 'suceeds when no dotnet platforms enabled' { + It 'succeeds when no dotnet platforms enabled' { $EnabledPlatforms = "" $config = Get-TestConfiguration ` @@ -217,6 +231,7 @@ Describe 'Get-TestConfiguration' { "cecil": { "LABEL": "cecil", "TESTS_LABELS": "extra-test-labels,run-cecil-tests", + "TEST_STAGE": "simulator", "LABEL_WITH_PLATFORM": "cecil", "STATUS_CONTEXT": "status-context - cecil", "TEST_PREFIX": "test-prefix_cecil", @@ -242,6 +257,7 @@ Describe 'Get-TestConfiguration' { "cecil": { "LABEL": "cecil", "TESTS_LABELS": "extra-test-labels,run-cecil-tests", + "TEST_STAGE": "simulator", "LABEL_WITH_PLATFORM": "cecil", "STATUS_CONTEXT": "status-context - cecil", "TEST_PREFIX": "test-prefix_cecil", @@ -250,6 +266,7 @@ Describe 'Get-TestConfiguration' { "dotnettests_iOS": { "LABEL": "dotnettests", "TESTS_LABELS": "extra-test-labels,run-dotnettests-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "dotnettests_iOS", "STATUS_CONTEXT": "status-context - dotnettests - iOS", "TEST_PREFIX": "test-prefix_dotnettests_iOS", @@ -259,6 +276,7 @@ Describe 'Get-TestConfiguration' { "dotnettests_macOS": { "LABEL": "dotnettests", "TESTS_LABELS": "extra-test-labels,run-dotnettests-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "dotnettests_macOS", "STATUS_CONTEXT": "status-context - dotnettests - macOS", "TEST_PREFIX": "test-prefix_dotnettests_macOS", @@ -268,6 +286,7 @@ Describe 'Get-TestConfiguration' { "dotnettests_MacCatalyst": { "LABEL": "dotnettests", "TESTS_LABELS": "extra-test-labels,run-dotnettests-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "dotnettests_MacCatalyst", "STATUS_CONTEXT": "status-context - dotnettests - MacCatalyst", "TEST_PREFIX": "test-prefix_dotnettests_MacCatalyst", @@ -277,6 +296,7 @@ Describe 'Get-TestConfiguration' { "dotnettests_Multiple": { "LABEL": "dotnettests", "TESTS_LABELS": "extra-test-labels,run-dotnettests-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "dotnettests_Multiple", "STATUS_CONTEXT": "status-context - dotnettests - Multiple", "TEST_PREFIX": "test-prefix_dotnettests_Multiple", @@ -286,6 +306,7 @@ Describe 'Get-TestConfiguration' { "monotouchtest_ios": { "LABEL": "monotouchtest", "TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "monotouchtest_iOS", "STATUS_CONTEXT": "status-context - monotouchtest - iOS", "TEST_PREFIX": "test-prefix_monotouchtest_ios", @@ -295,6 +316,7 @@ Describe 'Get-TestConfiguration' { "monotouchtest_macos": { "LABEL": "monotouchtest", "TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "monotouchtest_macOS", "STATUS_CONTEXT": "status-context - monotouchtest - macOS", "TEST_PREFIX": "test-prefix_monotouchtest_macos", @@ -304,6 +326,7 @@ Describe 'Get-TestConfiguration' { "monotouchtest_maccatalyst": { "LABEL": "monotouchtest", "TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests", + "TEST_STAGE": "test-prefix_", "LABEL_WITH_PLATFORM": "monotouchtest_MacCatalyst", "STATUS_CONTEXT": "status-context - monotouchtest - MacCatalyst", "TEST_PREFIX": "test-prefix_monotouchtest_maccatalyst", diff --git a/tools/devops/automation/scripts/TestConfiguration.psm1 b/tools/devops/automation/scripts/TestConfiguration.psm1 index df6a8cf3fa36..e5a81908f60b 100644 --- a/tools/devops/automation/scripts/TestConfiguration.psm1 +++ b/tools/devops/automation/scripts/TestConfiguration.psm1 @@ -26,11 +26,13 @@ class TestConfiguration { $underscoredLabel = $label.Replace('-','_') $splitByPlatforms = $config.splitByPlatforms $testPrefix = $config.testPrefix + $testStage = $config.testStage ? $config.testStage : $config.testPrefix $vars = [ordered]@{} # set common variables $vars["LABEL"] = $label $vars["TESTS_LABELS"] = "$($this.testsLabels),run-$($label)-tests" + $vars["TEST_STAGE"] = $testStage if ($splitByPlatforms -eq "True") { if ($enabledPlatformsForConfig.Length -eq 0) { Write-Host "No enabled platforms, skipping $label" @@ -111,7 +113,7 @@ function Get-TestConfiguration { $objTestConfigurations = ConvertFrom-Json -InputObject $TestConfigurations $objSupportedPlatforms = ConvertFrom-Json -InputObject $SupportedPlatforms - $arrEnabledPlatforms = -split $EnabledPlatforms + $arrEnabledPlatforms = -split $EnabledPlatforms | Where { $_ } $config = [TestConfiguration]::new($objTestConfigurations, $objSupportedPlatforms, $arrEnabledPlatforms, $TestsLabels, $StatusContext) return $config.Create() } diff --git a/tools/devops/automation/scripts/TestResults.Tests.ps1 b/tools/devops/automation/scripts/TestResults.Tests.ps1 index bf9c580358be..fbd65f49b905 100644 --- a/tools/devops/automation/scripts/TestResults.Tests.ps1 +++ b/tools/devops/automation/scripts/TestResults.Tests.ps1 @@ -11,7 +11,7 @@ Describe "TestResults tests" { $platform = "iOS" $resultContext = "tests" $suite = [TestSuite]::new($label) - $testConfig = [TestConfiguration]::new($suite, $title, $platform, $resultContext) + $testConfig = [TestConfiguration]::new($suite, $title, $platform, $resultContext, "testStage") $jobStatus = "Succeeded" $attempt = 1 $matrix = @" @@ -19,6 +19,7 @@ Describe "TestResults tests" { "dotnettests_ios": { "LABEL": "dotnettests", "TESTS_LABELS": "--label=skip-all-tests,run-ios-tests,run-ios-simulator-tests,run-tvos-tests,run-mac-tests,run-maccatalyst-tests,run-dotnet-tests,run-system-permission-tests,run-dotnettests-tests", + "TEST_STAGE": "testStage", "LABEL_WITH_PLATFORM": "dotnettests_iOS", "STATUS_CONTEXT": "VSTS: simulator tests - dotnettests - iOS", "TEST_simulator_tests": "simulator_dotnettests_ios", @@ -28,6 +29,7 @@ Describe "TestResults tests" { "dotnettests_tvos": { "LABEL": "dotnettests", "TESTS_LABELS": "--label=skip-all-tests,run-ios-tests,run-ios-simulator-tests,run-tvos-tests,run-mac-tests,run-maccatalyst-tests,run-dotnet-tests,run-system-permission-tests,run-dotnettests-tests", + "TEST_STAGE": "testStage", "LABEL_WITH_PLATFORM": "dotnettests_tvOS", "STATUS_CONTEXT": "VSTS: simulator tests - dotnettests - tvOS", "TEST_simulator_tests": "simulator_dotnettests_tvos", @@ -37,6 +39,7 @@ Describe "TestResults tests" { "dotnettests_maccatalyst": { "LABEL": "dotnettests", "TESTS_LABELS": "--label=skip-all-tests,run-ios-tests,run-ios-simulator-tests,run-tvos-tests,run-mac-tests,run-maccatalyst-tests,run-dotnet-tests,run-system-permission-tests,run-dotnettests-tests", + "TEST_STAGE": "testStage", "LABEL_WITH_PLATFORM": "dotnettests_MacCatalyst", "STATUS_CONTEXT": "VSTS: simulator tests - dotnettests - MacCatalyst", "TEST_simulator_tests": "simulator_dotnettests_maccatalyst", @@ -46,6 +49,7 @@ Describe "TestResults tests" { "dotnettests_macos": { "LABEL": "dotnettests", "TESTS_LABELS": "--label=skip-all-tests,run-ios-tests,run-ios-simulator-tests,run-tvos-tests,run-mac-tests,run-maccatalyst-tests,run-dotnet-tests,run-system-permission-tests,run-dotnettests-tests", + "TEST_STAGE": "testStage", "LABEL_WITH_PLATFORM": "dotnettests_macOS", "STATUS_CONTEXT": "VSTS: simulator tests - dotnettests - macOS", "TEST_simulator_tests": "simulator_dotnettests_macos", @@ -55,6 +59,7 @@ Describe "TestResults tests" { "dotnettests_multiple": { "LABEL": "dotnettests", "TESTS_LABELS": "--label=skip-all-tests,run-ios-tests,run-ios-simulator-tests,run-tvos-tests,run-mac-tests,run-maccatalyst-tests,run-dotnet-tests,run-system-permission-tests,run-dotnettests-tests", + "TEST_STAGE": "testStage", "LABEL_WITH_PLATFORM": "dotnettests_Multiple", "STATUS_CONTEXT": "VSTS: simulator tests - dotnettests - Multiple", "TEST_simulator_tests": "simulator_dotnettests_multiple", @@ -65,6 +70,7 @@ Describe "TestResults tests" { { "LABEL": "cecil", "TESTS_LABELS": "--label=skip-all-tests,run-ios-64-tests,run-ios-simulator-tests,run-tvos-tests,run-mac-tests,run-maccatalyst-tests,run-dotnet-tests,run-system-permission-tests,run-cecil-tests", + "TEST_STAGE": "testStage", "LABEL_WITH_PLATFORM": "cecil", "STATUS_CONTEXT": "VSTS: simulator tests - cecil", "TEST_simulator_tests": "simulator_cecil", @@ -478,15 +484,15 @@ Describe "TestResults tests" { -[Html Report (VSDrops)](vsdropsIndex/simulator_testsdotnettests_MacCatalyst-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-simulator_testsdotnettests_MacCatalyst-1&api-version=6.0&`$format=zip) +[Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_maccatalyst-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_maccatalyst-1&api-version=6.0&`$format=zip) ## Successes -:white_check_mark: cecil: All 1 tests passed. [Html Report (VSDrops)](vsdropsIndex/simulator_testscecil-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-simulator_testscecil-1&api-version=6.0&`$format=zip) -:white_check_mark: dotnettests (iOS): All 3 tests passed. [Html Report (VSDrops)](vsdropsIndex/simulator_testsdotnettests_iOS-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-simulator_testsdotnettests_iOS-1&api-version=6.0&`$format=zip) -:white_check_mark: dotnettests (macOS): All 6 tests passed. [Html Report (VSDrops)](vsdropsIndex/simulator_testsdotnettests_macOS-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-simulator_testsdotnettests_macOS-1&api-version=6.0&`$format=zip) -:white_check_mark: dotnettests (Multiple platforms): All 7 tests passed. [Html Report (VSDrops)](vsdropsIndex/simulator_testsdotnettests_Multiple-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-simulator_testsdotnettests_Multiple-1&api-version=6.0&`$format=zip) -:white_check_mark: dotnettests (tvOS): All 4 tests passed. [Html Report (VSDrops)](vsdropsIndex/simulator_testsdotnettests_tvOS-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-simulator_testsdotnettests_tvOS-1&api-version=6.0&`$format=zip) +:white_check_mark: cecil: All 1 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagececil-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagececil-1&api-version=6.0&`$format=zip) +:white_check_mark: dotnettests (iOS): All 3 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_ios-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_ios-1&api-version=6.0&`$format=zip) +:white_check_mark: dotnettests (macOS): All 6 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_macos-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_macos-1&api-version=6.0&`$format=zip) +:white_check_mark: dotnettests (Multiple platforms): All 7 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_multiple-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_multiple-1&api-version=6.0&`$format=zip) +:white_check_mark: dotnettests (tvOS): All 4 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_tvos-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_tvos-1&api-version=6.0&`$format=zip) [comment]: <> (This is a test result report added by Azure DevOps) " @@ -532,15 +538,15 @@ Describe "TestResults tests" { -[Html Report (VSDrops)](vsdropsIndex/simulator_testsdotnettests_MacCatalyst-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-simulator_testsdotnettests_MacCatalyst-1&api-version=6.0&`$format=zip) +[Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_maccatalyst-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_maccatalyst-1&api-version=6.0&`$format=zip) ## Successes -:white_check_mark: cecil: All 1 tests passed. [Html Report (VSDrops)](vsdropsIndex/simulator_testscecil-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-simulator_testscecil-1&api-version=6.0&`$format=zip) -:white_check_mark: dotnettests (iOS): All 3 tests passed. [Html Report (VSDrops)](vsdropsIndex/simulator_testsdotnettests_iOS-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-simulator_testsdotnettests_iOS-1&api-version=6.0&`$format=zip) -:white_check_mark: dotnettests (macOS): All 6 tests passed. [Html Report (VSDrops)](vsdropsIndex/simulator_testsdotnettests_macOS-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-simulator_testsdotnettests_macOS-1&api-version=6.0&`$format=zip) -:white_check_mark: dotnettests (Multiple platforms): All 7 tests passed. [Html Report (VSDrops)](vsdropsIndex/simulator_testsdotnettests_Multiple-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-simulator_testsdotnettests_Multiple-1&api-version=6.0&`$format=zip) -:white_check_mark: dotnettests (tvOS): All 4 tests passed. [Html Report (VSDrops)](vsdropsIndex/simulator_testsdotnettests_tvOS-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-simulator_testsdotnettests_tvOS-1&api-version=6.0&`$format=zip) +:white_check_mark: cecil: All 1 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagececil-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagececil-1&api-version=6.0&`$format=zip) +:white_check_mark: dotnettests (iOS): All 3 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_ios-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_ios-1&api-version=6.0&`$format=zip) +:white_check_mark: dotnettests (macOS): All 6 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_macos-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_macos-1&api-version=6.0&`$format=zip) +:white_check_mark: dotnettests (Multiple platforms): All 7 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_multiple-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_multiple-1&api-version=6.0&`$format=zip) +:white_check_mark: dotnettests (tvOS): All 4 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_tvos-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_tvos-1&api-version=6.0&`$format=zip) [comment]: <> (This is a test result report added by Azure DevOps) " diff --git a/tools/devops/automation/scripts/TestResults.psm1 b/tools/devops/automation/scripts/TestResults.psm1 index 9d276472f728..c06f0da24c57 100644 --- a/tools/devops/automation/scripts/TestResults.psm1 +++ b/tools/devops/automation/scripts/TestResults.psm1 @@ -22,23 +22,26 @@ class TestConfiguration { [string] $Title [string] $Platform [string] $Context + [string] $TestStage TestConfiguration ( [TestSuite] $suite, [string] $title, [string] $platform, - [string] $context + [string] $context, + [string] $testStage ) { $this.Suite = $suite $this.Title = $title $this.Platform = $platform $this.Context = $context + $this.TestStage = $testStage } [string] ToString() { - return "$($this.Suite.Label) : $($this.Title) - $($this.Platform) - $($this.Context)" + return "$($this.Suite.Label) : $($this.Title) - $($this.Platform) - $($this.Context) - $($this.TestStage)" } } @@ -51,6 +54,7 @@ class TestResult { [string] $Title [string] $Platform [string] $Context + [string] $TestStage hidden [int] $Passed hidden [int] $Failed hidden [string[]] $NotTestSummaryLabels = @() @@ -71,6 +75,7 @@ class TestResult { $this.Title = $testConfiguration.Title $this.Platform = $testConfiguration.Platform $this.Context = $testConfiguration.Context + $this.TestStage = $testConfiguration.TestStage Write-Host "TestsResult::new($path, $status, $testConfiguration, $attempt) Label: $($this.Label) Platform: $($this.Platform) Title: $($this.Title) Context: $($this.Context)" } @@ -249,19 +254,16 @@ class TestResult { class ParallelTestsResults { [string] $BuildFailureMessage [string] $Context - [string] $TestPrefix [string] $VSDropsIndex [TestResult[]] $Results ParallelTestsResults ( [TestResult[]] $results, [string] $context, - [string] $testPrefix, [string] $vsDropsIndex ) { $this.Results = $results $this.Context = $context - $this.TestPrefix = $testPrefix $this.VSDropsIndex = $vsDropsIndex } @@ -324,8 +326,8 @@ class ParallelTestsResults { } [string] GetDownloadLinks($testResult) { - $dropsIndex = "$($this.VSDropsIndex)/$($this.TestPrefix)$($testResult.Title)-$($testResult.Attempt)/;/tests/vsdrops_index.html" - $artifactUrl = "$Env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI$Env:SYSTEM_TEAMPROJECT/_apis/build/builds/$Env:BUILD_BUILDID/artifacts?artifactName=HtmlReport-$($this.TestPrefix)$($testResult.Title)-$($testResult.Attempt)&api-version=6.0&`$format=zip" + $dropsIndex = "$($this.VSDropsIndex)/$($testResult.TestStage)$($testResult.Title)-$($testResult.Attempt)/;/tests/vsdrops_index.html" + $artifactUrl = "$Env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI$Env:SYSTEM_TEAMPROJECT/_apis/build/builds/$Env:BUILD_BUILDID/artifacts?artifactName=HtmlReport-$($testResult.TestStage)$($testResult.Title)-$($testResult.Attempt)&api-version=6.0&`$format=zip" $downloadInfo = "[Html Report (VSDrops)]($dropsIndex) [Download]($artifactUrl)" return $downloadInfo } @@ -447,7 +449,9 @@ function New-TestResults { [string] $Context, [int] - $Attempt + $Attempt, + [string] + $TestStage ) return [TestResult]::new($Path, $Status, [TestConfiguration]::new($Label, $Title, $Platform, $Context), $Attempt) } @@ -486,6 +490,7 @@ function New-ParallelTestsResults { Write-Host "Got title: $title with entry: $( $entry | ConvertTo-Json -Depth 100 )" $platform = $entry["TEST_PLATFORM"] $label = $entry["LABEL"] + $testStage = $entry["TEST_STAGE"] if ($suites.Contains($label)) { $suite = $suites[$label] @@ -493,7 +498,7 @@ function New-ParallelTestsResults { $suite = [TestSuite]::new($label) $suites[$label] = $suite } - $testConfig = [TestConfiguration]::new($suite, $title, $platform, "$Context - $title") + $testConfig = [TestConfiguration]::new($suite, $title, $platform, "$Context - $title", $testStage) $suite.TestConfigurations += $testConfig Write-Host "Added test config: $( $testConfig.Title )" Write-Host "To suite: $( $suite.Label )" @@ -512,10 +517,10 @@ function New-ParallelTestsResults { } Write-Host "Stage dependency $($kvp.Name) is a test dependency" - $testPrefix = $kvp.Name + $testStage = $kvp.Name $outputs = $candidate.tests.outputs - Write-Host "Outputs for $($testPrefix):" + Write-Host "Outputs for $($testStage):" Write-Host $outputs $tests = [System.Collections.SortedList]::new() foreach ($name in $outputs.Keys) { @@ -538,6 +543,7 @@ function New-ParallelTestsResults { Bot = $bot Platform = $platform Attempt = $attempt + TestStage = $testStage } if ($tests.Contains($label)) { $testInfo = $tests[$label] @@ -549,54 +555,55 @@ function New-ParallelTestsResults { Write-Host "Added $label to tests ($title) Name: $name status: $($testResult.Status) bot: $($testResult.Bot) Platform: $($testResult.Platform) Attempt: $($testResult.Attempt)" } } + } - $testResults = [System.Collections.ArrayList]@() - foreach ($suite in $suites.Values | Sort-Object -Property Label) { - $label = $suite.Label - Write-Host "Processing results for $label with $($suite.TestConfigurations.Length) configurations" - foreach ($testConfig in $suite.TestConfigurations | Sort-Object -Property Title) { - $title = $testConfig.Title - $testResult = $null - Write-Host "`tProcessing config $title" - if ($tests.Contains($label)) { - Write-Host "`t`tFound results for label: $label" - $testInfo = $tests[$label] - if ($testInfo.Contains($title)) { - $testResult = $testInfo[$title] - Write-Host "`t`tFound results for title '$title': $testResult" - } else { - Write-Host "`t`tFound NO results for title: $title" - } + $testResults = [System.Collections.ArrayList]@() + foreach ($suite in $suites.Values | Sort-Object -Property Label) { + $label = $suite.Label + Write-Host "Processing results for $label with $($suite.TestConfigurations.Length) configurations" + foreach ($testConfig in $suite.TestConfigurations | Sort-Object -Property Title) { + $title = $testConfig.Title + $testResult = $null + Write-Host "`tProcessing config $title" + if ($tests.Contains($label)) { + Write-Host "`t`tFound results for label: $label" + $testInfo = $tests[$label] + if ($testInfo.Contains($title)) { + $testResult = $testInfo[$title] + Write-Host "`t`tFound results for title '$title': $testResult" } else { - Write-Host "`tFound NO results for label: $label" + Write-Host "`t`tFound NO results for title: $title" } + } else { + Write-Host "`tFound NO results for label: $label" + } - $platform = $testConfig.Platform - $title = $testConfig.Title - if ($null -eq $testResult) { - $result = [TestResult]::new($null, "None", $testConfig, -1) - } else { - $status = $testResult.Status - $testAttempt = $testResult.Attempt - - $testSummaryPath = Join-Path "$Path" "${UploadPrefix}TestSummary-$TestPrefix$($title.Replace('-','_'))-$testAttempt" "TestSummary.md" + $platform = $testConfig.Platform + $title = $testConfig.Title + if ($null -eq $testResult) { + $result = [TestResult]::new($null, "None", $testConfig, -1) + } else { + $status = $testResult.Status + $testAttempt = $testResult.Attempt + $testStage = $testResult.TestStage - Write-Host "`t`tTest results for $label on attempt $testAttempt is '$status' in $testSummaryPath" + $testSummaryPath = Join-Path "$Path" "${UploadPrefix}TestSummary-$testStage$($title.Replace('-','_'))-$testAttempt" "TestSummary.md" - if (-not (Test-Path -Path $testSummaryPath -PathType Leaf)) { - Write-Host "`t`tWARNING: Path $testSummaryPath does not exist" - } + Write-Host "`t`tTest results for $label on attempt $testAttempt is '$status' in $testSummaryPath" - $result = [TestResult]::new($testSummaryPath, $status, $testConfig, $testAttempt) + if (-not (Test-Path -Path $testSummaryPath -PathType Leaf)) { + Write-Host "`t`tWARNING: Path $testSummaryPath does not exist" } - $testResults += $result + $result = [TestResult]::new($testSummaryPath, $status, $testConfig, $testAttempt) } + + $testResults += $result } } - return [ParallelTestsResults]::new($testResults, $Context, $TestPrefix, $VSDropsIndex) + return [ParallelTestsResults]::new($testResults, $Context, $VSDropsIndex) } Export-ModuleMember -Function New-TestResults diff --git a/tools/devops/automation/scripts/VSTS.Tests.ps1 b/tools/devops/automation/scripts/VSTS.Tests.ps1 index 1f367ac0cdfb..70f3df437803 100644 --- a/tools/devops/automation/scripts/VSTS.Tests.ps1 +++ b/tools/devops/automation/scripts/VSTS.Tests.ps1 @@ -213,17 +213,10 @@ Describe 'New-BuildConfiguration' { $buildConfiguration | ConvertTo-Json | Should -Be "{ ""DOTNET_PLATFORMS"": ""iOS tvOS"", ""PARENT_BUILD_BUILD_BUILDID"": ""BUILD_BUILDID"", - ""PARENT_BUILD_BUILD_BUILDNUMBER"": null, - ""PARENT_BUILD_BUILD_BUILDURI"": null, - ""PARENT_BUILD_BUILD_BINARIESDIRECTORY"": null, - ""PARENT_BUILD_BUILD_DEFINITIONNAME"": null, ""PARENT_BUILD_BUILD_REASON"": ""BUILD_REASON"", - ""PARENT_BUILD_BUILD_REPOSITORY_ID"": null, - ""PARENT_BUILD_BUILD_REPOSITORY_NAME"": null, - ""PARENT_BUILD_BUILD_REPOSITORY_PROVIDER"": null, - ""PARENT_BUILD_BUILD_REPOSITORY_URI"": null, ""PARENT_BUILD_BUILD_SOURCEBRANCH"": ""BUILD_SOURCEBRANCH"", ""PARENT_BUILD_BUILD_SOURCEBRANCHNAME"": ""BUILD_SOURCEBRANCHNAME"", + ""ENABLE_DOTNET"": null, ""INCLUDE_DOTNET_IOS"": null, ""IOS_NUGET_VERSION_NO_METADATA"": null, ""IOS_NUGET_SDK_NAME"": ""iOSNuGetSdkName"", @@ -236,6 +229,19 @@ Describe 'New-BuildConfiguration' { ""TVOS_NUGET_REF_NAME"": ""tvOSNuGetRefName"", ""DOTNET_TVOS_RUNTIME_IDENTIFIERS"": ""tvos-arm64"", ""tvos-arm64_NUGET_RUNTIME_NAME"": null, + ""INCLUDE_XAMARIN_LEGACY"": null, + ""INCLUDE_LEGACY_IOS"": ""false"", + ""INCLUDE_LEGACY_MACOS"": ""false"", + ""INCLUDE_LEGACY_TVOS"": ""false"", + ""INCLUDE_LEGACY_MACCATALYST"": ""false"", + ""INCLUDE_IOS"": null, + ""IOS__NUGET_OS_VERSION"": null, + ""INCLUDE_MACOS"": null, + ""MACOS__NUGET_OS_VERSION"": null, + ""INCLUDE_TVOS"": null, + ""TVOS__NUGET_OS_VERSION"": null, + ""INCLUDE_MACCATALYST"": null, + ""MACCATALYST__NUGET_OS_VERSION"": null, ""Commit"": ""BUILD_SOURCEVERSION"", ""Tags"": [ ""ciBuild"", @@ -257,17 +263,10 @@ Describe 'New-BuildConfiguration' { $buildConfiguration | Should -Be "{ ""DOTNET_PLATFORMS"": ""iOS tvOS"", ""PARENT_BUILD_BUILD_BUILDID"": ""BUILD_BUILDID"", - ""PARENT_BUILD_BUILD_BUILDNUMBER"": null, - ""PARENT_BUILD_BUILD_BUILDURI"": null, - ""PARENT_BUILD_BUILD_BINARIESDIRECTORY"": null, - ""PARENT_BUILD_BUILD_DEFINITIONNAME"": null, ""PARENT_BUILD_BUILD_REASON"": ""BUILD_REASON"", - ""PARENT_BUILD_BUILD_REPOSITORY_ID"": null, - ""PARENT_BUILD_BUILD_REPOSITORY_NAME"": null, - ""PARENT_BUILD_BUILD_REPOSITORY_PROVIDER"": null, - ""PARENT_BUILD_BUILD_REPOSITORY_URI"": null, ""PARENT_BUILD_BUILD_SOURCEBRANCH"": ""BUILD_SOURCEBRANCH"", ""PARENT_BUILD_BUILD_SOURCEBRANCHNAME"": ""BUILD_SOURCEBRANCHNAME"", + ""ENABLE_DOTNET"": null, ""INCLUDE_DOTNET_IOS"": null, ""IOS_NUGET_VERSION_NO_METADATA"": null, ""IOS_NUGET_SDK_NAME"": ""iOSNuGetSdkName"", @@ -280,6 +279,19 @@ Describe 'New-BuildConfiguration' { ""TVOS_NUGET_REF_NAME"": ""tvOSNuGetRefName"", ""DOTNET_TVOS_RUNTIME_IDENTIFIERS"": ""tvos-arm64"", ""tvos-arm64_NUGET_RUNTIME_NAME"": null, + ""INCLUDE_XAMARIN_LEGACY"": null, + ""INCLUDE_LEGACY_IOS"": ""false"", + ""INCLUDE_LEGACY_MACOS"": ""false"", + ""INCLUDE_LEGACY_TVOS"": ""false"", + ""INCLUDE_LEGACY_MACCATALYST"": ""false"", + ""INCLUDE_IOS"": null, + ""IOS__NUGET_OS_VERSION"": null, + ""INCLUDE_MACOS"": null, + ""MACOS__NUGET_OS_VERSION"": null, + ""INCLUDE_TVOS"": null, + ""TVOS__NUGET_OS_VERSION"": null, + ""INCLUDE_MACCATALYST"": null, + ""MACCATALYST__NUGET_OS_VERSION"": null, ""Commit"": ""BUILD_SOURCEVERSION"", ""Tags"": [ ""ciBuild"", diff --git a/tools/devops/automation/templates/main-stage.yml b/tools/devops/automation/templates/main-stage.yml index 0238d26da1a4..056937c2979e 100644 --- a/tools/devops/automation/templates/main-stage.yml +++ b/tools/devops/automation/templates/main-stage.yml @@ -85,6 +85,12 @@ parameters: # label: bcl, # splitByPlatforms: false, # }, + { + label: windows, + splitByPlatforms: false, + testPrefix: 'windows_integration', + testStage: 'windows_integration' + }, { label: cecil, splitByPlatforms: false, diff --git a/tools/devops/automation/templates/pipelines/run-macos-tests-pipeline.yml b/tools/devops/automation/templates/pipelines/run-macos-tests-pipeline.yml index c596c8ebdab9..79627bf64fd6 100644 --- a/tools/devops/automation/templates/pipelines/run-macos-tests-pipeline.yml +++ b/tools/devops/automation/templates/pipelines/run-macos-tests-pipeline.yml @@ -108,6 +108,12 @@ parameters: # label: bcl, # splitByPlatforms: false, # }, + { + label: windows, + splitByPlatforms: false, + testPrefix: 'windows_integration', + testStage: 'windows_integration' + }, { label: cecil, splitByPlatforms: false, diff --git a/tools/devops/automation/templates/tests-stage.yml b/tools/devops/automation/templates/tests-stage.yml index f9f4eca20bb1..f33f77dff999 100644 --- a/tools/devops/automation/templates/tests-stage.yml +++ b/tools/devops/automation/templates/tests-stage.yml @@ -77,6 +77,12 @@ parameters: # label: bcl, # splitByPlatforms: false, # }, + { + label: windows, + splitByPlatforms: false, + testPrefix: 'windows_integration', + testStage: 'windows_integration' + }, { label: cecil, splitByPlatforms: false,