From 7361d854bbdb8681556ea579a5c1e2c4e1ffb051 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Mon, 9 Dec 2024 20:23:49 +0100 Subject: [PATCH] Fix BuildPass>1 artifacts and asset manifest content (#45323) --- eng/pipelines/templates/jobs/vmr-build.yml | 58 ++++-------- src/SourceBuild/content/Directory.Build.props | 9 +- .../content/Directory.Build.targets | 31 +++++++ src/SourceBuild/content/build.proj | 4 +- .../content/eng/extract-sdk-archive.proj | 2 +- .../content/eng/finish-source-only.proj | 51 ++++++++-- .../content/eng/init-cross-build.proj | 2 +- src/SourceBuild/content/eng/init-poison.proj | 2 +- .../content/eng/join-verticals.proj | 6 +- .../content/eng/merge-asset-manifests.proj | 10 +- .../GetKnownArtifactsFromAssetManifests.cs | 92 +++++++++++++++++++ .../GetKnownPackagesFromAssetManifests.cs | 46 ---------- .../MergeAssetManifests.cs | 5 +- .../repo-projects/Directory.Build.props | 2 +- .../repo-projects/Directory.Build.targets | 38 ++++++-- .../content/repo-projects/sdk.proj | 11 --- 16 files changed, 245 insertions(+), 124 deletions(-) create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/GetKnownArtifactsFromAssetManifests.cs delete mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/GetKnownPackagesFromAssetManifests.cs diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 4060893ef6c7..09910e116159 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -13,6 +13,10 @@ parameters: type: string default: '' +- name: configuration + type: string + default: 'Release' + - name: container type: string default: '' @@ -155,6 +159,11 @@ jobs: - name: sourcesPath value: $(vmrPath) + # Must be a path under $(sourcesPath). Inside the docker container, we mount $(sourcesPath) to /vmr + # and can't write outside of that folder. + - name: artifactsStagingDir + value: $(sourcesPath)/artifacts/staging + templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: @@ -166,14 +175,14 @@ jobs: sbomEnabled: false - output: pipelineArtifact - path: $(Build.ArtifactStagingDirectory)/publishing + path: $(artifactsStagingDir) artifact: $(Agent.JobName)_Artifacts displayName: Publish Artifacts sbomEnabled: true # Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs - output: buildArtifacts - PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml + PathtoPublish: $(artifactsStagingDir)/manifests/${{ parameters.configuration }}/$(Agent.JobName).xml ArtifactName: VerticalManifests displayName: Publish Vertical Manifest sbomEnabled: false @@ -276,13 +285,13 @@ jobs: if /I '${{ parameters.useDevVersions }}'=='True' set extraBuildArguments=%extraBuildArguments% -dev set extraBuildProperties= if not [${{ parameters.buildPass }}]==[] set extraBuildProperties=%extraBuildProperties% /p:DotNetBuildPass=${{ parameters.buildPass }} - call build.cmd -ci -cleanWhileBuilding -prepareMachine %extraBuildArguments% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) %extraBuildProperties% ${{ parameters.extraProperties }} + call build.cmd -ci -cleanWhileBuilding -prepareMachine %extraBuildArguments% -c ${{ parameters.configuration }} /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) %extraBuildProperties% ${{ parameters.extraProperties }} displayName: Build workingDirectory: ${{ variables.sourcesPath }} - ${{ if eq(parameters.runTests, 'True') }}: - script: | - call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) ${{ parameters.extraProperties }} + call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog -c ${{ parameters.configuration }} /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) ${{ parameters.extraProperties }} displayName: Run Tests workingDirectory: ${{ variables.sourcesPath }} timeoutInMinutes: ${{ variables.runTestsTimeout }} @@ -330,7 +339,8 @@ jobs: df -h customEnvVars="" - customBuildArgs="--ci --clean-while-building --prepareMachine" + customBuildArgs="--ci --clean-while-building --prepareMachine -c ${{ parameters.configuration }}" + if [[ '${{ parameters.runOnline }}' == 'True' ]]; then customBuildArgs="$customBuildArgs --online" fi @@ -473,10 +483,10 @@ jobs: # Only use Docker when a container is specified if [[ -n "${{ parameters.container }}" ]]; then - docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog $customBuildArgs $extraBuildProperties $(additionalBuildArgs) + docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog -c ${{ parameters.configuration }} $customBuildArgs $extraBuildProperties $(additionalBuildArgs) else cd $(sourcesPath) - ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog $customBuildArgs $extraBuildProperties $(additionalBuildArgs) + ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog -c ${{ parameters.configuration }} $customBuildArgs $extraBuildProperties $(additionalBuildArgs) fi displayName: Run Tests @@ -588,38 +598,8 @@ jobs: publishRunAttachments: true testRunTitle: ScenarioTests_$(Agent.JobName) - - task: CopyFiles@2 - inputs: - SourceFolder: $(sourcesPath)/artifacts - Contents: | - assets/** - manifests/** - TargetFolder: $(Build.ArtifactStagingDirectory)/publishing - displayName: Copy artifacts to Artifact Staging Directory - - - ${{ if eq(parameters.targetOS, 'windows') }}: - - powershell: | - $sourcePath = "$(sourcesPath)/artifacts/manifests/VerticalManifest.xml" - $targetPath = "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml" - New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)/manifests" -Force | Out-Null - Copy-Item $sourcePath -Destination $targetPath -Force - displayName: Copy vertical manifest to Artifact Staging Directory - - ${{ else }}: - - script: | - mkdir -p "$(Build.ArtifactStagingDirectory)/manifests" - cp "$(sourcesPath)/artifacts/manifests/VerticalManifest.xml" "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml" - displayName: Copy vertical manifest to Artifact Staging Directory - - # When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages - - ${{ if ne(parameters.buildSourceOnly, 'true') }}: - - task: CopyFiles@2 - inputs: - SourceFolder: $(sourcesPath)/artifacts/packages - TargetFolder: $(Build.ArtifactStagingDirectory)/publishing/packages - displayName: Copy packages to Artifact Staging Directory - - ${{ if or(ne(variables['System.TeamProject'], 'internal'), eq(variables['Build.Reason'], 'PullRequest')) }}: - - publish: $(Build.ArtifactStagingDirectory)/publishing + - publish: $(artifactsStagingDir) artifact: $(Agent.JobName)_Artifacts displayName: Publish Artifacts continueOnError: true @@ -627,6 +607,6 @@ jobs: # Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs - task: PublishBuildArtifacts@1 inputs: - PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml + PathtoPublish: $(artifactsStagingDir)/manifests/${{ parameters.configuration }}/$(Agent.JobName).xml ArtifactName: VerticalManifests displayName: Publish Vertical Manifest diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index d3ed10991b41..563f6f1e8bf5 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -162,8 +162,11 @@ $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'toolset', 'VSSdkResolvers')) $([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols')) - $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'manifests')) + + $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'manifests', '$(Configuration)')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'manifests', '$(Configuration)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'assets', '$(Configuration)')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'extracted-dotnet-sdk')) $([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt')) @@ -197,7 +200,9 @@ - $(AssetManifestsDir)VerticalManifest.xml + + $(ArtifactsAssetManifestsDir)$([MSBuild]::ValueOrDefault('$(VerticalName)', 'VerticalManifest')).xml + $(ArtifactsDir)staging diff --git a/src/SourceBuild/content/Directory.Build.targets b/src/SourceBuild/content/Directory.Build.targets index 95bafee1f633..01a45daba0cd 100644 --- a/src/SourceBuild/content/Directory.Build.targets +++ b/src/SourceBuild/content/Directory.Build.targets @@ -31,4 +31,35 @@ + + + + + + + + true + <_BinPlaceDir>%(BinPlaceDir.Identity) + + + + <_BinPlaceFileWithFullTargetPath Include="@(BinPlaceFile)"> + %(BinPlaceFile.TargetPath) + $([MSBuild]::MakeRelative('$(ArtifactsDir)', '%(BinPlaceFile.Identity)')) + + + + + + diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj index 29379c1ddac7..60665c44eafe 100644 --- a/src/SourceBuild/content/build.proj +++ b/src/SourceBuild/content/build.proj @@ -1,4 +1,4 @@ - + @@ -26,7 +26,7 @@ - + source-build non-source-build diff --git a/src/SourceBuild/content/eng/extract-sdk-archive.proj b/src/SourceBuild/content/eng/extract-sdk-archive.proj index 695965324244..f58c685f1558 100644 --- a/src/SourceBuild/content/eng/extract-sdk-archive.proj +++ b/src/SourceBuild/content/eng/extract-sdk-archive.proj @@ -5,7 +5,7 @@ diff --git a/src/SourceBuild/content/eng/finish-source-only.proj b/src/SourceBuild/content/eng/finish-source-only.proj index c00cc0d68089..c4750bb1d7bf 100644 --- a/src/SourceBuild/content/eng/finish-source-only.proj +++ b/src/SourceBuild/content/eng/finish-source-only.proj @@ -16,14 +16,14 @@ $(ArtifactsAssetsDir)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension) @@ -48,7 +48,7 @@ @@ -57,13 +57,18 @@ + + + + + @@ -79,6 +84,11 @@ + + + + + @@ -97,7 +107,7 @@ @@ -146,7 +156,7 @@ + + + + + @@ -216,7 +231,7 @@ @@ -252,7 +267,27 @@ + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/content/eng/init-cross-build.proj b/src/SourceBuild/content/eng/init-cross-build.proj index f017ae51d4a3..e229ad92e38a 100644 --- a/src/SourceBuild/content/eng/init-cross-build.proj +++ b/src/SourceBuild/content/eng/init-cross-build.proj @@ -5,7 +5,7 @@ + BeforeTargets="Build"> ROOTFS_DIR=$(ArtifactsObjDir)crossrootfs/arm ROOTFS_DIR=$(ArtifactsObjDir)crossrootfs/armel diff --git a/src/SourceBuild/content/eng/init-poison.proj b/src/SourceBuild/content/eng/init-poison.proj index d1497ccd9dfc..e2ccf13c1427 100644 --- a/src/SourceBuild/content/eng/init-poison.proj +++ b/src/SourceBuild/content/eng/init-poison.proj @@ -10,7 +10,7 @@ diff --git a/src/SourceBuild/content/eng/join-verticals.proj b/src/SourceBuild/content/eng/join-verticals.proj index 561464a10822..586d5f8eada4 100644 --- a/src/SourceBuild/content/eng/join-verticals.proj +++ b/src/SourceBuild/content/eng/join-verticals.proj @@ -5,11 +5,13 @@ - + - + diff --git a/src/SourceBuild/content/eng/merge-asset-manifests.proj b/src/SourceBuild/content/eng/merge-asset-manifests.proj index ae49332985b3..2ad3cf576d2b 100644 --- a/src/SourceBuild/content/eng/merge-asset-manifests.proj +++ b/src/SourceBuild/content/eng/merge-asset-manifests.proj @@ -10,10 +10,9 @@ - + - + @@ -24,6 +23,11 @@ MergedAssetManifestOutputPath="$(MergedAssetManifestOutputPath)" VmrBuildNumber="$(BUILD_BUILDNUMBER)" VerticalName="$(VerticalName)" /> + + + + + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/GetKnownArtifactsFromAssetManifests.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/GetKnownArtifactsFromAssetManifests.cs new file mode 100644 index 000000000000..daafec99de08 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/GetKnownArtifactsFromAssetManifests.cs @@ -0,0 +1,92 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#nullable enable + +using System.Collections.Generic; +using System.Linq; +using System.Xml.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.UnifiedBuild.Tasks +{ + /// + /// Get a list of MSBuild Items that represent the packages described in the asset manifests. + /// + public sealed class GetKnownArtifactsFromAssetManifests : Build.Utilities.Task + { + // Common metadata + private const string IdAttributeName = "Id"; + private const string RepoOriginAttributeName = "RepoOrigin"; + private const string NonShippingAttributeName = "NonShipping"; + private const string DotNetReleaseShippingAttributeName = "DotNetReleaseShipping"; + + // Package metadata + private const string PackageElementName = "Package"; + private const string PackageVersionAttributeName = "Version"; + + // Blob metadata + private const string BlobElementName = "Blob"; + + /// + /// A list of asset manifests to read. + /// + [Required] + public required ITaskItem[] AssetManifests { get; set; } + + /// + /// If provided, only artifacts from that repository will be returned. + /// + public string? RepoOrigin { get; set; } + + /// + /// The list of known packages including their versions as metadata. + /// + [Output] + public ITaskItem[]? KnownPackages { get; set; } + + /// + /// The list of known blobs. + /// + [Output] + public ITaskItem[]? KnownBlobs { get; set; } + + public override bool Execute() + { + XDocument[] xDocuments = AssetManifests + .Select(manifest => XDocument.Load(manifest.ItemSpec)) + .ToArray(); + + KnownPackages = xDocuments + .SelectMany(doc => doc.Root!.Descendants(PackageElementName)) + .Where(ShouldIncludeElement) + .Distinct() + .Select(package => new TaskItem(package.Attribute(IdAttributeName)!.Value, new Dictionary + { + { PackageVersionAttributeName, package.Attribute(PackageVersionAttributeName)!.Value }, + { RepoOriginAttributeName, package.Attribute(RepoOriginAttributeName)?.Value ?? string.Empty }, + { NonShippingAttributeName, package.Attribute(NonShippingAttributeName)?.Value ?? string.Empty }, + { DotNetReleaseShippingAttributeName, package.Attribute(DotNetReleaseShippingAttributeName)?.Value ?? string.Empty } + })) + .ToArray(); + + KnownBlobs = xDocuments + .SelectMany(doc => doc.Root!.Descendants(BlobElementName)) + .Where(ShouldIncludeElement) + .Distinct() + .Select(blob => new TaskItem(blob.Attribute(IdAttributeName)!.Value, new Dictionary + { + { RepoOriginAttributeName, blob.Attribute(RepoOriginAttributeName)?.Value ?? string.Empty }, + { NonShippingAttributeName, blob.Attribute(NonShippingAttributeName)?.Value ?? string.Empty }, + { DotNetReleaseShippingAttributeName, blob.Attribute(DotNetReleaseShippingAttributeName)?.Value ?? string.Empty } + })) + .ToArray(); + + return true; + } + + private bool ShouldIncludeElement(XElement element) => RepoOrigin == null || element.Attribute(RepoOriginAttributeName)?.Value == RepoOrigin; + } +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/GetKnownPackagesFromAssetManifests.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/GetKnownPackagesFromAssetManifests.cs deleted file mode 100644 index 4e5300f7def3..000000000000 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/GetKnownPackagesFromAssetManifests.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Collections.Generic; -using System.Linq; -using System.Xml.Linq; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Microsoft.DotNet.UnifiedBuild.Tasks -{ - /// - /// Get a list of MSBuild Items that represent the packages described in the asset manifests. - /// - public sealed class GetKnownArtifactsFromAssetManifests : Build.Utilities.Task - { - [Required] - public ITaskItem[] AssetManifests { get; set; } - - [Output] - public ITaskItem[] KnownPackages { get; set; } - - [Output] - public ITaskItem[] KnownBlobs { get; set; } - - public override bool Execute() - { - XDocument[] xDocuments = AssetManifests - .Select(manifest => XDocument.Load(manifest.ItemSpec)) - .ToArray(); - - KnownPackages = xDocuments - .SelectMany(doc => doc.Root!.Descendants("Package")) - .Select(package => new TaskItem(package.Attribute("Id")!.Value, new Dictionary { { "Version", package.Attribute("Version")!.Value } })) - .ToArray(); - - KnownBlobs = xDocuments - .SelectMany(doc => doc.Root!.Descendants("Blob")) - .Select(asset => new TaskItem(asset.Attribute("Id")!.Value)) - .ToArray(); - - return true; - } - } -} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs index c2b87171ad2c..fa06f60c52b0 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs @@ -76,7 +76,10 @@ public override bool Execute() XDocument verticalManifest = new(new XElement(mergedManifestRoot.Name, mergedManifestRoot.Attributes(), packageElements, blobElements)); - File.WriteAllText(MergedAssetManifestOutputPath, verticalManifest.ToString()); + FileInfo outputFileInfo = new(MergedAssetManifestOutputPath); + outputFileInfo.Directory!.Create(); + File.WriteAllText(outputFileInfo.FullName, verticalManifest.ToString()); + Log.LogMessage(MessageImportance.High, $"Merged asset manifest written to {MergedAssetManifestOutputPath}"); return !Log.HasLoggedErrors; diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.props b/src/SourceBuild/content/repo-projects/Directory.Build.props index c21f5ad17a46..74763050b7b3 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.props +++ b/src/SourceBuild/content/repo-projects/Directory.Build.props @@ -38,7 +38,7 @@ $(BaseIntermediateOutputPath)$([System.IO.Path]::GetFileName('$(OriginalNuGetConfigFile)')) - $([MSBuild]::NormalizeDirectory('$(AssetManifestsDir)', '$(RepositoryName)')) + $([MSBuild]::NormalizeDirectory('$(AssetManifestsIntermediateDir)', '$(RepositoryName)')) $([MSBuild]::NormalizeDirectory('$(IntermediateSymbolsRootDir)', '$(RepositoryName)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'source-built-sdks')) diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.targets b/src/SourceBuild/content/repo-projects/Directory.Build.targets index 70baaa2f7c44..a835f97ba84e 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.targets +++ b/src/SourceBuild/content/repo-projects/Directory.Build.targets @@ -528,24 +528,50 @@ - + Returns="@(ProducedPackage);@(ProducedPackageFromDependentVertical)"> - - + + true + + + + - + + + + + NonShipping + Shipping + + + + + + + + + + + + + + + + + + @@ -650,7 +676,7 @@ - diff --git a/src/SourceBuild/content/repo-projects/sdk.proj b/src/SourceBuild/content/repo-projects/sdk.proj index 43b19d1b5456..426902cd6945 100644 --- a/src/SourceBuild/content/repo-projects/sdk.proj +++ b/src/SourceBuild/content/repo-projects/sdk.proj @@ -86,15 +86,4 @@ - - - - -