Fix BuildPass>1 artifacts and asset manifest content (#45323)

eng/pipelines/templates/jobs/vmr-build.yml

@@ -13,6 +13,10 @@ parameters:
   type: string
   default: ''
 
+- name: configuration
+  type: string
+  default: 'Release'
+
 - name: container
   type: string
   default: ''
@@ -155,6 +159,11 @@ jobs:
     - name: sourcesPath
       value: $(vmrPath)
 
+  # Must be a path under $(sourcesPath). Inside the docker container, we mount $(sourcesPath) to /vmr
+  # and can't write outside of that folder.
+  - name: artifactsStagingDir
+    value: $(sourcesPath)/artifacts/staging
+
   templateContext:
     outputParentDirectory: $(Build.ArtifactStagingDirectory)
     outputs:
@@ -166,14 +175,14 @@ jobs:
       sbomEnabled: false
 
     - output: pipelineArtifact
-      path: $(Build.ArtifactStagingDirectory)/publishing
+      path: $(artifactsStagingDir)
       artifact: $(Agent.JobName)_Artifacts
       displayName: Publish Artifacts
       sbomEnabled: true
 
     # Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs
     - output: buildArtifacts
-      PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml
+      PathtoPublish: $(artifactsStagingDir)/manifests/${{ parameters.configuration }}/$(Agent.JobName).xml
       ArtifactName: VerticalManifests
       displayName: Publish Vertical Manifest
       sbomEnabled: false
@@ -276,13 +285,13 @@ jobs:
         if /I '${{ parameters.useDevVersions }}'=='True' set extraBuildArguments=%extraBuildArguments% -dev
         set extraBuildProperties=
         if not [${{ parameters.buildPass }}]==[] set extraBuildProperties=%extraBuildProperties% /p:DotNetBuildPass=${{ parameters.buildPass }}
-        call build.cmd -ci -cleanWhileBuilding -prepareMachine %extraBuildArguments% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) %extraBuildProperties% ${{ parameters.extraProperties }}
+        call build.cmd -ci -cleanWhileBuilding -prepareMachine %extraBuildArguments% -c ${{ parameters.configuration }} /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) %extraBuildProperties% ${{ parameters.extraProperties }}
       displayName: Build
       workingDirectory: ${{ variables.sourcesPath }}
 
     - ${{ if eq(parameters.runTests, 'True') }}:
       - script: |
-          call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) ${{ parameters.extraProperties }}
+          call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog -c ${{ parameters.configuration }} /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) ${{ parameters.extraProperties }}
         displayName: Run Tests
         workingDirectory: ${{ variables.sourcesPath }}
         timeoutInMinutes: ${{ variables.runTestsTimeout }}
@@ -330,7 +339,8 @@ jobs:
         df -h
 
         customEnvVars=""
-        customBuildArgs="--ci --clean-while-building --prepareMachine"
+        customBuildArgs="--ci --clean-while-building --prepareMachine -c ${{ parameters.configuration }}"
+
         if [[ '${{ parameters.runOnline }}' == 'True' ]]; then
           customBuildArgs="$customBuildArgs --online"
         fi
@@ -473,10 +483,10 @@ jobs:
 
           # Only use Docker when a container is specified
           if [[ -n "${{ parameters.container }}" ]]; then
-            docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog $customBuildArgs $extraBuildProperties $(additionalBuildArgs)
+            docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog -c ${{ parameters.configuration }} $customBuildArgs $extraBuildProperties $(additionalBuildArgs)
           else
             cd $(sourcesPath)
-            ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog $customBuildArgs $extraBuildProperties $(additionalBuildArgs)
+            ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog -c ${{ parameters.configuration }} $customBuildArgs $extraBuildProperties $(additionalBuildArgs)
           fi
 
         displayName: Run Tests
@@ -588,45 +598,15 @@ jobs:
         publishRunAttachments: true
         testRunTitle: ScenarioTests_$(Agent.JobName)
 
-  - task: CopyFiles@2
-    inputs:
-      SourceFolder: $(sourcesPath)/artifacts
-      Contents: |
-        assets/**
-        manifests/**
-      TargetFolder: $(Build.ArtifactStagingDirectory)/publishing
-    displayName: Copy artifacts to Artifact Staging Directory
-
-  - ${{ if eq(parameters.targetOS, 'windows') }}:
-    - powershell: |
-        $sourcePath = "$(sourcesPath)/artifacts/manifests/VerticalManifest.xml"
-        $targetPath = "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml"
-        New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)/manifests" -Force | Out-Null
-        Copy-Item $sourcePath -Destination $targetPath -Force
-      displayName: Copy vertical manifest to Artifact Staging Directory
-  - ${{ else }}:
-    - script: |
-        mkdir -p "$(Build.ArtifactStagingDirectory)/manifests"
-        cp "$(sourcesPath)/artifacts/manifests/VerticalManifest.xml" "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml"
-      displayName: Copy vertical manifest to Artifact Staging Directory
-
-  # When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages
-  - ${{ if ne(parameters.buildSourceOnly, 'true') }}:
-    - task: CopyFiles@2
-      inputs:
-        SourceFolder: $(sourcesPath)/artifacts/packages
-        TargetFolder: $(Build.ArtifactStagingDirectory)/publishing/packages
-      displayName: Copy packages to Artifact Staging Directory
-
   - ${{ if or(ne(variables['System.TeamProject'], 'internal'), eq(variables['Build.Reason'], 'PullRequest')) }}:
-    - publish: $(Build.ArtifactStagingDirectory)/publishing
+    - publish: $(artifactsStagingDir)
       artifact: $(Agent.JobName)_Artifacts
       displayName: Publish Artifacts
       continueOnError: true
 
     # Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs
     - task: PublishBuildArtifacts@1
       inputs:
-        PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml
+        PathtoPublish: $(artifactsStagingDir)/manifests/${{ parameters.configuration }}/$(Agent.JobName).xml
         ArtifactName: VerticalManifests
       displayName: Publish Vertical Manifest

src/SourceBuild/content/Directory.Build.props

@@ -162,8 +162,11 @@
 
     <VSMSBuildSdkResolversDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'toolset', 'VSSdkResolvers'))</VSMSBuildSdkResolversDir>
     <IntermediateSymbolsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols'))</IntermediateSymbolsRootDir>
-    <AssetManifestsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'manifests'))</AssetManifestsDir>
+
+    <ArtifactsAssetManifestsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'manifests', '$(Configuration)'))</ArtifactsAssetManifestsDir>
+    <AssetManifestsIntermediateDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'manifests', '$(Configuration)'))</AssetManifestsIntermediateDir>
     <ArtifactsAssetsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'assets', '$(Configuration)'))</ArtifactsAssetsDir>
+
     <DotNetSdkExtractDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'extracted-dotnet-sdk'))</DotNetSdkExtractDir>
 
     <PrebuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt'))</PrebuiltPackagesPath>
@@ -197,7 +200,9 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <MergedAssetManifestOutputPath>$(AssetManifestsDir)VerticalManifest.xml</MergedAssetManifestOutputPath>
+    <!-- Respect the VerticalName property which is set to a unique identifier in CI. Otherwise default to 'VerticalManifest'. -->
+    <MergedAssetManifestOutputPath>$(ArtifactsAssetManifestsDir)$([MSBuild]::ValueOrDefault('$(VerticalName)', 'VerticalManifest')).xml</MergedAssetManifestOutputPath>
+    <ArtifactsStagingDir Condition="'$(ContinuousIntegrationBuild)' == 'true'">$(ArtifactsDir)staging</ArtifactsStagingDir>
   </PropertyGroup>
 
 </Project>

src/SourceBuild/content/Directory.Build.targets

@@ -31,4 +31,35 @@
     </PropertyGroup>
   </Target>
 
+  <ItemGroup>
+    <!-- Binplace all assets and packages to ArtifactsStagingDir. -->
+    <BinPlaceDir Include="$(ArtifactsStagingDir)" Condition="'$(ArtifactsStagingDir)' != ''" />
+  </ItemGroup>
+
+  <Target Name="BinPlaceFiles"
+          AfterTargets="Build"
+          Condition="'@(BinPlaceDir)' != '' and '@(BinPlaceFile)' != ''"
+          Inputs="@(BinPlaceDir);%(BinPlaceDir.Identity)"
+          Outputs="unused">
+    <PropertyGroup>
+      <BinPlaceUseHardlinksIfPossible Condition="'$(BinPlaceUseHardlinksIfPossible)' == ''">true</BinPlaceUseHardlinksIfPossible>
+      <_BinPlaceDir>%(BinPlaceDir.Identity)</_BinPlaceDir>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <_BinPlaceFileWithFullTargetPath Include="@(BinPlaceFile)">
+        <TargetPath Condition="'%(BinPlaceFile.TargetPath)' != ''">%(BinPlaceFile.TargetPath)</TargetPath>
+        <TargetPath Condition="'%(BinPlaceFile.TargetPath)' == ''">$([MSBuild]::MakeRelative('$(ArtifactsDir)', '%(BinPlaceFile.Identity)'))</TargetPath>
+      </_BinPlaceFileWithFullTargetPath>
+    </ItemGroup>
+
+    <Copy SourceFiles="@(_BinPlaceFileWithFullTargetPath)"
+          DestinationFiles="@(_BinPlaceFileWithFullTargetPath -> '$([MSBuild]::EnsureTrailingSlash('$(_BinPlaceDir)'))%(TargetPath)')"
+          SkipUnchangedFiles="true"
+          OverwriteReadOnlyFiles="true"
+          Retries="$(CopyRetryCount)"
+          RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
+          UseHardlinksIfPossible="$(BinPlaceUseHardlinksIfPossible)" />
+  </Target>
+
 </Project>

src/SourceBuild/content/build.proj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.Build.Traversal" InitialTargets="PrintInfo">
+<Project Sdk="Microsoft.Build.Traversal">
 
 <!-- Default targets and parallelization -->
  <ItemDefinitionGroup>
@@ -26,7 +26,7 @@
     <ProjectReference Include="$(RepositoryEngineeringDir)finish-source-only.proj" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
   </ItemGroup>
 
-  <Target Name="PrintInfo">
+  <Target Name="PrintInfo" BeforeTargets="Build">
     <PropertyGroup>
       <BuildModeInfoText Condition="'$(DotNetBuildSourceOnly)' == 'true'">source-build</BuildModeInfoText>
       <BuildModeInfoText Condition="'$(DotNetBuildSourceOnly)' != 'true'">non-source-build</BuildModeInfoText>

src/SourceBuild/content/eng/extract-sdk-archive.proj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <Target Name="ExtractSdkArchive"
-          BeforeTargets="AfterBuild"
+          BeforeTargets="Build"
           DependsOnTargets="DetermineSourceBuiltSdkVersion"
           Inputs="$(SdkTarballPath)"
           Outputs="$(BaseIntermediateOutputPath)ExtractSdkArchive.complete">

src/SourceBuild/content/eng/finish-source-only.proj

@@ -16,14 +16,14 @@
 
   <!-- After building, generate a prebuilt usage report. -->
   <Target Name="ReportPrebuiltUsage"
-          AfterTargets="Build"
+          BeforeTargets="Build"
           Condition="'$(SkipReportPrebuiltUsage)' != 'true'">
     <MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
   </Target>
 
   <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.WriteSBRPUsageReport" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="ReportSbrpUsage"
-          AfterTargets="Build"
+          BeforeTargets="Build"
           Condition="'$(ReportSbrpUsage)' == 'true'">
     <WriteSbrpUsageReport SbrpRepoSrcPath="$(SbrpRepoSrcDir)"
                           SrcPath="$(SrcDir)"
@@ -35,7 +35,7 @@
     to be used as inputs and outputs of symbols repackaging targets.
   -->
   <Target Name="DetermineSymbolsTargetsInputsAndOutputs"
-          AfterTargets="Build"
+          BeforeTargets="Build"
           DependsOnTargets="DetermineSourceBuiltSdkVersion">
     <PropertyGroup>
       <UnifiedSymbolsTarball>$(ArtifactsAssetsDir)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</UnifiedSymbolsTarball>
@@ -48,7 +48,7 @@
 
   <!-- After building, repackage symbols into a single tarball. -->
   <Target Name="RepackageSymbols"
-          AfterTargets="Build"
+          BeforeTargets="Build"
           DependsOnTargets="DetermineSymbolsTargetsInputsAndOutputs"
           Inputs="@(IntermediateSymbol)"
           Outputs="$(UnifiedSymbolsTarball)">
@@ -57,13 +57,18 @@
     <Exec Command="tar --numeric-owner -czf $(UnifiedSymbolsTarball) *"
           WorkingDirectory="$(IntermediateSymbolsRootDir)" />
 
+    <!-- Mark the artifact so that it gets binplaced. -->
+    <ItemGroup>
+      <BinPlaceFile Include="$(UnifiedSymbolsTarball)" />
+    </ItemGroup>
+
     <Message Importance="High" Text="Packaged all symbols in '$(UnifiedSymbolsTarball)'" />
   </Target>
 
   <!-- After building, create the sdk symbols tarball. -->
   <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.CreateSdkSymbolsLayout" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="CreateSdkSymbolsTarball"
-          AfterTargets="Build"
+          BeforeTargets="Build"
           DependsOnTargets="RepackageSymbols"
           Inputs="@(IntermediateSymbol);$(SdkTarballPath)"
           Outputs="$(SdkSymbolsTarball)">
@@ -79,6 +84,11 @@
     <Exec Command="tar --numeric-owner -czf $(SdkSymbolsTarball) *"
           WorkingDirectory="$(IntermediateSdkSymbolsLayout)" />
 
+    <!-- Mark the artifact so that it gets binplaced. -->
+    <ItemGroup>
+      <BinPlaceFile Include="$(SdkSymbolsTarball)" />
+    </ItemGroup>
+
     <Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />
 
     <RemoveDir Directories="$(IntermediateSdkSymbolsLayout)" />
@@ -97,7 +107,7 @@
 
   <UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.CheckForPoison" AssemblyFile="$(MicrosoftDotNetSourceBuildTasksLeakDetectionAssembly)" TaskFactory="TaskHostFactory" Condition="'$(EnablePoison)' == 'true'" />
   <Target Name="ReportPoisonUsage"
-          AfterTargets="Build"
+          BeforeTargets="Build"
           Condition="'$(EnablePoison)' == 'true'"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)ReportPoisonUsage.complete" >
@@ -146,7 +156,7 @@
   </Target>
 
   <Target Name="CreatePrebuiltsTarball"
-          AfterTargets="Build"
+          BeforeTargets="Build"
           DependsOnTargets="
             CheckIfPrebuiltsExistToPack;
             CreatePrebuiltsTarballIfPrebuiltsExist;
@@ -172,6 +182,11 @@
     <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(PrebuiltsTarball)'))" />
     <Exec Command="tar --numeric-owner -zcf $(PrebuiltsTarball) -C $(PrebuiltsTarballWorkingDir) ." />
 
+    <!-- Mark the artifact so that it gets binplaced. -->
+    <ItemGroup>
+      <BinPlaceFile Include="$(PrebuiltsTarball)" />
+    </ItemGroup>
+
     <Message Text="Tarball '$(PrebuiltsTarball)' was successfully created from '$(PrebuiltsTarballWorkingDir)'" Importance="High" />
   </Target>
 
@@ -216,7 +231,7 @@
   <!-- Create the SourceBuilt.Private.Artifacts archive when building source-only. -->
   <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.WritePackageVersionsProps" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="CreatePrivateSourceBuiltArtifactsArchive"
-          AfterTargets="Build"
+          BeforeTargets="Build"
           DependsOnTargets="GetInputsOutputForCreatePrivateSourceBuiltArtifactsArchive;GetProducedPackages"
           Inputs="@(ArtifactsPackageToBundle);@(ReferencePackageToBundle);@(MergedAssetManifest)"
           Outputs="$(SourceBuiltTarballName);$(SourceBuiltVersionName);$(AllPackageVersionsPropsName);$(SourceBuiltMergedAssetManifestName)">
@@ -252,7 +267,27 @@
     <Exec Command="tar --numeric-owner -czhf $(SourceBuiltTarballName) $([System.IO.Path]::GetFileName('$(SourceBuiltVersionName)')) *"
           WorkingDirectory="$(SourceBuiltLayoutDir)" />
 
+    <!-- Mark the artifact so that it gets binplaced. -->
+    <ItemGroup>
+      <BinPlaceFile Include="$(SourceBuiltTarballName)" />
+    </ItemGroup>
+
     <Message Importance="High" Text="Packaged source-built artifacts to $(SourceBuiltTarballName)" />
   </Target>
 
+    <!-- Copy SDK archive to assets root as source-only build partners expect the file to be there. -->
+  <Target Name="CopySdkArchive"
+          BeforeTargets="Build"
+          DependsOnTargets="DetermineSourceBuiltSdkVersion">
+    <Copy SourceFiles="$(SdkTarballPath)"
+          DestinationFolder="$(ArtifactsAssetsDir)"
+          SkipUnchangedFiles="true"
+          UseHardlinksIfPossible="true" />
+
+    <!-- Mark the artifact so that it gets binplaced. -->
+    <ItemGroup>
+      <BinPlaceFile Include="$(ArtifactsAssetsDir)$([System.IO.Path]::GetFileName('$(SdkTarballPath)'))" />
+    </ItemGroup>
+  </Target>
+
 </Project>

src/SourceBuild/content/eng/init-cross-build.proj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <Target Name="GenerateRootFs"
-          AfterTargets="Build">
+          BeforeTargets="Build">
     <PropertyGroup>
       <ArmEnvironmentVariables Condition="'$(ArmEnvironmentVariables)' == ''">ROOTFS_DIR=$(ArtifactsObjDir)crossrootfs/arm</ArmEnvironmentVariables>
       <ArmEnvironmentVariables Condition="'$(Platform)' == 'armel'">ROOTFS_DIR=$(ArtifactsObjDir)crossrootfs/armel</ArmEnvironmentVariables>

src/SourceBuild/content/eng/init-poison.proj

@@ -10,7 +10,7 @@
 
   <UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.MarkAndCatalogPackages" AssemblyFile="$(MicrosoftDotNetSourceBuildTasksLeakDetectionAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="PoisonPrebuiltPackages"
-          AfterTargets="Build"
+          BeforeTargets="Build"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)PoisonPrebuiltPackages.complete">
     <PropertyGroup>

src/SourceBuild/content/eng/join-verticals.proj

@@ -5,11 +5,13 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <ProjectReference Include="$(TasksDir)Microsoft.DotNet.UnifiedBuild.Tasks\Microsoft.DotNet.UnifiedBuild.Tasks.csproj" BuildInParallel="true" />
+      <ProjectReference Include="$(TasksDir)Microsoft.DotNet.UnifiedBuild.Tasks\Microsoft.DotNet.UnifiedBuild.Tasks.csproj" />
     </ItemGroup>
 
     <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.JoinVerticals" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
-    <Target Name="JoinVerticals" DependsOnTargets="ResolveProjectReferences" AfterTargets="Build">
+    <Target Name="JoinVerticals"
+            DependsOnTargets="ResolveProjectReferences"
+            BeforeTargets="Build">
       <Error Condition="'$(MainVertical)' == ''" Text="MainVertical is not set." />
       <Error Condition="'$(VerticalManifestsPath)' == ''" Text="VerticalManifestsPath is not set." />
       <Error Condition="'$(BuildId)' == ''" Text="BuildId is not set." />