Merged PR 732516: Make sure enumerations are reported back when running front end related processes under a sandbox

Running graph construction tools under the frontend was missing directory enumerations, which was a reason for having underbuilds due to a wrongly cached build graph.
Make sure we report these accesses and consume them appropriately.

Author: smera

Public/Src/Engine/ProcessPipExecutor/PipEnvironment.cs

@@ -174,5 +174,10 @@ public static void ReportDuplicateVariable(LoggingContext loggingContext, string
                 existingValue,
                 ignoredValue);
         }
+
+        /// <summary>
+        /// The OS-dependent environment variables that should be always present in a process environment.
+        /// </summary>
+        public IBuildParameters GetBaseEnvironmentVariables() => m_baseEnvironmentVariables;
     }
 }

Public/Src/FrontEnd/CMake/CMakeWorkspaceResolver.cs

@@ -185,7 +185,7 @@ private async Task<Possible<Unit>> GenerateBuildDirectoryAsync()
                 return new CMakeGenerationError(m_resolverSettings.ModuleName, m_buildDirectory.ToString(m_context.PathTable));
             }
 
-            FrontEndUtilities.TrackToolFileAccesses(m_host.Engine, m_context, Name, result.AllUnexpectedFileAccesses, outputDirectory);
+            FrontEndUtilities.TrackToolFileAccesses(m_host.Engine, m_context, Name, result.AllUnexpectedFileAccesses.Union(result.ExplicitlyReportedFileAccesses), outputDirectory);
             return Possible.Create(Unit.Void);
         }
 

Public/Src/FrontEnd/JavaScript/ToolBasedJavaScriptWorkspaceResolver.cs

@@ -202,7 +202,7 @@ protected virtual string GetProjectNameForGroup(IReadOnlyCollection<JavaScriptPr
                     standardError);
             }
 
-            TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses, outputFile.GetParent(Context.PathTable));
+            TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses.Union(result.ExplicitlyReportedFileAccesses), outputFile.GetParent(Context.PathTable));
 
             JsonSerializer serializer = ConstructProjectGraphSerializer(JsonSerializerSettings);
 

Public/Src/FrontEnd/MsBuild/MsBuildWorkspaceResolver.cs

@@ -348,7 +348,7 @@ private async Task<Possible<ProjectGraphWithPredictionsResult<AbsolutePath>>> Co
                     standardError);
             }
 
-            TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses, outputFile.GetParent(Context.PathTable));
+            TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses.Union(result.ExplicitlyReportedFileAccesses), outputFile.GetParent(Context.PathTable));
             JsonSerializer serializer = ConstructProjectGraphSerializer(ProjectGraphSerializationSettings.Settings);
 
             using (var sr = new StreamReader(outputFile.ToString(Context.PathTable)))

Public/Src/FrontEnd/Ninja/NinjaWorkspaceResolver.cs

@@ -19,6 +19,7 @@
 using Newtonsoft.Json;
 using TypeScript.Net.Types;
 using static BuildXL.Utilities.Core.FormattableStringEx;
+using System.Linq;
 
 namespace BuildXL.FrontEnd.Ninja
 {
@@ -168,7 +169,7 @@ private async Task<Possible<NinjaGraphResult>> ComputeBuildGraphAsync()
                     standardError);
             }
 
-            TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses, outputFile.GetParent(Context.PathTable));
+            TrackFilesAndEnvironment(result.AllUnexpectedFileAccesses.Union(result.ExplicitlyReportedFileAccesses), outputFile.GetParent(Context.PathTable));
             var serializer = JsonSerializer.Create(GraphSerializationSettings.Settings);
 
             // Add custom deserializer for converting string arrays to AbsolutePath ReadOnlySets

Public/Src/FrontEnd/UnitTests/Utilities/FrontEndUtilitiesTest.cs

@@ -0,0 +1,88 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.IO;
+using System.Linq;
+using BuildXL.FrontEnd.Sdk;
+using BuildXL.FrontEnd.Utilities;
+using BuildXL.Processes;
+using BuildXL.ProcessPipExecutor;
+using BuildXL.Utilities.Configuration;
+using BuildXL.Utilities.Configuration.Mutable;
+using BuildXL.Utilities.Core;
+using Test.BuildXL.Processes;
+using Test.BuildXL.TestUtilities.Xunit;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Test.BuildXL.FrontEnd.Utilities
+{
+    public class FrontEndUtilitiesTest : TemporaryStorageTestBase
+    {
+        protected BuildXLContext Context { get; } 
+        protected FrontEndContext FrontEndContext { get; }
+        protected IConfiguration Configuration { get; }
+        protected PipEnvironment PipEnvironment { get; }
+
+        public FrontEndUtilitiesTest(ITestOutputHelper output) : base(output)
+        {
+            Configuration = new ConfigurationImpl();
+            Context = BuildXLContext.CreateInstanceForTesting();
+            FrontEndContext = FrontEndContext.CreateInstanceForTesting(
+                pathTable: Context.PathTable, symbolTable: Context.SymbolTable, qualifierTable: Context.QualifierTable, frontEndConfig: Configuration.FrontEnd, loggingContext: LoggingContext);
+            PipEnvironment = new PipEnvironment(LoggingContext);
+        }
+
+        [Fact]
+        public void RunSandboxedToolHasAllExpectedAccesses()
+        {
+            var input = GetFullPath("input");
+            var inputPath = AbsolutePath.Create(Context.PathTable, input);
+            File.WriteAllText(input, "some input");
+
+            var output = GetFullPath("output");
+            var outputPath = AbsolutePath.Create(Context.PathTable, output);
+
+            var temporaryDirectoryPath = AbsolutePath.Create(Context.PathTable, TemporaryDirectory);
+            var nonExistentPath = AbsolutePath.Create(Context.PathTable, GetFullPath("non-existent"));
+
+            string toolArguments;
+
+            // Let's test a read, an enumeration, a write and an absent probe
+            if (OperatingSystemHelper.IsWindowsOS)
+            {
+                toolArguments = "/C \"type input & dir & copy input output & if exist non-existent echo hi\"";
+            }
+            else
+            {
+                toolArguments = "-c \"cat input; ls; cp input output; if [ -e non-existent ]; then echo hi; fi\"";
+            }
+
+            var toolDirectory = AbsolutePath.Create(Context.PathTable, CmdHelper.OsShellExe).GetParent(Context.PathTable);
+
+            var result = FrontEndUtilities.RunSandboxedToolAsync(
+               FrontEndContext,
+               CmdHelper.OsShellExe,
+               buildStorageDirectory: TemporaryDirectory,
+               fileAccessManifest: FrontEndUtilities.GenerateToolFileAccessManifest(FrontEndContext, toolDirectory),
+               arguments: toolArguments,
+               workingDirectory: TemporaryDirectory,
+               description: $"Test sandboxed tool",
+               PipEnvironment.GetBaseEnvironmentVariables()).GetAwaiter().GetResult();
+
+            XAssert.AreEqual(0, result.ExitCode);
+            var allAccesses = result.AllUnexpectedFileAccesses.Union(result.ExplicitlyReportedFileAccesses);
+
+            // type input
+            XAssert.IsTrue(allAccesses.Any(a => a.RequestedAccess == RequestedAccess.Read && GetAbsolutePath(a) == inputPath));
+            // dir
+            XAssert.IsTrue(allAccesses.Any(a => a.RequestedAccess == RequestedAccess.Enumerate && GetAbsolutePath(a) == temporaryDirectoryPath));
+            // copy input output
+            XAssert.IsTrue(allAccesses.Any(a => a.RequestedAccess == RequestedAccess.Write && GetAbsolutePath(a) == outputPath));
+            // if exist non-existent echo hi
+            XAssert.IsTrue(allAccesses.Any(a => a.RequestedAccess == RequestedAccess.Probe && GetAbsolutePath(a) == nonExistentPath));
+        }
+
+        private AbsolutePath GetAbsolutePath(ReportedFileAccess access) => AbsolutePath.Create(Context.PathTable, access.GetPath(Context.PathTable));
+    }
+}

Public/Src/FrontEnd/UnitTests/Utilities/Test.BuildXL.FrontEnd.Utilities.dsc

@@ -0,0 +1,45 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import * as Managed from "Sdk.Managed";
+import * as MSBuild from "Sdk.Selfhost.MSBuild";
+import * as Frameworks from "Sdk.Managed.Frameworks";
+import {Node} from "Sdk.NodeJs";
+import {Transformer} from "Sdk.Transformers";
+
+namespace Test.Utilities {
+    
+    @@public
+    export const dll = BuildXLSdk.test({
+        runTestArgs: {
+            unsafeTestRunArguments: {
+                // These tests require Detours to run itself, so we won't detour the test runner process itself
+                runWithUntrackedDependencies: true
+            },
+        },
+        assemblyName: "Test.BuildXL.FrontEnd.Utilities",
+        sources: globR(d`.`, "*.cs"), 
+        references: [
+            Script.dll,
+            Core.dll,
+            importFrom("BuildXL.Core.UnitTests").EngineTestUtilities.dll,
+            importFrom("BuildXL.Engine").Engine.dll,
+            importFrom("BuildXL.Engine").Processes.dll,
+            importFrom("BuildXL.Engine").Scheduler.dll,
+            importFrom("BuildXL.Engine").ProcessPipExecutor.dll,
+            importFrom("BuildXL.FrontEnd").Core.dll,
+            importFrom("BuildXL.FrontEnd").Script.dll,
+            importFrom("BuildXL.FrontEnd").Sdk.dll,
+            importFrom("BuildXL.FrontEnd").JavaScript.dll,
+            importFrom("BuildXL.FrontEnd").TypeScript.Net.dll,
+            importFrom("BuildXL.FrontEnd").Utilities.dll,
+            importFrom("BuildXL.FrontEnd").SdkProjectGraph.dll,
+            importFrom("BuildXL.Pips").dll,
+            importFrom("BuildXL.Utilities").dll,
+            importFrom("BuildXL.Utilities").Configuration.dll,
+            importFrom("BuildXL.Utilities").Native.dll,
+            importFrom("BuildXL.Utilities").Script.Constants.dll,
+            importFrom("BuildXL.Utilities").Utilities.Core.dll,
+        ],
+    });
+}

Public/Src/FrontEnd/Utilities/FrontEndUtilities.cs

@@ -246,6 +246,12 @@ public static IDictionary<string, string> GetEngineEnvironment(FrontEndEngineAbs
         /// <summary>
         /// Generate a basic file access manifest for front end tools
         /// </summary>
+        /// <remarks>
+        /// The generated manifest will instruct the sandbox to not fail on unexpected file accesses, but to deny all of them.
+        /// Therefore, if the goal is collecting all file accesses, check <see cref="SandboxedProcessResult.AllUnexpectedFileAccesses"/> but
+        /// also check <see cref="SandboxedProcessResult.ExplicitlyReportedFileAccesses"/>, since accesses like enumerations
+        /// are never denied.
+        /// </remarks>
         public static FileAccessManifest GenerateToolFileAccessManifest(FrontEndContext context, AbsolutePath toolDirectory)
         {
             var pathTable = context.PathTable;
@@ -255,8 +261,6 @@ public static FileAccessManifest GenerateToolFileAccessManifest(FrontEndContext
             var fileAccessManifest = new FileAccessManifest(pathTable)
                                      {
                                          FailUnexpectedFileAccesses = false,
-                                         // The manifest is configured such that all relevant accesses are returned as unexpected ones, so we
-                                         // don't actually need to return all accesses
                                          ReportFileAccesses = false,
                                          MonitorNtCreateFile = true,
                                          MonitorZwCreateOpenQueryFile = true,
@@ -285,14 +289,17 @@ public static FileAccessManifest GenerateToolFileAccessManifest(FrontEndContext
 
             fileAccessManifest.AddScope(toolDirectory, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowReadAlways);
 
+            // Make sure the root node is configured so all accesses are reported
+            fileAccessManifest.AddScope(AbsolutePath.Invalid, FileAccessPolicy.MaskNothing, FileAccessPolicy.ReportAccess);
+
             return fileAccessManifest;
         }
 
         /// <summary>
         /// The FrontEnds that use an out-of-proc tool should sandbox that process and call this method
         /// in order to tack the tool's file accesses, enumerations, etc. in order to make graph caching sound
         /// </summary>
-        public static void TrackToolFileAccesses(FrontEndEngineAbstraction engine, FrontEndContext context, string frontEndName, ISet<ReportedFileAccess> fileAccesses, AbsolutePath frontEndFolder)
+        public static void TrackToolFileAccesses(FrontEndEngineAbstraction engine, FrontEndContext context, string frontEndName, IEnumerable<ReportedFileAccess> fileAccesses, AbsolutePath frontEndFolder)
         {
             // Compute all parseable paths
             // TODO: does it make sense to consider enumerations, or as a result the graph will be too unstable? Does it matter for MsBuild graph construction?

Public/Src/FrontEnd/Utilities/GenericProjectGraphResolver/ProjectGraphWorkspaceResolverBase.cs

@@ -276,7 +276,7 @@ protected BuildParameters.IBuildParameters RetrieveBuildParameters()
         /// <summary>
         /// Talks to the engine to register all accesses and build parameters
         /// </summary>
-        protected void TrackFilesAndEnvironment(ISet<ReportedFileAccess> fileAccesses, AbsolutePath frontEndFolder)
+        protected void TrackFilesAndEnvironment(IEnumerable<ReportedFileAccess> fileAccesses, AbsolutePath frontEndFolder)
         {
             // Register all build parameters passed to the graph construction process if they were retrieved from the process environment
             // Otherwise, if build parameters were defined by the main config file, then there is nothing to register: if the definition