Merged PR 741487: Make execute permission configurable and disable it for BXL internal repo

Introduced a flag to  enable/disable explicit setting of the execute permissions bit for the root process in linux builds.
Added a condition to explicitly set this bit for node and dotnet in the extractor.
This is done to obtain more information about the linux permissions bug.

Related work items: #2104538

Author: schandr78

.azdo/linux/job-ptrace.yml

@@ -96,7 +96,7 @@ jobs:
   # This step currently only builds selfhost with the --minimal flag, but will be extended in the future to run more unit tests with ptrace
   - bash: |
       set -eu
-      bash bxl.sh /ado /cacheMiss:"[Bxl.Selfhost.Linux.PTrace]" /logObservedFileAccesses /cacheConfigFilePath:Out/CacheConfig.json /logoutput:FullOutputOnError /logsToRetain:10 /exp:lazysodeletion- /logsDirectory:"Out/Logs/Build" --minimal --internal --use-dev "/p:BUILDXL_FINGERPRINT_SALT='Bxl.Selfhost.Linux.PTrace.CasingPR'" /forceEnableLinuxPTraceSandbox+ /injectCacheMisses:0.3
+      bash bxl.sh /ado /cacheMiss:"[Bxl.Selfhost.Linux.PTrace]" /logObservedFileAccesses /cacheConfigFilePath:Out/CacheConfig.json /logoutput:FullOutputOnError /logsToRetain:10 /exp:lazysodeletion- /logsDirectory:"Out/Logs/Build" --minimal --internal --use-dev /forceEnableLinuxPTraceSandbox+ /injectCacheMisses:0.3
     displayName: Build BXL with LKG and PTrace
     workingDirectory: /home/subst
     env:

.azdo/linux/job-selfhost.yml

@@ -149,7 +149,7 @@ jobs:
       # - the disks on Azure Pipeline VMs are too small to build everything, so let's instead run tests
       # - we also disable early worker release to avoid releasing a worker before attachment, which tends to happen
       #   when the build is highly cached: the intention is to have as much of a distributed build as possible for validation purposes
-      ./bxl.sh --use-dev --use-adobuildrunner ${{ parameters.BxlCommonArgs }} /logsDirectory:"Out/Logs/${{ parameters.validationName }}" ${{ parameters.bxlExtraArgs }} "/f:tag='test' /p:BUILDXL_FINGERPRINT_SALT='PrPipelineSalt_20230825'" /earlyWorkerRelease- /p:BuildXLWorkerAttachTimeoutMin=10 /logToKusto /logToKustoBlobUri:https://adomessages.blob.core.windows.net/adomessages /logToKustoIdentityId:6e0959cf-a9ba-4988-bbf1-7facd9deda51 /logToKustoTenantId:975f013f-7f24-47e8-a7d3-abc4752bf346 /historicMetadataCache-
+      ./bxl.sh --use-dev --use-adobuildrunner ${{ parameters.BxlCommonArgs }} /logsDirectory:"Out/Logs/${{ parameters.validationName }}" ${{ parameters.bxlExtraArgs }} "/f:tag='test'" /earlyWorkerRelease- /p:BuildXLWorkerAttachTimeoutMin=10 /logToKusto /logToKustoBlobUri:https://adomessages.blob.core.windows.net/adomessages /logToKustoIdentityId:6e0959cf-a9ba-4988-bbf1-7facd9deda51 /logToKustoTenantId:975f013f-7f24-47e8-a7d3-abc4752bf346 /historicMetadataCache-
     displayName: Test (${{ parameters.validationName }})
     workingDirectory: /home/subst
     env:

Public/Src/App/Bxl/Args.cs

@@ -381,6 +381,9 @@ public bool TryParse(string[] args, PathTable pathTable, out ICommandLineConfigu
                         OptionHandlerFactory.CreateBoolOption(
                             "disableIsObsoleteCheckDuringConversion",
                             sign => frontEndConfiguration.DisableIsObsoleteCheckDuringConversion = sign),
+                        OptionHandlerFactory.CreateBoolOption(
+                            "forceAddExecutionPermission",
+                            sign => sandboxConfiguration.ForceAddExecutionPermission = sign),
                         OptionHandlerFactory.CreateOption2(
                             "distributedBuildRole",
                             "dbr",

Public/Src/App/Bxl/HelpText.cs

@@ -922,6 +922,12 @@ public static void DisplayHelp(HelpLevel helpLevel)
                 HelpLevel.Verbose
                 );
 
+            hw.WriteOption(
+                "/forceAddExecutionPermission[+|-]",
+                Strings.HelpText_DisplayHelp_ForceAddExecutionPermission,
+                HelpLevel.Verbose
+                );
+
             #endregion
 
             hw.WriteBanner(

Public/Src/App/Bxl/Strings.resx

@@ -1168,4 +1168,7 @@ Example: ad2d42d2ec5d2ca0c0b7ad65402d07c7ef40b91e</value>
   <data name="HelpText_DisplayHelp_EnableVerboseProcessLogging" xml:space="preserve">
     <value>Enables verbose sandbox logging for specific pips based on their formatted semistable hashes. This is tantamount to switching /logObservedFileAccesses and /logProcesses for these pips, and also enabling verbose debug logging in the sandbox. A list of semistable hashes might be specified, separated by semicolons. If the special value "*" is given, sandbox logging will be enabled for every pip. Example: /debug_enableVerboseProcessLogging:Pip3855A4C7E1E820D0;PipE9638AD7DDD6AF67</value>
   </data>
+    <data name="HelpText_DisplayHelp_ForceAddExecutionPermission" xml:space="preserve">
+    <value>When set to true, it enables the execution permission for the root process of process pips in Linux builds. Defaults to true.</value>
+  </data>    
 </root>

Public/Src/Engine/ProcessPipExecutor/SandboxedProcessPipExecutor.cs

@@ -837,7 +837,8 @@ public async Task<SandboxedProcessPipExecutionResult> RunAsync(
                         sandboxConnection: sandboxConnection,
                         sidebandWriter: sidebandWriter,
                         detoursEventListener: m_detoursListener,
-                        fileSystemView: fileSystemView)
+                        fileSystemView: fileSystemView,
+                        forceAddExecutionPermission: m_sandboxConfig.ForceAddExecutionPermission)
                     {
                         Arguments = arguments,
                         WorkingDirectory = m_workingDirectory,

Public/Src/Engine/Processes/SandboxedProcessInfo.cs

@@ -132,7 +132,8 @@ public SandboxedProcessInfo(
              IDetoursEventListener? detoursEventListener = null,
              ISandboxConnection? sandboxConnection = null,
              bool createJobObjectForCurrentProcess = true,
-             SandboxedProcessResourceMonitoringConfig? monitoringConfig = null)
+             SandboxedProcessResourceMonitoringConfig? monitoringConfig = null,
+             bool forceAddExecutionPermission = true)
              : this(
                    new PathTable(),
                    fileStorage,
@@ -143,7 +144,8 @@ public SandboxedProcessInfo(
                    detoursEventListener,
                    sandboxConnection,
                    createJobObjectForCurrentProcess: createJobObjectForCurrentProcess,
-                   monitoringConfig: monitoringConfig)
+                   monitoringConfig: monitoringConfig,
+                   forceAddExecutionPermission: forceAddExecutionPermission)
         {
         }
 
@@ -163,7 +165,8 @@ public SandboxedProcessInfo(
             SidebandWriter? sidebandWriter = null,
             bool createJobObjectForCurrentProcess = true,
             ISandboxFileSystemView? fileSystemView = null,
-            SandboxedProcessResourceMonitoringConfig? monitoringConfig = null)
+            SandboxedProcessResourceMonitoringConfig? monitoringConfig = null,
+            bool forceAddExecutionPermission = true)
         {
             PathTable = pathTable;
             FileAccessManifest = fileAccessManifest ?? new FileAccessManifest(pathTable);
@@ -182,6 +185,7 @@ public SandboxedProcessInfo(
             CreateJobObjectForCurrentProcess = createJobObjectForCurrentProcess;
             FileSystemView = fileSystemView;
             MonitoringConfig = monitoringConfig;
+            ForceAddExecutionPermission = forceAddExecutionPermission;
         }
 
         /// <summary>
@@ -198,7 +202,8 @@ public SandboxedProcessInfo(
             ISandboxConnection? sandboxConnection = null,
             FileAccessManifest? fileAccessManifest = null,
             bool createJobObjectForCurrentProcess = true,
-            SandboxedProcessResourceMonitoringConfig? monitoringConfig = null
+            SandboxedProcessResourceMonitoringConfig? monitoringConfig = null,
+            bool forceAddExecutionPermission = true
             )
             : this(
                   pathTable,
@@ -211,7 +216,8 @@ public SandboxedProcessInfo(
                   detoursEventListener,
                   sandboxConnection,
                   createJobObjectForCurrentProcess: createJobObjectForCurrentProcess,
-                  monitoringConfig: monitoringConfig)
+                  monitoringConfig: monitoringConfig,
+                  forceAddExecutionPermission: forceAddExecutionPermission)
         {
         }
 
@@ -291,6 +297,11 @@ public SandboxedProcessInfo(
         /// </remarks>
         public int NumRetriesPipeReadOnCancel { get; set; } = DefaultPipeReadRetryOnCancellationCount;
 
+        /// <summary>
+        /// Force set the execute permission bit for the root process of process pips in Linux builds.
+        /// </summary>
+        public bool ForceAddExecutionPermission { get; }
+
         /// <summary>
         /// Encoded command line arguments
         /// </summary>
@@ -595,6 +606,7 @@ public void Serialize(Stream stream)
                 writer.WriteNullableString(DetoursFailureFile);
                 writer.WriteNullableReadOnlyList(ExternalVmSandboxStaleFilesToClean, (w, s) => w.Write(s));
                 writer.Write(CreateSandboxTraceFile);
+                writer.Write(ForceAddExecutionPermission);
 
                 // File access manifest should be serialized the last.
                 writer.Write(FileAccessManifest, (w, v) => FileAccessManifest.Serialize(stream));
@@ -643,9 +655,9 @@ public static SandboxedProcessInfo Deserialize(Stream stream, LoggingContext log
                 var detoursFailureFile = reader.ReadNullableString();
                 var externalVmSandboxStaleFilesToClean = reader.ReadNullableReadOnlyList(r => r.ReadString());
                 var createSandboxTraceFile = reader.ReadBoolean();
+                bool forceAddExecutionPermission = reader.ReadBoolean();
 
                 var fam = reader.ReadNullable(r => FileAccessManifest.Deserialize(stream));
-
                 return new SandboxedProcessInfo(
                     new PathTable(),
                     sandboxedProcessStandardFiles != null ? new StandardFileStorage(sandboxedProcessStandardFiles) : null,
@@ -655,7 +667,8 @@ public static SandboxedProcessInfo Deserialize(Stream stream, LoggingContext log
                     loggingContext: loggingContext,
                     sidebandWriter: sidebandWritter,
                     detoursEventListener: detoursEventListener,
-                    createJobObjectForCurrentProcess: createJobObjectForCurrentProcess)
+                    createJobObjectForCurrentProcess: createJobObjectForCurrentProcess,
+                    forceAddExecutionPermission: forceAddExecutionPermission)
                 {
                     m_arguments = arguments,
                     m_commandLine = commandLine,

Public/Src/Engine/Processes/SandboxedProcessUnix.cs

@@ -213,8 +213,11 @@ public SandboxedProcessUnix(SandboxedProcessInfo info, bool ignoreReportedAccess
                 DegreeOfParallelism: 1, // Must be one, otherwise SandboxedPipExecutor will fail asserting valid reports
                 SingleProducerConstrained: useSingleProducer);
 
-            m_pendingReports = ActionBlockSlim.Create<AccessReport>(configuration: executionOptions, HandleAccessReport);
-
+            m_pendingReports = ActionBlockSlim.Create<AccessReport>(configuration: executionOptions,
+                (accessReport) =>
+                {
+                    HandleAccessReport(accessReport, info.ForceAddExecutionPermission);
+                });
             // install 'ProcessReady' and 'ProcessStarted' handlers to inform the sandbox
             ProcessReady += () => SandboxConnection.NotifyPipReady(info.LoggingContext, info.FileAccessManifest, this, m_pendingReports.Completion);
             ProcessStarted += (pid) => OnProcessStartedAsync(info).GetAwaiter().GetResult();
@@ -280,7 +283,10 @@ protected override System.Diagnostics.Process CreateProcess(SandboxedProcessInfo
                 {
                     // This is a workaround for an issue that appears on Linux where some executables in the BuildXL
                     // nuget/npm package may not have the execute bit set causing a permission denied error
-                    _ = FileUtilities.TrySetExecutePermissionIfNeeded(process.StartInfo.FileName);
+                    if (info.ForceAddExecutionPermission)
+                    {
+                        _ = FileUtilities.TrySetExecutePermissionIfNeeded(process.StartInfo.FileName);
+                    }
 
                     process.StartInfo.Arguments = $"{process.StartInfo.FileName} {process.StartInfo.Arguments}";
                     process.StartInfo.FileName = EnvExecutable;
@@ -602,7 +608,7 @@ internal void PostAccessReport(AccessReport report)
 #endif
         }
 
-        private async Task FeedStdInAsync(SandboxedProcessInfo info, string? processStdinFileName)
+        private async Task FeedStdInAsync(SandboxedProcessInfo info, string? processStdinFileName, bool forceAddExecutionPermission = true)
         {
             Contract.Requires(info.RootJailInfo == null || !NeedsShellWrapping(), "Cannot run root jail on this OS");
 
@@ -651,7 +657,10 @@ private async Task FeedStdInAsync(SandboxedProcessInfo info, string? processStdi
 
             lines.Add($"exec {cmdLine}");
 
-            SetExecutePermissionIfNeeded(info.FileName, throwIfNotFound: false);
+            if (info.ForceAddExecutionPermission)
+            {
+                SetExecutePermissionIfNeeded(info.FileName, throwIfNotFound: false);
+            }
             foreach (string line in lines)
             {
                 await Process.StandardInput.WriteLineAsync(line);
@@ -836,7 +845,7 @@ private void UpdateAverageTimeSpentInReportQueue(AccessReportStatistics stats)
             m_sumOfReportQueueTimesUs += (stats.DequeueTime - stats.EnqueueTime) / 1000;
         }
 
-        private void HandleAccessReport(AccessReport report)
+        private void HandleAccessReport(AccessReport report, bool forceAddExecutionPermission)
         {
             if (ShouldReportFileAccesses && report.Operation != FileOperation.OpDebugMessage)
             {
@@ -869,7 +878,7 @@ private void HandleAccessReport(AccessReport report)
                 // The process is statically linked, a ptrace runner needs to be started up
                 if (report.Operation == FileOperation.OpStaticallyLinkedProcess)
                 {
-                    StartPTraceRunner(report.Pid, reportPath);
+                    StartPTraceRunner(report.Pid, reportPath, forceAddExecutionPermission);
                 }
 
                 var pathExists = true;
@@ -1067,7 +1076,7 @@ internal static string AccessReportToString(AccessReport report)
                 I($"{operation}:{pid}|{requestedAccess}|{status}|{explicitLogging}|{error}|{path}|{isDirectory}|e:{report.Statistics.EnqueueTime}|h:{processTime}us|q:{queueTime}us");
         }
 
-        private void StartPTraceRunner(int pid, string path)
+        private void StartPTraceRunner(int pid, string path, bool forceAddExecutionPermission)
         {
             var paths = SandboxConnectionLinuxDetours.GetPaths(RootJailInfo, UniqueName);
             var args = $"-c {pid} -x {path}";
@@ -1094,8 +1103,9 @@ private void StartPTraceRunner(int pid, string path)
                 // We will kill these manually if the pip is exiting
                 Timeout.InfiniteTimeSpan,
                 // The runner will only log to stderr if there's a problem, other logs go to the main log using the fifo
-                errorBuilder: line => { if (line != null) { Logger.Log.PTraceRunnerError(m_loggingContext, line); } }
-            );
+                errorBuilder: line => { if (line != null) { Logger.Log.PTraceRunnerError(m_loggingContext, line); } },
+                forceAddExecutionPermission: forceAddExecutionPermission
+           );
 
             ptraceRunner.Start();
             m_ptraceRunners.Add(runnerTask(ptraceRunner));

Public/Src/Engine/Processes/UnSandboxedProcess.cs

@@ -182,7 +182,8 @@ public UnsandboxedProcess(SandboxedProcessInfo info)
                     {
                         LogProcessState($"Unable to generate core dump: {m_dumpCreationException.GetLogEventMessage()}");
                     }
-                });
+                },
+                forceAddExecutionPermission: info.ForceAddExecutionPermission);
         }
 
         /// <summary>

Public/Src/FrontEnd/UnitTests/Lage/IntegrationTests/LageIntegrationTestBase.cs

@@ -75,12 +75,6 @@ protected LageIntegrationTestBase(ITestOutputHelper output) : base(output, true)
 
             SourceRoot = Path.Combine(TestRoot, RelativeSourceRoot);
             OutDir = "target";
-
-            // TODO Bug 2073919- this is a temporary workaround to tests that are flakey due to the execute permission
-            // not correctly transiting through cache
-            AssertTrue(FileUtilities.TrySetExecutePermissionIfNeeded(PathToYarn).Succeeded);
-            AssertTrue(FileUtilities.TrySetExecutePermissionIfNeeded(PathToLage).Succeeded);
-            AssertTrue(FileUtilities.TrySetExecutePermissionIfNeeded(PathToNode).Succeeded);
         }
 
         /// <nodoc/>