Disable core dumps in host tests that intentionally crash

src/installer/tests/Assets/Projects/AppWithCustomEntryPoints/AppWithCustomEntryPoints.csproj

@@ -5,4 +5,8 @@
     <OutputType>Exe</OutputType>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Compile Include="..\CoreDump.cs" />
+  </ItemGroup>
+
 </Project>

src/installer/tests/Assets/Projects/AppWithCustomEntryPoints/Program.cs

@@ -53,6 +53,9 @@ public static int ThrowException(IntPtr arg, int size)
         {
             functionPointerCallCount++;
             PrintFunctionPointerCallLog(nameof(ThrowException), arg, size);
+
+            // Disable core dumps - test is intentionally crashing
+            Utilities.CoreDump.Disable();
             throw new InvalidOperationException(nameof(ThrowException));
         }
 

src/installer/tests/Assets/Projects/Component/Component.cs

@@ -47,6 +47,9 @@ public static int ThrowException(IntPtr arg, int size)
         {
             componentCallCount++;
             PrintComponentCallLog(nameof(ThrowException), arg, size);
+
+            // Disable core dumps - test is intentionally crashing
+            Utilities.CoreDump.Disable();
             throw new InvalidOperationException(nameof(ThrowException));
         }
 

src/installer/tests/Assets/Projects/Component/Component.csproj

@@ -5,4 +5,8 @@
     <EnableDynamicLoading>true</EnableDynamicLoading>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Compile Include="..\CoreDump.cs" />
+  </ItemGroup>
+
 </Project>

src/installer/tests/Assets/Projects/CoreDump.cs

@@ -0,0 +1,51 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+//
+
+using System;
+using System.Runtime.InteropServices;
+
+namespace Utilities
+{
+    public static partial class CoreDump
+    {
+        public static void Disable()
+        {
+            if (OperatingSystem.IsLinux())
+            {
+                if (prctl(PR_SET_DUMPABLE, 0) != 0)
+                {
+                    throw new Exception($"Failed to disable core dump. Error: {Marshal.GetLastPInvokeError()}.");
+                }
+            }
+            else if (OperatingSystem.IsMacOS())
+            {
+                RLimit rlimit = new() { rlim_cur = 0, rlim_max = 0 };
+                if (setrlimit(RLIMIT_CORE, rlimit) != 0)
+                {
+                    throw new Exception($"Failed to disable core dump. Error: {Marshal.GetLastPInvokeError()}.");
+                }
+            }
+        }
+
+        [StructLayout(LayoutKind.Sequential)]
+        private struct RLimit
+        {
+            // These are rlim_t. All macOS platforms we use this on currently define it as unsigned 64-bit
+            public ulong rlim_cur; // Soft limit
+            public ulong rlim_max; // Hard limit
+        }
+
+        // Max core file size
+        private const int RLIMIT_CORE = 4;
+
+        [DllImport("libc", SetLastError = true)]
+        private static extern int setrlimit(int resource, in RLimit rlim);
+
+        // "dumpable" attribute of the calling process
+        private const int PR_SET_DUMPABLE = 4;
+
+        [DllImport("libc", SetLastError = true)]
+        private static extern int prctl(int option, int arg2);
+    }
+}

src/installer/tests/Assets/Projects/HelloWorld/HelloWorld.csproj

@@ -12,4 +12,8 @@
     </PropertyGroup>
   </Target>
 
+  <ItemGroup>
+    <Compile Include="..\CoreDump.cs" />
+  </ItemGroup>
+
 </Project>

src/installer/tests/Assets/Projects/HelloWorld/Program.cs

@@ -50,6 +50,8 @@ public static void Main(string[] args)
                     }
                     break;
                 case "throw_exception":
+                    // Disable core dumps - test is intentionally crashing
+                    Utilities.CoreDump.Disable();
                     throw new Exception("Goodbye World!");
                 default:
                     break;

src/installer/tests/Assets/Projects/HelloWorld/SelfContained.csproj

@@ -12,4 +12,9 @@
     <_SupportedArchitecture Condition="'$(TargetArchitecture)' == 'x64' or '$(TargetArchitecture)' == 'x86' or '$(TargetArchitecture)' == 'arm' or '$(TargetArchitecture)' == 'arm64'">true</_SupportedArchitecture>
     <RuntimeIdentifier Condition="'$(_SupportedPlatform)' == 'true' and '$(_SupportedArchitecture)' == 'true'">$(TargetRid)</RuntimeIdentifier>
   </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="..\CoreDump.cs" />
+  </ItemGroup>
+
 </Project>

src/installer/tests/HostActivation.Tests/Breadcrumbs.cs

@@ -37,7 +37,8 @@ public void UnhandledException_BreadcrumbThreadDoesNotFinish()
             TestContext.BuiltDotNet.Exec(sharedTestState.App.AppDll, "throw_exception")
                 .EnvironmentVariable(Constants.Breadcrumbs.EnvironmentVariable, sharedTestState.BreadcrumbLocation)
                 .EnableTracingAndCaptureOutputs()
-                .Execute(expectedToFail: true)
+                .DisableDumps() // Expected to throw an exception
+                .Execute()
                 .Should().Fail()
                 .And.HaveStdErrContaining("Unhandled exception.")
                 .And.HaveStdErrContaining("System.Exception: Goodbye World")

src/installer/tests/HostActivation.Tests/DependencyResolution/AdditionalDeps.cs

@@ -99,7 +99,7 @@ public void DepsFile(bool dependencyExists)
 
             CommandResult result = SharedState.DotNetWithNetCoreApp.Exec(Constants.AdditionalDeps.CommandLineArgument, additionalDepsFile, app.AppDll)
                 .EnableTracingAndCaptureOutputs()
-                .Execute(expectedToFail: !dependencyExists);
+                .Execute();
 
             result.Should().HaveUsedAdditionalDeps(additionalDepsFile);
             if (dependencyExists)
@@ -126,7 +126,7 @@ public void InvalidJson()
 
                 SharedState.DotNetWithNetCoreApp.Exec(Constants.AdditionalDeps.CommandLineArgument, invalidDepsFile, SharedState.FrameworkReferenceApp.AppDll)
                     .EnableTracingAndCaptureOutputs()
-                    .Execute(expectedToFail: true)
+                    .Execute()
                     .Should().Fail()
                     .And.HaveUsedAdditionalDeps(invalidDepsFile)
                     .And.HaveStdErrContaining($"Error initializing the dependency resolver: An error occurred while parsing: {invalidDepsFile}");

src/installer/tests/HostActivation.Tests/DotnetArgValidation.cs

@@ -25,7 +25,7 @@ public void MuxerExec_MissingAppAssembly_Fails()
             TestContext.BuiltDotNet.Exec("exec", assemblyName)
                 .CaptureStdOut()
                 .CaptureStdErr()
-                .Execute(expectedToFail: true)
+                .Execute()
                 .Should().Fail()
                 .And.HaveStdErrContaining($"The application to execute does not exist: '{assemblyName}'");
         }
@@ -37,7 +37,7 @@ public void MuxerExec_MissingAppAssembly_BadExtension_Fails()
             TestContext.BuiltDotNet.Exec("exec", assemblyName)
                 .CaptureStdOut()
                 .CaptureStdErr()
-                .Execute(expectedToFail: true)
+                .Execute()
                 .Should().Fail()
                 .And.HaveStdErrContaining($"The application to execute does not exist: '{assemblyName}'");
         }
@@ -52,7 +52,7 @@ public void MuxerExec_BadExtension_Fails()
             TestContext.BuiltDotNet.Exec("exec", assemblyName)
                 .CaptureStdOut()
                 .CaptureStdErr()
-                .Execute(expectedToFail: true)
+                .Execute()
                 .Should().Fail()
                 .And.HaveStdErrContaining($"dotnet exec needs a managed .dll or .exe extension. The application specified was '{assemblyName}'");
         }
@@ -63,7 +63,7 @@ public void MissingArgumentValue_Fails()
             TestContext.BuiltDotNet.Exec("--fx-version")
                 .CaptureStdOut()
                 .CaptureStdErr()
-                .Execute(expectedToFail: true)
+                .Execute()
                 .Should().Fail()
                 .And.HaveStdErrContaining($"Failed to parse supported options or their values:");
         }
@@ -76,7 +76,7 @@ public void InvalidFileOrCommand_NoSDK_ListsPossibleIssues()
                 .WorkingDirectory(sharedTestState.BaseDirectory.Location)
                 .CaptureStdOut()
                 .CaptureStdErr()
-                .Execute(expectedToFail: true)
+                .Execute()
                 .Should().Fail()
                 .And.HaveStdErrContaining($"The application '{fileName}' does not exist")
                 .And.FindAnySdk(false);

src/installer/tests/HostActivation.Tests/FrameworkDependentAppLaunch.cs

@@ -56,7 +56,7 @@ public void Muxer_AssemblyWithDifferentFileExtension_Fails()
 
             TestContext.BuiltDotNet.Exec(appOtherExt)
                 .CaptureStdErr()
-                .Execute(expectedToFail: true)
+                .Execute()
                 .Should().Fail()
                 .And.HaveStdErrContaining($"The application '{appOtherExt}' does not exist or is not a managed .dll or .exe");
         }
@@ -143,7 +143,8 @@ public void Muxer_NonAssemblyWithExeExtension()
             TestContext.BuiltDotNet.Exec(appExe)
                 .CaptureStdOut()
                 .CaptureStdErr()
-                .Execute(expectedToFail: true)
+                .DisableDumps() // Expected to throw an exception
+                .Execute()
                 .Should().Fail()
                 .And.HaveStdErrContaining("BadImageFormatException");
         }
@@ -429,7 +430,7 @@ public void AppHost_CLI_MissingRuntimeFramework_ErrorReportedInStdErr(bool missi
                     .EnableTracingAndCaptureOutputs()
                     .DotNetRoot(invalidDotNet.Location)
                     .MultilevelLookup(false)
-                    .Execute(expectedToFail: true);
+                    .Execute();
 
                 result.Should().Fail()
                     .And.HaveStdErrContaining($"https://aka.ms/dotnet-core-applaunch?{expectedUrlQuery}")

src/installer/tests/HostActivation.Tests/InvalidHost.cs

@@ -27,7 +27,7 @@ public void AppHost_NotBound()
             CommandResult result = Command.Create(sharedTestState.UnboundAppHost)
                 .CaptureStdErr()
                 .CaptureStdOut()
-                .Execute(expectedToFail: true);
+                .Execute();
 
             result.Should().Fail()
                 .And.HaveStdErrContaining("This executable is not bound to a managed DLL to execute.")
@@ -70,7 +70,7 @@ public void DotNet_Renamed()
             CommandResult result = Command.Create(sharedTestState.RenamedDotNet)
                 .CaptureStdErr()
                 .CaptureStdOut()
-                .Execute(expectedToFail: true);
+                .Execute();
 
             result.Should().Fail()
                 .And.HaveStdErrContaining($"Error: cannot execute dotnet when renamed to {Path.GetFileNameWithoutExtension(sharedTestState.RenamedDotNet)}")

src/installer/tests/HostActivation.Tests/NativeHosting/ApplicationExecution.cs

@@ -53,7 +53,8 @@ public void RunApp_UnhandledException()
             };
 
             sharedState.CreateNativeHostCommand(args, sharedState.DotNetRoot)
-                .Execute(expectedToFail: true)
+                .DisableDumps() // Expected to throw an exception
+                .Execute()
                 .Should().Fail()
                 .And.InitializeContextForApp(app.AppDll)
                 .And.ExecuteApplicationWithException(sharedState.NativeHostPath, app.AppDll);

src/installer/tests/HostActivation.Tests/NativeHosting/GetFunctionPointer.cs

@@ -205,7 +205,8 @@ public void CallDelegateOnApplicationContext_UnhandledException()
             };
 
             sharedState.CreateNativeHostCommand(args, sharedState.DotNetRoot)
-                .Execute(expectedToFail: true)
+                .DisableDumps() // Expected to throw an exception
+                .Execute()
                 .Should().Fail()
                 .And.InitializeContextForApp(app.AppDll)
                 .And.ExecuteFunctionPointerWithException(entryPoint, 1);

src/installer/tests/HostActivation.Tests/NativeHosting/LoadAssemblyAndGetFunctionPointer.cs

@@ -226,7 +226,8 @@ public void CallDelegateOnComponentContext_UnhandledException()
             };
 
             sharedState.CreateNativeHostCommand(args, sharedState.DotNetRoot)
-                .Execute(expectedToFail: true)
+                .DisableDumps() // Expected to throw an exception
+                .Execute()
                 .Should().Fail()
                 .And.InitializeContextForConfig(component.RuntimeConfigJson)
                 .And.ExecuteFunctionPointerWithException(entryPoint, 1);

src/installer/tests/HostActivation.Tests/SelfContainedAppLaunch.cs

@@ -151,7 +151,7 @@ public void DotNetRoot_IncorrectLayout_Fails()
             Command.Create(appExe)
                 .EnableTracingAndCaptureOutputs()
                 .DotNetRoot(app.Location)
-                .Execute(expectedToFail: true)
+                .Execute()
                 .Should().Fail()
                 .And.HaveUsedDotNetRootInstallLocation(Path.GetFullPath(app.Location), TestContext.BuildRID)
                 .And.HaveStdErrContaining($"The required library {Binaries.HostFxr.FileName} could not be found.");

src/installer/tests/HostActivation.Tests/StartupHooks.cs

@@ -96,7 +96,8 @@ public void Muxer_activation_of_StartupHook_With_Missing_Dependencies_Fails()
                 .EnvironmentVariable("TEST_STARTUPHOOK_USE_DEPENDENCY", true.ToString())
                 .CaptureStdOut()
                 .CaptureStdErr()
-                .Execute(expectedToFail: true)
+                .DisableDumps() // Expected to throw an exception
+                .Execute()
                 .Should().Fail()
                 .And.HaveStdErrContaining("System.IO.FileNotFoundException: Could not load file or assembly 'SharedLibrary");
         }

src/installer/tests/TestUtils/Command.cs

@@ -17,6 +17,7 @@ public class Command
         private StringWriter _stdOutCapture;
         private StringWriter _stdErrCapture;
 
+        private bool _disableDumps = false;
         private bool _running = false;
 
         public Process Process { get; }
@@ -133,6 +134,14 @@ private static bool ShouldUseCmd(string executable)
             return false;
         }
 
+        public Command DisableDumps()
+        {
+            _disableDumps = true;
+            RemoveEnvironmentVariable("COMPlus_DbgEnableMiniDump");
+            RemoveEnvironmentVariable("DOTNET_DbgEnableMiniDump");
+            return this;
+        }
+
         public Command Environment(IDictionary<string, string> env)
         {
             if (env == null)
@@ -153,16 +162,27 @@ public Command Environment(string key, string value)
             return this;
         }
 
-        public CommandResult Execute([CallerMemberName] string caller = "")
-        {
-            return Execute(false, caller);
-        }
-
         public Command Start([CallerMemberName] string caller = "")
         {
             ThrowIfRunning();
             _running = true;
 
+            if (_disableDumps && (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS()))
+            {
+                // Replace double quoted arguments with single quotes.
+                // We only want to replace non-escaped quotes - that is, ones not preceded by a backslash
+                // or preceded by an even number of backslashes.
+                string args = System.Text.RegularExpressions.Regex.Replace(
+                    Process.StartInfo.Arguments,
+                    @"((?:^|[^\\])(?:\\\\)*)""",
+                    m => m.Value.Substring(0, m.Value.Length - 1) + "'"
+                );
+
+                // Explicitly set the core file size to 0 before launching the process in the same shell
+                Process.StartInfo.Arguments = $"-c \"ulimit -c 0 && exec {Process.StartInfo.FileName} {args}\"";
+                Process.StartInfo.FileName = "/bin/bash";
+            }
+
             if (Process.StartInfo.RedirectStandardOutput)
             {
                 Process.OutputDataReceived += (sender, args) =>
@@ -245,17 +265,9 @@ public CommandResult WaitForExit(int timeoutMilliseconds = Timeout.Infinite, [Ca
         /// <summary>
         /// Execute the command and wait for it to exit.
         /// </summary>
-        /// <param name="expectedToFail">Whether or not the command is expected to fail (non-zero exit code)</param>
         /// <returns>Result of the command</returns>
-        public CommandResult Execute(bool expectedToFail, [CallerMemberName] string caller = "")
+        public CommandResult Execute([CallerMemberName] string caller = "")
         {
-            // Clear out any enabling of dump creation if failure is expected
-            if (expectedToFail)
-            {
-                RemoveEnvironmentVariable("COMPlus_DbgEnableMiniDump");
-                RemoveEnvironmentVariable("DOTNET_DbgEnableMiniDump");
-            }
-
             Start(caller);
             return WaitForExit(caller: caller);
         }