Disable core dumps in host tests that intentionally crash (#116725)

For tests that intentionally crash, we were already clearing out `DOTNET_DbgEnableMiniDump`/`COMPlus_DbgEnableMiniDump`, but OS core dumps were still enabled.
- In tests that are explicitly throwing an unhandled exception, call into OS APIs to disable dumps for the current process
- When launching a command that should have dumps disabled, call `ulimit -c 0` before the command (for cases where the test doesn't have control over the process)

For Unix, this avoids creating any dumps for intentional crashes. On Windows, we still have two created for `Muxer_NonAssemblyWithExeExtension` and `UnhandledException_BreadcrumbThreadDoesNotFinish`.

Author: elinor-fung

src/installer/tests/Assets/Projects/AppWithCustomEntryPoints/AppWithCustomEntryPoints.csproj

@@ -5,4 +5,8 @@
     <OutputType>Exe</OutputType>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Compile Include="..\CoreDump.cs" />
+  </ItemGroup>
+
 </Project>

src/installer/tests/Assets/Projects/AppWithCustomEntryPoints/Program.cs

@@ -53,6 +53,9 @@ public static int ThrowException(IntPtr arg, int size)
         {
             functionPointerCallCount++;
             PrintFunctionPointerCallLog(nameof(ThrowException), arg, size);
+
+            // Disable core dumps - test is intentionally crashing
+            Utilities.CoreDump.Disable();
             throw new InvalidOperationException(nameof(ThrowException));
         }
 

src/installer/tests/Assets/Projects/Component/Component.cs

@@ -47,6 +47,9 @@ public static int ThrowException(IntPtr arg, int size)
         {
             componentCallCount++;
             PrintComponentCallLog(nameof(ThrowException), arg, size);
+
+            // Disable core dumps - test is intentionally crashing
+            Utilities.CoreDump.Disable();
             throw new InvalidOperationException(nameof(ThrowException));
         }
 

src/installer/tests/Assets/Projects/Component/Component.csproj

@@ -5,4 +5,8 @@
     <EnableDynamicLoading>true</EnableDynamicLoading>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Compile Include="..\CoreDump.cs" />
+  </ItemGroup>
+
 </Project>

src/installer/tests/Assets/Projects/CoreDump.cs

@@ -0,0 +1,59 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+//
+
+using System;
+using System.Runtime.InteropServices;
+
+namespace Utilities
+{
+    public static partial class CoreDump
+    {
+        public static void Disable()
+        {
+            string? envValue = Environment.GetEnvironmentVariable("DOTNET_DbgEnableMiniDump");
+            if (envValue is not null && envValue != "0")
+                throw new InvalidOperationException("DOTNET_DbgEnableMiniDump is set and not 0. Ensure it is unset or set to 0 to disable dumps.");
+
+            envValue = Environment.GetEnvironmentVariable("COMPlus_DbgEnableMiniDump");
+            if (envValue is not null && envValue != "0")
+                throw new InvalidOperationException("COMPlus_DbgEnableMiniDump is set and not 0. Ensure it is unset or set to 0 to disable dumps.");
+
+            if (OperatingSystem.IsLinux())
+            {
+                if (prctl(PR_SET_DUMPABLE, 0) != 0)
+                {
+                    throw new InvalidOperationException($"Failed to disable core dump. Error: {Marshal.GetLastPInvokeError()}.");
+                }
+            }
+            else if (OperatingSystem.IsMacOS())
+            {
+                RLimit rlimit = new() { rlim_cur = 0, rlim_max = 0 };
+                if (setrlimit(RLIMIT_CORE, rlimit) != 0)
+                {
+                    throw new InvalidOperationException($"Failed to disable core dump. Error: {Marshal.GetLastPInvokeError()}.");
+                }
+            }
+        }
+
+        [StructLayout(LayoutKind.Sequential)]
+        private struct RLimit
+        {
+            // These are rlim_t. All macOS platforms we use this on currently define it as unsigned 64-bit
+            public ulong rlim_cur; // Soft limit
+            public ulong rlim_max; // Hard limit
+        }
+
+        // Max core file size
+        private const int RLIMIT_CORE = 4;
+
+        [DllImport("libc", SetLastError = true)]
+        private static extern int setrlimit(int resource, in RLimit rlim);
+
+        // "dumpable" attribute of the calling process
+        private const int PR_SET_DUMPABLE = 4;
+
+        [DllImport("libc", SetLastError = true)]
+        private static extern int prctl(int option, int arg2);
+    }
+}

src/installer/tests/Assets/Projects/HelloWorld/HelloWorld.csproj

@@ -12,4 +12,8 @@
     </PropertyGroup>
   </Target>
 
+  <ItemGroup>
+    <Compile Include="..\CoreDump.cs" />
+  </ItemGroup>
+
 </Project>

src/installer/tests/Assets/Projects/HelloWorld/Program.cs

@@ -50,6 +50,8 @@ public static void Main(string[] args)
                     }
                     break;
                 case "throw_exception":
+                    // Disable core dumps - test is intentionally crashing
+                    Utilities.CoreDump.Disable();
                     throw new Exception("Goodbye World!");
                 default:
                     break;

src/installer/tests/Assets/Projects/HelloWorld/SelfContained.csproj

@@ -12,4 +12,9 @@
     <_SupportedArchitecture Condition="'$(TargetArchitecture)' == 'x64' or '$(TargetArchitecture)' == 'x86' or '$(TargetArchitecture)' == 'arm' or '$(TargetArchitecture)' == 'arm64'">true</_SupportedArchitecture>
     <RuntimeIdentifier Condition="'$(_SupportedPlatform)' == 'true' and '$(_SupportedArchitecture)' == 'true'">$(TargetRid)</RuntimeIdentifier>
   </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="..\CoreDump.cs" />
+  </ItemGroup>
+
 </Project>

src/installer/tests/HostActivation.Tests/Breadcrumbs.cs

@@ -37,7 +37,8 @@ public void UnhandledException_BreadcrumbThreadDoesNotFinish()
             TestContext.BuiltDotNet.Exec(sharedTestState.App.AppDll, "throw_exception")
                 .EnvironmentVariable(Constants.Breadcrumbs.EnvironmentVariable, sharedTestState.BreadcrumbLocation)
                 .EnableTracingAndCaptureOutputs()
-                .Execute(expectedToFail: true)
+                .DisableDumps() // Expected to throw an exception
+                .Execute()
                 .Should().Fail()
                 .And.HaveStdErrContaining("Unhandled exception.")
                 .And.HaveStdErrContaining("System.Exception: Goodbye World")

src/installer/tests/HostActivation.Tests/DependencyResolution/AdditionalDeps.cs

@@ -99,7 +99,7 @@ public void DepsFile(bool dependencyExists)
 
             CommandResult result = SharedState.DotNetWithNetCoreApp.Exec(Constants.AdditionalDeps.CommandLineArgument, additionalDepsFile, app.AppDll)
                 .EnableTracingAndCaptureOutputs()
-                .Execute(expectedToFail: !dependencyExists);
+                .Execute();
 
             result.Should().HaveUsedAdditionalDeps(additionalDepsFile);
             if (dependencyExists)
@@ -126,7 +126,7 @@ public void InvalidJson()
 
                 SharedState.DotNetWithNetCoreApp.Exec(Constants.AdditionalDeps.CommandLineArgument, invalidDepsFile, SharedState.FrameworkReferenceApp.AppDll)
                     .EnableTracingAndCaptureOutputs()
-                    .Execute(expectedToFail: true)
+                    .Execute()
                     .Should().Fail()
                     .And.HaveUsedAdditionalDeps(invalidDepsFile)
                     .And.HaveStdErrContaining($"Error initializing the dependency resolver: An error occurred while parsing: {invalidDepsFile}");