Add timeouts and retries to LiveDebuggerTests when debugger is disabled (#7010)

## Summary of changes

Adds a timeout to the `DataTarget.CreateSnapshotAndAttach()` call, and
marks the `LiveDebuggerTests` as flaky

## Reason for change

The `LiveDebuggerTests` currently frequently hang, we need to understand
why, and mitigate the issue.

## Implementation details

- Add overall timeout to the `GetLiveObjectsByTypes` method
- Add some logs to try to understand what's happening
- Mark the test as flaky

## Test coverage

N/A

Example failures:
[Example1](https://dev.azure.com/datadoghq/dd-trace-dotnet/_build/results?buildId=178295&view=logs&j=5bbbfeff-80c6-5e3a-ed15-df19bc2221a9&t=f97d6e34-8c18-5125-1e10-f8ff1691d2d4),
[Example2](https://dev.azure.com/datadoghq/dd-trace-dotnet/_build/results?buildId=178288&view=logs&j=5a1533ef-d5b3-5244-8177-90f592ad9a02&t=889462ef-7425-5ee6-c218-8225a4de53e7),
[Example3](https://dev.azure.com/datadoghq/dd-trace-dotnet/_build/results?buildId=178279&view=logs&j=5bbbfeff-80c6-5e3a-ed15-df19bc2221a9&t=f97d6e34-8c18-5125-1e10-f8ff1691d2d4),
[Example4](https://dev.azure.com/datadoghq/dd-trace-dotnet/_build/results?buildId=178274&view=logs&j=dfbb55d7-038c-5432-f154-c7f295998250&t=cde1f8e2-299e-52ab-e384-edcffbdbb58d),

## Other details

I suspect there's a race condition - maybe the app exists before the
`DataTarget` can attach or something? _May_ want to rewrite to use the
existing memory dump capabilities we have built in?

If this continues to fail, we will likely need to skip it completely.

Author: andrewlock

tracer/test/Datadog.Trace.Debugger.IntegrationTests/Assertions/DumpHeapLive.cs

@@ -3,9 +3,13 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
 
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
 using Microsoft.Diagnostics.Runtime;
+using Xunit.Abstractions;
 
 namespace Datadog.Trace.Debugger.IntegrationTests.Assertions;
 
@@ -16,26 +20,55 @@ namespace Datadog.Trace.Debugger.IntegrationTests.Assertions;
 /// </summary>
 internal static class DumpHeapLive
 {
-    public static Dictionary<string, (int Live, int Disposed)> GetLiveObjectsByTypes(Process process)
+    public static async Task<Dictionary<string, (int Live, int Disposed)>> GetLiveObjectsByTypes(Process process, ITestOutputHelper output, TimeSpan timeout)
+    {
+        using var cts = new CancellationTokenSource(timeout);
+
+        // Run the task in a backgroun thread, and wait for it to complete or timeout.
+        return await Task.Run(() => GetLiveObjectsByTypes(process, output, cts.Token), cts.Token);
+    }
+
+    private static Dictionary<string, (int Live, int Disposed)> GetLiveObjectsByTypes(Process process, ITestOutputHelper output, CancellationToken ct)
     {
         var objCountByType = new Dictionary<string, (int Live, int Disposed)>();
         if (process.HasExited)
         {
             throw new MemoryAssertionException("Process has exited", string.Empty);
         }
 
+        output?.WriteLine($"Creating snapshot of process ID {process.Id}");
         using var target = DataTarget.CreateSnapshotAndAttach(process.Id);
+        if (ct.IsCancellationRequested)
+        {
+            throw new OperationCanceledException();
+        }
+
+        output?.WriteLine("Snapshot complete, analyzing heap...");
         var heap = target.ClrVersions[0].CreateRuntime().Heap;
         var considered = new ObjectSet(heap);
         var eval = new Stack<ulong>();
 
+        var rootCount = 0;
         foreach (var root in heap.EnumerateRoots())
         {
+            rootCount++;
             eval.Push(root.Object);
         }
 
+        output?.WriteLine($"Found {rootCount} roots in the heap");
+        var evalsComplete = 0;
         while (eval.Count > 0)
         {
+            if (ct.IsCancellationRequested)
+            {
+                throw new OperationCanceledException();
+            }
+
+            if (evalsComplete % 100 == 0)
+            {
+                output?.WriteLine($"Evaluated {evalsComplete} objects so far, {eval.Count} remaining.");
+            }
+
             var obj = eval.Pop();
             if (considered.Contains(obj))
             {
@@ -71,6 +104,7 @@ internal static class DumpHeapLive
             }
         }
 
+        output?.WriteLine($"Analysis complete, found {objCountByType.Count} object types in the heap");
         return objCountByType;
     }
 

tracer/test/Datadog.Trace.Debugger.IntegrationTests/Assertions/MemoryAssertions.cs

@@ -8,6 +8,8 @@
 using System.Diagnostics;
 using System.Reflection;
 using System.Runtime.InteropServices;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
 
 namespace Datadog.Trace.Debugger.IntegrationTests.Assertions;
 
@@ -32,15 +34,16 @@ private MemoryAssertions(Dictionary<string, (int Live, int Disposed)> liveObject
     /// which can perform assertions on said snapshot.
     /// </summary>
     /// <param name="process">Process to capture snapshot of</param>
+    /// <param name="output">The test output helper</param>
     /// <returns>MemoryAssertions</returns>
-    public static MemoryAssertions CaptureSnapshotToAssertOn(Process process)
+    public static async Task<MemoryAssertions> CaptureSnapshotToAssertOn(Process process, ITestOutputHelper output)
     {
         if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
         {
             throw new NotSupportedException("Arm64 is not supported for memory assertions");
         }
 
-        var liveObjectsByTypes = DumpHeapLive.GetLiveObjectsByTypes(process);
+        var liveObjectsByTypes = await DumpHeapLive.GetLiveObjectsByTypes(process, output, TimeSpan.FromSeconds(30));
         return new MemoryAssertions(liveObjectsByTypes);
     }
 

tracer/test/Datadog.Trace.Debugger.IntegrationTests/LiveDebuggerTests.cs

@@ -39,6 +39,7 @@ public LiveDebuggerTests(ITestOutputHelper output)
     [Trait("Category", "ArmUnsupported")]
     [Trait("RunOnWindows", "True")]
     [Trait("Category", "LinuxUnsupported")]
+    [Flaky("The explicitly disabled tests often hang on x86 .NET and .NET 8, when the debugger is disabled. Needs investigation.")]
     public async Task LiveDebuggerDisabled_DebuggerDisabledByDefault_NoDebuggerTypesCreated()
     {
         await RunTest();
@@ -49,6 +50,7 @@ public async Task LiveDebuggerDisabled_DebuggerDisabledByDefault_NoDebuggerTypes
     [Trait("Category", "ArmUnsupported")]
     [Trait("RunOnWindows", "True")]
     [Trait("Category", "LinuxUnsupported")]
+    [Flaky("The explicitly disabled tests often hang on x86 .NET and .NET 8, when the debugger is disabled. Needs investigation.")]
     public async Task LiveDebuggerDisabled_DebuggerExplicitlyDisabled_NoDebuggerTypesCreated()
     {
         SetEnvironmentVariable(ConfigurationKeys.Debugger.Enabled, "0");
@@ -68,7 +70,7 @@ private async Task RunTest()
 
         try
         {
-            var memoryAssertions = MemoryAssertions.CaptureSnapshotToAssertOn(sample);
+            var memoryAssertions = await MemoryAssertions.CaptureSnapshotToAssertOn(sample, Output);
 
             memoryAssertions.NoObjectsExist<SnapshotSink>();
             memoryAssertions.NoObjectsExist<LineProbeResolver>();