[TypeName] Nested types should respect MaxNode count

src/libraries/System.Private.CoreLib/src/System/Reflection/TypeNameResolver.cs

@@ -4,16 +4,13 @@
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.Reflection.Metadata;
-using System.Text;
 
 namespace System.Reflection.Metadata
 {
     internal struct TypeNameParseOptions
     {
         public TypeNameParseOptions() { }
 #pragma warning disable CA1822 // Mark members as static
-        // CoreLib does not enforce any limits
-        public bool IsMaxDepthExceeded(int _) => false;
         public int MaxNodes
         {
             get

src/libraries/System.Reflection.Metadata/src/System/Reflection/Metadata/TypeName.cs

@@ -301,24 +301,27 @@ public string Name
         /// </remarks>
         public int GetNodeCount()
         {
+            // This method does not check for integer overflows because it's impossible to parse
+            // a TypeName with NodeCount > int.MaxValue (TypeNameParseOptions.MaxNodes is an int).
             int result = 1;
 
             if (IsNested)
             {
                 result += DeclaringType.GetNodeCount();
             }
-            else if (IsConstructedGenericType)
-            {
-                result++;
-            }
-            else if (IsArray || IsPointer || IsByRef)
+
+            if (IsArray || IsPointer || IsByRef)
             {
                 result += GetElementType().GetNodeCount();
             }
-
-            foreach (TypeName genericArgument in GetGenericArguments())
+            else if (IsConstructedGenericType)
             {
-                result += genericArgument.GetNodeCount();
+                result += GetGenericTypeDefinition().GetNodeCount();
+
+                foreach (TypeName genericArgument in GetGenericArguments())
+                {
+                    result += genericArgument.GetNodeCount();
+                }
             }
 
             return result;

src/libraries/System.Reflection.Metadata/src/System/Reflection/Metadata/TypeNameParser.cs

@@ -49,7 +49,7 @@ private TypeNameParser(ReadOnlySpan<char> name, bool throwOnError, TypeNameParse
             {
                 if (throwOnError)
                 {
-                    if (parser._parseOptions.IsMaxDepthExceeded(recursiveDepth))
+                    if (IsMaxDepthExceeded(parser._parseOptions, recursiveDepth))
                     {
                         ThrowInvalidOperation_MaxNodesExceeded(parser._parseOptions.MaxNodes);
                     }
@@ -67,13 +67,13 @@ private TypeNameParser(ReadOnlySpan<char> name, bool throwOnError, TypeNameParse
         // this method should return null instead of throwing, so the caller can get errorIndex and include it in error msg
         private TypeName? ParseNextTypeName(bool allowFullyQualifiedName, ref int recursiveDepth)
         {
-            if (!TryDive(ref recursiveDepth))
+            if (!TryDive(_parseOptions, ref recursiveDepth))
             {
                 return null;
             }
 
             List<int>? nestedNameLengths = null;
-            if (!TryGetTypeNameInfo(ref _inputString, ref nestedNameLengths, out int fullTypeNameLength))
+            if (!TryGetTypeNameInfo(_parseOptions, ref _inputString, ref nestedNameLengths, ref recursiveDepth, out int fullTypeNameLength))
             {
                 return null;
             }
@@ -147,14 +147,38 @@ private TypeNameParser(ReadOnlySpan<char> name, bool throwOnError, TypeNameParse
             {
                 _inputString = capturedBeforeProcessing;
             }
+            else
+            {
+                // Every constructed generic type needs the generic type definition.
+                if (!TryDive(_parseOptions, ref recursiveDepth))
+                {
+                    return null;
+                }
+                // If that generic type is a nested type, this needs to be taken into account. Example:
+                // "Namespace.Declaring+NestedGeneric`1[GenericArg]" requires 5 TypeName instances:
+                // - constructed generic type: Namespace.Declaring+NestedGeneric`2[GenericArg] (+1, handled by ParseNextTypeName)
+                //   - declaring type: Namespace.Declaring (+1, handled by TryGetTypeNameInfo)
+                //   - generic type definition: Namespace.Declaring+NestedGeneric`1 (+1, handled by the TryDive above)
+                //      - declaring type: Namespace.Declaring (+1, handled by the if below)
+                //   - generic arguments:
+                //     - simple: GenericArg (+1, handled by ParseNextTypeName)
+                if (nestedNameLengths is not null)
+                {
+                    recursiveDepth += nestedNameLengths.Count;
+                    if (IsMaxDepthExceeded(_parseOptions, recursiveDepth))
+                    {
+                        return null;
+                    }
+                }
+            }
 
             int previousDecorator = default;
             // capture the current state so we can reprocess it again once we know the AssemblyName
             capturedBeforeProcessing = _inputString;
             // iterate over the decorators to ensure there are no illegal combinations
             while (TryParseNextDecorator(ref _inputString, out int parsedDecorator))
             {
-                if (!TryDive(ref recursiveDepth))
+                if (!TryDive(_parseOptions, ref recursiveDepth))
                 {
                     return null;
                 }
@@ -245,15 +269,5 @@ private bool TryParseAssemblyName(ref AssemblyNameInfo? assemblyName)
 
             return declaringType;
         }
-
-        private bool TryDive(ref int depth)
-        {
-            if (_parseOptions.IsMaxDepthExceeded(depth))
-            {
-                return false;
-            }
-            depth++;
-            return true;
-        }
     }
 }

src/libraries/System.Reflection.Metadata/src/System/Reflection/Metadata/TypeNameParserHelpers.cs

@@ -220,7 +220,8 @@ internal static bool IsBeginningOfGenericArgs(ref ReadOnlySpan<char> span, out b
             return false;
         }
 
-        internal static bool TryGetTypeNameInfo(ref ReadOnlySpan<char> input, ref List<int>? nestedNameLengths, out int totalLength)
+        internal static bool TryGetTypeNameInfo(TypeNameParseOptions options, ref ReadOnlySpan<char> input,
+            ref List<int>? nestedNameLengths, ref int recursiveDepth, out int totalLength)
         {
             bool isNestedType;
             totalLength = 0;
@@ -248,6 +249,11 @@ internal static bool TryGetTypeNameInfo(ref ReadOnlySpan<char> input, ref List<i
 #endif
                 if (isNestedType)
                 {
+                    if (!TryDive(options, ref recursiveDepth))
+                    {
+                        return false;
+                    }
+
                     (nestedNameLengths ??= new()).Add(length);
                     totalLength += 1; // skip the '+' sign in next search
                 }
@@ -389,6 +395,19 @@ internal static void ThrowInvalidOperation_HasToBeArrayClass()
 #endif
         }
 
+        internal static bool IsMaxDepthExceeded(TypeNameParseOptions options, int depth)
+#if SYSTEM_PRIVATE_CORELIB
+            => false; // CoreLib does not enforce any limits
+#else
+            => depth > options.MaxNodes;
+#endif
+
+        internal static bool TryDive(TypeNameParseOptions options, ref int depth)
+        {
+            depth++;
+            return !IsMaxDepthExceeded(options, depth);
+        }
+
 #if SYSTEM_REFLECTION_METADATA
         [DoesNotReturn]
         internal static void ThrowInvalidOperation_NotSimpleName(string fullName)

src/libraries/System.Reflection.Metadata/src/System/Reflection/Metadata/TypeNameParserOptions.cs

@@ -27,7 +27,5 @@ public int MaxNodes
                 _maxNodes = value;
             }
         }
-
-        internal bool IsMaxDepthExceeded(int depth) => depth >= _maxNodes;
     }
 }

src/libraries/System.Reflection.Metadata/tests/Metadata/TypeNameParserHelpersTests.cs

@@ -122,14 +122,17 @@ public void TryGetTypeNameInfoGetsAllTheInfo(string input, bool expectedResult,
         {
             List<int>? nestedNameLengths = null;
             ReadOnlySpan<char> span = input.AsSpan();
-            bool result = TypeNameParserHelpers.TryGetTypeNameInfo(ref span, ref nestedNameLengths, out int totalLength);
+            TypeNameParseOptions options = new();
+            int recursiveDepth = 0;
+            bool result = TypeNameParserHelpers.TryGetTypeNameInfo(options, ref span, ref nestedNameLengths, ref recursiveDepth, out int totalLength);
 
             Assert.Equal(expectedResult, result);
 
             if (expectedResult)
             {
                 Assert.Equal(expectedNestedNameLengths, nestedNameLengths?.ToArray());
                 Assert.Equal(expectedTotalLength, totalLength);
+                Assert.Equal(expectedNestedNameLengths?.Length ?? 0, recursiveDepth);
             }
         }