Add mechanism to allow CoreLib trimming to leverage whole program analysis (#118769)

A couple times in the past I needed a way to say "if X is not part of the program, eliminate the entire basic block". We can do this for allocated types (this is how branches under `is` checks elimination works), but we can't do this for more general "characteristics".

This introduces a mechanism where AOT compiler and CoreLib (or System.Private.* universe in general) can define whole program tags such as "whole program has X in it" and CoreLib can condition code on the presence of this tag.

This is easier shown than described, so I extracted the first use of this into a separate commit. In this commit, we eliminate code that tries looking for `StackTraceHiddenAttribute` if we know the whole program has no `StackTraceHiddenAttribute` in it. With this code eliminated, #118640 can then eliminate all custom attributes on methods, which in turn plays into #118718 and we can eliminate enum boxing even when StackTraceSupport is not set to false (right now #118718 really needs the StackTraceSupport=false to get rid of boxed enums; we get more boxed enums from method attributes).

We have a new node that represents the characteristic. The node can be dropped into the graph wherever needed. ILScanner then uses this to condition parts of the method body on this characteristic node. We need similar logic in the substitution IL provider because we need to guarantee that RyuJIT is not going to see basic blocks we didn't scan. So we treat it as a substitution during codegen phase too.

Author: MichalStrehovsky

src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/CompilerServices/AnalysisCharacteristicAttribute.cs

@@ -0,0 +1,11 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Runtime.CompilerServices
+{
+    // When applied to an intrinsic method, the method will become a characteristic check.
+    [AttributeUsage(AttributeTargets.Method, Inherited = false, AllowMultiple = false)]
+    internal class AnalysisCharacteristicAttribute : Attribute
+    {
+    }
+}

src/coreclr/nativeaot/System.Private.StackTraceMetadata/src/Internal/StackTraceMetadata/StackTraceMetadata.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Reflection.Runtime.General;
+using System.Runtime.CompilerServices;
 
 using Internal.Metadata.NativeFormat;
 using Internal.NativeFormat;
@@ -41,6 +42,10 @@ internal static void Initialize()
             RuntimeAugments.InitializeStackTraceMetadataSupport(new StackTraceMetadataCallbacksImpl());
         }
 
+        [Intrinsic]
+        [AnalysisCharacteristic]
+        internal static extern bool StackTraceHiddenMetadataPresent();
+
         /// <summary>
         /// Locate the containing module for a method and try to resolve its name based on start address.
         /// </summary>
@@ -75,21 +80,24 @@ public static unsafe string GetMethodNameFromStartAddressIfAvailable(IntPtr meth
                 out TypeDefinitionHandle typeHandle,
                 out MethodHandle methodHandle))
             {
-                foreach (CustomAttributeHandle cah in reader.GetTypeDefinition(typeHandle).CustomAttributes)
+                if (StackTraceHiddenMetadataPresent())
                 {
-                    if (cah.IsCustomAttributeOfType(reader, ["System", "Diagnostics"], "StackTraceHiddenAttribute"))
+                    foreach (CustomAttributeHandle cah in reader.GetTypeDefinition(typeHandle).CustomAttributes)
                     {
-                        isStackTraceHidden = true;
-                        break;
+                        if (cah.IsCustomAttributeOfType(reader, ["System", "Diagnostics"], "StackTraceHiddenAttribute"))
+                        {
+                            isStackTraceHidden = true;
+                            break;
+                        }
                     }
-                }
 
-                foreach (CustomAttributeHandle cah in reader.GetMethod(methodHandle).CustomAttributes)
-                {
-                    if (cah.IsCustomAttributeOfType(reader, ["System", "Diagnostics"], "StackTraceHiddenAttribute"))
+                    foreach (CustomAttributeHandle cah in reader.GetMethod(methodHandle).CustomAttributes)
                     {
-                        isStackTraceHidden = true;
-                        break;
+                        if (cah.IsCustomAttributeOfType(reader, ["System", "Diagnostics"], "StackTraceHiddenAttribute"))
+                        {
+                            isStackTraceHidden = true;
+                            break;
+                        }
                     }
                 }
 

src/coreclr/nativeaot/System.Private.StackTraceMetadata/src/System.Private.StackTraceMetadata.csproj

@@ -23,5 +23,7 @@
     <Compile Include="$(CompilerCommonPath)\Internal\Runtime\StackTraceData.cs">
       <Link>Internal\Runtime\StackTraceData.cs</Link>
     </Compile>
+    <Compile Include="..\..\System.Private.CoreLib\src\System\Runtime\CompilerServices\AnalysisCharacteristicAttribute.cs" />
+    <Compile Include="..\..\Runtime.Base\src\System\Runtime\CompilerServices\IntrinsicAttribute.cs" />
   </ItemGroup>
 </Project>

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/AnalysisCharacteristicNode.cs

@@ -0,0 +1,26 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+
+using ILCompiler.DependencyAnalysisFramework;
+
+namespace ILCompiler.DependencyAnalysis
+{
+    public class AnalysisCharacteristicNode : DependencyNodeCore<NodeFactory>
+    {
+        public AnalysisCharacteristicNode(string characteristic)
+            => Characteristic = characteristic;
+
+        public string Characteristic { get; }
+
+        public override bool InterestingForDynamicDependencyAnalysis => false;
+        public override bool HasDynamicDependencies => false;
+        public override bool HasConditionalStaticDependencies => false;
+        public override bool StaticDependenciesAreComputed => true;
+        public override IEnumerable<CombinedDependencyListEntry> GetConditionalStaticDependencies(NodeFactory context) => null;
+        public override IEnumerable<DependencyListEntry> GetStaticDependencies(NodeFactory context) => null;
+        public override IEnumerable<CombinedDependencyListEntry> SearchDynamicDependencies(List<DependencyNodeCore<NodeFactory>> markedNodes, int firstNode, NodeFactory context) => null;
+        protected override string GetName(NodeFactory context) => $"Analysis characteristic: {Characteristic}";
+    }
+}

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/MethodMetadataNode.cs

@@ -40,7 +40,9 @@ public MethodMetadataNode(MethodDesc method, bool isMinimal)
         public override IEnumerable<DependencyListEntry> GetStaticDependencies(NodeFactory factory)
         {
             DependencyList dependencies = new DependencyList();
-            dependencies.Add(factory.TypeMetadata((MetadataType)_method.OwningType), "Owning type metadata");
+
+            var owningType = (MetadataType)_method.OwningType;
+            dependencies.Add(factory.TypeMetadata(owningType), "Owning type metadata");
 
             if (!_isMinimal)
             {
@@ -77,6 +79,12 @@ public override IEnumerable<DependencyListEntry> GetStaticDependencies(NodeFacto
                 {
                     GenericArgumentDataFlow.ProcessGenericArgumentDataFlow(ref dependencies, factory, new MessageOrigin(_method), parameterType, _method);
                 }
+
+                if (_method.HasCustomAttribute("System.Diagnostics", "StackTraceHiddenAttribute")
+                    || owningType.HasCustomAttribute("System.Diagnostics", "StackTraceHiddenAttribute"))
+                {
+                    dependencies.Add(factory.AnalysisCharacteristic("StackTraceHiddenMetadataPresent"), "Method is StackTraceHidden");
+                }
             }
 
             return dependencies;

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/NodeFactory.cs

@@ -604,6 +604,11 @@ private void CreateNodeCaches()
                 return new ProxyTypeMapRequestNode(type);
             });
 
+            _analysisCharacteristics = new NodeCache<string, AnalysisCharacteristicNode>(c =>
+            {
+                return new AnalysisCharacteristicNode(c);
+            });
+
             NativeLayout = new NativeLayoutHelper(this);
         }
 
@@ -1526,6 +1531,12 @@ public ProxyTypeMapRequestNode ProxyTypeMapRequest(TypeDesc type)
             return _proxyTypeMapRequests.GetOrAdd(type);
         }
 
+        private NodeCache<string, AnalysisCharacteristicNode> _analysisCharacteristics;
+        public AnalysisCharacteristicNode AnalysisCharacteristic(string ch)
+        {
+            return _analysisCharacteristics.GetOrAdd(ch);
+        }
+
         /// <summary>
         /// Returns alternative symbol name that object writer should produce for given symbols
         /// in addition to the regular one.

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ILScanner.cs

@@ -310,6 +310,15 @@ public TypeMapManager GetTypeMapManager()
             return new ScannedTypeMapManager(_factory);
         }
 
+        public IEnumerable<string> GetAnalysisCharacteristics()
+        {
+            foreach (DependencyNodeCore<NodeFactory> n in MarkedNodes)
+            {
+                if (n is AnalysisCharacteristicNode acn)
+                    yield return acn.Characteristic;
+            }
+        }
+
         private sealed class ScannedVTableProvider : VTableSliceProvider
         {
             private readonly Dictionary<TypeDesc, MethodDesc[]> _vtableSlices = new Dictionary<TypeDesc, MethodDesc[]>();

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/SubstitutedILProvider.cs

@@ -10,6 +10,7 @@
 using ILCompiler.DependencyAnalysis;
 
 using Internal.IL;
+using Internal.IL.Stubs;
 using Internal.TypeSystem;
 using Internal.TypeSystem.Ecma;
 
@@ -24,13 +25,15 @@ public class SubstitutedILProvider : ILProvider
         private readonly SubstitutionProvider _substitutionProvider;
         private readonly DevirtualizationManager _devirtualizationManager;
         private readonly MetadataManager _metadataManager;
+        private readonly HashSet<string> _characteristics;
 
-        public SubstitutedILProvider(ILProvider nestedILProvider, SubstitutionProvider substitutionProvider, DevirtualizationManager devirtualizationManager, MetadataManager metadataManager = null)
+        public SubstitutedILProvider(ILProvider nestedILProvider, SubstitutionProvider substitutionProvider, DevirtualizationManager devirtualizationManager, MetadataManager metadataManager = null, IEnumerable<string> characteristics = null)
         {
             _nestedILProvider = nestedILProvider;
             _substitutionProvider = substitutionProvider;
             _devirtualizationManager = devirtualizationManager;
             _metadataManager = metadataManager;
+            _characteristics = characteristics != null ? new HashSet<string>(characteristics) : null;
         }
 
         public override MethodIL GetMethodIL(MethodDesc method)
@@ -41,6 +44,13 @@ public override MethodIL GetMethodIL(MethodDesc method)
                 return substitution.EmitIL(method);
             }
 
+            if (TryGetCharacteristicValue(method, out bool characteristicEnabled))
+            {
+                return new ILStubMethodIL(method,
+                    [characteristicEnabled ? (byte)ILOpCode.Ldc_i4_1 : (byte)ILOpCode.Ldc_i4_0, (byte)ILOpCode.Ret],
+                    [], []);
+            }
+
             // BEGIN TEMPORARY WORKAROUND
             //
             // The following lines should just be:
@@ -819,6 +829,11 @@ private bool TryGetConstantArgument(MethodIL methodIL, byte[] body, OpcodeFlags[
                         {
                             return true;
                         }
+                        else if (TryGetCharacteristicValue(method, out bool characteristic))
+                        {
+                            constant = characteristic ? 1 : 0;
+                            return true;
+                        }
                         else
                         {
                             constant = 0;
@@ -1127,6 +1142,19 @@ private static bool ReadGetTypeFromHandle(ref ILReader reader, MethodIL methodIL
             return true;
         }
 
+        private bool TryGetCharacteristicValue(MethodDesc maybeCharacteristicMethod, out bool value)
+        {
+            if (maybeCharacteristicMethod.IsIntrinsic
+                && maybeCharacteristicMethod.HasCustomAttribute("System.Runtime.CompilerServices", "AnalysisCharacteristicAttribute"))
+            {
+                value = _characteristics == null || _characteristics.Contains(maybeCharacteristicMethod.Name);
+                return true;
+            }
+
+            value = false;
+            return false;
+        }
+
         private sealed class SubstitutedMethodIL : MethodIL
         {
             private readonly byte[] _body;

src/coreclr/tools/aot/ILCompiler.Compiler/IL/ILImporter.Scanner.cs

@@ -61,6 +61,8 @@ public enum ImportState : byte
 
         private bool _isReadOnly;
         private TypeDesc _constrained;
+        private int _currentInstructionOffset;
+        private int _previousInstructionOffset;
 
         private DependencyList _dependencies;
         private BasicBlock _lateBasicBlocks;
@@ -258,6 +260,13 @@ private void StartImportingBasicBlock(BasicBlock basicBlock)
 
             _typeEqualityPatternAnalyzer = default;
             _isInstCheckPatternAnalyzer = default;
+            _currentInstructionOffset = 0;
+            _previousInstructionOffset = -1;
+        }
+
+        private void StartImportingInstruction()
+        {
+            _currentInstructionOffset = _currentOffset;
         }
 
         partial void StartImportingInstruction(ILOpcode opcode)
@@ -271,6 +280,8 @@ private void EndImportingInstruction()
             // The instruction should have consumed any prefixes.
             _constrained = null;
             _isReadOnly = false;
+
+            _previousInstructionOffset = _currentInstructionOffset;
         }
 
         private void ImportCasting(ILOpcode opcode, int token)
@@ -853,6 +864,17 @@ private void ImportBranch(ILOpcode opcode, BasicBlock target, BasicBlock fallthr
                 }
             }
 
+            if (opcode == ILOpcode.brfalse && _previousInstructionOffset >= 0)
+            {
+                var reader = new ILReader(_ilBytes, _previousInstructionOffset);
+                if (reader.ReadILOpcode() == ILOpcode.call
+                    && _methodIL.GetObject(reader.ReadILToken()) is MethodDesc { IsIntrinsic: true } intrinsicMethod
+                    && intrinsicMethod.HasCustomAttribute("System.Runtime.CompilerServices", "AnalysisCharacteristicAttribute"))
+                {
+                    condition = _factory.AnalysisCharacteristic(intrinsicMethod.Name);
+                }
+            }
+
             ImportFallthrough(target);
 
             if (fallthrough != null)
@@ -1531,7 +1553,6 @@ private DefType GetWellKnownType(WellKnownType wellKnownType)
             return _compilation.TypeSystemContext.GetWellKnownType(wellKnownType);
         }
 
-        private static void StartImportingInstruction() { }
         private static void ImportNop() { }
         private static void ImportBreak() { }
         private static void ImportLoadVar(int index, bool argument) { }

src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj

@@ -347,6 +347,7 @@
     <Compile Include="Compiler\BodySubstitution.cs" />
     <Compile Include="Compiler\BodySubstitutionParser.cs" />
     <Compile Include="Compiler\DependencyAnalysis\AddressTakenMethodNode.cs" />
+    <Compile Include="Compiler\DependencyAnalysis\AnalysisCharacteristicNode.cs" />
     <Compile Include="Compiler\DependencyAnalysis\AnalyzedExternalTypeMapNode.cs" />
     <Compile Include="Compiler\DependencyAnalysis\AnalyzedProxyTypeMapNode.cs" />
     <Compile Include="Compiler\DependencyAnalysis\ByRefTypeMapNode.cs" />