[CoreCLR] support for Debug managed-to-Java typemaps (#10065)

Context: 684ede6
Context: b11471b
Context: 7117414
Context: c227042
Context: a0a837a
Context: ea38c0c

From b11471b:

> `Java.Interop.JniRuntime.JniTypeManager` supports bi-directional
> mapping between JNI type signatures and managed `Type`s:
> 
>   * `GetTypeSignature(Type)` returns the JNI type signature that
>     corresponds to the `System.Type`.
>   * `GetType(JniTypeSignature)` returns the `Type` that corresponds
>     to a JNI type signature.

We have had *multiple* ways to perform such mappings, including:

  * Fully managed (Release config):
    Processes trimmed assemblies, creates app-global mappings
    within `Mono.Android.dll`.
    Required for NativeAOT, optional with CoreCLR, MonoVM.
    Commits 684ede6, b11471b

  * MonoVM Instant Run (Debug config):
    Per-assembly mapping data, fast deployed to to the target device.
    Removed in commits 8298b6f, 17e2e8e.

  * MonoVM libxamarin-app.so (Debug config):
    Processes *un-trimmed* assemblies, creates app-global
    *string-based* type mappings within LLVM-IR, so that mappings
    don't need to be rebuilt as often when assemblies are changed.
    Commits 7117414, c227042

  * MonoVM libxamarin-app.so (Release config):
    Processes trimmed assemblies, creates app-global
    *assembly-based* type mappings within LLVM-IR, relying on
    assembly name hashes, assembly MVIDs, and type token IDs.
    Commits c227042
  
  * CoreCLR libxamarin-app.so (Release config):
    Processes trimmed assemblies, creates app-global
    *assembly-based* type mappings within LLVM-IR, relying on
    assembly name hashes, assembly MVIDs, and type token IDs.
    Commits a0a837a, ea38c0c

"Missing" from this list is support for Debug config CoreCLR builds.

Begin implementing managed-to-Java lookups for Debug config CoreCLR
builds by

 1. Updating `libxamarin-app.so` generation to emit data similar to
    the "MonoVM libxamarin-app.so (Debug config)" data structures, &

 2. Update `clr_typemap_managed_to_java()` /
    `TypeMapper::typemap_managed_to_java()` within
    `libmono-android.debug.so` to use the Debug config data in (1).

This allows (mostly) untrimmed assemblies to be used with typemap data.

Additionally, the managed-to-Java lookup tables use the
assembly-qualified managed type name; introduce a step to "fix up"
the passed managed type name by appending the assembly name, which is
obtained by searching for MVID match in the MVID-to-name lookup table.

Author: grendello

src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/FindTypeMapObjectsStep.cs

@@ -37,13 +37,14 @@ public void ProcessAssembly (AssemblyDefinition assembly, StepContext context)
 
 		var xml = new TypeMapObjectsXmlFile {
 			AssemblyName = assembly.Name.Name,
+			AssemblyMvid = assembly.MainModule.Mvid,
 		};
 
 		if (Debug) {
-			var (javaToManaged, managedToJava, foundJniNativeRegistration) = TypeMapCecilAdapter.GetDebugNativeEntries (types, Context);
+			var (typeMapDebugSets, foundJniNativeRegistration) = TypeMapCecilAdapter.GetDebugNativeEntries (types, Context, needUniqueAssemblies: false);
 
-			xml.JavaToManagedDebugEntries.AddRange (javaToManaged);
-			xml.ManagedToJavaDebugEntries.AddRange (managedToJava);
+			xml.JavaToManagedDebugEntries.AddRange (typeMapDebugSets.JavaToManaged);
+			xml.ManagedToJavaDebugEntries.AddRange (typeMapDebugSets.ManagedToJava);
 			xml.FoundJniNativeRegistration = foundJniNativeRegistration;
 		} else {
 			var genState = TypeMapCecilAdapter.GetReleaseGenerationState (types, Context, out var foundJniNativeRegistration);

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AssetPackTests.cs

@@ -38,6 +38,7 @@ public void BuildLibraryWithAssetPack ([Values (true, false)] bool isRelease)
 		[Category ("SmokeTests")]
 		[TestCase (false, AndroidRuntime.MonoVM)]
 		[TestCase (true, AndroidRuntime.MonoVM)]
+		[TestCase (false, AndroidRuntime.CoreCLR)]
 		[TestCase (true, AndroidRuntime.CoreCLR)]
 		[TestCase (true, AndroidRuntime.NativeAOT)]
 		public void BuildApplicationWithAssetPackThatHasInvalidName (bool isRelease, AndroidRuntime runtime)
@@ -67,6 +68,7 @@ public void BuildApplicationWithAssetPackThatHasInvalidName (bool isRelease, And
 		[Category ("SmokeTests")]
 		[TestCase (false, AndroidRuntime.MonoVM)]
 		[TestCase (true, AndroidRuntime.MonoVM)]
+		[TestCase (false, AndroidRuntime.CoreCLR)]
 		[TestCase (true, AndroidRuntime.CoreCLR)]
 		[TestCase (true, AndroidRuntime.NativeAOT)]
 		public void BuildApplicationWithAssetPackOutsideProjectDirectory (bool isRelease, AndroidRuntime runtime)
@@ -117,6 +119,7 @@ public void BuildApplicationWithAssetPackOutsideProjectDirectory (bool isRelease
 		[Category ("SmokeTests")]
 		[TestCase (false, AndroidRuntime.MonoVM)]
 		[TestCase (true, AndroidRuntime.MonoVM)]
+		[TestCase (false, AndroidRuntime.CoreCLR)]
 		[TestCase (true, AndroidRuntime.CoreCLR)]
 		[TestCase (true, AndroidRuntime.NativeAOT)]
 		public void BuildApplicationWithAssetPackOverrides (bool isRelease, AndroidRuntime runtime)
@@ -160,6 +163,7 @@ public void BuildApplicationWithAssetPackOverrides (bool isRelease, AndroidRunti
 		[Category ("SmokeTests")]
 		[TestCase (false, AndroidRuntime.MonoVM)]
 		[TestCase (true, AndroidRuntime.MonoVM)]
+		[TestCase (false, AndroidRuntime.CoreCLR)]
 		[TestCase (true, AndroidRuntime.CoreCLR)]
 		[TestCase (true, AndroidRuntime.NativeAOT)]
 		public void BuildApplicationWithAssetPack (bool isRelease, AndroidRuntime runtime)

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs

@@ -132,6 +132,13 @@ public partial class BuildTest : BaseTest
 				/* usesAssemblyStore */  true,
 				/* runtime */            AndroidRuntime.CoreCLR,
 			},
+			new object [] {
+				/* runtimeIdentifiers */ "android-arm64",
+				/* isRelease */          false,
+				/* aot */                false,
+				/* usesAssemblyStore */  true,
+				/* runtime */            AndroidRuntime.CoreCLR,
+			},
 			new object [] {
 				/* runtimeIdentifiers */ "android-arm64",
 				/* isRelease */          true,

src/Xamarin.Android.Build.Tasks/Utilities/TypeMapCecilAdapter.cs

@@ -7,24 +7,27 @@
 using ReleaseGenerationState = Xamarin.Android.Tasks.TypeMapGenerator.ReleaseGenerationState;
 using TypeMapDebugEntry = Xamarin.Android.Tasks.TypeMapGenerator.TypeMapDebugEntry;
 using TypeMapReleaseEntry = Xamarin.Android.Tasks.TypeMapGenerator.TypeMapReleaseEntry;
+using TypeMapDebugDataSets = Xamarin.Android.Tasks.TypeMapGenerator.TypeMapDebugDataSets;
+using TypeMapDebugAssembly = Xamarin.Android.Tasks.TypeMapGenerator.TypeMapDebugAssembly;
 
 namespace Xamarin.Android.Tasks;
 
 // Converts types from Mono.Cecil to the format used by the typemap generator.
 class TypeMapCecilAdapter
 {
-	public static (List<TypeMapDebugEntry> javaToManaged, List<TypeMapDebugEntry> managedToJava) GetDebugNativeEntries (NativeCodeGenState state)
+	public static TypeMapDebugDataSets GetDebugNativeEntries (NativeCodeGenState state, bool needUniqueAssemblies)
 	{
-		var (javaToManaged, managedToJava, foundJniNativeRegistration) = GetDebugNativeEntries (state.AllJavaTypes, state.TypeCache);
+		var (dataSets, foundJniNativeRegistration) = GetDebugNativeEntries (state.AllJavaTypes, state.TypeCache, needUniqueAssemblies);
 
 		state.JniAddNativeMethodRegistrationAttributePresent = foundJniNativeRegistration;
 
-		return (javaToManaged, managedToJava);
+		return dataSets;
 	}
 
-	public static (List<TypeMapDebugEntry> javaToManaged, List<TypeMapDebugEntry> managedToJava, bool foundJniNativeRegistration) GetDebugNativeEntries (List<TypeDefinition> types, TypeDefinitionCache cache)
+	public static (TypeMapDebugDataSets dataSets, bool foundJniNativeRegistration) GetDebugNativeEntries (List<TypeDefinition> types, TypeDefinitionCache cache, bool needUniqueAssemblies)
 	{
 		var javaDuplicates = new Dictionary<string, List<TypeMapDebugEntry>> (StringComparer.Ordinal);
+		var uniqueAssemblies = needUniqueAssemblies ? new Dictionary<string, TypeMapDebugAssembly> (StringComparer.OrdinalIgnoreCase) : null;
 		var javaToManaged = new List<TypeMapDebugEntry> ();
 		var managedToJava = new List<TypeMapDebugEntry> ();
 		var foundJniNativeRegistration = false;
@@ -37,11 +40,31 @@ public static (List<TypeMapDebugEntry> javaToManaged, List<TypeMapDebugEntry> ma
 
 			javaToManaged.Add (entry);
 			managedToJava.Add (entry);
+
+			if (uniqueAssemblies == null) {
+				continue;
+			}
+
+			string? asmName = td.Module.Assembly.Name.Name;
+			if (String.IsNullOrEmpty (asmName) || uniqueAssemblies.ContainsKey (asmName)) {
+				continue;
+			}
+
+			var asmInfo = new TypeMapDebugAssembly {
+				MVID = td.Module.Mvid,
+				Name = asmName,
+			};
+			asmInfo.MVIDBytes = asmInfo.MVID.ToByteArray ();
+			uniqueAssemblies.Add (asmName, asmInfo);
 		}
 
 		SyncDebugDuplicates (javaDuplicates);
 
-		return (javaToManaged, managedToJava, foundJniNativeRegistration);
+		return (new TypeMapDebugDataSets {
+			JavaToManaged = javaToManaged,
+			ManagedToJava = managedToJava,
+			UniqueAssemblies = uniqueAssemblies != null ? new List<TypeMapDebugAssembly> (uniqueAssemblies.Values) : null
+		}, foundJniNativeRegistration);
 	}
 
 	public static ReleaseGenerationState GetReleaseGenerationState (NativeCodeGenState state)
@@ -204,7 +227,7 @@ static bool JniAddNativeMethodRegistrationAttributeFound (bool alreadyFound, Typ
 		if (alreadyFound || !javaType.HasCustomAttributes) {
 			return alreadyFound;
 		}
-		
+
 		foreach (CustomAttribute ca in javaType.CustomAttributes) {
 			if (string.Equals ("JniAddNativeMethodRegistrationAttribute", ca.AttributeType.Name, StringComparison.Ordinal)) {
 				return true;

src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs

@@ -83,12 +83,27 @@ public override string ToString ()
 			}
 		}
 
+		internal sealed class TypeMapDebugAssembly
+		{
+			public Guid MVID;
+			public byte[] MVIDBytes;
+			public string Name;
+		}
+
+		internal sealed class TypeMapDebugDataSets
+		{
+			public List<TypeMapDebugEntry> JavaToManaged ;
+			public List<TypeMapDebugEntry> ManagedToJava;
+			public List<TypeMapDebugAssembly>? UniqueAssemblies;
+		}
+
 		// Widths include the terminating nul character but not the padding!
 		internal sealed class ModuleDebugData
 		{
 			public uint EntryCount;
 			public List<TypeMapDebugEntry> JavaToManagedMap;
 			public List<TypeMapDebugEntry> ManagedToJavaMap;
+			public List<TypeMapDebugAssembly>? UniqueAssemblies;
 		}
 
 		internal sealed class ReleaseGenerationState
@@ -138,18 +153,23 @@ public void Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAt
 
 		void GenerateDebugNativeAssembly (string outputDirectory)
 		{
-			(var javaToManaged, var managedToJava) = state.GetDebugNativeEntries ();
+			TypeMapDebugDataSets dataSets = state.GetDebugNativeEntries (needUniqueAssemblies: runtime == AndroidRuntime.CoreCLR);
 
 			var data = new ModuleDebugData {
-				EntryCount = (uint)javaToManaged.Count,
-				JavaToManagedMap = javaToManaged,
-				ManagedToJavaMap = managedToJava,
+				EntryCount = (uint)dataSets.JavaToManaged.Count,
+				JavaToManagedMap = dataSets.JavaToManaged,
+				ManagedToJavaMap = dataSets.ManagedToJava,
+				UniqueAssemblies = dataSets.UniqueAssemblies,
 			};
 
 			data.JavaToManagedMap.Sort ((TypeMapDebugEntry a, TypeMapDebugEntry b) => String.Compare (a.JavaName, b.JavaName, StringComparison.Ordinal));
 			data.ManagedToJavaMap.Sort ((TypeMapDebugEntry a, TypeMapDebugEntry b) => String.Compare (a.ManagedName, b.ManagedName, StringComparison.Ordinal));
 
-			var composer = new TypeMappingDebugNativeAssemblyGenerator (log, data);
+			LLVMIR.LlvmIrComposer composer = runtime switch {
+				AndroidRuntime.MonoVM => new TypeMappingDebugNativeAssemblyGenerator (log, data),
+				AndroidRuntime.CoreCLR => new TypeMappingDebugNativeAssemblyGeneratorCLR (log, data),
+				_ => throw new NotSupportedException ($"Internal error: unsupported runtime {runtime}")
+			};
 			GenerateNativeAssembly (composer, composer.Construct (), outputDirectory);
 		}
 
@@ -216,7 +236,7 @@ interface ITypeMapGeneratorAdapter
 	{
 		AndroidTargetArch TargetArch { get; }
 		bool JniAddNativeMethodRegistrationAttributePresent { get; set; }
-		(List<TypeMapDebugEntry> javaToManaged, List<TypeMapDebugEntry> managedToJava) GetDebugNativeEntries ();
+		TypeMapDebugDataSets GetDebugNativeEntries (bool needUniqueAssemblies);
 		ReleaseGenerationState GetReleaseGenerationState ();
 	}
 
@@ -236,9 +256,9 @@ public bool JniAddNativeMethodRegistrationAttributePresent {
 			set => state.JniAddNativeMethodRegistrationAttributePresent = value;
 		}
 
-		public (List<TypeMapDebugEntry> javaToManaged, List<TypeMapDebugEntry> managedToJava) GetDebugNativeEntries ()
+		public TypeMapDebugDataSets GetDebugNativeEntries (bool needUniqueAssemblies)
 		{
-			return TypeMapCecilAdapter.GetDebugNativeEntries (state);
+			return TypeMapCecilAdapter.GetDebugNativeEntries (state, needUniqueAssemblies);
 		}
 
 		public ReleaseGenerationState GetReleaseGenerationState ()
@@ -259,12 +279,22 @@ public TypeMapObjectsFileAdapter (AndroidTargetArch targetArch)
 
 		public bool JniAddNativeMethodRegistrationAttributePresent { get; set; }
 
-		public (List<TypeMapDebugEntry> javaToManaged, List<TypeMapDebugEntry> managedToJava) GetDebugNativeEntries ()
+		public TypeMapDebugDataSets GetDebugNativeEntries (bool needUniqueAssemblies)
 		{
 			var javaToManaged = new List<TypeMapDebugEntry> ();
 			var managedToJava = new List<TypeMapDebugEntry> ();
+			var uniqueAssemblies = new Dictionary<string, TypeMapDebugAssembly> (StringComparer.OrdinalIgnoreCase);
 
 			foreach (var xml in XmlFiles) {
+				if (!uniqueAssemblies.ContainsKey (xml.AssemblyName)) {
+					var assm = new TypeMapDebugAssembly {
+						MVID = xml.AssemblyMvid,
+						MVIDBytes = xml.AssemblyMvid.ToByteArray (),
+						Name = xml.AssemblyName,
+					};
+					uniqueAssemblies.Add (xml.AssemblyName, assm);
+				}
+
 				javaToManaged.AddRange (xml.JavaToManagedDebugEntries);
 				managedToJava.AddRange (xml.ManagedToJavaDebugEntries);
 			}
@@ -273,7 +303,11 @@ public TypeMapObjectsFileAdapter (AndroidTargetArch targetArch)
 			GroupDuplicateDebugEntries (javaToManaged);
 			GroupDuplicateDebugEntries (managedToJava);
 
-			return (javaToManaged, managedToJava);
+			return new TypeMapDebugDataSets {
+				JavaToManaged = javaToManaged,
+				ManagedToJava = managedToJava,
+				UniqueAssemblies = uniqueAssemblies.Values.ToList (),
+			};
 		}
 
 		void GroupDuplicateDebugEntries (List<TypeMapDebugEntry> debugEntries)

src/Xamarin.Android.Build.Tasks/Utilities/TypeMapObjectsXmlFile.cs

@@ -25,6 +25,7 @@ class TypeMapObjectsXmlFile
 	static readonly TypeMapObjectsXmlFile unscanned = new TypeMapObjectsXmlFile { WasScanned = false };
 
 	public string? AssemblyName { get; set; }
+	public Guid AssemblyMvid { get; set; } = Guid.Empty;
 	public bool FoundJniNativeRegistration { get; set; }
 	public List<TypeMapDebugEntry> JavaToManagedDebugEntries { get; } = [];
 	public List<TypeMapDebugEntry> ManagedToJavaDebugEntries { get; } = [];
@@ -58,6 +59,10 @@ void Export (XmlWriter xml)
 		xml.WriteStartElement ("api");
 		xml.WriteAttributeString ("type", HasDebugEntries ? "debug" : "release");
 		xml.WriteAttributeStringIfNotDefault ("assembly-name", AssemblyName);
+
+		if (AssemblyMvid != Guid.Empty) {
+			xml.WriteAttributeString ("mvid", AssemblyMvid.ToString ("N"));
+		}
 		xml.WriteAttributeStringIfNotDefault ("found-jni-native-registration", FoundJniNativeRegistration);
 
 		if (HasDebugEntries)
@@ -173,11 +178,13 @@ public static TypeMapObjectsXmlFile Import (string filename)
 
 		var type = root.GetRequiredAttribute ("type");
 		var assemblyName = root.GetAttributeOrDefault ("assembly-name", (string?)null);
+		var mvid = Guid.Parse (root.GetAttributeOrDefault ("mvid", Guid.Empty.ToString ()));
 		var foundJniNativeRegistration = root.GetAttributeOrDefault ("found-jni-native-registration", false);
 
 		var file = new TypeMapObjectsXmlFile {
 			WasScanned = true,
 			AssemblyName = assemblyName,
+			AssemblyMvid = mvid,
 			FoundJniNativeRegistration = foundJniNativeRegistration,
 		};
 

src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingDebugNativeAssemblyGenerator.cs

@@ -13,7 +13,7 @@ class TypeMappingDebugNativeAssemblyGenerator : LlvmIrComposer
 	{
 		const string JavaToManagedSymbol = "map_java_to_managed";
 		const string ManagedToJavaSymbol = "map_managed_to_java";
-		const string TypeMapSymbol = "type_map"; // MUST match src/monodroid/xamarin-app.hh
+		const string TypeMapSymbol = "type_map"; // MUST match src/native/mono/xamarin-app-stub/xamarin-app.hh
 
 		sealed class TypeMapContextDataProvider : NativeAssemblerStructContextDataProvider
 		{
@@ -66,19 +66,19 @@ public override string GetComment (object data, string fieldName)
 				var entry = EnsureType<TypeMapEntry> (data);
 
 				if (String.Compare ("from", fieldName, StringComparison.Ordinal) == 0) {
-					return $"from: entry.from";
+					return $"from: {entry.from}";
 				}
 
 				if (String.Compare ("to", fieldName, StringComparison.Ordinal) == 0) {
-					return $"to: entry.to";
+					return $"to: {entry.to}";
 				}
 
 				return String.Empty;
 			}
 		}
 
 		// Order of fields and their type must correspond *exactly* to that in
-		// src/monodroid/jni/xamarin-app.hh TypeMapEntry structure
+		// src/native/mono/xamarin-app-stub/xamarin-app.hh TypeMapEntry structure
 		[NativeAssemblerStructContextDataProvider (typeof (TypeMapEntryContextDataProvider))]
 		sealed class TypeMapEntry
 		{
@@ -87,7 +87,7 @@ sealed class TypeMapEntry
 		};
 
 		// Order of fields and their type must correspond *exactly* to that in
-		// src/monodroid/jni/xamarin-app.hh TypeMap structure
+		// src/native/mono/xamarin-app-stub/xamarin-app.hh TypeMap structure
 		[NativeAssemblerStructContextDataProvider (typeof (TypeMapContextDataProvider))]
 		sealed class TypeMap
 		{