Bump to NDK r29

Documentation/workflow/HowToUpdateNDK.md

@@ -0,0 +1,64 @@
+# How to update Android NDK
+
+For the most part, update of the NDK version used to build this repository is
+very straightforward. The only complication arises from the fact that we carry
+a copy of some LLVM source files, for its libc++ and libc++abi libraries.
+The copied files are needed only by the `NativeAOT` host (see https://github.com/dotnet/runtime/issues/121172),
+the `MonoVM` and `CoreCLR` hosts link against the two libraries directly.
+
+Our copy of LLVM sources *must* be updated *every time* we update the NDK version.
+
+## Update NDK reference in `xaprepare`
+
+Visit https://developer.android.com/ndk/downloads/index.html to obtain NDK revision
+information then edit the `build-tools/xaprepare/xaprepare/ConfigAndData/BuildAndroidPlatforms.cs`
+file and update the `BuildAndroidPlatforms.AndroidNdkVersion` and `BuildAndroidPlatforms.AndroidNdkPkgRevision`
+properties with the information obtained from the NDK distribution URL.
+
+## Update LLVM sources
+
+The best way to do it is by using the `tools/update-llvm-sources` utility, after runing `xaprepare`.
+
+You can run the utility directly with `dotnet tools/update-llvm-sources` or, if you are on a Unix
+system, run `make update-llvm` from the top directory.
+
+### Details (should you need to update sources manually)
+
+Android NDK uses a fork of the upstream LLVM repository, currently
+https://android.googlesource.com/toolchain/llvm-project and this is the repository updated tool
+mentioned above uses to fetch the files.
+
+Android NDK has a manifest file for the LLVM toolchain which enumerates revisions of all the
+components, however that file changes name in each release, based on information it yet another
+manifest file, namely `${ANDROID_NDK_ROOT}/BUILD_INFO`. This is a JSON file, which contains a
+number of properties, we are however interested only in one of them, named `bid`. Its value
+is a string which is part of the second manifest, found in the `${ANDROID_NDK_ROOT}/manifest_${bid}.xml`
+file.
+
+In the XML manifest, we can find an element named `project`, with its `name` attribute set to
+`toolchain/llvm-project` - the `revision` attribute of that element is the Git revision we need
+in order to access sources from the Google's `llvm-project` fork.
+
+Once you have the revision, you can either clone the Android fork repository and checkout the
+revision, or visit the individual files in the browser. All the LLVM sources we copied are
+contained in the `src-ThirdParty/llvm/` directory, with the subdirectories reflecting exactly
+the `llvm-project` layout. This way, you can take a file path relative to `src-ThirdParty/llvm` and
+form the file's URL as follows:
+
+```
+https://android.googlesource.com/toolchain/llvm-project/+/${LLVM_REVISION}/${RELATIVE_FILE_PATH}
+```
+
+Visiting this url will show you the file with syntax higlighting and line numbers, however it's
+not the raw source, but rather its HTML rendering, useless for our purpose. In order to fetch the
+raw source, we need to append `?format=TEXT` to the URL. Once visited in the browser (or fetched
+using `curl` or `wget`), the resulting file will be downloaded but not yet ready for updating of
+our copy. The downloaded file is encoded in the `base64` encoding and must be decoded before use.
+
+On Unix systems this can be done using the following command:
+
+```shell
+$ base64 -d < downloaded_file.cpp > file.cpp
+```
+
+After that, the resulting file can be copied to its destination in our source tree.

build-tools/scripts/LlvmUpdateInfo.cs.in

@@ -0,0 +1,10 @@
+using System;
+
+namespace Xamarin.Android.Tools;
+
+static class LlvmUpdateInfo
+{
+	public const string Revision = "@LLVM_PROJECT_REVISION@";
+    public const string Version = "@LLVM_PROJECT_VERSION@";
+	public static readonly Uri BaseUrl = new Uri ("@LLVM_PROJECT_BASE_URL@");
+}

build-tools/xaprepare/xaprepare/ConfigAndData/BuildAndroidPlatforms.cs

@@ -5,8 +5,8 @@ namespace Xamarin.Android.Prepare
 {
 	class BuildAndroidPlatforms
 	{
-		public const string AndroidNdkVersion = "28c";
-		public const string AndroidNdkPkgRevision = "28.2.13676358";
+		public const string AndroidNdkVersion = "29";
+		public const string AndroidNdkPkgRevision = "29.0.14206865";
 		public const int NdkMinimumAPI = 21;
 		public const int NdkMinimumAPILegacy32 = 21;
 

build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs

@@ -36,6 +36,7 @@ public static partial class Urls
 			public static readonly Uri AndroidToolchain_AndroidUri = new Uri ("https://dl.google.com/android/repository/");
 
 			public static Uri BinutilsArchive = new Uri ($"https://github.com/dotnet/android-native-tools/releases/download/{BinutilsVersion}/xamarin-android-toolchain-{BinutilsVersion}.7z");
+			public static Uri GoogleSourcesBase = new Uri ("https://android.googlesource.com");
 		}
 
 		public static partial class Defaults

build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs

@@ -21,6 +21,7 @@ protected override void AddSteps (Context context)
 			Steps.Add (new Step_InstallDotNetPreview ());
 			Steps.Add (new Step_InstallMicrosoftOpenJDK ());
 			Steps.Add (new Step_Android_SDK_NDK ());
+			Steps.Add (new Step_Generate_LLVM_UpdateInfo ());
 			Steps.Add (new Step_GenerateFiles (atBuildStart: true));
 			Steps.Add (new Step_PrepareProps ());
 			Steps.Add (new Step_InstallGNUBinutils ());

build-tools/xaprepare/xaprepare/Steps/Step_Generate_LLVM_UpdateInfo.cs

@@ -0,0 +1,185 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using System.Text.Json;
+using System.Threading.Tasks;
+using System.Xml;
+
+namespace Xamarin.Android.Prepare;
+
+class Step_Generate_LLVM_UpdateInfo : Step
+{
+	static ReadOnlySpan<byte> Utf8Bom => new byte[] { 0xEF, 0xBB, 0xBF };
+	static readonly byte[] BidPropertyName = Encoding.UTF8.GetBytes ("bid");
+
+	public Step_Generate_LLVM_UpdateInfo ()
+		: base ("Generating LLVM source update information")
+	{}
+
+#pragma warning disable CS1998
+	protected override async Task<bool> Execute (Context context)
+	{
+		try {
+			if (!Generate (context)) {
+				Log.WarningLine ("Failed to generate LLVM update info. Attempt to update LLVM sources may fail.");
+			}
+		} catch (Exception ex) {
+			Log.WarningLine ($"Failed to generate LLVM update info. {ex.Message}");
+			Log.DebugLine ($"Exception was thrown while generating LLVM update info.");
+			Log.DebugLine (ex.ToString ());
+		}
+
+		// This step isn't critical, we never fail.
+		return true;
+	}
+#pragma warning restore CS1998
+
+	bool Generate (Context context)
+	{
+		// BUILD_INFO is a JSON document with build information, we need the "bid" component from there as it forms
+		// part of the toolchain manifest name
+		string? bid = GetBid (Path.Combine (Configurables.Paths.AndroidToolchainRootDirectory, "BUILD_INFO"));
+		if (String.IsNullOrEmpty (bid)) {
+			Log.DebugLine ("Unable to find LLVM toolchain bid information.");
+			return false;
+		}
+
+		// Manifest contains GIT revisions of various NDK components. We need the LLVM project's one from there.
+		string toolchainManifestPath = Path.Combine (Configurables.Paths.AndroidToolchainRootDirectory, $"manifest_{bid}.xml");
+		(string? llvmProjectPath, string? llvmProjectRevision) = GetLlvmProjectInfo (toolchainManifestPath);
+
+		if (String.IsNullOrEmpty (llvmProjectPath)) {
+			Log.DebugLine ("Failed to read LLVM project path from the manifest.");
+			return false;
+		}
+
+		if (String.IsNullOrEmpty (llvmProjectRevision)) {
+			Log.DebugLine ("Failed to read LLVM project GIT revision from the manifest.");
+			return false;
+		}
+
+		string? llvmProjectVersion = null;
+		string androidVersionPath = Path.Combine (Configurables.Paths.AndroidToolchainRootDirectory, "AndroidVersion.txt");
+		if (Path.Exists (androidVersionPath)) {
+			try {
+				foreach (string line in File.ReadLines (androidVersionPath)) {
+					// In NDK r29 LLVM version was on the first line
+					llvmProjectVersion = line.Trim ();
+					break;
+				}
+			} catch (Exception ex) {
+				Log.DebugLine ($"Failed to read LLVM Android version file '{androidVersionPath}'");
+				Log.DebugLine ("Exception was thrown:");
+				Log.DebugLine (ex.ToString ());
+			}
+		} else {
+			Log.WarningLine ($"LLVM Android version file not found at {androidVersionPath}");
+		}
+
+		if (String.IsNullOrEmpty (llvmProjectVersion)) {
+			llvmProjectVersion = "<unknown>";
+		}
+
+		Log.InfoLine ("LLVM project path: ", llvmProjectPath);
+		Log.InfoLine ("LLVM project revision: ", llvmProjectRevision);
+		Log.InfoLine ("LLVM project version: ", llvmProjectVersion);
+
+		// Manifest uses https://googleplex-android.googlesource.com/ which is not accessible for mere mortals,
+		// therefore we need to use the public URL
+		var baseURIBuilder = new UriBuilder (Configurables.Urls.GoogleSourcesBase);
+		baseURIBuilder.Path = $"{llvmProjectPath}/+/{llvmProjectRevision}";
+		Uri baseURI = baseURIBuilder.Uri;
+
+		const string updateSourcesInputName = "LlvmUpdateInfo.cs.in";
+		string updateInfoSourceInputPath = Path.Combine (Configurables.Paths.BuildToolsScriptsDir, updateSourcesInputName);
+		string updateInfoSourceOutputPath = Path.Combine (Configurables.Paths.BuildBinDir, Path.GetFileNameWithoutExtension (updateSourcesInputName));
+
+		Log.InfoLine ();
+		Log.InfoLine ($"Generating LLVM update info sources.");
+		var updateInfoSource = new GeneratedPlaceholdersFile (
+			new Dictionary <string, string> (StringComparer.Ordinal) {
+				{ "@LLVM_PROJECT_BASE_URL@", baseURI.ToString () },
+				{ "@LLVM_PROJECT_REVISION@", llvmProjectRevision },
+				{ "@LLVM_PROJECT_VERSION@", llvmProjectVersion },
+			},
+			updateInfoSourceInputPath,
+			updateInfoSourceOutputPath
+		);
+		updateInfoSource.Generate (context);
+
+		return true;
+	}
+
+	(string? path, string? revision) GetLlvmProjectInfo (string manifestPath)
+	{
+		Log.DebugLine ($"Reading LLVM toolchain manifest from '{manifestPath}'");
+
+		if (!File.Exists (manifestPath)) {
+			Log.DebugLine ($"NDK LLVM manifest '{manifestPath}' not found");
+			return (null, null);
+		}
+
+		var readerSettings = new XmlReaderSettings {
+			ValidationType = ValidationType.None,
+			DtdProcessing = DtdProcessing.Ignore,
+			IgnoreWhitespace = true,
+			IgnoreComments = true,
+			IgnoreProcessingInstructions = true,
+		};
+		using var reader = XmlReader.Create (manifestPath, readerSettings);
+		var doc = new XmlDocument ();
+		doc.Load (reader);
+
+		XmlNode? llvmToolchain = doc.SelectSingleNode ("//manifest/project[@name='toolchain/llvm-project']");
+		if (llvmToolchain == null) {
+			Log.DebugLine ("Failed to find LLVM toolchain info in the manifest.");
+			return (null, null);
+		}
+
+		if (llvmToolchain.Attributes == null) {
+			Log.DebugLine ("Unable to read path and revision info about the LLVM toolchain, no attributes on the element.");
+			return (null, null);
+		}
+
+		XmlAttribute? path = llvmToolchain.Attributes["path"];
+		XmlAttribute? revision = llvmToolchain.Attributes["revision"];
+
+		return (path?.Value, revision?.Value);
+	}
+
+	string? GetBid (string buildInfoPath)
+	{
+		Log.DebugLine ($"Reading LLVM toolchain build info from '{buildInfoPath}'");
+
+		ReadOnlySpan<byte> manifestBytes = File.ReadAllBytes (buildInfoPath);
+
+		if (manifestBytes.StartsWith (Utf8Bom)) {
+			manifestBytes = manifestBytes.Slice (Utf8Bom.Length);
+		}
+
+		string? bid = null;
+		var reader = new Utf8JsonReader (manifestBytes);
+		while (reader.Read ()) {
+			if (reader.TokenType != JsonTokenType.PropertyName) {
+				continue;
+			}
+
+			if (!reader.ValueTextEquals (BidPropertyName)) {
+				continue;
+			}
+
+			// let's assume the manifest document is formatted correctly
+			reader.Read ();
+			if (reader.TokenType != JsonTokenType.String) {
+				Log.DebugLine ($"Invalid token type '{reader.TokenType}' for the 'bid' property in LLVM manifest.");
+				return null;
+			}
+
+			bid = reader.GetString ();
+			break;
+		}
+
+		return bid;
+	}
+}